[Checkins] SVN: zodbupdate/branches/adamg-faster/src/zodbupdate/ lets see if ALAP unpickling speeds it up
Adam Groszer
agroszer at gmail.com
Tue Sep 28 07:05:18 EDT 2010
Log message for revision 117002:
lets see if ALAP unpickling speeds it up
Changed:
U zodbupdate/branches/adamg-faster/src/zodbupdate/serialize.py
U zodbupdate/branches/adamg-faster/src/zodbupdate/update.py
-=-
Modified: zodbupdate/branches/adamg-faster/src/zodbupdate/serialize.py
===================================================================
--- zodbupdate/branches/adamg-faster/src/zodbupdate/serialize.py 2010-09-28 10:59:27 UTC (rev 117001)
+++ zodbupdate/branches/adamg-faster/src/zodbupdate/serialize.py 2010-09-28 11:05:17 UTC (rev 117002)
@@ -193,7 +193,9 @@
unpickler = self.__unpickler(input_file)
class_meta = unpickler.load()
- data = unpickler.load()
+ if hasattr(unpickler, 'need_repickle'):
+ #repickle depends on the unpickle process
+ data = unpickler.load()
class_meta = self.__update_class_meta(class_meta)
@@ -206,6 +208,9 @@
pickler = self.__pickler(output_file)
try:
pickler.dump(class_meta)
+ if not hasattr(unpickler, 'need_repickle'):
+ #load data as late as possible
+ data = unpickler.load()
pickler.dump(data)
except cPickle.PicklingError:
# Could not pickle that record, likely due to a broken
Modified: zodbupdate/branches/adamg-faster/src/zodbupdate/update.py
===================================================================
--- zodbupdate/branches/adamg-faster/src/zodbupdate/update.py 2010-09-28 10:59:27 UTC (rev 117001)
+++ zodbupdate/branches/adamg-faster/src/zodbupdate/update.py 2010-09-28 11:05:17 UTC (rev 117002)
@@ -19,6 +19,7 @@
import ZODB.broken
import ZODB.utils
import cStringIO
+import datetime
import logging
import pickle
import pickletools
@@ -59,6 +60,9 @@
self.storage.tpc_finish(t)
def __call__(self):
+ startTime = datetime.datetime.now()
+ logger.debug('Started at %s' % startTime.isoformat())
+
try:
count = 0
t = self.__new_transaction()
@@ -89,6 +93,9 @@
del traceback
raise e
+ logger.debug('Finished at %s' % datetime.datetime.now().isoformat())
+ logger.debug('%s seconds' % (datetime.datetime.now() - startTime).seconds)
+
@property
def records(self):
next = ZODB.utils.repr_to_oid(self.start_at)
More information about the checkins
mailing list