[Checkins] SVN: zodbupdate/branches/adamg-faster/src/zodbupdate/ lets see if ALAP unpickling speeds it up

[Adam Groszer]

Log message for revision 117002:
  lets see if ALAP unpickling speeds it up

Changed:
  U   zodbupdate/branches/adamg-faster/src/zodbupdate/serialize.py
  U   zodbupdate/branches/adamg-faster/src/zodbupdate/update.py

-=-
Modified: zodbupdate/branches/adamg-faster/src/zodbupdate/serialize.py
===================================================================
--- zodbupdate/branches/adamg-faster/src/zodbupdate/serialize.py	2010-09-28 10:59:27 UTC (rev 117001)
+++ zodbupdate/branches/adamg-faster/src/zodbupdate/serialize.py	2010-09-28 11:05:17 UTC (rev 117002)
@@ -193,7 +193,9 @@
 
         unpickler = self.__unpickler(input_file)
         class_meta = unpickler.load()
-        data = unpickler.load()
+        if hasattr(unpickler, 'need_repickle'):
+            #repickle depends on the unpickle process
+            data = unpickler.load()
 
         class_meta = self.__update_class_meta(class_meta)
 
@@ -206,6 +208,9 @@
         pickler = self.__pickler(output_file)
         try:
             pickler.dump(class_meta)
+            if not hasattr(unpickler, 'need_repickle'):
+                #load data as late as possible
+                data = unpickler.load()
             pickler.dump(data)
         except cPickle.PicklingError:
             # Could not pickle that record, likely due to a broken

Modified: zodbupdate/branches/adamg-faster/src/zodbupdate/update.py
===================================================================
--- zodbupdate/branches/adamg-faster/src/zodbupdate/update.py	2010-09-28 10:59:27 UTC (rev 117001)
+++ zodbupdate/branches/adamg-faster/src/zodbupdate/update.py	2010-09-28 11:05:17 UTC (rev 117002)
@@ -19,6 +19,7 @@
 import ZODB.broken
 import ZODB.utils
 import cStringIO
+import datetime
 import logging
 import pickle
 import pickletools
@@ -59,6 +60,9 @@
             self.storage.tpc_finish(t)
 
     def __call__(self):
+        startTime = datetime.datetime.now()
+        logger.debug('Started at %s' % startTime.isoformat())
+
         try:
             count = 0
             t = self.__new_transaction()
@@ -89,6 +93,9 @@
             del traceback
             raise e
 
+        logger.debug('Finished at %s' % datetime.datetime.now().isoformat())
+        logger.debug('%s seconds' % (datetime.datetime.now() - startTime).seconds)
+
     @property
     def records(self):
         next = ZODB.utils.repr_to_oid(self.start_at)