[Checkins] SVN: relstorage/trunk/ Add an --incremental option to zodbconvert, allowing continueing a copy to a pre-existing destination

[Martijn Pieters]

Log message for revision 122744:
  Add an --incremental option to zodbconvert, allowing continueing a copy to a pre-existing destination

Changed:
  U   relstorage/trunk/CHANGES.txt
  U   relstorage/trunk/relstorage/zodbconvert.py

-=-
Modified: relstorage/trunk/CHANGES.txt
===================================================================
--- relstorage/trunk/CHANGES.txt	2011-09-07 14:05:30 UTC (rev 122743)
+++ relstorage/trunk/CHANGES.txt	2011-09-07 14:06:02 UTC (rev 122744)
@@ -5,6 +5,10 @@
   debug; this stage takes mere seconds, even in large sites, but could produce
   10s of thousands of lines of log output.
 
+- Add an --incremental option to the zodbconvert script, letting you convert
+  additional transactions at a later date, or update a non-live copy of your
+  database, copying over missing transactions.
+
 1.5.0 (2011-06-30)
 ------------------
 

Modified: relstorage/trunk/relstorage/zodbconvert.py
===================================================================
--- relstorage/trunk/relstorage/zodbconvert.py	2011-09-07 14:05:30 UTC (rev 122743)
+++ relstorage/trunk/relstorage/zodbconvert.py	2011-09-07 14:06:02 UTC (rev 122744)
@@ -23,6 +23,7 @@
 from StringIO import StringIO
 import sys
 import ZConfig
+from ZODB.utils import p64, readable_tid_repr
 
 schema_xml = """
 <schema>
@@ -61,6 +62,13 @@
     parser.add_option(
         "--clear", dest="clear", action="store_true",
         help="Clear the contents of the destination storage before copying")
+    parser.add_option(
+        "--incremental", dest="incremental", action="store_true",
+        help="Assume the destination contains a partial copy of the source "
+             "and resume copying from the last transaction. WARNING: no "
+             "effort is made to verify that the destination holds the same "
+             "transaction data before this point! Use at your own risk. "
+             "Currently only supports RelStorage destinations.")
     parser.set_defaults(dry_run=False, clear=False)
     options, args = parser.parse_args(argv[1:])
 
@@ -78,6 +86,22 @@
 
     log.info("Storages opened successfully.")
 
+    if options.incremental:
+        if not hasattr(destination, '_adapter'):
+            msg = ("Error: no API is known for determining the last committed "
+                   "transaction of the destination storage. Aborting "
+                   "conversion.")
+            sys.exit(msg)
+        if not storage_has_data(destination):
+            log.warning("Destination empty, start conversion from the beginning.")
+        else:
+            last_tid = destination._adapter.txncontrol.get_tid(
+                destination._load_cursor)
+            next_tid = p64(last_tid+1)
+            source = source.iterator(start=next_tid)
+            log.info("Resuming ZODB copy from %s", readable_tid_repr(next_tid))
+
+
     if options.dry_run:
         log.info("Dry run mode: not changing the destination.")
         if storage_has_data(destination):