[Checkins] SVN: zc.zodbdgc/trunk/src/zc/zodbdgc/ Deleting 10000 objects at a time was found to put too much stress on a

[Jim Fulton]

Log message for revision 103637:
  Deleting 10000 objects at a time was found to put too much stress on a
  heavily loaded storage server.
  
  - Add a sleep or allow the storahe to rest after a set of deletions.
    Sleep for twice the time taken to perform the deletions.
  
  - Adjust the deletion batch size to take about .5 seconds per
    batch of deletions, but do at least 10 at a time.
  

Changed:
  U   zc.zodbdgc/trunk/src/zc/zodbdgc/README.test
  U   zc.zodbdgc/trunk/src/zc/zodbdgc/__init__.py

-=-
Modified: zc.zodbdgc/trunk/src/zc/zodbdgc/README.test
===================================================================
--- zc.zodbdgc/trunk/src/zc/zodbdgc/README.test	2009-09-08 10:11:10 UTC (rev 103636)
+++ zc.zodbdgc/trunk/src/zc/zodbdgc/README.test	2009-09-08 10:31:39 UTC (rev 103637)
@@ -192,7 +192,7 @@
 Now let's perform gc.
 
     >>> import zc.zodbdgc
-    >>> bad = zc.zodbdgc.gc('config', days=2, batch_size=2)
+    >>> bad = zc.zodbdgc.gc('config', days=2)
 
     >>> for name, oid in sorted(bad.iterator()):
     ...     print name, u64(oid)

Modified: zc.zodbdgc/trunk/src/zc/zodbdgc/__init__.py
===================================================================
--- zc.zodbdgc/trunk/src/zc/zodbdgc/__init__.py	2009-09-08 10:11:10 UTC (rev 103636)
+++ zc.zodbdgc/trunk/src/zc/zodbdgc/__init__.py	2009-09-08 10:31:39 UTC (rev 103637)
@@ -89,16 +89,16 @@
               fs=dict(o.split('=') for o in options.fs or ()))
 
 
-def gc(conf, days=1, ignore=(), conf2=None, batch_size=10000, fs=()):
+def gc(conf, days=1, ignore=(), conf2=None, fs=()):
     close = []
     try:
-        return gc_(close, conf, days, ignore, conf2, batch_size, fs)
+        return gc_(close, conf, days, ignore, conf2, fs)
     finally:
         for db in close:
             for db in db.databases.itervalues():
                 db.close()
 
-def gc_(close, conf, days, ignore, conf2, batch_size, fs):
+def gc_(close, conf, days, ignore, conf2, fs):
     db1 = ZODB.config.databaseFromFile(open(conf))
     close.append(db1)
     if conf2 is None:
@@ -207,12 +207,14 @@
         close.pop()
 
     # Now, we have the garbage in bad.  Remove it.
+    batch_size = 100
     for name, db in sorted(db1.databases.iteritems()):
         logger.info("%s: remove garbage", name)
         storage = db.storage
         t = transaction.begin()
         storage.tpc_begin(t)
         nd = 0
+        start = time.time()
         for oid, tid in bad.iterator(name):
             try:
                 storage.deleteObject(oid, tid, t)
@@ -225,6 +227,9 @@
                 storage.tpc_finish(t)
                 t.commit()
                 logger.info("%s: deleted %s", name, nd)
+                duration = time.time()-start
+                time.sleep(duration*2)
+                batch_size = max(10, int(batch_size*.5/duration))
                 t = transaction.begin()
                 storage.tpc_begin(t)