[Checkins] SVN: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py Added missing error handling in delete code.

[Jim Fulton]

Log message for revision 102348:
  Added missing error handling in delete code.
  
  Added descriptive names to bdb database files.
  

Changed:
  U   zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py

-=-
Modified: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py
===================================================================
--- zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py	2009-07-27 13:16:50 UTC (rev 102347)
+++ zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py	2009-07-27 13:31:37 UTC (rev 102348)
@@ -34,6 +34,7 @@
 import ZODB.blob
 import ZODB.config
 import ZODB.FileStorage
+import ZODB.POSException
 import ZODB.TimeStamp
 
 def p64(v):
@@ -45,6 +46,7 @@
     return struct.unpack(">q", v)[0]
 
 logger = logging.getLogger(__name__)
+log_format = "%(asctime)s %(name)s %(levelname)s: %(message)s"
 
 def gc(conf, days=1, ignore=(), conf2=None, batch_size=10000):
     close = []
@@ -76,7 +78,7 @@
 
     good = oidset(databases)
     bad = Bad(databases)
-    deleted = oidset(databases)
+    deleted = Deleted(databases)
 
     for name, storage in storages:
         logger.info("%s: roots", name)
@@ -151,9 +153,6 @@
             db.close()
         close.pop()
 
-    good.close()
-    deleted.close()
-
     # Now, we have the garbage in bad.  Remove it.
     for name, db in sorted(db1.databases.iteritems()):
         logger.info("%s: remove garbage", name)
@@ -162,8 +161,11 @@
         storage.tpc_begin(t)
         nd = 0
         for oid in bad.iterator(name):
-            p, s = storage.load(oid, '')
-            storage.deleteObject(oid, s, t)
+            try:
+                p, s = storage.load(oid, '')
+                storage.deleteObject(oid, s, t)
+            except ZODB.POSException.POSKeyError:
+                continue
             nd += 1
             if (nd % batch_size) == 0:
                 storage.tpc_vote(t)
@@ -203,19 +205,21 @@
 
 class oidset:
 
+    type_ = 'good'
+
     def __init__(self, names):
         self._dbs = {}
         self._paths = []
         for name in names:
-            fd, path = tempfile.mkstemp(dir='.')
+            fd, path = tempfile.mkstemp(
+                dir='.', prefix='db-'+name.strip()+'-', suffix=self.type_)
             os.close(fd)
-            self._dbs[name] = bsddb3.hashopen(path)
+            self._dbs[name] = bsddb3.hashopen(path, cachesize=1<<24)
             self._paths.append(path)
 
     def close(self):
         for db in self._dbs.values():
             db.close()
-        self._dbs.clear()
         while self._paths:
             os.remove(self._paths.pop())
 
@@ -253,8 +257,14 @@
             for oid in self._dbs[name]:
                 yield oid
 
+class Deleted(oidset):
+
+    type_ = 'deleted'
+
 class Bad(oidset):
 
+    type_ = 'bad'
+
     def insert(self, name, oid, refs):
         db = self._dbs[name]
         old = db.get(oid)
@@ -308,7 +318,7 @@
             level = int(level)
         except ValueError:
             level = getattr(logging, level)
-        logging.basicConfig(level=level)
+        logging.basicConfig(level=level, format=log_format)
 
     return gc(args[0], options.days, options.ignore or (), *args[1:])
 
@@ -434,7 +444,7 @@
 def check_command(args=None):
     if args is None:
         args = sys.argv[1:]
-        logging.basicConfig(level=logging.WARNING)
+        logging.basicConfig(level=logging.WARNING, format=log_format)
 
     parser = optparse.OptionParser("usage: %prog [options] config")
     parser.add_option(