[Checkins] SVN: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py Added missing error handling in delete code.
Jim Fulton
jim at zope.com
Mon Jul 27 09:31:37 EDT 2009
Log message for revision 102348:
Added missing error handling in delete code.
Added descriptive names to bdb database files.
Changed:
U zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py
-=-
Modified: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py
===================================================================
--- zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py 2009-07-27 13:16:50 UTC (rev 102347)
+++ zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py 2009-07-27 13:31:37 UTC (rev 102348)
@@ -34,6 +34,7 @@
import ZODB.blob
import ZODB.config
import ZODB.FileStorage
+import ZODB.POSException
import ZODB.TimeStamp
def p64(v):
@@ -45,6 +46,7 @@
return struct.unpack(">q", v)[0]
logger = logging.getLogger(__name__)
+log_format = "%(asctime)s %(name)s %(levelname)s: %(message)s"
def gc(conf, days=1, ignore=(), conf2=None, batch_size=10000):
close = []
@@ -76,7 +78,7 @@
good = oidset(databases)
bad = Bad(databases)
- deleted = oidset(databases)
+ deleted = Deleted(databases)
for name, storage in storages:
logger.info("%s: roots", name)
@@ -151,9 +153,6 @@
db.close()
close.pop()
- good.close()
- deleted.close()
-
# Now, we have the garbage in bad. Remove it.
for name, db in sorted(db1.databases.iteritems()):
logger.info("%s: remove garbage", name)
@@ -162,8 +161,11 @@
storage.tpc_begin(t)
nd = 0
for oid in bad.iterator(name):
- p, s = storage.load(oid, '')
- storage.deleteObject(oid, s, t)
+ try:
+ p, s = storage.load(oid, '')
+ storage.deleteObject(oid, s, t)
+ except ZODB.POSException.POSKeyError:
+ continue
nd += 1
if (nd % batch_size) == 0:
storage.tpc_vote(t)
@@ -203,19 +205,21 @@
class oidset:
+ type_ = 'good'
+
def __init__(self, names):
self._dbs = {}
self._paths = []
for name in names:
- fd, path = tempfile.mkstemp(dir='.')
+ fd, path = tempfile.mkstemp(
+ dir='.', prefix='db-'+name.strip()+'-', suffix=self.type_)
os.close(fd)
- self._dbs[name] = bsddb3.hashopen(path)
+ self._dbs[name] = bsddb3.hashopen(path, cachesize=1<<24)
self._paths.append(path)
def close(self):
for db in self._dbs.values():
db.close()
- self._dbs.clear()
while self._paths:
os.remove(self._paths.pop())
@@ -253,8 +257,14 @@
for oid in self._dbs[name]:
yield oid
+class Deleted(oidset):
+
+ type_ = 'deleted'
+
class Bad(oidset):
+ type_ = 'bad'
+
def insert(self, name, oid, refs):
db = self._dbs[name]
old = db.get(oid)
@@ -308,7 +318,7 @@
level = int(level)
except ValueError:
level = getattr(logging, level)
- logging.basicConfig(level=level)
+ logging.basicConfig(level=level, format=log_format)
return gc(args[0], options.days, options.ignore or (), *args[1:])
@@ -434,7 +444,7 @@
def check_command(args=None):
if args is None:
args = sys.argv[1:]
- logging.basicConfig(level=logging.WARNING)
+ logging.basicConfig(level=logging.WARNING, format=log_format)
parser = optparse.OptionParser("usage: %prog [options] config")
parser.add_option(
More information about the Checkins
mailing list