[Checkins] SVN: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py Store the last tid of bad objects so we can avoid doing a load when it
Jim Fulton
jim at zope.com
Wed Jul 29 14:45:44 EDT 2009
Log message for revision 102366:
Store the last tid of bad objects so we can avoid doing a load when it
is time to delete them.
Changed:
U zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py
-=-
Modified: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py
===================================================================
--- zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py 2009-07-29 15:23:40 UTC (rev 102365)
+++ zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py 2009-07-29 18:45:43 UTC (rev 102366)
@@ -122,8 +122,7 @@
)
good = oidset(databases)
- bad = oidset(databases)
- badrefs = BadRefs(databases)
+ bad = Bad(databases)
deleted = oidset(databases)
for name, storage in storages:
@@ -192,16 +191,12 @@
if good.insert(*ref) and bad.has(*ref):
to_do = [ref]
while to_do:
- ref = to_do.pop()
- bad.remove(*ref)
- for ref in badrefs.pop(*ref):
+ for ref in bad.pop(*to_do.pop()):
if good.insert(*ref) and bad.has(*ref):
to_do.append(ref)
else:
- bad.insert(name, oid)
- refs = set(getrefs(data, name, ignore))
- if refs:
- badrefs.insert(name, oid, refs)
+ bad.insert(name, oid, record.tid,
+ getrefs(data, name, ignore))
else:
# deleted record
if good.has(name, oid):
@@ -210,8 +205,6 @@
bad.remove(name, oid)
deleted.insert(name, oid)
- badrefs.close()
-
if conf2 is not None:
for db in db2.databases.itervalues():
db.close()
@@ -224,11 +217,11 @@
t = transaction.begin()
storage.tpc_begin(t)
nd = 0
- for oid in bad.iterator(name):
+ for oid, tid in bad.iterator(name):
try:
- p, s = storage.load(oid, '')
- storage.deleteObject(oid, s, t)
- except ZODB.POSException.POSKeyError:
+ storage.deleteObject(oid, tid, t)
+ except (ZODB.POSException.POSKeyError,
+ ZODB.POSException.ConflictError):
continue
nd += 1
if (nd % batch_size) == 0:
@@ -331,7 +324,7 @@
for suffix in data:
yield prefix+suffix
-class BadRefs:
+class Bad:
def __init__(self, names):
self._dbs = {}
@@ -366,27 +359,24 @@
for oid in self._dbs[name]:
yield name, oid
else:
- for oid in self._dbs[name]:
- yield oid
+ for oid, data in self._dbs[name].iteritems():
+ tid = marshal.loads(data)[0]
+ yield oid, tid
- def insert(self, name, oid, refs):
- if not refs:
- return
-
+ def insert(self, name, oid, tid, refs):
db = self._dbs[name]
old = db.get(oid)
if old:
- old = set(marshal.loads(old))
- refs = old.union(refs)
- if refs != old:
- db[oid] = marshal.dumps(list(refs))
- else:
- db[oid] = marshal.dumps(list(refs))
+ oldtid, oldrefs = marshal.loads(old)
+ refs = set(oldrefs).union(refs)
+ tid = max(tid, oldtid)
+ db[oid] = marshal.dumps((tid, list(refs)))
+
def pop(self, name, oid):
refs = self._dbs[name].pop(oid, ())
if refs:
- return marshal.loads(refs)
+ return marshal.loads(refs)[1]
return ()
More information about the Checkins
mailing list