[Checkins] SVN: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/ Added the ability to do analysis against file-storage files directly.
Jim Fulton
jim at zope.com
Mon Jul 27 16:55:29 EDT 2009
Log message for revision 102353:
Added the ability to do analysis against file-storage files directly.
Changed:
U zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/README.test
U zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py
-=-
Modified: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/README.test
===================================================================
--- zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/README.test 2009-07-27 18:52:35 UTC (rev 102352)
+++ zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/README.test 2009-07-27 20:55:29 UTC (rev 102353)
@@ -291,6 +291,9 @@
-h, --help show this help message and exit
-d DAYS, --days=DAYS Number of trailing days (defaults to 1) to treat as
non-garbage
+ -f FS, --file-storage=FS
+ name=path, use the given file storage path for
+ analysis of the.named database
-i IGNORE, --ignore-database=IGNORE
Ignore references to the given database name.
-l LEVEL, --log-level=LEVEL
@@ -639,7 +642,74 @@
[('db1', '\x00\x00\x00\x00\x00\x00\x00\x02'),
('db1', '\x00\x00\x00\x00\x00\x00\x00\x03')]
+ >>> os.remove('one.fs')
+ >>> os.remove('two.fs')
+Using file-storage iterators directly
+-------------------------------------
+
+If the database under analysis is a file-storage, we can access teh
+files directly:
+
+
+ >>> open('config', 'w').write("""
+ ... <zodb db1>
+ ... <filestorage>
+ ... path one.fs
+ ... pack-gc false
+ ... </filestorage>
+ ... </zodb>
+ ... <zodb db2>
+ ... <filestorage>
+ ... path two.fs
+ ... pack-gc false
+ ... </filestorage>
+ ... </zodb>
+ ... """)
+ >>> db = ZODB.config.databaseFromFile(open('config'))
+ Ignoring index for one.fs
+ Ignoring index for two.fs
+
+ >>> conn = db.open()
+ >>> conn.get_connection('db2').root.x = C()
+ >>> transaction.commit()
+ >>> conn.root.x = C()
+ >>> conn.root.x.x = conn.get_connection('db2').root.x
+ >>> transaction.commit()
+ >>> conn.root.a = C()
+ >>> transaction.commit()
+ >>> conn.root.b = C()
+ >>> conn.root.a.b = conn.root.b
+ >>> conn.root.b.a = conn.root.a
+ >>> transaction.commit()
+ >>> del conn.root.a
+ >>> del conn.root.b
+ >>> transaction.commit()
+
+ >>> now += 2*86400
+
+ >>> db.pack()
+
+ >>> _ = [db.close() for db in db.databases.itervalues()]
+
+
+ >>> sorted(zc.zodbdgc.gc_command(['-fdb1=one.fs', '-fdb2=two.fs', 'config']
+ ... ).iterator())
+ ... # doctest: +NORMALIZE_WHITESPACE
+ db1: roots
+ db1: recent
+ db2: roots
+ db2: recent
+ db1: remove garbage
+ Removed 2 objects from db1
+ db2: remove garbage
+ Removed 0 objects from db2
+ [('db1', '\x00\x00\x00\x00\x00\x00\x00\x02'),
+ ('db1', '\x00\x00\x00\x00\x00\x00\x00\x03')]
+
+ >>> os.remove('one.fs')
+ >>> os.remove('two.fs')
+
.. cleanup
>>> logging.getLogger().setLevel(old_level)
Modified: zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py
===================================================================
--- zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py 2009-07-27 18:52:35 UTC (rev 102352)
+++ zc.zodbdgc/branches/jim-dev/src/zc/zodbdgc/__init__.py 2009-07-27 20:55:29 UTC (rev 102353)
@@ -48,16 +48,61 @@
logger = logging.getLogger(__name__)
log_format = "%(asctime)s %(name)s %(levelname)s: %(message)s"
-def gc(conf, days=1, ignore=(), conf2=None, batch_size=10000):
+def gc_command(args=None):
+ if args is None:
+ args = sys.argv[1:]
+ level = logging.WARNING
+ else:
+ level = None
+
+ parser = optparse.OptionParser("usage: %prog [options] config1 [config2]")
+ parser.add_option(
+ '-d', '--days', dest='days', type='int', default=1,
+ help='Number of trailing days (defaults to 1) to treat as non-garbage')
+ parser.add_option(
+ '-f', '--file-storage', dest='fs', action='append',
+ help='name=path, use the given file storage path for analysis of the.'
+ 'named database')
+ parser.add_option(
+ '-i', '--ignore-database', dest='ignore', action='append',
+ help='Ignore references to the given database name.')
+ parser.add_option(
+ '-l', '--log-level', dest='level',
+ help='The logging level. The default is WARNING.')
+
+ options, args = parser.parse_args(args)
+
+ if not args or len(args) > 2:
+ parser.parse_args(['-h'])
+ elif len(args) == 2:
+ conf2=args[1]
+ else:
+ conf2 = None
+
+ if options.level:
+ level = options.level
+
+ if level:
+ try:
+ level = int(level)
+ except ValueError:
+ level = getattr(logging, level)
+ logging.basicConfig(level=level, format=log_format)
+
+ return gc(args[0], options.days, options.ignore or (), conf2=conf2,
+ fs=dict(o.split('=') for o in options.fs or ()))
+
+
+def gc(conf, days=1, ignore=(), conf2=None, batch_size=10000, fs=()):
close = []
try:
- return gc_(close, conf, days, ignore, conf2, batch_size)
+ return gc_(close, conf, days, ignore, conf2, batch_size, fs)
finally:
for db in close:
for db in db.databases.itervalues():
db.close()
-def gc_(close, conf, days, ignore, conf2, batch_size):
+def gc_(close, conf, days, ignore, conf2, batch_size, fs):
db1 = ZODB.config.databaseFromFile(open(conf))
close.append(db1)
if conf2 is None:
@@ -93,7 +138,13 @@
if days:
# All non-deleted new records are good
logger.info("%s: recent", name)
- for trans in storage.iterator(ptid):
+
+ if name in fs:
+ it = ZODB.FileStorage.FileIterator(fs[name], ptid)
+ else:
+ it = storage.iterator(ptid)
+
+ for trans in it:
for record in trans:
if n and n%10000 == 0:
logger.info("%s: %s recent", name, n)
@@ -118,7 +169,12 @@
good.remove(name, oid)
# Now iterate over older records
- for trans in storage.iterator(None, ptid):
+ if name in fs:
+ it = ZODB.FileStorage.FileIterator(fs[name], None, ptid)
+ else:
+ it = storage.iterator(None, ptid)
+
+ for trans in it:
for record in trans:
if n and n%10000 == 0:
logger.info("%s: %s old", name, n)
@@ -334,43 +390,6 @@
return ()
-def gc_command(args=None):
- if args is None:
- args = sys.argv[1:]
- level = logging.WARNING
- else:
- level = None
-
- parser = optparse.OptionParser("usage: %prog [options] config1 [config2]")
- parser.add_option(
- '-d', '--days', dest='days', type='int', default=1,
- help='Number of trailing days (defaults to 1) to treat as non-garbage')
- parser.add_option(
- '-i', '--ignore-database', dest='ignore', action='append',
- help='Ignore references to the given database name.')
- parser.add_option(
- '-l', '--log-level', dest='level',
- help='The logging level. The default is WARNING.')
-
- options, args = parser.parse_args(args)
-
- if not args or len(args) > 2:
- parser.parse_args(['-h'])
-
- if options.level:
- level = options.level
-
- if level:
- try:
- level = int(level)
- except ValueError:
- level = getattr(logging, level)
- logging.basicConfig(level=level, format=log_format)
-
- return gc(args[0], options.days, options.ignore or (), *args[1:])
-
-
-
def check(config, refdb=None):
if refdb is None:
return check_(config)
More information about the Checkins
mailing list