[ZODB-Dev] List of references to all objects from ZODB?
Greg Ward
gward@mems-exchange.org
Wed, 30 Oct 2002 09:27:23 -0500
--82I3+IH0IqGh5yIs
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
On 30 October 2002, Magnus Lycka said:
> Is there a way to get a list of all objects that ZODB
> thinks exists?
Yes. The idea is to loop over all OIDs, ignoring those that no longer
have an object attached to them (presumably pack()'ed away some time in
the past). The attached zodb_iterate.py does precisely this; you'll
have to replace the main program with something that will work for you,
but it should be fairly clear.
I'll also attach zodb_iterator.py, which is a more modern/OO
implementation of the same idea.
Greg
--
Greg Ward - software developer gward@mems-exchange.org
MEMS Exchange http://www.mems-exchange.org
--82I3+IH0IqGh5yIs
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="zodb_iterate.py"
#!/www/python/bin/python
"""mems.tools.zodb_iterate
Iterate over all OIDs in a ZODB database, calling a user-supplied
function for each object found.
As an extra added bonus, when run as a script it prints the repr() of
each object.
"""
# created 2001/08/07, GPW (based on zodb_census.py)
__revision__ = "$Id: zodb_iterate.py,v 1.2 2001/08/14 18:50:21 gward Exp $"
import sys
from types import StringType
from struct import pack, unpack
def write_status (oid, objects_seen, newline=0):
sys.stdout.write("\rOID: %016x (objects seen: %d)" %
(oid, objects_seen))
if newline:
sys.stdout.write("\n")
sys.stdout.flush()
def pack_oid (oid_i):
"""pack_oid(oid_i : long|int) -> string
Pack an integer or long int into an 8-byte OID string, eg.
0x0001020304050607L -> "\0\1\2\3\4\5\6\7".
"""
return pack(">LL",
(oid_i & 0xffffffff00000000L) >> 32,
(oid_i & 0x00000000ffffffffL))
def unpack_oid (oid_s):
"""unpack_oid(oid_s : string) -> long
Unpack an 8-byte OID string to a long int, eg.
"\0\1\2\3\4\5\6\7" -> 0x0001020304050607L
"""
(hi, lo) = unpack(">LL", oid_s) # two 32-bit integers
return (hi << 32) | lo # a single 64-bit integer
def format_oid (oid):
"""format_oid(oid : long|int|string) -> string
Format an OID (either a packed 8-byte string or an integer) to
human readable form, eg.
"\0\1\2\3\4\5\6\7" -> "0001020304050607" or
0x0001020304050607L -> "0001020304050607"
"""
if type(oid) is StringType:
oid = unpack_oid(oid)
return "%016x" % oid
def parse_oid (oid):
"""parse_oid(oid : string) -> string
Turn a human-readable OID string into a packed 8-byte string, eg.
"0001020304050607" -> "\0\1\2\3\4\5\6\7"
"""
return pack_oid(long(oid, 16))
def _print(oid_i, oid_s, object):
print "%016x: %r" % (oid_i, object)
def iterate (database, connection,
callback=_print,
status_every=0x0800,
load_objects=1):
"""
Iterate over all OIDs in a ZODB database, calling a callback
function for every object found. (With load_objects false, this
changes; see below.) database and connection must be ZODB Database
and Connection instances. callback should be a function that
takes three arguments:
callback(oid_i : long, oid_s : string, object : instance)
oid_i is a long integer representation of the 64-bit OID.
oid_s is an 8-byte packed string, the OID representation used
by ZODB internally. object is the database object associated
with the OID. callback's return value is ignored.
If status_every is non-zero and stdout is a tty, iterate() prints a
status line to stdout every status_every OIDs (default 0x800, or
2048).
If load_objects is false, objects are not loaded from the database
and the callback signature must be
callback(oid_i : long, oid_s : string) -> boolean
callback must return true if there is an object associated with
this OID, false otherwise; if this is wrong, iterate() will terminate
early or never terminate at all.
"""
if not sys.stdout.isatty(): # no status reports when not a TTY
status_every = 0
oid_i = 0L
expected_count = database.objectCount()
objects_seen = 0L # number of actual objects seen
empty_slots = 0L # number of OIDs with no object
try:
while 1:
if status_every and (oid_i % status_every) == 0:
write_status(oid_i, objects_seen)
oid_s = pack_oid(oid_i)
if load_objects:
try:
object = connection[oid_s]
except KeyError:
empty_slots += 1
else:
objects_seen += 1
callback(oid_i, oid_s, object)
else:
object_seen = callback(oid_i, oid_s)
if object_seen:
objects_seen += 1
else:
empty_slots += 1
oid_i += 1
if objects_seen >= expected_count:
break
finally:
if status_every:
write_status(oid_i, objects_seen, newline=1)
return (oid_i, objects_seen, empty_slots)
if __name__ == "__main__":
from mems.lib import base
base.init_database()
iterate(base.get_database(), base.get_connection(), status_every=None)
--82I3+IH0IqGh5yIs
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="zodb_iterator.py"
#!/www/python/bin/python
"""mems.tools.zodb_iterator
Provides ZODBIterator and MXDBIterator.
"""
# created 2001/12/03, DMB (based on zodb_iterate.py)
__revision__ = "$Id: zodb_iterator.py,v 1.5 2002/02/11 23:42:44 nascheme Exp $"
import sys
from struct import pack
def pack_oid (oid_i):
"""pack_oid(oid_i : long|int) -> string
Pack an integer or long int into an 8-byte OID string, eg.
0x0001020304050607L -> "\0\1\2\3\4\5\6\7".
"""
return pack(">LL",
(oid_i & 0xffffffff00000000L) >> 32,
(oid_i & 0x00000000ffffffffL))
class ZODBIterator:
def __init__ (self, database, connection, verbose=0, fast=0):
self.index = -1L
self.oid_i = 0L
self.database = database
self.connection = connection
self.current = None
self.verbose = verbose
self.fast = 0
def __getitem__(self, i):
while i > self.index:
self.current=self.next(i)
return self.current
def next (self, index=0):
while self.index < self.database.objectCount()-1:
oid_s = pack_oid(self.oid_i)
self.oid_i += 1
try:
self.current = self.connection[oid_s]
self.index += 1
if (self.verbose and
(self.index+1) % 1000 == 0):
sys.stdout.write("\r%10d of %d" %
(self.index + 1,
self.database.objectCount()))
sys.stdout.flush()
if not self.fast:
dir(self.current)
return self.current
except:
pass
sys.stdout.write("\r%10d of %d" %
(self.index + 1,
self.database.objectCount()))
sys.stdout.flush()
sys.stdout.write("\n")
raise IndexError, index
class MXDBIterator(ZODBIterator):
def __init__(self, **kw):
from mems.lib import base
ZODBIterator.__init__(self,
base.get_database(),
base.get_connection(), **kw)
if __name__ == "__main__":
from mems.lib import base
base.init_database('file:/www/var/mxdb.fs')
from mems.lib.physical_unit import PhysicalUnit
physical_units = [p for p in MXDBIterator() if isinstance(p, PhysicalUnit)]
print physical_units
print len(physical_units)
--82I3+IH0IqGh5yIs--