[Checkins] SVN: zc.fsutils/branches/dev/ initial

Jim Fulton jim at zope.com
Tue Sep 25 18:40:24 EDT 2007


Log message for revision 80062:
  initial

Changed:
  _U  zc.fsutils/branches/dev/
  A   zc.fsutils/branches/dev/buildout.cfg
  A   zc.fsutils/branches/dev/setup.py
  A   zc.fsutils/branches/dev/src/
  A   zc.fsutils/branches/dev/src/zc/
  A   zc.fsutils/branches/dev/src/zc/__init__.py
  A   zc.fsutils/branches/dev/src/zc/fsutil/
  A   zc.fsutils/branches/dev/src/zc/fsutil/__init__.py
  A   zc.fsutils/branches/dev/src/zc/fsutil/references.py
  A   zc.fsutils/branches/dev/src/zc/fsutil/references.txt
  A   zc.fsutils/branches/dev/src/zc/fsutil/tests.py

-=-

Property changes on: zc.fsutils/branches/dev
___________________________________________________________________
Name: svn:ignore
   + develop-eggs
bin
parts


Added: zc.fsutils/branches/dev/buildout.cfg
===================================================================
--- zc.fsutils/branches/dev/buildout.cfg	                        (rev 0)
+++ zc.fsutils/branches/dev/buildout.cfg	2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,12 @@
+[buildout]
+parts = script
+develop = .
+
+[script]
+recipe = zc.recipe.egg
+eggs = ${test:eggs}
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = zc.fsutil
+


Property changes on: zc.fsutils/branches/dev/buildout.cfg
___________________________________________________________________
Name: svn:eol-style
   + native

Added: zc.fsutils/branches/dev/setup.py
===================================================================
--- zc.fsutils/branches/dev/setup.py	                        (rev 0)
+++ zc.fsutils/branches/dev/setup.py	2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,24 @@
+from setuptools import setup, find_packages
+
+entry_points="""
+[console_scripts]
+references = zc.fsutil.references:references_script
+
+"""
+
+setup(
+    name = "zc.fsutil",
+    description = "ZODB Database Utilities",
+    version = ".1",
+    license = "ZPL",
+    packages = find_packages('src'),
+    include_package_data = True,
+    zip_safe = False,
+    package_dir = {'':'src'},
+    namespace_packages = ['zc'],
+    entry_points = entry_points,
+    install_requires = [
+        'setuptools',
+        'ZODB3',
+        ],
+    )


Property changes on: zc.fsutils/branches/dev/setup.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Added: zc.fsutils/branches/dev/src/zc/__init__.py
===================================================================
--- zc.fsutils/branches/dev/src/zc/__init__.py	                        (rev 0)
+++ zc.fsutils/branches/dev/src/zc/__init__.py	2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)


Property changes on: zc.fsutils/branches/dev/src/zc/__init__.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Added: zc.fsutils/branches/dev/src/zc/fsutil/__init__.py
===================================================================
--- zc.fsutils/branches/dev/src/zc/fsutil/__init__.py	                        (rev 0)
+++ zc.fsutils/branches/dev/src/zc/fsutil/__init__.py	2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1 @@
+#


Property changes on: zc.fsutils/branches/dev/src/zc/fsutil/__init__.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Added: zc.fsutils/branches/dev/src/zc/fsutil/references.py
===================================================================
--- zc.fsutils/branches/dev/src/zc/fsutil/references.py	                        (rev 0)
+++ zc.fsutils/branches/dev/src/zc/fsutil/references.py	2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,103 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Analyze database references
+"""
+
+import cPickle, cStringIO, sys
+
+import ZODB.FileStorage.FileStorage
+import ZODB.utils
+
+
+def references(iterator):
+    """Create a database of database references.
+
+    Return a dictionary mapping oids to dictionaries with keys 'from'
+    and 'to' with values that are dictionaries mapping oids to lists
+    of references.
+    """
+    result = {}
+    for trans in iterator:
+        for record in trans:
+            from_oid = ZODB.utils.oid_repr(record.oid)
+            from_data = result.get(from_oid)
+            if from_data is None:
+                from_data = result[from_oid] = {
+                    'from': {}, 'to': {}, 'serials': [],
+                    }
+            from_data['serials'].append(record.tid)
+
+            refs = []
+            u = cPickle.Unpickler(cStringIO.StringIO(record.data))
+            u.persistent_load = refs
+            u.noload()
+            u.noload()
+            for ref in refs:
+                if isinstance(ref, tuple):
+                    to_oid = ZODB.utils.oid_repr(ref[0])
+                elif isinstance(ref, str):
+                    to_oid = ZODB.utils.oid_repr(ref)
+                elif isinstance(ref, list):
+                    if len(ref) == 1:
+                        to_oid = ZODB.utils.oid_repr(ref[0])
+                    else:
+                        try:
+                            reference_type, args = ref
+                        except ValueError:
+                            print 'wtf', ref
+                            continue
+
+                        if reference_type == 'w':
+                            to_oid = ZODB.utils.oid_repr(args[0])
+                        elif reference_type in 'nm':
+                            to_oid = args[0], ZODB.utils.oid_repr(args[1])
+                        else:
+                            print wtf, reference_type, args
+                else:
+                    print 'wtf', ref
+                    continue
+
+                ref = dict(ref=ref, tid=trans.tid, tpos=trans._tpos)
+                
+                from_to = from_data['to'].get(to_oid)
+                if from_to is None:
+                    from_to = from_data['to'][to_oid] = []
+                from_to.append(ref)
+
+                to_data = result.get(to_oid)
+                if to_data is None:
+                    to_data = result[to_oid] = {
+                        'to': {}, 'from': {}, 'serials': [],
+                        }
+                to_from = to_data['from'].get(from_oid)
+                if to_from is None:
+                    to_from = to_data['from'][from_oid] = []
+                to_from.append(ref)
+                
+    return result
+
+def references_script(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    [inp, outp] = args
+
+    iterator = sys.modules['ZODB.FileStorage.FileStorage' # :(
+                           ].FileIterator(inp)
+    data = references(iterator)
+    cPickle.Pickler(open(outp, 'w'), 1).dump(data)
+
+
+
+    


Property changes on: zc.fsutils/branches/dev/src/zc/fsutil/references.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native

Added: zc.fsutils/branches/dev/src/zc/fsutil/references.txt
===================================================================
--- zc.fsutils/branches/dev/src/zc/fsutil/references.txt	                        (rev 0)
+++ zc.fsutils/branches/dev/src/zc/fsutil/references.txt	2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,233 @@
+References analysis utility
+===========================
+
+The references module provides functions and scripts for analyzing
+database references.
+
+Let's create a couple of databases.
+
+    >>> from ZODB.FileStorage import FileStorage
+    >>> fs1 = FileStorage('fs1')
+    >>> fs2 = FileStorage('fs2')
+    >>> databases = {}
+    >>> from ZODB.DB import DB
+    >>> db1 = DB(fs1, databases=databases, database_name='db1')
+    >>> db2 = DB(fs2, databases=databases, database_name='db2')
+    >>> c1 = db1.open()
+    >>> c2 = c1.get_connection('db2')
+    >>> from ZODB.tests.util import P
+
+We create a simple chain of objects in each db.
+
+    >>> c1.root()['p'] = P('1.p')
+    >>> c1.root()['p'].p = P('1.p.p')
+    >>> c2.root()['p'] = P('2.p')
+    >>> c2.root()['p'].p = P('2.p.p')
+    >>> from transaction import commit
+    >>> commit()
+
+We create a cross database reference:
+
+    >>> c1.root()['p'].x = c2.root()['p'].p
+    >>> commit()
+
+Let's create a weak reference too:
+
+    >>> import persistent.wref
+    >>> c2.root()['w'] = persistent.wref.WeakRef(c2.root()['p'].p)
+    >>> commit()
+
+Now, let's be a bit abusive.  First, we'll delete the object we
+created the references to.  First, we'll create a Python reference to
+it.
+
+    >>> ob = c2.root()['p'].p
+    >>> del c2.root()['p'].p
+    >>> commit()
+
+Now, we'll pack  the second database:
+
+    >>> from ZODB.tests.util import pack
+    >>> pack(db2)
+
+and be evil by putting the object back. :)
+
+    >>> c2.root()['p'].p = ob
+    >>> del ob
+    >>> commit()
+
+    >>> c1.cacheMinimize()
+    >>> c2.cacheMinimize()
+
+Now, if we try to access the deleted object, we'll get a POSKeyError:
+
+    >>> c2.root()['p'].p
+    Traceback (most recent call last):
+    ...
+    POSKeyError: 0x02
+
+    >>> c1.root()['p'].x
+    Traceback (most recent call last):
+    ...
+    POSKeyError: 0x02
+
+    >>> c2.root()['w']()
+    Traceback (most recent call last):
+    ...
+    POSKeyError: 0x02
+
+OK, so we got these errors.  We know why because we created them on
+purpose. In practice, this isn't the case.  The references module can
+help us figure out what's going on.
+
+    >>> import zc.fsutil.references
+
+The references function computes a data structure from a storage
+iterator:
+
+    >>> refs1 = zc.fsutil.references.references(fs1.iterator())
+
+This data structure has information about all of the object ids that
+either had data or were references.
+
+    >>> from pprint import pprint
+    >>> pprint(refs1, width=1)
+    {'0x00': {'from': {},
+              'serials': ['\x03p\x98[\xe0C\xd73',
+                          '\x03p\x98[\xe0C\xd74'],
+              'to': {'0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x01',
+                                       None),
+                               'tid': '\x03p\x98[\xe0C\xd74',
+                               'tpos': 168L}]}},
+     '0x01': {'from': {'0x00': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x01',
+                                         None),
+                                 'tid': '\x03p\x98[\xe0C\xd74',
+                                 'tpos': 168L}]},
+              'serials': ['\x03p\x98[\xe0C\xd74',
+                          '\x03p\x98[\xe0C\xd75'],
+              'to': {'0x02': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+                                       None),
+                               'tid': '\x03p\x98[\xe0C\xd74',
+                               'tpos': 168L},
+                              {'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+                                       None),
+                               'tid': '\x03p\x98[\xe0C\xd75',
+                               'tpos': 535L}],
+                     ('db2', '0x02'): [{'ref': ['m',
+                                                ('db2',
+                                                 '\x00\x00\x00\x00\x00\x00\x00\x02',
+                                                 None)],
+                                        'tid': '\x03p\x98[\xe0C\xd75',
+                                        'tpos': 535L}]}},
+     '0x02': {'from': {'0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+                                         None),
+                                 'tid': '\x03p\x98[\xe0C\xd74',
+                                 'tpos': 168L},
+                                {'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+                                         None),
+                                 'tid': '\x03p\x98[\xe0C\xd75',
+                                 'tpos': 535L}]},
+              'serials': ['\x03p\x98[\xe0C\xd74'],
+              'to': {}},
+     ('db2', '0x02'): {'from': {'0x01': [{'ref': ['m',
+                                                  ('db2',
+                                                   '\x00\x00\x00\x00\x00\x00\x00\x02',
+                                                   None)],
+                                          'tid': '\x03p\x98[\xe0C\xd75',
+                                          'tpos': 535L}]},
+                       'serials': [],
+                       'to': {}}}
+
+There's an entry for each object referenced.  Each entry is a
+dictionary with 3 keys:
+
+from
+   Is a dictionary containing information about references to the entry's
+   object id. Each entry has as it's key, the refering object id and
+   as it's values, the list of references from that object id.  
+
+to
+   Is a dictionary containing information about references from the
+   entry's object id. Each entry has as it's key, the object id of the
+   object being referenced, and as it's values, the list of references
+   to that object id.
+
+serials
+   Is a list of serial ids (transaction ids) for the entry's object.
+
+   An entry with empty serials is a missing object. It is refered to,
+   but there are no records for it.
+
+Note that object 1 is refered to by object 0 and refers to object 2
+and to object 2 in the second database.  It has 2 database records,
+indicated by 2 values in it's serials.
+
+The references_script function is intended to be used as a setuptools
+entry point.  We'll call it directly, passing command line arguments,
+which are the name of an input file and the name of an output data file:
+
+    >>> zc.fsutil.references.references_script(['fs2', 'fs2.dat'])
+
+The data file is just a pickle file:
+
+    >>> import cPickle
+    >>> refs2 = cPickle.Unpickler(open('fs2.dat')).load()
+    >>> pprint(refs2, width=1)
+    {'0x00': {'from': {},
+              'serials': ['\x03p\x98[\xe0C\xd75'],
+              'to': {'0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x01',
+                                       None),
+                               'tid': '\x03p\x98[\xe0C\xd75',
+                               'tpos': 4L}],
+                     '0x02': [{'ref': ['w',
+                                       ('\x00\x00\x00\x00\x00\x00\x00\x02',)],
+                               'tid': '\x03p\x98[\xe0C\xd75',
+                               'tpos': 4L}]}},
+     '0x01': {'from': {'0x00': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x01',
+                                         None),
+                                 'tid': '\x03p\x98[\xe0C\xd75',
+                                 'tpos': 4L}]},
+              'serials': ['\x03p\x98[\xe0C\xd76',
+                          '\x03p\x98[\xe0C\xd77'],
+              'to': {'0x02': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+                                       None),
+                               'tid': '\x03p\x98[\xe0C\xd77',
+                               'tpos': 324L}]}},
+     '0x02': {'from': {'0x00': [{'ref': ['w',
+                                         ('\x00\x00\x00\x00\x00\x00\x00\x02',)],
+                                 'tid': '\x03p\x98[\xe0C\xd75',
+                                 'tpos': 4L}],
+                       '0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+                                         None),
+                                 'tid': '\x03p\x98[\xe0C\xd77',
+                                 'tpos': 324L}]},
+              'serials': [],
+              'to': {}}}
+
+In database 2, we see that object id is missing because it doesn't
+have any serials.
+
+We can query these data structures using Python.  For example, to dind
+missing objects (and the in-database objects that reference them:
+
+    >>> pprint([(oid, data) for (oid, data) in refs2.iteritems()
+    ...        if not data['serials']
+    ...        ], width=1)
+    [('0x02',
+      {'from': {'0x00': [{'ref': ['w',
+                                  ('\x00\x00\x00\x00\x00\x00\x00\x02',)],
+                          'tid': '\x03p\x98[\xe0C\xd75',
+                          'tpos': 4L}],
+                '0x01': [{'ref': ('\x00\x00\x00\x00\x00\x00\x00\x02',
+                                  None),
+                          'tid': '\x03p\x98[\xe0C\xd77',
+                          'tpos': 324L}]},
+       'serials': [],
+       'to': {}})]
+
+Here we see the broken weak reference from object 0 and the broken
+ordinary reference from object 1.
+
+Our data structure can't tell us about broken cross-database references
+directly.
+


Property changes on: zc.fsutils/branches/dev/src/zc/fsutil/references.txt
___________________________________________________________________
Name: svn:eol-style
   + native

Added: zc.fsutils/branches/dev/src/zc/fsutil/tests.py
===================================================================
--- zc.fsutils/branches/dev/src/zc/fsutil/tests.py	                        (rev 0)
+++ zc.fsutils/branches/dev/src/zc/fsutil/tests.py	2007-09-25 22:40:23 UTC (rev 80062)
@@ -0,0 +1,38 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import time, unittest
+from zope.testing import doctest
+import zope.testing.setupstack
+
+
+
+def setUp(test):
+    zope.testing.setupstack.setUpDirectory(test)
+    orig_time = time.time
+    def restore_time_time():
+        time.time = orig_time
+    zope.testing.setupstack.register(test, restore_time_time)
+    time.time = lambda : 1190753032.5621099
+    
+    
+
+
+def test_suite():
+    return unittest.TestSuite((
+        doctest.DocFileSuite(
+            'references.txt',
+            setUp=setUp, tearDown=zope.testing.setupstack.tearDown,
+            ),
+        ))
+


Property changes on: zc.fsutils/branches/dev/src/zc/fsutil/tests.py
___________________________________________________________________
Name: svn:keywords
   + Id
Name: svn:eol-style
   + native



More information about the Checkins mailing list