[Checkins] SVN: zc.zodbdgc/branches/dev/src/zc/zodbdgc/ Added blob file check.

Jim Fulton jim at zope.com
Tue May 26 13:59:36 EDT 2009


Log message for revision 100431:
  Added blob file check.
  
  Added check test that verifies finding dangling links.
  

Changed:
  U   zc.zodbdgc/branches/dev/src/zc/zodbdgc/README.test
  U   zc.zodbdgc/branches/dev/src/zc/zodbdgc/__init__.py

-=-
Modified: zc.zodbdgc/branches/dev/src/zc/zodbdgc/README.test
===================================================================
--- zc.zodbdgc/branches/dev/src/zc/zodbdgc/README.test	2009-05-26 17:43:23 UTC (rev 100430)
+++ zc.zodbdgc/branches/dev/src/zc/zodbdgc/README.test	2009-05-26 17:59:35 UTC (rev 100431)
@@ -26,6 +26,7 @@
     ...         pack-gc false
     ...         pack-keep-old false
     ...         path 1.fs
+    ...         blob-dir 1.blobs
     ...     </filestorage>
     ... </zodb>
     ... <zodb db2>
@@ -44,7 +45,7 @@
     ... </zodb>
     ... """)
 
-    >>> import ZODB.config, transaction
+    >>> import ZODB.blob, ZODB.config, transaction
     >>> db = ZODB.config.databaseFromFile(open('config'))
 
 And perform some updates:
@@ -72,9 +73,9 @@
     >>> transaction.commit()
 
     >>> conn1.root.x.y = conn2.root.y
-    >>> del conn2.root.y
+    >>> del conn2.root.y # conn2.root.y is now only referenced from db 1
     >>> conn1.root.x.y.z = conn3.root.z
-    >>> del conn3.root.z
+    >>> del conn3.root.z # conn3.root.z is now only referenced from db 2
 
 In db loops:
 
@@ -130,6 +131,19 @@
     >>> del conn2.root.b
     >>> transaction.commit()
 
+Let's add a blob.  We'll use it later:
+
+    >>> conn1.root.blob = ZODB.blob.Blob('some data')
+    >>> transaction.commit()
+    >>> blob_path = conn1.root.blob.committed()
+
+
+Save databases for later:
+
+    >>> import shutil
+    >>> for n in range(1, 4):
+    ...     shutil.copyfile('%s.fs' % n, '%s.fs-save' %n)
+
 More time passes.
 
     >>> now += 1
@@ -137,7 +151,7 @@
 The number of objecs in the databases now:
 
     >>> len(conn1._storage), len(conn2._storage), len(conn3._storage)
-    (2206, 5, 4)
+    (2207, 5, 4)
 
     >>> for d in db.databases.values():
     ...     d.pack()
@@ -145,7 +159,7 @@
 Packing doesn't change it:
 
     >>> len(conn1._storage), len(conn2._storage), len(conn3._storage)
-    (2206, 5, 4)
+    (2207, 5, 4)
 
     >>> _ = conn1._storage.load(p64(2))
     >>> _ = conn1._storage.load(p64(3))
@@ -160,11 +174,13 @@
     >>> import shutil
     >>> for n in range(1, 4):
     ...     shutil.copyfile('%s.fs' % n, '%s.fs-2' %n)
+    >>> shutil.copytree('1.blobs', '1.blobs-2')
+    >>> shutil.copymode('1.blobs', '1.blobs-2')
 
 Now let's perform gc.
 
     >>> import zc.zodbdgc
-    >>> bad = zc.zodbdgc.gc('config', days=2)
+    >>> bad = zc.zodbdgc.gc('config', days=2, batch_size=2)
 
     >>> for name, oid in sorted(bad.iterator()):
     ...     print name, u64(oid)
@@ -183,7 +199,7 @@
 haven't packed yet.
 
     >>> len(conn1._storage), len(conn2._storage), len(conn3._storage)
-    (2206, 5, 4)
+    (2207, 5, 4)
 
     >>> now += 1
 
@@ -191,7 +207,7 @@
     ...     d.pack()
 
     >>> len(conn1._storage), len(conn2._storage), len(conn3._storage)
-    (2204, 4, 2)
+    (2205, 4, 2)
 
     >>> import ZODB.POSException
     >>> for name, oid in bad.iterator():
@@ -218,22 +234,17 @@
     >>> open('config2', 'w').write("""
     ... <zodb db1>
     ...     <filestorage>
-    ...         pack-gc false
-    ...         pack-keep-old false
     ...         path 1.fs-2
+    ...         blob-dir 1.blobs-2
     ...     </filestorage>
     ... </zodb>
     ... <zodb db2>
     ...     <filestorage>
-    ...         pack-gc false
-    ...         pack-keep-old false
     ...         path 2.fs-2
     ...     </filestorage>
     ... </zodb>
     ... <zodb db3>
     ...     <filestorage>
-    ...         pack-gc false
-    ...         pack-keep-old false
     ...         path 3.fs-2
     ...     </filestorage>
     ... </zodb>
@@ -289,7 +300,7 @@
     ...     d.pack()
 
     >>> len(conn1._storage), len(conn2._storage), len(conn3._storage)
-    (2204, 4, 2)
+    (2205, 4, 2)
 
     >>> _ = [d.close() for d in db.databases.values()]
 
@@ -302,7 +313,7 @@
 
     >>> zc.zodbdgc.check_command(['config'])
 
-Make sure we can not specify days on the command line:
+Make sure we can omit days on the command line:
 
     >>> for n in range(1, 4):
     ...     shutil.copyfile('%s.fs' % n, '%s.fs-2' %n)
@@ -327,6 +338,43 @@
     Removed 2 objects from db2
     [('db2', 3L), ('db2', 4L)]
 
+
+Test the check command
+----------------------
+
+    If we pack the original files with gc enabled, we'll create
+    a missing link.  We'll also remove the blob file, to make sure we
+    catch that too.
+
+    >>> import os
+    >>> os.remove(blob_path.replace('1.blobs', '1.blobs-2'))
+
+    >>> for n in range(1, 4):
+    ...     shutil.copyfile('%s.fs-save' % n, '%s.fs-2' %n)
+    >>> db = ZODB.config.databaseFromFile(open('config2'))
+    Ignoring index for 1.fs-2
+    Ignoring index for 2.fs-2
+    Ignoring index for 3.fs-2
+
+    >>> for d in db.databases.values():
+    ...     d.pack()
+    >>> for d in db.databases.values():
+    ...     d.close()
+
+    >>> zc.zodbdgc.check_command(['config2'])
+    !!! db1 2206 db1 0
+    POSKeyError: 'No blob file'
+    !!! db2 2 db1 2
+    POSKeyError: 0x02
+
+Note that we see the missing link from db1 to db2.  There is not
+missing link from db1 to db3 because the referencing object was
+incorrectly removed as garbage.
+
+The check function also catches missing blob files:
+
+
+
 .. cleanup
 
     >>> logging.getLogger().setLevel(old_level)

Modified: zc.zodbdgc/branches/dev/src/zc/zodbdgc/__init__.py
===================================================================
--- zc.zodbdgc/branches/dev/src/zc/zodbdgc/__init__.py	2009-05-26 17:43:23 UTC (rev 100430)
+++ zc.zodbdgc/branches/dev/src/zc/zodbdgc/__init__.py	2009-05-26 17:59:35 UTC (rev 100431)
@@ -26,6 +26,7 @@
 import tempfile
 import time
 import transaction
+import ZODB.blob
 import ZODB.config
 import ZODB.TimeStamp
 
@@ -282,6 +283,8 @@
 
         try:
             p, tid = storages[name].load(oid, '')
+            if ZODB.blob.is_blob_record(p):
+                storages[name].loadBlob(oid, tid)
         except:
             print '!!!', name, u64(oid),
             referer = referers.pop((name, oid), None)
@@ -290,6 +293,8 @@
                 print rname, u64(roid)
             else:
                 print '?'
+            t, v = sys.exc_info()[:2]
+            print "%s: %s" % (t.__name__, v)
             continue
 
         referers.pop((name, oid), None)



More information about the Checkins mailing list