[Zodb-checkins] SVN: ZODB/branches/3.9/src/ Merged the test_repozo branch, which converted the old basic manual

Jim Fulton jim at zope.com
Thu Dec 3 14:50:05 EST 2009


Log message for revision 106199:
  Merged the test_repozo branch, which converted the old basic manual
  test into an automated test, and included Chris Wither's repozo fix
  (as monified by Godefroid Chapelle) to avoid a deprecation warning.
  

Changed:
  U   ZODB/branches/3.9/src/CHANGES.txt
  D   ZODB/branches/3.9/src/ZODB/scripts/fstail.txt
  D   ZODB/branches/3.9/src/ZODB/scripts/manual_tests/testrepozo.py
  D   ZODB/branches/3.9/src/ZODB/scripts/referrers.txt
  U   ZODB/branches/3.9/src/ZODB/scripts/repozo.py
  A   ZODB/branches/3.9/src/ZODB/scripts/tests/
  U   ZODB/branches/3.9/src/ZODB/scripts/tests/test_repozo.py
  D   ZODB/branches/3.9/src/ZODB/scripts/tests.py

-=-
Modified: ZODB/branches/3.9/src/CHANGES.txt
===================================================================
--- ZODB/branches/3.9/src/CHANGES.txt	2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/CHANGES.txt	2009-12-03 19:50:05 UTC (rev 106199)
@@ -16,6 +16,9 @@
 - Internal ZEO errors were logged at the INFO level, rather
   than at the error level.
 
+- The FileStorage backup and restore script, repozo, gave a
+  deprecation warning under Python 2.6.
+
 3.9.3 (2009-10-23)
 ==================
 

Deleted: ZODB/branches/3.9/src/ZODB/scripts/fstail.txt
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/fstail.txt	2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/fstail.txt	2009-12-03 19:50:05 UTC (rev 106199)
@@ -1,40 +0,0 @@
-====================
-The `fstail` utility
-====================
-
-The `fstail` utility shows information for a FileStorage about the last `n`
-transactions:
-
-We have to prepare a FileStorage first:
-
-  >>> from ZODB.FileStorage import FileStorage
-  >>> from ZODB.DB import DB
-  >>> import transaction
-  >>> from tempfile import mktemp
-  >>> storagefile = mktemp()
-  >>> base_storage = FileStorage(storagefile)
-  >>> database = DB(base_storage)
-  >>> connection1 = database.open()
-  >>> root = connection1.root()
-  >>> root['foo'] = 1
-  >>> transaction.commit()
-
-Now lets have a look at the last transactions of this FileStorage:
-
-  >>> from ZODB.scripts.fstail import main
-  >>> main(storagefile, 5)
-  2007-11-10 15:18:48.543001: hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3
-  user='' description='' length=132 offset=185
-  <BLANKLINE>
-  2007-11-10 15:18:48.543001: hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3
-  user='' description='initial database creation' length=150 offset=52
-  <BLANKLINE>
-
-Now clean up the storage again:
-
-  >>> import os
-  >>> base_storage.close()
-  >>> os.unlink(storagefile)
-  >>> os.unlink(storagefile+'.index')
-  >>> os.unlink(storagefile+'.lock')
-  >>> os.unlink(storagefile+'.tmp')

Deleted: ZODB/branches/3.9/src/ZODB/scripts/manual_tests/testrepozo.py
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/manual_tests/testrepozo.py	2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/manual_tests/testrepozo.py	2009-12-03 19:50:05 UTC (rev 106199)
@@ -1,164 +0,0 @@
-#!/usr/bin/env python
-##############################################################################
-#
-# Copyright (c) 2004 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-"""Test repozo.py.
-
-This is a by-hand test.  It succeeds iff it doesn't blow up.  Run it with
-its home directory as the current directory.  It will destroy all files
-matching Data.* and Copy.* in this directory, and anything in a
-subdirectory of name 'backup'.
-
-Usage:
-
-python testrepozo.py [repozo_script]
-
-  repozo_script, if provided, is a path to a script that runs repozo,
-  such as that generated by buildout.
-
-eg:
-$ ../../../../bin/py testrepozo.py ../../../../bin/repozo
-"""
-
-import os
-import random
-import time
-import glob
-import sys
-import shutil
-
-import ZODB
-from ZODB import FileStorage
-import transaction
-
-def cleanup():
-    for fname in glob.glob('Data.*') + glob.glob('Copy.*'):
-        os.remove(fname)
-
-    if os.path.isdir('backup'):
-        for fname in os.listdir('backup'):
-            os.remove(os.path.join('backup', fname))
-        os.rmdir('backup')
-
-class OurDB:
-    def __init__(self):
-        from BTrees.OOBTree import OOBTree
-        self.getdb()
-        conn = self.db.open()
-        conn.root()['tree'] = OOBTree()
-        transaction.commit()
-        self.close()
-
-    def getdb(self):
-        storage = FileStorage.FileStorage('Data.fs')
-        self.db = ZODB.DB(storage)
-
-    def gettree(self):
-        self.getdb()
-        conn = self.db.open()
-        return conn.root()['tree']
-
-    def pack(self):
-        self.getdb()
-        self.db.pack()
-
-    def close(self):
-        if self.db is not None:
-            self.db.close()
-            self.db = None
-
-# Do recovery to time 'when', and check that it's identical to correctpath.
-def check(correctpath='Data.fs', when=None):
-    if when is None:
-        extra = ''
-    else:
-        extra = ' -D ' + when
-    cmd = PYTHON + REPOZO + ' -vRr backup -o Copy.fs' + extra
-    os.system(cmd)
-    f = file(correctpath, 'rb')
-    g = file('Copy.fs', 'rb')
-    fguts = f.read()
-    gguts = g.read()
-    f.close()
-    g.close()
-    if fguts != gguts:
-        raise ValueError("guts don't match\n"
-                         "    correctpath=%r when=%r\n"
-                         "    cmd=%r" % (correctpath, when, cmd))
-
-def mutatedb(db):
-    # Make random mutations to the btree in the database.
-    tree = db.gettree()
-    for dummy in range(100):
-        if random.random() < 0.6:
-            tree[random.randrange(100000)] = random.randrange(100000)
-        else:
-            keys = tree.keys()
-            if keys:
-                del tree[keys[0]]
-    transaction.commit()
-    db.close()
-
-def main():
-    cleanup()
-    os.mkdir('backup')
-    d = OurDB()
-    # Every 9th time thru the loop, we save a full copy of Data.fs,
-    # and at the end we ensure we can reproduce those too.
-    saved_snapshots = []  # list of (name, time) pairs for copies.
-
-    for i in range(100):
-        # Make some mutations.
-        mutatedb(d)
-
-        # Pack about each tenth time.
-        if random.random() < 0.1:
-            print "packing"
-            d.pack()
-            d.close()
-
-        # Make an incremental backup, half the time with gzip (-z).
-        if random.random() < 0.5:
-            os.system(PYTHON + REPOZO + ' -vBQr backup -f Data.fs')
-        else:
-            os.system(PYTHON + REPOZO + ' -zvBQr backup -f Data.fs')
-
-        if i % 9 == 0:
-            copytime = '%04d-%02d-%02d-%02d-%02d-%02d' % (time.gmtime()[:6])
-            copyname = os.path.join('backup', "Data%d" % i) + '.fs'
-            shutil.copyfile('Data.fs', copyname)
-            saved_snapshots.append((copyname, copytime))
-
-        # Make sure the clock moves at least a second.
-        time.sleep(1.01)
-
-        # Verify current Data.fs can be reproduced exactly.
-        check()
-
-    # Verify snapshots can be reproduced exactly.
-    for copyname, copytime in saved_snapshots:
-        print "Checking that", copyname, "at", copytime, "is reproducible."
-        check(copyname, copytime)
-
-    # Tear it all down.
-    cleanup()
-    print 'Test passed!'
-
-if __name__ == '__main__':
-    PYTHON = sys.executable + ' '
-    if len(sys.argv)>1:
-        REPOZO = sys.argv[1]
-    else:
-        REPOZO = '../repozo.py'
-    main()

Deleted: ZODB/branches/3.9/src/ZODB/scripts/referrers.txt
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/referrers.txt	2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/referrers.txt	2009-12-03 19:50:05 UTC (rev 106199)
@@ -1,43 +0,0 @@
-Getting Object Referrers
-========================
-
-The referrers module provides a way to get object referrers.  It
-provides a referrers method that takes an iterable storage object.  It
-returns a dictionary mapping object ids to lists of referrer object
-versions, which each version is a tuple an object id nd serial
-nummber.
-
-To see how this works, we'll create a small database:
-
-    >>> import transaction
-    >>> from persistent.mapping import PersistentMapping
-    >>> from ZODB.FileStorage import FileStorage
-    >>> from ZODB.DB import DB
-    >>> import os, tempfile
-    >>> dest = tempfile.mkdtemp()
-    >>> fs = FileStorage(os.path.join(dest, 'Data.fs'))
-    >>> db = DB(fs)
-    >>> conn = db.open()
-    >>> conn.root()['a'] = PersistentMapping()
-    >>> conn.root()['b'] = PersistentMapping()
-    >>> transaction.commit()
-    >>> roid = conn.root()._p_oid
-    >>> aoid = conn.root()['a']._p_oid
-    >>> boid = conn.root()['b']._p_oid
-    >>> s1 = conn.root()['b']._p_serial
-
-    >>> conn.root()['a']['b'] = conn.root()['b']
-    >>> transaction.commit()
-    >>> s2 = conn.root()['a']._p_serial
-
-Now we'll get the storage and compute the referrers:
-
-    >>> import ZODB.scripts.referrers
-    >>> referrers = ZODB.scripts.referrers.referrers(fs)
-
-    >>> referrers[boid] == [(roid, s1), (aoid, s2)]
-    True
-
-.. Cleanup
-
-    >>> db.close()

Modified: ZODB/branches/3.9/src/ZODB/scripts/repozo.py
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/repozo.py	2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/repozo.py	2009-12-03 19:50:05 UTC (rev 106199)
@@ -65,7 +65,12 @@
 
 import os
 import sys
-import md5
+try:
+    # the hashlib package is available from Python 2.5
+    from hashlib import md5
+except ImportError:
+    # the md5 package is deprecated in Python 2.6
+    from md5 import new as md5
 import gzip
 import time
 import errno
@@ -101,10 +106,10 @@
         print >> sys.stderr, msg % args
 
 
-def parseargs():
+def parseargs(argv):
     global VERBOSE
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'BRvhf:r:FD:o:Qz',
+        opts, args = getopt.getopt(argv, 'BRvhf:r:FD:o:Qz',
                                    ['backup', 'recover', 'verbose', 'help',
                                     'file=', 'repository=', 'full', 'date=',
                                     'output=', 'quick', 'gzip'])
@@ -210,7 +215,7 @@
 
 def checksum(fp, n):
     # Checksum the first n bytes of the specified file
-    sum = md5.new()
+    sum = md5()
     def func(data):
         sum.update(data)
     dofile(func, fp, n)
@@ -221,7 +226,7 @@
     # Copy bytes from file src, to file dst, starting at offset start, for n
     # length of bytes.  For robustness, we first write, flush and fsync
     # to a temp file, then rename the temp file at the end.
-    sum = md5.new()
+    sum = md5()
     ifp = open(options.file, 'rb')
     ifp.seek(start)
     tempname = os.path.join(os.path.dirname(dst), 'tmp.tmp')
@@ -248,7 +253,7 @@
     # Concatenate a bunch of files from the repository, output to `outfile' if
     # given.  Return the number of bytes written and the md5 checksum of the
     # bytes.
-    sum = md5.new()
+    sum = md5()
     def func(data):
         sum.update(data)
         if ofp:
@@ -504,8 +509,10 @@
     log('Recovered %s bytes, md5: %s', reposz, reposum)
 
 
-def main():
-    options = parseargs()
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+    options = parseargs(argv)
     if options.mode == BACKUP:
         do_backup(options)
     else:

Modified: ZODB/branches/3.9/src/ZODB/scripts/tests/test_repozo.py
===================================================================
--- ZODB/branches/test_repozo/src/ZODB/scripts/tests/test_repozo.py	2009-12-03 17:41:51 UTC (rev 106197)
+++ ZODB/branches/3.9/src/ZODB/scripts/tests/test_repozo.py	2009-12-03 19:50:05 UTC (rev 106199)
@@ -15,6 +15,7 @@
 
 import unittest
 import os
+import ZODB.tests.util
 
 _NOISY = os.environ.get('NOISY_REPOZO_TEST_OUTPUT')
 
@@ -67,14 +68,10 @@
         self.close()
 
 
-class BasicRepozoTests(unittest.TestCase):
-
-    def test_importability(self):
-        from ZODB.scripts import repozo
-
-
 class RepozoTests(unittest.TestCase):
 
+    layer = ZODB.tests.util.MininalTestLayer('repozo')
+
     def setUp(self):
         # compute directory names
         import tempfile
@@ -176,7 +173,6 @@
 
 def test_suite():
     suite = unittest.TestSuite()
-    suite.addTest(unittest.makeSuite(BasicRepozoTests))
     suite.addTest(unittest.makeSuite(RepozoTests))
     return suite
 

Deleted: ZODB/branches/3.9/src/ZODB/scripts/tests.py
===================================================================
--- ZODB/branches/3.9/src/ZODB/scripts/tests.py	2009-12-03 19:39:52 UTC (rev 106198)
+++ ZODB/branches/3.9/src/ZODB/scripts/tests.py	2009-12-03 19:50:05 UTC (rev 106199)
@@ -1,35 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2004 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Test harness for scripts.
-
-$Id$
-"""
-import unittest
-import re
-from zope.testing import doctest, renormalizing
-import ZODB.tests.util
-
-checker = renormalizing.RENormalizing([
-    (re.compile('[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+'),
-     '2007-11-10 15:18:48.543001'),
-    (re.compile('hash=[0-9a-f]{40}'),
-     'hash=b16422d09fabdb45d4e4325e4b42d7d6f021d3c3')])
-
-def test_suite():
-    return unittest.TestSuite((
-        doctest.DocFileSuite(
-            'referrers.txt', 'fstail.txt',
-            setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown,
-            checker=checker),
-        ))



More information about the Zodb-checkins mailing list