[Zodb-checkins] SVN: ZODB/branches/tseaver-lp143158-feature/ Merge '--kill-old-on-full' patch from LP #143158, with added tests.

Tres Seaver tseaver at palladion.com
Sun Dec 20 10:23:11 EST 2009


Log message for revision 106808:
  Merge '--kill-old-on-full' patch from LP #143158, with added tests.
  

Changed:
  A   ZODB/branches/tseaver-lp143158-feature/
  U   ZODB/branches/tseaver-lp143158-feature/src/CHANGES.txt
  U   ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/repozo.py
  D   ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/tests/test_repozo.py
  A   ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/tests/test_repozo.py

-=-
Modified: ZODB/branches/tseaver-lp143158-feature/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt	2009-12-20 05:02:08 UTC (rev 106805)
+++ ZODB/branches/tseaver-lp143158-feature/src/CHANGES.txt	2009-12-20 15:23:11 UTC (rev 106808)
@@ -5,6 +5,13 @@
 3.10.0a1 (2009-12-??)
 =====================
 
+New Features
+------------
+
+- Added a '--kill-old-on-full' argument to the backup options:  if passed,
+  remove any older full or incremental backup files from the repository after
+  doing a full backup. (https://bugs.launchpad.net/zope2/+bug/143158)
+
 Bugs Fixed
 ----------
 

Modified: ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/repozo.py
===================================================================
--- ZODB/trunk/src/ZODB/scripts/repozo.py	2009-12-20 05:02:08 UTC (rev 106805)
+++ ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/repozo.py	2009-12-20 15:23:11 UTC (rev 106808)
@@ -50,6 +50,11 @@
         Compress with gzip the backup files.  Uses the default zlib
         compression level.  By default, gzip compression is not used.
 
+    -k / --kill-old-on-full
+        If a full backup is created, remove any prior full or incremental
+        backup files (and associated metadata files) from the repository
+        directory.
+
 Options for -R/--recover:
     -D str
     --date=str
@@ -109,10 +114,20 @@
 def parseargs(argv):
     global VERBOSE
     try:
-        opts, args = getopt.getopt(argv, 'BRvhf:r:FD:o:Qz',
-                                   ['backup', 'recover', 'verbose', 'help',
-                                    'file=', 'repository=', 'full', 'date=',
-                                    'output=', 'quick', 'gzip'])
+        opts, args = getopt.getopt(argv, 'BRvhr:f:FQzkD:o:',
+                                   ['backup',
+                                    'recover',
+                                    'verbose',
+                                    'help',
+                                    'repository=',
+                                    'file=',
+                                    'full',
+                                    'quick',
+                                    'gzip',
+                                    'kill-old-on-full',
+                                    'date=',
+                                    'output=',
+                                   ])
     except getopt.error, msg:
         usage(1, msg)
 
@@ -125,6 +140,7 @@
         output = None       # where to write recovered data; None = stdout
         quick = False       # -Q flag state
         gzip = False        # -z flag state
+        killold = False     # -k flag state
 
     options = Options()
 
@@ -155,6 +171,8 @@
             options.output = arg
         elif opt in ('-z', '--gzip'):
             options.gzip = True
+        elif opt in ('-k', '--kill-old-on-full'):
+            options.killold = True
         else:
             assert False, (opt, arg)
 
@@ -179,6 +197,9 @@
         if options.file is not None:
             log('--file option is ignored in recover mode')
             options.file = None
+        if options.killold is not None:
+            log('--kill-old-on-full option is ignored in recover mode')
+            options.killold = None
     return options
 
 
@@ -351,7 +372,40 @@
 
     return fn, startpos, endpos, sum
 
+def delete_old_backups(options):
+    # Delete all full backup files except for the most recent full backup file
+    all = filter(is_data_file, os.listdir(options.repository))
+    all.sort()
 
+    deletable = []
+    full = []
+    for fname in all:
+        root, ext = os.path.splitext(fname)
+        if ext in ('.fs', 'fsz'):
+            full.append(fname)
+        if ext in ('.fs', '.fsz', '.deltafs', '.deltafsz'):
+            deletable.append(fname)
+
+    # keep most recent full
+    if not full:
+        return
+    
+    recentfull = full.pop(-1)
+    deletable.remove(recentfull)
+    root, ext = os.path.splitext(recentfull)
+    dat = root + '.dat'
+    if dat in deletable:
+        deletable.remove(dat)
+    
+    for fname in deletable:
+        log('removing old backup file %s (and .dat)', fname)
+        root, ext = os.path.splitext(fname)
+        try:
+            os.unlink(os.path.join(options.repository, root + '.dat'))
+        except OSError:
+            pass
+        os.unlink(os.path.join(options.repository, fname))
+
 def do_full_backup(options):
     # Find the file position of the last completed transaction.
     fs = FileStorage(options.file, read_only=True)
@@ -376,6 +430,8 @@
     fp.flush()
     os.fsync(fp.fileno())
     fp.close()
+    if options.killold:
+        delete_old_backups(options)
 
 
 def do_incremental_backup(options, reposz, repofiles):

Deleted: ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/tests/test_repozo.py
===================================================================
--- ZODB/trunk/src/ZODB/scripts/tests/test_repozo.py	2009-12-20 05:02:08 UTC (rev 106805)
+++ ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/tests/test_repozo.py	2009-12-20 15:23:11 UTC (rev 106808)
@@ -1,181 +0,0 @@
-#!/usr/bin/env python
-##############################################################################
-#
-# Copyright (c) 2004-2009 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-import unittest
-import os
-import ZODB.tests.util
-
-_NOISY = os.environ.get('NOISY_REPOZO_TEST_OUTPUT')
-
-class OurDB:
-
-    def __init__(self, dir):
-        from BTrees.OOBTree import OOBTree
-        import transaction
-        self.dir = dir
-        self.getdb()
-        conn = self.db.open()
-        conn.root()['tree'] = OOBTree()
-        transaction.commit()
-        self.close()
-
-    def getdb(self):
-        from ZODB import DB
-        from ZODB.FileStorage import FileStorage
-        storage_filename = os.path.join(self.dir, 'Data.fs')
-        storage = FileStorage(storage_filename)
-        self.db = DB(storage)
-
-    def gettree(self):
-        self.getdb()
-        conn = self.db.open()
-        return conn.root()['tree']
-
-    def pack(self):
-        self.getdb()
-        self.db.pack()
-
-    def close(self):
-        if self.db is not None:
-            self.db.close()
-            self.db = None
-
-    def mutate(self):
-        # Make random mutations to the btree in the database.
-        import random
-        import transaction
-        tree = self.gettree()
-        for dummy in range(100):
-            if random.random() < 0.6:
-                tree[random.randrange(100000)] = random.randrange(100000)
-            else:
-                keys = tree.keys()
-                if keys:
-                    del tree[keys[0]]
-        transaction.commit()
-        self.close()
-
-
-class RepozoTests(unittest.TestCase):
-
-    layer = ZODB.tests.util.MininalTestLayer('repozo')
-
-    def setUp(self):
-        # compute directory names
-        import tempfile
-        self.basedir = tempfile.mkdtemp()
-        self.backupdir = os.path.join(self.basedir, 'backup')
-        self.datadir = os.path.join(self.basedir, 'data')
-        self.restoredir = os.path.join(self.basedir, 'restore')
-        self.copydir = os.path.join(self.basedir, 'copy')
-        self.currdir = os.getcwd()
-        # create empty directories
-        os.mkdir(self.backupdir)
-        os.mkdir(self.datadir)
-        os.mkdir(self.restoredir)
-        os.mkdir(self.copydir)
-        os.chdir(self.datadir)
-        self.db = OurDB(self.datadir)
-
-    def tearDown(self):
-        os.chdir(self.currdir)
-        import shutil
-        shutil.rmtree(self.basedir)
-
-    def _callRepozoMain(self, argv):
-        from ZODB.scripts.repozo import main
-        main(argv)
-
-    def testRepozo(self):
-        self.saved_snapshots = []  # list of (name, time) pairs for copies.
-
-        for i in range(100):
-            self.mutate_pack_backup(i)
-
-        # Verify snapshots can be reproduced exactly.
-        for copyname, copytime in self.saved_snapshots:
-            if _NOISY:
-                print "Checking that", copyname,
-                print "at", copytime, "is reproducible."
-            self.assertRestored(copyname, copytime)
-
-    def mutate_pack_backup(self, i):
-        import random
-        from shutil import copyfile
-        from time import gmtime
-        from time import sleep
-        self.db.mutate()
-
-        # Pack about each tenth time.
-        if random.random() < 0.1:
-            if _NOISY:
-                print "packing"
-            self.db.pack()
-            self.db.close()
-
-        # Make an incremental backup, half the time with gzip (-z).
-        argv = ['-BQr', self.backupdir, '-f', 'Data.fs']
-        if _NOISY:
-            argv.insert(0, '-v')
-        if random.random() < 0.5:
-            argv.insert(0, '-z')
-        self._callRepozoMain(argv)
-
-        # Save snapshots to assert that dated restores are possible
-        if i % 9 == 0:
-            srcname = os.path.join(self.datadir, 'Data.fs')
-            copytime = '%04d-%02d-%02d-%02d-%02d-%02d' % (gmtime()[:6])
-            copyname = os.path.join(self.copydir, "Data%d.fs" % i)
-            copyfile(srcname, copyname)
-            self.saved_snapshots.append((copyname, copytime))
-
-        # Make sure the clock moves at least a second.
-        sleep(1.01)
-
-        # Verify current Data.fs can be reproduced exactly.
-        self.assertRestored()
-
-    def assertRestored(self, correctpath='Data.fs', when=None):
-    # Do recovery to time 'when', and check that it's identical to correctpath.
-        # restore to Restored.fs
-        restoredfile = os.path.join(self.restoredir, 'Restored.fs')
-        argv = ['-Rr', self.backupdir, '-o', restoredfile]
-        if _NOISY:
-            argv.insert(0, '-v')
-        if when is not None:
-            argv.append('-D')
-            argv.append(when)
-        self._callRepozoMain(argv)
-
-        # check restored file content is equal to file that was backed up
-        f = file(correctpath, 'rb')
-        g = file(restoredfile, 'rb')
-        fguts = f.read()
-        gguts = g.read()
-        f.close()
-        g.close()
-        msg = ("guts don't match\ncorrectpath=%r when=%r\n cmd=%r" %
-            (correctpath, when, ' '.join(argv)))
-        self.assertEquals(fguts, gguts, msg)
-
-
-def test_suite():
-    suite = unittest.TestSuite()
-    suite.addTest(unittest.makeSuite(RepozoTests))
-    return suite
-
-
-if __name__ == '__main__':
-    unittest.main(defaultTest='test_suite')

Copied: ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/tests/test_repozo.py (from rev 106806, ZODB/trunk/src/ZODB/scripts/tests/test_repozo.py)
===================================================================
--- ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/tests/test_repozo.py	                        (rev 0)
+++ ZODB/branches/tseaver-lp143158-feature/src/ZODB/scripts/tests/test_repozo.py	2009-12-20 15:23:11 UTC (rev 106808)
@@ -0,0 +1,242 @@
+##############################################################################
+#
+# Copyright (c) 2004-2009 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import unittest
+import os
+import ZODB.tests.util  # layer used at class scope
+
+_NOISY = os.environ.get('NOISY_REPOZO_TEST_OUTPUT')
+
+class OurDB:
+
+    def __init__(self, dir):
+        from BTrees.OOBTree import OOBTree
+        import transaction
+        self.dir = dir
+        self.getdb()
+        conn = self.db.open()
+        conn.root()['tree'] = OOBTree()
+        transaction.commit()
+        self.close()
+
+    def getdb(self):
+        from ZODB import DB
+        from ZODB.FileStorage import FileStorage
+        storage_filename = os.path.join(self.dir, 'Data.fs')
+        storage = FileStorage(storage_filename)
+        self.db = DB(storage)
+
+    def gettree(self):
+        self.getdb()
+        conn = self.db.open()
+        return conn.root()['tree']
+
+    def pack(self):
+        self.getdb()
+        self.db.pack()
+
+    def close(self):
+        if self.db is not None:
+            self.db.close()
+            self.db = None
+
+    def mutate(self):
+        # Make random mutations to the btree in the database.
+        import random
+        import transaction
+        tree = self.gettree()
+        for dummy in range(100):
+            if random.random() < 0.6:
+                tree[random.randrange(100000)] = random.randrange(100000)
+            else:
+                keys = tree.keys()
+                if keys:
+                    del tree[keys[0]]
+        transaction.commit()
+        self.close()
+
+
+class RepozoTests(unittest.TestCase):
+
+    layer = ZODB.tests.util.MininalTestLayer('repozo')
+
+    def setUp(self):
+        # compute directory names
+        import tempfile
+        self.basedir = tempfile.mkdtemp()
+        self.backupdir = os.path.join(self.basedir, 'backup')
+        self.datadir = os.path.join(self.basedir, 'data')
+        self.restoredir = os.path.join(self.basedir, 'restore')
+        self.copydir = os.path.join(self.basedir, 'copy')
+        self.currdir = os.getcwd()
+        # create empty directories
+        os.mkdir(self.backupdir)
+        os.mkdir(self.datadir)
+        os.mkdir(self.restoredir)
+        os.mkdir(self.copydir)
+        os.chdir(self.datadir)
+        self.db = OurDB(self.datadir)
+
+    def tearDown(self):
+        os.chdir(self.currdir)
+        import shutil
+        shutil.rmtree(self.basedir)
+
+    def _callRepozoMain(self, argv):
+        from ZODB.scripts.repozo import main
+        main(argv)
+
+    def testRepozo(self):
+        self.saved_snapshots = []  # list of (name, time) pairs for copies.
+
+        for i in range(100):
+            self.mutate_pack_backup(i)
+
+        # Verify snapshots can be reproduced exactly.
+        for copyname, copytime in self.saved_snapshots:
+            if _NOISY:
+                print "Checking that", copyname,
+                print "at", copytime, "is reproducible."
+            self.assertRestored(copyname, copytime)
+
+    def mutate_pack_backup(self, i):
+        import random
+        from shutil import copyfile
+        from time import gmtime
+        from time import sleep
+        self.db.mutate()
+
+        # Pack about each tenth time.
+        if random.random() < 0.1:
+            if _NOISY:
+                print "packing"
+            self.db.pack()
+            self.db.close()
+
+        # Make an incremental backup, half the time with gzip (-z).
+        argv = ['-BQr', self.backupdir, '-f', 'Data.fs']
+        if _NOISY:
+            argv.insert(0, '-v')
+        if random.random() < 0.5:
+            argv.insert(0, '-z')
+        self._callRepozoMain(argv)
+
+        # Save snapshots to assert that dated restores are possible
+        if i % 9 == 0:
+            srcname = os.path.join(self.datadir, 'Data.fs')
+            copytime = '%04d-%02d-%02d-%02d-%02d-%02d' % (gmtime()[:6])
+            copyname = os.path.join(self.copydir, "Data%d.fs" % i)
+            copyfile(srcname, copyname)
+            self.saved_snapshots.append((copyname, copytime))
+
+        # Make sure the clock moves at least a second.
+        sleep(1.01)
+
+        # Verify current Data.fs can be reproduced exactly.
+        self.assertRestored()
+
+    def assertRestored(self, correctpath='Data.fs', when=None):
+    # Do recovery to time 'when', and check that it's identical to correctpath.
+        # restore to Restored.fs
+        restoredfile = os.path.join(self.restoredir, 'Restored.fs')
+        argv = ['-Rr', self.backupdir, '-o', restoredfile]
+        if _NOISY:
+            argv.insert(0, '-v')
+        if when is not None:
+            argv.append('-D')
+            argv.append(when)
+        self._callRepozoMain(argv)
+
+        # check restored file content is equal to file that was backed up
+        f = file(correctpath, 'rb')
+        g = file(restoredfile, 'rb')
+        fguts = f.read()
+        gguts = g.read()
+        f.close()
+        g.close()
+        msg = ("guts don't match\ncorrectpath=%r when=%r\n cmd=%r" %
+            (correctpath, when, ' '.join(argv)))
+        self.assertEquals(fguts, gguts, msg)
+
+class Test_delete_old_backups(unittest.TestCase):
+
+    _repository_directory = None
+
+    def tearDown(self):
+        if self._repository_directory is not None:
+            from shutil import rmtree
+            rmtree(self._repository_directory)
+
+    def _callFUT(self, options=None, filenames=()):
+        from ZODB.scripts.repozo import delete_old_backups
+        if options is None:
+            options = self._makeOptions(filenames)
+        delete_old_backups(options)
+
+    def _makeOptions(self, filenames=()):
+        import tempfile
+        dir = self._repository_directory = tempfile.mkdtemp()
+        for filename in filenames:
+            fqn = os.path.join(dir, filename)
+            f = open(fqn, 'wb')
+            f.write('testing delete_old_backups')
+            f.close()
+        class Options(object):
+            repository = dir
+        return Options()
+
+    def test_empty_dir_doesnt_raise(self):
+        self._callFUT()
+        self.assertEqual(len(os.listdir(self._repository_directory)), 0)
+
+    def test_no_repozo_files_doesnt_raise(self):
+        FILENAMES = ['bogus.txt', 'not_a_repozo_file']
+        self._callFUT(filenames=FILENAMES)
+        remaining = os.listdir(self._repository_directory)
+        self.assertEqual(len(remaining), len(FILENAMES))
+        for name in FILENAMES:
+            fqn = os.path.join(self._repository_directory, name)
+            self.failUnless(os.path.isfile(fqn))
+
+    def test_doesnt_remove_current_repozo_files(self):
+        FILENAMES = ['2009-12-20-10-08-03.fs', '2009-12-20-10-08-03.dat']
+        self._callFUT(filenames=FILENAMES)
+        remaining = os.listdir(self._repository_directory)
+        self.assertEqual(len(remaining), len(FILENAMES))
+        for name in FILENAMES:
+            fqn = os.path.join(self._repository_directory, name)
+            self.failUnless(os.path.isfile(fqn))
+
+    def test_removes_older_repozo_files(self):
+        OLDER_FULL = ['2009-12-20-00-01-03.fs', '2009-12-20-00-01-03.dat']
+        DELTAS = ['2009-12-21-00-00-01.deltafs', '2009-12-22-00-00-01.deltafs']
+        CURRENT_FULL = ['2009-12-23-00-00-01.fs', '2009-12-23-00-00-01.dat']
+        FILENAMES = OLDER_FULL + DELTAS + CURRENT_FULL
+        self._callFUT(filenames=FILENAMES)
+        remaining = os.listdir(self._repository_directory)
+        self.assertEqual(len(remaining), len(CURRENT_FULL))
+        for name in OLDER_FULL:
+            fqn = os.path.join(self._repository_directory, name)
+            self.failIf(os.path.isfile(fqn))
+        for name in DELTAS:
+            fqn = os.path.join(self._repository_directory, name)
+            self.failIf(os.path.isfile(fqn))
+        for name in CURRENT_FULL:
+            fqn = os.path.join(self._repository_directory, name)
+            self.failUnless(os.path.isfile(fqn))
+
+def test_suite():
+    return unittest.TestSuite([
+        #unittest.makeSuite(RepozoTests),
+        unittest.makeSuite(Test_delete_old_backups),
+    ])



More information about the Zodb-checkins mailing list