[Checkins] SVN: relstorage/trunk/ Added the zodbpack script.

Shane Hathaway shane at hathawaymix.org
Tue Nov 17 16:49:43 EST 2009


Log message for revision 105774:
  Added the zodbpack script.
  

Changed:
  U   relstorage/trunk/CHANGES.txt
  U   relstorage/trunk/README.txt
  U   relstorage/trunk/relstorage/tests/test_zodbconvert.py
  A   relstorage/trunk/relstorage/tests/test_zodbpack.py
  A   relstorage/trunk/relstorage/zodbpack.py
  U   relstorage/trunk/setup.py

-=-
Modified: relstorage/trunk/CHANGES.txt
===================================================================
--- relstorage/trunk/CHANGES.txt	2009-11-17 20:58:47 UTC (rev 105773)
+++ relstorage/trunk/CHANGES.txt	2009-11-17 21:49:42 UTC (rev 105774)
@@ -59,6 +59,11 @@
   failover to an out-of-date asynchronous slave database. For
   simplicity, invalidate the whole ZODB cache when this happens.
 
+- Replaced the speed test script with a separately distributed package,
+  ``zodbshootout``.
+
+- Added the ``zodbpack`` script.
+
 1.3.0b1 (2009-09-04)
 --------------------
 

Modified: relstorage/trunk/README.txt
===================================================================
--- relstorage/trunk/README.txt	2009-11-17 20:58:47 UTC (rev 105773)
+++ relstorage/trunk/README.txt	2009-11-17 21:49:42 UTC (rev 105774)
@@ -589,7 +589,32 @@
         The Oracle data source name.  The Oracle client library will
         normally expect to find the DSN in /etc/oratab.
 
+The ``zodbpack`` Script
+=======================
 
+RelStorage also comes with a script named ``zodbpack`` that packs any
+ZODB storage that allows concurrent connections (including RelStorage
+and ZEO, but not including FileStorage). Use ``zodbpack`` in ``cron``
+scripts. Pass the script the name of a configuration file that lists
+the storages to pack, in ZConfig format. An example configuration file::
+
+  <relstorage>
+    pack-gc true
+    pack-duty-cycle 0.9
+    <mysql>
+      db zodb
+    </mysql>
+  </relstorage>
+
+Options for ``zodbpack``
+------------------------
+
+  ``--days`` or ``-d``
+    Specifies how many days of historical data to keep. Defaults to 0,
+    meaning no history is kept. This is meaningful even for
+    history-free storages, since unreferenced objects are not removed
+    from the database until the specified number of days have passed.
+
 Development
 ===========
 

Modified: relstorage/trunk/relstorage/tests/test_zodbconvert.py
===================================================================
--- relstorage/trunk/relstorage/tests/test_zodbconvert.py	2009-11-17 20:58:47 UTC (rev 105773)
+++ relstorage/trunk/relstorage/tests/test_zodbconvert.py	2009-11-17 21:49:42 UTC (rev 105774)
@@ -47,6 +47,8 @@
             os.remove(self.destfile)
         if os.path.exists(self.srcfile):
             os.remove(self.srcfile)
+        if os.path.exists(self.cfgfile):
+            os.remove(self.cfgfile)
 
     def test_storage_has_data(self):
         from ZODB.DB import DB

Added: relstorage/trunk/relstorage/tests/test_zodbpack.py
===================================================================
--- relstorage/trunk/relstorage/tests/test_zodbpack.py	                        (rev 0)
+++ relstorage/trunk/relstorage/tests/test_zodbpack.py	2009-11-17 21:49:42 UTC (rev 105774)
@@ -0,0 +1,107 @@
+##############################################################################
+#
+# Copyright (c) 2009 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import unittest
+
+class ZODBPackScriptTests(unittest.TestCase):
+
+    def setUp(self):
+        import os
+        import tempfile
+
+        fd, self.db_fn = tempfile.mkstemp()
+        os.close(fd)
+
+        cfg = """
+        <filestorage>
+            path %s
+        </filestorage>
+        """ % self.db_fn
+
+        fd, self.cfg_fn = tempfile.mkstemp()
+        os.write(fd, cfg)
+        os.close(fd)
+
+    def tearDown(self):
+        import os
+        if os.path.exists(self.db_fn):
+            os.remove(self.db_fn)
+        if os.path.exists(self.cfg_fn):
+            os.remove(self.cfg_fn)
+
+    def test_pack_defaults(self):
+        from ZODB.DB import DB
+        from ZODB.FileStorage import FileStorage
+        from ZODB.POSException import POSKeyError
+        import time
+        import transaction
+        from relstorage.zodbpack import main
+
+        storage = FileStorage(self.db_fn, create=True)
+        db = DB(storage)
+        conn = db.open()
+        conn.root()['x'] = 1
+        transaction.commit()
+        oid = '\0' * 8
+        state, serial = storage.load(oid, '')
+        time.sleep(0.1)
+        conn.root()['x'] = 2
+        transaction.commit()
+        conn.close()
+        self.assertEqual(state, storage.loadSerial(oid, serial))
+        db.close()
+        storage = None
+
+        main(['', self.cfg_fn])
+
+        # packing should have removed the old state.
+        storage = FileStorage(self.db_fn)
+        self.assertRaises(POSKeyError, storage.loadSerial, oid, serial)
+        storage.close()
+
+    def test_pack_with_1_day(self):
+        from ZODB.DB import DB
+        from ZODB.FileStorage import FileStorage
+        from ZODB.POSException import POSKeyError
+        import time
+        import transaction
+        from relstorage.zodbpack import main
+
+        storage = FileStorage(self.db_fn, create=True)
+        db = DB(storage)
+        conn = db.open()
+        conn.root()['x'] = 1
+        transaction.commit()
+        oid = '\0' * 8
+        state, serial = storage.load(oid, '')
+        time.sleep(0.1)
+        conn.root()['x'] = 2
+        transaction.commit()
+        conn.close()
+        self.assertEqual(state, storage.loadSerial(oid, serial))
+        db.close()
+        storage = None
+
+        main(['', '--days=1', self.cfg_fn])
+
+        # packing should not have removed the old state.
+        storage = FileStorage(self.db_fn)
+        self.assertEqual(state, storage.loadSerial(oid, serial))
+        storage.close()
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(ZODBPackScriptTests))
+    return suite

Added: relstorage/trunk/relstorage/zodbpack.py
===================================================================
--- relstorage/trunk/relstorage/zodbpack.py	                        (rev 0)
+++ relstorage/trunk/relstorage/zodbpack.py	2009-11-17 21:49:42 UTC (rev 105774)
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+##############################################################################
+#
+# Copyright (c) 2009 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""ZODB storage packing utility.
+"""
+
+from StringIO import StringIO
+import optparse
+import sys
+import time
+import ZConfig
+import ZODB.serialize
+
+schema_xml = """
+<schema>
+  <import package="ZODB"/>
+  <import package="relstorage"/>
+  <multisection type="ZODB.storage" attribute="storages" />
+</schema>
+"""
+
+def main(argv=sys.argv):
+    parser = optparse.OptionParser(description=__doc__,
+        usage="%prog [options] config_file")
+    parser.add_option(
+        "-d", "--days", dest="days", default="0",
+        help="Days of history to keep (default 0)",
+        )
+    options, args = parser.parse_args(argv[1:])
+
+    if len(args) != 1:
+        parser.error("The name of one configuration file is required.")
+
+    schema = ZConfig.loadSchemaFile(StringIO(schema_xml))
+    config, handler = ZConfig.loadConfig(schema, args[0])
+
+    t = time.time() - float(options.days) * 86400.0
+    for s in config.storages:
+        storage = s.open()
+        storage.pack(t, ZODB.serialize.referencesf)
+        storage.close()
+
+if __name__ == '__main__':
+    main()

Modified: relstorage/trunk/setup.py
===================================================================
--- relstorage/trunk/setup.py	2009-11-17 20:58:47 UTC (rev 105773)
+++ relstorage/trunk/setup.py	2009-11-17 21:49:42 UTC (rev 105774)
@@ -53,6 +53,7 @@
             },
         entry_points = {'console_scripts': [
             'zodbconvert = relstorage.zodbconvert:main',
+            'zodbpack = relstorage.zodbpack:main',
             ]},
         test_suite='relstorage.tests.alltests.make_suite',
     )



More information about the checkins mailing list