[Checkins] SVN: Sandbox/J1m/zookeeperrecipes/ Recipe to set up zookeeper trees for development

Jim Fulton jim at zope.com
Fri Jan 27 21:57:17 UTC 2012


Log message for revision 124224:
  Recipe to set up zookeeper trees for development

Changed:
  D   Sandbox/J1m/zookeeperrecipes/README.txt
  A   Sandbox/J1m/zookeeperrecipes/README.txt
  U   Sandbox/J1m/zookeeperrecipes/buildout.cfg
  U   Sandbox/J1m/zookeeperrecipes/setup.py
  A   Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/
  A   Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt
  A   Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py
  A   Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py

-=-
Deleted: Sandbox/J1m/zookeeperrecipes/README.txt
===================================================================
--- Sandbox/J1m/zookeeperrecipes/README.txt	2012-01-27 21:53:11 UTC (rev 124223)
+++ Sandbox/J1m/zookeeperrecipes/README.txt	2012-01-27 21:57:17 UTC (rev 124224)
@@ -1,14 +0,0 @@
-Title Here
-**********
-
-
-To learn more, see
-
-
-Changes
-*******
-
-0.1.0 (yyyy-mm-dd)
-==================
-
-Initial release

Added: Sandbox/J1m/zookeeperrecipes/README.txt
===================================================================
--- Sandbox/J1m/zookeeperrecipes/README.txt	                        (rev 0)
+++ Sandbox/J1m/zookeeperrecipes/README.txt	2012-01-27 21:57:17 UTC (rev 124224)
@@ -0,0 +1 @@
+link ./src/zc/zookeeperrecipes/README.txt
\ No newline at end of file


Property changes on: Sandbox/J1m/zookeeperrecipes/README.txt
___________________________________________________________________
Added: svn:special
   + *

Modified: Sandbox/J1m/zookeeperrecipes/buildout.cfg
===================================================================
--- Sandbox/J1m/zookeeperrecipes/buildout.cfg	2012-01-27 21:53:11 UTC (rev 124223)
+++ Sandbox/J1m/zookeeperrecipes/buildout.cfg	2012-01-27 21:57:17 UTC (rev 124224)
@@ -4,7 +4,7 @@
 
 [test]
 recipe = zc.recipe.testrunner
-eggs = 
+eggs = zc.zookeeperrecipes [test]
 
 [py]
 recipe = zc.recipe.egg

Modified: Sandbox/J1m/zookeeperrecipes/setup.py
===================================================================
--- Sandbox/J1m/zookeeperrecipes/setup.py	2012-01-27 21:53:11 UTC (rev 124223)
+++ Sandbox/J1m/zookeeperrecipes/setup.py	2012-01-27 21:57:17 UTC (rev 124224)
@@ -11,12 +11,16 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-name, version = 'zc.', '0'
+name, version = 'zc.zookeeperrecipes', '0'
 
-install_requires = ['setuptools']
-extras_require = dict(test=['zope.testing'])
+install_requires = ['setuptools', 'zc.zk [static]']
+extras_require = dict(test=[
+    'zope.testing', 'zc.buildout', 'zc.zk [test]'
+    ])
 
 entry_points = """
+[zc.buildout]
+devtree = zc.zookeeperrecipes:DevTree
 """
 
 from setuptools import setup

Added: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt
===================================================================
--- Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt	                        (rev 0)
+++ Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt	2012-01-27 21:57:17 UTC (rev 124224)
@@ -0,0 +1,395 @@
+=================
+ZooKeeper Recipes
+=================
+
+devtree
+=======
+
+The devtree recipe sets up temporary ZooKeeper tree for a buildout::
+
+  [myproject]
+  recipe = zc.zookeeperrecipes:devtree
+  import-file = tree.txt
+
+.. -> conf
+
+    *** Basics, default path,  ***
+
+    >>> def write(name, text):
+    ...     with open(name, 'w') as f: f.write(text)
+
+    >>> write('tree.txt', """
+    ... x=1
+    ... type = 'foo'
+    ... /a
+    ...    /b
+    ... /c
+    ... """)
+
+    >>> import ConfigParser, StringIO, os
+    >>> from zc.zookeeperrecipes import DevTree
+
+    >>> here = os.getcwd()
+    >>> buildoutbuildout = {
+    ...     'directory': here,
+    ...     'parts-directory': os.path.join(here, 'parts'),
+    ...     }
+
+    >>> def buildout():
+    ...     parser = ConfigParser.RawConfigParser()
+    ...     parser.readfp(StringIO.StringIO(conf))
+    ...     buildout = dict((name, dict(parser.items(name)))
+    ...                     for name in parser.sections())
+    ...     [name] = buildout.keys()
+    ...     buildout['buildout'] = buildoutbuildout
+    ...     options = buildout[name]
+    ...     recipe = DevTree(buildout, name, options)
+    ...     return recipe, options
+
+
+    >>> import zc.zookeeperrecipes, mock
+    >>> with mock.patch('zc.zookeeperrecipes.timestamp') as ts:
+    ...     ts.return_value = '2012-01-26T14:50:14.864772'
+    ...     recipe, options = buildout()
+
+
+    >>> from pprint import pprint
+    >>> pprint(options)
+    {'effective-path': '/myproject2012-01-26T14:50:14.864772',
+     'import-file': 'tree.txt',
+     'import-text': "\nx=1\ntype = 'foo'\n/a\n   /b\n/c\n",
+     'location': '/testdirectory/parts/myproject',
+     'path': '/myproject',
+     'recipe': 'zc.zookeeperrecipes:devtree'}
+
+    >>> recipe.install()
+    ()
+
+    >>> def cat(*path):
+    ...     with open(os.path.join(*path)) as f:
+    ...          return f.read()
+
+    >>> cat('parts', 'myproject')
+    '/myproject2012-01-26T14:50:14.864772'
+
+  *** Test node name is persistent ***
+
+  Updating doesn't change the name:
+
+    >>> recipe, options = buildout()
+    >>> recipe.update()
+    ()
+    >>> options['effective-path'] == '/myproject2012-01-26T14:50:14.864772'
+    True
+    >>> cat('parts', 'myproject')
+    '/myproject2012-01-26T14:50:14.864772'
+
+    >>> import zc.zk
+    >>> zk = zc.zk.ZooKeeper('127.0.0.1:2181')
+    >>> zk.print_tree()
+    /myproject2012-01-26T14:50:14.864772 : foo
+      buildout:location = u'/testdirectory/parts/myproject'
+      x = 1
+      /a
+        /b
+      /c
+
+  *** Test updating tree source ***
+
+  If there are changes, we see them
+
+    >>> write('tree.txt', """
+    ... /a
+    ...   /d
+    ... /c
+    ... """)
+
+    >>> buildout()[0].install()
+    ()
+    >>> zk.print_tree()
+    /myproject2012-01-26T14:50:14.864772
+      buildout:location = u'/testdirectory/parts/myproject'
+      /a
+        /d
+      /c
+
+  Now, if there are ephemeral nodes:
+
+    >>> with mock.patch('os.getpid') as getpid:
+    ...     getpid.return_value = 42
+    ...     zk.register_server('/myproject2012-01-26T14:50:14.864772/a/d',
+    ...                        'x:y')
+
+    >>> write('tree.txt', """
+    ... /a
+    ...   /b
+    ... /c
+    ... """)
+
+    >>> buildout()[0].install() # doctest: +NORMALIZE_WHITESPACE
+    Not deleting /myproject2012-01-26T14:50:14.864772/a/d/x:y
+     because it's ephemeral.
+    /myproject2012-01-26T14:50:14.864772/a/d
+     not deleted due to ephemeral descendent.
+    ()
+
+    >>> zk.print_tree()
+    /myproject2012-01-26T14:50:14.864772
+      buildout:location = u'/testdirectory/parts/myproject'
+      /a
+        /b
+        /d
+          /x:y
+            pid = 42
+      /c
+
+  The ephemeral node, and the node containing it is left, but a
+  warning is issued.
+
+  *** Cleanup w different part name ***
+
+  Now, let's change out buildout to use a different part name:
+
+    >>> conf = """
+    ... [myproj]
+    ... recipe = zc.zookeeperrecipes:devtree
+    ... import-file = tree.txt
+    ... """
+
+    >>> os.remove(os.path.join('parts', 'myproject'))
+
+  Now, when we rerun the buildout, the old tree will get cleaned up:
+
+    >>> import signal
+    >>> with mock.patch('os.kill') as kill:
+    ...     with mock.patch('zc.zookeeperrecipes.timestamp') as ts:
+    ...         ts.return_value = '2012-01-26T14:50:15.864772'
+    ...         recipe, options = buildout()
+    ...     recipe.install()
+    ...     kill.assert_called_with(42, signal.SIGTERM)
+    ()
+
+    >>> pprint(options)
+    {'effective-path': '/myproj2012-01-26T14:50:15.864772',
+     'import-file': 'tree.txt',
+     'import-text': '\n/a\n  /b\n/c\n',
+     'location': '/testdirectory/parts/myproj',
+     'path': '/myproj',
+     'recipe': 'zc.zookeeperrecipes:devtree'}
+
+    >>> with mock.patch('os.getpid') as getpid:
+    ...     getpid.return_value = 42
+    ...     zk.register_server('/myproj2012-01-26T14:50:15.864772/a/b',
+    ...                        'x:y')
+
+    >>> zk.print_tree()
+    /myproj2012-01-26T14:50:15.864772
+      buildout:location = u'/testdirectory/parts/myproj'
+      /a
+        /b
+          /x:y
+            pid = 42
+      /c
+
+
+  *** Cleanup w different path and explicit path, and creation of base nodes ***
+
+    >>> conf = """
+    ... [myproj]
+    ... recipe = zc.zookeeperrecipes:devtree
+    ... import-file = tree.txt
+    ... path = /ztest/path
+    ... """
+
+    >>> with mock.patch('os.kill') as kill:
+    ...     with mock.patch('zc.zookeeperrecipes.timestamp') as ts:
+    ...         ts.return_value = '2012-01-26T14:50:16.864772'
+    ...         recipe, options = buildout()
+    ...     recipe.install()
+    ...     kill.assert_called_with(42, signal.SIGTERM)
+    ()
+
+    >>> pprint(options)
+    {'effective-path': '/ztest/path2012-01-26T14:50:16.864772',
+     'import-file': 'tree.txt',
+     'import-text': '\n/a\n  /b\n/c\n',
+     'location': '/tmp/tmpZ3mohq/testdirectory/parts/myproj',
+     'path': '/ztest/path',
+     'recipe': 'zc.zookeeperrecipes:devtree'}
+
+    >>> with mock.patch('os.getpid') as getpid:
+    ...     getpid.return_value = 42
+    ...     zk.register_server('/ztest/path2012-01-26T14:50:16.864772/a/b',
+    ...                        'x:y')
+
+    >>> zk.print_tree()
+    /ztest
+      /path2012-01-26T14:50:16.864772
+        buildout:location = u'/tmp/tmpZ3mohq/testdirectory/parts/myproj'
+        /a
+          /b
+            /x:y
+              pid = 42
+        /c
+
+  *** explicit effective-path ***
+
+  We can control the effective-path directly:
+
+    >>> conf = """
+    ... [myproj]
+    ... recipe = zc.zookeeperrecipes:devtree
+    ... effective-path = /my/path
+    ... import-file = tree.txt
+    ... """
+
+  This time, we'll also check
+  that kill fail handlers are handled properly.
+
+    >>> with mock.patch('os.kill') as kill:
+    ...     def noway(pid, sig):
+    ...         raise OSError
+    ...     kill.side_effect = noway
+    ...     recipe, options = buildout()
+    ...     recipe.install()
+    ...     kill.assert_called_with(42, signal.SIGTERM)
+    ()
+
+    >>> pprint(options)
+    {'effective-path': '/my/path',
+     'import-file': 'tree.txt',
+     'import-text': '\n/a\n  /b\n/c\n',
+     'location': '/testdirectory/parts/myproj',
+     'recipe': 'zc.zookeeperrecipes:devtree'}
+
+    >>> zk.print_tree() # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+    /my
+      /path
+        buildout:location = u'/tmp/tmpiKIi2U/testdirectory/parts/myproj'
+        /a
+          /b
+        /c
+    /ztest
+
+    >>> zk.close()
+
+  *** Non-local zookeeper no cleanup no/explicit import text ***
+
+    **Note** because of the way zookeeper testing works, there's
+    really only one zookeeper "server", so even though we're using a
+    different connection string, we get the same db.
+
+    >>> conf = """
+    ... [myproj]
+    ... recipe = zc.zookeeperrecipes:devtree
+    ... effective-path = /my/path
+    ... zookeeper = zookeeper.example.com:2181
+    ... """
+
+    >>> recipe, options = buildout()
+    >>> recipe.install()
+    ()
+
+    >>> pprint(options)
+    {'effective-path': '/my/path',
+     'location': '/tmp/tmpUAkJkK/testdirectory/parts/myproj',
+     'recipe': 'zc.zookeeperrecipes:devtree',
+     'zookeeper': 'zookeeper.example.com:2181'}
+
+    >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+    >>> with mock.patch('os.getpid') as getpid:
+    ...     getpid.return_value = 42
+    ...     zk.register_server('/my/path', 'a:b')
+    >>> zk.print_tree()
+    /my
+      /path
+        buildout:location = u'/tmp/tmp2Qp4qX/testdirectory/parts/myproj'
+        /a:b
+          pid = 42
+    /ztest
+
+    >>> conf = """
+    ... [myproj]
+    ... recipe = zc.zookeeperrecipes:devtree
+    ... import-text = /a
+    ... zookeeper = zookeeper.example.com:2181
+    ... path =
+    ... """
+
+
+    >>> with mock.patch('os.kill') as kill:
+    ...     def noway(pid, sig):
+    ...         print 'wtf killed'
+    ...     kill.side_effect = noway
+    ...     with mock.patch('zc.zookeeperrecipes.timestamp') as ts:
+    ...         ts.return_value = '2012-01-26T14:50:24.864772'
+    ...         recipe, options = buildout()
+    ...     recipe.install()
+    ()
+
+    >>> zk.print_tree()
+    /2012-01-26T14:50:24.864772
+      buildout:location = u'/tmp/tmpxh1XPP/testdirectory/parts/myproj'
+      /a
+    /my
+      /path
+        buildout:location = u'/tmp/tmpxh1XPP/testdirectory/parts/myproj'
+        /a:b
+          pid = 42
+    /ztest
+
+
+In this example, we're creating a ZooKeeper tree at the path
+``/myprojectYYYY-MM-DDTHH:MM:SS.SSSSSS`` with data imported from the
+buildout-local file ``tree.txt``, where YYYY-MM-DDTHH:MM:SS.SSSSSS is
+the ISO date-time when the node was created.
+
+The ``tree`` recipe options are:
+
+zookeeper
+   Optional ZooKeeper connection string.
+
+   It defaults to '127.0.0.1:2181'.
+
+path
+   Optional path at which to create the tree.
+
+   If not provided, the part name is used, with a leading ``/`` added.
+
+   When a ``devtree`` part is installed, a path is created at a path
+   derived from the given (or implied) path by adding the current date
+   and time to the path in ISO date-time format
+   (YYYY-MM-DDTHH:MM:SS.SSSSSS).  The derived path is stored a file in
+   the buildout parts directory with a name equal to the section name.
+
+effective-path
+   Optional path to be used as is.
+
+   This option is normally computed by the recipe and is queryable
+   from other recipes, but it may also be set explicitly.
+
+import-file
+   Optional import file.
+
+   This is the name of a file containing tree-definition text. See the
+   ``zc.zk`` documentation for information on the format of this file.
+
+import-text
+   Optional import text.
+
+   Unfortunately, because of the way buildout parsers configuration
+   files, leading whitespace is stripped, making this option hard to
+   specify.
+
+Cleanup
+-------
+
+We don't want trees to accumulate indefinately.  When using a local
+zookeeper (default), when the recipe is run, the entire tree is
+scanned looking for nodes that have ``buildout:location`` properties
+with paths that no-longer exist in the local file system paths that
+contain different ZooKeeper paths.
+
+If such nodes are found, then the nodes are removed and, if the nodes
+had any ephemeral subnodes with pids, those pids are sent a SIGTERM
+signal.


Property changes on: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/README.txt
___________________________________________________________________
Added: svn:eol-style
   + native

Added: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py
===================================================================
--- Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py	                        (rev 0)
+++ Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py	2012-01-27 21:57:17 UTC (rev 124224)
@@ -0,0 +1,127 @@
+##############################################################################
+#
+# Copyright (c) 2011 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import datetime
+import logging
+import os
+import re
+import signal
+import textwrap
+import zc.buildout
+import zc.zk
+
+logger = logging.getLogger(__name__)
+
+def timestamp():
+    return datetime.datetime.now().isoformat()
+
+timestamped = re.compile(r'(.+)\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+$').match
+
+class DevTree:
+
+    def __init__(self, buildout, name, options):
+        self.name, self.options = name, options
+        if 'import-file' in options:
+            options['import-text'] = open(os.path.join(
+                buildout['buildout']['directory'],
+                options['import-file'])).read()
+
+        options['location'] = location = os.path.join(
+            buildout['buildout']['parts-directory'], name)
+
+        if 'effective-path' not in options:
+            path = options.get('path', name)
+            if not path.startswith('/'):
+                path = '/' + path
+            options['path'] = path
+
+            if os.path.exists(location):
+                with open(location) as f:
+                    epath = f.read()
+                    m = timestamped(epath)
+                    if m and m.group(1) == path:
+                        options['effective-path'] = epath
+                    else:
+                        options['effective-path'] = path + timestamp()
+            else:
+                options['effective-path'] = path + timestamp()
+
+        if not options.get('clean', 'auto') in ('auto', 'yes', 'no'):
+            raise zc.buildout.UserError(
+                'clean must be one of "auto", "yes", or "no"')
+
+
+    def install(self):
+        options = self.options
+        connection = options.get('zookeeper', '127.0.0.1:2181')
+        zk = zc.zk.ZooKeeper(connection)
+        location = options['location']
+        path = options['effective-path']
+        base, name = path.rsplit('/', 1)
+        if base:
+            zk.create_recursive(base, '', zc.zk.OPEN_ACL_UNSAFE)
+        zk.import_tree(
+            '/'+name+'\n  buildout:location = %r\n  ' % location +
+            textwrap.dedent(
+                self.options.get('import-text', '')
+                ).replace('\n', '\n  '),
+            base, trim=True)
+
+        with open(location, 'w') as f:
+            f.write(path)
+
+        clean = options.get('clean', 'auto')
+        if (clean == 'yes' or
+            (clean == 'auto' and (
+                connection.startswith('localhost:')
+                or
+                connection.startswith('127.')
+                )
+             )
+            ):
+            self.clean(zk)
+
+        return ()
+
+    update = install
+
+    def clean(self, zk):
+        for name in zk.get_children('/'):
+            if name == 'zookeeper':
+                continue
+            self._clean(zk, '/'+name)
+
+    def _clean(self, zk, path):
+        location = zk.get_properties(path).get('buildout:location')
+        if location is not None:
+            if not os.path.exists(location) or readfile(location) != path:
+                pids = []
+                for spath in zk.walk(path):
+                    if spath != path:
+                        if zk.is_ephemeral(spath):
+                            pid = zk.get_properties(spath).get('pid')
+                            if pid:
+                                pids.append(pid)
+                zk.delete_recursive(path, force=True)
+                for pid in pids:
+                    try:
+                        os.kill(pid, signal.SIGTERM)
+                    except OSError:
+                        logger.warn('Failed to kill')
+        else:
+            for name in zk.get_children(path):
+                self._clean(zk, path + '/' + name)
+
+def readfile(path):
+    with open(path) as f:
+        return f.read()


Property changes on: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/__init__.py
___________________________________________________________________
Added: svn:keywords
   + Id
Added: svn:eol-style
   + native

Added: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py
===================================================================
--- Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py	                        (rev 0)
+++ Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py	2012-01-27 21:57:17 UTC (rev 124224)
@@ -0,0 +1,52 @@
+##############################################################################
+#
+# Copyright (c) 2010 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import manuel.capture
+import manuel.doctest
+import manuel.testing
+import mock
+import os
+import re
+import unittest
+import zc.zk.testing
+import zope.testing.setupstack
+import zope.testing.renormalizing
+
+def setUp(test):
+    zope.testing.setupstack.setUpDirectory(test)
+    os.mkdir('testdirectory')
+    os.chdir('testdirectory')
+    os.mkdir('parts')
+    zc.zk.testing.setUp(test, tree='/zookeeper\n  buildout:path="/xxxxxxxxx"')
+    test.globs['ZooKeeper'].connection_strings.add('127.0.0.1:2181')
+    test.globs['ZooKeeper'].connection_strings.add('localhost:2181')
+
+
+def tearDown(test):
+    zc.zk.testing.tearDown(test)
+    zope.testing.setupstack.tearDown(test)
+
+def test_suite():
+    checker = zope.testing.renormalizing.RENormalizing([
+        (re.compile(r'(/\w+)+/testdirectory/'), '/testdirectory/'),
+        # (re.compile(r''), ''),
+        # (re.compile(r''), ''),
+        ])
+    return unittest.TestSuite((
+        manuel.testing.TestSuite(
+            manuel.doctest.Manuel(checker=checker) + manuel.capture.Manuel(),
+            'README.txt',
+            setUp=setUp, tearDown=tearDown,
+            ),
+        ))
+


Property changes on: Sandbox/J1m/zookeeperrecipes/src/zc/zookeeperrecipes/tests.py
___________________________________________________________________
Added: svn:keywords
   + Id
Added: svn:eol-style
   + native



More information about the checkins mailing list