[Checkins] SVN: zc.zk/trunk/s - Added tree import and export.
Jim Fulton
jim at zope.com
Thu Dec 1 23:38:54 UTC 2011
Log message for revision 123556:
- Added tree import and export.
- Added recursive node-deletion API.
- Added symbolic-link representation.
- Added convenience access to low-level ZooKeeper APIs.
- Localized data encode and decode logic.
- Added a ZooKeeper model for testing.
- Added OPEN_ACL_UNSAFE and READ_ACL_UNSAFE, which are mentioned by
the ZooKeeper docs. but not included in the ``zookeeper`` module.
Changed:
U zc.zk/trunk/setup.py
U zc.zk/trunk/src/zc/zk/README.txt
U zc.zk/trunk/src/zc/zk/__init__.py
U zc.zk/trunk/src/zc/zk/tests.py
-=-
Modified: zc.zk/trunk/setup.py
===================================================================
--- zc.zk/trunk/setup.py 2011-11-30 11:03:35 UTC (rev 123555)
+++ zc.zk/trunk/setup.py 2011-12-01 23:38:52 UTC (rev 123556)
@@ -15,7 +15,7 @@
install_requires = ['setuptools', 'zc.thread']
extras_require = dict(
- test=['zope.testing', 'zc-zookeeper-static', 'mock'],
+ test=['zope.testing', 'zc-zookeeper-static', 'mock', 'manuel'],
static=['zc-zookeeper-static'],
)
Modified: zc.zk/trunk/src/zc/zk/README.txt
===================================================================
--- zc.zk/trunk/src/zc/zk/README.txt 2011-11-30 11:03:35 UTC (rev 123555)
+++ zc.zk/trunk/src/zc/zk/README.txt 2011-12-01 23:38:52 UTC (rev 123556)
@@ -36,25 +36,8 @@
Instantiating a ZooKeeper helper
--------------------------------
-To use the helper API, create a ZooKeeper instance:
+To use the helper API, create a ZooKeeper instance::
-.. test
-
- >>> import zookeeper
- >>> @side_effect(init)
- ... def _(addr, func):
- ... global session_watch
- ... session_watch = func
- ... func(0, zookeeper.SESSION_EVENT, zookeeper.CONNECTED_STATE, '')
- ... assert_(addr=='zookeeper.example.com:2181', addr)
-
- >>> @side_effect(state)
- ... def _(handle):
- ... assert_(handle==0)
- ... return zookeeper.CONNECTED_STATE
-
-::
-
>>> import zc.zk
>>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
@@ -66,34 +49,19 @@
--------------------------------------
To register a server, use the ``register_server`` method, which takes
-a service path and the address a server is listing on
+a service path and the address a server is listing on::
+ >>> zk.register_server('/fooservice/providers', ('192.168.0.42', 8080))
+
.. test
- >>> import os, json, zookeeper
- >>> path = '/fooservice/servers'
- >>> addrs = []
- >>> child_handler = None
- >>> @side_effect(create)
- ... def _(handle, path_, data, acl, flags):
- ... assert_(handle == 0)
- ... path_, addr = path_.rsplit('/', 1)
- ... assert_(path_ == path)
- ... assert_(json.loads(data) == dict(pid=os.getpid()))
- ... addrs.append(addr)
- ... assert_(acl == [zc.zk.world_permission()])
- ... assert_(flags == zookeeper.EPHEMERAL)
- ... global child_handler
- ... if child_handler is not None:
- ... child_handler(handle, zookeeper.CHILD_EVENT,
- ... zookeeper.CONNECTED_STATE, path_)
- ... child_handler = None
+ >>> import os
+ >>> zc.zk.decode(ZooKeeper.get(
+ ... 0, '/fooservice/providers/192.168.0.42:8080')[0]
+ ... ) == dict(pid=os.getpid())
+ True
-::
- >>> zk.register_server('/fooservice/servers', ('192.168.0.42', 8080))
-
-
``register_server`` creates a read-only ephemeral ZooKeeper node as a
child of the given service path. The name of the new node is the
given address. This allows clients to get the list of addresses by
@@ -108,40 +76,28 @@
the process id is set as the ``pid`` server key. This is useful to
tracking down the server process.
-Get the addresses of servers providing a service.
--------------------------------------------------
+Get the addresses of service providers.
+---------------------------------------
Getting the adresses providing a service is accomplished by getting the
-children of a service node.
+children of a service node::
-.. test
-
- >>> @side_effect(get_children)
- ... def _(handle, path, handler):
- ... assert_(handle == 0, handle)
- ... assert_(path == '/fooservice/servers', path)
- ... global child_handler
- ... child_handler = handler
- ... return addrs
-
-::
-
- >>> addresses = zk.children('/fooservice/servers')
+ >>> addresses = zk.children('/fooservice/providers')
>>> sorted(addresses)
['192.168.0.42:8080']
The ``children`` method returns an iterable of names of child nodes of
the node specified by the given path. The iterable is automatically
-updated when new servers are registered::
+updated when new providers are registered::
- >>> zk.register_server('/fooservice/servers', ('192.168.0.42', 8081))
+ >>> zk.register_server('/fooservice/providers', ('192.168.0.42', 8081))
>>> sorted(addresses)
['192.168.0.42:8080', '192.168.0.42:8081']
You can call the iterable with a callback function that is called
whenenever the list of children changes::
- >>> @zk.children('/fooservice/servers')
+ >>> @zk.children('/fooservice/providers')
... def addresses_updated(addresses):
... print 'addresses changed'
... print sorted(addresses)
@@ -151,7 +107,7 @@
The callback is called immediately with the children. When we add
another child, it'll be called again::
- >>> zk.register_server('/fooservice/servers', ('192.168.0.42', 8082))
+ >>> zk.register_server('/fooservice/providers', ('192.168.0.42', 8082))
addresses changed
['192.168.0.42:8080', '192.168.0.42:8081', '192.168.0.42:8082']
@@ -160,25 +116,8 @@
You get service configuration data by getting data associated with a
ZooKeeper node. The interface for getting data is similar to the
-interface for getting children:
+interface for getting children::
-
-.. test
-
- >>> node_data = json.dumps(dict(
- ... database = "/databases/foomain",
- ... threads = 1,
- ... favorite_color= "red"))
- >>> @side_effect(get)
- ... def _(handle, path, handler):
- ... assert_(handle == 0)
- ... assert_(path == '/fooservice')
- ... global get_handler
- ... get_handler = handler
- ... return node_data, {}
-
-::
-
>>> data = zk.properties('/fooservice')
>>> data['database']
u'/databases/foomain'
@@ -209,19 +148,8 @@
------------------
You can't set data properties, but you can update data by calling it's
-``update`` method:
+``update`` method::
-.. test
-
- >>> @side_effect(set)
- ... def _(handle, path, data):
- ... global node_data
- ... node_data = data
- ... get_handler(handle, zookeeper.CHANGED_EVENT,
- ... zookeeper.CONNECTED_STATE, path)
-
-::
-
>>> data.update(threads=2, secret='123')
data updated
database: u'/databases/foomain'
@@ -251,6 +179,244 @@
logging API. ZooKeeper log messages are forwarded to the Python
``'ZooKeeper'`` logger.
+Tree-definition format, import, and export
+------------------------------------------
+
+You can describe a ZooKeeper tree using a textual tree
+representation. You can then populate the tree by importing the
+representation. Heres an example::
+
+ /lb
+ /pools
+ /cms
+ # The address is fixed because it's
+ # exposed externally
+ address = '1.2.3.4:80'
+ providers -> /cms/providers
+ /retail
+ address = '1.2.3.5:80'
+ providers -> /cms/retail
+
+ /cms
+ threads = 3
+ /providers
+ /databases
+ /main
+ /providers
+
+ /retail
+ threads = 1
+ /providers
+ /databases
+ main -> /cms/databases/main
+ /ugc
+ /providers
+
+.. -> tree_text
+
+This example defines a tree with 3 top nodes, ``lb`` and ``cms``, and
+``retail``. The ``retail`` node has two subnodes, ``providers`` and
+``databases`` and a property ``threads``. The ``/retail/databases``
+node has symbolic link, ``main`` and a ``ugc`` subnode. The symbolic
+link is implemented as a property named ``main ->``. We'll say more
+about symbolic links in a later section.
+
+You can import a tree definition with the ``import_tree`` method:
+
+ >>> zk.import_tree(tree_text)
+
+This imports the tree at the top pf the ZooKeeper tree.
+
+We can also export a ZooKeeper tree:
+
+ >>> print zk.export_tree(),
+ /cms
+ threads = 3
+ /databases
+ /main
+ /providers
+ /providers
+ /fooservice
+ secret = u'1234'
+ threads = 3
+ /providers
+ /lb
+ /pools
+ /cms
+ address = u'1.2.3.4:80'
+ providers -> /cms/providers
+ /retail
+ address = u'1.2.3.5:80'
+ providers -> /cms/retail
+ /retail
+ threads = 1
+ /databases
+ main -> /cms/databases/main
+ /ugc
+ /providers
+ /providers
+
+Note that when we export a tree:
+
+- The special reserverd top-level zookeeper node is omitted.
+- Ephemeral nodes are ommitted.
+- Each node's information is sorted by type (properties, then links,
+- then subnodes) and then by name,
+
+You can export just a portion of a tree:
+
+ >>> print zk.export_tree('/fooservice'),
+ /fooservice
+ secret = u'1234'
+ threads = 3
+ /providers
+
+You can optionally see ephemeral nodes:
+
+ >>> print zk.export_tree('/fooservice', ephemeral=True),
+ /fooservice
+ secret = u'1234'
+ threads = 3
+ /providers
+ /192.168.0.42:8080
+ pid = 81176
+ /192.168.0.42:8081
+ pid = 81176
+ /192.168.0.42:8082
+ pid = 81176
+
+We can import a tree over an existing tree and changes will be
+applied. Let's update our textual description::
+
+ /lb
+ /pools
+ /cms
+ # The address is fixed because it's
+ # exposed externally
+ address = '1.2.3.4:80'
+ providers -> /cms/providers
+
+ /cms
+ threads = 4
+ /providers
+ /databases
+ /main
+ /providers
+
+.. -> tree_text
+
+and reimport::
+
+ >>> zk.import_tree(tree_text)
+ extra path not trimmed: /lb/pools/retail
+
+We got a warning about nodes left over from the old tree. We can see
+this if we export the tree:
+
+ >>> print zk.export_tree(),
+ /cms
+ threads = 4
+ /databases
+ /main
+ /providers
+ /providers
+ /fooservice
+ secret = u'1234'
+ threads = 3
+ /providers
+ /lb
+ /pools
+ /cms
+ address = u'1.2.3.4:80'
+ providers -> /cms/providers
+ /retail
+ address = u'1.2.3.5:80'
+ providers -> /cms/retail
+ /retail
+ threads = 1
+ /databases
+ main -> /cms/databases/main
+ /ugc
+ /providers
+ /providers
+
+If we want to trim these, we can add a ``trim`` option. This is a
+little scary, so we'll use the dry-run option to see what it's going
+to do::
+
+ >>> zk.import_tree(tree_text, trim=True, dry_run=True)
+ would delete /lb/pools/retail.
+
+That's what we'd expect, so we go ahead:
+
+ >>> zk.import_tree(tree_text, trim=True)
+ >>> print zk.export_tree(),
+ /cms
+ threads = 4
+ /databases
+ /main
+ /providers
+ /providers
+ /fooservice
+ secret = u'1234'
+ threads = 3
+ /providers
+ /lb
+ /pools
+ /cms
+ address = u'1.2.3.4:80'
+ providers -> /cms/providers
+ /retail
+ threads = 1
+ /databases
+ main -> /cms/databases/main
+ /ugc
+ /providers
+ /providers
+
+Note that nodes containing (directly or recursively) ephemeral nodes
+will never be trimmed. Also node that top-level nodes are never
+automatically trimmed. So we weren't warned about the unreferenced
+top-level nodes in the import.
+
+Recursice Deletion
+------------------
+
+ZooKeeper only allows deletion of nodes without children.
+The ``delete_recursive`` method automates removing a node and all of
+it's children.
+
+If we want to remove the ``retail`` top-level node, we can use
+delete_recursive::
+
+ >>> zk.delete_recursive('/retail')
+ >>> print zk.export_tree(),
+ /cms
+ threads = 4
+ /databases
+ /main
+ /providers
+ /providers
+ /fooservice
+ secret = u'1234'
+ threads = 3
+ /providers
+ /lb
+ /pools
+ /cms
+ address = u'1.2.3.4:80'
+ providers -> /cms/providers
+
+You can't delete nodes ephemeral nodes, or nodes that contain them:
+
+ >>> zk.delete_recursive('/fooservice')
+ Not deleting /fooservice/providers/192.168.0.42:8080 because it's ephemeral.
+ Not deleting /fooservice/providers/192.168.0.42:8081 because it's ephemeral.
+ Not deleting /fooservice/providers/192.168.0.42:8082 because it's ephemeral.
+ /fooservice/providers not deleted due to ephemeral descendent.
+ /fooservice not deleted due to ephemeral descendent.
+
+
``zc.zk.ZooKeeper``
-------------------
@@ -269,6 +435,46 @@
This attribute can be used to call the lower-level API provided by
the ``zookeeper`` extension.
+``import_tree(text[, path='/'[, trim[, acl[, dry_run]]]])``
+ Create tree nodes by importing a textual tree representation.
+
+ text
+ A textual representation of the tree.
+
+ path
+ The path at which to create the top-level nodes.
+
+ trim
+ Boolean, defaulting to false, indicating whether nodes not in
+ the textual representation should be removed.
+
+ acl
+ An access control-list to use for imported nodes. If not
+ specifuied, then full access is allowed to everyone.
+
+ dry_run
+ Boolean, defaulting to false, indicating whether to do a dry
+ run of the import, without applying any changes.
+
+``export_tree(path[, include_ephemeral])``
+ Export a tree to a text representation.
+
+ path
+ The path to export.
+
+ include_ephemeral
+ Boolean, defaulting to false, indicating whether to include
+ ephemeral nodes in the export. Including ephemeral nodes is
+ mainly useful for visualizing the tree state.
+
+``delete_recursive(path[, dry_run])``
+ Delete a node and all of it's subnodes.
+
+ Ephemeral nodes or nodes containing them are not deleted.
+
+ The dry_run option causes a summary of what would be deleted to be
+ printed without actually deleting anything.
+
``register_server(path, address, **data)``
Register a server at a path with the address.
@@ -285,6 +491,17 @@
This should be called when cleanly shutting down servers to more
quickly remove ephemeral nodes.
+In addition, ``ZooKeeper`` instances provide access to the following
+ZooKeeper functions as methods: ``acreate``, ``add_auth``,
+``adelete``, ``aexists``, ``aget``, ``aget_acl``, ``aget_children``,
+``aset``, ``aset_acl``, ``async``, ``client_id``, ``create``,
+``delete``, ``exists``, ``get``, ``get_acl``, ``get_children``,
+``is_unrecoverable``, ``recv_timeout``, ``set``, ``set2``,
+``set_acl``, ``set_debug_level``, ``set_log_stream``, ``set_watcher``,
+and ``zerror``. When calling these as methods on ``ZooKeeper``
+instances, it isn't necessary to pass a handle, as that is provided
+automatically.
+
zc.zk.Children
--------------
@@ -328,6 +545,17 @@
Changes
-------
+0.2.0 (2011-12-??)
+~~~~~~~~~~~~~~~~~~
+
+- Added tree import and export.
+- Added recursive node-deletion API.
+- Added symbolic-links.
+- Added convenience access to low-level ZooKeeper APIs.
+- Added ``OPEN_ACL_UNSAFE`` and ``READ_ACL_UNSAFE`` (in ``zc.zk``),
+ which are mentioned by the ZooKeeper docs. but not included in the
+ ``zookeeper`` module.
+
0.1.0 (2011-11-27)
~~~~~~~~~~~~~~~~~~
Modified: zc.zk/trunk/src/zc/zk/__init__.py
===================================================================
--- zc.zk/trunk/src/zc/zk/__init__.py 2011-11-30 11:03:35 UTC (rev 123555)
+++ zc.zk/trunk/src/zc/zk/__init__.py 2011-12-01 23:38:52 UTC (rev 123556)
@@ -1,3 +1,16 @@
+##############################################################################
+#
+# Copyright (c) Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
import collections
import json
import logging
@@ -2,2 +15,3 @@
import os
+import re
import sys
@@ -41,9 +55,53 @@
host, port = addr.split(':')
return host, int(port)
+def encode(props):
+ if len(props) == 1 and 'string_value' in props:
+ return props['string_value']
+
+ if props:
+ return json.dumps(props, separators=(',',':'))
+ else:
+ return ''
+
+def decode(sdata, path='?'):
+ s = sdata.strip()
+ if not s:
+ data = {}
+ elif s.startswith('{') and s.endswith('}'):
+ try:
+ data = json.loads(s)
+ except:
+ logger.exception('bad json data in node at %r', path)
+ data = dict(string_value = sdata)
+ else:
+ data = dict(string_value = sdata)
+ return data
+
+def join(*args):
+ return '/'.join(args)
+
def world_permission(perms=zookeeper.PERM_READ):
return dict(perms=perms, scheme='world', id='anyone')
+OPEN_ACL_UNSAFE = [world_permission(zookeeper.PERM_ALL)]
+READ_ACL_UNSAFE = [world_permission()]
+
+
+_text_is_node = re.compile(r'/(?P<name>\S+)$').match
+_text_is_property = re.compile(
+ r'(?P<name>\S+)'
+ '\s*=\s*'
+ '(?P<expr>\S.*)'
+ '$'
+ ).match
+_text_is_link = re.compile(
+ r'(?P<name>\S+)'
+ '\s*->\s*'
+ '(?P<target>/\S+)'
+ '$'
+ ).match
+
class CancelWatch(Exception):
pass
@@ -87,8 +145,10 @@
def register_server(self, path, addr, **kw):
kw['pid'] = os.getpid()
+ if not isinstance(addr, str):
+ addr = '%s:%s' % addr
self.connected.wait()
- zookeeper.create(self.handle, path + '/%s:%s' % addr, json.dumps(kw),
+ zookeeper.create(self.handle, path + '/' + addr, encode(kw),
[world_permission()], zookeeper.EPHEMERAL)
def _watch(self, watch, wait=True):
@@ -115,6 +175,183 @@
def children(self, path):
return Children(self, path)
+ def import_tree(self, text, path='/', trim=False, acl=OPEN_ACL_UNSAFE,
+ dry_run=False):
+ # Step 1, build up internal tree repesentation:
+ root = _Tree()
+ indents = [(-1, root)] # sorted [(indent, node)]
+ lineno = 0
+ for line in text.split('\n'):
+ lineno += 1
+ line = line.rstrip()
+ if not line:
+ continue
+ data = line.strip()
+ if data[0] == '#':
+ continue
+ indent = len(line) - len(data)
+
+ m = _text_is_property(data)
+ if m:
+ expr = m.group('expr')
+ try:
+ data = eval(expr, {})
+ except Exception, v:
+ raise ValueError("Error %s in expression: %r" % (v, expr))
+ data = m.group('name'), data
+ else:
+ m = _text_is_link(data)
+ if m:
+ data = (m.group('name') + ' ->'), m.group('target')
+ else:
+ m = _text_is_node(data)
+ if m:
+ data = _Tree(m.group('name'))
+ else:
+ if '->' in data:
+ raise ValueError(lineno, data, "Bad link format")
+ else:
+ raise ValueError(lineno, data, "Unrecognized data")
+
+ if indent > indents[-1][0]:
+ if not isinstance(indents[-1][1], _Tree):
+ raise ValueError(
+ lineno, line,
+ "Can't indent under properties")
+ indents.append((indent, data))
+ else:
+ while indent < indents[-1][0]:
+ indents.pop()
+
+ if indent > indents[-1][0]:
+ raise ValueError(lineno, data, "Invalid indentation")
+
+ if isinstance(data, _Tree):
+ children = indents[-2][1].children
+ if data.name in children:
+ raise ValueError(lineno, data, 'duplicate node')
+ children[data.name] = data
+ indents[-1] = indent, data
+ else:
+ if indents[-2][1] is root:
+ raise ValueError("Can't above imported nodes.")
+ properties = indents[-2][1].properties
+ name, value = data
+ if name in properties:
+ raise ValueError(lineno, data, 'duplicate property')
+ properties[name] = value
+
+ # Step 2 Create The nodes
+ while path.endswith('/'):
+ path = path[:-1] # Mainly to deal w root: /
+ self._import_tree(path, root, acl, trim, dry_run, True)
+
+ def _import_tree(self, path, node, acl, trim, dry_run, top=False):
+ self.connected.wait()
+ if not top:
+ new_children = set(node.children)
+ for name in self.get_children(path):
+ if name in new_children:
+ continue
+ cpath = join(path, name)
+ if trim:
+ self.delete_recursive(cpath, dry_run)
+ else:
+ print 'extra path not trimmed:', cpath
+
+ for name, child in node.children.iteritems():
+ cpath = path + '/' + name
+ data = encode(child.properties)
+ if self.exists(cpath):
+ if dry_run:
+ new = child.properties
+ old = decode(self.get(cpath)[0])
+ for n, v in sorted(old.items()):
+ if n not in new:
+ if n.endswith(' ->'):
+ print '%s remove link %s %s' % (cpath, n, v)
+ else:
+ print '%s remove property %s = %s' % (
+ cpath, n, v)
+ elif new[n] != v:
+ if n.endswith(' ->'):
+ print '%s %s link change from %s to %s' % (
+ cpath, n[:-3], v, new[n])
+ else:
+ print '%s %s change from %s to %s' % (
+ cpath, n, v, new[n])
+ for n, v in sorted(new.items()):
+ if n not in old:
+ if n.endswith(' ->'):
+ print '%s add link %s %s' % (cpath, n, v)
+ else:
+ print '%s add property %s = %s' % (
+ cpath, n, v)
+ else:
+ self.set(cpath, data)
+ meta, oldacl = self.get_acl(cpath)
+ if acl != oldacl:
+ self.set_acl(cpath, meta['aversion'], acl)
+ else:
+ if dry_run:
+ print 'add', cpath
+ continue
+ else:
+ self.create(cpath, data, acl)
+ self._import_tree(cpath, child, acl, trim, dry_run)
+
+ def delete_recursive(self, path, dry_run=False):
+ for name in self.get_children(path):
+ self.delete_recursive(join(path, name))
+
+ if self.get_children(path):
+ print "%s not deleted due to ephemeral descendent." % path
+ return
+
+ ephemeral = self.get(path)[1]['ephemeralOwner']
+ if dry_run:
+ if ephemeral:
+ print "wouldn't delete %s because it's ephemeral." % path
+ else:
+ print "would delete %s." % path
+ else:
+ if ephemeral:
+ print "Not deleting %s because it's ephemeral." % path
+ else:
+ logger.info('deleting %s', path)
+ self.delete(path)
+
+ def export_tree(self, path='/', ephemeral=False):
+ output = []
+ out = output.append
+
+ def export_tree(path, indent):
+ children = self.get_children(path)
+ if path == '/':
+ path = ''
+ if 'zookeeper' in children:
+ children.remove('zookeeper')
+ else:
+ data, meta = self.get(path)
+ if meta['ephemeralOwner'] and not ephemeral:
+ return
+ out(indent+'/'+path.rsplit('/', 1)[1])
+ indent += ' '
+ links = []
+ for i in sorted(decode(data).iteritems()):
+ if i[0].endswith(' ->'):
+ links.append(i)
+ else:
+ out(indent+"%s = %r" % i)
+ for i in links:
+ out(indent+"%s %s" % i)
+
+ for name in children:
+ export_tree(path+'/'+name, indent)
+
+ export_tree(path, '')
+ return '\n'.join(output)+'\n'
+
def properties(self, path):
return Properties(self, path)
@@ -122,26 +359,6 @@
self.connected.wait()
return zookeeper.set(self.handle, path, data)
- def print_tree(self, path='/', indent=0):
- self.connected.wait()
- prefix = ' '*indent
- print prefix + path.split('/')[-1]+'/'
- indent += 2
- prefix += ' '
- data = zookeeper.get(self.handle, path)[0].strip()
- if data:
- if data.startswith('{') and data.endswith('}'):
- data = json.loads(data)
- import pprint
- print prefix+pprint.pformat(data).replace(
- '\n', prefix+'\n')
- else:
- print prefix + repr(data)
- for p in zookeeper.get_children(self.handle, path):
- if not path.endswith('/'):
- p = '/'+p
- self.print_tree(path+p, indent)
-
def close(self):
zookeeper.close(self.handle)
del self.handle
@@ -153,6 +370,23 @@
return zookeeper.state(self.handle)
+def _make_method(name):
+ return (lambda self, *a, **kw:
+ getattr(zookeeper, name)(self.handle, *a, **kw))
+
+for name in (
+ 'acreate', 'add_auth', 'adelete', 'aexists', 'aget', 'aget_acl',
+ 'aget_children', 'aset', 'aset_acl', 'async', 'client_id',
+ 'create', 'delete', 'exists', 'get', 'get_acl',
+ 'get_children', 'is_unrecoverable', 'recv_timeout', 'set',
+ 'set2', 'set_acl', 'set_debug_level', 'set_log_stream',
+ 'set_watcher', 'zerror',
+ ):
+ setattr(ZooKeeper, name, _make_method(name))
+
+del _make_method
+
+
class NodeInfo:
def __init__(self, session, path):
@@ -213,20 +447,8 @@
def setData(self, data):
sdata, self.meta_data = data
- s = sdata.strip()
- if not s:
- data = {}
- elif s.startswith('{') and s.endswith('}'):
- try:
- data = json.loads(s)
- except:
- logger.exception('bad json data in node at %r', self.path)
- data = dict(string_value = sdata)
- else:
- data = dict(string_value = sdata)
+ self.data = decode(sdata, self.path)
- self.data = data
-
def __getitem__(self, key):
return self.data[key]
@@ -240,14 +462,8 @@
return self.data.copy()
def _set(self, data):
- if not data:
- sdata = ''
- elif len(data) == 1 and 'string_value' in data:
- sdata = data['string_value']
- else:
- sdata = json.dumps(data)
self.data = data
- zookeeper.set(self.session.handle, self.path, sdata)
+ zookeeper.set(self.session.handle, self.path, encode(data))
def set(self, **data):
self._set(data)
@@ -260,3 +476,13 @@
def __hash__(self):
# Gaaaa, collections.Mapping
return hash(id(self))
+
+class _Tree:
+ # Internal tree rep for import/export
+
+ def __init__(self, name='', properties=None, **children):
+ self.name = name
+ self.properties = properties or {}
+ self.children = children
+ for name, child in children.iteritems():
+ child.name = name
Modified: zc.zk/trunk/src/zc/zk/tests.py
===================================================================
--- zc.zk/trunk/src/zc/zk/tests.py 2011-11-30 11:03:35 UTC (rev 123555)
+++ zc.zk/trunk/src/zc/zk/tests.py 2011-12-01 23:38:52 UTC (rev 123556)
@@ -1,8 +1,25 @@
+##############################################################################
+#
+# Copyright (c) Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
import doctest
import json
import logging
+import manuel.capture
+import manuel.doctest
+import manuel.testing
import mock
import os
+import re
import StringIO
import time
import zc.zk
@@ -10,6 +27,7 @@
import zookeeper
import zope.testing.loggingsupport
import zope.testing.setupstack
+import zope.testing.renormalizing
import unittest
def wait_until(func, timeout=9):
@@ -28,10 +46,13 @@
f = StringIO.StringIO()
h = logging.StreamHandler(f)
logger.addHandler(h)
- logger.setLevel(logging.ERROR)
- handle = zookeeper.init('zookeeper.example.com:2181')
- wait_until(lambda : 'error' in f.getvalue())
- zookeeper.close(handle)
+ logger.setLevel(logging.DEBUG)
+ try:
+ handle = zookeeper.init('zookeeper.example.com:2181')
+ zookeeper.close(handle)
+ except:
+ pass
+ wait_until(lambda : 'environment' in f.getvalue())
logger.setLevel(logging.NOTSET)
logger.removeHandler(h)
@@ -299,7 +320,7 @@
self.assertEqual((handle, path_), (0, path))
properties.set(b=2)
- self.assertEqual(self.__set_data, '{"b": 2}')
+ self.assertEqual(self.__set_data, '{"b":2}')
properties.set()
self.assertEqual(self.__set_data, '')
properties.set(string_value='xxx')
@@ -344,6 +365,104 @@
self.assertEqual(list(children), [])
ccb.assert_called_with()
+def resilient_import():
+ """
+We can use vatious spacing in properties and links:
+
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> zk.import_tree('''
+ ... /test
+ ... a=1
+ ... b =1
+ ... c= 1
+ ... ad->/x
+ ... af ->/x
+ ... ae-> /x
+ ... ''')
+
+ >>> print zk.export_tree('/test'),
+ /test
+ a = 1
+ b = 1
+ c = 1
+ ad -> /x
+ ae -> /x
+ af -> /x
+
+When an expression is messed up, we get sane errors:
+
+ >>> zk.import_tree('''
+ ... /test
+ ... a= 1+
+ ... ''') # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ ValueError: Error unexpected EOF while parsing (<string>, line 1)
+ in expression: '1+'
+
+ >>> zk.import_tree('''
+ ... /test
+ ... a ->
+ ... ''') # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ ValueError: (3, 'a ->', 'Bad link format')
+
+ >>> zk.import_tree('''
+ ... /test
+ ... a -> 1
+ ... ''') # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ValueError: (3, 'a -> 1', 'Bad link format')
+ """
+
+def import_dry_run():
+ """
+
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> zk.import_tree('''
+ ... /test
+ ... a=1
+ ... b = 2
+ ... /c1
+ ... /c12
+ ... /c2
+ ... ae-> /x
+ ... ad-> /y
+ ... ''')
+
+ >>> zk.import_tree('''
+ ... /test
+ ... a=2
+ ... /c1
+ ... /c12
+ ... a = 1
+ ... b -> /b
+ ... /c123
+ ... ae-> /z
+ ... ''', dry_run=True)
+ /test a change from 1 to 2
+ /test remove link ad -> /y
+ /test ae link change from /x to /z
+ /test remove property b = 2
+ extra path not trimmed: /test/c2
+ /test/c1/c12 add property a = 1
+ /test/c1/c12 add link b -> /b
+ add /test/c1/c12/c123
+
+ >>> print zk.export_tree('/test'),
+ /test
+ a = 1
+ b = 2
+ ad -> /y
+ ae -> /x
+ /c1
+ /c12
+ /c2
+
+ """
+
+
def assert_(cond, mess=''):
if not cond:
print 'assertion failed: ', mess
@@ -351,17 +470,155 @@
def setup(test):
test.globs['side_effect'] = side_effect
test.globs['assert_'] = assert_
- for name in 'state', 'init', 'create', 'get', 'set', 'get_children':
+ test.globs['ZooKeeper'] = zk = ZooKeeper(
+ Node(
+ fooservice = Node(
+ json.dumps(dict(
+ database = "/databases/foomain",
+ threads = 1,
+ favorite_color= "red",
+ )),
+ providers = Node()
+ ),
+ zookeeper = Node('', quota=Node()),
+ ),
+ )
+ for name in ('state', 'init', 'create', 'get', 'set', 'get_children',
+ 'exists', 'get_acl', 'set_acl', 'delete'):
cm = mock.patch('zookeeper.'+name)
- test.globs[name] = cm.__enter__()
+ test.globs[name] = m = cm.__enter__()
+ m.side_effect = getattr(zk, name)
zope.testing.setupstack.register(test, cm.__exit__)
+class ZooKeeper:
+
+ def __init__(self, tree):
+ self.root = tree
+
+ def init(self, addr, watch=None):
+ self.handle = 0
+ assert_(addr=='zookeeper.example.com:2181', addr)
+ if watch:
+ watch(0, zookeeper.SESSION_EVENT, zookeeper.CONNECTED_STATE, '')
+
+ def state(self, handle):
+ self.check_handle(handle)
+ return zookeeper.CONNECTED_STATE
+
+ def check_handle(self, handle):
+ if handle != self.handle:
+ raise zookeeper.ZooKeeperException('handle out of range')
+
+ def traverse(self, path):
+ node = self.root
+ for name in path.split('/')[1:]:
+ if not name:
+ continue
+ try:
+ node = node.children[name]
+ except KeyError:
+ raise zookeeper.NoNodeException('no node')
+
+ return node
+
+ def create(self, handle, path, data, acl, flags=0):
+ self.check_handle(handle)
+ base, name = path.rsplit('/', 1)
+ node = self.traverse(base)
+ if name in node.children:
+ raise zookeeper.NodeExistsException()
+ node.children[name] = newnode = Node(data)
+ newnode.acls = acl
+ newnode.flags = flags
+ node.children_changed(self.handle, zookeeper.CONNECTED_STATE, base)
+ return path
+
+ def delete(self, handle, path):
+ self.check_handle(handle)
+ self.traverse(path) # seeif it's there
+ base, name = path.rsplit('/', 1)
+ node = self.traverse(base)
+ del node.children[name]
+ node.children_changed(self.handle, zookeeper.CONNECTED_STATE, base)
+
+ def exists(self, handle, path):
+ self.check_handle(handle)
+ try:
+ self.traverse(path)
+ return True
+ except zookeeper.NoNodeException:
+ return False
+
+ def get_children(self, handle, path, watch=None):
+ self.check_handle(handle)
+ node = self.traverse(path)
+ if watch:
+ node.child_watchers += (watch, )
+ return sorted(node.children)
+
+ def get(self, handle, path, watch=None):
+ self.check_handle(handle)
+ node = self.traverse(path)
+ if watch:
+ node.watchers += (watch, )
+ return node.data, dict(
+ ephemeralOwner=(1 if node.flags & zookeeper.EPHEMERAL else 0),
+ )
+
+ def set(self, handle, path, data):
+ self.check_handle(handle)
+ node = self.traverse(path)
+ node.data = data
+ node.changed(self.handle, zookeeper.CONNECTED_STATE, path)
+
+ def get_acl(self, handle, path):
+ self.check_handle(handle)
+ node = self.traverse(path)
+ return dict(aversion=node.aversion), node.acl
+
+ def set_acl(self, handle, path, aversion, acl):
+ self.check_handle(handle)
+ node = self.traverse(path)
+ if aversion != node.aversion:
+ raise zookeeper.BadVersionException("bad version")
+ node.aversion += 1
+ node.acl = acl
+
+class Node:
+ watchers = child_watchers = ()
+ flags = 0
+ aversion = 0
+ acl = zc.zk.OPEN_ACL_UNSAFE
+
+ def __init__(self, data='', **children):
+ self.data = data
+ self.children = children
+
+ def children_changed(self, handle, state, path):
+ watchers = self.child_watchers
+ self.child_watchers = ()
+ for w in watchers:
+ w(handle, zookeeper.CHILD_EVENT, state, path)
+
+ def changed(self, handle, state, path):
+ watchers = self.watchers
+ self.watchers = ()
+ for w in watchers:
+ w(handle, zookeeper.CHANGED_EVENT, state, path)
+
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(Tests),
- doctest.DocFileSuite(
+ doctest.DocTestSuite(
+ setUp=setup, tearDown=zope.testing.setupstack.tearDown,
+ ),
+ manuel.testing.TestSuite(
+ manuel.doctest.Manuel(
+ checker = zope.testing.renormalizing.RENormalizing([
+ (re.compile('pid = \d+'), 'pid = 9999')
+ ])) + manuel.capture.Manuel(),
'README.txt',
- setUp=setup, tearDown=zope.testing.setupstack.tearDown
+ setUp=setup, tearDown=zope.testing.setupstack.tearDown,
),
unittest.makeSuite(LoggingTests),
))
More information about the checkins
mailing list