[Checkins] SVN: zc.zk/trunk/src/zc/zk/ - Symbolic links can now be relative and use ``.`` and ``..`` in the
Jim Fulton
jim at zope.com
Mon Dec 26 23:36:01 UTC 2011
Log message for revision 123862:
- Symbolic links can now be relative and use ``.`` and ``..`` in the
usual way.
- Added property links.
Changed:
U zc.zk/trunk/src/zc/zk/README.txt
U zc.zk/trunk/src/zc/zk/__init__.py
U zc.zk/trunk/src/zc/zk/tests.py
-=-
Modified: zc.zk/trunk/src/zc/zk/README.txt
===================================================================
--- zc.zk/trunk/src/zc/zk/README.txt 2011-12-22 19:48:48 UTC (rev 123861)
+++ zc.zk/trunk/src/zc/zk/README.txt 2011-12-26 23:36:00 UTC (rev 123862)
@@ -536,6 +536,102 @@
>>> dict(main_properties)
{u'a': 1}
+Symbolic links can be relative. If a link doesn't start with a slash,
+it's interpreted relative to the node the link occurs in. The special
+names ``.`` and ``..`` have their usual meanings.
+
+So, in::
+
+ /a
+ /b
+ l -> c
+ l2 -> ../c
+ /c
+ /c
+
+.. -> relative_link_source
+
+ >>> zk.import_tree(relative_link_source)
+ >>> zk.resolve('/a/b/l')
+ u'/a/b/c'
+ >>> zk.resolve('/a/b/l2')
+ u'/a/c'
+
+ >>> zk.delete_recursive('/a')
+
+The link at ``/a/b/l`` resolves to ``/a/b/c`` and ``/a/b/l2`` resolves
+to ``/a/c``.
+
+Property links
+==============
+
+In addition to symbolic links between nodes, you can have links
+between properties. In our earlier example, both the ``/cms`` and
+``/fooservice`` nodes had ``threads`` properties::
+
+ /cms : z4m cms
+ threads = 4
+ /databases
+ /main
+ /providers
+ /providers
+ /fooservice
+ secret = u'1234'
+ threads = 3
+ /providers
+ /lb : ipvs
+ /pools
+ /cms
+ address = u'1.2.3.4:80'
+ providers -> /cms/providers
+
+If we wanted ``/cms`` to have the same ``threads`` settings, we could
+use a property link::
+
+ /cms : z4m cms
+ threads => /fooservice threads
+ /databases
+ /main
+ /providers
+ /providers
+ /fooservice
+ secret = u'1234'
+ threads = 3
+ /providers
+ /lb : ipvs
+ /pools
+ /cms
+ address = u'1.2.3.4:80'
+ providers -> /cms/providers
+
+.. -> property_link_source
+
+ >>> _ = zk.create('/test-propery-links', '', zc.zk.OPEN_ACL_UNSAFE)
+
+ >>> zk.import_tree(property_link_source, '/test-propery-links')
+ >>> properties = zk.properties('/test-propery-links/cms')
+ >>> properties['threads =>']
+ u'/fooservice threads'
+ >>> properties['threads']
+ 3
+
+ >>> zk.import_tree('/cms\n threads => /fooservice\n',
+ ... '/test-propery-links')
+ extra path not trimmed: /test-propery-links/cms/databases
+ extra path not trimmed: /test-propery-links/cms/providers
+ >>> properties['threads =>']
+ u'/fooservice'
+ >>> properties['threads']
+ 3
+
+ >>> zk.delete_recursive('/test-propery-links')
+
+Property links are indicated with ``=>``. The value is a node path and
+optional property name, separated by whitespace. If the name is
+ommitted, then the refering name is used. For example, the name could
+be left off of the property link above.
+
+
Node deletion
=============
@@ -893,6 +989,14 @@
Change History
==============
+0.5.0 (2011-12-??)
+------------------
+
+- Symbolic links can now be relative and use ``.`` and ``..`` in the
+ usual way.
+
+- Added property links.
+
0.4.0 (2011-12-12)
------------------
Modified: zc.zk/trunk/src/zc/zk/__init__.py
===================================================================
--- zc.zk/trunk/src/zc/zk/__init__.py 2011-12-22 19:48:48 UTC (rev 123861)
+++ zc.zk/trunk/src/zc/zk/__init__.py 2011-12-26 23:36:00 UTC (rev 123862)
@@ -93,24 +93,6 @@
OPEN_ACL_UNSAFE = [world_permission(zookeeper.PERM_ALL)]
READ_ACL_UNSAFE = [world_permission()]
-
-_text_is_node = re.compile(
- r'/(?P<name>\S+)'
- '(\s*:\s*(?P<type>\S.*))?'
- '$').match
-_text_is_property = re.compile(
- r'(?P<name>\S+)'
- '\s*=\s*'
- '(?P<expr>\S.*)'
- '$'
- ).match
-_text_is_link = re.compile(
- r'(?P<name>\S+)'
- '\s*->\s*'
- '(?P<target>/\S+)'
- '$'
- ).match
-
class CancelWatch(Exception):
pass
@@ -120,6 +102,9 @@
class FailedConnect(Exception):
pass
+class BadPropertyLink(Exception):
+ pass
+
class ZooKeeper:
def __init__(self, connection_string="127.0.0.1:2181", session_timeout=None,
@@ -447,8 +432,18 @@
print self.export_tree(path, True),
def resolve(self, path, seen=()):
+ if path.endswith('/.'):
+ return self.resolve(path[:-2])
+ if path.endswith('/..'):
+ base = self.resolve(path[:-3])
+ if '/' in base:
+ return base.rsplit('/', 1)[0]
+ else:
+ raise zookeeper.NoNodeException(path)
+
if self.exists(path):
return path
+
if path in seen:
seen += (path,)
raise LinkLoop(seen)
@@ -464,6 +459,9 @@
if not newpath:
raise zookeeper.NoNodeException()
+ if not newpath[0] == '/':
+ newpath = base + '/' + newpath
+
seen += (path,)
return self.resolve(newpath, seen)
except zookeeper.NoNodeException:
@@ -610,7 +608,8 @@
self.session.handle, self.path)
def _notify(self, data):
- self.setData(data)
+ if data is not None:
+ self.setData(data)
for callback in list(self.callbacks):
try:
callback(self)
@@ -640,24 +639,111 @@
event_type = zookeeper.CHANGED_EVENT
+ def __init__(self, *args):
+ self._linked_properties = {}
+ NodeInfo.__init__(self, *args)
+
+ def _setData(self, data, handle_errors=False):
+ # Save a mapping as our data.
+ # Set up watchers for any property links.
+ _linked_properties = {}
+ for name in data:
+ if name.endswith(' =>') and name[:-3] not in data:
+ link = data[name].strip().split()
+ try:
+ if not (1 <= len(link) <= 2):
+ raise ValueError('Bad link data')
+ path = link.pop(0)
+ if path[0] != '/':
+ path = self.path + '/' + path
+ path = self.session.resolve(path)
+ properties = self._setup_link(path, _linked_properties)
+ properties[link and link[0] or name[:-3]]
+ except Exception, v:
+ if handle_errors:
+ logger.exception(
+ 'Bad property link %r %r', name, data[name])
+ else:
+ raise ValueError("Bad property link",
+ name, data[name], v)
+
+ # Only after processing all links, do we update instance attrs:
+ self.data = data
+ self._linked_properties = _linked_properties
+
+ def _setup_link(self, path, _linked_properties=None):
+ if _linked_properties is None:
+ _linked_properties = self._linked_properties
+
+ props = _linked_properties.get(path, self)
+ if props is not self:
+ return props
+
+ props = self.session.properties(path)
+
+ _linked_properties[path] = props
+
+ def notify(properties=None):
+ if properties is None:
+ # A node we were watching was deleted. We shuld try to
+ # re-resolve it. This doesn't happen often, let's just reset
+ # everything.
+ self._setData(self.data)
+ elif self._linked_properties.get(path) is properties:
+ self._notify(None)
+ else:
+ # We must not care about it anymore.
+ raise CancelWatch()
+
+ props.callbacks.append(notify)
+ return props
+
def setData(self, data):
+ # Called by upstream watchers.
sdata, self.meta_data = data
- self.data = decode(sdata, self.path)
+ self._setData(decode(sdata, self.path), True)
- def __getitem__(self, key):
- return self.data[key]
+ def __getitem__(self, key, seen=()):
+ try:
+ return self.data[key]
+ except KeyError:
+ link = self.data.get(key + ' =>', self)
+ if link is self:
+ raise
+ try:
+ data = link.split()
+ if len(data) > 2:
+ raise ValueError('Invalid property link')
+ path = data.pop(0)
+ if not path[0] == '/':
+ path = self.path + '/' + path
+ path = self.session.resolve(path)
+ if path in seen:
+ raise LinkLoop(seen+(path,))
+ seen += (path,)
+ properties = self._linked_properties.get(path)
+ if properties is None:
+ properties = self._setup_link(path)
+ name = data and data[0] or key
+ return properties.__getitem__(name, seen)
+ except Exception, v:
+ raise BadPropertyLink(
+ v, 'in %r: %r' %
+ (key + ' =>', self.data[key + ' =>'])
+ )
+
def __len__(self):
return len(self.data)
def __contains__(self, key):
- return key in self.data
+ return key in self.data or (key + ' =>') in self.data
def copy(self):
return self.data.copy()
def _set(self, data):
- self.data = data
+ self._setData(data)
self.session._set(self.path, encode(data))
def set(self, data=None, **properties):
@@ -676,6 +762,30 @@
# Gaaaa, collections.Mapping
return hash(id(self))
+_text_is_node = re.compile(
+ r'/(?P<name>\S+)'
+ '(\s*:\s*(?P<type>\S.*))?'
+ '$').match
+_text_is_property = re.compile(
+ r'(?P<name>\S+)'
+ '\s*=\s*'
+ '(?P<expr>\S.*)'
+ '$'
+ ).match
+_text_is_link = re.compile(
+ r'(?P<name>\S+)'
+ '\s*->\s*'
+ '(?P<target>\S+)'
+ '$'
+ ).match
+_text_is_plink = re.compile(
+ r'(?P<name>\S+)'
+ '\s*=>\s*'
+ '(?P<target>\S+(\s+\S+)?)'
+ '(\s+(?P<pname>/\S+))?'
+ '$'
+ ).match
+
def parse_tree(text):
root = _Tree()
indents = [(-1, root)] # sorted [(indent, node)]
@@ -685,34 +795,46 @@
line = line.rstrip()
if not line:
continue
- data = line.strip()
- if data[0] == '#':
+ stripped = line.strip()
+ if stripped[0] == '#':
continue
- indent = len(line) - len(data)
+ indent = len(line) - len(stripped)
- m = _text_is_property(data)
+ data = None
+
+ m = _text_is_plink(stripped)
if m:
- expr = m.group('expr')
- try:
- data = eval(expr, {})
- except Exception, v:
- raise ValueError("Error %s in expression: %r" % (v, expr))
- data = m.group('name'), data
- else:
- m = _text_is_link(data)
+ data = (m.group('name') + ' =>'), m.group('target')
+
+ if data is None:
+ m = _text_is_property(stripped)
if m:
+ expr = m.group('expr')
+ try:
+ data = eval(expr, {})
+ except Exception, v:
+ raise ValueError(
+ "Error %s in expression: %r in line %s" %
+ (v, expr, lineno))
+ data = m.group('name'), data
+
+ if data is None:
+ m = _text_is_link(stripped)
+ if m:
data = (m.group('name') + ' ->'), m.group('target')
+
+ if data is None:
+ m = _text_is_node(stripped)
+ if m:
+ data = _Tree(m.group('name'))
+ if m.group('type'):
+ data.properties['type'] = m.group('type')
+
+ if data is None:
+ if '->' in stripped:
+ raise ValueError(lineno, stripped, "Bad link format")
else:
- m = _text_is_node(data)
- if m:
- data = _Tree(m.group('name'))
- if m.group('type'):
- data.properties['type'] = m.group('type')
- else:
- if '->' in data:
- raise ValueError(lineno, data, "Bad link format")
- else:
- raise ValueError(lineno, data, "Unrecognized data")
+ raise ValueError(lineno, stripped, "Unrecognized data")
if indent > indents[-1][0]:
if not isinstance(indents[-1][1], _Tree):
Modified: zc.zk/trunk/src/zc/zk/tests.py
===================================================================
--- zc.zk/trunk/src/zc/zk/tests.py 2011-12-22 19:48:48 UTC (rev 123861)
+++ zc.zk/trunk/src/zc/zk/tests.py 2011-12-26 23:36:00 UTC (rev 123862)
@@ -11,6 +11,7 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
+from pprint import pprint
import doctest
import json
import logging
@@ -19,7 +20,6 @@
import manuel.testing
import mock
import os
-import pprint
import re
import StringIO
import sys
@@ -634,7 +634,7 @@
Traceback (most recent call last):
...
ValueError: Error unexpected EOF while parsing (<string>, line 1)
- in expression: '1+'
+ in expression: '1+' in line 3
>>> zk.import_tree('''
... /test
@@ -644,13 +644,6 @@
...
ValueError: (3, 'a ->', 'Bad link format')
- >>> zk.import_tree('''
- ... /test
- ... a -> 1
- ... ''') # doctest: +NORMALIZE_WHITESPACE
- Traceback (most recent call last):
- ValueError: (3, 'a -> 1', 'Bad link format')
-
>>> zk.close()
"""
@@ -707,7 +700,7 @@
>>> data = zk.properties('/fooservice')
>>> @data
... def _(data):
- ... pprint.pprint(dict(data), width=70)
+ ... pprint(dict(data), width=70)
{u'database': u'/databases/foomain',
u'favorite_color': u'red',
u'threads': 1}
@@ -780,6 +773,9 @@
...
LinkLoop: ('/top/a/loop', u'/top/a/b/loop', u'/top/a/loop')
+ >>> zk.resolve('/top/a/b/c/d/./../..')
+ '/top/a/b'
+
>>> zk.close()
"""
@@ -787,7 +783,7 @@
"""
>>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
>>> zk.ln('/databases/main', '/fooservice/')
- >>> pprint.pprint(zk.get_properties('/fooservice'))
+ >>> pprint(zk.get_properties('/fooservice'))
{u' ->': u'/databases/main',
u'database': u'/databases/foomain',
u'favorite_color': u'red',
@@ -1036,6 +1032,239 @@
>>> zk.close()
"""
+relative_property_links_data = """
+
+/a
+ /b
+ x => c x
+ xx => ./c x
+ x2 => .. x
+ x3 => ../c x
+ x22 => ./.. x
+ x33 => ./../c x
+ x333 => .././c x
+ /c
+ x=1
+ /c
+ x = 3
+ x = 2
+
+"""
+def relative_property_links():
+ """
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> zk.import_tree(relative_property_links_data)
+ >>> p = zk.properties('/a/b')
+ >>> p['x']
+ 1
+ >>> p['xx']
+ 1
+ >>> p['x2']
+ 2
+ >>> p['x22']
+ 2
+ >>> p['x3']
+ 3
+ >>> p['x33']
+ 3
+ >>> p['x333']
+ 3
+ """
+
+def property_links_expand_callbacks_to_linked_nodes():
+ """
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> zk.import_tree('''
+ ... /a
+ ... /b
+ ... x => c
+ ... xx => ../c
+ ... /c
+ ... x = 1
+ ... /c
+ ... x = 2
+ ... ''')
+
+ >>> ab = zk.properties('/a/b')
+
+ >>> @ab
+ ... def _(properties):
+ ... print 'updated'
+ updated
+
+ >>> ac = zk.properties('/a/c')
+ >>> ac.update(x=3)
+ updated
+
+ >>> ab.update(xx=2)
+ updated
+
+ >>> ac.update(x=4)
+
+ """
+
+def bad_links_are_reported_and_prevent_updates():
+ """
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> properties = zk.properties('/fooservice')
+
+ >>> properties.update({'c =>': '/a/b/c d'}, a=1, b=2, d=3, e=4)
+ ... # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Bad property link', 'c =>', '/a/b/c d',
+ NoNodeException('/a/b/c',))
+ >>> pprint(dict(properties), width=70)
+ {u'database': u'/databases/foomain',
+ u'favorite_color': u'red',
+ u'threads': 1}
+
+ >>> properties.update({'c =>': ''}, a=1, b=2, d=3, e=4)
+ ... # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Bad property link', 'c =>', '',
+ ValueError('Bad link data',))
+ >>> pprint(dict(properties), width=70)
+ {u'database': u'/databases/foomain',
+ u'favorite_color': u'red',
+ u'threads': 1}
+
+ >>> properties.update({'c =>': '/fooservice x'}, a=1, b=2, d=3, e=4)
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Bad property link', 'c =>', '/fooservice x', KeyError('x',))
+ >>> pprint(dict(properties), width=70)
+ {u'database': u'/databases/foomain',
+ u'favorite_color': u'red',
+ u'threads': 1}
+
+ >>> properties.update({'c =>': '/fooservice threads x'}, a=1, b=2, d=3, e=4)
+ ... # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Bad property link', 'c =>', '/fooservice threads x',
+ ValueError('Bad link data',))
+ >>> pprint(dict(properties), width=70)
+ {u'database': u'/databases/foomain',
+ u'favorite_color': u'red',
+ u'threads': 1}
+
+ >>> properties.set({'c =>': '/a/b/c d'}, a=1, b=2, d=3, e=4)
+ ... # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Bad property link', 'c =>', '/a/b/c d',
+ NoNodeException('/a/b/c',))
+ >>> pprint(dict(properties), width=70)
+ {u'database': u'/databases/foomain',
+ u'favorite_color': u'red',
+ u'threads': 1}
+
+ >>> properties.set({'c =>': ''}, a=1, b=2, d=3, e=4)
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Bad property link', 'c =>', '', ValueError('Bad link data',))
+ >>> pprint(dict(properties), width=70)
+ {u'database': u'/databases/foomain',
+ u'favorite_color': u'red',
+ u'threads': 1}
+ """
+
+def contains_w_property_link():
+ """
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> properties = zk.properties('/fooservice/providers')
+ >>> properties.update({'c =>': '.. threads'})
+ >>> 'c' in properties
+ True
+
+ """
+
+def property_getitem_error_handling():
+ """
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> _ = zk.set('/fooservice/providers', json.dumps({
+ ... 'a =>': '/a/b',
+ ... 'b =>': '/fooservice threads x',
+ ... 'c =>': '',
+ ... }))
+ >>> properties = zk.properties('/fooservice/providers')
+ >>> properties['a']
+ Traceback (most recent call last):
+ ...
+ BadPropertyLink: (NoNodeException(u'/a/b',), "in 'a =>': u'/a/b'")
+ >>> properties['b'] # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ BadPropertyLink: (ValueError('Invalid property link',),
+ "in 'b =>': u'/fooservice threads x'")
+ >>> properties['c']
+ Traceback (most recent call last):
+ ...
+ BadPropertyLink: (IndexError('pop from empty list',), "in 'c =>': u''")
+ """
+
+def property_link_loops():
+ """
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> zk.import_tree('''
+ ... /a
+ ... x => ../b x
+ ... /b
+ ... x => ../a x
+ ... ''')
+ >>> properties = zk.properties('/a')
+ >>> properties['x'] # doctest: +NORMALIZE_WHITESPACE
+ Traceback (most recent call last):
+ ...
+ BadPropertyLink:
+ (BadPropertyLink(BadPropertyLink(LinkLoop((u'/b', u'/a', u'/b'),),
+ "in u'x =>': u'../b x'"), "in u'x =>': u'../a x'"), "in 'x =>': u'../b x'")
+
+ """
+
+def deleting_linked_nodes():
+ """
+ Links are reestablished after deleting a linked node.
+
+ >>> zk = zc.zk.ZooKeeper('zookeeper.example.com:2181')
+ >>> zk.import_tree('''
+ ... /a
+ ... /b
+ ... x => ../c
+ ... /c
+ ... x = 1
+ ... ''')
+
+ >>> ab = zk.properties('/a/b')
+
+ >>> @ab
+ ... def _(properties):
+ ... print 'updated'
+ updated
+
+ >>> ab['x']
+ 1
+
+ >>> zk.import_tree('''
+ ... /d
+ ... x = 2
+ ... ''')
+
+ >>> _ = zk.set('/a', '{"c ->": "/d"}')
+ >>> _ = zk.delete('/a/c')
+ updated
+ >>> ab['x']
+ 2
+
+ """
+
+
+
+# XXX
+# deleting linked node
+
event = threading.Event()
def check_async(show=True, expected_status=0):
event.clear()
More information about the checkins
mailing list