[Checkins] SVN: keas.pbstate/tags/ Tagged 0.1
Shane Hathaway
shane at hathawaymix.org
Tue Jan 27 18:52:08 EST 2009
Log message for revision 95270:
Tagged 0.1
Changed:
A keas.pbstate/tags/
A keas.pbstate/tags/0.1/
D keas.pbstate/tags/0.1/buildout.cfg
A keas.pbstate/tags/0.1/buildout.cfg
D keas.pbstate/tags/0.1/setup.py
A keas.pbstate/tags/0.1/setup.py
A keas.pbstate/tags/0.1/src/keas/pbstate/README.txt
D keas.pbstate/tags/0.1/src/keas/pbstate/meta.py
A keas.pbstate/tags/0.1/src/keas/pbstate/meta.py
D keas.pbstate/tags/0.1/src/keas/pbstate/state.py
A keas.pbstate/tags/0.1/src/keas/pbstate/state.py
A keas.pbstate/tags/0.1/src/keas/pbstate/testclasses.proto
A keas.pbstate/tags/0.1/src/keas/pbstate/testclasses_pb2.py
A keas.pbstate/tags/0.1/src/keas/pbstate/tests.py
-=-
Property changes on: keas.pbstate/tags/0.1
___________________________________________________________________
Added: svn:ignore
+ .installed.cfg
bin
coverage
develop-eggs
dist
parts
.builder
Added: svn:mergeinfo
+
Deleted: keas.pbstate/tags/0.1/buildout.cfg
===================================================================
--- keas.pbstate/trunk/buildout.cfg 2009-01-13 00:38:50 UTC (rev 94713)
+++ keas.pbstate/tags/0.1/buildout.cfg 2009-01-27 23:52:08 UTC (rev 95270)
@@ -1,12 +0,0 @@
-[buildout]
-develop = .
-parts = test python
-
-[test]
-recipe = zc.recipe.testrunner
-eggs = keas.pbstate
-
-[python]
-recipe = zc.recipe.egg
-eggs = keas.pbstate
-interpreter = python
Copied: keas.pbstate/tags/0.1/buildout.cfg (from rev 94717, keas.pbstate/trunk/buildout.cfg)
===================================================================
--- keas.pbstate/tags/0.1/buildout.cfg (rev 0)
+++ keas.pbstate/tags/0.1/buildout.cfg 2009-01-27 23:52:08 UTC (rev 95270)
@@ -0,0 +1,23 @@
+[buildout]
+develop = .
+parts = test python coverage-test coverage-report
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = keas.pbstate
+
+[python]
+recipe = zc.recipe.egg
+eggs = keas.pbstate
+interpreter = python
+
+[coverage-test]
+recipe = zc.recipe.testrunner
+eggs = keas.pbstate
+defaults = ['--coverage', '../../coverage']
+
+[coverage-report]
+recipe = zc.recipe.egg
+eggs = z3c.coverage
+scripts = coverage=coverage-report
+arguments = ('coverage', 'coverage/report')
Deleted: keas.pbstate/tags/0.1/setup.py
===================================================================
--- keas.pbstate/trunk/setup.py 2009-01-13 00:38:50 UTC (rev 94713)
+++ keas.pbstate/tags/0.1/setup.py 2009-01-27 23:52:08 UTC (rev 95270)
@@ -1,31 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2008 Zope Foundation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-
-from setuptools import setup
-
-setup(
- name='keas.pbstate',
- version='0.1dev',
- author='Shane Hathaway and the Zope Community',
- author_email='zope-dev at zope.org',
- description='Method of storing object state in a Google Protocol Buffer',
- license='ZPL 2.1',
-
- package_dir={'': 'src'},
- packages=['keas.pbstate'],
- namespace_packages=['keas'],
- install_requires=[
- 'protobuf',
- ],
-)
Copied: keas.pbstate/tags/0.1/setup.py (from rev 95269, keas.pbstate/trunk/setup.py)
===================================================================
--- keas.pbstate/tags/0.1/setup.py (rev 0)
+++ keas.pbstate/tags/0.1/setup.py 2009-01-27 23:52:08 UTC (rev 95270)
@@ -0,0 +1,42 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""An object database foundation based on Google's Protocol Buffers"""
+
+import os
+from setuptools import setup
+
+VERSION='0.1'
+
+def read_file(*path):
+ base_dir = os.path.dirname(__file__)
+ file_path = (base_dir, ) + tuple(path)
+ return file(os.path.join(*file_path)).read()
+
+setup(
+ name='keas.pbstate',
+ version=VERSION,
+ author='Shane Hathaway and the Zope Community',
+ author_email='zope-dev at zope.org',
+ description=__doc__,
+ license='ZPL 2.1',
+
+ package_dir={'': 'src'},
+ packages=['keas.pbstate'],
+ namespace_packages=['keas'],
+ install_requires=[
+ 'setuptools',
+ 'protobuf',
+ ],
+ long_description = read_file('src', 'keas', 'pbstate', 'README.txt'),
+)
Copied: keas.pbstate/tags/0.1/src/keas/pbstate/README.txt (from rev 95268, keas.pbstate/trunk/src/keas/pbstate/README.txt)
===================================================================
--- keas.pbstate/tags/0.1/src/keas/pbstate/README.txt (rev 0)
+++ keas.pbstate/tags/0.1/src/keas/pbstate/README.txt 2009-01-27 23:52:08 UTC (rev 95270)
@@ -0,0 +1,373 @@
+
+Overview
+========
+
+Google's Protocol Buffers project provides an interesting way to serialize
+data. Protocol Buffer messages are efficient to produce and parse, flexible
+enough to weather schema changes, fairly expressive, and are usable in
+several programming languages.
+
+What if we combined those properties with an object database? Object
+databases often provide an excellent software foundation. Unfortunately,
+object databases are generally either joined to a complex object-relational
+mapper, or are bound to a single programming language due to the
+object serialization format. This package uses the latter strategy, but
+uses Protocol Buffers as the serialization format, conceivably making
+it possible to build an object database that multiple programming languages
+can access.
+
+Using this package also provides schema documentation. The Protocol Buffers
+package requires programmers to write the schema of their data in a
+concise form that also serves as documentation of the schema. While it's
+usually possible to guess at the schema by looking at application
+code, having it written out in Protocol Buffer format is much more
+direct and informative.
+
+This package is designed to be combined with an object database such as
+ZODB, but this package does not require ZODB.
+
+
+Tests
+=====
+
+The tests below describe how to use this package.
+These tests depend on the module named testclasses_pb2.py, which
+is generated from testclasses.proto using the following command,
+available once the Google Protocol Buffers package is installed::
+
+ protoc --python_out . *.proto
+
+Create a Contact class. Notice its metaclass. The metaclass adds
+properties to the class so that you can read and write protocol
+buffer message fields using simple attribute access. The
+'create_time' attribute is one such field.
+
+ >>> import time
+ >>> from keas.pbstate.meta import ProtobufState
+ >>> from keas.pbstate.testclasses_pb2 import ContactPB
+ >>> class Contact(object):
+ ... __metaclass__ = ProtobufState
+ ... protobuf_type = ContactPB
+ ... def __init__(self):
+ ... self.create_time = int(time.time())
+ ...
+
+Create an instance of this class and verify the instance has the expected
+attributes. These attributes are all described in the .proto file.
+
+ >>> c = Contact()
+ >>> c.create_time > 0
+ True
+ >>> c.name
+ u''
+ >>> c.address.line1
+ u''
+ >>> c.address.country
+ u'United States'
+
+The instance also provides access to the protobuf message, its type (inherited
+from the class), and the references from the message. References will be
+discussed later.
+
+ >>> c.protobuf
+ <keas.pbstate.testclasses_pb2.ContactPB object at ...>
+ >>> c.protobuf_type
+ <class 'keas.pbstate.testclasses_pb2.ContactPB'>
+ >>> c.protobuf_refs
+ <keas.pbstate.meta.ProtobufReferences object at ...>
+
+Set and retrieve some of the attributes.
+
+ >>> c.name = u'John Doe'
+ >>> c.address.line1 = u'100 First Avenue'
+ >>> c.address.country = u'Canada'
+ >>> c.name
+ u'John Doe'
+ >>> c.address.country
+ u'Canada'
+
+Try to set one of the attributes to a value the protobuf message can't
+serialize.
+
+ >>> c.name = 100
+ Traceback (most recent call last):
+ ...
+ TypeError: 100 has type <type 'int'>, but expected one of: (<type 'str'>, <type 'unicode'>)
+ >>> c.name
+ u'John Doe'
+
+Try to set an attribute not declared in the .proto file.
+
+ >>> c.phone = u'555-1234'
+ Traceback (most recent call last):
+ ...
+ AttributeError: 'Contact' object has no attribute 'phone'
+
+
+Mixins
+------
+
+A class can mix in properties that access sub-messages. This is
+useful when subclassing (although subclassing should be avoided in general).
+
+Here is a class that mixes the ContactPB properties and the AddressPB
+properties in a single class.
+
+ >>> class MixedContact(object):
+ ... __metaclass__ = ProtobufState
+ ... protobuf_type = ContactPB
+ ... protobuf_mixins = ('address',)
+
+ >>> mc = MixedContact()
+ >>> mc.line1 = u'180 Market St.'
+ >>> mc.line1
+ u'180 Market St.'
+ >>> mc.address.line1
+ u'180 Market St.'
+
+
+Serialization
+-------------
+
+The ProtobufState also provides __getstate__ and __setstate__ methods,
+which Python uses for serialization purposes.
+
+Try to serialize the object without providing all of the required fields.
+
+ >>> c.__getstate__()
+ Traceback (most recent call last):
+ ...
+ EncodeError: Required field AddressPB.city is not set.
+
+Finish filling out the required fields, then serialize.
+
+ >>> c.address.city = u'Toronto'
+ >>> c.create_time = 1001
+ >>> c.__getstate__()
+ ('\x08\xe9\x07\x12\x08John Doe\x1a#\n\x10100 First Avenue\x1a\x07Toronto2\x06Canada', {})
+
+Create a contact and copy its state from c.
+
+ >>> c_dup = Contact.__new__(Contact)
+ >>> c_dup.__setstate__(c.__getstate__())
+ >>> c_dup.name
+ u'John Doe'
+ >>> c_dup.address.country
+ u'Canada'
+
+Create another contact, but this time provide no address information.
+
+ >>> c2 = Contact()
+ >>> c2.create_time = 1002
+ >>> c2.name = u'Mary Anne'
+ >>> c2.__getstate__()
+ ('\x08\xea\x07\x12\tMary Anne', {})
+
+
+Object References
+-----------------
+
+Classes using the ProtobufState metaclass support references to arbitrary
+objects through the use of the 'protobuf_refs' attribute.
+
+Our Contact class has a 'guardians' attribute that contains a list of
+references. The ProtobufState metaclass treats any message or sub-message
+with a _p_refid field as a reference.
+
+Add a guardian to c2, but don't say who the guardian is yet.
+
+ >>> guardian_ref = c2.guardians.add()
+
+Call protobuf_refs.set() to make guardian_ref refer to c.
+
+ >>> c2.protobuf_refs.set(guardian_ref, c)
+
+Let's go over what happened. The set method generated a reference ID, then
+that ID was assigned to guardian_ref._p_refid, and the refid and target object
+were added to the internal state of the protobuf_refs instance. Any message
+with a _p_refid field is a reference. Every _p_refid field should be
+of type uint32.
+
+Read the reference.
+
+ >>> c2.protobuf_refs.get(guardian_ref) is c
+ True
+
+Verify the reference gets serialized correctly.
+
+ >>> data, targets = c2.__getstate__()
+ >>> targets[c2.guardians[0]._p_refid] is c
+ True
+
+Delete the reference.
+
+ >>> c2.protobuf_refs.delete(guardian_ref)
+ >>> c2.protobuf_refs.get(guardian_ref, 'gone')
+ 'gone'
+
+Verify the reference is no longer contained in the serialized state.
+
+ >>> data, targets = c2.__getstate__()
+ >>> len(targets)
+ 0
+
+
+Features Designed for ZODB
+--------------------------
+
+This package provides enough features for storing ProtobufState objects
+in ZODB, although without the keas.pbpersist package, the stored objects
+will still be wrapped inside a Python pickle, making them hard for
+languages other than Python to access. See the keas.pbpersist package
+for a straightforward method of storing ProtobufState objects in ZODB
+without using Python pickles.
+
+In ZODB, objects have a _p_changed attribute to indicate when they
+are dirty. The ProtobufState metaclass causes instances to modify
+the _p_changed attribute if it exists; it is set to True whenever the
+message changes.
+
+Here is a PersistentContact class, which has a _p_changed attribute.
+(We also define a FakePersistent base class in order to avoid
+depending on ZODB.)
+
+ >>> class FakePersistent(object):
+ ... __slots__ = ('_changed',)
+ ... def _get_changed(self):
+ ... return getattr(self, '_changed', False)
+ ... def _set_changed(self, value):
+ ... self._changed = value
+ ... if not value:
+ ... # reset the _cache_byte_size_dirty flags
+ ... self.protobuf.ByteSize()
+ ... _p_changed = property(_get_changed, _set_changed)
+ ...
+ >>> class PersistentContact(FakePersistent):
+ ... __metaclass__ = ProtobufState
+ ... protobuf_type = ContactPB
+ ...
+
+ >>> c3 = PersistentContact()
+ >>> c3._p_changed
+ False
+ >>> c3.create_time = 1003
+ >>> c3.name = u'Snoopy'
+ >>> c3._p_changed = False
+
+Reading an attribute does not set _p_changed.
+
+ >>> c3.name
+ u'Snoopy'
+ >>> c3._p_changed
+ False
+
+Writing an attribute sets _p_changed.
+
+ >>> c3.name = u'Woodstock'
+ >>> c3._p_changed
+ True
+
+Adding to a repeated element sets _p_changed.
+
+ >>> c3._p_changed = False
+ >>> c3._p_changed
+ False
+ >>> c3.guardians.add()
+ <keas.pbstate.testclasses_pb2.Ref object at ...>
+ >>> c3._p_changed
+ True
+ >>> del c3.guardians[0]
+
+A copy of c3 should initially have _p_changed = False; setting an attribute
+should set _p_changed to true.
+
+ >>> c4 = PersistentContact.__new__(PersistentContact)
+ >>> c4.__setstate__(c3.__getstate__())
+ >>> c4._p_changed
+ False
+ >>> c4.name = u'Linus'
+ >>> c4._p_changed
+ True
+
+The tuple returned by __getstate__ is actually a subclass of tuple. The
+StateTuple suggests to the ZODB serializer that it can save the state
+in a different format than the default pickle format.
+
+ >>> type(c.__getstate__())
+ <class 'keas.pbstate.state.StateTuple'>
+ >>> c.__getstate__().serial_format
+ 'protobuf'
+
+
+Edge Cases
+----------
+
+Synthesize a refid hash collision. This is a rare occurrence, but
+this package should handle it transparently as long as no single object
+holds more than about one billion (2**30) references to other objects.
+
+First make a reference:
+
+ >>> guardian_ref = c2.guardians.add()
+ >>> c2.protobuf_refs.set(guardian_ref, c)
+
+Covertly change the target of that reference:
+
+ >>> c2.protobuf_refs._targets[guardian_ref._p_refid] = mc
+
+Add a new reference to the original target. The first generated refid
+will collide, but he protobuf_refs should should choose a different
+refid automatically.
+
+ >>> guardian2_ref = c2.guardians.add()
+ >>> c2.protobuf_refs.set(guardian2_ref, c)
+ >>> guardian_ref._p_refid == guardian2_ref._p_refid
+ False
+
+
+Exception Conditions
+--------------------
+
+Deleting message attributes is not allowed.
+
+ >>> del c.name
+ Traceback (most recent call last):
+ ...
+ AttributeError: can't delete attribute
+ >>> del mc.line1
+ Traceback (most recent call last):
+ ...
+ AttributeError: can't delete attribute
+
+Mixin names are checked.
+
+ >>> class MixedUpContact(object):
+ ... __metaclass__ = ProtobufState
+ ... protobuf_type = ContactPB
+ ... protobuf_mixins = ('bogus',)
+ Traceback (most recent call last):
+ ...
+ AttributeError: Field 'bogus' not defined for protobuf type <...>
+
+Create a broken reference by setting a reference using the wrong
+protobuf_refs. To prevent this condition, the protobuf_refs attribute
+and the first argument to protobuf_refs.set() must descend from the
+same containing object.
+
+ >>> c.guardians.add()
+ <keas.pbstate.testclasses_pb2.Ref object at ...>
+ >>> c2.protobuf_refs.set(c.guardians[0], c)
+ >>> c.__getstate__()
+ Traceback (most recent call last):
+ ...
+ KeyError: 'Object contains broken references: <Contact object at ...>'
+ >>> del c.guardians[0]
+
+Don't omit the protobuf_type attribute.
+
+ >>> class FailedContact(object):
+ ... __metaclass__ = ProtobufState
+ Traceback (most recent call last):
+ ...
+ TypeError: Class ...FailedContact needs a protobuf_type attribute
+
\ No newline at end of file
Deleted: keas.pbstate/tags/0.1/src/keas/pbstate/meta.py
===================================================================
--- keas.pbstate/trunk/src/keas/pbstate/meta.py 2009-01-13 00:38:50 UTC (rev 94713)
+++ keas.pbstate/tags/0.1/src/keas/pbstate/meta.py 2009-01-27 23:52:08 UTC (rev 95270)
@@ -1,261 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2008 Zope Foundation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Provides the ProtobufState metaclass."""
-
-from google.protobuf.descriptor import FieldDescriptor
-
-from keas.pbstate.state import StateTuple
-
-
-def _protobuf_property(name):
- """A property that delegates to a protobuf message"""
- def get(self):
- return getattr(self.protobuf, name)
- def set(self, value):
- setattr(self.protobuf, name, value)
- def delete(self):
- delattr(self.protobuf, name)
- return property(get, set, delete)
-
-
-def _protobuf_mixin_property(container_getter, name):
- """A property that delegates to a protobuf sub-message"""
- def get_(self):
- return getattr(container_getter(self), name)
- def set_(self, value):
- setattr(container_getter(self), name, value)
- def del_(self):
- delattr(container_getter(self), name)
- return property(get_, set_, del_)
-
-
-def _add_mixin(created_class, mixin_name):
- """Add the properties from a protobuf sub-message to a class"""
- main_desc = created_class.protobuf_type.DESCRIPTOR
-
- # traverse to the named descriptor
- descriptor = main_desc
- parts = mixin_name.split('.')
- for name in parts:
- # find the named field
- for f in descriptor.fields:
- if f.name == name:
- descriptor = f.message_type
- break
- else:
- raise AttributeError(
- "Field %r not defined for protobuf type %r" %
- (mixin_name, main_desc))
-
- container_getter = eval(
- 'lambda self: self.protobuf.%s' % mixin_name)
-
- for field in descriptor.fields:
- setattr(created_class, field.name, _protobuf_mixin_property(
- container_getter, field.name))
-
-
-class MessageRefidGatherer:
- """Returns the set of used refids from a message of a certain type."""
-
- def __init__(self, descriptor):
- self.attrs = {} # {attr: gather callable}
- self.has_refs = False
-
- # Visit certain fields in the message.
- for field in descriptor.fields:
-
- # set up a 'gather' callable that iterates refids,
- if field.message_type is not None:
- gather = MessageRefidGatherer(field.message_type)
- if gather.has_refs:
- self.has_refs = True
- else:
- # don't bother visiting this part of the message
- continue
- elif field.name == '_p_refid':
- self.has_refs = True
- def gather(value):
- if value:
- yield value
- else:
- # other scalars don't matter
- continue
-
- if field.label == FieldDescriptor.LABEL_REPEATED:
- def gather_all(container, gather=gather):
- for obj in container:
- for refid in gather(obj):
- yield refid
- gather = gather_all
-
- self.attrs[field.name] = gather
-
- def __call__(self, container):
- for name, gather in self.attrs.iteritems():
- for refid in gather(getattr(container, name)):
- yield refid
-
-
-class ProtobufReferences(object):
- """A mapping-like object that gets or sets the target of references.
-
- A reference is a protobuf message with a _p_refid field.
- The target can be any kind of pickleable object, including derivatives
- of ZODB.Persistent.
- """
- __slots__ = ('_targets',)
-
- def __init__(self, targets):
- self._targets = targets # {refid -> target}
-
- def __getitem__(self, message):
- """Get a reference target"""
- refid = message._p_refid
- if not refid:
- raise KeyError("No reference set")
- return self._targets[refid]
-
- def __setitem__(self, message, target):
- """Set the target of a reference message"""
- targets = self._targets
- refid = id(target) % 0xffffffff
- if refid not in targets or targets[refid] is not target:
- while not refid or refid in targets:
- refid = (refid + 1) % 0xffffffff
- targets[refid] = target
- message._p_refid = refid
-
- def __delitem__(self, message):
- """Unlink a target from a reference message"""
- # We can't actually remove the reference from the reference mapping
- # because something else might still have a reference. __getstate__
- # will remove unused references.
- message._p_refid = 0
-
- def _get_targets(self, obj, used):
- """Clean out unused reference targets, then return the target dict.
-
- Also raises an error if there are broken references.
- obj is the object that contains this references object. used
- is the set of _p_refids in use by the protobuf.
- """
- targets = self._targets
- current = set(targets)
- broken = used.difference(current)
- if broken:
- raise KeyError("Object contains broken references: %s" % repr(obj))
- for key in current.difference(used):
- del targets[key]
- return targets
-
-
-class StateClassMethods(object):
- """Contains methods that get copied into classes using ProtobufState"""
-
- def __getstate__(self):
- """Encode the entire state of the object in a ProtobufState."""
- # Clean up unused references and get the targets mapping.
- used = set(self._protobuf_find_refids(self.protobuf))
- targets = self.protobuf_refs._get_targets(self, used)
- # Return the state and all reference targets.
- return StateTuple((self.protobuf.SerializeToString(), targets))
-
- def __setstate__(self, state):
- """Set the state of the object.
-
- Accepts a StateTuple or any two item sequence containing
- data (a byte string) and targets (a map of refid to object).
- """
- data, targets = state
- self.protobuf_refs = ProtobufReferences(targets)
- self.protobuf = self.protobuf_type()
- self.protobuf.MergeFromString(data)
- if hasattr(self, '_p_changed'):
- self.protobuf._SetListener(PersistentChangeListener(self))
-
-
-class PersistentChangeListener(object):
- """Propagates protobuf change notifications to a Persistent container.
-
- Implements the interface described by
- google.protobuf.internal.message_listener.MessageListener.
- """
- __slots__ = ('obj',)
-
- def __init__(self, obj):
- self.obj = obj
-
- def TransitionToNonempty(self):
- self.obj._p_changed = True
-
- def ByteSizeDirty(self):
- self.obj._p_changed = True
-
-
-class ProtobufState(type):
- """Metaclass for classes using a protobuf for state and serialization."""
-
- def __new__(metaclass, name, bases, dct):
- """Set up a new class."""
-
- # Arrange for class instances to always have the 'protobuf' and
- # 'protobuf_refs' instance attributes. This is done by creating
- # or overriding the __new__() method of the new class.
- parent__new__ = dct.get('__new__')
- def __new__(subclass, *args, **kw):
- # subclass is either created_class or a subclass of created_class.
- create = parent__new__
- if create is None:
- create = super(created_class, subclass).__new__
- instance = create(subclass, *args, **kw)
- instance.protobuf_refs = ProtobufReferences({})
- instance.protobuf = subclass.protobuf_type()
- if hasattr(instance, '_p_changed'):
- instance.protobuf._SetListener(
- PersistentChangeListener(instance))
- return instance
- dct['__new__'] = __new__
-
- # Limit instance attributes to avoid partial serialization.
- # Note that classes can still store temporary state by
- # declaring other attribute names in the class' __slots__
- # attribute.
- dct['__slots__'] = dct.get('__slots__', ()) + (
- 'protobuf', 'protobuf_refs')
-
- created_class = type.__new__(metaclass, name, bases, dct)
- if not hasattr(created_class, 'protobuf_type'):
- raise TypeError("Class %s.%s needs a protobuf_type attribute"
- % (created_class.__module__, created_class.__name__))
- descriptor = created_class.protobuf_type.DESCRIPTOR
-
- # Copy methods into the class
- for method_name in ('__getstate__', '__setstate__'):
- setattr(created_class, method_name,
- StateClassMethods.__dict__[method_name])
-
- # Set up the refid gatherer
- created_class._protobuf_find_refids = MessageRefidGatherer(descriptor)
-
- # Create properties that delegate storage to the protobuf
- for field in descriptor.fields:
- setattr(created_class, field.name,
- _protobuf_property(field.name))
-
- # Create properties that delegate to mixed in attributes
- for mixin_name in dct.get('protobuf_mixins', ()):
- _add_mixin(created_class, mixin_name)
-
- return created_class
Copied: keas.pbstate/tags/0.1/src/keas/pbstate/meta.py (from rev 94718, keas.pbstate/trunk/src/keas/pbstate/meta.py)
===================================================================
--- keas.pbstate/tags/0.1/src/keas/pbstate/meta.py (rev 0)
+++ keas.pbstate/tags/0.1/src/keas/pbstate/meta.py 2009-01-27 23:52:08 UTC (rev 95270)
@@ -0,0 +1,263 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Provides the ProtobufState metaclass."""
+
+from google.protobuf.descriptor import FieldDescriptor
+
+from keas.pbstate.state import StateTuple
+
+
+def _protobuf_property(name):
+ """A property that delegates to a protobuf message"""
+ def get(self):
+ return getattr(self.protobuf, name)
+ def set(self, value):
+ setattr(self.protobuf, name, value)
+ def delete(self):
+ delattr(self.protobuf, name)
+ return property(get, set, delete)
+
+
+def _protobuf_mixin_property(container_getter, name):
+ """A property that delegates to a protobuf sub-message"""
+ def get_(self):
+ return getattr(container_getter(self), name)
+ def set_(self, value):
+ setattr(container_getter(self), name, value)
+ def del_(self):
+ delattr(container_getter(self), name)
+ return property(get_, set_, del_)
+
+
+def _add_mixin(created_class, mixin_name):
+ """Add the properties from a protobuf sub-message to a class"""
+ main_desc = created_class.protobuf_type.DESCRIPTOR
+
+ # traverse to the named descriptor
+ descriptor = main_desc
+ parts = mixin_name.split('.')
+ for name in parts:
+ # find the named field
+ for f in descriptor.fields:
+ if f.name == name:
+ descriptor = f.message_type
+ break
+ else:
+ raise AttributeError(
+ "Field %r not defined for protobuf type %r" %
+ (mixin_name, main_desc))
+
+ container_getter = eval(
+ 'lambda self: self.protobuf.%s' % mixin_name)
+
+ for field in descriptor.fields:
+ setattr(created_class, field.name, _protobuf_mixin_property(
+ container_getter, field.name))
+
+
+class MessageRefidGatherer:
+ """Returns the set of used refids from a message of a certain type."""
+
+ def __init__(self, descriptor):
+ self.attrs = {} # {attr: gather callable}
+ self.has_refs = False
+
+ # Visit certain fields in the message.
+ for field in descriptor.fields:
+
+ # set up a 'gather' callable that iterates refids,
+ if field.message_type is not None:
+ gather = MessageRefidGatherer(field.message_type)
+ if gather.has_refs:
+ self.has_refs = True
+ else:
+ # don't bother visiting this part of the message
+ continue
+ elif field.name == '_p_refid':
+ self.has_refs = True
+ def gather(value):
+ if value:
+ yield value
+ else:
+ # other scalars don't matter
+ continue
+
+ if field.label == FieldDescriptor.LABEL_REPEATED:
+ def gather_all(container, gather=gather):
+ for obj in container:
+ for refid in gather(obj):
+ yield refid
+ gather = gather_all
+
+ self.attrs[field.name] = gather
+
+ def __call__(self, container):
+ for name, gather in self.attrs.iteritems():
+ for refid in gather(getattr(container, name)):
+ yield refid
+
+
+class ProtobufReferences(object):
+ """Gets or sets the target of references.
+
+ A reference is a protobuf message with a _p_refid field.
+ The target can be any kind of pickleable object, including derivatives
+ of ZODB.Persistent.
+ """
+ __slots__ = ('_targets',)
+
+ def __init__(self, targets):
+ self._targets = targets # {refid -> target}
+
+ def get(self, ref_message, default=None):
+ """Get a reference target"""
+ refid = ref_message._p_refid
+ if not refid:
+ return default
+ return self._targets[refid]
+
+ def set(self, ref_message, target):
+ """Set the target of a reference message"""
+ targets = self._targets
+ refid = id(target) % 0xffffffff
+ if refid not in targets or targets[refid] is not target:
+ while not refid or refid in targets:
+ refid = (refid + 1) % 0xffffffff
+ targets[refid] = target
+ ref_message._p_refid = refid
+
+ def delete(self, ref_message):
+ """Unlink a target from a reference message"""
+ # We can't actually remove the reference from the reference mapping
+ # because something else might still have a reference. __getstate__
+ # will remove unused references.
+ ref_message._p_refid = 0
+
+ def _get_targets(self, obj, used):
+ """Clean out unused reference targets, then return the target dict.
+
+ Also raises an error if there are broken references.
+ obj is the object that contains this references object. used
+ is the set of _p_refids in use by the protobuf.
+ """
+ targets = self._targets
+ current = set(targets)
+ broken = used.difference(current)
+ if broken:
+ raise KeyError("Object contains broken references: %s" % repr(obj))
+ for key in current.difference(used):
+ del targets[key]
+ return targets
+
+
+class StateClassMethods(object):
+ """Contains methods that get copied into classes using ProtobufState"""
+
+ def __getstate__(self):
+ """Encode the entire state of the object in a ProtobufState."""
+ # Clean up unused references and get the targets mapping.
+ used = set(self._protobuf_find_refids(self.protobuf))
+ targets = self.protobuf_refs._get_targets(self, used)
+ if hasattr(self, '_p_changed'):
+ # Reset the message's internal _cache_byte_size_dirty flag
+ self.protobuf.ByteSize()
+ # Return the state and all reference targets.
+ return StateTuple((self.protobuf.SerializeToString(), targets))
+
+ def __setstate__(self, state):
+ """Set the state of the object.
+
+ Accepts a StateTuple or any two item sequence containing
+ data (a byte string) and targets (a map of refid to object).
+ """
+ data, targets = state
+ self.protobuf_refs = ProtobufReferences(targets)
+ self.protobuf = self.protobuf_type()
+ self.protobuf.MergeFromString(data)
+ if hasattr(self, '_p_changed'):
+ self.protobuf._SetListener(PersistentChangeListener(self))
+ # Reset the message's internal _cache_byte_size_dirty flag
+ self.protobuf.ByteSize()
+
+
+class PersistentChangeListener(object):
+ """Propagates protobuf change notifications to a Persistent container.
+
+ Implements the interface described by
+ google.protobuf.internal.message_listener.MessageListener.
+ """
+ __slots__ = ('obj',)
+
+ def __init__(self, obj):
+ self.obj = obj
+
+ def TransitionToNonempty(self):
+ self.obj._p_changed = True
+
+ def ByteSizeDirty(self):
+ self.obj._p_changed = True
+
+
+class ProtobufState(type):
+ """Metaclass for classes using a protobuf for state and serialization."""
+
+ def __new__(metaclass, name, bases, dct):
+ """Set up a new class."""
+
+ # Arrange for class instances to always have the 'protobuf' and
+ # 'protobuf_refs' instance attributes. This is done by creating
+ # or overriding the __new__() method of the new class.
+ def __new__(subclass, *args, **kw):
+ # subclass is either created_class or a subclass of created_class.
+ super_new = super(created_class, subclass).__new__
+ instance = super_new(subclass, *args, **kw)
+ instance.protobuf_refs = ProtobufReferences({})
+ instance.protobuf = subclass.protobuf_type()
+ if hasattr(instance, '_p_changed'):
+ instance.protobuf._SetListener(
+ PersistentChangeListener(instance))
+ return instance
+ dct['__new__'] = __new__
+
+ # Limit instance attributes to avoid partial serialization.
+ # Note that classes can still store temporary state by
+ # declaring other attribute names in the class' __slots__
+ # attribute.
+ dct['__slots__'] = dct.get('__slots__', ()) + (
+ 'protobuf', 'protobuf_refs')
+
+ created_class = type.__new__(metaclass, name, bases, dct)
+ if not hasattr(created_class, 'protobuf_type'):
+ raise TypeError("Class %s.%s needs a protobuf_type attribute"
+ % (created_class.__module__, created_class.__name__))
+ descriptor = created_class.protobuf_type.DESCRIPTOR
+
+ # Copy methods into the class
+ for method_name in ('__getstate__', '__setstate__'):
+ setattr(created_class, method_name,
+ StateClassMethods.__dict__[method_name])
+
+ # Set up the refid gatherer
+ created_class._protobuf_find_refids = MessageRefidGatherer(descriptor)
+
+ # Create properties that delegate storage to the protobuf
+ for field in descriptor.fields:
+ setattr(created_class, field.name,
+ _protobuf_property(field.name))
+
+ # Create properties that delegate to mixed in attributes
+ for mixin_name in dct.get('protobuf_mixins', ()):
+ _add_mixin(created_class, mixin_name)
+
+ return created_class
Deleted: keas.pbstate/tags/0.1/src/keas/pbstate/state.py
===================================================================
--- keas.pbstate/trunk/src/keas/pbstate/state.py 2009-01-13 00:38:50 UTC (rev 94713)
+++ keas.pbstate/tags/0.1/src/keas/pbstate/state.py 2009-01-27 23:52:08 UTC (rev 95270)
@@ -1,7 +0,0 @@
-
-
-class StateTuple(tuple):
- """Contains the persistent state of a ProtobufState object.
-
- Contains data (a byte string) and targets (a map of refid to object).
- """
Copied: keas.pbstate/tags/0.1/src/keas/pbstate/state.py (from rev 94923, keas.pbstate/trunk/src/keas/pbstate/state.py)
===================================================================
--- keas.pbstate/tags/0.1/src/keas/pbstate/state.py (rev 0)
+++ keas.pbstate/tags/0.1/src/keas/pbstate/state.py 2009-01-27 23:52:08 UTC (rev 95270)
@@ -0,0 +1,10 @@
+
+
+class StateTuple(tuple):
+ """Contains the persistent state of a ProtobufState object.
+
+ Contains data (a byte string) and targets (a map of refid to object).
+ This state class has an attribute, 'serial_format', that provides
+ a hint on how the state ought to be serialized.
+ """
+ serial_format = 'protobuf'
Copied: keas.pbstate/tags/0.1/src/keas/pbstate/testclasses.proto (from rev 94715, keas.pbstate/trunk/src/keas/pbstate/testclasses.proto)
===================================================================
--- keas.pbstate/tags/0.1/src/keas/pbstate/testclasses.proto (rev 0)
+++ keas.pbstate/tags/0.1/src/keas/pbstate/testclasses.proto 2009-01-27 23:52:08 UTC (rev 95270)
@@ -0,0 +1,20 @@
+
+message Ref {
+ required uint32 _p_refid = 1;
+}
+
+message AddressPB {
+ required string line1 = 1;
+ optional string line2 = 2;
+ required string city = 3;
+ optional string state = 4;
+ optional string postal_code = 5;
+ required string country = 6 [default = 'United States'];
+}
+
+message ContactPB {
+ required uint64 create_time = 1;
+ required string name = 2;
+ optional AddressPB address = 3;
+ repeated Ref guardians = 4;
+}
Copied: keas.pbstate/tags/0.1/src/keas/pbstate/testclasses_pb2.py (from rev 94715, keas.pbstate/trunk/src/keas/pbstate/testclasses_pb2.py)
===================================================================
--- keas.pbstate/tags/0.1/src/keas/pbstate/testclasses_pb2.py (rev 0)
+++ keas.pbstate/tags/0.1/src/keas/pbstate/testclasses_pb2.py 2009-01-27 23:52:08 UTC (rev 95270)
@@ -0,0 +1,149 @@
+#!/usr/bin/python2.4
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+from google.protobuf import descriptor
+from google.protobuf import message
+from google.protobuf import reflection
+from google.protobuf import service
+from google.protobuf import service_reflection
+from google.protobuf import descriptor_pb2
+
+
+
+_REF = descriptor.Descriptor(
+ name='Ref',
+ full_name='Ref',
+ filename='src/keas/pbstate/testclasses.proto',
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='_p_refid', full_name='Ref._p_refid', index=0,
+ number=1, type=13, cpp_type=3, label=2,
+ default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[
+ ],
+ nested_types=[], # TODO(robinson): Implement.
+ enum_types=[
+ ],
+ options=None)
+
+
+_ADDRESSPB = descriptor.Descriptor(
+ name='AddressPB',
+ full_name='AddressPB',
+ filename='src/keas/pbstate/testclasses.proto',
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='line1', full_name='AddressPB.line1', index=0,
+ number=1, type=9, cpp_type=9, label=2,
+ default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='line2', full_name='AddressPB.line2', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='city', full_name='AddressPB.city', index=2,
+ number=3, type=9, cpp_type=9, label=2,
+ default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='state', full_name='AddressPB.state', index=3,
+ number=4, type=9, cpp_type=9, label=1,
+ default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='postal_code', full_name='AddressPB.postal_code', index=4,
+ number=5, type=9, cpp_type=9, label=1,
+ default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='country', full_name='AddressPB.country', index=5,
+ number=6, type=9, cpp_type=9, label=2,
+ default_value=unicode("United States", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[
+ ],
+ nested_types=[], # TODO(robinson): Implement.
+ enum_types=[
+ ],
+ options=None)
+
+
+_CONTACTPB = descriptor.Descriptor(
+ name='ContactPB',
+ full_name='ContactPB',
+ filename='src/keas/pbstate/testclasses.proto',
+ containing_type=None,
+ fields=[
+ descriptor.FieldDescriptor(
+ name='create_time', full_name='ContactPB.create_time', index=0,
+ number=1, type=4, cpp_type=4, label=2,
+ default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='name', full_name='ContactPB.name', index=1,
+ number=2, type=9, cpp_type=9, label=2,
+ default_value=unicode("", "utf-8"),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='address', full_name='ContactPB.address', index=2,
+ number=3, type=11, cpp_type=10, label=1,
+ default_value=None,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ descriptor.FieldDescriptor(
+ name='guardians', full_name='ContactPB.guardians', index=3,
+ number=4, type=11, cpp_type=10, label=3,
+ default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ options=None),
+ ],
+ extensions=[
+ ],
+ nested_types=[], # TODO(robinson): Implement.
+ enum_types=[
+ ],
+ options=None)
+
+
+_CONTACTPB.fields_by_name['address'].message_type = _ADDRESSPB
+_CONTACTPB.fields_by_name['guardians'].message_type = _REF
+
+class Ref(message.Message):
+ __metaclass__ = reflection.GeneratedProtocolMessageType
+ DESCRIPTOR = _REF
+
+class AddressPB(message.Message):
+ __metaclass__ = reflection.GeneratedProtocolMessageType
+ DESCRIPTOR = _ADDRESSPB
+
+class ContactPB(message.Message):
+ __metaclass__ = reflection.GeneratedProtocolMessageType
+ DESCRIPTOR = _CONTACTPB
+
Copied: keas.pbstate/tags/0.1/src/keas/pbstate/tests.py (from rev 94715, keas.pbstate/trunk/src/keas/pbstate/tests.py)
===================================================================
--- keas.pbstate/tags/0.1/src/keas/pbstate/tests.py (rev 0)
+++ keas.pbstate/tags/0.1/src/keas/pbstate/tests.py 2009-01-27 23:52:08 UTC (rev 95270)
@@ -0,0 +1,27 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import unittest
+
+from zope.testing import doctest
+
+def test_suite():
+ return unittest.TestSuite([
+ doctest.DocFileSuite(
+ 'README.txt',
+ optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS),
+ ])
+
+if __name__ == '__main__':
+ unittest.main()
More information about the Checkins
mailing list