[Checkins] SVN: z3c.dobbin/trunk/ Polymorphic attributes are now
stored using pickles.
Malthe Borch
mborch at gmail.com
Thu Jul 17 12:55:06 EDT 2008
Log message for revision 88457:
Polymorphic attributes are now stored using pickles.
Changed:
U z3c.dobbin/trunk/CHANGES.txt
U z3c.dobbin/trunk/setup.py
U z3c.dobbin/trunk/src/z3c/dobbin/README.txt
U z3c.dobbin/trunk/src/z3c/dobbin/bootstrap.py
U z3c.dobbin/trunk/src/z3c/dobbin/mapper.py
U z3c.dobbin/trunk/src/z3c/dobbin/session.py
U z3c.dobbin/trunk/src/z3c/dobbin/soup.py
U z3c.dobbin/trunk/src/z3c/dobbin/zs2sa.py
-=-
Modified: z3c.dobbin/trunk/CHANGES.txt
===================================================================
--- z3c.dobbin/trunk/CHANGES.txt 2008-07-17 16:05:56 UTC (rev 88456)
+++ z3c.dobbin/trunk/CHANGES.txt 2008-07-17 16:55:04 UTC (rev 88457)
@@ -1,9 +1,15 @@
Change log
==========
-0.3.2 dev
----------
+0.3.2
+-----
+- Use pickles to store polymorphic attributes; there's no benefit in
+ using native columns for amorphic data.
+
+- Dobbin now uses ``zope.sqlalchemy`` for transaction and session
+ glue.
+
0.3.1
-----
Modified: z3c.dobbin/trunk/setup.py
===================================================================
--- z3c.dobbin/trunk/setup.py 2008-07-17 16:05:56 UTC (rev 88456)
+++ z3c.dobbin/trunk/setup.py 2008-07-17 16:55:04 UTC (rev 88457)
@@ -18,7 +18,7 @@
return "\n".join((open(f).read() for f in files))
setup(name='z3c.dobbin',
- version='0.3.2dev',
+ version='0.3.2',
license='ZPL',
author = "Malthe Borch, Stefan Eletzhofer and the Zope Community",
author_email = "zope-dev at zope.org",
Modified: z3c.dobbin/trunk/src/z3c/dobbin/README.txt
===================================================================
--- z3c.dobbin/trunk/src/z3c/dobbin/README.txt 2008-07-17 16:05:56 UTC (rev 88456)
+++ z3c.dobbin/trunk/src/z3c/dobbin/README.txt 2008-07-17 16:55:04 UTC (rev 88457)
@@ -485,14 +485,28 @@
... title=u"Discographies by artist",
... value_type=schema.List())
-Polymorphic relations
----------------------
+Polymorphic structures
+----------------------
-We can create relations to instances as well as immutable objects
-(rocks).
+We can use weak typing to store (almost) any kind of structure. Values
+are kept as Python pickles.
+ >>> class IPolyFavorite(interface.Interface):
+ ... item = interface.Attribute(u"Any kind of favorite")
+
+ >>> __builtin__.IPolyFavorite = IPolyFavorite
+ >>> favorite = create(IPolyFavorite)
+
+A transaction hook makes sure that assigned values are transient
+during a session.
+
+ >>> obj = object()
+ >>> favorite.item = obj
+ >>> favorite.item is obj
+ True
+
Integers, floats and unicode strings are straight-forward.
-
+
>>> favorite.item = 42; transaction.commit()
>>> favorite.item
42
@@ -511,7 +525,7 @@
>>> str(favorite.item)
'My favorite number is 42.'
-Or sequences of relations.
+Or sequences of items.
>>> favorite.item = (u"green", u"blue", u"red"); transaction.commit()
>>> favorite.item
@@ -525,23 +539,51 @@
{u'blue': 255, u'green': 65280, u'red': 16711680}
>>> favorite.item[u"black"] = 0x000000
- >>> favorite.item
- {u'blue': 255, u'green': 65280, u'black': 0, u'red': 16711680}
+ >>> sorted(favorite.item.items())
+ [(u'black', 0), (u'blue', 255), (u'green', 65280), (u'red', 16711680)]
+
+We do need explicitly set the dirty bit of this instance.
+
+ >>> favorite.item = favorite.item
+ >>> transaction.commit()
+
+Clear the object cache and verify value.
+ >>> del favorite._v_cached_item_pickle
+ >>> sorted(favorite.item.items())
+ [(u'black', 0), (u'blue', 255), (u'green', 65280), (u'red', 16711680)]
+
When we create relations to mutable objects, a hook is made into the
transaction machinery to keep track of the pending state.
- >>> some_list = [u"green", u"blue", u"red"]; transaction.commit()
+ >>> some_list = [u"green", u"blue"]
>>> favorite.item = some_list
+ >>> some_list.append(u"red"); transaction.commit()
>>> favorite.item
[u'green', u'blue', u'red']
-Amorphic relations.
+Amorphic structures.
>>> favorite.item = ((1, u"green"), (2, u"blue"), (3, u"red")); transaction.commit()
>>> favorite.item
((1, u'green'), (2, u'blue'), (3, u'red'))
+Structures involving relations to other instances.
+
+ >>> favorite.item = vinyl; transaction.commit()
+ >>> del favorite._v_cached_item_pickle
+ >>> favorite.item
+ <Vinyl Diana Ross and The Supremes: Taking Care of Business (@ 45 RPM)>
+
+Self-referencing works because polymorphic attributes are lazy.
+
+ >>> session.save(favorite)
+
+ >>> favorite.item = favorite; transaction.commit()
+ >>> del favorite._v_cached_item_pickle
+ >>> favorite.item
+ <Mapper (__builtin__.IPolyFavorite) at ...>
+
Security
--------
Modified: z3c.dobbin/trunk/src/z3c/dobbin/bootstrap.py
===================================================================
--- z3c.dobbin/trunk/src/z3c/dobbin/bootstrap.py 2008-07-17 16:05:56 UTC (rev 88456)
+++ z3c.dobbin/trunk/src/z3c/dobbin/bootstrap.py 2008-07-17 16:55:04 UTC (rev 88457)
@@ -3,17 +3,19 @@
import sqlalchemy as rdb
from sqlalchemy import orm
-from z3c.saconfig.interfaces import IEngineFactory
+from z3c.saconfig import Session
+import soup
import relations
+import interfaces
class UUID(rdb.types.TypeEngine):
def get_col_spec(self):
return "UUID"
def bootstrapDatabaseEngine(event=None):
- factory = component.getUtility(IEngineFactory)
- engine = factory()
+ session = Session()
+ engine = session.bind
engine.metadata = metadata = rdb.MetaData(engine)
# setup metadata
@@ -63,5 +65,14 @@
class Soup(object):
"""Soup class.
- This stub is used as the mapper for the soup table.
+ This is the base object of all mappers.
"""
+
+ def __cmp__(self, other):
+ if interfaces.IMapped.providedBy(other):
+ return cmp(self.id, other.id)
+
+ return -1
+
+ def __reduce__(self):
+ return (soup.lookup, (self.uuid,))
Modified: z3c.dobbin/trunk/src/z3c/dobbin/mapper.py
===================================================================
--- z3c.dobbin/trunk/src/z3c/dobbin/mapper.py 2008-07-17 16:05:56 UTC (rev 88456)
+++ z3c.dobbin/trunk/src/z3c/dobbin/mapper.py 2008-07-17 16:55:04 UTC (rev 88457)
@@ -16,108 +16,16 @@
from sqlalchemy.orm.attributes import proxied_attribute_factory
from z3c.saconfig import Session
-from zs2sa import FieldTranslator, StringTranslator
from uuid import uuid1
from random import randint
+from itertools import chain
import bootstrap
-import relations
-import collections
-import factory
import soup
-
-from itertools import chain
-
+import zs2sa
import types
-class ObjectTranslator(object):
- def __init__(self, column_type=None):
- self.column_type = column_type
-
- def __call__(self, field, metadata):
- return rdb.Column(
- field.__name__+'_uuid', bootstrap.UUID, nullable=False)
-
-class ObjectProperty(object):
- """Object property.
-
- We're not checking type here, because we'll only be creating
- relations to items that are joined with the soup.
- """
-
- def __call__(self, field, column, metadata):
- relation = relations.RelationProperty(field)
-
- return {
- field.__name__: relation,
- relation.name: orm.relation(
- bootstrap.Soup,
- primaryjoin=bootstrap.Soup.c.uuid==column,
- foreign_keys=[column],
- enable_typechecks=False,
- lazy=True)
- }
-
-class CollectionProperty(object):
- """A collection property."""
-
- collection_class = None
- relation_class = None
-
- def __call__(self, field, column, metadata):
- return {
- field.__name__: orm.relation(
- self.relation_class,
- primaryjoin=self.getPrimaryJoinCondition(),
- collection_class=self.collection_class,
- enable_typechecks=False)
- }
-
- def getPrimaryJoinCondition(self):
- return NotImplementedError("Must be implemented by subclass.")
-
-class ListProperty(CollectionProperty):
- collection_class = collections.OrderedList
- relation_class = relations.OrderedRelation
-
- def getPrimaryJoinCondition(self):
- return bootstrap.Soup.c.uuid==relations.OrderedRelation.c.left
-
-class TupleProperty(ListProperty):
- collection_class = collections.Tuple
-
-class DictProperty(CollectionProperty):
- collection_class = collections.Dict
- relation_class = relations.KeyRelation
-
- def getPrimaryJoinCondition(self):
- return bootstrap.Soup.c.uuid==relations.KeyRelation.c.left
-
-fieldmap = {
- schema.ASCII: StringTranslator(),
- schema.ASCIILine: StringTranslator(),
- schema.Bool: FieldTranslator(rdb.BOOLEAN),
- schema.Bytes: FieldTranslator(rdb.BLOB),
- schema.BytesLine: FieldTranslator(rdb.CLOB),
- schema.Choice: StringTranslator(rdb.Unicode),
- schema.Date: FieldTranslator(rdb.DATE),
- schema.Dict: (None, DictProperty()),
- schema.DottedName: StringTranslator(),
- schema.Float: FieldTranslator(rdb.Float),
- schema.Id: StringTranslator(rdb.Unicode),
- schema.Int: FieldTranslator(rdb.Integer),
- schema.List: (None, ListProperty()),
- schema.Tuple: (None, TupleProperty()),
- schema.Object: (ObjectTranslator(), ObjectProperty()),
- schema.Password: StringTranslator(rdb.Unicode),
- schema.SourceText: StringTranslator(rdb.UnicodeText),
- schema.Text: StringTranslator(rdb.UnicodeText),
- schema.TextLine: StringTranslator(rdb.Unicode),
- schema.URI: StringTranslator(rdb.Unicode),
- interface.interface.Method: None,
-}
-
def decode(name):
return resolve(name.replace(':', '.'))
@@ -150,10 +58,10 @@
metadata = engine.metadata
exclude = ['__name__']
-
+
# expand specification
if interface.interfaces.IInterface.providedBy(spec):
- ifaces = set([spec.get(name).interface for name in schema.getFields(spec)])
+ ifaces = set([spec.get(name).interface for name in spec.names(True)])
kls = object
else:
implemented = interface.implementedBy(spec)
@@ -165,6 +73,8 @@
if isinstance(value, property):
exclude.append(name)
+ assert ifaces, "Specification must declare at least one field."
+
# create joined table
properties = {}
first_table = None
@@ -190,12 +100,6 @@
self.uuid = "{%s}" % uuid1()
self.spec = self.__spec__
- def __cmp__(self, other):
- if IMapped.providedBy(other):
- return cmp(self.id, other.id)
- else:
- return -1
-
# if the specification is an interface class, try to look up a
# security checker and define it on the mapper
if interface.interfaces.IInterface.providedBy(spec):
@@ -243,6 +147,7 @@
def removeMapper(spec):
del spec.mapper
interface.noLongerProvides(spec, IMapped)
+
def getTable(iface, metadata, ignore=()):
name = encode(iface)
@@ -257,7 +162,7 @@
continue
try:
- factories = fieldmap[type(field)]
+ factories = zs2sa.fieldmap[type(field)]
except KeyError:
# raise NotImplementedError("Field type unsupported (%s)." % field)
continue
Modified: z3c.dobbin/trunk/src/z3c/dobbin/session.py
===================================================================
--- z3c.dobbin/trunk/src/z3c/dobbin/session.py 2008-07-17 16:05:56 UTC (rev 88456)
+++ z3c.dobbin/trunk/src/z3c/dobbin/session.py 2008-07-17 16:55:04 UTC (rev 88457)
@@ -2,32 +2,32 @@
from z3c.saconfig import Session
-import soup
import transaction
-def beforeCommitHook(obj, uuid):
- # unset pending state
- session = Session()
- del session._d_pending[uuid]
+def COPY_CONCRETE_TO_INSTANCE(uuid):
+ return COPY_CONCRETE_TO_INSTANCE, uuid
- # build instance
- instance = soup.lookup(uuid)
+def COPY_VALUE_TO_INSTANCE(uuid, name):
+ return COPY_VALUE_TO_INSTANCE, uuid, name
- # update attributes
- soup.update(instance, obj)
-
-def registerObject(obj, uuid):
+def addBeforeCommitHook(token, value, hook):
session = Session()
try:
pending = session._d_pending.keys()
except AttributeError:
pending = ()
-
- if obj not in pending:
+
+ if token in pending:
+ session._d_pending[token] = value
+ else:
try:
- session._d_pending[uuid] = obj
+ session._d_pending[token] = value
except AttributeError:
- session._d_pending = {uuid: obj}
+ session._d_pending = {token: value}
- transaction.get().addBeforeCommitHook(beforeCommitHook, (obj, uuid))
+ def remove_and_call(*args):
+ hook(*args)
+ del session._d_pending[token]
+
+ transaction.get().addBeforeCommitHook(remove_and_call, ())
Modified: z3c.dobbin/trunk/src/z3c/dobbin/soup.py
===================================================================
--- z3c.dobbin/trunk/src/z3c/dobbin/soup.py 2008-07-17 16:05:56 UTC (rev 88456)
+++ z3c.dobbin/trunk/src/z3c/dobbin/soup.py 2008-07-17 16:55:04 UTC (rev 88457)
@@ -3,27 +3,26 @@
from interfaces import IMapper
from interfaces import IMapped
-from session import registerObject
-
from zope.dottedname.resolve import resolve
from z3c.saconfig import Session
import factory
import bootstrap
import interfaces
+import session as tx
import types
BASIC_TYPES = (int, float, str, unicode, tuple, list, set, dict)
+IMMUTABLE_TYPES = (int, float, str, unicode, tuple)
-IMMUTABLE_TYPES = (int, float, str, unicode, tuple)
-
def lookup(uuid, ignore_pending=False):
session = Session()
# check if object is in pending session objects
if not ignore_pending:
try:
- return session._d_pending[uuid]
+ token = tx.COPY_CONCRETE_TO_INSTANCE(uuid)
+ return session._d_pending[token]
except (AttributeError, KeyError):
pass
@@ -56,7 +55,18 @@
# register mutable objects with transaction manager
if type(item) not in IMMUTABLE_TYPES:
- registerObject(item, instance.uuid)
+ uuid = instance.uuid
+
+ def copy_concrete_to_mapped():
+ # build instance
+ instance = lookup(uuid)
+
+ # update attributes
+ update(instance, item)
+
+ # add transaction hook
+ tx.addBeforeCommitHook(
+ tx.COPY_CONCRETE_TO_INSTANCE(uuid), item, copy_concrete_to_mapped)
return instance
Modified: z3c.dobbin/trunk/src/z3c/dobbin/zs2sa.py
===================================================================
--- z3c.dobbin/trunk/src/z3c/dobbin/zs2sa.py 2008-07-17 16:05:56 UTC (rev 88456)
+++ z3c.dobbin/trunk/src/z3c/dobbin/zs2sa.py 2008-07-17 16:55:04 UTC (rev 88457)
@@ -1,7 +1,7 @@
##############################################################################
#
-# Copyright (c) 2008 Kapil Thangavelu <kapil.foss at gmail.com>
-# All Rights Reserved.
+# Parts of this module is copyright (c) 2008 Kapil Thangavelu
+# <kapil.foss at gmail.com>. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
@@ -11,15 +11,21 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-"""
-Zope3 Schemas to SQLAlchemy
-$Id: sa2zs.py 1710 2006-10-26 17:39:37Z hazmat $
-"""
+from zope import interface
+from zope import schema
-from zope import schema
import sqlalchemy as rdb
+from sqlalchemy import orm
+from z3c.saconfig import Session
+import session as tx
+import bootstrap
+import relations
+import collections
+
+import cPickle as Pickle
+
class FieldTranslator( object ):
""" Translate a zope schema field to an sa column
"""
@@ -57,31 +63,146 @@
return d
class ObjectTranslator(object):
-
def __call__(self, field, metadata):
- table = transmute(field.schema, metadata)
- pk = get_pk_name(table.name)
- field_name = "%s.%s" % table.name, pk
- return rdb.Column(pk, rdb.Integer, rdb.ForeignKey(field_name),
- nullable=False)
+ return rdb.Column(
+ field.__name__+'_uuid', bootstrap.UUID, nullable=False)
+class PickleTranslator(object):
+ def __call__(self, field, metadata):
+ return rdb.Column(
+ field.__name__+'_pickle', rdb.BLOB, nullable=True)
+
+class PickleProperty(property):
+ def __init__(self, name):
+ self.name = name
+ self.cache = '_v_cached_'+name
+ property.__init__(self, self._get, self._set)
+
+ def _get(self, obj, type=None):
+ session = Session()
+ name = self.name
+ cache = self.cache
+
+ token = tx.COPY_VALUE_TO_INSTANCE(obj.uuid, name)
+
+ # check pending objects
+ try:
+ return session._d_pending[token]
+ except (AttributeError, KeyError):
+ pass
+
+ # check object cache
+ value = getattr(obj, cache, None)
+ if value is not None:
+ return value
+
+ # load pickle
+ pickle = getattr(obj, name)
+ value = pickle and Pickle.loads(pickle)
+
+ # update cache
+ if value is not None:
+ setattr(obj, cache, value)
+
+ return value
+
+ def _set(self, obj, value):
+ name = self.name
+ token = tx.COPY_VALUE_TO_INSTANCE(obj.uuid, name)
+
+ def copy_value_to_instance():
+ value = Session()._d_pending[token]
+ pickle = Pickle.dumps(value)
+ setattr(obj, name, pickle)
+
+ # add transaction hook
+ tx.addBeforeCommitHook(
+ token, value, copy_value_to_instance)
+
+ # update cache
+ if value is not None:
+ setattr(obj, self.cache, value)
+
+class PicklePropertyFactory(object):
+ def __call__(self, field, column, metadata):
+ return {field.__name__: PickleProperty(column.name)}
+
+class ObjectProperty(object):
+ """Object property.
+
+ We're not checking type here, because we'll only be creating
+ relations to items that are joined with the soup.
+ """
+
+ def __call__(self, field, column, metadata):
+ relation = relations.RelationProperty(field)
+
+ return {
+ field.__name__: relation,
+ relation.name: orm.relation(
+ bootstrap.Soup,
+ primaryjoin=bootstrap.Soup.c.uuid==column,
+ foreign_keys=[column],
+ enable_typechecks=False,
+ lazy=True)
+ }
+
+class CollectionProperty(object):
+ """A collection property."""
+
+ collection_class = None
+ relation_class = None
+
+ def __call__(self, field, column, metadata):
+ return {
+ field.__name__: orm.relation(
+ self.relation_class,
+ primaryjoin=self.getPrimaryJoinCondition(),
+ collection_class=self.collection_class,
+ enable_typechecks=False)
+ }
+
+ def getPrimaryJoinCondition(self):
+ return NotImplementedError("Must be implemented by subclass.")
+
+class ListProperty(CollectionProperty):
+ collection_class = collections.OrderedList
+ relation_class = relations.OrderedRelation
+
+ def getPrimaryJoinCondition(self):
+ return bootstrap.Soup.c.uuid==relations.OrderedRelation.c.left
+
+class TupleProperty(ListProperty):
+ collection_class = collections.Tuple
+
+class DictProperty(CollectionProperty):
+ collection_class = collections.Dict
+ relation_class = relations.KeyRelation
+
+ def getPrimaryJoinCondition(self):
+ return bootstrap.Soup.c.uuid==relations.KeyRelation.c.left
+
fieldmap = {
- 'ASCII': StringTranslator(),
- 'ASCIILine': StringTranslator(),
- 'Bool': FieldTranslator(rdb.BOOLEAN),
- 'Bytes': FieldTranslator(rdb.BLOB),
- 'BytesLine': FieldTranslator(rdb.BLOB),
- 'Choice': StringTranslator(),
- 'Date': FieldTranslator(rdb.DATE),
- 'Datetime': FieldTranslator(rdb.DATE),
- 'DottedName': StringTranslator(),
- 'Float': FieldTranslator(rdb.Float),
- 'Id': StringTranslator(),
- 'Int': FieldTranslator(rdb.Integer),
- 'Object': ObjectTranslator(),
- 'Password': StringTranslator(),
- 'SourceText': StringTranslator(),
- 'Text': StringTranslator(),
- 'TextLine': StringTranslator(),
- 'URI': StringTranslator(),
+ schema.ASCII: StringTranslator(),
+ schema.ASCIILine: StringTranslator(),
+ schema.Bool: FieldTranslator(rdb.BOOLEAN),
+ schema.Bytes: FieldTranslator(rdb.BLOB),
+ schema.BytesLine: FieldTranslator(rdb.CLOB),
+ schema.Choice: StringTranslator(rdb.Unicode),
+ schema.Date: FieldTranslator(rdb.DATE),
+ schema.Dict: (None, DictProperty()),
+ schema.DottedName: StringTranslator(),
+ schema.Float: FieldTranslator(rdb.Float),
+ schema.Id: StringTranslator(rdb.Unicode),
+ schema.Int: FieldTranslator(rdb.Integer),
+ schema.List: (None, ListProperty()),
+ schema.Tuple: (None, TupleProperty()),
+ schema.Object: (ObjectTranslator(), ObjectProperty()),
+ schema.Password: StringTranslator(rdb.Unicode),
+ schema.SourceText: StringTranslator(rdb.UnicodeText),
+ schema.Text: StringTranslator(rdb.UnicodeText),
+ schema.TextLine: StringTranslator(rdb.Unicode),
+ schema.URI: StringTranslator(rdb.Unicode),
+ interface.Attribute: (PickleTranslator(), PicklePropertyFactory()),
+ interface.interface.Method: None,
}
More information about the Checkins
mailing list