[Checkins] SVN: zope.bforest/trunk/src/zope/bforest/ Add a period
kind of bforest with automatic rotation
Gary Poster
gary at zope.com
Tue Sep 12 13:17:59 EDT 2006
Log message for revision 70132:
Add a period kind of bforest with automatic rotation
Changed:
U zope.bforest/trunk/src/zope/bforest/bforest.py
A zope.bforest/trunk/src/zope/bforest/period.py
A zope.bforest/trunk/src/zope/bforest/period.txt
U zope.bforest/trunk/src/zope/bforest/tests.py
A zope.bforest/trunk/src/zope/bforest/utils.py
-=-
Modified: zope.bforest/trunk/src/zope/bforest/bforest.py
===================================================================
--- zope.bforest/trunk/src/zope/bforest/bforest.py 2006-09-12 16:30:04 UTC (rev 70131)
+++ zope.bforest/trunk/src/zope/bforest/bforest.py 2006-09-12 17:17:59 UTC (rev 70132)
@@ -82,7 +82,7 @@
buckets = self.buckets
if len(buckets) == 1:
res = buckets[0].keys()
- else:
+ else: # TODO: use multiunion for I* flavors
res = union(buckets[0], buckets[1])
for b in buckets[2:]:
res = union(res, b)
@@ -212,8 +212,9 @@
# bucket. If you want a dict, cast it to a dict, or if you want
# another one of these but with all of the keys in the first bucket,
# call obj.__class__(obj)
- copy = self.__class__(count=0)
- copy.buckets = [self._treeclass(t) for t in self.buckets]
+ copy = self.__class__(count=len(self.buckets))
+ for i in range(len(self.buckets)):
+ copy.buckets[i].update(self.buckets[i])
return copy
def clear(self):
Added: zope.bforest/trunk/src/zope/bforest/period.py
===================================================================
--- zope.bforest/trunk/src/zope/bforest/period.py 2006-09-12 16:30:04 UTC (rev 70131)
+++ zope.bforest/trunk/src/zope/bforest/period.py 2006-09-12 17:17:59 UTC (rev 70132)
@@ -0,0 +1,60 @@
+import datetime
+from BTrees import IOBTree, OOBTree, OIBTree, IIBTree
+from zope.bforest import bforest, utils
+
+def mutating(name):
+ def mutate(self, *args, **kwargs):
+ if (datetime.datetime.now(utils.UTC) -
+ self.last_rotation) >= self._inner_period:
+ self.rotateBucket()
+ return getattr(super(Abstract, self), name)(*args, **kwargs)
+ return mutate
+
+class Abstract(bforest.AbstractBForest):
+ def __init__(self, period, d=None, count=2):
+ super(Abstract, self).__init__(d, count)
+ self.period = period
+ self.last_rotation = datetime.datetime.now(utils.UTC)
+
+ _inner_period = _period = None
+ def period(self, value):
+ self._period = value
+ self._inner_period = self._period / len(self.buckets)
+ period = property(
+ lambda self: self._period, period)
+
+ def copy(self):
+ # this makes an exact copy, including the individual state of each
+ # bucket. If you want a dict, cast it to a dict, or if you want
+ # another one of these but with all of the keys in the first bucket,
+ # call obj.__class__(obj)
+ copy = self.__class__(self.period, count=len(self.buckets))
+ for i in range(len(self.buckets)):
+ copy.buckets[i].update(self.buckets[i])
+ return copy
+
+ def rotateBucket(self):
+ super(Abstract, self).rotateBucket()
+ self.last_rotation = datetime.datetime.now(utils.UTC)
+
+ __setitem__ = mutating('__setitem__')
+ __delitem__ = mutating('__delitem__')
+ pop = mutating('pop')
+ popitem = mutating('popitem')
+ update = mutating('update')
+
+class IOBForest(Abstract):
+ _treemodule = IOBTree
+ _treeclass = IOBTree.IOBTree
+
+class OIBForest(Abstract):
+ _treemodule = OIBTree
+ _treeclass = OIBTree.OIBTree
+
+class OOBForest(Abstract):
+ _treemodule = OOBTree
+ _treeclass = OOBTree.OOBTree
+
+class IIBForest(Abstract):
+ _treemodule = IIBTree
+ _treeclass = IIBTree.IIBTree
Added: zope.bforest/trunk/src/zope/bforest/period.txt
===================================================================
--- zope.bforest/trunk/src/zope/bforest/period.txt 2006-09-12 16:30:04 UTC (rev 70131)
+++ zope.bforest/trunk/src/zope/bforest/period.txt 2006-09-12 17:17:59 UTC (rev 70132)
@@ -0,0 +1,87 @@
+The bforests in the period module have simple policies for automatic
+tree rotation based on time. Each take one additional argument beyond
+the usual bforest initialization: period. This is effectively the
+minimum amount of time a given value will be kept in the bforest.
+
+For example, consider a period bforest with three trees and a period of three
+hours.
+
+As with normal bforests, period bforests come in four flavors:
+Integer-Integer (IIBForest), Integer-Object (IOBForest), Object-Integer
+(OIBForest), and Object-Object (OOBForest). The examples here will deal
+with them in the abstract: we will create classes from the imaginary and
+representative BForest class, and generate keys from KeyGenerator and
+values from ValueGenerator. From the examples you should be able to
+extrapolate usage of all four types.
+
+We will also imagine that we control the time with a function called setNow.
+
+ >>> import datetime
+ >>> from zope.bforest import utils
+ >>> setNow(datetime.datetime(2006, 9, 11, 22, 51, tzinfo=utils.UTC))
+ >>> d = BForest(datetime.timedelta(hours=3), count=3)
+ >>> d.last_rotation
+ datetime.datetime(2006, 9, 11, 22, 51, tzinfo=<UTC>)
+
+Now let's put in some keys and advance time.
+
+ >>> first_hour = {KeyGenerator(): ValueGenerator()}
+ >>> d.update(first_hour)
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> second_hour = {KeyGenerator(): ValueGenerator()}
+ >>> d.update(second_hour)
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> third_hour = {KeyGenerator(): ValueGenerator()}
+ >>> d.update(third_hour)
+ >>> current = first_hour.copy()
+ >>> current.update(second_hour)
+ >>> current.update(third_hour)
+ >>> d == current
+ True
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> d == current
+ True
+ >>> fourth_hour = {KeyGenerator(): ValueGenerator()}
+ >>> d.update(fourth_hour)
+ >>> current.update(fourth_hour)
+ >>> del current[first_hour.keys()[0]]
+ >>> d == current
+ True
+
+Updating isn't the only way to rotate the trees though. Basically
+any non-clear mutating operation will cause a rotation: __setitem__,
+update, __delitem__, pop, and popitem. We've seen update; here are examples
+of the rest.
+
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> fifth_hour = {KeyGenerator(): ValueGenerator()}
+ >>> d[fifth_hour.keys()[0]] = fifth_hour.values()[0] # __setitem__
+ >>> current.update(fifth_hour)
+ >>> del current[second_hour.keys()[0]]
+ >>> d == current
+ True
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> del d[fifth_hour.keys()[0]] # __delitem__
+ >>> d == fourth_hour
+ True
+ >>> sixth_hour = {KeyGenerator(): ValueGenerator()}
+ >>> d[sixth_hour.keys()[0]] = sixth_hour.values()[0]
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> d.pop(sixth_hour.keys()[0]) == sixth_hour.values()[0] # pop
+ True
+ >>> d == {}
+ True
+ >>> seventh_hour = {KeyGenerator(): ValueGenerator()}
+ >>> d[seventh_hour.keys()[0]] = seventh_hour.values()[0]
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> d.rotateBucket()
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> ninth_hour = {KeyGenerator(): ValueGenerator()}
+ >>> d[ninth_hour.keys()[0]] = ninth_hour.values()[0]
+ >>> setNow(datetime.datetime.now(utils.UTC) + datetime.timedelta(hours=1))
+ >>> out = dict((d.popitem(),))
+ >>> out == ninth_hour
+ True
+ >>> d == {}
+ True
+
Property changes on: zope.bforest/trunk/src/zope/bforest/period.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Modified: zope.bforest/trunk/src/zope/bforest/tests.py
===================================================================
--- zope.bforest/trunk/src/zope/bforest/tests.py 2006-09-12 16:30:04 UTC (rev 70131)
+++ zope.bforest/trunk/src/zope/bforest/tests.py 2006-09-12 17:17:59 UTC (rev 70132)
@@ -16,7 +16,9 @@
"""
import unittest
+import datetime
from zope import bforest
+from zope.bforest import period
from zope.testing import doctest
def StringGenerator(src='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'):
@@ -33,34 +35,97 @@
yield number
number += interval
+oldDatetime = datetime.datetime
+class _datetime(oldDatetime):
+ _now = None
+ @classmethod
+ def now(klass, tzinfo=None):
+ if tzinfo is None:
+ return klass._now.replace(tzinfo=None)
+ else:
+ return klass._now.astimezone(tzinfo)
+ def astimezone(self, tzinfo):
+ return _datetime(
+ *super(_datetime,self).astimezone(tzinfo).__reduce__()[1])
+ def replace(self, *args, **kwargs):
+ return _datetime(
+ *super(_datetime,self).replace(
+ *args, **kwargs).__reduce__()[1])
+ def __repr__(self):
+ raw = super(_datetime, self).__repr__()
+ return "datetime.datetime%s" % (
+ raw[raw.index('('):],)
+ def __reduce__(self):
+ return (argh, super(_datetime, self).__reduce__()[1])
+def setNow(dt):
+ _datetime._now = _datetime(*dt.__reduce__()[1])
+def argh(*args, **kwargs):
+ return _datetime(*args, **kwargs)
+
+def setUp(test):
+
+ datetime.datetime = _datetime
+ test.globs['setNow'] = setNow
+
+def tearDown(test):
+ datetime.datetime = oldDatetime # cleanup
+
def test_suite():
suite = unittest.TestSuite()
numgen = iter(NumberGenerator()).next
strgen = iter(StringGenerator()).next
suite.addTest(
doctest.DocFileSuite(
- 'bforest.txt',
+ 'bforest.txt',
globs={'BForest': bforest.IOBForest,
'KeyGenerator': numgen,
'ValueGenerator': strgen}))
suite.addTest(
doctest.DocFileSuite(
- 'bforest.txt',
+ 'bforest.txt',
globs={'BForest': bforest.OIBForest,
'KeyGenerator': strgen,
'ValueGenerator': numgen}))
suite.addTest(
doctest.DocFileSuite(
- 'bforest.txt',
+ 'bforest.txt',
globs={'BForest': bforest.IIBForest,
'KeyGenerator': numgen,
'ValueGenerator': numgen}))
suite.addTest(
doctest.DocFileSuite(
- 'bforest.txt',
+ 'bforest.txt',
globs={'BForest': bforest.OOBForest,
'KeyGenerator': strgen,
'ValueGenerator': strgen}))
+ suite.addTest(
+ doctest.DocFileSuite(
+ 'period.txt',
+ globs={'BForest': period.IOBForest,
+ 'KeyGenerator': numgen,
+ 'ValueGenerator': strgen},
+ setUp=setUp, tearDown=tearDown))
+ suite.addTest(
+ doctest.DocFileSuite(
+ 'period.txt',
+ globs={'BForest': period.OIBForest,
+ 'KeyGenerator': strgen,
+ 'ValueGenerator': numgen},
+ setUp=setUp, tearDown=tearDown))
+ suite.addTest(
+ doctest.DocFileSuite(
+ 'period.txt',
+ globs={'BForest': period.IIBForest,
+ 'KeyGenerator': numgen,
+ 'ValueGenerator': numgen},
+ setUp=setUp, tearDown=tearDown))
+ suite.addTest(
+ doctest.DocFileSuite(
+ 'period.txt',
+ globs={'BForest': period.OOBForest,
+ 'KeyGenerator': strgen,
+ 'ValueGenerator': strgen},
+ setUp=setUp, tearDown=tearDown))
return suite
if __name__ == '__main__':
Added: zope.bforest/trunk/src/zope/bforest/utils.py
===================================================================
--- zope.bforest/trunk/src/zope/bforest/utils.py 2006-09-12 16:30:04 UTC (rev 70131)
+++ zope.bforest/trunk/src/zope/bforest/utils.py 2006-09-12 17:17:59 UTC (rev 70132)
@@ -0,0 +1,82 @@
+try:
+ from pytz import UTC, _UTC # we import _UTC so that pickles made by the
+ # fallback code below can still be reinstantiated if pytz is added in.
+except ImportError:
+ import datetime
+ class UTC(datetime.tzinfo):
+ """UTC
+
+ Identical to the reference UTC implementation given in Python docs except
+ that it unpickles using the single module global instance defined beneath
+ this class declaration.
+
+ Also contains extra attributes and methods to match other pytz tzinfo
+ instances.
+ """
+ zone = "UTC"
+
+ def utcoffset(self, dt):
+ return ZERO
+
+ def tzname(self, dt):
+ return "UTC"
+
+ def dst(self, dt):
+ return ZERO
+
+ def __reduce__(self):
+ return _UTC, ()
+
+ def localize(self, dt, is_dst=False):
+ '''Convert naive time to local time'''
+ if dt.tzinfo is not None:
+ raise ValueError, 'Not naive datetime (tzinfo is already set)'
+ return dt.replace(tzinfo=self)
+
+ def normalize(self, dt, is_dst=False):
+ '''Correct the timezone information on the given datetime'''
+ if dt.tzinfo is None:
+ raise ValueError, 'Naive time - no tzinfo set'
+ return dt.replace(tzinfo=self)
+
+ def __repr__(self):
+ return "<UTC>"
+
+ def __str__(self):
+ return "UTC"
+
+
+ UTC = utc = UTC() # UTC is a singleton
+
+
+ def _UTC():
+ """Factory function for utc unpickling.
+
+ Makes sure that unpickling a utc instance always returns the same
+ module global.
+
+ These examples belong in the UTC class above, but it is obscured; or in
+ the README.txt, but we are not depending on Python 2.4 so integrating
+ the README.txt examples with the unit tests is not trivial.
+
+ >>> import datetime, pickle
+ >>> dt = datetime.datetime(2005, 3, 1, 14, 13, 21, tzinfo=utc)
+ >>> naive = dt.replace(tzinfo=None)
+ >>> p = pickle.dumps(dt, 1)
+ >>> naive_p = pickle.dumps(naive, 1)
+ >>> len(p), len(naive_p), len(p) - len(naive_p)
+ (60, 43, 17)
+ >>> new = pickle.loads(p)
+ >>> new == dt
+ True
+ >>> new is dt
+ False
+ >>> new.tzinfo is dt.tzinfo
+ True
+ >>> utc is UTC is timezone('UTC')
+ True
+ >>> utc is timezone('GMT')
+ False
+ """
+ return utc
+ _UTC.__safe_for_unpickling__ = True
More information about the Checkins
mailing list