[Checkins] SVN: appendonly/trunk/ Added an 'Archive' class
Tres Seaver
tseaver at palladion.com
Tue Feb 21 21:47:09 UTC 2012
Log message for revision 124448:
Added an 'Archive' class
Instances of this class are intended to support long-term storage of the
layer data pruned from a companion 'AppendStack'.
Changed:
U appendonly/trunk/CHANGES.txt
U appendonly/trunk/README.txt
U appendonly/trunk/appendonly/__init__.py
U appendonly/trunk/appendonly/tests.py
U appendonly/trunk/setup.py
-=-
Modified: appendonly/trunk/CHANGES.txt
===================================================================
--- appendonly/trunk/CHANGES.txt 2012-02-21 20:29:28 UTC (rev 124447)
+++ appendonly/trunk/CHANGES.txt 2012-02-21 21:47:08 UTC (rev 124448)
@@ -1,10 +1,11 @@
``appendonly`` Changelog
========================
-0.9.1 (unreleased)
+0.10 (unreleased)
------------------
-- TBD
+- Added an 'Archive' class, intended to support long-term storage of the
+ layer data pruned from an 'AppendStack'.
0.9 (2010-08-09)
Modified: appendonly/trunk/README.txt
===================================================================
--- appendonly/trunk/README.txt 2012-02-21 20:29:28 UTC (rev 124447)
+++ appendonly/trunk/README.txt 2012-02-21 21:47:08 UTC (rev 124448)
@@ -31,3 +31,28 @@
The stack is implemented as a single persistent record, with custom
ZODB conflict resolution code.
+
+
+``appendonly.Archive``
+----------------------
+
+This class provides a linked list of separately-persisted copies of layer
+data pruned from an ``AppendStack``. The intended use would be something
+like:
+
+ from appendonly import AppendStack
+ from appendonly import Archive
+
+ class RecentItems(object):
+ def __init__(self):
+ self._recent = AppendStack()
+ self._archive = Archive()
+
+ def pushItem(object):
+ self._stack.push(object, self._archive.addLayer)
+
+ def __iter__(self):
+ for generation, index, item in self._stack:
+ yield item
+ for generation, index, item in self._archive:
+ yield items
Modified: appendonly/trunk/appendonly/__init__.py
===================================================================
--- appendonly/trunk/appendonly/__init__.py 2012-02-21 20:29:28 UTC (rev 124447)
+++ appendonly/trunk/appendonly/__init__.py 2012-02-21 21:47:08 UTC (rev 124448)
@@ -21,15 +21,8 @@
pass
-class _Layer(object):
- """ Append-only list with maximum length.
-
- - Raise `_LayerFull` on attempts to exceed that length.
-
- - Iteration occurs in reverse order of appends, and yields (index, object)
- tuples.
-
- - Hold generation (a sequence number) on behalf of `AppendStack`.
+class _LayerBase(object):
+ """ Base for both _Layer and _ArchiveLayer.
"""
def __init__(self, max_length=100, generation=0):
self._stack = []
@@ -53,6 +46,18 @@
break
yield index, obj
+
+class _Layer(_LayerBase):
+ """ Append-only list with maximum length.
+
+ - Raise `_LayerFull` on attempts to exceed that length.
+
+ - Iteration occurs in reverse order of appends, and yields (index, object)
+ tuples.
+
+ - Hold generation (a sequence number) on behalf of `AppendStack`.
+ """
+
def push(self, obj):
if len(self._stack) >= self._max_length:
raise _LayerFull()
@@ -179,3 +184,59 @@
m_layers[0][1].append(to_push)
return c_m_layers, c_m_length, m_layers[:c_m_layers]
+
+
+class _ArchiveLayer(Persistent, _LayerBase):
+ """ Allow saving layer info in separate persistent sub-objects.
+
+ Archive layers don't support 'push' (they are conceptually immuatble).
+
+ These layers will be kept in a linked list in an archive.
+ """
+ _next = None
+
+ @classmethod
+ def fromLayer(klass, layer):
+ copy = klass(layer._max_length, layer._generation)
+ copy._stack[:] = layer._stack
+ return copy
+
+
+class Archive(Persistent):
+ """ Manage layers discarded from an AppendStack as a persistent linked list.
+ """
+ _head = None
+ _generation = -1
+
+ def __iter__(self):
+ current = self._head
+ while current is not None:
+ for index, item in current:
+ yield current._generation, index, item
+ current = current._next
+
+ def addLayer(self, generation, items):
+ if generation <= self._generation:
+ raise ValueError(
+ "Cannot add older layers to an already-populated archive")
+ copy = _ArchiveLayer(generation=generation)
+ copy._stack[:] = items
+ self._head, copy._next = copy, self._head
+ self._generation = generation
+
+ #
+ # ZODB Conflict resolution
+ #
+ # Archive is a simpler problem, because the only mutation occurs when
+ # adding a layer. We can resolve IFF the committed version and the new
+ # version have the same generation: in that case, we can just keep the
+ # committed version, because the two layers are equivalent.
+ #
+ # This neglects the case of independently-constructed layers: we presume
+ # that the source layers are coming from the same AppendStack, in which
+ # case they will be identical.
+ #
+ def _p_resolveConflict(self, old, committed, new):
+ if committed['_generation'] == new['_generation']:
+ return committed
+ raise ConflictError('Conflicting generations')
Modified: appendonly/trunk/appendonly/tests.py
===================================================================
--- appendonly/trunk/appendonly/tests.py 2012-02-21 20:29:28 UTC (rev 124447)
+++ appendonly/trunk/appendonly/tests.py 2012-02-21 21:47:08 UTC (rev 124448)
@@ -12,11 +12,8 @@
##############################################################################
import unittest
-class LayerTests(unittest.TestCase):
- def _getTargetClass(self):
- from appendonly import _Layer
- return _Layer
+class _LayerTestBase(object):
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)
@@ -49,6 +46,23 @@
layer = self._makeOne()
self.assertEqual(list(layer.newer(0)), [])
+
+class LayerTests(unittest.TestCase, _LayerTestBase):
+
+ def _getTargetClass(self):
+ from appendonly import _Layer
+ return _Layer
+
+ def test___iter___filled(self):
+ layer = self._makeOne()
+ OBJ1 = object()
+ OBJ2 = object()
+ OBJ3 = object()
+ layer.push(OBJ1)
+ layer.push(OBJ2)
+ layer.push(OBJ3)
+ self.assertEqual(list(layer), [(2, OBJ3), (1, OBJ2), (0, OBJ1)])
+
def test_newer_miss(self):
layer = self._makeOne()
layer.push(object())
@@ -92,6 +106,7 @@
(0, OBJ1),
])
+
class AppendStackTests(unittest.TestCase):
def _getTargetClass(self):
@@ -429,3 +444,131 @@
stack = self._makeOne()
merged = stack._p_resolveConflict(O_STATE, C_STATE, N_STATE)
self.assertEqual(merged, M_STATE)
+
+
+class ArchiveLayerTests(unittest.TestCase, _LayerTestBase):
+
+ def _getTargetClass(self):
+ from appendonly import _ArchiveLayer
+ return _ArchiveLayer
+
+ def test_is_persistent(self):
+ from persistent import Persistent
+ self.failUnless(issubclass(self._getTargetClass(), Persistent))
+
+ def test_fromLayer(self):
+ from appendonly import _Layer
+ klass = self._getTargetClass()
+ source = _Layer(max_length=42, generation=13)
+ for i in range(25):
+ obj = object()
+ source.push(obj)
+ copied = klass.fromLayer(source)
+ self.assertEqual(copied._max_length, 42)
+ self.assertEqual(copied._generation, 13)
+ self.assertEqual(len(copied._stack), len(source._stack))
+ for s_obj, c_obj in zip(source._stack, copied._stack):
+ self.failUnless(s_obj is c_obj)
+
+ def test___iter___filled(self):
+ from appendonly import _Layer
+ klass = self._getTargetClass()
+ source = _Layer()
+ OBJ1 = object()
+ OBJ2 = object()
+ OBJ3 = object()
+ source.push(OBJ1)
+ source.push(OBJ2)
+ source.push(OBJ3)
+ copied = klass.fromLayer(source)
+ self.assertEqual(list(copied), [(2, OBJ3), (1, OBJ2), (0, OBJ1)])
+
+ def test_newer_miss(self):
+ from appendonly import _Layer
+ klass = self._getTargetClass()
+ source = _Layer()
+ source.push(object())
+ copied = klass.fromLayer(source)
+ self.assertEqual(list(copied.newer(0)), [])
+
+ def test_newer_hit(self):
+ from appendonly import _Layer
+ klass = self._getTargetClass()
+ source = _Layer()
+ OBJ1 = object()
+ OBJ2 = object()
+ OBJ3 = object()
+ source.push(OBJ1)
+ source.push(OBJ2)
+ source.push(OBJ3)
+ copied = klass.fromLayer(source)
+ self.assertEqual(list(copied.newer(0)),
+ [(2, OBJ3), (1, OBJ2)])
+
+
+class ArchiveTests(unittest.TestCase):
+
+ def _getTargetClass(self):
+ from appendonly import Archive
+ return Archive
+
+ def _makeOne(self):
+ return self._getTargetClass()()
+
+ def test_ctor(self):
+ archive = self._makeOne()
+ self.failUnless(archive._head is None)
+ self.assertEqual(archive._generation, -1)
+
+ def test___iter___empty(self):
+ archive = self._makeOne()
+ self.assertEqual(list(archive), [])
+
+ def test___iter___filled(self):
+ archive = self._makeOne()
+ created = []
+ layer0 = []
+ for i in range(3):
+ obj = object()
+ created.append(obj)
+ layer0.append(obj)
+ archive.addLayer(0, layer0)
+ layer1 = []
+ for i in range(2):
+ obj = object()
+ created.append(obj)
+ layer1.append(obj)
+ archive.addLayer(1, layer1)
+ found = []
+ gen_ndx = []
+ for generation, index, obj in archive:
+ gen_ndx.append((generation, index))
+ found.append(obj)
+ self.assertEqual(gen_ndx, [(1, 1), (1, 0), (0, 2), (0, 1), (0, 0)])
+ self.assertEqual(found, list(reversed(created)))
+
+ def test_addLayer_older(self):
+ archive = self._makeOne()
+ archive.addLayer(0, [])
+ self.assertRaises(ValueError, archive.addLayer, 0, [])
+
+ def test__p_resolveConflict_w_same_generation(self):
+ O_STATE = {'_generation': -1, '_head': None}
+ c_obj = object()
+ C_STATE = {'_generation': 0, '_head': c_obj}
+ n_obj = object()
+ N_STATE = {'_generation': 0, '_head': n_obj}
+ archive = self._makeOne()
+ resolved = archive._p_resolveConflict(O_STATE, C_STATE, N_STATE)
+ self.assertEqual(resolved, C_STATE)
+
+ def test__p_resolveConflict_w_different_generation(self):
+ from ZODB.POSException import ConflictError
+ O_STATE = {'_generation': -1, '_head': None}
+ c_obj = object()
+ C_STATE = {'_generation': 0, '_head': c_obj}
+ n_obj = object()
+ N_STATE = {'_generation': 1, '_head': n_obj}
+ archive = self._makeOne()
+ self.assertRaises(ConflictError, archive._p_resolveConflict,
+ O_STATE, C_STATE, N_STATE)
Modified: appendonly/trunk/setup.py
===================================================================
--- appendonly/trunk/setup.py 2012-02-21 20:29:28 UTC (rev 124447)
+++ appendonly/trunk/setup.py 2012-02-21 21:47:08 UTC (rev 124448)
@@ -1,6 +1,6 @@
##############################################################################
#
-# Copyright (c) 2010 Zope Foundation and Contributors.
+# Copyright (c) 2010, 2012 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
@@ -20,7 +20,7 @@
CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()
setup(name='appendonly',
- version='0.9.1dev',
+ version='0.10dev',
description='Persistent append-only data structures.',
long_description=README + '\n\n' + CHANGES,
classifiers=[
More information about the checkins
mailing list