[Checkins] SVN: Products.ZCatalog/trunk/ Added a new UUIDIndex, based on the common UnIndex. It behaves like a FieldIndex, but can only store one document id per value, so there's a 1:1 mapping from value to document id. A ValueError is raised if a different document id is indexed for an already taken value. The internal data structures are optimized for this and avoid storing one IITreeSet per value.
Hanno Schlichting
hannosch at hannosch.eu
Fri Jan 28 14:23:01 EST 2011
Log message for revision 119991:
Added a new UUIDIndex, based on the common UnIndex. It behaves like a FieldIndex, but can only store one document id per value, so there's a 1:1 mapping from value to document id. A ValueError is raised if a different document id is indexed for an already taken value. The internal data structures are optimized for this and avoid storing one IITreeSet per value.
Changed:
U Products.ZCatalog/trunk/CHANGES.txt
A Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/
A Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/UUIDIndex.py
A Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/__init__.py
A Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/
A Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/addUUIDIndex.dtml
A Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/manageUUIDIndex.dtml
A Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/tests.py
U Products.ZCatalog/trunk/src/Products/PluginIndexes/__init__.py
-=-
Modified: Products.ZCatalog/trunk/CHANGES.txt
===================================================================
--- Products.ZCatalog/trunk/CHANGES.txt 2011-01-28 08:00:26 UTC (rev 119990)
+++ Products.ZCatalog/trunk/CHANGES.txt 2011-01-28 19:23:01 UTC (rev 119991)
@@ -4,6 +4,12 @@
2.13.4 (unreleased)
-------------------
+- Added a new UUIDIndex, based on the common UnIndex. It behaves like a
+ FieldIndex, but can only store one document id per value, so there's a 1:1
+ mapping from value to document id. A ValueError is raised if a different
+ document id is indexed for an already taken value. The internal data
+ structures are optimized for this and avoid storing one IITreeSet per value.
+
- Optimize sorting in presence of batching arguments. If a batch from the end
of the result set is requested, we internally reverse the sorting order and
at the end reverse the lazy sequence again. In a sequence with 100 entries,
Added: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/UUIDIndex.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/UUIDIndex.py (rev 0)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/UUIDIndex.py 2011-01-28 19:23:01 UTC (rev 119991)
@@ -0,0 +1,90 @@
+##############################################################################
+#
+# Copyright (c) 2011 Zope Foundation and Contributors.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from App.special_dtml import DTMLFile
+from BTrees.IOBTree import IOBTree
+from BTrees.Length import Length
+from BTrees.OIBTree import OIBTree
+
+from Products.PluginIndexes.common.UnIndex import UnIndex
+
+_marker = []
+
+
+class UUIDIndex(UnIndex):
+ """Index for uuid fields with an unique value per key.
+
+ The internal structure is:
+
+ self._index = {datum:documentId]}
+ self._unindex = {documentId:datum}
+
+ For each datum only one documentId can exist.
+ """
+
+ meta_type= " UUIDIndex"
+
+ manage_options= (
+ {'label': 'Settings', 'action': 'manage_main'},
+ {'label': 'Browse', 'action': 'manage_browse'},
+ )
+
+ query_options = ["query", "range"]
+
+ manage = manage_main = DTMLFile('dtml/manageUUIDIndex', globals())
+ manage_main._setName('manage_main')
+ manage_browse = DTMLFile('../dtml/browseIndex', globals())
+
+ def clear(self):
+ self._length = Length()
+ self._index = OIBTree()
+ self._unindex = IOBTree()
+
+ def numObjects(self):
+ """Return the number of indexed objects. Since we have a 1:1 mapping
+ from documents to values, we can reuse the stored length.
+ """
+ return self.indexSize()
+
+ def insertForwardIndexEntry(self, entry, documentId):
+ """Take the entry provided and put it in the correct place
+ in the forward index.
+ """
+ if entry is None:
+ return
+
+ old_docid = self._index.get(entry, _marker)
+ if old_docid is _marker:
+ self._index[entry] = documentId
+ self._length.change(1)
+ elif old_docid != documentId:
+ raise ValueError("A different document with value '%s' already "
+ "exists in the index.'")
+
+ def removeForwardIndexEntry(self, entry, documentId):
+ """Take the entry provided and remove any reference to documentId
+ in its entry in the index.
+ """
+ old_docid = self._index.get(entry, _marker)
+ if old_docid is not _marker:
+ del self._index[entry]
+ self._length.change(-1)
+
+manage_addUUIDIndexForm = DTMLFile('dtml/addUUIDIndex', globals())
+
+
+def manage_addUUIDIndex(self, id, extra=None,
+ REQUEST=None, RESPONSE=None, URL3=None):
+ """Add an uuid index"""
+ return self.manage_addIndex(id, 'UUIDIndex', extra=extra, \
+ REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
Property changes on: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/UUIDIndex.py
___________________________________________________________________
Added: svn:eol-style
+ native
Property changes on: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/__init__.py
___________________________________________________________________
Added: svn:eol-style
+ native
Added: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/addUUIDIndex.dtml
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/addUUIDIndex.dtml (rev 0)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/addUUIDIndex.dtml 2011-01-28 19:23:01 UTC (rev 119991)
@@ -0,0 +1,57 @@
+<dtml-var manage_page_header>
+
+<dtml-var "manage_form_title(this(), _, form_title='Add UUIDIndex')">
+
+<p class="form-help">
+An <strong>UUIDIndex</strong> can index uuid or similar unique values.
+</p>
+
+<form action="manage_addUUIDIndex" method="post" enctype="multipart/form-data">
+<table cellspacing="0" cellpadding="2" border="0">
+ <tr>
+ <td align="left" valign="top">
+ <div class="form-label">
+ Id
+ </div>
+ </td>
+ <td align="left" valign="top">
+ <input type="text" name="id" size="40" />
+ </td>
+ </tr>
+
+ <tr>
+ <td align="left" valign="top">
+ <div class="form-label">
+ Indexed attributes
+ </div>
+ </td>
+ <td align="left" valign="top">
+ <input type="text" name="extra.indexed_attrs:record:string" size="40" />
+ <em>attribute1,attribute2,...</em> or leave empty
+ </td>
+ </tr>
+
+ <tr>
+ <td align="left" valign="top">
+ <div class="form-optional">
+ Type
+ </div>
+ </td>
+ <td align="left" valign="top">
+ UUID Index
+ </td>
+ </tr>
+ <tr>
+ <td align="left" valign="top">
+ </td>
+ <td align="left" valign="top">
+ <div class="form-element">
+ <input class="form-element" type="submit" name="submit"
+ value=" Add " />
+ </div>
+ </td>
+ </tr>
+</table>
+</form>
+
+<dtml-var manage_page_footer>
Property changes on: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/addUUIDIndex.dtml
___________________________________________________________________
Added: svn:eol-style
+ native
Added: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/manageUUIDIndex.dtml
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/manageUUIDIndex.dtml (rev 0)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/manageUUIDIndex.dtml 2011-01-28 19:23:01 UTC (rev 119991)
@@ -0,0 +1,10 @@
+<dtml-var manage_page_header>
+<dtml-var manage_tabs>
+
+<p class="form-help">
+Objects indexed: <dtml-var numObjects>
+<br>
+Distinct values: <dtml-var indexSize>
+</p>
+
+<dtml-var manage_page_footer>
Property changes on: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/dtml/manageUUIDIndex.dtml
___________________________________________________________________
Added: svn:eol-style
+ native
Added: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/tests.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/tests.py (rev 0)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/tests.py 2011-01-28 19:23:01 UTC (rev 119991)
@@ -0,0 +1,136 @@
+##############################################################################
+#
+# Copyright (c) 2011 Zope Foundation and Contributors.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import unittest
+
+from Products.PluginIndexes.UUIDIndex.UUIDIndex import UUIDIndex
+
+
+class Dummy:
+
+ def __init__(self, foo):
+ self._foo = foo
+
+ def foo(self):
+ return self._foo
+
+ def __str__(self):
+ return '<Dummy: %s>' % self._foo
+
+ __repr__ = __str__
+
+
+class UUIDIndexTests(unittest.TestCase):
+
+ def setUp(self):
+ self._index = UUIDIndex('foo')
+ self._marker = []
+ self._values = [
+ (0, Dummy('a')),
+ (1, Dummy('ab')),
+ (2, Dummy(123)),
+ (3, Dummy(234)),
+ (4, Dummy(0))]
+ self._forward = {}
+ self._backward = {}
+ for k, v in self._values:
+ self._backward[k] = v
+ keys = self._forward.get(v, [])
+ self._forward[v] = keys
+
+ def tearDown(self):
+ self._index.clear()
+
+ def _populateIndex(self):
+ for k, v in self._values:
+ self._index.index_object(k, v)
+
+ def _checkApply(self, req, expectedValues):
+ result, used = self._index._apply_index(req)
+ if hasattr(result, 'keys'):
+ result = result.keys()
+ self.assertEqual(used, ('foo', ))
+ self.assertEqual(len(result), len(expectedValues))
+ for k, v in expectedValues:
+ self.assertTrue(k in result)
+
+ def test_interfaces(self):
+ from Products.PluginIndexes.interfaces import IPluggableIndex
+ from Products.PluginIndexes.interfaces import ISortIndex
+ from Products.PluginIndexes.interfaces import IUniqueValueIndex
+ from zope.interface.verify import verifyClass
+
+ verifyClass(IPluggableIndex, UUIDIndex)
+ verifyClass(ISortIndex, UUIDIndex)
+ verifyClass(IUniqueValueIndex, UUIDIndex)
+
+ def test_empty(self):
+ self.assertEqual(len(self._index), 0)
+ self.assertEqual(len(self._index.referencedObjects()), 0)
+ self.assertEqual(self._index.numObjects(), 0)
+ self._checkApply({'foo': 'a'}, [])
+
+ def test_populated(self):
+ self._populateIndex()
+ values = self._values
+ self.assertEqual(len(self._index), len(values))
+ self.assertEqual(self._index.indexSize(), len(values))
+ self.assertTrue(self._index.getEntryForObject(10) is None)
+ self._checkApply({'foo': 'not'}, [])
+
+ self._index.unindex_object(10) # nothrow
+
+ for k, v in values:
+ self.assertEqual(self._index.getEntryForObject(k), v.foo())
+
+ self._checkApply({'foo': 'a'}, [values[0]])
+ self._checkApply({'foo': 0}, [values[4]])
+ self._checkApply({'foo': ['a', 'ab']}, values[:2])
+
+ def test_none(self):
+ self._index.index_object(10, Dummy(None))
+ self._checkApply({'foo': None}, [])
+ self.assertFalse(None in self._index.uniqueValues('foo'))
+
+ def test_reindex(self):
+ self._populateIndex()
+ self._checkApply({'foo': 'a'}, [self._values[0]])
+ d = Dummy('world')
+ self._index.index_object(0, d)
+ self._checkApply({'foo': 'a'}, [])
+ self._checkApply({'foo': 'world'}, [(0, d)])
+ self.assertEqual(self._index.keyForDocument(0), 'world')
+ del d._foo
+ self._index.index_object(0, d)
+ self._checkApply({'foo': 'world'}, [])
+ self.assertRaises(KeyError, self._index.keyForDocument, 0)
+
+ def test_range(self):
+ values = []
+ for i in range(100):
+ obj = (i, Dummy(i))
+ self._index.index_object(*obj)
+ values.append(obj)
+
+ query = {'foo': {'query': [10, 20], 'range': 'min:max'}}
+ self._checkApply(query, values[10:21])
+
+ def test_non_unique(self):
+ self._index.index_object(0, Dummy('a'))
+ self.assertRaises(ValueError, self._index.index_object, 1, Dummy('a'))
+
+
+def test_suite():
+ return unittest.TestSuite((
+ unittest.makeSuite(UUIDIndexTests),
+ ))
Property changes on: Products.ZCatalog/trunk/src/Products/PluginIndexes/UUIDIndex/tests.py
___________________________________________________________________
Added: svn:eol-style
+ native
Modified: Products.ZCatalog/trunk/src/Products/PluginIndexes/__init__.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/__init__.py 2011-01-28 08:00:26 UTC (rev 119990)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/__init__.py 2011-01-28 19:23:01 UTC (rev 119991)
@@ -105,3 +105,17 @@
icon='www/index.gif',
visibility=None,
)
+
+ from Products.PluginIndexes.UUIDIndex.UUIDIndex import UUIDIndex
+ from Products.PluginIndexes.UUIDIndex.UUIDIndex import \
+ manage_addUUIDIndex
+ from Products.PluginIndexes.UUIDIndex.UUIDIndex import \
+ manage_addUUIDIndexForm
+
+ context.registerClass(UUIDIndex,
+ permission='Add Pluggable Index',
+ constructors=(manage_addUUIDIndexForm,
+ manage_addUUIDIndex),
+ icon='www/index.gif',
+ visibility=None,
+ )
More information about the checkins
mailing list