[Checkins] SVN: zc.catalog/trunk/src/zc/catalog/ Converted
zc.catalog to 64-bit BTrees.
Albertas Agejevas
alga at pov.lt
Fri Feb 2 14:06:24 EST 2007
Log message for revision 72334:
Converted zc.catalog to 64-bit BTrees.
(The flavour of BTrees to use can be registered as a utility).
Changed:
U zc.catalog/trunk/src/zc/catalog/extentcatalog.py
U zc.catalog/trunk/src/zc/catalog/extentcatalog.txt
U zc.catalog/trunk/src/zc/catalog/index.py
U zc.catalog/trunk/src/zc/catalog/setindex.txt
U zc.catalog/trunk/src/zc/catalog/tests.py
-=-
Modified: zc.catalog/trunk/src/zc/catalog/extentcatalog.py
===================================================================
--- zc.catalog/trunk/src/zc/catalog/extentcatalog.py 2007-02-02 18:12:49 UTC (rev 72333)
+++ zc.catalog/trunk/src/zc/catalog/extentcatalog.py 2007-02-02 19:06:24 UTC (rev 72334)
@@ -21,7 +21,10 @@
from zope import interface, component
from zope.app.catalog import catalog
from zope.app.intid.interfaces import IIntIds
+
import zope.component
+from zope.component.interfaces import IFactory
+from BTrees.Interfaces import IMerge
from zc.catalog import interfaces
@@ -30,9 +33,20 @@
__parent__ = None
def __init__(self, filter):
- self.set = IFBTree.IFTreeSet()
+ treeset = zope.component.queryUtility(IFactory, name='IFTreeSet',
+ default=IFBTree.IFTreeSet)
+ self.set = treeset()
self.filter = filter
+ @property
+ def IFBTree(self):
+ """Get the [IL]BTree module of the correct flavor.
+
+ Used for set operations.
+ """
+ return zope.component.queryUtility(IMerge, name='IFBTree',
+ default=IFBTree)
+
def addable(self, uid, obj):
return self.filter(self, uid, obj)
@@ -46,7 +60,7 @@
__ror__ = __or__
def union(self, other, self_weight=1, other_weight=1):
- return IFBTree.weightedUnion(
+ return self.IFBTree.weightedUnion(
self.set, other, self_weight, other_weight)[1]
def __and__(self, other):
@@ -56,7 +70,7 @@
__rand__ = __and__
def intersection(self, other, self_weight=1, other_weight=1):
- return IFBTree.weightedIntersection(
+ return self.IFBTree.weightedIntersection(
self.set, other, self_weight, other_weight)[1]
def __sub__(self, other):
@@ -64,14 +78,14 @@
return self.difference(other)
def difference(self, other):
- return IFBTree.difference(self.set, other)
+ return self.IFBTree.difference(self.set, other)
def __rsub__(self, other):
"set - extent"
return self.rdifference(other)
def rdifference(self, other):
- return IFBTree.difference(other, self.set)
+ return self.IFBTree.difference(other, self.set)
def __iter__(self):
return iter(self.set)
Modified: zc.catalog/trunk/src/zc/catalog/extentcatalog.txt
===================================================================
--- zc.catalog/trunk/src/zc/catalog/extentcatalog.txt 2007-02-02 18:12:49 UTC (rev 72333)
+++ zc.catalog/trunk/src/zc/catalog/extentcatalog.txt 2007-02-02 19:06:24 UTC (rev 72334)
@@ -15,13 +15,16 @@
>>> root = makeRoot()
>>> intid = zope.component.getUtility(
... zope.app.intid.interfaces.IIntIds, context=root)
+ >>> IFTreeSet = component.queryUtility(component.interfaces.IFactory,
+ ... name="IFTreeSet",
+ ... default=BTrees.IFBTree.IFTreeSet)
>>> from zope.app.container.interfaces import IContained
>>> class DummyIndex(persistent.Persistent):
... interface.implements(IContained)
... __parent__ = __name__ = None
... def __init__(self):
- ... self.uids = BTrees.IFBTree.IFTreeSet()
+ ... self.uids = IFTreeSet()
... def unindex_doc(self, uid):
... if uid in self.uids:
... self.uids.remove(uid)
@@ -181,8 +184,7 @@
>>> for i in range(1, 100, 2):
... extent.add(i, None)
...
- >>> from BTrees import IFBTree
- >>> alt_set = IFBTree.IFTreeSet()
+ >>> alt_set = IFTreeSet()
>>> alt_set.update(range(0, 166, 33)) # return value is unimportant here
6
>>> sorted(alt_set)
Modified: zc.catalog/trunk/src/zc/catalog/index.py
===================================================================
--- zc.catalog/trunk/src/zc/catalog/index.py 2007-02-02 18:12:49 UTC (rev 72333)
+++ zc.catalog/trunk/src/zc/catalog/index.py 2007-02-02 19:06:24 UTC (rev 72334)
@@ -19,8 +19,10 @@
import pytz.reference
import persistent
from BTrees import IFBTree, OOBTree, IOBTree, Length
+from BTrees.Interfaces import IMerge
from zope import component, interface
+import zope.component.interfaces
import zope.interface.common.idatetime
import zope.index.interfaces
import zope.security.management
@@ -29,6 +31,7 @@
import zc.catalog.interfaces
from zc.catalog.i18n import _
+
class AbstractIndex(persistent.Persistent):
interface.implements(zope.index.interfaces.IInjection,
@@ -40,9 +43,38 @@
def __init__(self):
self.clear()
+ @property
+ def IFBTree(self):
+ """Get the [IL]FBTree module of the flavour we're using"""
+ return component.queryUtility(IMerge, name='IFBTree',
+ default=IFBTree)
+
+ @property
+ def IFTreeSet(self):
+ """Get the [IL]FTreeSet class of the flavour we're using"""
+ return component.queryUtility(zope.component.interfaces.IFactory,
+ name='IFTreeSet',
+ default=IFBTree.IFTreeSet)
+
+ @property
+ def IFSet(self):
+ """Get the [IL]FSet class of the flavour we're using"""
+ return component.queryUtility(zope.component.interfaces.IFactory,
+ name='IFSet',
+ default=IFBTree.IFSet)
+
+ @property
+ def IFBucket(self):
+ """Get the [IL]FBucket class of the flavour we're using"""
+ return component.queryUtility(zope.component.interfaces.IFactory,
+ name='IFBucket',
+ default=IFBTree.IFBucket)
+
def clear(self):
self.values_to_documents = OOBTree.OOBTree()
- self.documents_to_values = IOBTree.IOBTree()
+ self.documents_to_values = component.queryUtility(
+ zope.component.interfaces.IFactory,
+ name='IOBTree', default=IOBTree.IOBTree)()
self.documentCount = Length.Length(0)
self.wordCount = Length.Length(0)
@@ -97,7 +129,7 @@
values_to_documents = self.values_to_documents
docs = values_to_documents.get(added)
if docs is None:
- values_to_documents[added] = IFBTree.IFTreeSet((doc_id,))
+ values_to_documents[added] = self.IFTreeSet((doc_id,))
self.wordCount.change(1)
else:
docs.insert(doc_id)
@@ -139,23 +171,23 @@
if query_type is None:
res = None
elif query_type == 'any_of':
- res = IFBTree.multiunion(
+ res = self.IFBTree.multiunion(
[s for s in (values_to_documents.get(v) for v in query)
if s is not None])
elif query_type == 'any':
if query is None:
- res = IFBTree.IFSet(self.ids())
+ res = self.IFSet(self.ids())
else:
assert zc.catalog.interfaces.IExtent.providedBy(query)
- res = query & IFBTree.IFSet(self.ids())
+ res = query & self.IFSet(self.ids())
elif query_type == 'between':
- res = IFBTree.multiunion(
+ res = self.IFBTree.multiunion(
[s for s in (values_to_documents.get(v) for v in
values_to_documents.keys(*query))
if s is not None])
elif query_type == 'none':
assert zc.catalog.interfaces.IExtent.providedBy(query)
- res = query - IFBTree.IFSet(self.ids())
+ res = query - self.IFSet(self.ids())
else:
raise ValueError(
"unknown query type", query_type)
@@ -186,7 +218,7 @@
for v in added:
docs = values_to_documents.get(v)
if docs is None:
- values_to_documents[v] = IFBTree.IFTreeSet((doc_id,))
+ values_to_documents[v] = self.IFTreeSet((doc_id,))
self.wordCount.change(1)
else:
docs.insert(doc_id)
@@ -236,36 +268,38 @@
if query_type is None:
res = None
elif query_type == 'any_of':
- res = IFBTree.IFBucket()
+ res = self.IFBucket()
for v in query:
- _, res = IFBTree.weightedUnion(
+ _, res = self.IFBTree.weightedUnion(
res, values_to_documents.get(v))
elif query_type == 'any':
if query is None:
- res = IFBTree.IFSet(self.ids())
+ res = self.IFSet(self.ids())
else:
assert zc.catalog.interfaces.IExtent.providedBy(query)
- res = query & IFBTree.IFSet(self.ids())
+ res = query & self.IFSet(self.ids())
elif query_type == 'all_of':
res = None
values = iter(query)
try:
res = values_to_documents.get(values.next())
except StopIteration:
- res = IFBTree.IFTreeSet()
+ res = self.IFTreeSet()
while res:
try:
v = values.next()
except StopIteration:
break
- res = IFBTree.intersection(res, values_to_documents.get(v))
+ res = self.IFBTree.intersection(res,
+ values_to_documents.get(v))
elif query_type == 'between':
- res = IFBTree.IFBucket()
+ res = self.IFBucket()
for v in values_to_documents.keys(*query):
- _, res = IFBTree.weightedUnion(res, values_to_documents.get(v))
+ _, res = self.IFBTree.weightedUnion(res,
+ values_to_documents.get(v))
elif query_type == 'none':
assert zc.catalog.interfaces.IExtent.providedBy(query)
- res = query - IFBTree.IFSet(self.ids())
+ res = query - self.IFSet(self.ids())
else:
raise ValueError(
"unknown query type", query_type)
Modified: zc.catalog/trunk/src/zc/catalog/setindex.txt
===================================================================
--- zc.catalog/trunk/src/zc/catalog/setindex.txt 2007-02-02 18:12:49 UTC (rev 72333)
+++ zc.catalog/trunk/src/zc/catalog/setindex.txt 2007-02-02 19:06:24 UTC (rev 72334)
@@ -74,8 +74,8 @@
[1, 2, 3, 4, 6, 8]
>>> list(index.apply({'any_of':(42,)}))
[]
- >>> index.apply({'any_of': ('a', 3, 7)})
- BTrees._IFBTree.IFBucket([(1, 1.0), (2, 3.0), (5, 1.0), (6, 1.0), (9, 2.0)])
+ >>> index.apply({'any_of': ('a', 3, 7)}) # doctest: +ELLIPSIS
+ BTrees...FBucket([(1, 1.0), (2, 3.0), (5, 1.0), (6, 1.0), (9, 2.0)])
Another query is 'any'. If the key is None, all indexed document ids with any
values are returned. If the key is an extent, the intersection of the extent
@@ -122,8 +122,8 @@
[2, 4, 7, 9]
>>> list(index.apply({'between': (1, 7, True, True)}))
[2, 4, 6, 8, 9]
- >>> index.apply({'between': (2, 6)})
- BTrees._IFBTree.IFBucket([(2, 2.0), (4, 1.0), (6, 2.0), (8, 1.0), (9, 4.0)])
+ >>> index.apply({'between': (2, 6)}) # doctest: +ELLIPSIS
+ BTrees...FBucket([(2, 2.0), (4, 1.0), (6, 2.0), (8, 1.0), (9, 4.0)])
The 'none' argument takes an extent and returns the ids in the extent
that are not indexed; it is intended to be used to return docids that have
Modified: zc.catalog/trunk/src/zc/catalog/tests.py
===================================================================
--- zc.catalog/trunk/src/zc/catalog/tests.py 2007-02-02 18:12:49 UTC (rev 72333)
+++ zc.catalog/trunk/src/zc/catalog/tests.py 2007-02-02 19:06:24 UTC (rev 72334)
@@ -19,7 +19,15 @@
import unittest
from zope.testing import doctest, module
import zope.component.testing
+import zope.component.factory
+import zope.component.interfaces
+import BTrees.Interfaces
+import BTrees.LOBTree
+import BTrees.OLBTree
+import BTrees.LFBTree
+
+
def modSetUp(test):
zope.component.testing.setUp(test)
module.setUp(test, 'zc.catalog.doctest_test')
@@ -28,8 +36,45 @@
module.tearDown(test)
zope.component.testing.tearDown(test)
+
+def setUp64bit(test):
+ zope.component.testing.setUp(test)
+ zope.component.provideUtility(
+ zope.component.factory.Factory(BTrees.LFBTree.LFBTree),
+ name='IFBTree')
+ zope.component.provideUtility(
+ zope.component.factory.Factory(BTrees.LFBTree.LFTreeSet),
+ name='IFTreeSet')
+ zope.component.provideUtility(
+ zope.component.factory.Factory(BTrees.LFBTree.LFSet),
+ name='IFSet')
+ zope.component.provideUtility(
+ zope.component.factory.Factory(BTrees.LFBTree.LFBucket),
+ name='IFBucket')
+ zope.component.provideUtility(
+ zope.component.factory.Factory(BTrees.LOBTree.LOBTree),
+ name='IOBTree')
+ zope.component.provideUtility(
+ zope.component.factory.Factory(BTrees.OLBTree.OLBTree),
+ name='OIBTree')
+ zope.component.provideUtility(
+ BTrees.LFBTree,
+ provides=BTrees.Interfaces.IMerge,
+ name='IFBTree')
+
+
+def tearDown64bit(test):
+ zope.component.testing.tearDown(test)
+
+
+def modSetUp64bit(test):
+ setUp64bit(test)
+ module.setUp(test, 'zc.catalog.doctest_test')
+
+
def test_suite():
tests = unittest.TestSuite((
+ # 32 bits
doctest.DocFileSuite(
'extentcatalog.txt', setUp=modSetUp, tearDown=modTearDown,
optionflags=doctest.INTERPRET_FOOTNOTES),
@@ -37,6 +82,19 @@
doctest.DocFileSuite('valueindex.txt'),
doctest.DocFileSuite('normalizedindex.txt'),
doctest.DocFileSuite('globber.txt'),
+
+ # 64 bits
+ doctest.DocFileSuite(
+ 'extentcatalog.txt', setUp=modSetUp64bit, tearDown=tearDown64bit,
+ optionflags=doctest.INTERPRET_FOOTNOTES),
+ doctest.DocFileSuite('setindex.txt', setUp=setUp64bit,
+ tearDown=tearDown64bit),
+ doctest.DocFileSuite('valueindex.txt', setUp=setUp64bit,
+ tearDown=tearDown64bit),
+ doctest.DocFileSuite('normalizedindex.txt', setUp=setUp64bit,
+ tearDown=tearDown64bit),
+ doctest.DocFileSuite('globber.txt', setUp=setUp64bit,
+ tearDown=tearDown64bit),
))
import zc.catalog.stemmer
if not zc.catalog.stemmer.broken:
More information about the Checkins
mailing list