[Checkins] SVN: Zope3/trunk/src/ use the new flavor of the BTree
family interface
Fred L. Drake, Jr.
fdrake at gmail.com
Thu Apr 26 10:52:39 EDT 2007
Log message for revision 74783:
use the new flavor of the BTree family interface
Changed:
_U Zope3/trunk/src/
U Zope3/trunk/src/zope/app/catalog/catalog.py
U Zope3/trunk/src/zope/app/intid/__init__.py
U Zope3/trunk/src/zope/index/field/index.py
U Zope3/trunk/src/zope/index/keyword/index.py
U Zope3/trunk/src/zope/index/text/baseindex.py
U Zope3/trunk/src/zope/index/text/cosineindex.py
U Zope3/trunk/src/zope/index/text/okapiindex.py
U Zope3/trunk/src/zope/index/text/parsetree.py
U Zope3/trunk/src/zope/index/text/setops.py
U Zope3/trunk/src/zope/index/text/tests/test_queryengine.py
U Zope3/trunk/src/zope/index/topic/filter.py
U Zope3/trunk/src/zope/index/topic/index.py
-=-
Property changes on: Zope3/trunk/src
___________________________________________________________________
Name: svn:externals
- docutils svn://svn.zope.org/repos/main/docutils/tags/0.4.0
ZConfig svn://svn.zope.org/repos/main/ZConfig/trunk/ZConfig
BTrees -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/BTrees
persistent -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/persistent
ThreadedAsync -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/ThreadedAsync
transaction -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/transaction
ZEO -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/ZEO
ZODB -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/ZODB
twisted svn://svn.twistedmatrix.com/svn/Twisted/tags/releases/twisted-core-2.5.0/twisted
zdaemon -r 40792 svn://svn.zope.org/repos/main/zdaemon/trunk/src/zdaemon
+ docutils svn://svn.zope.org/repos/main/docutils/tags/0.4.0
ZConfig svn://svn.zope.org/repos/main/ZConfig/trunk/ZConfig
BTrees -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/BTrees
persistent -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/persistent
ThreadedAsync -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/ThreadedAsync
transaction -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/transaction
ZEO -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/ZEO
ZODB -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/ZODB
twisted svn://svn.twistedmatrix.com/svn/Twisted/tags/releases/twisted-core-2.5.0/twisted
zdaemon -r 40792 svn://svn.zope.org/repos/main/zdaemon/trunk/src/zdaemon
Modified: Zope3/trunk/src/zope/app/catalog/catalog.py
===================================================================
--- Zope3/trunk/src/zope/app/catalog/catalog.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/app/catalog/catalog.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -129,7 +129,7 @@
_, result = results.pop(0)
for _, r in results:
- _, result = self.family.IFModule.weightedIntersection(result, r)
+ _, result = self.family.IF.weightedIntersection(result, r)
return result
Modified: Zope3/trunk/src/zope/app/intid/__init__.py
===================================================================
--- Zope3/trunk/src/zope/app/intid/__init__.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/app/intid/__init__.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -60,8 +60,8 @@
def __init__(self, family=None):
if family is not None:
self.family = family
- self.ids = self.family.OIModule.BTree()
- self.refs = self.family.IOModule.BTree()
+ self.ids = self.family.OI.BTree()
+ self.refs = self.family.IO.BTree()
def __len__(self):
return len(self.ids)
Modified: Zope3/trunk/src/zope/index/field/index.py
===================================================================
--- Zope3/trunk/src/zope/index/field/index.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/field/index.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -19,7 +19,6 @@
import BTrees
-from BTrees.OOBTree import OOBTree
from BTrees.Length import Length
import zope.interface
@@ -44,9 +43,9 @@
def clear(self):
"""Initialize forward and reverse mappings."""
# The forward index maps indexed values to a sequence of docids
- self._fwd_index = OOBTree()
+ self._fwd_index = self.family.OO.BTree()
# The reverse index maps a docid to its index value
- self._rev_index = self.family.IOModule.BTree()
+ self._rev_index = self.family.IO.BTree()
self._num_docs = Length(0)
def documentCount(self):
@@ -70,7 +69,7 @@
# Insert into forward index.
set = self._fwd_index.get(value)
if set is None:
- set = self.family.IFModule.TreeSet()
+ set = self.family.IF.TreeSet()
self._fwd_index[value] = set
set.insert(docid)
@@ -106,5 +105,5 @@
def apply(self, query):
if len(query) != 2 or not isinstance(query, tuple):
raise TypeError("two-length tuple expected", query)
- return self.family.IFModule.multiunion(
+ return self.family.IF.multiunion(
self._fwd_index.values(*query))
Modified: Zope3/trunk/src/zope/index/keyword/index.py
===================================================================
--- Zope3/trunk/src/zope/index/keyword/index.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/keyword/index.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -19,10 +19,8 @@
import BTrees
-from BTrees.OOBTree import OOBTree, OOSet, difference
from BTrees.Length import Length
-from types import StringTypes
from zope.index.interfaces import IInjection, IStatistics
from zope.index.keyword.interfaces import IKeywordQuerying
from zope.interface import implements
@@ -44,12 +42,12 @@
"""Initialize forward and reverse mappings."""
# The forward index maps index keywords to a sequence of docids
- self._fwd_index = OOBTree()
+ self._fwd_index = self.family.OO.BTree()
# The reverse index maps a docid to its keywords
# TODO: Using a vocabulary might be the better choice to store
# keywords since it would allow use to use integers instead of strings
- self._rev_index = self.family.IOModule.BTree()
+ self._rev_index = self.family.IO.BTree()
self._num_docs = Length(0)
def documentCount(self):
@@ -64,7 +62,7 @@
return bool(self._rev_index.has_key(docid))
def index_doc(self, docid, seq):
- if isinstance(seq, StringTypes):
+ if isinstance(seq, basestring):
raise TypeError('seq argument must be a list/tuple of strings')
if not seq:
@@ -74,7 +72,7 @@
seq = [w.lower() for w in seq]
old_kw = self._rev_index.get(docid, None)
- new_kw = OOSet(seq)
+ new_kw = self.family.OO.Set(seq)
if old_kw is None:
self._insert_forward(docid, new_kw)
@@ -83,8 +81,8 @@
else:
# determine added and removed keywords
- kw_added = difference(new_kw, old_kw)
- kw_removed = difference(old_kw, new_kw)
+ kw_added = self.family.OO.difference(new_kw, old_kw)
+ kw_removed = self.family.OO.difference(old_kw, new_kw)
# removed keywords are removed from the forward index
for word in kw_removed:
@@ -119,7 +117,7 @@
has_key = idx.has_key
for word in words:
if not has_key(word):
- idx[word] = self.family.IIModule.Set()
+ idx[word] = self.family.II.Set()
idx[word].insert(docid)
def _insert_reverse(self, docid, words):
@@ -130,25 +128,25 @@
def search(self, query, operator='and'):
"""Execute a search given by 'query'."""
- if isinstance(query, StringTypes):
+ if isinstance(query, basestring):
query = [query]
if self.normalize:
query = [w.lower() for w in query]
- f = {'and' : self.family.IIModule.intersection,
- 'or' : self.family.IIModule.union,
+ f = {'and' : self.family.II.intersection,
+ 'or' : self.family.II.union,
}[operator]
rs = None
for word in query:
- docids = self._fwd_index.get(word, self.family.IIModule.Set())
+ docids = self._fwd_index.get(word, self.family.II.Set())
rs = f(rs, docids)
if rs:
return rs
else:
- return self.family.IIModule.Set()
+ return self.family.II.Set()
class CaseSensitiveKeywordIndex(KeywordIndex):
""" A case-sensitive keyword index """
Modified: Zope3/trunk/src/zope/index/text/baseindex.py
===================================================================
--- Zope3/trunk/src/zope/index/text/baseindex.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/baseindex.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -64,11 +64,11 @@
# Different indexers have different notions of doc weight, but we
# expect each indexer to use ._docweight to map docids to its
# notion of what a doc weight is.
- self._docweight = self.family.IFModule.BTree()
+ self._docweight = self.family.IF.BTree()
# docid -> WidCode'd list of wids
# Used for un-indexing, and for phrase search.
- self._docwords = self.family.IOModule.BTree()
+ self._docwords = self.family.IO.BTree()
# Use a BTree length for efficient length computation w/o conflicts
self.wordCount = Length.Length()
@@ -118,13 +118,13 @@
old_wid2w, old_docw = self._get_frequencies(old_wids)
new_wid2w, new_docw = self._get_frequencies(new_wids)
- old_widset = self.family.IFModule.TreeSet(old_wid2w.keys())
- new_widset = self.family.IFModule.TreeSet(new_wid2w.keys())
+ old_widset = self.family.IF.TreeSet(old_wid2w.keys())
+ new_widset = self.family.IF.TreeSet(new_wid2w.keys())
- IFModule = self.family.IFModule
- in_both_widset = IFModule.intersection(old_widset, new_widset)
- only_old_widset = IFModule.difference(old_widset, in_both_widset)
- only_new_widset = IFModule.difference(new_widset, in_both_widset)
+ IF = self.family.IF
+ in_both_widset = IF.intersection(old_widset, new_widset)
+ only_old_widset = IF.difference(old_widset, in_both_widset)
+ only_new_widset = IF.difference(new_widset, in_both_widset)
del old_widset, new_widset
for wid in only_old_widset.keys():
@@ -164,7 +164,7 @@
def unindex_doc(self, docid):
if docid not in self._docwords:
return
- for wid in self.family.IFModule.TreeSet(self.get_words(docid)).keys():
+ for wid in self.family.IF.TreeSet(self.get_words(docid)).keys():
self._del_wordinfo(wid, docid)
del self._docwords[docid]
del self._docweight[docid]
@@ -186,13 +186,13 @@
cleaned_wids = self._remove_oov_wids(wids)
if len(wids) != len(cleaned_wids):
# At least one wid was OOV: can't possibly find it.
- return self.family.IFModule.BTree()
+ return self.family.IF.BTree()
scores = self._search_wids(wids)
hits = mass_weightedIntersection(scores, self.family)
if not hits:
return hits
code = widcode.encode(wids)
- result = self.family.IFModule.BTree()
+ result = self.family.IF.BTree()
for docid, weight in hits.items():
docwords = self._docwords[docid]
if docwords.find(code) >= 0:
@@ -257,7 +257,7 @@
# len(IFBTree).
if (isinstance(doc2score, type({})) and
len(doc2score) == self.DICT_CUTOFF):
- doc2score = self.family.IFModule.BTree(doc2score)
+ doc2score = self.family.IF.BTree(doc2score)
doc2score[docid] = f
self._wordinfo[wid] = doc2score # not redundant: Persistency!
@@ -280,7 +280,7 @@
new_word_count += 1
elif (isinstance(doc2score, dicttype) and
len(doc2score) == self.DICT_CUTOFF):
- doc2score = self.family.IFModule.BTree(doc2score)
+ doc2score = self.family.IF.BTree(doc2score)
doc2score[docid] = weight
self._wordinfo[wid] = doc2score # not redundant: Persistency!
self.wordCount.change(new_word_count)
Modified: Zope3/trunk/src/zope/index/text/cosineindex.py
===================================================================
--- Zope3/trunk/src/zope/index/text/cosineindex.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/cosineindex.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -73,7 +73,7 @@
idf = inverse_doc_frequency(len(d2w), N) # an unscaled float
#print "idf = %.3f" % idf
if isinstance(d2w, DictType):
- d2w = self.family.IFModule.Bucket(d2w)
+ d2w = self.family.IF.Bucket(d2w)
L.append((d2w, idf))
return L
Modified: Zope3/trunk/src/zope/index/text/okapiindex.py
===================================================================
--- Zope3/trunk/src/zope/index/text/okapiindex.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/okapiindex.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -265,7 +265,7 @@
for t in wids:
d2f = self._wordinfo[t] # map {docid -> f(docid, t)}
idf = inverse_doc_frequency(len(d2f), N) # an unscaled float
- result = self.family.IFModule.Bucket()
+ result = self.family.IF.Bucket()
for docid, f in d2f.items():
lenweight = B_from1 + B * docid2len[docid] / meandoclen
tf = f * K1_plus1 / (f + K1 * lenweight)
@@ -309,7 +309,7 @@
for t in wids:
d2f = self._wordinfo[t] # map {docid -> f(docid, t)}
idf = inverse_doc_frequency(len(d2f), N) # an unscaled float
- result = self.family.IFModule.Bucket()
+ result = self.family.IF.Bucket()
score(result, d2f.items(), docid2len, idf, meandoclen)
L.append((result, 1))
return L
Modified: Zope3/trunk/src/zope/index/text/parsetree.py
===================================================================
--- Zope3/trunk/src/zope/index/text/parsetree.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/parsetree.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -89,7 +89,7 @@
set = mass_weightedIntersection(L, index.family)
if Nots:
notset = mass_weightedUnion(Nots, index.family)
- set = index.family.IFModule.difference(set, notset)
+ set = index.family.IF.difference(set, notset)
return set
class OrNode(ParseTreeNode):
Modified: Zope3/trunk/src/zope/index/text/setops.py
===================================================================
--- Zope3/trunk/src/zope/index/text/setops.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/setops.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -30,9 +30,9 @@
# (len(Bucket) is fast; len(BTree) is slow).
L.sort(lambda x, y: cmp(len(x[0]), len(y[0])))
(x, wx), (y, wy) = L[:2]
- dummy, result = family.IFModule.weightedIntersection(x, y, wx, wy)
+ dummy, result = family.IF.weightedIntersection(x, y, wx, wy)
for x, wx in L[2:]:
- dummy, result = family.IFModule.weightedIntersection(result, x, 1, wx)
+ dummy, result = family.IF.weightedIntersection(result, x, 1, wx)
return result
def mass_weightedUnion(L, family=BTrees.family32):
@@ -47,7 +47,7 @@
# Merge the two smallest so far, and add back to the queue.
(x, wx), dummy = merge.pop_smallest()
(y, wy), dummy = merge.pop_smallest()
- dummy, z = family.IFModule.weightedUnion(x, y, wx, wy)
+ dummy, z = family.IF.weightedUnion(x, y, wx, wy)
merge.add((z, 1), len(z))
(result, weight), dummy = merge.pop_smallest()
return result
@@ -57,9 +57,9 @@
# pair, we may still need to multiply the mapping by its weight.
assert len(L) <= 1
if len(L) == 0:
- return family.IFModule.Bucket()
+ return family.IF.Bucket()
[(result, weight)] = L
if weight != 1:
- dummy, result = family.IFModule.weightedUnion(
- family.IFModule.Bucket(), result, 0, weight)
+ dummy, result = family.IF.weightedUnion(
+ family.IF.Bucket(), result, 0, weight)
return result
Modified: Zope3/trunk/src/zope/index/text/tests/test_queryengine.py
===================================================================
--- Zope3/trunk/src/zope/index/text/tests/test_queryengine.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/tests/test_queryengine.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -28,7 +28,7 @@
family = BTrees.family32
def search(self, term):
- b = self.family.IFModule.Bucket()
+ b = self.family.IF.Bucket()
if term == "foo":
b[1] = b[3] = 1
elif term == "bar":
Modified: Zope3/trunk/src/zope/index/topic/filter.py
===================================================================
--- Zope3/trunk/src/zope/index/topic/filter.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/topic/filter.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -41,7 +41,7 @@
self.clear()
def clear(self):
- self._ids = self.family.IIModule.Set()
+ self._ids = self.family.II.Set()
def index_doc(self, docid, context):
raise NotImplementedError
Modified: Zope3/trunk/src/zope/index/topic/index.py
===================================================================
--- Zope3/trunk/src/zope/index/topic/index.py 2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/topic/index.py 2007-04-26 14:52:38 UTC (rev 74783)
@@ -19,8 +19,6 @@
import BTrees
-from BTrees.OOBTree import OOBTree
-
from zope.interface import implements
from zope.index.interfaces import IInjection
@@ -40,7 +38,7 @@
def clear(self):
# mapping filter id -> filter
- self._filters = OOBTree()
+ self._filters = self.family.OO.BTree()
def addFilter(self, f):
""" Add filter 'f' with ID 'id' """
@@ -68,8 +66,9 @@
raise TypeError(
'query argument must be a list/tuple of filter ids')
- IIModule = self.family.IIModule
- f = {'and': IIModule.intersection, 'or': IIModule.union}[operator]
+ f = {'and': self.family.II.intersection,
+ 'or': self.family.II.union,
+ }[operator]
rs = None
for id in self._filters.keys():
@@ -78,4 +77,4 @@
rs = f(rs, docids)
if rs: return rs
- else: return self.family.IIModule.Set()
+ else: return self.family.II.Set()
More information about the Checkins
mailing list