[Zope3-checkins] SVN: Zope3/trunk/src/ use the new flavor of the BTree family interface

Fred L. Drake, Jr. fdrake at gmail.com
Thu Apr 26 10:52:39 EDT 2007


Log message for revision 74783:
  use the new flavor of the BTree family interface

Changed:
  _U  Zope3/trunk/src/
  U   Zope3/trunk/src/zope/app/catalog/catalog.py
  U   Zope3/trunk/src/zope/app/intid/__init__.py
  U   Zope3/trunk/src/zope/index/field/index.py
  U   Zope3/trunk/src/zope/index/keyword/index.py
  U   Zope3/trunk/src/zope/index/text/baseindex.py
  U   Zope3/trunk/src/zope/index/text/cosineindex.py
  U   Zope3/trunk/src/zope/index/text/okapiindex.py
  U   Zope3/trunk/src/zope/index/text/parsetree.py
  U   Zope3/trunk/src/zope/index/text/setops.py
  U   Zope3/trunk/src/zope/index/text/tests/test_queryengine.py
  U   Zope3/trunk/src/zope/index/topic/filter.py
  U   Zope3/trunk/src/zope/index/topic/index.py

-=-

Property changes on: Zope3/trunk/src
___________________________________________________________________
Name: svn:externals
   - docutils       svn://svn.zope.org/repos/main/docutils/tags/0.4.0
ZConfig        svn://svn.zope.org/repos/main/ZConfig/trunk/ZConfig
BTrees         -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/BTrees
persistent     -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/persistent
ThreadedAsync  -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/ThreadedAsync
transaction    -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/transaction
ZEO            -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/ZEO
ZODB           -r 74768 svn://svn.zope.org/repos/main/ZODB/trunk/src/ZODB
twisted        svn://svn.twistedmatrix.com/svn/Twisted/tags/releases/twisted-core-2.5.0/twisted
zdaemon        -r 40792 svn://svn.zope.org/repos/main/zdaemon/trunk/src/zdaemon


   + docutils       svn://svn.zope.org/repos/main/docutils/tags/0.4.0
ZConfig        svn://svn.zope.org/repos/main/ZConfig/trunk/ZConfig
BTrees         -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/BTrees
persistent     -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/persistent
ThreadedAsync  -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/ThreadedAsync
transaction    -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/transaction
ZEO            -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/ZEO
ZODB           -r 74780 svn://svn.zope.org/repos/main/ZODB/trunk/src/ZODB
twisted        svn://svn.twistedmatrix.com/svn/Twisted/tags/releases/twisted-core-2.5.0/twisted
zdaemon        -r 40792 svn://svn.zope.org/repos/main/zdaemon/trunk/src/zdaemon



Modified: Zope3/trunk/src/zope/app/catalog/catalog.py
===================================================================
--- Zope3/trunk/src/zope/app/catalog/catalog.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/app/catalog/catalog.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -129,7 +129,7 @@
 
         _, result = results.pop(0)
         for _, r in results:
-            _, result = self.family.IFModule.weightedIntersection(result, r)
+            _, result = self.family.IF.weightedIntersection(result, r)
 
         return result
 

Modified: Zope3/trunk/src/zope/app/intid/__init__.py
===================================================================
--- Zope3/trunk/src/zope/app/intid/__init__.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/app/intid/__init__.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -60,8 +60,8 @@
     def __init__(self, family=None):
         if family is not None:
             self.family = family
-        self.ids = self.family.OIModule.BTree()
-        self.refs = self.family.IOModule.BTree()
+        self.ids = self.family.OI.BTree()
+        self.refs = self.family.IO.BTree()
 
     def __len__(self):
         return len(self.ids)

Modified: Zope3/trunk/src/zope/index/field/index.py
===================================================================
--- Zope3/trunk/src/zope/index/field/index.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/field/index.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -19,7 +19,6 @@
 
 import BTrees
 
-from BTrees.OOBTree import OOBTree
 from BTrees.Length import Length
 
 import zope.interface
@@ -44,9 +43,9 @@
     def clear(self):
         """Initialize forward and reverse mappings."""
         # The forward index maps indexed values to a sequence of docids
-        self._fwd_index = OOBTree()
+        self._fwd_index = self.family.OO.BTree()
         # The reverse index maps a docid to its index value
-        self._rev_index = self.family.IOModule.BTree()
+        self._rev_index = self.family.IO.BTree()
         self._num_docs = Length(0)
 
     def documentCount(self):
@@ -70,7 +69,7 @@
         # Insert into forward index.
         set = self._fwd_index.get(value)
         if set is None:
-            set = self.family.IFModule.TreeSet()
+            set = self.family.IF.TreeSet()
             self._fwd_index[value] = set
         set.insert(docid)
 
@@ -106,5 +105,5 @@
     def apply(self, query):
         if len(query) != 2 or not isinstance(query, tuple):
             raise TypeError("two-length tuple expected", query)
-        return self.family.IFModule.multiunion(
+        return self.family.IF.multiunion(
             self._fwd_index.values(*query))

Modified: Zope3/trunk/src/zope/index/keyword/index.py
===================================================================
--- Zope3/trunk/src/zope/index/keyword/index.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/keyword/index.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -19,10 +19,8 @@
 
 import BTrees
 
-from BTrees.OOBTree import OOBTree, OOSet, difference 
 from BTrees.Length import Length
 
-from types import StringTypes
 from zope.index.interfaces import IInjection, IStatistics
 from zope.index.keyword.interfaces import IKeywordQuerying
 from zope.interface import implements
@@ -44,12 +42,12 @@
         """Initialize forward and reverse mappings."""
 
         # The forward index maps index keywords to a sequence of docids
-        self._fwd_index = OOBTree()
+        self._fwd_index = self.family.OO.BTree()
 
         # The reverse index maps a docid to its keywords
         # TODO: Using a vocabulary might be the better choice to store
         # keywords since it would allow use to use integers instead of strings
-        self._rev_index = self.family.IOModule.BTree()
+        self._rev_index = self.family.IO.BTree()
         self._num_docs = Length(0)
 
     def documentCount(self):
@@ -64,7 +62,7 @@
         return bool(self._rev_index.has_key(docid))
 
     def index_doc(self, docid, seq):
-        if isinstance(seq, StringTypes):
+        if isinstance(seq, basestring):
             raise TypeError('seq argument must be a list/tuple of strings')
     
         if not seq:
@@ -74,7 +72,7 @@
             seq = [w.lower() for w in seq]
 
         old_kw = self._rev_index.get(docid, None)
-        new_kw = OOSet(seq)
+        new_kw = self.family.OO.Set(seq)
 
         if old_kw is None:
             self._insert_forward(docid, new_kw)
@@ -83,8 +81,8 @@
         else:
 
             # determine added and removed keywords
-            kw_added = difference(new_kw, old_kw)
-            kw_removed = difference(old_kw, new_kw)
+            kw_added = self.family.OO.difference(new_kw, old_kw)
+            kw_removed = self.family.OO.difference(old_kw, new_kw)
 
             # removed keywords are removed from the forward index
             for word in kw_removed:
@@ -119,7 +117,7 @@
         has_key = idx.has_key
         for word in words:
             if not has_key(word):
-                idx[word] = self.family.IIModule.Set()
+                idx[word] = self.family.II.Set()
             idx[word].insert(docid)
 
     def _insert_reverse(self, docid, words):
@@ -130,25 +128,25 @@
 
     def search(self, query, operator='and'):
         """Execute a search given by 'query'."""
-        if isinstance(query, StringTypes):
+        if isinstance(query, basestring):
             query = [query]
 
         if self.normalize:
             query = [w.lower() for w in query]
 
-        f = {'and' : self.family.IIModule.intersection,
-             'or' : self.family.IIModule.union,
+        f = {'and' : self.family.II.intersection,
+             'or' : self.family.II.union,
              }[operator]
     
         rs = None
         for word in query:
-            docids = self._fwd_index.get(word, self.family.IIModule.Set())
+            docids = self._fwd_index.get(word, self.family.II.Set())
             rs = f(rs, docids)
             
         if rs:
             return rs
         else:
-            return self.family.IIModule.Set()
+            return self.family.II.Set()
 
 class CaseSensitiveKeywordIndex(KeywordIndex):
     """ A case-sensitive keyword index """

Modified: Zope3/trunk/src/zope/index/text/baseindex.py
===================================================================
--- Zope3/trunk/src/zope/index/text/baseindex.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/baseindex.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -64,11 +64,11 @@
         # Different indexers have different notions of doc weight, but we
         # expect each indexer to use ._docweight to map docids to its
         # notion of what a doc weight is.
-        self._docweight = self.family.IFModule.BTree()
+        self._docweight = self.family.IF.BTree()
 
         # docid -> WidCode'd list of wids
         # Used for un-indexing, and for phrase search.
-        self._docwords = self.family.IOModule.BTree()
+        self._docwords = self.family.IO.BTree()
 
         # Use a BTree length for efficient length computation w/o conflicts
         self.wordCount = Length.Length()
@@ -118,13 +118,13 @@
         old_wid2w, old_docw = self._get_frequencies(old_wids)
         new_wid2w, new_docw = self._get_frequencies(new_wids)
 
-        old_widset = self.family.IFModule.TreeSet(old_wid2w.keys())
-        new_widset = self.family.IFModule.TreeSet(new_wid2w.keys())
+        old_widset = self.family.IF.TreeSet(old_wid2w.keys())
+        new_widset = self.family.IF.TreeSet(new_wid2w.keys())
 
-        IFModule = self.family.IFModule
-        in_both_widset = IFModule.intersection(old_widset, new_widset)
-        only_old_widset = IFModule.difference(old_widset, in_both_widset)
-        only_new_widset = IFModule.difference(new_widset, in_both_widset)
+        IF = self.family.IF
+        in_both_widset = IF.intersection(old_widset, new_widset)
+        only_old_widset = IF.difference(old_widset, in_both_widset)
+        only_new_widset = IF.difference(new_widset, in_both_widset)
         del old_widset, new_widset
 
         for wid in only_old_widset.keys():
@@ -164,7 +164,7 @@
     def unindex_doc(self, docid):
         if docid not in self._docwords:
             return
-        for wid in self.family.IFModule.TreeSet(self.get_words(docid)).keys():
+        for wid in self.family.IF.TreeSet(self.get_words(docid)).keys():
             self._del_wordinfo(wid, docid)
         del self._docwords[docid]
         del self._docweight[docid]
@@ -186,13 +186,13 @@
         cleaned_wids = self._remove_oov_wids(wids)
         if len(wids) != len(cleaned_wids):
             # At least one wid was OOV:  can't possibly find it.
-            return self.family.IFModule.BTree()
+            return self.family.IF.BTree()
         scores = self._search_wids(wids)
         hits = mass_weightedIntersection(scores, self.family)
         if not hits:
             return hits
         code = widcode.encode(wids)
-        result = self.family.IFModule.BTree()
+        result = self.family.IF.BTree()
         for docid, weight in hits.items():
             docwords = self._docwords[docid]
             if docwords.find(code) >= 0:
@@ -257,7 +257,7 @@
             # len(IFBTree).
             if (isinstance(doc2score, type({})) and
                 len(doc2score) == self.DICT_CUTOFF):
-                doc2score = self.family.IFModule.BTree(doc2score)
+                doc2score = self.family.IF.BTree(doc2score)
         doc2score[docid] = f
         self._wordinfo[wid] = doc2score # not redundant:  Persistency!
 
@@ -280,7 +280,7 @@
                 new_word_count += 1
             elif (isinstance(doc2score, dicttype) and
                   len(doc2score) == self.DICT_CUTOFF):
-                doc2score = self.family.IFModule.BTree(doc2score)
+                doc2score = self.family.IF.BTree(doc2score)
             doc2score[docid] = weight
             self._wordinfo[wid] = doc2score # not redundant:  Persistency!
         self.wordCount.change(new_word_count)

Modified: Zope3/trunk/src/zope/index/text/cosineindex.py
===================================================================
--- Zope3/trunk/src/zope/index/text/cosineindex.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/cosineindex.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -73,7 +73,7 @@
             idf = inverse_doc_frequency(len(d2w), N)  # an unscaled float
             #print "idf = %.3f" % idf
             if isinstance(d2w, DictType):
-                d2w = self.family.IFModule.Bucket(d2w)
+                d2w = self.family.IF.Bucket(d2w)
             L.append((d2w, idf))
         return L
 

Modified: Zope3/trunk/src/zope/index/text/okapiindex.py
===================================================================
--- Zope3/trunk/src/zope/index/text/okapiindex.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/okapiindex.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -265,7 +265,7 @@
         for t in wids:
             d2f = self._wordinfo[t] # map {docid -> f(docid, t)}
             idf = inverse_doc_frequency(len(d2f), N)  # an unscaled float
-            result = self.family.IFModule.Bucket()
+            result = self.family.IF.Bucket()
             for docid, f in d2f.items():
                 lenweight = B_from1 + B * docid2len[docid] / meandoclen
                 tf = f * K1_plus1 / (f + K1 * lenweight)
@@ -309,7 +309,7 @@
         for t in wids:
             d2f = self._wordinfo[t] # map {docid -> f(docid, t)}
             idf = inverse_doc_frequency(len(d2f), N)  # an unscaled float
-            result = self.family.IFModule.Bucket()
+            result = self.family.IF.Bucket()
             score(result, d2f.items(), docid2len, idf, meandoclen)
             L.append((result, 1))
         return L

Modified: Zope3/trunk/src/zope/index/text/parsetree.py
===================================================================
--- Zope3/trunk/src/zope/index/text/parsetree.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/parsetree.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -89,7 +89,7 @@
         set = mass_weightedIntersection(L, index.family)
         if Nots:
             notset = mass_weightedUnion(Nots, index.family)
-            set = index.family.IFModule.difference(set, notset)
+            set = index.family.IF.difference(set, notset)
         return set
 
 class OrNode(ParseTreeNode):

Modified: Zope3/trunk/src/zope/index/text/setops.py
===================================================================
--- Zope3/trunk/src/zope/index/text/setops.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/setops.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -30,9 +30,9 @@
     # (len(Bucket) is fast; len(BTree) is slow).
     L.sort(lambda x, y: cmp(len(x[0]), len(y[0])))
     (x, wx), (y, wy) = L[:2]
-    dummy, result = family.IFModule.weightedIntersection(x, y, wx, wy)
+    dummy, result = family.IF.weightedIntersection(x, y, wx, wy)
     for x, wx in L[2:]:
-        dummy, result = family.IFModule.weightedIntersection(result, x, 1, wx)
+        dummy, result = family.IF.weightedIntersection(result, x, 1, wx)
     return result
 
 def mass_weightedUnion(L, family=BTrees.family32):
@@ -47,7 +47,7 @@
         # Merge the two smallest so far, and add back to the queue.
         (x, wx), dummy = merge.pop_smallest()
         (y, wy), dummy = merge.pop_smallest()
-        dummy, z = family.IFModule.weightedUnion(x, y, wx, wy)
+        dummy, z = family.IF.weightedUnion(x, y, wx, wy)
         merge.add((z, 1), len(z))
     (result, weight), dummy = merge.pop_smallest()
     return result
@@ -57,9 +57,9 @@
     # pair, we may still need to multiply the mapping by its weight.
     assert len(L) <= 1
     if len(L) == 0:
-        return family.IFModule.Bucket()
+        return family.IF.Bucket()
     [(result, weight)] = L
     if weight != 1:
-        dummy, result = family.IFModule.weightedUnion(
-            family.IFModule.Bucket(), result, 0, weight)
+        dummy, result = family.IF.weightedUnion(
+            family.IF.Bucket(), result, 0, weight)
     return result

Modified: Zope3/trunk/src/zope/index/text/tests/test_queryengine.py
===================================================================
--- Zope3/trunk/src/zope/index/text/tests/test_queryengine.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/text/tests/test_queryengine.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -28,7 +28,7 @@
     family = BTrees.family32
 
     def search(self, term):
-        b = self.family.IFModule.Bucket()
+        b = self.family.IF.Bucket()
         if term == "foo":
             b[1] = b[3] = 1
         elif term == "bar":

Modified: Zope3/trunk/src/zope/index/topic/filter.py
===================================================================
--- Zope3/trunk/src/zope/index/topic/filter.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/topic/filter.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -41,7 +41,7 @@
         self.clear()
 
     def clear(self):
-        self._ids = self.family.IIModule.Set()
+        self._ids = self.family.II.Set()
 
     def index_doc(self, docid, context):
         raise NotImplementedError

Modified: Zope3/trunk/src/zope/index/topic/index.py
===================================================================
--- Zope3/trunk/src/zope/index/topic/index.py	2007-04-26 14:51:17 UTC (rev 74782)
+++ Zope3/trunk/src/zope/index/topic/index.py	2007-04-26 14:52:38 UTC (rev 74783)
@@ -19,8 +19,6 @@
 
 import BTrees
 
-from BTrees.OOBTree import OOBTree
-
 from zope.interface import implements
 
 from zope.index.interfaces import IInjection
@@ -40,7 +38,7 @@
 
     def clear(self):
         # mapping filter id -> filter
-        self._filters = OOBTree()
+        self._filters = self.family.OO.BTree()
 
     def addFilter(self, f):
         """ Add filter 'f' with ID 'id' """
@@ -68,8 +66,9 @@
             raise TypeError(
                 'query argument must be a list/tuple of filter ids')
 
-        IIModule = self.family.IIModule
-        f = {'and': IIModule.intersection, 'or': IIModule.union}[operator]
+        f = {'and': self.family.II.intersection,
+             'or': self.family.II.union,
+             }[operator]
     
         rs = None
         for id in self._filters.keys():
@@ -78,4 +77,4 @@
                 rs = f(rs, docids)
             
         if rs: return rs
-        else: return self.family.IIModule.Set()
+        else: return self.family.II.Set()



More information about the Zope3-Checkins mailing list