[Checkins] SVN: zc.catalog/trunk/src/zc/catalog/ Separated a simple extent from a filter extent.

Albertas Agejevas alga at pov.lt
Mon Feb 5 14:21:48 EST 2007


Log message for revision 72378:
  Separated a simple extent from a filter extent.
  

Changed:
  U   zc.catalog/trunk/src/zc/catalog/extentcatalog.py
  U   zc.catalog/trunk/src/zc/catalog/extentcatalog.txt
  U   zc.catalog/trunk/src/zc/catalog/interfaces.py

-=-
Modified: zc.catalog/trunk/src/zc/catalog/extentcatalog.py
===================================================================
--- zc.catalog/trunk/src/zc/catalog/extentcatalog.py	2007-02-05 17:44:10 UTC (rev 72377)
+++ zc.catalog/trunk/src/zc/catalog/extentcatalog.py	2007-02-05 19:21:48 UTC (rev 72378)
@@ -30,21 +30,20 @@
 from zc.catalog import interfaces
 
 
-class FilterExtent(persistent.Persistent):
-    interface.implements(interfaces.IFilterExtent)
+class Extent(persistent.Persistent):
+    interface.implements(interfaces.IExtent)
     __parent__ = None
 
     BTreeAPI = zc.catalog.BTreeAPI32
 
-    def __init__(self, filter):
+    def __init__(self):
         self.BTreeAPI = zope.component.queryUtility(
             interfaces.IBTreeAPI,
             default=zc.catalog.BTreeAPI32)
         self.set = self.BTreeAPI.TreeSet()
-        self.filter = filter
 
-    def addable(self, uid, obj):
-        return self.filter(self, uid, obj)
+    def add(self, uid, obj):
+        self.set.insert(uid)
 
     def clear(self):
         self.set.clear()
@@ -92,12 +91,6 @@
     def __contains__(self, uid):
         return self.set.has_key(uid)
 
-    def add(self, uid, obj):
-        if not self.addable(uid, obj):
-            raise ValueError
-        else:
-            self.set.insert(uid)
-
     def remove(self, uid):
         self.set.remove(uid)
 
@@ -107,6 +100,24 @@
         except KeyError:
             pass
 
+
+class FilterExtent(Extent):
+    interface.implements(interfaces.IFilterExtent)
+
+    def __init__(self, filter):
+        super(FilterExtent, self).__init__()
+        self.filter = filter
+
+    def add(self, uid, obj):
+        if not self.addable(uid, obj):
+            raise ValueError
+        else:
+            self.set.insert(uid)
+
+    def addable(self, uid, obj):
+        return self.filter(self, uid, obj)
+
+
 class Catalog(catalog.Catalog):
     interface.implements(interfaces.IExtentCatalog)
     

Modified: zc.catalog/trunk/src/zc/catalog/extentcatalog.txt
===================================================================
--- zc.catalog/trunk/src/zc/catalog/extentcatalog.txt	2007-02-05 17:44:10 UTC (rev 72377)
+++ zc.catalog/trunk/src/zc/catalog/extentcatalog.txt	2007-02-05 19:21:48 UTC (rev 72378)
@@ -1,10 +1,12 @@
-An extent catalog is very similar to a normal catalog except that it only
-indexes items addable to its extent.  The extent is both a filter and a
-set that may be merged with other result sets.
+An extent catalog is very similar to a normal catalog except that it
+only indexes items addable to its extent.  The extent is both a filter
+and a set that may be merged with other result sets.  The filtering is
+an additional feature we will discuss below; we'll begin with a simple
+"do nothing" extent that only supports the second use case.
 
-To show the extent catalog at work, we need an intid utility, an index,
-some items to index, and a filter that determines what the extent accepts.
-We'll do this within a real ZODB and a real intid utility [#setup]_.
+To show the extent catalog at work, we need an intid utility, an
+index, some items to index.  We'll do this within a real ZODB and a
+real intid utility [#setup]_.
 
     >>> import zc.catalog
     >>> import zc.catalog.interfaces
@@ -39,8 +41,139 @@
     ...         self.id = name
     ...         self.__parent__ = parent
     ...
+
+    >>> extent = extentcatalog.Extent()
+    >>> verify.verifyObject(interfaces.IExtent, extent)
+    True
+    >>> root['catalog'] = catalog = extentcatalog.Catalog(extent)
+    >>> verify.verifyObject(interfaces.IExtentCatalog, catalog)
+    True
+    >>> index = DummyIndex()
+    >>> catalog['index'] = index
+    >>> transaction.commit()
+
+Now we have a catalog set up with an index and an extent.  We can add
+some data to the extent:
+
+    >>> matches = []
+    >>> for i in range(100):
+    ...     c = DummyContent(i, root)
+    ...     root[i] = c
+    ...     doc_id = intid.register(c)
+    ...     catalog.index_doc(doc_id, c)
+    ...     matches.append(doc_id)
+    >>> matches.sort()
+    >>> sorted(extent) == sorted(index.uids) == matches
+    True
+
+Unindexing an object that is in the catalog should simply remove it from the
+catalog and index as usual.
+
+    >>> matches[0] in catalog.extent
+    True
+    >>> matches[0] in catalog['index'].uids
+    True
+    >>> catalog.unindex_doc(matches[0])
+    >>> matches[0] in catalog.extent
+    False
+    >>> matches[0] in catalog['index'].uids
+    False
+    >>> doc_id = matches.pop(0)
+    >>> sorted(extent) == sorted(index.uids) == matches
+    True
+
+Clearing the catalog clears both the extent and the contained indexes.
+
+    >>> catalog.clear()
+    >>> list(catalog.extent) == list(catalog['index'].uids) == []
+    True
+
+Updating all indexes and an individual index both also update the extent.
+
+    >>> catalog.updateIndexes()
+    >>> matches.insert(0, doc_id)
+    >>> sorted(extent) == sorted(index.uids) == matches
+    True
+
+    >>> index2 = DummyIndex()
+    >>> catalog['index2'] = index2
+    >>> index2.__parent__ == catalog
+    True
+    >>> index.uids.remove(matches[0]) # to confirm that only index 2 is touched
+    >>> catalog.updateIndex(index2)
+    >>> sorted(extent) == sorted(index2.uids) == matches
+    True
+    >>> matches[0] in index.uids
+    False
+    >>> matches[0] in index2.uids
+    True
+    >>> res = index.uids.insert(matches[0])
+
+But so why have an extent in the first place?  It allows indices to
+operate against a reliable collection of the full indexed data;
+therefore, it allows the indices in zc.catalog to perform NOT
+operations.
+
+The extent itself provides a number of merging features to allow its
+values to be merged with other BTrees.IFBTree data structures.  These
+include intersection, union, difference, and reverse difference.
+Given an extent named 'extent' and another IFBTree data structure
+named 'data', intersections can be spelled "extent & data" or "data &
+extent"; unions can be spelled "extent | data" or "data | extent";
+differences can be spelled "extent - data"; and reverse differences
+can be spelled "data - extent".  Unions and intersections are
+weighted.
+
+    >>> extent = extentcatalog.Extent()
+    >>> for i in range(1, 100, 2):
+    ...     extent.add(i, None)
+    ...
+    >>> alt_set = BTreeAPI.TreeSet()
+    >>> alt_set.update(range(0, 166, 33)) # return value is unimportant here
+    6
+    >>> sorted(alt_set)
+    [0, 33, 66, 99, 132, 165]
+    >>> sorted(extent & alt_set)
+    [33, 99]
+    >>> sorted(alt_set & extent)
+    [33, 99]
+    >>> sorted(extent.intersection(alt_set))
+    [33, 99]
+    >>> original = set(extent)
+    >>> union_matches = original.copy()
+    >>> union_matches.update(alt_set)
+    >>> union_matches = sorted(union_matches)
+    >>> sorted(alt_set | extent) == union_matches
+    True
+    >>> sorted(extent | alt_set) == union_matches
+    True
+    >>> sorted(extent.union(alt_set)) == union_matches
+    True
+    >>> sorted(alt_set - extent)
+    [0, 66, 132, 165]
+    >>> sorted(extent.rdifference(alt_set))
+    [0, 66, 132, 165]
+    >>> original.remove(33)
+    >>> original.remove(99)
+    >>> set(extent - alt_set) == original
+    True
+    >>> set(extent.difference(alt_set)) == original
+    True
+
+[#cleanup]_
+
+
+Catalog with a filter extent
+----------------------------
+
+As discussed at the beginning of this document, extents can not only help
+with index operations, but also act as a filter, so that a given catalog
+can answer questions about a subset of the objects contained in the intids.
+
+The filter extent only stores objects that match a given filter.
+
     >>> def filter(extent, uid, ob):
-    ...     assert interfaces.IExtent.providedBy(extent)
+    ...     assert interfaces.IFilterExtent.providedBy(extent)
     ...     # This is an extent of objects with odd-numbered uids without a
     ...     # True ignore attribute
     ...     return uid % 2 and not getattr(ob, 'ignore', False)
@@ -48,7 +181,7 @@
     >>> extent = extentcatalog.FilterExtent(filter)
     >>> verify.verifyObject(interfaces.IFilterExtent, extent)
     True
-    >>> root['catalog'] = catalog = extentcatalog.Catalog(extent)
+    >>> root['catalog1'] = catalog = extentcatalog.Catalog(extent)
     >>> verify.verifyObject(interfaces.IExtentCatalog, catalog)
     True
     >>> index = DummyIndex()
@@ -95,25 +228,8 @@
     >>> sorted(extent) == sorted(index.uids) == matches
     True
 
-Unindexing an object that is in the catalog should simply remove it from the
-catalog and index as usual.
+Unindexing an object that is not in the catalog should be a no-op.
 
-    >>> matches[0] in catalog.extent
-    True
-    >>> matches[0] in catalog['index'].uids
-    True
-    >>> catalog.unindex_doc(matches[0])
-    >>> matches[0] in catalog.extent
-    False
-    >>> matches[0] in catalog['index'].uids
-    False
-    >>> doc_id = matches.pop(0)
-    >>> sorted(extent) == sorted(index.uids) == matches
-    True
-
-And similarly, unindexing an object that is not in the catalog should be a
-no-op.
-
     >>> fails[0] in catalog.extent
     False
     >>> catalog.unindex_doc(fails[0])
@@ -122,26 +238,15 @@
     >>> sorted(extent) == sorted(index.uids) == matches
     True
 
-Clearing the catalog clears both the extent and the contained indexes.
-
-    >>> catalog.clear()
-    >>> list(catalog.extent) == list(catalog['index'].uids) == []
-    True
-
 Updating all indexes and an individual index both also update the extent.
 
-    >>> catalog.updateIndexes()
-    >>> matches.insert(0, doc_id)
-    >>> sorted(extent) == sorted(index.uids) == matches
-    True
-
     >>> index2 = DummyIndex()
     >>> catalog['index2'] = index2
     >>> index2.__parent__ == catalog
     True
     >>> index.uids.remove(matches[0]) # to confirm that only index 2 is touched
     >>> catalog.updateIndex(index2)
-    >>> sorted(extent) == sorted(index2.uids) == matches
+    >>> sorted(extent) == sorted(index2.uids)
     True
     >>> matches[0] in index.uids
     False
@@ -172,52 +277,7 @@
     ...  == sorted(index2.uids))
     True
 
-The extent itself provides a number of merging features to allow its values to
-be merged with other BTrees.IFBTree data structures.  These include
-intersection, union, difference, and reverse difference.  Given an extent
-named 'extent' and another IFBTree data structure named 'data', intersections
-can be spelled "extent & data" or "data & extent"; unions can be spelled
-"extent | data" or "data | extent"; differences can be spelled "extent - data";
-and reverse differences can be spelled "data - extent".  Unions and
-intersections are weighted.
 
-    >>> extent = extentcatalog.FilterExtent(filter)
-    >>> for i in range(1, 100, 2):
-    ...     extent.add(i, None)
-    ...
-    >>> alt_set = BTreeAPI.TreeSet()
-    >>> alt_set.update(range(0, 166, 33)) # return value is unimportant here
-    6
-    >>> sorted(alt_set)
-    [0, 33, 66, 99, 132, 165]
-    >>> sorted(extent & alt_set)
-    [33, 99]
-    >>> sorted(alt_set & extent)
-    [33, 99]
-    >>> sorted(extent.intersection(alt_set))
-    [33, 99]
-    >>> original = set(extent)
-    >>> union_matches = original.copy()
-    >>> union_matches.update(alt_set)
-    >>> union_matches = sorted(union_matches)
-    >>> sorted(alt_set | extent) == union_matches
-    True
-    >>> sorted(extent | alt_set) == union_matches
-    True
-    >>> sorted(extent.union(alt_set)) == union_matches
-    True
-    >>> sorted(alt_set - extent)
-    [0, 66, 132, 165]
-    >>> sorted(extent.rdifference(alt_set))
-    [0, 66, 132, 165]
-    >>> original.remove(33)
-    >>> original.remove(99)
-    >>> set(extent - alt_set) == original
-    True
-    >>> set(extent.difference(alt_set)) == original
-    True
-
-
 Self-populating extents
 -----------------------
 
@@ -400,3 +460,12 @@
     ...     return obj._p_jar.root()['components']
     ...
     >>> zope.component.provideAdapter(getComponentLookup)
+
+
+.. [#cleanup] Unregister the objects of the previous tests from intid utility:
+
+    >>> intid = zope.component.getUtility(
+    ...     zope.app.intid.interfaces.IIntIds, context=root)
+    >>> for doc_id in matches:
+    ...     intid.unregister(intid.queryObject(doc_id))
+

Modified: zc.catalog/trunk/src/zc/catalog/interfaces.py
===================================================================
--- zc.catalog/trunk/src/zc/catalog/interfaces.py	2007-02-05 17:44:10 UTC (rev 72377)
+++ zc.catalog/trunk/src/zc/catalog/interfaces.py	2007-02-05 19:21:48 UTC (rev 72378)
@@ -23,16 +23,16 @@
 from zc.catalog.i18n import _
 import BTrees.Interfaces
 
+
 class IExtent(interface.Interface):
+    """An extent represents the full set of objects indexed by a catalog.
+    It is useful for a variety of index operations and catalog queries.
+    """
 
     __parent__ = interface.Attribute(
         """The catalog for which this is an extent; must be None before it is
         set to a catalog""")
 
-    def addable(uid, obj):
-        """returns True or False, indicating whether the obj may be added to
-        the extent"""
-
     def add(uid, obj):
         """add uid to extent; raise ValueError if it is not addable.
 
@@ -88,6 +88,7 @@
     def __contains__(uid):
         "return boolean indicating if uid is in set"
 
+
 class IFilterExtent(IExtent):
 
     filter = interface.Attribute(
@@ -95,7 +96,11 @@
         associated obj and should return a boolean True (is member of extent)
         or False (is not member of extent).""")
 
+    def addable(uid, obj):
+        """returns True or False, indicating whether the obj may be added to
+        the extent"""
 
+
 class ISelfPopulatingExtent(IExtent):
     """An extent that knows how to create it's own initial population."""
 
@@ -126,6 +131,7 @@
     extent = interface.Attribute(
         """An IExtent of the objects cataloged""")
 
+
 class IIndexValues(interface.Interface):
     """An index that allows introspection of the indexed values"""
 
@@ -175,6 +181,7 @@
         IIndexValues.values(doc_id=id).
         """
 
+
 class ISetIndex(interface.Interface):
 
     def apply(query):
@@ -206,6 +213,7 @@
         the extent that do not have any values in the index.
         """
 
+
 class IValueIndex(interface.Interface):
 
     def apply(query):
@@ -234,16 +242,19 @@
         the extent that do not have any values in the index.
         """
 
+
 class ICatalogValueIndex(zope.app.catalog.interfaces.IAttributeIndex,
                          zope.app.catalog.interfaces.ICatalogIndex):
     """Interface-based catalog value index
     """
 
+
 class ICatalogSetIndex(zope.app.catalog.interfaces.IAttributeIndex,
                        zope.app.catalog.interfaces.ICatalogIndex):
     """Interface-based catalog set index
     """
 
+
 class INormalizationWrapper(zope.index.interfaces.IInjection,
                             zope.index.interfaces.IIndexSearch,
                             zope.index.interfaces.IStatistics,
@@ -262,6 +273,7 @@
         (each composite value normalized) or not (original value is
         normalized)""")
 
+
 class INormalizer(interface.Interface):
 
     def value(value):
@@ -288,6 +300,7 @@
     (_('day'), _('hour'), _('minute'), _('second'), _('microsecond')))])
     #  0         1          2            3            4
 
+
 class IDateTimeNormalizer(INormalizer):
     resolution = schema.Choice(
         vocabulary=resolution_vocabulary,



More information about the Checkins mailing list