[Checkins] SVN: z3c.indexer/trunk/ see CHANGES.txt
Roger Ineichen
roger at projekt01.ch
Mon Dec 1 19:08:26 EST 2008
Log message for revision 93523:
see CHANGES.txt
The main goal for this refactoring was indexing speedup and avoid duplicated un/indexing calls
Changed:
_U z3c.indexer/trunk/
U z3c.indexer/trunk/CHANGES.txt
U z3c.indexer/trunk/setup.py
U z3c.indexer/trunk/src/z3c/indexer/README.txt
A z3c.indexer/trunk/src/z3c/indexer/_bbb.py
A z3c.indexer/trunk/src/z3c/indexer/collector.py
U z3c.indexer/trunk/src/z3c/indexer/index.py
U z3c.indexer/trunk/src/z3c/indexer/indexer.py
U z3c.indexer/trunk/src/z3c/indexer/interfaces.py
U z3c.indexer/trunk/src/z3c/indexer/performance.py
A z3c.indexer/trunk/src/z3c/indexer/sample.txt
U z3c.indexer/trunk/src/z3c/indexer/search.py
U z3c.indexer/trunk/src/z3c/indexer/subscriber.py
-=-
Property changes on: z3c.indexer/trunk
___________________________________________________________________
Modified: svn:ignore
- bin
develop-eggs
parts
.installed.cfg
+ bin
develop-eggs
parts
.installed.cfg
coverage
Modified: z3c.indexer/trunk/CHANGES.txt
===================================================================
--- z3c.indexer/trunk/CHANGES.txt 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/CHANGES.txt 2008-12-02 00:08:26 UTC (rev 93523)
@@ -5,26 +5,31 @@
Version 0.5.1dev (unreleased)
-----------------------------
-- bugfix: SearchQuery wasn't able to act correct if initialized with query=None
+- Bugfix: SearchQuery wasn't able to act correct if initialized with query=None
because of an empty result setup. The And() and Not() methods could not act
correctly with this an empty initilized result.
-- bugfix: SearchQuery.And() and Not() didn't return an empty result if previous
- query result was empty. And() method didn't return a empty result if given
- query was empty. The previous result was returned.
+- Bugfix: SearchQuery.And() and Not() didn't return an empty result if previous
+ or given query result was empty. The previous result was returned.
-- bugfix: performance tests, not query did not show the right timer in catalog
+- Feature: implemented new indexing strategy which uses transaction and thread
+ local for prevent to index an object more then once per transaction.
+
+- Bugfix: performance tests, not query did not show the right timer in catalog
cell.
-- feature: Expose SearchQuery.results property. Ensure that we return always an
+- Feature: added optional searchResultFactory. This class get used as a search
+ result wrapper. By default the SearchResult class get used.
+
+- Feature: Expose SearchQuery.results property. Ensure that we return always an
empty result if None is given and allow to override existing results as a
part of the SearchQuery API.
-- feature: Implemented optional intids argument in SearchQuery.searchResults
+- Feature: Implemented optional intids argument in SearchQuery.searchResults
method. This intids is used instead of query the IntIds util. This is usefull
if you use builtin IIntIds objects for optimized access.
-- Added __repr__ for ResultSet with result lenght
+- Added __repr__ for ResultSet with result lenght.
- Optimized SearchQuery.And() and Not() methods. Skip given query processing
if previous result is empty.
Modified: z3c.indexer/trunk/setup.py
===================================================================
--- z3c.indexer/trunk/setup.py 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/setup.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -55,29 +55,25 @@
'z3c.testing',
'zope.testing',
'zope.app.keyreference',
+ 'zope.app.component',
],
performance = [
+ 'zc.catalog',
+ 'zope.app.catalog',
'zope.app.component',
- 'zope.app.pagetemplate',
- 'zope.app.publisher',
- 'zope.app.publication',
'zope.app.container',
+ 'zope.app.intid',
+ 'zope.app.keyreference',
'zope.app.testing',
- 'zope.app.zapi',
- 'zope.app.pagetemplate',
- 'zope.contentprovider',
- 'zope.i18n',
- 'zope.i18nmessageid',
+ 'zope.cachedescriptors',
+ 'zope.component',
+ 'zope.deferredimport',
'zope.event',
- 'zope.i18n',
- 'zope.i18nmessageid',
- 'zope.lifecycleevent',
+ 'zope.index',
'zope.interface',
+ 'zope.lifecycleevent',
+ 'zope.location',
'zope.schema',
- 'zope.security',
- 'zope.testing',
- 'zope.traversing',
- 'zope.viewlet',
],
),
install_requires = [
@@ -86,10 +82,13 @@
'zope.app.container',
'zope.app.intid',
'zope.app.keyreference',
+ 'zope.cachedescriptors',
'zope.component',
- 'zope.configuration',
+ 'zope.deferredimport',
+ 'zope.event',
'zope.index',
'zope.interface',
+ 'zope.lifecycleevent',
'zope.location',
'zope.schema',
],
Modified: z3c.indexer/trunk/src/z3c/indexer/README.txt
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/README.txt 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/src/z3c/indexer/README.txt 2008-12-02 00:08:26 UTC (rev 93523)
@@ -56,43 +56,98 @@
Performeance
------------
-See also the performance test located in this package. Here is a sample output
-from my 2 GHz Duo Core Laptop:
+See also the performance test located in this package. The interesting part is
+the speedup different for a larger amount of indexes. Here is a sample output
+from my 2,53 GHz Thinkpad W500:
-- 1000 x repeat tests
-
+Run test with
+-------------
+- 100 x repeat tests
- 10000 objects
-
- 3 relevant indexes
+- 25 other indexes
+Note, the index update get only processed one time.
-- 50 other indexes
+z3c.indexer
+indexer based indexing time: 2.47 s
+indexer based query time: 0.72 s
+indexer based not query time: 0.73 s
+indexer based update time: 1.05 s
+indexer object modified time: 0.02 s
+indexer parent modified time: 0.00 s
+indexer object moved time: 0.00 s
+indexer parent moved time: 15.00 s
+indexer object remove time: 2.08 s
+zope.app.catalog
+catalog based indexing time: 14.41 s
+catalog based query time: 0.97 s
+catalog based not query time: 3.63 s
+catalog based update time: 7.72 s
+catalog object modified time: 0.09 s
+catalog parent modified time: 0.02 s
+catalog object moved time: 0.00 s
+catalog parent moved time: 15.11 s
+catalog object remove time: 8.47 s
+
+Result for 10000 objects with 3 relevant and 25 other indexes
+ ----------------------------------------------------------------------------------
+| type | indexing | query | not query | update | modify | moved | remove |
+ ----------------------------------------------------------------------------------
+| catalog | 14.41s | 0.97s | 3.63s | 7.72s | 0.09s | 0.00s | 8.47s |
+ ----------------------------------------------------------------------------------
+| indexer | 2.47s | 0.72s | 0.73s | 1.05s | 0.02s | 0.00s | 2.08s |
+ ----------------------------------------------------------------------------------
+| speedup | 11.94s | 0.25s | 2.89s | 6.67s | 0.08s | 0.00s | 6.39s |
+ ----------------------------------------------------------------------------------
+| speedup | 483% | 35% | 393% | 638% | 487% | 0% | 308% |
+ ----------------------------------------------------------------------------------
+
+
+Run test with
+-------------
+- 100 x repeat tests
+- 10000 objects
+- 3 relevant indexes
+- 75 other indexes
Note, the index update get only processed one time.
-Result for 10000 objects with 3 relevant and 50 other indexes
- ------------------------------------------------------------------------
-| type | indexing | query | not query | update | modify | remove |
- ------------------------------------------------------------------------
-| catalog | 37.39s | 17.83s | 17.83s | 31.14s | 0.05s | 28.53s |
- ------------------------------------------------------------------------
-| indexer | 4.84s | 10.75s | 10.84s | 1.66s | 0.03s | 3.73s |
- ------------------------------------------------------------------------
-| speedup | 32.55s | 7.08s | 70.03s | 29.48s | 0.02s | 24.80s |
- ------------------------------------------------------------------------
-| speedup | 672% | 66% | 646% | 1780% | 47% | 664% |
- ------------------------------------------------------------------------
+z3c.indexer
+indexer based indexing time: 2.63 s
+indexer based query time: 0.72 s
+indexer based not query time: 0.73 s
+indexer based update time: 1.03 s
+indexer object modified time: 0.02 s
+indexer parent modified time: 0.00 s
+indexer object moved time: 0.00 s
+indexer parent moved time: 15.09 s
+indexer object remove time: 2.09 s
-This speedup tests shows that the indexing and object remove time get improved
-by more then 6 times. This is a very common usecase in many application. The
-indexer update test is probably not really comparable since we do not update
-all indexes. We only update the relevant objects in the index which makes
-it very fast. But that's the goal of this package, it offers the concepts
-for doing things which can prevent to run into performance problems. Which
-means we can compare the update speedup because the indexes get updated
-with the relevant objects. I've you can agree on this, the speedup for
-index update is more then 17 times ;-)
+zope.app.catalog
+catalog based indexing time: 36.92 s
+catalog based query time: 0.75 s
+catalog based not query time: 2.66 s
+catalog based update time: 22.33 s
+catalog object modified time: 0.22 s
+catalog parent modified time: 0.02 s
+catalog object moved time: 0.00 s
+catalog parent moved time: 15.22 s
+catalog object remove time: 19.53 s
+Result for 10000 objects with 3 relevant and 75 other indexes
+ ----------------------------------------------------------------------------------
+| type | indexing | query | not query | update | modify | moved | remove |
+ ----------------------------------------------------------------------------------
+| catalog | 36.92s | 0.75s | 2.66s | 22.33s | 0.22s | 0.00s | 19.53s |
+ ----------------------------------------------------------------------------------
+| indexer | 2.63s | 0.72s | 0.73s | 1.03s | 0.02s | 0.00s | 2.09s |
+ ----------------------------------------------------------------------------------
+| speedup | 34.30s | 0.03s | 1.92s | 21.30s | 0.20s | 0.00s | 17.44s |
+ ----------------------------------------------------------------------------------
+| speedup | 1307% | 4% | 261% | 2066% | 1269% | 0% | 833% |
+ ----------------------------------------------------------------------------------
+
Goals
-----
@@ -101,18 +156,14 @@
indexing
~~~~~~~~
- - Allow to explicit define of what, where get indexed
+ - Allow to explicit define of what, where and when get indexed
- Reduce index calls. The existing zope.app.catalog implementation forces to
index every object which raises a ObjectAddedEvent. This ends in trying to
- index each object on every existing index contained in a catalog.
+ index each object on every existing index contained in a catalog.
- - The existing container add item concept forces to reindex every item
- in a container which is most the time not needed. This is because the
- existing event dispatching to it's sublocation.
+ - Get rid of persistent catalog as a single indexing utility.
- - Get rid of persistent catalog and it's index items.
-
- Use indexes as utilities
searching
@@ -404,127 +455,7 @@
1
-Auto indexing
--------------
-Sometimes, you like to ensure that each object get indexed or updated in the
-index like we us to do in the default zope.app.catalog implementation. This
-means each object get index after adding or updated on object modification.
-We offer a solution for this behavior with the IAutoIndexer adapter call.
-On each object added event or object modified event, a subscriber tries to
-lookup an IAutoIndexer which could index or update the object values in the
-relevant indexes. Since the subscriber calls getAdapters, it's allowed to
-have more then one such indexer adapter.
-
-First register a new index:
-
- >>> from z3c.indexer.index import TextIndex
- >>> autoIndex = TextIndex()
- >>> sm['default']['autoIndex'] = textIndex
- >>> sm.registerUtility(autoIndex, interfaces.IIndex, name='autoIndex')
-
-The new auto (text) index does not contain any indexed objects right now:
-
- >>> autoIndex.documentCount()
- 0
-
-Let's now define a IAutoIndexer adapter:
-
- >>> from z3c.indexer.indexer import ValueAutoIndexer
- >>> class MyDemoContentAutoIndexer(ValueAutoIndexer):
- ... zope.component.adapts(IDemoContent)
- ...
- ... indexName = 'autoIndex'
- ...
- ... @property
- ... def value(self):
- ... """Get the value form context."""
- ... return 'auto indexed value: %s %s' % (self.context.title,
- ... self.context.body)
-
-and register them:
-
- >>> zope.component.provideAdapter(MyDemoContentAutoIndexer, name='Auto')
-
-Now we need to register our subscriber which calls the IAutoIndexer adapters:
-
- >>> from z3c.indexer import subscriber
- >>> zope.component.provideHandler(subscriber.autoIndexSubscriber)
- >>> zope.component.provideHandler(subscriber.autoUnindexSubscriber)
-
-and we also need to register the intid subscribers:
-
- >>> from zope.app.intid import addIntIdSubscriber
- >>> from zope.app.intid import removeIntIdSubscriber
- >>> zope.component.provideHandler(addIntIdSubscriber)
- >>> zope.component.provideHandler(removeIntIdSubscriber)
-
-If we now add a new object it get auomaticly indexed without to call the
-indexer method explicit:
-
- >>> autoIndex.documentCount()
- 0
-
- >>> textIndex.documentCount()
- 1
-
- >>> setIndex.documentCount()
- 1
-
- >>> valueIndex.documentCount()
- 1
-
-Let's now add a new content object:
-
- >>> autoDemo = DemoContent(u'Auto')
- >>> autoDemo.description = u'Auto Demo'
- >>> autoDemo.field = u'Auto field'
- >>> autoDemo.value = u'auto value'
- >>> autoDemo.iterable = (1, 2, 'Iterable')
- >>> site['autoDemo'] = autoDemo
-
-You can see that we've got a key reference for the new object:
-
- >>> intids.getId(autoDemo) is not None
- True
-
-You can see that the new object get added without to call the index method
-defined in indexer module:
-
- >>> autoIndex.documentCount()
- 1
-
- >>> textIndex.documentCount()
- 1
-
- >>> setIndex.documentCount()
- 1
-
- >>> valueIndex.documentCount()
- 1
-
-As you can see only the ``autoIndex`` get used.
-
-Of corse there is also a ``auto`` unindex implementation which get used if we
-remove the object:
-
- >>> del site['autoDemo']
-
-As you can see the object got removed from the ``autoIndex``:
-
- >>> autoIndex.documentCount()
- 0
-
- >>> textIndex.documentCount()
- 1
-
- >>> setIndex.documentCount()
- 1
-
- >>> valueIndex.documentCount()
- 1
-
-
SearchQuery
-----------
@@ -1116,8 +1047,14 @@
Another use case which we didn't test is that a applyIn can contain the
same object twice with different values. Let's test the built in union
-which removes such duplications:
+which removes such duplications. Since this is the first test which uses
+IIntId subscribers let' register them:
+ >>> from zope.app.intid import addIntIdSubscriber
+ >>> from zope.app.intid import removeIntIdSubscriber
+ >>> zope.component.provideHandler(addIntIdSubscriber)
+ >>> zope.component.provideHandler(removeIntIdSubscriber)
+
>>> demo1 = DemoContent(u'Demo 1')
>>> demo1.description = u'Description'
>>> demo1.field = u'Field'
Added: z3c.indexer/trunk/src/z3c/indexer/_bbb.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/_bbb.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/_bbb.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -0,0 +1,87 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import zope.interface
+import zope.component
+from zope.app.intid.interfaces import IIntIdAddedEvent
+from zope.app.intid.interfaces import IIntIdRemovedEvent
+
+from z3c.indexer import interfaces
+from z3c.indexer import indexer
+
+
+# Remove in 1.0.0 release
+class IAutoIndexer(zope.interface.Interface):
+ """Auto indexer adapter get called by the auto indexer subscriber."""
+
+ oid = zope.interface.Attribute("""IIntId of context.""")
+
+ def doIndex():
+ """Index the context in the relevant index."""
+
+ def doUnIndex():
+ """Unindex the context in the relevant index."""
+
+
+class IValueAutoIndexer(IAutoIndexer):
+ """Value (auto) indexer."""
+
+ index = zope.interface.Attribute("""The named index utility.""")
+
+ value = zope.interface.Attribute("""Value for context/index combination.""")
+
+
+class IMultiAutoIndexer(IAutoIndexer):
+ """Multi (auto) indexer"""
+
+ def getIndex(indexName):
+ """The named index utility getter method."""
+
+
+# implicit indexing pattern (generic)
+# IAutoIndexer adapter
+ at zope.component.adapter(IIntIdAddedEvent)
+def autoIndexSubscriber(event):
+ """Index all objects which get added to the intids utility."""
+ adapters = zope.component.getAdapters((event.object,),
+ interfaces.IAutoIndexer)
+ for name, adapter in adapters:
+ adapter.doIndex()
+
+
+# IAutoIndexer adapter
+ at zope.component.adapter(IIntIdRemovedEvent)
+def autoUnindexSubscriber(event):
+ """Unindex all objects which get added to the intids utility."""
+ adapters = zope.component.getAdapters((event.object,),
+ interfaces.IAutoIndexer)
+ for name, adapter in adapters:
+ adapter.doUnIndex()
+
+
+class ValueAutoIndexer(indexer.ValueIndexerBase):
+ """Value (auto) indexer implementation."""
+
+ zope.interface.implements(interfaces.IValueAutoIndexer)
+
+
+class MultiAutoIndexer(indexer.MultiIndexerBase):
+ """Can be used as base for index a object in more then one index."""
+
+ zope.interface.implements(interfaces.IMultiAutoIndexer)
+
Added: z3c.indexer/trunk/src/z3c/indexer/collector.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/collector.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/collector.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -0,0 +1,153 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import threading
+import transaction
+
+import zope.interface
+
+from z3c.indexer import interfaces
+
+
+class IndexerCollector(object):
+ """The collector stores pending indexer and their indexing state.
+
+ This indexer get calculated and the relevant indexer get processed at the
+ end of the transaction.
+ """
+ zope.interface.implements(interfaces.IIndexerCollector)
+
+ def __init__(self, transaction):
+ self.transaction = transaction
+ self._pending = []
+ self._prepared = False
+
+ def addIndexer(self, name, indexer):
+ """Index a value by it's id."""
+ self._pending.append((name, indexer.oid, indexer, True))
+
+ def addUnIndexer(self, name, indexer):
+ # Since the oid is Lazy, we ensure by accessing it that the oid is
+ # available after remove the object from the IntIds utility.
+ self._pending.append((name, indexer.oid, indexer, False))
+
+ def abort(self, transaction):
+ self._checkTransaction(transaction)
+ if self.transaction is not None:
+ self.transaction = None
+ self._pending = []
+ self._prepared = False
+
+ def tpc_begin(self, transaction):
+ if self._prepared:
+ raise TypeError('Already prepared')
+ self._checkTransaction(transaction)
+ self.transaction = transaction
+ self._prepared = True
+
+ def tpc_vote(self, transaction):
+ """unify pending indexer calls.
+
+ Some sample use cases are:
+
+ index -> index
+ index, index -> index
+ index, index, index -> index
+
+ unindex -> unindex
+ index, unindex -> unindex
+ index, index, unindex -> unindex
+
+ The following usecases will work but should not happen:
+
+ unindex, unindex -> unindex (1)
+
+ The following usecases will work but should not happen:
+
+ unindex, index -> works but should not happen (2)
+ index, unindex, index -> works but should not happen (2)
+
+ (1) calling unindex two times without our indexer collector will end
+ in a KeyError in IIntIds because of the already removed object. We
+ silently ignore this fact and just unindex the object once because it
+ should never happen.
+
+ (2) index after unindex means that the object get removed from the
+ IIntIds utility. This should never happen but will work. The following
+ is going on in such a scenario. The last unindex will remove the oid
+ from the indexes and the first index call will index the new oid in the
+ relevant indexes.
+
+ """
+ self._checkTransaction(transaction)
+ # find obsolate objects
+ doIndex = []
+ doUnIndex = []
+ seen = {}
+ # build a dict with final indexing states based on our chronological
+ # ordered (name, oid, indexer, state) list
+
+ for name, oid, indexer, state in self._pending:
+ seen[(name, oid, indexer.__class__)] = (indexer, state)
+
+ for (name, oid, cls), (indexer, state) in seen.items():
+ if state:
+ indexer.doIndex()
+ else:
+ indexer.doUnIndex()
+ self._pending = []
+
+ def commit(self, transaction):
+ pass
+
+ def tpc_finish(self, transaction):
+ pass
+
+ def tpc_abort(self, transaction):
+ self.abort(transaction)
+
+ def _checkTransaction(self, transaction):
+ if (self.transaction is not None and
+ self.transaction is not transaction):
+ raise TypeError("Transaction missmatch",
+ transaction, self.transaction)
+
+ def sortKey(self):
+ return str(id(self))
+
+
+_collector = threading.local()
+
+def get():
+ """Returns a index collector
+
+ Threading local provides a storage for our index collector. This will
+ ensure taht we never use more then one collector in a transaction.
+ """
+ txn = transaction.manager.get()
+ indexerCollector = getattr(_collector, 'z3cIndexerCollector', None)
+ if indexerCollector is None:
+ indexerCollector = IndexerCollector(txn)
+ _collector.z3cIndexerCollector = indexerCollector
+ txn.join(indexerCollector)
+ elif txn != indexerCollector.transaction:
+ # can happen in testing where we use more then one setUp, tearDown
+ indexerCollector = IndexerCollector(txn)
+ _collector.z3cIndexerCollector = indexerCollector
+ txn.join(indexerCollector)
+ return indexerCollector
Modified: z3c.indexer/trunk/src/z3c/indexer/index.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/index.py 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/src/z3c/indexer/index.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -39,8 +39,7 @@
self.unindex_doc(oid)
-class TextIndex(IndexMixin, textindex.TextIndex,
- contained.Contained):
+class TextIndex(IndexMixin, textindex.TextIndex, contained.Contained):
"""Text index based on zope.index.text.textindex.TextIndex"""
zope.interface.implements(interfaces.ITextIndex)
Modified: z3c.indexer/trunk/src/z3c/indexer/indexer.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/indexer.py 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/src/z3c/indexer/indexer.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -18,12 +18,22 @@
import zope.interface
import zope.component
+import zope.deferredimport
from zope.cachedescriptors.property import Lazy
from zope.app.intid.interfaces import IIntIds
from z3c.indexer import interfaces
+# deprecated IAutoIndexer (remove in 1.0.0 release)
+zope.deferredimport.deprecated(
+ "IAutoIndexer will go away in 1.0.0 release. Implement IIndexerSubscriber "
+ "and use them within the new intIdAddedEventDispatcher and "
+ "intIdRemovedEventDispatcher subscribers.",
+ ValueAutoIndexer = 'z3c.indexer._bbb:ValueAutoIndexer',
+ MultiAutoIndexer = 'z3c.indexer._bbb:MultiAutoIndexer',
+ )
+
def index(context):
"""Index object.
@@ -68,8 +78,17 @@
@Lazy
def oid(self):
- """Get IntId for the adapted context."""
- intids = zope.component.getUtility(IIntIds, context=self.context)
+ """Get IntId for the adapted context.
+
+ This attribute must be Lazy stored and the oid must get accessed right
+ after we collect the indexer in our transaction data manager. The
+ reason is; the oid will not be available at the end of our transaction
+ if we remove objects because the object get removed from the IIntids
+ utility before we process the indexer. If you access other objects
+ during unindexing make sure they still exist at the end of the
+ transactions or keep a reference to them in the indexer adapter.
+ """
+ intids = zope.component.getUtility(IIntIds)
return intids.getId(self.context)
def doIndex(self):
@@ -89,8 +108,7 @@
@Lazy
def index(self):
- return zope.component.getUtility(interfaces.IIndex, self.indexName,
- context=self.context)
+ return zope.component.getUtility(interfaces.IIndex, self.indexName)
@property
def value(self):
@@ -110,12 +128,6 @@
zope.interface.implements(interfaces.IValueIndexer)
-class ValueAutoIndexer(ValueIndexerBase):
- """Value (auto) indexer implementation."""
-
- zope.interface.implements(interfaces.IValueAutoIndexer)
-
-
# multi indexer
class MultiIndexerBase(IndexerBase):
"""Can be used a s base for index a object in more then one index."""
@@ -125,8 +137,7 @@
self.context = context
def getIndex(self, indexName):
- return zope.component.getUtility(interfaces.IIndex, indexName,
- context=self.context)
+ return zope.component.getUtility(interfaces.IIndex, indexName)
def doIndex(self):
"""Implement your own indexing pattern."""
@@ -138,12 +149,6 @@
class MultiIndexer(MultiIndexerBase):
- """Can be used a s base for index a object in more then one index."""
+ """Can be used as base for index a object in more then one index."""
zope.interface.implements(interfaces.IMultiIndexer)
-
-
-class MultiAutoIndexer(MultiIndexerBase):
- """Can be used a s base for index a object in more then one index."""
-
- zope.interface.implements(interfaces.IMultiAutoIndexer)
Modified: z3c.indexer/trunk/src/z3c/indexer/interfaces.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/interfaces.py 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/src/z3c/indexer/interfaces.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -18,7 +18,9 @@
import zope.interface
+import zope.deferredimport
+from transaction.interfaces import IDataManager
from zope.index.interfaces import IInjection
from zope.index.interfaces import IIndexSearch
from zope.index.interfaces import IStatistics
@@ -26,7 +28,16 @@
NOVALUE = object()
+# deprecated auto index subscribers
+zope.deferredimport.deprecated(
+ "IAutoIndexer will go away in 1.0.0 release. Use the new optimized "
+ "intIdAddedEventDispatcher and intIdRemovedEventDispatcher subscribers.",
+ IAutoIndexer = 'z3c.indexer._bbb:IAutoIndexer',
+ IValueAutoIndexer = 'z3c.indexer._bbb:IValueAutoIndexer',
+ IMultiAutoIndexer = 'z3c.indexer._bbb:IMultiAutoIndexer',
+ )
+
class ISearchQuery(zope.interface.Interface):
"""Chainable search query."""
@@ -69,6 +80,7 @@
"""
+# adapter based indexing concept
class IIndexer(zope.interface.Interface):
"""An Indexer knows how to index objects."""
@@ -96,33 +108,6 @@
"""The named index utility getter method."""
-class IAutoIndexer(zope.interface.Interface):
- """Auto indexer adapter get called by the auto indexer subscriber."""
-
- oid = zope.interface.Attribute("""IIntId of context.""")
-
- def doIndex():
- """Index the context in the relevant index."""
-
- def doUnIndex():
- """Unindex the context in the relevant index."""
-
-
-class IValueAutoIndexer(IAutoIndexer):
- """Value (auto) indexer."""
-
- index = zope.interface.Attribute("""The named index utility.""")
-
- value = zope.interface.Attribute("""Value for context/index combination.""")
-
-
-class IMultiAutoIndexer(IAutoIndexer):
- """Multi (auto) indexer"""
-
- def getIndex(indexName):
- """The named index utility getter method."""
-
-
# index base interface
class IIndex(zope.interface.Interface, IContained):
"""An Index marker interface."""
@@ -317,3 +302,14 @@
"""Knows how to lookup index values."""
value = zope.interface.Attribute("""Index value of context.""")
+
+
+# transaction based indexing concept
+class IIndexerCollector(IDataManager):
+ """Collects IIndexer which get processed at the end of the transaction."""
+
+ def addIndexer(obj):
+ """Add an object indexer."""
+
+ def addUnIndexer(obj):
+ """Add an object un-indexer."""
Modified: z3c.indexer/trunk/src/z3c/indexer/performance.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/performance.py 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/src/z3c/indexer/performance.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -17,6 +17,7 @@
__docformat__ = "reStructuredText"
import time
+import transaction
import persistent
import zope.interface
import zope.component
@@ -24,9 +25,7 @@
import zope.event
import zope.lifecycleevent
import zope.location.interfaces
-from ZODB.interfaces import IConnection
from BTrees.IFBTree import difference, IFBTree
-from persistent.interfaces import IPersistent
from zope.index.text.textindex import TextIndex as ZTextIndex
from zope.app.keyreference.interfaces import IKeyReference
@@ -34,7 +33,6 @@
from zope.app.component import hooks
from zope.app.catalog.interfaces import ICatalog
-from zope.app.catalog.text import ITextIndex
from zope.app.catalog.field import FieldIndex as ZFieldIndex
from zope.app.catalog.interfaces import ICatalogIndex
from zope.app.catalog.catalog import Catalog
@@ -59,17 +57,18 @@
from z3c.indexer import interfaces
from z3c.indexer import subscriber
+from z3c.indexer import collector
from z3c.indexer.query import TextQuery
from z3c.indexer.query import Eq
from z3c.indexer.index import TextIndex
from z3c.indexer.index import FieldIndex
from z3c.indexer.index import ValueIndex
-from z3c.indexer.indexer import MultiAutoIndexer
+from z3c.indexer.indexer import MultiIndexer
from z3c.indexer.search import SearchQuery
-
timeResult = None
+
def timeTest(function, counter=10000, *args, **kw):
"""timer"""
res = []
@@ -81,28 +80,45 @@
total_time = time.time() - start_time
global timeResult
timeResult = total_time
+ # commit transaction forces indexer collector indexing our objects
+ transaction.commit()
return res
return wrapper(*args, **kw)
+#def calcSpeedUp(catalogTime, indexerTime):
+# # adjust time and prevent zero division errors
+# if float(catalogTime) == 0:
+# catalogTime = 0.001
+# if float(indexerTime) == 0:
+# indexerTime = 0.001
+# if catalogTime > indexerTime:
+# # faster
+# precent = int('%.0f' % \
+# float(float(catalogTime) / float(indexerTime) * 100 -100))
+# return '%4d%s' % (precent, '%')
+# else:
+# # slower
+# precent = int('%.0f' % \
+# float(float(indexerTime) / float(catalogTime) * 100 -100))
+# return '-%3d%s' % (precent, '%')
def calcSpeedUp(catalogTime, indexerTime):
+ # adjust time and prevent zero division errors
+ indexerTime = float(indexerTime) or 0.001
+ catalogTime = float(catalogTime) or 0.001
try:
- if catalogTime > indexerTime:
+ if catalogTime < indexerTime:
+ # slower
+ precent = int('%.0f' % float(indexerTime / catalogTime * 100 -100))
+ return '-%3d%s' % (precent, '%')
+ else:
# faster
- precent = int('%.0f' % \
- float(float(catalogTime) / float(indexerTime) * 100 -100))
+ precent = int('%.0f' % float(catalogTime / indexerTime * 100 -100))
return '%4d%s' % (precent, '%')
- else:
- # slower
- precent = int('%.0f' % \
- float(float(indexerTime) / float(catalogTime) * 100 -100))
- return '-%3d%s' % (precent, '%')
except ZeroDivisionError:
return ' 0'
-
-
##############################################################################
#
# test components
@@ -155,7 +171,7 @@
# indexer
-class ContentIndexer(MultiAutoIndexer):
+class ContentIndexer(MultiIndexer):
"""Multi indexer for IContent."""
zope.component.adapts(IContent)
@@ -203,21 +219,28 @@
# provide sub location adapter
zope.component.provideAdapter(contained.ContainerSublocations,
- (zope.app.container.interfaces.IReadContainer,),
- zope.location.interfaces.ISublocations)
+ (IReadContainer,), zope.location.interfaces.ISublocations)
-def setUpSubscribers(gsm):
- gsm.registerHandler(indexAdded, (ICatalogIndex, IObjectAddedEvent))
- gsm.registerHandler(indexDocSubscriber, (IIntIdAddedEvent,))
- gsm.registerHandler(reindexDocSubscriber, (ICatalogIndex, IObjectModifiedEvent))
- gsm.registerHandler(unindexDocSubscriber, (IIntIdRemovedEvent,))
+def setUpSharedSubscribers(gsm):
gsm.registerHandler(addIntIdSubscriber)
gsm.registerHandler(removeIntIdSubscriber)
gsm.registerHandler(contained.dispatchToSublocations,
(zope.location.interfaces.ILocation, IObjectMovedEvent))
+def setUpCatalogSubscribers(gsm):
+ gsm.registerHandler(indexAdded, (ICatalogIndex, IObjectAddedEvent))
+ gsm.registerHandler(indexDocSubscriber, (IIntIdAddedEvent,))
+ gsm.registerHandler(reindexDocSubscriber, (IObjectModifiedEvent,))
+ gsm.registerHandler(unindexDocSubscriber, (IIntIdRemovedEvent,))
+def setUpindexerSubscribers(gsm):
+ gsm.registerHandler(subscriber.intIdAddedEventDispatcher)
+ gsm.registerHandler(subscriber.objectModifiedDispatcher)
+ gsm.registerHandler(subscriber.intIdRemovedEventDispatcher)
+
+
+
def setUpCatalog(amountOfIndexes=0):
site = setup.placefulSetUp(True)
setUpAdapter()
@@ -242,7 +265,8 @@
# now we are ready and can setup the event subscribers
gsm = zope.component.getGlobalSiteManager()
- setUpSubscribers(gsm)
+ setUpSharedSubscribers(gsm)
+ setUpCatalogSubscribers(gsm)
# return site
return site
@@ -269,17 +293,16 @@
# add additional not IContent relevant indexes
for i in range(amountOfIndexes):
idxName = 'index-%i' % i
- sm[idxName] = CatalogTextIndex()
+ sm[idxName] = TextIndex()
sm.registerUtility(sm[idxName], interfaces.IIndex, name=idxName)
- # provide IAutoIndexer for IContent
+ # provide IIndexer for IContent
zope.component.provideAdapter(ContentIndexer)
# now we are ready and can setup the event subscribers
gsm = zope.component.getGlobalSiteManager()
- setUpSubscribers(gsm)
- zope.component.provideHandler(subscriber.autoIndexSubscriber)
- zope.component.provideHandler(subscriber.autoUnindexSubscriber)
+ setUpSharedSubscribers(gsm)
+ setUpindexerSubscribers(gsm)
# return site
return site
@@ -313,10 +336,9 @@
def runIndexerUpdate(site):
- for content in site.values():
- interfaces.IAutoIndexer(content).doIndex()
+ [interfaces.IIndexer(content).doIndex() for content in site.values()]
+
-
def runCatalogQuery():
ctlg = zope.component.getUtility(ICatalog)
return ctlg.searchResults(textIndex='Number 42', fieldIndex=(41, 43))
@@ -325,8 +347,7 @@
def runIndexerQuery():
textQuery = TextQuery('textIndex', 'Number 42')
fieldQuery = Eq('fieldIndex', 42)
- searchQuery = SearchQuery(textQuery)
- searchQuery = searchQuery.Or(fieldQuery)
+ searchQuery = SearchQuery(textQuery).Or(fieldQuery)
return searchQuery.apply()
@@ -348,18 +369,17 @@
def runIndexerNotQuery():
textQuery = TextQuery('textIndex', 'Number 42')
fieldQuery = Eq('fieldIndex', 1)
- searchQuery = SearchQuery(textQuery)
- searchQuery = searchQuery.Not(fieldQuery)
+ searchQuery = SearchQuery(textQuery).Not(fieldQuery)
return searchQuery.apply()
def runObjectModifiedEvent(obj):
- obj.title = u'New'
zope.event.notify(zope.lifecycleevent.ObjectModifiedEvent(obj))
-def runParentModifiedEvent(site):
- zope.event.notify(zope.lifecycleevent.ObjectModifiedEvent(site))
+def runObjectMovedEvent(obj):
+ zope.event.notify(contained.ObjectMovedEvent(obj, None, None,
+ obj.__parent__, obj.__name__))
def runObjectRemove(site):
@@ -368,6 +388,7 @@
del site[item.__name__]
return
+
##############################################################################
#
# performance test
@@ -390,7 +411,73 @@
print "- 3 relevant indexes"
print "- %i other indexes" % amountOfIndexes
print "Note, the index update get only processed one time."
+
+ # setup z3c.indexer based performance test
+ indexerSite = setUpIndexer(amountOfIndexes)
+
print ""
+ print "z3c.indexer"
+ # time indexer indexing
+ timeTest(runIndexerIndexing, 1, indexerSite, amountOfObjects)
+ runIndexerIndexingTime = timeResult
+ print "indexer based indexing time: %.2f s" % runIndexerIndexingTime
+
+ # now get the first obejct
+ firstObj = indexerSite['1']
+
+ # time indexer query
+ res = timeTest(runIndexerQuery, repeatTimes)
+ runIndexerQueryTime = timeResult
+ print "indexer based query time: %.2f s" % runIndexerQueryTime
+ if len(res.pop(0)) != 1:
+ print "...bad query result returned"
+
+ # time indexer query
+ res = timeTest(runIndexerNotQuery, repeatTimes)
+ runIndexerNotQueryTime = timeResult
+ print "indexer based not query time: %.2f s" % runIndexerNotQueryTime
+ if len(res.pop(0)) != 1:
+ print "...bad query result returned"
+
+ # time indexer indexes update
+ timeTest(runIndexerUpdate, 1, indexerSite)
+ runIndexerUpdateTime = timeResult
+ print "indexer based update time: %.2f s" % runIndexerUpdateTime
+
+ # time object modified event
+ timeTest(runObjectModifiedEvent, repeatTimes, firstObj)
+ runIndexerModifiedTime = timeResult
+ print "indexer object modified time: %.2f s" % runIndexerModifiedTime
+
+ # time parent modified event
+ timeTest(runObjectModifiedEvent, repeatTimes, indexerSite)
+ runIndexerParentModifiedTime = timeResult
+ print "indexer parent modified time: %.2f s" % runIndexerParentModifiedTime
+
+ # time object moved event
+ timeTest(runObjectMovedEvent, repeatTimes, firstObj)
+ runIndexerObjectMovedTime = timeResult
+ print "indexer object moved time: %.2f s" % runIndexerObjectMovedTime
+
+ # time parent moved event
+ timeTest(runObjectMovedEvent, repeatTimes, indexerSite)
+ runIndexerParentObjectMovedTime = timeResult
+ print "indexer parent moved time: %.2f s" % runIndexerParentObjectMovedTime
+
+ # time object remove (IObjectMovedEvent)
+ timeTest(runObjectRemove, amountOfObjects, indexerSite)
+ runIndexerObjectRemoveTime = timeResult
+ print "indexer object remove time: %.2f s" % runIndexerObjectRemoveTime
+
+ # check if all intids get removed
+ intids = zope.component.getUtility(IIntIds)
+ if len(intids) > 0:
+ print "...Not all objects get removed from IIntIds"
+ # tear down
+ tearDownIndexer()
+
+
+ print ""
print "zope.app.catalog"
# setup zope.app.catalog based performance test
catalogSite = setUpCatalog(amountOfIndexes)
@@ -399,6 +486,9 @@
timeTest(runCatalogIndexing, 1, catalogSite, amountOfObjects)
runCatalogIndexingTime = timeResult
print "catalog based indexing time: %.2f s" % runCatalogIndexingTime
+
+ # now get the first obejct
+ firstObj = catalogSite['1']
# time catalog query
res = timeTest(runCatalogQuery, repeatTimes)
@@ -420,112 +510,77 @@
print "catalog based update time: %.2f s" % runCatalogUpdateTime
# time object modified event
- ctlgObj = catalogSite['1']
- timeTest(runObjectModifiedEvent, repeatTimes, ctlgObj)
+ timeTest(runObjectModifiedEvent, repeatTimes, firstObj)
runCatalogModifiedTime = timeResult
print "catalog object modified time: %.2f s" % runCatalogModifiedTime
# time parent modified event
- timeTest(runParentModifiedEvent, repeatTimes, catalogSite)
+ timeTest(runObjectModifiedEvent, repeatTimes, catalogSite)
runCatalogParentModifiedTime = timeResult
print "catalog parent modified time: %.2f s" % runCatalogParentModifiedTime
+ # time object moved event
+ timeTest(runObjectMovedEvent, repeatTimes, firstObj)
+ runCatalogObjectMovedTime = timeResult
+ print "catalog object moved time: %.2f s" % runCatalogObjectMovedTime
+
+ # time parent moved event
+ timeTest(runObjectMovedEvent, repeatTimes, catalogSite)
+ runCatalogParentObjectMovedTime = timeResult
+ print "catalog parent moved time: %.2f s" % runCatalogParentObjectMovedTime
+
# time object remove (IObjectMovedEvent)
timeTest(runObjectRemove, amountOfObjects, catalogSite)
runCatalogObjectRemoveTime = timeResult
print "catalog object remove time: %.2f s" % runCatalogObjectRemoveTime
+ # check if all intids get removed
+ intids = zope.component.getUtility(IIntIds)
+ if len(intids) > 0:
+ print "...Not all objects get removed from IIntIds"
# tear down
tearDownCatalog()
- # setup z3c.indexer based performance test
- indexerSite = setUpIndexer(amountOfIndexes)
-
print ""
- print "z3c.indexer"
- # time indexer indexing
- timeTest(runIndexerIndexing, 1, indexerSite, amountOfObjects)
- runIndexerIndexingTime = timeResult
- print "indexer based indexing time: %.2f s" % runIndexerIndexingTime
-
- # time indexer query
- res = timeTest(runIndexerQuery, repeatTimes)
- runIndexerQueryTime = timeResult
- print "indexer based query time: %.2f s" % runIndexerQueryTime
- if len(res.pop(0)) != 1:
- print "...bad query result returned"
-
- # time indexer query
- res = timeTest(runIndexerNotQuery, repeatTimes)
- runIndexerNotQueryTime = timeResult
- print "indexer based not query time: %.2f s" % runIndexerNotQueryTime
- if len(res.pop(0)) != 1:
- print "...bad query result returned"
-
- # time indexer indexes update
- timeTest(runIndexerUpdate, 1, indexerSite)
- runIndexerUpdateTime = timeResult
- print "indexer based update time: %.2f s" % runIndexerUpdateTime
-
- # time object modified event
- idObj = indexerSite['1']
- timeTest(runObjectModifiedEvent, repeatTimes, idObj)
- runIndexerModifiedTime = timeResult
- print "indexer object modified time: %.2f s" % runIndexerModifiedTime
-
- # time parent modified event
- timeTest(runParentModifiedEvent, repeatTimes, indexerSite)
- runIndexerParentModifiedTime = timeResult
- print "indexer parent modified time: %.2f s" % runIndexerParentModifiedTime
-
- # time object remove (IObjectMovedEvent)
- timeTest(runObjectRemove, amountOfObjects, indexerSite)
- runIndexerObjectRemoveTime = timeResult
- print "indexer object remove time: %.2f s" % runIndexerObjectRemoveTime
-
- # tear down
- tearDownIndexer()
-
- print ""
print "Result for %i objects with 3 relevant and %i other indexes" % (
amountOfObjects, amountOfIndexes)
- print " ------------------------------------------------------------------------"
- print "| type | indexing | query | not query | update | modify | remove |"
- print " ------------------------------------------------------------------------"
- print "| catalog | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
+ print " ----------------------------------------------------------------------------------"
+ print "| type | indexing | query | not query | update | modify | moved | remove |"
+ print " ----------------------------------------------------------------------------------"
+ print "| catalog | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
runCatalogIndexingTime, runCatalogQueryTime, runCatalogNotQueryTime,
- runCatalogUpdateTime, runCatalogModifiedTime,
+ runCatalogUpdateTime, runCatalogModifiedTime, runCatalogObjectMovedTime,
runCatalogObjectRemoveTime)
- print " ------------------------------------------------------------------------"
- print "| indexer | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
+ print " ----------------------------------------------------------------------------------"
+ print "| indexer | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
runIndexerIndexingTime, runIndexerQueryTime, runIndexerNotQueryTime,
- runIndexerUpdateTime, runIndexerModifiedTime,
+ runIndexerUpdateTime, runIndexerModifiedTime, runIndexerObjectMovedTime,
runIndexerObjectRemoveTime)
- print " ------------------------------------------------------------------------"
- print "| speedup | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
+ print " ----------------------------------------------------------------------------------"
+ print "| speedup | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
(runCatalogIndexingTime - runIndexerIndexingTime),
(runCatalogQueryTime - runIndexerQueryTime),
(runCatalogNotQueryTime - runIndexerNotQueryTime),
(runCatalogUpdateTime - runIndexerUpdateTime),
(runCatalogModifiedTime - runIndexerModifiedTime),
+ (runCatalogObjectMovedTime - runIndexerObjectMovedTime),
(runCatalogObjectRemoveTime - runIndexerObjectRemoveTime))
- print " ------------------------------------------------------------------------"
- print "| speedup | %s | %s | %s | %s | %s | %s |" %(
+ print " ----------------------------------------------------------------------------------"
+ print "| speedup | %s | %s | %s | %s | %s | %s | %s |" %(
calcSpeedUp(runCatalogIndexingTime, runIndexerIndexingTime),
calcSpeedUp(runCatalogQueryTime, runIndexerQueryTime),
calcSpeedUp(runCatalogNotQueryTime, runIndexerNotQueryTime),
calcSpeedUp(runCatalogUpdateTime, runIndexerUpdateTime),
calcSpeedUp(runCatalogModifiedTime, runIndexerModifiedTime),
+ calcSpeedUp(runCatalogObjectMovedTime, runIndexerObjectMovedTime),
calcSpeedUp(runCatalogObjectRemoveTime, runIndexerObjectRemoveTime))
- print " ------------------------------------------------------------------------"
+ print " ----------------------------------------------------------------------------------"
print ""
def main():
- runTest(1, 100, 10)
- runTest(1, 100, 50)
- runTest(1000, 100, 10)
- runTest(1000, 100, 50)
- runTest(1000, 10000, 10)
- runTest(1000, 10000, 50)
+ runTest(1000, 1000, 25)
+ runTest(1000, 1000, 75)
+ runTest(100, 10000, 25)
+ runTest(100, 10000, 75)
Added: z3c.indexer/trunk/src/z3c/indexer/sample.txt
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/sample.txt (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/sample.txt 2008-12-02 00:08:26 UTC (rev 93523)
@@ -0,0 +1,271 @@
+======
+Sample
+======
+
+Sample for an index and unindex story dring the mypypi package development. As
+you can see below, many objects (containers) get index more then one time.
+E.g. the z3c.formdemo MirrorPackage get indexed 15 times. That's very bad and
+just overhead.
+
+The fazit durung this development was: We really need to find a way to index
+an object just one time. We also need to make sure that we unindex if indexed
+or index an object if not indexed yet.
+
+I think an index queue which is controlled by a transaction is the right concept
+for indexing. But anyway this package can be used without that indexing concept.
+
+Initial sample
+--------------
+
+Our indexing story produces the following output before we implemented the
+transaction supported index collector:
+
+index story
+
+intIdAddedEventDispatcher: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <MirrorLogger u'7'> 7
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x035E1F70> 1
+intIdAddedEventDispatcher: <MirrorRelease u'1.5.3'> 1.5.3
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x036442F0> 2
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.5.3.tar.gz'> z3c.formdemo-1.5.3.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.5.3'> 1.5.3
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x036444F0> 3
+objectModifiedHandler: <MirrorRelease u'1.5.3'> 1.5.3
+intIdAddedEventDispatcher: <MirrorRelease u'1.5.2'> 1.5.2
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644670> 4
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.5.2.tar.gz'> z3c.formdemo-1.5.2.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.5.2'> 1.5.2
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644830> 5
+objectModifiedHandler: <MirrorRelease u'1.5.2'> 1.5.2
+intIdAddedEventDispatcher: <MirrorRelease u'1.5.1'> 1.5.1
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x036449F0> 6
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.5.1.tar.gz'> z3c.formdemo-1.5.1.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.5.1'> 1.5.1
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644BB0> 7
+objectModifiedHandler: <MirrorRelease u'1.5.1'> 1.5.1
+intIdAddedEventDispatcher: <MirrorRelease u'1.5.0'> 1.5.0
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644D70> 8
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.5.0.tar.gz'> z3c.formdemo-1.5.0.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.5.0'> 1.5.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644F30> 9
+objectModifiedHandler: <MirrorRelease u'1.5.0'> 1.5.0
+intIdAddedEventDispatcher: <MirrorRelease u'1.4.0'> 1.4.0
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364B130> 10
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.4.0-py2.4.egg'> z3c.formdemo-1.4.0-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.4.0'> 1.4.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364B2F0> 11
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.4.0.tar.gz'> z3c.formdemo-1.4.0.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.4.0'> 1.4.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364B4F0> 12
+objectModifiedHandler: <MirrorRelease u'1.4.0'> 1.4.0
+intIdAddedEventDispatcher: <MirrorRelease u'1.3.0b1'> 1.3.0b1
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644DB0> 13
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.3.0b1.tar.gz'> z3c.formdemo-1.3.0b1.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.3.0b1'> 1.3.0b1
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644AF0> 14
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.3.0b1-py2.4.egg'> z3c.formdemo-1.3.0b1-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.3.0b1'> 1.3.0b1
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644330> 15
+objectModifiedHandler: <MirrorRelease u'1.3.0b1'> 1.3.0b1
+intIdAddedEventDispatcher: <MirrorRelease u'1.3.0'> 1.3.0
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x035E1DB0> 16
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.3.0.tar.gz'> z3c.formdemo-1.3.0.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.3.0'> 1.3.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364B370> 17
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.3.0-py2.4.egg'> z3c.formdemo-1.3.0-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.3.0'> 1.3.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364B730> 18
+objectModifiedHandler: <MirrorRelease u'1.3.0'> 1.3.0
+intIdAddedEventDispatcher: <MirrorRelease u'1.2.0'> 1.2.0
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364B8F0> 19
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.2.0-py2.4.egg'> z3c.formdemo-1.2.0-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.2.0'> 1.2.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364BAB0> 20
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.2.0.tar.gz'> z3c.formdemo-1.2.0.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.2.0'> 1.2.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364BCB0> 21
+objectModifiedHandler: <MirrorRelease u'1.2.0'> 1.2.0
+intIdAddedEventDispatcher: <MirrorRelease u'1.1.2'> 1.1.2
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364BE70> 22
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.2.tar.gz'> z3c.formdemo-1.1.2.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.1.2'> 1.1.2
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03654070> 23
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.2-py2.4.egg'> z3c.formdemo-1.1.2-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.1.2'> 1.1.2
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03654270> 24
+objectModifiedHandler: <MirrorRelease u'1.1.2'> 1.1.2
+intIdAddedEventDispatcher: <MirrorRelease u'1.1.1'> 1.1.1
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03654430> 25
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.1-py2.4.egg'> z3c.formdemo-1.1.1-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.1.1'> 1.1.1
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x036545F0> 26
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.1.tar.gz'> z3c.formdemo-1.1.1.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.1.1'> 1.1.1
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x036547F0> 27
+objectModifiedHandler: <MirrorRelease u'1.1.1'> 1.1.1
+intIdAddedEventDispatcher: <MirrorRelease u'1.1.0'> 1.1.0
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x036549B0> 28
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.0.tar.gz'> z3c.formdemo-1.1.0.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.1.0'> 1.1.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03654B70> 29
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.0-py2.4.egg'> z3c.formdemo-1.1.0-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.1.0'> 1.1.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03654D70> 30
+objectModifiedHandler: <MirrorRelease u'1.1.0'> 1.1.0
+intIdAddedEventDispatcher: <MirrorRelease u'1.0.0c2'> 1.0.0c2
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364BF30> 31
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0c2-py2.4.egg'> z3c.formdemo-1.0.0c2-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.0.0c2'> 1.0.0c2
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364BB30> 32
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0c2.tar.gz'> z3c.formdemo-1.0.0c2.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.0.0c2'> 1.0.0c2
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x0364B670> 33
+objectModifiedHandler: <MirrorRelease u'1.0.0c2'> 1.0.0c2
+intIdAddedEventDispatcher: <MirrorRelease u'1.0.0c1'> 1.0.0c1
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03644730> 34
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0c1-py2.4.egg'> z3c.formdemo-1.0.0c1-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.0.0c1'> 1.0.0c1
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x02A383F0> 35
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0c1.tar.gz'> z3c.formdemo-1.0.0c1.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.0.0c1'> 1.0.0c1
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03654A70> 36
+objectModifiedHandler: <MirrorRelease u'1.0.0c1'> 1.0.0c1
+intIdAddedEventDispatcher: <MirrorRelease u'1.0.0'> 1.0.0
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03654670> 37
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0-py2.4.egg'> z3c.formdemo-1.0.0-py2.4.egg
+objectModifiedHandler: <MirrorRelease u'1.0.0'> 1.0.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03654130> 38
+intIdAddedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0.tar.gz'> z3c.formdemo-1.0.0.tar.gz
+objectModifiedHandler: <MirrorRelease u'1.0.0'> 1.0.0
+intIdAddedEventDispatcher: <mypypi.logger.HistoryEntry object at 0x03661070> 39
+objectModifiedHandler: <MirrorRelease u'1.0.0'> 1.0.0
+objectModifiedHandler: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+
+unindex story:
+
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0-py2.4.egg'> z3c.formdemo-1.0.0-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0.tar.gz'> z3c.formdemo-1.0.0.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.0.0'> 1.0.0
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0c1-py2.4.egg'> z3c.formdemo-1.0.0c1-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0c1.tar.gz'> z3c.formdemo-1.0.0c1.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.0.0c1'> 1.0.0c1
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0c2-py2.4.egg'> z3c.formdemo-1.0.0c2-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.0.0c2.tar.gz'> z3c.formdemo-1.0.0c2.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.0.0c2'> 1.0.0c2
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.0-py2.4.egg'> z3c.formdemo-1.1.0-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.0.tar.gz'> z3c.formdemo-1.1.0.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.1.0'> 1.1.0
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.1-py2.4.egg'> z3c.formdemo-1.1.1-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.1.tar.gz'> z3c.formdemo-1.1.1.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.1.1'> 1.1.1
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.2-py2.4.egg'> z3c.formdemo-1.1.2-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.1.2.tar.gz'> z3c.formdemo-1.1.2.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.1.2'> 1.1.2
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.2.0-py2.4.egg'> z3c.formdemo-1.2.0-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.2.0.tar.gz'> z3c.formdemo-1.2.0.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.2.0'> 1.2.0
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.3.0-py2.4.egg'> z3c.formdemo-1.3.0-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.3.0.tar.gz'> z3c.formdemo-1.3.0.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.3.0'> 1.3.0
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.3.0b1-py2.4.egg'> z3c.formdemo-1.3.0b1-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.3.0b1.tar.gz'> z3c.formdemo-1.3.0b1.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.3.0b1'> 1.3.0b1
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.4.0-py2.4.egg'> z3c.formdemo-1.4.0-py2.4.egg
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.4.0.tar.gz'> z3c.formdemo-1.4.0.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.4.0'> 1.4.0
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.5.0.tar.gz'> z3c.formdemo-1.5.0.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.5.0'> 1.5.0
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.5.1.tar.gz'> z3c.formdemo-1.5.1.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.5.1'> 1.5.1
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.5.2.tar.gz'> z3c.formdemo-1.5.2.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.5.2'> 1.5.2
+intIdRemovedEventDispatcher: <ReleaseFile u'z3c.formdemo-1.5.3.tar.gz'> z3c.formdemo-1.5.3.tar.gz
+intIdRemovedEventDispatcher: <MirrorRelease u'1.5.3'> 1.5.3
+intIdRemovedEventDispatcher: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+
+New concept
+-----------
+
+after implement a transaction based indexer queue we have to following indexing
+story:
+
+
+start transaction
+start index collector
+collect objects which should get indexed
+collect objects which should get un-indexed
+before end transaction
+process index collector
+ - find out which object whould get indexed or un-indexed or skipped
+end transaction
+
+This produces the following output:
+
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4B830> 39
+doIndex: <MirrorRelease u'1.2.0'> 1.2.0
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F4870> 4
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A452F0> 12
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45DF0> 18
+doIndex: <MirrorRelease u'1.3.0b1'> 1.3.0b1
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F4A30> 5
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A454B0> 13
+doIndex: <MirrorRelease u'1.5.3'> 1.5.3
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A64130> 34
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4BBB0> 38
+doIndex: <MirrorRelease u'1.1.1'> 1.1.1
+doIndex: <MirrorRelease u'1.5.0'> 1.5.0
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F46F0> 3
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4B5B0> 28
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45BF0> 17
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F2DF0> 1
+doIndex: <MirrorRelease u'1.1.0'> 1.1.0
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45870> 15
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A644F0> 36
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45670> 14
+doIndex: <MirrorRelease u'1.3.0'> 1.3.0
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4B1B0> 20
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A450F0> 11
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45FB0> 19
+doIndex: <MirrorRelease u'1.0.0c2'> 1.0.0c2
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F47F0> 9
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4B2B0> 27
+doIndex: <MirrorRelease u'1.1.2'> 1.1.2
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45F30> 22
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45AF0> 23
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4B9B0> 30
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4BB70> 31
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F4CB0> 8
+doIndex: <MirrorRelease u'1.0.0'> 1.0.0
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A642F0> 35
+doIndex: <MirrorRelease u'1.0.0c1'> 1.0.0c1
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A454F0> 24
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4B3B0> 21
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45170> 25
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F44B0> 2
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F4BF0> 6
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A646B0> 37
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F4EF0> 10
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F4DB0> 7
+doIndex: <MirrorRelease u'1.4.0'> 1.4.0
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A45A30> 16
+doIndex: <MirrorPackage u'z3c.formdemo'> z3c.formdemo
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4BF30> 33
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4BD30> 32
+doIndex: <MirrorRelease u'1.5.1'> 1.5.1
+doIndex: <mypypi.logger.HistoryEntry object at 0x02A4B7B0> 29
+doIndex: <mypypi.logger.HistoryEntry object at 0x029F4AB0> 26
+doIndex: <MirrorRelease u'1.5.2'> 1.5.2
Modified: z3c.indexer/trunk/src/z3c/indexer/search.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/search.py 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/src/z3c/indexer/search.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -16,7 +16,6 @@
"""
__docformat__ = "reStructuredText"
-import BTrees
import zope.interface
import zope.component
from zope.app.intid.interfaces import IIntIds
@@ -60,9 +59,6 @@
return '<%s len: %s>' %(self.__class__.__name__, len(self.uids))
-# TODO: allow to initialize with query=None and set the first given query as
-# the initial one. If we don't do that, And & Or are never return
-# something because of the initial empty IF.Set()
class SearchQuery(object):
"""Chainable query processor.
@@ -76,6 +72,7 @@
zope.interface.implements(interfaces.ISearchQuery)
family = FamilyProperty()
+ searchResultFactory = ResultSet
_results = None
def __init__(self, query=None, family=None):
@@ -104,12 +101,16 @@
def apply(self):
return self.results
- def searchResults(self, intids=None):
+ def searchResults(self, intids=None, searchResultFactory=None):
+ if searchResultFactory is None:
+ searchResultFactory = self.searchResultFactory
+ else:
+ searchResultFactory = searchResultFactory
results = []
if len(self.results) > 0:
if intids is None:
intids = zope.component.getUtility(IIntIds)
- results = ResultSet(self.results, intids)
+ results = searchResultFactory(self.results, intids)
return results
def Or(self, query):
Modified: z3c.indexer/trunk/src/z3c/indexer/subscriber.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/subscriber.py 2008-12-01 23:57:32 UTC (rev 93522)
+++ z3c.indexer/trunk/src/z3c/indexer/subscriber.py 2008-12-02 00:08:26 UTC (rev 93523)
@@ -11,32 +11,108 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-"""
+"""Indexing pattern
+
+Configure one of this different indexing pattern or use the
+z3c.indexer.indexer.index method explicit if needed.
+
$Id:$
"""
__docformat__ = "reStructuredText"
import zope.component
-from zope.app.intid.interfaces import IIntIdAddedEvent
-from zope.app.intid.interfaces import IIntIdRemovedEvent
+import zope.deferredimport
+import zope.lifecycleevent.interfaces
+import zope.app.intid.interfaces
+from zope.app.intid.interfaces import IIntIds
+
from z3c.indexer import interfaces
+from z3c.indexer import collector
- at zope.component.adapter(IIntIdAddedEvent)
-def autoIndexSubscriber(event):
- """Index all objects which get added to the intids utility."""
- adapters = zope.component.getAdapters((event.object,),
- interfaces.IAutoIndexer)
- for name, adapter in adapters:
- adapter.doIndex()
+###############################################################################
+#
+# Auto indexing strategy
+#
+#
+# The first indexing pattern offers a python method called index which will
+# lookup one or more object IIndexer adapters which will index the object based
+# on this IIndexer adapters. In the initial release we offered subscribers
+# for this indexing methods. This subscribers will get deprecated in the
+# release 1.0.0 because this indexing strategy is to slow because it will
+# index one object more then one time if more then one time if too much events
+# are involved. e.g. adding more then one object.
+#
+# The auto indexer implementation will be removed in 1.0.0 release.
+#
+###############################################################################
+# deprecated auto index subscribers
+zope.deferredimport.deprecated(
+ "IAutoIndexer will go away in 1.0.0 release. Implement IIndexerSubscriber "
+ "and use them within the new intIdAddedEventDispatcher and "
+ "intIdRemovedEventDispatcher subscribers.",
+ autoIndexSubscriber = 'z3c.indexer._bbb:autoIndexSubscriber',
+ autoUnindexSubscriber = 'z3c.indexer._bbb:autoUnindexSubscriber',
+ )
+###############################################################################
+#
+# Unified indexing strategy
+#
+#
+# The second indexing pattern registers indexer with a transaction data manager.
+# This allows us to filter multiple index calls and only use the latest which
+# is very useful because object modified event will force to index an object
+# more then one time during a transaction.
+#
+###############################################################################
+# object added
+ at zope.component.adapter(zope.app.intid.interfaces.IIntIdAddedEvent)
+def intIdAddedEventDispatcher(event):
+ """Event subscriber to dispatch IntIdAddedEvent to IIndexer adapters.
- at zope.component.adapter(IIntIdRemovedEvent)
-def autoUnindexSubscriber(event):
- """Unindex all objects which get added to the intids utility."""
- adapters = zope.component.getAdapters((event.object,),
- interfaces.IAutoIndexer)
- for name, adapter in adapters:
- adapter.doUnIndex()
\ No newline at end of file
+ This event subscriber allows to use IIndexer adapters and collects them
+ in the transaction data manager. At the end of the transaction the different
+ registered adapters get calculated and only relevant indexer get processed.
+
+ This will ensure that we never process an indexer more then one time per
+ transaction.
+ """
+ adapters = zope.component.getAdapters((event.object,), interfaces.IIndexer)
+ if adapters:
+ # collect indexer
+ iCollector = collector.get()
+ for name, indexer in adapters:
+ iCollector.addIndexer(name, indexer)
+
+
+# object modified
+ at zope.component.adapter(zope.lifecycleevent.interfaces.IObjectModifiedEvent)
+def objectModifiedDispatcher(event):
+ """Event subscriber to dispatch IObjectModifiedEvent to interested adapters.
+ """
+ adapters = zope.component.getAdapters((event.object,), interfaces.IIndexer)
+ if adapters:
+ # collect indexer
+ iCollector = collector.get()
+ for name, indexer in adapters:
+ iCollector.addIndexer(name, indexer)
+
+
+# object removed
+ at zope.component.adapter(zope.app.intid.interfaces.IIntIdRemovedEvent)
+def intIdRemovedEventDispatcher(event):
+ """Event subscriber to dispatch IIntIdRemovedEvent to interested adapters.
+
+ This event subscriber allows to use subscription adapters for
+ (object, IIntIdRemovedEvent) which reduces simple (IIntIdRemovedEvent)
+ subscription adapter calls and makes the indexing story more explicit.
+ """
+ adapters = zope.component.getAdapters((event.object,), interfaces.IIndexer)
+ if adapters:
+ # collect indexer
+ iCollector = collector.get()
+ for name, indexer in adapters:
+ iCollector.addUnIndexer(name, indexer)
More information about the Checkins
mailing list