[Checkins] SVN: z3c.indexer/trunk/ Added initial implementation
Roger Ineichen
roger at projekt01.ch
Thu May 1 21:25:19 EDT 2008
Log message for revision 86039:
Added initial implementation
Added buildout folders and files to ignore file list
Changed:
_U z3c.indexer/trunk/
A z3c.indexer/trunk/CHANGES.txt
A z3c.indexer/trunk/LICENSE.txt
A z3c.indexer/trunk/README.txt
A z3c.indexer/trunk/bootstrap.py
A z3c.indexer/trunk/buildout.cfg
A z3c.indexer/trunk/setup.py
_U z3c.indexer/trunk/src/
A z3c.indexer/trunk/src/z3c/
A z3c.indexer/trunk/src/z3c/__init__.py
A z3c.indexer/trunk/src/z3c/indexer/
A z3c.indexer/trunk/src/z3c/indexer/README.txt
A z3c.indexer/trunk/src/z3c/indexer/__init__.py
A z3c.indexer/trunk/src/z3c/indexer/index.py
A z3c.indexer/trunk/src/z3c/indexer/indexer.py
A z3c.indexer/trunk/src/z3c/indexer/interfaces.py
A z3c.indexer/trunk/src/z3c/indexer/performance.py
A z3c.indexer/trunk/src/z3c/indexer/query.py
A z3c.indexer/trunk/src/z3c/indexer/search.py
A z3c.indexer/trunk/src/z3c/indexer/subscriber.py
A z3c.indexer/trunk/src/z3c/indexer/testing.py
A z3c.indexer/trunk/src/z3c/indexer/tests.py
A z3c.indexer/trunk/src/z3c/indexer/value.py
-=-
Property changes on: z3c.indexer/trunk
___________________________________________________________________
Name: svn:ignore
+ bin
develop-eggs
parts
.installed.cfg
Added: z3c.indexer/trunk/CHANGES.txt
===================================================================
--- z3c.indexer/trunk/CHANGES.txt (rev 0)
+++ z3c.indexer/trunk/CHANGES.txt 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,8 @@
+=======
+CHANGES
+=======
+
+Version 0.5.0 (unreleased)
+--------------------------
+
+- Initial release
Property changes on: z3c.indexer/trunk/CHANGES.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/LICENSE.txt
===================================================================
--- z3c.indexer/trunk/LICENSE.txt (rev 0)
+++ z3c.indexer/trunk/LICENSE.txt 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,54 @@
+Zope Public License (ZPL) Version 2.1
+-------------------------------------
+
+A copyright notice accompanies this license document that
+identifies the copyright holders.
+
+This license has been certified as open source. It has also
+been designated as GPL compatible by the Free Software
+Foundation (FSF).
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the
+following conditions are met:
+
+1. Redistributions in source code must retain the
+ accompanying copyright notice, this list of conditions,
+ and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the accompanying
+ copyright notice, this list of conditions, and the
+ following disclaimer in the documentation and/or other
+ materials provided with the distribution.
+
+3. Names of the copyright holders must not be used to
+ endorse or promote products derived from this software
+ without prior written permission from the copyright
+ holders.
+
+4. The right to distribute this software or to use it for
+ any purpose does not give you the right to use
+ Servicemarks (sm) or Trademarks (tm) of the copyright
+ holders. Use of them is covered by separate agreement
+ with the copyright holders.
+
+5. If any files are modified, you must cause the modified
+ files to carry prominent notices stating that you changed
+ the files and the date of any change.
+
+Disclaimer
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS''
+ AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT
+ NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
+ NO EVENT SHALL THE COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ DAMAGE.
Property changes on: z3c.indexer/trunk/LICENSE.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/README.txt
===================================================================
--- z3c.indexer/trunk/README.txt (rev 0)
+++ z3c.indexer/trunk/README.txt 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,2 @@
+This package provides an implementation for indexing objects and query indexes
+for Zope3. This will make zope.app.catalog obsolate.
Property changes on: z3c.indexer/trunk/README.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/bootstrap.py
===================================================================
--- z3c.indexer/trunk/bootstrap.py (rev 0)
+++ z3c.indexer/trunk/bootstrap.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,52 @@
+##############################################################################
+#
+# Copyright (c) 2007 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+
+$Id: bootstrap.py 75940 2007-05-24 14:45:00Z srichter $
+"""
+
+import os, shutil, sys, tempfile, urllib2
+
+tmpeggs = tempfile.mkdtemp()
+
+ez = {}
+exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
+ ).read() in ez
+ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
+
+import pkg_resources
+
+cmd = 'from setuptools.command.easy_install import main; main()'
+if sys.platform == 'win32':
+ cmd = '"%s"' % cmd # work around spawn lamosity on windows
+
+ws = pkg_resources.working_set
+assert os.spawnle(
+ os.P_WAIT, sys.executable, sys.executable,
+ '-c', cmd, '-mqNxd', tmpeggs, 'zc.buildout',
+ dict(os.environ,
+ PYTHONPATH=
+ ws.find(pkg_resources.Requirement.parse('setuptools')).location
+ ),
+ ) == 0
+
+ws.add_entry(tmpeggs)
+ws.require('zc.buildout')
+import zc.buildout.buildout
+zc.buildout.buildout.main(sys.argv[1:] + ['bootstrap'])
+shutil.rmtree(tmpeggs)
Property changes on: z3c.indexer/trunk/bootstrap.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/buildout.cfg
===================================================================
--- z3c.indexer/trunk/buildout.cfg (rev 0)
+++ z3c.indexer/trunk/buildout.cfg 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,33 @@
+[buildout]
+develop = .
+parts = test checker coverage-test coverage-report performance
+
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = z3c.indexer [test]
+
+
+[checker]
+recipe = lovely.recipe:importchecker
+path = src/z3c/indexer
+
+
+[coverage-test]
+recipe = zc.recipe.testrunner
+eggs = z3c.indexer [test]
+defaults = ['--coverage', '../../coverage']
+
+
+[coverage-report]
+recipe = zc.recipe.egg
+eggs = z3c.coverage
+scripts = coverage=coverage-report
+arguments = ('coverage', 'coverage/report')
+
+
+[performance]
+recipe = z3c.recipe.dev:script
+eggs = z3c.indexer [performance]
+method = main
+module = z3c.indexer.performance
Added: z3c.indexer/trunk/setup.py
===================================================================
--- z3c.indexer/trunk/setup.py (rev 0)
+++ z3c.indexer/trunk/setup.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,97 @@
+##############################################################################
+#
+# Copyright (c) 2007 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Setup
+
+$Id:$
+"""
+import os
+from setuptools import setup, find_packages
+
+def read(*rnames):
+ return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+
+setup (
+ name='z3c.indexer',
+ version='0.5.0dev',
+ author = "Roger Ineichen and the Zope Community",
+ author_email = "zope3-dev at zope.org",
+ description = "A new way to index objects for Zope3",
+ long_description=(
+ read('README.txt')
+ + '\n\n' +
+ read('CHANGES.txt')
+ ),
+ license = "ZPL 2.1",
+ keywords = "zope3 z3c catalog indexer index indexing",
+ classifiers = [
+ 'Development Status :: 4 - Beta',
+ 'Environment :: Web Environment',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: Zope Public License',
+ 'Programming Language :: Python',
+ 'Natural Language :: English',
+ 'Operating System :: OS Independent',
+ 'Topic :: Internet :: WWW/HTTP',
+ 'Framework :: Zope3'],
+ url = 'http://cheeseshop.python.org/pypi/z3c.indexer',
+ packages = find_packages('src'),
+ include_package_data = True,
+ package_dir = {'':'src'},
+ namespace_packages = ['z3c',],
+ extras_require = dict(
+ test = [
+ 'z3c.coverage',
+ 'z3c.testing',
+ 'zope.testing',
+ 'zope.app.keyreference',
+ ],
+ performance = [
+ 'zope.app.component',
+ 'zope.app.pagetemplate',
+ 'zope.app.publisher',
+ 'zope.app.publication',
+ 'zope.app.container',
+ 'zope.app.testing',
+ 'zope.app.zapi',
+ 'zope.app.pagetemplate',
+ 'zope.contentprovider',
+ 'zope.i18n',
+ 'zope.i18nmessageid',
+ 'zope.event',
+ 'zope.i18n',
+ 'zope.i18nmessageid',
+ 'zope.lifecycleevent',
+ 'zope.interface',
+ 'zope.schema',
+ 'zope.security',
+ 'zope.testing',
+ 'zope.traversing',
+ 'zope.viewlet',
+ ],
+ ),
+ install_requires = [
+ 'setuptools',
+ 'zc.catalog',
+ 'zope.app.container',
+ 'zope.app.intid',
+ 'zope.app.keyreference',
+ 'zope.component',
+ 'zope.configuration',
+ 'zope.index',
+ 'zope.interface',
+ 'zope.location',
+ 'zope.schema',
+ ],
+ zip_safe = False,
+)
Property changes on: z3c.indexer/trunk/setup.py
___________________________________________________________________
Name: svn:eol-style
+ native
Property changes on: z3c.indexer/trunk/src
___________________________________________________________________
Name: svn:ignore
+ z3c.indexer.egg-info
Added: z3c.indexer/trunk/src/z3c/__init__.py
===================================================================
--- z3c.indexer/trunk/src/z3c/__init__.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/__init__.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,7 @@
+# this is a namespace package
+try:
+ import pkg_resources
+ pkg_resources.declare_namespace(__name__)
+except ImportError:
+ import pkgutil
+ __path__ = pkgutil.extend_path(__path__, __name__)
Property changes on: z3c.indexer/trunk/src/z3c/__init__.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/README.txt
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/README.txt (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/README.txt 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,1145 @@
+======
+README
+======
+
+Intro
+-----
+
+The existing catalog and index implementation get a little out of sync
+because there are at leasst 3 different type of indexes. This are FieldIndex,
+ValueIndex and SetIndex. Each of them uses a different API if it comes to the
+search part. This implementation will make it easier to use all the relevant
+catalog index concepts. It also makes the catalog itself obsolate because
+the search query API interacts directly with indexes.
+
+The explicit usage of this components will also allow you to index object
+if you need them and not because a container likes to do it explicit because
+of some default registered generic catalog index subscribers.
+
+This packages tries to improve the different search query concept as well. This
+is done in two different levels. The And,Or and Not query are implemented as
+a chainable query processor which allows to apply intersections and union etc.
+based on previous results form the query chain. Other query hooks are directly
+implemented at the index level. This allows us to improve the query concept
+directly at the index implementation level for each query index combination.
+
+The implementation of different query types at the index level allows us
+to reuse the search query API which makes it easy to reuse for new indexes.
+This means you don't have to learn a new search query API if somebody provides
+the search query API defined in this package.
+
+
+Note
+----
+
+This package does nothing out of the box, everything is explicit and you have
+to decide what you like to do and how. The package offers you a ready to use
+API and a nice set of adapters and utilities which you can register and use.
+Especialy there is no subscriber which does something for you. You need to
+find your own pattern how and when you need to index your objects. Nothing will
+index or unindex objects by default. This is probably not the right concept
+for every project but it's needed for one of my project which started having
+performance issues during a search. So I started to review and rethink about
+the existing patterns. As far as I can see, there is not much potential for
+improvments in the indexing implementation, but the search and indexing concept
+offered by this package provide a hugh performance win. One of my page in a
+project which uses a complex query for list some items is now 9 times faster
+because of the new chainable search query. Also note, indexing and build
+inteligent search query patterns are not a easy part and this package can
+only help to improve your work if you know what you are doing ;-)
+
+Let me say it again, you can get very quick into trouble if you chain the
+And, Or and Not parts in a bad order. But on the other hand this concept can
+incredible speedup your search query because it compares combined queries
+against each other and not against all available objects.
+
+Performeance
+------------
+
+See also the performance test located in this package. Here is a sample output
+from my 2 GHz Duo Core Laptop:
+
+- 1000 x repeat tests
+
+- 10000 objects
+
+- 3 relevant indexes
+
+- 50 other indexes
+
+Note, the index update get only processed one time.
+
+Result for 10000 objects with 3 relevant and 50 other indexes
+ ------------------------------------------------------------------------
+| type | indexing | query | not query | update | modify | remove |
+ ------------------------------------------------------------------------
+| catalog | 37.39s | 17.83s | 17.83s | 31.14s | 0.05s | 28.53s |
+ ------------------------------------------------------------------------
+| indexer | 4.84s | 10.75s | 10.84s | 1.66s | 0.03s | 3.73s |
+ ------------------------------------------------------------------------
+| speedup | 32.55s | 7.08s | 70.03s | 29.48s | 0.02s | 24.80s |
+ ------------------------------------------------------------------------
+| speedup | 672% | 66% | 646% | 1780% | 47% | 664% |
+ ------------------------------------------------------------------------
+
+This speedup tests shows that the indexing and object remove time get improved
+by more then 6 times. This is a very common usecase in many application. The
+indexer update test is probably not really comparable since we do not update
+all indexes. We only update the relevant objects in the index which makes
+it very fast. But that's the goal of this package, it offers the concepts
+for doing things which can prevent to run into performance problems. Which
+means we can compare the update speedup because the indexes get updated
+with the relevant objects. I've you can agree on this, the speedup for
+index update is more then 17 times ;-)
+
+
+Goals
+-----
+
+The goals of this package are:
+
+indexing
+~~~~~~~~
+
+ - Allow to explicit define of what, where get indexed
+
+ - Reduce index calls. The existing zope.app.catalog implementation forces to
+ index every object which raises a ObjectAddedEvent. This ends in trying to
+ index each object on every existing index contained in a catalog.
+
+ - The existing container add item concept forces to reindex every item
+ in a container which is most the time not needed. This is because the
+ existing event dispatching to it's sublocation.
+
+ - Get rid of persistent catalog and it's index items.
+
+ - Use indexes as utilities
+
+searching
+~~~~~~~~~
+
+ - Offer a optimized query function for build search queries. Implement a
+ chainable search query builder
+
+
+Concepts
+--------
+
+The concepts used in this package are:
+
+- Each index is a utility
+
+- The IIndexer can index objects in one or more index
+
+- The default IIndexer adapter will lookup a IIndexValue multi adapter for each
+ (object, index) tuple and get the right value from this adapter. You can
+ register custom IIndexer adapters for your objects if you like to avoid this
+ additional adapter call.
+
+- Each obj, index pair can have a IIndexValue multi adapter which knows how
+ to get the value which get indexed. Only needed if no IIndexer adapter is
+ available for your custom object.
+
+- Everything is explicit. This means it does not base on IntIdAddedEvent by
+ default. But you can write your own subscriber if you need to use it.
+
+
+This will allow you to
+----------------------
+
+- choose when you index
+
+- choose how you index, e.g.
+
+ - in AddForm
+
+ - call index per object
+
+ - in large data imports
+
+ - call update on index after import all objects
+
+
+And you can make custom speedup improvments like
+
+- index per object or update the index after adding large data sets without
+ indexing on each object added event
+
+- write index with built in value getter
+
+
+Start a simple test setup
+-------------------------
+
+Setup some helpers:
+
+ >>> import zope.component
+ >>> from zope.app import folder
+ >>> from zope.app.component.site import LocalSiteManager
+ >>> from z3c.indexer import interfaces
+ >>> from z3c.indexer import testing
+
+Setup a site
+
+ >>> class SiteStub(folder.Folder):
+ ... """Sample site."""
+ >>> site = SiteStub()
+
+ >>> root['site'] = site
+ >>> sm = LocalSiteManager(site)
+ >>> site.setSiteManager(sm)
+
+And set the site as the current site. This is normaly done by traversing to a
+site:
+
+ >>> from zope.app.component import hooks
+ >>> hooks.setSite(site)
+
+Setup a IIntIds utility:
+
+ >>> from zope.app.intid import IntIds
+ >>> from zope.app.intid.interfaces import IIntIds
+ >>> intids = IntIds()
+ >>> sm['default']['intids'] = intids
+ >>> sm.registerUtility(intids, IIntIds)
+
+
+TextIndex
+---------
+
+Setup a text index:
+
+ >>> from z3c.indexer.index import TextIndex
+ >>> textIndex = TextIndex()
+ >>> sm['default']['textIndex'] = textIndex
+ >>> sm.registerUtility(textIndex, interfaces.IIndex, name='textIndex')
+
+
+FieldIndex
+----------
+
+Setup a field index:
+
+ >>> from z3c.indexer.index import FieldIndex
+ >>> fieldIndex = FieldIndex()
+ >>> sm['default']['fieldIndex'] = fieldIndex
+ >>> sm.registerUtility(fieldIndex, interfaces.IIndex, name='fieldIndex')
+
+
+ValueIndex
+----------
+
+Setup a value index:
+
+ >>> from z3c.indexer.index import ValueIndex
+ >>> valueIndex = ValueIndex()
+ >>> sm['default']['valueIndex'] = valueIndex
+ >>> sm.registerUtility(valueIndex, interfaces.IIndex, name='valueIndex')
+
+
+SetIndex
+--------
+
+Setup a set index:
+
+ >>> from z3c.indexer.index import SetIndex
+ >>> setIndex = SetIndex()
+ >>> sm['default']['setIndex'] = setIndex
+ >>> sm.registerUtility(setIndex, interfaces.IIndex, name='setIndex')
+
+
+DemoContent
+-----------
+
+Now we define a content object:
+
+ >>> import persistent
+ >>> import zope.interface
+ >>> from zope.app.container import contained
+ >>> from zope.schema.fieldproperty import FieldProperty
+
+ >>> class IDemoContent(zope.interface.Interface):
+ ... """Demo content."""
+ ... title = zope.schema.TextLine(
+ ... title=u'Title',
+ ... default=u'')
+ ...
+ ... body = zope.schema.TextLine(
+ ... title=u'Body',
+ ... default=u'')
+ ...
+ ... field = zope.schema.TextLine(
+ ... title=u'a field',
+ ... default=u'')
+ ...
+ ... value = zope.schema.TextLine(
+ ... title=u'A value',
+ ... default=u'')
+ ...
+ ... iterable = zope.schema.Tuple(
+ ... title=u'A sequence of values',
+ ... default=())
+
+ >>> class DemoContent(persistent.Persistent, contained.Contained):
+ ... """Demo content."""
+ ... zope.interface.implements(IDemoContent)
+ ...
+ ... title = FieldProperty(IDemoContent['title'])
+ ... body = FieldProperty(IDemoContent['body'])
+ ... field = FieldProperty(IDemoContent['field'])
+ ... value = FieldProperty(IDemoContent['value'])
+ ... iterable = FieldProperty(IDemoContent['iterable'])
+ ...
+ ... def __init__(self, title=u''):
+ ... self.title = title
+ ...
+ ... def __repr__(self):
+ ... return '<%s %r>' % (self.__class__.__name__, self.title)
+
+And we create and add the content object to the site:
+
+ >>> demo = DemoContent(u'Title')
+ >>> demo.description = u'Description'
+ >>> demo.field = u'Field'
+ >>> demo.value = u'Value'
+ >>> demo.iterable = (1, 2, 'Iterable')
+ >>> site['demo'] = demo
+
+The zope event subscriber for __setitem__ whould call the IIntIds register
+method for our content object. But we didn't setup the relevant subscribers, so
+we do this here:
+
+ >>> uid = intids.register(demo)
+
+
+Indexer
+-------
+
+Setup a indexer adapter for our content object. Let's define a IIndexer class
+which knows how to index text given from body and description attribute:
+
+ >>> from z3c.indexer.indexer import ValueIndexer
+ >>> class DemoValueIndexer(ValueIndexer):
+ ... zope.component.adapts(IDemoContent)
+ ...
+ ... indexName = 'textIndex'
+ ...
+ ... @property
+ ... def value(self):
+ ... """Get the value form context."""
+ ... return '%s %s' % (self.context.title, self.context.body)
+
+Register the adapter as a named adapter:
+
+ >>> zope.component.provideAdapter(DemoValueIndexer, name='textIndex')
+
+We can also use a indexer wich knows how to index the object in different
+indexes.
+
+ >>> from z3c.indexer.indexer import MultiIndexer
+ >>> class DemoMultiIndexer(MultiIndexer):
+ ... zope.component.adapts(IDemoContent)
+ ...
+ ... def doIndex(self):
+ ...
+ ... # index context in fieldIndex
+ ... fieldIndex = self.getIndex('fieldIndex')
+ ... fieldIndex.doIndex(self.oid, self.context.field)
+ ...
+ ... # index context in setIndex
+ ... setIndex = self.getIndex('setIndex')
+ ... setIndex.doIndex(self.oid, self.context.iterable)
+ ...
+ ... # index context in valueIndex
+ ... valueIndex = self.getIndex('valueIndex')
+ ... valueIndex.doIndex(self.oid, self.context.value)
+ ...
+ ... def doUnIndex(self):
+ ...
+ ... # index context in fieldIndex
+ ... fieldIndex = self.getIndex('fieldIndex')
+ ... fieldIndex.doUnIndex(self.oid)
+ ...
+ ... # index context in setIndex
+ ... setIndex = self.getIndex('setIndex')
+ ... setIndex.doUnIndex(self.oid)
+ ...
+ ... # index context in valueIndex
+ ... valueIndex = self.getIndex('valueIndex')
+ ... valueIndex.doUnIndex(self.oid)
+
+Register the adapter as a named adapter:
+
+ >>> zope.component.provideAdapter(DemoMultiIndexer, name='DemoMultiIndexer')
+
+
+Indexing
+--------
+
+Before we start indexing, we check the index:
+
+ >>> textIndex.documentCount()
+ 0
+
+ >>> setIndex.documentCount()
+ 0
+
+ >>> valueIndex.documentCount()
+ 0
+
+Now we can index our demo object:
+
+ >>> from z3c.indexer.indexer import index
+ >>> index(demo)
+
+And check our indexes:
+
+ >>> textIndex.documentCount()
+ 1
+
+ >>> setIndex.documentCount()
+ 1
+
+ >>> valueIndex.documentCount()
+ 1
+
+
+Auto indexing
+-------------
+
+Sometimes, you like to ensure that each object get indexed or updated in the
+index like we us to do in the default zope.app.catalog implementation. This
+means each object get index after adding or updated on object modification.
+We offer a solution for this behavior with the IAutoIndexer adapter call.
+On each object added event or object modified event, a subscriber tries to
+lookup an IAutoIndexer which could index or update the object values in the
+relevant indexes. Since the subscriber calls getAdapters, it's allowed to
+have more then one such indexer adapter.
+
+First register a new index:
+
+ >>> from z3c.indexer.index import TextIndex
+ >>> autoIndex = TextIndex()
+ >>> sm['default']['autoIndex'] = textIndex
+ >>> sm.registerUtility(autoIndex, interfaces.IIndex, name='autoIndex')
+
+The new auto (text) index does not contain any indexed objects right now:
+
+ >>> autoIndex.documentCount()
+ 0
+
+Let's now define a IAutoIndexer adapter:
+
+ >>> from z3c.indexer.indexer import ValueAutoIndexer
+ >>> class MyDemoContentAutoIndexer(ValueAutoIndexer):
+ ... zope.component.adapts(IDemoContent)
+ ...
+ ... indexName = 'autoIndex'
+ ...
+ ... @property
+ ... def value(self):
+ ... """Get the value form context."""
+ ... return 'auto indexed value: %s %s' % (self.context.title,
+ ... self.context.body)
+
+and register them:
+
+ >>> zope.component.provideAdapter(MyDemoContentAutoIndexer, name='Auto')
+
+Now we need to register our subscriber which calls the IAutoIndexer adapters:
+
+ >>> from z3c.indexer import subscriber
+ >>> zope.component.provideHandler(subscriber.autoIndexSubscriber)
+ >>> zope.component.provideHandler(subscriber.autoUnindexSubscriber)
+
+and we also need to register the intid subscribers:
+
+ >>> from zope.app.intid import addIntIdSubscriber
+ >>> from zope.app.intid import removeIntIdSubscriber
+ >>> zope.component.provideHandler(addIntIdSubscriber)
+ >>> zope.component.provideHandler(removeIntIdSubscriber)
+
+If we now add a new object it get auomaticly indexed without to call the
+indexer method explicit:
+
+ >>> autoIndex.documentCount()
+ 0
+
+ >>> textIndex.documentCount()
+ 1
+
+ >>> setIndex.documentCount()
+ 1
+
+ >>> valueIndex.documentCount()
+ 1
+
+Let's now add a new content object:
+
+ >>> autoDemo = DemoContent(u'Auto')
+ >>> autoDemo.description = u'Auto Demo'
+ >>> autoDemo.field = u'Auto field'
+ >>> autoDemo.value = u'auto value'
+ >>> autoDemo.iterable = (1, 2, 'Iterable')
+ >>> site['autoDemo'] = autoDemo
+
+You can see that we've got a key reference for the new object:
+
+ >>> intids.getId(autoDemo) is not None
+ True
+
+You can see that the new object get added without to call the index method
+defined in indexer module:
+
+ >>> autoIndex.documentCount()
+ 1
+
+ >>> textIndex.documentCount()
+ 1
+
+ >>> setIndex.documentCount()
+ 1
+
+ >>> valueIndex.documentCount()
+ 1
+
+As you can see only the ``autoIndex`` get used.
+
+Of corse there is also a ``auto`` unindex implementation which get used if we
+remove the object:
+
+ >>> del site['autoDemo']
+
+As you can see the object got removed from the ``autoIndex``:
+
+ >>> autoIndex.documentCount()
+ 0
+
+ >>> textIndex.documentCount()
+ 1
+
+ >>> setIndex.documentCount()
+ 1
+
+ >>> valueIndex.documentCount()
+ 1
+
+
+SearchQuery
+-----------
+
+Text Index Query
+~~~~~~~~~~~~~~~~
+
+Build a simple text search query:
+
+ >>> from z3c.indexer.search import SearchQuery
+ >>> from z3c.indexer.query import TextQuery
+ >>> textQuery = TextQuery('textIndex', 'Title')
+ >>> query = SearchQuery(textQuery)
+
+Now let's see if we get ``uid`` from the content object:
+
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A none existent value will return a emtpy result:
+
+ >>> textQuery = TextQuery('textIndex', 'bad')
+ >>> query = SearchQuery(textQuery)
+ >>> query.apply()
+ IFSet([])
+
+
+Field Index Query
+~~~~~~~~~~~~~~~~~
+
+Search with a Eq query:
+
+ >>> from z3c.indexer.query import Eq
+ >>> eqQuery = Eq('fieldIndex', 'Field')
+ >>> query = SearchQuery(eqQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A none existent value will return a emtpy result:
+
+ >>> eqQuery = Eq('fieldIndex', 'bad')
+ >>> query = SearchQuery(eqQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a NotEq query:
+
+ >>> from z3c.indexer.query import NotEq
+ >>> notEqQuery = NotEq('fieldIndex', 'bad')
+ >>> query = SearchQuery(notEqQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A existent value will return a emtpy result:
+
+ >>> notEqQuery = NotEq('fieldIndex', 'Field')
+ >>> query = SearchQuery(notEqQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Between query:
+
+ >>> from z3c.indexer.query import Between
+ >>> betweenQuery = Between('fieldIndex', 'Fiel', 'Fielder')
+ >>> query = SearchQuery(betweenQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong min and max value will return a emtpy result:
+
+ >>> betweenQuery = Between('fieldIndex', 'Fielder', 'Fiel')
+ >>> query = SearchQuery(betweenQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Ge query:
+
+ >>> from z3c.indexer.query import Ge
+ >>> geQuery = Ge('fieldIndex', 'Fiel')
+ >>> query = SearchQuery(geQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong max value will return a emtpy result:
+
+ >>> geQuery = Ge('fieldIndex', 'Fielder')
+ >>> query = SearchQuery(geQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Le query:
+
+ >>> from z3c.indexer.query import Le
+ >>> leQuery = Le('fieldIndex', 'Fielder')
+ >>> query = SearchQuery(leQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong min value will return a emtpy result:
+
+ >>> leQuery = Le('fieldIndex', 'Fiel')
+ >>> query = SearchQuery(leQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a In query:
+
+ >>> from z3c.indexer.query import In
+ >>> inQuery = In('fieldIndex', ['Field', 1, 2])
+ >>> query = SearchQuery(inQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A list of none existent values will return a emtpy result:
+
+ >>> inQuery = In('fieldIndex', ['Fielder', 1, 2])
+ >>> query = SearchQuery(inQuery)
+ >>> query.apply()
+ IFSet([])
+
+
+Value Index Query
+~~~~~~~~~~~~~~~~~
+
+Search with a Eq query:
+
+ >>> eqQuery = Eq('valueIndex', 'Value')
+ >>> query = SearchQuery(eqQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A none existent value will return a emtpy result:
+
+ >>> eqQuery = Eq('valueIndex', 'bad')
+ >>> query = SearchQuery(eqQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a NotEq query:
+
+ >>> notEqQuery = NotEq('valueIndex', 'bad')
+ >>> query = SearchQuery(notEqQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A existent value will return a emtpy result:
+
+ >>> notEqQuery = NotEq('valueIndex', 'Value')
+ >>> query = SearchQuery(notEqQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Between query:
+
+ >>> betweenQuery = Between('valueIndex', 'Val', 'Values')
+ >>> query = SearchQuery(betweenQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong min and max value will return a emtpy result:
+
+ >>> betweenQuery = Between('valueIndex', 'Values', 'Val')
+ >>> query = SearchQuery(betweenQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Ge query:
+
+ >>> geQuery = Ge('valueIndex', 'Val')
+ >>> query = SearchQuery(geQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong max value will return a emtpy result:
+
+ >>> geQuery = Ge('valueIndex', 'Values')
+ >>> query = SearchQuery(geQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Le query:
+
+ >>> leQuery = Le('valueIndex', 'Values')
+ >>> query = SearchQuery(leQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong min value will return a emtpy result:
+
+ >>> leQuery = Le('valueIndex', 'Val')
+ >>> query = SearchQuery(leQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a In query:
+
+ >>> inQuery = In('valueIndex', ['Value', 1, 2])
+ >>> query = SearchQuery(inQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A list of none existent values will return a emtpy result:
+
+ >>> inQuery = In('valueIndex', ['Values', 1, 2])
+ >>> query = SearchQuery(inQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a ExtentAny query:
+
+ >>> from zc.catalog.extentcatalog import Extent
+ >>> from z3c.indexer.query import ExtentAny
+ >>> extent = Extent()
+ >>> extent.add(uid, ['Values', 1, 2])
+
+ >>> extentAnyQuery = ExtentAny('valueIndex', extent)
+ >>> query = SearchQuery(extentAnyQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+Search with a ExtentNone query:
+
+ >>> from z3c.indexer.query import ExtentNone
+ >>> extentNoneQuery = ExtentNone('valueIndex', extent)
+ >>> query = SearchQuery(extentNoneQuery)
+ >>> res = query.apply()
+ >>> len(res)
+ 0
+
+
+Set Index Query
+~~~~~~~~~~~~~~~
+
+Search with a AnyOf query:
+
+ >>> from z3c.indexer.query import AnyOf
+ >>> anyOfQuery = AnyOf('setIndex', ['Iterable', 1])
+ >>> query = SearchQuery(anyOfQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A list of none existent values will return a emtpy result:
+
+ >>> anyOfQuery = AnyOf('setIndex', ['Iter', 3])
+ >>> query = SearchQuery(anyOfQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a AllOf query:
+
+ >>> from z3c.indexer.query import AllOf
+ >>> allOfQuery = AllOf('setIndex', ['Iterable', 1, 2])
+ >>> query = SearchQuery(allOfQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A list of to less values will return the same result:
+
+ >>> from z3c.indexer.query import AllOf
+ >>> allOfQuery = AllOf('setIndex', ['Iterable', 1])
+ >>> query = SearchQuery(allOfQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A list of to much values will return a emtpy result:
+
+ >>> allOfQuery = AllOf('setIndex', ['Iterable', 1, 2, 3])
+ >>> query = SearchQuery(allOfQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Between query:
+
+ >>> betweenQuery = Between('setIndex', 'Iter', 'Iterables')
+ >>> query = SearchQuery(betweenQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong min and max value will return a emtpy result:
+
+ >>> betweenQuery = Between('setIndex', 'Iterables', 'Iter')
+ >>> query = SearchQuery(betweenQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Ge query:
+
+ >>> geQuery = Ge('valueIndex', 'Iter')
+ >>> query = SearchQuery(geQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong max value will return a emtpy result:
+
+ >>> geQuery = Ge('setIndex', 'Iterables')
+ >>> query = SearchQuery(geQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a Le query:
+
+ >>> leQuery = Le('setIndex', 'Iterables')
+ >>> query = SearchQuery(leQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+A wrong min value will return a emtpy result:
+
+ >>> leQuery = Le('setIndex', 0)
+ >>> query = SearchQuery(leQuery)
+ >>> query.apply()
+ IFSet([])
+
+Search with a ExtentAny query:
+
+ >>> extentAnyQuery = ExtentAny('setIndex', extent)
+ >>> query = SearchQuery(extentAnyQuery)
+ >>> res = query.apply()
+ >>> res[0] == uid
+ True
+
+Search with a ExtentNone query:
+
+ >>> extent = Extent()
+ >>> extent.add(uid, ['Iterables'])
+ >>> extentNoneQuery = ExtentNone('setIndex', extent)
+ >>> query = SearchQuery(extentNoneQuery)
+ >>> query.apply()
+ IFSet([])
+
+
+Chainable Search Query
+----------------------
+
+A search query is chainable. This means we can append queries to queries
+itself. The result of a previous query will be used for the next query in the
+chain. Note, this pattern can give you hugh speedup but you have to take care
+on what you chain in which order or you will very quickly get a wrong result.
+But the speedup can be hugh, I optimized one of my application with this
+pattern and got a speedup by 900%.
+
+Let's cleanup the text index first:
+
+ >>> textIndex.clear()
+
+And add some more demo object with different values:
+
+ >>> apple = DemoContent(u'Apple')
+ >>> site['apple'] = apple
+ >>> appleId = intids.register(apple)
+
+ >>> house = DemoContent(u'House')
+ >>> site['house'] = house
+ >>> houseId = intids.register(house)
+
+ >>> tower = DemoContent(u'Tower')
+ >>> site['tower'] = tower
+ >>> towerId = intids.register(tower)
+
+ >>> every = DemoContent(u'Apple House Tower')
+ >>> site['every'] = every
+ >>> everyId = intids.register(every)
+
+And register them in the text index:
+
+ >>> index(apple)
+ >>> index(house)
+ >>> index(tower)
+ >>> index(every)
+
+Now we can see that we have 3 items in the text index:
+
+ >>> textIndex.documentCount()
+ 4
+
+Let's buold some query:
+
+ >>> appleQuery = TextQuery('textIndex', 'Apple')
+ >>> houseQuery = TextQuery('textIndex', 'House')
+ >>> towerQuery = TextQuery('textIndex', 'Tower')
+
+
+And SearchQuery
+---------------
+
+Now we can build a search query chain with this queries. The following sample
+will return all items which are returned by the 'Apple' and the 'House' query
+This is only the case for the ``every`` object:
+
+ >>> query = SearchQuery(appleQuery).And(houseQuery)
+ >>> res = query.apply()
+ >>> res[0] == everyId
+ True
+
+ >>> intids.getObject(res[0])
+ <DemoContent u'Apple House Tower'>
+
+
+Or SearchQuery
+--------------
+
+A Or search query will return all object which are contained in each query. The
+search query below will return all 4 objects becaues each of them get found by
+one of the existing queries. And the ``every`` object will only get listed once:
+
+ >>> allQuery = SearchQuery(appleQuery).Or(houseQuery).Or(towerQuery)
+ >>> res = allQuery.apply()
+ >>> len(res)
+ 4
+
+
+Not SearchQuery
+---------------
+
+A Not search query will return all object which are not contained in the given
+query. The search query below will return all objects except the ones which
+contains the word ``Apple`` becaues we exclude them within the appleQuery.
+Another interesting thing is, that we can use the previous query and simple
+add another query to the chain. This is a very interesting pattern for filters.
+
+ >>> query = allQuery.Not(appleQuery)
+ >>> res = query.apply()
+ >>> len(res)
+ 2
+
+ >>> intids.getObject(sorted(res)[0])
+ <DemoContent u'House'>
+
+ >>> intids.getObject(sorted(res)[1])
+ <DemoContent u'Tower'>
+
+
+ResultSet
+---------
+
+The SearchQuery provides also a ResultSet wrapper. We can get an iterable
+ResultSet instance if we call ``searchResults`` from the search query:
+
+ >>> allQuery = SearchQuery(appleQuery).Or(houseQuery).Or(towerQuery)
+ >>> resultSet = allQuery.searchResults()
+ >>> len(resultSet)
+ 4
+
+Or we can get a slice from the ResultSet:
+
+ >>> resultSet[-1:]
+ [<DemoContent u'Apple House Tower'>]
+
+ >>> resultSet[0:]
+ [<DemoContent u'Apple'>, <DemoContent u'House'>, <DemoContent u'Tower'>,
+ <DemoContent u'Apple House Tower'>]
+
+ >>> resultSet[1:]
+ [<DemoContent u'House'>, <DemoContent u'Tower'>,
+ <DemoContent u'Apple House Tower'>]
+
+ >>> resultSet[:-2]
+ [<DemoContent u'Apple'>, <DemoContent u'House'>]
+
+Or we can iterate over the ResultSet:
+
+ >>> list(resultSet)
+ [<DemoContent u'Apple'>, <DemoContent u'House'>, <DemoContent u'Tower'>,
+ <DemoContent u'Apple House Tower'>]
+
+Or check if a item is a part of the result set:
+
+ >>> resultSet.__contains__(object())
+ False
+
+ >>> resultSet.__contains__(apple)
+ True
+
+
+
+Batching
+--------
+
+This ResultSet described above can be used together with the BAtch
+implementation defined in the z3c.batching package.
+
+
+unindex
+-------
+
+Now after the different index and serch tests we are ready to unindex our
+indexed objects. Let's see what we have in the indexes:
+
+ >>> textIndex.documentCount()
+ 4
+
+ >>> setIndex.documentCount()
+ 1
+
+ >>> valueIndex.documentCount()
+ 5
+
+Now let's use our unindex method from the module indexer. This will call our
+Indexer adapter and delegate the unindex call to the doUnIndex method of such
+a IIndexer adapter. Let's unindex our demo object:
+
+ >>> from z3c.indexer.indexer import unindex
+ >>> unindex(demo)
+
+Now you can see that the dome object get reomved:
+
+ >>> textIndex.documentCount()
+ 4
+
+ >>> setIndex.documentCount()
+ 0
+
+ >>> valueIndex.documentCount()
+ 4
+
+
+Coverage
+--------
+
+Let's test some line of code which are not used right now. First test if the
+base class IndexerBase raises a NotImplementedError if we do not implement the
+methods:
+
+ >>> from z3c.indexer.indexer import IndexerBase
+ >>> indexerBase = IndexerBase(object())
+ >>> indexerBase.doIndex()
+ Traceback (most recent call last):
+ ...
+ NotImplementedError: Subclass must implement doIndex method.
+
+ >>> indexerBase.doUnIndex()
+ Traceback (most recent call last):
+ ...
+ NotImplementedError: Subclass must implement doUnIndex method.
+
+The MutliIndexer does also not implement this methods:
+
+ >>> from z3c.indexer.indexer import MultiIndexer
+ >>> multiIndexer = MultiIndexer(object())
+ >>> multiIndexer.doIndex()
+ Traceback (most recent call last):
+ ...
+ NotImplementedError: Subclass must implement doIndex method.
+
+ >>> multiIndexer.doUnIndex()
+ Traceback (most recent call last):
+ ...
+ NotImplementedError: Subclass must implement doUnIndex method.
+
+And the ValueIndexer does not implement the value attribute:
+
+ >>> valueIndexer = ValueIndexer(object())
+ >>> valueIndexer.indexName = 'textIndex'
+ >>> valueIndexer.value
+ Traceback (most recent call last):
+ ...
+ NotImplementedError: Subclass must implement value property.
+
+Another use case which we didn't test is that a applyIn can contain the
+same object twice with different values. Let's test the built in union
+which removes such duplications:
+
+ >>> demo1 = DemoContent(u'Demo 1')
+ >>> demo1.description = u'Description'
+ >>> demo1.field = u'Field'
+ >>> demo1.value = u'Value'
+ >>> demo1.iterable = (1, 2, 'Iterable')
+ >>> site['demo1'] = demo1
+ >>> index(demo1)
+
+ >>> demo2 = DemoContent(u'Demo 2')
+ >>> demo2.description = u'Description'
+ >>> demo2.field = u'Field'
+ >>> demo2.value = u'Value'
+ >>> demo2.iterable = (1, 2, 'Iterable')
+ >>> site['demo2'] = demo2
+ >>> index(demo2)
+
+ >>> inQuery = In('fieldIndex', ['Field', 'Field'])
+ >>> query = SearchQuery(inQuery)
+ >>> resultSet = query.searchResults()
+ >>> resultSet.__contains__(demo1)
+ True
+ >>> resultSet.__contains__(demo2)
+ True
+
+A SearchQuery allows us to set a BTrees family argument:
+
+ >>> import BTrees
+
+ >>> textDemo = DemoContent(u'Text Demo')
+ >>> textDemo.description = u'Description'
+ >>> site['textDemo'] = textDemo
+ >>> index(textDemo)
+
+ >>> textQuery = TextQuery('textIndex', 'Text Demo')
+ >>> query = SearchQuery(textQuery, BTrees.family32)
+ >>> resultSet.__contains__(demo1)
+ True
Property changes on: z3c.indexer/trunk/src/z3c/indexer/README.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/__init__.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/__init__.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/__init__.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1 @@
+# make a package
\ No newline at end of file
Property changes on: z3c.indexer/trunk/src/z3c/indexer/__init__.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/index.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/index.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/index.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,156 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+from BTrees.IFBTree import IFBTree
+from BTrees.IFBTree import union
+from BTrees.IFBTree import difference
+
+import zope.interface
+from zope.index.field import index as fieldindex
+from zope.index.text import textindex
+from zope.app.container import contained
+from zc.catalog import index as zcindex
+from z3c.indexer import interfaces
+
+
+class IndexMixin(object):
+ """Text index based on zope.index.text.textindex.TextIndex"""
+
+ def doIndex(self, oid, value):
+ """Index a value by it's id."""
+ self.index_doc(oid, value)
+
+ def doUnIndex(self, oid):
+ self.unindex_doc(oid)
+
+
+class TextIndex(IndexMixin, textindex.TextIndex,
+ contained.Contained):
+ """Text index based on zope.index.text.textindex.TextIndex"""
+
+ zope.interface.implements(interfaces.ITextIndex)
+
+
+class FieldIndex(IndexMixin, fieldindex.FieldIndex,
+ contained.Contained):
+ """Text index based on zope.index.text.textindex.TextIndex
+
+ Field index will use tuple in it's base apply method.
+ """
+
+ zope.interface.implements(interfaces.IFieldIndex)
+
+ def applyEq(self, value):
+ return self.apply((value, value))
+
+ def applyNotEq(self, not_value):
+ all = self.apply((None, None))
+ r = self.apply((not_value, not_value))
+ return difference(all, r)
+
+ def applyBetween(self, min_value, max_value, exclude_min=False,
+ exclude_max=False):
+ return self.apply((min_value, max_value))
+
+ def applyGe(self, min_value, exclude_min=False):
+ return self.apply((min_value, None))
+
+ def applyLe(self, max_value, exclude_max=False):
+ return self.apply((None, max_value))
+
+ def applyIn(self, values):
+ results = []
+ for value in values:
+ res = self.apply((value, value))
+ # empty results
+ if not res:
+ continue
+ results.append(res)
+
+ if not results:
+ # no applicable terms at all
+ return IFBTree()
+
+ result = results.pop(0)
+ for res in results:
+ result = union(result, res)
+ return result
+
+
+class ValueIndex(IndexMixin, zcindex.ValueIndex, contained.Contained):
+ """Value index based on zc.catalog.index.ValueIndex"""
+
+ zope.interface.implements(interfaces.IValueIndex)
+
+ def applyEq(self, value):
+ return self.apply({'any_of': (value,)})
+
+ def applyNotEq(self, not_value):
+ values = list(self.values())
+ if not_value in values:
+ values.remove(not_value)
+ return self.apply({'any_of': values})
+
+ def applyBetween(self, min_value, max_value, exclude_min, exclude_max):
+ return self.apply(
+ {'between': (min_value, max_value, exclude_min, exclude_max)})
+
+ def applyGe(self, min_value, exclude_min=False):
+ return self.apply({'between': (min_value, None, exclude_min, False)})
+
+ def applyLe(self, max_value, exclude_max=False):
+ return self.apply({'between': (None, max_value, False, exclude_max)})
+
+ def applyIn(self, values):
+ return self.apply({'any_of': values})
+
+ def applyExtentAny(self, extent):
+ return self.apply({'any': extent})
+
+ def applyExtentNone(self, extent):
+ return self.apply({'none': extent})
+
+
+class SetIndex(IndexMixin, zcindex.SetIndex, contained.Contained):
+ """Set index based on zc.catalog.index.SetIndex"""
+
+ zope.interface.implements(interfaces.ISetIndex)
+
+ def applyAnyOf(self, values):
+ return self.apply({'any_of': values})
+
+ def applyAllOf(self, values):
+ return self.apply({'all_of': values})
+
+ def applyBetween(self, min_value, max_value, exclude_min, exclude_max):
+ self.tuple = (min_value, max_value, exclude_min, exclude_max)
+ return self.apply({'between': self.tuple})
+
+ def applyGe(self, min_value, exclude_min=False):
+ self.tuple = (min_value, None, exclude_min, False)
+ return self.apply({'between': self.tuple})
+
+ def applyLe(self, max_value, exclude_max=False):
+ self.tuple = (None, max_value, False, exclude_max)
+ return self.apply({'between': self.tuple})
+
+ def applyExtentAny(self, extent):
+ return self.apply({'any': extent})
+
+ def applyExtentNone(self, extent):
+ return self.apply({'none': extent})
\ No newline at end of file
Property changes on: z3c.indexer/trunk/src/z3c/indexer/index.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/indexer.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/indexer.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/indexer.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,149 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import zope.interface
+import zope.component
+from zope.cachedescriptors.property import Lazy
+from zope.app.intid.interfaces import IIntIds
+
+from z3c.indexer import interfaces
+
+
+def index(context):
+ """Index object.
+
+ This will only call indexes which object have IIndexer adapter for and will
+ avoid calling all indexes for each object.
+
+ If you use the old indexing pattern, implement a own subscriber which uses
+ this pattern for z3c.indexer based indexes.
+ """
+ adapters = zope.component.getAdapters((context,), interfaces.IIndexer)
+ for name, adapter in adapters:
+ adapter.doIndex()
+
+
+def unindex(context):
+ """Unindex object.
+
+ This will only call indexes which object have IIndexer adapter for and will
+ avoid calling all indexes for each object.
+
+ If you use the old unindexing pattern, implement a own subscriber which
+ uses this pattern for z3c.indexer based indexes.
+ """
+ adapters = zope.component.getAdapters((context,), interfaces.IIndexer)
+ for name, adapter in adapters:
+ adapter.doUnIndex()
+
+
+class IndexerBase(object):
+ """Indexer base class.
+
+ A IIndexer is registered as a named adapter. This makes it possible to
+ provide more then one adapter per object. You can implement one single
+ IIndexer adapter for your object or more then one. It depends on how you
+ like to call them. The global helper method called ``index`` and
+ ``unindex`` above can be used for call all IIndexer in a single call.
+ """
+
+ def __init__(self, context):
+ """Registered as named index adapter"""
+ self.context = context
+
+ @Lazy
+ def oid(self):
+ """Get IntId for the adapted context."""
+ intids = zope.component.getUtility(IIntIds, context=self.context)
+ return intids.getId(self.context)
+
+ def doIndex(self):
+ """Implement your own indexing pattern."""
+ raise NotImplementedError("Subclass must implement doIndex method.")
+
+ def doUnIndex(self):
+ """Implement your own un-indexing pattern."""
+ raise NotImplementedError("Subclass must implement doUnIndex method.")
+
+
+# value indexer
+class ValueIndexerBase(IndexerBase):
+ """Value indexer implementation."""
+
+ indexName = ''
+
+ @Lazy
+ def index(self):
+ return zope.component.getUtility(interfaces.IIndex, self.indexName,
+ context=self.context)
+
+ @property
+ def value(self):
+ """Get the index value for the adapted context and relevant index."""
+ raise NotImplementedError("Subclass must implement value property.")
+
+ def doIndex(self):
+ self.index.doIndex(self.oid, self.value)
+
+ def doUnIndex(self):
+ self.index.doUnIndex(self.oid)
+
+
+class ValueIndexer(ValueIndexerBase):
+ """Value indexer implementation."""
+
+ zope.interface.implements(interfaces.IValueIndexer)
+
+
+class ValueAutoIndexer(ValueIndexerBase):
+ """Value (auto) indexer implementation."""
+
+ zope.interface.implements(interfaces.IValueAutoIndexer)
+
+
+# multi indexer
+class MultiIndexerBase(IndexerBase):
+ """Can be used a s base for index a object in more then one index."""
+
+ def __init__(self, context):
+ """Registered as named index adapter"""
+ self.context = context
+
+ def getIndex(self, indexName):
+ return zope.component.getUtility(interfaces.IIndex, indexName,
+ context=self.context)
+
+ def doIndex(self):
+ """Implement your own indexing pattern."""
+ raise NotImplementedError("Subclass must implement doIndex method.")
+
+ def doUnIndex(self):
+ """Implement your own un-indexing pattern."""
+ raise NotImplementedError("Subclass must implement doUnIndex method.")
+
+
+class MultiIndexer(MultiIndexerBase):
+ """Can be used a s base for index a object in more then one index."""
+
+ zope.interface.implements(interfaces.IMultiIndexer)
+
+
+class MultiAutoIndexer(MultiIndexerBase):
+ """Can be used a s base for index a object in more then one index."""
+
+ zope.interface.implements(interfaces.IMultiAutoIndexer)
Property changes on: z3c.indexer/trunk/src/z3c/indexer/indexer.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/interfaces.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/interfaces.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/interfaces.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,315 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+
+import zope.interface
+
+from zope.index.interfaces import IInjection
+from zope.index.interfaces import IIndexSearch
+from zope.index.interfaces import IStatistics
+from zope.app.container.interfaces import IContained
+
+NOVALUE = object()
+
+
+class ISearchQuery(zope.interface.Interface):
+ """Chainable search query."""
+
+ def __init__(query=None, family=None):
+ """Initialize with none or existing query."""
+
+ def apply():
+ """Return iterable search result wrapper."""
+
+ def searchResults():
+ """Retruns iterable search result objects."""
+
+ def Or(query):
+ """Enhance search results. (union)
+
+ The result will contain intids which exist in the existing result
+ and/or in the result from te given query.
+ """
+
+ def And(query):
+ """Restrict search results. (intersection)
+
+ The result will only contain intids which exist in the existing
+ result and in the result from te given query. (union)
+ """
+
+ def Not(query):
+ """Exclude search results. (difference)
+
+ The result will only contain intids which exist in the existing
+ result but do not exist in the result from te given query.
+
+ This is faster if the existing result is small. But note, it get
+ processed in a chain, results added after this query get added again.
+ So probably you need to call this at the end of the chain.
+ """
+
+
+class IIndexer(zope.interface.Interface):
+ """An Indexer knows how to index objects."""
+
+ oid = zope.interface.Attribute("""IIntId of context.""")
+
+ def doIndex():
+ """Index the context in the relevant index."""
+
+ def doUnIndex():
+ """Unindex the context in the relevant index."""
+
+
+class IValueIndexer(IIndexer):
+ """An Indexer which knows the value used for indexing."""
+
+ index = zope.interface.Attribute("""The named index utility.""")
+
+ value = zope.interface.Attribute("""Value for context/index combination.""")
+
+
+class IMultiIndexer(IIndexer):
+ """Can be used a s base for index a object in more then one index."""
+
+ def getIndex(indexName):
+ """The named index utility getter method."""
+
+
+class IAutoIndexer(zope.interface.Interface):
+ """Auto indexer adapter get called by the auto indexer subscriber."""
+
+ oid = zope.interface.Attribute("""IIntId of context.""")
+
+ def doIndex():
+ """Index the context in the relevant index."""
+
+ def doUnIndex():
+ """Unindex the context in the relevant index."""
+
+
+class IValueAutoIndexer(IAutoIndexer):
+ """Value (auto) indexer."""
+
+ index = zope.interface.Attribute("""The named index utility.""")
+
+ value = zope.interface.Attribute("""Value for context/index combination.""")
+
+
+class IMultiAutoIndexer(IAutoIndexer):
+ """Multi (auto) indexer"""
+
+ def getIndex(indexName):
+ """The named index utility getter method."""
+
+
+# index base interface
+class IIndex(zope.interface.Interface, IContained):
+ """An Index marker interface."""
+
+ def doIndex(obj, value):
+ """Index a object."""
+
+ def doUnIndex(obj):
+ """Unindex a object."""
+
+ def apply(query):
+ """Simple apply method with single argument."""
+
+
+# indexes
+class ITextIndex(IIndex, IInjection, IIndexSearch, IStatistics):
+ """Text index."""
+
+ def apply(value):
+ """Apply text query."""
+
+
+class IFieldIndex(IIndex):
+ """Value index."""
+
+ def applyEq(value):
+ """Apply equals query."""
+
+ def applyNotEq(not_value):
+ """Apply not equals query."""
+
+ def applyBetween(min_value, max_value, exclude_min=False,
+ exclude_max=False):
+ """Apply between query."""
+
+ def applyGe(min_value, exclude_min=False):
+ """Apply greater query."""
+
+ def applyLe(max_value, exclude_max=False):
+ """Apply less query."""
+
+ def applyIn(values):
+ """Apply in query."""
+
+
+class IValueIndex(IIndex):
+ """Value index."""
+
+ def applyEq(value):
+ """Apply equals query."""
+
+ def applyNotEq(not_value):
+ """Apply not equals query."""
+
+ def applyBetween(min_value, max_value, exclude_min, exclude_max):
+ """Apply between query."""
+
+ def applyGe(min_value, exclude_min=False):
+ """Apply greater query."""
+
+ def applyLe(max_value, exclude_max=False):
+ """Apply less query."""
+
+ def applyIn(values):
+ """Apply in query."""
+
+ def applyExtentAny(extent):
+ """Apply extent any query."""
+
+ def applyExtentNone(extent):
+ """Apply extent query."""
+
+
+class ISetIndex(IIndex):
+ """Value index."""
+
+ def applyAnyOf(values):
+ """Apply any of query."""
+
+ def applyAllOf(values):
+ """Apply all of query."""
+
+ def applyBetween(min_value, max_value, exclude_min, exclude_max):
+ """Apply between query."""
+
+ def applyGe(min_value, exclude_min=False):
+ """Apply greater query."""
+
+ def applyLe(max_value, exclude_max=False):
+ """Apply less query."""
+
+ def applyExtentAny(extent):
+ """Apply extent any query."""
+
+ def applyExtentNone(extent):
+ """Apply extent None query."""
+
+
+
+# queries
+class IQuery(zope.interface.Interface):
+ """Search query."""
+
+ def apply():
+ """Apply query with predefined query value."""
+
+
+class ITextQuery(IQuery):
+ """Text index query."""
+
+ def __init__(indexOrName, value):
+ """Query signature."""
+
+
+class IEqQuery(IQuery):
+ """Equal query."""
+
+ def __init__(indexOrName, value):
+ """Query signature."""
+
+
+class INotEqQuery(IQuery):
+ """Not equal query."""
+
+ def __init__(indexOrName, not_value):
+ """Query signature."""
+
+
+class IBetweenQuery(IQuery):
+ """Between query."""
+
+ def __init__(indexOrName, min_value, max_value):
+ """Query signature."""
+
+
+class IGeQuery(IQuery):
+ """Greater query."""
+
+ def __init__(indexOrName, min_value):
+ """Query signature."""
+
+
+class ILeQuery(IQuery):
+ """Less query."""
+
+ def __init__(indexOrName, max_value):
+ """Query signature."""
+
+
+class IInQuery(IQuery):
+ """In query."""
+
+ def __init__(indexOrName, values):
+ """Query signature."""
+
+
+class IAnyOfQuery(IQuery):
+ """AnyOf query.
+
+ The result will be the docids whose values contain any of the given values.
+ """
+
+ def __init__(indexOrName, values):
+ """Query signature."""
+
+
+class IAllOfQuery(IQuery):
+ """AllOf query.
+
+ The result will be the docids whose values contain all of the given values.
+ """
+
+ def __init__(indexOrName, values):
+ """Query signature."""
+
+
+class IExtentAnyQuery(IQuery):
+ """ExtentAny query."""
+
+ def __init__(indexOrName, values):
+ """Query signature."""
+
+
+class IExtentNoneQuery(IQuery):
+ """ExtentNone query."""
+
+ def __init__(indexOrName, values):
+ """Query signature."""
+
+
+class IIndexValue(zope.interface.Interface):
+ """Knows how to lookup index values."""
+
+ value = zope.interface.Attribute("""Index value of context.""")
Property changes on: z3c.indexer/trunk/src/z3c/indexer/interfaces.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/performance.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/performance.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/performance.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,531 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import time
+import persistent
+import zope.interface
+import zope.component
+import zope.schema
+import zope.event
+import zope.lifecycleevent
+import zope.location.interfaces
+from ZODB.interfaces import IConnection
+from BTrees.IFBTree import difference, IFBTree
+from persistent.interfaces import IPersistent
+from zope.index.text.textindex import TextIndex as ZTextIndex
+
+from zope.app.keyreference.interfaces import IKeyReference
+from zope.app.keyreference.testing import SimpleKeyReference
+
+from zope.app.component import hooks
+from zope.app.catalog.interfaces import ICatalog
+from zope.app.catalog.text import ITextIndex
+from zope.app.catalog.field import FieldIndex as ZFieldIndex
+from zope.app.catalog.interfaces import ICatalogIndex
+from zope.app.catalog.catalog import Catalog
+from zope.app.catalog.catalog import indexAdded
+from zope.app.catalog.catalog import indexDocSubscriber
+from zope.app.catalog.catalog import reindexDocSubscriber
+from zope.app.catalog.catalog import unindexDocSubscriber
+from zope.app.container.interfaces import IReadContainer
+from zope.app.container.interfaces import IObjectAddedEvent
+from zope.app.container.interfaces import IObjectModifiedEvent
+from zope.app.container.interfaces import IObjectMovedEvent
+from zope.app.container import contained
+from zope.app.intid import IntIds
+from zope.app.intid import addIntIdSubscriber
+from zope.app.intid import removeIntIdSubscriber
+from zope.app.intid.interfaces import IIntIds
+from zope.app.intid.interfaces import IIntIdAddedEvent
+from zope.app.intid.interfaces import IIntIdRemovedEvent
+from zope.app.testing import setup
+
+from zc.catalog import index as zcindex
+
+from z3c.indexer import interfaces
+from z3c.indexer import subscriber
+from z3c.indexer.query import TextQuery
+from z3c.indexer.query import Eq
+from z3c.indexer.index import TextIndex
+from z3c.indexer.index import FieldIndex
+from z3c.indexer.index import ValueIndex
+from z3c.indexer.indexer import MultiAutoIndexer
+from z3c.indexer.search import SearchQuery
+
+
+timeResult = None
+
+def timeTest(function, counter=10000, *args, **kw):
+ """timer"""
+ res = []
+ append = res.append
+ def wrapper(*args, **kw):
+ start_time = time.time()
+ for i in range(counter):
+ append(function(*args, **kw))
+ total_time = time.time() - start_time
+ global timeResult
+ timeResult = total_time
+ return res
+ return wrapper(*args, **kw)
+
+
+def calcSpeedUp(catalogTime, indexerTime):
+ try:
+ if catalogTime > indexerTime:
+ # faster
+ precent = int('%.0f' % \
+ float(float(catalogTime) / float(indexerTime) * 100 -100))
+ return '%4d%s' % (precent, '%')
+ else:
+ # slower
+ precent = int('%.0f' % \
+ float(float(indexerTime) / float(catalogTime) * 100 -100))
+ return '-%3d%s' % (precent, '%')
+ except ZeroDivisionError:
+ return ' 0'
+
+
+
+
+##############################################################################
+#
+# test components
+#
+##############################################################################
+
+class IContent(zope.interface.Interface):
+ """Sample content."""
+
+ title = zope.schema.TextLine(title=u'Title')
+ counter = zope.schema.Int(title=u'Counter')
+ time = zope.schema.Datetime(title=u'Time')
+
+
+class Content(persistent.Persistent, contained.Contained):
+ """Sample content."""
+
+ zope.interface.implements(IContent)
+
+ def __init__(self, counter):
+ self.title = u'Number %s' % counter
+ self.counter = counter
+ self.time = time.time()
+
+
+# catalog index
+class CatalogTextIndex(ZTextIndex, contained.Contained):
+ """Test index for catalog."""
+
+ def index_doc(self, docid, obj):
+ if not IContent.providedBy(obj):
+ return
+ return super(CatalogTextIndex, self).index_doc(docid, obj.title)
+
+
+class CatalogValueIndex(zcindex.ValueIndex, contained.Contained):
+ """Value Index for catalog setup."""
+
+ def index_doc(self, docid, obj):
+ if not IContent.providedBy(obj):
+ return
+ return super(CatalogValueIndex, self).index_doc(docid, obj.time)
+
+
+class CatalogFieldIndex(ZFieldIndex, contained.Contained):
+ """Value Index for catalog setup."""
+
+ default_field_name = 'counter'
+ default_interface = IContent
+
+
+# indexer
+class ContentIndexer(MultiAutoIndexer):
+ """Multi indexer for IContent."""
+
+ zope.component.adapts(IContent)
+
+ def __init__(self, context):
+ """Registered as named index adapter"""
+ super(ContentIndexer, self).__init__(context)
+ site = hooks.getSite()
+ sm = site.getSiteManager()
+ self.textIndex = sm['textIndex']
+ self.valueIndex = sm['valueIndex']
+ self.fieldIndex = sm['fieldIndex']
+
+ def doIndex(self):
+ # index context in textIndex
+ self.textIndex.doIndex(self.oid, self.context.title)
+
+ # index context in valueIndex
+ self.valueIndex.doIndex(self.oid, self.context.time)
+
+ # index context in fieldIndex
+ self.fieldIndex.doIndex(self.oid, self.context.counter)
+
+ def doUnIndex(self):
+
+ # index context in setIndex
+ self.textIndex.doUnIndex(self.oid)
+
+ # index context in valueIndex
+ self.valueIndex.doUnIndex(self.oid)
+
+ # index context in fieldIndex
+ self.fieldIndex.doUnIndex(self.oid)
+
+
+def setUpIntIds(sm):
+ intids = IntIds()
+ sm['intids'] = intids
+ zope.component.provideUtility(intids, IIntIds)
+
+
+def setUpAdapter():
+ # setup key reference
+ zope.component.provideAdapter(SimpleKeyReference, (None,), IKeyReference)
+
+ # provide sub location adapter
+ zope.component.provideAdapter(contained.ContainerSublocations,
+ (zope.app.container.interfaces.IReadContainer,),
+ zope.location.interfaces.ISublocations)
+
+
+def setUpSubscribers(gsm):
+ gsm.registerHandler(indexAdded, (ICatalogIndex, IObjectAddedEvent))
+ gsm.registerHandler(indexDocSubscriber, (IIntIdAddedEvent,))
+ gsm.registerHandler(reindexDocSubscriber, (ICatalogIndex, IObjectModifiedEvent))
+ gsm.registerHandler(unindexDocSubscriber, (IIntIdRemovedEvent,))
+ gsm.registerHandler(addIntIdSubscriber)
+ gsm.registerHandler(removeIntIdSubscriber)
+ gsm.registerHandler(contained.dispatchToSublocations,
+ (zope.location.interfaces.ILocation, IObjectMovedEvent))
+
+
+def setUpCatalog(amountOfIndexes=0):
+ site = setup.placefulSetUp(True)
+ setUpAdapter()
+
+ # setup IIntIds
+ sm = site.getSiteManager()
+ setUpIntIds(sm)
+
+ # setup catalog
+ ctlg = Catalog()
+ sm['catalog'] = ctlg
+ sm.registerUtility(ctlg, ICatalog)
+
+ # create and add indexes
+ ctlg['textIndex'] = CatalogTextIndex()
+ ctlg['valueIndex'] = CatalogValueIndex()
+ ctlg['fieldIndex'] = CatalogFieldIndex()
+ # add additional not IContent relevant indexes
+ for i in range(amountOfIndexes):
+ idxName = 'index-%i' % i
+ ctlg[idxName] = CatalogTextIndex()
+
+ # now we are ready and can setup the event subscribers
+ gsm = zope.component.getGlobalSiteManager()
+ setUpSubscribers(gsm)
+
+ # return site
+ return site
+
+
+def setUpIndexer(amountOfIndexes=0):
+ site = setup.placefulSetUp(True)
+ setUpAdapter()
+
+ # setup IIntIds
+ sm = site.getSiteManager()
+ setUpIntIds(sm)
+
+ # create and add indexes
+ sm['textIndex'] = textIndex = TextIndex()
+ sm['valueIndex'] = valueIndex = ValueIndex()
+ sm['fieldIndex'] = fieldIndex = FieldIndex()
+
+ # register indexes as utilities
+ sm.registerUtility(textIndex, interfaces.IIndex, name='textIndex')
+ sm.registerUtility(valueIndex, interfaces.IIndex, name='valueIndex')
+ sm.registerUtility(fieldIndex, interfaces.IIndex, name='fieldIndex')
+
+ # add additional not IContent relevant indexes
+ for i in range(amountOfIndexes):
+ idxName = 'index-%i' % i
+ sm[idxName] = CatalogTextIndex()
+ sm.registerUtility(sm[idxName], interfaces.IIndex, name=idxName)
+
+ # provide IAutoIndexer for IContent
+ zope.component.provideAdapter(ContentIndexer)
+
+ # now we are ready and can setup the event subscribers
+ gsm = zope.component.getGlobalSiteManager()
+ setUpSubscribers(gsm)
+ zope.component.provideHandler(subscriber.autoIndexSubscriber)
+ zope.component.provideHandler(subscriber.autoUnindexSubscriber)
+
+ # return site
+ return site
+
+def tearDownCatalog():
+ setup.placefulTearDown()
+
+def tearDownIndexer():
+ setup.placefulTearDown()
+
+
+def runCatalogIndexing(site, amountOfObjects=1000):
+ # setup component
+ for i in range(amountOfObjects):
+ content = Content(i)
+ zope.event.notify(zope.lifecycleevent.ObjectCreatedEvent(content))
+ site[unicode(i)] = content
+
+
+def runIndexerIndexing(site, amountOfObjects=1000):
+ # setup component
+ for i in range(amountOfObjects):
+ content = Content(i)
+ zope.event.notify(zope.lifecycleevent.ObjectCreatedEvent(content))
+ site[unicode(i)] = content
+
+
+def runCatalogUpdate():
+ ctlg = zope.component.getUtility(ICatalog)
+ ctlg.updateIndexes()
+
+
+def runIndexerUpdate(site):
+ for content in site.values():
+ interfaces.IAutoIndexer(content).doIndex()
+
+
+def runCatalogQuery():
+ ctlg = zope.component.getUtility(ICatalog)
+ return ctlg.searchResults(textIndex='Number 42', fieldIndex=(41, 43))
+
+
+def runIndexerQuery():
+ textQuery = TextQuery('textIndex', 'Number 42')
+ fieldQuery = Eq('fieldIndex', 42)
+ searchQuery = SearchQuery(textQuery)
+ searchQuery = searchQuery.Or(fieldQuery)
+ return searchQuery.apply()
+
+
+def runCatalogNotQuery():
+ ctlg = zope.component.getUtility(ICatalog)
+ query = ({'textIndex':'Number', 'fieldIndex':(1, 999999999999999999)})
+ found = ctlg.apply(query)
+
+ # get all objects
+ intids = zope.component.getUtility(IIntIds)
+
+ # and remove the given result
+ result = IFBTree()
+ for uid in intids:
+ result.insert(uid, 0)
+ return difference(result, found)
+
+
+def runIndexerNotQuery():
+ textQuery = TextQuery('textIndex', 'Number 42')
+ fieldQuery = Eq('fieldIndex', 1)
+ searchQuery = SearchQuery(textQuery)
+ searchQuery = searchQuery.Not(fieldQuery)
+ return searchQuery.apply()
+
+
+def runObjectModifiedEvent(obj):
+ obj.title = u'New'
+ zope.event.notify(zope.lifecycleevent.ObjectModifiedEvent(obj))
+
+
+def runParentModifiedEvent(site):
+ zope.event.notify(zope.lifecycleevent.ObjectModifiedEvent(site))
+
+
+def runObjectRemove(site):
+ # remove one item
+ for item in site.values():
+ del site[item.__name__]
+ return
+
+##############################################################################
+#
+# performance test
+#
+# This performance test will show that the default zope.app.catalog based
+# indexing pattern is very bad on a large set of indexes. Because the default
+# catalog pattern tries to index every object in every index.
+#
+# The z3c.indexer pattern avoids this because you can implement optimized
+# indexing pattern which only index objects in the relevant indexes.
+#
+##############################################################################
+
+def runTest(repeatTimes, amountOfObjects, amountOfIndexes=0):
+ print ""
+ print "Run test with"
+ print "-------------"
+ print "- %i x repeat tests" % repeatTimes
+ print "- %i objects" % amountOfObjects
+ print "- 3 relevant indexes"
+ print "- %i other indexes" % amountOfIndexes
+ print "Note, the index update get only processed one time."
+ print ""
+ print "zope.app.catalog"
+ # setup zope.app.catalog based performance test
+ catalogSite = setUpCatalog(amountOfIndexes)
+
+ # time catalog indexing
+ timeTest(runCatalogIndexing, 1, catalogSite, amountOfObjects)
+ runCatalogIndexingTime = timeResult
+ print "catalog based indexing time: %.2f s" % runCatalogIndexingTime
+
+ # time catalog query
+ res = timeTest(runCatalogQuery, repeatTimes)
+ runCatalogQueryTime = timeResult
+ print "catalog based query time: %.2f s" % runCatalogQueryTime
+ if len(res.pop(0)) != 1:
+ print "...bad query result returned"
+
+ # time catalog not query
+ res = timeTest(runCatalogNotQuery, repeatTimes)
+ runCatalogNotQueryTime = timeResult
+ print "catalog based not query time: %.2f s" % runCatalogNotQueryTime
+ if len(res.pop(0)) != 1:
+ print "...bad query result returned"
+
+ # time catalog indexes update
+ timeTest(runCatalogUpdate, 1)
+ runCatalogUpdateTime = timeResult
+ print "catalog based update time: %.2f s" % runCatalogUpdateTime
+
+ # time object modified event
+ ctlgObj = catalogSite['1']
+ timeTest(runObjectModifiedEvent, repeatTimes, ctlgObj)
+ runCatalogModifiedTime = timeResult
+ print "catalog object modified time: %.2f s" % runCatalogModifiedTime
+
+ # time parent modified event
+ timeTest(runParentModifiedEvent, repeatTimes, catalogSite)
+ runCatalogParentModifiedTime = timeResult
+ print "catalog parent modified time: %.2f s" % runCatalogParentModifiedTime
+
+ # time object remove (IObjectMovedEvent)
+ timeTest(runObjectRemove, amountOfObjects, catalogSite)
+ runCatalogObjectRemoveTime = timeResult
+ print "catalog object remove time: %.2f s" % runCatalogObjectRemoveTime
+
+
+ # tear down
+ tearDownCatalog()
+
+ # setup z3c.indexer based performance test
+ indexerSite = setUpIndexer(amountOfIndexes)
+
+ print ""
+ print "z3c.indexer"
+ # time indexer indexing
+ timeTest(runIndexerIndexing, 1, indexerSite, amountOfObjects)
+ runIndexerIndexingTime = timeResult
+ print "indexer based indexing time: %.2f s" % runIndexerIndexingTime
+
+ # time indexer query
+ res = timeTest(runIndexerQuery, repeatTimes)
+ runIndexerQueryTime = timeResult
+ print "indexer based query time: %.2f s" % runIndexerQueryTime
+ if len(res.pop(0)) != 1:
+ print "...bad query result returned"
+
+ # time indexer query
+ res = timeTest(runIndexerNotQuery, repeatTimes)
+ runIndexerNotQueryTime = timeResult
+ print "indexer based not query time: %.2f s" % runIndexerNotQueryTime
+ if len(res.pop(0)) != 1:
+ print "...bad query result returned"
+
+ # time indexer indexes update
+ timeTest(runIndexerUpdate, 1, indexerSite)
+ runIndexerUpdateTime = timeResult
+ print "indexer based update time: %.2f s" % runIndexerUpdateTime
+
+ # time object modified event
+ idObj = indexerSite['1']
+ timeTest(runObjectModifiedEvent, repeatTimes, idObj)
+ runIndexerModifiedTime = timeResult
+ print "indexer object modified time: %.2f s" % runIndexerModifiedTime
+
+ # time parent modified event
+ timeTest(runParentModifiedEvent, repeatTimes, indexerSite)
+ runIndexerParentModifiedTime = timeResult
+ print "indexer parent modified time: %.2f s" % runIndexerParentModifiedTime
+
+ # time object remove (IObjectMovedEvent)
+ timeTest(runObjectRemove, amountOfObjects, indexerSite)
+ runIndexerObjectRemoveTime = timeResult
+ print "indexer object remove time: %.2f s" % runIndexerObjectRemoveTime
+
+ # tear down
+ tearDownIndexer()
+
+ print ""
+ print "Result for %i objects with 3 relevant and %i other indexes" % (
+ amountOfObjects, amountOfIndexes)
+ print " ------------------------------------------------------------------------"
+ print "| type | indexing | query | not query | update | modify | remove |"
+ print " ------------------------------------------------------------------------"
+ print "| catalog | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
+ runCatalogIndexingTime, runCatalogQueryTime, runCatalogQueryTime,
+ runCatalogUpdateTime, runCatalogModifiedTime,
+ runCatalogObjectRemoveTime)
+ print " ------------------------------------------------------------------------"
+ print "| indexer | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
+ runIndexerIndexingTime, runIndexerQueryTime, runIndexerNotQueryTime,
+ runIndexerUpdateTime, runIndexerModifiedTime,
+ runIndexerObjectRemoveTime)
+ print " ------------------------------------------------------------------------"
+ print "| speedup | % 7.2fs |% 7.2fs | % 7.2fs |% 7.2fs |% 7.2fs |% 7.2fs |" % (
+ (runCatalogIndexingTime - runIndexerIndexingTime),
+ (runCatalogQueryTime - runIndexerQueryTime),
+ (runCatalogNotQueryTime - runIndexerNotQueryTime),
+ (runCatalogUpdateTime - runIndexerUpdateTime),
+ (runCatalogModifiedTime - runIndexerModifiedTime),
+ (runCatalogObjectRemoveTime - runIndexerObjectRemoveTime))
+ print " ------------------------------------------------------------------------"
+ print "| speedup | %s | %s | %s | %s | %s | %s |" %(
+ calcSpeedUp(runCatalogIndexingTime, runIndexerIndexingTime),
+ calcSpeedUp(runCatalogQueryTime, runIndexerQueryTime),
+ calcSpeedUp(runCatalogNotQueryTime, runIndexerNotQueryTime),
+ calcSpeedUp(runCatalogUpdateTime, runIndexerUpdateTime),
+ calcSpeedUp(runCatalogModifiedTime, runIndexerModifiedTime),
+ calcSpeedUp(runCatalogObjectRemoveTime, runIndexerObjectRemoveTime))
+ print " ------------------------------------------------------------------------"
+ print ""
+
+
+def main():
+ runTest(1, 100, 10)
+ runTest(1, 100, 50)
+ runTest(1000, 100, 10)
+ runTest(1000, 100, 50)
+ runTest(1000, 10000, 10)
+ runTest(1000, 10000, 50)
Property changes on: z3c.indexer/trunk/src/z3c/indexer/performance.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/query.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/query.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/query.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,190 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import zope.component
+from z3c.indexer import interfaces
+
+
+class QueryMixin(object):
+ """Index query."""
+
+ def __init__(self, indexOrName):
+ if isinstance(indexOrName, str):
+ self.index = zope.component.getUtility(interfaces.IIndex,
+ name=indexOrName)
+ else:
+ # indexOrName is a index
+ self.index = indexOrName
+
+
+class TextQuery(QueryMixin):
+ """Text query."""
+
+ zope.interface.implements(interfaces.ITextQuery)
+
+ def __init__(self, indexOrName, value):
+ super(TextQuery, self).__init__(indexOrName)
+ self.value = value
+
+ def apply(self):
+ return self.index.apply(self.value)
+
+
+class Eq(QueryMixin):
+ """Equal query."""
+
+ zope.interface.implements(interfaces.IEqQuery)
+
+ def __init__(self, indexOrName, value):
+ assert value is not None
+ super(Eq, self).__init__(indexOrName)
+ self.value = value
+
+ def apply(self):
+ return self.index.applyEq(self.value)
+
+
+class NotEq(QueryMixin):
+ """Not equal query."""
+
+ zope.interface.implements(interfaces.INotEqQuery)
+
+ def __init__(self, indexOrName, value):
+ assert value is not None
+ super(NotEq, self).__init__(indexOrName)
+ self.value = value
+
+ def apply(self):
+ return self.index.applyNotEq(self.value)
+
+
+class Between(QueryMixin):
+ """Between query."""
+
+ zope.interface.implements(interfaces.IBetweenQuery)
+
+ def __init__(self, indexOrName, min_value, max_value, exclude_min=False,
+ exclude_max=False):
+ super(Between, self).__init__(indexOrName)
+ self.min_value = min_value
+ self.max_value = max_value
+ self.exclude_min = exclude_min
+ self.exclude_max = exclude_max
+
+ def apply(self):
+ return self.index.applyBetween(self.min_value, self.max_value,
+ self.exclude_min, self.exclude_max)
+
+
+class Ge(QueryMixin):
+ """Between query."""
+
+ zope.interface.implements(interfaces.IGeQuery)
+
+ def __init__(self, indexOrName, min_value, exclude_min=False):
+ super(Ge, self).__init__(indexOrName)
+ self.min_value = min_value
+ self.exclude_min = exclude_min
+
+ def apply(self):
+ return self.index.applyGe(self.min_value, self.exclude_min)
+
+
+class Le(QueryMixin):
+ """Between query."""
+
+ zope.interface.implements(interfaces.ILeQuery)
+
+ def __init__(self, indexOrName, max_value, exclude_max=False):
+ super(Le, self).__init__(indexOrName)
+ self.max_value = max_value
+ self.exclude_max = exclude_max
+
+ def apply(self):
+ return self.index.applyLe(self.max_value, self.exclude_max)
+
+
+class In(QueryMixin):
+ """In query."""
+
+ zope.interface.implements(interfaces.IInQuery)
+
+ def __init__(self, indexOrName, values):
+ super(In, self).__init__(indexOrName)
+ self.values = values
+
+ def apply(self):
+ return self.index.applyIn(self.values)
+
+
+class AnyOf(QueryMixin):
+ """Any of query.
+
+ The result will be the docids whose values contain any of the given values.
+ """
+
+ zope.interface.implements(interfaces.IAnyOfQuery)
+
+ def __init__(self, indexOrName, values):
+ super(AnyOf, self).__init__(indexOrName)
+ self.values = values
+
+ def apply(self):
+ return self.index.applyAnyOf(self.values)
+
+
+class AllOf(QueryMixin):
+ """Any of query.
+
+ The result will be the docids whose values contain all of the given values.
+ """
+
+ zope.interface.implements(interfaces.IAllOfQuery)
+
+ def __init__(self, indexOrName, values):
+ super(AllOf, self).__init__(indexOrName)
+ self.values = values
+
+ def apply(self):
+ return self.index.applyAllOf(self.values)
+
+
+class ExtentAny(QueryMixin):
+ """ExtentAny query."""
+
+ zope.interface.implements(interfaces.IExtentAnyQuery)
+
+ def __init__(self, indexOrName, extent):
+ super(ExtentAny, self).__init__(indexOrName)
+ self.extent = extent
+
+ def apply(self):
+ return self.index.applyExtentAny(self.extent)
+
+
+class ExtentNone(QueryMixin):
+ """ExtentNone query."""
+
+ zope.interface.implements(interfaces.IExtentNoneQuery)
+
+ def __init__(self, indexOrName, extent):
+ super(ExtentNone, self).__init__(indexOrName)
+ self.extent = extent
+
+ def apply(self):
+ return self.index.applyExtentNone(self.extent)
Property changes on: z3c.indexer/trunk/src/z3c/indexer/query.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/search.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/search.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/search.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,122 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import BTrees
+import zope.interface
+import zope.component
+from zope.app.intid.interfaces import IIntIds
+
+from z3c.indexer import interfaces
+
+
+class ResultSet:
+ """Lazily accessed set of objects."""
+
+ def __init__(self, uids, intids):
+ self.uids = uids
+ self.intids = intids
+
+ def __len__(self):
+ return len(self.uids)
+
+ def __contains__(self, item):
+ idx = self.intids.queryId(item)
+ if idx and idx in self.uids:
+ return True
+ else:
+ return False
+
+ def __getitem__(self, slice):
+ start = slice.start
+ stop = slice.stop
+ if stop > len(self):
+ stop = len(self)
+ return [self.intids.getObject(self.uids[idx])
+ for idx in range(start, stop)]
+
+ def __iter__(self):
+ for uid in self.uids:
+ obj = self.intids.getObject(uid)
+ yield obj
+
+
+class SearchQuery(object):
+ """Chainable query processor."""
+
+ zope.interface.implements(interfaces.ISearchQuery)
+
+ family = BTrees.family32
+
+ def __init__(self, query=None, family=None):
+ """Initialize with none or existing query."""
+ res= None
+ if query is not None:
+ res = query.apply()
+ if family is not None:
+ self.family = family
+ if res:
+ self.results = self.family.IF.Set(res)
+ else:
+ self.results = self.family.IF.Set()
+
+ def apply(self):
+ return self.results
+
+ def searchResults(self):
+ results = []
+ if self.results is not None:
+ intids = zope.component.getUtility(IIntIds)
+ results = ResultSet(self.results, intids)
+ return results
+
+ def Or(self, query):
+ """Enhance search results. (union)
+
+ The result will contain intids which exist in the existing result
+ and/or in the result from te given query.
+ """
+ res = query.apply()
+ if res:
+ self.results = self.family.IF.union(self.results, res)
+ return self
+
+ def And(self, query):
+ """Restrict search results. (intersection)
+
+ The result will only contain intids which exist in the existing
+ result and in the result from te given query. (union)
+ """
+ res = query.apply()
+ if res:
+ self.results = self.family.IF.intersection(self.results, res)
+ return self
+
+ def Not(self, query):
+ """Exclude search results. (difference)
+
+ The result will only contain intids which exist in the existing
+ result but do not exist in the result from te given query.
+
+ This is faster if the existing result is small. But note, it get
+ processed in a chain, results added after this query get added again.
+ So probably you need to call this at the end of the chain.
+ """
+ res = query.apply()
+ if res:
+ self.results = self.family.IF.difference(self.results, res)
+ return self
Property changes on: z3c.indexer/trunk/src/z3c/indexer/search.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/subscriber.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/subscriber.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/subscriber.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,42 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import zope.component
+from zope.app.intid.interfaces import IIntIdAddedEvent
+from zope.app.intid.interfaces import IIntIdRemovedEvent
+
+from z3c.indexer import interfaces
+
+
+ at zope.component.adapter(IIntIdAddedEvent)
+def autoIndexSubscriber(event):
+ """Index all objects which get added to the intids utility."""
+ adapters = zope.component.getAdapters((event.object,),
+ interfaces.IAutoIndexer)
+ for name, adapter in adapters:
+ adapter.doIndex()
+
+
+
+ at zope.component.adapter(IIntIdRemovedEvent)
+def autoUnindexSubscriber(event):
+ """Unindex all objects which get added to the intids utility."""
+ adapters = zope.component.getAdapters((event.object,),
+ interfaces.IAutoIndexer)
+ for name, adapter in adapters:
+ adapter.doUnIndex()
\ No newline at end of file
Property changes on: z3c.indexer/trunk/src/z3c/indexer/subscriber.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/testing.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/testing.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/testing.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,47 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import zope.component
+from zope.app import folder
+from zope.app.keyreference.testing import SimpleKeyReference
+from zope.app.testing import setup
+
+
+###############################################################################
+#
+# Test Component
+#
+###############################################################################
+
+class SiteStub(folder.Folder):
+ """Sample site."""
+
+
+###############################################################################
+#
+# setUp helper
+#
+###############################################################################
+
+def setUp(test):
+ test.globs = {'root': setup.placefulSetUp(True)}
+ zope.component.provideAdapter(SimpleKeyReference)
+
+
+def tearDown(test):
+ setup.placefulTearDown()
Property changes on: z3c.indexer/trunk/src/z3c/indexer/testing.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/tests.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/tests.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/tests.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,291 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import unittest
+import zope.component
+from zope.testing import doctest
+from zope.app.intid import IntIds
+from zope.app.intid.interfaces import IIntIds
+from zope.app.keyreference.interfaces import IKeyReference
+
+import z3c.testing
+from z3c.indexer import interfaces
+from z3c.indexer import testing
+from z3c.indexer import index
+from z3c.indexer import indexer
+from z3c.indexer import query
+from z3c.indexer import search
+
+
+class FakeKeyReference(object):
+ """Fake keyref for testing"""
+ def __init__(self, object):
+ self.object = object
+
+ def __call__(self):
+ return self.object
+
+ def __hash__(self):
+ return id(self.object)
+
+ def __cmp__(self, other):
+ return cmp(id(self.object), id(other.object))
+
+
+class ValueIndexerStub(indexer.ValueIndexer):
+ value = u'ignored'
+
+# IIndex
+class TestTextIndex(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.ITextIndex
+
+ def getTestClass(self):
+ return index.TextIndex
+
+
+class TestFieldIndex(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IFieldIndex
+
+ def getTestClass(self):
+ return index.FieldIndex
+
+
+class TestValueIndex(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IValueIndex
+
+ def getTestClass(self):
+ return index.ValueIndex
+
+
+class TestSetIndex(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.ISetIndex
+
+ def getTestClass(self):
+ return index.SetIndex
+
+
+# IIndexer
+class TestValueIndexer(z3c.testing.InterfaceBaseTest):
+
+ def setUp(test):
+ zope.component.provideAdapter(FakeKeyReference, (None,), IKeyReference)
+ intids = IntIds()
+ zope.component.provideUtility(intids, IIntIds)
+ intids.register(None)
+ valueIndex = index.ValueIndex()
+ zope.component.provideUtility(valueIndex, interfaces.IIndex)
+
+ def getTestInterface(self):
+ return interfaces.IValueIndexer
+
+ def getTestClass(self):
+ return ValueIndexerStub
+
+ def getTestPos(self):
+ return (None,)
+
+
+class TestMultiIndexer(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IIndexer
+
+ def getTestClass(self):
+ return indexer.MultiIndexer
+
+ def getTestPos(self):
+ return (None,)
+
+
+class TestTextQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.ITextQuery
+
+ def getTestClass(self):
+ return query.TextQuery
+
+ def getTestPos(self):
+ return (None, None)
+
+
+class TestEqQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IEqQuery
+
+ def getTestClass(self):
+ return query.Eq
+
+ def getTestPos(self):
+ return (None, 'not None')
+
+
+class TestNotEqQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.INotEqQuery
+
+ def getTestClass(self):
+ return query.NotEq
+
+ def getTestPos(self):
+ return (None, 'not None')
+
+
+class TestBetweenQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IBetweenQuery
+
+ def getTestClass(self):
+ return query.Between
+
+ def getTestPos(self):
+ return (None, None, None)
+
+
+class TestGeQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IGeQuery
+
+ def getTestClass(self):
+ return query.Ge
+
+ def getTestPos(self):
+ return (None, None)
+
+
+class TestLeQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.ILeQuery
+
+ def getTestClass(self):
+ return query.Le
+
+ def getTestPos(self):
+ return (None, None)
+
+
+class TestInQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IInQuery
+
+ def getTestClass(self):
+ return query.In
+
+ def getTestPos(self):
+ return (None, None)
+
+
+class TestAnyOfQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IAnyOfQuery
+
+ def getTestClass(self):
+ return query.AnyOf
+
+ def getTestPos(self):
+ return (None, None)
+
+
+class TestAllOfQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IAllOfQuery
+
+ def getTestClass(self):
+ return query.AllOf
+
+ def getTestPos(self):
+ return (None, None)
+
+
+class TestExtentAnyQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IExtentAnyQuery
+
+ def getTestClass(self):
+ return query.ExtentAny
+
+ def getTestPos(self):
+ return (None, None)
+
+
+class TestExtentNoneQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.IExtentNoneQuery
+
+ def getTestClass(self):
+ return query.ExtentNone
+
+ def getTestPos(self):
+ return (None, None)
+
+
+class TestSearchQuery(z3c.testing.InterfaceBaseTest):
+
+ def getTestInterface(self):
+ return interfaces.ISearchQuery
+
+ def getTestClass(self):
+ return search.SearchQuery
+
+
+def test_suite():
+ return unittest.TestSuite((
+ doctest.DocFileSuite('README.txt',
+ setUp=testing.setUp, tearDown=testing.tearDown,
+ optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
+ ),
+ unittest.makeSuite(TestTextIndex),
+ unittest.makeSuite(TestFieldIndex),
+ unittest.makeSuite(TestValueIndex),
+ unittest.makeSuite(TestSetIndex),
+ unittest.makeSuite(TestValueIndexer),
+ unittest.makeSuite(TestMultiIndexer),
+ unittest.makeSuite(TestTextQuery),
+ unittest.makeSuite(TestEqQuery),
+ unittest.makeSuite(TestNotEqQuery),
+ unittest.makeSuite(TestBetweenQuery),
+ unittest.makeSuite(TestGeQuery),
+ unittest.makeSuite(TestLeQuery),
+ unittest.makeSuite(TestInQuery),
+ unittest.makeSuite(TestAnyOfQuery),
+ unittest.makeSuite(TestExtentAnyQuery),
+ unittest.makeSuite(TestExtentNoneQuery),
+ unittest.makeSuite(TestSearchQuery),
+ ))
+
+
+if __name__ == '__main__':
+ unittest.main(defaultTest='test_suite')
Property changes on: z3c.indexer/trunk/src/z3c/indexer/tests.py
___________________________________________________________________
Name: svn:eol-style
+ native
Added: z3c.indexer/trunk/src/z3c/indexer/value.py
===================================================================
--- z3c.indexer/trunk/src/z3c/indexer/value.py (rev 0)
+++ z3c.indexer/trunk/src/z3c/indexer/value.py 2008-05-02 01:25:18 UTC (rev 86039)
@@ -0,0 +1,34 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+$Id:$
+"""
+__docformat__ = "reStructuredText"
+
+import zope.interface
+from z3c.indexer import interfaces
+
+
+class IIndexValue(object):
+ """Index value adapter"""
+
+ zope.interface.implements(interfaces.IIndexValue)
+
+ def __init__self(self, context, index):
+ self.context = context
+ self.index = index
+
+ @property
+ def value(self):
+ raise NotImplementedError('Sub class must implement value attribute')
Property changes on: z3c.indexer/trunk/src/z3c/indexer/value.py
___________________________________________________________________
Name: svn:eol-style
+ native
More information about the Checkins
mailing list