[Checkins] SVN: z3c.indexing.xapian/ Initial import.
Malthe Borch
mborch at gmail.com
Sat Mar 29 07:33:24 EDT 2008
Log message for revision 85011:
Initial import.
Changed:
A z3c.indexing.xapian/
A z3c.indexing.xapian/trunk/
A z3c.indexing.xapian/trunk/AUTHOR.txt
A z3c.indexing.xapian/trunk/README.txt
A z3c.indexing.xapian/trunk/bootstrap.py
A z3c.indexing.xapian/trunk/buildout.cfg
A z3c.indexing.xapian/trunk/setup.py
A z3c.indexing.xapian/trunk/src/
A z3c.indexing.xapian/trunk/src/z3c/
A z3c.indexing.xapian/trunk/src/z3c/__init__.py
A z3c.indexing.xapian/trunk/src/z3c/indexing/
A z3c.indexing.xapian/trunk/src/z3c/indexing/__init__.py
A z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/
A z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/README.txt
A z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/__init__.py
A z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/connection.py
A z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/dispatcher.py
A z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/interfaces.py
A z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/resolver.py
A z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/tests.py
-=-
Added: z3c.indexing.xapian/trunk/AUTHOR.txt
===================================================================
--- z3c.indexing.xapian/trunk/AUTHOR.txt (rev 0)
+++ z3c.indexing.xapian/trunk/AUTHOR.txt 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,4 @@
+Authors
+=======
+
+Malthe Borch, Sylvian Viollon, Kapil Thangavelu
Added: z3c.indexing.xapian/trunk/README.txt
===================================================================
--- z3c.indexing.xapian/trunk/README.txt (rev 0)
+++ z3c.indexing.xapian/trunk/README.txt 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,8 @@
+Overview
+--------
+
+This package provides a dispatcher that dispatches indexing to
+Xapian.
+
+A ``zope.app.intid``-resolver is provided. Other resolvers can be
+registered as components.
Added: z3c.indexing.xapian/trunk/bootstrap.py
===================================================================
--- z3c.indexing.xapian/trunk/bootstrap.py (rev 0)
+++ z3c.indexing.xapian/trunk/bootstrap.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,52 @@
+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+
+$Id: bootstrap.py 71627 2006-12-20 16:46:11Z jim $
+"""
+
+import os, shutil, sys, tempfile, urllib2
+
+tmpeggs = tempfile.mkdtemp()
+
+ez = {}
+exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
+ ).read() in ez
+ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
+
+import pkg_resources
+
+cmd = 'from setuptools.command.easy_install import main; main()'
+if sys.platform == 'win32':
+ cmd = '"%s"' % cmd # work around spawn lamosity on windows
+
+ws = pkg_resources.working_set
+assert os.spawnle(
+ os.P_WAIT, sys.executable, sys.executable,
+ '-c', cmd, '-mqNxd', tmpeggs, 'zc.buildout',
+ dict(os.environ,
+ PYTHONPATH=
+ ws.find(pkg_resources.Requirement.parse('setuptools')).location
+ ),
+ ) == 0
+
+ws.add_entry(tmpeggs)
+ws.require('zc.buildout')
+import zc.buildout.buildout
+zc.buildout.buildout.main(sys.argv[1:] + ['bootstrap'])
+shutil.rmtree(tmpeggs)
Added: z3c.indexing.xapian/trunk/buildout.cfg
===================================================================
--- z3c.indexing.xapian/trunk/buildout.cfg (rev 0)
+++ z3c.indexing.xapian/trunk/buildout.cfg 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,39 @@
+[buildout]
+parts = xapian xapian-bindings interpreter test
+find-links =
+ http://download.zope.org/distribution/
+ http://xappy.googlecode.com/svn/trunk#egg=xappy-0.5
+
+index = http://download.zope.org/ppix/
+develop = . ../z3c.indexing.dispatch
+
+[xapian]
+recipe = zc.recipe.cmmi
+url = http://xappy.googlecode.com/files/xapian-core-10121.tgz
+
+[xapian-bindings]
+recipe = zc.recipe.cmmi
+url = http://xappy.googlecode.com/files/xapian-bindings-10121.tgz
+extra_options =
+ PYTHON_LIB=${xapian:location}/lib/python
+ XAPIAN_CONFIG=${xapian:location}/bin/xapian-config
+ --with-python
+ --with-php=no
+ --with-ruby=no
+ --with-java=no
+ --with-csharp=no
+
+[interpreter]
+# python interpreter w/ app eggs for all entry points found in these eggs
+recipe = zc.recipe.egg
+eggs = xappy
+extra-paths = ${xapian:location}/lib/python
+interpreter = python
+
+[test]
+recipe = zc.recipe.testrunner
+eggs =
+ xappy
+ z3c.indexing.xapian
+extra-paths= ${xapian:location}/lib/python
+defaults = ['--tests-pattern', '^f?tests$', '-v']
\ No newline at end of file
Added: z3c.indexing.xapian/trunk/setup.py
===================================================================
--- z3c.indexing.xapian/trunk/setup.py (rev 0)
+++ z3c.indexing.xapian/trunk/setup.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,41 @@
+from setuptools import setup, find_packages
+import sys, os
+
+version = '0.1'
+
+setup(name='z3c.indexing.xapian',
+ version=version,
+ description="Xapian indexing dispatcher.",
+ long_description="""\
+""",
+ # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers
+ classifiers=[
+ "Framework :: Plone",
+ "Framework :: Zope2",
+ "Framework :: Zope3",
+ "Programming Language :: Python",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+ ],
+ keywords='',
+ author='Zope Corporation and Contributors',
+ author_email='zope3-dev at zope.org',
+ url='',
+ license='ZPL',
+ packages=find_packages('src'),
+ package_dir={'': 'src'},
+ namespace_packages=['z3c', 'z3c.indexing'],
+ include_package_data=True,
+ zip_safe=False,
+ install_requires=[
+ 'setuptools',
+ 'zope.interface',
+ 'zope.component',
+ 'zope.testing',
+ 'zope.app.intid',
+ 'ZODB3',
+ 'z3c.indexing.dispatch',
+ ],
+ entry_points="""
+ # -*- Entry points: -*-
+ """,
+ )
Added: z3c.indexing.xapian/trunk/src/z3c/__init__.py
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/__init__.py (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/__init__.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+ __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+ from pkgutil import extend_path
+ __path__ = extend_path(__path__, __name__)
Added: z3c.indexing.xapian/trunk/src/z3c/indexing/__init__.py
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/indexing/__init__.py (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/indexing/__init__.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+ __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+ from pkgutil import extend_path
+ __path__ = extend_path(__path__, __name__)
Added: z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/README.txt
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/README.txt (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/README.txt 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,167 @@
+z3c.indexing.xapian
+===================
+
+Let's set up some content.
+
+ >>> rabbit = Content(title=u"rabbit", description="furry little creatures")
+ >>> elephant = Content(title=u"elephant", description="large mammals with memory")
+ >>> snake = Content(title=u"snake", description="reptile with scales")
+
+Set up component lookup.
+
+ >>> from zope.component.interfaces import IComponentLookup
+ >>> component.provideAdapter(
+ ... lambda context: component.getSiteManager(), (Content,), IComponentLookup)
+
+Setting up an indexer connection
+--------------------------------
+
+To index a given object, the dispatcher needs to be able to locate an
+indexer connection utility, identified by the interface
+``IIndexerConnection``.
+
+We'll provide the utility globally:
+
+ >>> import xappy
+ >>> indexer = xappy.IndexerConnection('tmp.idx')
+
+ >>> from z3c.indexing.xapian.interfaces import IIndexerConnection
+ >>> component.provideUtility(indexer, IIndexerConnection)
+
+Let's add field actions corresponding to our example content object.
+
+ >>> indexer.add_field_action('title', xappy.FieldActions.INDEX_FREETEXT )
+ >>> indexer.add_field_action('title', xappy.FieldActions.STORE_CONTENT )
+ >>> indexer.add_field_action('description', xappy.FieldActions.INDEX_FREETEXT )
+
+Next we set up an adapter that provides a Xapian document for our content.
+
+ >>> def createXapianDocument(obj):
+ ... doc = xappy.UnprocessedDocument()
+ ... doc.id = obj.title
+ ... doc.fields.append(xappy.Field('title', obj.title))
+ ... doc.fields.append(xappy.Field('description', obj.description))
+ ... return doc
+
+ >>> from z3c.indexing.xapian.interfaces import IDocument
+ >>> component.provideAdapter(createXapianDocument, (Content,), IDocument)
+
+ >>> IDocument(rabbit)
+ UnprocessedDocument(u'rabbit',
+ [Field('title', u'rabbit'), Field('description', 'furry little creatures')])
+
+Dispatch
+--------
+
+ >>> from z3c.indexing.xapian.dispatcher import XapianDispatcher
+ >>> dispatcher = XapianDispatcher()
+
+Let's index some content.
+
+ >>> dispatcher.index(rabbit)
+ >>> dispatcher.index(elephant)
+ >>> dispatcher.index(snake)
+
+Modification and deletion.
+
+ >>> rabbit.description = 'cute little creatures'
+ >>> dispatcher.reindex(rabbit)
+ >>> dispatcher.unindex(snake)
+
+Flush queue.
+
+ >>> dispatcher.flush()
+
+Searching
+---------
+
+Now we can start querying the index.
+
+ >>> from z3c.indexing.xapian.connection import ConnectionHub
+ >>> hub = ConnectionHub('tmp.idx')
+
+We should be able to find the "rabbit" item.
+
+ >>> searcher = hub.get()
+ >>> query = searcher.query_parse('rabbit')
+ >>> len(searcher.search( query, 0, 30))
+ 1
+
+Also by searching for the modified description.
+
+ >>> searcher = hub.get()
+ >>> query = searcher.query_parse('cute little creatures')
+ >>> len(searcher.search( query, 0, 30))
+ 1
+
+But the "snake" item shouldn't be there.
+
+ >>> searcher = hub.get()
+ >>> query = searcher.query_parse('snake')
+ >>> len(searcher.search( query, 0, 30))
+ 0
+
+Resolving
+---------
+
+ >>> from ZODB.tests.util import DB
+ >>> db = DB()
+
+Opened database and acquire root object:
+
+ >>> conn = db.open()
+ >>> root = conn.root()
+
+Add these objects to the root object.
+
+ >>> root['content'] = dict(rabbit=rabbit, elephant=elephant, snake=snake)
+
+Commit transaction, so objects are added to the ZODB.
+
+ >>> import transaction
+ >>> transaction.commit()
+
+Setup an intid service and an ``IKeyReference`` adapter that uses the
+persistent object id to uniquely identify an object.
+
+ >>> import zope.app.intid
+ >>> component.provideUtility(zope.app.intid.IntIds(), zope.app.intid.interfaces.IIntIds)
+
+ >>> import zope.app.keyreference.persistent
+ >>> from persistent.interfaces import IPersistent
+ >>> component.provideAdapter(
+ ... zope.app.keyreference.persistent.KeyReferenceToPersistent, (IPersistent,))
+
+Register our content objects with the intid machinery.
+
+ >>> from zope.app.container.contained import ObjectAddedEvent
+ >>> zope.app.intid.addIntIdSubscriber(rabbit, ObjectAddedEvent(rabbit))
+ >>> zope.app.intid.addIntIdSubscriber(elephant, ObjectAddedEvent(elephant))
+ >>> zope.app.intid.addIntIdSubscriber(snake, ObjectAddedEvent(snake))
+
+Provide the intid resolver as utility:
+
+ >>> from z3c.indexing.xapian.resolver import IntIdResolver
+ >>> component.provideUtility(IntIdResolver(), name='intid')
+
+ >>> from zope.event import notify
+ >>> from zope.app.container.contained import ObjectAddedEvent
+
+Notify the intid framework that our content was added.
+
+ >>> notify(ObjectAddedEvent(rabbit))
+ >>> notify(ObjectAddedEvent(elephant))
+ >>> notify(ObjectAddedEvent(snake))
+
+Let's try out the resolver.
+
+ >>> from z3c.indexing.xapian.interfaces import IResolver
+ >>> resolver = component.getUtility(IResolver, name='intid')
+ >>> id = resolver.getId(rabbit)
+ >>> resolver.getObject(id) is rabbit
+ True
+
+Cleanup
+-------
+
+ >>> conn.close()
Added: z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/__init__.py
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/__init__.py (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/__init__.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1 @@
+#
Added: z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/connection.py
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/connection.py (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/connection.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,38 @@
+from zope import interface
+
+import threading
+import interfaces
+import time
+import xappy
+
+class ConnectionHub(object):
+ interface.implements(interfaces.IConnectionHub)
+
+ auto_refresh_delta = 20 # max time in seconds till we refresh a connection
+
+ def __init__(self, index_path):
+ self.store = threading.local()
+ self.modified = time.time()
+ self.index_path = index_path
+
+ def invalidate(self):
+ self.modified = time.time()
+
+ def get(self):
+ conn = getattr(self.store, 'connection', None)
+
+ now = time.time()
+ if self.modified + self.auto_refresh_delta < now:
+ self.modified = now
+
+ if conn is None:
+ self.store.connection = conn = xappy.SearchConnection(self.index_path)
+ self.store.opened = now
+
+ opened = getattr(self.store, 'opened')
+
+ if opened < self.modified:
+ conn.reopen()
+ self.store.opened = now
+
+ return conn
Added: z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/dispatcher.py
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/dispatcher.py (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/dispatcher.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,40 @@
+from zope import interface
+from zope import component
+
+from z3c.indexing.dispatch.interfaces import IDispatcher
+
+import interfaces
+
+class XapianDispatcher(object):
+ """Xapian dispatcher."""
+
+ interface.implements(IDispatcher)
+
+ def __init__(self):
+ self._connections = set()
+
+ def index(self, obj, attributes=None):
+ self._get_connection(obj).add(self._get_document(obj))
+
+ def reindex(self, obj, attributes=None):
+ self._get_connection(obj).replace(self._get_document(obj))
+
+ def unindex(self, obj):
+ self._get_connection(obj).delete(self._get_document(obj).id)
+
+ def flush(self):
+ for conn in self._connections:
+ conn.flush()
+
+ self._connections.clear()
+
+ def _get_document(self, obj):
+ return component.getAdapter(obj, interfaces.IDocument, context=obj)
+
+ def _get_connection(self, obj):
+ conn = component.getUtility(interfaces.IIndexerConnection, context=obj)
+
+ # keep connection to be able to flush it later
+ self._connections.add(conn)
+
+ return conn
Added: z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/interfaces.py
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/interfaces.py (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/interfaces.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,51 @@
+from zope import interface
+
+class IResolver(interface.Interface):
+ """An object resolver.
+
+ Xapian identifies searched documents with an identification
+ string.
+ """
+
+ interface.Attribute(
+ """Protocol identifier.""")
+
+ def getId(obj):
+ """Return an object identification string."""
+
+ def getObject(id):
+ """Return an object given an identification string."""
+
+class IDocument(interface.Interface):
+ """A Xapian indexing document."""
+
+ id = interface.Attribute(
+ """Document id.""")
+
+class IConnectionHub(interface.Interface):
+ """Search connection storage and retrieval.
+
+ Automatic reconnections with connection aging. Connections are all
+ thread local.
+ """
+
+ def invalidate():
+ """Invalidate existing connection."""
+
+ def get():
+ """Retrieves a connection."""
+
+class ISearchConnection(interface.Interface):
+ """A search connection to Xapian."""
+
+class IIndexerConnection(interface.Interface):
+ """An indexer connection to Xapian."""
+
+ def add(document):
+ """Adds document to database."""
+
+ def replace(document):
+ """Replaces existing document."""
+
+ def delete(document_id):
+ """Removed document from database."""
Added: z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/resolver.py
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/resolver.py (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/resolver.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,26 @@
+from zope import interface
+from zope import component
+
+import interfaces
+
+from zope.app.intid.interfaces import IIntIdsQuery
+
+class IntIdResolver(object):
+ interface.implements(interfaces.IResolver)
+
+ protocol = 'intid'
+
+ def getId(self, obj):
+ query = component.getUtility(IIntIdsQuery)
+ return "%s://%s" % (self.protocol, query.getId(obj))
+
+ def getObject(self, id):
+ protocol, intid = id.split('://')
+
+ try:
+ intid = int(intid)
+ except TypeError:
+ raise TypeError("Must be an integer id.")
+
+ query = component.getUtility(IIntIdsQuery)
+ return query.getObject(intid)
Added: z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/tests.py
===================================================================
--- z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/tests.py (rev 0)
+++ z3c.indexing.xapian/trunk/src/z3c/indexing/xapian/tests.py 2008-03-29 11:33:23 UTC (rev 85011)
@@ -0,0 +1,50 @@
+from zope import interface
+from zope import component
+
+import zope.component.testing
+import zope.testing
+
+import unittest
+import shutil
+
+from persistent import Persistent
+
+class Content(Persistent):
+ __parent__ = None
+
+ @property
+ def __name__(self):
+ return self.title
+
+ def __init__(self, **kw):
+ self.__dict__.update(kw)
+
+ def __hash__(self):
+ return hash(self.title)
+
+def setUp(test):
+ zope.component.testing.setUp(test)
+
+def tearDown(test):
+ zope.component.testing.tearDown(test)
+ test.globs['db'].close()
+
+ try:
+ shutil.rmtree('tmp.idx')
+ except OSError:
+ pass
+
+def test_suite():
+ globs = dict(interface=interface, component=component, Content=Content)
+
+ return unittest.TestSuite((
+ zope.testing.doctest.DocFileSuite(
+ 'README.txt',
+ setUp=setUp, tearDown=tearDown,
+ globs=globs,
+ optionflags=zope.testing.doctest.NORMALIZE_WHITESPACE,
+ ),
+ ))
+
+if __name__ == '__main__':
+ unittest.main(defaultTest='test_suite')
More information about the Checkins
mailing list