[Checkins] SVN: zc.relationship/trunk/ Checkpoint on way to 2.0
release. See CHANGES.txt for details.
Gary Poster
gary at zope.com
Tue Jun 19 22:51:29 EDT 2007
Log message for revision 76822:
Checkpoint on way to 2.0 release. See CHANGES.txt for details.
Changed:
D zc.relationship/trunk/CHANGES.txt
A zc.relationship/trunk/CHANGES.txt
U zc.relationship/trunk/buildout.cfg
U zc.relationship/trunk/setup.py
A zc.relationship/trunk/src/zc/relationship/CHANGES.txt
U zc.relationship/trunk/src/zc/relationship/README.txt
U zc.relationship/trunk/src/zc/relationship/index.py
U zc.relationship/trunk/src/zc/relationship/interfaces.py
U zc.relationship/trunk/src/zc/relationship/tests.py
-=-
Deleted: zc.relationship/trunk/CHANGES.txt
===================================================================
--- zc.relationship/trunk/CHANGES.txt 2007-06-20 01:25:11 UTC (rev 76821)
+++ zc.relationship/trunk/CHANGES.txt 2007-06-20 02:51:27 UTC (rev 76822)
@@ -1,39 +0,0 @@
-=======
-Changes
-=======
-
-1.1
-===
-
-(supports Zope 3.4/Zope 2.11/ZODB 3.8, unreleased)
-
-1.1.0 alpha
------------
-
-(unreleased)
-
-- adjust to BTrees changes in ZODB 3.8 (thanks Juergen Kartnaller)
-
-1.0
-===
-
-(supports Zope 3.3/Zope 2.10/ZODB 3.7)
-
-1.0.1
------
-
-- Incorporated test and bug fix from Gabriel Shaar::
-
- if the target parameter is a container with no objects, then
- `shared.AbstractContainer.isLinked` resolves to False in a bool context and
- tokenization fails. `target and tokenize({'target': target})` returns the
- target instead of the result of the tokenize function.
-
-- Made README.txt tests pass on hopefully wider set of machines (this was a
- test improvement; the relationship index did not have the fragility).
- Reported by Gabriel Shaar.
-
-1.0.0
------
-
-Initial release
\ No newline at end of file
Added: zc.relationship/trunk/CHANGES.txt
===================================================================
--- zc.relationship/trunk/CHANGES.txt (rev 0)
+++ zc.relationship/trunk/CHANGES.txt 2007-06-20 02:51:27 UTC (rev 76822)
@@ -0,0 +1 @@
+Please see CHANGES.txt in src/zc/relationship.
Modified: zc.relationship/trunk/buildout.cfg
===================================================================
--- zc.relationship/trunk/buildout.cfg 2007-06-20 01:25:11 UTC (rev 76821)
+++ zc.relationship/trunk/buildout.cfg 2007-06-20 02:51:27 UTC (rev 76822)
@@ -1,15 +1,15 @@
[buildout]
develop = .
-parts = zope3 test
+parts = test py
find-links = http://download.zope.org/distribution/
[test]
recipe = zc.recipe.testrunner
eggs = zc.relationship
-extra-paths = parts/zope3/src
+defaults = "--tests-pattern [fn]?tests --exit-with-status".split()
-[zope3]
-recipe = zc.recipe.zope3checkout
-url = svn://svn.zope.org/repos/main/Zope3/trunk
-
+[py]
+recipe = zc.recipe.egg
+eggs = zc.relationship
+interpreter = py
Modified: zc.relationship/trunk/setup.py
===================================================================
--- zc.relationship/trunk/setup.py 2007-06-20 01:25:11 UTC (rev 76821)
+++ zc.relationship/trunk/setup.py 2007-06-20 02:51:27 UTC (rev 76822)
@@ -2,7 +2,7 @@
setup(
name="zc.relationship",
- version="1.1a",
+ version="1.1a1",
packages=find_packages('src'),
include_package_data=True,
package_dir= {'':'src'},
@@ -14,8 +14,23 @@
author_email='zope3-dev at zope.org',
description=open("README.txt").read(),
long_description=(
- open('CHANGES.txt').read() + '\n========\nOverview\n========\n\n' +
+ open('src/zc/relationship/CHANGES.txt').read() +
+ '\n========\nOverview\n========\n\n' +
open("src/zc/relationship/README.txt").read()),
license='ZPL 2.1',
keywords="zope zope3",
+ install_requires=[
+ 'ZODB3 >= 3.8dev',
+ 'zope.app.container', # would be nice to remove this
+ 'zope.app.intid',
+ 'zope.interface',
+ 'zope.component',
+ 'zope.app.keyreference',
+ 'zope.location',
+ 'zope.index',
+
+ 'zope.app.testing', # TODO remove this
+ 'zope.app.component', # TODO remove this
+ 'zope.testing',
+ ],
)
Copied: zc.relationship/trunk/src/zc/relationship/CHANGES.txt (from rev 76636, zc.relationship/trunk/CHANGES.txt)
===================================================================
--- zc.relationship/trunk/src/zc/relationship/CHANGES.txt (rev 0)
+++ zc.relationship/trunk/src/zc/relationship/CHANGES.txt 2007-06-20 02:51:27 UTC (rev 76822)
@@ -0,0 +1,101 @@
+=======
+Changes
+=======
+
+2.0
+===
+
+supports Zope 3.4/Zope 2.11/ZODB 3.8
+
+New Requirements
+----------------
+
+- ZODB 3.8
+
+Incompatibilities with 1.0
+--------------------------
+
+- `findRelationships` will now use the defaultTransitiveQueriesFactory if it
+ is set.
+
+- `deactivateSets` is no longer an instantiation option (it was broken because
+ of a ZODB bug anyway, as had been described in the documentation).
+
+Changes in 2.0 alpha
+--------------------
+
+- adjust to BTrees changes in ZODB 3.8 (thanks Juergen Kartnaller)
+
+- support both 64-bit and 32-bit BTree families
+
+- support specifying indexed values by passing callables rather than
+ interface elements (which are also still supported).
+
+- in findValues and findValueTokens, `query` argument is now optional. If
+ the query evaluates to False in a boolean context, all values, or value
+ tokens, are returned. Value tokens are explicitly returned using the
+ underlying BTree storage. This can then be used directly for other BTree
+ operations.
+
+ In these and other cases, you should not ever mutate returned results!
+ They may be internal data structures (and are intended to be so, so
+ that they can be used for efficient set operations for other uses).
+ The interfaces hopefully clarify what calls will return an internal
+ data structure.
+
+- README has a new beginning, which both demonstrates some of the new features
+ and tries to be a bit simpler than the later sections.
+
+- `findRelationships` and new method `findRelationshipTokens` can find
+ relationships transitively and intransitively. `findRelationshipTokens`
+ when used intransitively repeats the behavior of `findRelationshipTokenSet`.
+ (`findRelationshipTokenSet` remains in the API, not deprecated, a companion
+ to `findValueTokenSet`.)
+
+- 100% test coverage (per the usual misleading line analysis :-) of index
+ module. (Note that the significantly lower test coverage of the container
+ code is unlikely to change without contributions: I use the index
+ exclusively. See plone.relations for a zc.relationship container with
+ very good test coverage.
+
+- converted buildout to rely exclusively on eggs
+
+TODO in 2.0 alpha
+-----------------
+
+- XXX You can now instantiate the index with a new argument,
+ `transitiveIndexes`. This specifies zero or more indices, each for a single
+ value or for relationships, for a given transitive factory, for unlimited
+ depth queries. These indices will be used transparently, if appropriate,
+ in calls to find Values, findValueTokens, findRelationships, and
+ findRelationshipTokens. In order to support this, transitive query
+ factories must implement a new interface, IXXX, that allows them to be
+ compared for equality.
+
+- XXX make it possible to have a query that takes multiple tokens for a given
+ argument, with the semantics of a union of the results. PROBABLY WON'T
+ HAPPEN FOR 2.0; SPELLING UNCLEAR
+
+1.0
+===
+
+(supports Zope 3.3/Zope 2.10/ZODB 3.7)
+
+1.0.1
+-----
+
+- Incorporated test and bug fix from Gabriel Shaar::
+
+ if the target parameter is a container with no objects, then
+ `shared.AbstractContainer.isLinked` resolves to False in a bool context and
+ tokenization fails. `target and tokenize({'target': target})` returns the
+ target instead of the result of the tokenize function.
+
+- Made README.txt tests pass on hopefully wider set of machines (this was a
+ test improvement; the relationship index did not have the fragility).
+ Reported by Gabriel Shaar.
+
+1.0.0
+-----
+
+Initial release
Modified: zc.relationship/trunk/src/zc/relationship/README.txt
===================================================================
--- zc.relationship/trunk/src/zc/relationship/README.txt 2007-06-20 01:25:11 UTC (rev 76821)
+++ zc.relationship/trunk/src/zc/relationship/README.txt 2007-06-20 02:51:27 UTC (rev 76822)
@@ -1,9 +1,11 @@
-The Relationship package currently contains two main types of components: a
-relationship index, and some relationship containers. Both are designed for
-use within the ZODB. They share the model that relationships are full-fledged
-objects that are indexed for optimized searches. They also share the ability
-to perform optimized intransitive and transitive relationship searches, and
-to support arbitrary filter searches on relationship tokens.
+The zc.relationship package currently contains two main types of
+components: a relationship index, and some relationship containers.
+Both are designed to be used within the ZODB, although the index is
+flexible enough to be used in other contexts. They share the model that
+relationships are full-fledged objects that are indexed for optimized
+searches. They also share the ability to perform optimized intransitive
+and transitive relationship searches, and to support arbitrary filter
+searches on relationship tokens.
The index is a very generic component that can be used to optimize searches
for N-ary relationships, can be used standalone or within a catalog, can be
@@ -13,26 +15,278 @@
The relationship containers use the index to manage two-way
relationships, using a derived mapping interface. It is a reasonable
-example of the index in standalone use.
+example of the index in standalone use.
-This document describes the relationship index. See container.txt for
-documentation of the relationship container.
+Another example, using the container model but supporting five-way
+relationships ("sources", "targets", "relation", "getContext", "state"), can
+be found in plone.relations. Its README is a good read.
+http://dev.plone.org/plone/browser/plone.relations/trunk/plone/relations
+
+This current document describes the relationship index. See
+container.txt for documentation of the relationship container.
+
=====
Index
=====
-The index interface searches for object and relationship tokens. To use a
-relationship index, you need to have interface attributes, or methods callable
-with no arguments, that are treated as relationship pointers. The pointers
-may be a collection of items or a single item.
+Overview
+========
-To exercise the index, we'll come up with a somewhat complex relationship to
-index. Let's say we are modeling a generic set-up like SUBJECT
-RELATIONSHIPTYPE OBJECT in CONTEXT. This could let you let users define
-relationship types, then index them on the fly. The context can be something
-like a project, so we could say
+The index takes a very precise view of the world: instantiation requires
+multiple arguments specifying the configuration; and using the index
+requires that you acknowledge that the relationships and their
+associated indexed values are usually tokenized within the index. This
+precision trades some ease-of-use for the possibility of flexibility,
+power, and efficiency. That said, the index's API is intended to be
+consistent, and to largely adhere to "there's only one way to do it"
+[#apply]_.
+Simplest Example
+----------------
+
+Before diving into the N-way flexibility and the other more complex
+bits, then, let's have a quick basic demonstration: a two way
+relationship from one value to another. This will give you a taste of
+the relationship index, and let you use it reasonably well for
+light-to-medium usage. If you are going to use more of its features or
+use it more in a potentially high-volume capacity, please consider
+trying to understand the entire document.
+
+Let's say that we are modeling a relationship of people to their
+supervisors: an employee may have a single supervisor.
+
+Let's say further that employee names are unique and can be used to
+represent employees. We can use names as our "tokens". Tokens are
+similar to the primary key in a relational database, or in intid or
+keyreference in Zope 3--some way to uniquely identify an object, which
+sorts reliably and can be resolved to the object given the right context.
+
+ >>> employees = {} # we'll use this to resolve the "name" tokens
+ >>> class Employee(object):
+ ... def __init__(self, name, supervisor=None):
+ ... if name in employees:
+ ... raise ValueError('employee with same name already exists')
+ ... self.name = name # expect this to be readonly
+ ... self.supervisor = supervisor
+ ... employees[name] = self
+ ... def __repr__(self): # to make the tests prettier...
+ ... return '<' + self.name + '>'
+ ... def __cmp__(self, other): # to make the tests prettier...
+ ... # pukes if other doesn't have name
+ ... return cmp(self.name, other.name)
+ ...
+
+So, we need to define how to turn employees into their tokens. That's
+trivial. (We explain the arguments to this function in detail below,
+but for now we're aiming for "breezy overview".)
+
+ >>> def dumpEmployees(emp, index, cache):
+ ... return emp.name
+ ...
+
+We also need a way to turn tokens into employees. We use our dict for that.
+
+ >>> def loadEmployees(token, index, cache):
+ ... return employees[token]
+ ...
+
+We also need a way to tell the index to find the supervisor for indexing:
+
+ >>> def supervisor(emp, index):
+ ... return emp.supervisor # None or another employee
+ ...
+
+Now we have enough to get started with an index. The first argument to
+Index is the attributes to index: we pass the `supervisor` function
+(which is also used in this case to define the index's name, since we do
+not pass one explicitly), the dump and load functions, and a BTree
+module that specifies sets that can hold our tokens (OO or OL should
+also work). As keyword arguments, we tell the index how to dump and
+load our relationship tokens--the same functions in this case--and what
+a reasonable BTree module is for sets (again, we choose OI, but OO or OL
+should work).
+
+ >>> from zc.relationship import index
+ >>> import BTrees
+ >>> ix = index.Index(
+ ... ({'callable': supervisor, 'dump': dumpEmployees,
+ ... 'load': loadEmployees, 'btree': BTrees.family32.OI},),
+ ... dumpRel=dumpEmployees, loadRel=loadEmployees,
+ ... relFamily=BTrees.family32.OI)
+
+Now let's create a few employees.
+
+ >>> a = Employee('Alice')
+ >>> b = Employee('Betty', a)
+ >>> c = Employee('Chuck', a)
+ >>> d = Employee('Duane', b)
+ >>> e = Employee('Edgar', b)
+ >>> f = Employee('Frank', c)
+ >>> g = Employee('Grant', c)
+ >>> h = Employee('Howie', d)
+
+In a diagram style with which you will become familiar if you make it to
+the end of this document, let's show the hierarchy.
+
+::
+
+ Alice
+ __/ \__
+ Betty Chuck
+ / \ / \
+ Duane Edgar Frank Grant
+ |
+ Howie
+
+So who works for Alice? To ask the index, we need to tell it about them.
+
+ >>> for emp in (a,b,c,d,e,f,g,h):
+ ... ix.index(emp)
+ ...
+
+Now we can ask. We always need to ask with tokens. The index provides
+a method to try and make this more convenient: `tokenizeQuery`
+[#resolveQuery]_. The spelling of the query is described in more detail
+later, but the idea is simply that keys in a dictionary specify
+attribute names, and the values specify the constraints.
+
+ >>> t = ix.tokenizeQuery
+ >>> sorted(ix.findRelationshipTokens(t({'supervisor': a})))
+ ['Betty', 'Chuck']
+ >>> sorted(ix.findRelationships(t({'supervisor': a})))
+ [<Betty>, <Chuck>]
+
+How do we find what the employee's supervisor is? Well, in this case,
+look at the attribute! If you can use an attribute that will usually be
+a win in the ZODB. If you want to look at the data in the index,
+though, that's easy enough. Who is Howie's supervisor? The None key in
+the query indicates that we are matching against the relationship token
+itself [#None_details]_.
+
+ >>> h.supervisor
+ <Duane>
+ >>> list(ix.findValueTokens('supervisor', t({None: h})))
+ ['Duane']
+ >>> list(ix.findValues('supervisor', t({None: h})))
+ [<Duane>]
+
+What about transitive searching? Well, you need to tell the index how to
+walk the tree. In simple cases like this, the index's
+TransposingTransitiveQueriesFactory will do the trick. We just want to tell
+the factory to transpose the two keys, None and 'supervisor'. We can then use
+it in queries for transitive searches.
+
+ >>> factory = index.TransposingTransitiveQueriesFactory(None, 'supervisor')
+
+Who are all of Howie's supervisors transitively (this looks up in the
+diagram)?
+
+ >>> list(ix.findValueTokens('supervisor', t({None: h}),
+ ... transitiveQueriesFactory=factory))
+ ['Duane', 'Betty', 'Alice']
+ >>> list(ix.findValues('supervisor', t({None: h}),
+ ... transitiveQueriesFactory=factory))
+ [<Duane>, <Betty>, <Alice>]
+
+Who are all of the people Betty supervises transitively, breadth first (this
+looks down in the diagram)?
+
+ >>> people = list(ix.findRelationshipTokens(
+ ... t({'supervisor': b}), transitiveQueriesFactory=factory))
+ >>> sorted(people[:2])
+ ['Duane', 'Edgar']
+ >>> people[2]
+ 'Howie'
+ >>> people = list(ix.findRelationships(
+ ... t({'supervisor': b}), transitiveQueriesFactory=factory))
+ >>> sorted(people[:2])
+ [<Duane>, <Edgar>]
+ >>> people[2]
+ <Howie>
+
+This transitive search is really the only transitive factory you would want
+here, so it probably is safe to wire it in as a default. While most
+attributes on the index must be set at instantiation, this happens to be one
+we can set after the fact.
+
+ >>> ix.defaultTransitiveQueriesFactory = factory
+
+Now all searches are transitive.
+
+ >>> list(ix.findValueTokens('supervisor', t({None: h})))
+ ['Duane', 'Betty', 'Alice']
+ >>> list(ix.findValues('supervisor', t({None: h})))
+ [<Duane>, <Betty>, <Alice>]
+ >>> people = list(ix.findRelationshipTokens(t({'supervisor': b})))
+ >>> sorted(people[:2])
+ ['Duane', 'Edgar']
+ >>> people[2]
+ 'Howie'
+ >>> people = list(ix.findRelationships(t({'supervisor': b})))
+ >>> sorted(people[:2])
+ [<Duane>, <Edgar>]
+ >>> people[2]
+ <Howie>
+
+We can force a non-transitive search, or a specific search depth, with
+maxDepth [#needs_a_transitive_queries_factory]_.
+
+ >>> list(ix.findValueTokens('supervisor', t({None: h}), maxDepth=1))
+ ['Duane']
+ >>> list(ix.findValues('supervisor', t({None: h}), maxDepth=1))
+ [<Duane>]
+ >>> sorted(ix.findRelationshipTokens(t({'supervisor': b}), maxDepth=1))
+ ['Duane', 'Edgar']
+ >>> sorted(ix.findRelationships(t({'supervisor': b}), maxDepth=1))
+ [<Duane>, <Edgar>]
+
+Transitive searches can handle recursive loops and have other features as
+discussed in the larger example and the interface.
+
+Our last two introductory examples show off three other methods: `isLinked`
+`findRelationshipTokenChains` and `findRelationshipChains`.
+
+isLinked lets you answer whether two queries are linked. Is Alice a
+supervisor of Howie? What about Chuck? (Note that, if your
+relationships describe a hierarchy, searching up a hierarchy is usually
+more efficient, so the second pair of questions is generally preferable
+to the first in that case.)
+
+ >>> ix.isLinked(t({'supervisor': a}), targetQuery=t({None: h}))
+ True
+ >>> ix.isLinked(t({'supervisor': c}), targetQuery=t({None: h}))
+ False
+ >>> ix.isLinked(t({None: h}), targetQuery=t({'supervisor': a}))
+ True
+ >>> ix.isLinked(t({None: h}), targetQuery=t({'supervisor': c}))
+ False
+
+`findRelationshipTokenChains` and `findRelationshipChains` help you discover
+*how* things are transitively related. A "chain" is a transitive path of
+relationships. For instance, what's the chain of command between Alice and
+Howie?
+
+ >>> list(ix.findRelationshipTokenChains(
+ ... t({'supervisor': a}), targetQuery=t({None: h})))
+ [('Betty', 'Duane', 'Howie')]
+ >>> list(ix.findRelationshipChains(
+ ... t({'supervisor': a}), targetQuery=t({None: h})))
+ [(<Betty>, <Duane>, <Howie>)]
+
+This gives you a quick overview of the basic index features. This should be
+enough to get you going. Now we'll dig in some more, if you want to know the
+details.
+
+Starting the N-Way Examples
+===========================
+
+To exercise the index further, we'll come up with a somewhat complex
+relationship to index. Let's say we are modeling a generic set-up like
+SUBJECT RELATIONSHIPTYPE OBJECT in CONTEXT. This could let you let
+users define relationship types, then index them on the fly. The
+context can be something like a project, so we could say
+
"Fred" "has the role of" "Project Manager" on the "zope.org redesign project".
Mapped to the parts of the relationship object, that's
@@ -44,8 +298,9 @@
["Ygritte" (SUBJECT)] ["manages" (RELATIONSHIPTYPE)] ["Uther" (OBJECT)]
-So let's define a basic interface without the context, and then an extended
-interface with the context.
+In our new example, we'll leverage the fact that the index can accept
+interface attributes to index. So let's define a basic interface
+without the context, and then an extended interface with the context.
>>> from zope import interface
>>> class IRelationship(interface.Interface):
@@ -63,16 +318,20 @@
... '''return a context for the relationship'''
...
-Now we'll create an index. To do that, we must minimally pass in an iterable
-describing the indexed values. Each item in the iterable must either be an
-interface element (a zope.interface.Attribute or zope.interface.Method
-associated with an interface, typically obtained using a spelling like
-`IRelationship['subjects']`) or a dict. Each dict must have at least one key:
-'element', which is the interface element to be indexed. It then can contain
-other keys to override the default indexing behavior for the element.
+Now we'll create an index. To do that, we must minimally pass in an
+iterable describing the indexed values. Each item in the iterable must
+either be an interface element (a zope.interface.Attribute or
+zope.interface.Method associated with an interface, typically obtained
+using a spelling like `IRelationship['subjects']`) or a dict. Each dict
+must have either the 'element' key, which is the interface element to be
+indexed; or the 'callable' key, which is the callable shown in the
+simpler, introductory example above [#there_can_be_only_one]_. It then
+can contain other keys to override the default indexing behavior for the
+element.
-The element's __name__ will be used to refer to this element in queries, unless
-the dict has a 'name' key, which must be a non-empty string.
+The element's or callable's __name__ will be used to refer to this
+element in queries, unless the dict has a 'name' key, which must be a
+non-empty string [#name_errors]_.
The element is assumed to be a single value, unless the dict has a 'multiple'
key with a value equivalent True. In our example, "subjects" and "objects" are
@@ -108,6 +367,11 @@
also may return the same value as the other tokenizers to mean different
objects: the stores are separate.
+Note that both dump and load may also be explicitly None in the dictionary:
+this will mean that the values are already appropriate to be used as tokens.
+It enables an optimization described in the
+`Optimizing relationship index use`_ section [#neither_or_both]_.
+
In addition to the one required argument to the class, the signature contains
four optional arguments. The 'defaultTransitiveQueriesFactory' is the next,
and allows you to specify a callable as described in
@@ -121,7 +385,7 @@
manages Emily (OBJECT), a search for all those transitively managed by Ygritte
will transpose Uther from OBJECT to SUBJECT and find that Uther manages Emily.
Similarly, to find all transitive managers of Emily, Uther will change place
-from SUBJECT to OBJECT in the search.
+from SUBJECT to OBJECT in the search [#TransposingTransitiveQueriesFactory]_.
The next three arguments, 'dumpRel', 'loadRel' and 'relFamily', have
to do with the relationship tokens. The default values assume that you will
@@ -140,21 +404,14 @@
If you are unable or unwilling to use intid relationship tokens, tokens must
still be homogenous and immutable as described above for indexed values tokens.
-The last argument is 'deactivateSets', which defaults to False. This is an
-optimization to try and keep relationship index searches from consuming too
-much of the ZODB's object cache. It can cause inefficiency under some
-usages--if queries against the relationship index are very frequent and often
-use the same sets, for instance--and it exposes a bug in the ZODB at the time
-of this writing (_p_deactivate on a new object that has been given an _p_oid
-but has not yet been committed will irretrievably snuff out the object before
-it has had a chance to be committed, so if you add and query in the same
-transaction you will have trouble). Pass True to this argument to enable
-this optimization.
+The last argument is 'family', which effectively defaults to BTrees.family32.
+If you don't expicitly specify BTree modules for your value and relationship
+sets, this value will determine whether you use the 32 bit or the 64 bit
+IFBTrees [#family64]_.
If we had an IIntId utility registered and wanted to use the defaults, then
instantiation of an index for our relationship would look like this:
- >>> from zc.relationship import index
>>> ix = index.Index(
... ({'element': IRelationship['subjects'], 'multiple': True},
... IRelationship['relationshiptype'],
@@ -194,8 +451,7 @@
We will also use the intid utility to resolve relationship tokens. See the
relationship container (and container.txt) for examples of changing the
-relationship type, especially in keyref.py. The example also turns on the
-'deactivateSets' optimization.
+relationship type, especially in keyref.py.
Here are the methods we'll use for the 'subjects' and 'objects' tokens,
followed by the methods we'll use for the 'relationshiptypes' tokens.
@@ -231,7 +487,7 @@
...
>>> def relTypeLoad(token, index, cache):
... assert token in relTypes, 'unknown relationshiptype'
- ... return obj
+ ... return token
...
Note that these implementations are completely silly if we actually cared about
@@ -472,13 +728,14 @@
objects to a dict with tokens. You'll see below that we shorten our calls by
stashing `tokenizeQuery` away in the 'q' name.
+ >>> q = ix.tokenizeQuery
+
We have indexed our first example relationship--"Fred has the role of project
manager in the zope.org redesign"--so we can search for it. We'll first look
at `findValues` and `findValueTokens`. Here, we ask 'who has the role of
project manager in the zope.org redesign?'. We do it first with findValues
-and then with findTokenValues.
+and then with findValueTokens [#findValue_errors]_.
- >>> q = ix.tokenizeQuery
>>> list(ix.findValues(
... 'subjects',
... q({'reltype': 'has the role of',
@@ -493,6 +750,18 @@
... 'context': projects['zope.org redesign']}))]
[<Person 'Fred'>]
+If you don't pass a query to these methods, you get all indexed values for the
+given name in a BTree (don't modify this! this is an internal data structure--
+we pass it out directly because you can do efficient things with it with BTree
+set operations). In this case, we've only indexed a single relationship,
+so its subjects are the subjects in this result.
+
+ >>> res = ix.findValueTokens('subjects', maxDepth=1)
+ >>> res # doctest: +ELLIPSIS
+ <BTrees.IOBTree.IOBTree object at ...>
+ >>> [load(t, ix, {}) for t in res]
+ [<Person 'Fred'>]
+
If we want to find all the relationships for which Fred is a subject, we can
use `findRelationshipTokenSet`. It, combined with `findValueTokenSet`, is
useful for querying the index data structures at a fairly low level, when you
@@ -508,21 +777,69 @@
>>> [intids.getObject(t) for t in res]
[<(<Person 'Fred'>,) has the role of (<Role 'Project Manager'>,)>]
-`findRelationships` does the same thing but with resolving the relationships.
+It is in fact equivalent to `findRelationshipTokens` called without
+transitivity and without any filtering.
+ >>> res2 = ix.findRelationshipTokens(
+ ... q({'subjects': people['Fred']}), maxDepth=1)
+ >>> res2 is res
+ True
+
+The `findRelationshipTokenSet` method always returns a set, even if the
+query does not have any results.
+
+ >>> res = ix.findRelationshipTokenSet(q({'subjects': people['Ygritte']}))
+ >>> res # doctest: +ELLIPSIS
+ <BTrees.IFBTree.IFTreeSet object at ...>
+ >>> list(res)
+ []
+
+An empty query returns all relationships in the index (this is true of other
+search methods as well).
+
+ >>> res = ix.findRelationshipTokenSet({})
+ >>> res # doctest: +ELLIPSIS
+ <BTrees.IFBTree.IFTreeSet object at ...>
+ >>> len(res) == ix.documentCount()
+ True
+ >>> for r in ix.resolveRelationshipTokens(res):
+ ... if r not in ix:
+ ... print 'oops'
+ ... break
+ ... else:
+ ... print 'correct'
+ ...
+ correct
+
+`findRelationships` can do the same thing but with resolving the relationships.
+
>>> list(ix.findRelationships(q({'subjects': people['Fred']})))
[<(<Person 'Fred'>,) has the role of (<Role 'Project Manager'>,)>]
+
+However, like `findRelationshipTokens` and unlike
+`findRelationshipTokenSet`, `findRelationships` can be used
+transitively, as shown in the introductory section of this document.
`findValueTokenSet`, given a relationship token and a value name, returns a
set (based on the btree family for the value) of value tokens for that
relationship.
- >>> res = ix.findValueTokenSet(list(res)[0], 'subjects')
+ >>> src = ix.findRelationshipTokenSet(q({'subjects': people['Fred']}))
+
+ >>> res = ix.findValueTokenSet(list(src)[0], 'subjects')
>>> res # doctest: +ELLIPSIS
<BTrees.IFBTree.IFTreeSet object at ...>
>>> [load(t, ix, {}) for t in res]
[<Person 'Fred'>]
+Like `findRelationshipTokenSet` and `findRelationshipTokens`,
+`findValueTokenSet` is equivalent to `findValueTokens` without a
+transitive search or filtering.
+
+ >>> res2 = ix.findValueTokenSet(list(src)[0], 'subjects')
+ >>> res2 is res
+ True
+
The apply method, part of the zope.index.interfaces.IIndexSearch interface,
can essentially only duplicate the `findValueTokens` and
`findRelationshipTokenSet` search calls. The only additional functionality
@@ -551,14 +868,13 @@
>>> res = ix.apply({'relationships':
... q({'reltype': 'has the role of'})})
- ... doctest: +ELLIPSIS
>>> res # doctest: +ELLIPSIS
<BTrees.IFBTree.IFTreeSet object at ...>
>>> [intids.getObject(t) for t in res]
[<(<Person 'Fred'>,) has the role of (<Role 'Project Manager'>,)>]
-Here, we ask for the known relationships for the zope.org redesign. It
-will fail, because the result cannot be expressed as an IFBTree.IFTreeSet
+Here, we ask for the known relationships types for the zope.org redesign. It
+will fail, because the result cannot be expressed as an IFBTree.IFTreeSet.
>>> res = ix.apply({'values':
... {'resultName': 'reltype', 'query':
@@ -567,8 +883,11 @@
Traceback (most recent call last):
...
ValueError: cannot fulfill `apply` interface because cannot return an
- IFBTree-based result
+ (I|L)FBTree-based result
+The same kind of error will be raised if you request relationships and the
+relationships are not stored in IFBTree or LFBTree structures [#apply_errors]_.
+
The last basic search methods, `isLinked`, `findRelationshipTokenChains`, and
`findRelationshipChains`, are most useful for transitive searches. We
have not yet created any relationships that we can use transitively. They
@@ -896,7 +1215,7 @@
targetFilter can be used for many tasks, such as only returning values that
are in specially annotated relationships, or only returning values that have
traversed a certain hinge relationship in a two-part search, or other tasks.
-A very simply one, though, is to effectively specify a minimum traversal depth.
+A very simple one, though, is to effectively specify a minimum traversal depth.
Here, we find the people who are precisely two steps down from Bran, no more
and no less. We do it twice, once with findValueTokens and once with
findValues.
@@ -1119,6 +1438,22 @@
... targetQuery=q({'objects': companies['Zookd']}))))
0
+You can combine targetQuery with targetFilter. Here we arbitrarily say we
+are looking for a path between Rob and Ygritte that is at least 3 links long.
+
+ >>> res = [repr([intids.getObject(t) for t in path]) for path in
+ ... ix.findRelationshipTokenChains(
+ ... q({'reltype': 'manages', 'subjects': people['Rob']}),
+ ... targetQuery=q({'objects': people['Ygritte']}),
+ ... targetFilter=lambda relchain, q, i, c: len(relchain)>=3)]
+ >>> len(res)
+ 1
+ >>> res # doctest: +NORMALIZE_WHITESPACE
+ ["[<(<Person 'Rob'>,) manages
+ (<Person 'Sam'>, <Person 'Terry'>, <Person 'Uther'>)>,
+ <(<Person 'Terry'>,) manages (<Person 'Xen'>,)>,
+ <(<Person 'Uther'>, <Person 'Xen'>) manages (<Person 'Ygritte'>,)>]"]
+
`isLinked` takes the same arguments as all of the other transitive-aware
methods. For instance, Rob and Ygritte are transitively linked, but Abe and
Zane are not.
@@ -1170,13 +1505,43 @@
8
>>> interfaces.ICircularRelationshipPath.providedBy(res[7])
True
- >>> [sorted(
- ... (nm, nm == 'objects' and load(t, ix, {}) or t)
- ... for nm, t in search.items()) for search in res[7].cycled]
+ >>> [sorted(ix.resolveQuery(search).items()) for search in res[7].cycled]
+ [[('objects', <Person 'Abe'>), ('reltype', 'manages')]]
+ >>> tuple(ix.resolveRelationshipTokens(res[7]))
... # doctest: +NORMALIZE_WHITESPACE
- [[('objects', <Person 'Abe'>),
- ('reltype', 'manages')]]
+ (<(<Person 'Heather'>,) manages (<Person 'Ingrid'>,)>,
+ <(<Person 'David'>,) manages (<Person 'Heather'>,)>,
+ <(<Person 'Bran'>,) manages (<Person 'David'>,)>,
+ <(<Person 'Abe'>,) manages (<Person 'Bran'>,)>,
+ <(<Person 'Gary'>,) manages (<Person 'Abe'>,)>,
+ <(<Person 'Fred'>,) manages (<Person 'Gary'>,)>,
+ <(<Person 'Cathy'>,) manages (<Person 'Fred'>,)>,
+ <(<Person 'Abe'>,) manages (<Person 'Cathy'>,)>)
+The same kind of thing works for `findRelationshipChains`. Notice that the
+query in the .cycled attribute is not resolved: it is still the query that
+would be needed to continue the cycle.
+
+ >>> res = list(ix.findRelationshipChains(
+ ... q({'objects': people['Ingrid'], 'reltype': 'manages'})))
+ >>> len(res)
+ 8
+ >>> len(res[7])
+ 8
+ >>> interfaces.ICircularRelationshipPath.providedBy(res[7])
+ True
+ >>> [sorted(ix.resolveQuery(search).items()) for search in res[7].cycled]
+ [[('objects', <Person 'Abe'>), ('reltype', 'manages')]]
+ >>> res[7] # doctest: +NORMALIZE_WHITESPACE
+ cycle(<(<Person 'Heather'>,) manages (<Person 'Ingrid'>,)>,
+ <(<Person 'David'>,) manages (<Person 'Heather'>,)>,
+ <(<Person 'Bran'>,) manages (<Person 'David'>,)>,
+ <(<Person 'Abe'>,) manages (<Person 'Bran'>,)>,
+ <(<Person 'Gary'>,) manages (<Person 'Abe'>,)>,
+ <(<Person 'Fred'>,) manages (<Person 'Gary'>,)>,
+ <(<Person 'Cathy'>,) manages (<Person 'Fred'>,)>,
+ <(<Person 'Abe'>,) manages (<Person 'Cathy'>,)>)
+
Notice that there is nothing special about the new relationship, by the way.
If we had started to look for Fred's supervisors, the cycle marker would have
been given for the relationship that points back to Fred as a supervisor to
@@ -1386,9 +1751,49 @@
>>> list(ix.findValueTokens('objects', {'subjects': 1}))
[2]
-Remember that BTrees (not just BTreeSets) can be used for these values:
-the keys are used as the set members in that case.
+Reindexing is where some of the big improvements can happen. The following
+gyrations exercise the optimization code.
+ >>> rel.subjects = IFBTree.IFTreeSet((3,4,5))
+ >>> ix.index(rel)
+ >>> list(ix.findValueTokens('objects', {'subjects': 3}))
+ [2]
+
+ >>> rel.subjects.insert(6)
+ 1
+ >>> ix.index(rel)
+ >>> list(ix.findValueTokens('objects', {'subjects': 6}))
+ [2]
+
+ >>> rel.subjects.update(range(100, 200))
+ 100
+ >>> ix.index(rel)
+ >>> list(ix.findValueTokens('objects', {'subjects': 100}))
+ [2]
+
+ >>> rel.subjects = IFBTree.IFTreeSet((3,4,5,6))
+ >>> ix.index(rel)
+ >>> list(ix.findValueTokens('objects', {'subjects': 3}))
+ [2]
+
+ >>> rel.subjects = IFBTree.IFTreeSet(())
+ >>> ix.index(rel)
+ >>> list(ix.findValueTokens('objects', {'subjects': 3}))
+ []
+
+ >>> rel.subjects = IFBTree.IFTreeSet((3,4,5))
+ >>> ix.index(rel)
+ >>> list(ix.findValueTokens('objects', {'subjects': 3}))
+ [2]
+
+tokenizeValues and resolveValueTokens work correctly without loaders and
+dumpers--that is, they do nothing.
+
+ >>> ix.tokenizeValues((3,4,5), 'subjects')
+ (3, 4, 5)
+ >>> ix.resolveValueTokens((3,4,5), 'subjects')
+ (3, 4, 5)
+
__contains__ and Unindexing
=============================
@@ -1451,3 +1856,325 @@
>>> ix.unindex(app['abeAndBran'])
>>> ix.unindex_doc(ix.tokenizeRelationship(app['abeAndBran']))
+
+
+
+..[#apply] `apply` and the other zope.index-related methods are the obvious
+ exceptions.
+
+..[#resolveQuery] You can also resolve queries.
+
+ >>> ix.resolveQuery({None: 'Alice'})
+ {None: <Alice>}
+ >>> ix.resolveQuery({'supervisor': 'Alice'})
+ {'supervisor': <Alice>}
+
+..[#None_details] You can search for relations that haven't been indexed.
+
+ >>> list(ix.findRelationshipTokens({None: 'Ygritte'}))
+ []
+
+ You can also combine searches with None, just for completeness.
+
+ >>> list(ix.findRelationshipTokens({None: 'Alice', 'supervisor': None}))
+ ['Alice']
+ >>> list(ix.findRelationshipTokens({None: 'Alice', 'supervisor': 'Betty'}))
+ []
+ >>> list(ix.findRelationshipTokens({None: 'Betty', 'supervisor': 'Alice'}))
+ ['Betty']
+
+..[#needs_a_transitive_queries_factory] A search with a maxDepth > 1 but
+ no transitiveQueriesFactory raises an error.
+
+ >>> ix.defaultTransitiveQueriesFactory = None
+ >>> ix.findRelationshipTokens({'supervisor': 'Duane'}, maxDepth=3)
+ Traceback (most recent call last):
+ ...
+ ValueError: if maxDepth != 1, transitiveQueriesFactory must be available
+
+ >>> ix.defaultTransitiveQueriesFactory = factory
+
+..[#there_can_be_only_one] instantiating an index with a dictionary containing
+ both the 'element' and the 'callable' key is an error:
+
+ >>> def subjects(obj, index, cache):
+ ... return obj.subjects
+ ...
+ >>> ix = index.Index(
+ ... ({'element': IRelationship['subjects'],
+ ... 'callable': subjects, 'multiple': True},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ Traceback (most recent call last):
+ ...
+ ValueError: cannot provide both callable and element
+
+ While we're at it, as you might expect, you must provide one of them.
+
+ >>> ix = index.Index(
+ ... ({'multiple': True},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ Traceback (most recent call last):
+ ...
+ ValueError: must provide element or callable
+
+..[#name_errors] It's possible to pass a callable without a name, in which
+ case you must explicitly specify a name.
+
+ >>> class AttrGetter(object):
+ ... def __init__(self, attr):
+ ... self.attr = attr
+ ... def __call__(self, obj, index, cache):
+ ... return getattr(obj, self.attr, None)
+ ...
+ >>> subjects = AttrGetter('subjects')
+ >>> ix = index.Index(
+ ... ({'callable': subjects, 'multiple': True},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ Traceback (most recent call last):
+ ...
+ ValueError: no name specified
+ >>> ix = index.Index(
+ ... ({'callable': subjects, 'multiple': True, 'name': subjects},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+
+ It's also an error to specify the same name or element twice,
+ however you do it.
+
+ >>> ix = index.Index(
+ ... ({'callable': subjects, 'multiple': True, 'name': 'objects'},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ ... # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Duplicate in attrs', 'objects', <...Attribute ...>)
+
+ >>> ix = index.Index(
+ ... ({'callable': subjects, 'multiple': True, 'name': 'subjects'},
+ ... IRelationship['relationshiptype'],
+ ... {'callable': subjects, 'multiple': True, 'name': 'objects'},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ ... # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Duplicate in attrs', 'objects', <...AttrGetter ...>)
+
+ >>> ix = index.Index(
+ ... ({'element': IRelationship['objects'], 'multiple': True,
+ ... 'name': 'subjects'},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ ... # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ValueError: ('Duplicate in attrs', 'objects', <...Attribute ...>)
+
+..[#neither_or_both] It is not allowed to provide only one or the other of
+ 'load' and 'dump'.
+
+ >>> ix = index.Index(
+ ... ({'element': IRelationship['objects'], 'multiple': True,
+ ... 'name': 'subjects','dump': None},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ ... # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ValueError: either both of 'dump' and 'load' must be None, or neither
+
+ >>> ix = index.Index(
+ ... ({'element': IRelationship['objects'], 'multiple': True,
+ ... 'name': 'subjects','load': None},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ ... # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ValueError: either both of 'dump' and 'load' must be None, or neither
+
+..[#TransposingTransitiveQueriesFactory] The factory lets you specify two
+ names, which are transposed for transitive walks. This is usually what
+ you want for a hierarchy and similar variations: as the text describes
+ later, more complicated traversal might be desired in more complicated
+ relationships, as found in genealogy.
+
+ It supports both transposing values and relationship tokens, as seen in
+ the text.
+
+ In this footnote, we'll explore the factory in the small, with index
+ stubs.
+
+ >>> factory = index.TransposingTransitiveQueriesFactory(
+ ... 'subjects', 'objects')
+ >>> class StubIndex(object):
+ ... def findValueTokenSet(self, rel, name):
+ ... return {
+ ... ('foo', 'objects'): ('bar',),
+ ... ('bar', 'subjects'): ('foo',)}[(rel, name)]
+ ...
+ >>> ix = StubIndex()
+ >>> list(factory(['foo'], {'subjects': 'foo'}, ix, {}))
+ [{'subjects': 'bar'}]
+ >>> list(factory(['bar'], {'objects': 'bar'}, ix, {}))
+ [{'objects': 'foo'}]
+
+ If you specify both fields then it won't transpose.
+
+ >>> list(factory(['foo'], {'objects': 'bar', 'subjects': 'foo'}, ix, {}))
+ []
+
+ If you specify additional fields then it keeps them statically.
+
+ >>> list(factory(['foo'], {'subjects': 'foo', 'getContext': 'shazam'},
+ ... ix, {})) == [{'subjects': 'bar', 'getContext': 'shazam'}]
+ True
+
+..[#family64] Here's an example of specifying the family64. This is a "white
+ box" demonstration that looks at some of the internals.
+
+ >>> ix = index.Index( # 32 bit default
+ ... ({'element': IRelationship['subjects'], 'multiple': True},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'))
+ >>> ix._relTools['BTree'] is BTrees.family32.IF.BTree
+ True
+ >>> ix._attrs['subjects']['BTree'] is BTrees.family32.IF.BTree
+ True
+ >>> ix._attrs['objects']['BTree'] is BTrees.family32.IF.BTree
+ True
+ >>> ix._attrs['getContext']['BTree'] is BTrees.family32.IF.BTree
+ True
+
+ >>> ix = index.Index( # explicit 32 bit
+ ... ({'element': IRelationship['subjects'], 'multiple': True},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'),
+ ... family=BTrees.family32)
+ >>> ix._relTools['BTree'] is BTrees.family32.IF.BTree
+ True
+ >>> ix._attrs['subjects']['BTree'] is BTrees.family32.IF.BTree
+ True
+ >>> ix._attrs['objects']['BTree'] is BTrees.family32.IF.BTree
+ True
+ >>> ix._attrs['getContext']['BTree'] is BTrees.family32.IF.BTree
+ True
+
+ >>> ix = index.Index( # explicit 64 bit
+ ... ({'element': IRelationship['subjects'], 'multiple': True},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'),
+ ... family=BTrees.family64)
+ >>> ix._relTools['BTree'] is BTrees.family64.IF.BTree
+ True
+ >>> ix._attrs['subjects']['BTree'] is BTrees.family64.IF.BTree
+ True
+ >>> ix._attrs['objects']['BTree'] is BTrees.family64.IF.BTree
+ True
+ >>> ix._attrs['getContext']['BTree'] is BTrees.family64.IF.BTree
+ True
+
+..[#findValue_errors] `findValueTokens` and `findValues` raise errors if
+ you try to get a value that is not indexed.
+
+ >>> list(ix.findValues(
+ ... 'folks',
+ ... q({'reltype': 'has the role of',
+ ... 'objects': roles['Project Manager'],
+ ... 'context': projects['zope.org redesign']})))
+ Traceback (most recent call last):
+ ...
+ ValueError: ('name not indexed', 'folks')
+
+ >>> list(ix.findValueTokens(
+ ... 'folks',
+ ... q({'reltype': 'has the role of',
+ ... 'objects': roles['Project Manager'],
+ ... 'context': projects['zope.org redesign']})))
+ Traceback (most recent call last):
+ ...
+ ValueError: ('name not indexed', 'folks')
+
+..[#apply_errors] Only one key may be in the dictionary.
+
+ >>> res = ix.apply({'values':
+ ... {'resultName': 'objects', 'query':
+ ... q({'reltype': 'has the role of',
+ ... 'context': projects['zope.org redesign']})},
+ ... 'relationships': q({'reltype': 'has the role of'})})
+ Traceback (most recent call last):
+ ...
+ ValueError: one key in the primary query dictionary
+
+ The keys must be one of 'values' or 'relationships'.
+
+ >>> res = ix.apply({'kumquats':
+ ... {'resultName': 'objects', 'query':
+ ... q({'reltype': 'has the role of',
+ ... 'context': projects['zope.org redesign']})}})
+ Traceback (most recent call last):
+ ...
+ ValueError: ('unknown query type', 'kumquats')
+
+ If a relationship uses LFBTrees, searches are fine.
+
+ >>> ix2 = index.Index( # explicit 64 bit
+ ... ({'element': IRelationship['subjects'], 'multiple': True},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'),
+ ... family=BTrees.family64)
+
+ >>> list(ix2.apply({'values':
+ ... {'resultName': 'objects', 'query':
+ ... q({'subjects': people['Gary']})}}))
+ []
+
+ >>> list(ix2.apply({'relationships':
+ ... q({'subjects': people['Gary']})}))
+ []
+
+ But, as with shown in the main text for values, if you are using another
+ BTree module for relationships, you'll get an error.
+
+ >>> ix2 = index.Index( # explicit 64 bit
+ ... ({'element': IRelationship['subjects'], 'multiple': True},
+ ... IRelationship['relationshiptype'],
+ ... {'element': IRelationship['objects'], 'multiple': True},
+ ... IContextAwareRelationship['getContext']),
+ ... index.TransposingTransitiveQueriesFactory('subjects', 'objects'),
+ ... relFamily=BTrees.OIBTree)
+
+ >>> list(ix2.apply({'relationships':
+ ... q({'subjects': people['Gary']})}))
+ Traceback (most recent call last):
+ ...
+ ValueError: cannot fulfill `apply` interface because cannot return an (I|L)FBTree-based result
+
Modified: zc.relationship/trunk/src/zc/relationship/index.py
===================================================================
--- zc.relationship/trunk/src/zc/relationship/index.py 2007-06-20 01:25:11 UTC (rev 76821)
+++ zc.relationship/trunk/src/zc/relationship/index.py 2007-06-20 02:51:27 UTC (rev 76822)
@@ -3,7 +3,8 @@
import persistent
import persistent.interfaces
-from BTrees import OOBTree, IFBTree, IOBTree, Length
+import BTrees
+from BTrees import Length
from zope import interface, component
import zope.interface.interfaces
@@ -56,7 +57,12 @@
else:
static = cache['static']
if dynamic:
- for r in index.findValueTokenSet(relchain[-1], dynamic[1]):
+ name = dynamic[1]
+ if name is None:
+ rels = (relchain[-1],)
+ else:
+ rels = index.findValueTokenSet(relchain[-1], name)
+ for r in rels:
res = {dynamic[0]: r}
res.update(static)
yield res
@@ -88,42 +94,39 @@
class Index(persistent.Persistent, zope.app.container.contained.Contained):
interface.implements(interfaces.IIndex)
- def _deactivate(self, ob):
- if (getattr(ob, '_p_jar', None) is not None and
- persistent.interfaces.IPersistent.providedBy(ob)):
- ob._p_deactivate()
+ family = BTrees.family32
def __init__(self, attrs, defaultTransitiveQueriesFactory=None,
dumpRel=generateToken, loadRel=resolveToken,
- relFamily=IFBTree, deactivateSets=False):
- self._name_TO_mapping = OOBTree.BTree()
+ relFamily=None, family=None):
+ if family is not None:
+ self.family = family
+ else:
+ family = self.family
+ self._name_TO_mapping = family.OO.BTree()
# held mappings are objtoken to (relcount, relset)
- self._EMPTY_name_TO_relcount_relset = OOBTree.BTree()
- self._reltoken_name_TO_objtokenset = OOBTree.BTree()
+ self._EMPTY_name_TO_relcount_relset = family.OO.BTree()
+ self._reltoken_name_TO_objtokenset = family.OO.BTree()
self.defaultTransitiveQueriesFactory = defaultTransitiveQueriesFactory
+ if relFamily is None:
+ relFamily = family.IF
self._relTools = getModuleTools(relFamily)
self._relTools['load'] = loadRel
self._relTools['dump'] = dumpRel
self._relLength = Length.Length()
self._relTokens = self._relTools['TreeSet']()
- self.deactivateSets = deactivateSets
self._attrs = _attrs = {} # this is private, and it's not expected to
# mutate after this initial setting.
seen = set()
for data in attrs:
# see README.txt for description of attrs.
+
if zope.interface.interfaces.IElement.providedBy(data):
data = {'element': data}
- res = getModuleTools(data.get('btree', IFBTree))
- res['element'] = val = data['element']
- res['attrname'] = val.__name__
- res['name'] = data.get('name', res['attrname'])
- if res['name'] in _attrs or val in seen:
- raise ValueError('Duplicate in attrs', name, val)
- seen.add(val)
- _attrs[res['name']] = res
+ res = getModuleTools(data.get('btree', family.IF))
res['dump'] = data.get('dump', generateToken)
res['load'] = data.get('load', resolveToken)
+ res['multiple'] = data.get('multiple', False)
if (res['dump'] is None) ^ (res['load'] is None):
raise ValueError(
"either both of 'dump' and 'load' must be None, or neither")
@@ -131,36 +134,60 @@
# optimization that can be a large optimization if the returned
# value is one of the main four options of the selected btree
# family (BTree, TreeSet, Set, Bucket).
- res['interface'] = val.interface
- res['multiple'] = data.get('multiple', False)
- res['call'] = zope.interface.interfaces.IMethod.providedBy(val)
- if res['TreeSet'].__name__.startswith('I'):
- Mapping = IOBTree.BTree
+
+ if 'element' in data:
+ if 'callable' in data:
+ raise ValueError(
+ 'cannot provide both callable and element')
+ res['element'] = val = data['element']
+ name = res['attrname'] = val.__name__
+ res['interface'] = val.interface
+ res['call'] = zope.interface.interfaces.IMethod.providedBy(val)
+ elif 'callable' not in data:
+ raise ValueError('must provide element or callable')
else:
+ # must return iterable or None
+ val = res['callable'] = data['callable']
+ name = getattr(res['callable'], '__name__', None)
+ res['name'] = data.get('name', name)
+ if res['name'] is None:
+ raise ValueError('no name specified')
+ if res['name'] in _attrs or val in seen:
+ raise ValueError('Duplicate in attrs', res['name'], val)
+ if res['TreeSet'].__name__[0] == 'I':
+ Mapping = BTrees.family32.IO.BTree
+ elif res['TreeSet'].__name__[0] == 'L':
+ Mapping = BTrees.family64.IO.BTree
+ else:
assert res['TreeSet'].__name__.startswith('O')
- Mapping = OOBTree.BTree
+ Mapping = family.OO.BTree
self._name_TO_mapping[res['name']] = Mapping()
# these are objtoken to (relcount, relset)
+ seen.add(val)
+ _attrs[res['name']] = res
def _getValuesAndTokens(self, rel, data):
- valueSource = data['interface'](rel, None)
- if valueSource is not None:
- values = getattr(valueSource, data['attrname'])
- if data['call']:
- values = values()
- if not data['multiple'] and values is not None:
- # None is a marker for no value
- values = (values,)
- elif not values:
- values = None
+ values = None
+ if 'interface' in data:
+ valueSource = data['interface'](rel, None)
+ if valueSource is not None:
+ values = getattr(valueSource, data['attrname'])
+ if data['call']:
+ values = values()
else:
- values = None
- if values is None:
- return values, values, False
- elif data['dump'] is None and isinstance(values, (
- data['TreeSet'], data['BTree'], data['Bucket'], data['Set'])):
+ values = data['callable'](rel, self)
+ if not data['multiple'] and values is not None:
+ # None is a marker for no value
+ values = (values,)
+ optimization = data['dump'] is None and (
+ values is None or
+ isinstance(values, (
+ data['TreeSet'], data['BTree'], data['Bucket'], data['Set'])))
+ if not values:
+ return None, None, optimization
+ elif optimization:
# this is the optimization story (see _add)
- return values, values, True
+ return values, values, optimization
else:
cache = {}
if data['dump'] is None:
@@ -244,7 +271,7 @@
else:
removed = oldTokens
added = newTokens
- if optimization:
+ if optimization and newTokens is not None:
newTokens = data['TreeSet'](newTokens)
self._remove(relToken, removed, data['name'])
self._add(relToken, added, data['name'], newTokens)
@@ -297,20 +324,20 @@
raise ValueError('one key in the primary query dictionary')
(searchType, query) = query.items()[0]
if searchType=='relationships':
- if self._relTools['TreeSet'] is not IFBTree.TreeSet:
+ if self._relTools['TreeSet'].__name__[:2] not in ('IF', 'LF'):
raise ValueError(
'cannot fulfill `apply` interface because cannot return '
- 'an IFBTree-based result')
+ 'an (I|L)FBTree-based result')
res = self._relData(query)
if res is None:
res = self._relTools['TreeSet']()
return res
elif searchType=='values':
data = self._attrs[query['resultName']]
- if data['TreeSet'] is not IFBTree.TreeSet:
+ if data['TreeSet'].__name__[:2] not in ('IF', 'LF'):
raise ValueError(
'cannot fulfill `apply` interface because cannot return '
- 'an IFBTree-based result')
+ 'an (I|L)FBTree-based result')
iterable = self._yieldValueTokens(
query['resultName'], *(self._parse(
query['query'], query.get('maxDepth'),
@@ -318,31 +345,36 @@
query.get('targetFilter'),
query.get('transitiveQueriesFactory')) +
(True,)))
- if data['multiunion'] is not None:
- res = data['multiunion'](tuple(iterable))
- else:
- res = data['TreeSet']()
- for s in iterable:
- res = data['union'](res, s)
- return res
+ # IF and LF have multiunion; can demand its presence
+ return data['multiunion'](tuple(iterable))
else:
raise ValueError('unknown query type', searchType)
def tokenizeQuery(self, query):
res = {}
- for k, v in query.items():
- data = self._attrs[k]
- if v is not None and data['dump'] is not None:
- v = data['dump'](v, self, {})
+ if getattr(query, 'items', None) is not None:
+ query = query.items()
+ for k, v in query:
+ if k is None:
+ v = self._relTools['dump'](v, self, {})
+ else:
+ data = self._attrs[k]
+ if v is not None and data['dump'] is not None:
+ v = data['dump'](v, self, {})
res[k] = v
return res
def resolveQuery(self, query):
res = {}
- for k, v in query.items():
- data = self._attrs[k]
- if v is not None and data['load'] is not None:
- v = data['load'](v, self, {})
+ if getattr(query, 'items', None) is not None:
+ query = query.items()
+ for k, v in query:
+ if k is None:
+ v = self._relTools['load'](v, self, {})
+ else:
+ data = self._attrs[k]
+ if v is not None and data['load'] is not None:
+ v = data['load'](v, self, {})
res[k] = v
return res
@@ -356,7 +388,7 @@
def resolveValueTokens(self, tokens, name):
load = self._attrs[name]['load']
if load is None:
- return values
+ return tokens
cache = {}
return (load(t, self, cache) for t in tokens)
@@ -374,17 +406,17 @@
cache = {}
return (self._relTools['load'](t, self, cache) for t in tokens)
- def findRelationships(self, query):
- return self.resolveRelationshipTokens(
- self.findRelationshipTokenSet(query))
-
def findRelationshipTokenSet(self, query):
+ # equivalent to, and used by, non-transitive
+ # findRelationshipTokens(query)
res = self._relData(query)
if res is None:
res = self._relTools['TreeSet']()
return res
def findValueTokenSet(self, reltoken, name):
+ # equivalent to, and used by, non-transitive
+ # findValueTokens(name, {None: reltoken})
res = self._reltoken_name_TO_objtokenset.get((reltoken, name))
if res is None:
res = self._attrs[name]['TreeSet']()
@@ -394,14 +426,28 @@
data = []
if getattr(searchTerms, 'items', None) is not None:
searchTerms = searchTerms.items()
+ searchTerms = tuple(searchTerms)
+ if not searchTerms:
+ return self._relTokens
+ rel = None
for nm, token in searchTerms:
- if token is None:
- relData = self._EMPTY_name_TO_relcount_relset.get(nm)
+ if nm is None:
+ rel = token
+ if rel not in self._relTokens:
+ return None
else:
- relData = self._name_TO_mapping[nm].get(token)
- if relData is None or relData[0].value == 0:
- return None
- data.append((relData[0].value, nm, token, relData[1]))
+ if token is None:
+ relData = self._EMPTY_name_TO_relcount_relset.get(nm)
+ else:
+ relData = self._name_TO_mapping[nm].get(token)
+ if relData is None or relData[0].value == 0:
+ return None
+ data.append((relData[0].value, nm, token, relData[1]))
+ if rel is not None:
+ for ct, nm, tk, st in data:
+ if rel not in st:
+ return None
+ return self._relTools['TreeSet']((rel,))
data.sort()
while len(data) > 1:
first_count, _ignore1, _ignore2, first_set = data[0]
@@ -423,9 +469,6 @@
else:
intersection = self._relTools['intersection'](
first_set, second_set)
- if self.deactivateSets:
- self._deactivate(first_set)
- self._deactivate(second_set)
if not intersection:
return None
data = data[2:]
@@ -479,39 +522,55 @@
return (query, relData, maxDepth, checkFilter, checkTargetFilter,
getQueries)
- def findValueTokens(self, resultName, query, maxDepth=None, filter=None,
- targetQuery=None, targetFilter=None,
+ def findValueTokens(self, resultName, query=(), maxDepth=None,
+ filter=None, targetQuery=None, targetFilter=None,
transitiveQueriesFactory=None):
- if resultName not in self._attrs:
- raise ValueError('name not indexed', nm)
+ data = self._attrs.get(resultName)
+ if data is None:
+ raise ValueError('name not indexed', resultName)
+ if (((maxDepth is None and transitiveQueriesFactory is None and
+ self.defaultTransitiveQueriesFactory is None)
+ or maxDepth==1)
+ and filter is None and not targetQuery and targetFilter is None):
+ if not query:
+ return self._name_TO_mapping[resultName]
+ rels = self._relData(query)
+ if not rels:
+ return data['TreeSet']()
+ elif len(rels) == 1:
+ return self.findValueTokenSet(iter(rels).next(), resultName)
+ else:
+ iterable = (
+ self._reltoken_name_TO_objtokenset.get((r, resultName))
+ for r in rels)
+ if data['multiunion'] is not None:
+ res = data['multiunion'](tuple(iterable))
+ else:
+ res = data['TreeSet']()
+ for s in iterable:
+ res = data['union'](res, s)
+ return res
return self._yieldValueTokens(
resultName, *self._parse(
query, maxDepth, filter, targetQuery, targetFilter,
transitiveQueriesFactory))
- def findValues(self, resultName, query, maxDepth=None, filter=None,
+ def findValues(self, resultName, query=(), maxDepth=None, filter=None,
targetQuery=None, targetFilter=None,
transitiveQueriesFactory=None):
- resolve = self._attrs[resultName]['load']
+ data = self._attrs.get(resultName)
+ if data is None:
+ raise ValueError('name not indexed', resultName)
+ resolve = data['load']
+ res = self.findValueTokens(resultName, query, maxDepth, filter,
+ targetQuery, targetFilter,
+ transitiveQueriesFactory)
if resolve is None:
- return self._yieldValueTokens(
- resultName, *self._parse(
- query, maxDepth, filter, targetQuery, targetFilter,
- transitiveQueriesFactory))
- return self._yieldValues(
- resultName, *self._parse(
- query, maxDepth, filter, targetQuery, targetFilter,
- transitiveQueriesFactory))
+ return res
+ else:
+ cache = {}
+ return (resolve(t, self, cache) for t in res)
- def _yieldValues(self, resultName, query, relData, maxDepth, checkFilter,
- checkTargetFilter, getQueries):
- resolve = self._attrs[resultName]['load']
- cache = {}
- for t in self._yieldValueTokens(resultName, query, relData, maxDepth,
- checkFilter, checkTargetFilter,
- getQueries):
- yield resolve(t, self, cache)
-
def _yieldValueTokens(
self, resultName, query, relData, maxDepth, checkFilter,
checkTargetFilter, getQueries, yieldSets=False):
@@ -533,9 +592,30 @@
if token not in objSeen:
yield token
objSeen.add(token)
- if self.deactivateSets:
- self._deactivate(outputSet)
+ def findRelationshipTokens(self, query=(), maxDepth=None, filter=None,
+ targetQuery=None, targetFilter=None,
+ transitiveQueriesFactory=None):
+ if (((maxDepth is None and transitiveQueriesFactory is None and
+ self.defaultTransitiveQueriesFactory is None)
+ or maxDepth==1)
+ and filter is None and not targetQuery and targetFilter is None):
+ return self.findRelationshipTokenSet(query)
+ seen = self._relTools['TreeSet']()
+ return (res[-1] for res in self._yieldRelationshipTokenChains(
+ *self._parse(query, maxDepth, filter, targetQuery,
+ targetFilter, transitiveQueriesFactory) +
+ (False,))
+ if seen.insert(res[-1]))
+
+ def findRelationships(self, query=(), maxDepth=None, filter=None,
+ targetQuery=None, targetFilter=None,
+ transitiveQueriesFactory=None):
+ return self.resolveRelationshipTokens(
+ self.findRelationshipTokens(
+ query, maxDepth, filter, targetQuery, targetFilter,
+ transitiveQueriesFactory))
+
def findRelationshipChains(self, query, maxDepth=None, filter=None,
targetQuery=None, targetFilter=None,
transitiveQueriesFactory=None):
@@ -543,8 +623,6 @@
same arguments as findValueTokens except no resultName.
"""
- if self._relTools['load'] is None:
- raise RuntimeError('not configured to resolve relationship tokens')
return self._yieldRelationshipChains(*self._parse(
query, maxDepth, filter, targetQuery, targetFilter,
transitiveQueriesFactory))
@@ -606,8 +684,6 @@
cycled.append(q)
elif walkFurther:
next.update(relData)
- if self.deactivateSets:
- self._deactivate(relData)
if walkFurther and next:
stack.append((tokenChain, iter(next)))
if cycled:
Modified: zc.relationship/trunk/src/zc/relationship/interfaces.py
===================================================================
--- zc.relationship/trunk/src/zc/relationship/interfaces.py 2007-06-20 01:25:11 UTC (rev 76821)
+++ zc.relationship/trunk/src/zc/relationship/interfaces.py 2007-06-20 02:51:27 UTC (rev 76822)
@@ -60,15 +60,18 @@
def __contains__(relationship):
"""returns whether the relationship is in the index"""
- def findValueTokens(resultName, query, maxDepth=None, filter=None,
+ def findValueTokens(resultName, query=None, maxDepth=None, filter=None,
targetQuery=None, targetFilter=None,
transitiveQueriesFactory=None):
"""find token results for searchTerms.
- resultName is the index name wanted for results.
+ - if query is None (or evaluates to boolean False), returns the
+ underlying btree data structure; which is an iterable result but
+ can also be used with BTree operations
Otherwise, same arguments as findRelationshipChains.
"""
- def findValues(resultName, query, maxDepth=None, filter=None,
+ def findValues(resultName, query=None, maxDepth=None, filter=None,
targetQuery=None, targetFilter=None,
transitiveQueriesFactory=None):
"""Like findValueTokens, but resolves value tokens"""
Modified: zc.relationship/trunk/src/zc/relationship/tests.py
===================================================================
--- zc.relationship/trunk/src/zc/relationship/tests.py 2007-06-20 01:25:11 UTC (rev 76821)
+++ zc.relationship/trunk/src/zc/relationship/tests.py 2007-06-20 02:51:27 UTC (rev 76822)
@@ -101,7 +101,8 @@
res = unittest.TestSuite((
doctest.DocFileSuite(
'README.txt',
- setUp=READMESetUp, tearDown=READMETearDown),
+ setUp=READMESetUp, tearDown=READMETearDown,
+ optionflags=doctest.INTERPRET_FOOTNOTES),
doctest.DocFileSuite(
'container.txt', setUp=keyrefSetUp, tearDown=tearDown),
doctest.DocFileSuite(
More information about the Checkins
mailing list