[Checkins] SVN: Products.ZCatalog/trunk/ Added support for `not` queries to FieldIndexes - note that these aren't optimized in any way and could be slow.
Hano Schlichting
cvs-admin at zope.org
Sun Mar 25 12:49:56 UTC 2012
Log message for revision 124718:
Added support for `not` queries to FieldIndexes - note that these aren't optimized in any way and could be slow.
Changed:
U Products.ZCatalog/trunk/CHANGES.txt
U Products.ZCatalog/trunk/setup.py
U Products.ZCatalog/trunk/src/Products/PluginIndexes/FieldIndex/FieldIndex.py
U Products.ZCatalog/trunk/src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py
U Products.ZCatalog/trunk/src/Products/PluginIndexes/common/UnIndex.py
U Products.ZCatalog/trunk/src/Products/PluginIndexes/common/tests/test_util.py
U Products.ZCatalog/trunk/src/Products/PluginIndexes/common/util.py
-=-
Modified: Products.ZCatalog/trunk/CHANGES.txt
===================================================================
--- Products.ZCatalog/trunk/CHANGES.txt 2012-03-25 10:55:59 UTC (rev 124717)
+++ Products.ZCatalog/trunk/CHANGES.txt 2012-03-25 12:49:53 UTC (rev 124718)
@@ -1,9 +1,15 @@
Changelog
=========
-2.13.23 (unreleased)
---------------------
+3.0 (unreleased)
+----------------
+- Added support for `not` queries to FieldIndexes. Both restrictions of normal
+ queries and range queries are supported, as well as purely exclusive
+ queries. For example: `{'foo': {'query': ['a', 'ab'], 'not': 'a'}}`,
+ `{'query': 'a', 'range': 'min', 'not': ['a', 'e', 'f']}}` and
+ `{'foo': {'not': ['a', 'b']}}`.
+
- Updated deprecation warnings to point to Zope 4 instead of 2.14.
2.13.22 (2011-11-17)
Modified: Products.ZCatalog/trunk/setup.py
===================================================================
--- Products.ZCatalog/trunk/setup.py 2012-03-25 10:55:59 UTC (rev 124717)
+++ Products.ZCatalog/trunk/setup.py 2012-03-25 12:49:53 UTC (rev 124718)
@@ -15,7 +15,7 @@
from setuptools import setup, find_packages
setup(name='Products.ZCatalog',
- version = '2.13.23dev',
+ version = '3.0dev',
url='http://pypi.python.org/pypi/Products.ZCatalog',
license='ZPL 2.1',
description="Zope 2's indexing and search solution.",
Modified: Products.ZCatalog/trunk/src/Products/PluginIndexes/FieldIndex/FieldIndex.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/FieldIndex/FieldIndex.py 2012-03-25 10:55:59 UTC (rev 124717)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/FieldIndex/FieldIndex.py 2012-03-25 12:49:53 UTC (rev 124718)
@@ -29,7 +29,7 @@
{'label': 'Browse', 'action': 'manage_browse'},
)
- query_options = ["query","range"]
+ query_options = ["query", "range", "not"]
manage = manage_main = DTMLFile('dtml/manageFieldIndex', globals())
manage_main._setName('manage_main')
Modified: Products.ZCatalog/trunk/src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py 2012-03-25 10:55:59 UTC (rev 124717)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py 2012-03-25 12:49:53 UTC (rev 124718)
@@ -37,8 +37,6 @@
"""
def setUp( self ):
- """
- """
self._index = FieldIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( 'a' ) )
@@ -57,24 +55,31 @@
keys = self._forward.get( v, [] )
self._forward[v] = keys
- self._noop_req = { 'bar': 123 }
- self._request = { 'foo': 'abce' }
- self._min_req = { 'foo': {'query': 'abc'
- , 'range': 'min'}
- }
- self._max_req = { 'foo': {'query': 'abc'
- , 'range': 'max' }
- }
- self._range_req = { 'foo': {'query': ( 'abc', 'abcd' )
- , 'range': 'min:max' }
- }
- self._zero_req = { 'foo': 0 }
- self._none_req = { 'foo': None }
+ self._noop_req = {'bar': 123}
+ self._request = {'foo': 'abce'}
+ self._min_req = {'foo':
+ {'query': 'abc', 'range': 'min'}}
+ self._min_req_n = {'foo':
+ {'query': 'abc', 'range': 'min', 'not': 'abca'}}
+ self._max_req = {'foo':
+ {'query': 'abc', 'range': 'max'}}
+ self._max_req_n = {'foo':
+ {'query': 'abc', 'range': 'max', 'not': ['a', 'b', None, 0]}}
+ self._range_req = {'foo':
+ {'query': ( 'abc', 'abcd' ), 'range': 'min:max'}}
+ self._range_ren = {'foo':
+ {'query': ( 'abc', 'abcd' ), 'range': 'min:max', 'not': 'abcd'}}
+ self._range_non = {'foo':
+ {'query': ( 'a', 'aa' ), 'range': 'min:max', 'not': 'a'}}
+ self._zero_req = {'foo': 0 }
+ self._none_req = {'foo': None }
+ self._not_1 = {'foo': {'query': 'a', 'not': 'a'}}
+ self._not_2 = {'foo': {'query': ['a', 'ab'], 'not': 'a'}}
+ self._not_3 = {'foo': {'not': 'a'}}
+ self._not_4 = {'foo': {'not': [0, None]}}
+ self._not_5 = {'foo': {'not': ['a', 'b']}}
+ self._not_6 = {'foo': 'a', 'bar': {'query': 123, 'not': 1}}
- def tearDown( self ):
- """
- """
-
def _populateIndex( self ):
for k, v in self._values:
self._index.index_object( k, v )
@@ -115,11 +120,15 @@
assert not self._index.hasUniqueValuesFor( 'bar' )
assert len( self._index.uniqueValues( 'foo' ) ) == 0
- assert self._index._apply_index( self._noop_req ) is None
- self._checkApply( self._request, [] )
- self._checkApply( self._min_req, [] )
- self._checkApply( self._max_req, [] )
- self._checkApply( self._range_req, [] )
+ assert self._index._apply_index(self._noop_req) is None
+ self._checkApply(self._request, [])
+ self._checkApply(self._min_req, [])
+ self._checkApply(self._min_req_n, [])
+ self._checkApply(self._max_req, [])
+ self._checkApply(self._max_req_n, [])
+ self._checkApply(self._range_req, [])
+ self._checkApply(self._range_ren, [])
+ self._checkApply(self._range_non, [])
def testPopulated( self ):
""" Test a populated FieldIndex """
@@ -142,11 +151,22 @@
assert self._index._apply_index( self._noop_req ) is None
- self._checkApply( self._request, values[ -4:-2 ] )
- self._checkApply( self._min_req, values[ 2:-2 ] )
- self._checkApply( self._max_req, values[ :3 ] + values[ -2: ] )
- self._checkApply( self._range_req, values[ 2:5 ] )
+ self._checkApply(self._request, values[-4:-2])
+ self._checkApply(self._min_req, values[2:-2])
+ self._checkApply(self._min_req_n, values[2:3] + values[4:-2])
+ self._checkApply(self._max_req, values[:3] + values[-2:])
+ self._checkApply(self._max_req_n, values[1:3])
+ self._checkApply(self._range_req, values[2:5])
+ self._checkApply(self._range_ren, values[2:4])
+ self._checkApply(self._range_non, [])
+ self._checkApply(self._not_1, [])
+ self._checkApply(self._not_2, values[1:2])
+ self._checkApply(self._not_3, values[1:])
+ self._checkApply(self._not_4, values[:7])
+ self._checkApply(self._not_5, values[1:])
+ self._checkApply(self._not_6, values[0:1])
+
def testZero( self ):
""" Make sure 0 gets indexed """
self._populateIndex()
Modified: Products.ZCatalog/trunk/src/Products/PluginIndexes/common/UnIndex.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/common/UnIndex.py 2012-03-25 10:55:59 UTC (rev 124717)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/common/UnIndex.py 2012-03-25 12:49:53 UTC (rev 124718)
@@ -17,6 +17,7 @@
from logging import getLogger
import sys
+from BTrees.IIBTree import difference
from BTrees.IIBTree import intersection
from BTrees.IIBTree import IITreeSet
from BTrees.IIBTree import IISet
@@ -293,6 +294,18 @@
LOG.debug('Attempt to unindex nonexistent document'
' with id %s' % documentId,exc_info=True)
+ def _apply_not(self, not_parm, resultset=None):
+ index = self._index
+ setlist = []
+ for k in not_parm:
+ s = index.get(k, None)
+ if s is None:
+ continue
+ elif isinstance(s, int):
+ s = IISet((s, ))
+ setlist.append(s)
+ return multiunion(setlist)
+
def _apply_index(self, request, resultset=None):
"""Apply the index to query parameters given in the request arg.
@@ -336,6 +349,13 @@
r = None
opr = None
+ # not / exclude parameter
+ not_parm = record.get('not', None)
+ if not record.keys and not_parm:
+ # we have only a 'not' query
+ record.keys = [k for k in index.keys() if k not in not_parm]
+ not_parm = None
+
# experimental code for specifing the operator
operator = record.get('operator',self.useOperator)
if not operator in self.operators :
@@ -371,6 +391,9 @@
result = setlist[0]
if isinstance(result, int):
result = IISet((result,))
+ if not_parm:
+ exclude = self._apply_not(not_parm, resultset)
+ result = difference(result, exclude)
return result, (self.id,)
if operator == 'or':
@@ -417,6 +440,9 @@
result = setlist[0]
if isinstance(result, int):
result = IISet((result,))
+ if not_parm:
+ exclude = self._apply_not(not_parm, resultset)
+ result = difference(result, exclude)
return result, (self.id,)
if operator == 'or':
@@ -442,8 +468,10 @@
r = IISet((r, ))
if r is None:
return IISet(), (self.id,)
- else:
- return r, (self.id,)
+ if not_parm:
+ exclude = self._apply_not(not_parm, resultset)
+ r = difference(r, exclude)
+ return r, (self.id,)
def hasUniqueValuesFor(self, name):
"""has unique values for column name"""
Modified: Products.ZCatalog/trunk/src/Products/PluginIndexes/common/tests/test_util.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/common/tests/test_util.py 2012-03-25 10:55:59 UTC (rev 124717)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/common/tests/test_util.py 2012-03-25 12:49:53 UTC (rev 124718)
@@ -53,7 +53,31 @@
self.assertEqual(parser.get('level'), 0)
self.assertEqual(parser.get('operator'), 'and')
+ def test_get_not_dict(self):
+ request = {'path': {'query': 'foo', 'not': 'bar'}}
+ parser = self._makeOne(request, 'path', ('query', 'not'))
+ self.assertEqual(parser.get('keys'), ['foo'])
+ self.assertEqual(parser.get('not'), ['bar'])
+ def test_get_not_dict_list(self):
+ request = {'path': {'query': 'foo', 'not': ['bar', 'baz']}}
+ parser = self._makeOne(request, 'path', ('query', 'not'))
+ self.assertEqual(parser.get('keys'), ['foo'])
+ self.assertEqual(parser.get('not'), ['bar', 'baz'])
+
+ def test_get_not_string(self):
+ request = {'path': 'foo', 'path_not': 'bar'}
+ parser = self._makeOne(request, 'path', ('query', 'not'))
+ self.assertEqual(parser.get('keys'), ['foo'])
+ self.assertEqual(parser.get('not'), ['bar'])
+
+ def test_get_not_string_list(self):
+ request = {'path': 'foo', 'path_not': ['bar', 'baz']}
+ parser = self._makeOne(request, 'path', ('query', 'not'))
+ self.assertEqual(parser.get('keys'), ['foo'])
+ self.assertEqual(parser.get('not'), ['bar', 'baz'])
+
+
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(parseIndexRequestTests))
Modified: Products.ZCatalog/trunk/src/Products/PluginIndexes/common/util.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/PluginIndexes/common/util.py 2012-03-25 10:55:59 UTC (rev 124717)
+++ Products.ZCatalog/trunk/src/Products/PluginIndexes/common/util.py 2012-03-25 12:49:53 UTC (rev 124718)
@@ -118,6 +118,11 @@
setattr(self, op, request[field])
self.keys = keys
+ not_value = getattr(self, 'not', None)
+ if not_value is not None:
+ if isinstance(not_value, basestring):
+ not_value = [not_value]
+ setattr(self, 'not', not_value)
def get(self, k, default_v=None):
if hasattr(self, k):
More information about the checkins
mailing list