[Checkins] SVN: Products.ZCatalog/trunk/ Extend sort_on queries to an arbitary number of indexes - the more the slower it gets though.
Hano Schlichting
cvs-admin at zope.org
Sat Apr 7 23:21:49 UTC 2012
Log message for revision 125093:
Extend sort_on queries to an arbitary number of indexes - the more the slower it gets though.
Changed:
U Products.ZCatalog/trunk/CHANGES.txt
U Products.ZCatalog/trunk/src/Products/ZCatalog/Catalog.py
U Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_catalog.py
-=-
Modified: Products.ZCatalog/trunk/CHANGES.txt
===================================================================
--- Products.ZCatalog/trunk/CHANGES.txt 2012-04-07 20:55:27 UTC (rev 125092)
+++ Products.ZCatalog/trunk/CHANGES.txt 2012-04-07 23:21:45 UTC (rev 125093)
@@ -11,10 +11,11 @@
methods and used it for a progress handler. Also optimized some of their
internals.
-- Added support for simple `sort_on` queries with two sort indexes and
+- Added support for `sort_on` queries with any number of sort indexes and
differing `sort_order` values. For example:
`{'foo': 'a', 'sort_on': ('foo', 'bar')}`
`{'foo': 'a', 'sort_on': ('foo', 'bar'), 'sort_order': ('', 'reverse')}`
+ `{'foo': 'a', 'sort_on': ('foo', 'bar', 'baz')}`
- Change some internal wiring and let multiple sort_on values pass through.
`Catalog.sortResults` can now be called with a list of search indexes.
Modified: Products.ZCatalog/trunk/src/Products/ZCatalog/Catalog.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/ZCatalog/Catalog.py 2012-04-07 20:55:27 UTC (rev 125092)
+++ Products.ZCatalog/trunk/src/Products/ZCatalog/Catalog.py 2012-04-07 23:21:45 UTC (rev 125093)
@@ -667,17 +667,19 @@
# Return a lazy result set in sorted order if merge is true otherwise
# returns a list of (sortkey, uid, getter_function) tuples, where
# sortkey can be a tuple on its own.
- index2 = None
+ second_indexes = None
+ second_indexes_key_map = None
sort_index_length = 1
if isinstance(sort_index, list):
sort_index_length = len(sort_index)
if sort_index_length > 1:
- index2 = sort_index[1]
+ second_indexes = sort_index[1:]
+ second_indexes_key_map = []
+ for si in second_indexes:
+ second_indexes_key_map.append(si.documentToKeyMap())
sort_index = sort_index[0]
_self__getitem__ = self.__getitem__
index_key_map = sort_index.documentToKeyMap()
- index2_key_map = (index2 is not None and
- index2.documentToKeyMap() or None)
result = []
append = result.append
if hasattr(rs, 'keys'):
@@ -769,10 +771,13 @@
# sort on secondary index
keysets = defaultdict(list)
for i in intset:
- try:
- keysets[(k, index2_key_map[i])].append(i)
- except KeyError:
- pass
+ full_key = [k]
+ for km in second_indexes_key_map:
+ try:
+ full_key.append(km[i])
+ except KeyError:
+ pass
+ keysets[tuple(full_key)].append(i)
for k2, v2 in keysets.items():
append((k2, v2, _self__getitem__))
result = multisort(result, sort_spec)
@@ -799,13 +804,14 @@
else:
for did in rs:
try:
- key = index_key_map[did]
- key2 = index2_key_map[did]
+ full_key = [index_key_map[did]]
+ for km in second_indexes_key_map:
+ full_key.append(km[did])
except KeyError:
# This document is not in the sort key index, skip it.
pass
else:
- append(((key, key2), did, _self__getitem__))
+ append((tuple(full_key), did, _self__getitem__))
if merge:
result = multisort(result, sort_spec)
if merge:
@@ -849,7 +855,9 @@
for did in rs:
try:
key = index_key_map[did]
- key2 = index2_key_map[did]
+ full_key = [key]
+ for km in second_indexes_key_map:
+ full_key.append(km[did])
except KeyError:
# This document is not in the sort key index, skip it.
pass
@@ -858,7 +866,8 @@
continue
i = bisect(keys, key)
keys.insert(i, key)
- result.insert(i, ((key, key2), did, _self__getitem__))
+ result.insert(i,
+ (tuple(full_key), did, _self__getitem__))
if n == limit:
del keys[0], result[0]
else:
@@ -901,7 +910,9 @@
for did in rs:
try:
key = index_key_map[did]
- key2 = index2_key_map[did]
+ full_key = [key]
+ for km in second_indexes_key_map:
+ full_key.append(km[did])
except KeyError:
# This document is not in the sort key index, skip it.
pass
@@ -910,7 +921,8 @@
continue
i = bisect(keys, key)
keys.insert(i, key)
- result.insert(i, ((key, key2), did, _self__getitem__))
+ result.insert(i,
+ (tuple(full_key), did, _self__getitem__))
if n == limit:
del keys[-1], result[-1]
else:
@@ -966,9 +978,6 @@
'not capable of being used as a sort index: '
'%s' % repr(name))
sort_indexes.append(sort_index)
- if len(sort_indexes) > 2:
- raise CatalogError('Two sort indexes are supported at max, '
- 'got: %s' % repr(name))
if len(sort_indexes) == 1:
# be nice and keep the old API intact for single sort_on's
return sort_indexes[0]
Modified: Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_catalog.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_catalog.py 2012-04-07 20:55:27 UTC (rev 125092)
+++ Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_catalog.py 2012-04-07 23:21:45 UTC (rev 125093)
@@ -696,7 +696,61 @@
for x in range(99):
self.assertTrue(a[x].num > a[x + 1].num)
+ def test_sort_on_three(self):
+ def extra(catalog):
+ col2 = FieldIndex('col2')
+ catalog.addIndex('col2', col2)
+ catalog = self._make_one(extra)
+ a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1')
+ self.assertEqual(len(a), self.upper)
+ for x in range(self.upper):
+ self.assertEqual(a[x].num, x)
+ def test_sort_on_three_reverse(self):
+ def extra(catalog):
+ col2 = FieldIndex('col2')
+ catalog.addIndex('col2', col2)
+ catalog = self._make_one(extra)
+ a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1',
+ sort_order='reverse')
+ self.assertEqual(len(a), self.upper)
+ for x in range(self.upper - 1):
+ self.assertTrue(a[x].num > a[x + 1].num)
+
+ def test_sort_on_three_reverse_last(self):
+ def extra(catalog):
+ col2 = FieldIndex('col2')
+ catalog.addIndex('col2', col2)
+ catalog = self._make_one(extra)
+ a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1',
+ sort_order=('', '', 'reverse'))
+ self.assertEqual(len(a), self.upper)
+ for x in range(self.upper - 1):
+ self.assertTrue(a[x].num > a[x + 1].num)
+
+ def test_sort_on_three_small_limit(self):
+ def extra(catalog):
+ col2 = FieldIndex('col2')
+ catalog.addIndex('col2', col2)
+ catalog = self._make_one(extra)
+ a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1',
+ sort_limit=10)
+ self.assertEqual(len(a), 10)
+ for x in range(9):
+ self.assertTrue(a[x].num < a[x + 1].num)
+
+ def test_sort_on_three_big_limit(self):
+ def extra(catalog):
+ col2 = FieldIndex('col2')
+ catalog.addIndex('col2', col2)
+ catalog = self._make_one(extra)
+ a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1',
+ sort_limit=self.upper * 3)
+ self.assertEqual(len(a), 100)
+ for x in range(99):
+ self.assertTrue(a[x].num < a[x + 1].num)
+
+
class TestUnCatalog(unittest.TestCase):
upper = 5
More information about the checkins
mailing list