[Checkins] SVN: Products.ZCatalog/trunk/ Extend sort_on queries to an arbitary number of indexes - the more the slower it gets though.

Hano Schlichting cvs-admin at zope.org
Sat Apr 7 23:21:49 UTC 2012


Log message for revision 125093:
  Extend sort_on queries to an arbitary number of indexes - the more the slower it gets though.
  

Changed:
  U   Products.ZCatalog/trunk/CHANGES.txt
  U   Products.ZCatalog/trunk/src/Products/ZCatalog/Catalog.py
  U   Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_catalog.py

-=-
Modified: Products.ZCatalog/trunk/CHANGES.txt
===================================================================
--- Products.ZCatalog/trunk/CHANGES.txt	2012-04-07 20:55:27 UTC (rev 125092)
+++ Products.ZCatalog/trunk/CHANGES.txt	2012-04-07 23:21:45 UTC (rev 125093)
@@ -11,10 +11,11 @@
   methods and used it for a progress handler. Also optimized some of their
   internals.
 
-- Added support for simple `sort_on` queries with two sort indexes and
+- Added support for `sort_on` queries with any number of sort indexes and
   differing `sort_order` values. For example:
   `{'foo': 'a', 'sort_on': ('foo', 'bar')}`
   `{'foo': 'a', 'sort_on': ('foo', 'bar'), 'sort_order': ('', 'reverse')}`
+  `{'foo': 'a', 'sort_on': ('foo', 'bar', 'baz')}`
 
 - Change some internal wiring and let multiple sort_on values pass through.
   `Catalog.sortResults` can now be called with a list of search indexes.

Modified: Products.ZCatalog/trunk/src/Products/ZCatalog/Catalog.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/ZCatalog/Catalog.py	2012-04-07 20:55:27 UTC (rev 125092)
+++ Products.ZCatalog/trunk/src/Products/ZCatalog/Catalog.py	2012-04-07 23:21:45 UTC (rev 125093)
@@ -667,17 +667,19 @@
         # Return a lazy result set in sorted order if merge is true otherwise
         # returns a list of (sortkey, uid, getter_function) tuples, where
         # sortkey can be a tuple on its own.
-        index2 = None
+        second_indexes = None
+        second_indexes_key_map = None
         sort_index_length = 1
         if isinstance(sort_index, list):
             sort_index_length = len(sort_index)
             if sort_index_length > 1:
-                index2 = sort_index[1]
+                second_indexes = sort_index[1:]
+                second_indexes_key_map = []
+                for si in second_indexes:
+                    second_indexes_key_map.append(si.documentToKeyMap())
             sort_index = sort_index[0]
         _self__getitem__ = self.__getitem__
         index_key_map = sort_index.documentToKeyMap()
-        index2_key_map = (index2 is not None and
-            index2.documentToKeyMap() or None)
         result = []
         append = result.append
         if hasattr(rs, 'keys'):
@@ -769,10 +771,13 @@
                         # sort on secondary index
                         keysets = defaultdict(list)
                         for i in intset:
-                            try:
-                                keysets[(k, index2_key_map[i])].append(i)
-                            except KeyError:
-                                pass
+                            full_key = [k]
+                            for km in second_indexes_key_map:
+                                try:
+                                    full_key.append(km[i])
+                                except KeyError:
+                                    pass
+                            keysets[tuple(full_key)].append(i)
                         for k2, v2 in keysets.items():
                             append((k2, v2, _self__getitem__))
                 result = multisort(result, sort_spec)
@@ -799,13 +804,14 @@
             else:
                 for did in rs:
                     try:
-                        key = index_key_map[did]
-                        key2 = index2_key_map[did]
+                        full_key = [index_key_map[did]]
+                        for km in second_indexes_key_map:
+                            full_key.append(km[did])
                     except KeyError:
                         # This document is not in the sort key index, skip it.
                         pass
                     else:
-                        append(((key, key2), did, _self__getitem__))
+                        append((tuple(full_key), did, _self__getitem__))
                 if merge:
                     result = multisort(result, sort_spec)
             if merge:
@@ -849,7 +855,9 @@
                 for did in rs:
                     try:
                         key = index_key_map[did]
-                        key2 = index2_key_map[did]
+                        full_key = [key]
+                        for km in second_indexes_key_map:
+                            full_key.append(km[did])
                     except KeyError:
                         # This document is not in the sort key index, skip it.
                         pass
@@ -858,7 +866,8 @@
                             continue
                         i = bisect(keys, key)
                         keys.insert(i, key)
-                        result.insert(i, ((key, key2), did, _self__getitem__))
+                        result.insert(i,
+                            (tuple(full_key), did, _self__getitem__))
                         if n == limit:
                             del keys[0], result[0]
                         else:
@@ -901,7 +910,9 @@
                 for did in rs:
                     try:
                         key = index_key_map[did]
-                        key2 = index2_key_map[did]
+                        full_key = [key]
+                        for km in second_indexes_key_map:
+                            full_key.append(km[did])
                     except KeyError:
                         # This document is not in the sort key index, skip it.
                         pass
@@ -910,7 +921,8 @@
                             continue
                         i = bisect(keys, key)
                         keys.insert(i, key)
-                        result.insert(i, ((key, key2), did, _self__getitem__))
+                        result.insert(i,
+                            (tuple(full_key), did, _self__getitem__))
                         if n == limit:
                             del keys[-1], result[-1]
                         else:
@@ -966,9 +978,6 @@
                             'not capable of being used as a sort index: '
                             '%s' % repr(name))
                 sort_indexes.append(sort_index)
-            if len(sort_indexes) > 2:
-                raise CatalogError('Two sort indexes are supported at max, '
-                    'got: %s' % repr(name))
             if len(sort_indexes) == 1:
                 # be nice and keep the old API intact for single sort_on's
                 return sort_indexes[0]

Modified: Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_catalog.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_catalog.py	2012-04-07 20:55:27 UTC (rev 125092)
+++ Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_catalog.py	2012-04-07 23:21:45 UTC (rev 125093)
@@ -696,7 +696,61 @@
         for x in range(99):
             self.assertTrue(a[x].num > a[x + 1].num)
 
+    def test_sort_on_three(self):
+        def extra(catalog):
+            col2 = FieldIndex('col2')
+            catalog.addIndex('col2', col2)
+        catalog = self._make_one(extra)
+        a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1')
+        self.assertEqual(len(a), self.upper)
+        for x in range(self.upper):
+            self.assertEqual(a[x].num, x)
 
+    def test_sort_on_three_reverse(self):
+        def extra(catalog):
+            col2 = FieldIndex('col2')
+            catalog.addIndex('col2', col2)
+        catalog = self._make_one(extra)
+        a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1',
+            sort_order='reverse')
+        self.assertEqual(len(a), self.upper)
+        for x in range(self.upper - 1):
+            self.assertTrue(a[x].num > a[x + 1].num)
+
+    def test_sort_on_three_reverse_last(self):
+        def extra(catalog):
+            col2 = FieldIndex('col2')
+            catalog.addIndex('col2', col2)
+        catalog = self._make_one(extra)
+        a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1',
+            sort_order=('', '', 'reverse'))
+        self.assertEqual(len(a), self.upper)
+        for x in range(self.upper - 1):
+            self.assertTrue(a[x].num > a[x + 1].num)
+
+    def test_sort_on_three_small_limit(self):
+        def extra(catalog):
+            col2 = FieldIndex('col2')
+            catalog.addIndex('col2', col2)
+        catalog = self._make_one(extra)
+        a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1',
+            sort_limit=10)
+        self.assertEqual(len(a), 10)
+        for x in range(9):
+            self.assertTrue(a[x].num < a[x + 1].num)
+
+    def test_sort_on_three_big_limit(self):
+        def extra(catalog):
+            col2 = FieldIndex('col2')
+            catalog.addIndex('col2', col2)
+        catalog = self._make_one(extra)
+        a = catalog(sort_on=('att1', 'col2', 'num'), att1='att1',
+            sort_limit=self.upper * 3)
+        self.assertEqual(len(a), 100)
+        for x in range(99):
+            self.assertTrue(a[x].num < a[x + 1].num)
+
+
 class TestUnCatalog(unittest.TestCase):
 
     upper = 5



More information about the checkins mailing list