[Checkins] SVN: Products.ZCatalog/trunk/ Removed tracking of result length from the query plan. The calculation of the length of an intermediate index result can itself be expensive.
Hanno Schlichting
hannosch at hannosch.eu
Sun Apr 10 11:43:34 EDT 2011
Log message for revision 121369:
Removed tracking of result length from the query plan. The calculation of the length of an intermediate index result can itself be expensive.
Changed:
U Products.ZCatalog/trunk/CHANGES.txt
U Products.ZCatalog/trunk/src/Products/ZCatalog/dtml/catalogReport.dtml
U Products.ZCatalog/trunk/src/Products/ZCatalog/plan.py
U Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_plan.py
-=-
Modified: Products.ZCatalog/trunk/CHANGES.txt
===================================================================
--- Products.ZCatalog/trunk/CHANGES.txt 2011-04-10 11:45:46 UTC (rev 121368)
+++ Products.ZCatalog/trunk/CHANGES.txt 2011-04-10 15:43:34 UTC (rev 121369)
@@ -4,6 +4,8 @@
2.13.10 (unreleased)
--------------------
+- Removed tracking of result length from the query plan. The calculation of the
+ length of an intermediate index result can itself be expensive.
2.13.9 (2011-04-10)
-------------------
Modified: Products.ZCatalog/trunk/src/Products/ZCatalog/dtml/catalogReport.dtml
===================================================================
--- Products.ZCatalog/trunk/src/Products/ZCatalog/dtml/catalogReport.dtml 2011-04-10 11:45:46 UTC (rev 121368)
+++ Products.ZCatalog/trunk/src/Products/ZCatalog/dtml/catalogReport.dtml 2011-04-10 15:43:34 UTC (rev 121369)
@@ -56,8 +56,7 @@
<dtml-var expr="'%3.2f' % last['duration']">ms
[<dtml-in expr="last['details']" sort mapping>
&dtml-id;:
- <dtml-var expr="'%3.2f' % duration">ms /
- &dtml-length; objects,
+ <dtml-var expr="'%3.2f' % duration">ms,
</dtml-in>]
</div>
</td>
Modified: Products.ZCatalog/trunk/src/Products/ZCatalog/plan.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/ZCatalog/plan.py 2011-04-10 11:45:46 UTC (rev 121368)
+++ Products.ZCatalog/trunk/src/Products/ZCatalog/plan.py 2011-04-10 15:43:34 UTC (rev 121369)
@@ -27,8 +27,8 @@
Duration = namedtuple('Duration', ['start', 'end'])
IndexMeasurement = namedtuple('IndexMeasurement',
- ['name', 'duration', 'num', 'limit'])
-Benchmark = namedtuple('Benchmark', ['num', 'duration', 'hits', 'limit'])
+ ['name', 'duration', 'limit'])
+Benchmark = namedtuple('Benchmark', ['duration', 'hits', 'limit'])
RecentQuery = namedtuple('RecentQuery', ['duration', 'details'])
Report = namedtuple('Report', ['hits', 'duration', 'last'])
@@ -241,8 +241,9 @@
if not benchmark:
return None
- # sort indexes on (mean result length, mean search time)
- ranking = [((value.limit, value.num, value.duration), name)
+ # sort indexes on (limited result index, mean search time)
+ # skip internal ('#') bookkeeping records
+ ranking = [((value.limit, value.duration), name)
for name, value in benchmark.items()]
ranking.sort()
return [r[1] for r in ranking]
@@ -257,14 +258,10 @@
def stop_split(self, name, result=None, limit=False):
current = time.time()
start_time, stop_time = self.interim.get(name, Duration(None, None))
- length = 0
- if result is not None:
- # TODO: calculating the length can be expensive
- length = len(result)
self.interim[name] = Duration(start_time, current)
dt = current - start_time
self.res.append(IndexMeasurement(
- name=name, duration=dt, num=length, limit=limit))
+ name=name, duration=dt, limit=limit))
if name == 'sort_on':
# sort_on isn't an index. We only do time reporting on it
@@ -273,17 +270,16 @@
# remember index's hits, search time and calls
benchmark = self.benchmark
if name not in benchmark:
- benchmark[name] = Benchmark(num=length, duration=dt,
+ benchmark[name] = Benchmark(duration=dt,
hits=1, limit=limit)
else:
- num, duration, hits, limit = benchmark[name]
- num = int(((num * hits) + length) / float(hits + 1))
+ duration, hits, limit = benchmark[name]
duration = ((duration * hits) + dt) / float(hits + 1)
# reset adaption
if hits % REFRESH_RATE == 0:
hits = 0
hits += 1
- benchmark[name] = Benchmark(num, duration, hits, limit)
+ benchmark[name] = Benchmark(duration, hits, limit)
def stop(self):
self.end_time = time.time()
@@ -291,7 +287,7 @@
# Make absolutely sure we never omit query keys from the plan
for key in self.query.keys():
if key not in self.benchmark.keys():
- self.benchmark[key] = Benchmark(0, 0, 0, False)
+ self.benchmark[key] = Benchmark(0, 0, False)
PriorityMap.set_entry(self.cid, self.key, self.benchmark)
self.log()
@@ -328,8 +324,7 @@
'duration': report.duration * 1000,
'last': {'duration': last.duration * 1000,
'details': [dict(id=d.name,
- duration=d.duration * 1000,
- length=d.num)
+ duration=d.duration * 1000)
for d in last.details],
},
}
Modified: Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_plan.py
===================================================================
--- Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_plan.py 2011-04-10 11:45:46 UTC (rev 121368)
+++ Products.ZCatalog/trunk/src/Products/ZCatalog/tests/test_plan.py 2011-04-10 15:43:34 UTC (rev 121369)
@@ -33,8 +33,8 @@
TESTMAP = {
'/folder/catalog': {
'index1 index2': {
- 'index1': (10, 2.0, 3, True),
- 'index2': (15, 1.5, 2, False),
+ 'index1': (2.0, 3, True),
+ 'index2': (1.5, 2, False),
}
}
}
@@ -128,8 +128,8 @@
'Products.ZCatalog.tests.test_plan.TESTMAP'
self.pmap.load_default()
expected = {'/folder/catalog': {'index1 index2': {
- 'index1': Benchmark(num=10, duration=2.0, hits=3, limit=True),
- 'index2': Benchmark(num=15, duration=1.5, hits=2, limit=False),
+ 'index1': Benchmark(duration=2.0, hits=3, limit=True),
+ 'index2': Benchmark(duration=1.5, hits=2, limit=False),
}}}
self.assertEquals(self.pmap.get_value(), expected)
finally:
More information about the checkins
mailing list