[Zope-Checkins] CVS: Products/Transience -
Transience.py:1.32.12.8.2.7
Chris McDonough
chrism at plope.com
Tue Sep 14 17:40:22 EDT 2004
Update of /cvs-repository/Products/Transience
In directory cvs.zope.org:/tmp/cvs-serv6499
Modified Files:
Tag: chrism-pre273-branch
Transience.py
Log Message:
Replace BTrees.Length.Length with dunny's Length (not _p_independent) in
order to get "number of objects in data container" right.
Point out some ReadConflictError hotspots in the code.
=== Products/Transience/Transience.py 1.32.12.8.2.6 => 1.32.12.8.2.7 ===
--- Products/Transience/Transience.py:1.32.12.8.2.6 Mon Sep 13 12:27:42 2004
+++ Products/Transience/Transience.py Tue Sep 14 17:40:22 2004
@@ -30,10 +30,9 @@
TTWDictionary, ImmutablyValuedMappingOfPickleableObjects,\
StringKeyedHomogeneousItemContainer, TransientItemContainer
-from BTrees.Length import Length
+from BTrees.Length import Length as BTreesLength
from BTrees.OOBTree import OOBTree
from BTrees.IOBTree import IOBTree
-from ZODB.POSException import ConflictError, ReadConflictError
from Persistence import Persistent
from OFS.SimpleItem import SimpleItem
@@ -245,8 +244,10 @@
# we need to maintain the length of the index structure separately
# because getting the length of a BTree is very expensive, and it
# doesn't really tell us which ones are "active" anyway.
- try: self._length.set(0)
- except AttributeError: self._length = self.getLen = Length()
+ try:
+ self._length.set(0)
+ except AttributeError:
+ self._length = self.getLen = Length2()
def _getCurrentSlices(self, now):
if self._timeout_slices:
@@ -293,7 +294,8 @@
found_ts = None
for ts in current_slices:
- abucket = self._data.get(ts, None)
+ abucket = self._data.get(ts, None) # XXX ReadConflictError hotspot
+
if abucket is None:
DEBUG and TLOG('_move_item: no bucket for ts %s' % ts)
continue
@@ -429,15 +431,20 @@
STRICT and _assert(self._data.has_key(current_ts))
if item is _marker:
# the key didnt already exist, this is a new item
- if self._limit and len(self) >= self._limit:
+
+ length = self._length() # XXX ReadConflictError hotspot
+
+ if self._limit and length >= self._limit:
LOG('Transience', WARNING,
('Transient object container %s max subobjects '
'reached' % self.getId())
)
raise MaxTransientObjectsExceeded, (
"%s exceeds maximum number of subobjects %s" %
- (len(self), self._limit))
- self._length.change(1)
+ (length, self._limit))
+
+ self._length.increment(1)
+
DEBUG and TLOG('__setitem__: placing value for key %s in bucket %s' %
(k, current_ts))
current_bucket = self._data[current_ts]
@@ -464,7 +471,11 @@
if not issubclass(BUCKET_CLASS, Persistent):
# tickle persistence machinery
self._data[current_ts] = bucket
- self._length.change(-1)
+
+ # XXX does increment(-1) make any sense here?
+ # rationale from dunny: we are removing an item rather than simply
+ # declaring it to be unused?
+ self._length.increment(-1)
return current_ts, item
def __len__(self):
@@ -535,9 +546,6 @@
DEBUG and TLOG('_finalize: lock acquired successfully')
last_finalized = self._last_finalized_timeslice()
- if now is None:
- now = getCurrentTimeslice(self._period) # for unit tests
-
# we want to start finalizing from one timeslice after the
# timeslice which we last finalized.
@@ -580,7 +588,8 @@
for key in to_finalize:
- assert(start_finalize <= key <= max_ts)
+ _assert(start_finalize <= key)
+ _assert(key <= max_ts)
STRICT and _assert(self._data.has_key(key))
values = list(self._data[key].values())
DEBUG and TLOG('_do_finalize_work: values to notify from ts %s '
@@ -592,13 +601,21 @@
self.notifyDel(v)
if delta:
- self._length.change(-delta)
+ self._length.decrement(delta)
DEBUG and TLOG('_do_finalize_work: setting _last_finalized_timeslice '
'to max_ts of %s' % max_ts)
self._last_finalized_timeslice.set(max_ts)
+ def _invoke_finalize_and_gc(self):
+ # for unit testing purposes only!
+ last_finalized = self._last_finalized_timeslice()
+ now = getCurrentTimeslice(self._period) # for unit tests
+ start_finalize = last_finalized + self._period
+ max_ts = self._get_max_expired_ts(now)
+ self._do_finalize_work(now, max_ts, start_finalize)
+ self._do_gc_work(now)
def _replentish(self, now):
""" Add 'fresh' future or current buckets """
@@ -695,14 +712,7 @@
% new_buckets)
for k in new_buckets:
STRICT and _assert(not self._data.has_key(k))
-
- # this is a conflict hotspot
- try:
- self._data[k] = BUCKET_CLASS()
- except ConflictError:
- DEBUG and TLOG('_do_replentish_work: conflict when adding %s' %
- k)
- raise
+ self._data[k] = BUCKET_CLASS() # XXX ReadConflictError hotspot
self._max_timeslice.set(max(new_buckets))
@@ -756,7 +766,7 @@
DEBUG and TLOG('_do_gc_work: to_gc is: %s' % str(to_gc))
for key in to_gc:
- assert(key <= max_ts)
+ _assert(key <= max_ts)
STRICT and _assert(self._data.has_key(key))
DEBUG and TLOG('_do_gc_work: deleting %s from _data' % key)
del self._data[key]
@@ -961,9 +971,17 @@
# f/w compat: 2.8 cannot use __len__ as an instance variable
if not state.has_key('_length'):
- length = state.get('__len__', Length())
+ length = state.get('__len__', Length2())
self._length = self.getLen = length
+ oldlength = state['_length']
+ if isinstance(oldlength, BTreesLength):
+ # TOCS prior to 2.7.3 had a BTrees.Length.Length object as
+ # the TOC length object, replace it with our own Length2
+ # that does our conflict resolution correctly:
+ sz = oldlength()
+ self._length = self.getLen = Length2(sz)
+
# TOCs prior to 2.7.1 took their period from a global
if not state.has_key('_period'):
self._period = 20 # this was the default for all prior releases
@@ -1060,4 +1078,51 @@
def _p_resolveConflict(self, old, state1, state2):
return max(old, state1, state2)
+
+class Length2(Persistent):
+ """
+ A persistent object responsible for maintaining a repesention of
+ the number of current transient objects.
+
+ Conflict resolution is sensitive to which methods are used to
+ change the length.
+ """
+ def __init__(self, value=0):
+ self.set(value)
+
+ def set(self, value):
+ self.value = value
+ self.floor = 0
+ self.ceiling = 0
+
+ def increment(self, delta):
+ """Increase the length by delta.
+
+ Conflict resolution will take the sum of all the increments."""
+ self.ceiling += delta
+ self.value += delta
+
+ def decrement(self, delta):
+ """Decrease the length by delta.
+
+ Conflict resolution will take the highest decrement."""
+ self.floor += delta
+ self.value -= delta
+
+ def __getstate__(self):
+ return self.__dict__
+
+ def __setstate__(self, state):
+ self.__dict__.update(state)
+
+ def __call__(self):
+ return self.value
+
+ def _p_resolveConflict(self, old, saved, new):
+ new['ceiling'] = saved['ceiling'] + new['ceiling'] - old['ceiling']
+ new['floor'] = max(old['floor'], saved['floor'], new['floor'])
+ new['value'] = new['ceiling'] - new['floor']
+ return new
+
Globals.InitializeClass(TransientObjectContainer)
+
More information about the Zope-Checkins
mailing list