[Checkins] SVN: relstorage/trunk/ - Revised the way RelStorage uses memcached. Minimized the number of
Shane Hathaway
shane at hathawaymix.org
Wed Oct 7 16:55:28 EDT 2009
Log message for revision 104897:
- Revised the way RelStorage uses memcached. Minimized the number of
trips to both the cache server and the database.
- Added a wrapper for pylibmc.
- The get_current_tid method of IObjectMover is no longer needed.
Changed:
U relstorage/trunk/CHANGES.txt
U relstorage/trunk/relstorage/adapters/interfaces.py
U relstorage/trunk/relstorage/adapters/mover.py
A relstorage/trunk/relstorage/pylibmc_wrapper.py
U relstorage/trunk/relstorage/storage.py
U relstorage/trunk/relstorage/tests/fakecache.py
U relstorage/trunk/relstorage/tests/reltestbase.py
-=-
Modified: relstorage/trunk/CHANGES.txt
===================================================================
--- relstorage/trunk/CHANGES.txt 2009-10-07 20:48:05 UTC (rev 104896)
+++ relstorage/trunk/CHANGES.txt 2009-10-07 20:55:27 UTC (rev 104897)
@@ -11,6 +11,11 @@
- Expanded the option documentation in README.txt.
+- Revised the way RelStorage uses memcached. Minimized the number of
+ trips to both the cache server and the database.
+
+- Added a wrapper module for pylibmc.
+
- Renamed relstorage.py to storage.py to overcome import issues.
Also moved the Options class to options.py.
Modified: relstorage/trunk/relstorage/adapters/interfaces.py
===================================================================
--- relstorage/trunk/relstorage/adapters/interfaces.py 2009-10-07 20:48:05 UTC (rev 104896)
+++ relstorage/trunk/relstorage/adapters/interfaces.py 2009-10-07 20:55:27 UTC (rev 104897)
@@ -182,12 +182,6 @@
class IObjectMover(Interface):
"""Move object states to/from the database and within the database."""
- def get_current_tid(cursor, oid):
- """Returns the current integer tid for an object.
-
- oid is an integer. Returns None if object does not exist.
- """
-
def load_current(cursor, oid):
"""Returns the current state and integer tid for an object.
Modified: relstorage/trunk/relstorage/adapters/mover.py
===================================================================
--- relstorage/trunk/relstorage/adapters/mover.py 2009-10-07 20:48:05 UTC (rev 104896)
+++ relstorage/trunk/relstorage/adapters/mover.py 2009-10-07 20:55:27 UTC (rev 104897)
@@ -37,7 +37,6 @@
implements(IObjectMover)
_method_names = (
- 'get_current_tid',
'load_current',
'load_revision',
'exists',
@@ -69,56 +68,6 @@
- def generic_get_current_tid(self, cursor, oid):
- """Returns the current integer tid for an object.
-
- oid is an integer. Returns None if object does not exist.
- """
- if self.keep_history:
- stmt = """
- SELECT tid
- FROM current_object
- WHERE zoid = %s
- """
- else:
- stmt = """
- SELECT tid
- FROM object_state
- WHERE zoid = %s
- """
- cursor.execute(stmt, (oid,))
- for (tid,) in cursor:
- return tid
- return None
-
- postgresql_get_current_tid = generic_get_current_tid
- mysql_get_current_tid = generic_get_current_tid
-
- def oracle_get_current_tid(self, cursor, oid):
- """Returns the current integer tid for an object.
-
- oid is an integer. Returns None if object does not exist.
- """
- if self.keep_history:
- stmt = """
- SELECT tid
- FROM current_object
- WHERE zoid = :1
- """
- else:
- stmt = """
- SELECT tid
- FROM object_state
- WHERE zoid = :1
- """
- cursor.execute(stmt, (oid,))
- for (tid,) in cursor:
- return tid
- return None
-
-
-
-
def postgresql_load_current(self, cursor, oid):
"""Returns the current pickle and integer tid for an object.
Added: relstorage/trunk/relstorage/pylibmc_wrapper.py
===================================================================
--- relstorage/trunk/relstorage/pylibmc_wrapper.py (rev 0)
+++ relstorage/trunk/relstorage/pylibmc_wrapper.py 2009-10-07 20:55:27 UTC (rev 104897)
@@ -0,0 +1,73 @@
+##############################################################################
+#
+# Copyright (c) 2009 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""A wrapper around pylibmc to make it not raise memcache errors.
+
+One way to use this is to add 'cache-module-name relstorage.pylibmc_wrapper'
+to zope.conf and set the 'cache-servers' parameter as well.
+"""
+
+import pylibmc
+
+# Get the MemcachedError class. XXX Report to the pylibmc author that
+# pylibmc should at least export the exception class!
+_c = pylibmc.Client([])
+try:
+ _c.get('foo')
+except Exception, e:
+ MemcachedError = type(e)
+ del _c
+else:
+ raise ImportError("Could not get MemcachedError")
+
+
+class Client(object):
+
+ def __init__(self, servers):
+ self._client = pylibmc.Client(servers, binary=True)
+
+ def get(self, key):
+ try:
+ return self._client.get(key)
+ except MemcachedError:
+ return None
+
+ def get_multi(self, keys):
+ try:
+ return self._client.get_multi(keys)
+ except MemcachedError:
+ return None
+
+ def set(self, key, value):
+ try:
+ return self._client.set(key, value)
+ except MemcachedError:
+ return None
+
+ def set_multi(self, d):
+ try:
+ return self._client.set_multi(d)
+ except MemcachedError:
+ return None
+
+ def add(self, key, value):
+ try:
+ return self._client.add(key, value)
+ except MemcachedError:
+ return None
+
+ def incr(self, key):
+ try:
+ return self._client.incr(key)
+ except MemcachedError:
+ return None
Modified: relstorage/trunk/relstorage/storage.py
===================================================================
--- relstorage/trunk/relstorage/storage.py 2009-10-07 20:48:05 UTC (rev 104896)
+++ relstorage/trunk/relstorage/storage.py 2009-10-07 20:55:27 UTC (rev 104897)
@@ -344,13 +344,6 @@
logfunc('; '.join(msg))
- def _get_oid_cache_key(self, oid_int):
- """Return the cache key for finding the current tid."""
- my_tid = self._prev_polled_tid
- if my_tid is None:
- return None
- return 'tid:%d:%d' % (oid_int, my_tid)
-
def load(self, oid, version):
oid_int = u64(oid)
cache = self._cache_client
@@ -360,38 +353,57 @@
if not self._load_transaction_open:
self._restart_load()
cursor = self._load_cursor
+
if cache is None:
state, tid_int = self._adapter.mover.load_current(
cursor, oid_int)
+ state = str(state or '')
+
else:
- # get tid_int from the cache or the database
- cachekey = self._get_oid_cache_key(oid_int)
- if cachekey:
- tid_int = cache.get(cachekey)
- if not cachekey or not tid_int:
- tid_int = self._adapter.mover.get_current_tid(
+ # try to load from cache
+ state_key = 'state:%d' % oid_int
+ my_tid = self._prev_polled_tid
+ if my_tid:
+ backptr_key = 'back:%d:%d' % (my_tid, oid_int)
+ v = cache.get_multi([state_key, backptr_key])
+ if v is not None:
+ cache_data = v.get(state_key)
+ backptr = v.get(backptr_key)
+ else:
+ cache_data = None
+ backptr = None
+ else:
+ cache_data = cache.get(state_key)
+ backptr = None
+
+ state = None
+ if cache_data and len(cache_data) >= 8:
+ # validate the cache result
+ tid = cache_data[:8]
+ tid_int = u64(tid)
+ if tid_int == my_tid or tid == backptr:
+ # the cached data is current.
+ state = cache_data[8:]
+
+ if state is None:
+ # could not load from cache, so get from the database
+ state, tid_int = self._adapter.mover.load_current(
cursor, oid_int)
- if cachekey and tid_int is not None:
- cache.set(cachekey, tid_int)
- if tid_int is None:
- self._log_keyerror(oid_int, "no tid found(1)")
- raise POSKeyError(oid)
-
- # get state from the cache or the database
- cachekey = 'state:%d:%d' % (oid_int, tid_int)
- state = cache.get(cachekey)
- if not state:
- state = self._adapter.mover.load_revision(
- cursor, oid_int, tid_int)
- if state:
- state = str(state)
- cache.set(cachekey, state)
+ state = str(state or '')
+ if tid_int is not None:
+ # cache the result
+ tid = p64(tid_int)
+ if my_tid and my_tid != tid_int:
+ cache.set_multi({
+ state_key: tid + state,
+ backptr_key: tid,
+ })
+ else:
+ cache.set(state_key, tid + state)
finally:
self._lock_release()
if tid_int is not None:
- if state:
- state = str(state)
if not state:
# This can happen if something attempts to load
# an object whose creation has been undone.
@@ -399,7 +411,7 @@
raise POSKeyError(oid)
return state, p64(tid_int)
else:
- self._log_keyerror(oid_int, "no tid found(2)")
+ self._log_keyerror(oid_int, "no tid found")
raise POSKeyError(oid)
def loadEx(self, oid, version):
@@ -411,12 +423,6 @@
"""Load a specific revision of an object"""
oid_int = u64(oid)
tid_int = u64(serial)
- cache = self._cache_client
- if cache is not None:
- cachekey = 'state:%d:%d' % (oid_int, tid_int)
- state = cache.get(cachekey)
- if state:
- return state
self._lock_acquire()
try:
@@ -436,8 +442,6 @@
state = str(state)
if not state:
raise POSKeyError(oid)
- if cache is not None:
- cache.set(cachekey, state)
return state
else:
raise POSKeyError(oid)
Modified: relstorage/trunk/relstorage/tests/fakecache.py
===================================================================
--- relstorage/trunk/relstorage/tests/fakecache.py 2009-10-07 20:48:05 UTC (rev 104896)
+++ relstorage/trunk/relstorage/tests/fakecache.py 2009-10-07 20:55:27 UTC (rev 104897)
@@ -24,9 +24,15 @@
def get(self, key):
return data.get(key)
+ def get_multi(self, keys):
+ return dict((key, data.get(key)) for key in keys)
+
def set(self, key, value):
data[key] = value
+ def set_multi(self, d):
+ data.update(d)
+
def add(self, key, value):
if key not in data:
data[key] = value
Modified: relstorage/trunk/relstorage/tests/reltestbase.py
===================================================================
--- relstorage/trunk/relstorage/tests/reltestbase.py 2009-10-07 20:48:05 UTC (rev 104896)
+++ relstorage/trunk/relstorage/tests/reltestbase.py 2009-10-07 20:55:27 UTC (rev 104897)
@@ -235,24 +235,30 @@
self.assertEqual(c1._storage._cache_client.servers, ['x:1', 'y:2'])
fakecache.data.clear()
r1 = c1.root()
- # the root tid and state should now be cached
- self.assertEqual(len(fakecache.data), 2)
+ # the root state should now be cached
+ self.assertEqual(fakecache.data.keys(), ['state:0'])
r1['alpha'] = PersistentMapping()
self.assertFalse('commit_count' in fakecache.data)
transaction.commit()
self.assertTrue('commit_count' in fakecache.data)
- self.assertEqual(len(fakecache.data), 3)
+ self.assertEqual(sorted(fakecache.data.keys()),
+ ['commit_count', 'state:0'])
oid = r1['alpha']._p_oid
- self.assertEqual(len(fakecache.data), 3)
+ self.assertEqual(sorted(fakecache.data.keys()),
+ ['commit_count', 'state:0'])
got, serial = c1._storage.load(oid, '')
- # another tid and state should now be cached
- self.assertEqual(len(fakecache.data), 5)
+ # another state should now be cached
+ self.assertEqual(len(fakecache.data.keys()), 3)
- # load the object via loadSerial()
- got2 = c1._storage.loadSerial(oid, serial)
- self.assertEqual(got, got2)
+ # make a change
+ r1['beta'] = 0
+ transaction.commit()
+ got, serial = c1._storage.load(oid, '')
+ # a backpointer should now be cached
+ self.assertEqual(len(fakecache.data.keys()), 4)
+
# try to load an object that doesn't exist
self.assertRaises(KeyError, c1._storage.load, 'bad.oid.', '')
finally:
More information about the checkins
mailing list