[Checkins] SVN: lovely.memcached/trunk/src/lovely/memcached/
intermediat safety checkin for key method implementation,
have to try another way to do it
Bernd Dorn
bernd.dorn at lovelysystems.com
Mon Apr 2 06:30:37 EDT 2007
Log message for revision 73978:
intermediat safety checkin for key method implementation, have to try another way to do it
Changed:
U lovely.memcached/trunk/src/lovely/memcached/README.txt
U lovely.memcached/trunk/src/lovely/memcached/interfaces.py
U lovely.memcached/trunk/src/lovely/memcached/utility.py
-=-
Modified: lovely.memcached/trunk/src/lovely/memcached/README.txt
===================================================================
--- lovely.memcached/trunk/src/lovely/memcached/README.txt 2007-04-02 08:31:27 UTC (rev 73977)
+++ lovely.memcached/trunk/src/lovely/memcached/README.txt 2007-04-02 10:30:35 UTC (rev 73978)
@@ -75,6 +75,79 @@
>>> util1.query(1) is util2.query(2) is None
True
+Getting existing keys
+=====================
+
+The memcached daemon does not provide the ability to retrieve a list
+of all keys that are stored. In the utility this is implemented.
+
+ >>> util1.keys()
+ Traceback (most recent call last):
+ ...
+ NotImplementedError: trackKeys not enabled
+
+The key tracking adds on overhead so it must be enabled explicitly.
+
+ >>> util3 = MemcachedClient(trackKeys=True)
+ >>> util3.set(1,1)
+ >>> sorted(util3.keys())
+ [1]
+ >>> util3.set(2,2)
+ >>> sorted(util3.keys())
+ [1, 2]
+
+Keys are global on memcached daemons. In order to test this we need to
+have multiple threads.
+
+ >>> import threading
+ >>> log = []
+
+Each thread has a differnt thread.
+
+ >>> def differentConn():
+ ... util3.set(3,3)
+ ... log.append(sorted(util3.keys()))
+ ...
+ >>> thread = threading.Thread(target=differentConn)
+ >>> thread.start()
+ >>> thread.join()
+ >>> log
+ [[1, 2, 3]]
+
+Keys expire too
+
+ >>> util3.set(4, 4, lifetime=1)
+ >>> sorted(util3.keys())
+ [1, 2, 3, 4]
+ >>> import time
+ >>> time.sleep(2)
+ >>> sorted(util3.keys())
+ [1, 2, 3]
+ >>> util3.query(4) is None
+ True
+
+Keys are always bound to a namespace.
+
+ >>> util3.set(5, 5, ns=u'3')
+
+If not give the ``None`` namespace is used.
+
+ >>> sorted(util3.keys())
+ [1, 2, 3]
+ >>> sorted(util3.keys(u'3'))
+ [5]
+
+ >>> t = time.time()
+ >>> for i in range(1000):
+ ... util3.set(i, i, ns=u'speed')
+ >>> #print time.time()-t
+ >>> thread = threading.Thread(target=util3.keys, args=(u'speed',))
+ >>> t = time.time()
+ >>> thread.start()
+ >>> thread.join()
+ >>> print time.time()-t
+
+
Statistics
==========
Modified: lovely.memcached/trunk/src/lovely/memcached/interfaces.py
===================================================================
--- lovely.memcached/trunk/src/lovely/memcached/interfaces.py 2007-04-02 08:31:27 UTC (rev 73977)
+++ lovely.memcached/trunk/src/lovely/memcached/interfaces.py 2007-04-02 10:30:35 UTC (rev 73978)
@@ -44,6 +44,12 @@
default = 3600,
)
+ trackKeys = schema.Bool(
+ title = u'Track Keys',
+ description = u'Enable the keys method',
+ required = False,
+ default = False,
+ )
def getStatistics():
"""returns the memcached stats"""
Modified: lovely.memcached/trunk/src/lovely/memcached/utility.py
===================================================================
--- lovely.memcached/trunk/src/lovely/memcached/utility.py 2007-04-02 08:31:27 UTC (rev 73977)
+++ lovely.memcached/trunk/src/lovely/memcached/utility.py 2007-04-02 10:30:35 UTC (rev 73978)
@@ -18,6 +18,9 @@
import md5
+import random
+import sys
+import time
import logging
import memcache
import cPickle
@@ -30,21 +33,27 @@
log = logging.getLogger('lovely.memcached')
+NS = 'lovely.memcached'
+
class MemcachedClient(persistent.Persistent):
interface.implements(IMemcachedClient)
defaultNS = FieldProperty(IMemcachedClient['defaultNS'])
servers = FieldProperty(IMemcachedClient['servers'])
- defaultLifetime = FieldProperty(IMemcachedClient['defaultLifetime'])
+ defaultLifetime = FieldProperty(
+ IMemcachedClient['defaultLifetime'])
+ trackKeys = FieldProperty(IMemcachedClient['trackKeys'])
def __init__(self, servers=None, defaultAge=None,
- defaultNS=None):
+ defaultNS=None, trackKeys=None):
if servers is not None:
self.servers = servers
if defaultAge is not None:
self.defaultAge = defaultAge
if defaultNS is not None:
self.defaultNS = defaultNS
+ if trackKeys is not None:
+ self.trackKeys = trackKeys
def getStatistics(self):
return self.client.get_stats()
@@ -55,9 +64,13 @@
ns = ns or self.defaultNS or None
data = cPickle.dumps(data)
- log.debug('set: %r, %r, %r, %r' % (key, len(data), ns, lifetime))
+ log.debug('set: %r, %r, %r, %r' % (key,
+ len(data), ns,
+ lifetime))
+
self.client.set(self._buildKey(key, ns), data, lifetime)
-
+ self._keysSet(key, ns, lifetime)
+
def query(self, key, default=None, ns=None):
ns = ns or self.defaultNS or None
res = self.client.get(self._buildKey(key, ns))
@@ -142,5 +155,61 @@
# thread.
if not hasattr(self, '_v_storage'):
self._v_storage = local()
+ if self.trackKeys and not hasattr(self._v_storage, 'keys'):
+ self._keysInit(self._v_storage)
return self._v_storage
+
+ def _keysInit(self, storage):
+ storage.keys = {}
+ storage.uid = random.randint(0, sys.maxint)
+ clients = self._getClients()
+ if not storage.uid in clients:
+ clients.add(storage.uid)
+ self.set(clients, 'clients', lifetime=0, ns=NS)
+
+
+ def _keysSet(self, key, ns, lifetime):
+ """track a key"""
+ if not self.trackKeys or ns==NS: return
+ s = self.storage
+ keys = s.keys.get(ns)
+ t = time.time()
+ if lifetime!=0:
+ tEnd = t + lifetime
+ else:
+ tEnd = 0
+ if keys is None:
+ keys = set([(key, tEnd)])
+ s.keys[ns] = keys
+ elif key in keys:
+ return
+ else:
+ keys.add((key, tEnd))
+ for key, eol in keys:
+ if eol == 0 or eol>t:
+ continue
+ s.remove((key, eol))
+ self.set(keys, (s.uid, ns), lifetime=0, ns=NS)
+
+ def _getClients(self):
+ return self.query('clients', set(), ns=NS)
+
+ def keys(self, ns=None):
+ if not self.trackKeys:
+ raise NotImplementedError, "trackKeys not enabled"
+ res = set()
+ s = self.storage
+ t = time.time()
+ for client in self._getClients():
+ if client == s.uid:
+ v = s.keys.get(ns, [])
+ else:
+ v = self.query((client, ns), default=[], ns=NS)
+ for k, eol in v:
+ if eol == 0 or eol>t:
+ res.add(k)
+ return res
+
+
+
More information about the Checkins
mailing list