[Checkins] SVN: lovely.memcached/trunk/src/lovely/memcached/ keys method impl

[Bernd Dorn]

Log message for revision 73979:
  keys method impl

Changed:
  U   lovely.memcached/trunk/src/lovely/memcached/README.txt
  U   lovely.memcached/trunk/src/lovely/memcached/interfaces.py
  U   lovely.memcached/trunk/src/lovely/memcached/utility.py

-=-
Modified: lovely.memcached/trunk/src/lovely/memcached/README.txt
===================================================================
--- lovely.memcached/trunk/src/lovely/memcached/README.txt	2007-04-02 10:30:35 UTC (rev 73978)
+++ lovely.memcached/trunk/src/lovely/memcached/README.txt	2007-04-02 11:43:54 UTC (rev 73979)
@@ -137,16 +137,10 @@
   >>> sorted(util3.keys(u'3'))
   [5]
 
-  >>> t = time.time()
-  >>> for i in range(1000):
-  ...     util3.set(i, i, ns=u'speed')
-  >>> #print time.time()-t
-  >>> thread = threading.Thread(target=util3.keys, args=(u'speed',))
-  >>> t = time.time()
-  >>> thread.start()
-  >>> thread.join()
-  >>> print time.time()-t
+This is just for an internal test, it updates the key records on the
+server.
 
+  >>> util3._keysUpdate([1,2], u'speed')
 
 Statistics
 ==========

Modified: lovely.memcached/trunk/src/lovely/memcached/interfaces.py
===================================================================
--- lovely.memcached/trunk/src/lovely/memcached/interfaces.py	2007-04-02 10:30:35 UTC (rev 73978)
+++ lovely.memcached/trunk/src/lovely/memcached/interfaces.py	2007-04-02 11:43:54 UTC (rev 73979)
@@ -70,3 +70,8 @@
     def invalidateAll():
         """invalidates all data of the memcached servers, not that all
         namespaces are invalidated"""
+
+    def keys(ns=None):
+        """if trackKeys is True, returns the keys defined in the
+        namespace"""
+        

Modified: lovely.memcached/trunk/src/lovely/memcached/utility.py
===================================================================
--- lovely.memcached/trunk/src/lovely/memcached/utility.py	2007-04-02 10:30:35 UTC (rev 73978)
+++ lovely.memcached/trunk/src/lovely/memcached/utility.py	2007-04-02 11:43:54 UTC (rev 73979)
@@ -33,7 +33,10 @@
 
 log = logging.getLogger('lovely.memcached')
 
+# base namespace for key management
 NS = 'lovely.memcached'
+# namespace for key timesamps
+STAMP_NS = NS + '.stamps'
 
 class MemcachedClient(persistent.Persistent):
     interface.implements(IMemcachedClient)
@@ -163,6 +166,8 @@
     def _keysInit(self, storage):
         storage.keys = {}
         storage.uid = random.randint(0, sys.maxint)
+        storage.dirtyKeys = set()
+        storage.lastUpdates = {}
         clients = self._getClients()
         if not storage.uid in clients:
             clients.add(storage.uid)
@@ -171,25 +176,15 @@
 
     def _keysSet(self, key, ns, lifetime):
         """track a key"""
-        if not self.trackKeys or ns==NS: return
+        if not self.trackKeys or ns in (NS, STAMP_NS): return
         s = self.storage
         keys = s.keys.get(ns)
-        t = time.time()
-        if lifetime!=0:
-            tEnd = t + lifetime
-        else:
-            tEnd = 0
         if keys is None:
-            keys = set([(key, tEnd)])
+            keys = set([key])
             s.keys[ns] = keys
-        elif key in keys:
-            return
-        else:
-            keys.add((key, tEnd))
-        for key, eol in keys:
-            if eol == 0 or eol>t:
-                continue
-            s.remove((key, eol))
+        elif not key in keys:
+            keys.add(key)
+        self.set(s.uid, (ns, key), lifetime=lifetime, ns=STAMP_NS)
         self.set(keys, (s.uid, ns), lifetime=0, ns=NS)
 
     def _getClients(self):
@@ -201,15 +196,42 @@
         res = set()
         s = self.storage
         t = time.time()
+        localKeys = s.keys.get(ns, set())
         for client in self._getClients():
             if client == s.uid:
-                v = s.keys.get(ns, [])
+                v = localKeys
             else:
-                v = self.query((client, ns), default=[], ns=NS)
-            for k, eol in v:
-                if eol == 0 or eol>t:
-                    res.add(k)
+                v = self.query((client, ns), default=set(), ns=NS)
+            res.update(v)
+        # look at the timestamps
+        changed = False
+        for k in list(res):
+            uid = self.query((ns, k), ns=STAMP_NS)
+            if uid is None:
+                if localKeys:
+                    changed=True
+                    localKeys.discard(k)
+                res.discard(k)
+        if changed:
+            # update the server, we do this just here, because the
+            # keys method always looks at the stamps, so it is not
+            # required to delete keys from the stored keylist at key
+            # setting time
+            s.dirtyKeys.add(ns)
+            self._keysUpdate(localKeys, ns)
         return res
-                
-    
         
+
+    def _keysUpdate(self, keys, ns):
+        # updates the key set of this thread on server
+        s = self.storage
+        if not ns in s.dirtyKeys:
+            return
+        t = time.time()
+        # we update only every 5 minutes
+        if s.lastUpdates.get(ns, 0) + 300 > t:
+            return
+        self.set(keys, (s.uid, ns), lifetime=0, ns=NS)
+        s.dirtyKeys.discard(ns)
+        s.lastUpdates[ns] = t
+