[Zodb-checkins] CVS: ZODB3/ZEO - cache.py:1.1.2.7 stats.py:1.21.20.1

Jeremy Hylton cvs-admin at zope.org
Wed Nov 19 13:38:49 EST 2003


Update of /cvs-repository/ZODB3/ZEO
In directory cvs.zope.org:/tmp/cvs-serv28651

Modified Files:
      Tag: ZODB3-mvcc-2-branch
	cache.py stats.py 
Log Message:
Initial attempt to implement a cache trace for MVCC.

The cache implementation is very different, so the cache trace format
is too.  The trace stores start and end transaction ids, if present.
There are many fewer codes, because version and non-version data are
handled independently.


=== ZODB3/ZEO/cache.py 1.1.2.6 => 1.1.2.7 ===
--- ZODB3/ZEO/cache.py:1.1.2.6	Thu Nov 13 13:44:58 2003
+++ ZODB3/ZEO/cache.py	Wed Nov 19 13:38:48 2003
@@ -14,8 +14,10 @@
 """Example client cache that stores multiple revisions of an object."""
 
 import bisect
+import logging
 import os
 import struct
+import time
 
 from sets import Set
 
@@ -67,9 +69,15 @@
     # @param path path of persistent snapshot of cache state
     # @param size maximum size of object data, in bytes
 
-    def __init__(self, path=None, size=None):
+    def __init__(self, path=None, size=None, trace=True):
         self.path = path
         self.size = size
+        self.log = logging.getLogger("zeo.cache")
+
+        if trace:
+            self._setup_trace()
+        else:
+            self._trace = self._notrace
         
         # Last transaction seen by the cache, either via setLastTid()
         # or by invalidate().
@@ -176,10 +184,12 @@
         if tid is None:
             tid = self.current.get(oid)
         if tid is None:
+            self._trace(0x20, oid, version)
             return None
         o = self.dll.access((oid, tid))
         if o is None:
             return None
+        self._trace(0x22, oid, version)
         return o.data, o.serialno, tid
 
     ##
@@ -192,6 +202,7 @@
     def loadNonCurrent(self, oid, tid):
         L = self.noncurrent.get(oid)
         if L is None:
+            self._trace(0x24, oid, tid)
             return None
         # A pair with None as the second element will always be less
         # than any pair with the same first tid.
@@ -199,12 +210,15 @@
         # The least element left of tid was written before tid.  If
         # there is no element, the cache doesn't have old enough data.
         if i == 0:
+            self._trace(0x24, oid, tid)
             return
         lo, hi = L[i-1]
         # XXX lo should always be less than tid
         if not lo < tid <= hi:
+            self._trace(0x24, oid, tid)
             return None
         o = self.dll.access((oid, lo))
+        self._trace(0x26, oid, tid)
         return o.data, o.serialno, o.start_tid, o.end_tid
 
     ##
@@ -240,15 +254,29 @@
         if version:
             if end_tid is not None:
                 raise ValueError("cache only stores current version data")
+            if oid in self.version:
+                if self.version[oid] != (version, start_tid):
+                    raise ValueError("data already exists for version %r"
+                                     % self.version[oid][0])
             self.version[oid] = version, start_tid
+            self._trace(0x50, oid, version, start_tid)
         else:
             if end_tid is None:
-                if oid in self.current:
-                    raise ValueError("already have current data for oid")
+                _cur_start = self.current.get(oid)
+                if _cur_start:
+                    if _cur_start != start_tid:
+                        raise ValueError("already have current data for oid")
+                    else:
+                        return
                 self.current[oid] = start_tid
+                self._trace(0x52, oid, version, start_tid)
             else:
                 L = self.noncurrent.setdefault(oid, [])
+                p = start_tid, end_tid
+                if p in L:
+                    return # duplicate store
                 bisect.insort_left(L, (start_tid, end_tid))
+                self._trace(0x54, oid, version, start_end, end_tid)
         self.dll.add(o)
 
     ##
@@ -263,16 +291,20 @@
             self.tid = tid
         if version:
             if oid in self.version:
+                self._trace(0x1A, oid, version, tid)
                 del self.version[oid]
             return
         if oid not in self.current:
+            self._trace(0x10, oid, version, tid)
             return
         cur_tid = self.current.pop(oid)
         # XXX Want to fetch object without marking it as accessed
         o = self.dll.access((oid, cur_tid))
         if o is None:
+            # XXX is this possible?
             return None
         o.end_tid = tid
+        self._trace(0x1C, oid, version, tid)
         L = self.noncurrent.setdefault(oid, [])
         bisect.insort_left(L, (cur_tid, tid))
 
@@ -310,6 +342,39 @@
             L = self.noncurrent[oid]
             L.remove((o.start_tid, o.end_tid))
 
+    def _setup_trace(self):
+        tfn = self.path + ".trace"
+        self.tracefile = None
+        try:
+            self.tracefile = open(tfn, "ab")
+            self._trace(0x00)
+        except IOError, msg:
+            self.tracefile = None
+            self.log.warning("Could not write to trace file %s: %s",
+                             tfn, msg)
+
+    def _notrace(self, *arg):
+        pass
+
+    def _trace(self,
+               code, oid="", version="", tid="", end_tid="", dlen=0,
+               # The next two are just speed hacks.
+               time_time=time.time, struct_pack=struct.pack):
+        # The code argument is two hex digits; bits 0 and 7 must be zero.
+        # The first hex digit shows the operation, the second the outcome.
+        # If the second digit is in "02468" then it is a 'miss'.
+        # If it is in "ACE" then it is a 'hit'.
+        # This method has been carefully tuned to be as fast as possible.
+        # Note: when tracing is disabled, this method is hidden by a dummy.
+        if version:
+            code |= 0x80
+        encoded = (dlen + 255) & 0x7fffff00 | code 
+        self.tracefile.write(
+            struct_pack(">iiH8s8s",
+                        time_time(),
+                        encoded,
+                        len(oid),
+                        tid, end_tid) + oid)
 ##
 # An object that's part of a headed, circular, doubly-linked list.
 # Because it's doubly linked, an object can be removed from the list
@@ -585,3 +650,4 @@
         self.worthsets[obj.worth].discard(obj)
         del self.key2object[obj.key]
         self._unlink(obj)
+


=== ZODB3/ZEO/stats.py 1.21 => 1.21.20.1 ===
--- ZODB3/ZEO/stats.py:1.21	Tue Jun 10 13:08:10 2003
+++ ZODB3/ZEO/stats.py	Wed Nov 19 13:38:48 2003
@@ -157,12 +157,12 @@
                 if not quiet:
                     print "Skipping 8 bytes at offset", offset-8
                 continue
-            r = f_read(10)
+            r = f_read(18)
             if len(r) < 10:
                 break
             offset += 10
             records += 1
-            oidlen, serial = struct_unpack(">H8s", r)
+            oidlen, start_tid, end_tid = struct_unpack(">H8s8s", r)
             oid = f_read(oidlen)
             if len(oid) != oidlen:
                 break
@@ -203,12 +203,13 @@
                     bysizew[dlen] = d = bysizew.get(dlen) or {}
                     d[oid] = d.get(oid, 0) + 1
             if verbose:
-                print "%s %d %02x %s %016x %1s %s" % (
+                print "%s %d %02x %s %016x %016x %1s %s" % (
                     time.ctime(ts)[4:-5],
                     current,
                     code,
                     oid_repr(oid),
-                    U64(serial),
+                    U64(start_tid),
+                    U64(end_tid),
                     version,
                     dlen and str(dlen) or "")
             if code & 0x70 == 0x20:
@@ -376,31 +377,18 @@
     0x00: "_setup_trace (initialization)",
 
     0x10: "invalidate (miss)",
-    0x1A: "invalidate (hit, version, writing 'n')",
-    0x1C: "invalidate (hit, writing 'i')",
+    0x1A: "invalidate (hit, version)",
+    0x1C: "invalidate (hit, saving non-current)",
 
     0x20: "load (miss)",
-    0x22: "load (miss, version, status 'n')",
-    0x24: "load (miss, deleting index entry)",
-    0x26: "load (miss, no non-version data)",
-    0x28: "load (miss, version mismatch, no non-version data)",
-    0x2A: "load (hit, returning non-version data)",
-    0x2C: "load (hit, version mismatch, returning non-version data)",
-    0x2E: "load (hit, returning version data)",
-
-    0x3A: "update",
-
-    0x40: "modifiedInVersion (miss)",
-    0x4A: "modifiedInVersion (hit, return None, status 'n')",
-    0x4C: "modifiedInVersion (hit, return '')",
-    0x4E: "modifiedInVersion (hit, return version)",
-
-    0x5A: "store (non-version data present)",
-    0x5C: "store (only version data present)",
-
-    0x6A: "_copytocurrent",
-
-    0x70: "checkSize (cache flip)",
+    0x22: "load (hit)",
+    0x24: "load (non-current, miss)",
+    0x26: "load (non-current, hit)",
+
+    0x50: "store (version)",
+    0x52: "store (current, non-version)",
+    0x54: "store (non-current)",
+    
     }
 
 if __name__ == "__main__":




More information about the Zodb-checkins mailing list