[Zodb-checkins] SVN: ZODB/branches/perfmetrics/s Send pool sizes, load/store counts, and connection durations to Statsd.

Shane Hathaway cvs-admin at zope.org
Mon Sep 3 20:35:59 UTC 2012


Log message for revision 127681:
  Send pool sizes, load/store counts, and connection durations to Statsd.
  

Changed:
  U   ZODB/branches/perfmetrics/setup.py
  U   ZODB/branches/perfmetrics/src/ZODB/DB.py
  U   ZODB/branches/perfmetrics/src/ZODB/tests/testDB.py

-=-
Modified: ZODB/branches/perfmetrics/setup.py
===================================================================
--- ZODB/branches/perfmetrics/setup.py	2012-09-03 18:12:38 UTC (rev 127680)
+++ ZODB/branches/perfmetrics/setup.py	2012-09-03 20:35:55 UTC (rev 127681)
@@ -20,7 +20,7 @@
 interface, rich transaction support, and undo.
 """
 
-VERSION = "3.10dev"
+VERSION = "3.10.5-perfmetrics"
 
 from ez_setup import use_setuptools
 use_setuptools()
@@ -200,6 +200,7 @@
       tests_require = ['zope.testing', manuel_version],
       extras_require = dict(test=['zope.testing', manuel_version]),
       install_requires = [
+        'perfmetrics',
         transaction_version,
         'zc.lockfile',
         'ZConfig',

Modified: ZODB/branches/perfmetrics/src/ZODB/DB.py
===================================================================
--- ZODB/branches/perfmetrics/src/ZODB/DB.py	2012-09-03 18:12:38 UTC (rev 127680)
+++ ZODB/branches/perfmetrics/src/ZODB/DB.py	2012-09-03 20:35:55 UTC (rev 127681)
@@ -35,6 +35,7 @@
 from ZODB.interfaces import IMVCCStorage
 
 import transaction
+from perfmetrics import statsd_client
 
 from persistent.TimeStamp import TimeStamp
 
@@ -319,6 +320,20 @@
     return before
 
 
+class StatsdActivityMonitor(object):
+    # Implements the ActivityMonitor interface.
+
+    def closedConnection(self, connection):
+        client = statsd_client()
+        if client is not None:
+            database_name = connection.db().database_name or '_'
+            loads, stores = connection.getTransferCounts(True)
+            buf = []
+            client.incr('zodb.%s.loads' % database_name, loads, buf=buf)
+            client.incr('zodb.%s.stores' % database_name, stores, buf=buf)
+            client.sendbuf(buf)
+
+
 class DB(object):
     """The Object Database
     -------------------
@@ -362,7 +377,8 @@
     implements(IDatabase)
 
     klass = Connection  # Class to use for connections
-    _activity_monitor = next = previous = None
+    _activity_monitor = StatsdActivityMonitor()
+    next = previous = None
 
     def __init__(self, storage,
                  pool_size=7,
@@ -489,12 +505,23 @@
         self._a()
         try:
             assert connection._db is self
+
+            client = statsd_client()
+            if client is not None:
+                if connection.opened:
+                    elapsed = int((time.time() - connection.opened) * 1000.0)
+                    dbname = self.database_name or '_'
+                    client.timing('zodb.%s.duration' % dbname, elapsed)
+
             connection.opened = None
 
             if connection.before:
                 self.historical_pool.repush(connection, connection.before)
             else:
                 self.pool.repush(connection)
+
+            if client is not None:
+                self._reportPoolInfo()
         finally:
             self._r()
 
@@ -755,6 +782,9 @@
             self.pool.availableGC()
             self.historical_pool.availableGC()
 
+            if statsd_client() is not None:
+                self._reportPoolInfo()
+
             return result
 
         finally:
@@ -790,6 +820,38 @@
     def getActivityMonitor(self):
         return self._activity_monitor
 
+    def _reportPoolInfo(self):
+        """Report pool stats to statsd.
+
+        The lock must be acquired when this is called.
+        """
+        client = statsd_client()
+        if client is None:
+            return
+
+        buf = []
+        dbname = self.database_name or '_'
+
+        for pool, prefix in [(self.pool, ''),
+                             (self.historical_pool, 'hist_')]:
+            name = 'zodb.%s.%s' % (dbname, prefix)
+            conns = []
+            pool.all.map(conns.append)
+            opened = 0
+            estsize = 0
+            nonghosts = 0
+            for c in conns:
+                if c.opened:
+                    opened += 1
+                estsize += c._cache.total_estimated_size
+                nonghosts += c._cache.cache_non_ghost_count
+            client.gauge(name + 'pool', len(conns), buf=buf)
+            client.gauge(name + 'opened', opened, buf=buf)
+            client.gauge(name + 'estsize', estsize, buf=buf)
+            client.gauge(name + 'nonghosts', nonghosts, buf=buf)
+
+        client.sendbuf(buf)
+
     def pack(self, t=None, days=0):
         """Pack the storage, deleting unused object revisions.
 

Modified: ZODB/branches/perfmetrics/src/ZODB/tests/testDB.py
===================================================================
--- ZODB/branches/perfmetrics/src/ZODB/tests/testDB.py	2012-09-03 18:12:38 UTC (rev 127680)
+++ ZODB/branches/perfmetrics/src/ZODB/tests/testDB.py	2012-09-03 20:35:55 UTC (rev 127681)
@@ -64,7 +64,64 @@
         import ZODB.serialize
         self.assert_(self.db.references is ZODB.serialize.referencesf)
 
+    def test_open_and_close_with_statsd_client(self):
+        from perfmetrics import statsd_client_stack
+        sent = []
 
+        class DummyStatsdClient:
+            def gauge(self, name, value, buf):
+                buf.append('%s:%s|g' % (name, value))
+
+            def incr(self, name, value, buf):
+                buf.append('%s:%s|c' % (name, value))
+
+            def timing(self, name, value):
+                sent.append('%s:%s|ms' % (name, value))
+
+            def sendbuf(self, buf):
+                sent.append(buf)
+
+        client = DummyStatsdClient()
+        statsd_client_stack.push(client)
+        try:
+            c = self.db.open()
+            c.root().get('x')
+            time.sleep(0.1)
+            c.close()
+        finally:
+            statsd_client_stack.pop()
+
+        self.maxDiff = 10000
+        self.assertEqual(len(sent), 4)
+        self.assertEqual(sent[0],
+                         ['zodb.unnamed.pool:1|g',
+                          'zodb.unnamed.opened:1|g',
+                          'zodb.unnamed.estsize:0|g',
+                          'zodb.unnamed.nonghosts:0|g',
+                          'zodb.unnamed.hist_pool:0|g',
+                          'zodb.unnamed.hist_opened:0|g',
+                          'zodb.unnamed.hist_estsize:0|g',
+                          'zodb.unnamed.hist_nonghosts:0|g'])
+
+        import re
+        mo = re.match(r'zodb.unnamed.duration:([0-9\.]+)|ms', sent[1])
+        self.assertTrue(mo is not None, 'Regex failed to match %r' % sent[1])
+        self.assertTrue(int(mo.group(1)) > 0)
+
+        self.assertEqual(sent[2],
+                         ['zodb.unnamed.pool:1|g',
+                          'zodb.unnamed.opened:0|g',
+                          'zodb.unnamed.estsize:64|g',
+                          'zodb.unnamed.nonghosts:1|g',
+                          'zodb.unnamed.hist_pool:0|g',
+                          'zodb.unnamed.hist_opened:0|g',
+                          'zodb.unnamed.hist_estsize:0|g',
+                          'zodb.unnamed.hist_nonghosts:0|g'])
+        self.assertEqual(sent[3],
+                         ['zodb.unnamed.loads:1|c',
+                          'zodb.unnamed.stores:0|c'])
+
+
 def test_invalidateCache():
     """The invalidateCache method invalidates a connection caches for all of
     the connections attached to a database::



More information about the Zodb-checkins mailing list