[ZODB-Dev] Errors running a Stress Test

[Brandon Ehle]

I've used ZODB w/ ZEO in the past, and I wrote a quick stress test to try
and test for bugs that I've encountered in the past.  I'm currently
running into lots of bugs that I haven't seen before, so perhaps I'm doing
something wrong that I didn't notice, but I was curious if anyone else was
seeing these problems.

After running for a couple hours I got all sorts of errors.  Am I doing
something obviously wrong, or am I just tickling ZODB in the wrong way?


***** Client side errors

Exception in thread Thread-1:
Traceback (most recent call last):
  File "C:\Python23\lib\threading.py", line 436, in __bootstrap
    self.run()
  File "C:\Python23\lib\threading.py", line 416, in run
    self.__target(*self.__args, **self.__kwargs)
  File "nightrun.py", line 55, in hammer_the_db
    random.choice([commit] + [abort])()
  File "nightrun.py", line 28, in commit
    get_transaction().commit()
  File "C:\Python23\Lib\site-packages\ZODB\Transaction.py", line 254, in
commit
    self._cleanup(objects, ncommitted, jars, subjars)
  File "C:\Python23\Lib\site-packages\ZODB\Transaction.py", line 250, in
commit
    self._finish_one(jars[0])
  File "C:\Python23\Lib\site-packages\ZODB\Transaction.py", line 359, in
_finish_one
    jar.tpc_finish(self)
  File "C:\Python23\Lib\site-packages\ZODB\Connection.py", line 678, in
tpc_finish
    self._storage.tpc_finish(transaction, callback)
  File "C:\Python23\Lib\site-packages\ZEO\ClientStorage.py", line 962, in
tpc_finish
    self._update_cache(tid)
  File "C:\Python23\Lib\site-packages\ZEO\ClientStorage.py", line 994, in
_update_cache
    self._cache.store(oid, version, s, None, data)
  File "C:\Python23\Lib\site-packages\ZEO\cache.py", line 276, in store
    self.fc.add(o)
  File "C:\Python23\Lib\site-packages\ZEO\cache.py", line 781, in add
    available = self._makeroom(size)
  File "C:\Python23\Lib\site-packages\ZEO\cache.py", line 725, in _makeroom
    size, e = self.filemap.pop(ofs)
KeyError: 301258

***** Server side errors

2004-03-15T11:46:52 ERROR(200) Conflict Resolution Unexpected error
Traceback (most recent call last):
  File "C:\Python23\Lib\site-packages\ZODB\ConflictResolution.py", line
108, in tryToResolveConflict
    old = state(self, oid, oldSerial, prfactory)
  File "C:\Python23\Lib\site-packages\ZODB\ConflictResolution.py", line
38, in state
    p = p or self.loadSerial(oid, serial)
  File "C:\Python23\Lib\site-packages\ZODB\FileStorage\FileStorage.py",
line 581, in loadSerial
    raise POSKeyError(oid)
POSKeyError: 0000000000000001

***** Test Setup

Win2K SP4
Python-2.3.3
ZODB-3.3a2

Running the server:

  Python23/Lib/site-packages/ZEO/runzeo.py -C server.conf

Running the client on multiple machines with 5 threads:

  stress.py 5

***** server.conf

<zeo>
    address 9999
</zeo>

<filestorage 1>
    path zodb.fs
</filestorage>

***** client.conf

<zodb>
    <zeoclient>
        server behle731.rws.ad.ea.com:9999
        client zodb-cache
    </zeoclient>
</zodb>

***** stress.py

import ZODB.config
from ZODB.POSException import ConflictError
from ZEO.Exceptions import ClientDisconnected

from persistent import Persistent
from persistent.list import PersistentList
from persistent.mapping import PersistentMapping
from persistent.TimeStamp import TimeStamp
from BTrees.OOBTree import OOBTree

import sys
import os
import random
import thread
import threading
import time
import traceback

db = ZODB.config.databaseFromURL('client.conf')

# Stress test the database

def hammer_the_db():
    def commit():
        get_transaction().commit()
        tid = thread.get_ident()
        sys.stdout.write(chr(tid%26+97))

    def abort():
        get_transaction().abort()
        sys.stdout.write('.')

    conn = db.open()
    if 'test' not in conn.root():
        conn.root()['test'] = OOBTree()
        get_transaction().commit()

    test = conn.root()['test']

    while 1:
        try:
            time.sleep(random.random())
            conn.sync()
            get_transaction().begin()
            keys = ("yourmom", "yourdad", "yourbrother", "yoursister")
            test[random.choice(keys)] = "hi"
            random.choice([commit] + [abort] * 10)()

            get_transaction().begin()
            for k,v in test.items():
                test[k] = v+"bye"
                random.choice([commit] + [abort])()
        except ConflictError:
            sys.stdout.write('C')
            continue
        except ClientDisconnected:
            sys.stdout.write('D')
            continue

for i in range(0, int(sys.argv[1])):
    t = threading.Thread(target=hammer_the_db)
    t.start()

conn = db.open()
while threading.activeCount()>1:
    try:
        sys.stdout.write("%d/%d" % (threading.activeCount()-1, db.getSize()))
        db.pack()
        sys.stdout.write("/%d" % db.getSize())
        time.sleep(random.uniform(60, 500))
    except ClientDisconnected:
        sys.stdout.write('D')
        continue
    except:
        traceback.print_exc()