[Checkins] SVN: relstorage/trunk/relstorage/ Oracle history-free storage now works.
Shane Hathaway
shane at hathawaymix.org
Wed Sep 30 18:52:22 EDT 2009
Log message for revision 104662:
Oracle history-free storage now works.
Also copied the MySQL-specific pack optimizations to the history-free
packer and updated the speed tests to use the newest test database
connection conventions.
Changed:
U relstorage/trunk/relstorage/adapters/mover.py
U relstorage/trunk/relstorage/adapters/mysql.py
U relstorage/trunk/relstorage/adapters/oracle.py
U relstorage/trunk/relstorage/adapters/packundo.py
U relstorage/trunk/relstorage/adapters/txncontrol.py
U relstorage/trunk/relstorage/tests/README.txt
U relstorage/trunk/relstorage/tests/packstresstest.py
U relstorage/trunk/relstorage/tests/speedtest.py
U relstorage/trunk/relstorage/tests/testoracle.py
-=-
Modified: relstorage/trunk/relstorage/adapters/mover.py
===================================================================
--- relstorage/trunk/relstorage/adapters/mover.py 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/adapters/mover.py 2009-09-30 22:52:21 UTC (rev 104662)
@@ -467,20 +467,20 @@
if len(data) <= 2000:
# Send data inline for speed. Oracle docs say maximum size
# of a RAW is 2000 bytes. inputsize_BINARY corresponds with RAW.
- cursor.setinputsizes(rawdata=self.inputsize_BINARY)
stmt = """
INSERT INTO temp_store (zoid, prev_tid, md5, state)
VALUES (:oid, :prev_tid, :md5sum, :rawdata)
"""
+ cursor.setinputsizes(rawdata=self.inputsize_BINARY)
cursor.execute(stmt, oid=oid, prev_tid=prev_tid,
md5sum=md5sum, rawdata=data)
else:
# Send data as a BLOB
- cursor.setinputsizes(blobdata=self.inputsize_BLOB)
stmt = """
INSERT INTO temp_store (zoid, prev_tid, md5, state)
VALUES (:oid, :prev_tid, :md5sum, :blobdata)
"""
+ cursor.setinputsizes(blobdata=self.inputsize_BLOB)
cursor.execute(stmt, oid=oid, prev_tid=prev_tid,
md5sum=md5sum, blobdata=data)
@@ -532,16 +532,16 @@
md5sum = compute_md5sum(data)
else:
md5sum = None
- cursor.setinputsizes(data=self.inputsize_BLOB)
stmt = """
UPDATE temp_store SET
prev_tid = :prev_tid,
md5 = :md5sum,
- state = :data
+ state = :blobdata
WHERE zoid = :oid
"""
+ cursor.setinputsizes(blobdata=self.inputsize_BLOB)
cursor.execute(stmt, oid=oid, prev_tid=prev_tid,
- md5sum=md5sum, data=self.Binary(data))
+ md5sum=md5sum, blobdata=self.Binary(data))
@@ -575,13 +575,13 @@
WHERE zoid = %s
"""
cursor.execute(stmt, (oid,))
+ if data:
+ stmt = """
+ INSERT INTO object_state (zoid, tid, state)
+ VALUES (%s, %s, decode(%s, 'base64'))
+ """
+ cursor.execute(stmt, (oid, tid, encoded))
- stmt = """
- INSERT INTO object_state (zoid, tid, state)
- VALUES (%s, %s, decode(%s, 'base64'))
- """
- cursor.execute(stmt, (oid, tid, encoded))
-
def mysql_restore(self, cursor, oid, tid, data):
"""Store an object directly, without conflict detection.
@@ -606,18 +606,18 @@
"""
cursor.execute(stmt, (oid, tid, oid, md5sum, encoded))
else:
- if not data:
+ if data:
stmt = """
+ REPLACE INTO object_state (zoid, tid, state)
+ VALUES (%s, %s, %s)
+ """
+ cursor.execute(stmt, (oid, tid, encoded))
+ else:
+ stmt = """
DELETE FROM object_state
WHERE zoid = %s
"""
cursor.execute(stmt, (oid,))
- else:
- stmt = """
- REPLACE INTO object_state (zoid, tid, state)
- VALUES (%s, %s, %s)
- """
- cursor.execute(stmt, (oid, tid, encoded))
def oracle_restore(self, cursor, oid, tid, data):
"""Store an object directly, without conflict detection.
@@ -628,16 +628,12 @@
md5sum = compute_md5sum(data)
else:
md5sum = None
- stmt = """
- DELETE FROM object_state
- WHERE zoid = %s
- """
+ stmt = "DELETE FROM object_state WHERE zoid = :1"
cursor.execute(stmt, (oid,))
if not data or len(data) <= 2000:
# Send data inline for speed. Oracle docs say maximum size
# of a RAW is 2000 bytes. inputsize_BINARY corresponds with RAW.
- cursor.setinputsizes(rawdata=self.inputsize_BINARY)
if self.keep_history:
stmt = """
INSERT INTO object_state (zoid, tid, prev_tid, md5, state)
@@ -646,17 +642,19 @@
(SELECT tid FROM current_object WHERE zoid = :oid), 0),
:md5sum, :rawdata)
"""
+ cursor.setinputsizes(rawdata=self.inputsize_BINARY)
cursor.execute(stmt, oid=oid, tid=tid,
md5sum=md5sum, rawdata=data)
else:
- stmt = """
- INSERT INTO object_state (zoid, tid, state)
- VALUES (:oid, :tid, :rawdata)
- """
- cursor.execute(stmt, oid=oid, tid=tid, rawdata=data)
+ if data:
+ stmt = """
+ INSERT INTO object_state (zoid, tid, state)
+ VALUES (:oid, :tid, :rawdata)
+ """
+ cursor.setinputsizes(rawdata=self.inputsize_BINARY)
+ cursor.execute(stmt, oid=oid, tid=tid, rawdata=data)
else:
# Send data as a BLOB
- cursor.setinputsizes(blobdata=self.inputsize_BLOB)
if self.keep_history:
stmt = """
INSERT INTO object_state (zoid, tid, prev_tid, md5, state)
@@ -665,13 +663,16 @@
(SELECT tid FROM current_object WHERE zoid = :oid), 0),
:md5sum, :blobdata)
"""
+ cursor.setinputsizes(blobdata=self.inputsize_BLOB)
cursor.execute(stmt, oid=oid, tid=tid,
md5sum=md5sum, blobdata=data)
else:
+ cursor.execute(stmt, (oid,))
stmt = """
INSERT INTO object_state (zoid, tid, state)
VALUES (:oid, :tid, :blobdata)
"""
+ cursor.setinputsizes(blobdata=self.inputsize_BLOB)
cursor.execute(stmt, oid=oid, tid=tid, blobdata=data)
@@ -802,11 +803,18 @@
"""
cursor.execute(stmt)
- stmt = """
- INSERT INTO object_state (zoid, tid, state)
- SELECT zoid, %s, state
- FROM temp_store
- """
+ if self.database_name == 'oracle':
+ stmt = """
+ INSERT INTO object_state (zoid, tid, state)
+ SELECT zoid, :1, state
+ FROM temp_store
+ """
+ else:
+ stmt = """
+ INSERT INTO object_state (zoid, tid, state)
+ SELECT zoid, %s, state
+ FROM temp_store
+ """
cursor.execute(stmt, (tid,))
stmt = """
Modified: relstorage/trunk/relstorage/adapters/mysql.py
===================================================================
--- relstorage/trunk/relstorage/adapters/mysql.py 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/adapters/mysql.py 2009-09-30 22:52:21 UTC (rev 104662)
@@ -59,7 +59,7 @@
from relstorage.adapters.locker import MySQLLocker
from relstorage.adapters.mover import ObjectMover
from relstorage.adapters.oidallocator import MySQLOIDAllocator
-from relstorage.adapters.packundo import HistoryFreePackUndo
+from relstorage.adapters.packundo import MySQLHistoryFreePackUndo
from relstorage.adapters.packundo import MySQLHistoryPreservingPackUndo
from relstorage.adapters.poller import Poller
from relstorage.adapters.schema import MySQLSchemaInstaller
@@ -127,7 +127,7 @@
runner=self.runner,
)
else:
- self.packundo = HistoryFreePackUndo(
+ self.packundo = MySQLHistoryFreePackUndo(
connmanager=self.connmanager,
runner=self.runner,
locker=self.locker,
Modified: relstorage/trunk/relstorage/adapters/oracle.py
===================================================================
--- relstorage/trunk/relstorage/adapters/oracle.py 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/adapters/oracle.py 2009-09-30 22:52:21 UTC (rev 104662)
@@ -24,7 +24,7 @@
from relstorage.adapters.locker import OracleLocker
from relstorage.adapters.mover import ObjectMover
from relstorage.adapters.oidallocator import OracleOIDAllocator
-from relstorage.adapters.packundo import HistoryFreePackUndo
+from relstorage.adapters.packundo import OracleHistoryFreePackUndo
from relstorage.adapters.packundo import OracleHistoryPreservingPackUndo
from relstorage.adapters.poller import Poller
from relstorage.adapters.schema import OracleSchemaInstaller
@@ -127,7 +127,7 @@
runner=self.runner,
)
else:
- self.packundo = HistoryFreePackUndo(
+ self.packundo = OracleHistoryFreePackUndo(
connmanager=self.connmanager,
runner=self.runner,
locker=self.locker,
Modified: relstorage/trunk/relstorage/adapters/packundo.py
===================================================================
--- relstorage/trunk/relstorage/adapters/packundo.py 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/adapters/packundo.py 2009-09-30 22:52:21 UTC (rev 104662)
@@ -47,7 +47,7 @@
else:
stmt = """
SELECT transaction.tid
- FROM (SELECT DISTINCT tid FROM object_state) AS transaction
+ FROM (SELECT DISTINCT tid FROM object_state) transaction
LEFT JOIN object_refs_added
ON (transaction.tid = object_refs_added.tid)
WHERE object_refs_added.tid IS NULL
@@ -1014,3 +1014,39 @@
%(TRUNCATE)s pack_object
"""
self.runner.run_script(cursor, stmt)
+
+
+class MySQLHistoryFreePackUndo(HistoryFreePackUndo):
+
+ _script_create_temp_pack_visit = """
+ CREATE TEMPORARY TABLE temp_pack_visit (
+ zoid BIGINT NOT NULL,
+ keep_tid BIGINT
+ );
+ CREATE UNIQUE INDEX temp_pack_visit_zoid ON temp_pack_visit (zoid);
+ CREATE TEMPORARY TABLE temp_pack_child (
+ zoid BIGINT NOT NULL
+ );
+ CREATE UNIQUE INDEX temp_pack_child_zoid ON temp_pack_child (zoid);
+ """
+
+ # Note: UPDATE must be the last statement in the script
+ # because it returns a value.
+ _script_pre_pack_follow_child_refs = """
+ %(TRUNCATE)s temp_pack_child;
+
+ INSERT INTO temp_pack_child
+ SELECT DISTINCT to_zoid
+ FROM object_ref
+ JOIN temp_pack_visit USING (zoid);
+
+ -- MySQL-specific syntax for table join in update
+ UPDATE pack_object, temp_pack_child SET keep = %(TRUE)s
+ WHERE keep = %(FALSE)s
+ AND pack_object.zoid = temp_pack_child.zoid;
+ """
+
+
+class OracleHistoryFreePackUndo(HistoryFreePackUndo):
+
+ _script_create_temp_pack_visit = None
Modified: relstorage/trunk/relstorage/adapters/txncontrol.py
===================================================================
--- relstorage/trunk/relstorage/adapters/txncontrol.py 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/adapters/txncontrol.py 2009-09-30 22:52:21 UTC (rev 104662)
@@ -225,6 +225,8 @@
assert len(rows) == 1
tid, now = rows[0]
+ if tid is None:
+ tid = 0
return tid, self._parse_dsinterval(now)
def add_transaction(self, cursor, tid, username, description, extension,
Modified: relstorage/trunk/relstorage/tests/README.txt
===================================================================
--- relstorage/trunk/relstorage/tests/README.txt 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/tests/README.txt 2009-09-30 22:52:21 UTC (rev 104662)
@@ -6,6 +6,7 @@
databases. Use or adapt the SQL statements below to create the
databases.
+
PostgreSQL
----------
@@ -37,3 +38,20 @@
CREATE DATABASE relstoragetest2_hf;
GRANT ALL ON relstoragetest2_hf.* TO 'relstoragetest'@'localhost';
FLUSH PRIVILEGES;
+
+
+Oracle
+------
+
+Execute these commands as the 'SYSTEM' user. When running the tests,
+you can use the environment variable ORACLE_TEST_DSN to override the
+data source name, which defaults to "XE" (for Oracle 10g XE).
+
+CREATE USER relstoragetest IDENTIFIED BY relstoragetest;
+GRANT CONNECT, RESOURCE, CREATE TABLE, CREATE SEQUENCE TO relstoragetest;
+CREATE USER relstoragetest2 IDENTIFIED BY relstoragetest;
+GRANT CONNECT, RESOURCE, CREATE TABLE, CREATE SEQUENCE TO relstoragetest2;
+CREATE USER relstoragetest_hf IDENTIFIED BY relstoragetest;
+GRANT CONNECT, RESOURCE, CREATE TABLE, CREATE SEQUENCE TO relstoragetest_hf;
+CREATE USER relstoragetest2_hf IDENTIFIED BY relstoragetest;
+GRANT CONNECT, RESOURCE, CREATE TABLE, CREATE SEQUENCE TO relstoragetest2_hf;
Modified: relstorage/trunk/relstorage/tests/packstresstest.py
===================================================================
--- relstorage/trunk/relstorage/tests/packstresstest.py 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/tests/packstresstest.py 2009-09-30 22:52:21 UTC (rev 104662)
@@ -6,23 +6,39 @@
import transaction
from persistent.mapping import PersistentMapping
import random
+import os
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
use = 'oracle'
+keep_history=True
if use == 'mysql':
from relstorage.adapters.mysql import MySQLAdapter
- a = MySQLAdapter(db='packtest')
+ a = MySQLAdapter(
+ db='packtest',
+ user='relstoragetest',
+ passwd='relstoragetest',
+ keep_history=keep_history,
+ )
elif use == 'postgresql':
from relstorage.adapters.postgresql import PostgreSQLAdapter
- a = PostgreSQLAdapter(dsn="dbname='packtest'")
+ a = PostgreSQLAdapter(dsn=
+ "dbname='packtest' "
+ 'user=relstoragetest '
+ 'password=relstoragetest',
+ keep_history=keep_history,
+ )
elif use == 'oracle':
from relstorage.adapters.oracle import OracleAdapter
- from relstorage.tests.testoracle import getOracleParams
- user, password, dsn = getOracleParams()
- a = OracleAdapter(user, password, dsn)
+ dsn = os.environ.get('ORACLE_TEST_DSN', 'XE')
+ a = OracleAdapter(
+ user='packtest',
+ password='relstoragetest',
+ dsn=dsn,
+ keep_history=keep_history,
+ )
else:
raise AssertionError("which database?")
Modified: relstorage/trunk/relstorage/tests/speedtest.py
===================================================================
--- relstorage/trunk/relstorage/tests/speedtest.py 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/tests/speedtest.py 2009-09-30 22:52:21 UTC (rev 104662)
@@ -46,6 +46,7 @@
]
repetitions = 3
max_attempts = 20
+keep_history = True
class ChildProcessError(Exception):
@@ -195,7 +196,12 @@
def postgres_test(self):
from relstorage.adapters.postgresql import PostgreSQLAdapter
- adapter = PostgreSQLAdapter('dbname=relstoragetest')
+ if keep_history:
+ db = 'relstoragetest'
+ else:
+ db = 'relstoragetest_hf'
+ dsn = 'dbname=%s user=relstoragetest password=relstoragetest' % db
+ adapter = PostgreSQLAdapter(dsn=dsn, keep_history=keep_history)
adapter.schema.prepare()
adapter.schema.zap_all()
def make_storage():
@@ -204,9 +210,17 @@
def oracle_test(self):
from relstorage.adapters.oracle import OracleAdapter
- from relstorage.tests.testoracle import getOracleParams
- user, password, dsn = getOracleParams()
- adapter = OracleAdapter(user, password, dsn)
+ dsn = os.environ.get('ORACLE_TEST_DSN', 'XE')
+ if keep_history:
+ db = 'relstoragetest'
+ else:
+ db = 'relstoragetest_hf'
+ adapter = OracleAdapter(
+ user=db,
+ password='relstoragetest',
+ dsn=dsn,
+ keep_history=keep_history,
+ )
adapter.schema.prepare()
adapter.schema.zap_all()
def make_storage():
@@ -215,7 +229,16 @@
def mysql_test(self):
from relstorage.adapters.mysql import MySQLAdapter
- adapter = MySQLAdapter(db='relstoragetest')
+ if keep_history:
+ db = 'relstoragetest'
+ else:
+ db = 'relstoragetest_hf'
+ adapter = MySQLAdapter(
+ db=db,
+ user='relstoragetest',
+ passwd='relstoragetest',
+ keep_history=keep_history,
+ )
adapter.schema.prepare()
adapter.schema.zap_all()
def make_storage():
Modified: relstorage/trunk/relstorage/tests/testoracle.py
===================================================================
--- relstorage/trunk/relstorage/tests/testoracle.py 2009-09-30 19:30:10 UTC (rev 104661)
+++ relstorage/trunk/relstorage/tests/testoracle.py 2009-09-30 22:52:21 UTC (rev 104662)
@@ -13,37 +13,30 @@
##############################################################################
"""Tests of relstorage.adapters.oracle"""
+from relstorage.tests.hftestbase import HistoryFreeFromFileStorage
+from relstorage.tests.hftestbase import HistoryFreeRelStorageTests
+from relstorage.tests.hftestbase import HistoryFreeToFileStorage
+from relstorage.tests.hptestbase import HistoryPreservingFromFileStorage
+from relstorage.tests.hptestbase import HistoryPreservingRelStorageTests
+from relstorage.tests.hptestbase import HistoryPreservingToFileStorage
import logging
import os
-import re
import unittest
-from relstorage.tests.hptestbase import HistoryPreservingFromFileStorage
-from relstorage.tests.hptestbase import HistoryPreservingRelStorageTests
-from relstorage.tests.hptestbase import HistoryPreservingToFileStorage
-
-
-def getOracleParams():
- # Expect an environment variable that specifies how to connect.
- # A more secure way of providing the password would be nice,
- # if anyone wants to tackle it.
- connect_string = os.environ.get('ORACLE_CONNECT')
- if not connect_string:
- raise KeyError("An ORACLE_CONNECT environment variable is "
- "required to run OracleTests")
- mo = re.match('([^/]+)/([^@]+)@(.*)', connect_string)
- if mo is None:
- raise KeyError("The ORACLE_CONNECT environment variable must "
- "be of the form 'user/password at dsn'")
- user, password, dsn = mo.groups()
- return user, password, dsn
-
-
class UseOracleAdapter:
def make_adapter(self):
from relstorage.adapters.oracle import OracleAdapter
- user, password, dsn = getOracleParams()
- return OracleAdapter(user, password, dsn)
+ dsn = os.environ.get('ORACLE_TEST_DSN', 'XE')
+ if self.keep_history:
+ db = 'relstoragetest'
+ else:
+ db = 'relstoragetest_hf'
+ return OracleAdapter(
+ keep_history=self.keep_history,
+ user=db,
+ password='relstoragetest',
+ dsn=dsn,
+ )
class HPOracleTests(UseOracleAdapter, HistoryPreservingRelStorageTests):
pass
@@ -54,17 +47,78 @@
class HPOracleFromFile(UseOracleAdapter, HistoryPreservingFromFileStorage):
pass
+class HFOracleTests(UseOracleAdapter, HistoryFreeRelStorageTests):
+ pass
+class HFOracleToFile(UseOracleAdapter, HistoryFreeToFileStorage):
+ pass
+
+class HFOracleFromFile(UseOracleAdapter, HistoryFreeFromFileStorage):
+ pass
+
+db_names = {
+ 'data': 'relstoragetest',
+ '1': 'relstoragetest',
+ '2': 'relstoragetest2',
+ 'dest': 'relstoragetest2',
+ }
+
def test_suite():
suite = unittest.TestSuite()
for klass in [
HPOracleTests,
HPOracleToFile,
HPOracleFromFile,
+ HFOracleTests,
+ HFOracleToFile,
+ HFOracleFromFile,
]:
suite.addTest(unittest.makeSuite(klass, "check"))
+
+ try:
+ import ZODB.blob
+ except ImportError:
+ # ZODB < 3.8
+ pass
+ else:
+ from relstorage.tests.blob.testblob import storage_reusable_suite
+ dsn = os.environ.get('ORACLE_TEST_DSN', 'XE')
+ for keep_history in (False, True):
+ def create_storage(name, blob_dir, keep_history=keep_history):
+ from relstorage.storage import RelStorage
+ from relstorage.adapters.oracle import OracleAdapter
+ db = db_names[name]
+ if not keep_history:
+ db += '_hf'
+ adapter = OracleAdapter(
+ keep_history=keep_history,
+ user=db,
+ password='relstoragetest',
+ dsn=dsn,
+ )
+ storage = RelStorage(adapter, name=name, create=True,
+ blob_dir=os.path.abspath(blob_dir))
+ storage.zap_all()
+ return storage
+
+ if keep_history:
+ prefix = 'HPOracle'
+ pack_test_name = 'blob_packing.txt'
+ else:
+ prefix = 'HFOracle'
+ pack_test_name = 'blob_packing_history_free.txt'
+
+ suite.addTest(storage_reusable_suite(
+ prefix, create_storage,
+ test_blob_storage_recovery=True,
+ test_packing=True,
+ test_undo=keep_history,
+ pack_test_name=pack_test_name,
+ ))
+
return suite
if __name__=='__main__':
logging.basicConfig()
unittest.main(defaultTest="test_suite")
+
More information about the checkins
mailing list