[Checkins] SVN: relstorage/trunk/ - Install the PL/pgSQL language as necessary. Also tidied schema.py.
Shane Hathaway
shane at hathawaymix.org
Sat Jun 25 16:32:51 EDT 2011
Log message for revision 121975:
- Install the PL/pgSQL language as necessary. Also tidied schema.py.
- zodbconvert: When copying a database containing blobs, ensure the source
blob file exists long enough to copy it.
Changed:
U relstorage/trunk/CHANGES.txt
U relstorage/trunk/notes/migrate-to-1.5.txt
U relstorage/trunk/relstorage/adapters/schema.py
U relstorage/trunk/relstorage/storage.py
-=-
Modified: relstorage/trunk/CHANGES.txt
===================================================================
--- relstorage/trunk/CHANGES.txt 2011-06-25 13:51:27 UTC (rev 121974)
+++ relstorage/trunk/CHANGES.txt 2011-06-25 20:32:50 UTC (rev 121975)
@@ -1,23 +1,26 @@
Next Release
------------
-- Fixed another minor compatibility issue with PostgreSQL 9.0. Packing
- raised an error when the client used old an version of libpq.
+- PostgreSQL: Fixed another minor compatibility issue with PostgreSQL 9.0.
+ Packing raised an error when the client used old an version of libpq.
- Delete empty transactions in batches of 1000 rows instead of all in one
go, to prevent holding the transaction lock for longer than absolutely
necessary.
-- Fix object reference downloading performance for large Oracle RelStorage
- database during the garbage collection phase of a pack.
+- Oracle: Fix object reference downloading performance for large RelStorage
+ databases during the garbage collection phase of a pack.
-- On Oracle and PostgreSQL, switch to storing ZODB blob in chunks up to 4GB
+- Oracle, PostgreSQL: Switch to storing ZODB blob in chunks up to 4GB
(the maximum supported by cx_Oracle) or 2GB (PostgreSQL maximum blob size)
to maximize blob reading and writing performance.
The PostgreSQL blob_chunk schema changed to support this, see
notes/migrate-to-1.5.txt to update existing databases.
+- zodbconvert: When copying a database containing blobs, ensure the source
+ blob file exists long enough to copy it.
+
1.5.0b2 (2011-03-02)
--------------------
Modified: relstorage/trunk/notes/migrate-to-1.5.txt
===================================================================
--- relstorage/trunk/notes/migrate-to-1.5.txt 2011-06-25 13:51:27 UTC (rev 121974)
+++ relstorage/trunk/notes/migrate-to-1.5.txt 2011-06-25 20:32:50 UTC (rev 121975)
@@ -15,13 +15,15 @@
PostgreSQL
----------
+1. Migrate the object_state table.
+
BEGIN;
ALTER TABLE object_state ADD COLUMN state_size BIGINT;
UPDATE object_state SET state_size = COALESCE(LENGTH(state), 0);
ALTER TABLE object_state ALTER COLUMN state_size SET NOT NULL;
COMMIT;
-If you used RelStorage a 1.5.0 version before version b3 you'll need to
+2. If you used a RelStorage 1.5.0 version before version b3 you'll need to
migrate your blob_chunk table schema:
BEGIN;
Modified: relstorage/trunk/relstorage/adapters/schema.py
===================================================================
--- relstorage/trunk/relstorage/adapters/schema.py 2011-06-25 13:51:27 UTC (rev 121974)
+++ relstorage/trunk/relstorage/adapters/schema.py 2011-06-25 20:32:50 UTC (rev 121975)
@@ -19,7 +19,10 @@
from zope.interface import implements
import re
-relstorage_op_version = '1.5A'
+# Version of installed stored procedures
+oracle_sproc_version = '1.5A'
+postgresql_sproc_version = '1.5A'
+
log = logging.getLogger("relstorage")
history_preserving_schema = """
@@ -399,7 +402,7 @@
CREATE OR REPLACE FUNCTION blob_chunk_delete_trigger() RETURNS TRIGGER
AS $blob_chunk_delete_trigger$
-- Version: %s
- -- Unlink large object data file after blob_chunck row deletion
+ -- Unlink large object data file after blob_chunk row deletion
DECLARE
expect integer;
cnt integer;
@@ -425,7 +428,7 @@
FOR EACH ROW
EXECUTE PROCEDURE blob_chunk_delete_trigger();
/
-""" % relstorage_op_version
+""" % postgresql_sproc_version
oracle_history_preserving_plsql = """
CREATE OR REPLACE PACKAGE relstorage_op AS
@@ -481,7 +484,7 @@
END restore;
END relstorage_op;
/
-""" % relstorage_op_version
+""" % oracle_sproc_version
history_free_schema = """
@@ -530,6 +533,7 @@
chunk OID NOT NULL
);
CREATE INDEX blob_chunk_lookup ON blob_chunk (zoid);
+ CREATE INDEX blob_chunk_loid ON blob_chunk (chunk);
ALTER TABLE blob_chunk ADD CONSTRAINT blob_chunk_fk
FOREIGN KEY (zoid)
REFERENCES object_state (zoid)
@@ -757,7 +761,7 @@
END restore;
END relstorage_op;
/
-""" % relstorage_op_version
+""" % oracle_sproc_version
def filter_script(script, database_name):
@@ -880,20 +884,20 @@
existent = set(self.list_tables(cursor))
todo = list(self.all_tables)
todo.reverse()
- log.info("Checking tables: %r", todo)
+ log.debug("Checking tables: %r", todo)
for table in todo:
- log.info("Considering table %s", table)
+ log.debug("Considering table %s", table)
if table.startswith('temp_'):
continue
if table in existent:
- log.info("Deleting table %s...", table)
+ log.debug("Deleting from table %s...", table)
cursor.execute("DELETE FROM %s" % table)
- log.info("Done deleting tables.")
+ log.debug("Done deleting from tables.")
script = filter_script(self.init_script, self.database_name)
if script:
- log.info("Running init script.")
+ log.debug("Running init script.")
self.runner.run_script(cursor, script)
- log.info("Done running init script.")
+ log.debug("Done running init script.")
self.connmanager.open_and_call(callback)
def drop_all(self):
@@ -930,30 +934,36 @@
self.check_compatibility(cursor, tables)
self.update_schema(cursor, tables)
triggers = self.list_triggers(cursor)
- if triggers.get('blob_chunk_delete_trigger') != relstorage_op_version:
+ trigger_name = 'blob_chunk_delete_trigger'
+ if triggers.get(trigger_name) != postgresql_sproc_version:
self.install_triggers(cursor)
triggers = self.list_triggers(cursor)
- if triggers.get('blob_chunk_delete_trigger') != relstorage_op_version:
+ if triggers.get(trigger_name) != postgresql_sproc_version:
raise AssertionError(
"Could not get version information after "
- "installing the blob_chunk_delete_trigger trigger.")
+ "installing %s." % trigger_name)
self.connmanager.open_and_call(callback)
+ def list_languages(self, cursor):
+ cursor.execute("SELECT lanname FROM pg_catalog.pg_language")
+ return [name for (name,) in cursor]
+
+ def install_languages(self, cursor):
+ if 'plpgsql' not in self.list_languages(cursor):
+ cursor.execute("CREATE LANGUAGE plpgsql")
+
def install_triggers(self, cursor):
"""Install the PL/pgSQL triggers"""
+ self.install_languages(cursor)
+
if self.keep_history:
plpgsql = postgresql_history_preserving_plpgsql
else:
plpgsql = postgresql_history_free_plpgsql
- lines = []
- for line in plpgsql.splitlines():
- if line.strip() == '/':
- # end of a statement
- cursor.execute('\n'.join(lines))
- lines = []
- elif line.strip():
- lines.append(line)
+ for stmt in plpgsql.split('\n/\n'):
+ if stmt.strip():
+ cursor.execute(stmt)
def create(self, cursor):
"""Create the database tables."""
@@ -1038,13 +1048,14 @@
self.check_compatibility(cursor, tables)
self.update_schema(cursor, tables)
packages = self.list_packages(cursor)
- if packages.get('relstorage_op') != relstorage_op_version:
+ package_name = 'relstorage_op'
+ if packages.get(package_name) != oracle_sproc_version:
self.install_plsql(cursor)
packages = self.list_packages(cursor)
- if packages.get('relstorage_op') != relstorage_op_version:
+ if packages.get(package_name) != oracle_sproc_version:
raise AssertionError(
"Could not get version information after "
- "installing the relstorage_op package.")
+ "installing the %s package." % package_name)
self.connmanager.open_and_call(callback)
def install_plsql(self, cursor):
@@ -1054,14 +1065,9 @@
else:
plsql = oracle_history_free_plsql
- lines = []
- for line in plsql.splitlines():
- if line.strip() == '/':
- # end of a statement
- cursor.execute('\n'.join(lines))
- lines = []
- elif line.strip():
- lines.append(line)
+ for stmt in plsql.split('\n/\n'):
+ if stmt.strip():
+ cursor.execute(stmt)
def list_tables(self, cursor):
cursor.execute("SELECT table_name FROM user_tables")
Modified: relstorage/trunk/relstorage/storage.py
===================================================================
--- relstorage/trunk/relstorage/storage.py 2011-06-25 13:51:27 UTC (rev 121974)
+++ relstorage/trunk/relstorage/storage.py 2011-06-25 20:32:50 UTC (rev 121975)
@@ -1331,20 +1331,23 @@
self.tpc_begin(trans, trans.tid, trans.status)
for record in trans:
- blobfilename = None
+ blobfile = None
if self.blobhelper is not None:
if is_blob_record(record.data):
try:
- blobfilename = other.loadBlob(
+ blobfile = other.openCommittedBlobFile(
record.oid, record.tid)
except POSKeyError:
pass
- if blobfilename is not None:
+ if blobfile is not None:
fd, name = tempfile.mkstemp(
suffix='.tmp',
dir=self.blobhelper.temporaryDirectory())
os.close(fd)
- ZODB.utils.cp(open(blobfilename, 'rb'), open(name, 'wb'))
+ target = open(name, 'wb')
+ ZODB.utils.cp(blobfile, target)
+ blobfile.close()
+ target.close()
self.restoreBlob(record.oid, record.tid, record.data,
name, record.data_txn, trans)
else:
More information about the checkins
mailing list