[Checkins] SVN: relstorage/trunk/ According to the transaction
interface, transaction metadata is always a string but may
Shane Hathaway
shane at hathawaymix.org
Wed Feb 27 14:39:05 EST 2008
Log message for revision 84349:
According to the transaction interface, transaction metadata is always a string but may
have encoded characters. Fixed accordingly and provided migration scripts.
Changed:
A relstorage/trunk/notes/migrate-1.0-beta.txt
U relstorage/trunk/relstorage/adapters/common.py
U relstorage/trunk/relstorage/adapters/mysql.py
U relstorage/trunk/relstorage/adapters/oracle.py
U relstorage/trunk/relstorage/adapters/postgresql.py
U relstorage/trunk/relstorage/relstorage.py
U relstorage/trunk/relstorage/tests/reltestbase.py
-=-
Added: relstorage/trunk/notes/migrate-1.0-beta.txt
===================================================================
--- relstorage/trunk/notes/migrate-1.0-beta.txt (rev 0)
+++ relstorage/trunk/notes/migrate-1.0-beta.txt 2008-02-27 19:39:04 UTC (rev 84349)
@@ -0,0 +1,40 @@
+1.0 Beta Migration
+
+
+PostgreSQL 8.3:
+
+ ALTER TABLE transaction
+ ALTER username TYPE BYTEA USING (convert_to(username, 'UTF-8')),
+ ALTER description TYPE BYTEA USING (convert_to(description, 'UTF-8'));
+
+PostgreSQL 8.2 and below:
+
+ ALTER TABLE transaction
+ ALTER username TYPE BYTEA USING
+ (decode(replace(convert(username, 'UTF-8'), '\\', '\\\\'), 'escape')),
+ ALTER description TYPE BYTEA USING
+ (decode(replace(convert(description, 'UTF-8'), '\\', '\\\\'), 'escape'));
+
+MySQL:
+
+ ALTER TABLE transaction
+ MODIFY username BLOB NOT NULL,
+ MODIFY description BLOB NOT NULL;
+
+Oracle:
+
+ ALTER TABLE transaction ADD (
+ new_username RAW(255),
+ new_description RAW(2000),
+ new_extension RAW(2000));
+
+ UPDATE transaction
+ SET new_username = UTL_I18N.STRING_TO_RAW(username, 'UTF8'),
+ new_description = UTL_I18N.STRING_TO_RAW(description, 'UTF8'),
+ new_extension = extension;
+
+ ALTER TABLE transaction DROP (username, description, extension);
+ ALTER TABLE transaction RENAME COLUMN new_username TO username;
+ ALTER TABLE transaction RENAME COLUMN new_description TO description;
+ ALTER TABLE transaction RENAME COLUMN new_extension TO extension;
+
Modified: relstorage/trunk/relstorage/adapters/common.py
===================================================================
--- relstorage/trunk/relstorage/adapters/common.py 2008-02-27 18:39:03 UTC (rev 84348)
+++ relstorage/trunk/relstorage/adapters/common.py 2008-02-27 19:39:04 UTC (rev 84349)
@@ -135,11 +135,18 @@
Each row begins with (tid, username, description, extension)
and may have other columns.
-
- The default implementation returns the cursor unmodified.
- Subclasses can override this.
"""
- return cursor
+ for row in cursor:
+ tid, username, description = row[:3]
+ if username is None:
+ username = ''
+ else:
+ username = str(username)
+ if description is None:
+ description = ''
+ else:
+ description = str(description)
+ yield (tid, username, description) + tuple(row[3:])
def iter_transactions(self, cursor):
Modified: relstorage/trunk/relstorage/adapters/mysql.py
===================================================================
--- relstorage/trunk/relstorage/adapters/mysql.py 2008-02-27 18:39:03 UTC (rev 84348)
+++ relstorage/trunk/relstorage/adapters/mysql.py 2008-02-27 19:39:04 UTC (rev 84349)
@@ -75,10 +75,10 @@
CREATE TABLE transaction (
tid BIGINT NOT NULL PRIMARY KEY,
packed BOOLEAN NOT NULL DEFAULT FALSE,
- username VARCHAR(255) NOT NULL,
- description TEXT NOT NULL,
+ username BLOB NOT NULL,
+ description BLOB NOT NULL,
extension BLOB
- ) ENGINE = InnoDB CHARACTER SET utf8;
+ ) ENGINE = InnoDB;
-- Create a special transaction to represent object creation. This
-- row is often referenced by object_state.prev_tid, but never by
@@ -434,7 +434,8 @@
VALUES (%s, %s, %s, %s, %s)
"""
cursor.execute(stmt, (
- tid, packed, username, description, MySQLdb.Binary(extension)))
+ tid, packed, MySQLdb.Binary(username),
+ MySQLdb.Binary(description), MySQLdb.Binary(extension)))
def detect_conflict(self, cursor):
"""Find one conflict in the data about to be committed.
Modified: relstorage/trunk/relstorage/adapters/oracle.py
===================================================================
--- relstorage/trunk/relstorage/adapters/oracle.py 2008-02-27 18:39:03 UTC (rev 84348)
+++ relstorage/trunk/relstorage/adapters/oracle.py 2008-02-27 19:39:04 UTC (rev 84349)
@@ -105,8 +105,8 @@
CREATE TABLE transaction (
tid NUMBER(20) NOT NULL PRIMARY KEY,
packed CHAR DEFAULT 'N' CHECK (packed IN ('N', 'Y')),
- username NVARCHAR2(255),
- description NVARCHAR2(2000),
+ username RAW(500),
+ description RAW(2000),
extension RAW(2000)
);
@@ -114,7 +114,10 @@
-- row is often referenced by object_state.prev_tid, but never by
-- object_state.tid.
INSERT INTO transaction (tid, username, description)
- VALUES (0, 'system', 'special transaction for object creation');
+ VALUES (0,
+ UTL_I18N.STRING_TO_RAW('system', 'US7ASCII'),
+ UTL_I18N.STRING_TO_RAW(
+ 'special transaction for object creation', 'US7ASCII'));
CREATE SEQUENCE zoid_seq;
@@ -239,7 +242,9 @@
DELETE FROM transaction;
-- Create a transaction to represent object creation.
INSERT INTO transaction (tid, username, description) VALUES
- (0, 'system', 'special transaction for object creation');
+ (0, UTL_I18N.STRING_TO_RAW('system', 'US7ASCII'),
+ UTL_I18N.STRING_TO_RAW(
+ 'special transaction for object creation', 'US7ASCII'));
DROP SEQUENCE zoid_seq;
CREATE SEQUENCE zoid_seq;
"""
@@ -514,8 +519,8 @@
"""
encoding = cursor.connection.encoding
cursor.execute(stmt, (
- tid, packed and 'Y' or 'N', username.encode(encoding),
- description.encode(encoding), cx_Oracle.Binary(extension)))
+ tid, packed and 'Y' or 'N', cx_Oracle.Binary(username),
+ cx_Oracle.Binary(description), cx_Oracle.Binary(extension)))
def detect_conflict(self, cursor):
"""Find one conflict in the data about to be committed.
@@ -627,24 +632,6 @@
return cursor.fetchone()[0]
- def _transaction_iterator(self, cursor):
- """Iterate over a list of transactions returned from the database.
-
- Each row begins with (tid, username, description, extension)
- and may have other columns.
-
- This overrides the default implementation.
- """
- encoding = cursor.connection.encoding
- for row in cursor:
- tid, username, description = row[:3]
- if username is not None:
- username = username.decode(encoding)
- if description is not None:
- description = description.decode(encoding)
- yield (tid, username, description) + tuple(row[3:])
-
-
def hold_pack_lock(self, cursor):
"""Try to acquire the pack lock.
Modified: relstorage/trunk/relstorage/adapters/postgresql.py
===================================================================
--- relstorage/trunk/relstorage/adapters/postgresql.py 2008-02-27 18:39:03 UTC (rev 84348)
+++ relstorage/trunk/relstorage/adapters/postgresql.py 2008-02-27 19:39:04 UTC (rev 84349)
@@ -41,8 +41,8 @@
CREATE TABLE transaction (
tid BIGINT NOT NULL PRIMARY KEY,
packed BOOLEAN NOT NULL DEFAULT FALSE,
- username VARCHAR(255) NOT NULL,
- description TEXT NOT NULL,
+ username BYTEA NOT NULL,
+ description BYTEA NOT NULL,
extension BYTEA
);
@@ -436,10 +436,12 @@
stmt = """
INSERT INTO transaction
(tid, packed, username, description, extension)
- VALUES (%s, %s, %s, %s, decode(%s, 'base64'))
+ VALUES (%s, %s,
+ decode(%s, 'base64'), decode(%s, 'base64'), decode(%s, 'base64'))
"""
- cursor.execute(stmt, (
- tid, packed, username, description, encodestring(extension)))
+ cursor.execute(stmt, (tid, packed,
+ encodestring(username), encodestring(description),
+ encodestring(extension)))
def detect_conflict(self, cursor):
"""Find one conflict in the data about to be committed.
Modified: relstorage/trunk/relstorage/relstorage.py
===================================================================
--- relstorage/trunk/relstorage/relstorage.py 2008-02-27 18:39:03 UTC (rev 84348)
+++ relstorage/trunk/relstorage/relstorage.py 2008-02-27 19:39:04 UTC (rev 84349)
@@ -327,8 +327,8 @@
self._transaction = transaction
self._clear_temp()
- user = transaction.user
- desc = transaction.description
+ user = str(transaction.user)
+ desc = str(transaction.description)
ext = transaction._extension
if ext:
ext = cPickle.dumps(ext, 1)
Modified: relstorage/trunk/relstorage/tests/reltestbase.py
===================================================================
--- relstorage/trunk/relstorage/tests/reltestbase.py 2008-02-27 18:39:03 UTC (rev 84348)
+++ relstorage/trunk/relstorage/tests/reltestbase.py 2008-02-27 19:39:04 UTC (rev 84349)
@@ -376,22 +376,22 @@
def checkNonASCIITransactionMetadata(self):
# Verify the database stores and retrieves non-ASCII text
# in transaction metadata.
+ ugly_string = ''.join(chr(c) for c in range(256))
+
db = DB(self._storage)
try:
c1 = db.open()
r1 = c1.root()
r1['alpha'] = 1
- user = u"\u65e5\u672c\u8e86"
- transaction.get().setUser(user)
+ transaction.get().setUser(ugly_string)
transaction.commit()
r1['alpha'] = 2
- desc = u"Japanese: \u65e5\u672c\u8e86"
- transaction.get().note(desc)
+ transaction.get().note(ugly_string)
transaction.commit()
info = self._storage.undoInfo()
- self.assertEqual(info[0]['description'], desc)
- self.assertEqual(info[1]['user_name'], '/ ' + user)
+ self.assertEqual(info[0]['description'], ugly_string)
+ self.assertEqual(info[1]['user_name'], '/ ' + ugly_string)
finally:
db.close()
More information about the Checkins
mailing list