[Checkins] SVN: relstorage/trunk/relstorage/adapters/ More logging when analyzing transactions; use unsigned integers for OIDs and tids in MySQL in order to allow the full 2**64 range.
Shane Hathaway
shane at hathawaymix.org
Wed Feb 2 04:06:52 EST 2011
Log message for revision 120041:
More logging when analyzing transactions; use unsigned integers for OIDs and tids in MySQL in order to allow the full 2**64 range.
Changed:
U relstorage/trunk/relstorage/adapters/mover.py
U relstorage/trunk/relstorage/adapters/packundo.py
U relstorage/trunk/relstorage/adapters/schema.py
-=-
Modified: relstorage/trunk/relstorage/adapters/mover.py
===================================================================
--- relstorage/trunk/relstorage/adapters/mover.py 2011-02-01 16:37:45 UTC (rev 120040)
+++ relstorage/trunk/relstorage/adapters/mover.py 2011-02-02 09:06:51 UTC (rev 120041)
@@ -409,8 +409,8 @@
# note that the md5 column is not used if self.keep_history == False.
stmt = """
CREATE TEMPORARY TABLE temp_store (
- zoid BIGINT NOT NULL PRIMARY KEY,
- prev_tid BIGINT NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
+ prev_tid BIGINT UNSIGNED NOT NULL,
md5 CHAR(32),
state LONGBLOB
) ENGINE MyISAM
@@ -419,8 +419,8 @@
stmt = """
CREATE TEMPORARY TABLE temp_blob_chunk (
- zoid BIGINT NOT NULL,
- chunk_num BIGINT NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL,
+ chunk_num BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (zoid, chunk_num),
chunk LONGBLOB
) ENGINE MyISAM
Modified: relstorage/trunk/relstorage/adapters/packundo.py
===================================================================
--- relstorage/trunk/relstorage/adapters/packundo.py 2011-02-01 16:37:45 UTC (rev 120040)
+++ relstorage/trunk/relstorage/adapters/packundo.py 2011-02-02 09:06:51 UTC (rev 120041)
@@ -346,19 +346,25 @@
"""
self.runner.run_script_stmt(cursor, stmt)
tids = [tid for (tid,) in cursor]
+ log_at = time.time() + 60
if tids:
self.on_filling_object_refs()
- added = 0
- log.info("discovering references from objects in %d "
- "transaction(s)" % len(tids))
+ tid_count = len(tids)
+ txns_done = 0
+ log.info("analyzing references from objects in %d "
+ "transaction(s)", tid_count)
for tid in tids:
- added += self._add_refs_for_tid(cursor, tid, get_references)
- if added >= 10000:
+ self._add_refs_for_tid(cursor, tid, get_references)
+ txns_done += 1
+ now = time.time()
+ if now >= log_at:
# save the work done so far
conn.commit()
- added = 0
- if added:
- conn.commit()
+ log_at = now + 60
+ log.info(
+ "transactions analyzed: %d/%d", txns_done, tid_count)
+ conn.commit()
+ log.info("transactions analyzed: %d/%d", txns_done, tid_count)
def _add_refs_for_tid(self, cursor, tid, get_references):
"""Fill object_refs with all states for a transaction.
@@ -394,10 +400,12 @@
for to_oid in to_oids:
add_rows.append((from_oid, tid, to_oid))
- if not self.keep_history:
- stmt = "DELETE FROM object_ref WHERE zoid = %s"
- self.runner.run_many(cursor, stmt, replace_rows)
+ # A previous pre-pack may have been interrupted. Delete rows
+ # from the interrupted attempt.
+ stmt = "DELETE FROM object_ref WHERE tid = %(tid)s"
+ self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
+ # Add the new references.
stmt = """
INSERT INTO object_ref (zoid, tid, to_zoid)
VALUES (%s, %s, %s)
@@ -747,12 +755,12 @@
# http://bugs.mysql.com/bug.php?id=28257
_script_create_temp_pack_visit = """
CREATE TEMPORARY TABLE temp_pack_visit (
- zoid BIGINT NOT NULL,
- keep_tid BIGINT
+ zoid BIGINT UNSIGNED NOT NULL,
+ keep_tid BIGINT UNSIGNED
);
CREATE UNIQUE INDEX temp_pack_visit_zoid ON temp_pack_visit (zoid);
CREATE TEMPORARY TABLE temp_pack_child (
- zoid BIGINT NOT NULL
+ zoid BIGINT UNSIGNED NOT NULL
);
CREATE UNIQUE INDEX temp_pack_child_zoid ON temp_pack_child (zoid);
"""
@@ -801,7 +809,15 @@
WHERE transaction.empty = true
"""
+ _script_create_temp_undo = """
+ CREATE TEMPORARY TABLE temp_undo (
+ zoid BIGINT UNSIGNED NOT NULL,
+ prev_tid BIGINT UNSIGNED NOT NULL
+ );
+ CREATE UNIQUE INDEX temp_undo_zoid ON temp_undo (zoid)
+ """
+
class OracleHistoryPreservingPackUndo(HistoryPreservingPackUndo):
_script_choose_pack_transaction = """
@@ -900,22 +916,28 @@
"""
self.runner.run_script_stmt(cursor, stmt)
oids = [oid for (oid,) in cursor]
+ log_at = time.time() + 60
if oids:
if attempt == 1:
self.on_filling_object_refs()
- added = 0
- log.info("discovering references from %d objects", len(oids))
+ oid_count = len(oids)
+ oids_done = 0
+ log.info("analyzing references from %d object(s)", oid_count)
while oids:
batch = oids[:100]
oids = oids[100:]
- added += self._add_refs_for_oids(
- cursor, batch, get_references)
- if added >= 10000:
+ self._add_refs_for_oids(cursor, batch, get_references)
+ oids_done += len(batch)
+ now = time.time()
+ if now >= log_at:
# Save the work done so far.
conn.commit()
- added = 0
- if added:
- conn.commit()
+ log_at = now + 60
+ log.info(
+ "objects analyzed: %d/%d", oids_done, oid_count)
+ conn.commit()
+ log.info(
+ "objects analyzed: %d/%d", oids_done, oid_count)
else:
# No changes since last pass.
break
@@ -1136,12 +1158,12 @@
_script_create_temp_pack_visit = """
CREATE TEMPORARY TABLE temp_pack_visit (
- zoid BIGINT NOT NULL,
- keep_tid BIGINT
+ zoid BIGINT UNSIGNED NOT NULL,
+ keep_tid BIGINT UNSIGNED
);
CREATE UNIQUE INDEX temp_pack_visit_zoid ON temp_pack_visit (zoid);
CREATE TEMPORARY TABLE temp_pack_child (
- zoid BIGINT NOT NULL
+ zoid BIGINT UNSIGNED NOT NULL
);
CREATE UNIQUE INDEX temp_pack_child_zoid ON temp_pack_child (zoid);
"""
Modified: relstorage/trunk/relstorage/adapters/schema.py
===================================================================
--- relstorage/trunk/relstorage/adapters/schema.py 2011-02-01 16:37:45 UTC (rev 120040)
+++ relstorage/trunk/relstorage/adapters/schema.py 2011-02-02 09:06:51 UTC (rev 120041)
@@ -50,7 +50,7 @@
mysql:
CREATE TABLE transaction (
- tid BIGINT NOT NULL PRIMARY KEY,
+ tid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
packed BOOLEAN NOT NULL DEFAULT FALSE,
empty BOOLEAN NOT NULL DEFAULT FALSE,
username BLOB NOT NULL,
@@ -75,7 +75,7 @@
mysql:
CREATE TABLE new_oid (
- zoid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT
+ zoid BIGINT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT
) ENGINE = MyISAM;
oracle:
@@ -113,11 +113,11 @@
mysql:
CREATE TABLE object_state (
- zoid BIGINT NOT NULL,
- tid BIGINT NOT NULL REFERENCES transaction,
+ zoid BIGINT UNSIGNED NOT NULL,
+ tid BIGINT UNSIGNED NOT NULL REFERENCES transaction,
PRIMARY KEY (zoid, tid),
CHECK (tid > 0),
- prev_tid BIGINT NOT NULL REFERENCES transaction,
+ prev_tid BIGINT UNSIGNED NOT NULL REFERENCES transaction,
md5 CHAR(32) CHARACTER SET ascii,
state LONGBLOB
) ENGINE = InnoDB;
@@ -125,9 +125,9 @@
CREATE INDEX object_state_prev_tid ON object_state (prev_tid);
CREATE TABLE blob_chunk (
- zoid BIGINT NOT NULL,
- tid BIGINT NOT NULL,
- chunk_num BIGINT NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL,
+ tid BIGINT UNSIGNED NOT NULL,
+ chunk_num BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (zoid, tid, chunk_num),
chunk LONGBLOB NOT NULL
) ENGINE = InnoDB;
@@ -176,8 +176,8 @@
mysql:
CREATE TABLE current_object (
- zoid BIGINT NOT NULL PRIMARY KEY,
- tid BIGINT NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
+ tid BIGINT UNSIGNED NOT NULL,
FOREIGN KEY (zoid, tid)
REFERENCES object_state (zoid, tid)
) ENGINE = InnoDB;
@@ -208,9 +208,9 @@
mysql:
CREATE TABLE object_ref (
- zoid BIGINT NOT NULL,
- tid BIGINT NOT NULL,
- to_zoid BIGINT NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL,
+ tid BIGINT UNSIGNED NOT NULL,
+ to_zoid BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (tid, zoid, to_zoid)
) ENGINE = MyISAM;
@@ -236,7 +236,7 @@
mysql:
CREATE TABLE object_refs_added (
- tid BIGINT NOT NULL PRIMARY KEY
+ tid BIGINT UNSIGNED NOT NULL PRIMARY KEY
) ENGINE = MyISAM;
oracle:
@@ -265,9 +265,9 @@
mysql:
CREATE TABLE pack_object (
- zoid BIGINT NOT NULL PRIMARY KEY,
+ zoid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
keep BOOLEAN NOT NULL,
- keep_tid BIGINT NOT NULL,
+ keep_tid BIGINT UNSIGNED NOT NULL,
visited BOOLEAN NOT NULL DEFAULT FALSE
) ENGINE = MyISAM;
CREATE INDEX pack_object_keep_zoid ON pack_object (keep, zoid);
@@ -293,8 +293,8 @@
mysql:
CREATE TABLE pack_state (
- tid BIGINT NOT NULL,
- zoid BIGINT NOT NULL,
+ tid BIGINT UNSIGNED NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (tid, zoid)
) ENGINE = MyISAM;
@@ -315,7 +315,7 @@
mysql:
CREATE TABLE pack_state_tid (
- tid BIGINT NOT NULL PRIMARY KEY
+ tid BIGINT UNSIGNED NOT NULL PRIMARY KEY
) ENGINE = MyISAM;
oracle:
@@ -467,7 +467,7 @@
mysql:
CREATE TABLE new_oid (
- zoid BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT
+ zoid BIGINT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT
) ENGINE = MyISAM;
oracle:
@@ -498,18 +498,18 @@
mysql:
CREATE TABLE object_state (
- zoid BIGINT NOT NULL PRIMARY KEY,
- tid BIGINT NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
+ tid BIGINT UNSIGNED NOT NULL,
CHECK (tid > 0),
state LONGBLOB
) ENGINE = InnoDB;
CREATE INDEX object_state_tid ON object_state (tid);
CREATE TABLE blob_chunk (
- zoid BIGINT NOT NULL,
- chunk_num BIGINT NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL,
+ chunk_num BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (zoid, chunk_num),
- tid BIGINT NOT NULL,
+ tid BIGINT UNSIGNED NOT NULL,
chunk LONGBLOB NOT NULL
) ENGINE = InnoDB;
CREATE INDEX blob_chunk_lookup ON blob_chunk (zoid);
@@ -553,9 +553,9 @@
mysql:
CREATE TABLE object_ref (
- zoid BIGINT NOT NULL,
- to_zoid BIGINT NOT NULL,
- tid BIGINT NOT NULL,
+ zoid BIGINT UNSIGNED NOT NULL,
+ to_zoid BIGINT UNSIGNED NOT NULL,
+ tid BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (zoid, to_zoid)
) ENGINE = MyISAM;
@@ -578,8 +578,8 @@
mysql:
CREATE TABLE object_refs_added (
- zoid BIGINT NOT NULL PRIMARY KEY,
- tid BIGINT NOT NULL
+ zoid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
+ tid BIGINT UNSIGNED NOT NULL
) ENGINE = MyISAM;
oracle:
@@ -608,9 +608,9 @@
mysql:
CREATE TABLE pack_object (
- zoid BIGINT NOT NULL PRIMARY KEY,
+ zoid BIGINT UNSIGNED NOT NULL PRIMARY KEY,
keep BOOLEAN NOT NULL,
- keep_tid BIGINT NOT NULL,
+ keep_tid BIGINT UNSIGNED NOT NULL,
visited BOOLEAN NOT NULL DEFAULT FALSE
) ENGINE = MyISAM;
CREATE INDEX pack_object_keep_zoid ON pack_object (keep, zoid);
More information about the checkins
mailing list