[Checkins] SVN: relstorage/ Refinement of history-free pack bugfix: ensure a consistent state when filling object_refs by using multiple passes.
Shane Hathaway
shane at hathawaymix.org
Tue Feb 1 02:00:41 EST 2011
Log message for revision 120033:
Refinement of history-free pack bugfix: ensure a consistent state when filling object_refs by using multiple passes.
Changed:
U relstorage/branches/1.4/relstorage/adapters/packundo.py
U relstorage/trunk/relstorage/adapters/packundo.py
-=-
Modified: relstorage/branches/1.4/relstorage/adapters/packundo.py
===================================================================
--- relstorage/branches/1.4/relstorage/adapters/packundo.py 2011-02-01 06:18:00 UTC (rev 120032)
+++ relstorage/branches/1.4/relstorage/adapters/packundo.py 2011-02-01 07:00:40 UTC (rev 120033)
@@ -865,30 +865,51 @@
"""Test injection point"""
def fill_object_refs(self, conn, cursor, get_references):
- """Update the object_refs table by analyzing new object states."""
- stmt = """
- SELECT object_state.zoid FROM object_state
- LEFT JOIN object_refs_added USING (zoid)
- WHERE object_refs_added.tid IS NULL
- OR object_refs_added.tid != object_state.tid
- ORDER BY object_state.zoid
+ """Update the object_refs table by analyzing new object states.
+
+ Note that ZODB connections can change the object states while this
+ method is running, possibly obscuring object references,
+ so this method runs repeatedly until it detects no changes between
+ two passes.
"""
- self.runner.run_script_stmt(cursor, stmt)
- oids = [oid for (oid,) in cursor]
- if oids:
- self.on_filling_object_refs()
- added = 0
- log.info("discovering references from %d objects", len(oids))
- while oids:
- batch = oids[:100]
- oids = oids[100:]
- added += self._add_refs_for_oids(cursor, batch, get_references)
- if added >= 10000:
- # save the work done so far
+ holding_commit = False
+ attempt = 0
+ while True:
+ attempt += 1
+ if attempt >= 3 and not holding_commit:
+ # Starting with the third attempt, hold the commit lock
+ # to prevent changes.
+ holding_commit = True
+ self.locker.hold_commit_lock(cursor)
+
+ stmt = """
+ SELECT object_state.zoid FROM object_state
+ LEFT JOIN object_refs_added USING (zoid)
+ WHERE object_refs_added.tid IS NULL
+ OR object_refs_added.tid != object_state.tid
+ ORDER BY object_state.zoid
+ """
+ self.runner.run_script_stmt(cursor, stmt)
+ oids = [oid for (oid,) in cursor]
+ if oids:
+ if attempt == 1:
+ self.on_filling_object_refs()
+ added = 0
+ log.info("discovering references from %d objects", len(oids))
+ while oids:
+ batch = oids[:100]
+ oids = oids[100:]
+ added += self._add_refs_for_oids(
+ cursor, batch, get_references)
+ if added >= 10000:
+ # Save the work done so far.
+ conn.commit()
+ added = 0
+ if added:
conn.commit()
- added = 0
- if added:
- conn.commit()
+ else:
+ # No changes since last pass.
+ break
def _add_refs_for_oids(self, cursor, oids, get_references):
"""Fill object_refs with the states for some objects.
Modified: relstorage/trunk/relstorage/adapters/packundo.py
===================================================================
--- relstorage/trunk/relstorage/adapters/packundo.py 2011-02-01 06:18:00 UTC (rev 120032)
+++ relstorage/trunk/relstorage/adapters/packundo.py 2011-02-01 07:00:40 UTC (rev 120033)
@@ -873,30 +873,51 @@
"""Test injection point"""
def fill_object_refs(self, conn, cursor, get_references):
- """Update the object_refs table by analyzing new object states."""
- stmt = """
- SELECT object_state.zoid FROM object_state
- LEFT JOIN object_refs_added USING (zoid)
- WHERE object_refs_added.tid IS NULL
- OR object_refs_added.tid != object_state.tid
- ORDER BY object_state.zoid
+ """Update the object_refs table by analyzing new object states.
+
+ Note that ZODB connections can change the object states while this
+ method is running, possibly obscuring object references,
+ so this method runs repeatedly until it detects no changes between
+ two passes.
"""
- self.runner.run_script_stmt(cursor, stmt)
- oids = [oid for (oid,) in cursor]
- if oids:
- self.on_filling_object_refs()
- added = 0
- log.info("discovering references from %d objects", len(oids))
- while oids:
- batch = oids[:100]
- oids = oids[100:]
- added += self._add_refs_for_oids(cursor, batch, get_references)
- if added >= 10000:
- # save the work done so far
+ holding_commit = False
+ attempt = 0
+ while True:
+ attempt += 1
+ if attempt >= 3 and not holding_commit:
+ # Starting with the third attempt, hold the commit lock
+ # to prevent changes.
+ holding_commit = True
+ self.locker.hold_commit_lock(cursor)
+
+ stmt = """
+ SELECT object_state.zoid FROM object_state
+ LEFT JOIN object_refs_added USING (zoid)
+ WHERE object_refs_added.tid IS NULL
+ OR object_refs_added.tid != object_state.tid
+ ORDER BY object_state.zoid
+ """
+ self.runner.run_script_stmt(cursor, stmt)
+ oids = [oid for (oid,) in cursor]
+ if oids:
+ if attempt == 1:
+ self.on_filling_object_refs()
+ added = 0
+ log.info("discovering references from %d objects", len(oids))
+ while oids:
+ batch = oids[:100]
+ oids = oids[100:]
+ added += self._add_refs_for_oids(
+ cursor, batch, get_references)
+ if added >= 10000:
+ # Save the work done so far.
+ conn.commit()
+ added = 0
+ if added:
conn.commit()
- added = 0
- if added:
- conn.commit()
+ else:
+ # No changes since last pass.
+ break
def _add_refs_for_oids(self, cursor, oids, get_references):
"""Fill object_refs with the states for some objects.
More information about the checkins
mailing list