[Checkins] SVN: relstorage/trunk/relstorage/adapters/ Added unit tests of the row batchers
Shane Hathaway
shane at hathawaymix.org
Fri Oct 9 19:44:43 EDT 2009
Log message for revision 104978:
Added unit tests of the row batchers
Changed:
U relstorage/trunk/relstorage/adapters/batch.py
A relstorage/trunk/relstorage/adapters/tests/test_batch.py
-=-
Modified: relstorage/trunk/relstorage/adapters/batch.py
===================================================================
--- relstorage/trunk/relstorage/adapters/batch.py 2009-10-09 19:41:15 UTC (rev 104977)
+++ relstorage/trunk/relstorage/adapters/batch.py 2009-10-09 23:44:43 UTC (rev 104978)
@@ -59,22 +59,24 @@
def flush(self):
if self.deletes:
- self.do_deletes()
+ self._do_deletes()
self.deletes.clear()
if self.inserts:
- self.do_inserts()
+ self._do_inserts()
self.inserts.clear()
self.rows_added = 0
self.size_added = 0
- def do_deletes(self):
+ def _do_deletes(self):
for (table, varname), values in sorted(self.deletes.items()):
- value_str = ','.join(values)
+ v = list(values)
+ v.sort()
+ value_str = ','.join(v)
stmt = "DELETE FROM %s WHERE %s IN (%s)" % (
table, varname, value_str)
self.cursor.execute(stmt)
- def do_inserts(self):
+ def _do_inserts(self):
items = sorted(self.inserts.items())
for (command, header, row_schema), rows in items:
if self.support_batch_insert:
@@ -85,7 +87,7 @@
parts.append(s)
params.extend(row)
parts = ',\n'.join(parts)
- stmt = "%s INTO %s VALUES %s" % (command, header, parts)
+ stmt = "%s INTO %s VALUES\n%s" % (command, header, parts)
self.cursor.execute(stmt, tuple(params))
else:
for row in rows.values():
@@ -118,7 +120,7 @@
super(OracleRowBatcher, self).__init__(cursor)
self.inputsizes = inputsizes
- def do_inserts(self):
+ def _do_inserts(self):
def replace_var(match):
name = match.group(1)
Added: relstorage/trunk/relstorage/adapters/tests/test_batch.py
===================================================================
--- relstorage/trunk/relstorage/adapters/tests/test_batch.py (rev 0)
+++ relstorage/trunk/relstorage/adapters/tests/test_batch.py 2009-10-09 23:44:43 UTC (rev 104978)
@@ -0,0 +1,293 @@
+##############################################################################
+#
+# Copyright (c) 2009 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import unittest
+
+class RowBatcherTests(unittest.TestCase):
+
+ def getClass(self):
+ from relstorage.adapters.batch import RowBatcher
+ return RowBatcher
+
+ def test_delete_defer(self):
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor)
+ batcher.delete_from("mytable", "id", 2)
+ self.assertEqual(cursor.executed, [])
+ self.assertEqual(batcher.rows_added, 1)
+ self.assertEqual(batcher.size_added, 0)
+ self.assertEqual(batcher.deletes, {('mytable', 'id'): set(["2"])})
+
+ def test_delete_auto_flush(self):
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor)
+ batcher.row_limit = 2
+ batcher.delete_from("mytable", "id", 2)
+ batcher.delete_from("mytable", "id", 1)
+ self.assertEqual(cursor.executed,
+ [('DELETE FROM mytable WHERE id IN (1,2)', None)])
+ self.assertEqual(batcher.rows_added, 0)
+ self.assertEqual(batcher.size_added, 0)
+ self.assertEqual(batcher.deletes, {})
+
+ def test_insert_defer(self):
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor)
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (1, 'a'),
+ rowkey=1,
+ size=3,
+ )
+ self.assertEqual(cursor.executed, [])
+ self.assertEqual(batcher.rows_added, 1)
+ self.assertEqual(batcher.size_added, 3)
+ self.assertEqual(batcher.inserts, {
+ ('INSERT', 'mytable (id, name)', '%s, id || %s'): {1: (1, 'a')}
+ })
+
+ def test_insert_replace(self):
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor)
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (1, 'a'),
+ rowkey=1,
+ size=3,
+ command='REPLACE',
+ )
+ self.assertEqual(cursor.executed, [])
+ self.assertEqual(batcher.rows_added, 1)
+ self.assertEqual(batcher.size_added, 3)
+ self.assertEqual(batcher.inserts, {
+ ('REPLACE', 'mytable (id, name)', '%s, id || %s'): {1: (1, 'a')}
+ })
+
+ def test_insert_duplicate(self):
+ # A second insert on the same rowkey replaces the first insert.
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor)
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (1, 'a'),
+ rowkey=1,
+ size=3,
+ )
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (1, 'b'),
+ rowkey=1,
+ size=3,
+ )
+ self.assertEqual(cursor.executed, [])
+ self.assertEqual(batcher.rows_added, 2)
+ self.assertEqual(batcher.size_added, 6)
+ self.assertEqual(batcher.inserts, {
+ ('INSERT', 'mytable (id, name)', '%s, id || %s'): {1: (1, 'b')}
+ })
+
+ def test_insert_auto_flush(self):
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor)
+ batcher.size_limit = 10
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (1, 'a'),
+ rowkey=1,
+ size=5,
+ )
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (2, 'B'),
+ rowkey=2,
+ size=5,
+ )
+ self.assertEqual(cursor.executed, [(
+ 'INSERT INTO mytable (id, name) VALUES\n'
+ '(%s, id || %s),\n'
+ '(%s, id || %s)',
+ (1, 'a', 2, 'B'))
+ ])
+ self.assertEqual(batcher.rows_added, 0)
+ self.assertEqual(batcher.size_added, 0)
+ self.assertEqual(batcher.inserts, {})
+
+ def test_flush(self):
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor)
+ batcher.delete_from("mytable", "id", 1)
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (1, 'a'),
+ rowkey=1,
+ size=5,
+ )
+ batcher.flush()
+ self.assertEqual(cursor.executed, [
+ ('DELETE FROM mytable WHERE id IN (1)', None),
+ ('INSERT INTO mytable (id, name) VALUES\n(%s, id || %s)',
+ (1, 'a')),
+ ])
+
+
+class PostgreSQLRowBatcherTests(unittest.TestCase):
+
+ def getClass(self):
+ from relstorage.adapters.batch import PostgreSQLRowBatcher
+ return PostgreSQLRowBatcher
+
+ def test_insert_postgresql_8_1(self):
+ class MockVersionDetector:
+ def get_version(self, cursor):
+ return (8, 1)
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor, MockVersionDetector())
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (1, 'a'),
+ rowkey=1,
+ size=3,
+ )
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (2, 'b'),
+ rowkey=2,
+ size=3,
+ )
+ self.assertEqual(cursor.executed, [])
+ batcher.flush()
+ self.assertEqual(cursor.executed, [
+ ('INSERT INTO mytable (id, name) VALUES (%s, id || %s)', (1, 'a')),
+ ('INSERT INTO mytable (id, name) VALUES (%s, id || %s)', (2, 'b'))
+ ])
+
+
+class OracleRowBatcherTests(unittest.TestCase):
+
+ def getClass(self):
+ from relstorage.adapters.batch import OracleRowBatcher
+ return OracleRowBatcher
+
+ def test_insert_one_row(self):
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor, {})
+ batcher.insert_into(
+ "mytable (id, name)",
+ "%s, id || %s",
+ (1, 'a'),
+ rowkey=1,
+ size=3,
+ )
+ self.assertEqual(cursor.executed, [])
+ batcher.flush()
+ self.assertEqual(cursor.executed, [
+ ('INSERT INTO mytable (id, name) VALUES (%s, id || %s)', (1, 'a')),
+ ])
+
+ def test_insert_two_rows(self):
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor, {})
+ batcher.insert_into(
+ "mytable (id, name)",
+ ":id, :id || :name",
+ {'id': 1, 'name': 'a'},
+ rowkey=1,
+ size=3,
+ )
+ batcher.insert_into(
+ "mytable (id, name)",
+ ":id, :id || :name",
+ {'id': 2, 'name': 'b'},
+ rowkey=2,
+ size=3,
+ )
+ self.assertEqual(cursor.executed, [])
+ batcher.flush()
+ self.assertEqual(cursor.executed, [(
+ 'INSERT ALL\n'
+ 'INTO mytable (id, name) VALUES (:id_0, :id_0 || :name_0)\n'
+ 'INTO mytable (id, name) VALUES (:id_1, :id_1 || :name_1)\n'
+ 'SELECT * FROM DUAL',
+ {'id_0': 1, 'id_1': 2, 'name_1': 'b', 'name_0': 'a'})
+ ])
+
+ def test_insert_one_raw_row(self):
+ class MockRawType:
+ pass
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor, {'rawdata': MockRawType})
+ batcher.insert_into(
+ "mytable (id, data)",
+ ":id, :rawdata",
+ {'id': 1, 'rawdata': 'xyz'},
+ rowkey=1,
+ size=3,
+ )
+ batcher.flush()
+ self.assertEqual(cursor.executed, [
+ ('INSERT INTO mytable (id, data) VALUES (:id, :rawdata)',
+ {'id': 1, 'rawdata': 'xyz'})
+ ])
+ self.assertEqual(cursor.inputsizes, {'rawdata': MockRawType})
+
+ def test_insert_two_raw_rows(self):
+ class MockRawType:
+ pass
+ cursor = MockCursor()
+ batcher = self.getClass()(cursor, {'rawdata': MockRawType})
+ batcher.insert_into(
+ "mytable (id, data)",
+ ":id, :rawdata",
+ {'id': 1, 'rawdata': 'xyz'},
+ rowkey=1,
+ size=3,
+ )
+ batcher.insert_into(
+ "mytable (id, data)",
+ ":id, :rawdata",
+ {'id': 2, 'rawdata': 'abc'},
+ rowkey=2,
+ size=3,
+ )
+ batcher.flush()
+ self.assertEqual(cursor.executed, [(
+ 'INSERT ALL\n'
+ 'INTO mytable (id, data) VALUES (:id_0, :rawdata_0)\n'
+ 'INTO mytable (id, data) VALUES (:id_1, :rawdata_1)\n'
+ 'SELECT * FROM DUAL',
+ {'id_0': 1, 'id_1': 2, 'rawdata_0': 'xyz', 'rawdata_1': 'abc'})
+ ])
+ self.assertEqual(cursor.inputsizes, {
+ 'rawdata_0': MockRawType,
+ 'rawdata_1': MockRawType,
+ })
+
+
+class MockCursor:
+ def __init__(self):
+ self.executed = []
+ self.inputsizes = {}
+ def setinputsizes(self, **kw):
+ self.inputsizes.update(kw)
+ def execute(self, stmt, params=None):
+ self.executed.append((stmt, params))
More information about the checkins
mailing list