[Checkins] SVN: zodbupgrade/trunk/src/zodbupgrade/picklefilter.py Started working on code that allows modifying pickles directly.

Christian Theune ct at gocept.com
Fri Jun 12 12:24:04 EDT 2009


Log message for revision 100898:
  Started working on code that allows modifying pickles directly.
  

Changed:
  A   zodbupgrade/trunk/src/zodbupgrade/picklefilter.py

-=-
Added: zodbupgrade/trunk/src/zodbupgrade/picklefilter.py
===================================================================
--- zodbupgrade/trunk/src/zodbupgrade/picklefilter.py	                        (rev 0)
+++ zodbupgrade/trunk/src/zodbupgrade/picklefilter.py	2009-06-12 16:24:04 UTC (rev 100898)
@@ -0,0 +1,105 @@
+##############################################################################
+#
+# Copyright (c) 2009 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Tools for filtering a pickle opcode stream (as generated by
+pickletools.genops) and reassemblying the pickle.
+"""
+
+import ZODB
+import sys
+import struct
+import pickle
+import pickletools
+import StringIO
+
+
+# The following functions were created on the basis of the code found in
+# pickle.py. They reflect all opcodes that pickle knows about and how they get
+# written to the output stream under the knowledge how pickletools.genops
+# parses the opcode arguments.
+
+packi = lambda arg:struct.pack('<i', arg)
+reprn = lambda arg:repr(arg)+'\n'
+fact_ref = lambda arg:arg.replace(' ','\n')+'\n'
+arg_len = lambda arg:packi(len(arg))+arg
+
+noargs = '().NQR]abdeostu}l\x81\x85\x86\x87\x88\x89210'
+generators = {
+    'G': lambda arg:struct.pack('>d', arg),
+    'I': lambda arg:reprn(arg) if type(arg) is int else '0%s\n' % int(arg),
+    'J': packi,
+    'K': chr,
+    'L': reprn,
+    'M': lambda arg:"%c%c" % (arg&0xff, arg>>8),
+    'S': reprn,
+    'T': arg_len,
+    'U': lambda arg:chr(len(arg)) + arg,
+    'X': lambda arg:arg_len(arg.encode('utf-8')),
+    'c': fact_ref,
+    'i': fact_ref,
+    'h': chr,
+    'j': packi,
+    'p': reprn,
+    'g': reprn,
+    'q': chr,
+    'r': packi,
+    'P': lambda arg:str(arg)+'\n',
+    'V': lambda arg:arg.replace('\\', '\\u005c').replace('\n', '\\u000a').encode('raw-unicode-escape')+'\n',
+    '\x80': chr,
+    '\x82': chr,
+    '\x83': lambda arg:"%c%c" % (arg&0xff, arg>>8),
+    '\x84': packi,
+    '\x8a': lambda arg:chr(len(pickle.encode_long(arg)))+pickle.encode_long(arg),
+    '\x8b': lambda arg:arg_len(pickle.encode_long(arg)),
+}
+
+
+def analyze(p):
+    new = ''
+    for op, arg, pos in pickletools.genops(p):
+        new += op.code
+        if op.code in noargs:
+            pass
+        elif op.code in generators:
+            generated = generators[op.code](arg)
+            new += generated
+        else:
+            print "+", op.code, arg, pos
+            raise SystemExit()
+    return new
+
+total = root._p_jar._db.objectCount()
+storage = root._p_jar._storage
+count = 0
+next = None
+while True:
+    last = next
+    count += 1
+    oid, tid, data, next = storage.record_iternext(next)
+    pickle_data = StringIO.StringIO(data)
+    # ZODB records consist of two concatenated pickles, so the following
+    # needs to be done twice:
+    if not count % 5000:
+        print '%s objects (%s%%)' % (count, float(count)/total*100.0)
+        print ZODB.utils.oid_repr(oid), ZODB.utils.oid_repr(tid)
+    new = ''
+    for i in range(2):
+        new += analyze(pickle_data)
+    if new != data:
+        print repr(last)
+        print repr(new)
+        print "="*80
+        print repr(data)
+        raise SystemExit()
+    if next is None:
+        break


Property changes on: zodbupgrade/trunk/src/zodbupgrade/picklefilter.py
___________________________________________________________________
Added: svn:eol-style
   + native



More information about the Checkins mailing list