[Checkins] SVN: zodbupgrade/trunk/src/zodbupgrade/picklefilter.py Started working on code that allows modifying pickles directly.
Christian Theune
ct at gocept.com
Fri Jun 12 12:24:04 EDT 2009
Log message for revision 100898:
Started working on code that allows modifying pickles directly.
Changed:
A zodbupgrade/trunk/src/zodbupgrade/picklefilter.py
-=-
Added: zodbupgrade/trunk/src/zodbupgrade/picklefilter.py
===================================================================
--- zodbupgrade/trunk/src/zodbupgrade/picklefilter.py (rev 0)
+++ zodbupgrade/trunk/src/zodbupgrade/picklefilter.py 2009-06-12 16:24:04 UTC (rev 100898)
@@ -0,0 +1,105 @@
+##############################################################################
+#
+# Copyright (c) 2009 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Tools for filtering a pickle opcode stream (as generated by
+pickletools.genops) and reassemblying the pickle.
+"""
+
+import ZODB
+import sys
+import struct
+import pickle
+import pickletools
+import StringIO
+
+
+# The following functions were created on the basis of the code found in
+# pickle.py. They reflect all opcodes that pickle knows about and how they get
+# written to the output stream under the knowledge how pickletools.genops
+# parses the opcode arguments.
+
+packi = lambda arg:struct.pack('<i', arg)
+reprn = lambda arg:repr(arg)+'\n'
+fact_ref = lambda arg:arg.replace(' ','\n')+'\n'
+arg_len = lambda arg:packi(len(arg))+arg
+
+noargs = '().NQR]abdeostu}l\x81\x85\x86\x87\x88\x89210'
+generators = {
+ 'G': lambda arg:struct.pack('>d', arg),
+ 'I': lambda arg:reprn(arg) if type(arg) is int else '0%s\n' % int(arg),
+ 'J': packi,
+ 'K': chr,
+ 'L': reprn,
+ 'M': lambda arg:"%c%c" % (arg&0xff, arg>>8),
+ 'S': reprn,
+ 'T': arg_len,
+ 'U': lambda arg:chr(len(arg)) + arg,
+ 'X': lambda arg:arg_len(arg.encode('utf-8')),
+ 'c': fact_ref,
+ 'i': fact_ref,
+ 'h': chr,
+ 'j': packi,
+ 'p': reprn,
+ 'g': reprn,
+ 'q': chr,
+ 'r': packi,
+ 'P': lambda arg:str(arg)+'\n',
+ 'V': lambda arg:arg.replace('\\', '\\u005c').replace('\n', '\\u000a').encode('raw-unicode-escape')+'\n',
+ '\x80': chr,
+ '\x82': chr,
+ '\x83': lambda arg:"%c%c" % (arg&0xff, arg>>8),
+ '\x84': packi,
+ '\x8a': lambda arg:chr(len(pickle.encode_long(arg)))+pickle.encode_long(arg),
+ '\x8b': lambda arg:arg_len(pickle.encode_long(arg)),
+}
+
+
+def analyze(p):
+ new = ''
+ for op, arg, pos in pickletools.genops(p):
+ new += op.code
+ if op.code in noargs:
+ pass
+ elif op.code in generators:
+ generated = generators[op.code](arg)
+ new += generated
+ else:
+ print "+", op.code, arg, pos
+ raise SystemExit()
+ return new
+
+total = root._p_jar._db.objectCount()
+storage = root._p_jar._storage
+count = 0
+next = None
+while True:
+ last = next
+ count += 1
+ oid, tid, data, next = storage.record_iternext(next)
+ pickle_data = StringIO.StringIO(data)
+ # ZODB records consist of two concatenated pickles, so the following
+ # needs to be done twice:
+ if not count % 5000:
+ print '%s objects (%s%%)' % (count, float(count)/total*100.0)
+ print ZODB.utils.oid_repr(oid), ZODB.utils.oid_repr(tid)
+ new = ''
+ for i in range(2):
+ new += analyze(pickle_data)
+ if new != data:
+ print repr(last)
+ print repr(new)
+ print "="*80
+ print repr(data)
+ raise SystemExit()
+ if next is None:
+ break
Property changes on: zodbupgrade/trunk/src/zodbupgrade/picklefilter.py
___________________________________________________________________
Added: svn:eol-style
+ native
More information about the Checkins
mailing list