[Checkins] SVN: Sandbox/J1m/s3_blob_hack/ hack

Jim Fulton jim at zope.com
Tue Jul 19 16:01:59 EDT 2011


Log message for revision 122295:
  hack

Changed:
  U   Sandbox/J1m/s3_blob_hack/buildout.cfg
  U   Sandbox/J1m/s3_blob_hack/setup.py
  A   Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/
  A   Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py

-=-
Modified: Sandbox/J1m/s3_blob_hack/buildout.cfg
===================================================================
--- Sandbox/J1m/s3_blob_hack/buildout.cfg	2011-07-19 18:26:20 UTC (rev 122294)
+++ Sandbox/J1m/s3_blob_hack/buildout.cfg	2011-07-19 20:01:59 UTC (rev 122295)
@@ -1,12 +1,23 @@
 [buildout]
 develop = .
-parts = test py
+parts = py zodb
 
 [test]
-recipe = zc.recipe.testrunner
-eggs = 
+eggs = zc.s3blobhack
+       zope.testing
 
 [py]
 recipe = zc.recipe.egg
 eggs = ${test:eggs}
 interpreter = py
+
+[zodb]
+recipe = zc.recipe.egg
+eggs = ZODB3
+       zc.s3blobhack
+s =
+initialization =
+   ${:s}import os
+   ${:s}if 'AWS_SECRET_ACCESS_KEY' in os.environ:
+   ${:s}    import zc.s3blobhack
+   ${:s}    zc.s3blobhack.patch()

Modified: Sandbox/J1m/s3_blob_hack/setup.py
===================================================================
--- Sandbox/J1m/s3_blob_hack/setup.py	2011-07-19 18:26:20 UTC (rev 122294)
+++ Sandbox/J1m/s3_blob_hack/setup.py	2011-07-19 20:01:59 UTC (rev 122295)
@@ -11,12 +11,14 @@
 # FOR A PARTICULAR PURPOSE.
 #
 ##############################################################################
-name, version = 'zc.', '0'
+name, version = 'zc.s3blobhack', '0'
 
-install_requires = ['setuptools']
+install_requires = ['setuptools', 'ZODB3', 'boto']
 extras_require = dict(test=['zope.testing'])
 
 entry_points = """
+[console_scripts]
+copy-blobs-to-s3 = zc.s3blobhack:copy_blobs
 """
 
 from setuptools import setup

Added: Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py
===================================================================
--- Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py	                        (rev 0)
+++ Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py	2011-07-19 20:01:59 UTC (rev 122295)
@@ -0,0 +1,83 @@
+# S3 blob test
+import boto.s3.connection
+import boto.s3.key
+import os
+import re
+import sys
+import tempfile
+import time
+import ZEO.StorageServer
+
+# copy script that gets around the fact that we haven't implemented
+# committing blobs.
+
+hexmatch = re.compile('0x(\d\d)$').match
+
+def copy_blobs(args=None):
+    if args is None:
+        args = sys.argv[1:]
+    [blob_dir] = args
+
+    bucket_name, folder = os.environ['S3_FOLDER'].split('/', 1)
+
+    conn = boto.s3.connection.S3Connection()
+    bucket = conn.get_bucket(bucket_name)
+    prefix = blob_dir
+    if not prefix.endswith('/'):
+        prefix += '/'
+    lprefix = len(prefix)
+
+    key = boto.s3.key.Key(bucket)
+
+    logfile = open("/tmp/copy_blobs_to_s3-%s-%s-%s.log"
+                   % (bucket_name, folder, time.time()),
+                   'w')
+
+    for dirpath, dirs, files in os.walk(blob_dir):
+        for n in files:
+            if not n.endswith('.blob'):
+                continue
+            p = os.path.join(dirpath, n)
+            oid = ''.join(hexmatch(seg).group(1)
+                          for seg in dirpath[lprefix:].split('/')
+                          )
+            serial = n[2:-5]
+            key.key = "%s/%s/%s" % (folder, oid, serial)
+            t = time.time()
+            key.set_contents_from_filename(p)
+            sz = os.stat(p).st_size
+            print >>logfile, int((time.time()-t)*1000000), sz
+
+# Monkey patch that lods blobs from s
+
+def patch():
+
+    bucket_name, folder = os.environ['S3_FOLDER'].split('/', 1)
+
+    def sendBlob(self, oid, serial):
+        try:
+            key = self._s3key
+        except AttributeError:
+            conn = boto.s3.connection.S3Connection()
+            bucket = self._s3bucket = conn.get_bucket(bucket_name)
+            key = self._s3key = boto.s3.key.Key(
+                boto.s3.connection.S3Connection().get_bucket(bucket_name)
+                )
+        key.key = "%s/%s/%s" % (folder, oid.encode('hex'), serial.encode('hex'))
+        f = tempfile.TemporaryFile()
+        key.get_contents_to_file(f)
+        f.seek(0)
+
+        def store():
+            yield ('receiveBlobStart', (oid, serial))
+            while 1:
+                chunk = f.read(59000)
+                if not chunk:
+                    break
+                yield ('receiveBlobChunk', (oid, serial, chunk, ))
+            f.close()
+            yield ('receiveBlobStop', (oid, serial))
+
+        self.client.rpc.callAsyncIterator(store())
+
+    ZEO.StorageServer.ZEOStorage.sendBlob = sendBlob


Property changes on: Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py
___________________________________________________________________
Added: svn:keywords
   + Id
Added: svn:eol-style
   + native



More information about the checkins mailing list