[Checkins] SVN: Sandbox/J1m/s3_blob_hack/ hack
Jim Fulton
jim at zope.com
Tue Jul 19 16:01:59 EDT 2011
Log message for revision 122295:
hack
Changed:
U Sandbox/J1m/s3_blob_hack/buildout.cfg
U Sandbox/J1m/s3_blob_hack/setup.py
A Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/
A Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py
-=-
Modified: Sandbox/J1m/s3_blob_hack/buildout.cfg
===================================================================
--- Sandbox/J1m/s3_blob_hack/buildout.cfg 2011-07-19 18:26:20 UTC (rev 122294)
+++ Sandbox/J1m/s3_blob_hack/buildout.cfg 2011-07-19 20:01:59 UTC (rev 122295)
@@ -1,12 +1,23 @@
[buildout]
develop = .
-parts = test py
+parts = py zodb
[test]
-recipe = zc.recipe.testrunner
-eggs =
+eggs = zc.s3blobhack
+ zope.testing
[py]
recipe = zc.recipe.egg
eggs = ${test:eggs}
interpreter = py
+
+[zodb]
+recipe = zc.recipe.egg
+eggs = ZODB3
+ zc.s3blobhack
+s =
+initialization =
+ ${:s}import os
+ ${:s}if 'AWS_SECRET_ACCESS_KEY' in os.environ:
+ ${:s} import zc.s3blobhack
+ ${:s} zc.s3blobhack.patch()
Modified: Sandbox/J1m/s3_blob_hack/setup.py
===================================================================
--- Sandbox/J1m/s3_blob_hack/setup.py 2011-07-19 18:26:20 UTC (rev 122294)
+++ Sandbox/J1m/s3_blob_hack/setup.py 2011-07-19 20:01:59 UTC (rev 122295)
@@ -11,12 +11,14 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-name, version = 'zc.', '0'
+name, version = 'zc.s3blobhack', '0'
-install_requires = ['setuptools']
+install_requires = ['setuptools', 'ZODB3', 'boto']
extras_require = dict(test=['zope.testing'])
entry_points = """
+[console_scripts]
+copy-blobs-to-s3 = zc.s3blobhack:copy_blobs
"""
from setuptools import setup
Added: Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py
===================================================================
--- Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py (rev 0)
+++ Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py 2011-07-19 20:01:59 UTC (rev 122295)
@@ -0,0 +1,83 @@
+# S3 blob test
+import boto.s3.connection
+import boto.s3.key
+import os
+import re
+import sys
+import tempfile
+import time
+import ZEO.StorageServer
+
+# copy script that gets around the fact that we haven't implemented
+# committing blobs.
+
+hexmatch = re.compile('0x(\d\d)$').match
+
+def copy_blobs(args=None):
+ if args is None:
+ args = sys.argv[1:]
+ [blob_dir] = args
+
+ bucket_name, folder = os.environ['S3_FOLDER'].split('/', 1)
+
+ conn = boto.s3.connection.S3Connection()
+ bucket = conn.get_bucket(bucket_name)
+ prefix = blob_dir
+ if not prefix.endswith('/'):
+ prefix += '/'
+ lprefix = len(prefix)
+
+ key = boto.s3.key.Key(bucket)
+
+ logfile = open("/tmp/copy_blobs_to_s3-%s-%s-%s.log"
+ % (bucket_name, folder, time.time()),
+ 'w')
+
+ for dirpath, dirs, files in os.walk(blob_dir):
+ for n in files:
+ if not n.endswith('.blob'):
+ continue
+ p = os.path.join(dirpath, n)
+ oid = ''.join(hexmatch(seg).group(1)
+ for seg in dirpath[lprefix:].split('/')
+ )
+ serial = n[2:-5]
+ key.key = "%s/%s/%s" % (folder, oid, serial)
+ t = time.time()
+ key.set_contents_from_filename(p)
+ sz = os.stat(p).st_size
+ print >>logfile, int((time.time()-t)*1000000), sz
+
+# Monkey patch that lods blobs from s
+
+def patch():
+
+ bucket_name, folder = os.environ['S3_FOLDER'].split('/', 1)
+
+ def sendBlob(self, oid, serial):
+ try:
+ key = self._s3key
+ except AttributeError:
+ conn = boto.s3.connection.S3Connection()
+ bucket = self._s3bucket = conn.get_bucket(bucket_name)
+ key = self._s3key = boto.s3.key.Key(
+ boto.s3.connection.S3Connection().get_bucket(bucket_name)
+ )
+ key.key = "%s/%s/%s" % (folder, oid.encode('hex'), serial.encode('hex'))
+ f = tempfile.TemporaryFile()
+ key.get_contents_to_file(f)
+ f.seek(0)
+
+ def store():
+ yield ('receiveBlobStart', (oid, serial))
+ while 1:
+ chunk = f.read(59000)
+ if not chunk:
+ break
+ yield ('receiveBlobChunk', (oid, serial, chunk, ))
+ f.close()
+ yield ('receiveBlobStop', (oid, serial))
+
+ self.client.rpc.callAsyncIterator(store())
+
+ ZEO.StorageServer.ZEOStorage.sendBlob = sendBlob
Property changes on: Sandbox/J1m/s3_blob_hack/src/zc/s3blobhack/__init__.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
More information about the checkins
mailing list