[Checkins] SVN: Sandbox/J1m/s3uploadqueue/ checkpoint
jim
cvs-admin at zope.org
Fri Jun 15 13:34:41 UTC 2012
Log message for revision 126861:
checkpoint
Changed:
U Sandbox/J1m/s3uploadqueue/buildout.cfg
U Sandbox/J1m/s3uploadqueue/setup.py
A Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/
A Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/README.txt
A Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/__init__.py
A Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/tests.py
-=-
Modified: Sandbox/J1m/s3uploadqueue/buildout.cfg
===================================================================
--- Sandbox/J1m/s3uploadqueue/buildout.cfg 2012-06-15 13:33:51 UTC (rev 126860)
+++ Sandbox/J1m/s3uploadqueue/buildout.cfg 2012-06-15 13:34:38 UTC (rev 126861)
@@ -4,7 +4,7 @@
[test]
recipe = zc.recipe.testrunner
-eggs =
+eggs = zc.s3uploadqueue [test]
[py]
recipe = zc.recipe.egg
Modified: Sandbox/J1m/s3uploadqueue/setup.py
===================================================================
--- Sandbox/J1m/s3uploadqueue/setup.py 2012-06-15 13:33:51 UTC (rev 126860)
+++ Sandbox/J1m/s3uploadqueue/setup.py 2012-06-15 13:34:38 UTC (rev 126861)
@@ -11,10 +11,10 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-name, version = 'zc.', '0'
+name, version = 'zc.s3uploadqueue', '0'
-install_requires = ['setuptools']
-extras_require = dict(test=['zope.testing'])
+install_requires = ['setuptools', 'boto', 'zc.testing']
+extras_require = dict(test=['zope.testing', 'mock'])
entry_points = """
"""
Added: Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/README.txt
===================================================================
--- Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/README.txt (rev 0)
+++ Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/README.txt 2012-06-15 13:34:38 UTC (rev 126861)
@@ -0,0 +1,117 @@
+Use a file-system directory as an queue for uploading files to S3
+=================================================================
+
+This package provides support for implementing an S3 upload queue
+using a file directory.
+
+An application that wants to upload to S3 writes a temporary file and
+then moved it into a directory and then goes on about it's business.
+
+Separately, either in a daemon, or in a separate application thread,
+zc.s3uploadqueue.main is run, which watches the directory and moves
+files to S3 as they appear.
+
+The main function takes the name of a configuration file.
+
+
+The configuration file has 2 sections:
+
+Credentials
+ The AWS credentials, with options:
+
+ aws_access_key_id
+ Your AWS access key
+
+ aws_secret_access_key
+ Your AWS secret access key
+
+Queue
+ Option controlling the queue processing:
+
+ directory
+ The directory containing files to be moved to S3
+
+ bucket
+ The S3 bucket name
+
+ poll-interval
+ How often to poll for files.
+
+ This defaults to 9 seconds.
+
+ threads
+ The number of threads to use for processing files.
+
+ Defaults to 9.
+
+ log-level
+ Basic logging log level.
+
+ Defaults to INFO.
+
+The file names in the directory must be URL-encoded S3 Keys. This
+allows '/'s to be included without using directories.
+
+The ``main`` function will poll the directory every poll interval and
+upload any files found. As it uploads each file, it will remove it
+from the source directory. After uploading all of the files found, it
+will poll immediately, in case new files were added while processing
+files.
+
+Here's a sample configuration::
+
+ [Credentials]
+ aws_access_key_id = 42
+ aws_secret_access_key = k3y
+
+ [Queue]
+ directory = test
+ bucket = testbucket
+ poll-interval = 1
+ threads = 2
+
+.. -> config
+
+If we were to put files in S3, we'd place them in the ``test``
+directory. For example, to put data at the key:
+``2012-06-14/01.txt``, we'd store the data in the file::
+
+ 2012-06-14%2F01.txt
+
+.. -> name
+
+This is the key name urlquoted with no safe characters::
+
+ urllib.quote(key, safe='')
+
+We can run the updater as a long-running script (using something like
+`zdaemon <http://pypi.python.org/pypi/zdaemon>`_ or `supervisor
+<http://pypi.python.org/pypi/supervisor>`_) or as a from an existing
+process. To run from an existing process, just import the ``process``
+function and run it::
+
+ import zc.s3uploadqueue
+ zc.s3uploadqueue.process(config)
+
+.. -> src
+
+This starts a daemonic threads that upload any files placed in the directory.
+
+.. basic test with above data and code
+
+ Place some data:
+
+ >>> write(name.strip(), '')
+
+ Start the processor:
+
+ >>> exec(src)
+
+ Wait a bit and see that boto was called:
+
+ >>> import time
+ >>> time.sleep(.1)
+
+ >>> import boto.s3.connection, pprint
+ >>> pprint(boto.s3.connection.S3Connection.calls)
+
Property changes on: Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/README.txt
___________________________________________________________________
Added: svn:eol-style
+ native
Added: Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/__init__.py
===================================================================
--- Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/__init__.py (rev 0)
+++ Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/__init__.py 2012-06-15 13:34:38 UTC (rev 126861)
@@ -0,0 +1,102 @@
+##############################################################################
+#
+# Copyright (c) 2011 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import boto.s3.connection
+import boto.s3.key
+import ConfigParser
+import logging
+import os
+import Queue
+import sys
+import time
+import urllib
+import zc.thread
+
+logger = logging.getLogger(__name__)
+
+levels = dict(
+ CRITICAL=50
+ ERROR=40
+ WARNING=30
+ INFO=20
+ DEBUG=10
+ NOTSET=0
+ )
+
+def process(config=None):
+ if config is None:
+ [config] = sys.argv[1:]
+
+ parser = ConfigParser.RawConfigParser()
+ parser.read(config)
+ options = dict(dict(parser.items(section)) for section in parser.sections)
+
+ logging.basicConfig(level=levels[options['Queue'].get('log-level', 'INFO')])
+
+ queue = queue.Queue
+ srcdir = options['Queue']['directory']
+ nthreads = int(options['Queue'].get('threads', 9))
+ poll_interval = int(options['Queue'].get('poll-interval', 9))
+
+ def process_queue():
+ conn = boto.s3.connection.S3Connection(
+ options['Credentials']['aws_access_key_id'],
+ options['Credentials']['aws_secret_access_key'],
+ )
+ bucket = conn.get_bucket(options['Queue']['bucket'])
+ key = boto.s3.key.Key(bucket)
+
+ while 1:
+ try:
+ name = queue.get()
+ if name is None:
+ # stop
+ return
+ key.key = urllib.unquote(name)
+ path = os.path.join(srcdir, name)
+ key.set_contents_from_filename(path)
+ os.remove(path)
+ logger.info('Transferred %r', name)
+ except Exception:
+ logger.exception('processing %r', name)
+ finally:
+ queue.task_done()
+
+ theads = [zc.thread.Thread(process_queue) for i in range(nthreads)]
+ running = [1]
+
+ def stop():
+ running.pop()
+ for thread in threads:
+ thread.join(1)
+
+ @zc.thread.Thread(daemon=False)
+ def poll():
+ while running:
+ files = os.listdir(srcdir)
+ if files:
+ for name in files:
+ queue.put(name)
+ queue.join()
+ else:
+ time.sleep(poll_interval)
+ for thread in threads:
+ queue.put(None)
+
+ if testing:
+ return stop
+
+if __name__ == '__main__':
+ main()
+
Property changes on: Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/__init__.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Added: Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/tests.py
===================================================================
--- Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/tests.py (rev 0)
+++ Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/tests.py 2012-06-15 13:34:38 UTC (rev 126861)
@@ -0,0 +1,73 @@
+##############################################################################
+#
+# Copyright (c) 2010 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+from zope.testing import setupstack
+import doctest
+import manuel.capture
+import manuel.doctest
+import manuel.testing
+import os
+import unittest
+import zc.s3uploadqueue
+
+def write(path, s):
+ with open(path, 'w') as f:
+ f.write(s)
+
+def basic():
+ """
+
+ >>> os.mkdir('test')
+ >>> write('queue.cfg', '''
+ ... [Credentials]
+ ... aws_access_key_id = 42
+ ... aws_secret_access_key = k3y
+ ...
+ ... [Queue]
+ ... directory = test
+ ... bucket = testbucket
+ ... poll-interval = 1
+ ... ''')
+
+ >>>
+
+ """
+
+
+
+
+def setup(test):
+ setupstack.setUpDirectory(test)
+ s3conn = setupstack.context_manager(
+ test, mock.patch('boto.s3.connection.S3Connection'))
+ s3conn = setupstack.context_manager(
+ test, mock.patch('boto.s3.key.Key'))
+
+
+def test_suite():
+ return unittest.TestSuite((
+ manuel.testing.TestSuite(
+ manuel.doctest.Manuel() + manuel.capture.Manuel(),
+ 'README.txt',
+ setUp=setup, tearDown=setupstack.tearDown,
+ ),
+ # doctest.DocTestSuite(
+ # setUp=setup, tearDown=setupstack.tearDown()
+ # ),
+ ))
+
+
+
+
+def test_suite():
+
Property changes on: Sandbox/J1m/s3uploadqueue/src/zc/s3uploadqueue/tests.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
More information about the checkins
mailing list