[Checkins] SVN: z3c.extfile/trunk/ see CHANGES.txt
Bernd Dorn
bernd.dorn at lovelysystems.com
Thu Jan 17 12:45:36 EST 2008
Log message for revision 82936:
see CHANGES.txt
Changed:
U z3c.extfile/trunk/CHANGES.txt
U z3c.extfile/trunk/buildout.cfg
U z3c.extfile/trunk/setup.py
U z3c.extfile/trunk/src/z3c/extfile/filter.py
A z3c.extfile/trunk/src/z3c/extfile/filter.txt
U z3c.extfile/trunk/src/z3c/extfile/hashdir.py
U z3c.extfile/trunk/src/z3c/extfile/hashdir.txt
U z3c.extfile/trunk/src/z3c/extfile/interfaces.py
U z3c.extfile/trunk/src/z3c/extfile/processor.py
A z3c.extfile/trunk/src/z3c/extfile/testdata/paste.ini
U z3c.extfile/trunk/src/z3c/extfile/testing.py
U z3c.extfile/trunk/src/z3c/extfile/tests.py
U z3c.extfile/trunk/src/z3c/extfile/utility.py
-=-
Modified: z3c.extfile/trunk/CHANGES.txt
===================================================================
--- z3c.extfile/trunk/CHANGES.txt 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/CHANGES.txt 2008-01-17 17:45:35 UTC (rev 82936)
@@ -5,6 +5,13 @@
After
=====
+- TODO: set content-type in wsgi filter if info is present when
+ delivering files
+
+- tell and abort methods on WriteFile
+
+- content type recognition/restrictions in wsgi filter see filter.txt
+
- allow relative paths in directory argument for wsgi filter
- updated config example in README.txt for paste deployment
Modified: z3c.extfile/trunk/buildout.cfg
===================================================================
--- z3c.extfile/trunk/buildout.cfg 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/buildout.cfg 2008-01-17 17:45:35 UTC (rev 82936)
@@ -4,4 +4,5 @@
[test]
recipe = zc.recipe.testrunner
-eggs = z3c.extfile [test]
\ No newline at end of file
+eggs = z3c.extfile [test]
+defaults = ['--auto-color']
Modified: z3c.extfile/trunk/setup.py
===================================================================
--- z3c.extfile/trunk/setup.py 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/setup.py 2008-01-17 17:45:35 UTC (rev 82936)
@@ -37,5 +37,7 @@
'zope.schema',
'zope.security',
'zope.traversing',
+ 'Paste',
+ 'PasteDeploy',
],
)
Modified: z3c.extfile/trunk/src/z3c/extfile/filter.py
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/filter.py 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/src/z3c/extfile/filter.py 2008-01-17 17:45:35 UTC (rev 82936)
@@ -2,6 +2,7 @@
from z3c.extfile import processor, hashdir
from cStringIO import StringIO
import mimetypes
+import interfaces
BLOCK_SIZE = 1024*128
@@ -20,16 +21,25 @@
self.hd = hashdir.HashDir(self.dir)
def __call__(self, env, start_response):
- if env.get('REQUEST_METHOD')=='POST' and \
+ if env.get('X-EXTFILE-HANDLE') and env.get('REQUEST_METHOD')=='POST' and \
env.get('CONTENT_TYPE','').startswith('multipart/form-data;'):
fp = env['wsgi.input']
out = StringIO()
- proc = processor.Processor(self.hd)
+ proc = processor.Processor(
+ self.hd,
+ contentInfo=env.has_key('X-EXTFILE-INFO'),
+ allowedTypes=env.get('X-EXTFILE-TYPES'),
+ )
cl = env.get('CONTENT_LENGTH')
if not cl:
raise RuntimeError, "No content-length header found"
cl = int(cl)
- proc.pushInput(fp, out, cl)
+ try:
+ proc.pushInput(fp, out, cl)
+ except interfaces.TypeNotAllowed:
+ start_response("400 Bad Request", [
+ ('Content-Type', 'text/plain')])
+ return []
env['CONTENT_LENGTH'] = out.tell()
out.seek(0)
env['wsgi.input'] = out
@@ -38,6 +48,8 @@
return resp(env, start_response)
return self.app(env, start_response)
+
+
class FileResponse(object):
def __init__(self, app, hd):
Added: z3c.extfile/trunk/src/z3c/extfile/filter.txt
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/filter.txt (rev 0)
+++ z3c.extfile/trunk/src/z3c/extfile/filter.txt 2008-01-17 17:45:35 UTC (rev 82936)
@@ -0,0 +1,134 @@
+==================
+Upload WSGI Filter
+==================
+
+This wsgi filter replaces any multipart file upload with a hash by
+using the processor (see processor.txt).
+
+Our testing application runs through our extfile filter and returns the input
+stream.
+
+ >>> app.post('/', params=dict(x=1)).body
+ 'x=1'
+
+We take the testfiles from z3c.filetype for
+this. We see that only the hash gets sent to the application.
+
+ >>> import z3c.filetype, os
+ >>> def testFile(name):
+ ... return os.path.join(os.path.dirname(z3c.filetype.__file__),
+ ... 'testdata', name)
+
+
+So let us upload a file. We need a specific header to be present
+otherwise the filter is disabled.
+
+ >>> print app.post('/', params=dict(x=1),
+ ... upload_files=(('myfile',testFile('test.html')),),).body
+ ------------a_...$
+ Content-Disposition: form-data; name="x"
+ 1
+ ------------a_...$
+ Content-Disposition: form-data; name="myfile"; filename="...test.html"
+ Content-Type: text/html
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <title>Title</title>
+ <BLANKLINE>
+ <body>
+ body
+ </body>
+ </html>
+ ------------a_...$--
+
+So let us set the header, this should be done by the frontend server normally.
+
+ >>> env = {'X-EXTFILE-HANDLE':'on'}
+ >>> print app.post('/', params=dict(x=1), extra_environ=env,
+ ... upload_files=(('myfile',testFile('test.png')),),).body
+ ------------a_...$
+ Content-Disposition: form-data; name="x"
+ 1
+ ------------a_...$
+ Content-Disposition: form-data; name="myfile"; filename="...test.png"
+ Content-Type: application/x-z3c.extfile-info
+ z3c.extfile.digest:8154ea0062bc100c0de661cba37740863b34e79f
+ ------------...$--
+
+Filetype recognition
+====================
+
+When enabled the upload filter adds the content type and length to the
+uplaad data. We can enable this by setting an additional header.
+
+ >>> env['X-EXTFILE-INFO'] = 'on'
+ >>> print app.post('/', params=dict(x=1), extra_environ=env,
+ ... upload_files=(('myfile',testFile('test.png')),),).body
+ ---...
+ z3c.extfile.digest:8154ea0062bc100c0de661cba37740863b34e79f:image/png:4412
+ ...$--
+
+
+We can also restrict types by setting a regex in a header. If the type
+does not match a 400 is raised.
+
+ >>> env['X-EXTFILE-TYPES'] = 'text/html'
+ >>> print app.post('/', params=dict(x=1), extra_environ=env,
+ ... upload_files=(('myfile',testFile('test.png')),),).body
+ Traceback (most recent call last):
+ ...
+ AppError: Bad response: 400 Bad Request (not 200 OK or 3xx redirect for /)
+
+Let us upload a html file.
+
+ >>> print app.post('/', params=dict(x=1), extra_environ=env,
+ ... upload_files=(('myfile',testFile('test.html')),),).body
+ -----...
+ Content-Type: application/x-z3c.extfile-info
+ z3c.extfile.digest:9a2e5260cd0001b96b623d25b01194ca7d8008db:text/html:126
+ ...
+
+For example we can allow only jpegs and pngs to be uploaded.
+
+ >>> env['X-EXTFILE-TYPES'] = 'image/((jpe?g)|(png))'
+ >>> print app.post('/', extra_environ=env,
+ ... upload_files=(('myfile',testFile('test.png')),),).status
+ 200
+ >>> print app.post('/', extra_environ=env,
+ ... upload_files=(('myfile',testFile('test.html')),),).status
+ Traceback (most recent call last):
+ ...
+ AppError: Bad response: 400 Bad Request (not 200 OK or 3xx redirect for /)
+
+ >>> del env['X-EXTFILE-TYPES']
+ >>> print app.post('/', extra_environ=env,
+ ... upload_files=(('myfile',testFile('ipod.mp4')),),).body
+ ---...
+ z3c.extfile.digest:4934828cf300711df0af9879b0b479c1c18e5707:video/mp4:1026603
+ ...
+
+Let us check the size of the file.
+
+ >>> from z3c.extfile.hashdir import HashDir
+ >>> path = os.environ['EXTFILE_STORAGEDIR']
+ >>> hd = HashDir(path)
+ >>> hd.getSize('4934828cf300711df0af9879b0b479c1c18e5707')
+ 1026603L
+
+Some more type tests because the filter only looks at the first line,
+we have to make sure that the types are recognized.
+
+ >>> files = [(name, testFile(name)) for name in \
+ ... ('IMG_0504.JPG', 'faces_gray.avi', 'jumps.mov', 'test.tgz')]
+
+ >>> print app.post('/', extra_environ=env, upload_files=files).body
+ ---...
+ z3c...:e641782f446534f6c4d8ae2ce2ae4e6ad0e13738:image/jpeg:511110
+ ...
+ z3c...:f53154408fec55e610a9d48e2ebe6f0eb981ce6c:video/x-msvideo:196608
+ ...
+ z3c...:27f8cd025077e08228ca34c37cc0c7536e592e0d:video/quicktime:77449
+ ...
+ z3c...:3641323866cd50dd809a926cee773849cb6c8a85:application/x-gzip:4552
+ ...
+
Property changes on: z3c.extfile/trunk/src/z3c/extfile/filter.txt
___________________________________________________________________
Name: svn:eol-style
+ native
Modified: z3c.extfile/trunk/src/z3c/extfile/hashdir.py
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/hashdir.py 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/src/z3c/extfile/hashdir.py 2008-01-17 17:45:35 UTC (rev 82936)
@@ -156,13 +156,23 @@
self.handle = handle
self.path = path
self.sha = sha.new()
+ self._pos = 0
def write(self, s):
self.sha.update(s)
os.write(self.handle, s)
+ self._pos += len(s)
def commit(self):
"""returns the sha digest and saves the file"""
os.close(self.handle)
return self.hd.commit(self)
+ def tell(self):
+ """see file.tell"""
+ return self._pos
+
+ def abort(self):
+ """abort the write and delete file"""
+ os.close(self.handle)
+ os.unlink(self.path)
Modified: z3c.extfile/trunk/src/z3c/extfile/hashdir.txt
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/hashdir.txt 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/src/z3c/extfile/hashdir.txt 2008-01-17 17:45:35 UTC (rev 82936)
@@ -142,7 +142,7 @@
>>> hd.getPath('abc')
Traceback (most recent call last):
...
- ValueError: abc
+ ValueError: 'abc'
If we have a valid digest but it is not there a KeyError is raised.
>>> hd.getPath('da39a3ee5e6b4b0d3255bfef95601890afd80700')
Modified: z3c.extfile/trunk/src/z3c/extfile/interfaces.py
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/interfaces.py 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/src/z3c/extfile/interfaces.py 2008-01-17 17:45:35 UTC (rev 82936)
@@ -1,6 +1,9 @@
from zope import interface, schema
from zope.schema.interfaces import IBytes
+class TypeNotAllowed(Exception):
+ pass
+
class IHashDir(interface.Interface):
"""a hashdir utility"""
@@ -54,5 +57,11 @@
"""commits the file to be stored with the digest"""
-
+ def abort():
+ """aborts the writing of file and deletes it"""
+
+ def tell():
+
+ """returns the current position, same as file.tell"""
+
Modified: z3c.extfile/trunk/src/z3c/extfile/processor.py
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/processor.py 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/src/z3c/extfile/processor.py 2008-01-17 17:45:35 UTC (rev 82936)
@@ -1,6 +1,12 @@
import time
import os
import stat
+from z3c.filetype import api
+from z3c.filetype.interfaces import filetypes
+from zope import interface
+from cStringIO import StringIO
+import interfaces
+import re
BLOCK_SIZE = 1024*128
@@ -22,9 +28,13 @@
class Processor:
- def __init__(self, hd):
+ def __init__(self, hd, contentInfo=False, allowedTypes=None):
self.hd = hd
self._incoming = []
+ self.contentInfo = contentInfo
+ self.allowedTypes = allowedTypes and re.compile(allowedTypes)
+ self._ct = None
+ self._len = None
# we use a state pattern where the handle method gets
# replaced by the current handle method for this state.
self.handle = self.handle_first_boundary
@@ -112,8 +122,11 @@
def _end():
# write last line, but without \r\n
self._f.write(self._previous_line[:-2])
+ size = self._f.tell()
digest = self._f.commit()
out.write('z3c.extfile.digest:%s' % digest)
+ if self.contentInfo:
+ out.write(':%s:%s' % (self._ct, size))
out.write('\r\n')
out.write(line)
self._f = None
@@ -125,8 +138,36 @@
elif line == self._last_boundary:
_end()
self._f = None
+ self._ct = None
self.handle = None # shouldn't be called again
else:
if self._previous_line is not None:
self._f.write(self._previous_line)
+ elif self.contentInfo:
+ ct = self._sniffType(line)
+ if self.allowedTypes is not None:
+ if self.allowedTypes.match(ct) is None:
+ self._f.abort()
+ raise interfaces.TypeNotAllowed, repr(ct)
+ self._ct = ct
self._previous_line = line
+
+ def _sniffType(self, sample):
+ f = StringIO(sample)
+ ifaces = api.getInterfacesFor(f)
+ decl = interface.Declaration(ifaces)
+ for iface in decl.flattened():
+ mt = iface.queryTaggedValue(filetypes.MT)
+ if mt is not None:
+ return mt
+
+ def _getInfo(self, digest):
+ f = self.hd.open(digest)
+ ifaces = api.getInterfacesFor(f)
+ decl = interface.Declaration(ifaces)
+ for iface in decl.flattened():
+ mt = iface.queryTaggedValue(filetypes.MT)
+ if mt is not None:
+ break
+ return (mt, int(len(f)))
+
Added: z3c.extfile/trunk/src/z3c/extfile/testdata/paste.ini
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/testdata/paste.ini (rev 0)
+++ z3c.extfile/trunk/src/z3c/extfile/testdata/paste.ini 2008-01-17 17:45:35 UTC (rev 82936)
@@ -0,0 +1,7 @@
+[app:main]
+paste.app_factory = z3c.extfile.testing:app_factory
+filter-with = fs
+
+[filter:fs]
+paste.filter_factory = z3c.extfile.filter:filter_factory
+
Modified: z3c.extfile/trunk/src/z3c/extfile/testing.py
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/testing.py 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/src/z3c/extfile/testing.py 2008-01-17 17:45:35 UTC (rev 82936)
@@ -23,3 +23,15 @@
extfileDir = tempfile.mkdtemp()
os.environ['EXTFILE_STORAGEDIR'] = tempfile.mkdtemp()
+class In2OutApplication(object):
+ """
+ returns the input stream
+ """
+ def __call__(self, environ, start_response):
+ #import pdb;pdb.set_trace()
+ start_response("200 OK", [('Content-Type', 'text/plain')])
+ for l in environ.get('wsgi.input'):
+ yield l
+
+def app_factory(global_conf, **local_conf):
+ return In2OutApplication()
Modified: z3c.extfile/trunk/src/z3c/extfile/tests.py
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/tests.py 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/src/z3c/extfile/tests.py 2008-01-17 17:45:35 UTC (rev 82936)
@@ -21,8 +21,19 @@
from zope.testing.doctestunit import DocFileSuite, DocTestSuite
from zope.app.testing import setup
+from paste.fixture import setup_module
+from paste.fixture import TestApp
+import os
+here = os.path.dirname(__file__)
+def setUpWSGI(test):
+ test.globs['app'] = TestApp('config:testdata/paste.ini',
+ relative_to=here)
+def setUpNoEnv(test):
+ if os.environ['EXTFILE_STORAGEDIR']:
+ del os.environ['EXTFILE_STORAGEDIR']
+
def test_suite():
return unittest.TestSuite(
@@ -38,6 +49,9 @@
DocFileSuite('processor.txt',
optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
),
+ DocFileSuite('filter.txt', setUp=setUpWSGI,
+ optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
+ ),
DocFileSuite('property.txt',
optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
),
@@ -50,8 +64,7 @@
DocTestSuite('z3c.extfile.utility',
optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
),
- # this needs to be the last test !
- DocFileSuite('testing.txt',
+ DocFileSuite('testing.txt', setUp=setUpNoEnv,
optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
),
))
Modified: z3c.extfile/trunk/src/z3c/extfile/utility.py
===================================================================
--- z3c.extfile/trunk/src/z3c/extfile/utility.py 2008-01-17 12:49:39 UTC (rev 82935)
+++ z3c.extfile/trunk/src/z3c/extfile/utility.py 2008-01-17 17:45:35 UTC (rev 82936)
@@ -12,6 +12,7 @@
If not path is defined None is returned
+ >>> del os.environ['EXTFILE_STORAGEDIR']
>>> getPath() is None
True
More information about the Checkins
mailing list