[Checkins] SVN: zc.resumelb/trunk/src/zc/resumelb/ Fixed: Workers buffered large request bodies in memory. Now large

[jim]

Log message for revision 125279:
  Fixed: Workers buffered large request bodies in memory.  Now large
  request bodies are buffered to disk.
  

Changed:
  U   zc.resumelb/trunk/src/zc/resumelb/README.txt
  U   zc.resumelb/trunk/src/zc/resumelb/tests.py
  U   zc.resumelb/trunk/src/zc/resumelb/worker.py
  U   zc.resumelb/trunk/src/zc/resumelb/worker.test

-=-
Modified: zc.resumelb/trunk/src/zc/resumelb/README.txt
===================================================================
--- zc.resumelb/trunk/src/zc/resumelb/README.txt	2012-04-24 17:38:56 UTC (rev 125278)
+++ zc.resumelb/trunk/src/zc/resumelb/README.txt	2012-04-24 18:59:10 UTC (rev 125279)
@@ -242,7 +242,7 @@
 Change History
 ==============
 
-0.4.0 (2012-04-23)
+0.4.0 (2012-04-24)
 ------------------
 
 - Change the load-balancing algorithm to take backlogs of
@@ -263,6 +263,9 @@
   start time of the oldest request for each worker, to be used for
   monitoring.
 
+- Fixed: Workers buffered large request bodies in memory.  Now large
+  request bodies are buffered to disk.
+
 0.3.0 (2012-03-28)
 ------------------
 

Modified: zc.resumelb/trunk/src/zc/resumelb/tests.py
===================================================================
--- zc.resumelb/trunk/src/zc/resumelb/tests.py	2012-04-24 17:38:56 UTC (rev 125278)
+++ zc.resumelb/trunk/src/zc/resumelb/tests.py	2012-04-24 18:59:10 UTC (rev 125279)
@@ -24,6 +24,7 @@
 import time
 import unittest
 import webob
+import zc.resumelb.worker
 import zc.zk.testing
 import zope.testing.setupstack
 import zope.testing.wait
@@ -91,6 +92,11 @@
     global pid
     pid = 6115
     test.globs['wait'] = zope.testing.wait.Wait(getsleep=lambda : gevent.sleep)
+    old_STRING_BUFFER_SIZE = zc.resumelb.worker.STRING_BUFFER_SIZE
+    zope.testing.setupstack.register(
+        test, setattr, zc.resumelb.worker,
+        'STRING_BUFFER_SIZE', old_STRING_BUFFER_SIZE)
+    zc.resumelb.worker.STRING_BUFFER_SIZE = 9999
 
 def zkSetUp(test):
     setUp(test)

Modified: zc.resumelb/trunk/src/zc/resumelb/worker.py
===================================================================
--- zc.resumelb/trunk/src/zc/resumelb/worker.py	2012-04-24 17:38:56 UTC (rev 125278)
+++ zc.resumelb/trunk/src/zc/resumelb/worker.py	2012-04-24 18:59:10 UTC (rev 125279)
@@ -10,6 +10,7 @@
 import marshal
 import os
 import sys
+import tempfile
 import time
 import zc.parse_addr
 import zc.resumelb.util
@@ -24,6 +25,8 @@
     print >>sys.stderr, mess
     traceback.print_exc()
 
+STRING_BUFFER_SIZE = 9999999
+
 class Worker:
 
     def __init__(self, app, addr,
@@ -197,7 +200,11 @@
             env['wsgi.errors'] = sys.stderr
 
             # XXX We're buffering input. Maybe should to have option not to.
-            f = cStringIO.StringIO()
+            content_length = int(env.get('CONTENT_LENGTH', 0))
+            if content_length > STRING_BUFFER_SIZE:
+                f = tempfile.TemporaryFile(suffix='.rlbwi')
+            else:
+                f = cStringIO.StringIO()
             while 1:
                 data = get()
                 if data:

Modified: zc.resumelb/trunk/src/zc/resumelb/worker.test
===================================================================
--- zc.resumelb/trunk/src/zc/resumelb/worker.test	2012-04-24 17:38:56 UTC (rev 125278)
+++ zc.resumelb/trunk/src/zc/resumelb/worker.test	2012-04-24 18:59:10 UTC (rev 125279)
@@ -116,9 +116,12 @@
 
 We can have multiple outstanding requests:
 
-    >>> write_message(worker_socket, 2, newenv('', '/hi.html'))
-    >>> write_message(worker_socket, 3, newenv('1', '/hi.html'))
-    >>> write_message(worker_socket, 4, newenv('1', '/hi.html'))
+    >>> write_message(worker_socket, 2, newenv('', '/hi.html', dict(
+    ...     REQUEST_METHOD='POST', CONTENT_LENGTH='1000')))
+    >>> write_message(worker_socket, 3, newenv('1', '/hi.html', dict(
+    ...     REQUEST_METHOD='POST', CONTENT_LENGTH='10000')))
+    >>> write_message(worker_socket, 4, newenv('1', '/hi.html', dict(
+    ...     REQUEST_METHOD='POST', CONTENT_LENGTH='100000')))
 
 At this point, we have 3 outstanding requests.  Let's create 3 bodies: