[Checkins] SVN: Sandbox/J1m/resumelb/ Freshened basic (non ZooKeeper) worker and lb support.
jim
cvs-admin at zope.org
Fri Mar 9 20:00:43 UTC 2012
Log message for revision 124541:
Freshened basic (non ZooKeeper) worker and lb support.
Changed:
U Sandbox/J1m/resumelb/buildout.cfg
U Sandbox/J1m/resumelb/src/zc/resumelb/lb.py
U Sandbox/J1m/resumelb/src/zc/resumelb/worker.py
U Sandbox/J1m/resumelb/src/zc/resumelb/zk.py
U Sandbox/J1m/resumelb/src/zc/resumelb/zk.test
-=-
Modified: Sandbox/J1m/resumelb/buildout.cfg
===================================================================
--- Sandbox/J1m/resumelb/buildout.cfg 2012-03-09 15:22:41 UTC (rev 124540)
+++ Sandbox/J1m/resumelb/buildout.cfg 2012-03-09 20:00:39 UTC (rev 124541)
@@ -11,7 +11,7 @@
[ctl]
recipe = zc.recipe.rhrc
dest = ${buildout:bin-directory}
-parts = lb worker
+parts = lb worker basic-lb basic-worker
[test]
recipe = zc.recipe.testrunner
@@ -30,7 +30,8 @@
[lb]
recipe = zc.zdaemonrecipe
path = ${zookeeper:effective-path}/lb
-program = ${buildout:bin-directory}/zkresumelb -d ${zookeeper:zookeeper} ${:path}
+program =
+ ${buildout:bin-directory}/zkresumelb -d ${zookeeper:zookeeper} ${:path}
[worker.ini]
recipe = zc.recipe.deployment:configuration
@@ -50,3 +51,26 @@
[worker]
recipe = zc.zdaemonrecipe
program = ${buildout:bin-directory}/paster serve ${worker.ini:location}
+
+[basic-lb]
+recipe = zc.zdaemonrecipe
+path = ${zookeeper:effective-path}/lb
+program = ${buildout:bin-directory}/resumelb -LINFO -a :8080 ${basic-worker:address}
+
+[basic-worker.ini]
+recipe = zc.recipe.deployment:configuration
+text =
+ [app:main]
+ use = egg:bobo
+ bobo_resources = zc.resumelb.tests
+
+ [server:main]
+ use = egg:zc.resumelb
+ threads = 1
+ tracelog = tracelog
+ address = ${basic-worker:address}
+
+[basic-worker]
+address = 127.0.0.1:8000
+recipe = zc.zdaemonrecipe
+program = ${buildout:bin-directory}/paster serve ${basic-worker.ini:location}
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/lb.py
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/lb.py 2012-03-09 15:22:41 UTC (rev 124540)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/lb.py 2012-03-09 20:00:39 UTC (rev 124541)
@@ -8,6 +8,7 @@
import re
import sys
import webob
+import zc.parse_addr
import zc.resumelb.util
block_size = 1<<16
@@ -351,14 +352,99 @@
return classifier
def main(args=None):
+ """%prog [options] worker_addresses
+
+ Run a resume-based load balancer on the give addresses.
+ """
if args is None:
args = sys.argv[1:]
- logging.basicConfig(level=logging.INFO)
+ import optparse
+ parser = optparse.OptionParser(main.__doc__)
+ parser.add_option(
+ '-a', '--address', default=':0',
+ help="Address to listed on for web requests"
+ )
+ parser.add_option(
+ '-l', '--access-log', default='-',
+ help='Access-log path.\n\n'
+ 'Use - (default) for standard output.\n'
+ )
+ parser.add_option(
+ '-b', '--backlog', type='int',
+ help="Server backlog setting.")
+ parser.add_option(
+ '-m', '--max-connections', type='int',
+ help="Maximum number of simultanious accepted connections.")
+ parser.add_option(
+ '-L', '--logger-configuration',
+ help=
+ "Read logger configuration from the given configuration file path.\n"
+ "\n"
+ "The configuration file must be in ZConfig logger configuration syntax."
+ )
+ parser.add_option(
+ '-r', '--request-classifier', default='zc.resumelb.lb:host_classifier',
+ help="Request classification function (module:expr)"
+ )
+ parser.add_option(
+ '-e', '--disconnect-message',
+ help="Path to error page to use when a request is lost due to "
+ "worker disconnection"
+ )
+ parser.add_option(
+ '-v', '--variance', type='float', default=4.0,
+ help="Maximum ration of a worker backlog to the mean backlog"
+ )
+ parser.add_option(
+ '--backlog-history', type='int', default=9,
+ help="Rough numner of requests to average worker backlogs over"
+ )
+
+ parser.add_option(
+ '--unskilled-score', type='float', default=1.0,
+ help="Score (requests/second) to assign to new workers."
+ )
+
+ options, args = parser.parse_args(args)
+ if not args:
+ print 'Error: must supply one or more worker addresses.'
+ parser.parse_args(['-h'])
+
+ if options.logger_configuration:
+ logger_config = options.logger_configuration
+ if re.match(r'\d+$', logger_config):
+ logging.basicConfig(level=int(logger_config))
+ elif logger_config in ('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'):
+ logging.basicConfig(level=getattr(logging, logger_config))
+ else:
+ import ZConfig
+ with open(logger_config) as f:
+ ZConfig.configureLoggers(f.read())
+
addrs = map(parse_addr, args)
- wsgi_addr = addrs.pop(0)
+ wsgi_addr = parse_addr(options.address)
- lb = LB(addrs, host_classifier)
+ lb = LB(addrs, host_classifier,
+ variance=options.variance,
+ backlog_history=options.backlog_history,
+ unskilled_score=options.unskilled_score)
+
+ # Now, start a wsgi server
+ addr = zc.parse_addr.parse_addr(options.address)
+ if options.max_connections:
+ spawn= gevent.pool.Pool(options.max_connections)
+ else:
+ spawn = 'default'
+
+ accesslog = options.access_log
+ if isinstance(accesslog, str):
+ accesslog = sys.stdout if accesslog == '-' else open(accesslog, 'a')
+
+ gevent.pywsgi.WSGIServer(
+ wsgi_addr, lb.handle_wsgi, backlog = options.backlog,
+ spawn = spawn, log = accesslog)
+
gevent.pywsgi.WSGIServer(wsgi_addr, lb.handle_wsgi).serve_forever()
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/worker.py
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/worker.py 2012-03-09 15:22:41 UTC (rev 124540)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/worker.py 2012-03-09 20:00:39 UTC (rev 124541)
@@ -27,6 +27,7 @@
def __init__(self, app, addr,
history=9999, max_skill_age=None,
resume_file=None, threads=None, tracelog=None):
+ history = int(history)
self.app = app
self.history = history
self.max_skill_age = max_skill_age or history * 10
@@ -49,7 +50,7 @@
self.connections = set()
if threads:
- self.threadpool = gevent.threadpool.ThreadPool(threads)
+ self.threadpool = gevent.threadpool.ThreadPool(int(threads))
pool_apply = self.threadpool.apply
else:
pool_apply = None
@@ -251,11 +252,9 @@
pass
-def server_runner(app, global_conf, address, history=500, threads=0, **kw):
+def server_runner(app, global_conf, address, **kw):
# paste deploy hook
logging.basicConfig(level=logging.INFO)
host, port = address.split(':')
- Worker(app, (host, int(port)), dict(history=history),
- threads=threads and int(threads),
- **kw).server.serve_forever()
+ Worker(app, (host, int(port)), **kw).server.serve_forever()
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/zk.py
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/zk.py 2012-03-09 15:22:41 UTC (rev 124540)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/zk.py 2012-03-09 20:00:39 UTC (rev 124541)
@@ -101,11 +101,13 @@
'-m', '--max-connections', type='int',
help="Maximum number of simultanious accepted connections.")
parser.add_option(
- '--logger-configuration',
+ '-L', '--logger-configuration',
help=
"Read logger configuration from the given configuration file path.\n"
"\n"
"The configuration file must be in ZConfig logger configuration syntax."
+ "\n"
+ "Alternatively, you can give a Python logger level name or number."
)
parser.add_option(
'-r', '--request-classifier', default='zc.resumelb.lb:host_classifier',
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/zk.test
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/zk.test 2012-03-09 15:22:41 UTC (rev 124540)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/zk.test 2012-03-09 20:00:39 UTC (rev 124541)
@@ -171,11 +171,12 @@
-m MAX_CONNECTIONS, --max-connections=MAX_CONNECTIONS
Maximum number of simultanious accepted
connections.
- --logger-configuration=LOGGER_CONFIGURATION
+ -L LOGGER_CONFIGURATION, --logger-configuration=LOGGER_CONFIGURATION
Read logger configuration from the given
configuration file path. The configuration
file must be in ZConfig logger configuration
- syntax.
+ syntax. Alternatively, you can give a Python
+ logger level name or number.
-r REQUEST_CLASSIFIER, --request-classifier=REQUEST_CLASSIFIER
Request classification function
(module:expr)
More information about the checkins
mailing list