[Checkins] SVN: Sandbox/J1m/resumelb/src/zc/resumelb/ Refactored to:
Jim Fulton
jim at zope.com
Sat Jan 21 16:57:50 UTC 2012
Log message for revision 124124:
Refactored to:
- Change the connection direction between workers and lbs.
Now lbs connect to workers. Before, it seemd more attractibe for
workers to connect to lbs, since there'd be fewer lbs to keep track
of.
Now that I've discovered zookeeper, keeping track of processes is a
lot easier and having workers behave more like servers is cleaner.
- Multiple lbs are now supported.
This, of course is supported both for lb availability and
balancing of load on lbs.
(Also simplified worker tests a little.)
Changed:
U Sandbox/J1m/resumelb/src/zc/resumelb/lb.py
U Sandbox/J1m/resumelb/src/zc/resumelb/lb.test
U Sandbox/J1m/resumelb/src/zc/resumelb/tests.py
U Sandbox/J1m/resumelb/src/zc/resumelb/worker.py
U Sandbox/J1m/resumelb/src/zc/resumelb/worker.test
-=-
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/lb.py
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/lb.py 2012-01-21 12:28:18 UTC (rev 124123)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/lb.py 2012-01-21 16:57:50 UTC (rev 124124)
@@ -2,7 +2,7 @@
import gevent
import gevent.hub
import gevent.pywsgi
-import gevent.server
+import gevent.socket
import llist
import logging
import sys
@@ -23,21 +23,24 @@
class LB:
- def __init__(self, worker_addr, classifier,
+ def __init__(self, worker_addrs, classifier,
settings=None,
disconnect_message=default_disconnect_message,
):
self.classifier = classifier
self.disconnect_message = disconnect_message
self.pool = Pool(settings)
- self.worker_server = gevent.server.StreamServer(
- worker_addr, self.handle_worker)
- self.worker_server.start()
- def handle_worker(self, socket, addr):
- logger.info('new worker')
- Worker(self.pool, socket, addr)
+ self.workletts = dict(
+ (addr, gevent.spawn(self.connect, addr))
+ for addr in worker_addrs
+ )
+ def connect(self, addr):
+ while 1:
+ socket = gevent.socket.create_connection(addr)
+ Worker(self.pool, socket, addr)
+
def handle_wsgi(self, env, start_response):
rclass = self.classifier(env)
logger.debug('wsgi: %s', rclass)
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/lb.test
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/lb.test 2012-01-21 12:28:18 UTC (rev 124123)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/lb.test 2012-01-21 16:57:50 UTC (rev 124124)
@@ -1,24 +1,48 @@
Basic LB tests
==============
-The LB algorithm us tested in pool.test. This file aims to test the
+The LB algorithm is tested in pool.test. This file aims to test the
networking aspects of the LB.
-We'll start by creating a load balencer:
+An lb takes a set of worker addresses, which it connects to. It
+listens on an address for incoming web requests. You can updare it's
+set of worker addresses at any time.
+To test lb behavior, we'll create faux workers the lb can connect to.
+
+ >>> import gevent.server
+ >>> class Worker:
+ ... def __init__(self):
+ ... server = gevent.server.StreamServer(
+ ... ('127.0.0.1', 0), self.handle)
+ ... server.start()
+ ... self.addr = '127.0.0.1', server.server_port
+ ... def handle(self, socket, addr):
+ ... self.socket = socket
+
+
+ >>> workers = [Worker() for i in range(2)]
+
+We have some workers running. Now, let's create a load balancer:
+
>>> import zc.resumelb.lb
- >>> lb = zc.resumelb.lb.LB(('127.0.0.1', 0), zc.resumelb.lb.host_classifier)
+ >>> lb = zc.resumelb.lb.LB([w.addr for w in workers],
+ ... zc.resumelb.lb.host_classifier)
-Now, we'll create a couple of sockets representing workers:
+We pass the constructor an iterable of addresses. The lb will connect
+to these addresses. Let's wait for it to do so:
- >>> import gevent.socket
+ >>> wait_until(
+ ... lambda :
+ ... len([w for w in workers if hasattr(w, 'socket')]) == len(workers)
+ ... )
+
+When the workers get connections, they send the lb their resumes:
+
+ >>> worker1, worker2 = [w.socket for w in workers]
>>> from zc.resumelb.util import read_message, write_message
- >>> worker1 = gevent.socket.create_connection(
- ... ('127.0.0.1', lb.worker_server.server_port))
+
>>> write_message(worker1, 0, {'h1.com': 10.0})
-
- >>> worker2 = gevent.socket.create_connection(
- ... ('127.0.0.1', lb.worker_server.server_port))
>>> write_message(worker2, 0, {'h2.com': 10.0})
Now, let's make a request and make sure the data gets where it's
@@ -251,3 +275,24 @@
Please try again.
</body></html>
+Automatic reconnection
+======================
+
+Meanwhile, since worker1 disconnected, the load balancer reconnected
+to the same address. We can see this because the first worker (server)
+has a new socket:
+
+ >>> workers[0].socket != worker1
+ True
+
+It's not in the lb pool yet, because we haven't sent it's resume yet:
+
+ >>> len(lb.pool.workers)
+ 1
+
+But if we send a resume, it will be:
+
+ >>> write_message(workers[0].socket, 0, {})
+ >>> gevent.sleep(.01)
+ >>> len(lb.pool.workers)
+ 2
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/tests.py
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/tests.py 2012-01-21 12:28:18 UTC (rev 124123)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/tests.py 2012-01-21 16:57:50 UTC (rev 124124)
@@ -46,9 +46,20 @@
def app():
return bobo.Application(bobo_resources=__name__)
+def wait_until(func=None, timeout=9):
+ if func is None:
+ return lambda f: wait_until(f, timeout)
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ if func():
+ return
+ gevent.sleep(.01)
+ raise ValueError('timeout')
+
def setUp(test):
global pid
pid = 6115
+ test.globs['wait_until'] = wait_until
def test_suite():
return unittest.TestSuite((
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/worker.py
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/worker.py 2012-01-21 12:28:18 UTC (rev 124123)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/worker.py 2012-01-21 16:57:50 UTC (rev 124124)
@@ -2,9 +2,8 @@
import errno
import gevent
import gevent.hub
-import gevent.socket
+import gevent.server
import logging
-import socket
import sys
import time
import zc.mappingobject
@@ -13,14 +12,24 @@
logger = logging.getLogger(__name__)
-class Worker(zc.resumelb.util.Worker):
+def error(mess):
+ logger.exception(mess)
+import traceback
+def error(mess):
+ print >>sys.stderr, mess
+ traceback.print_exc()
+
+class Worker:
+
def __init__(self, app, addr, settings):
self.app = app
self.settings = zc.mappingobject.mappingobject(settings)
+ self.worker_request_number = 0
self.resume = {}
self.time_ring = []
self.time_ring_pos = 0
+ self.connections = set()
if settings.get('threads'):
pool = zc.resumelb.thread.Pool(self.settings.threads)
@@ -28,95 +37,104 @@
else:
self.apply = lambda f, *a: f(*a)
- while 1:
- try:
- self.connect(addr)
- except socket.error, err:
- if err.args[0] == errno.ECONNREFUSED:
- gevent.sleep(1)
- else:
- raise
+ self.server = gevent.server.StreamServer(addr, self.handle_connection)
+ self.server.start()
+ self.addr = addr[0], self.server.server_port
- def connect(self, addr):
- socket = gevent.socket.create_connection(addr)
- readers = self.connected(socket)
- self.put((0, self.resume))
+ def handle_connection(self, sock, addr):
+ try:
+ conn = zc.resumelb.util.Worker()
+ self.connections.add(conn)
+ readers = conn.connected(sock, addr)
+ conn.put((0, self.resume))
+ while conn.is_connected:
+ try:
+ rno, data = zc.resumelb.util.read_message(sock)
+ except gevent.GreenletExit:
+ conn.disconnected()
+ self.connections.remove(conn)
+ return
- while self.is_connected:
- try:
- rno, data = zc.resumelb.util.read_message(socket)
- except gevent.GreenletExit:
- self.disconnected()
- return
+ rput = readers.get(rno)
+ if rput is None:
+ env = data
+ env['zc.resumelb.time'] = time.time()
+ env['zc.resumelb.lb_addr'] = addr
+ gevent.spawn(self.handle, conn, rno, conn.start(rno), env)
+ else:
+ rput(data)
+ except:
+ error('handle_connection')
- rput = readers.get(rno)
- if rput is None:
- env = data
- env['zc.resumelb.time'] = time.time()
- env['zc.resumelb.lb_addr'] = self.addr
- gevent.spawn(self.handle, rno, self.start(rno), env)
- else:
- rput(data)
+ def handle(self, conn, rno, get, env):
+ try:
+ f = cStringIO.StringIO()
+ env['wsgi.input'] = f
+ env['wsgi.errors'] = sys.stderr
- def handle(self, rno, get, env):
- f = cStringIO.StringIO()
- env['wsgi.input'] = f
- env['wsgi.errors'] = sys.stderr
+ # XXX We're buffering input. It maybe should to have option not to.
+ while 1:
+ data = get()
+ if data:
+ f.write(data)
+ elif data is None:
+ # Request cancelled (or worker disconnected)
+ return
+ else:
+ break
+ f.seek(0)
- # XXX We're buffering input. It maybe should to have option not to.
- while 1:
- data = get()
- if data:
- f.write(data)
- elif data is None:
- # Request cancelled (or worker disconnected)
- self.end(rno)
- return
- else:
- break
- f.seek(0)
+ def start_response(status, headers, exc_info=None):
+ assert not exc_info # XXX
+ conn.put((rno, (status, headers)))
- def start_response(status, headers, exc_info=None):
- assert not exc_info # XXX
- self.put((rno, (status, headers)))
+ try:
+ for data in self.apply(self.app, env, start_response):
+ conn.put((rno, data))
- try:
- for data in self.apply(self.app, env, start_response):
- self.put((rno, data))
+ conn.put((rno, ''))
- self.put((rno, ''))
- self.readers.pop(rno)
+ elapsed = max(time.time() - env['zc.resumelb.time'], 1e-9)
+ time_ring = self.time_ring
+ time_ring_pos = rno % self.settings.history
+ rclass = env['zc.resumelb.request_class']
+ try:
+ time_ring[time_ring_pos] = rclass, elapsed
+ except IndexError:
+ while len(time_ring) <= time_ring_pos:
+ time_ring.append((rclass, elapsed))
- elapsed = max(time.time() - env['zc.resumelb.time'], 1e-9)
- time_ring = self.time_ring
- time_ring_pos = rno % self.settings.history
- rclass = env['zc.resumelb.request_class']
- try:
- time_ring[time_ring_pos] = rclass, elapsed
- except IndexError:
- while len(time_ring) <= time_ring_pos:
- time_ring.append((rclass, elapsed))
+ worker_request_number = self.worker_request_number + 1
+ self.worker_request_number = worker_request_number
+ if worker_request_number % self.settings.history == 0:
+ byrclass = {}
+ for rclass, elapsed in time_ring:
+ sumn = byrclass.get(rclass)
+ if sumn:
+ sumn[0] += elapsed
+ sumn[1] += 1
+ else:
+ byrclass[rclass] = [elapsed, 1]
+ self.new_resume(dict(
+ (rclass, n/sum)
+ for (rclass, (sum, n)) in byrclass.iteritems()
+ ))
- if rno % self.settings.history == 0:
- byrclass = {}
- for rclass, elapsed in time_ring:
- sumn = byrclass.get(rclass)
- if sumn:
- sumn[0] += elapsed
- sumn[1] += 1
- else:
- byrclass[rclass] = [elapsed, 1]
- self.new_resume(dict(
- (rclass, n/sum)
- for (rclass, (sum, n)) in byrclass.iteritems()
- ))
+ except conn.Disconnected:
+ return # whatever
+ except:
+ error('handle_connection')
+ finally:
+ conn.end(rno)
- except self.Disconnected:
- return # whatever
-
def new_resume(self, resume):
self.resume = resume
- self.put((0, resume))
+ for conn in self.connections:
+ if conn.is_connected:
+ try:
+ conn.put((0, resume))
+ except conn.Disconnected:
+ pass
def server_runner(app, global_conf, lb, history=500): # paste deploy hook
Modified: Sandbox/J1m/resumelb/src/zc/resumelb/worker.test
===================================================================
--- Sandbox/J1m/resumelb/src/zc/resumelb/worker.test 2012-01-21 12:28:18 UTC (rev 124123)
+++ Sandbox/J1m/resumelb/src/zc/resumelb/worker.test 2012-01-21 16:57:50 UTC (rev 124124)
@@ -1,42 +1,34 @@
-Workers act as wsgi servers, but rather than listenting for HTTP
-requests, they connect to a resumelb to request work.
+Basic, single-connection tests
+==============================
-To test the worker, we'll start a testing lb that let's us control
-data going to and from the worker.
+Workers act as wsgi servers, accepting requests and sending responses
+using serialized wsgi (not web requests).
- >>> import gevent.server, gevent.event
- >>> worker_socket_result = gevent.event.AsyncResult()
- >>> def handle(sock, addr):
- ... worker_socket_result.set(sock)
+Now, let's create a worker:
- >>> server = gevent.server.StreamServer(('127.0.0.1', 0), handle)
- >>> server.start()
- >>> server_port = server.server_port
-
-Now, we can start a worker:
-
>>> import zc.resumelb.worker, zc.resumelb.tests
- >>> worker = gevent.spawn(
- ... zc.resumelb.worker.Worker,
- ... zc.resumelb.tests.app(),
- ... ('127.0.0.1', server_port),
- ... dict(history=5))
+ >>> worker = zc.resumelb.worker.Worker(
+ ... zc.resumelb.tests.app(), ('127.0.0.1', 0), dict(history=5))
Here we created a worker using a test application, telling it to
-connect to our server address and to update it's resume after every
+an address to listen on and to update it's resume after every
five requests.
-Now, wait for the worker to connect:
+Note that we passed 0 as the address port. This causes an ephemeral
+port to be used. We can get the actual address using ``worker.addr``.
- >>> worker_socket = worker_socket_result.get()
+Now, we'll connect to the worker:
+ >>> import gevent.socket
+ >>> worker_socket = gevent.socket.create_connection(worker.addr)
+
Workers and the lb communicate via sized messages.
Each message consists of binary request numbers, data size and a
marshalled data string. Helper functions help us read and write
-messages. When workers connect, they send their resume and then wait
-for work to do. Because our worker has no experience :), it's resume
-is empty:
+messages. When workers accept a connection, they send their resume
+and then wait for work to do. Because our worker has no experience
+:), it's resume is empty:
>>> from zc.resumelb.util import read_message, write_message
>>> read_message(worker_socket)
@@ -54,9 +46,9 @@
... inp = env.pop('wsgi.input')
... del env['wsgi.errors']
... env['zc.resumelb.request_class'] = rclass
- ... return env, inp
+ ... return env
- >>> env, _ = newenv('', '/hi.html')
+ >>> env = newenv('', '/hi.html')
The newenv helper:
@@ -78,7 +70,12 @@
- an empty end-of-body message.
>>> def print_response(worker_socket, rno, size_only=False):
- ... rn, (status, headers) = read_message(worker_socket)
+ ... d = read_message(worker_socket)
+ ... try: rn, (status, headers) = d
+ ... except:
+ ... print 'wtf', `d`
+ ... return
+ ... #rn, (status, headers) = read_message(worker_socket)
... if rn != rno:
... raise AssertionError("Bad request numbers", rno, rn)
... print rno, status
@@ -112,12 +109,9 @@
We can have multiple outstanding requests:
- >>> env, _ = newenv('', '/hi.html')
- >>> write_message(worker_socket, 2, env)
- >>> env, inp = newenv('1', '/hi.html')
- >>> write_message(worker_socket, 3, env)
- >>> env, inp = newenv('1', '/hi.html')
- >>> write_message(worker_socket, 4, env)
+ >>> write_message(worker_socket, 2, newenv('', '/hi.html'))
+ >>> write_message(worker_socket, 3, newenv('1', '/hi.html'))
+ >>> write_message(worker_socket, 4, newenv('1', '/hi.html'))
At this point, we have 3 outstading requests. Let's create 3 bodies:
@@ -189,8 +183,7 @@
will keep track of times for the last 5 requests and compute a new
resume after 5 requests. Let's test that by making a 5th request.
- >>> env, _ = newenv('2', '/sleep.html?dur=.11')
- >>> write_message(worker_socket, 5, env, '')
+ >>> write_message(worker_socket, 5, newenv('2', '/sleep.html?dur=.11'), '')
>>> print_response(worker_socket, 5)
5 200 OK
Content-Length: 12
@@ -212,11 +205,10 @@
would be less than 10.
We can reuse request numbers. We normally don't reuse request numbers
-until we get to 4 billion or so., But lots make sure we can reuse
+until we get to 4 billion or so., But lets make sure we can reuse
them:
- >>> env, _ = newenv('', '/gen.html?size=100')
- >>> write_message(worker_socket, 1, env, '')
+ >>> write_message(worker_socket, 1, newenv('', '/gen.html?size=100'), '')
In this example, we've also requested a very large output.
@@ -227,35 +219,141 @@
<BLANKLINE>
1200000
-Reconnection
-============
+Multiple connections (multiple load balancers)
+==============================================
-If a worker is disconnected, it will automatically reconnect:
+In a production deployment, there will likely be multiple load
+balancers for redundancy. In this case, there are multiple
+connections to workers. Let's excercise that and make sure it works
+properly.
- >>> worker_socket_result = gevent.event.AsyncResult()
- >>> worker_socket.close()
- >>> worker_socket = worker_socket_result.get()
+Open a second connection:
-and send it's resume:
+ >>> worker_socket2 = gevent.socket.create_connection(worker.addr)
+We're immediately send the worker's resume -- same as the old.
+
+ >>> read_message(worker_socket2) == (zero, resume)
+ True
+
+And send simultaneous requests to each connection:
+
+ >>> write_message(worker_socket, 2, newenv('1', '/hi.html'))
+ >>> write_message(worker_socket2, 2, newenv('2', '/hi.html'))
+ >>> write_message(worker_socket, 3, newenv('1', '/hi.html'))
+ >>> write_message(worker_socket2, 3, newenv('2', '/hi.html'))
+ >>> write_message(worker_socket, 4, newenv('1', '/hi.html'))
+ >>> write_message(worker_socket2, 4, newenv('2', '/hi.html'))
+
+And pretty much repeat the multi-connection test we did for one worker:
+
+ >>> b22 = 'i'*1000
+ >>> b32 = 'j'*10000
+ >>> b42 = 'k'*100000
+
+ >>> sha1(b22)
+ 'bbf78f200f29636bb75c85467c1319e57a0d4149'
+ >>> sha1(b32)
+ '44bd0dbf8208fea52dc6180376d14798b86847bd'
+ >>> sha1(b42)
+ 'd10948c2d88f13ea561a09157dd263c38527b2b6'
+
+
+ >>> write_message(worker_socket, 2, b2)
+ >>> write_message(worker_socket2, 2, b22)
+ >>> write_message(worker_socket, 3, b3[:5000])
+ >>> write_message(worker_socket2, 3, b32[:5000])
+ >>> write_message(worker_socket, 4, b4[:5000])
+ >>> write_message(worker_socket2, 4, b42[:5000])
+ >>> write_message(worker_socket, 4, b4[5000:10000])
+ >>> write_message(worker_socket2, 4, b42[5000:10000])
+ >>> write_message(worker_socket, 3, b3[5000:10000])
+ >>> write_message(worker_socket2, 3, b32[5000:10000])
+ >>> for i in range(1, 10):
+ ... write_message(worker_socket, 4, b4[i*10000:(i+1)*10000])
+ ... write_message(worker_socket2, 4, b42[i*10000:(i+1)*10000])
+
+ >>> write_message(worker_socket, 4, '')
+ >>> print_response(worker_socket, 4) # doctest: +NORMALIZE_WHITESPACE
+ 4 200 OK
+ Content-Length: 84
+ Content-Type: text/html; charset=UTF-8; charset=UTF-8
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ http://localhost/hi.html ->
+ 6115 100000 3235771c66bf77697df635e1bce4173668d2ea32
+ <BLANKLINE>
+
+And on with our simultaneous request on multiple worker test.
+
+ >>> write_message(worker_socket2, 4, '')
+ >>> print_response(worker_socket2, 4) # doctest: +NORMALIZE_WHITESPACE
+ 4 200 OK
+ Content-Length: 84
+ Content-Type: text/html; charset=UTF-8; charset=UTF-8
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ http://localhost/hi.html ->
+ 6115 100000 d10948c2d88f13ea561a09157dd263c38527b2b6
+ <BLANKLINE>
+
+ >>> write_message(worker_socket, 2, '')
+ >>> print_response(worker_socket, 2) # doctest: +NORMALIZE_WHITESPACE
+ 2 200 OK
+ Content-Length: 82
+ Content-Type: text/html; charset=UTF-8; charset=UTF-8
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ http://localhost/hi.html ->
+ 6115 1000 c3efa690fa3fdd2e2526853eed670538ea127638
+ <BLANKLINE>
+
+ >>> write_message(worker_socket2, 2, '')
+ >>> print_response(worker_socket2, 2) # doctest: +NORMALIZE_WHITESPACE
+ 2 200 OK
+ Content-Length: 82
+ Content-Type: text/html; charset=UTF-8; charset=UTF-8
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ http://localhost/hi.html ->
+ 6115 1000 bbf78f200f29636bb75c85467c1319e57a0d4149
+ <BLANKLINE>
+
+
+We're due for another resume. We should get it on both sockets!
+
>>> zero, resume = read_message(worker_socket)
>>> zero, resume.keys(), [x for x in resume.values() if type(x) != float]
(0, ['', '1', '2'], [])
- >>> resume[''] > 10, resume['1'] > 10, resume['2'] < 10
- (True, True, True)
-We can even stop the server for a while, and the worker will
-reconnect:
+ >>> read_message(worker_socket2) == (zero, resume)
+ True
- >>> server.stop()
- >>> worker_socket.close()
- >>> gevent.sleep(3)
- >>> worker_socket_result = gevent.event.AsyncResult()
- >>> server = gevent.server.StreamServer(('127.0.0.1', server_port), handle)
- >>> server.start()
- >>> worker_socket = worker_socket_result.get()
- >>> zero, resume = read_message(worker_socket)
- >>> zero, resume.keys(), [x for x in resume.values() if type(x) != float]
- (0, ['', '1', '2'], [])
- >>> resume[''] > 10, resume['1'] > 10, resume['2'] < 10
- (True, True, True)
+
+ >>> write_message(worker_socket, 3, '')
+ >>> print_response(worker_socket, 3) # doctest: +NORMALIZE_WHITESPACE
+ 3 200 OK
+ Content-Length: 83
+ Content-Type: text/html; charset=UTF-8; charset=UTF-8
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ http://localhost/hi.html ->
+ 6115 10000 c1d5e830a1027a7b5de9e0620f3a2497d6b60c3e
+ <BLANKLINE>
+
+ >>> write_message(worker_socket2, 3, '')
+ >>> print_response(worker_socket2, 3) # doctest: +NORMALIZE_WHITESPACE
+ 3 200 OK
+ Content-Length: 83
+ Content-Type: text/html; charset=UTF-8; charset=UTF-8
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ http://localhost/hi.html ->
+ 6115 10000 44bd0dbf8208fea52dc6180376d14798b86847bd
+ <BLANKLINE>
More information about the checkins
mailing list