[Checkins] SVN: zc.zservertracelog/trunk/src/zc/zservertracelog/ Added possibility to specify date range to parse from trace log file.
Mantas Zimnickas
cvs-admin at zope.org
Fri Jun 29 14:27:50 UTC 2012
Log message for revision 127189:
Added possibility to specify date range to parse from trace log file.
Some times trace log file can become very huge, and parsing whore file takes
some time. With this added feature one can specify date range and only log
lines that are in this range will be parsed and processed.
Changed:
A zc.zservertracelog/trunk/src/zc/zservertracelog/fseek.py
U zc.zservertracelog/trunk/src/zc/zservertracelog/tests.py
U zc.zservertracelog/trunk/src/zc/zservertracelog/tracereport.py
U zc.zservertracelog/trunk/src/zc/zservertracelog/tracereport.txt
-=-
Added: zc.zservertracelog/trunk/src/zc/zservertracelog/fseek.py
===================================================================
--- zc.zservertracelog/trunk/src/zc/zservertracelog/fseek.py (rev 0)
+++ zc.zservertracelog/trunk/src/zc/zservertracelog/fseek.py 2012-06-29 14:27:46 UTC (rev 127189)
@@ -0,0 +1,123 @@
+import StringIO
+import string
+import unittest
+
+
+def fseek(f, size, search, getkey=string.strip):
+ # Check first line
+ key = getkey(f.readline())
+ if key >= search:
+ f.seek(0)
+ return 0
+
+ seen = 0
+ position = 0
+ seek = chunk = size / 2
+ while chunk > 0:
+ f.seek(seek)
+ line = f.readline() # maybe incomplete
+ position = f.tell()
+ line = f.readline() # complete
+ key = getkey(line)
+
+ chunk /= 2
+
+ if key >= search:
+ seek -= chunk
+ seen = position
+ else:
+ seek += chunk
+
+ if seen and key < search:
+ position = seen
+
+ f.seek(position)
+ return position
+
+
+def _get_btree_args(*lines):
+ content = '\n'.join(lines)
+ f = StringIO.StringIO(content)
+ size = len(content)
+ return (f, size)
+
+
+def _get_colon_key(line):
+ key, x = line.split(':', 1)
+ return key.strip()
+
+
+class FSeekTest(unittest.TestCase):
+ def test_1(self):
+ f, size = _get_btree_args(
+ '0',
+ '1',
+ '2',
+ '3',
+ '4',
+ )
+
+ p = fseek(f, size, '0')
+ self.assertEqual(p, 0)
+
+ f.seek(0) ; p = fseek(f, size, '1')
+ self.assertEqual(p, 2)
+
+ f.seek(0) ; p = fseek(f, size, '2')
+ self.assertEqual(p, 4)
+
+ f.seek(0) ; p = fseek(f, size, '3')
+ self.assertEqual(p, 6)
+
+ f.seek(0) ; p = fseek(f, size, '4')
+ self.assertEqual(p, 8)
+
+ def test_2(self):
+ f, size = _get_btree_args(
+ '0',
+ '0',
+ '0',
+ '1',
+ '1',
+ '1',
+ '2',
+ '2',
+ '2',
+ )
+
+ p = fseek(f, size, '0')
+ self.assertEqual(p, 0)
+
+ f.seek(0) ; p = fseek(f, size, '1')
+ self.assertEqual(p, 6)
+
+ f.seek(0) ; p = fseek(f, size, '2')
+ self.assertEqual(p, 12)
+
+ def test_3(self):
+ f, size = _get_btree_args(
+ '0: sdf asdfasd ',
+ '1: dffwef',
+ '2: afwefwfaf adfa sdf wef aefasdfa sdfae',
+ '3: afefw',
+ '4:',
+ )
+
+ p = fseek(f, size, '0')
+ self.assertEqual(p, 0)
+
+ f.seek(0) ; p = fseek(f, size, '1', _get_colon_key)
+ self.assertEqual(p, 16)
+
+ f.seek(0) ; p = fseek(f, size, '2', _get_colon_key)
+ self.assertEqual(p, 26)
+
+ f.seek(0) ; p = fseek(f, size, '3', _get_colon_key)
+ self.assertEqual(p, 67)
+
+ f.seek(0) ; p = fseek(f, size, '4', _get_colon_key)
+ self.assertEqual(p, 76)
+
+
+if __name__ == '__main__':
+ unittest.main()
Modified: zc.zservertracelog/trunk/src/zc/zservertracelog/tests.py
===================================================================
--- zc.zservertracelog/trunk/src/zc/zservertracelog/tests.py 2012-06-29 09:21:31 UTC (rev 127188)
+++ zc.zservertracelog/trunk/src/zc/zservertracelog/tests.py 2012-06-29 14:27:46 UTC (rev 127189)
@@ -22,6 +22,8 @@
import unittest
import zope.testing.renormalizing
+from zc.zservertracelog.fseek import FSeekTest
+
here = os.path.dirname(os.path.abspath(__file__))
checker = zope.testing.renormalizing.RENormalizing([
@@ -68,6 +70,7 @@
'tracereport.txt',
checker=checker,
setUp=analysis_setUp),
- ]
+ unittest.makeSuite(FSeekTest),
+ ]
return unittest.TestSuite(tests)
Modified: zc.zservertracelog/trunk/src/zc/zservertracelog/tracereport.py
===================================================================
--- zc.zservertracelog/trunk/src/zc/zservertracelog/tracereport.py 2012-06-29 09:21:31 UTC (rev 127188)
+++ zc.zservertracelog/trunk/src/zc/zservertracelog/tracereport.py 2012-06-29 14:27:46 UTC (rev 127189)
@@ -17,9 +17,23 @@
import datetime
import optparse
import sys
-import time
+import os.path
+from zc.zservertracelog.fseek import fseek
+
+# Date format and default offset.
+DATE_FORMATS = (
+ ('%Y-%m-%d %H:%M:%S', ''),
+ ('%Y-%m-%d %H:%M', 'seconds=59'),
+ ('%Y-%m-%d %H', 'minutes=59, seconds=59'),
+ ('%Y-%m-%d', 'hours=23, minutes=59, seconds=59'),
+ ('%Y-%m', 'days=31, hours=23, minutes=59, seconds=59'),
+)
+
+OFFSET_VALID_KEYS = ('days', 'minutes', 'seconds')
+
+
class Request(object):
output_bytes = '-'
@@ -148,11 +162,93 @@
return datetime.datetime(*args)
+def parse_offset(offset):
+ offset = offset.replace(' ', '')
+ if offset:
+ items = []
+ for item in offset.split(','):
+ key, val = item.split('=')
+ items.append((key, int(val)))
+ kwargs = dict(items)
+ else:
+ kwargs = dict()
+ kwargs['microseconds'] = 999999
+ return datetime.timedelta(**kwargs)
+
+
+def parse_date_interval(interval_string):
+ # Get offset
+ offset = ''
+ negative_offset = False
+ if ' +' in interval_string:
+ date_string, offset = interval_string.split(' +')
+ elif ' -' in interval_string:
+ date_string, offset = interval_string.split(' -')
+ negative_offset = True
+ else:
+ date_string = interval_string
+
+ # Get datetime
+ date = None
+ date_string = date_string.strip()
+ for date_format, date_format_offset in DATE_FORMATS:
+ try:
+ date = datetime.datetime.strptime(date_string, date_format)
+ except (ValueError, OverflowError):
+ pass
+ else:
+ break
+ if date is None:
+ raise TypeError("Unknown date format of '%s'." % interval_string)
+
+ # Return datetime interval tuple
+ offset = offset or date_format_offset
+ offset = parse_offset(offset)
+ if negative_offset:
+ return (date - offset, date)
+ else:
+ return (date, date + offset)
+
+
+def time_from_line(line):
+ x, x, strtime, x = parse_line(line)
+ return parse_datetime(strtime)
+
+
+def iterlog(file, date_interval=None):
+ size = 0
+ if file == '-':
+ file = sys.stdin
+ else:
+ size = os.path.getsize(file)
+ file = open(file)
+
+ date_from, date_to = date_interval or (None, None)
+ if date_from and size:
+ fseek(file, size, date_from, time_from_line)
+
+ for line in file:
+ typ, rid, strtime, msg = parse_line(line)
+ dt = parse_datetime(strtime)
+ if date_to and date_to < dt:
+ break
+ elif date_from and date_from > dt:
+ continue
+ else:
+ yield line, dt, typ, rid, strtime, msg
+
+
def main(args=None):
if args is None:
args = sys.argv[1:]
options, args = parser.parse_args(args)
+
+ if options.date:
+ date_interval = parse_date_interval(options.date)
+ else:
+ date_interval = None
+
if options.event_log:
restarts = find_restarts(options.event_log)
else:
@@ -188,17 +284,11 @@
restart = restarts.pop(0)
minutes_header()
remove_prefix = options.remove_prefix
+ dt = None
- if file == '-':
- file = sys.stdin
- else:
- file = open(file)
-
- for record in file:
- typ, rid, strtime, msg = parse_line(record)
+ for record, dt, typ, rid, strtime, msg in iterlog(file, date_interval):
min = strtime.split('.')[0]
min = min[:-3]
- dt = parse_datetime(strtime)
if dt == restart:
continue
while dt > restart:
@@ -295,11 +385,12 @@
print 'WTF', record
record_hung(urls, requests)
- print_app_requests(requests, dt,
- options.old_requests,
- options.app_requests,
- urls,
- "Left over:")
+ if dt:
+ print_app_requests(requests, dt,
+ options.old_requests,
+ options.app_requests,
+ urls,
+ "Left over:")
minutes_footer()
@@ -545,6 +636,8 @@
parser.add_option("--summary-lines", dest='summary_lines',
type="int", default=999999999,
help="""The number of summary lines to show""")
+parser.add_option("--date", "-d", dest='date', default=None,
+ help="""Date range that will be parsed from log""")
Modified: zc.zservertracelog/trunk/src/zc/zservertracelog/tracereport.txt
===================================================================
--- zc.zservertracelog/trunk/src/zc/zservertracelog/tracereport.txt 2012-06-29 09:21:31 UTC (rev 127188)
+++ zc.zservertracelog/trunk/src/zc/zservertracelog/tracereport.txt 2012-06-29 14:27:46 UTC (rev 127189)
@@ -63,8 +63,8 @@
--summary-only Only output summary lines.
--summary-lines=SUMMARY_LINES
The number of summary lines to show
+ -d DATE, --date=DATE Date range that will be parsed from log
-
Here, we display the summarized report for a sample trace log.
>>> zc.zservertracelog.tracereport.main(
@@ -620,3 +620,43 @@
</tr>
</table>
</body></html>
+
+Report by given time interval.
+
+ >>> zc.zservertracelog.tracereport.main(
+ ... ['--date', '2009-07-30 15:48 +minutes=5', sample_log])
+ <BLANKLINE>
+ minute req input wait app output
+ ================ ===== ===== ===== ===== ======
+ 2009-07-30 15:48 2 I= 0 W= 1 A= 1 O= 0 N= 2 0.43 0.35
+ 2009-07-30 15:49 0 I= 0 W= 0 A= 0 O= 0 N= 4 31.81 15.85
+ 2009-07-30 15:50 2 I= 0 W= 0 A= 2 O= 0 N= 2 0.34 0.17
+ 2009-07-30 15:51 3 I= 0 W= 1 A= 1 O= 1 N= 5 12.31 12.25
+ Left over:
+ 140.94385 /submit-photo
+ <BLANKLINE>
+ <BLANKLINE>
+ URL statistics:
+ Impact count min median mean max hangs
+ ========= ===== ====== ====== ====== ====== =====
+ 62.4 1 62.42 62.42 62.42 62.42 0 /constellations/andromeda.html
+ 60.1 1 60.13 60.13 60.13 60.13 0 /favicon.png
+ 9.7 1 9.69 9.69 9.69 9.69 0 /planets/saturn.html
+ 8.3 1 8.30 8.30 8.30 8.30 0 /moons/io.html
+ 0.8 2 0.35 0.39 0.39 0.42 0 /space-travel/plans/launchpad.html
+ 0.7 2 0.35 0.35 0.35 0.35 0 /space-travel/plans/moon-base.jpg
+ 0.4 1 0.40 0.40 0.40 0.40 0 /space-travel/plans/moon-buggy.jpg
+ 0.4 1 0.38 0.38 0.38 0.38 0 /space-travel/plans/space-logs.txt
+ 0.4 1 0.38 0.38 0.38 0.38 0 /space-travel/plans/space-diary.txt
+ 0.4 2 0.18 0.19 0.19 0.19 0 /faqs/how-to-recognize-lazer-fire.html
+ 0.4 1 0.35 0.35 0.35 0.35 0 /space-travel/plans/lunar-cycles.html
+ 0.3 1 0.34 0.34 0.34 0.34 0 /space-travel/plans/supplies.txt
+ 0.3 1 0.33 0.33 0.33 0.33 0 /space-travel/plans/cryostasis.txt
+ 0.3 1 0.32 0.32 0.32 0.32 0 /space-travel/plans/space-suit.jpg
+ 0.3 1 0.32 0.32 0.32 0.32 0 /space-travel/ships/tardis.html
+ 0.3 1 0.25 0.25 0.25 0.25 0 /approve-photo
+ 0.2 1 0.18 0.18 0.18 0.18 0 /space-travelers/famous/kirk.jpg
+ 0.2 1 0.16 0.16 0.16 0.16 0 /login
+ 0 1 /submit-photo
+ 0 1 /stars/alpha-centauri.html
+ 0 1 /faqs/how-to-charge-lazers.html
More information about the checkins
mailing list