Andrew's git - logparse.git/blob

0abf6edeb2a05432d59d05489b7fe3759e9f32de
   1#
   2#   httpd.py
   3#   
   4#   Analyse Apache (httpd) server logs, including data transferred, requests,
   5#   clients, and errors. Note that Apache's logs can get filled up very quickly
   6#   with the default verbosity, leading to logparse taking a very long time to
   7#   analyse them. In general the default verbosity is good, but logs should be
   8#   cleared as soon as they are analysed (make sure 'rotate' is set to 'y'). 
   9#
  10
  11import re
  12
  13from logparse.formatting import *
  14from logparse.util import readlog, resolve
  15from logparse import config
  16
  17import logging
  18logger = logging.getLogger(__name__)
  19
  20ACCESS_REGEX = "^\s*(\S+).*\"GET (\S+) HTTP(?:\/\d\.\d)?\" (\d{3}) (\d*) \".+\" \"(.*)\""
  21
  22class AccessLine(object):
  23
  24    def __init__(self, line):
  25        self.line = line
  26        fields = re.search(ACCESS_REGEX, line)
  27        
  28        self.client = fields.group(1)
  29        self.file = fields.group(2)
  30        self.statuscode = int(fields.group(3))
  31        self.bytes = int(fields.group(4))
  32        self.useragent = fields.group(5)
  33
  34def parse_log():
  35
  36    logger.debug("Starting httpd section")
  37    section = Section("httpd")
  38
  39    accesslog = readlog(prefs("logs", "httpd-access"))
  40
  41    errorlog= readlog(prefs("logs", "httpd-error"))
  42    total_errors = len(errorlog.splitlines())
  43
  44    logger.debug("Retrieved log data")
  45
  46    logger.debug("Searching through access log")
  47
  48    accesses = []
  49
  50    for line in accesslog.splitlines():
  51        if "GET" in line:
  52            accesses.append(AccessLine(line))
  53
  54    total_requests = len(accesses)
  55    
  56    section.append_data(Data("Total of " + plural("request", total_requests)))
  57    section.append_data(Data(plural("error", total_errors)))
  58
  59    size = Data()
  60    size.subtitle = "Transferred " + parsesize(sum([ac.bytes for ac in accesses]))
  61    section.append_data(size)
  62
  63    clients = Data()
  64    clients.items = [resolve(ac.client, config.prefs.get("httpd", "resolve-domains")) for ac in accesses]
  65    clients.orderbyfreq()
  66    clients.subtitle = "Received requests from " + plural("client", len(clients.items))
  67    clients.truncl(config.prefs.getint("logparse", "maxlist"))
  68    section.append_data(clients)
  69
  70    files = Data()
  71    files.items = [ac.file for ac in accesses]
  72    files.orderbyfreq()
  73    files.subtitle = plural("file", len(files.items)) + " requested"
  74    files.truncl(config.prefs.getint("logparse", "maxlist"))
  75    section.append_data(files)
  76
  77    useragents = Data()
  78    useragents.items = [ac.useragent for ac in accesses]
  79    useragents.orderbyfreq()
  80    useragents.subtitle = plural("user agent", len(useragents.items))
  81    useragents.truncl(config.prefs.getint("logparse", "maxlist"))
  82    section.append_data(useragents)
  83
  84    logger.info("httpd has received " + str(total_requests) + " requests with " + str(total_errors) + " errors")
  85
  86
  87    logger.info("Finished httpd section")
  88    return section