Andrew's git - logparse.git/blob

243af06be1900ea745c8d17ee0db3425e1f3af87
   1#
   2#   httpd.py
   3#   
   4#   Analyse Apache (httpd) server logs, including data transferred, requests,
   5#   clients, and errors. Note that Apache's logs can get filled up very quickly
   6#   with the default verbosity, leading to logparse taking a very long time to
   7#   analyse them. In general the default verbosity is good, but logs should be
   8#   cleared as soon as they are analysed (make sure 'rotate' is set to 'y'). 
   9#
  10
  11import re
  12
  13from ..formatting import *
  14from ..util import readlog, resolve
  15from .. import config
  16
  17import logging
  18logger = logging.getLogger(__name__)
  19
  20def parse_log():
  21    output = ''
  22    logger.debug("Starting httpd section")
  23    output += opentag('div', 1, 'httpd', 'section')
  24    accesslog = readlog(config['logs']['httpd'] + '/access.log')
  25    a = len(accesslog.split('\n'))
  26    errorlog = readlog(config['logs']['httpd'] + '/error.log')
  27    e = len(errorlog.split('\n'))
  28    data_b = 0
  29    ips = []
  30    files = []
  31    useragents = []
  32    errors = []
  33    notfound = []
  34    unprivileged = []
  35
  36    logger.debug("Searching through access log")
  37    for line in accesslog.split('\n'):
  38        fields = re.search('^(\S*) .*GET (\/.*) HTTP/\d\.\d\" 200 (\d*) \"(.*)\".*\((.*)\;', line)
  39        try:
  40            ips.append(resolve(fields.group(1), fqdn=config.prefs['httpd']['resolve-domains']))
  41            files.append(fields.group(2))
  42            useragents.append(fields.group(5))
  43            data_b += int(fields.group(3))
  44        except Exception as error:
  45            if type(error) is AttributeError: # this line is not an access log
  46                pass
  47            else:
  48                logger.warning("Error processing httpd access log: " + str(error))
  49                traceback.print_exc()
  50    data_h = parsesize(data_b)
  51    output += writetitle("apache")
  52
  53    logger.info("httpd has transferred " + str(data_b) + " bytes in response to " + str(a) + " requests with " + str(e) + " errors")
  54    if (a > 0):
  55        logger.debug("Parsing request statistics (this might take a while)")
  56        files = addtag(files, 'code')
  57        files = orderbyfreq(files)
  58        files = truncl(files, config.prefs['maxlist'])
  59        output += writedata(plural(" request", a), files)
  60    if (ips != None):
  61        logger.debug("Parsing client statistics")
  62        ips = addtag(ips, 'code')
  63        ips = orderbyfreq(ips)
  64        n_ip = str(len(ips))
  65        ips = truncl(ips, config.prefs['maxlist'])
  66        output += writedata(plural(" client", n_ip), ips)
  67    if (useragents != None):
  68        logger.debug("Parsing user agent statistics")
  69        useragents = addtag(useragents, 'code')
  70        useragents = orderbyfreq(useragents)
  71        n_ua = str(len(useragents))
  72        useragents = truncl(useragents, config.prefs['maxlist'])
  73        output += writedata(plural(" device", n_ua), useragents)
  74
  75    output += writedata(data_h + " transferred")
  76    output += writedata(plural(" error", e))
  77
  78    output += closetag('div', 1)
  79    logger.info("Finished httpd section")
  80    return output