Andrew's git - logparse.git/blob - logparse/parsers/httpd.py

   1#
   2#   httpd.py
   3#   
   4#   Analyse Apache (httpd) server logs, including data transferred, requests,
   5#   clients, and errors. Note that Apache's logs can get filled up very quickly
   6#   with the default verbosity, leading to logparse taking a very long time to
   7#   analyse them. In general the default verbosity is good, but logs should be
   8#   cleared as soon as they are analysed (make sure 'rotate' is set to 'y'). 
   9#
  10
  11import re
  12
  13from ..formatting import *
  14from ..util import readlog, resolve
  15from .. import config
  16
  17import logging
  18logger = logging.getLogger(__name__)
  19
  20def parse_log():
  21    logger.debug("Starting httpd section")
  22    section = Section("httpd")
  23    accesslog = readlog(config.prefs['logs']['httpd'] + '/access.log')
  24    a = len(accesslog.split('\n'))
  25    errorlog = readlog(config.prefs['logs']['httpd'] + '/error.log')
  26    e = len(errorlog.split('\n'))
  27    data_b = 0
  28    ips = []
  29    files = []
  30    useragents = []
  31    errors = []
  32    notfound = []
  33    unprivileged = []
  34
  35    logger.debug("Searching through access log")
  36    for line in accesslog.split('\n'):
  37        fields = re.search('^(\S*) .*GET (\/.*) HTTP/\d\.\d\" 200 (\d*) \"(.*)\".*\((.*)\;', line)
  38        try:
  39            ips.append(resolve(fields.group(1), fqdn=config.prefs['httpd']['resolve-domains']))
  40            files.append(fields.group(2))
  41            useragents.append(fields.group(5))
  42            data_b += int(fields.group(3))
  43        except Exception as error:
  44            if type(error) is AttributeError: # this line is not an access log
  45                pass
  46            else:
  47                logger.warning("Error processing httpd access log: " + str(error))
  48                traceback.print_exc()
  49    data_h = parsesize(data_b)
  50
  51    logger.info("httpd has transferred " + str(data_b) + " bytes in response to " + str(a) + " requests with " + str(e) + " errors")
  52    if (a > 0):
  53        logger.debug("Parsing request statistics (this might take a while)")
  54        request_data = Data()
  55        request_data.items = backticks(files)
  56        request_data.orderbyfreq()
  57        request_data.truncl(config.prefs['maxlist'])
  58        request_data.subtitle = plural(" request", a)
  59        section.append_data(request_data)
  60    if (ips != None):
  61        logger.debug("Parsing client statistics")
  62        client_data = Data()
  63        client_data.items = orderbyfreq(ips)
  64        client_data.subtitlte = plural(" client", str(len(ips)))
  65        client_data.truncl(config.prefs['maxlist'])
  66        section.append_data(client_data)
  67    if (useragents != None):
  68        logger.debug("Parsing user agent statistics")
  69        ua_data = Data()
  70        ua_data.items = orderbyfreq(useragents)
  71        n_ua = str(len(ua_data.items))
  72        ua_data.truncl(config.prefs['maxlist'])
  73        ua_data.subtitle = plural(" user agent", n_ua)
  74        section.append_data(client_data)
  75
  76    section.append_data(Data(data_h + " transferred"))
  77    section.append_data(Data(plural(" error", e)))
  78
  79    logger.info("Finished httpd section")
  80    return section