logparse / interface.pyon commit update readme & docs (29c0dc8)
   1#!/usr/bin/env python
   2# -*- coding: utf-8 -*-
   3"""
   4This module is the entrypoint of the `logparse` shell command and also contains
   5single-use functions which don't fit elsewhere. All user interaction with
   6logparse should be through this module.
   7
   8This module provides the following methods:
   9    - `main()`:         Set up arguments, config, logging, and execute parsers
  10    - `rotate()`:       Rotate logs using systemd logrotate
  11    - `rotate_sim()`:   Simulate log rotation
  12"""
  13
  14import logging, logging.handlers
  15import argparse
  16import os
  17from sys import stdin, version
  18from subprocess import check_output
  19from datetime import datetime
  20
  21import logparse
  22from logparse import formatting, mail, config, load_parsers
  23
  24
  25def main():
  26    """
  27    Initialisation and general management of logparse functionaliy.
  28    """
  29
  30    # Get arguments
  31
  32    global argparser
  33    argparser = get_argparser()
  34
  35    # Load config
  36
  37    config.prefs = config.loadconf(argparser.parse_args().config)
  38    
  39    # Set up logging
  40
  41    logger = logging.getLogger(__name__)
  42    loghandler = logging.handlers.SysLogHandler(address = '/dev/log')
  43    loghandler.setFormatter(logging.Formatter(fmt='logparse[' + str(os.getpid()) + ']: %(message)s'))
  44    loghandler.setLevel(logging.INFO)   # don't spam syslog with debug messages
  45    if argparser.parse_args().quiet or config.prefs.getboolean("logparse", "quiet"):
  46        logging.basicConfig(level=logging.CRITICAL)
  47    elif argparser.parse_args().verbose or config.prefs.getboolean("logparse", "verbose"):
  48        logging.basicConfig(level=logging.DEBUG)
  49        logger.debug("Verbose mode turned on")
  50    else:
  51        logging.basicConfig(level=logging.INFO)
  52    logger.addHandler(loghandler)
  53
  54    # Time analysis
  55
  56    global start
  57    start = datetime.now()
  58    logger.info("Beginning log analysis at {0} {1}".format(start.strftime(formatting.DATEFMT), start.strftime(formatting.TIMEFMT)))
  59    logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, version.replace('\n', '')))
  60     
  61    # Write header
  62
  63    formatting.init_var()
  64
  65    if argparser.parse_args().plain:
  66        output = formatting.PlaintextOutput(linewidth=config.prefs.getint("plain", "linewidth"))
  67        output.append_header()
  68    else:
  69        output = formatting.HtmlOutput()
  70        output.append_header(config.prefs.get("html", "header"))
  71
  72    # Find parsers
  73    
  74    parser_names = []
  75    ignore_logs = []
  76    if argparser.parse_args().logs:
  77        parser_names = set(argparser.parse_args().logs.split())
  78    elif config.prefs.get("logparse", "parsers"):
  79        parser_names = set(config.prefs.get("logparse", "parsers").split())
  80
  81    if argparser.parse_args().ignore_logs:
  82        ignore_logs = argparser.parse_args().ignore_logs.split()
  83    elif config.prefs.get("logparse", "ignore-parsers"):
  84        ignore_logs = config.prefs.get("logparse", "ignore-parsers").split()
  85
  86    # Set up parsers
  87
  88    loader = load_parsers.ParserLoader()
  89    if parser_names:
  90        for parser_name in parser_names:
  91            if parser_name not in ignore_logs:
  92                loader.search(parser_name)
  93    else:
  94        loader.load_pkg()
  95        if ignore_logs:
  96            loader.ignore(ignore_logs)
  97
  98    # Execute parsers
  99
 100    for parser in loader.parsers:
 101        output.append_section(parser.parse_log())
 102
 103    # Write footer
 104
 105    output.append_footer()
 106
 107    # Write output
 108
 109    if (argparser.parse_args().destination or config.prefs.get("logparse", "output")) and not argparser.parse_args().no_write:
 110        if argparser.parse_args().destination:
 111            dest_path = argparser.parse_args().destination
 112        else:
 113            dest_path = config.prefs.get("logparse", "output")
 114        logger.debug("Outputting to {0}".format(dest_path))
 115        if (argparser.parse_args().embed_styles or config.prefs.getboolean("html", "embed-styles")) and not (argparser.parse_args().plain or config.prefs.getboolean("plain", "plain")):
 116            output.embed_css(config.prefs.get("html", "css"))
 117        if (not os.path.isfile(dest_path)) and not (argparser.parse_args().overwrite or config.prefs.getboolean("logparse", "overwrite")):
 118            output.write(dest_path)
 119        elif logging.root.level == logging.CRITICAL:
 120            pass
 121        else:
 122            logger.warning("Destination file already exists")
 123            if input("Would you like to overwrite {0}? (y/n) [n] ".format(dest_path)) == 'y':
 124                output.write(dest_path)
 125            else:
 126                logger.warning("No output written")
 127
 128    # Send email if requested
 129
 130    if (str(argparser.parse_args().to) or str(config.prefs.get("mail", "to"))) and not argparser.parse_args().no_mail:
 131        if str(argparser.parse_args().to):
 132            to = argparser.parse_args().to
 133        else:
 134            to = config.prefs.get("mail", "to")
 135        mail.sendmail(
 136            mailbin=config.prefs.get("mail", "mailbin"),
 137            body=(output.embed_css(config.prefs.get("html", "css")) if isinstance(output, formatting.HtmlOutput) else output.content),
 138            recipient=to,
 139            subject=formatting.fsubject(config.prefs.get("mail", "subject")),
 140            html=isinstance(output, formatting.HtmlOutput),
 141            sender=config.prefs.get("mail", "from"))
 142
 143    # Rotate logs if requested
 144
 145    if not argparser.parse_args().no_rotate:
 146        if argparser.parse_args().simulate or config.prefs.getboolean("logparse", "rotate"):
 147            rotate_sim()
 148        elif config.prefs.getboolean("logparse", "rotate") or argparser.parse_args().rotate:
 149            rotate()
 150        else:
 151            logger.debug("User doesn't want to rotate logs")
 152    else:
 153        logger.debug("User doesn't want to rotate logs")
 154
 155    # Finish up
 156
 157    finish = datetime.now()
 158    logger.info("Finished parsing logs at {0} {1} (total time: {2})".format(finish.strftime(formatting.DATEFMT), finish.strftime(formatting.TIMEFMT), finish - start))
 159
 160    if argparser.parse_args().printout:
 161        output.print_stdout()
 162
 163    return
 164
 165def get_argparser():
 166    """
 167    Initialise arguments (in a separate function for documentation purposes)
 168    """
 169    argparser = argparse.ArgumentParser(description='grab logs of some common services and send them by email')
 170    argparser.add_argument('-t','--to', help='mail recipient (\"to\" address)', required=False)
 171    argparser.add_argument('-c', '--config', help='path to config file', required=False, default="/etc/logparse/logparse.conf")
 172    argparser.add_argument('-p', '--print', help='print HTML to stdout', required=False, dest='printout', action='store_true', default=False)
 173    argparser.add_argument('-d', '--destination', help='file to output HTML', required=False)
 174    argparser.add_argument('-f', '--overwrite', help='force overwrite an existing output file', required=False, action='store_true', default=False)
 175    argparser.add_argument('-v', '--verbose', help='verbose console/syslog output (for debugging)', required=False, default=False, action='store_true')
 176    argparser.add_argument('-r', '--rotate', help='force rotate log files using systemd logrotate (overrides --rotate and "rotate" in logparse.conf)', required=False, default=False, action='store_true')
 177    argparser.add_argument('-nr', '--no-rotate', help='do not rotate logfiles (overrides --rotate and logparse.conf)', required=False, default=False, action='store_true')
 178    argparser.add_argument('-s', '--simulate', help="test run logrotate (do not actually change files)", required=False, default=False, action="store_true")
 179    argparser.add_argument('-l', '--logs', help='services to analyse', required=False)
 180    argparser.add_argument('-nl', '--ignore-logs', help='skip these services (takes precedence over -l)', required=False)
 181    argparser.add_argument('-es', '--embed-styles', help='make CSS rules inline rather than linking the file', required=False, default=False, action='store_true')
 182    argparser.add_argument('-nh', '--plain', help='write/send plain text rather than HTML', required=False, default=False, action='store_true')
 183    argparser.add_argument('-q', '--quiet', help='no output to stdout', required=False, default=False, action='store_true')
 184    argparser.add_argument('-nm', '--no-mail', help="do not send email (overrides config file)", required=False, default=False, action="store_true")
 185    argparser.add_argument('-nw', '--no-write', help="do not write output file (overrides config file)", required=False, default=False, action="store_true")
 186    return argparser
 187
 188
 189
 190
 191def rotate():
 192    """
 193    Rotate logs using systemd logrotate. This requires root privileges, and a
 194    basic check for this is attempted below. Root password will be prompted
 195    for if permissions are not automatically granted.
 196    """
 197
 198    try:
 199        if not os.geteuid() == 0:
 200            if stdin.isatty():
 201                logger.warning("Not running as root, using sudo (may require password to be entered)")
 202                rotate_shell = check_output("sudo logrotate /etc/logrotate.conf", shell=True)
 203            else:
 204                raise PermissionError("Root priviliges are required to run logrotate but are not provided")
 205        else:
 206            rotate_shell = check_output("/usr/sbin/logrotate /etc/logrotate.conf", shell=True)
 207        logger.info("Rotated logfiles")
 208        logger.debug("logrotate output: " + rotate_shell)
 209    except Exception as e:
 210        logger.warning("Failed to rotate log files: " + str(e))
 211
 212
 213def rotate_sim():   # Simulate log rotation
 214    """
 215    Simulate log rotation using logrotate's -d flag. This does not require root
 216    privileges, but permission errors will be shown in the output without it.
 217    """
 218
 219    try:
 220        if not os.geteuid() == 0:
 221            logger.warning("Cannot run logrotate as root - you will see permission errors in the output below")
 222        sim_cmd = "logrotate -d /etc/logrotate.conf"
 223        logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd))
 224        sim = check_output(sim_cmd, shell=True)
 225        logger.debug(sim)
 226    except Exception as e:
 227        logger.warning("Failed to get logrotate simulation: " + str(e))