Andrew's git - logparse.git/blob

770640e30baff9a7d9c552b8140f23dca147569f
   1#!/usr/bin/env python
   2# -*- coding: utf-8 -*-
   3"""
   4This module is the entrypoint of the `logparse` shell command and also contains
   5single-use functions which don't fit elsewhere. All user interaction with
   6logparse should be through this module.
   7
   8This module provides the following methods:
   9    - `main()`:         Set up arguments, config, logging, and execute parsers
  10    - `rotate()`:       Rotate logs using systemd logrotate
  11    - `rotate_sim()`:   Simulate log rotation
  12"""
  13
  14import logging, logging.handlers
  15import argparse
  16import os
  17from sys import stdin, version
  18from subprocess import check_output
  19from datetime import datetime
  20
  21import logparse
  22from logparse import formatting, mail, config, load_parsers
  23
  24
  25def main():
  26    """
  27    Initialisation and general management of logparse functionaliy.
  28    """
  29
  30    # Get arguments
  31
  32    global argparser
  33    argparser = get_argparser()
  34
  35    # Load config
  36
  37    config.prefs = config.loadconf(argparser.parse_args().config)
  38    if argparser.parse_args().time_period:
  39        config.prefs.set("logparse", "period",
  40                argparser.parse_args().time_period)
  41    
  42    # Set up logging
  43
  44    logger = logging.getLogger(__name__)
  45    loghandler = logging.handlers.SysLogHandler(address = '/dev/log')
  46    loghandler.setFormatter(logging.Formatter(
  47        fmt='logparse[' + str(os.getpid()) + ']: %(message)s'))
  48    loghandler.setLevel(logging.INFO)   # don't spam syslog with debug messages
  49
  50    if (argparser.parse_args().quiet
  51            or config.prefs.getboolean("logparse", "quiet")):
  52        logging.basicConfig(level=logging.CRITICAL)
  53    elif (argparser.parse_args().verbose 
  54            or config.prefs.getboolean("logparse", "verbose")):
  55        logging.basicConfig(level=logging.DEBUG)
  56        logger.debug("Verbose mode turned on")
  57    else:
  58        logging.basicConfig(level=logging.INFO)
  59
  60    logger.addHandler(loghandler)
  61
  62    # Time analysis
  63
  64    global start
  65    start = datetime.now()
  66    logger.info("Beginning log analysis at {0} {1}".format(
  67        start.strftime(formatting.DATEFMT), start.strftime(formatting.TIMEFMT)))
  68    logger.debug("This is {0} version {1}, running on Python {2}".format(
  69        logparse.__name__, logparse.__version__, version.replace('\n', '')))
  70     
  71    # Write header
  72
  73    formatting.init_var()
  74
  75    if argparser.parse_args().plain:
  76        output = formatting.PlaintextOutput(
  77                linewidth=config.prefs.getint("plain", "linewidth"))
  78        output.append_header()
  79    else:
  80        output = formatting.HtmlOutput()
  81        output.append_header(config.prefs.get("html", "header"))
  82
  83    # Find parsers
  84    
  85    parser_names = []
  86    ignore_logs = []
  87    if argparser.parse_args().logs:
  88        parser_names = set(argparser.parse_args().logs.split())
  89    elif config.prefs.get("logparse", "parsers"):
  90        parser_names = set(config.prefs.get("logparse", "parsers").split())
  91
  92    if argparser.parse_args().ignore_logs:
  93        ignore_logs = argparser.parse_args().ignore_logs.split()
  94    elif config.prefs.get("logparse", "ignore-parsers"):
  95        ignore_logs = config.prefs.get("logparse", "ignore-parsers").split()
  96
  97    # Set up parsers
  98
  99    loader = load_parsers.ParserLoader()
 100    if parser_names:
 101        for parser_name in parser_names:
 102            if parser_name not in ignore_logs:
 103                loader.search(parser_name)
 104    else:
 105        loader.load_pkg()
 106        if ignore_logs:
 107            loader.ignore(ignore_logs)
 108
 109    # Execute parsers
 110
 111    for parser in loader.parsers:
 112        output.append_section(parser.parse_log())
 113
 114    # Write footer
 115    output.append_footer()
 116
 117    # Write output
 118    if ((argparser.parse_args().destination
 119        or config.prefs.get("logparse", "output"))
 120        and not argparser.parse_args().no_write):
 121
 122        # Determine destination path
 123        if argparser.parse_args().destination:
 124            dest_path = argparser.parse_args().destination
 125        else:
 126            dest_path = config.prefs.get("logparse", "output")
 127
 128        logger.debug("Outputting to {0}".format(dest_path))
 129
 130        # Determine whether to clobber old file
 131        if (not os.path.isfile(dest_path)) \
 132        and not (argparser.parse_args().overwrite
 133                or config.prefs.getboolean("logparse", "overwrite")):
 134
 135            if (argparser.parse_args().embed_styles
 136                    or config.prefs.getboolean("html", "embed-styles")) \
 137                and not (argparser.parse_args().plain
 138                        or config.prefs.getboolean("plain", "plain")):
 139                # Embed CSS stylesheet
 140                output.embed_css(config.prefs.get("html", "css"))
 141                output.write_embedded(dest_path)
 142            else:
 143                output.write(dest_path)
 144
 145        elif logging.root.level == logging.CRITICAL:
 146
 147            # Don't write output if running in quiet mode (only stdout)
 148            pass
 149
 150        else:
 151
 152            logger.warning("Destination file already exists")
 153            if input("Would you like to overwrite {0}? (y/n) [n] "
 154                    .format(dest_path)) == 'y':
 155                if (argparser.parse_args().embed_styles
 156                        or config.prefs.getboolean("html", "embed-styles")) \
 157                    and not (argparser.parse_args().plain
 158                        or config.prefs.getboolean("plain", "plain")):
 159
 160                    output.embed_css(config.prefs.get("html", "css"))
 161                    output.write_embedded(dest_path)
 162
 163                else:
 164                    output.write(dest_path)
 165            else:
 166                logger.warning("No output written")
 167
 168    # Send email if requested
 169
 170    if (str(argparser.parse_args().to) or str(config.prefs.get("mail", "to"))) \
 171            and not argparser.parse_args().no_mail:
 172
 173        if str(argparser.parse_args().to):
 174            to = argparser.parse_args().to
 175        else:
 176            to = config.prefs.get("mail", "to")
 177
 178        mail.sendmail(
 179            mailbin=config.prefs.get("mail", "mailbin"),
 180            body=(output.embed_css(config.prefs.get("html", "css"))
 181                if isinstance(output, formatting.HtmlOutput) else output.content),
 182            recipient=to,
 183            subject=formatting.fsubject(config.prefs.get("mail", "subject")),
 184            html=isinstance(output, formatting.HtmlOutput),
 185            sender=config.prefs.get("mail", "from"))
 186
 187    # Rotate logs if requested
 188
 189    if not argparser.parse_args().no_rotate:
 190        if (argparser.parse_args().simulate
 191                or config.prefs.getboolean("logparse", "rotate")):
 192            rotate_sim()
 193        elif (config.prefs.getboolean("logparse", "rotate")
 194                or argparser.parse_args().rotate):
 195            rotate()
 196        else:
 197            logger.debug("User doesn't want to rotate logs")
 198    else:
 199        logger.debug("User doesn't want to rotate logs")
 200
 201    # Finish up
 202
 203    finish = datetime.now()
 204    logger.info("Finished parsing logs at {0} {1} (total time: {2})".format(
 205        finish.strftime(formatting.DATEFMT),
 206        finish.strftime(formatting.TIMEFMT),
 207        finish - start))
 208
 209    if argparser.parse_args().printout:
 210        if isinstance(output, formatting.HtmlOutput) \
 211                and argparser.parse_args().embed_styles \
 212                or config.prefs.getboolean("html", "embed-styles"):
 213            output.print_stdout_embedded()
 214        else:
 215            output.print_stdout()
 216
 217    return
 218
 219def get_argparser():
 220    """
 221    Initialise arguments (in a separate function for documentation purposes)
 222    """
 223
 224    argparser = argparse.ArgumentParser(description=
 225            'Grab logs of some common services and send them by email')
 226    argparser.add_argument('-t','--to', required=False,
 227            help='mail recipient (\"to\" address)')
 228    argparser.add_argument('-c', '--config', required=False,
 229            default="/etc/logparse/logparse.conf",
 230            help='path to config file')
 231    argparser.add_argument('-p', '--print', required=False, dest='printout',
 232            action='store_true', default=False,
 233            help='print HTML to stdout')
 234    argparser.add_argument('-d', '--destination', required=False, 
 235            help='file to output HTML')
 236    argparser.add_argument('-f', '--overwrite', required=False,
 237            action='store_true', default=False, 
 238            help='force overwrite an existing output file')
 239    argparser.add_argument('-v', '--verbose', required=False, default=False,
 240            action='store_true',
 241            help='verbose console/syslog output (for debugging)')
 242    argparser.add_argument('-r', '--rotate', required=False, default=False,
 243            action='store_true',
 244            help='force rotate log files using systemd logrotate (overrides \
 245            --rotate and "rotate" in logparse.conf)')
 246    argparser.add_argument('-nr', '--no-rotate', required=False, default=False,
 247            action='store_true', 
 248            help='do not rotate log files (overrides config)')
 249    argparser.add_argument('-s', '--simulate', required=False, default=False,
 250            action="store_true",
 251            help="test run logrotate (do not actually change files)")
 252    argparser.add_argument('-l', '--logs', required=False, 
 253            help='services to analyse')
 254    argparser.add_argument('-nl', '--ignore-logs', required=False,
 255            help='skip these services (takes precedence over -l)')
 256    argparser.add_argument('-es', '--embed-styles', required=False,
 257            default=False, action='store_true',
 258            help='make CSS rules inline rather than linking the file')
 259    argparser.add_argument('-nh', '--plain', required=False, default=False,
 260            action='store_true', help='write/send plain text rather than HTML')
 261    argparser.add_argument('-q', '--quiet', required=False, default=False,
 262            action='store_true', help='no output to stdout')
 263    argparser.add_argument('-nm', '--no-mail', required=False, default=False,
 264            action="store_true",
 265            help="do not send email (overrides config file)")
 266    argparser.add_argument('-nw', '--no-write', required=False, default=False,
 267            action="store_true",
 268            help="do not write output file (overrides config file)")
 269    argparser.add_argument('-tp', '--time-period', required=False,
 270            help="time period to analyse logs for (applies to all parsers)")
 271
 272    return argparser
 273
 274
 275
 276
 277def rotate():
 278    """
 279    Rotate logs using systemd logrotate. This requires root privileges, and a
 280    basic check for this is attempted below. Root password will be prompted
 281    for if permissions are not automatically granted.
 282    """
 283
 284    logger = logging.getLogger(__name__)
 285    try:
 286        if not os.geteuid() == 0:
 287            if stdin.isatty():
 288                logger.warning("Not running as root, using sudo \
 289                        (may require password to be entered)")
 290                rotate_shell = check_output(
 291                        "sudo logrotate /etc/logrotate.conf", shell=True)
 292            else:
 293                raise PermissionError("Root priviliges are required to run \
 294                        logrotate but were not provided")
 295        else:
 296            rotate_shell = check_output(
 297                    "/usr/sbin/logrotate /etc/logrotate.conf", shell=True)
 298        logger.info("Rotated logfiles")
 299        logger.debug("logrotate output: " + rotate_shell)
 300    except Exception as e:
 301        logger.warning("Failed to rotate log files: " + str(e))
 302
 303
 304def rotate_sim():   # Simulate log rotation
 305    """
 306    Simulate log rotation using logrotate's -d flag. This does not require root
 307    privileges, but permission errors will be shown in the output without it.
 308    """
 309
 310    logger = logging.getLogger(__name__)
 311    try:
 312        if not os.geteuid() == 0:
 313            logger.warning("Cannot run logrotate as root - \
 314                    you will see permission errors in the output below")
 315        sim_cmd = "logrotate -d /etc/logrotate.conf"
 316        logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd))
 317        sim = check_output(sim_cmd, shell=True)
 318        logger.debug(sim)
 319    except Exception as e:
 320        logger.warning("Failed to get logrotate simulation: " + str(e))