Andrew's git - logparse.git/blob

bee8d49f4ef46274e6f8d0b87b44b96e0d8c9e00
   1#!/usr/bin/env python3
   2# -*- coding: utf-8 -*-
   3
   4"""
   5This module is the entrypoint of the `logparse` shell command and also contains
   6single-use functions which don't fit elsewhere. All user interaction with
   7logparse should be through this module.
   8
   9This module provides the following methods:
  10    - `main()`:         Set up arguments, config, logging, and execute parsers
  11    - `rotate()`:       Rotate logs using systemd logrotate
  12    - `rotate_sim()`:   Simulate log rotation
  13"""
  14
  15import argparse
  16from copy import copy
  17import logging
  18import logging.handlers
  19import os
  20from sys import stdin, version
  21from subprocess import check_output
  22from datetime import datetime
  23
  24import logparse
  25from logparse import formatting, mail, config, load_parsers, util
  26
  27
  28def main():
  29    """
  30    Initialisation and general management of logparse functionaliy.
  31    """
  32
  33    # Get arguments
  34
  35    global argparser
  36    argparser = get_argparser()
  37
  38    # Load config
  39
  40    config.prefs = config.loadconf(argparser.parse_args().config)
  41    if argparser.parse_args().time_period:
  42        config.prefs.set("logparse", "period",
  43                argparser.parse_args().time_period)
  44    
  45    # Set up logging
  46    logger = logging.getLogger(__name__)
  47    if (argparser.parse_args().quiet
  48            or config.prefs.getboolean("logparse", "quiet")):
  49        logparse.logger.setLevel(logging.CRITICAL)
  50    elif (argparser.parse_args().verbose 
  51            or config.prefs.getboolean("logparse", "verbose")):
  52        logparse.logger.setLevel(logging.DEBUG)
  53        logger.debug("Verbose mode turned on")
  54    else:
  55        logparse.logger.setLevel(logging.INFO)
  56
  57
  58
  59    # Time analysis
  60
  61    global start
  62    start = datetime.now()
  63    logger.info("Beginning log analysis at {0} {1}".format(
  64        start.strftime(formatting.DATEFMT), start.strftime(formatting.TIMEFMT)))
  65    logger.debug("This is {0} version {1}, running on Python {2}".format(
  66        logparse.__name__, logparse.__version__, version.replace('\n', '')))
  67     
  68    # Write header
  69
  70    formatting.init_var()
  71
  72    if argparser.parse_args().plain:
  73        output = formatting.PlaintextOutput(
  74                linewidth=config.prefs.getint("plain", "linewidth"))
  75        output.append_header()
  76    else:
  77        output = formatting.HtmlOutput()
  78        output.append_header(config.prefs.get("html", "header"))
  79
  80    # Find parsers
  81    
  82    parser_names = []
  83    ignore_logs = []
  84    if argparser.parse_args().logs:
  85        parser_names = set(argparser.parse_args().logs.split())
  86    elif config.prefs.get("logparse", "parsers"):
  87        parser_names = set(config.prefs.get("logparse", "parsers").split())
  88
  89    if argparser.parse_args().ignore_logs:
  90        ignore_logs = argparser.parse_args().ignore_logs.split()
  91    elif config.prefs.get("logparse", "ignore-parsers"):
  92        ignore_logs = config.prefs.get("logparse", "ignore-parsers").split()
  93
  94    # Set up parsers
  95
  96    loader = load_parsers.ParserLoader()
  97    if parser_names:
  98        for parser_name in parser_names:
  99            if parser_name not in ignore_logs:
 100                loader.search(parser_name)
 101    else:
 102        loader.load_pkg()
 103        if ignore_logs:
 104            loader.ignore(ignore_logs)
 105
 106    # Execute parsers
 107
 108    for parser in loader.parsers:
 109        output.append_section(parser.parse_log())
 110
 111    # Write footer
 112    output.append_footer()
 113
 114    # Write output
 115    if ((argparser.parse_args().destination
 116        or config.prefs.get("logparse", "output"))
 117        and not argparser.parse_args().no_write):
 118
 119        # Determine destination path
 120        if argparser.parse_args().destination:
 121            dest_path = argparser.parse_args().destination
 122        else:
 123            dest_path = config.prefs.get("logparse", "output")
 124
 125        logger.debug("Outputting to {0}".format(dest_path))
 126
 127        # Determine whether to clobber old file
 128        if (not os.path.isfile(dest_path)) \
 129        and not (argparser.parse_args().overwrite
 130                or config.prefs.getboolean("logparse", "overwrite")):
 131
 132            if (argparser.parse_args().embed_styles
 133                    or config.prefs.getboolean("html", "embed-styles")) \
 134                and not (argparser.parse_args().plain
 135                        or config.prefs.getboolean("plain", "plain")):
 136                # Embed CSS stylesheet
 137                output.embed_css(config.prefs.get("html", "css"))
 138                output.write_embedded(dest_path)
 139            else:
 140                output.write(dest_path)
 141
 142        elif logging.root.level == logging.CRITICAL:
 143
 144            # Don't write output if running in quiet mode (only stdout)
 145            pass
 146
 147        else:
 148
 149            logger.warning("Destination file already exists")
 150            if input("Would you like to overwrite {0}? (y/n) [n] "
 151                    .format(dest_path)) == 'y':
 152                if (argparser.parse_args().embed_styles
 153                        or config.prefs.getboolean("html", "embed-styles")) \
 154                    and not (argparser.parse_args().plain
 155                        or config.prefs.getboolean("plain", "plain")):
 156
 157                    output.embed_css(config.prefs.get("html", "css"))
 158                    output.write_embedded(dest_path)
 159
 160                else:
 161                    output.write(dest_path)
 162            else:
 163                logger.warning("No output written")
 164
 165    # Send email if requested
 166
 167    if (str(argparser.parse_args().to) or str(config.prefs.get("mail", "to"))) \
 168            and not argparser.parse_args().no_mail:
 169
 170        if str(argparser.parse_args().to):
 171            to = argparser.parse_args().to
 172        else:
 173            to = config.prefs.get("mail", "to")
 174
 175        mail.sendmail(
 176            mailbin=config.prefs.get("mail", "mailbin"),
 177            body=(output.embed_css(config.prefs.get("html", "css"))
 178                if isinstance(output, formatting.HtmlOutput) else output.content),
 179            recipient=to,
 180            subject=formatting.fsubject(config.prefs.get("mail", "subject")),
 181            html=isinstance(output, formatting.HtmlOutput),
 182            sender=config.prefs.get("mail", "from"))
 183
 184    # Rotate logs if requested
 185
 186    if not argparser.parse_args().no_rotate:
 187        if (argparser.parse_args().simulate
 188                or config.prefs.getboolean("logparse", "rotate")):
 189            rotate_sim()
 190        elif (config.prefs.getboolean("logparse", "rotate")
 191                or argparser.parse_args().rotate):
 192            rotate()
 193        else:
 194            logger.debug("User doesn't want to rotate logs")
 195    else:
 196        logger.debug("User doesn't want to rotate logs")
 197
 198    # Finish up
 199
 200    finish = datetime.now()
 201    logger.info("Finished parsing logs at {0} {1} (total time: {2})".format(
 202        finish.strftime(formatting.DATEFMT),
 203        finish.strftime(formatting.TIMEFMT),
 204        finish - start))
 205
 206    if argparser.parse_args().printout:
 207        if isinstance(output, formatting.HtmlOutput) \
 208                and argparser.parse_args().embed_styles \
 209                or config.prefs.getboolean("html", "embed-styles"):
 210            output.print_stdout_embedded()
 211        else:
 212            output.print_stdout()
 213
 214    return
 215
 216def get_argparser():
 217    """
 218    Initialise arguments (in a separate function for documentation purposes)
 219    """
 220
 221    argparser = argparse.ArgumentParser(description=
 222            'Grab logs of some common services and send them by email')
 223    argparser.add_argument('-t','--to', required=False,
 224            help='mail recipient (\"to\" address)')
 225    argparser.add_argument('-c', '--config', required=False,
 226            default="/etc/logparse/logparse.conf",
 227            help='path to config file')
 228    argparser.add_argument('-p', '--print', required=False, dest='printout',
 229            action='store_true', default=False,
 230            help='print HTML to stdout')
 231    argparser.add_argument('-d', '--destination', required=False, 
 232            help='file to output HTML')
 233    argparser.add_argument('-f', '--overwrite', required=False,
 234            action='store_true', default=False, 
 235            help='force overwrite an existing output file')
 236    argparser.add_argument('-v', '--verbose', required=False, default=False,
 237            action='store_true',
 238            help='verbose console/syslog output (for debugging)')
 239    argparser.add_argument('-r', '--rotate', required=False, default=False,
 240            action='store_true',
 241            help='force rotate log files using systemd logrotate (overrides \
 242            --rotate and "rotate" in logparse.conf)')
 243    argparser.add_argument('-nr', '--no-rotate', required=False, default=False,
 244            action='store_true', 
 245            help='do not rotate log files (overrides config)')
 246    argparser.add_argument('-s', '--simulate', required=False, default=False,
 247            action="store_true",
 248            help="test run logrotate (do not actually change files)")
 249    argparser.add_argument('-l', '--logs', required=False, 
 250            help='services to analyse')
 251    argparser.add_argument('-nl', '--ignore-logs', required=False,
 252            help='skip these services (takes precedence over -l)')
 253    argparser.add_argument('-es', '--embed-styles', required=False,
 254            default=False, action='store_true',
 255            help='make CSS rules inline rather than linking the file')
 256    argparser.add_argument('-nh', '--plain', required=False, default=False,
 257            action='store_true', help='write/send plain text rather than HTML')
 258    argparser.add_argument('-q', '--quiet', required=False, default=False,
 259            action='store_true', help='no output to stdout')
 260    argparser.add_argument('-nm', '--no-mail', required=False, default=False,
 261            action="store_true",
 262            help="do not send email (overrides config file)")
 263    argparser.add_argument('-nw', '--no-write', required=False, default=False,
 264            action="store_true",
 265            help="do not write output file (overrides config file)")
 266    argparser.add_argument('-tp', '--time-period', required=False,
 267            help="time period to analyse logs for (applies to all parsers)")
 268
 269    return argparser
 270
 271
 272
 273
 274def rotate():
 275    """
 276    Rotate logs using systemd logrotate. This requires root privileges, and a
 277    basic check for this is attempted below. Root password will be prompted
 278    for if permissions are not automatically granted.
 279    """
 280
 281    logger = logging.getLogger(__name__)
 282    try:
 283        if not os.geteuid() == 0:
 284            if stdin.isatty():
 285                logger.warning("Not running as root, using sudo \
 286                        (may require password to be entered)")
 287                rotate_shell = check_output(
 288                        "sudo logrotate /etc/logrotate.conf", shell=True)
 289            else:
 290                raise PermissionError("Root priviliges are required to run \
 291                        logrotate but were not provided")
 292        else:
 293            rotate_shell = check_output(
 294                    "/usr/sbin/logrotate /etc/logrotate.conf", shell=True)
 295        logger.info("Rotated logfiles")
 296        logger.debug("logrotate output: " + rotate_shell)
 297    except Exception as e:
 298        logger.warning("Failed to rotate log files: " + str(e))
 299
 300
 301def rotate_sim():   # Simulate log rotation
 302    """
 303    Simulate log rotation using logrotate's -d flag. This does not require root
 304    privileges, but permission errors will be shown in the output without it.
 305    """
 306
 307    logger = logging.getLogger(__name__)
 308    try:
 309        if not os.geteuid() == 0:
 310            logger.warning("Cannot run logrotate as root - \
 311                    you will see permission errors in the output below")
 312        sim_cmd = "logrotate -d /etc/logrotate.conf"
 313        logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd))
 314        sim = check_output(sim_cmd, shell=True)
 315        logger.debug(sim)
 316    except Exception as e:
 317        logger.warning("Failed to get logrotate simulation: " + str(e))