-#
-# __main__.py
-#
-# This module is the entrypoint of the `logparse` shell command and also
-# contains single-use functions which don't fit elsewhere.
-#
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+This module is the entrypoint of the `logparse` shell command and also contains
+single-use functions which don't fit elsewhere. All user interaction with
+logparse should be through this module.
+
+This module provides the following methods:
+ - `main()`: Set up arguments, config, logging, and execute parsers
+ - `rotate()`: Rotate logs using systemd logrotate
+ - `rotate_sim()`: Simulate log rotation
+"""
import logging, logging.handlers
import argparse
import os
-import glob
-import sys
+from sys import stdin, version
from subprocess import check_output
from datetime import datetime
import logparse
-import logparse.config
-from logparse.config import prefs, loadconf
-from logparse import formatting, mail, config
-from .parsers import load_parsers
-
-global argparser
-
-def rotate(): # Rotate logs using systemd logrotate
- try:
- if not os.geteuid() == 0:
- if sys.stdin.isatty():
- logger.warning("Not running as root, using sudo (may require password to be entered)")
- rotate_shell = check_output("sudo logrotate /etc/logrotate.conf", shell=True)
- else:
- raise PermissionError("Root priviliges are required to run logrotate but are not provided")
- else:
- rotate_shell = check_output("/usr/sbin/logrotate /etc/logrotate.conf", shell=True)
- logger.info("Rotated logfiles")
- logger.debug("logrotate output: " + rotate_shell)
- except Exception as e:
- logger.warning("Failed to rotate log files: " + str(e))
-
-def rotate_sim(): # Simulate log rotation
- try:
- if not os.geteuid() == 0:
- logger.warning("Cannot run logrotate as root - you will see permission errors in the output below")
- sim_cmd = "logrotate -d /etc/logrotate.conf"
- logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd))
- sim = check_output(sim_cmd, shell=True)
- logger.debug(sim)
- except Exception as e:
- logger.warning("Failed to get logrotate simulation: " + str(e))
+from logparse import formatting, mail, config, load_parsers
def main():
+ """
+ Initialisation and general management of logparse functionaliy.
+ """
+
# Get arguments
+
global argparser
- argparser = argparse.ArgumentParser(description='grab logs of some common services and send them by email')
- argparser.add_argument('-t','--to', help='mail recipient (\"to\" address)', required=False)
- argparser.add_argument('-c', '--config', help='path to config file', required=False, default="/etc/logparse/logparse.conf")
- argparser.add_argument('-p', '--print', help='print HTML to stdout', required=False, dest='printout', action='store_true', default=False)
- argparser.add_argument('-d', '--destination', help='file to output HTML', required=False)
- argparser.add_argument('-f', '--overwrite', help='force overwrite an existing output file', required=False, action='store_true', default=False)
- argparser.add_argument('-v', '--verbose', help='verbose console/syslog output (for debugging)', required=False, default=False, action='store_true')
- argparser.add_argument('-r', '--rotate', help='force rotate log files using systemd logrotate (overrides --rotate and "rotate" in logparse.conf)', required=False, default=False, action='store_true')
- argparser.add_argument('-nr', '--no-rotate', help='do not rotate logfiles (overrides --rotate and logparse.conf)', required=False, default=False, action='store_true')
- argparser.add_argument('-s', '--simulate', help="test run logrotate (do not actually change files)", required=False, default=False, action="store_true")
- argparser.add_argument('-l', '--logs', help='services to analyse', required=False)
- argparser.add_argument('-nl', '--ignore-logs', help='skip these services (takes precedence over -l)', required=False)
- argparser.add_argument('-es', '--embed-styles', help='make CSS rules inline rather than linking the file', required=False, default=False, action='store_true')
- argparser.add_argument('-nh', '--plain', help='write/send plain text rather than HTML', required=False, default=False, action='store_true')
- argparser.add_argument('-q', '--quiet', help='no output to stdout', required=False, default=False, action='store_true')
- argparser.add_argument('-nm', '--no-mail', help="do not send email (overrides config file)", required=False, default=False, action="store_true")
- argparser.add_argument('-nw', '--no-write', help="do not write output file (overrides config file)", required=False, default=False, action="store_true")
+ argparser = get_argparser()
# Load config
- config.prefs = loadconf(argparser.parse_args().config)
+
+ config.prefs = config.loadconf(argparser.parse_args().config)
# Set up logging
+
logger = logging.getLogger(__name__)
loghandler = logging.handlers.SysLogHandler(address = '/dev/log')
loghandler.setFormatter(logging.Formatter(fmt='logparse[' + str(os.getpid()) + ']: %(message)s'))
logging.basicConfig(level=logging.INFO)
logger.addHandler(loghandler)
- logger.debug([x for x in config.prefs.sections()])
- logger.debug(config.prefs.get("logparse", "output"))
- logger.debug("Config test: " + config.prefs.get("logparse", "output"))
-
# Time analysis
+
global start
start = datetime.now()
logger.info("Beginning log analysis at {0} {1}".format(start.strftime(formatting.DATEFMT), start.strftime(formatting.TIMEFMT)))
- logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, sys.version.replace('\n', '')))
+ logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, version.replace('\n', '')))
# Write header
output = formatting.HtmlOutput()
output.append_header(config.prefs.get("html", "header"))
-
# Find parsers
- parser_providers = []
+ parser_names = []
+ ignore_logs = []
if argparser.parse_args().logs:
- log_src = argparser.parse_args().logs.split()
+ parser_names = set(argparser.parse_args().logs.split())
elif config.prefs.get("logparse", "parsers"):
- log_src = config.prefs.get("logparse", "parsers").split()
- else:
- log_src = load_parsers.default_parsers
-
- for parser_name in log_src:
- parser = load_parsers.search(parser_name)
- if parser == None:
- logger.warning("Can't find parser {0}".format(parser_name))
- continue
- else:
- parser_providers.append(load_parsers.load(parser))
+ parser_names = set(config.prefs.get("logparse", "parsers").split())
if argparser.parse_args().ignore_logs:
- ignore_src = argparser.parse_args().ignore_logs.split()
+ ignore_logs = argparser.parse_args().ignore_logs.split()
elif config.prefs.get("logparse", "ignore-parsers"):
- ignore_src = config.prefs.get("logparse", "ignore-parsers").split()
+ ignore_logs = config.prefs.get("logparse", "ignore-parsers").split()
+
+ # Set up parsers
+
+ loader = load_parsers.ParserLoader()
+ if parser_names:
+ for parser_name in parser_names:
+ if parser_name not in ignore_logs:
+ loader.search(parser_name)
else:
- ignore_src = []
- if len(ignore_src) > 0:
- for parser_name in ignore_src:
- if parser_name in [x.__name__.rpartition('.')[2] for x in parser_providers]:
- logger.info("Ignoring default parser {0}".format(parser_name))
- parser_providers_new = []
- for p in parser_providers:
- if p.__name__.rpartition('.')[2] != parser_name:
- parser_providers_new.append(p)
- parser_providers = parser_providers_new
- continue
+ loader.load_pkg()
+ if ignore_logs:
+ loader.ignore(ignore_logs)
# Execute parsers
- logger.debug(str(parser_providers))
- for parser in parser_providers:
+ for parser in loader.parsers:
output.append_section(parser.parse_log())
- # Write HTML footer
+ # Write footer
+
output.append_footer()
+ # Write output
+
if (argparser.parse_args().destination or config.prefs.get("logparse", "output")) and not argparser.parse_args().no_write:
if argparser.parse_args().destination:
dest_path = argparser.parse_args().destination
else:
logger.warning("No output written")
+ # Send email if requested
+
if (str(argparser.parse_args().to) or str(config.prefs.get("mail", "to"))) and not argparser.parse_args().no_mail:
if str(argparser.parse_args().to):
to = argparser.parse_args().to
html=isinstance(output, formatting.HtmlOutput),
sender=config.prefs.get("mail", "from"))
+ # Rotate logs if requested
+
if not argparser.parse_args().no_rotate:
if argparser.parse_args().simulate or config.prefs.getboolean("logparse", "rotate"):
rotate_sim()
else:
logger.debug("User doesn't want to rotate logs")
- # Print end message
+ # Finish up
+
finish = datetime.now()
logger.info("Finished parsing logs at {0} {1} (total time: {2})".format(finish.strftime(formatting.DATEFMT), finish.strftime(formatting.TIMEFMT), finish - start))
output.print_stdout()
return
+
+def get_argparser():
+ """
+ Initialise arguments (in a separate function for documentation purposes)
+ """
+ argparser = argparse.ArgumentParser(description='grab logs of some common services and send them by email')
+ argparser.add_argument('-t','--to', help='mail recipient (\"to\" address)', required=False)
+ argparser.add_argument('-c', '--config', help='path to config file', required=False, default="/etc/logparse/logparse.conf")
+ argparser.add_argument('-p', '--print', help='print HTML to stdout', required=False, dest='printout', action='store_true', default=False)
+ argparser.add_argument('-d', '--destination', help='file to output HTML', required=False)
+ argparser.add_argument('-f', '--overwrite', help='force overwrite an existing output file', required=False, action='store_true', default=False)
+ argparser.add_argument('-v', '--verbose', help='verbose console/syslog output (for debugging)', required=False, default=False, action='store_true')
+ argparser.add_argument('-r', '--rotate', help='force rotate log files using systemd logrotate (overrides --rotate and "rotate" in logparse.conf)', required=False, default=False, action='store_true')
+ argparser.add_argument('-nr', '--no-rotate', help='do not rotate logfiles (overrides --rotate and logparse.conf)', required=False, default=False, action='store_true')
+ argparser.add_argument('-s', '--simulate', help="test run logrotate (do not actually change files)", required=False, default=False, action="store_true")
+ argparser.add_argument('-l', '--logs', help='services to analyse', required=False)
+ argparser.add_argument('-nl', '--ignore-logs', help='skip these services (takes precedence over -l)', required=False)
+ argparser.add_argument('-es', '--embed-styles', help='make CSS rules inline rather than linking the file', required=False, default=False, action='store_true')
+ argparser.add_argument('-nh', '--plain', help='write/send plain text rather than HTML', required=False, default=False, action='store_true')
+ argparser.add_argument('-q', '--quiet', help='no output to stdout', required=False, default=False, action='store_true')
+ argparser.add_argument('-nm', '--no-mail', help="do not send email (overrides config file)", required=False, default=False, action="store_true")
+ argparser.add_argument('-nw', '--no-write', help="do not write output file (overrides config file)", required=False, default=False, action="store_true")
+ return argparser
+
+
+
+
+def rotate():
+ """
+ Rotate logs using systemd logrotate. This requires root privileges, and a
+ basic check for this is attempted below. Root password will be prompted
+ for if permissions are not automatically granted.
+ """
+
+ try:
+ if not os.geteuid() == 0:
+ if stdin.isatty():
+ logger.warning("Not running as root, using sudo (may require password to be entered)")
+ rotate_shell = check_output("sudo logrotate /etc/logrotate.conf", shell=True)
+ else:
+ raise PermissionError("Root priviliges are required to run logrotate but are not provided")
+ else:
+ rotate_shell = check_output("/usr/sbin/logrotate /etc/logrotate.conf", shell=True)
+ logger.info("Rotated logfiles")
+ logger.debug("logrotate output: " + rotate_shell)
+ except Exception as e:
+ logger.warning("Failed to rotate log files: " + str(e))
+
+
+def rotate_sim(): # Simulate log rotation
+ """
+ Simulate log rotation using logrotate's -d flag. This does not require root
+ privileges, but permission errors will be shown in the output without it.
+ """
+
+ try:
+ if not os.geteuid() == 0:
+ logger.warning("Cannot run logrotate as root - you will see permission errors in the output below")
+ sim_cmd = "logrotate -d /etc/logrotate.conf"
+ logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd))
+ sim = check_output(sim_cmd, shell=True)
+ logger.debug(sim)
+ except Exception as e:
+ logger.warning("Failed to get logrotate simulation: " + str(e))