From c9a3c26f5e8819a62b01ec4854f3b6b2ae4b4c5e Mon Sep 17 00:00:00 2001 From: Andrew Lorimer Date: Thu, 5 Sep 2019 21:06:00 +1000 Subject: [PATCH] rework parser loading interface --- logparse/interface.py | 146 ++++++++++++-------- logparse/load_parsers.py | 252 ++++++++++++++++++++++------------- logparse/parsers/__init__.py | 1 - logparse/parsers/cron.py | 1 + logparse/parsers/httpd.py | 4 +- logparse/parsers/sshd.py | 1 + 6 files changed, 251 insertions(+), 154 deletions(-) diff --git a/logparse/interface.py b/logparse/interface.py index 8d08c48..90b3ea8 100644 --- a/logparse/interface.py +++ b/logparse/interface.py @@ -1,54 +1,32 @@ -# -# __main__.py -# -# This module is the entrypoint of the `logparse` shell command and also -# contains single-use functions which don't fit elsewhere. -# +""" +This module is the entrypoint of the `logparse shell command and also contains +single-use functions which don't fit elsewhere. All user interaction with +logparse should be through this module. + +This module provides the following methods: + main Set up arguments, config, logging, and execute parsers + rotate Rotate logs using systemd logrotate + rotate_sim Simulate log rotation +""" import logging, logging.handlers import argparse import os -import glob -import sys +from sys import stdin, version from subprocess import check_output from datetime import datetime import logparse -import logparse.config -from logparse.config import prefs, loadconf from logparse import formatting, mail, config, load_parsers -global argparser - -def rotate(): # Rotate logs using systemd logrotate - try: - if not os.geteuid() == 0: - if sys.stdin.isatty(): - logger.warning("Not running as root, using sudo (may require password to be entered)") - rotate_shell = check_output("sudo logrotate /etc/logrotate.conf", shell=True) - else: - raise PermissionError("Root priviliges are required to run logrotate but are not provided") - else: - rotate_shell = check_output("/usr/sbin/logrotate /etc/logrotate.conf", shell=True) - logger.info("Rotated logfiles") - logger.debug("logrotate output: " + rotate_shell) - except Exception as e: - logger.warning("Failed to rotate log files: " + str(e)) - -def rotate_sim(): # Simulate log rotation - try: - if not os.geteuid() == 0: - logger.warning("Cannot run logrotate as root - you will see permission errors in the output below") - sim_cmd = "logrotate -d /etc/logrotate.conf" - logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd)) - sim = check_output(sim_cmd, shell=True) - logger.debug(sim) - except Exception as e: - logger.warning("Failed to get logrotate simulation: " + str(e)) - def main(): + """ + Initialisation and general management of logparse functionaliy. + """ + # Get arguments + global argparser argparser = argparse.ArgumentParser(description='grab logs of some common services and send them by email') argparser.add_argument('-t','--to', help='mail recipient (\"to\" address)', required=False) @@ -69,9 +47,11 @@ def main(): argparser.add_argument('-nw', '--no-write', help="do not write output file (overrides config file)", required=False, default=False, action="store_true") # Load config - config.prefs = loadconf(argparser.parse_args().config) + + config.prefs = config.loadconf(argparser.parse_args().config) # Set up logging + logger = logging.getLogger(__name__) loghandler = logging.handlers.SysLogHandler(address = '/dev/log') loghandler.setFormatter(logging.Formatter(fmt='logparse[' + str(os.getpid()) + ']: %(message)s')) @@ -85,15 +65,12 @@ def main(): logging.basicConfig(level=logging.INFO) logger.addHandler(loghandler) - logger.debug([x for x in config.prefs.sections()]) - logger.debug(config.prefs.get("logparse", "output")) - logger.debug("Config test: " + config.prefs.get("logparse", "output")) - # Time analysis + global start start = datetime.now() logger.info("Beginning log analysis at {0} {1}".format(start.strftime(formatting.DATEFMT), start.strftime(formatting.TIMEFMT))) - logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, sys.version.replace('\n', ''))) + logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, version.replace('\n', ''))) # Write header @@ -106,32 +83,43 @@ def main(): output = formatting.HtmlOutput() output.append_header(config.prefs.get("html", "header")) - # Find parsers - loader = load_parsers.ParserLoader("logparse.parsers") - parser_names = set([x.name for x in loader.parsers]) - + parser_names = [] + ignore_logs = [] if argparser.parse_args().logs: - parser_names = parser_names.intersection(set(argparser.parse_args().logs.split())) + parser_names = set(argparser.parse_args().logs.split()) elif config.prefs.get("logparse", "parsers"): - parser_names = parser_names.intersection(set(config.prefs.get("logparse", "parsers").split())) + parser_names = set(config.prefs.get("logparse", "parsers").split()) if argparser.parse_args().ignore_logs: - parser_names = parser_names.difference(set(argparser.parse_args().ignore_logs.split())) + ignore_logs = argparser.parse_args().ignore_logs.split() elif config.prefs.get("logparse", "ignore-parsers"): - parser_names = parser_names.difference(set(config.prefs.get("logparse", "ignore-parsers").split())) + ignore_logs = config.prefs.get("logparse", "ignore-parsers").split() + + # Set up parsers + + loader = load_parsers.ParserLoader() + if parser_names: + for parser_name in parser_names: + if parser_name not in ignore_logs: + loader.search(parser_name) + else: + loader.load_pkg() + if ignore_logs: + loader.ignore(ignore_logs) # Execute parsers - logger.debug("Queued the following parsers: " + str(loader.parsers)) for parser in loader.parsers: - if parser.name in parser_names: - output.append_section(parser.parse_log()) + output.append_section(parser.parse_log()) + + # Write footer - # Write HTML footer output.append_footer() + # Write output + if (argparser.parse_args().destination or config.prefs.get("logparse", "output")) and not argparser.parse_args().no_write: if argparser.parse_args().destination: dest_path = argparser.parse_args().destination @@ -151,6 +139,8 @@ def main(): else: logger.warning("No output written") + # Send email if requested + if (str(argparser.parse_args().to) or str(config.prefs.get("mail", "to"))) and not argparser.parse_args().no_mail: if str(argparser.parse_args().to): to = argparser.parse_args().to @@ -164,6 +154,8 @@ def main(): html=isinstance(output, formatting.HtmlOutput), sender=config.prefs.get("mail", "from")) + # Rotate logs if requested + if not argparser.parse_args().no_rotate: if argparser.parse_args().simulate or config.prefs.getboolean("logparse", "rotate"): rotate_sim() @@ -174,7 +166,8 @@ def main(): else: logger.debug("User doesn't want to rotate logs") - # Print end message + # Finish up + finish = datetime.now() logger.info("Finished parsing logs at {0} {1} (total time: {2})".format(finish.strftime(formatting.DATEFMT), finish.strftime(formatting.TIMEFMT), finish - start)) @@ -182,3 +175,42 @@ def main(): output.print_stdout() return + + +def rotate(): + """ + Rotate logs using systemd logrotate. This requires root privileges, and a + basic check for this is attempted below. Root password will be prompted + for if permissions are not automatically granted. + """ + + try: + if not os.geteuid() == 0: + if stdin.isatty(): + logger.warning("Not running as root, using sudo (may require password to be entered)") + rotate_shell = check_output("sudo logrotate /etc/logrotate.conf", shell=True) + else: + raise PermissionError("Root priviliges are required to run logrotate but are not provided") + else: + rotate_shell = check_output("/usr/sbin/logrotate /etc/logrotate.conf", shell=True) + logger.info("Rotated logfiles") + logger.debug("logrotate output: " + rotate_shell) + except Exception as e: + logger.warning("Failed to rotate log files: " + str(e)) + + +def rotate_sim(): # Simulate log rotation + """ + Simulate log rotation using logrotate's -d flag. This does not require root + privileges, but permission errors will be shown in the output without it. + """ + + try: + if not os.geteuid() == 0: + logger.warning("Cannot run logrotate as root - you will see permission errors in the output below") + sim_cmd = "logrotate -d /etc/logrotate.conf" + logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd)) + sim = check_output(sim_cmd, shell=True) + logger.debug(sim) + except Exception as e: + logger.warning("Failed to get logrotate simulation: " + str(e)) diff --git a/logparse/load_parsers.py b/logparse/load_parsers.py index 8ea7a03..44651fc 100644 --- a/logparse/load_parsers.py +++ b/logparse/load_parsers.py @@ -1,136 +1,200 @@ -# -# load_parsers.py -# -# Search for and load files which parse logs for particular services -# +""" +A basic "plugin loader" implementation which searches for default packaged and +user-supplied parser modules and verifies them so they can be executed by +logparse.interface. The requirements for parser modules and classes are +specified in the docstring of the Parser class. + +Classes in this module: + Parser Base class that every parser should inherit + ParserLoader Class used internally by logparse.interface to load parsers +""" -import imp import importlib -import os -import glob -import pkgutil +from os.path import dirname +from pkgutil import iter_modules import inspect from pathlib import Path -from sys import path -from typing import NamedTuple - -parser_dir = "/usr/share/logparse/" -main_module = "__init__" -default_parsers = ["cron_journald", "httpd", "mem", "postfix", "smbd", "sshd_journald", "sudo", "sysinfo", "temperature", "zfs"] -deprecated_parsers = ["sshd", "cron"] +from typing import get_type_hints import logging logger = logging.getLogger(__name__) + +PARSER_DIR = "/usr/share/logparse/user-parsers" +PARSER_PKG = "logparse.parsers" + + class Parser(): """ - Base class that every parser should inherit + This is the base class that every parser should inherit. Parsers should + each exist in their own module and contain a Parser class whose name is the + same as the parser (e.g. `example.py` contains class `Example(Parser)`). + Each parser module must contain exactly one Parser class definition, and + this class cannot be a redefinition of the base Parser class (i.e. this + class). This class must provide the parse_log() method which returns a + logparse.formatting.Section object. """ - def __init__(self, name=None, path=None, info=None): + + def __init__(self, name=None, path=None, info=None, deprecated=False): + """ + The following variables can be set to display information about the + parser. The object `self.logger` can be used as for outputting messages + to whatever sink is set up in logparse.interface (no setup required in + the parser module itself). + """ self.name = str(name) if name else None self.path = Path(path) if path else None self.info = dict(info) if info else None self.logger = logging.getLogger(__name__) + self.deprecated = deprecated def load(self): + """ + A generic loading method to import a parser, only used for debugging + """ logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults")) return importlib.import_module(self.name) - def parse_log(self, **args): + def parse_log(self, **args) -> None: """ Every parser should provide the parse_log method which is executed at - runtime to analyse logs. + runtime to analyse logs. Verification checks should prevent the below + exception from ever being raised. """ - raise NotImplementedError("Failed to find an entry point for parser " + self.name) + raise NotImplementedError("Failed to find an entry point for parser") + + class ParserLoader: """ - This class searches for parsers in the main logparse package and - optionally in another external package (default /usr/share/logparse). + This class searches for parsers in the main logparse package + (logparser.parsers) and optionally in another external package (default + /usr/share/logparse). """ - def __init__(self, pkg): + def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR): """ - Initiate search for parsers + The pkg and path attributes shouldn't need to be set on object + creation, the default values should work fine. They are hard-coded here + for security so that a module can't force-load a module from another + package/location, e.g. from the internet. """ - self.pkg = pkg - self.parsers= [] - self.reload() + self.pkg = pkg + self.path = path + self.parsers = [] - def reload(self): + def search(self, pattern): """ - Reset parsers list and iterate through package modules + Basic wrapper for the two search functions below. """ - self.parsers= [] - self.seen_paths = [] - logger.debug("Looking for parsers in package {0}".format(str(self.pkg))) - self.walk_package(self.pkg) - def walk_package(self, package): + default_parser = self._search_default(pattern) + if default_parser != None: + self.parsers.append(default_parser) + return default_parser + else: + user_parser = self._search_user(pattern) + if user_parser != None: + self.parsers.append(user_parser) + return user_parser + else: + logger.warning("Couldn't find a matching parser module for search term {0}".format(pattern)) + return None + + def _search_user(self, pattern): """ - Check package and subdirectories for loadable modules + Search for a parser name `pattern` in the user-managed parser directory """ - imported_package = __import__(package, fromlist=["null"]) # fromlist must be non-empty to load target module rather than parent package + logger.debug("Searching for {0} in {1}".format(pattern, self.path)) + try: + spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path]) + parser_module = spec.loader.load_module(spec.name) + return self._validate_module(parser_module) + except Exception as e: + logger.debug("Couldn't find parser {0} in {1}: {2}".format(pattern, self.path, str(e))) + return None - for _, parser_name, ispkg in pkgutil.iter_modules(imported_package.__path__, imported_package.__name__ + '.'): - if not ispkg: - parser_module = __import__(parser_name, fromlist=["null"]) - clsmembers = inspect.getmembers(parser_module, inspect.isclass) - for (_, c) in clsmembers: - # Ignore the base Parser class - if issubclass(c, Parser) & (c is not Parser): - logger.debug("Found parser {0}.{1}".format(c.__module__, c.__name__)) - self.parsers.append(c()) + def _search_default(self, pattern): + """ + Search for a parser name `pattern` in the default parser package + """ + + # TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below + logger.debug("Searching for {0} in default parsers".format(pattern)) + try: + parser_module = importlib.import_module(self.pkg + "." + pattern) + return self._validate_module(parser_module) + except Exception as e: + return None + + def _validate_module(self, parser_module): + """ + Some basic security tests for candidate modules: + 1. Must contain exactly one Parser object + 3. This class cannot be a redefinition of the base Parser class + 4. Must provide the parse_log() method + 5. Must not return None + 6. Must not match an already-loaded class + """ + logger.debug("Checking validity of module {0} at {1}".format(parser_module.__name__, parser_module.__file__)) + available_parsers = [] + clsmembers = inspect.getmembers(parser_module, inspect.isclass) + + # Check individual classes + for (_, c) in clsmembers: + if not (issubclass(c, Parser) & (c is not Parser)): + continue + if c in self.parsers: + logger.warning("Parser class {0} has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__)) + if not inspect.isroutine(c.parse_log): + logger.warning("Parser class {0} in {1} does not contain a parse_log() method".format(c.__class__.__name__, c.__file__)) + continue + if None in get_type_hints(c): + logger.warning("Parser class {0} in {1} contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__)) + continue + logger.debug("Found parser {0}.{1}".format(c.__module__, c.__class__.__name__)) + available_parsers.append(c()) + + # Check module structure + if len(available_parsers) == 1: + logger.debug("Parser module {0} at {1} passed validity checks".format(parser_module.__name__, parser_module.__file__)) + return available_parsers[0] + elif len(available_parsers) == 0: + logger.warning("No valid classes in {0} at {1}".format(parser_module.__name__, parser_module.__file__)) + return None + elif len(available_parsers) > 1: + logger.warning("Found multiple valid parser classes in {0} at {1} - ignoring this module".format(parser_module.__name__, parser_module.__file__)) + return None + + def load_pkg(self): + """ + Clear the list of currently loaded packages and load all valid and + non-deprecated parser classes from self.pkg using importlib. + """ - # Recurse subpackages + available_parsers = [name for _, name, _ in iter_modules([dirname(importlib.import_module(self.pkg).__file__)])] + for parser_name in available_parsers: + parser_module = importlib.import_module("logparse.parsers." + parser_name) + parser_class = self._validate_module(parser_module) + if parser_class == None: + continue + parser_obj = parser_class + if parser_obj.deprecated: + logger.debug("Ignoring parser {0} because it is deprecated".format(parser_class.__class__.__name__)) + continue + self.parsers.append(parser_class) + return self.parsers + + def ignore(self, pattern): + """ + Remove a parser from the list of currently loaded parsers + """ - all_current_paths = [] - if isinstance(imported_package.__path__, str): - all_current_paths.append(imported_package.__path__) - else: - all_current_paths.extend([x for x in imported_package.__path__]) - - for pkg_path in all_current_paths: - if pkg_path not in self.seen_paths: - self.seen_paths.append(pkg_path) - - # Get subdirectories of package - child_pkgs = [p for p in os.listdir(pkg_path) if os.path.isdir(os.path.join(pkg_path, p))] - - # Walk through each subdirectory - for child_pkg in child_pkgs: - self.walk_package(package + '.' + child_pkg) - -def findall(): - logger.debug("Searching for parsers in {0}".format(parser_dir)) - path.append(os.path.abspath(parser_dir)) - parsers = [] - parser_candidates = os.listdir(parser_dir) - for parser_name in parser_candidates: - location = os.path.join(parser_dir, parser_name) - if not os.path.isdir(location) or not main_module + '.py' in os.listdir(location): - logger.warning("Rejecting parser {0} due to invalid structure".format(location)) - continue - info = imp.find_module(main_module, [location]) - parser_obj = Parser(parser_name, location, info) - parsers.append(parser_obj) - logger.debug("Added parser {0}".format(parser_obj.name)) - return parsers - -def search(name): - logger.debug("Searching for parser {0}".format(name)) - if name in default_parsers: - logger.debug("Found parser {0} in default modules".format(name)) - return Parser('.'.join(__name__.split('.')[:-1] + [name])) - elif name in deprecated_parsers: - logger.debug("Found parser {0} in deprecated modules".format(name)) - return Parser('.'.join(__name__.split('.')[:-1] + [name])) - else: - return None - -def load(parser): - logger.debug("Loading parser {0} from {1}".format(parser.name, parser.path if parser.path != None else "defaults")) - return importlib.import_module(parser.name) + for parser in self.parsers: + if parser.__module__ == pattern: + self.parsers.remove(parser) + logger.debug("Ignoring parser {0}".format(parser.__name__)) + return self.parsers diff --git a/logparse/parsers/__init__.py b/logparse/parsers/__init__.py index 5aa0845..e69de29 100644 --- a/logparse/parsers/__init__.py +++ b/logparse/parsers/__init__.py @@ -1 +0,0 @@ -__all__ = ["load_parsers", "sudo", "sshd"] diff --git a/logparse/parsers/cron.py b/logparse/parsers/cron.py index 02ea2dd..731732d 100644 --- a/logparse/parsers/cron.py +++ b/logparse/parsers/cron.py @@ -22,6 +22,7 @@ class Cron(Parser): super().__init__() self.name = "cron" self.info = "List the logged (executed) cron jobs and their commands (uses static syslog file)" + self.deprecated = True def parse_log(self): diff --git a/logparse/parsers/httpd.py b/logparse/parsers/httpd.py index 2e6ae2f..b86f1c1 100644 --- a/logparse/parsers/httpd.py +++ b/logparse/parsers/httpd.py @@ -41,9 +41,9 @@ class Httpd(Parser): logger.debug("Starting httpd section") section = Section("httpd") - accesslog = readlog(prefs("logs", "httpd-access")) + accesslog = readlog(config.prefs.get("logs", "httpd-access")) - errorlog= readlog(prefs("logs", "httpd-error")) + errorlog= readlog(config.prefs.get("logs", "httpd-error")) total_errors = len(errorlog.splitlines()) logger.debug("Retrieved log data") diff --git a/logparse/parsers/sshd.py b/logparse/parsers/sshd.py index a20ec61..b7fd2c3 100644 --- a/logparse/parsers/sshd.py +++ b/logparse/parsers/sshd.py @@ -21,6 +21,7 @@ class Sshd(Parser): super().__init__() self.name = "sshd" self.info = "Find number of ssh logins and authorised users (uses /var/log/auth.log)" + self.deprecated = True def parse_log(self): -- 2.43.2