-#
-# __main__.py
-#
-# This module is the entrypoint of the `logparse` shell command and also
-# contains single-use functions which don't fit elsewhere.
-#
+"""
+This module is the entrypoint of the `logparse shell command and also contains
+single-use functions which don't fit elsewhere. All user interaction with
+logparse should be through this module.
+
+This module provides the following methods:
+ main Set up arguments, config, logging, and execute parsers
+ rotate Rotate logs using systemd logrotate
+ rotate_sim Simulate log rotation
+"""
import logging, logging.handlers
import argparse
import os
-import glob
-import sys
+from sys import stdin, version
from subprocess import check_output
from datetime import datetime
import logparse
-import logparse.config
-from logparse.config import prefs, loadconf
from logparse import formatting, mail, config, load_parsers
-global argparser
-
-def rotate(): # Rotate logs using systemd logrotate
- try:
- if not os.geteuid() == 0:
- if sys.stdin.isatty():
- logger.warning("Not running as root, using sudo (may require password to be entered)")
- rotate_shell = check_output("sudo logrotate /etc/logrotate.conf", shell=True)
- else:
- raise PermissionError("Root priviliges are required to run logrotate but are not provided")
- else:
- rotate_shell = check_output("/usr/sbin/logrotate /etc/logrotate.conf", shell=True)
- logger.info("Rotated logfiles")
- logger.debug("logrotate output: " + rotate_shell)
- except Exception as e:
- logger.warning("Failed to rotate log files: " + str(e))
-
-def rotate_sim(): # Simulate log rotation
- try:
- if not os.geteuid() == 0:
- logger.warning("Cannot run logrotate as root - you will see permission errors in the output below")
- sim_cmd = "logrotate -d /etc/logrotate.conf"
- logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd))
- sim = check_output(sim_cmd, shell=True)
- logger.debug(sim)
- except Exception as e:
- logger.warning("Failed to get logrotate simulation: " + str(e))
-
def main():
+ """
+ Initialisation and general management of logparse functionaliy.
+ """
+
# Get arguments
+
global argparser
argparser = argparse.ArgumentParser(description='grab logs of some common services and send them by email')
argparser.add_argument('-t','--to', help='mail recipient (\"to\" address)', required=False)
argparser.add_argument('-nw', '--no-write', help="do not write output file (overrides config file)", required=False, default=False, action="store_true")
# Load config
- config.prefs = loadconf(argparser.parse_args().config)
+
+ config.prefs = config.loadconf(argparser.parse_args().config)
# Set up logging
+
logger = logging.getLogger(__name__)
loghandler = logging.handlers.SysLogHandler(address = '/dev/log')
loghandler.setFormatter(logging.Formatter(fmt='logparse[' + str(os.getpid()) + ']: %(message)s'))
logging.basicConfig(level=logging.INFO)
logger.addHandler(loghandler)
- logger.debug([x for x in config.prefs.sections()])
- logger.debug(config.prefs.get("logparse", "output"))
- logger.debug("Config test: " + config.prefs.get("logparse", "output"))
-
# Time analysis
+
global start
start = datetime.now()
logger.info("Beginning log analysis at {0} {1}".format(start.strftime(formatting.DATEFMT), start.strftime(formatting.TIMEFMT)))
- logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, sys.version.replace('\n', '')))
+ logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, version.replace('\n', '')))
# Write header
output = formatting.HtmlOutput()
output.append_header(config.prefs.get("html", "header"))
-
# Find parsers
- loader = load_parsers.ParserLoader("logparse.parsers")
- parser_names = set([x.name for x in loader.parsers])
-
+ parser_names = []
+ ignore_logs = []
if argparser.parse_args().logs:
- parser_names = parser_names.intersection(set(argparser.parse_args().logs.split()))
+ parser_names = set(argparser.parse_args().logs.split())
elif config.prefs.get("logparse", "parsers"):
- parser_names = parser_names.intersection(set(config.prefs.get("logparse", "parsers").split()))
+ parser_names = set(config.prefs.get("logparse", "parsers").split())
if argparser.parse_args().ignore_logs:
- parser_names = parser_names.difference(set(argparser.parse_args().ignore_logs.split()))
+ ignore_logs = argparser.parse_args().ignore_logs.split()
elif config.prefs.get("logparse", "ignore-parsers"):
- parser_names = parser_names.difference(set(config.prefs.get("logparse", "ignore-parsers").split()))
+ ignore_logs = config.prefs.get("logparse", "ignore-parsers").split()
+
+ # Set up parsers
+
+ loader = load_parsers.ParserLoader()
+ if parser_names:
+ for parser_name in parser_names:
+ if parser_name not in ignore_logs:
+ loader.search(parser_name)
+ else:
+ loader.load_pkg()
+ if ignore_logs:
+ loader.ignore(ignore_logs)
# Execute parsers
- logger.debug("Queued the following parsers: " + str(loader.parsers))
for parser in loader.parsers:
- if parser.name in parser_names:
- output.append_section(parser.parse_log())
+ output.append_section(parser.parse_log())
+
+ # Write footer
- # Write HTML footer
output.append_footer()
+ # Write output
+
if (argparser.parse_args().destination or config.prefs.get("logparse", "output")) and not argparser.parse_args().no_write:
if argparser.parse_args().destination:
dest_path = argparser.parse_args().destination
else:
logger.warning("No output written")
+ # Send email if requested
+
if (str(argparser.parse_args().to) or str(config.prefs.get("mail", "to"))) and not argparser.parse_args().no_mail:
if str(argparser.parse_args().to):
to = argparser.parse_args().to
html=isinstance(output, formatting.HtmlOutput),
sender=config.prefs.get("mail", "from"))
+ # Rotate logs if requested
+
if not argparser.parse_args().no_rotate:
if argparser.parse_args().simulate or config.prefs.getboolean("logparse", "rotate"):
rotate_sim()
else:
logger.debug("User doesn't want to rotate logs")
- # Print end message
+ # Finish up
+
finish = datetime.now()
logger.info("Finished parsing logs at {0} {1} (total time: {2})".format(finish.strftime(formatting.DATEFMT), finish.strftime(formatting.TIMEFMT), finish - start))
output.print_stdout()
return
+
+
+def rotate():
+ """
+ Rotate logs using systemd logrotate. This requires root privileges, and a
+ basic check for this is attempted below. Root password will be prompted
+ for if permissions are not automatically granted.
+ """
+
+ try:
+ if not os.geteuid() == 0:
+ if stdin.isatty():
+ logger.warning("Not running as root, using sudo (may require password to be entered)")
+ rotate_shell = check_output("sudo logrotate /etc/logrotate.conf", shell=True)
+ else:
+ raise PermissionError("Root priviliges are required to run logrotate but are not provided")
+ else:
+ rotate_shell = check_output("/usr/sbin/logrotate /etc/logrotate.conf", shell=True)
+ logger.info("Rotated logfiles")
+ logger.debug("logrotate output: " + rotate_shell)
+ except Exception as e:
+ logger.warning("Failed to rotate log files: " + str(e))
+
+
+def rotate_sim(): # Simulate log rotation
+ """
+ Simulate log rotation using logrotate's -d flag. This does not require root
+ privileges, but permission errors will be shown in the output without it.
+ """
+
+ try:
+ if not os.geteuid() == 0:
+ logger.warning("Cannot run logrotate as root - you will see permission errors in the output below")
+ sim_cmd = "logrotate -d /etc/logrotate.conf"
+ logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd))
+ sim = check_output(sim_cmd, shell=True)
+ logger.debug(sim)
+ except Exception as e:
+ logger.warning("Failed to get logrotate simulation: " + str(e))
-#
-# load_parsers.py
-#
-# Search for and load files which parse logs for particular services
-#
+"""
+A basic "plugin loader" implementation which searches for default packaged and
+user-supplied parser modules and verifies them so they can be executed by
+logparse.interface. The requirements for parser modules and classes are
+specified in the docstring of the Parser class.
+
+Classes in this module:
+ Parser Base class that every parser should inherit
+ ParserLoader Class used internally by logparse.interface to load parsers
+"""
-import imp
import importlib
-import os
-import glob
-import pkgutil
+from os.path import dirname
+from pkgutil import iter_modules
import inspect
from pathlib import Path
-from sys import path
-from typing import NamedTuple
-
-parser_dir = "/usr/share/logparse/"
-main_module = "__init__"
-default_parsers = ["cron_journald", "httpd", "mem", "postfix", "smbd", "sshd_journald", "sudo", "sysinfo", "temperature", "zfs"]
-deprecated_parsers = ["sshd", "cron"]
+from typing import get_type_hints
import logging
logger = logging.getLogger(__name__)
+
+PARSER_DIR = "/usr/share/logparse/user-parsers"
+PARSER_PKG = "logparse.parsers"
+
+
class Parser():
"""
- Base class that every parser should inherit
+ This is the base class that every parser should inherit. Parsers should
+ each exist in their own module and contain a Parser class whose name is the
+ same as the parser (e.g. `example.py` contains class `Example(Parser)`).
+ Each parser module must contain exactly one Parser class definition, and
+ this class cannot be a redefinition of the base Parser class (i.e. this
+ class). This class must provide the parse_log() method which returns a
+ logparse.formatting.Section object.
"""
- def __init__(self, name=None, path=None, info=None):
+
+ def __init__(self, name=None, path=None, info=None, deprecated=False):
+ """
+ The following variables can be set to display information about the
+ parser. The object `self.logger` can be used as for outputting messages
+ to whatever sink is set up in logparse.interface (no setup required in
+ the parser module itself).
+ """
self.name = str(name) if name else None
self.path = Path(path) if path else None
self.info = dict(info) if info else None
self.logger = logging.getLogger(__name__)
+ self.deprecated = deprecated
def load(self):
+ """
+ A generic loading method to import a parser, only used for debugging
+ """
logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults"))
return importlib.import_module(self.name)
- def parse_log(self, **args):
+ def parse_log(self, **args) -> None:
"""
Every parser should provide the parse_log method which is executed at
- runtime to analyse logs.
+ runtime to analyse logs. Verification checks should prevent the below
+ exception from ever being raised.
"""
- raise NotImplementedError("Failed to find an entry point for parser " + self.name)
+ raise NotImplementedError("Failed to find an entry point for parser")
+
+
class ParserLoader:
"""
- This class searches for parsers in the main logparse package and
- optionally in another external package (default /usr/share/logparse).
+ This class searches for parsers in the main logparse package
+ (logparser.parsers) and optionally in another external package (default
+ /usr/share/logparse).
"""
- def __init__(self, pkg):
+ def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR):
"""
- Initiate search for parsers
+ The pkg and path attributes shouldn't need to be set on object
+ creation, the default values should work fine. They are hard-coded here
+ for security so that a module can't force-load a module from another
+ package/location, e.g. from the internet.
"""
- self.pkg = pkg
- self.parsers= []
- self.reload()
+ self.pkg = pkg
+ self.path = path
+ self.parsers = []
- def reload(self):
+ def search(self, pattern):
"""
- Reset parsers list and iterate through package modules
+ Basic wrapper for the two search functions below.
"""
- self.parsers= []
- self.seen_paths = []
- logger.debug("Looking for parsers in package {0}".format(str(self.pkg)))
- self.walk_package(self.pkg)
- def walk_package(self, package):
+ default_parser = self._search_default(pattern)
+ if default_parser != None:
+ self.parsers.append(default_parser)
+ return default_parser
+ else:
+ user_parser = self._search_user(pattern)
+ if user_parser != None:
+ self.parsers.append(user_parser)
+ return user_parser
+ else:
+ logger.warning("Couldn't find a matching parser module for search term {0}".format(pattern))
+ return None
+
+ def _search_user(self, pattern):
"""
- Check package and subdirectories for loadable modules
+ Search for a parser name `pattern` in the user-managed parser directory
"""
- imported_package = __import__(package, fromlist=["null"]) # fromlist must be non-empty to load target module rather than parent package
+ logger.debug("Searching for {0} in {1}".format(pattern, self.path))
+ try:
+ spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path])
+ parser_module = spec.loader.load_module(spec.name)
+ return self._validate_module(parser_module)
+ except Exception as e:
+ logger.debug("Couldn't find parser {0} in {1}: {2}".format(pattern, self.path, str(e)))
+ return None
- for _, parser_name, ispkg in pkgutil.iter_modules(imported_package.__path__, imported_package.__name__ + '.'):
- if not ispkg:
- parser_module = __import__(parser_name, fromlist=["null"])
- clsmembers = inspect.getmembers(parser_module, inspect.isclass)
- for (_, c) in clsmembers:
- # Ignore the base Parser class
- if issubclass(c, Parser) & (c is not Parser):
- logger.debug("Found parser {0}.{1}".format(c.__module__, c.__name__))
- self.parsers.append(c())
+ def _search_default(self, pattern):
+ """
+ Search for a parser name `pattern` in the default parser package
+ """
+
+ # TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below
+ logger.debug("Searching for {0} in default parsers".format(pattern))
+ try:
+ parser_module = importlib.import_module(self.pkg + "." + pattern)
+ return self._validate_module(parser_module)
+ except Exception as e:
+ return None
+
+ def _validate_module(self, parser_module):
+ """
+ Some basic security tests for candidate modules:
+ 1. Must contain exactly one Parser object
+ 3. This class cannot be a redefinition of the base Parser class
+ 4. Must provide the parse_log() method
+ 5. Must not return None
+ 6. Must not match an already-loaded class
+ """
+ logger.debug("Checking validity of module {0} at {1}".format(parser_module.__name__, parser_module.__file__))
+ available_parsers = []
+ clsmembers = inspect.getmembers(parser_module, inspect.isclass)
+
+ # Check individual classes
+ for (_, c) in clsmembers:
+ if not (issubclass(c, Parser) & (c is not Parser)):
+ continue
+ if c in self.parsers:
+ logger.warning("Parser class {0} has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__))
+ if not inspect.isroutine(c.parse_log):
+ logger.warning("Parser class {0} in {1} does not contain a parse_log() method".format(c.__class__.__name__, c.__file__))
+ continue
+ if None in get_type_hints(c):
+ logger.warning("Parser class {0} in {1} contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__))
+ continue
+ logger.debug("Found parser {0}.{1}".format(c.__module__, c.__class__.__name__))
+ available_parsers.append(c())
+
+ # Check module structure
+ if len(available_parsers) == 1:
+ logger.debug("Parser module {0} at {1} passed validity checks".format(parser_module.__name__, parser_module.__file__))
+ return available_parsers[0]
+ elif len(available_parsers) == 0:
+ logger.warning("No valid classes in {0} at {1}".format(parser_module.__name__, parser_module.__file__))
+ return None
+ elif len(available_parsers) > 1:
+ logger.warning("Found multiple valid parser classes in {0} at {1} - ignoring this module".format(parser_module.__name__, parser_module.__file__))
+ return None
+
+ def load_pkg(self):
+ """
+ Clear the list of currently loaded packages and load all valid and
+ non-deprecated parser classes from self.pkg using importlib.
+ """
- # Recurse subpackages
+ available_parsers = [name for _, name, _ in iter_modules([dirname(importlib.import_module(self.pkg).__file__)])]
+ for parser_name in available_parsers:
+ parser_module = importlib.import_module("logparse.parsers." + parser_name)
+ parser_class = self._validate_module(parser_module)
+ if parser_class == None:
+ continue
+ parser_obj = parser_class
+ if parser_obj.deprecated:
+ logger.debug("Ignoring parser {0} because it is deprecated".format(parser_class.__class__.__name__))
+ continue
+ self.parsers.append(parser_class)
+ return self.parsers
+
+ def ignore(self, pattern):
+ """
+ Remove a parser from the list of currently loaded parsers
+ """
- all_current_paths = []
- if isinstance(imported_package.__path__, str):
- all_current_paths.append(imported_package.__path__)
- else:
- all_current_paths.extend([x for x in imported_package.__path__])
-
- for pkg_path in all_current_paths:
- if pkg_path not in self.seen_paths:
- self.seen_paths.append(pkg_path)
-
- # Get subdirectories of package
- child_pkgs = [p for p in os.listdir(pkg_path) if os.path.isdir(os.path.join(pkg_path, p))]
-
- # Walk through each subdirectory
- for child_pkg in child_pkgs:
- self.walk_package(package + '.' + child_pkg)
-
-def findall():
- logger.debug("Searching for parsers in {0}".format(parser_dir))
- path.append(os.path.abspath(parser_dir))
- parsers = []
- parser_candidates = os.listdir(parser_dir)
- for parser_name in parser_candidates:
- location = os.path.join(parser_dir, parser_name)
- if not os.path.isdir(location) or not main_module + '.py' in os.listdir(location):
- logger.warning("Rejecting parser {0} due to invalid structure".format(location))
- continue
- info = imp.find_module(main_module, [location])
- parser_obj = Parser(parser_name, location, info)
- parsers.append(parser_obj)
- logger.debug("Added parser {0}".format(parser_obj.name))
- return parsers
-
-def search(name):
- logger.debug("Searching for parser {0}".format(name))
- if name in default_parsers:
- logger.debug("Found parser {0} in default modules".format(name))
- return Parser('.'.join(__name__.split('.')[:-1] + [name]))
- elif name in deprecated_parsers:
- logger.debug("Found parser {0} in deprecated modules".format(name))
- return Parser('.'.join(__name__.split('.')[:-1] + [name]))
- else:
- return None
-
-def load(parser):
- logger.debug("Loading parser {0} from {1}".format(parser.name, parser.path if parser.path != None else "defaults"))
- return importlib.import_module(parser.name)
+ for parser in self.parsers:
+ if parser.__module__ == pattern:
+ self.parsers.remove(parser)
+ logger.debug("Ignoring parser {0}".format(parser.__name__))
+ return self.parsers