logparse / load_parsers.pyon commit bugfixing & add smbd_journald (1e68934)
   1#!/usr/bin/env python
   2# -*- coding: utf-8 -*-
   3"""
   4A basic "plugin loader" implementation which searches for default packaged and
   5user-supplied parser modules and verifies them so they can be executed by
   6logparse.interface. The requirements for parser modules and classes are
   7specified in the docstring of the Parser class.
   8
   9Classes in this module:
  10    - `Parser`:         Base class that every parser should inherit
  11    - `ParserLoader`:   Class used internally by `interface.py` to load parsers
  12"""
  13
  14import importlib
  15from os.path import dirname
  16from pkgutil import iter_modules
  17import inspect
  18from pathlib import Path
  19from typing import get_type_hints
  20
  21import logging
  22logger = logging.getLogger(__name__)
  23
  24
  25PARSER_DIR = "/usr/share/logparse/user-parsers"
  26PARSER_PKG = "logparse.parsers"
  27
  28
  29class Parser():
  30    """
  31    This is the base class that every parser should inherit. Parsers should
  32    each exist in their own module and contain a Parser class whose name is the
  33    same as the parser (e.g. `example.py` contains class `Example(Parser)`).
  34    Each parser module must contain exactly one Parser class definition, and
  35    this class cannot be a redefinition of the base Parser class (i.e. this
  36    class). This class must provide the parse_log() method which returns a
  37    logparse.formatting.Section object.
  38    """
  39
  40    def __init__(self, name=None, path=None, info=None, deprecated=False, successor=""):
  41        """
  42        The following variables can be set to display information about the
  43        parser. The object `self.logger` can be used as for outputting messages
  44        to whatever sink is set up in logparse.interface (no setup required in
  45        the parser module itself).
  46        """
  47        self.name = str(name) if name else None
  48        self.path = Path(path) if path else None
  49        self.info = dict(info) if info else None
  50        self.logger = logging.getLogger(__name__)
  51        self.deprecated = deprecated
  52        self.successor = successor
  53
  54    def load(self):
  55        """
  56        A generic loading method to import a parser, only used for debugging
  57        """
  58        logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults"))
  59        return importlib.import_module(self.name)
  60
  61    def parse_log(self, **args) -> None:
  62        """
  63        Every parser should provide the parse_log method which is executed at
  64        runtime to analyse logs. Verification checks should prevent the below
  65        exception from ever being raised.
  66        """
  67        raise NotImplementedError("Failed to find an entry point for parser")
  68
  69
  70
  71class ParserLoader:
  72    """
  73    This class searches for parsers in the main logparse package
  74    (logparser.parsers) and optionally in another external package (default
  75    /usr/share/logparse).
  76    """
  77
  78    def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR):
  79        """
  80        The pkg and path attributes shouldn't need to be set on object
  81        creation, the default values should work fine. They are hard-coded here
  82        for security so that a module can't force-load a module from another
  83        package/location, e.g. from the internet.
  84        """
  85
  86        self.pkg = pkg
  87        self.path = path
  88        self.parsers = []
  89
  90    def search(self, pattern):
  91        """
  92        Basic wrapper for the two search functions below.
  93        """
  94
  95        default_parser = self._search_default(pattern)
  96        if default_parser != None:
  97            self.parsers.append(default_parser)
  98            return default_parser
  99        else:
 100            user_parser = self._search_user(pattern)
 101            if user_parser != None:
 102                self.parsers.append(user_parser)
 103                return user_parser
 104            else:
 105                logger.warning("Couldn't find a matching parser module for search term {0}".format(pattern))
 106                return None
 107
 108    def _search_user(self, pattern):
 109        """
 110        Search for a parser name `pattern` in the user-managed parser directory
 111        """
 112
 113        logger.debug("Searching for {0} in {1}".format(pattern, self.path))
 114        try:
 115            spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path])
 116            parser_module = spec.loader.load_module(spec.name)
 117            return self._validate_module(parser_module)
 118        except Exception as e:
 119            logger.debug("Couldn't find parser {0} in {1}".format(pattern, self.path))
 120            return None
 121
 122    def _search_default(self, pattern):
 123        """
 124        Search for a parser name `pattern` in the default parser package
 125        """
 126        
 127        # TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below
 128        logger.debug("Searching for {0} in default parsers".format(pattern))
 129        try:
 130            parser_module = importlib.import_module(self.pkg + "." + pattern)
 131            return self._validate_module(parser_module)
 132        except Exception as e:
 133            return None 
 134
 135    def _validate_module(self, parser_module):
 136        """
 137        Some basic security tests for candidate modules:
 138            1. Must contain exactly one Parser object
 139            3. This class cannot be a redefinition of the base Parser class
 140            4. Must provide the parse_log() method
 141            5. Must not return None
 142            6. Must not match an already-loaded class
 143        """
 144
 145        logger.debug("Checking validity of module {0} at {1}".format(parser_module.__name__, parser_module.__file__))
 146        available_parsers = []
 147        clsmembers = inspect.getmembers(parser_module, inspect.isclass)
 148
 149        # Check individual classes
 150        for (_, c) in clsmembers:
 151            if not (issubclass(c, Parser) & (c is not Parser)):
 152                continue
 153            if c in self.parsers:
 154                logger.warning("Parser class {0} has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__))
 155            if not inspect.isroutine(c.parse_log):
 156                logger.warning("Parser class {0} in {1} does not contain a parse_log() method".format(c.__class__.__name__, c.__file__))
 157                continue
 158            if None in get_type_hints(c):
 159                logger.warning("Parser class {0} in {1} contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__))
 160                continue
 161            parser_obj = c()
 162            if parser_obj.deprecated:
 163                logger.warning("Parser {0} is deprecated - use {1} instead".format(parser_obj.name, parser_obj.successor))
 164            logger.debug("Found parser {0}.{1}".format(c.__module__, c.__class__.__name__))
 165            available_parsers.append(c())
 166
 167        # Check module structure
 168        if len(available_parsers) == 1:
 169            logger.debug("Parser module {0} at {1} passed validity checks".format(parser_module.__name__, parser_module.__file__))
 170            return available_parsers[0]
 171        elif len(available_parsers) == 0:
 172            logger.warning("No valid classes in {0} at {1}".format(parser_module.__name__, parser_module.__file__))
 173            return None
 174        elif len(available_parsers) > 1:
 175            logger.warning("Found multiple valid parser classes in {0} at {1} - ignoring this module".format(parser_module.__name__, parser_module.__file__))
 176            return None
 177
 178    def load_pkg(self):
 179        """
 180        Clear the list of currently loaded packages and load all valid and
 181        non-deprecated parser classes from self.pkg using importlib.
 182        """
 183
 184        available_parsers = [name for _, name, _ in iter_modules([dirname(importlib.import_module(self.pkg).__file__)])]
 185        for parser_name in available_parsers:
 186            parser_module = importlib.import_module("logparse.parsers." + parser_name)
 187            parser_class = self._validate_module(parser_module)
 188            if parser_class == None:
 189                continue
 190            parser_obj = parser_class
 191            if parser_obj.deprecated:
 192                logger.debug("Ignoring parser {0} because it is deprecated".format(parser_class.__class__.__name__))
 193                continue
 194            self.parsers.append(parser_class)
 195        return self.parsers
 196
 197    def ignore(self, pattern):
 198        """
 199        Remove a parser from the list of currently loaded parsers
 200        """
 201
 202        for parser in self.parsers:
 203            if parser.__module__ == pattern:
 204                self.parsers.remove(parser)
 205                logger.debug("Ignoring parser {0}".format(parser.__name__))
 206        return self.parsers