logparse / load_parsers.pyon commit rework parser loading interface (c9a3c26)
   1"""
   2A basic "plugin loader" implementation which searches for default packaged and
   3user-supplied parser modules and verifies them so they can be executed by
   4logparse.interface. The requirements for parser modules and classes are
   5specified in the docstring of the Parser class.
   6
   7Classes in this module:
   8    Parser          Base class that every parser should inherit
   9    ParserLoader    Class used internally by logparse.interface to load parsers
  10"""
  11
  12import importlib
  13from os.path import dirname
  14from pkgutil import iter_modules
  15import inspect
  16from pathlib import Path
  17from typing import get_type_hints
  18
  19import logging
  20logger = logging.getLogger(__name__)
  21
  22
  23PARSER_DIR = "/usr/share/logparse/user-parsers"
  24PARSER_PKG = "logparse.parsers"
  25
  26
  27class Parser():
  28    """
  29    This is the base class that every parser should inherit. Parsers should
  30    each exist in their own module and contain a Parser class whose name is the
  31    same as the parser (e.g. `example.py` contains class `Example(Parser)`).
  32    Each parser module must contain exactly one Parser class definition, and
  33    this class cannot be a redefinition of the base Parser class (i.e. this
  34    class). This class must provide the parse_log() method which returns a
  35    logparse.formatting.Section object.
  36    """
  37
  38    def __init__(self, name=None, path=None, info=None, deprecated=False):
  39        """
  40        The following variables can be set to display information about the
  41        parser. The object `self.logger` can be used as for outputting messages
  42        to whatever sink is set up in logparse.interface (no setup required in
  43        the parser module itself).
  44        """
  45        self.name = str(name) if name else None
  46        self.path = Path(path) if path else None
  47        self.info = dict(info) if info else None
  48        self.logger = logging.getLogger(__name__)
  49        self.deprecated = deprecated
  50
  51    def load(self):
  52        """
  53        A generic loading method to import a parser, only used for debugging
  54        """
  55        logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults"))
  56        return importlib.import_module(self.name)
  57
  58    def parse_log(self, **args) -> None:
  59        """
  60        Every parser should provide the parse_log method which is executed at
  61        runtime to analyse logs. Verification checks should prevent the below
  62        exception from ever being raised.
  63        """
  64        raise NotImplementedError("Failed to find an entry point for parser")
  65
  66
  67
  68class ParserLoader:
  69    """
  70    This class searches for parsers in the main logparse package
  71    (logparser.parsers) and optionally in another external package (default
  72    /usr/share/logparse).
  73    """
  74
  75    def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR):
  76        """
  77        The pkg and path attributes shouldn't need to be set on object
  78        creation, the default values should work fine. They are hard-coded here
  79        for security so that a module can't force-load a module from another
  80        package/location, e.g. from the internet.
  81        """
  82
  83        self.pkg = pkg
  84        self.path = path
  85        self.parsers = []
  86
  87    def search(self, pattern):
  88        """
  89        Basic wrapper for the two search functions below.
  90        """
  91
  92        default_parser = self._search_default(pattern)
  93        if default_parser != None:
  94            self.parsers.append(default_parser)
  95            return default_parser
  96        else:
  97            user_parser = self._search_user(pattern)
  98            if user_parser != None:
  99                self.parsers.append(user_parser)
 100                return user_parser
 101            else:
 102                logger.warning("Couldn't find a matching parser module for search term {0}".format(pattern))
 103                return None
 104
 105    def _search_user(self, pattern):
 106        """
 107        Search for a parser name `pattern` in the user-managed parser directory
 108        """
 109
 110        logger.debug("Searching for {0} in {1}".format(pattern, self.path))
 111        try:
 112            spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path])
 113            parser_module = spec.loader.load_module(spec.name)
 114            return self._validate_module(parser_module)
 115        except Exception as e:
 116            logger.debug("Couldn't find parser {0} in {1}: {2}".format(pattern, self.path, str(e)))
 117            return None
 118
 119    def _search_default(self, pattern):
 120        """
 121        Search for a parser name `pattern` in the default parser package
 122        """
 123        
 124        # TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below
 125        logger.debug("Searching for {0} in default parsers".format(pattern))
 126        try:
 127            parser_module = importlib.import_module(self.pkg + "." + pattern)
 128            return self._validate_module(parser_module)
 129        except Exception as e:
 130            return None 
 131
 132    def _validate_module(self, parser_module):
 133        """
 134        Some basic security tests for candidate modules:
 135            1. Must contain exactly one Parser object
 136            3. This class cannot be a redefinition of the base Parser class
 137            4. Must provide the parse_log() method
 138            5. Must not return None
 139            6. Must not match an already-loaded class
 140        """
 141
 142        logger.debug("Checking validity of module {0} at {1}".format(parser_module.__name__, parser_module.__file__))
 143        available_parsers = []
 144        clsmembers = inspect.getmembers(parser_module, inspect.isclass)
 145
 146        # Check individual classes
 147        for (_, c) in clsmembers:
 148            if not (issubclass(c, Parser) & (c is not Parser)):
 149                continue
 150            if c in self.parsers:
 151                logger.warning("Parser class {0} has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__))
 152            if not inspect.isroutine(c.parse_log):
 153                logger.warning("Parser class {0} in {1} does not contain a parse_log() method".format(c.__class__.__name__, c.__file__))
 154                continue
 155            if None in get_type_hints(c):
 156                logger.warning("Parser class {0} in {1} contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__))
 157                continue
 158            logger.debug("Found parser {0}.{1}".format(c.__module__, c.__class__.__name__))
 159            available_parsers.append(c())
 160
 161        # Check module structure
 162        if len(available_parsers) == 1:
 163            logger.debug("Parser module {0} at {1} passed validity checks".format(parser_module.__name__, parser_module.__file__))
 164            return available_parsers[0]
 165        elif len(available_parsers) == 0:
 166            logger.warning("No valid classes in {0} at {1}".format(parser_module.__name__, parser_module.__file__))
 167            return None
 168        elif len(available_parsers) > 1:
 169            logger.warning("Found multiple valid parser classes in {0} at {1} - ignoring this module".format(parser_module.__name__, parser_module.__file__))
 170            return None
 171
 172    def load_pkg(self):
 173        """
 174        Clear the list of currently loaded packages and load all valid and
 175        non-deprecated parser classes from self.pkg using importlib.
 176        """
 177
 178        available_parsers = [name for _, name, _ in iter_modules([dirname(importlib.import_module(self.pkg).__file__)])]
 179        for parser_name in available_parsers:
 180            parser_module = importlib.import_module("logparse.parsers." + parser_name)
 181            parser_class = self._validate_module(parser_module)
 182            if parser_class == None:
 183                continue
 184            parser_obj = parser_class
 185            if parser_obj.deprecated:
 186                logger.debug("Ignoring parser {0} because it is deprecated".format(parser_class.__class__.__name__))
 187                continue
 188            self.parsers.append(parser_class)
 189        return self.parsers
 190
 191    def ignore(self, pattern):
 192        """
 193        Remove a parser from the list of currently loaded parsers
 194        """
 195
 196        for parser in self.parsers:
 197            if parser.__module__ == pattern:
 198                self.parsers.remove(parser)
 199                logger.debug("Ignoring parser {0}".format(parser.__name__))
 200        return self.parsers