Andrew's git - logparse.git/blob - logparse/load

   1# -*- coding: utf-8 -*-
   2
   3"""
   4A basic "plugin loader" implementation which searches for default packaged and
   5user-supplied parser modules and verifies them so they can be executed by
   6logparse.interface. The requirements for parser modules and classes are
   7specified in the docstring of the Parser class.
   8
   9Classes in this module:
  10    - `Parser`:         Base class that every parser should inherit
  11    - `ParserLoader`:   Class used internally by `interface.py` to load parsers
  12"""
  13
  14import importlib
  15from os.path import dirname
  16from pkgutil import iter_modules
  17import inspect
  18from pathlib import Path
  19from typing import get_type_hints
  20
  21import logging
  22logger = logging.getLogger(__name__)
  23
  24
  25PARSER_DIR = "/usr/share/logparse/user-parsers"
  26PARSER_PKG = "logparse.parsers"
  27
  28
  29class Parser():
  30    """
  31    This is the base class that every parser should inherit. Parsers should
  32    each exist in their own module and contain a Parser class whose name is the
  33    same as the parser (e.g. `example.py` contains class `Example(Parser)`).
  34    Each parser module must contain exactly one Parser class definition, and
  35    this class cannot be a redefinition of the base Parser class (i.e. this
  36    class). This class must provide the parse_log() method which returns a
  37    logparse.formatting.Section object.
  38    """
  39
  40    def __init__(self, name=None, path=None, info=None, deprecated=False,
  41            successor=""):
  42        """
  43        The following variables can be set to display information about the
  44        parser. The object `self.logger` can be used for outputting messages to
  45        to whatever sink is set up in __init__.py (no setup required in the
  46        parser module itself).
  47        """
  48        self.name = str(name) if name else None
  49        self.path = Path(path) if path else None
  50        self.info = dict(info) if info else None
  51        self.logger = logging.getLogger(self.__module__)
  52        self.deprecated = deprecated
  53        self.successor = successor
  54
  55    def load(self):
  56        """
  57        A generic loading method to import a parser, only used for debugging
  58        """
  59        logger.debug("Loading parser {0} from {1}".format(
  60            self.name, str(self.path) if self.path != None else "defaults"))
  61        return importlib.import_module(self.name)
  62
  63    def parse_log(self, **args) -> None:
  64        """
  65        Every parser should provide the parse_log method which is executed at
  66        runtime to analyse logs. Verification checks should prevent the below
  67        exception from ever being raised.
  68        """
  69        raise NotImplementedError("Failed to find an entry point for parser")
  70
  71
  72
  73class ParserLoader:
  74    """
  75    This class searches for parsers in the main logparse package
  76    (logparser.parsers) and optionally in another external package (default
  77    /usr/share/logparse).
  78    """
  79
  80    def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR):
  81        """
  82        The pkg and path attributes shouldn't need to be set on object
  83        creation, the default values should work fine. They are hard-coded here
  84        for security so that a module can't force-load a module from another
  85        package/location, e.g. from the internet.
  86        """
  87
  88        self.pkg = pkg
  89        self.path = path
  90        self.parsers = []
  91
  92    def search(self, pattern):
  93        """
  94        Basic wrapper for the two search functions below.
  95        """
  96
  97        default_parser = self._search_default(pattern)
  98        if default_parser != None:
  99            self.parsers.append(default_parser)
 100            return default_parser
 101        else:
 102            user_parser = self._search_user(pattern)
 103            if user_parser != None:
 104                self.parsers.append(user_parser)
 105                return user_parser
 106            else:
 107                logger.warning("Couldn't find a matching parser module "
 108                    "for search term {0}".format(pattern))
 109                return None
 110
 111    def _search_user(self, pattern):
 112        """
 113        Search for a parser name `pattern` in the user-managed parser directory
 114        """
 115
 116        logger.debug("Searching for {0} in {1}".format(pattern, self.path))
 117        try:
 118            spec = importlib.machinery.PathFinder.find_spec(
 119                    pattern, path=[self.path])
 120            parser_module = spec.loader.load_module(spec.name)
 121            return self._validate_module(parser_module)
 122        except Exception as e:
 123            return None
 124
 125    def _search_default(self, pattern):
 126        """
 127        Search for a parser name `pattern` in the default parser package
 128        TODO use importlib.resources.is_resources() once there is a backport
 129        to Python 3.6 or below
 130        """
 131        
 132        logger.debug("Searching for {0} in default parsers".format(pattern))
 133        try:
 134            parser_module = importlib.import_module(self.pkg + "." + pattern)
 135            return self._validate_module(parser_module)
 136        except Exception as e:
 137            return None 
 138
 139    def _validate_module(self, parser_module):
 140        """
 141        Some basic security tests for candidate modules:
 142            1. Must contain exactly one Parser object
 143            3. This class cannot be a redefinition of the base Parser class
 144            4. Must provide the parse_log() method
 145            5. Must not return None
 146            6. Must not match an already-loaded class
 147        """
 148
 149        logger.debug("Checking validity of module {0} at {1}".format(
 150            parser_module.__name__, parser_module.__file__))
 151        available_parsers = []
 152        clsmembers = inspect.getmembers(parser_module, inspect.isclass)
 153
 154        # Check individual classes
 155        for (_, c) in clsmembers:
 156            if not (issubclass(c, Parser) & (c is not Parser)):
 157                continue
 158            if c in self.parsers:
 159                logger.warning("Parser class {0} has already been loaded "
 160                    "from another source, ignoring it".format(
 161                        c.__class__.__name__, c.__file__))
 162            if not inspect.isroutine(c.parse_log):
 163                logger.warning("Parser class {0} in {1} does not contain a "
 164                    "parse_log() method".format(
 165                        c.__class__.__name__, c.__file__))
 166                continue
 167            if None in get_type_hints(c):
 168                logger.warning("Parser class {0} in {1} contains a "
 169                    "null-returning parse_log() method".format(
 170                        c.__class__.__name__, c.__file__))
 171                continue
 172            parser_obj = c()
 173            if parser_obj.deprecated:
 174                logger.warning("Parser {0} is deprecated - "
 175                    "use {1} instead".format(
 176                        parser_obj.name, parser_obj.successor))
 177            logger.debug("Found parser {0}.{1}".format(
 178                c.__module__, c.__class__.__name__))
 179            available_parsers.append(parser_obj)
 180
 181        # Check module structure
 182        if len(available_parsers) == 1:
 183            logger.debug("Parser module {0} at {1} passed validity checks"
 184                    .format(parser_module.__name__, parser_module.__file__))
 185            return available_parsers[0]
 186        elif len(available_parsers) == 0:
 187            logger.warning("No valid classes in {0} at {1}".
 188                    format(parser_module.__name__, parser_module.__file__))
 189            return None
 190        elif len(available_parsers) > 1:
 191            logger.warning("Found multiple valid parser classes in {0} at {1} "
 192                "- ignoring this module"
 193                .format(parser_module.__name__, parser_module.__file__))
 194            return None
 195
 196    def load_pkg(self):
 197        """
 198        Clear the list of currently loaded packages and load all valid and
 199        non-deprecated parser classes from self.pkg using importlib.
 200        """
 201
 202        available_parsers = [name for _, name, _ in iter_modules(
 203            [dirname(importlib.import_module(self.pkg).__file__)])]
 204        for parser_name in available_parsers:
 205            parser_module = importlib.import_module(
 206                    "logparse.parsers." + parser_name)
 207            parser_class = self._validate_module(parser_module)
 208            if parser_class == None:
 209                continue
 210            if parser_class.deprecated:
 211                logger.debug("Ignoring parser {0} because it is deprecated"
 212                        .format(parser_class.__class__.__name__))
 213                continue
 214            self.parsers.append(parser_class)
 215        return self.parsers
 216
 217    def ignore(self, pattern):
 218        """
 219        Remove a parser from the list of currently loaded parsers
 220        """
 221
 222        for parser in self.parsers:
 223            if parser.__module__ == pattern:
 224                self.parsers.remove(parser)
 225                logger.debug("Ignoring parser {0}".format(parser.__name__))
 226        return self.parsers