Andrew's git - logparse.git/blob - logparse/load

   1#!/usr/bin/env python
   2# -*- coding: utf-8 -*-
   3"""
   4A basic "plugin loader" implementation which searches for default packaged and
   5user-supplied parser modules and verifies them so they can be executed by
   6logparse.interface. The requirements for parser modules and classes are
   7specified in the docstring of the Parser class.
   8
   9Classes in this module:
  10    - `Parser`:         Base class that every parser should inherit
  11    - `ParserLoader`:   Class used internally by `interface.py` to load parsers
  12"""
  13
  14import importlib
  15from os.path import dirname
  16from pkgutil import iter_modules
  17import inspect
  18from pathlib import Path
  19from typing import get_type_hints
  20
  21import logging
  22logger = logging.getLogger(__name__)
  23
  24
  25PARSER_DIR = "/usr/share/logparse/user-parsers"
  26PARSER_PKG = "logparse.parsers"
  27
  28
  29class Parser():
  30    """
  31    This is the base class that every parser should inherit. Parsers should
  32    each exist in their own module and contain a Parser class whose name is the
  33    same as the parser (e.g. `example.py` contains class `Example(Parser)`).
  34    Each parser module must contain exactly one Parser class definition, and
  35    this class cannot be a redefinition of the base Parser class (i.e. this
  36    class). This class must provide the parse_log() method which returns a
  37    logparse.formatting.Section object.
  38    """
  39
  40    def __init__(self, name=None, path=None, info=None, deprecated=False, successor=""):
  41        """
  42        The following variables can be set to display information about the
  43        parser. The object `self.logger` can be used as for outputting messages
  44        to whatever sink is set up in logparse.interface (no setup required in
  45        the parser module itself).
  46        """
  47        self.name = str(name) if name else None
  48        self.path = Path(path) if path else None
  49        self.info = dict(info) if info else None
  50        self.logger = logging.getLogger(__name__)
  51        self.deprecated = deprecated
  52        self.successor = successor
  53
  54    def load(self):
  55        """
  56        A generic loading method to import a parser, only used for debugging
  57        """
  58        logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults"))
  59        return importlib.import_module(self.name)
  60
  61    def parse_log(self, **args) -> None:
  62        """
  63        Every parser should provide the parse_log method which is executed at
  64        runtime to analyse logs. Verification checks should prevent the below
  65        exception from ever being raised.
  66        """
  67        raise NotImplementedError("Failed to find an entry point for parser")
  68
  69
  70
  71class ParserLoader:
  72    """
  73    This class searches for parsers in the main logparse package
  74    (logparser.parsers) and optionally in another external package (default
  75    /usr/share/logparse).
  76    """
  77
  78    def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR):
  79        """
  80        The pkg and path attributes shouldn't need to be set on object
  81        creation, the default values should work fine. They are hard-coded here
  82        for security so that a module can't force-load a module from another
  83        package/location, e.g. from the internet.
  84        """
  85
  86        self.pkg = pkg
  87        self.path = path
  88        self.parsers = []
  89
  90    def search(self, pattern):
  91        """
  92        Basic wrapper for the two search functions below.
  93        """
  94
  95        default_parser = self._search_default(pattern)
  96        if default_parser != None:
  97            self.parsers.append(default_parser)
  98            return default_parser
  99        else:
 100            user_parser = self._search_user(pattern)
 101            if user_parser != None:
 102                self.parsers.append(user_parser)
 103                return user_parser
 104            else:
 105                logger.warning("Couldn't find a matching parser module for search term {0}".format(pattern))
 106                return None
 107
 108    def _search_user(self, pattern):
 109        """
 110        Search for a parser name `pattern` in the user-managed parser directory
 111        """
 112
 113        logger.debug("Searching for {0} in {1}".format(pattern, self.path))
 114        try:
 115            spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path])
 116            parser_module = spec.loader.load_module(spec.name)
 117            return self._validate_module(parser_module)
 118        except Exception as e:
 119            return None
 120
 121    def _search_default(self, pattern):
 122        """
 123        Search for a parser name `pattern` in the default parser package
 124        """
 125        
 126        # TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below
 127        logger.debug("Searching for {0} in default parsers".format(pattern))
 128        try:
 129            parser_module = importlib.import_module(self.pkg + "." + pattern)
 130            return self._validate_module(parser_module)
 131        except Exception as e:
 132            return None 
 133
 134    def _validate_module(self, parser_module):
 135        """
 136        Some basic security tests for candidate modules:
 137            1. Must contain exactly one Parser object
 138            3. This class cannot be a redefinition of the base Parser class
 139            4. Must provide the parse_log() method
 140            5. Must not return None
 141            6. Must not match an already-loaded class
 142        """
 143
 144        logger.debug("Checking validity of module {0} at {1}".format(parser_module.__name__, parser_module.__file__))
 145        available_parsers = []
 146        clsmembers = inspect.getmembers(parser_module, inspect.isclass)
 147
 148        # Check individual classes
 149        for (_, c) in clsmembers:
 150            if not (issubclass(c, Parser) & (c is not Parser)):
 151                continue
 152            if c in self.parsers:
 153                logger.warning("Parser class {0} has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__))
 154            if not inspect.isroutine(c.parse_log):
 155                logger.warning("Parser class {0} in {1} does not contain a parse_log() method".format(c.__class__.__name__, c.__file__))
 156                continue
 157            if None in get_type_hints(c):
 158                logger.warning("Parser class {0} in {1} contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__))
 159                continue
 160            parser_obj = c()
 161            if parser_obj.deprecated:
 162                logger.warning("Parser {0} is deprecated - use {1} instead".format(parser_obj.name, parser_obj.successor))
 163            logger.debug("Found parser {0}.{1}".format(c.__module__, c.__class__.__name__))
 164            available_parsers.append(c())
 165
 166        # Check module structure
 167        if len(available_parsers) == 1:
 168            logger.debug("Parser module {0} at {1} passed validity checks".format(parser_module.__name__, parser_module.__file__))
 169            return available_parsers[0]
 170        elif len(available_parsers) == 0:
 171            logger.warning("No valid classes in {0} at {1}".format(parser_module.__name__, parser_module.__file__))
 172            return None
 173        elif len(available_parsers) > 1:
 174            logger.warning("Found multiple valid parser classes in {0} at {1} - ignoring this module".format(parser_module.__name__, parser_module.__file__))
 175            return None
 176
 177    def load_pkg(self):
 178        """
 179        Clear the list of currently loaded packages and load all valid and
 180        non-deprecated parser classes from self.pkg using importlib.
 181        """
 182
 183        available_parsers = [name for _, name, _ in iter_modules([dirname(importlib.import_module(self.pkg).__file__)])]
 184        for parser_name in available_parsers:
 185            parser_module = importlib.import_module("logparse.parsers." + parser_name)
 186            parser_class = self._validate_module(parser_module)
 187            if parser_class == None:
 188                continue
 189            parser_obj = parser_class
 190            if parser_obj.deprecated:
 191                logger.debug("Ignoring parser {0} because it is deprecated".format(parser_class.__class__.__name__))
 192                continue
 193            self.parsers.append(parser_class)
 194        return self.parsers
 195
 196    def ignore(self, pattern):
 197        """
 198        Remove a parser from the list of currently loaded parsers
 199        """
 200
 201        for parser in self.parsers:
 202            if parser.__module__ == pattern:
 203                self.parsers.remove(parser)
 204                logger.debug("Ignoring parser {0}".format(parser.__name__))
 205        return self.parsers