Andrew's git - logparse.git/blob - logparse/load

   1#!/usr/bin/env python
   2# -*- coding: utf-8 -*-
   3"""
   4A basic "plugin loader" implementation which searches for default packaged and
   5user-supplied parser modules and verifies them so they can be executed by
   6logparse.interface. The requirements for parser modules and classes are
   7specified in the docstring of the Parser class.
   8
   9Classes in this module:
  10    - `Parser`:         Base class that every parser should inherit
  11    - `ParserLoader`:   Class used internally by `interface.py` to load parsers
  12"""
  13
  14import importlib
  15from os.path import dirname
  16from pkgutil import iter_modules
  17import inspect
  18from pathlib import Path
  19from typing import get_type_hints
  20
  21import logging
  22logger = logging.getLogger(__name__)
  23
  24
  25PARSER_DIR = "/usr/share/logparse/user-parsers"
  26PARSER_PKG = "logparse.parsers"
  27
  28
  29class Parser():
  30    """
  31    This is the base class that every parser should inherit. Parsers should
  32    each exist in their own module and contain a Parser class whose name is the
  33    same as the parser (e.g. `example.py` contains class `Example(Parser)`).
  34    Each parser module must contain exactly one Parser class definition, and
  35    this class cannot be a redefinition of the base Parser class (i.e. this
  36    class). This class must provide the parse_log() method which returns a
  37    logparse.formatting.Section object.
  38    """
  39
  40    def __init__(self, name=None, path=None, info=None, deprecated=False):
  41        """
  42        The following variables can be set to display information about the
  43        parser. The object `self.logger` can be used as for outputting messages
  44        to whatever sink is set up in logparse.interface (no setup required in
  45        the parser module itself).
  46        """
  47        self.name = str(name) if name else None
  48        self.path = Path(path) if path else None
  49        self.info = dict(info) if info else None
  50        self.logger = logging.getLogger(__name__)
  51        self.deprecated = deprecated
  52
  53    def load(self):
  54        """
  55        A generic loading method to import a parser, only used for debugging
  56        """
  57        logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults"))
  58        return importlib.import_module(self.name)
  59
  60    def parse_log(self, **args) -> None:
  61        """
  62        Every parser should provide the parse_log method which is executed at
  63        runtime to analyse logs. Verification checks should prevent the below
  64        exception from ever being raised.
  65        """
  66        raise NotImplementedError("Failed to find an entry point for parser")
  67
  68
  69
  70class ParserLoader:
  71    """
  72    This class searches for parsers in the main logparse package
  73    (logparser.parsers) and optionally in another external package (default
  74    /usr/share/logparse).
  75    """
  76
  77    def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR):
  78        """
  79        The pkg and path attributes shouldn't need to be set on object
  80        creation, the default values should work fine. They are hard-coded here
  81        for security so that a module can't force-load a module from another
  82        package/location, e.g. from the internet.
  83        """
  84
  85        self.pkg = pkg
  86        self.path = path
  87        self.parsers = []
  88
  89    def search(self, pattern):
  90        """
  91        Basic wrapper for the two search functions below.
  92        """
  93
  94        default_parser = self._search_default(pattern)
  95        if default_parser != None:
  96            self.parsers.append(default_parser)
  97            return default_parser
  98        else:
  99            user_parser = self._search_user(pattern)
 100            if user_parser != None:
 101                self.parsers.append(user_parser)
 102                return user_parser
 103            else:
 104                logger.warning("Couldn't find a matching parser module for search term {0}".format(pattern))
 105                return None
 106
 107    def _search_user(self, pattern):
 108        """
 109        Search for a parser name `pattern` in the user-managed parser directory
 110        """
 111
 112        logger.debug("Searching for {0} in {1}".format(pattern, self.path))
 113        try:
 114            spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path])
 115            parser_module = spec.loader.load_module(spec.name)
 116            return self._validate_module(parser_module)
 117        except Exception as e:
 118            logger.debug("Couldn't find parser {0} in {1}: {2}".format(pattern, self.path, str(e)))
 119            return None
 120
 121    def _search_default(self, pattern):
 122        """
 123        Search for a parser name `pattern` in the default parser package
 124        """
 125        
 126        # TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below
 127        logger.debug("Searching for {0} in default parsers".format(pattern))
 128        try:
 129            parser_module = importlib.import_module(self.pkg + "." + pattern)
 130            return self._validate_module(parser_module)
 131        except Exception as e:
 132            return None 
 133
 134    def _validate_module(self, parser_module):
 135        """
 136        Some basic security tests for candidate modules:
 137            1. Must contain exactly one Parser object
 138            3. This class cannot be a redefinition of the base Parser class
 139            4. Must provide the parse_log() method
 140            5. Must not return None
 141            6. Must not match an already-loaded class
 142        """
 143
 144        logger.debug("Checking validity of module {0} at {1}".format(parser_module.__name__, parser_module.__file__))
 145        available_parsers = []
 146        clsmembers = inspect.getmembers(parser_module, inspect.isclass)
 147
 148        # Check individual classes
 149        for (_, c) in clsmembers:
 150            if not (issubclass(c, Parser) & (c is not Parser)):
 151                continue
 152            if c in self.parsers:
 153                logger.warning("Parser class {0} has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__))
 154            if not inspect.isroutine(c.parse_log):
 155                logger.warning("Parser class {0} in {1} does not contain a parse_log() method".format(c.__class__.__name__, c.__file__))
 156                continue
 157            if None in get_type_hints(c):
 158                logger.warning("Parser class {0} in {1} contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__))
 159                continue
 160            logger.debug("Found parser {0}.{1}".format(c.__module__, c.__class__.__name__))
 161            available_parsers.append(c())
 162
 163        # Check module structure
 164        if len(available_parsers) == 1:
 165            logger.debug("Parser module {0} at {1} passed validity checks".format(parser_module.__name__, parser_module.__file__))
 166            return available_parsers[0]
 167        elif len(available_parsers) == 0:
 168            logger.warning("No valid classes in {0} at {1}".format(parser_module.__name__, parser_module.__file__))
 169            return None
 170        elif len(available_parsers) > 1:
 171            logger.warning("Found multiple valid parser classes in {0} at {1} - ignoring this module".format(parser_module.__name__, parser_module.__file__))
 172            return None
 173
 174    def load_pkg(self):
 175        """
 176        Clear the list of currently loaded packages and load all valid and
 177        non-deprecated parser classes from self.pkg using importlib.
 178        """
 179
 180        available_parsers = [name for _, name, _ in iter_modules([dirname(importlib.import_module(self.pkg).__file__)])]
 181        for parser_name in available_parsers:
 182            parser_module = importlib.import_module("logparse.parsers." + parser_name)
 183            parser_class = self._validate_module(parser_module)
 184            if parser_class == None:
 185                continue
 186            parser_obj = parser_class
 187            if parser_obj.deprecated:
 188                logger.debug("Ignoring parser {0} because it is deprecated".format(parser_class.__class__.__name__))
 189                continue
 190            self.parsers.append(parser_class)
 191        return self.parsers
 192
 193    def ignore(self, pattern):
 194        """
 195        Remove a parser from the list of currently loaded parsers
 196        """
 197
 198        for parser in self.parsers:
 199            if parser.__module__ == pattern:
 200                self.parsers.remove(parser)
 201                logger.debug("Ignoring parser {0}".format(parser.__name__))
 202        return self.parsers