1""" 2A basic "plugin loader" implementation which searches for default packaged and 3user-supplied parser modules and verifies them so they can be executed by 4logparse.interface. The requirements for parser modules and classes are 5specified in the docstring of the Parser class. 6 7Classes in this module: 8 Parser Base class that every parser should inherit 9 ParserLoader Class used internally by logparse.interface to load parsers 10""" 11 12import importlib 13from os.path import dirname 14from pkgutil import iter_modules 15import inspect 16from pathlib import Path 17from typing import get_type_hints 18 19import logging 20logger = logging.getLogger(__name__) 21 22 23PARSER_DIR = "/usr/share/logparse/user-parsers" 24PARSER_PKG = "logparse.parsers" 25 26 27class Parser(): 28 """ 29 This is the base class that every parser should inherit. Parsers should 30 each exist in their own module and contain a Parser class whose name is the 31 same as the parser (e.g. `example.py` contains class `Example(Parser)`). 32 Each parser module must contain exactly one Parser class definition, and 33 this class cannot be a redefinition of the base Parser class (i.e. this 34 class). This class must provide the parse_log() method which returns a 35 logparse.formatting.Section object. 36 """ 37 38 def __init__(self, name=None, path=None, info=None, deprecated=False): 39 """ 40 The following variables can be set to display information about the 41 parser. The object `self.logger` can be used as for outputting messages 42 to whatever sink is set up in logparse.interface (no setup required in 43 the parser module itself). 44 """ 45 self.name = str(name) if name else None 46 self.path = Path(path) if path else None 47 self.info = dict(info) if info else None 48 self.logger = logging.getLogger(__name__) 49 self.deprecated = deprecated 50 51 def load(self): 52 """ 53 A generic loading method to import a parser, only used for debugging 54 """ 55 logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults")) 56 return importlib.import_module(self.name) 57 58 def parse_log(self, **args) -> None: 59 """ 60 Every parser should provide the parse_log method which is executed at 61 runtime to analyse logs. Verification checks should prevent the below 62 exception from ever being raised. 63 """ 64 raise NotImplementedError("Failed to find an entry point for parser") 65 66 67 68class ParserLoader: 69 """ 70 This class searches for parsers in the main logparse package 71 (logparser.parsers) and optionally in another external package (default 72 /usr/share/logparse). 73 """ 74 75 def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR): 76 """ 77 The pkg and path attributes shouldn't need to be set on object 78 creation, the default values should work fine. They are hard-coded here 79 for security so that a module can't force-load a module from another 80 package/location, e.g. from the internet. 81 """ 82 83 self.pkg = pkg 84 self.path = path 85 self.parsers = [] 86 87 def search(self, pattern): 88 """ 89 Basic wrapper for the two search functions below. 90 """ 91 92 default_parser = self._search_default(pattern) 93 if default_parser != None: 94 self.parsers.append(default_parser) 95 return default_parser 96 else: 97 user_parser = self._search_user(pattern) 98 if user_parser != None: 99 self.parsers.append(user_parser) 100 return user_parser 101 else: 102 logger.warning("Couldn't find a matching parser module for search term {0}".format(pattern)) 103 return None 104 105 def _search_user(self, pattern): 106 """ 107 Search for a parser name `pattern` in the user-managed parser directory 108 """ 109 110 logger.debug("Searching for {0} in {1}".format(pattern, self.path)) 111 try: 112 spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path]) 113 parser_module = spec.loader.load_module(spec.name) 114 return self._validate_module(parser_module) 115 except Exception as e: 116 logger.debug("Couldn't find parser {0} in {1}: {2}".format(pattern, self.path, str(e))) 117 return None 118 119 def _search_default(self, pattern): 120 """ 121 Search for a parser name `pattern` in the default parser package 122 """ 123 124 # TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below 125 logger.debug("Searching for {0} in default parsers".format(pattern)) 126 try: 127 parser_module = importlib.import_module(self.pkg + "." + pattern) 128 return self._validate_module(parser_module) 129 except Exception as e: 130 return None 131 132 def _validate_module(self, parser_module): 133 """ 134 Some basic security tests for candidate modules: 135 1. Must contain exactly one Parser object 136 3. This class cannot be a redefinition of the base Parser class 137 4. Must provide the parse_log() method 138 5. Must not return None 139 6. Must not match an already-loaded class 140 """ 141 142 logger.debug("Checking validity of module {0} at {1}".format(parser_module.__name__, parser_module.__file__)) 143 available_parsers = [] 144 clsmembers = inspect.getmembers(parser_module, inspect.isclass) 145 146 # Check individual classes 147 for (_, c) in clsmembers: 148 if not (issubclass(c, Parser) & (c is not Parser)): 149 continue 150 if c in self.parsers: 151 logger.warning("Parser class {0} has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__)) 152 if not inspect.isroutine(c.parse_log): 153 logger.warning("Parser class {0} in {1} does not contain a parse_log() method".format(c.__class__.__name__, c.__file__)) 154 continue 155 if None in get_type_hints(c): 156 logger.warning("Parser class {0} in {1} contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__)) 157 continue 158 logger.debug("Found parser {0}.{1}".format(c.__module__, c.__class__.__name__)) 159 available_parsers.append(c()) 160 161 # Check module structure 162 if len(available_parsers) == 1: 163 logger.debug("Parser module {0} at {1} passed validity checks".format(parser_module.__name__, parser_module.__file__)) 164 return available_parsers[0] 165 elif len(available_parsers) == 0: 166 logger.warning("No valid classes in {0} at {1}".format(parser_module.__name__, parser_module.__file__)) 167 return None 168 elif len(available_parsers) > 1: 169 logger.warning("Found multiple valid parser classes in {0} at {1} - ignoring this module".format(parser_module.__name__, parser_module.__file__)) 170 return None 171 172 def load_pkg(self): 173 """ 174 Clear the list of currently loaded packages and load all valid and 175 non-deprecated parser classes from self.pkg using importlib. 176 """ 177 178 available_parsers = [name for _, name, _ in iter_modules([dirname(importlib.import_module(self.pkg).__file__)])] 179 for parser_name in available_parsers: 180 parser_module = importlib.import_module("logparse.parsers." + parser_name) 181 parser_class = self._validate_module(parser_module) 182 if parser_class == None: 183 continue 184 parser_obj = parser_class 185 if parser_obj.deprecated: 186 logger.debug("Ignoring parser {0} because it is deprecated".format(parser_class.__class__.__name__)) 187 continue 188 self.parsers.append(parser_class) 189 return self.parsers 190 191 def ignore(self, pattern): 192 """ 193 Remove a parser from the list of currently loaded parsers 194 """ 195 196 for parser in self.parsers: 197 if parser.__module__ == pattern: 198 self.parsers.remove(parser) 199 logger.debug("Ignoring parser {0}".format(parser.__name__)) 200 return self.parsers