1# -*- coding: utf-8 -*- 2 3""" 4A basic "plugin loader" implementation which searches for default packaged and 5user-supplied parser modules and verifies them so they can be executed by 6logparse.interface. The requirements for parser modules and classes are 7specified in the docstring of the Parser class. 8 9Classes in this module: 10 - `Parser`: Base class that every parser should inherit 11 - `ParserLoader`: Class used internally by `interface.py` to load parsers 12""" 13 14import importlib 15from os.path import dirname 16from pkgutil import iter_modules 17import inspect 18from pathlib import Path 19from typing import get_type_hints 20 21import logging 22logger = logging.getLogger(__name__) 23 24 25PARSER_DIR = "/usr/share/logparse/user-parsers" 26PARSER_PKG = "logparse.parsers" 27 28 29class Parser(): 30 """ 31 This is the base class that every parser should inherit. Parsers should 32 each exist in their own module and contain a Parser class whose name is the 33 same as the parser (e.g. `example.py` contains class `Example(Parser)`). 34 Each parser module must contain exactly one Parser class definition, and 35 this class cannot be a redefinition of the base Parser class (i.e. this 36 class). This class must provide the parse_log() method which returns a 37 logparse.formatting.Section object. 38 """ 39 40 def __init__(self, name=None, path=None, info=None, deprecated=False, 41 successor=""): 42 """ 43 The following variables can be set to display information about the 44 parser. The object `self.logger` can be used for outputting messages to 45 to whatever sink is set up in __init__.py (no setup required in the 46 parser module itself). 47 """ 48 self.name = str(name) if name else None 49 self.path = Path(path) if path else None 50 self.info = dict(info) if info else None 51 self.logger = logging.getLogger(self.__module__) 52 self.deprecated = deprecated 53 self.successor = successor 54 55 def load(self): 56 """ 57 A generic loading method to import a parser, only used for debugging 58 """ 59 logger.debug("Loading parser {0} from {1}".format( 60 self.name, str(self.path) if self.path != None else "defaults")) 61 return importlib.import_module(self.name) 62 63 def parse_log(self, **args) -> None: 64 """ 65 Every parser should provide the parse_log method which is executed at 66 runtime to analyse logs. Verification checks should prevent the below 67 exception from ever being raised. 68 """ 69 raise NotImplementedError("Failed to find an entry point for parser") 70 71 72 73class ParserLoader: 74 """ 75 This class searches for parsers in the main logparse package 76 (logparser.parsers) and optionally in another external package (default 77 /usr/share/logparse). 78 """ 79 80 def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR): 81 """ 82 The pkg and path attributes shouldn't need to be set on object 83 creation, the default values should work fine. They are hard-coded here 84 for security so that a module can't force-load a module from another 85 package/location, e.g. from the internet. 86 """ 87 88 self.pkg = pkg 89 self.path = path 90 self.parsers = [] 91 92 def search(self, pattern): 93 """ 94 Basic wrapper for the two search functions below. 95 """ 96 97 default_parser = self._search_default(pattern) 98 if default_parser != None: 99 self.parsers.append(default_parser) 100 return default_parser 101 else: 102 user_parser = self._search_user(pattern) 103 if user_parser != None: 104 self.parsers.append(user_parser) 105 return user_parser 106 else: 107 logger.warning("Couldn't find a matching parser module " 108 "for search term {0}".format(pattern)) 109 return None 110 111 def _search_user(self, pattern): 112 """ 113 Search for a parser name `pattern` in the user-managed parser directory 114 """ 115 116 logger.debug("Searching for {0} in {1}".format(pattern, self.path)) 117 try: 118 spec = importlib.machinery.PathFinder.find_spec( 119 pattern, path=[self.path]) 120 parser_module = spec.loader.load_module(spec.name) 121 return self._validate_module(parser_module) 122 except Exception as e: 123 return None 124 125 def _search_default(self, pattern): 126 """ 127 Search for a parser name `pattern` in the default parser package 128 TODO use importlib.resources.is_resources() once there is a backport 129 to Python 3.6 or below 130 """ 131 132 logger.debug("Searching for {0} in default parsers".format(pattern)) 133 try: 134 parser_module = importlib.import_module(self.pkg + "." + pattern) 135 return self._validate_module(parser_module) 136 except Exception as e: 137 return None 138 139 def _validate_module(self, parser_module): 140 """ 141 Some basic security tests for candidate modules: 142 1. Must contain exactly one Parser object 143 3. This class cannot be a redefinition of the base Parser class 144 4. Must provide the parse_log() method 145 5. Must not return None 146 6. Must not match an already-loaded class 147 """ 148 149 logger.debug("Checking validity of module {0} at {1}".format( 150 parser_module.__name__, parser_module.__file__)) 151 available_parsers = [] 152 clsmembers = inspect.getmembers(parser_module, inspect.isclass) 153 154 # Check individual classes 155 for (_, c) in clsmembers: 156 if not (issubclass(c, Parser) & (c is not Parser)): 157 continue 158 if c in self.parsers: 159 logger.warning("Parser class {0} has already been loaded " 160 "from another source, ignoring it".format( 161 c.__class__.__name__, c.__file__)) 162 if not inspect.isroutine(c.parse_log): 163 logger.warning("Parser class {0} in {1} does not contain a " 164 "parse_log() method".format( 165 c.__class__.__name__, c.__file__)) 166 continue 167 if None in get_type_hints(c): 168 logger.warning("Parser class {0} in {1} contains a " 169 "null-returning parse_log() method".format( 170 c.__class__.__name__, c.__file__)) 171 continue 172 parser_obj = c() 173 if parser_obj.deprecated: 174 logger.warning("Parser {0} is deprecated - " 175 "use {1} instead".format( 176 parser_obj.name, parser_obj.successor)) 177 logger.debug("Found parser {0}.{1}".format( 178 c.__module__, c.__class__.__name__)) 179 available_parsers.append(parser_obj) 180 181 # Check module structure 182 if len(available_parsers) == 1: 183 logger.debug("Parser module {0} at {1} passed validity checks" 184 .format(parser_module.__name__, parser_module.__file__)) 185 return available_parsers[0] 186 elif len(available_parsers) == 0: 187 logger.warning("No valid classes in {0} at {1}". 188 format(parser_module.__name__, parser_module.__file__)) 189 return None 190 elif len(available_parsers) > 1: 191 logger.warning("Found multiple valid parser classes in {0} at {1} " 192 "- ignoring this module" 193 .format(parser_module.__name__, parser_module.__file__)) 194 return None 195 196 def load_pkg(self): 197 """ 198 Clear the list of currently loaded packages and load all valid and 199 non-deprecated parser classes from self.pkg using importlib. 200 """ 201 202 available_parsers = [name for _, name, _ in iter_modules( 203 [dirname(importlib.import_module(self.pkg).__file__)])] 204 for parser_name in available_parsers: 205 parser_module = importlib.import_module( 206 "logparse.parsers." + parser_name) 207 parser_class = self._validate_module(parser_module) 208 if parser_class == None: 209 continue 210 if parser_class.deprecated: 211 logger.debug("Ignoring parser {0} because it is deprecated" 212 .format(parser_class.__class__.__name__)) 213 continue 214 self.parsers.append(parser_class) 215 return self.parsers 216 217 def ignore(self, pattern): 218 """ 219 Remove a parser from the list of currently loaded parsers 220 """ 221 222 for parser in self.parsers: 223 if parser.__module__ == pattern: 224 self.parsers.remove(parser) 225 logger.debug("Ignoring parser {0}".format(parser.__name__)) 226 return self.parsers