1""" 2A basic "plugin loader" implementation which searches for default packaged and 3user-supplied parser modules and verifies them so they can be executed by 4logparse.interface. The requirements for parser modules and classes are 5specified in the docstring of the Parser class. 6 7Classes in this module: 8 Parser Base class that every parser should inherit 9 ParserLoader Class used internally by logparse.interface to load parsers 10""" 11 12import importlib 13from os.path import dirname 14from pkgutil import iter_modules 15import inspect 16from pathlib import Path 17from typing import get_type_hints 18 19import logging 20logger = logging.getLogger(__name__) 21 22 23PARSER_DIR ="/usr/share/logparse/user-parsers" 24PARSER_PKG ="logparse.parsers" 25 26 27classParser(): 28""" 29 This is the base class that every parser should inherit. Parsers should 30 each exist in their own module and contain a Parser class whose name is the 31 same as the parser (e.g. `example.py` contains class `Example(Parser)`). 32 Each parser module must contain exactly one Parser class definition, and 33 this class cannot be a redefinition of the base Parser class (i.e. this 34 class). This class must provide the parse_log() method which returns a 35 logparse.formatting.Section object. 36 """ 37 38def__init__(self, name=None, path=None, info=None, deprecated=False): 39""" 40 The following variables can be set to display information about the 41 parser. The object `self.logger` can be used as for outputting messages 42 to whatever sink is set up in logparse.interface (no setup required in 43 the parser module itself). 44 """ 45 self.name =str(name)if name else None 46 self.path =Path(path)if path else None 47 self.info =dict(info)if info else None 48 self.logger = logging.getLogger(__name__) 49 self.deprecated = deprecated 50 51defload(self): 52""" 53 A generic loading method to import a parser, only used for debugging 54 """ 55 logger.debug("Loading parser{0}from{1}".format(self.name,str(self.path)if self.path !=None else"defaults")) 56return importlib.import_module(self.name) 57 58defparse_log(self, **args) ->None: 59""" 60 Every parser should provide the parse_log method which is executed at 61 runtime to analyse logs. Verification checks should prevent the below 62 exception from ever being raised. 63 """ 64raiseNotImplementedError("Failed to find an entry point for parser") 65 66 67 68class ParserLoader: 69""" 70 This class searches for parsers in the main logparse package 71 (logparser.parsers) and optionally in another external package (default 72 /usr/share/logparse). 73 """ 74 75def__init__(self, pkg=PARSER_PKG, path=PARSER_DIR): 76""" 77 The pkg and path attributes shouldn't need to be set on object 78 creation, the default values should work fine. They are hard-coded here 79 for security so that a module can't force-load a module from another 80 package/location, e.g. from the internet. 81 """ 82 83 self.pkg = pkg 84 self.path = path 85 self.parsers = [] 86 87defsearch(self, pattern): 88""" 89 Basic wrapper for the two search functions below. 90 """ 91 92 default_parser = self._search_default(pattern) 93if default_parser !=None: 94 self.parsers.append(default_parser) 95return default_parser 96else: 97 user_parser = self._search_user(pattern) 98if user_parser !=None: 99 self.parsers.append(user_parser) 100return user_parser 101else: 102 logger.warning("Couldn't find a matching parser module for search term{0}".format(pattern)) 103return None 104 105def_search_user(self, pattern): 106""" 107 Search for a parser name `pattern` in the user-managed parser directory 108 """ 109 110 logger.debug("Searching for{0}in{1}".format(pattern, self.path)) 111try: 112 spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path]) 113 parser_module = spec.loader.load_module(spec.name) 114return self._validate_module(parser_module) 115exceptExceptionas e: 116 logger.debug("Couldn't find parser{0}in{1}:{2}".format(pattern, self.path,str(e))) 117return None 118 119def_search_default(self, pattern): 120""" 121 Search for a parser name `pattern` in the default parser package 122 """ 123 124# TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below 125 logger.debug("Searching for{0}in default parsers".format(pattern)) 126try: 127 parser_module = importlib.import_module(self.pkg +"."+ pattern) 128return self._validate_module(parser_module) 129exceptExceptionas e: 130return None 131 132def_validate_module(self, parser_module): 133""" 134 Some basic security tests for candidate modules: 135 1. Must contain exactly one Parser object 136 3. This class cannot be a redefinition of the base Parser class 137 4. Must provide the parse_log() method 138 5. Must not return None 139 6. Must not match an already-loaded class 140 """ 141 142 logger.debug("Checking validity of module{0}at{1}".format(parser_module.__name__, parser_module.__file__)) 143 available_parsers = [] 144 clsmembers = inspect.getmembers(parser_module, inspect.isclass) 145 146# Check individual classes 147for(_, c)in clsmembers: 148if not(issubclass(c, Parser) & (c is not Parser)): 149continue 150if c in self.parsers: 151 logger.warning("Parser class{0}has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__)) 152if not inspect.isroutine(c.parse_log): 153 logger.warning("Parser class{0}in{1}does not contain a parse_log() method".format(c.__class__.__name__, c.__file__)) 154continue 155if None inget_type_hints(c): 156 logger.warning("Parser class{0}in{1}contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__)) 157continue 158 logger.debug("Found parser{0}.{1}".format(c.__module__, c.__class__.__name__)) 159 available_parsers.append(c()) 160 161# Check module structure 162iflen(available_parsers) ==1: 163 logger.debug("Parser module{0}at{1}passed validity checks".format(parser_module.__name__, parser_module.__file__)) 164return available_parsers[0] 165eliflen(available_parsers) ==0: 166 logger.warning("No valid classes in{0}at{1}".format(parser_module.__name__, parser_module.__file__)) 167return None 168eliflen(available_parsers) >1: 169 logger.warning("Found multiple valid parser classes in{0}at{1}- ignoring this module".format(parser_module.__name__, parser_module.__file__)) 170return None 171 172defload_pkg(self): 173""" 174 Clear the list of currently loaded packages and load all valid and 175 non-deprecated parser classes from self.pkg using importlib. 176 """ 177 178 available_parsers = [name for _, name, _ initer_modules([dirname(importlib.import_module(self.pkg).__file__)])] 179for parser_name in available_parsers: 180 parser_module = importlib.import_module("logparse.parsers."+ parser_name) 181 parser_class = self._validate_module(parser_module) 182if parser_class ==None: 183continue 184 parser_obj = parser_class 185if parser_obj.deprecated: 186 logger.debug("Ignoring parser{0}because it is deprecated".format(parser_class.__class__.__name__)) 187continue 188 self.parsers.append(parser_class) 189return self.parsers 190 191defignore(self, pattern): 192""" 193 Remove a parser from the list of currently loaded parsers 194 """ 195 196for parser in self.parsers: 197if parser.__module__== pattern: 198 self.parsers.remove(parser) 199 logger.debug("Ignoring parser{0}".format(parser.__name__)) 200return self.parsers