1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3""" 4A basic "plugin loader" implementation which searches for default packaged and 5user-supplied parser modules and verifies them so they can be executed by 6logparse.interface. The requirements for parser modules and classes are 7specified in the docstring of the Parser class. 8 9Classes in this module: 10 - `Parser`: Base class that every parser should inherit 11 - `ParserLoader`: Class used internally by `interface.py` to load parsers 12""" 13 14import importlib 15from os.path import dirname 16from pkgutil import iter_modules 17import inspect 18from pathlib import Path 19from typing import get_type_hints 20 21import logging 22logger = logging.getLogger(__name__) 23 24 25PARSER_DIR ="/usr/share/logparse/user-parsers" 26PARSER_PKG ="logparse.parsers" 27 28 29classParser(): 30""" 31 This is the base class that every parser should inherit. Parsers should 32 each exist in their own module and contain a Parser class whose name is the 33 same as the parser (e.g. `example.py` contains class `Example(Parser)`). 34 Each parser module must contain exactly one Parser class definition, and 35 this class cannot be a redefinition of the base Parser class (i.e. this 36 class). This class must provide the parse_log() method which returns a 37 logparse.formatting.Section object. 38 """ 39 40def__init__(self, name=None, path=None, info=None, deprecated=False, successor=""): 41""" 42 The following variables can be set to display information about the 43 parser. The object `self.logger` can be used as for outputting messages 44 to whatever sink is set up in logparse.interface (no setup required in 45 the parser module itself). 46 """ 47 self.name =str(name)if name else None 48 self.path =Path(path)if path else None 49 self.info =dict(info)if info else None 50 self.logger = logging.getLogger(__name__) 51 self.deprecated = deprecated 52 self.successor = successor 53 54defload(self): 55""" 56 A generic loading method to import a parser, only used for debugging 57 """ 58 logger.debug("Loading parser{0}from{1}".format(self.name,str(self.path)if self.path !=None else"defaults")) 59return importlib.import_module(self.name) 60 61defparse_log(self, **args) ->None: 62""" 63 Every parser should provide the parse_log method which is executed at 64 runtime to analyse logs. Verification checks should prevent the below 65 exception from ever being raised. 66 """ 67raiseNotImplementedError("Failed to find an entry point for parser") 68 69 70 71class ParserLoader: 72""" 73 This class searches for parsers in the main logparse package 74 (logparser.parsers) and optionally in another external package (default 75 /usr/share/logparse). 76 """ 77 78def__init__(self, pkg=PARSER_PKG, path=PARSER_DIR): 79""" 80 The pkg and path attributes shouldn't need to be set on object 81 creation, the default values should work fine. They are hard-coded here 82 for security so that a module can't force-load a module from another 83 package/location, e.g. from the internet. 84 """ 85 86 self.pkg = pkg 87 self.path = path 88 self.parsers = [] 89 90defsearch(self, pattern): 91""" 92 Basic wrapper for the two search functions below. 93 """ 94 95 default_parser = self._search_default(pattern) 96if default_parser !=None: 97 self.parsers.append(default_parser) 98return default_parser 99else: 100 user_parser = self._search_user(pattern) 101if user_parser !=None: 102 self.parsers.append(user_parser) 103return user_parser 104else: 105 logger.warning("Couldn't find a matching parser module for search term{0}".format(pattern)) 106return None 107 108def_search_user(self, pattern): 109""" 110 Search for a parser name `pattern` in the user-managed parser directory 111 """ 112 113 logger.debug("Searching for{0}in{1}".format(pattern, self.path)) 114try: 115 spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path]) 116 parser_module = spec.loader.load_module(spec.name) 117return self._validate_module(parser_module) 118exceptExceptionas e: 119 logger.debug("Couldn't find parser{0}in{1}".format(pattern, self.path)) 120return None 121 122def_search_default(self, pattern): 123""" 124 Search for a parser name `pattern` in the default parser package 125 """ 126 127# TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below 128 logger.debug("Searching for{0}in default parsers".format(pattern)) 129try: 130 parser_module = importlib.import_module(self.pkg +"."+ pattern) 131return self._validate_module(parser_module) 132exceptExceptionas e: 133return None 134 135def_validate_module(self, parser_module): 136""" 137 Some basic security tests for candidate modules: 138 1. Must contain exactly one Parser object 139 3. This class cannot be a redefinition of the base Parser class 140 4. Must provide the parse_log() method 141 5. Must not return None 142 6. Must not match an already-loaded class 143 """ 144 145 logger.debug("Checking validity of module{0}at{1}".format(parser_module.__name__, parser_module.__file__)) 146 available_parsers = [] 147 clsmembers = inspect.getmembers(parser_module, inspect.isclass) 148 149# Check individual classes 150for(_, c)in clsmembers: 151if not(issubclass(c, Parser) & (c is not Parser)): 152continue 153if c in self.parsers: 154 logger.warning("Parser class{0}has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__)) 155if not inspect.isroutine(c.parse_log): 156 logger.warning("Parser class{0}in{1}does not contain a parse_log() method".format(c.__class__.__name__, c.__file__)) 157continue 158if None inget_type_hints(c): 159 logger.warning("Parser class{0}in{1}contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__)) 160continue 161 parser_obj =c() 162if parser_obj.deprecated: 163 logger.warning("Parser{0}is deprecated - use{1}instead".format(parser_obj.name, parser_obj.successor)) 164 logger.debug("Found parser{0}.{1}".format(c.__module__, c.__class__.__name__)) 165 available_parsers.append(c()) 166 167# Check module structure 168iflen(available_parsers) ==1: 169 logger.debug("Parser module{0}at{1}passed validity checks".format(parser_module.__name__, parser_module.__file__)) 170return available_parsers[0] 171eliflen(available_parsers) ==0: 172 logger.warning("No valid classes in{0}at{1}".format(parser_module.__name__, parser_module.__file__)) 173return None 174eliflen(available_parsers) >1: 175 logger.warning("Found multiple valid parser classes in{0}at{1}- ignoring this module".format(parser_module.__name__, parser_module.__file__)) 176return None 177 178defload_pkg(self): 179""" 180 Clear the list of currently loaded packages and load all valid and 181 non-deprecated parser classes from self.pkg using importlib. 182 """ 183 184 available_parsers = [name for _, name, _ initer_modules([dirname(importlib.import_module(self.pkg).__file__)])] 185for parser_name in available_parsers: 186 parser_module = importlib.import_module("logparse.parsers."+ parser_name) 187 parser_class = self._validate_module(parser_module) 188if parser_class ==None: 189continue 190 parser_obj = parser_class 191if parser_obj.deprecated: 192 logger.debug("Ignoring parser{0}because it is deprecated".format(parser_class.__class__.__name__)) 193continue 194 self.parsers.append(parser_class) 195return self.parsers 196 197defignore(self, pattern): 198""" 199 Remove a parser from the list of currently loaded parsers 200 """ 201 202for parser in self.parsers: 203if parser.__module__== pattern: 204 self.parsers.remove(parser) 205 logger.debug("Ignoring parser{0}".format(parser.__name__)) 206return self.parsers