1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3""" 4A basic "plugin loader" implementation which searches for default packaged and 5user-supplied parser modules and verifies them so they can be executed by 6logparse.interface. The requirements for parser modules and classes are 7specified in the docstring of the Parser class. 8 9Classes in this module: 10 - `Parser`: Base class that every parser should inherit 11 - `ParserLoader`: Class used internally by `interface.py` to load parsers 12""" 13 14import importlib 15from os.path import dirname 16from pkgutil import iter_modules 17import inspect 18from pathlib import Path 19from typing import get_type_hints 20 21import logging 22logger = logging.getLogger(__name__) 23 24 25PARSER_DIR ="/usr/share/logparse/user-parsers" 26PARSER_PKG ="logparse.parsers" 27 28 29classParser(): 30""" 31 This is the base class that every parser should inherit. Parsers should 32 each exist in their own module and contain a Parser class whose name is the 33 same as the parser (e.g. `example.py` contains class `Example(Parser)`). 34 Each parser module must contain exactly one Parser class definition, and 35 this class cannot be a redefinition of the base Parser class (i.e. this 36 class). This class must provide the parse_log() method which returns a 37 logparse.formatting.Section object. 38 """ 39 40def__init__(self, name=None, path=None, info=None, deprecated=False, successor=""): 41""" 42 The following variables can be set to display information about the 43 parser. The object `self.logger` can be used as for outputting messages 44 to whatever sink is set up in logparse.interface (no setup required in 45 the parser module itself). 46 """ 47 self.name =str(name)if name else None 48 self.path =Path(path)if path else None 49 self.info =dict(info)if info else None 50 self.logger = logging.getLogger(__name__) 51 self.deprecated = deprecated 52 self.successor = successor 53 54defload(self): 55""" 56 A generic loading method to import a parser, only used for debugging 57 """ 58 logger.debug("Loading parser{0}from{1}".format(self.name,str(self.path)if self.path !=None else"defaults")) 59return importlib.import_module(self.name) 60 61defparse_log(self, **args) ->None: 62""" 63 Every parser should provide the parse_log method which is executed at 64 runtime to analyse logs. Verification checks should prevent the below 65 exception from ever being raised. 66 """ 67raiseNotImplementedError("Failed to find an entry point for parser") 68 69 70 71class ParserLoader: 72""" 73 This class searches for parsers in the main logparse package 74 (logparser.parsers) and optionally in another external package (default 75 /usr/share/logparse). 76 """ 77 78def__init__(self, pkg=PARSER_PKG, path=PARSER_DIR): 79""" 80 The pkg and path attributes shouldn't need to be set on object 81 creation, the default values should work fine. They are hard-coded here 82 for security so that a module can't force-load a module from another 83 package/location, e.g. from the internet. 84 """ 85 86 self.pkg = pkg 87 self.path = path 88 self.parsers = [] 89 90defsearch(self, pattern): 91""" 92 Basic wrapper for the two search functions below. 93 """ 94 95 default_parser = self._search_default(pattern) 96if default_parser !=None: 97 self.parsers.append(default_parser) 98return default_parser 99else: 100 user_parser = self._search_user(pattern) 101if user_parser !=None: 102 self.parsers.append(user_parser) 103return user_parser 104else: 105 logger.warning("Couldn't find a matching parser module for search term{0}".format(pattern)) 106return None 107 108def_search_user(self, pattern): 109""" 110 Search for a parser name `pattern` in the user-managed parser directory 111 """ 112 113 logger.debug("Searching for{0}in{1}".format(pattern, self.path)) 114try: 115 spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path]) 116 parser_module = spec.loader.load_module(spec.name) 117return self._validate_module(parser_module) 118exceptExceptionas e: 119return None 120 121def_search_default(self, pattern): 122""" 123 Search for a parser name `pattern` in the default parser package 124 """ 125 126# TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below 127 logger.debug("Searching for{0}in default parsers".format(pattern)) 128try: 129 parser_module = importlib.import_module(self.pkg +"."+ pattern) 130return self._validate_module(parser_module) 131exceptExceptionas e: 132return None 133 134def_validate_module(self, parser_module): 135""" 136 Some basic security tests for candidate modules: 137 1. Must contain exactly one Parser object 138 3. This class cannot be a redefinition of the base Parser class 139 4. Must provide the parse_log() method 140 5. Must not return None 141 6. Must not match an already-loaded class 142 """ 143 144 logger.debug("Checking validity of module{0}at{1}".format(parser_module.__name__, parser_module.__file__)) 145 available_parsers = [] 146 clsmembers = inspect.getmembers(parser_module, inspect.isclass) 147 148# Check individual classes 149for(_, c)in clsmembers: 150if not(issubclass(c, Parser) & (c is not Parser)): 151continue 152if c in self.parsers: 153 logger.warning("Parser class{0}has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__)) 154if not inspect.isroutine(c.parse_log): 155 logger.warning("Parser class{0}in{1}does not contain a parse_log() method".format(c.__class__.__name__, c.__file__)) 156continue 157if None inget_type_hints(c): 158 logger.warning("Parser class{0}in{1}contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__)) 159continue 160 parser_obj =c() 161if parser_obj.deprecated: 162 logger.warning("Parser{0}is deprecated - use{1}instead".format(parser_obj.name, parser_obj.successor)) 163 logger.debug("Found parser{0}.{1}".format(c.__module__, c.__class__.__name__)) 164 available_parsers.append(c()) 165 166# Check module structure 167iflen(available_parsers) ==1: 168 logger.debug("Parser module{0}at{1}passed validity checks".format(parser_module.__name__, parser_module.__file__)) 169return available_parsers[0] 170eliflen(available_parsers) ==0: 171 logger.warning("No valid classes in{0}at{1}".format(parser_module.__name__, parser_module.__file__)) 172return None 173eliflen(available_parsers) >1: 174 logger.warning("Found multiple valid parser classes in{0}at{1}- ignoring this module".format(parser_module.__name__, parser_module.__file__)) 175return None 176 177defload_pkg(self): 178""" 179 Clear the list of currently loaded packages and load all valid and 180 non-deprecated parser classes from self.pkg using importlib. 181 """ 182 183 available_parsers = [name for _, name, _ initer_modules([dirname(importlib.import_module(self.pkg).__file__)])] 184for parser_name in available_parsers: 185 parser_module = importlib.import_module("logparse.parsers."+ parser_name) 186 parser_class = self._validate_module(parser_module) 187if parser_class ==None: 188continue 189 parser_obj = parser_class 190if parser_obj.deprecated: 191 logger.debug("Ignoring parser{0}because it is deprecated".format(parser_class.__class__.__name__)) 192continue 193 self.parsers.append(parser_class) 194return self.parsers 195 196defignore(self, pattern): 197""" 198 Remove a parser from the list of currently loaded parsers 199 """ 200 201for parser in self.parsers: 202if parser.__module__== pattern: 203 self.parsers.remove(parser) 204 logger.debug("Ignoring parser{0}".format(parser.__name__)) 205return self.parsers