1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3""" 4A basic "plugin loader" implementation which searches for default packaged and 5user-supplied parser modules and verifies them so they can be executed by 6logparse.interface. The requirements for parser modules and classes are 7specified in the docstring of the Parser class. 8 9Classes in this module: 10 - `Parser`: Base class that every parser should inherit 11 - `ParserLoader`: Class used internally by `interface.py` to load parsers 12""" 13 14import importlib 15from os.path import dirname 16from pkgutil import iter_modules 17import inspect 18from pathlib import Path 19from typing import get_type_hints 20 21import logging 22logger = logging.getLogger(__name__) 23 24 25PARSER_DIR ="/usr/share/logparse/user-parsers" 26PARSER_PKG ="logparse.parsers" 27 28 29classParser(): 30""" 31 This is the base class that every parser should inherit. Parsers should 32 each exist in their own module and contain a Parser class whose name is the 33 same as the parser (e.g. `example.py` contains class `Example(Parser)`). 34 Each parser module must contain exactly one Parser class definition, and 35 this class cannot be a redefinition of the base Parser class (i.e. this 36 class). This class must provide the parse_log() method which returns a 37 logparse.formatting.Section object. 38 """ 39 40def__init__(self, name=None, path=None, info=None, deprecated=False): 41""" 42 The following variables can be set to display information about the 43 parser. The object `self.logger` can be used as for outputting messages 44 to whatever sink is set up in logparse.interface (no setup required in 45 the parser module itself). 46 """ 47 self.name =str(name)if name else None 48 self.path =Path(path)if path else None 49 self.info =dict(info)if info else None 50 self.logger = logging.getLogger(__name__) 51 self.deprecated = deprecated 52 53defload(self): 54""" 55 A generic loading method to import a parser, only used for debugging 56 """ 57 logger.debug("Loading parser{0}from{1}".format(self.name,str(self.path)if self.path !=None else"defaults")) 58return importlib.import_module(self.name) 59 60defparse_log(self, **args) ->None: 61""" 62 Every parser should provide the parse_log method which is executed at 63 runtime to analyse logs. Verification checks should prevent the below 64 exception from ever being raised. 65 """ 66raiseNotImplementedError("Failed to find an entry point for parser") 67 68 69 70class ParserLoader: 71""" 72 This class searches for parsers in the main logparse package 73 (logparser.parsers) and optionally in another external package (default 74 /usr/share/logparse). 75 """ 76 77def__init__(self, pkg=PARSER_PKG, path=PARSER_DIR): 78""" 79 The pkg and path attributes shouldn't need to be set on object 80 creation, the default values should work fine. They are hard-coded here 81 for security so that a module can't force-load a module from another 82 package/location, e.g. from the internet. 83 """ 84 85 self.pkg = pkg 86 self.path = path 87 self.parsers = [] 88 89defsearch(self, pattern): 90""" 91 Basic wrapper for the two search functions below. 92 """ 93 94 default_parser = self._search_default(pattern) 95if default_parser !=None: 96 self.parsers.append(default_parser) 97return default_parser 98else: 99 user_parser = self._search_user(pattern) 100if user_parser !=None: 101 self.parsers.append(user_parser) 102return user_parser 103else: 104 logger.warning("Couldn't find a matching parser module for search term{0}".format(pattern)) 105return None 106 107def_search_user(self, pattern): 108""" 109 Search for a parser name `pattern` in the user-managed parser directory 110 """ 111 112 logger.debug("Searching for{0}in{1}".format(pattern, self.path)) 113try: 114 spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path]) 115 parser_module = spec.loader.load_module(spec.name) 116return self._validate_module(parser_module) 117exceptExceptionas e: 118 logger.debug("Couldn't find parser{0}in{1}:{2}".format(pattern, self.path,str(e))) 119return None 120 121def_search_default(self, pattern): 122""" 123 Search for a parser name `pattern` in the default parser package 124 """ 125 126# TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below 127 logger.debug("Searching for{0}in default parsers".format(pattern)) 128try: 129 parser_module = importlib.import_module(self.pkg +"."+ pattern) 130return self._validate_module(parser_module) 131exceptExceptionas e: 132return None 133 134def_validate_module(self, parser_module): 135""" 136 Some basic security tests for candidate modules: 137 1. Must contain exactly one Parser object 138 3. This class cannot be a redefinition of the base Parser class 139 4. Must provide the parse_log() method 140 5. Must not return None 141 6. Must not match an already-loaded class 142 """ 143 144 logger.debug("Checking validity of module{0}at{1}".format(parser_module.__name__, parser_module.__file__)) 145 available_parsers = [] 146 clsmembers = inspect.getmembers(parser_module, inspect.isclass) 147 148# Check individual classes 149for(_, c)in clsmembers: 150if not(issubclass(c, Parser) & (c is not Parser)): 151continue 152if c in self.parsers: 153 logger.warning("Parser class{0}has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__)) 154if not inspect.isroutine(c.parse_log): 155 logger.warning("Parser class{0}in{1}does not contain a parse_log() method".format(c.__class__.__name__, c.__file__)) 156continue 157if None inget_type_hints(c): 158 logger.warning("Parser class{0}in{1}contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__)) 159continue 160 logger.debug("Found parser{0}.{1}".format(c.__module__, c.__class__.__name__)) 161 available_parsers.append(c()) 162 163# Check module structure 164iflen(available_parsers) ==1: 165 logger.debug("Parser module{0}at{1}passed validity checks".format(parser_module.__name__, parser_module.__file__)) 166return available_parsers[0] 167eliflen(available_parsers) ==0: 168 logger.warning("No valid classes in{0}at{1}".format(parser_module.__name__, parser_module.__file__)) 169return None 170eliflen(available_parsers) >1: 171 logger.warning("Found multiple valid parser classes in{0}at{1}- ignoring this module".format(parser_module.__name__, parser_module.__file__)) 172return None 173 174defload_pkg(self): 175""" 176 Clear the list of currently loaded packages and load all valid and 177 non-deprecated parser classes from self.pkg using importlib. 178 """ 179 180 available_parsers = [name for _, name, _ initer_modules([dirname(importlib.import_module(self.pkg).__file__)])] 181for parser_name in available_parsers: 182 parser_module = importlib.import_module("logparse.parsers."+ parser_name) 183 parser_class = self._validate_module(parser_module) 184if parser_class ==None: 185continue 186 parser_obj = parser_class 187if parser_obj.deprecated: 188 logger.debug("Ignoring parser{0}because it is deprecated".format(parser_class.__class__.__name__)) 189continue 190 self.parsers.append(parser_class) 191return self.parsers 192 193defignore(self, pattern): 194""" 195 Remove a parser from the list of currently loaded parsers 196 """ 197 198for parser in self.parsers: 199if parser.__module__== pattern: 200 self.parsers.remove(parser) 201 logger.debug("Ignoring parser{0}".format(parser.__name__)) 202return self.parsers