1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3"""
4A basic "plugin loader" implementation which searches for default packaged and
5user-supplied parser modules and verifies them so they can be executed by
6logparse.interface. The requirements for parser modules and classes are
7specified in the docstring of the Parser class.
89
Classes in this module:
10- `Parser`: Base class that every parser should inherit
11- `ParserLoader`: Class used internally by `interface.py` to load parsers
12"""
1314
import importlib
15from os.path import dirname
16from pkgutil import iter_modules
17import inspect
18from pathlib import Path
19from typing import get_type_hints
2021
import logging
22logger = logging.getLogger(__name__)
2324
25
PARSER_DIR = "/usr/share/logparse/user-parsers"
26PARSER_PKG = "logparse.parsers"
2728
29
class Parser():
30"""
31This is the base class that every parser should inherit. Parsers should
32each exist in their own module and contain a Parser class whose name is the
33same as the parser (e.g. `example.py` contains class `Example(Parser)`).
34Each parser module must contain exactly one Parser class definition, and
35this class cannot be a redefinition of the base Parser class (i.e. this
36class). This class must provide the parse_log() method which returns a
37logparse.formatting.Section object.
38"""
3940
def __init__(self, name=None, path=None, info=None, deprecated=False, successor=""):
41"""
42The following variables can be set to display information about the
43parser. The object `self.logger` can be used as for outputting messages
44to whatever sink is set up in logparse.interface (no setup required in
45the parser module itself).
46"""
47self.name = str(name) if name else None
48self.path = Path(path) if path else None
49self.info = dict(info) if info else None
50self.logger = logging.getLogger(__name__)
51self.deprecated = deprecated
52self.successor = successor
5354
def load(self):
55"""
56A generic loading method to import a parser, only used for debugging
57"""
58logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults"))
59return importlib.import_module(self.name)
6061
def parse_log(self, **args) -> None:
62"""
63Every parser should provide the parse_log method which is executed at
64runtime to analyse logs. Verification checks should prevent the below
65exception from ever being raised.
66"""
67raise NotImplementedError("Failed to find an entry point for parser")
6869
70
71
class ParserLoader:
72"""
73This class searches for parsers in the main logparse package
74(logparser.parsers) and optionally in another external package (default
75/usr/share/logparse).
76"""
7778
def __init__(self, pkg=PARSER_PKG, path=PARSER_DIR):
79"""
80The pkg and path attributes shouldn't need to be set on object
81creation, the default values should work fine. They are hard-coded here
82for security so that a module can't force-load a module from another
83package/location, e.g. from the internet.
84"""
8586
self.pkg = pkg
87self.path = path
88self.parsers = []
8990
def search(self, pattern):
91"""
92Basic wrapper for the two search functions below.
93"""
9495
default_parser = self._search_default(pattern)
96if default_parser != None:
97self.parsers.append(default_parser)
98return default_parser
99else:
100user_parser = self._search_user(pattern)
101if user_parser != None:
102self.parsers.append(user_parser)
103return user_parser
104else:
105logger.warning("Couldn't find a matching parser module for search term {0}".format(pattern))
106return None
107108
def _search_user(self, pattern):
109"""
110Search for a parser name `pattern` in the user-managed parser directory
111"""
112113
logger.debug("Searching for {0} in {1}".format(pattern, self.path))
114try:
115spec = importlib.machinery.PathFinder.find_spec(pattern, path=[self.path])
116parser_module = spec.loader.load_module(spec.name)
117return self._validate_module(parser_module)
118except Exception as e:
119logger.debug("Couldn't find parser {0} in {1}".format(pattern, self.path))
120return None
121122
def _search_default(self, pattern):
123"""
124Search for a parser name `pattern` in the default parser package
125"""
126127
# TODO use importlib.resources.is_resources() once there is a backport to Python 3.6 or below
128logger.debug("Searching for {0} in default parsers".format(pattern))
129try:
130parser_module = importlib.import_module(self.pkg + "." + pattern)
131return self._validate_module(parser_module)
132except Exception as e:
133return None
134135
def _validate_module(self, parser_module):
136"""
137Some basic security tests for candidate modules:
1381. Must contain exactly one Parser object
1393. This class cannot be a redefinition of the base Parser class
1404. Must provide the parse_log() method
1415. Must not return None
1426. Must not match an already-loaded class
143"""
144145
logger.debug("Checking validity of module {0} at {1}".format(parser_module.__name__, parser_module.__file__))
146available_parsers = []
147clsmembers = inspect.getmembers(parser_module, inspect.isclass)
148149
# Check individual classes
150for (_, c) in clsmembers:
151if not (issubclass(c, Parser) & (c is not Parser)):
152continue
153if c in self.parsers:
154logger.warning("Parser class {0} has already been loaded from another source, ignoring it".format(c.__class__.__name__, c.__file__))
155if not inspect.isroutine(c.parse_log):
156logger.warning("Parser class {0} in {1} does not contain a parse_log() method".format(c.__class__.__name__, c.__file__))
157continue
158if None in get_type_hints(c):
159logger.warning("Parser class {0} in {1} contains a null-returning parse_log() method".format(c.__class__.__name__, c.__file__))
160continue
161parser_obj = c()
162if parser_obj.deprecated:
163logger.warning("Parser {0} is deprecated - use {1} instead".format(parser_obj.name, parser_obj.successor))
164logger.debug("Found parser {0}.{1}".format(c.__module__, c.__class__.__name__))
165available_parsers.append(c())
166167
# Check module structure
168if len(available_parsers) == 1:
169logger.debug("Parser module {0} at {1} passed validity checks".format(parser_module.__name__, parser_module.__file__))
170return available_parsers[0]
171elif len(available_parsers) == 0:
172logger.warning("No valid classes in {0} at {1}".format(parser_module.__name__, parser_module.__file__))
173return None
174elif len(available_parsers) > 1:
175logger.warning("Found multiple valid parser classes in {0} at {1} - ignoring this module".format(parser_module.__name__, parser_module.__file__))
176return None
177178
def load_pkg(self):
179"""
180Clear the list of currently loaded packages and load all valid and
181non-deprecated parser classes from self.pkg using importlib.
182"""
183184
available_parsers = [name for _, name, _ in iter_modules([dirname(importlib.import_module(self.pkg).__file__)])]
185for parser_name in available_parsers:
186parser_module = importlib.import_module("logparse.parsers." + parser_name)
187parser_class = self._validate_module(parser_module)
188if parser_class == None:
189continue
190parser_obj = parser_class
191if parser_obj.deprecated:
192logger.debug("Ignoring parser {0} because it is deprecated".format(parser_class.__class__.__name__))
193continue
194self.parsers.append(parser_class)
195return self.parsers
196197
def ignore(self, pattern):
198"""
199Remove a parser from the list of currently loaded parsers
200"""
201202
for parser in self.parsers:
203if parser.__module__ == pattern:
204self.parsers.remove(parser)
205logger.debug("Ignoring parser {0}".format(parser.__name__))
206return self.parsers