Andrew's git - logparse.git/blob - logparse/load

   1#
   2#   load_parsers.py
   3#   
   4#   Search for and load files which parse logs for particular services
   5#
   6
   7import imp
   8import importlib
   9import os
  10import glob
  11import pkgutil
  12import inspect
  13from pathlib import Path
  14from sys import path
  15from typing import NamedTuple
  16
  17parser_dir = "/usr/share/logparse/"
  18main_module = "__init__"
  19default_parsers = ["cron_journald", "httpd", "mem", "postfix", "smbd", "sshd_journald", "sudo", "sysinfo", "temperature", "zfs"]
  20deprecated_parsers = ["sshd", "cron"]
  21
  22import logging
  23logger = logging.getLogger(__name__)
  24
  25class Parser():
  26    """
  27    Base class that every parser should inherit
  28    """
  29    def __init__(self, name=None, path=None, info=None):
  30        self.name = str(name) if name else None
  31        self.path = Path(path) if path else None
  32        self.info = dict(info) if info else None
  33        self.logger = logging.getLogger(__name__)
  34
  35    def load(self):
  36        logger.debug("Loading parser {0} from {1}".format(self.name, str(self.path) if self.path != None else "defaults"))
  37        return importlib.import_module(self.name)
  38
  39    def parse_log(self, **args):
  40        """
  41        Every parser should provide the parse_log method which is executed at
  42        runtime to analyse logs.
  43        """
  44        raise NotImplementedError("Failed to find an entry point for parser " + self.name)
  45
  46class ParserLoader:
  47    """
  48    This class searches for parsers in the main logparse package and
  49    optionally in another external package (default /usr/share/logparse).
  50    """
  51
  52    def __init__(self, pkg):
  53        """
  54        Initiate search for parsers
  55        """
  56        self.pkg = pkg 
  57        self.parsers= []
  58        self.reload()
  59
  60
  61    def reload(self):
  62        """
  63        Reset parsers list and iterate through package modules
  64        """
  65        self.parsers= []
  66        self.seen_paths = []
  67        logger.debug("Looking for parsers in package {0}".format(str(self.pkg)))
  68        self.walk_package(self.pkg)
  69
  70    def walk_package(self, package):
  71        """
  72        Check package and subdirectories for loadable modules
  73        """
  74
  75        imported_package = __import__(package, fromlist=["null"]) # fromlist must be non-empty to load target module rather than parent package
  76
  77        for _, parser_name, ispkg in pkgutil.iter_modules(imported_package.__path__, imported_package.__name__ + '.'):
  78            if not ispkg:
  79                parser_module = __import__(parser_name, fromlist=["null"])
  80                clsmembers = inspect.getmembers(parser_module, inspect.isclass)
  81                for (_, c) in clsmembers:
  82                    # Ignore the base Parser class
  83                    if issubclass(c, Parser) & (c is not Parser):
  84                        logger.debug("Found parser {0}.{1}".format(c.__module__, c.__name__))
  85                        self.parsers.append(c())
  86
  87
  88        # Recurse subpackages
  89
  90        all_current_paths = []
  91        if isinstance(imported_package.__path__, str):
  92            all_current_paths.append(imported_package.__path__)
  93        else:
  94            all_current_paths.extend([x for x in imported_package.__path__])
  95
  96        for pkg_path in all_current_paths:
  97            if pkg_path not in self.seen_paths:
  98                self.seen_paths.append(pkg_path)
  99
 100                # Get subdirectories of package
 101                child_pkgs = [p for p in os.listdir(pkg_path) if os.path.isdir(os.path.join(pkg_path, p))]
 102
 103                # Walk through each subdirectory
 104                for child_pkg in child_pkgs:
 105                    self.walk_package(package + '.' + child_pkg)
 106
 107def findall():
 108    logger.debug("Searching for parsers in {0}".format(parser_dir))
 109    path.append(os.path.abspath(parser_dir))
 110    parsers = []
 111    parser_candidates = os.listdir(parser_dir)
 112    for parser_name in parser_candidates:
 113        location = os.path.join(parser_dir, parser_name)
 114        if not os.path.isdir(location) or not main_module + '.py' in os.listdir(location):
 115            logger.warning("Rejecting parser {0} due to invalid structure".format(location))
 116            continue
 117        info = imp.find_module(main_module, [location])
 118        parser_obj = Parser(parser_name, location, info)
 119        parsers.append(parser_obj)
 120        logger.debug("Added parser {0}".format(parser_obj.name))
 121    return parsers
 122
 123def search(name):
 124    logger.debug("Searching for parser {0}".format(name))
 125    if name in default_parsers:
 126        logger.debug("Found parser {0} in default modules".format(name))
 127        return Parser('.'.join(__name__.split('.')[:-1] + [name]))
 128    elif name in deprecated_parsers:
 129        logger.debug("Found parser {0} in deprecated modules".format(name))
 130        return Parser('.'.join(__name__.split('.')[:-1] + [name]))
 131    else:
 132        return None
 133
 134def load(parser):
 135    logger.debug("Loading parser {0} from {1}".format(parser.name, parser.path if parser.path != None else "defaults"))
 136    return importlib.import_module(parser.name)