From 66a8e81e6538712971a2229ba8de0074db4b21b5 Mon Sep 17 00:00:00 2001 From: Andrew Lorimer Date: Mon, 2 Sep 2019 19:50:00 +1000 Subject: [PATCH] migrate configuration system to the stdlib ConfigParser --- logparse/config.py | 209 ++++++++++++++---------------- logparse/formatting.py | 73 +++++++---- logparse/interface.py | 83 +++++++----- logparse/parsers/cron-journald.py | 2 +- logparse/parsers/cron.py | 13 +- logparse/parsers/httpd.py | 22 ++-- logparse/parsers/journaltest.py | 11 -- logparse/parsers/mem.py | 6 +- logparse/parsers/postfix.py | 10 +- logparse/parsers/smbd.py | 14 +- logparse/parsers/sshd-journald.py | 12 +- logparse/parsers/sshd.py | 20 +-- logparse/parsers/sudo.py | 14 +- logparse/parsers/sysinfo.py | 8 +- logparse/parsers/temperature.py | 17 ++- logparse/parsers/zfs.py | 8 +- logparse/util.py | 4 +- setup.py | 2 +- 18 files changed, 266 insertions(+), 262 deletions(-) delete mode 100644 logparse/parsers/journaltest.py diff --git a/logparse/config.py b/logparse/config.py index 8b7d798..53f332d 100644 --- a/logparse/config.py +++ b/logparse/config.py @@ -1,136 +1,115 @@ # # config.py # -# Default config values and basic wrapper for PyYaml. New config options +# Default config values and basic wrapper for ConfigParser. New config options # should be added to the dictionary below, along with appropriate defaults. # +# Runtime configuration is done through /etc/logparse/logparse.conf (default) +# or the path specified in the "--config" argument. The file uses the INI +# syntax, with general options being declared in the [logparse] section and +# parser-specific options declared in their own sections. +# -import yaml -import types -import os +from configparser import ConfigParser from pkg_resources import Requirement, resource_filename -from types import SimpleNamespace import logparse import logging logger = logging.getLogger(__name__) +global prefs +prefs = None + +defaults = { + 'logparse': { + 'output': '', + 'overwrite': False, + 'title': logparse.__name__, + 'maxlist': 10, + 'maxcmd': 6, + 'resolve-domains': 'fqdn', + 'rotate': False, + 'verbose': False, + 'quiet': False, + 'hostname-path': '/etc/hostname', + 'parsers': '', + 'ignore-parsers': '' + }, + 'html': { + 'header': '/etc/logparse/header.html', + 'css': '/etc/logparse/main.css', + 'embed-styles': False, + 'css-relpath': True + }, + 'plain': { + 'plain': False, + 'linewidth': 80 + }, + 'logs': { + 'auth': '/var/log/auth.log', + 'cron': '/var/log/cron.log', + 'cpuinfo': '/proc/cpuinfo', + 'meminfo': '/proc/meminfo', + 'sys': '/var/log/syslog', + 'smbd': '/var/log/samba', + 'zfs': '/var/log/zpool.log', + 'alloc': '/var/log/du.log', + 'postfix': '/var/log/mail.log', + 'httpd-access': '/var/log/apache2/access.log', + 'httpd-error': '/var/log/apache2/error.log' + }, + 'mail': { + 'to': '', + 'from': '', + 'subject': 'logparse from $hostname', + 'mailbin': '/usr/bin/mail' + }, + 'temperatures': { + 'drives': ['/dev/sda'], + 'host': '127.0.0.1', + 'separator': '|', + 'timeout': 10, + 'port': 7634, + 'show-model': False, + }, + 'sshd': { + 'resolve-domains': '' + }, + 'smbd': { + 'resolve-domains': '' + }, + 'httpd': { + 'resolve-domains': '' + }, + 'du': { + 'paths': ['/', '/etc', '/home'], + 'force-write': False + } +} + def locate(filename): + """ + DEPRECATED: draft method for what is now parsers/load_parsers.py. Kept here + for historical purposes. + """ logger.debug("Searching for {0}".format(filename)) loc = resource_filename(Requirement.parse(__package__), filename) logger.debug("Found {0}".format(loc)) return loc -class Configuration(dict): - - def __init__(self, *arg, **kw): - super(Configuration, self).__init__(*arg, **kw) - - def _str2bool(x): - positives = ["yes", "true", "1", "y"] - negatives = ["no", "false", "0", "n"] - x = x.lower() - if x in positives: - return True - elif x in negatives: - return False - else: - raise ValueError("Unknown option %s" % x) - -defaults = Configuration({ - 'output': '', - 'header': '/etc/logparse/header.html', - 'css': '/etc/logparse/main.css', - 'linewidth': 80, - 'embed-styles': False, - 'plain': False, - 'overwrite': False, - 'title': logparse.__name__, - 'maxlist': 10, - 'maxcmd': 6, - 'resolve-domains': 'fqdn', - 'mail': { - 'to': '', - 'from': '', - 'subject': 'logparse from $hostname$', - 'mailbin': '/usr/bin/mail', - }, - 'rotate': False, - 'verbose': False, - 'quiet': False, - 'hddtemp': { - 'drives': ['/dev/sda'], - 'host': '127.0.0.1', - 'separator': '|', - 'timeout': 10, - 'port': 7634, - 'show-model': False, - }, - 'apache': { - 'resolve-domains': '', - }, - 'sshd': { - 'resolve-domains': '', - }, - 'smbd': { - 'resolve-domains': '', - }, - 'httpd': { - 'resolve-domains': '', - }, - 'du': { - 'paths': ['/', '/etc', '/home'], - 'force-write': False, - }, - 'hostname-path': '/etc/hostname', - 'parsers': {}, - 'ignore-parsers': {}, - 'logs': { - 'auth': '/var/log/auth.log', - 'cron': '/var/log/cron.log', - 'cpuinfo': '/proc/cpuinfo', - 'meminfo': '/proc/meminfo', - 'sys': '/var/log/syslog', - 'smb': '/var/log/samba', - 'zfs': '/var/log/zpool.log', - 'alloc': '/var/log/du.log', - 'postfix': '/var/log/mail.log', - 'httpd': '/var/log/apache2' - } -}) - -def verify(raw_dict, defaults): - for key, value in raw_dict.items(): - if key in defaults: # valid key - logger.debug("Found valid key {0} with value {1}".format(key, value)) - if (isinstance(value, dict)): - verify(value, defaults[key]) # recurse nested dictionaries - - else: # invalid key - logger.warning("Invalid key {0} with value {1}".format(key, value)) - -def loadconf(argparser, configfile = "/etc/logparse/logparse.conf"): - logger.debug("Getting config from {0}".format(configfile)) +def loadconf(configpaths): + """ + Initial setup for a ConfigParser object. `configpaths` should be a list of + configuration files to load (typically only one). To use the generated + ConfigParser, use `import logparse.config` and then `config.prefs.get(..)`. + The prefs object is returned after creation as a convenience but this method + should only be called once per runtime. + """ + prefs= ConfigParser() + prefs.read_dict(defaults) try: - raw_dict = yaml.safe_load(open(configfile)) - # verify fields - verify(raw_dict, defaults) - prefs = defaults - for value in raw_dict: - if(isinstance(raw_dict[value], dict)): - for key in raw_dict[value].items(): - logger.debug("Inserting key {0} with value {1}".format(key[0], key[1])) - if not value in prefs: - prefs[value] = {} - prefs[value][key[0]] = key[1] - else: - prefs[value] = raw_dict[value] - if argparser.parse_args().to is not None: - prefs['mail']['to'] = argparser.parse_args().to - if not prefs['mail']['to']: - logger.info("No recipient address provided, outputting to stdout") - else: - logger.info("Email will be sent to " + prefs['mail']['to']) - return prefs + success = prefs.read(configpaths) + logger.debug("Loaded {0} config file(s): {1}".format(str(len(success)), str(success))) except Exception as e: logger.warning("Error processing config: " + str(e)) + return prefs diff --git a/logparse/formatting.py b/logparse/formatting.py index ab6266e..7b666d8 100644 --- a/logparse/formatting.py +++ b/logparse/formatting.py @@ -15,7 +15,7 @@ from math import floor, ceil from tabulate import tabulate import logparse -from . import interface, util, config, mail +from logparse import interface, util, mail, config import logging logger = logging.getLogger(__name__) @@ -36,6 +36,27 @@ JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼'] BULLET = "• " INDENT = " " +global VARSUBST + +def init_var(): + global VARSUBST + css_path = config.prefs.get("html", "css") + if config.prefs.getboolean("html", "css-relpath"): + if interface.argparser.parse_args().no_write: + css_path = os.path.relpath(css_path, ".") + elif interface.argparser.parse_args().destination: + css_path = os.path.relpath(css_path, interface.argparser.parse_args().destination()) + elif config.prefs.get("logparse", "output"): + css_path = os.path.relpath(css_path, config.prefs.get("logparse", "output")) + VARSUBST = { + "title": config.prefs.get("logparse", "title"), + "date": interface.start.strftime(DATEFMT), + "time": interface.start.strftime(TIMEFMT), + "hostname": util.hostname(config.prefs.get("logparse", "hostname-path")), + "version": logparse.__version__, + "css": css_path + } + class Output: """ @@ -65,6 +86,20 @@ class Output: f.write(self.content) logger.info("Written output to {}".format(destination)) + def print_stdout(self, lines=False): + """ + Echo the contents to the console + """ + print() + if lines: + line = PlaintextLine(linewidth=config.prefs.getint("plain", "linewidth"), double=True) + print(line.draw()) + print(self.content) + if lines: + print(line.draw()) + print() + + class PlaintextOutput(Output): """ @@ -80,8 +115,7 @@ class PlaintextOutput(Output): """ Print details with some primitive formatting """ - init_varfilter() - box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth']) + box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(VARSUBST), vpadding=2, hpadding="\t\t", linewidth=self.linewidth) line = PlaintextLine(self.linewidth) self.append(box.draw() + line.draw()) @@ -89,9 +123,8 @@ class PlaintextOutput(Output): """ Append a horizontal line and some details """ - init_varfilter() self.append(PlaintextLine(self.linewidth, vpadding=1).draw()) - self.append(Template("$hostname $time $date").safe_substitute(varsubst)) + self.append(Template("$hostname $time $date").safe_substitute(VARSUBST)) def append_section(self, section): """ @@ -128,13 +161,13 @@ class PlaintextOutput(Output): itemoutput = subtitle + '\n' for datum in data: datum = BULLET + datum - if len(datum) > config.prefs['linewidth'] - 3: + if len(datum) > self.linewidth - 3: words = datum.split() - if max(map(len, words)) > config.prefs['linewidth'] - len(INDENT): + if max(map(len, words)) > self.linewidth - len(INDENT): continue res, part, others = [], words[0], words[1:] for word in others: - if 1 + len(word) > config.prefs['linewidth'] - len(part): + if 1 + len(word) > self.linewidth - len(part): res.append(part) part = word else: @@ -156,21 +189,22 @@ class HtmlOutput(Output): self.content = "" self.destination = "" self.css = "" + self._embedded = "" def embed_css(self, css): """ Convert stylesheet to inline tags """ - self.content = mail.mailprep(self.content, css) - return self.content + if not self._embedded: + self._embedded = mail.mailprep(self.content, css) + return self._embedded def append_header(self, template): """ Insert variables into header template file and append HTML tags """ - init_varfilter() headercontent = Template(open(template, 'r').read()) - self.append(headercontent.safe_substitute(varsubst)) + self.append(headercontent.safe_substitute(VARSUBST)) self.append(opentag('div', id='main')) def append_footer(self): @@ -491,7 +525,7 @@ class PlaintextLine: def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""): self.linewidth = linewidth - self.double = False + self.double = double self.vpadding = vpadding self.hpadding = hpadding @@ -576,15 +610,6 @@ class PlaintextBox: return ('\n').join(contentlines) -def init_varfilter(): - global varfilter - global varpattern - global varsubst - varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))} - varfilter = dict((re.escape(k), v) for k, v in varfilter.items()) - varpattern = re.compile("|".join(varfilter.keys())) - varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))) - def backticks(l): return ["`" + x + "`" for x in l] @@ -616,11 +641,11 @@ def parsesize(num, suffix='B'): return "%.1f%s%s" % (num, 'Yi', suffix) -def fsubject(template): +def fsubject(subject): """ Replace variables in the title template provided in config """ - r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template) + r = Template(subject).safe_substitute(VARSUBST) logger.debug("Returning subject line " + r) return r diff --git a/logparse/interface.py b/logparse/interface.py index 583d6b6..ef8d7fc 100644 --- a/logparse/interface.py +++ b/logparse/interface.py @@ -14,10 +14,13 @@ from subprocess import check_output from datetime import datetime import logparse -from .config import * +import logparse.config +from logparse.config import prefs, loadconf from logparse import formatting, mail, config from .parsers import load_parsers +global argparser + def rotate(): # Rotate logs using systemd logrotate try: if not os.geteuid() == 0: @@ -38,7 +41,7 @@ def rotate_sim(): # Simulate log rotation if not os.geteuid() == 0: logger.warning("Cannot run logrotate as root - you will see permission errors in the output below") sim_cmd = "logrotate -d /etc/logrotate.conf" - logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd) + logger.debug("Here is the output of `{0}` (simulated):".format(sim_cmd)) sim = check_output(sim_cmd, shell=True) logger.debug(sim) except Exception as e: @@ -47,9 +50,10 @@ def rotate_sim(): # Simulate log rotation def main(): # Get arguments + global argparser argparser = argparse.ArgumentParser(description='grab logs of some common services and send them by email') argparser.add_argument('-t','--to', help='mail recipient (\"to\" address)', required=False) - argparser.add_argument('-c', '--config', help='path to config file', required=False) + argparser.add_argument('-c', '--config', help='path to config file', required=False, default="/etc/logparse/logparse.conf") argparser.add_argument('-p', '--print', help='print HTML to stdout', required=False, dest='printout', action='store_true', default=False) argparser.add_argument('-d', '--destination', help='file to output HTML', required=False) argparser.add_argument('-f', '--overwrite', help='force overwrite an existing output file', required=False, action='store_true', default=False) @@ -62,29 +66,29 @@ def main(): argparser.add_argument('-es', '--embed-styles', help='make CSS rules inline rather than linking the file', required=False, default=False, action='store_true') argparser.add_argument('-nh', '--plain', help='write/send plain text rather than HTML', required=False, default=False, action='store_true') argparser.add_argument('-q', '--quiet', help='no output to stdout', required=False, default=False, action='store_true') + argparser.add_argument('-nm', '--no-mail', help="do not send email (overrides config file)", required=False, default=False, action="store_true") + argparser.add_argument('-nw', '--no-write', help="do not write output file (overrides config file)", required=False, default=False, action="store_true") # Load config - if argparser.parse_args().config: - config.prefs = config.loadconf(argparser.parse_args().config, argparser) - else: - config.prefs = config.loadconf(argparser=argparser) - prefs = config.prefs + config.prefs = loadconf(argparser.parse_args().config) # Set up logging logger = logging.getLogger(__name__) loghandler = logging.handlers.SysLogHandler(address = '/dev/log') loghandler.setFormatter(logging.Formatter(fmt='logparse[' + str(os.getpid()) + ']: %(message)s')) loghandler.setLevel(logging.INFO) # don't spam syslog with debug messages - if argparser.parse_args().quiet or config.prefs['quiet']: + if argparser.parse_args().quiet or config.prefs.getboolean("logparse", "quiet"): logging.basicConfig(level=logging.CRITICAL) - elif argparser.parse_args().verbose or config.prefs['verbose']: + elif argparser.parse_args().verbose or config.prefs.getboolean("logparse", "verbose"): logging.basicConfig(level=logging.DEBUG) logger.debug("Verbose mode turned on") else: logging.basicConfig(level=logging.INFO) logger.addHandler(loghandler) - logger.debug("Finished loading config") + logger.debug([x for x in config.prefs.sections()]) + logger.debug(config.prefs.get("logparse", "output")) + logger.debug("Config test: " + config.prefs.get("logparse", "output")) # Time analysis global start @@ -94,21 +98,23 @@ def main(): # Write header - global output + formatting.init_var() + if argparser.parse_args().plain: - output = formatting.PlaintextOutput(linewidth=prefs['linewidth']) + output = formatting.PlaintextOutput(linewidth=config.prefs.getint("plain", "linewidth")) + output.append_header() else: output = formatting.HtmlOutput() + output.append_header(config.prefs.get("html", "header")) - output.append_header(prefs['header']) # Find parsers parser_providers = [] if argparser.parse_args().logs: log_src = argparser.parse_args().logs.split() - elif len(prefs['parsers']) > 0: - log_src = prefs['parsers'] + elif config.prefs.get("logparse", "parsers"): + log_src = config.prefs.get("logparse", "parsers").split() else: log_src = load_parsers.default_parsers @@ -120,11 +126,13 @@ def main(): else: parser_providers.append(load_parsers.load(parser)) - if argparser.parse_args().ignore_logs or len(prefs['ignore-parsers']) > 0: - if argparser.parse_args().ignore_logs: - ignore_src = argparser.parse_args().ignore_logs.split() - else: - ignore_src = prefs['ignore-parsers'] + if argparser.parse_args().ignore_logs: + ignore_src = argparser.parse_args().ignore_logs.split() + elif config.prefs.get("logparse", "ignore-parsers"): + ignore_src = config.prefs.get("logparse", "ignore-parsers").split() + else: + ignore_src = [] + if len(ignore_src) > 0: for parser_name in ignore_src: if parser_name in [x.__name__.rpartition('.')[2] for x in parser_providers]: logger.info("Ignoring default parser {0}".format(parser_name)) @@ -144,17 +152,15 @@ def main(): # Write HTML footer output.append_footer() - if argparser.parse_args().printout: - print(output) - if argparser.parse_args().destination or prefs['output']: + if (argparser.parse_args().destination or config.prefs.get("logparse", "output")) and not argparser.parse_args().no_write: if argparser.parse_args().destination: dest_path = argparser.parse_args().destination else: - dest_path = prefs['output'] + dest_path = config.prefs.get("logparse", "output") logger.debug("Outputting to {0}".format(dest_path)) - if (argparser.parse_args().embed_styles or prefs['embed-styles']) and not (argparser.parse_args().plain or prefs['plain']): - output.embed_css(prefs['css']) - if (not os.path.isfile(dest_path)) and not (argparser.parse_args().overwrite or config['overwrite']): + if (argparser.parse_args().embed_styles or config.prefs.getboolean("html", "embed-styles")) and not (argparser.parse_args().plain or config.prefs.getboolean("plain", "plain")): + output.embed_css(config.prefs.get("html", "css")) + if (not os.path.isfile(dest_path)) and not (argparser.parse_args().overwrite or config.prefs.getboolean("logparse", "overwrite")): output.write(dest_path) elif logging.root.level == logging.CRITICAL: pass @@ -165,17 +171,23 @@ def main(): else: logger.warning("No output written") - if argparser.parse_args().to or prefs['mail']['to']: - if argparser.parse_args().to: + if (str(argparser.parse_args().to) or str(config.prefs.get("mail", "to"))) and not argparser.parse_args().no_mail: + if str(argparser.parse_args().to): to = argparser.parse_args().to else: - to = prefs['mail']['to'] - mail.sendmail(mailbin=prefs['mail']['mailbin'], body=(output.embed_css(prefs['css']) if isinstance(output, formatting.HtmlOutput) else output.content), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject']), html=isinstance(output, formatting.HtmlOutput), sender=prefs['mail']['from']) + to = config.prefs.get("mail", "to") + mail.sendmail( + mailbin=config.prefs.get("mail", "mailbin"), + body=(output.embed_css(config.prefs.get("html", "css")) if isinstance(output, formatting.HtmlOutput) else output.content), + recipient=to, + subject=formatting.fsubject(config.prefs.get("mail", "subject")), + html=isinstance(output, formatting.HtmlOutput), + sender=config.prefs.get("mail", "from")) if not argparser.parse_args().no_rotate: - if argparser.parse_args().simulate or prefs['rotate'] == 's': + if argparser.parse_args().simulate or config.prefs.getboolean("logparse", "rotate"): rotate_sim() - elif prefs['rotate'] or argparser.parse_args().rotate: + elif config.prefs.getboolean("logparse", "rotate") or argparser.parse_args().rotate: rotate() else: logger.debug("User doesn't want to rotate logs") @@ -186,4 +198,7 @@ def main(): finish = datetime.now() logger.info("Finished parsing logs at {0} {1} (total time: {2})".format(finish.strftime(formatting.DATEFMT), finish.strftime(formatting.TIMEFMT), finish - start)) + if argparser.parse_args().printout: + output.print_stdout() + return diff --git a/logparse/parsers/cron-journald.py b/logparse/parsers/cron-journald.py index b085aea..63a0e95 100644 --- a/logparse/parsers/cron-journald.py +++ b/logparse/parsers/cron-journald.py @@ -54,7 +54,7 @@ def parse_log(): user_data.subtitle = plural("cron session", len(cmdlist)) + " for " + usr user_data.items = ("`{0}`".format(cmd) for cmd in cmdlist) user_data.orderbyfreq() - user_data.truncl(config.prefs['maxcmd']) + user_data.truncl(config.prefs.getint("logparse", "maxcmd")) section.append_data(user_data) logger.info("Finished cron section") diff --git a/logparse/parsers/cron.py b/logparse/parsers/cron.py index 4408ba2..83cb1cf 100644 --- a/logparse/parsers/cron.py +++ b/logparse/parsers/cron.py @@ -10,10 +10,9 @@ import re -from ..formatting import * -from ..util import readlog, resolve -from .. import config -from .. import util +from logparse.formatting import * +from logparse.util import readlog +from logparse import config import logging logger = logging.getLogger(__name__) @@ -25,13 +24,11 @@ def parse_log(): logger.debug("Starting cron section") section = Section("cron") - matches = re.findall('.*CMD\s*\(\s*(?!.*cd)(.*)\)', readlog(config.prefs['logs']['cron'])) + matches = re.findall('.*CMD\s*\(\s*(?!.*cd)(.*)\)', readlog(config.prefs.get("logs", "cron"))) num = len(matches) commands = [] for match in matches: commands.append(str(match)) - # commands.append([str(match)for match in matches]) - #logger.debug("found cron command " + str(commands)) logger.info("Found " + str(num) + " cron jobs") jobs_data = Data(str(num) + " cron jobs run") section.append_data(jobs_data) @@ -41,7 +38,7 @@ def parse_log(): cmd_data = Data("Top cron commands") cmd_data.items = ("`{0}`".format(x) for x in commands) cmd_data.orderbyfreq() - cmd_data.truncl(config.prefs['maxcmd']) + cmd_data.truncl(config.prefs.getint("logparse", "maxcmd")) section.append_data(cmd_data) logger.info("Finished cron section") diff --git a/logparse/parsers/httpd.py b/logparse/parsers/httpd.py index 518af7a..0abf6ed 100644 --- a/logparse/parsers/httpd.py +++ b/logparse/parsers/httpd.py @@ -10,9 +10,9 @@ import re -from ..formatting import * -from ..util import readlog, resolve -from .. import config +from logparse.formatting import * +from logparse.util import readlog, resolve +from logparse import config import logging logger = logging.getLogger(__name__) @@ -36,17 +36,13 @@ def parse_log(): logger.debug("Starting httpd section") section = Section("httpd") - accesslog = readlog(config.prefs['logs']['httpd'] + '/access.log') + accesslog = readlog(prefs("logs", "httpd-access")) - errorlog = readlog(config.prefs['logs']['httpd'] + '/error.log') + errorlog= readlog(prefs("logs", "httpd-error")) total_errors = len(errorlog.splitlines()) logger.debug("Retrieved log data") - errors = [] - notfound = [] - unprivileged = [] - logger.debug("Searching through access log") accesses = [] @@ -65,24 +61,24 @@ def parse_log(): section.append_data(size) clients = Data() - clients.items = [resolve(ac.client, "fqdn") for ac in accesses] + clients.items = [resolve(ac.client, config.prefs.get("httpd", "resolve-domains")) for ac in accesses] clients.orderbyfreq() clients.subtitle = "Received requests from " + plural("client", len(clients.items)) - clients.truncl(config.prefs['maxlist']) + clients.truncl(config.prefs.getint("logparse", "maxlist")) section.append_data(clients) files = Data() files.items = [ac.file for ac in accesses] files.orderbyfreq() files.subtitle = plural("file", len(files.items)) + " requested" - files.truncl(config.prefs['maxlist']) + files.truncl(config.prefs.getint("logparse", "maxlist")) section.append_data(files) useragents = Data() useragents.items = [ac.useragent for ac in accesses] useragents.orderbyfreq() useragents.subtitle = plural("user agent", len(useragents.items)) - useragents.truncl(config.prefs['maxlist']) + useragents.truncl(config.prefs.getint("logparse", "maxlist")) section.append_data(useragents) logger.info("httpd has received " + str(total_requests) + " requests with " + str(total_errors) + " errors") diff --git a/logparse/parsers/journaltest.py b/logparse/parsers/journaltest.py deleted file mode 100644 index 2d0016f..0000000 --- a/logparse/parsers/journaltest.py +++ /dev/null @@ -1,11 +0,0 @@ -from ..formatting import * -from .. import config - -import logging -logger = logging.getLogger(__name__) - -def parse_log(): - - parser = util.JournalParser() - parser.parse() - diff --git a/logparse/parsers/mem.py b/logparse/parsers/mem.py index 20baa85..82b2e50 100644 --- a/logparse/parsers/mem.py +++ b/logparse/parsers/mem.py @@ -6,8 +6,8 @@ import re -from ..formatting import * -from .. import config +from logparse.formatting import * +from logparse import config import logging logger = logging.getLogger(__name__) @@ -22,7 +22,7 @@ def parse_log(): ram_b = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') table.add_row(Row([Column("Installed"), Column(parsesize(ram_b))])) - raw_mem = util.readlog(config.prefs['logs']['meminfo']) + raw_mem = util.readlog(config.prefs.get("logs", "meminfo")) line_regex = re.compile("^Mem(\w+):\s*(\d*)\s*kB$") for line in raw_mem.splitlines(): diff --git a/logparse/parsers/postfix.py b/logparse/parsers/postfix.py index ea4bac9..1b7ff80 100644 --- a/logparse/parsers/postfix.py +++ b/logparse/parsers/postfix.py @@ -6,9 +6,9 @@ import re -from ..formatting import * -from ..util import readlog, resolve -from .. import config +from logparse.formatting import * +from logparse.util import readlog +from logparse import config import logging logger = logging.getLogger(__name__) @@ -17,7 +17,7 @@ def parse_log(): section = Section("postfix") logger.debug("Starting postfix section") logger.debug("Searching through postfix logs") - messages = re.findall('.*from\=<(.*)>, size\=(\d*),.*\n.*to=<(.*)>', readlog(config.prefs['logs']['postfix'])) + messages = re.findall('.*from\=<(.*)>, size\=(\d*),.*\n.*to=<(.*)>', readlog(config.prefs.get("logs", "postfix"))) r = [] s = [] size = 0 @@ -37,7 +37,7 @@ def parse_log(): if (len(s) > 1): rec_data.items = r rec_data.orderbyfreq() - rec_data.truncl(config.prefs['maxlist']) + rec_data.truncl(config.prefs.getint("logparse", "maxlist")) rec_data.subtitle = n + " messages sent to" else: rec_data.subtitle = n + " messages sent to " + r[0] diff --git a/logparse/parsers/smbd.py b/logparse/parsers/smbd.py index efe48cd..133f8da 100644 --- a/logparse/parsers/smbd.py +++ b/logparse/parsers/smbd.py @@ -8,9 +8,9 @@ import re import glob -from ..formatting import * -from ..util import readlog, resolve -from .. import config +from logparse.formatting import * +from logparse.util import readlog, resolve +from logparse import config import logging logger = logging.getLogger(__name__) @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) def parse_log(): logger.debug("Starting smbd section") section = Section("smbd") - files = glob.glob(config.prefs['logs']['smb'] + "/log.*[!\.gz][!\.old]") # find list of logfiles + files = glob.glob(config.prefs.get("logs", "smbd") + "/log.*[!\.gz][!\.old]") # find list of logfiles # for f in files: # file_mod_time = os.stat(f).st_mtime @@ -46,8 +46,8 @@ def parse_log(): # find the machine (ip or hostname) that this file represents ip = re.search('log\.(.*)', file).group(1) # get ip or hostname from file path (/var/log/samba/log.host) - host = resolve(ip, fqdn=config.prefs['smbd']['resolve-domains']) - if (host == ip and (config.prefs['smbd']['resolve-domains'] or config.prefs['resolve-domains']) != 'ip'): # if ip has disappeared, fall back to a hostname from logfile + host = resolve(ip, fqdn=config.prefs.get("smbd", "resolve-domains")) + if host == ip and (config.prefs.get("smbd", "resolve-domains") != "ip" or config.prefs.get("logparse", "resolve-domains") != "ip"): # if ip has disappeared, fall back to a hostname from logfile newhost = re.findall('.*\]\@\[(.*)\]', readlog(file)) if (len(set(newhost)) == 1): # all hosts in one file should be the same host = newhost[0].lower() @@ -70,7 +70,7 @@ def parse_log(): else: # multiple users auth_data.items = sigma_auths auth_data.orderbyfreq() - auth_data.truncl(config.prefs['maxlist']) + auth_data.truncl(config.prefs.getint("logparse", "maxlist")) logger.debug("Found {0} samba logins".format(str(n_auths))) section.append_data(auth_data) logger.info("Finished smbd section") diff --git a/logparse/parsers/sshd-journald.py b/logparse/parsers/sshd-journald.py index 8b58b0c..179fc2b 100644 --- a/logparse/parsers/sshd-journald.py +++ b/logparse/parsers/sshd-journald.py @@ -37,7 +37,7 @@ def parse_log(): user = entry.group(1) ip = entry.group(2) - userhost = user + '@' + resolve(ip, fqdn=config.prefs['sshd']['resolve-domains']) + userhost = user + '@' + resolve(ip, fqdn=config.prefs.get("sshd", "resolve-domains")) login_data.items.append(userhost) elif "Connection closed by authenticating user root" in msg: @@ -45,7 +45,7 @@ def parse_log(): user = entry.group(1) ip = entry.group(2) - userhost = user + '@' + resolve(ip, fqdn=config.prefs['sshd']['resolve-domains']) + userhost = user + '@' + resolve(ip, fqdn=config.prefs.get("sshd", "resolve-domains")) failed_data.items.append(userhost) elif "Invalid user" in msg: @@ -53,21 +53,21 @@ def parse_log(): user = entry.group(1) ip = entry.group(2) - userhost = user + '@' + resolve(ip, fqdn=config.prefs['sshd']['resolve-domains']) + userhost = user + '@' + resolve(ip, fqdn=config.prefs.get("sshd", "resolve-domains")) invalid_data.items.append(userhost) login_data.subtitle = plural("successful login", len(login_data.items)) + " from" login_data.orderbyfreq() - login_data.truncl(config.prefs['maxlist']) + login_data.truncl(config.prefs.getint("logparse", "maxlist")) invalid_data.subtitle = plural("attempted login", len(invalid_data.items)) invalid_data.orderbyfreq() invalid_data.subtitle += plural(" from invalid user", len(invalid_data.items), False) - invalid_data.truncl(config.prefs['maxlist']) + invalid_data.truncl(config.prefs.getint("logparse", "maxlist")) failed_data.subtitle = plural("failed login", len(failed_data.items)) + " from" failed_data.orderbyfreq() - failed_data.truncl(config.prefs['maxlist']) + failed_data.truncl(config.prefs.getint("logparse", "maxlist")) section.append_data(login_data) section.append_data(invalid_data) diff --git a/logparse/parsers/sshd.py b/logparse/parsers/sshd.py index 18b1799..8a7ed48 100644 --- a/logparse/parsers/sshd.py +++ b/logparse/parsers/sshd.py @@ -10,9 +10,9 @@ import re -from ..formatting import * -from ..util import readlog, resolve -from .. import config +from logparse.formatting import * +from logparse.util import readlog, resolve +from logparse import config import logging logger = logging.getLogger(__name__) @@ -23,12 +23,12 @@ def parse_log(): logger.debug("Starting sshd section") section = Section("ssh") - logger.debug("Searching for matches in {0}".format(config.prefs['logs']['auth'])) - matches = re.findall('.*sshd.*Accepted publickey for .* from .*', readlog(config.prefs['logs']['auth'])) # get all logins + logger.debug("Searching for matches in {0}".format(config.prefs.get("logs", "auth"))) + matches = re.findall('.*sshd.*Accepted publickey for .* from .*', readlog(config.prefs.get("logs", "auth"))) # get all logins logger.debug("Finished searching for logins") - logger.debug("Searching for matches in {0}".format(config.prefs['logs']['auth'])) - authlog = readlog(config.prefs['logs']['auth']) + logger.debug("Searching for matches in {0}".format(config.prefs.get("logs", "auth"))) + authlog = readlog(config.prefs.get("logs", "auth")) matches = re.findall('.*sshd.*Accepted publickey for .* from .*', authlog) # get all logins invalid_matches = re.findall(".*sshd.*Invalid user .* from .*", authlog) @@ -44,7 +44,7 @@ def parse_log(): user = entry.group(1) ip = entry.group(2) - userhost = user + '@' + resolve(ip, fqdn=config.prefs['sshd']['resolve-domains']) + userhost = user + '@' + resolve(ip, fqdn=config.prefs.get("sshd", "resolve-domains")) users.append(userhost) logger.debug("Parsed list of authorised users") @@ -54,7 +54,7 @@ def parse_log(): logger.debug("found " + str(len(matches)) + " ssh logins for user " + users[0]) auth_data.subtitle += ' ' + auth_data.items[0] auth_data.orderbyfreq() - auth_data.truncl(config.prefs['maxlist']) + auth_data.truncl(config.prefs.getint("logparse", "maxlist")) logger.debug("Found " + str(len(matches)) + " ssh logins for users " + str(data)) section.append_data(auth_data) @@ -76,7 +76,7 @@ def parse_log(): logger.debug("Found " + str(len(invalid_matches)) + " SSH login attempts for invalid user " + invalid_users[0]) invalid_data.subtitle += ' ' + invalid_data.items[0] invalid_data.orderbyfreq() - invalid_data.truncl(config.prefs['maxlist']) + invalid_data.truncl(config.prefs.get("logparse", "maxlist")) logger.debug("Found " + str(len(invalid_matches)) + " SSH login attempts for invalid users " + str(data)) section.append_data(invalid_data) diff --git a/logparse/parsers/sudo.py b/logparse/parsers/sudo.py index 87d4be7..77feaa6 100644 --- a/logparse/parsers/sudo.py +++ b/logparse/parsers/sudo.py @@ -6,9 +6,9 @@ import re -from ..formatting import * -from ..util import readlog, resolve -from .. import config +from logparse..formatting import * +from logparse.util import readlog +from logparse.config import prefs import logging logger = logging.getLogger(__name__) @@ -16,8 +16,8 @@ logger = logging.getLogger(__name__) def parse_log(): logger.debug("Starting sudo section") section = Section("sudo") - logger.debug("Searching for matches in {0}".format(config.prefs['logs']['auth'])) - umatches = re.findall('.*sudo:session\): session opened.*', readlog(config.prefs['logs']['auth'])) + logger.debug("Searching for matches in {0}".format(prefs.get("logs", "auth"))) + umatches = re.findall('.*sudo:session\): session opened.*', readlog(prefs.get("logs", "auth"))) num = sum(1 for line in umatches) # total number of sessions users = [] data = [] @@ -29,7 +29,7 @@ def parse_log(): else: users[exists[0]][1] += 1 commands = [] - cmatches = re.findall('sudo:.*COMMAND\=(.*)', readlog(config.prefs['logs']['auth'])) + cmatches = re.findall('sudo:.*COMMAND\=(.*)', readlog(prefs.get("logs", "auth"))) for cmd in cmatches: commands.append(cmd) logger.debug("Finished parsing sudo sessions") @@ -50,7 +50,7 @@ def parse_log(): commands = backticks(commands) command_data.items = commands command_data.orderbyfreq() - command_data.truncl(config.prefs['maxcmd']) + command_data.truncl(prefs.getint("logparse", "maxcmd")) section.append_data(command_data) logger.info("Finished sudo section") diff --git a/logparse/parsers/sysinfo.py b/logparse/parsers/sysinfo.py index 01a37c2..e0168b8 100644 --- a/logparse/parsers/sysinfo.py +++ b/logparse/parsers/sysinfo.py @@ -9,8 +9,8 @@ import subprocess import os import re -from ..formatting import * -from .. import config +from logparse.formatting import * +from logparse.config import prefs import logging logger = logging.getLogger(__name__) @@ -21,13 +21,13 @@ def parse_log(): section = Section("system") table = Table() - table.add_row(Row([Column("Hostname"), Column(util.hostname(config.prefs['hostname-path']))])) + table.add_row(Row([Column("Hostname"), Column(util.hostname(prefs.get("logparse", "hostname-path")))])) table.add_row(Row([Column("OS"), Column(platform.platform())])) table.add_row(Row([Column("OS version"), Column(platform.version())])) table.add_row(Row([Column("Platform"), Column(platform.system() + " " + platform.machine())])) processors = [] - raw_proc = util.readlog(config.prefs['logs']['cpuinfo']) + raw_proc = util.readlog(prefs.get("logs", "cpuinfo")) line_regex = re.compile(".*model name.*:\s*") proc_regex = re.compile("\s*(\(R\)|\(TM\)|CPU)") for line in raw_proc.splitlines(): diff --git a/logparse/parsers/temperature.py b/logparse/parsers/temperature.py index d1d4058..ebca1ac 100644 --- a/logparse/parsers/temperature.py +++ b/logparse/parsers/temperature.py @@ -18,10 +18,9 @@ import socket, sys from telnetlib import Telnet from typing import List, Dict, NamedTuple -from logparse import formatting -from ..formatting import * -from ..util import readlog, resolve -from ..config import * +from logparse.formatting import * +from logparse.util import readlog +from logparse import config import logging logger = logging.getLogger(__name__) @@ -124,14 +123,18 @@ def parse_log(): data = "" hddtemp_data = Data("Disks") - client = HddtempClient(host=config.prefs['hddtemp']['host'], port=int(config.prefs['hddtemp']['port']), sep=config.prefs['hddtemp']['separator'], timeout=int(config.prefs['hddtemp']['timeout'])) + client = HddtempClient( + host=config.prefs.get("temperatures", "host"), + port=config.prefs.getint("temperatures", "port"), + sep=config.prefs.get("temperatures", "separator"), + timeout=int(config.prefs.get("temperatures", "timeout"))) drives = client.get_drives() logger.debug("Received drive info: " + str(drives)) for drive in sorted(drives, key=lambda x: x.path): - if drive.path in config.prefs['hddtemp']['drives']: + if drive.path in config.prefs.get("temperatures", "drives").split(): sumtemp += drive.temperature - hddtemp_data.items.append(("{0} ({1})".format(drive.path, drive.model) if config.prefs['hddtemp']['show-model'] else drive.path) + ": {0}{1}{2}".format(drive.temperature, DEG, drive.units)) + hddtemp_data.items.append(("{0} ({1})".format(drive.path, drive.model) if config.prefs.getboolean("temperatures", "show-model") else drive.path) + ": {0}{1}{2}".format(drive.temperature, DEG, drive.units)) else: drives.remove(drive) logger.debug("Ignoring drive {0} ({1}) due to config".format(drive.path, drive.model)) diff --git a/logparse/parsers/zfs.py b/logparse/parsers/zfs.py index 5e805a2..75d414e 100644 --- a/logparse/parsers/zfs.py +++ b/logparse/parsers/zfs.py @@ -18,9 +18,9 @@ import re import sys, traceback -from ..formatting import * -from ..util import readlog, resolve -from .. import config +from formatting import * +from logparse.util import readlog +from logparse.config import prefs import logging logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def parse_log(): logger.debug("Starting zfs section") section = Section("zfs") - zfslog = readlog(config.prefs['logs']['zfs']) + zfslog = readlog(prefs.get("logs", "zfs")) logger.debug("Analysing zpool log") pool = re.search('.*---\n(\w*)', zfslog).group(1) diff --git a/logparse/util.py b/logparse/util.py index 2f30637..ae27eee 100644 --- a/logparse/util.py +++ b/logparse/util.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) from pkg_resources import Requirement, resource_filename -from . import config +from logparse.config import prefs def hostname(path): # get the hostname of current server hnfile = open(path, 'r') @@ -40,7 +40,7 @@ def resolve(ip, fqdn=None): # try to resolve an ip to hostname # resolve-domains defined in individual sections of the config take priority over global config if not fqdn: - fqdn = config.prefs['resolve-domains'] + fqdn = prefs.getboolean("logparse", "resolve-domains") if fqdn == 'ip': return(ip) diff --git a/setup.py b/setup.py index b439f41..7dd8d3e 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( keywords='logparse log parse analysis summary monitor email server', packages=['logparse', 'logparse.parsers'], python_requires='>=3', # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires - install_requires=['premailer', 'requests', 'pyyaml', 'tabulate'], # https://packaging.python.org/en/latest/requirements.html + install_requires=['premailer', 'requests', 'tabulate'], # https://packaging.python.org/en/latest/requirements.html data_files=[('/etc/logparse', ['logparse.conf', 'header.html', 'main.css'])], # installed to /etc/logparse project_urls={ 'Readme': 'https://git.lorimer.id.au/logparse.git/about', -- 2.43.2