From d8d26ab0d070bc4704036a6ef0aaa844ad8bd270 Mon Sep 17 00:00:00 2001 From: Andrew Lorimer Date: Mon, 26 Aug 2019 21:43:39 +1000 Subject: [PATCH 1/1] finish initial plugin system, get email and other features working --- header.html | 10 +-- logparse/config.py | 6 +- logparse/formatting.py | 10 +-- logparse/interface.py | 123 ++++++++++++++++++++----------- logparse/mail.py | 9 ++- logparse/parsers/load_parsers.py | 43 ++++++++--- 6 files changed, 133 insertions(+), 68 deletions(-) diff --git a/header.html b/header.html index 53e4ea2..b5da1fe 100755 --- a/header.html +++ b/header.html @@ -1,14 +1,14 @@ - $title$ $version$ on $hostname$ ($date$) - - + $title $version on $hostname ($date) + + - - + +

$title$

$hostname$
$date$ $time$

$title

$hostname
$date $time

diff --git a/logparse/config.py b/logparse/config.py index 13d6d07..3e61050 100644 --- a/logparse/config.py +++ b/logparse/config.py @@ -38,9 +38,11 @@ class Configuration(dict): raise ValueError("Unknown option %s" % x) defaults = Configuration({ - 'output': '/var/www/logparse/summary.html', + 'output': '', 'header': '/etc/logparse/header.html', 'css': '/etc/logparse/main.css', + 'embed-styles': False, + 'overwrite': False, 'title': logparse.__name__, 'maxlist': 10, 'maxcmd': 3, @@ -78,6 +80,8 @@ defaults = Configuration({ 'force-write': 'n', }, 'hostname-path': '/etc/hostname', + 'parsers': {}, + 'ignore-parsers': {}, 'logs': { 'auth': '/var/log/auth.log', 'cron': '/var/log/cron.log', diff --git a/logparse/formatting.py b/logparse/formatting.py index b3ae5f0..aa4b71f 100644 --- a/logparse/formatting.py +++ b/logparse/formatting.py @@ -10,9 +10,8 @@ import os import re import locale +from string import Template -#import util -#import interface import logparse from . import interface, util, config @@ -29,9 +28,11 @@ DATEFMT = "%x" def init_varfilter(): global varfilter global varpattern + global varsubst varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))} varfilter = dict((re.escape(k), v) for k, v in varfilter.items()) varpattern = re.compile("|".join(varfilter.keys())) + varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))) def writetitle(title): # write title for a section if (title == '' or '\n' in title): @@ -72,9 +73,8 @@ def header(template): # return a parsed html header from file # except Exception as e: # logger.warning("could not copy main.css - " + str(e)) init_varfilter() - headercontent = open(template, 'r').read() - headercontent = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], headercontent) - return headercontent + headercontent = Template(open(template, 'r').read()) + return headercontent.safe_substitute(varsubst) def orderbyfreq(l): # order a list by the frequency of its elements and remove duplicates temp_l = l[:] diff --git a/logparse/interface.py b/logparse/interface.py index d93ca92..f2d5c7f 100644 --- a/logparse/interface.py +++ b/logparse/interface.py @@ -15,11 +15,11 @@ from datetime import datetime import logparse from .config import * from logparse import formatting, mail, config -from .parsers import load_parsers, sudo, sshd, cron, httpd, smbd, postfix, zfs, temperature +from .parsers import load_parsers def rotate(): # rotate logs using systemd logrotate - if parser.parse_args().function is None: + if argparser.parse_args().function is None: if (prefs['rotate'] == 'y'): subprocess.call("/usr/sbin/logrotate -f /etc/logrotate.conf", shell=True) logger.info("rotated logfiles") @@ -34,22 +34,24 @@ def rotate(): def main(): # Get arguments - parser = argparse.ArgumentParser(description='grab logs of some common services and send them by email') - parser.add_argument('-t','--to', help='mail recipient (\"to\" address)', required=False) - parser.add_argument('-c', '--config', help='path to config file', required=False) - parser.add_argument('-p', '--print', help='print HTML to stdout', required=False, dest='printout', action='store_true', default=False) - parser.add_argument('-d', '--destination', help='file to output HTML', required=False) - parser.add_argument('-f', '--overwrite', help='force overwrite an existing output file', required=False, action='store_true', default=False) - parser.add_argument('-v', '--verbose', help='verbose console/syslog output (for debugging)', required=False, default=False, action='store_true') - parser.add_argument('-r', '--rotate', help='force rotate log files using systemd logrotate', required=False, default=False, action='store_true') - parser.add_argument('-nr', '--no-rotate', help='do not rotate logfiles (overrides logparse.conf)', required=False, default=False, action='store_true') - parser.add_argument('-l', '--logs', help='services to analyse', required=False) + argparser = argparse.ArgumentParser(description='grab logs of some common services and send them by email') + argparser.add_argument('-t','--to', help='mail recipient (\"to\" address)', required=False) + argparser.add_argument('-c', '--config', help='path to config file', required=False) + argparser.add_argument('-p', '--print', help='print HTML to stdout', required=False, dest='printout', action='store_true', default=False) + argparser.add_argument('-d', '--destination', help='file to output HTML', required=False) + argparser.add_argument('-f', '--overwrite', help='force overwrite an existing output file', required=False, action='store_true', default=False) + argparser.add_argument('-v', '--verbose', help='verbose console/syslog output (for debugging)', required=False, default=False, action='store_true') + argparser.add_argument('-r', '--rotate', help='force rotate log files using systemd logrotate', required=False, default=False, action='store_true') + argparser.add_argument('-nr', '--no-rotate', help='do not rotate logfiles (overrides logparse.conf)', required=False, default=False, action='store_true') + argparser.add_argument('-l', '--logs', help='services to analyse', required=False) + argparser.add_argument('-nl', '--ignore-logs', help='skip these services (takes precedence over -l)', required=False) + argparser.add_argument('-es', '--embed-styles', help='make CSS rules inline rather than linking the file', required=False, default=False, action='store_true') # Load config - if parser.parse_args().config: - config.prefs = config.loadconf(parser.parse_args().config, parser) + if argparser.parse_args().config: + config.prefs = config.loadconf(argparser.parse_args().config, argparser) else: - config.prefs = config.loadconf(argparser=parser) + config.prefs = config.loadconf(argparser=argparser) prefs = config.prefs # Set up logging @@ -57,8 +59,7 @@ def main(): loghandler = logging.handlers.SysLogHandler(address = '/dev/log') loghandler.setFormatter(logging.Formatter(fmt='logparse.py[' + str(os.getpid()) + ']: %(message)s')) loghandler.setLevel(logging.WARNING) # don't spam syslog with debug messages - if parser.parse_args().verbose or (config.prefs['verbose'] == 'y' or config.prefs['verbose'] == 'yes'): - print("Verbose mode is on") + if argparser.parse_args().verbose or (config.prefs['verbose'] == 'y' or config.prefs['verbose'] == 'yes'): logging.basicConfig(level=logging.DEBUG) logger.debug("Verbose mode turned on") else: @@ -73,42 +74,80 @@ def main(): logger.info("Beginning log analysis at {0} {1}".format(start.strftime(formatting.DATEFMT), start.strftime(formatting.TIMEFMT))) logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, sys.version.replace('\n', ''))) -# for l in parser.parse_args().logs.split(' '): -# eval(l) -# sys.exit() + # Write HTML header - print(load_parsers.search()); - # Write HTML document global output_html output_html = formatting.header(prefs['header']) - output_html += sudo.parse_log() - output_html += sshd.parse_log() - output_html += cron.parse_log() - output_html += httpd.parse_log() - output_html += smbd.parse_log() - output_html += postfix.parse_log() - output_html += zfs.parse_log() - output_html += temperature.parse_log() - output_html += formatting.closetag('body') + formatting.closetag('html') - if parser.parse_args().printout: + output_html += formatting.opentag('div', id='main') + + # Find parsers + + parser_providers = [] + if argparser.parse_args().logs: + log_src = argparser.parse_args().logs.split() + elif len(prefs['parsers']) > 0: + log_src = prefs['parsers'] + else: + log_src = load_parsers.default_parsers + + for parser_name in log_src: + parser = load_parsers.search(parser_name) + if parser == None: + logger.warning("Can't find parser {0}".format(parser_name)) + continue + else: + parser_providers.append(load_parsers.load(parser)) + + if argparser.parse_args().ignore_logs or len(prefs['ignore-parsers']) > 0: + if argparser.parse_args().ignore_logs: + ignore_src = argparser.parse_args().ignore_logs.split() + else: + ignore_src = prefs['ignore-parsers'] + for parser_name in ignore_src: + if parser_name in [x.__name__.rpartition('.')[2] for x in parser_providers]: + logger.info("Ignoring default parser {0}".format(parser_name)) + parser_providers_new = [] + for p in parser_providers: + if p.__name__.rpartition('.')[2] != parser_name: + parser_providers_new.append(p) + parser_providers = parser_providers_new + continue + + # Execute parsers + + logger.debug(str(parser_providers)) + for parser in parser_providers: + output_html += parser.parse_log() + + # Write HTML footer + + output_html += formatting.closetag('div') + formatting.closetag('body') + formatting.closetag('html') + + if argparser.parse_args().printout: print(output_html) - if parser.parse_args().destination: - logger.debug("Outputting to {0}".format(parser.parse_args().destination)) - if not os.path.isfile(parser.parse_args().destination) and not parser.parse_args().overwrite: - with open(parser.parse_args().destination, 'w') as f: + if argparser.parse_args().destination or prefs['output']: + if argparser.parse_args().destination: + dest_path = argparser.parse_args().destination + else: + dest_path = prefs['output'] + logger.debug("Outputting to {0}".format(dest_path)) + if argparser.parse_args().embed_styles or prefs['embed-styles']: + output_html = mail.mailprep(output_html, prefs['css']) + if not os.path.isfile(dest_path) and not (argparser.parse_args().overwrite or config['overwrite']): + with open(dest_path, 'w') as f: f.write(output_html) - logger.info("Written output to {}".format(parser.parse_args().destination)) + logger.info("Written output to {}".format(dest_path)) else: logger.warning("Destination file already exists") - if input("Would you like to overwrite {0}? (y/n) [n] ".format(parser.parse_args().destination)) == 'y': - with open(parser.parse_args().destination, 'w') as f: + if input("Would you like to overwrite {0}? (y/n) [n] ".format(dest_path)) == 'y': + with open(dest_path, 'w') as f: f.write(output_html) - logger.debug("Written output to {}".format(parser.parse_args().destination)) + logger.debug("Written output to {}".format(dest_path)) else: logger.warning("No output written") - if parser.parse_args().to: - mail.sendmail(mailbin=prefs['mail']['mailbin'], body=output_html, recipient=parser.parse_args().to, subject="logparse test") + if argparser.parse_args().to: + mail.sendmail(mailbin=prefs['mail']['mailbin'], body=mail.mailprep(output_html, prefs['css']), recipient=argparser.parse_args().to, subject=formatting.fsubject(config.prefs['mail']['subject'])) # Print end message finish = datetime.now() diff --git a/logparse/mail.py b/logparse/mail.py index 9e840a4..ecd7afc 100644 --- a/logparse/mail.py +++ b/logparse/mail.py @@ -9,7 +9,7 @@ import logging logger = logging.getLogger(__name__) from os.path import isfile -from premailer import transform +import premailer from email.mime.text import MIMEText import subprocess @@ -26,17 +26,18 @@ def mailprep(htmlin, stylesheet): def sendmail(mailbin, body, recipient, subject, *sender): logger.debug("Sending email") - msg = MIMEText(body) + msg = MIMEText(body, 'html') if sender: msg["From"] = sender msg["To"] = recipient + msg["Content-type"] = "text/html: te: text/html" msg["Subject"] = subject - mailproc = subprocess.Popen([mailbin, '--debug-level=10', '-t'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + mailproc = subprocess.Popen([mailbin, "--debug-level=" + str(10 if logging.root.level == logging.DEBUG else 0), "-t"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) logger.debug("Compiled message and opened process") try: stdout = mailproc.communicate(msg.as_bytes(), timeout=15) logger.debug("sendmail output: {}".format(stdout)) - logger.info("Sent email") + logger.info("Sent email to {0}".format(recipient)) return 0 # except TimeoutExpired: # mailproc.kill() diff --git a/logparse/parsers/load_parsers.py b/logparse/parsers/load_parsers.py index f282110..e2e6ab4 100644 --- a/logparse/parsers/load_parsers.py +++ b/logparse/parsers/load_parsers.py @@ -5,29 +5,50 @@ # import imp +import importlib import os import glob +from pathlib import Path +from sys import path +from typing import NamedTuple parser_dir = "/usr/share/logparse/" -main_module = "__init__.py" +main_module = "__init__" +default_parsers = ["cron", "httpd", "postfix", "smbd", "sshd", "sudo", "temperature", "zfs"] import logging logger = logging.getLogger(__name__) -def search(): +class Parser(): + def __init__(self, name, path=None, info=None): + self.name = str(name) + self.path = Path(path) if path else None + self.info = dict(info) if info else None + +def findall(): logger.debug("Searching for parsers in {0}".format(parser_dir)) + path.append(os.path.abspath(parser_dir)) parsers = [] - parser_candidates = glob.glob(os.path.join(os.path.dirname(parser_dir), "*")) - logger.debug("Found parser candidates {0}".format(str(parser_candidates))) - for p in parser_candidates: - location = os.path.join(parser_dir, p) - if not os.path.isdir(parser_dir) or not main_module + '.py' in os.listdir(location): + parser_candidates = os.listdir(parser_dir) + for parser_name in parser_candidates: + location = os.path.join(parser_dir, parser_name) + if not os.path.isdir(location) or not main_module + '.py' in os.listdir(location): + logger.warning("Rejecting parser {0} due to invalid structure".format(location)) continue info = imp.find_module(main_module, [location]) - parsers.append({"name": p, "info": info}) + parser_obj = Parser(parser_name, location, info) + parsers.append(parser_obj) + logger.debug("Added parser {0}".format(parser_obj.name)) return parsers -def load(parser): - logger.debug("Loading {0}".format(parser["name"])) - return imp.load_module(parser["name"], *parser["info"]) +def search(name): + logger.debug("Searching for parser {0}".format(name)) + if name in default_parsers: + logger.debug("Found parser {0} in default modules".format(name)) + return Parser('.'.join(__name__.split('.')[:-1] + [name])) + else: + return None +def load(parser): + logger.debug("Loading parser {0} from {1}".format(parser.name, parser.path if parser.path != None else "defaults")) + return importlib.import_module(parser.name) -- 2.47.0