$title$ $version$ on $hostname$ ($date$)

From: Andrew Lorimer Date: Sat, 18 Aug 2018 13:08:10 +0000 (+1000) Subject: bugfixing & config file X-Git-Url: https://git.lorimer.id.au/logparse.git/diff_plain/76fdd81d1361022c73f2aa24f51214c83efd035b bugfixing & config file --- diff --git a/header.html b/header.html index 0ca9aaa..9480535 100755 --- a/header.html +++ b/header.html @@ -3,6 +3,7 @@ $title$ $version$ on $hostname$ ($date$) + diff --git a/logparse.py b/logparse.py index 02dbfeb..0ec21b5 100755 --- a/logparse.py +++ b/logparse.py @@ -3,39 +3,68 @@ import argparse, logging, os, shutil, re, subprocess, sys, requests, glob, socket, sensors, datetime, time, operator, premailer from sys import stdin from collections import namedtuple, defaultdict +from shutil import copyfile +import yaml + +reload(sys) +sys.setdefaultencoding('utf-8') + +scriptdir = os.path.dirname(os.path.realpath(__file__)) + diskstat = namedtuple('diskstat', ['cap', 'alloc', 'free', 'ratio']) drivetemp = namedtuple('drivetemp', ['name', 'temp', 'units']) +config = { + 'output': '~/var/www/logparse/summary.html', + 'header': scriptdir + '/header.html', + 'css': scriptdir + '/main.css', + 'title': 'logparse', + 'maxlist': 10, + 'maxcmd': 3, + 'mail': { + 'to': '', + 'from': '', + 'subject': 'logparse from $hostname$' + }, + 'hddtemp': { + 'drives': ['/dev/sda'], + 'port': 7634 + }, + 'du-paths': ['/', '/etc', '/home'], + 'hostname-path': '/etc/hostname', + 'logs': { + 'auth': '/var/log/auth.log', + 'cron': '/var/log/cron.log', + 'sys': '/var/log/syslog', + 'smb': '/var/log/samba', + 'zfs': '/var/log/zpool.log', + 'alloc': '/tmp/alloc', + 'postfix': '/var/log/mail.log', + 'httpd': '/var/log/apache2' + } +} -AUTHPATH = "/var/log/auth.log" -CRONPATH = "/var/log/cron.log" -SYSPATH = "/var/log/syslog" -SMBDDIR = "/var/log/samba" -ZFSPATH = "/var/log/zpool.log" -ALLOCPATH = "/tmp/alloc" -POSTFIXPATH = "/var/log/mail.log" HTTPDSTATUS = "http://localhost/server-status" -HTTPDDIR = "/var/log/apache2" -HOSTNAMEPATH = "/etc/hostname" -DUPATHS = ["/home/andrew", "/mnt/andrew"] -HDDTEMPS = ["/dev/sda", "/dev/sdc", "/dev/sdd", "/dev/sde"] -HDDTEMPPORT = 7634 -SUMMARYPATH = "/mnt/andrew/temp/logparse-test.html" -OUTPUTPATH = "/mnt/andrew/temp/logparse-test2.html" -MAILPATH = "/mnt/andrew/temp/log-parse-test-3.html" -HEADERPATH = os.path.dirname(os.path.realpath(__file__)) + "/header.html" -STYLEPATH = os.path.dirname(os.path.realpath(__file__)) + "/main.css" +# config['du-paths'] = ["/home/andrew", "/mnt/andrew"] +# config['hddtemp']['drives'] = ["/dev/sda", "/dev/sdc", "/dev/sdd", "/dev/sde"] +# config['hddtemp']['port'] = 7634 +# config['output'] = "/mnt/andrew/temp/logparse/summary.html" +# config['output'] = "/mnt/andrew/temp/logparse/out.html" +MAILPATH = "/mnt/andrew/temp/logparse/mail.html" +# config['dest'] = "/mnt/andrew/temp/logparse" +# config['header'] = os.path.dirname(os.path.realpath(__file__)) + "/header.html" +# config['css'] = os.path.dirname(os.path.realpath(__file__)) + "/main.css" MAILOUT = "" HTMLOUT = "" TXTOUT = "" -TITLE = "logparse" -MAXLIST = 10 -CMDNO = 3 -MAILSUBJECT = "logparse from $hostname$" +# config['title'] = "logparse" +# config['maxlist'] = 10 +# config['maxcmd'] = 3 +# config['mail']['subject'] = "logparse from $hostname$" VERSION = "v0.1" -# DEG = u'\N{DEGREE SIGN}'.encode('utf-8') -DEG = 'C' +DEG = u'\N{DEGREE SIGN}'.encode('utf-8') +DEG = " °C".encode('unicode_escape') # Set up logging logging.basicConfig(level=logging.DEBUG) @@ -53,9 +82,11 @@ def __main__(): else: logger.info("email will be sent to " + to) + loadconf(scriptdir + "/logparse.yaml") + global tempfile - tempfile = open(SUMMARYPATH, 'w+') - tempfile.write(header(HEADERPATH)) + tempfile = open(config['output'], 'w+') + tempfile.write(header(config['header'])) opentag('div', 1, 'main') sshd() sudo() @@ -70,10 +101,10 @@ def __main__(): for tag in ['div', 'body', 'html']: closetag(tag, 1) tempfile.close() - mailprep(SUMMARYPATH, MAILPATH) + mailprep(config['output'], MAILPATH) if (to != None): logger.debug("sending email") - ms = subject(MAILSUBJECT) + ms = subject(config['mail']['subject']) cmd = "cat " + MAILPATH + " | mail --debug-level=10 -a 'Content-type: text/html' -s '" + ms + "' " + to logger.debug(cmd) subprocess.call(cmd, shell=True) @@ -92,7 +123,7 @@ def writedata(subtitle, data = None): # write title and data to tempfile loggger.warning("no subtitle provided.. skipping section") return - if (data == None): + if (data == None or len(data) == 0): logger.debug("no data provided.. just printing subtitle") tag('p', 0, subtitle) else: @@ -132,6 +163,11 @@ def tag(tag, block = 0, content = ""): # write html opening tag, content, and h closetag(tag, block) def header(template): # return a parsed html header from file + try: + copyfile(config['css'], config['dest'] + '/' + os.path.basename(config['css'])) + logger.debug("copied main.css") + except Exception as e: + logger.warning("could not copy main.css - " + str(e)) headercontent = open(template, 'r').read() headercontent = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], headercontent) return headercontent @@ -142,7 +178,7 @@ def subject(template): return r def hostname(): # get the hostname - hnfile = open(HOSTNAMEPATH, 'r') + hnfile = open(config['hostname-path'], 'r') hn = re.search('^(.*)\n*', hnfile.read()).group(1) return hn @@ -222,13 +258,13 @@ def truncl(input, limit): # truncate list else: return(input) -def mailprep(inputpath, outputpath, *stylesheet): +def mailprep(inputpath, output, *stylesheet): logger.debug("converting stylesheet to inline tags") old = readlog(inputpath) - pm = premailer.Premailer(old, external_styles=STYLEPATH) + pm = premailer.Premailer(old, external_styles=config['css']) MAILOUT = pm.transform() logger.info("converted stylesheet to inline tags") - file = open(outputpath, 'w') + file = open(output, 'w') file.write(MAILOUT) file.close() logger.info("written to temporary mail file") @@ -268,8 +304,8 @@ def sshd(): else: for user in users: data.append(user[0] + ' (' + str(user[1]) + ')') - if len(data) > MAXLIST: # if there are lots of users, truncate them - data.append('+ ' + str(len(users) - MAXLIST - 1) + " more") + if len(data) > config['maxlist']: # if there are lots of users, truncate them + data.append('+ ' + str(len(users) - config['maxlist'] - 1) + " more") break logger.debug("found " + str(len(matches)) + " ssh logins for users " + str(data)) writedata(subtitle, data) @@ -314,7 +350,7 @@ def sudo(): if (len(commands) > 0): commands = addtag(commands, 'code') commands = orderbyfreq(commands) - commands = truncl(commands, CMDNO) + commands = truncl(commands, config['maxcmd']) writedata("top sudo commands", [c for c in commands]) closetag('div', 1) logger.info("finished sudo section") @@ -340,7 +376,7 @@ def cron(): if (matches > 0): commands = addtag(commands, 'code') commands = orderbyfreq(commands) - commands = truncl(commands, CMDNO) + commands = truncl(commands, config['maxcmd']) writedata("top cron commands", [c for c in commands]) closetag('div', 1) logger.info("finished cron section") @@ -367,8 +403,8 @@ def nameget(): logger.debug("the following downloads succeeded: " + str(l_f)) logger.debug("found " + str(n_s) + " successful downloads, and " + str(n_f) + " failed attempts") writetitle("nameget") - writedata(str(n_s) + " succeeded", truncl(orderbyfreq(l_s), CMDNO)) - writedata(str(n_f) + " failed", truncl(orderbyfreq(l_f), CMDNO)) + writedata(str(n_s) + " succeeded", truncl(l_s, config['maxcmd'])) + writedata(str(n_f) + " failed", truncl(l_f, config['maxcmd'])) closetag('div', 1) logger.info("finished nameget section") @@ -414,22 +450,22 @@ def httpd(): logger.debug("found the following requests: " + str(files)) files = addtag(files, 'code') files = orderbyfreq(files) - files = truncl(files, CMDNO) + files = truncl(files, config['maxcmd']) writedata(str(a) + " requests", files) if (ips != None): logger.debug("found the following ips: " + str(ips)) ips = addtag(ips, 'code') ips = orderbyfreq(ips) n_ip = str(len(ips)) - ips = truncl(ips, CMDNO) - writedata(n_ip + " unique clients", ips) + ips = truncl(ips, config['maxcmd']) + writedata(n_ip + " clients", ips) if (useragents != None): logger.debug("found the following useragents: " + str(useragents)) useragents = addtag(useragents, 'code') useragents = orderbyfreq(useragents) n_ua = str(len(useragents)) - useragents = truncl(useragents, CMDNO) - writedata(n_ua + " unique devices", useragents) + useragents = truncl(useragents, config['maxcmd']) + writedata(n_ua + " devices", useragents) writedata(data_h + " transferred") writedata(str(e) + " errors") @@ -463,7 +499,7 @@ def httpdsession(): def smbd(): logger.debug("starting smbd section") opentag('div', 1, 'smbd', 'section') - files = glob.glob(SMBDDIR + "/log.*[!\.gz][!\.old]") # find list of logfiles + files = glob.glob(config['logs']['smb'] + "/log.*[!\.gz][!\.old]") # find list of logfiles logger.debug("found log files " + str(files)) n_auths = 0 # total number of logins from all users sigma_auths = [] # contains users @@ -495,7 +531,7 @@ def smbd(): writedata(subtitle) else: # multiple users sigma_auths = orderbyfreq(sigma_auths) - sigma_auths = truncl(sigma_auths, CMDNO) + sigma_auths = truncl(sigma_auths, config['maxcmd']) logger.debug("found " + str(n_auths) + " samba logins for users " + str(sigma_auths)) writedata(subtitle, sigma_auths) closetag('div', 1) @@ -525,7 +561,7 @@ def postfix(): s = list(set(r)) # unique recipients if (len(s) > 1): r = orderbyfreq(r) - r = truncl(r, CMDNO) + r = truncl(r, config['maxcmd']) writedata(n + " messages sent to", r) else: writedata(n + " messages sent to " + r[0]) @@ -603,29 +639,29 @@ def temp(): # For this to work, `hddtemp` must be running in daemon mode. # Start it like this (bash): sudo hddtemp -d /dev/sda /dev/sdX... s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(('localhost',HDDTEMPPORT)) + s.connect(('localhost',config['hddtemp']['port'])) output = s.recv(4096) output += s.recv(4096) s.close() - hddtemps = [] - for drive in re.split('\|{2}', output): + config['hddtemp']['drives'] = [] + for drive in re.split('\|1}', output): try: fields = re.search('\|*(/dev/sd.)\|.*\|(\d+)\|(.)', drive) name = fields.group(1) temp = float(fields.group(2)) units = fields.group(3) - hddtemps.append(drivetemp(name, temp, units)) + config['hddtemp']['drives'].append(drivetemp(name, temp, DEG)) except: pass hddtotal = 0 data = [] - for drive in hddtemps: + for drive in config['hddtemp']['drives']: data.append(drive.name + ': ' + str(drive.temp) + drive.units) logger.debug("found disk " + drive.name + " at " + str(drive.temp)) hddtotal += drive.temp - logger.debug("found " + str(len(hddtemps)) + " disks") + logger.debug("found " + str(len(config['hddtemp']['drives'])) + " disks") logger.debug("sum of disk temps is " + str(hddtotal)) - hddavg = hddtotal/float(len(hddtemps)) + hddavg = "{0:.2f}".format(hddtotal/float(len(config['hddtemp']['drives']))) + DEG logger.debug("avg disk temp is " + str(hddavg)) data.append("avg: " + str(hddavg)) writetitle("temperatures") @@ -633,7 +669,7 @@ def temp(): writedata("sys: " + str(systemp) + DEG) if (coretemps != ''): writedata("cores", coretemps) - if (hddtemps != ''): + if (config['hddtemp']['drives'] != ''): writedata("disks", data) closetag('div', 1) @@ -649,7 +685,7 @@ def du(): out = [] content = readlog('alloc') contentnew = "" - for p in DUPATHS: + for p in config['du-paths']: alloc_f = getusage(p).alloc delta = None try: @@ -680,13 +716,30 @@ def du(): timenow = time.strftime("%H:%M:%S") datenow = time.strftime("%x") -pathfilter = {"auth": AUTHPATH, "cron": CRONPATH, "sys": SYSPATH, "postfix": POSTFIXPATH, "smb": SMBDDIR, "zfs": ZFSPATH, "alloc": ALLOCPATH, "httpd": HTTPDDIR, "header": HEADERPATH} +pathfilter = {"auth": config['logs']['auth'], "cron": config['logs']['cron'], "sys": config['logs']['sys'], "postfix": config['logs']['postfix'], "smb": config['logs']['smb'], "zfs": config['logs']['zfs'], "alloc": config['logs']['alloc'], "httpd": config['logs']['httpd'], "header": config['header']} pathfilter = dict((re.escape(k), v) for k, v in pathfilter.iteritems()) pathpattern = re.compile("|".join(pathfilter.keys())) -varfilter = {"$title$": TITLE, "$date$": datenow, "$time$": timenow, "$hostname$": hostname(), "$version$": VERSION} +varfilter = {"$title$": config['title'], "$date$": datenow, "$time$": timenow, "$hostname$": hostname(), "$version$": VERSION, "$css$": os.path.basename(config['css'])} varfilter = dict((re.escape(k), v) for k, v in varfilter.iteritems()) varpattern = re.compile("|".join(varfilter.keys())) - -__main__() +def loadconf(configfile): + try: + data = yaml.safe_load(open(configfile)) + for value in data: + if (type(value) == dict): + config[value][key] = (data[value][key] for key in value) + else: + config[value] = data[value] + config['dest'] = os.path.dirname(config['output']) + logger.debug(str(config)) + except Exception as e: + logger.warning("error processing config: " + str(e)) + + +try: + __main__() +finally: + subprocess.call("logrotate -f /etc/logrotate.conf", shell=True) + logger.info("rotated logfiles") diff --git a/logparse.yaml b/logparse.yaml new file mode 100644 index 0000000..ee81a36 --- /dev/null +++ b/logparse.yaml @@ -0,0 +1,14 @@ + +output: /mnt/andrew/temp/logparse/summary.html +mail: + to: andrew@lorimer.id.au +hddtemp: + drives: + - /dev/sda + - /dev/sdc + - /dev/sdd + - /dev/sde + port: 7634 +du-paths: + - /home/andrew + - /mnt/andrew \ No newline at end of file