import argparse, logging, os, shutil, re, subprocess, sys, requests, glob, socket, sensors, datetime, time, operator, premailer
from sys import stdin
from collections import namedtuple, defaultdict
+from shutil import copyfile
+import yaml
+
+reload(sys)
+sys.setdefaultencoding('utf-8')
+
+scriptdir = os.path.dirname(os.path.realpath(__file__))
+
diskstat = namedtuple('diskstat', ['cap', 'alloc', 'free', 'ratio'])
drivetemp = namedtuple('drivetemp', ['name', 'temp', 'units'])
+config = {
+ 'output': '~/var/www/logparse/summary.html',
+ 'header': scriptdir + '/header.html',
+ 'css': scriptdir + '/main.css',
+ 'title': 'logparse',
+ 'maxlist': 10,
+ 'maxcmd': 3,
+ 'mail': {
+ 'to': '',
+ 'from': '',
+ 'subject': 'logparse from $hostname$'
+ },
+ 'hddtemp': {
+ 'drives': ['/dev/sda'],
+ 'port': 7634
+ },
+ 'du-paths': ['/', '/etc', '/home'],
+ 'hostname-path': '/etc/hostname',
+ 'logs': {
+ 'auth': '/var/log/auth.log',
+ 'cron': '/var/log/cron.log',
+ 'sys': '/var/log/syslog',
+ 'smb': '/var/log/samba',
+ 'zfs': '/var/log/zpool.log',
+ 'alloc': '/tmp/alloc',
+ 'postfix': '/var/log/mail.log',
+ 'httpd': '/var/log/apache2'
+ }
+}
-AUTHPATH = "/var/log/auth.log"
-CRONPATH = "/var/log/cron.log"
-SYSPATH = "/var/log/syslog"
-SMBDDIR = "/var/log/samba"
-ZFSPATH = "/var/log/zpool.log"
-ALLOCPATH = "/tmp/alloc"
-POSTFIXPATH = "/var/log/mail.log"
HTTPDSTATUS = "http://localhost/server-status"
-HTTPDDIR = "/var/log/apache2"
-HOSTNAMEPATH = "/etc/hostname"
-DUPATHS = ["/home/andrew", "/mnt/andrew"]
-HDDTEMPS = ["/dev/sda", "/dev/sdc", "/dev/sdd", "/dev/sde"]
-HDDTEMPPORT = 7634
-SUMMARYPATH = "/mnt/andrew/temp/logparse-test.html"
-OUTPUTPATH = "/mnt/andrew/temp/logparse-test2.html"
-MAILPATH = "/mnt/andrew/temp/log-parse-test-3.html"
-HEADERPATH = os.path.dirname(os.path.realpath(__file__)) + "/header.html"
-STYLEPATH = os.path.dirname(os.path.realpath(__file__)) + "/main.css"
+# config['du-paths'] = ["/home/andrew", "/mnt/andrew"]
+# config['hddtemp']['drives'] = ["/dev/sda", "/dev/sdc", "/dev/sdd", "/dev/sde"]
+# config['hddtemp']['port'] = 7634
+# config['output'] = "/mnt/andrew/temp/logparse/summary.html"
+# config['output'] = "/mnt/andrew/temp/logparse/out.html"
+MAILPATH = "/mnt/andrew/temp/logparse/mail.html"
+# config['dest'] = "/mnt/andrew/temp/logparse"
+# config['header'] = os.path.dirname(os.path.realpath(__file__)) + "/header.html"
+# config['css'] = os.path.dirname(os.path.realpath(__file__)) + "/main.css"
MAILOUT = ""
HTMLOUT = ""
TXTOUT = ""
-TITLE = "logparse"
-MAXLIST = 10
-CMDNO = 3
-MAILSUBJECT = "logparse from $hostname$"
+# config['title'] = "logparse"
+# config['maxlist'] = 10
+# config['maxcmd'] = 3
+# config['mail']['subject'] = "logparse from $hostname$"
VERSION = "v0.1"
-# DEG = u'\N{DEGREE SIGN}'.encode('utf-8')
-DEG = 'C'
+DEG = u'\N{DEGREE SIGN}'.encode('utf-8')
+DEG = " °C".encode('unicode_escape')
# Set up logging
logging.basicConfig(level=logging.DEBUG)
else:
logger.info("email will be sent to " + to)
+ loadconf(scriptdir + "/logparse.yaml")
+
global tempfile
- tempfile = open(SUMMARYPATH, 'w+')
- tempfile.write(header(HEADERPATH))
+ tempfile = open(config['output'], 'w+')
+ tempfile.write(header(config['header']))
opentag('div', 1, 'main')
sshd()
sudo()
for tag in ['div', 'body', 'html']:
closetag(tag, 1)
tempfile.close()
- mailprep(SUMMARYPATH, MAILPATH)
+ mailprep(config['output'], MAILPATH)
if (to != None):
logger.debug("sending email")
- ms = subject(MAILSUBJECT)
+ ms = subject(config['mail']['subject'])
cmd = "cat " + MAILPATH + " | mail --debug-level=10 -a 'Content-type: text/html' -s '" + ms + "' " + to
logger.debug(cmd)
subprocess.call(cmd, shell=True)
loggger.warning("no subtitle provided.. skipping section")
return
- if (data == None):
+ if (data == None or len(data) == 0):
logger.debug("no data provided.. just printing subtitle")
tag('p', 0, subtitle)
else:
closetag(tag, block)
def header(template): # return a parsed html header from file
+ try:
+ copyfile(config['css'], config['dest'] + '/' + os.path.basename(config['css']))
+ logger.debug("copied main.css")
+ except Exception as e:
+ logger.warning("could not copy main.css - " + str(e))
headercontent = open(template, 'r').read()
headercontent = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], headercontent)
return headercontent
return r
def hostname(): # get the hostname
- hnfile = open(HOSTNAMEPATH, 'r')
+ hnfile = open(config['hostname-path'], 'r')
hn = re.search('^(.*)\n*', hnfile.read()).group(1)
return hn
else:
return(input)
-def mailprep(inputpath, outputpath, *stylesheet):
+def mailprep(inputpath, output, *stylesheet):
logger.debug("converting stylesheet to inline tags")
old = readlog(inputpath)
- pm = premailer.Premailer(old, external_styles=STYLEPATH)
+ pm = premailer.Premailer(old, external_styles=config['css'])
MAILOUT = pm.transform()
logger.info("converted stylesheet to inline tags")
- file = open(outputpath, 'w')
+ file = open(output, 'w')
file.write(MAILOUT)
file.close()
logger.info("written to temporary mail file")
else:
for user in users:
data.append(user[0] + ' (' + str(user[1]) + ')')
- if len(data) > MAXLIST: # if there are lots of users, truncate them
- data.append('+ ' + str(len(users) - MAXLIST - 1) + " more")
+ if len(data) > config['maxlist']: # if there are lots of users, truncate them
+ data.append('+ ' + str(len(users) - config['maxlist'] - 1) + " more")
break
logger.debug("found " + str(len(matches)) + " ssh logins for users " + str(data))
writedata(subtitle, data)
if (len(commands) > 0):
commands = addtag(commands, 'code')
commands = orderbyfreq(commands)
- commands = truncl(commands, CMDNO)
+ commands = truncl(commands, config['maxcmd'])
writedata("top sudo commands", [c for c in commands])
closetag('div', 1)
logger.info("finished sudo section")
if (matches > 0):
commands = addtag(commands, 'code')
commands = orderbyfreq(commands)
- commands = truncl(commands, CMDNO)
+ commands = truncl(commands, config['maxcmd'])
writedata("top cron commands", [c for c in commands])
closetag('div', 1)
logger.info("finished cron section")
logger.debug("the following downloads succeeded: " + str(l_f))
logger.debug("found " + str(n_s) + " successful downloads, and " + str(n_f) + " failed attempts")
writetitle("nameget")
- writedata(str(n_s) + " succeeded", truncl(orderbyfreq(l_s), CMDNO))
- writedata(str(n_f) + " failed", truncl(orderbyfreq(l_f), CMDNO))
+ writedata(str(n_s) + " succeeded", truncl(l_s, config['maxcmd']))
+ writedata(str(n_f) + " failed", truncl(l_f, config['maxcmd']))
closetag('div', 1)
logger.info("finished nameget section")
logger.debug("found the following requests: " + str(files))
files = addtag(files, 'code')
files = orderbyfreq(files)
- files = truncl(files, CMDNO)
+ files = truncl(files, config['maxcmd'])
writedata(str(a) + " requests", files)
if (ips != None):
logger.debug("found the following ips: " + str(ips))
ips = addtag(ips, 'code')
ips = orderbyfreq(ips)
n_ip = str(len(ips))
- ips = truncl(ips, CMDNO)
- writedata(n_ip + " unique clients", ips)
+ ips = truncl(ips, config['maxcmd'])
+ writedata(n_ip + " clients", ips)
if (useragents != None):
logger.debug("found the following useragents: " + str(useragents))
useragents = addtag(useragents, 'code')
useragents = orderbyfreq(useragents)
n_ua = str(len(useragents))
- useragents = truncl(useragents, CMDNO)
- writedata(n_ua + " unique devices", useragents)
+ useragents = truncl(useragents, config['maxcmd'])
+ writedata(n_ua + " devices", useragents)
writedata(data_h + " transferred")
writedata(str(e) + " errors")
def smbd():
logger.debug("starting smbd section")
opentag('div', 1, 'smbd', 'section')
- files = glob.glob(SMBDDIR + "/log.*[!\.gz][!\.old]") # find list of logfiles
+ files = glob.glob(config['logs']['smb'] + "/log.*[!\.gz][!\.old]") # find list of logfiles
logger.debug("found log files " + str(files))
n_auths = 0 # total number of logins from all users
sigma_auths = [] # contains users
writedata(subtitle)
else: # multiple users
sigma_auths = orderbyfreq(sigma_auths)
- sigma_auths = truncl(sigma_auths, CMDNO)
+ sigma_auths = truncl(sigma_auths, config['maxcmd'])
logger.debug("found " + str(n_auths) + " samba logins for users " + str(sigma_auths))
writedata(subtitle, sigma_auths)
closetag('div', 1)
s = list(set(r)) # unique recipients
if (len(s) > 1):
r = orderbyfreq(r)
- r = truncl(r, CMDNO)
+ r = truncl(r, config['maxcmd'])
writedata(n + " messages sent to", r)
else:
writedata(n + " messages sent to " + r[0])
# For this to work, `hddtemp` must be running in daemon mode.
# Start it like this (bash): sudo hddtemp -d /dev/sda /dev/sdX...
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- s.connect(('localhost',HDDTEMPPORT))
+ s.connect(('localhost',config['hddtemp']['port']))
output = s.recv(4096)
output += s.recv(4096)
s.close()
- hddtemps = []
- for drive in re.split('\|{2}', output):
+ config['hddtemp']['drives'] = []
+ for drive in re.split('\|1}', output):
try:
fields = re.search('\|*(/dev/sd.)\|.*\|(\d+)\|(.)', drive)
name = fields.group(1)
temp = float(fields.group(2))
units = fields.group(3)
- hddtemps.append(drivetemp(name, temp, units))
+ config['hddtemp']['drives'].append(drivetemp(name, temp, DEG))
except:
pass
hddtotal = 0
data = []
- for drive in hddtemps:
+ for drive in config['hddtemp']['drives']:
data.append(drive.name + ': ' + str(drive.temp) + drive.units)
logger.debug("found disk " + drive.name + " at " + str(drive.temp))
hddtotal += drive.temp
- logger.debug("found " + str(len(hddtemps)) + " disks")
+ logger.debug("found " + str(len(config['hddtemp']['drives'])) + " disks")
logger.debug("sum of disk temps is " + str(hddtotal))
- hddavg = hddtotal/float(len(hddtemps))
+ hddavg = "{0:.2f}".format(hddtotal/float(len(config['hddtemp']['drives']))) + DEG
logger.debug("avg disk temp is " + str(hddavg))
data.append("avg: " + str(hddavg))
writetitle("temperatures")
writedata("sys: " + str(systemp) + DEG)
if (coretemps != ''):
writedata("cores", coretemps)
- if (hddtemps != ''):
+ if (config['hddtemp']['drives'] != ''):
writedata("disks", data)
closetag('div', 1)
out = []
content = readlog('alloc')
contentnew = ""
- for p in DUPATHS:
+ for p in config['du-paths']:
alloc_f = getusage(p).alloc
delta = None
try:
timenow = time.strftime("%H:%M:%S")
datenow = time.strftime("%x")
-pathfilter = {"auth": AUTHPATH, "cron": CRONPATH, "sys": SYSPATH, "postfix": POSTFIXPATH, "smb": SMBDDIR, "zfs": ZFSPATH, "alloc": ALLOCPATH, "httpd": HTTPDDIR, "header": HEADERPATH}
+pathfilter = {"auth": config['logs']['auth'], "cron": config['logs']['cron'], "sys": config['logs']['sys'], "postfix": config['logs']['postfix'], "smb": config['logs']['smb'], "zfs": config['logs']['zfs'], "alloc": config['logs']['alloc'], "httpd": config['logs']['httpd'], "header": config['header']}
pathfilter = dict((re.escape(k), v) for k, v in pathfilter.iteritems())
pathpattern = re.compile("|".join(pathfilter.keys()))
-varfilter = {"$title$": TITLE, "$date$": datenow, "$time$": timenow, "$hostname$": hostname(), "$version$": VERSION}
+varfilter = {"$title$": config['title'], "$date$": datenow, "$time$": timenow, "$hostname$": hostname(), "$version$": VERSION, "$css$": os.path.basename(config['css'])}
varfilter = dict((re.escape(k), v) for k, v in varfilter.iteritems())
varpattern = re.compile("|".join(varfilter.keys()))
-
-__main__()
+def loadconf(configfile):
+ try:
+ data = yaml.safe_load(open(configfile))
+ for value in data:
+ if (type(value) == dict):
+ config[value][key] = (data[value][key] for key in value)
+ else:
+ config[value] = data[value]
+ config['dest'] = os.path.dirname(config['output'])
+ logger.debug(str(config))
+ except Exception as e:
+ logger.warning("error processing config: " + str(e))
+
+
+try:
+ __main__()
+finally:
+ subprocess.call("logrotate -f /etc/logrotate.conf", shell=True)
+ logger.info("rotated logfiles")