import argparse, logging, os, shutil, re, subprocess, sys, requests, glob, socket, sensors, datetime, time, operator, premailer
from sys import stdin
from collections import namedtuple, defaultdict
+from shutil import copyfile
+import yaml
+
+reload(sys)
+sys.setdefaultencoding('utf-8')
+
+scriptdir = os.path.dirname(os.path.realpath(__file__))
+
diskstat = namedtuple('diskstat', ['cap', 'alloc', 'free', 'ratio'])
drivetemp = namedtuple('drivetemp', ['name', 'temp', 'units'])
+config = {
+ 'output': '~/var/www/logparse/summary.html',
+ 'header': scriptdir + '/header.html',
+ 'css': scriptdir + '/main.css',
+ 'title': 'logparse',
+ 'maxlist': 10,
+ 'maxcmd': 3,
+ 'mail': {
+ 'to': '',
+ 'from': '',
+ 'subject': 'logparse from $hostname$'
+ },
+ 'hddtemp': {
+ 'drives': ['/dev/sda'],
+ 'port': 7634
+ },
+ 'du-paths': ['/', '/etc', '/home'],
+ 'hostname-path': '/etc/hostname',
+ 'logs': {
+ 'auth': '/var/log/auth.log',
+ 'cron': '/var/log/cron.log',
+ 'sys': '/var/log/syslog',
+ 'smb': '/var/log/samba',
+ 'zfs': '/var/log/zpool.log',
+ 'alloc': '/tmp/alloc',
+ 'postfix': '/var/log/mail.log',
+ 'httpd': '/var/log/apache2'
+ }
+}
-AUTHPATH = "/var/log/auth.log"
-CRONPATH = "/var/log/cron.log"
-SYSPATH = "/var/log/syslog"
-SMBDDIR = "/var/log/samba"
-ZFSPATH = "/var/log/zpool.log"
-ALLOCPATH = "/tmp/alloc"
-POSTFIXPATH = "/var/log/mail.log"
HTTPDSTATUS = "http://localhost/server-status"
-HTTPDDIR = "/var/log/apache2"
-HOSTNAMEPATH = "/etc/hostname"
-DUPATHS = ["/home/andrew", "/mnt/andrew"]
-HDDTEMPS = ["/dev/sda", "/dev/sdc", "/dev/sdd", "/dev/sde"]
-HDDTEMPPORT = 7634
-SUMMARYPATH = "/mnt/andrew/temp/logparse-test.html"
-OUTPUTPATH = "/mnt/andrew/temp/logparse-test2.html"
-MAILPATH = "/mnt/andrew/temp/log-parse-test-3.html"
-HEADERPATH = "header.html"
-STYLEPATH = "main.css"
+# config['du-paths'] = ["/home/andrew", "/mnt/andrew"]
+# config['hddtemp']['drives'] = ["/dev/sda", "/dev/sdc", "/dev/sdd", "/dev/sde"]
+# config['hddtemp']['port'] = 7634
+# config['output'] = "/mnt/andrew/temp/logparse/summary.html"
+# config['output'] = "/mnt/andrew/temp/logparse/out.html"
+MAILPATH = "/mnt/andrew/temp/logparse/mail.html"
+# config['dest'] = "/mnt/andrew/temp/logparse"
+# config['header'] = os.path.dirname(os.path.realpath(__file__)) + "/header.html"
+# config['css'] = os.path.dirname(os.path.realpath(__file__)) + "/main.css"
MAILOUT = ""
HTMLOUT = ""
TXTOUT = ""
-TITLE = "logparse"
-MAXLIST = 10
-CMDNO = 3
-MAILSUBJECT = "logparse from $hostname$"
+# config['title'] = "logparse"
+# config['maxlist'] = 10
+# config['maxcmd'] = 3
+# config['mail']['subject'] = "logparse from $hostname$"
VERSION = "v0.1"
-# DEG = u'\N{DEGREE SIGN}'.encode('utf-8')
-DEG = 'C'
+DEG = u'\N{DEGREE SIGN}'.encode('utf-8')
+DEG = " °C".encode('unicode_escape')
# Set up logging
logging.basicConfig(level=logging.DEBUG)
else:
logger.info("email will be sent to " + to)
+ loadconf(scriptdir + "/logparse.yaml")
+
global tempfile
- tempfile = open(SUMMARYPATH, 'w+')
- tempfile.write(header(HEADERPATH))
+ tempfile = open(config['output'], 'w+')
+ tempfile.write(header(config['header']))
opentag('div', 1, 'main')
sshd()
sudo()
for tag in ['div', 'body', 'html']:
closetag(tag, 1)
tempfile.close()
- mailprep(SUMMARYPATH, MAILPATH)
+ mailprep(config['output'], MAILPATH)
if (to != None):
logger.debug("sending email")
- ms = subject(MAILSUBJECT)
+ ms = subject(config['mail']['subject'])
cmd = "cat " + MAILPATH + " | mail --debug-level=10 -a 'Content-type: text/html' -s '" + ms + "' " + to
logger.debug(cmd)
subprocess.call(cmd, shell=True)
loggger.warning("no subtitle provided.. skipping section")
return
- tag('p', 0, subtitle)
- if (data == None):
+ if (data == None or len(data) == 0):
logger.debug("no data provided.. just printing subtitle")
+ tag('p', 0, subtitle)
else:
logger.debug("received data " + str(data))
- opentag('ul', 1)
- for datum in data:
- logger.debug("printing datum " + datum)
- tag('li', 0, datum)
- closetag('ul', 1)
+ subtitle += ':'
+ if (len(data) == 1):
+ tag('p', 0, subtitle + ' ' + data[0])
+ else:
+ tag('p', 0, subtitle)
+ opentag('ul', 1)
+ for datum in data:
+ logger.debug("printing datum " + datum)
+ tag('li', 0, datum)
+ closetag('ul', 1)
def opentag(tag, block = 0, id = None, cl = None): # write html opening tag
if (block == 1):
closetag(tag, block)
def header(template): # return a parsed html header from file
+ try:
+ copyfile(config['css'], config['dest'] + '/' + os.path.basename(config['css']))
+ logger.debug("copied main.css")
+ except Exception as e:
+ logger.warning("could not copy main.css - " + str(e))
headercontent = open(template, 'r').read()
headercontent = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], headercontent)
return headercontent
return r
def hostname(): # get the hostname
- hnfile = open(HOSTNAMEPATH, 'r')
+ hnfile = open(config['hostname-path'], 'r')
hn = re.search('^(.*)\n*', hnfile.read()).group(1)
return hn
return
else:
path = pathpattern.sub(lambda m: pathfilter[re.escape(m.group(0))], path)
- return open(path, mode).read()
+ if (os.path.isfile(path) is False):
+ logger.error(path + " does not exist")
+ return ''
+ else:
+ return open(path, mode).read()
def writelog(path = None, content = "", mode = 'w'): # read file, substituting known paths
if (path == None or content == None):
def orderbyfreq(l): # order a list by the frequency of its elements and remove duplicates
temp_l = l[:]
l = list(set(l))
- l.sort(key=lambda x:temp_l.count(x))
+ l = [[i, temp_l.count(i)] for i in l] # add count of each element
+ l.sort(key=lambda x:temp_l.count(x[0])) # sort by count
+ l = [i[0] + ' (' + str(i[1]) + ')' for i in l] # put element and count into string
+ l = l[::-1] # reverse
return l
def addtag(l, tag): # add prefix and suffix tags to each item in a list
def truncl(input, limit): # truncate list
if (len(input) > limit):
more = str(len(input) - limit)
- output = input[-limit:]
+ output = input[:limit]
output.append("+ " + more + " more")
return(output)
else:
return(input)
-def mailprep(inputpath, outputpath, *stylesheet):
+def mailprep(inputpath, output, *stylesheet):
logger.debug("converting stylesheet to inline tags")
old = readlog(inputpath)
- pm = premailer.Premailer(old, external_styles=STYLEPATH)
+ pm = premailer.Premailer(old, external_styles=config['css'])
MAILOUT = pm.transform()
logger.info("converted stylesheet to inline tags")
- file = open(outputpath, 'w')
+ file = open(output, 'w')
file.write(MAILOUT)
file.close()
logger.info("written to temporary mail file")
subtitle += ' ' + users[0][0]
writedata(subtitle)
else:
- subtitle += ':'
for user in users:
data.append(user[0] + ' (' + str(user[1]) + ')')
- if len(data) > MAXLIST: # if there are lots of users, truncate them
- data.append('+ ' + str(len(users) - MAXLIST - 1) + " more")
+ if len(data) > config['maxlist']: # if there are lots of users, truncate them
+ data.append('+ ' + str(len(users) - config['maxlist'] - 1) + " more")
break
logger.debug("found " + str(len(matches)) + " ssh logins for users " + str(data))
writedata(subtitle, data)
for cmd in cmatches:
commands.append(cmd)
logger.debug("found the following commands: " + str(commands))
- # temp_cmd=commands[:]
- # commands = list(set(commands))
- # commands.sort(key=lambda x:temp_cmd.count(x))
- commands = orderbyfreq(commands)
- logger.debug("top 3 sudo commands: " + str(commands[-3:]))
writetitle("sudo")
subtitle = plural("sudo session", num) + " for"
subtitle += ' ' + users[0][0]
writedata(subtitle)
else:
- subtitle += ':'
for user in users:
data.append(user[0] + ' (' + str(user[1]) + ')')
- if len(data) > 3:
- data.append('+ ' + str(len(users) - 2) + " more")
- break
- logger.debug("found " + str(len(matches)) + " sudo sessions for users " + str(data))
+ logger.debug("found " + str(num) + " sudo sessions for users " + str(data))
writedata(subtitle, data)
if (len(commands) > 0):
commands = addtag(commands, 'code')
- commands = truncl(commands, CMDNO)
+ commands = orderbyfreq(commands)
+ commands = truncl(commands, config['maxcmd'])
writedata("top sudo commands", [c for c in commands])
closetag('div', 1)
logger.info("finished sudo section")
writetitle("cron")
writedata(subtitle)
if (matches > 0):
- commands = orderbyfreq(commands)
commands = addtag(commands, 'code')
- commands = truncl(commands, CMDNO)
+ commands = orderbyfreq(commands)
+ commands = truncl(commands, config['maxcmd'])
writedata("top cron commands", [c for c in commands])
closetag('div', 1)
logger.info("finished cron section")
for i in failed:
l_f.append(i)
logger.debug("the following downloads failed: " + str(l_f))
- succ = re.findall('.*nameget.*downloaded.*', syslog)
+ succ = re.findall('.*nameget.*downloaded\s(.*)', syslog)
n_s = sum(1 for i in succ)
l_s = []
for i in succ:
logger.debug("the following downloads succeeded: " + str(l_f))
logger.debug("found " + str(n_s) + " successful downloads, and " + str(n_f) + " failed attempts")
writetitle("nameget")
- writedata(str(n_s) + " succeeded", truncl(orderbyfreq(l_s), CMDNO))
- writedata(str(n_f) + " failed", truncl(orderbyfreq(l_f), CMDNO))
+ writedata(str(n_s) + " succeeded", truncl(l_s, config['maxcmd']))
+ writedata(str(n_f) + " failed", truncl(l_f, config['maxcmd']))
closetag('div', 1)
logger.info("finished nameget section")
logger.info("starting httpd section")
opentag('div', 1, 'httpd', 'section')
accesslog = readlog("httpd/access.log")
- a = len(accesslog)
+ a = len(accesslog.split('\n'))
errorlog = readlog("httpd/error.log")
- e = len(errorlog)
+ e = len(errorlog.split('\n'))
data_b = 0
+ ips = []
+ files = []
+ useragents = []
+ errors = []
+ notfound = []
+ unprivileged = []
for line in accesslog.split('\n'):
+ fields = re.search('^(\S*) .*GET (\/.*) HTTP/\d\.\d\" 200 (\d*) \"(.*)\".*\((.*)\;', line)
try:
- data_b += int(re.search('.*HTTP/\d\.\d\" 200 (\d*) ', line).group(1))
+ ips.append(fields.group(1))
+ files.append(fields.group(2))
+ useragents.append(fields.group(5))
+ logger.debug("transferred " + fields.group(3) + " bytes in this request")
+ data_b += int(fields.group(3))
+ logger.debug("data_b is now " + str(data_b))
except Exception as error:
if type(error) is AttributeError:
- pass
+ logger.debug("attributeerrror: " + str(error))
else:
logger.warning("error processing httpd access log: " + str(error))
+ logger.debug(str(data_b) + " bytes transferred")
data_h = parsesize(data_b)
+ writetitle("apache")
logger.debug("httpd has transferred " + str(data_b) + " bytes in response to " + str(a) + " requests with " + str(e) + " errors")
+ if (a > 0):
+ logger.debug("found the following requests: " + str(files))
+ files = addtag(files, 'code')
+ files = orderbyfreq(files)
+ files = truncl(files, config['maxcmd'])
+ writedata(str(a) + " requests", files)
+ if (ips != None):
+ logger.debug("found the following ips: " + str(ips))
+ ips = addtag(ips, 'code')
+ ips = orderbyfreq(ips)
+ n_ip = str(len(ips))
+ ips = truncl(ips, config['maxcmd'])
+ writedata(n_ip + " clients", ips)
+ if (useragents != None):
+ logger.debug("found the following useragents: " + str(useragents))
+ useragents = addtag(useragents, 'code')
+ useragents = orderbyfreq(useragents)
+ n_ua = str(len(useragents))
+ useragents = truncl(useragents, config['maxcmd'])
+ writedata(n_ua + " devices", useragents)
- writetitle("apache")
writedata(data_h + " transferred")
- writedata(str(a) + " requests")
writedata(str(e) + " errors")
closetag('div', 1)
def smbd():
logger.debug("starting smbd section")
opentag('div', 1, 'smbd', 'section')
- files = glob.glob(SMBDDIR + "/log.*[!\.gz][!\.old]") # find list of logfiles
+ files = glob.glob(config['logs']['smb'] + "/log.*[!\.gz][!\.old]") # find list of logfiles
+ logger.debug("found log files " + str(files))
n_auths = 0 # total number of logins from all users
- sigma_auths = [] # contains users and their respective no. of logins
+ sigma_auths = [] # contains users
output = ""
for file in files: # one log file for each client
+ logger.debug("looking at file " + file)
+
# find the machine (ip or hostname) that this file represents
ip = re.search('log\.(.*)', file).group(1) # get ip or hostname from file path (/var/log/samba/log.host)
host = resolve(ip)
matches = re.findall('.*sam authentication for user \[(.*)\] succeeded.*', readlog(file))
for match in matches:
userhost = match + "@" + host
- exists = [i for i, item in enumerate(sigma_auths) if re.search(userhost, item[0])]
- if (exists == []):
- sigma_auths.append([userhost, 1])
- else:
- sigma_auths[exists[0]][1] += 1
+ sigma_auths.append(userhost)
+ # exists = [i for i, item in enumerate(sigma_auths) if re.search(userhost, item[0])]
+ # if (exists == []):
+ # sigma_auths.append([userhost, 1])
+ # else:
+ # sigma_auths[exists[0]][1] += 1
n_auths += 1
writetitle("samba")
subtitle = plural("login", n_auths) + " from"
- data = []
if (len(sigma_auths) == 1): # if only one user, do not display no of logins for this user
subtitle += ' ' + sigma_auths[0][0]
writedata(subtitle)
else: # multiple users
- subtitle += ':'
- for x in sigma_auths:
- data.append((str(x[0])) + " (" + str(x[1]) + ")")
- if len(data) > MAXLIST: # if many users, truncate them
- data.append('+ ' + str(len(sigma_auths) - MAXLIST - 1) + " more")
- break
+ sigma_auths = orderbyfreq(sigma_auths)
+ sigma_auths = truncl(sigma_auths, config['maxcmd'])
logger.debug("found " + str(n_auths) + " samba logins for users " + str(sigma_auths))
- writedata(subtitle, data)
+ writedata(subtitle, sigma_auths)
closetag('div', 1)
logger.info("finished smbd section")
def postfix():
logger.debug("starting postfix section")
opentag('div', 1, 'postfix', 'section')
- messages = re.findall('.*from\=<.*>, size\=(\d*),.*\n.*\n.*\: removed\n.*', readlog('postfix'))
- size = sum([int(x) for x in messages])
+ messages = re.findall('.*from\=<(.*)>, size\=(\d*),.*\n.*to=<(.*)>', readlog('postfix'))
+ r = []
+ s = []
+ size = 0
+ for message in messages:
+ r.append(message[2])
+ s.append(message[0])
+ size += int(message[1])
+ # size = sum([int(x) for x in messages])
size = parsesize(size)
n = str(len(messages))
writetitle("postfix")
- writedata(n + " messages sent")
+
+ if (len(r) > 0):
+ s = list(set(r)) # unique recipients
+ if (len(s) > 1):
+ r = orderbyfreq(r)
+ r = truncl(r, config['maxcmd'])
+ writedata(n + " messages sent to", r)
+ else:
+ writedata(n + " messages sent to " + r[0])
+ else:
+ writedata(n + " messages sent")
writedata("total of " + size)
closetag('div', 1)
logger.info("finished postfix section")
logger.debug("starting zfs section")
opentag('div', 1, 'zfs', 'section')
zfslog = readlog('zfs')
+ logger.debug("zfs log is " + zfslog)
logger.debug("got zfs logfile\n" + zfslog + "---end log---")
pool = re.search('.*---\n(\w*)', zfslog).group(1)
scrub = re.search('.*scrub repaired (\d*) in \d*h\d*m with (\d*) errors on (\S*\s)(\S*)\s(\d+\s)', zfslog)
iostat = re.search('.*---\n\w*\s*(\S*)\s*(\S*)\s', zfslog)
- scrubrepairs = scrub.group(1)
- scruberrors = scrub.group(2)
- scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4)
+ scrubrepairs = scruberrors = scrubdate = None
+ try:
+ scrubrepairs = scrub.group(1)
+ scruberrors = scrub.group(2)
+ scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4)
+ except:
+ logger.debug("error getting scrub data")
alloc = iostat.group(1)
free = iostat.group(2)
writetitle("zfs")
- subtitle = "Scrub on " + scrubdate + ": "
- data = [scrubrepairs + " repaired", scruberrors + " errors", alloc + " used", free + " free"]
+ if (scrubdate != None):
+ subtitle = "Scrub of " + pool + " on " + scrubdate
+ data = [scrubrepairs + " repaired", scruberrors + " errors", alloc + " used", free + " free"]
+ else:
+ subtitle = pool
+ data = [alloc + " used", free + " free"]
writedata(subtitle, data)
closetag('div', 1)
logger.info("finished zfs section")
# For this to work, `hddtemp` must be running in daemon mode.
# Start it like this (bash): sudo hddtemp -d /dev/sda /dev/sdX...
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- s.connect(('localhost',HDDTEMPPORT))
+ s.connect(('localhost',config['hddtemp']['port']))
output = s.recv(4096)
output += s.recv(4096)
s.close()
- hddtemps = []
- for drive in re.split('\|{2}', output):
+ config['hddtemp']['drives'] = []
+ for drive in re.split('\|1}', output):
try:
fields = re.search('\|*(/dev/sd.)\|.*\|(\d+)\|(.)', drive)
name = fields.group(1)
temp = float(fields.group(2))
units = fields.group(3)
- hddtemps.append(drivetemp(name, temp, units))
+ config['hddtemp']['drives'].append(drivetemp(name, temp, DEG))
except:
pass
hddtotal = 0
data = []
- for drive in hddtemps:
+ for drive in config['hddtemp']['drives']:
data.append(drive.name + ': ' + str(drive.temp) + drive.units)
logger.debug("found disk " + drive.name + " at " + str(drive.temp))
hddtotal += drive.temp
- logger.debug("found " + str(len(hddtemps)) + " disks")
+ logger.debug("found " + str(len(config['hddtemp']['drives'])) + " disks")
logger.debug("sum of disk temps is " + str(hddtotal))
- hddavg = hddtotal/float(len(hddtemps))
+ hddavg = "{0:.2f}".format(hddtotal/float(len(config['hddtemp']['drives']))) + DEG
logger.debug("avg disk temp is " + str(hddavg))
data.append("avg: " + str(hddavg))
writetitle("temperatures")
writedata("sys: " + str(systemp) + DEG)
if (coretemps != ''):
writedata("cores", coretemps)
- if (hddtemps != ''):
+ if (config['hddtemp']['drives'] != ''):
writedata("disks", data)
closetag('div', 1)
out = []
content = readlog('alloc')
contentnew = ""
- for p in DUPATHS:
+ for p in config['du-paths']:
alloc_f = getusage(p).alloc
delta = None
try:
timenow = time.strftime("%H:%M:%S")
datenow = time.strftime("%x")
-pathfilter = {"auth": AUTHPATH, "cron": CRONPATH, "sys": SYSPATH, "postfix": POSTFIXPATH, "smb": SMBDDIR, "zfs": ZFSPATH, "alloc": ALLOCPATH, "httpd": HTTPDDIR, "header": HEADERPATH}
+pathfilter = {"auth": config['logs']['auth'], "cron": config['logs']['cron'], "sys": config['logs']['sys'], "postfix": config['logs']['postfix'], "smb": config['logs']['smb'], "zfs": config['logs']['zfs'], "alloc": config['logs']['alloc'], "httpd": config['logs']['httpd'], "header": config['header']}
pathfilter = dict((re.escape(k), v) for k, v in pathfilter.iteritems())
pathpattern = re.compile("|".join(pathfilter.keys()))
-varfilter = {"$title$": TITLE, "$date$": datenow, "$time$": timenow, "$hostname$": hostname(), "$version$": VERSION}
+varfilter = {"$title$": config['title'], "$date$": datenow, "$time$": timenow, "$hostname$": hostname(), "$version$": VERSION, "$css$": os.path.basename(config['css'])}
varfilter = dict((re.escape(k), v) for k, v in varfilter.iteritems())
varpattern = re.compile("|".join(varfilter.keys()))
-
-__main__()
+def loadconf(configfile):
+ try:
+ data = yaml.safe_load(open(configfile))
+ for value in data:
+ if (type(value) == dict):
+ config[value][key] = (data[value][key] for key in value)
+ else:
+ config[value] = data[value]
+ config['dest'] = os.path.dirname(config['output'])
+ logger.debug(str(config))
+ except Exception as e:
+ logger.warning("error processing config: " + str(e))
+
+
+try:
+ __main__()
+finally:
+ subprocess.call("logrotate -f /etc/logrotate.conf", shell=True)
+ logger.info("rotated logfiles")