From 0bd8554d80b6e6089afce88562cf12e847f7d7c4 Mon Sep 17 00:00:00 2001 From: Andrew Lorimer Date: Fri, 17 Aug 2018 23:59:48 +1000 Subject: [PATCH] fix bugs --- logparse.py | 148 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 100 insertions(+), 48 deletions(-) diff --git a/logparse.py b/logparse.py index cd1e632..b416218 100755 --- a/logparse.py +++ b/logparse.py @@ -92,16 +92,21 @@ def writedata(subtitle, data = None): # write title and data to tempfile loggger.warning("no subtitle provided.. skipping section") return - tag('p', 0, subtitle) if (data == None): logger.debug("no data provided.. just printing subtitle") + tag('p', 0, subtitle) else: logger.debug("received data " + str(data)) - opentag('ul', 1) - for datum in data: - logger.debug("printing datum " + datum) - tag('li', 0, datum) - closetag('ul', 1) + subtitle += ':' + if (len(data) == 1): + tag('p', 0, subtitle + ' ' + data[0]) + else: + tag('p', 0, subtitle) + opentag('ul', 1) + for datum in data: + logger.debug("printing datum " + datum) + tag('li', 0, datum) + closetag('ul', 1) def opentag(tag, block = 0, id = None, cl = None): # write html opening tag if (block == 1): @@ -194,7 +199,10 @@ def getusage(path): # Get disk usage statistics def orderbyfreq(l): # order a list by the frequency of its elements and remove duplicates temp_l = l[:] l = list(set(l)) - l.sort(key=lambda x:temp_l.count(x)) + l = [[i, temp_l.count(i)] for i in l] # add count of each element + l.sort(key=lambda x:temp_l.count(x[0])) # sort by count + l = [i[0] + ' (' + str(i[1]) + ')' for i in l] # put element and count into string + l = l[::-1] # reverse return l def addtag(l, tag): # add prefix and suffix tags to each item in a list @@ -204,7 +212,7 @@ def addtag(l, tag): # add prefix and suffix tags to each item in a list def truncl(input, limit): # truncate list if (len(input) > limit): more = str(len(input) - limit) - output = input[-limit:] + output = input[:limit] output.append("+ " + more + " more") return(output) else: @@ -254,7 +262,6 @@ def sshd(): subtitle += ' ' + users[0][0] writedata(subtitle) else: - subtitle += ':' for user in users: data.append(user[0] + ' (' + str(user[1]) + ')') if len(data) > MAXLIST: # if there are lots of users, truncate them @@ -288,11 +295,6 @@ def sudo(): for cmd in cmatches: commands.append(cmd) logger.debug("found the following commands: " + str(commands)) - # temp_cmd=commands[:] - # commands = list(set(commands)) - # commands.sort(key=lambda x:temp_cmd.count(x)) - commands = orderbyfreq(commands) - logger.debug("top 3 sudo commands: " + str(commands[-3:])) writetitle("sudo") subtitle = plural("sudo session", num) + " for" @@ -301,16 +303,13 @@ def sudo(): subtitle += ' ' + users[0][0] writedata(subtitle) else: - subtitle += ':' for user in users: data.append(user[0] + ' (' + str(user[1]) + ')') - if len(data) > 3: - data.append('+ ' + str(len(users) - 2) + " more") - break logger.debug("found " + str(len(matches)) + " sudo sessions for users " + str(data)) writedata(subtitle, data) if (len(commands) > 0): commands = addtag(commands, 'code') + commands = orderbyfreq(commands) commands = truncl(commands, CMDNO) writedata("top sudo commands", [c for c in commands]) closetag('div', 1) @@ -335,8 +334,8 @@ def cron(): writetitle("cron") writedata(subtitle) if (matches > 0): - commands = orderbyfreq(commands) commands = addtag(commands, 'code') + commands = orderbyfreq(commands) commands = truncl(commands, CMDNO) writedata("top cron commands", [c for c in commands]) closetag('div', 1) @@ -356,7 +355,7 @@ def nameget(): for i in failed: l_f.append(i) logger.debug("the following downloads failed: " + str(l_f)) - succ = re.findall('.*nameget.*downloaded.*', syslog) + succ = re.findall('.*nameget.*downloaded\s(.*)', syslog) n_s = sum(1 for i in succ) l_s = [] for i in succ: @@ -377,26 +376,58 @@ def httpd(): logger.info("starting httpd section") opentag('div', 1, 'httpd', 'section') accesslog = readlog("httpd/access.log") - a = len(accesslog) + a = len(accesslog.split('\n')) errorlog = readlog("httpd/error.log") - e = len(errorlog) + e = len(errorlog.split('\n')) data_b = 0 + ips = [] + files = [] + useragents = [] + errors = [] + notfound = [] + unprivileged = [] for line in accesslog.split('\n'): + fields = re.search('^(\S*) .*GET (\/.*) HTTP/\d\.\d\" 200 (\d*) \"(.*)\".*\((.*)\;', line) try: - data_b += int(re.search('.*HTTP/\d\.\d\" 200 (\d*) ', line).group(1)) + ips.append(fields.group(1)) + files.append(fields.group(2)) + useragents.append(fields.group(5)) + logger.debug("transferred " + fields.group(3) + " bytes in this request") + data_b += int(fields.group(3)) + logger.debug("data_b is now " + str(data_b)) except Exception as error: if type(error) is AttributeError: - pass + logger.debug("attributeerrror: " + str(error)) else: logger.warning("error processing httpd access log: " + str(error)) + logger.debug(str(data_b) + " bytes transferred") data_h = parsesize(data_b) + writetitle("apache") logger.debug("httpd has transferred " + str(data_b) + " bytes in response to " + str(a) + " requests with " + str(e) + " errors") + if (a > 0): + logger.debug("found the following requests: " + str(files)) + files = addtag(files, 'code') + files = orderbyfreq(files) + files = truncl(files, CMDNO) + writedata(str(a) + " requests", files) + if (ips != None): + logger.debug("found the following ips: " + str(ips)) + ips = addtag(ips, 'code') + ips = orderbyfreq(ips) + n_ip = str(len(ips)) + ips = truncl(ips, CMDNO) + writedata(n_ip + " unique clients", ips) + if (useragents != None): + logger.debug("found the following useragents: " + str(useragents)) + useragents = addtag(useragents, 'code') + useragents = orderbyfreq(useragents) + n_ua = str(len(useragents)) + useragents = truncl(useragents, CMDNO) + writedata(n_ua + " unique devices", useragents) - writetitle("apache") writedata(data_h + " transferred") - writedata(str(a) + " requests") writedata(str(e) + " errors") closetag('div', 1) @@ -430,7 +461,7 @@ def smbd(): opentag('div', 1, 'smbd', 'section') files = glob.glob(SMBDDIR + "/log.*[!\.gz][!\.old]") # find list of logfiles n_auths = 0 # total number of logins from all users - sigma_auths = [] # contains users and their respective no. of logins + sigma_auths = [] # contains users output = "" for file in files: # one log file for each client @@ -443,27 +474,23 @@ def smbd(): matches = re.findall('.*sam authentication for user \[(.*)\] succeeded.*', readlog(file)) for match in matches: userhost = match + "@" + host - exists = [i for i, item in enumerate(sigma_auths) if re.search(userhost, item[0])] - if (exists == []): - sigma_auths.append([userhost, 1]) - else: - sigma_auths[exists[0]][1] += 1 + sigma_auths.append(userhost) + # exists = [i for i, item in enumerate(sigma_auths) if re.search(userhost, item[0])] + # if (exists == []): + # sigma_auths.append([userhost, 1]) + # else: + # sigma_auths[exists[0]][1] += 1 n_auths += 1 writetitle("samba") subtitle = plural("login", n_auths) + " from" - data = [] if (len(sigma_auths) == 1): # if only one user, do not display no of logins for this user subtitle += ' ' + sigma_auths[0][0] writedata(subtitle) else: # multiple users - subtitle += ':' - for x in sigma_auths: - data.append((str(x[0])) + " (" + str(x[1]) + ")") - if len(data) > MAXLIST: # if many users, truncate them - data.append('+ ' + str(len(sigma_auths) - MAXLIST - 1) + " more") - break + sigma_auths = orderbyfreq(sigma_auths) + sigma_auths = truncl(sigma_auths, CMDNO) logger.debug("found " + str(n_auths) + " samba logins for users " + str(sigma_auths)) - writedata(subtitle, data) + writedata(subtitle, sigma_auths) closetag('div', 1) logger.info("finished smbd section") @@ -474,12 +501,29 @@ def smbd(): def postfix(): logger.debug("starting postfix section") opentag('div', 1, 'postfix', 'section') - messages = re.findall('.*from\=<.*>, size\=(\d*),.*\n.*\n.*\: removed\n.*', readlog('postfix')) - size = sum([int(x) for x in messages]) + messages = re.findall('.*from\=<(.*)>, size\=(\d*),.*\n.*to=<(.*)>', readlog('postfix')) + r = [] + s = [] + size = 0 + for message in messages: + r.append(message[2]) + s.append(message[0]) + size += int(message[1]) + # size = sum([int(x) for x in messages]) size = parsesize(size) n = str(len(messages)) writetitle("postfix") - writedata(n + " messages sent") + + if (len(r) > 0): + s = list(set(r)) # unique recipients + if (len(s) > 1): + r = orderbyfreq(r) + r = truncl(r, CMDNO) + writedata(n + " messages sent to", r) + else: + writedata(n + " messages sent to " + r[0]) + else: + writedata(n + " messages sent") writedata("total of " + size) closetag('div', 1) logger.info("finished postfix section") @@ -496,14 +540,22 @@ def zfs(): pool = re.search('.*---\n(\w*)', zfslog).group(1) scrub = re.search('.*scrub repaired (\d*) in \d*h\d*m with (\d*) errors on (\S*\s)(\S*)\s(\d+\s)', zfslog) iostat = re.search('.*---\n\w*\s*(\S*)\s*(\S*)\s', zfslog) - scrubrepairs = scrub.group(1) - scruberrors = scrub.group(2) - scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4) + scrubrepairs = scruberrors = scrubdate = None + try: + scrubrepairs = scrub.group(1) + scruberrors = scrub.group(2) + scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4) + except: + logger.debug("error getting scrub data") alloc = iostat.group(1) free = iostat.group(2) writetitle("zfs") - subtitle = "Scrub on " + scrubdate + ": " - data = [scrubrepairs + " repaired", scruberrors + " errors", alloc + " used", free + " free"] + if (scrubdate != None): + subtitle = "Scrub of " + pool + " on " + scrubdate + data = [scrubrepairs + " repaired", scruberrors + " errors", alloc + " used", free + " free"] + else: + subtitle = pool + data = [alloc + " used", free + " free"] writedata(subtitle, data) closetag('div', 1) logger.info("finished zfs section") -- 2.43.2