From cbb12b929cdb657d13ab53f09c121f1ccca7be88 Mon Sep 17 00:00:00 2001 From: Andrew Lorimer Date: Thu, 29 Aug 2019 17:03:32 +1000 Subject: [PATCH] further bugfixing in parsers & clean up formatting.py --- logparse/formatting.py | 150 ++++++++++++++++++++++---------- logparse/interface.py | 7 +- logparse/mail.py | 6 +- logparse/parsers/httpd.py | 10 ++- logparse/parsers/temperature.py | 40 ++++----- logparse/parsers/zfs.py | 28 +++--- 6 files changed, 151 insertions(+), 90 deletions(-) diff --git a/logparse/formatting.py b/logparse/formatting.py index 19eda99..45d2e3f 100644 --- a/logparse/formatting.py +++ b/logparse/formatting.py @@ -13,11 +13,12 @@ import locale from string import Template import logparse -from . import interface, util, config +from . import interface, util, config, mail import logging logger = logging.getLogger(__name__) + locale.setlocale(locale.LC_ALL, '') # inherit system locale #DEG = "°".encode('unicode_escape') DEG = u'\N{DEGREE SIGN}' @@ -28,8 +29,10 @@ CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔'] CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌'] LINECHARS_DOUBLE = ['║', '═'] LINECHARS_SINGLE = ['│', '─'] +BULLET = "• " INDENT = " " + class Output: """ Base class for a data processor. @@ -40,9 +43,15 @@ class Output: self.destination = "" def append(self, content): + """ + Add a string + """ self.content += content def write(self, destination=""): + """ + Output contents into a file + """ if destination == "": destination = self.destination if destination == "": @@ -64,17 +73,27 @@ class PlaintextOutput(Output): self.linewidth = linewidth; def append_header(self, template=''): + """ + Print details with some primitive formatting + """ init_varfilter() box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth']) line = PlaintextLine(self.linewidth) self.append(box.draw() + line.draw()) def append_footer(self): + """ + Append a horizontal line and some details + """ init_varfilter() self.append(PlaintextLine(self.linewidth, vpadding=1).draw()) self.append(Template("$hostname $time $date").safe_substitute(varsubst)) def append_section(self, section): + """ + Call the appropriate methods to format a section (provided by a parser). + This should be run by interface.py after every instance of parse_log(). + """ self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw()) self.append('\n'*2) for data in section.data: @@ -82,6 +101,10 @@ class PlaintextOutput(Output): self.append('\n') def _fmt_data(self, subtitle, data = None): # write title and data + """ + Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks. + Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed. + """ if (subtitle == ""): logger.warning("No subtitle provided.. skipping section") return @@ -97,7 +120,7 @@ class PlaintextOutput(Output): else: itemoutput = subtitle + '\n' for datum in data: - datum = '• ' + datum + datum = BULLET + datum if len(datum) > config.prefs['linewidth'] - 3: words = datum.split() if max(map(len, words)) > config.prefs['linewidth'] - len(INDENT): @@ -129,58 +152,83 @@ class HtmlOutput(Output): self.css = "" def embed_css(self, css): + """ + Convert stylesheet to inline tags + """ self.content = mail.mailprep(self.content, css) + return self.content - def append_header(self, template): # insert variables into header template file + def append_header(self, template): + """ + Insert variables into header template file and append HTML tags + """ init_varfilter() headercontent = Template(open(template, 'r').read()) self.append(headercontent.safe_substitute(varsubst)) self.append(self.opentag('div', id='main')) def append_footer(self): - self.append(closetag('div') + closetag('body') + closetag('html')) + """ + Close HTML tags that were opened in the template. + TODO: add footer template similar to header template. + """ + self.append(self.closetag('div') + self.closetag('body') + self.closetag('html')) def append_section(self, section): + """ + Call the appropriate methods to generate HTML tags for a section (provided by a parser). + This should be run by interface.py after every instance of parse_log(). + """ self.append(self.opentag('div', 1, section.title, 'section')) self.append(self._gen_title(section.title)) for data in section.data: self.append(self._fmt_data(data.subtitle, data.items)) - self.append(closetag('div', 1)) + self.append(self.closetag('div', 1)) - def _gen_title(self, title): # write title for a section + def _gen_title(self, title): + """ + Format the title for a section + """ if (title == '' or '\n' in title): logger.error("Invalid title") raise ValueError logger.debug("Writing title for " + title) - return tag('h2', 0, title) + return self.tag('h2', False, title) - def _fmt_data(self, subtitle, data = None): # write title and data + def _fmt_data(self, subtitle, data = None): + """ + Format the properties of a data object into usable HTML tags. + Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed. + """ if (subtitle == ""): logger.warning("No subtitle provided.. skipping section") return if (data == None or len(data) == 0): logger.debug("No data provided.. just printing subtitle") - return tag('p', 0, subtitle) + return self.tag('p', False, subtitle) else: logger.debug("Received data " + str(data)) subtitle += ':' if (len(data) == 1): - return tag('p', 0, subtitle + ' ' + data[0]) + return self.tag('p', False, subtitle + ' ' + data[0]) else: output = "" - output += tag('p', 0, subtitle) + output += self.tag('p', False, subtitle) output += self.opentag('ul', 1) coderegex = re.compile('`(.*)`') for datum in data: if datum == "" or datum == None: continue - datum = coderegex.sub(r'{\1}', str(datum)) - output += tag('li', 0, datum) - output += closetag('ul', 1) + datum = coderegex.sub(r"\1", str(datum)) + output += self.tag('li', False, datum) + output += self.closetag('ul', True) return output - def opentag(self, tag, block = 0, id = None, cl = None): # write html opening tag + def opentag(self, tag, block=False, id=None, cl=None): + """ + Write HTML opening tag + """ output = "" if (block): output += '\n' @@ -194,19 +242,24 @@ class HtmlOutput(Output): output += '\n' return output - def closetag(self, tag, block = 0): # write html closing tag - if (block == 0): - return "" - else: + def closetag(self, tag, block=False): + """ + Write HTML closing tag + """ + if block: return "\n\n" + else: + return "" - def tag(self, tag, block = 0, content = ""): # write html opening tag, content, and html closing tag + def tag(self, tag, block=False, content=""): + """ + Write HTML opening tag, content, and closing tag + """ o = self.opentag(tag, block) c = self.closetag(tag, block) return o + content + c - class Section: """ Each parser should output a Section() which contains the title and returned data. @@ -219,6 +272,7 @@ class Section: def append_data(self, data): self.data.append(data) + class Data: """ Each section (parser) can have one or more Data() objects which are essentially glorified lists. @@ -229,26 +283,25 @@ class Data: self.items = items def truncl(self, limit): # truncate list + """ + Truncate self.items to a specified value and state how many items are hidden. + """ if (len(self.items) > limit): more = str(len(self.items) - limit) self.items = self.items[:limit] - self.items.append("+ " + more + " more") - - def orderbyfreq(self): # order a list by the frequency of its elements and remove duplicates -# temp = list(self.items)[:] -# logger.debug(self.items) -# self.items = list(set(self.items)) -# self.items = [[i, temp.count(i)] for i in self.items] # add count of each element -# self.items.sort(key=lambda x:temp.count(x[0])) # sort by count -# self.items = [i[0] + ' (' + str(i[1]) + ')' for i in self.items] # put element and count into string -# self.items = self.items[::-1] # reverse + self.items.append("+ {0} more".format(more)) + + def orderbyfreq(self): + """ + Order a list by frequency of each item, then remove duplicates and append frequency in parentheses. + """ unsorted = list(self.items) - self.items = [ "{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x)) ] + self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))] class PlaintextLine: """ - Draw a horizontal line for plain text format, with optional padding/styling + Draw a horizontal line for plain text format, with optional padding/styling. """ def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""): @@ -261,6 +314,7 @@ class PlaintextLine: line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1]) return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding + class PlaintextBox: """ Draw a rectangular box around text, with customisable padding/size/style @@ -336,6 +390,7 @@ class PlaintextBox: contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1]) return ('\n').join(contentlines) + def init_varfilter(): global varfilter global varpattern @@ -345,34 +400,35 @@ def init_varfilter(): varpattern = re.compile("|".join(varfilter.keys())) varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))) -def writetitle(title): # write title for a section - if (title == '' or '\n' in title): - logger.error("Invalid title") - raise ValueError - logger.debug("Writing title for " + title) - return tag('h2', 0, title) - -def addtag(l, tag): # add prefix and suffix tags to each item in a list - l2 = ['<' + tag + '>' + i + '' for i in l] - return l2 - def backticks(l): return ["`" + x + "`" for x in l] -def plural(noun, quantity): # return "1 noun" or "n nouns" + +def plural(noun, quantity): + """ + Return "1 noun" or "n nouns" + """ if (quantity == 1): return(str(quantity) + " " + noun) else: return(str(quantity) + " " + noun + "s") -def parsesize(num, suffix='B'): # return human-readable size from number of bytes + +def parsesize(num, suffix='B'): + """ + Return human-readable size from number of bytes + """ for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: if abs(num) < 1024.0: return "%3.1f %s%s" % (num, unit, suffix) num /= 1024.0 return "%.1f%s%s" % (num, 'Yi', suffix) -def fsubject(template): # Replace variables in the title template provided in config + +def fsubject(template): + """ + Replace variables in the title template provided in config + """ r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template) logger.debug("Returning subject line " + r) return r diff --git a/logparse/interface.py b/logparse/interface.py index a9bd2e1..a4cb5ca 100644 --- a/logparse/interface.py +++ b/logparse/interface.py @@ -135,7 +135,7 @@ def main(): else: dest_path = prefs['output'] logger.debug("Outputting to {0}".format(dest_path)) - if (argparser.parse_args().embed_styles or prefs['embed-styles']) and not (argparser.parse_args.plain or prefs['plain']): + if (argparser.parse_args().embed_styles or prefs['embed-styles']) and not (argparser.parse_args().plain or prefs['plain']): output.embed_css(prefs['css']) if (not os.path.isfile(dest_path)) and not (argparser.parse_args().overwrite or config['overwrite']): output.write(dest_path) @@ -151,9 +151,8 @@ def main(): to = argparser.parse_args().to else: to = prefs['mail']['to'] - if argparser.parse_args().plain or prefs['plain']: - mail.sendmail(mailbin=prefs['mail']['mailbin'], body=(output.embed_css(prefs['css']) if isinstance(output, formatting.HtmlOutput) else output.content), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject'])) - + mail.sendmail(mailbin=prefs['mail']['mailbin'], body=(output.embed_css(prefs['css']) if isinstance(output, formatting.HtmlOutput) else output.content), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject']), html=isinstance(output, formatting.HtmlOutput)) + # Print end message finish = datetime.now() logger.info("Finished parsing logs at {0} {1} (total time: {2})".format(finish.strftime(formatting.DATEFMT), finish.strftime(formatting.TIMEFMT), finish - start)) diff --git a/logparse/mail.py b/logparse/mail.py index ecd7afc..439b9c7 100644 --- a/logparse/mail.py +++ b/logparse/mail.py @@ -24,13 +24,13 @@ def mailprep(htmlin, stylesheet): return htmlout -def sendmail(mailbin, body, recipient, subject, *sender): +def sendmail(mailbin, body, recipient, subject, html=True, *sender): logger.debug("Sending email") - msg = MIMEText(body, 'html') + msg = MIMEText(body, 'html' if html else 'plain') if sender: msg["From"] = sender msg["To"] = recipient - msg["Content-type"] = "text/html: te: text/html" + msg["Content-type"] = "text/html: te: text/html" if html else "text/plain: te: text/plain" msg["Subject"] = subject mailproc = subprocess.Popen([mailbin, "--debug-level=" + str(10 if logging.root.level == logging.DEBUG else 0), "-t"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) logger.debug("Compiled message and opened process") diff --git a/logparse/parsers/httpd.py b/logparse/parsers/httpd.py index 7175ea8..d0e800c 100644 --- a/logparse/parsers/httpd.py +++ b/logparse/parsers/httpd.py @@ -60,18 +60,20 @@ def parse_log(): if (ips != None): logger.debug("Parsing client statistics") client_data = Data() - client_data.items = orderbyfreq(ips) - client_data.subtitlte = plural(" client", str(len(ips))) + client_data.items = ips + client_data.orderbyfreq() + client_data.subtitle = plural(" client", str(len(ips))) client_data.truncl(config.prefs['maxlist']) section.append_data(client_data) if (useragents != None): logger.debug("Parsing user agent statistics") ua_data = Data() - ua_data.items = orderbyfreq(useragents) + ua_data.items = useragents + ua_data.orderbyfreq() n_ua = str(len(ua_data.items)) ua_data.truncl(config.prefs['maxlist']) ua_data.subtitle = plural(" user agent", n_ua) - section.append_data(client_data) + section.append_data(ua_data) section.append_data(Data(data_h + " transferred")) section.append_data(Data(plural(" error", e))) diff --git a/logparse/parsers/temperature.py b/logparse/parsers/temperature.py index dfebb9c..9d08119 100644 --- a/logparse/parsers/temperature.py +++ b/logparse/parsers/temperature.py @@ -18,6 +18,7 @@ import socket, sys from telnetlib import Telnet from typing import List, Dict, NamedTuple +from logparse import formatting from ..formatting import * from ..util import readlog, resolve from ..config import * @@ -77,36 +78,35 @@ def parse_log(): sensors.init() - coretemps = [] - pkgtemp = 0 - systemp = 0 - - systemp_data = Data("Sys") - coretemp_data = Data("Cores") - pkgtemp_data = Data("Processor") + systemp = Data("Sys", []) + coretemp = Data("Cores", []) + pkgtemp = Data("Processor", []) try: - for chip in sensors.iter_detected_chips(): for feature in chip: if "Core" in feature.label: - coretemp_data.items.append([feature.label, feature.get_value()]) - logger.debug("Found core " + feature.label + " at temp " + str(feature.get_value())) + coretemp.items.append([feature.label, float(feature.get_value())]) + continue if "CPUTIN" in feature.label: - pkgtem_data.items.append([feature.label, str(feature.get_value())]) - logger.debug("Found CPU package at temp" + str(feature.get_value())) + pkgtemp.items.append([feature.label, float(feature.get_value())]) + continue if "SYS" in feature.label: - systemp_data.items.append([feature.label, str(feature.get_value())]) - logger.debug("Found sys input " + feature.label + " at temp " + str(feature.get_value())) - - for temp_data in [systemp_data, coretemp_data, pkgtemp_data]: + systemp.items.append([feature.label, float(feature.get_value())]) + continue + + logger.debug("Core data is {0}".format(str(coretemp.items))) + logger.debug("Sys data is {0}".format(str(systemp.items))) + logger.debug("Pkg data is {0}".format(str(pkgtemp.items))) + for temp_data in [systemp, coretemp, pkgtemp]: + logger.debug("Looking at temp data {0}".format(str(temp_data.items))) if len(temp_data.items) > 1: - avg = sum(feature[1] for feature in temp_data.items) / len(temp_data.items) + avg = float(sum(feature[1] for feature in temp_data.items)) / len(temp_data.items) logger.debug("Avg temp for {0} is {1} {2}{3}".format(temp_data.subtitle, str(avg), DEG, CEL)) - temp_data.subtitle += " (avg {0}{1}{2}):".format(str(avg), DEG, CEL) - temp_data.items = ["{0}: {1}{2}{3}".format(feature[0], feature[1], DEG, CEL) for feature in temp_data] + temp_data.subtitle += " (avg {0}{1}{2})".format(str(avg), DEG, CEL) + temp_data.items = ["{0}: {1}{2}{3}".format(feature[0], str(feature[1]), DEG, CEL) for feature in temp_data.items] else: - temp_data.items = temp_data[0][1] + DEG + CEL + temp_data.items = [str(temp_data.items[0][1]) + DEG + CEL] section.append_data(temp_data) finally: diff --git a/logparse/parsers/zfs.py b/logparse/parsers/zfs.py index 09db33e..5e805a2 100644 --- a/logparse/parsers/zfs.py +++ b/logparse/parsers/zfs.py @@ -26,16 +26,21 @@ import logging logger = logging.getLogger(__name__) def parse_log(): - output = '' + logger.debug("Starting zfs section") - output += opentag('div', 1, 'zfs', 'section') + section = Section("zfs") + zfslog = readlog(config.prefs['logs']['zfs']) + logger.debug("Analysing zpool log") pool = re.search('.*---\n(\w*)', zfslog).group(1) scrub = re.search('.* scrub repaired (\d+\s*\w+) in .* with (\d+) errors on (\w+)\s+(\w+)\s+(\d+)\s+(\d{1,2}:\d{2}):\d+\s+(\d{4})', zfslog) logger.debug("Found groups {0}".format(scrub.groups())) iostat = re.search('.*---\n\w*\s*(\S*)\s*(\S*)\s', zfslog) scrubrepairs = scruberrors = scrubdate = None + alloc = iostat.group(1) + free = iostat.group(2) + try: scrubrepairs = scrub.group(1) scruberrors = scrub.group(2) @@ -43,16 +48,15 @@ def parse_log(): except Exception as e: logger.debug("Error getting scrub data: " + str(e)) traceback.print_exc(limit=2, file=sys.stdout) - alloc = iostat.group(1) - free = iostat.group(2) - output += writetitle("zfs") + if (scrubdate != None): - subtitle = "Scrub of " + pool + " on " + scrubdate - data = [scrubrepairs + " repaired", scruberrors + " errors", alloc + " used", free + " free"] + scrub_data = Data("Scrub of " + pool + " on " + scrubdate) + scrub_data.items = [scrubrepairs + " repaired", scruberrors + " errors", alloc + " used", free + " free"] else: - subtitle = pool - data = [alloc + " used", free + " free"] - output += writedata(subtitle, data) - output += closetag('div', 1) + scrub_data = Data(pool) + scrub_data.items = [alloc + " used", free + " free"] + + section.append_data(scrub_data) + logger.info("Finished zfs section") - return output + return section -- 2.43.2