From 3dc3cc563cbaf5608719a40129d9846032724c8a Mon Sep 17 00:00:00 2001 From: Andrew Lorimer Date: Fri, 30 Aug 2019 14:20:30 +1000 Subject: [PATCH] add table implementation --- logparse/config.py | 2 + logparse/formatting.py | 314 ++++++++++++++++++++++++++----- logparse/interface.py | 2 +- logparse/mail.py | 2 +- logparse/parsers/load_parsers.py | 2 +- setup.py | 2 +- 6 files changed, 274 insertions(+), 50 deletions(-) diff --git a/logparse/config.py b/logparse/config.py index 3dc2e46..442de85 100644 --- a/logparse/config.py +++ b/logparse/config.py @@ -87,6 +87,8 @@ defaults = Configuration({ 'logs': { 'auth': '/var/log/auth.log', 'cron': '/var/log/cron.log', + 'cpuinfo': '/proc/cpuinfo', + 'meminfo': '/proc/meminfo', 'sys': '/var/log/syslog', 'smb': '/var/log/samba', 'zfs': '/var/log/zpool.log', diff --git a/logparse/formatting.py b/logparse/formatting.py index 45d2e3f..c7b7c30 100644 --- a/logparse/formatting.py +++ b/logparse/formatting.py @@ -11,6 +11,8 @@ import os import re import locale from string import Template +from math import floor, ceil +from tabulate import tabulate import logparse from . import interface, util, config, mail @@ -29,6 +31,8 @@ CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔'] CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌'] LINECHARS_DOUBLE = ['║', '═'] LINECHARS_SINGLE = ['│', '─'] +JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬'] +JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼'] BULLET = "• " INDENT = " " @@ -99,6 +103,9 @@ class PlaintextOutput(Output): for data in section.data: self.append(self._fmt_data(data.subtitle, data.items)) self.append('\n') + for table in section.tables: + self.append(table.draw_plain()) + self.append("\n") def _fmt_data(self, subtitle, data = None): # write title and data """ @@ -139,7 +146,6 @@ class PlaintextOutput(Output): itemoutput += datum + '\n' return itemoutput - class HtmlOutput(Output): """ Process and output data in HTML format. @@ -165,25 +171,27 @@ class HtmlOutput(Output): init_varfilter() headercontent = Template(open(template, 'r').read()) self.append(headercontent.safe_substitute(varsubst)) - self.append(self.opentag('div', id='main')) + self.append(opentag('div', id='main')) def append_footer(self): """ Close HTML tags that were opened in the template. TODO: add footer template similar to header template. """ - self.append(self.closetag('div') + self.closetag('body') + self.closetag('html')) + self.append(closetag('div') + closetag('body') + closetag('html')) def append_section(self, section): """ Call the appropriate methods to generate HTML tags for a section (provided by a parser). This should be run by interface.py after every instance of parse_log(). """ - self.append(self.opentag('div', 1, section.title, 'section')) + self.append(opentag('div', 1, section.title, 'section')) self.append(self._gen_title(section.title)) for data in section.data: self.append(self._fmt_data(data.subtitle, data.items)) - self.append(self.closetag('div', 1)) + for table in section.tables: + self.append(table.draw_html()) + self.append(closetag('div', 1)) def _gen_title(self, title): """ @@ -193,7 +201,7 @@ class HtmlOutput(Output): logger.error("Invalid title") raise ValueError logger.debug("Writing title for " + title) - return self.tag('h2', False, title) + return tag('h2', False, title) def _fmt_data(self, subtitle, data = None): """ @@ -206,59 +214,25 @@ class HtmlOutput(Output): if (data == None or len(data) == 0): logger.debug("No data provided.. just printing subtitle") - return self.tag('p', False, subtitle) + return tag('p', False, subtitle) else: logger.debug("Received data " + str(data)) subtitle += ':' if (len(data) == 1): - return self.tag('p', False, subtitle + ' ' + data[0]) + return tag('p', False, subtitle + ' ' + data[0]) else: output = "" - output += self.tag('p', False, subtitle) - output += self.opentag('ul', 1) + output += tag('p', False, subtitle) + output += opentag('ul', 1) coderegex = re.compile('`(.*)`') for datum in data: if datum == "" or datum == None: continue datum = coderegex.sub(r"\1", str(datum)) - output += self.tag('li', False, datum) - output += self.closetag('ul', True) + output += tag('li', False, datum) + output += closetag('ul', True) return output - def opentag(self, tag, block=False, id=None, cl=None): - """ - Write HTML opening tag - """ - output = "" - if (block): - output += '\n' - output += '<' + tag - if (id != None): - output += " id='" + id + "'" - if (cl != None): - output += " class='" + cl + "'" - output += '>' - if (block): - output += '\n' - return output - - def closetag(self, tag, block=False): - """ - Write HTML closing tag - """ - if block: - return "\n\n" - else: - return "" - - def tag(self, tag, block=False, content=""): - """ - Write HTML opening tag, content, and closing tag - """ - o = self.opentag(tag, block) - c = self.closetag(tag, block) - return o + content + c - class Section: """ @@ -268,10 +242,14 @@ class Section: def __init__(self, title): self.title = title self.data = [] + self.tables = [] def append_data(self, data): self.data.append(data) + def append_table(self, table): + self.tables.append(table) + class Data: """ @@ -298,6 +276,211 @@ class Data: unsorted = list(self.items) self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))] +class Table(object): + """ + A wrapper for python-tabulate's Tabulate type. + """ + + def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]): + self.rows = [] # List of row objects + self.keymap = {} # For fast lookup of row by value of first column + self.double = double + self.borders = borders + self.align_cols = [] + self.hpadding = hpadding + self.maxwidth = maxwidth + self.headers = headers + self._align_cols = [] + + def add_row(self, row): + + self.rows.append(row) + if len(row.columns) > 0: + self.keymap[row.columns[0]] = row + + logger.debug("Added row with {0} columns".format(str(len(row.columns)))) + + def align_column(self, i, align): + while len(self._align_cols) -1 < i: + self._align_cols.append("") + self._align_cols[i] = align + for row in self.rows: + row.columns[i].align = align + logger.debug("Column alignment is now {0}".format(str(self._align_cols))) + + def _gen_list(self): + hierarchy = [] + for row in self.rows: + row_data = [] + for column in row.columns: + row_data.append(column.content) + hierarchy.append(row_data) + return hierarchy + + def draw_html(self): + output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols)) + return output + + def draw_plain(self): + output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols)) + return output + + +class Table0(object): + """ + A two-dimensional information display. + This is a hacky implementation - Table() now relies on the Tabulate package which is much more reliable. + """ + + def __init__(self, double=False, borders=True, hpadding="+", maxwidth=80): + self.rows = [] # List of row objects + self.keymap = {} # For fast lookup of row by value of first column + self.double = double + self.borders = borders + self.align_cols = [] + self.hpadding = hpadding + self.maxwidth = maxwidth + self._colwidths = [] + + def add_row(self, row): + self.rows.append(row) + for i, col in enumerate(row.columns): + if len(self._colwidths) >= i + 1: + self._colwidths[i] = max([self._colwidths[i], len(col.content)]) + else: + self._colwidths.append(len(col.content)) + logger.debug("Added row with {0} columns. Column widths are now {1}.".format(str(len(row.columns)), str(self._colwidths))) + if len(row.columns) > 0: + self.keymap[row.columns[0]] = row + + def align_column(self, i, align): + for row in self.rows: + row.columns[i].align = align + + + def draw_html(self): + output = "" + output += opentag("table", True, cl="data_table") + for row in self.rows: + if row.header: + output += opentag("th", block=True, cl="header") + else: + output += opentag("tr", block=True) + for column in row.columns: + output += tag("td", content=column.content, style={"text-align": column.align} if column.align else {}) + if row.header: + output += closetag("th", True) + else: + output += closetag("tr", True) + output += closetag("table", True) + logger.debug("Built table with {0} rows and {1} columns".format(str(len(self.rows)), str(max([x.n for x in self.rows])))) + return output + + def draw_plain(self): + output = "" + cols = [list(x) for x in zip(self.rows)] + logger.debug("Cols are " + str(cols)) + + if self.double == True: + cornerchars = CORNERCHARS_DOUBLE + linechars = LINECHARS_DOUBLE + jxnchars = JXNCHARS_DOUBLE + else: + cornerchars = CORNERCHARS_SINGLE + linechars = LINECHARS_SINGLE + jxnchars = JXNCHARS_SINGLE + + lengths = [] + row_lengths = [] + for row in self.rows: + for i, col in enumerate(row.columns): + if len(lengths) >= i + 1: + lengths[i] = max([lengths[i], len(col.content)]) + else: + lengths.append(len(col.content)) + + logger.debug("Lengths are " + str(lengths)) + + for i, row in enumerate(self.rows): + l = (len(INDENT) + len(self.hpadding)*2*len(row.columns) + ((1+len(row.columns)) if self.borders else 0) + sum([len(col.content) for col in row.columns])) + if l > self.maxwidth: + logger.debug("Line overflow for cell in row {0} of table".format(str(i))) + words = row.columns[-1].content.split() + if max(map(len, words)) > self.maxwidth: + continue + res, part, others = [], words[0], words[1:] + for word in others: + if l - len(word) < self.maxwidth: + res.append(part) + part = word + else: + part += ' ' + word + if part: + res.append(part) + self._colwidths[-1] = max([len(f) for f in res] + [len(r.columns[-1].content) for r in self.rows if r != row]) + if self.borders: + row.columns[-1].content = res[0][:-1] + self.hpadding + " "*(self._colwidths[-1]-len(res[0])+1) + linechars[0] + for fragment in res[1:]: + row.columns[-1].content += "\n" + INDENT + "".join([(linechars[0] + self.hpadding + " "*x + self.hpadding) for x in lengths]) + linechars[0] + self.hpadding + fragment + " " * (max(self._colwidths) - len(fragment)) + + if self.borders: + top = INDENT + cornerchars[3] + jxnchars[2].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[2] + "\n" + bottom = INDENT + cornerchars[0] + jxnchars[3].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[1] + "\n" + rowtext = INDENT + linechars[0] + linechars[0].join("{:>%d}" % l for l in self._colwidths) + linechars[0] + "\n" + line = INDENT + jxnchars[0] + jxnchars[4].join(linechars[1] * (l+2) for l in self._colwidths) + jxnchars[1] + "\n" + else: + top = bottom = line = "" + rowtext = " ".join("{:>%d}" % l for l in self._colwidths) + "\n" + + for i, row in enumerate(self.rows): + logger.debug("Processing row {0} of {1}".format(str(i), str(len(self.rows)-1))) + row_output = "" + if i == 0: + row_output += top + row_output += (INDENT + linechars[0] if self.borders else "") + for j, column in enumerate(row.columns): + if column.align == "right": + cell_output = self.hpadding + " "*(self._colwidths[j]-len(column.content)) + column.content + self.hpadding + (linechars[0] if self.borders else "") + elif column.align == "left": + cell_output = self.hpadding + column.content + " "*(self._colwidths[j]-len(column.content)) + self.hpadding + (linechars[0] if self.borders else "") + elif column.align == "center": + n_whitespace = (self._colwidths[j]-len(column.content))/2 + cell_output = self.hpadding + " "*(floor(n_whitespace) if len(column.content) % 2 == 0 else ceil(n_whitespace)) + column.content + " "*(ceil(n_whitespace) if len(column.content) % 2 == 0 else floor(n_whitespace)) + self.hpadding + (linechars[0] if self.borders else "") + else: + logger.warning("Couldn't find alignment value for cell {0} of row {1} with content \"{2}\"".format(str(j), str(i), column.content())) + continue + row_output += cell_output + if len(row_output) > self.maxwidth: + logger.warning("Line overflow for row {0} of table".format(str(i))) + + output += row_output + "\n" + if i == len(self.rows)-1: + output += (bottom if self.borders else "") + else: + output += (line if self.borders else "") + + return output + +class Row(object): + + def __init__(self, columns=[], header=False): + self.columns = columns + self.header = header + self.n = len(self.columns) + + def add_column(self, column): + self.columns.append(column) + self.n += 1 + + def rm_column(self, column): + self.remove(column) + self.n -= 1 + +class Column(object): + + def __init__(self, content="", align="right"): + self.content = content + self.align = align class PlaintextLine: """ @@ -432,3 +615,42 @@ def fsubject(template): r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template) logger.debug("Returning subject line " + r) return r + +def opentag(tag, block=False, id=None, cl=None, style=None): + """ + Write HTML opening tag + """ + output = "" + if block: + output += '\n' + output += '<' + tag + if id: + output += " id='" + id + "'" + if cl: + output += " class='" + cl + "'" + if style: + output += " style='" + output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items()) + output += "'" + output += '>' + if block: + output += '\n' + return output + +def closetag(tag, block=False): + """ + Write HTML closing tag + """ + if block: + return "\n\n" + else: + return "" + +def tag(tag, block=False, content="", id=None, cl=None, style=None): + """ + Write HTML opening tag, content, and closing tag + """ + o = opentag(tag, block, id, cl, style) + c = closetag(tag, block) + return o + content + c + diff --git a/logparse/interface.py b/logparse/interface.py index a4cb5ca..3c2b1a2 100644 --- a/logparse/interface.py +++ b/logparse/interface.py @@ -151,7 +151,7 @@ def main(): to = argparser.parse_args().to else: to = prefs['mail']['to'] - mail.sendmail(mailbin=prefs['mail']['mailbin'], body=(output.embed_css(prefs['css']) if isinstance(output, formatting.HtmlOutput) else output.content), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject']), html=isinstance(output, formatting.HtmlOutput)) + mail.sendmail(mailbin=prefs['mail']['mailbin'], body=(output.embed_css(prefs['css']) if isinstance(output, formatting.HtmlOutput) else output.content), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject']), html=isinstance(output, formatting.HtmlOutput), sender=prefs['mail']['from']) # Print end message finish = datetime.now() diff --git a/logparse/mail.py b/logparse/mail.py index 439b9c7..26a402e 100644 --- a/logparse/mail.py +++ b/logparse/mail.py @@ -24,7 +24,7 @@ def mailprep(htmlin, stylesheet): return htmlout -def sendmail(mailbin, body, recipient, subject, html=True, *sender): +def sendmail(mailbin, body, recipient, subject, html=True, sender=""): logger.debug("Sending email") msg = MIMEText(body, 'html' if html else 'plain') if sender: diff --git a/logparse/parsers/load_parsers.py b/logparse/parsers/load_parsers.py index e2e6ab4..0dc291a 100644 --- a/logparse/parsers/load_parsers.py +++ b/logparse/parsers/load_parsers.py @@ -14,7 +14,7 @@ from typing import NamedTuple parser_dir = "/usr/share/logparse/" main_module = "__init__" -default_parsers = ["cron", "httpd", "postfix", "smbd", "sshd", "sudo", "temperature", "zfs"] +default_parsers = ["cron", "httpd", "mem", "postfix", "smbd", "sshd", "sudo", "sysinfo", "temperature", "zfs"] import logging logger = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index 5790025..b439f41 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( keywords='logparse log parse analysis summary monitor email server', packages=['logparse', 'logparse.parsers'], python_requires='>=3', # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires - install_requires=['premailer', 'requests', 'pyyaml'], # https://packaging.python.org/en/latest/requirements.html + install_requires=['premailer', 'requests', 'pyyaml', 'tabulate'], # https://packaging.python.org/en/latest/requirements.html data_files=[('/etc/logparse', ['logparse.conf', 'header.html', 'main.css'])], # installed to /etc/logparse project_urls={ 'Readme': 'https://git.lorimer.id.au/logparse.git/about', -- 2.43.2