add table implementation
authorAndrew Lorimer <andrew@charles.cortex>
Fri, 30 Aug 2019 04:20:30 +0000 (14:20 +1000)
committerAndrew Lorimer <andrew@charles.cortex>
Fri, 30 Aug 2019 04:20:30 +0000 (14:20 +1000)
logparse/config.py
logparse/formatting.py
logparse/interface.py
logparse/mail.py
logparse/parsers/load_parsers.py
setup.py
index 3dc2e46a3eaaa1a7f13971cb2277ee01a9c23c6b..442de85bb80be292e21ab1643f940345fa175603 100644 (file)
@@ -87,6 +87,8 @@ defaults = Configuration({
     'logs': {
         'auth': '/var/log/auth.log',
         'cron': '/var/log/cron.log',
+        'cpuinfo': '/proc/cpuinfo',
+        'meminfo': '/proc/meminfo',
         'sys': '/var/log/syslog',
         'smb': '/var/log/samba',
         'zfs': '/var/log/zpool.log',
index 45d2e3fa383583397d158d05d6c83d7a0c5426aa..c7b7c30388a025aa612623ec4a9f5bab59903ff9 100644 (file)
@@ -11,6 +11,8 @@ import os
 import re
 import locale
 from string import Template
+from math import floor, ceil
+from tabulate import tabulate
 
 import logparse
 from . import interface, util, config, mail
@@ -29,6 +31,8 @@ CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
 CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
 LINECHARS_DOUBLE = ['║', '═']
 LINECHARS_SINGLE = ['│', '─']
+JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
+JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
 BULLET = "• "
 INDENT = "  "
 
@@ -99,6 +103,9 @@ class PlaintextOutput(Output):
         for data in section.data:
             self.append(self._fmt_data(data.subtitle, data.items))
             self.append('\n')
+        for table in section.tables:
+            self.append(table.draw_plain())
+        self.append("\n")
 
     def _fmt_data(self, subtitle, data = None):   # write title and data
         """
@@ -139,7 +146,6 @@ class PlaintextOutput(Output):
                     itemoutput += datum + '\n'
                 return itemoutput
 
-
 class HtmlOutput(Output):
     """
     Process and output data in HTML format.
@@ -165,25 +171,27 @@ class HtmlOutput(Output):
         init_varfilter()
         headercontent = Template(open(template, 'r').read())
         self.append(headercontent.safe_substitute(varsubst))
-        self.append(self.opentag('div', id='main'))
+        self.append(opentag('div', id='main'))
 
     def append_footer(self):
         """
         Close HTML tags that were opened in the template.
         TODO: add footer template similar to header template.
         """
-        self.append(self.closetag('div') + self.closetag('body') + self.closetag('html'))
+        self.append(closetag('div') + closetag('body') + closetag('html'))
 
     def append_section(self, section):
         """
         Call the appropriate methods to generate HTML tags for a section (provided by a parser).
         This should be run by interface.py after every instance of parse_log().
         """
-        self.append(self.opentag('div', 1, section.title, 'section'))
+        self.append(opentag('div', 1, section.title, 'section'))
         self.append(self._gen_title(section.title))
         for data in section.data:
             self.append(self._fmt_data(data.subtitle, data.items))
-        self.append(self.closetag('div', 1))
+        for table in section.tables:
+            self.append(table.draw_html())
+        self.append(closetag('div', 1))
 
     def _gen_title(self, title):
         """
@@ -193,7 +201,7 @@ class HtmlOutput(Output):
             logger.error("Invalid title")
             raise ValueError 
         logger.debug("Writing title for " + title)
-        return self.tag('h2', False, title)
+        return tag('h2', False, title)
 
     def _fmt_data(self, subtitle, data = None):
         """
@@ -206,59 +214,25 @@ class HtmlOutput(Output):
 
         if (data == None or len(data) == 0):
             logger.debug("No data provided.. just printing subtitle")
-            return self.tag('p', False, subtitle)
+            return tag('p', False, subtitle)
         else:
             logger.debug("Received data " + str(data))
             subtitle += ':'
             if (len(data) == 1):
-                return self.tag('p', False, subtitle + ' ' + data[0])
+                return tag('p', False, subtitle + ' ' + data[0])
             else:
                 output = ""
-                output += self.tag('p', False, subtitle)
-                output += self.opentag('ul', 1)
+                output += tag('p', False, subtitle)
+                output += opentag('ul', 1)
                 coderegex = re.compile('`(.*)`')
                 for datum in data:
                     if datum == "" or datum == None:
                         continue
                     datum = coderegex.sub(r"<code>\1</code>", str(datum))
-                    output += self.tag('li', False, datum)
-                output += self.closetag('ul', True)
+                    output += tag('li', False, datum)
+                output += closetag('ul', True)
                 return output
 
-    def opentag(self, tag, block=False, id=None, cl=None):
-        """
-        Write HTML opening tag
-        """
-        output = ""
-        if (block):
-            output += '\n'
-        output += '<' + tag
-        if (id != None):
-            output += " id='" + id + "'"
-        if (cl != None):
-            output += " class='" + cl + "'"
-        output += '>'
-        if (block):
-            output += '\n'
-        return output
-
-    def closetag(self, tag, block=False):
-        """
-        Write HTML closing tag
-        """
-        if block:
-            return "\n</" + tag + ">\n"
-        else:
-            return "</" + tag + ">"
-
-    def tag(self, tag, block=False, content=""):
-        """
-        Write HTML opening tag, content, and closing tag
-        """
-        o = self.opentag(tag, block)
-        c = self.closetag(tag, block)
-        return o + content + c
-
 
 class Section:
     """
@@ -268,10 +242,14 @@ class Section:
     def __init__(self, title):
         self.title = title
         self.data = []
+        self.tables = []
 
     def append_data(self, data):
         self.data.append(data)
 
+    def append_table(self, table):
+        self.tables.append(table)
+
 
 class Data:
     """
@@ -298,6 +276,211 @@ class Data:
         unsorted = list(self.items)
         self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
 
+class Table(object):
+    """
+    A wrapper for python-tabulate's Tabulate type.
+    """
+    
+    def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]):
+        self.rows =  []     # List of row objects
+        self.keymap = {}    # For fast lookup of row by value of first column 
+        self.double = double
+        self.borders = borders
+        self.align_cols = []
+        self.hpadding = hpadding
+        self.maxwidth = maxwidth
+        self.headers = headers
+        self._align_cols = []
+
+    def add_row(self, row):
+
+        self.rows.append(row)
+        if len(row.columns) > 0:
+            self.keymap[row.columns[0]] = row
+
+        logger.debug("Added row with {0} columns".format(str(len(row.columns))))
+
+    def align_column(self, i, align):
+        while len(self._align_cols) -1 < i:
+            self._align_cols.append("")
+        self._align_cols[i] = align
+        for row in self.rows:
+            row.columns[i].align = align
+        logger.debug("Column alignment is now {0}".format(str(self._align_cols)))
+
+    def _gen_list(self):
+        hierarchy = []
+        for row in self.rows:
+            row_data = []
+            for column in row.columns:
+                row_data.append(column.content)
+            hierarchy.append(row_data)
+        return hierarchy
+
+    def draw_html(self):
+        output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols))
+        return output
+
+    def draw_plain(self):
+        output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols))
+        return output
+
+
+class Table0(object):
+    """
+    A two-dimensional information display.
+    This is a hacky implementation - Table() now relies on the Tabulate package which is much more reliable.
+    """
+
+    def __init__(self, double=False, borders=True, hpadding="+", maxwidth=80):
+        self.rows =  []     # List of row objects
+        self.keymap = {}    # For fast lookup of row by value of first column 
+        self.double = double
+        self.borders = borders
+        self.align_cols = []
+        self.hpadding = hpadding
+        self.maxwidth = maxwidth
+        self._colwidths = []
+
+    def add_row(self, row):
+        self.rows.append(row)
+        for i, col in enumerate(row.columns):
+            if len(self._colwidths) >= i + 1:
+                self._colwidths[i] = max([self._colwidths[i], len(col.content)])
+            else:
+                self._colwidths.append(len(col.content))
+        logger.debug("Added row with {0} columns. Column widths are now {1}.".format(str(len(row.columns)), str(self._colwidths)))
+        if len(row.columns) > 0:
+            self.keymap[row.columns[0]] = row
+    
+    def align_column(self, i, align):
+        for row in self.rows:
+            row.columns[i].align = align
+        
+
+    def draw_html(self):
+        output = ""
+        output += opentag("table", True, cl="data_table") 
+        for row in self.rows:
+            if row.header:
+                output += opentag("th", block=True, cl="header")
+            else:
+                output += opentag("tr", block=True)
+            for column in row.columns:
+                output += tag("td", content=column.content, style={"text-align": column.align} if column.align else {})
+            if row.header:
+                output += closetag("th", True)
+            else:
+                output += closetag("tr", True)
+        output += closetag("table", True)
+        logger.debug("Built table with {0} rows and {1} columns".format(str(len(self.rows)), str(max([x.n for x in self.rows]))))
+        return output
+
+    def draw_plain(self):
+        output = ""
+        cols = [list(x) for x in zip(self.rows)]
+        logger.debug("Cols are " + str(cols))
+
+        if self.double == True:
+            cornerchars = CORNERCHARS_DOUBLE
+            linechars = LINECHARS_DOUBLE
+            jxnchars = JXNCHARS_DOUBLE
+        else:
+            cornerchars = CORNERCHARS_SINGLE
+            linechars = LINECHARS_SINGLE
+            jxnchars = JXNCHARS_SINGLE
+        
+        lengths = []
+        row_lengths = []
+        for row in self.rows:
+            for i, col in enumerate(row.columns):
+                if len(lengths) >= i + 1:
+                    lengths[i] = max([lengths[i], len(col.content)])
+                else:
+                    lengths.append(len(col.content))
+
+        logger.debug("Lengths are " + str(lengths))
+
+        for i, row in enumerate(self.rows):
+            l = (len(INDENT) + len(self.hpadding)*2*len(row.columns) + ((1+len(row.columns)) if self.borders else 0) + sum([len(col.content) for col in row.columns]))
+            if l > self.maxwidth:
+                logger.debug("Line overflow for cell in row {0} of table".format(str(i)))
+                words = row.columns[-1].content.split()
+                if max(map(len, words)) > self.maxwidth:
+                    continue
+                res, part, others = [], words[0], words[1:]
+                for word in others:
+                    if l - len(word) < self.maxwidth:
+                        res.append(part)
+                        part = word
+                    else:
+                        part += ' ' + word
+                if part:
+                    res.append(part)
+                self._colwidths[-1] = max([len(f) for f in res] + [len(r.columns[-1].content) for r in self.rows if r != row])
+                if self.borders:
+                    row.columns[-1].content = res[0][:-1] + self.hpadding + " "*(self._colwidths[-1]-len(res[0])+1) + linechars[0]
+                    for fragment in res[1:]:
+                        row.columns[-1].content += "\n" + INDENT + "".join([(linechars[0] + self.hpadding + " "*x + self.hpadding) for x in lengths]) + linechars[0] + self.hpadding + fragment + " " * (max(self._colwidths) - len(fragment))
+
+        if self.borders:
+            top = INDENT + cornerchars[3] + jxnchars[2].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[2] + "\n"
+            bottom = INDENT + cornerchars[0] + jxnchars[3].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[1] + "\n"
+            rowtext = INDENT + linechars[0] + linechars[0].join("{:>%d}" % l for l in self._colwidths) + linechars[0] + "\n"
+            line = INDENT + jxnchars[0] + jxnchars[4].join(linechars[1] * (l+2) for l in self._colwidths) + jxnchars[1] + "\n"
+        else:
+            top = bottom = line = ""
+            rowtext = " ".join("{:>%d}" % l for l in self._colwidths) + "\n"
+
+        for i, row in enumerate(self.rows):
+            logger.debug("Processing row {0} of {1}".format(str(i), str(len(self.rows)-1)))
+            row_output = ""
+            if i == 0:
+                row_output += top
+            row_output += (INDENT + linechars[0] if self.borders else "")
+            for j, column in enumerate(row.columns):
+                if column.align == "right":
+                    cell_output = self.hpadding + " "*(self._colwidths[j]-len(column.content)) + column.content + self.hpadding + (linechars[0] if self.borders else "")
+                elif column.align == "left":
+                    cell_output = self.hpadding + column.content + " "*(self._colwidths[j]-len(column.content)) + self.hpadding + (linechars[0] if self.borders else "")
+                elif column.align == "center":
+                    n_whitespace = (self._colwidths[j]-len(column.content))/2
+                    cell_output = self.hpadding + " "*(floor(n_whitespace) if len(column.content) % 2 == 0 else ceil(n_whitespace)) + column.content + " "*(ceil(n_whitespace) if len(column.content) % 2 == 0 else floor(n_whitespace)) + self.hpadding + (linechars[0] if self.borders else "")
+                else:
+                    logger.warning("Couldn't find alignment value for cell {0} of row {1} with content \"{2}\"".format(str(j), str(i), column.content()))
+                    continue
+                row_output += cell_output
+                if len(row_output) > self.maxwidth:
+                    logger.warning("Line overflow for row {0} of table".format(str(i)))
+
+            output += row_output + "\n"
+            if i == len(self.rows)-1:
+                output += (bottom if self.borders else "")
+            else:
+                output += (line if self.borders else "")
+
+        return output
+
+class Row(object):
+    
+    def __init__(self, columns=[], header=False):
+        self.columns = columns
+        self.header = header
+        self.n = len(self.columns)
+
+    def add_column(self, column):
+        self.columns.append(column)
+        self.n += 1
+
+    def rm_column(self, column):
+        self.remove(column)
+        self.n -= 1
+
+class Column(object):
+
+    def __init__(self, content="", align="right"):
+        self.content = content
+        self.align = align
 
 class PlaintextLine:
     """
@@ -432,3 +615,42 @@ def fsubject(template):
     r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template)
     logger.debug("Returning subject line " + r)
     return r
+
+def opentag(tag, block=False, id=None, cl=None, style=None):
+    """
+    Write HTML opening tag
+    """
+    output = ""
+    if block:
+        output += '\n'
+    output += '<' + tag
+    if id:
+        output += " id='" + id + "'"
+    if cl:
+        output += " class='" + cl + "'"
+    if style:
+        output += " style='"
+        output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items())
+        output += "'"
+    output += '>'
+    if block:
+        output += '\n'
+    return output
+
+def closetag(tag, block=False):
+    """
+    Write HTML closing tag
+    """
+    if block:
+        return "\n</" + tag + ">\n"
+    else:
+        return "</" + tag + ">"
+
+def tag(tag, block=False, content="", id=None, cl=None, style=None):
+    """
+    Write HTML opening tag, content, and closing tag
+    """
+    o = opentag(tag, block, id, cl, style)
+    c = closetag(tag, block)
+    return o + content + c
+
index a4cb5cae2521266b834fe15899ebd37341ed885a..3c2b1a27351805ce6170f6b07082b4bc8ab5c980 100644 (file)
@@ -151,7 +151,7 @@ def main():
             to = argparser.parse_args().to
         else:
             to = prefs['mail']['to']
-        mail.sendmail(mailbin=prefs['mail']['mailbin'], body=(output.embed_css(prefs['css']) if isinstance(output, formatting.HtmlOutput) else output.content), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject']), html=isinstance(output, formatting.HtmlOutput))
+        mail.sendmail(mailbin=prefs['mail']['mailbin'], body=(output.embed_css(prefs['css']) if isinstance(output, formatting.HtmlOutput) else output.content), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject']), html=isinstance(output, formatting.HtmlOutput), sender=prefs['mail']['from'])
 
     # Print end message
     finish = datetime.now()
index 439b9c79c5a0b6f1a2121c2bc33580ca87b876e3..26a402e44eacd14b5d353e34623c9808016becdf 100644 (file)
@@ -24,7 +24,7 @@ def mailprep(htmlin, stylesheet):
     return htmlout
 
 
-def sendmail(mailbin, body, recipient, subject, html=True, *sender):
+def sendmail(mailbin, body, recipient, subject, html=True, sender=""):
     logger.debug("Sending email")
     msg = MIMEText(body, 'html' if html else 'plain')
     if sender:
index e2e6ab433f0764958e5f9f402a6d5a1b67fe7d86..0dc291ad4db3800af2b35c385d99dce77315d9f6 100644 (file)
@@ -14,7 +14,7 @@ from typing import NamedTuple
 
 parser_dir = "/usr/share/logparse/"
 main_module = "__init__"
-default_parsers = ["cron", "httpd", "postfix", "smbd", "sshd", "sudo", "temperature", "zfs"]
+default_parsers = ["cron", "httpd", "mem", "postfix", "smbd", "sshd", "sudo", "sysinfo", "temperature", "zfs"]
 
 import logging
 logger = logging.getLogger(__name__)
index 5790025acc6ec76a04687df9c89d4080cc27a5c8..b439f41321297db78c2d23423b2d8b74005dd8df 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,7 @@ setup(
     keywords='logparse log parse analysis summary monitor email server',
     packages=['logparse', 'logparse.parsers'],
     python_requires='>=3',              # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires
-    install_requires=['premailer', 'requests', 'pyyaml'],   # https://packaging.python.org/en/latest/requirements.html
+    install_requires=['premailer', 'requests', 'pyyaml', 'tabulate'],   # https://packaging.python.org/en/latest/requirements.html
     data_files=[('/etc/logparse', ['logparse.conf', 'header.html', 'main.css'])],   # installed to /etc/logparse
     project_urls={
         'Readme': 'https://git.lorimer.id.au/logparse.git/about',