From: Andrew Lorimer Date: Mon, 9 Sep 2019 07:09:17 +0000 (+1000) Subject: add more docstrings X-Git-Url: https://git.lorimer.id.au/logparse.git/diff_plain/ab9387721f75e245a0d4b32a64e462b0be8c657a?hp=29c0dc87b75c70ee2ca543506262c43d64d7b183 add more docstrings --- diff --git a/logparse/__init__.py b/logparse/__init__.py index ce4b891..36a590d 100644 --- a/logparse/__init__.py +++ b/logparse/__init__.py @@ -1,2 +1,2 @@ -__version__ = '1.0' +__version__ = '2.0' __name__ = 'logparse' diff --git a/logparse/__main__.py b/logparse/__main__.py index 55f225d..2be6d19 100644 --- a/logparse/__main__.py +++ b/logparse/__main__.py @@ -1,7 +1,6 @@ -# -# __main__.py -# -# Executed when the logparse directory is executed as a script +""" +Executed when the logparse directory is executed as a script +""" from .interface import main main() diff --git a/logparse/config.py b/logparse/config.py index 8d1b21e..df81c4e 100644 --- a/logparse/config.py +++ b/logparse/config.py @@ -1,14 +1,15 @@ -# -# config.py -# -# Default config values and basic wrapper for ConfigParser. New config options -# should be added to the dictionary below, along with appropriate defaults. -# -# Runtime configuration is done through /etc/logparse/logparse.conf (default) -# or the path specified in the "--config" argument. The file uses the INI -# syntax, with general options being declared in the [logparse] section and -# parser-specific options declared in their own sections. -# +""" +This modules contains default config values and basic wrapper for ConfigParser. +New config options should be added to the dictionary below, along with +appropriate defaults. Runtime configuration is done through the config file at +/etc/logparse/logparse.conf (default) or the path specified in the "--config" +argument. The file uses the INI syntax, with general options being declared in +the [logparse] section and parser-specific options declared in their own +sections. + +This module provides the following methods: + - `loadconf()`: set up ConfigParser and process config file +""" from configparser import ConfigParser from pkg_resources import Requirement, resource_filename @@ -87,15 +88,6 @@ defaults = { } } -def locate(filename): - """ - DEPRECATED: draft method for what is now parsers/load_parsers.py. Kept here - for historical purposes. - """ - logger.debug("Searching for {0}".format(filename)) - loc = resource_filename(Requirement.parse(__package__), filename) - logger.debug("Found {0}".format(loc)) - return loc def loadconf(configpaths): """ diff --git a/logparse/formatting.py b/logparse/formatting.py index 7e647a9..acdba9f 100644 --- a/logparse/formatting.py +++ b/logparse/formatting.py @@ -1,13 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# -# format.py -# -# This file contains global functions for formatting and printing data. This -# file should be imported into individual log-parsing scripts located in -# logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or -# emailng data is left to __main__.py. -# + +""" +This file contains global functions for formatting and printing data. This file +should be imported into individual log-parsing scripts located in logs/*. Data +is formatted in HTML or plaintext. Writing to disk and/or emailng data is left +to interface.py. +""" import os import re @@ -24,7 +23,8 @@ logger = logging.getLogger(__name__) locale.setlocale(locale.LC_ALL, '') # inherit system locale -#DEG = "°".encode('unicode_escape') + + DEG = u'\N{DEGREE SIGN}' CEL = "C" TIMEFMT = "%X" @@ -38,9 +38,16 @@ JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼'] BULLET = "• " INDENT = " " + global VARSUBST + def init_var(): + """ + Initialise variable substitution templates (should be called before doing + any substitutions) + """ + global VARSUBST css_path = config.prefs.get("html", "css") if config.prefs.getboolean("html", "css-relpath"): @@ -73,12 +80,14 @@ class Output: """ Add a string """ + self.content += content def write(self, destination=""): """ Output contents into a file """ + if destination == "": destination = self.destination if destination == "": @@ -92,6 +101,7 @@ class Output: """ Echo the contents to the console """ + print() if lines: line = PlaintextLine(linewidth=config.prefs.getint("plain", "linewidth"), double=True) @@ -102,7 +112,6 @@ class Output: print() - class PlaintextOutput(Output): """ Processes & outputs data in a plaintext form which can be read with cat or plaintext email. @@ -133,6 +142,7 @@ class PlaintextOutput(Output): Call the appropriate methods to format a section (provided by a parser). This should be run by interface.py after every instance of parse_log(). """ + self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw()) self.append('\n'*2) for data in section.data: @@ -144,9 +154,12 @@ class PlaintextOutput(Output): def _fmt_data(self, subtitle, data = None): # write title and data """ - Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks. - Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed. + Format the properties of a data object into usable plaintext form with + a few fancy symbols/formatting tricks. Subtitle is required, data is + not. If only subtitle is supplied or subtitle + one data item, a single + line will be printed. """ + if (subtitle == ""): logger.warning("No subtitle provided.. skipping section") return @@ -183,11 +196,15 @@ class PlaintextOutput(Output): class HtmlOutput(Output): """ - Process and output data in HTML format. - All HTML formatting functions now reside in this class to differentiate them from plaintext. + Process and output data in HTML format. All HTML formatting functions now + reside in this class to differentiate them from plain text. """ def __init__(self): + """ + Initialise variables (no parameters required for initialisation) + """ + self.content = "" self.destination = "" self.css = "" @@ -197,6 +214,7 @@ class HtmlOutput(Output): """ Convert stylesheet to inline tags """ + if not self._embedded: self._embedded = mail.mailprep(self.content, css) return self._embedded @@ -205,6 +223,7 @@ class HtmlOutput(Output): """ Insert variables into header template file and append HTML tags """ + headercontent = Template(open(template, 'r').read()) self.append(headercontent.safe_substitute(VARSUBST)) self.append(opentag('div', id='main')) @@ -214,6 +233,7 @@ class HtmlOutput(Output): Close HTML tags that were opened in the template. TODO: add footer template similar to header template. """ + self.append(closetag('div') + closetag('body') + closetag('html')) def append_section(self, section): @@ -221,6 +241,7 @@ class HtmlOutput(Output): Call the appropriate methods to generate HTML tags for a section (provided by a parser). This should be run by interface.py after every instance of parse_log(). """ + self.append(opentag('div', 1, section.title, 'section')) self.append(self._gen_title(section.title)) for data in section.data: @@ -233,6 +254,7 @@ class HtmlOutput(Output): """ Format the title for a section """ + if (title == '' or '\n' in title): logger.error("Invalid title") raise ValueError @@ -244,6 +266,7 @@ class HtmlOutput(Output): Format the properties of a data object into usable HTML tags. Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed. """ + if (subtitle == ""): logger.warning("No subtitle provided.. skipping section") return @@ -289,10 +312,16 @@ class Section: class Data: """ - Each section (parser) can have one or more Data() objects which are essentially glorified lists. + Each section (parser) can have one or more Data() objects which are + essentially glorified lists. """ def __init__(self, subtitle="", items=[]): + """ + Initialise variables. No parameters are enforced upon initialisation, + but at least the subtitle is required for valid output. + """ + self.subtitle = subtitle self.items = items @@ -300,6 +329,7 @@ class Data: """ Truncate self.items to a specified value and state how many items are hidden. """ + if (len(self.items) > limit): more = len(self.items) - limit if more == 1: @@ -309,17 +339,25 @@ class Data: def orderbyfreq(self): """ - Order a list by frequency of each item, then remove duplicates and append frequency in parentheses. + Order a list by frequency of each item, then remove duplicates and + append frequency in parentheses. """ + unsorted = list(self.items) self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))] + class Table(object): """ A wrapper for python-tabulate's Tabulate type. """ def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]): + """ + Initialise variables. Note the keymap is used for a faster index map, + but is not currently used anywhere (may be removed in future). + """ + self.rows = [] # List of row objects self.keymap = {} # For fast lookup of row by value of first column self.double = double @@ -331,6 +369,9 @@ class Table(object): self._align_cols = [] def add_row(self, row): + """ + Append a row to the list and amend index mapping + """ self.rows.append(row) if len(row.columns) > 0: @@ -339,6 +380,10 @@ class Table(object): logger.debug("Added row with {0} columns".format(str(len(row.columns)))) def align_column(self, i, align): + """ + Set alignment for the 'i'th column (`align` should be 'l', 'c' or 'r') + """ + while len(self._align_cols) -1 < i: self._align_cols.append("") self._align_cols[i] = align @@ -347,6 +392,10 @@ class Table(object): logger.debug("Column alignment is now {0}".format(str(self._align_cols))) def _gen_list(self): + """ + Used locally for organising rows and columns into a 2D list structure + """ + hierarchy = [] for row in self.rows: row_data = [] @@ -356,182 +405,92 @@ class Table(object): return hierarchy def draw_html(self): + """ + Output HTML string (wrapper for tabulate) + """ + output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols)) return output def draw_plain(self): + """ + Output plain text string (wrapper for tabulate) + """ + output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols)) return output + "\n"*2 -class Table0(object): +class Row(object): """ - A two-dimensional information display. - This is a hacky implementation - Table() now relies on the Tabulate package which is much more reliable. + Object representing a literal row in a 2D table with the individual cells + in the row represented by columns[]. """ - - def __init__(self, double=False, borders=True, hpadding="+", maxwidth=80): - self.rows = [] # List of row objects - self.keymap = {} # For fast lookup of row by value of first column - self.double = double - self.borders = borders - self.align_cols = [] - self.hpadding = hpadding - self.maxwidth = maxwidth - self._colwidths = [] - - def add_row(self, row): - self.rows.append(row) - for i, col in enumerate(row.columns): - if len(self._colwidths) >= i + 1: - self._colwidths[i] = max([self._colwidths[i], len(col.content)]) - else: - self._colwidths.append(len(col.content)) - logger.debug("Added row with {0} columns. Column widths are now {1}.".format(str(len(row.columns)), str(self._colwidths))) - if len(row.columns) > 0: - self.keymap[row.columns[0]] = row - - def align_column(self, i, align): - for row in self.rows: - row.columns[i].align = align - - - def draw_html(self): - output = "" - output += opentag("table", True, cl="data_table") - for row in self.rows: - if row.header: - output += opentag("th", block=True, cl="header") - else: - output += opentag("tr", block=True) - for column in row.columns: - output += tag("td", content=column.content, style={"text-align": column.align} if column.align else {}) - if row.header: - output += closetag("th", True) - else: - output += closetag("tr", True) - output += closetag("table", True) - logger.debug("Built table with {0} rows and {1} columns".format(str(len(self.rows)), str(max([x.n for x in self.rows])))) - return output - - def draw_plain(self): - output = "" - cols = [list(x) for x in zip(self.rows)] - logger.debug("Cols are " + str(cols)) - - if self.double == True: - cornerchars = CORNERCHARS_DOUBLE - linechars = LINECHARS_DOUBLE - jxnchars = JXNCHARS_DOUBLE - else: - cornerchars = CORNERCHARS_SINGLE - linechars = LINECHARS_SINGLE - jxnchars = JXNCHARS_SINGLE - - lengths = [] - row_lengths = [] - for row in self.rows: - for i, col in enumerate(row.columns): - if len(lengths) >= i + 1: - lengths[i] = max([lengths[i], len(col.content)]) - else: - lengths.append(len(col.content)) - - logger.debug("Lengths are " + str(lengths)) - - for i, row in enumerate(self.rows): - l = (len(INDENT) + len(self.hpadding)*2*len(row.columns) + ((1+len(row.columns)) if self.borders else 0) + sum([len(col.content) for col in row.columns])) - if l > self.maxwidth: - logger.debug("Line overflow for cell in row {0} of table".format(str(i))) - words = row.columns[-1].content.split() - if max(map(len, words)) > self.maxwidth: - continue - res, part, others = [], words[0], words[1:] - for word in others: - if l - len(word) < self.maxwidth: - res.append(part) - part = word - else: - part += ' ' + word - if part: - res.append(part) - self._colwidths[-1] = max([len(f) for f in res] + [len(r.columns[-1].content) for r in self.rows if r != row]) - if self.borders: - row.columns[-1].content = res[0][:-1] + self.hpadding + " "*(self._colwidths[-1]-len(res[0])+1) + linechars[0] - for fragment in res[1:]: - row.columns[-1].content += "\n" + INDENT + "".join([(linechars[0] + self.hpadding + " "*x + self.hpadding) for x in lengths]) + linechars[0] + self.hpadding + fragment + " " * (max(self._colwidths) - len(fragment)) - - if self.borders: - top = INDENT + cornerchars[3] + jxnchars[2].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[2] + "\n" - bottom = INDENT + cornerchars[0] + jxnchars[3].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[1] + "\n" - rowtext = INDENT + linechars[0] + linechars[0].join("{:>%d}" % l for l in self._colwidths) + linechars[0] + "\n" - line = INDENT + jxnchars[0] + jxnchars[4].join(linechars[1] * (l+2) for l in self._colwidths) + jxnchars[1] + "\n" - else: - top = bottom = line = "" - rowtext = " ".join("{:>%d}" % l for l in self._colwidths) + "\n" - - for i, row in enumerate(self.rows): - logger.debug("Processing row {0} of {1}".format(str(i), str(len(self.rows)-1))) - row_output = "" - if i == 0: - row_output += top - row_output += (INDENT + linechars[0] if self.borders else "") - for j, column in enumerate(row.columns): - if column.align == "right": - cell_output = self.hpadding + " "*(self._colwidths[j]-len(column.content)) + column.content + self.hpadding + (linechars[0] if self.borders else "") - elif column.align == "left": - cell_output = self.hpadding + column.content + " "*(self._colwidths[j]-len(column.content)) + self.hpadding + (linechars[0] if self.borders else "") - elif column.align == "center": - n_whitespace = (self._colwidths[j]-len(column.content))/2 - cell_output = self.hpadding + " "*(floor(n_whitespace) if len(column.content) % 2 == 0 else ceil(n_whitespace)) + column.content + " "*(ceil(n_whitespace) if len(column.content) % 2 == 0 else floor(n_whitespace)) + self.hpadding + (linechars[0] if self.borders else "") - else: - logger.warning("Couldn't find alignment value for cell {0} of row {1} with content \"{2}\"".format(str(j), str(i), column.content())) - continue - row_output += cell_output - if len(row_output) > self.maxwidth: - logger.warning("Line overflow for row {0} of table".format(str(i))) - - output += row_output + "\n" - if i == len(self.rows)-1: - output += (bottom if self.borders else "") - else: - output += (line if self.borders else "") - - return output - -class Row(object): def __init__(self, columns=[], header=False): + """ + Initialise variables. The variable n is used locally to keep track of + the row width. + """ + self.columns = columns self.header = header self.n = len(self.columns) def add_column(self, column): + """ + Append a single cell horizontally and increment the cell count + """ + self.columns.append(column) self.n += 1 def rm_column(self, column): + """ + Remove the specified column object and decrement the cell count + """ + self.remove(column) self.n -= 1 + class Column(object): + """ + Object representing a single table cell. This is somewhat of a misnomer - + one column object exists for each cell in the table. Columns are children + of rows. + """ def __init__(self, content="", align="right"): + """ + Initialise variables. The align property sets the alignment of a single + cell ('l', 'c', or 'r'). + """ + self.content = content self.align = align + class PlaintextLine: """ Draw a horizontal line for plain text format, with optional padding/styling. """ def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""): + """ + Initialise variables + """ + self.linewidth = linewidth self.double = double self.vpadding = vpadding self.hpadding = hpadding def draw(self): + """ + Output a plain text string based on the current object parameters + """ + line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1]) return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding @@ -542,6 +501,9 @@ class PlaintextBox: """ def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1): + """ + Initialise variables + """ self.content = content self.fullwidth = fullwidth self.linewidth = linewidth @@ -550,6 +512,11 @@ class PlaintextBox: self.double = double def draw(self): + """ + Output a plain text string based on the current object parameters. This + involves calculating the text width, breaking text at the maximum line + length, and then drawing a box around it all. + """ if self.double == True: cornerchars = CORNERCHARS_DOUBLE @@ -613,6 +580,11 @@ class PlaintextBox: def backticks(l): + """ + Surround every item in a list by backticks. Used for showing code in both + HTML and plain text formats (converted to tags for HTML) + """ + return ["`" + x + "`" for x in l] @@ -620,6 +592,7 @@ def plural(noun, quantity, print_quantity=True): """ Return "1 noun" or "n nouns" """ + if print_quantity: if (quantity == 1): return(str(quantity) + " " + noun) @@ -636,6 +609,7 @@ def parsesize(num, suffix='B'): """ Return human-readable size from number of bytes """ + for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: if abs(num) < 1024.0: return "%3.1f %s%s" % (num, unit, suffix) @@ -647,14 +621,17 @@ def fsubject(subject): """ Replace variables in the title template provided in config """ + r = Template(subject).safe_substitute(VARSUBST) logger.debug("Returning subject line " + r) return r + def opentag(tag, block=False, id=None, cl=None, style=None): """ Write HTML opening tag """ + output = "" if block: output += '\n' @@ -672,19 +649,23 @@ def opentag(tag, block=False, id=None, cl=None, style=None): output += '\n' return output + def closetag(tag, block=False): """ Write HTML closing tag """ + if block: return "\n\n" else: return "" + def tag(tag, block=False, content="", id=None, cl=None, style=None): """ Write HTML opening tag, content, and closing tag """ + o = opentag(tag, block, id, cl, style) c = closetag(tag, block) return o + content + c diff --git a/logparse/logio.py b/logparse/logio.py deleted file mode 100644 index e69de29..0000000 diff --git a/logparse/mail.py b/logparse/mail.py index 26a402e..82e3ddb 100644 --- a/logparse/mail.py +++ b/logparse/mail.py @@ -1,9 +1,12 @@ -# -# email.py -# -# This module is essentially a wrapper for Python's premailer and whatever -# the default mail handler is. -# +""" +This module is essentially a wrapper for Python's premailer and whatever the +default mail transfer is (usually Postfix). Note that the premailer package +(https://pypi.org/project/premailer/) is required for style embedding. + +This module provides the following methods: + - `mailprep`: embed CSS styles into inline HTML tags + - `sendmail`: send HTML or plaintext email using default mail handler +""" import logging logger = logging.getLogger(__name__) @@ -13,7 +16,13 @@ import premailer from email.mime.text import MIMEText import subprocess + def mailprep(htmlin, stylesheet): + """ + Embed CSS styles from a file into inline HTML tags. Requires the premailer + package (https://pypi.org/project/premailer/). + """ + logger.debug("Converting stylesheet " + stylesheet + " to inline tags") if not isfile(stylesheet): logger.warning("Cannot read stylesheet {}: file does not exist".format(stylesheet)) @@ -25,6 +34,12 @@ def mailprep(htmlin, stylesheet): def sendmail(mailbin, body, recipient, subject, html=True, sender=""): + """ + Prepare and send an email in either HTML or plain text format. The default + MTA path is usually correct, but can be modified in the config option + "mailbin" in the [mail] section. + """ + logger.debug("Sending email") msg = MIMEText(body, 'html' if html else 'plain') if sender: @@ -39,12 +54,6 @@ def sendmail(mailbin, body, recipient, subject, html=True, sender=""): logger.debug("sendmail output: {}".format(stdout)) logger.info("Sent email to {0}".format(recipient)) return 0 -# except TimeoutExpired: -# mailproc.kill() -# stdout = mailproc.communicate() -# logger.debug("Timeout expired: {}".format(stdout)) -# raise subprocess.TimeoutError except Exception as e: mailproc.kill() logger.warning("Failed to send message: {0}".format(str(e))) -# raise ChildProcessError diff --git a/logparse/util.py b/logparse/util.py index 8f905f4..3aca904 100644 --- a/logparse/util.py +++ b/logparse/util.py @@ -1,43 +1,65 @@ -# -# utilities.py -# -# Commonly used general functions -# +""" +Commonly used general functions. -import re +This module provides the following methods: + - `hostname`: get the current machine's hostname + - `getlocaldomain`: get the current machine's domain name + - `resolve`: attempt to convert a local/public IP to hostname + - `readlog`: read contents of a log file from disk +""" + +from datetime import datetime, timedelta +import inspect +import logging import os +from pkg_resources import Requirement, resource_filename +import re import socket -import inspect from systemd import journal -from datetime import datetime, timedelta -import logging -logger = logging.getLogger(__name__) +from logparse import config -from pkg_resources import Requirement, resource_filename -from logparse import config +logger = logging.getLogger(__name__) + def hostname(path): # get the hostname of current server + """ + Get the hostname of the current machine using the file supplied in the + `hostname-path` config option. + """ + hnfile = open(path, 'r') hn = re.search('^(\w*)\n*', hnfile.read()).group(1) return hn + def getlocaldomain(): # get the parent fqdn of current server - domain = socket.getfqdn().split('.', 1) # Note: if socket.fetfqdn() returns localhost, make sure the first entry in /etc/hosts contains the fqdn + """ + Get parent domain name (possibly FQDN) of the current machine. Note: if + `socket.fetfqdn()` returns localhost, make sure the first entry in the + hostname file includes the FQDN. + """ + + domain = socket.getfqdn().split('.', 1) if len(domain) != 2: - logger.warning('Could not get domain of this server, only hostname. Please consider updating /etc/hosts') + logger.warning("Could not get domain of this server, only hostname. Please consider updating the hostname file at {0}".format(config.prefs.get("logparse", "hostname-path"))) return 'localdomain' else: return domain[-1] + def resolve(ip, fqdn=None): # try to resolve an ip to hostname - # Possible values for fqdn: - # fqdn show full hostname and domain - # fqdn-implicit show hostname and domain unless local - # host-only only show hostname - # ip never resolve anything - # resolve-domains defined in individual sections of the config take priority over global config + """ + Attempt to resolve an IP into a hostname or FQDN. + Possible values for fqdn: + - fqdn show full hostname and domain + - fqdn-implicit show hostname and domain unless local + - host-only only show hostname + - ip never resolve anything + Note resolve-domains settings defined in individual sections of the config + take priority over the global config (this is enforced in parser modules) + """ if not fqdn: fqdn = config.prefs.get("logparse", "resolve-domains") @@ -55,7 +77,7 @@ def resolve(ip, fqdn=None): # try to resolve an ip to hostname elif fqdn == 'host-only': return(hn.split('.')[0]) else: - logger.warning("invalid value for fqdn config") + logger.warning("Invalid value for FQDN config") return(hn) except socket.herror: # cannot resolve ip @@ -69,10 +91,15 @@ def resolve(ip, fqdn=None): # try to resolve an ip to hostname logger.warning("failed to resolve hostname for " + ip + ": " + str(err)) return(ip) # return ip if no hostname exists -def readlog(path = None, mode = 'r'): # read file + +def readlog(path = None, mode = 'r'): + """ + Read a logfile from disk and return string + """ + if (path == None): - logger.error("no path provided") - return + logger.error("No path provided") + return 1 else: if (os.path.isfile(path) is False): logger.error("Log at {0} was requested but does not exist".format(path))