#!/usr/bin/env python
# -*- coding: utf-8 -*-
-#
-# format.py
-#
-# This file contains global functions for formatting and printing data. This
-# file should be imported into individual log-parsing scripts located in
-# logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or
-# emailng data is left to __main__.py.
-#
+
+"""
+This file contains global functions for formatting and printing data. This file
+should be imported into individual log-parsing scripts located in logs/*. Data
+is formatted in HTML or plaintext. Writing to disk and/or emailng data is left
+to interface.py.
+"""
import os
import re
locale.setlocale(locale.LC_ALL, '') # inherit system locale
-#DEG = "°".encode('unicode_escape')
+
+
DEG = u'\N{DEGREE SIGN}'
CEL = "C"
TIMEFMT = "%X"
BULLET = "• "
INDENT = " "
+
global VARSUBST
+
def init_var():
+ """
+ Initialise variable substitution templates (should be called before doing
+ any substitutions)
+ """
+
global VARSUBST
css_path = config.prefs.get("html", "css")
if config.prefs.getboolean("html", "css-relpath"):
"""
Add a string
"""
+
self.content += content
def write(self, destination=""):
"""
Output contents into a file
"""
+
if destination == "":
destination = self.destination
if destination == "":
"""
Echo the contents to the console
"""
+
print()
if lines:
line = PlaintextLine(linewidth=config.prefs.getint("plain", "linewidth"), double=True)
print()
-
class PlaintextOutput(Output):
"""
Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
Call the appropriate methods to format a section (provided by a parser).
This should be run by interface.py after every instance of parse_log().
"""
+
self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
self.append('\n'*2)
for data in section.data:
def _fmt_data(self, subtitle, data = None): # write title and data
"""
- Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks.
- Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
+ Format the properties of a data object into usable plaintext form with
+ a few fancy symbols/formatting tricks. Subtitle is required, data is
+ not. If only subtitle is supplied or subtitle + one data item, a single
+ line will be printed.
"""
+
if (subtitle == ""):
logger.warning("No subtitle provided.. skipping section")
return
class HtmlOutput(Output):
"""
- Process and output data in HTML format.
- All HTML formatting functions now reside in this class to differentiate them from plaintext.
+ Process and output data in HTML format. All HTML formatting functions now
+ reside in this class to differentiate them from plain text.
"""
def __init__(self):
+ """
+ Initialise variables (no parameters required for initialisation)
+ """
+
self.content = ""
self.destination = ""
self.css = ""
"""
Convert stylesheet to inline tags
"""
+
if not self._embedded:
self._embedded = mail.mailprep(self.content, css)
return self._embedded
"""
Insert variables into header template file and append HTML tags
"""
+
headercontent = Template(open(template, 'r').read())
self.append(headercontent.safe_substitute(VARSUBST))
self.append(opentag('div', id='main'))
Close HTML tags that were opened in the template.
TODO: add footer template similar to header template.
"""
+
self.append(closetag('div') + closetag('body') + closetag('html'))
def append_section(self, section):
Call the appropriate methods to generate HTML tags for a section (provided by a parser).
This should be run by interface.py after every instance of parse_log().
"""
+
self.append(opentag('div', 1, section.title, 'section'))
self.append(self._gen_title(section.title))
for data in section.data:
"""
Format the title for a section
"""
+
if (title == '' or '\n' in title):
logger.error("Invalid title")
raise ValueError
Format the properties of a data object into usable HTML tags.
Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
"""
+
if (subtitle == ""):
logger.warning("No subtitle provided.. skipping section")
return
class Data:
"""
- Each section (parser) can have one or more Data() objects which are essentially glorified lists.
+ Each section (parser) can have one or more Data() objects which are
+ essentially glorified lists.
"""
def __init__(self, subtitle="", items=[]):
+ """
+ Initialise variables. No parameters are enforced upon initialisation,
+ but at least the subtitle is required for valid output.
+ """
+
self.subtitle = subtitle
self.items = items
"""
Truncate self.items to a specified value and state how many items are hidden.
"""
+
if (len(self.items) > limit):
more = len(self.items) - limit
if more == 1:
def orderbyfreq(self):
"""
- Order a list by frequency of each item, then remove duplicates and append frequency in parentheses.
+ Order a list by frequency of each item, then remove duplicates and
+ append frequency in parentheses.
"""
+
unsorted = list(self.items)
self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
+
class Table(object):
"""
A wrapper for python-tabulate's Tabulate type.
"""
def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]):
+ """
+ Initialise variables. Note the keymap is used for a faster index map,
+ but is not currently used anywhere (may be removed in future).
+ """
+
self.rows = [] # List of row objects
self.keymap = {} # For fast lookup of row by value of first column
self.double = double
self._align_cols = []
def add_row(self, row):
+ """
+ Append a row to the list and amend index mapping
+ """
self.rows.append(row)
if len(row.columns) > 0:
logger.debug("Added row with {0} columns".format(str(len(row.columns))))
def align_column(self, i, align):
+ """
+ Set alignment for the 'i'th column (`align` should be 'l', 'c' or 'r')
+ """
+
while len(self._align_cols) -1 < i:
self._align_cols.append("")
self._align_cols[i] = align
logger.debug("Column alignment is now {0}".format(str(self._align_cols)))
def _gen_list(self):
+ """
+ Used locally for organising rows and columns into a 2D list structure
+ """
+
hierarchy = []
for row in self.rows:
row_data = []
return hierarchy
def draw_html(self):
+ """
+ Output HTML string (wrapper for tabulate)
+ """
+
output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols))
return output
def draw_plain(self):
+ """
+ Output plain text string (wrapper for tabulate)
+ """
+
output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols))
return output + "\n"*2
-class Table0(object):
+class Row(object):
"""
- A two-dimensional information display.
- This is a hacky implementation - Table() now relies on the Tabulate package which is much more reliable.
+ Object representing a literal row in a 2D table with the individual cells
+ in the row represented by columns[].
"""
-
- def __init__(self, double=False, borders=True, hpadding="+", maxwidth=80):
- self.rows = [] # List of row objects
- self.keymap = {} # For fast lookup of row by value of first column
- self.double = double
- self.borders = borders
- self.align_cols = []
- self.hpadding = hpadding
- self.maxwidth = maxwidth
- self._colwidths = []
-
- def add_row(self, row):
- self.rows.append(row)
- for i, col in enumerate(row.columns):
- if len(self._colwidths) >= i + 1:
- self._colwidths[i] = max([self._colwidths[i], len(col.content)])
- else:
- self._colwidths.append(len(col.content))
- logger.debug("Added row with {0} columns. Column widths are now {1}.".format(str(len(row.columns)), str(self._colwidths)))
- if len(row.columns) > 0:
- self.keymap[row.columns[0]] = row
-
- def align_column(self, i, align):
- for row in self.rows:
- row.columns[i].align = align
-
-
- def draw_html(self):
- output = ""
- output += opentag("table", True, cl="data_table")
- for row in self.rows:
- if row.header:
- output += opentag("th", block=True, cl="header")
- else:
- output += opentag("tr", block=True)
- for column in row.columns:
- output += tag("td", content=column.content, style={"text-align": column.align} if column.align else {})
- if row.header:
- output += closetag("th", True)
- else:
- output += closetag("tr", True)
- output += closetag("table", True)
- logger.debug("Built table with {0} rows and {1} columns".format(str(len(self.rows)), str(max([x.n for x in self.rows]))))
- return output
-
- def draw_plain(self):
- output = ""
- cols = [list(x) for x in zip(self.rows)]
- logger.debug("Cols are " + str(cols))
-
- if self.double == True:
- cornerchars = CORNERCHARS_DOUBLE
- linechars = LINECHARS_DOUBLE
- jxnchars = JXNCHARS_DOUBLE
- else:
- cornerchars = CORNERCHARS_SINGLE
- linechars = LINECHARS_SINGLE
- jxnchars = JXNCHARS_SINGLE
-
- lengths = []
- row_lengths = []
- for row in self.rows:
- for i, col in enumerate(row.columns):
- if len(lengths) >= i + 1:
- lengths[i] = max([lengths[i], len(col.content)])
- else:
- lengths.append(len(col.content))
-
- logger.debug("Lengths are " + str(lengths))
-
- for i, row in enumerate(self.rows):
- l = (len(INDENT) + len(self.hpadding)*2*len(row.columns) + ((1+len(row.columns)) if self.borders else 0) + sum([len(col.content) for col in row.columns]))
- if l > self.maxwidth:
- logger.debug("Line overflow for cell in row {0} of table".format(str(i)))
- words = row.columns[-1].content.split()
- if max(map(len, words)) > self.maxwidth:
- continue
- res, part, others = [], words[0], words[1:]
- for word in others:
- if l - len(word) < self.maxwidth:
- res.append(part)
- part = word
- else:
- part += ' ' + word
- if part:
- res.append(part)
- self._colwidths[-1] = max([len(f) for f in res] + [len(r.columns[-1].content) for r in self.rows if r != row])
- if self.borders:
- row.columns[-1].content = res[0][:-1] + self.hpadding + " "*(self._colwidths[-1]-len(res[0])+1) + linechars[0]
- for fragment in res[1:]:
- row.columns[-1].content += "\n" + INDENT + "".join([(linechars[0] + self.hpadding + " "*x + self.hpadding) for x in lengths]) + linechars[0] + self.hpadding + fragment + " " * (max(self._colwidths) - len(fragment))
-
- if self.borders:
- top = INDENT + cornerchars[3] + jxnchars[2].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[2] + "\n"
- bottom = INDENT + cornerchars[0] + jxnchars[3].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[1] + "\n"
- rowtext = INDENT + linechars[0] + linechars[0].join("{:>%d}" % l for l in self._colwidths) + linechars[0] + "\n"
- line = INDENT + jxnchars[0] + jxnchars[4].join(linechars[1] * (l+2) for l in self._colwidths) + jxnchars[1] + "\n"
- else:
- top = bottom = line = ""
- rowtext = " ".join("{:>%d}" % l for l in self._colwidths) + "\n"
-
- for i, row in enumerate(self.rows):
- logger.debug("Processing row {0} of {1}".format(str(i), str(len(self.rows)-1)))
- row_output = ""
- if i == 0:
- row_output += top
- row_output += (INDENT + linechars[0] if self.borders else "")
- for j, column in enumerate(row.columns):
- if column.align == "right":
- cell_output = self.hpadding + " "*(self._colwidths[j]-len(column.content)) + column.content + self.hpadding + (linechars[0] if self.borders else "")
- elif column.align == "left":
- cell_output = self.hpadding + column.content + " "*(self._colwidths[j]-len(column.content)) + self.hpadding + (linechars[0] if self.borders else "")
- elif column.align == "center":
- n_whitespace = (self._colwidths[j]-len(column.content))/2
- cell_output = self.hpadding + " "*(floor(n_whitespace) if len(column.content) % 2 == 0 else ceil(n_whitespace)) + column.content + " "*(ceil(n_whitespace) if len(column.content) % 2 == 0 else floor(n_whitespace)) + self.hpadding + (linechars[0] if self.borders else "")
- else:
- logger.warning("Couldn't find alignment value for cell {0} of row {1} with content \"{2}\"".format(str(j), str(i), column.content()))
- continue
- row_output += cell_output
- if len(row_output) > self.maxwidth:
- logger.warning("Line overflow for row {0} of table".format(str(i)))
-
- output += row_output + "\n"
- if i == len(self.rows)-1:
- output += (bottom if self.borders else "")
- else:
- output += (line if self.borders else "")
-
- return output
-
-class Row(object):
def __init__(self, columns=[], header=False):
+ """
+ Initialise variables. The variable n is used locally to keep track of
+ the row width.
+ """
+
self.columns = columns
self.header = header
self.n = len(self.columns)
def add_column(self, column):
+ """
+ Append a single cell horizontally and increment the cell count
+ """
+
self.columns.append(column)
self.n += 1
def rm_column(self, column):
+ """
+ Remove the specified column object and decrement the cell count
+ """
+
self.remove(column)
self.n -= 1
+
class Column(object):
+ """
+ Object representing a single table cell. This is somewhat of a misnomer -
+ one column object exists for each cell in the table. Columns are children
+ of rows.
+ """
def __init__(self, content="", align="right"):
+ """
+ Initialise variables. The align property sets the alignment of a single
+ cell ('l', 'c', or 'r').
+ """
+
self.content = content
self.align = align
+
class PlaintextLine:
"""
Draw a horizontal line for plain text format, with optional padding/styling.
"""
def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
+ """
+ Initialise variables
+ """
+
self.linewidth = linewidth
self.double = double
self.vpadding = vpadding
self.hpadding = hpadding
def draw(self):
+ """
+ Output a plain text string based on the current object parameters
+ """
+
line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
"""
def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
+ """
+ Initialise variables
+ """
self.content = content
self.fullwidth = fullwidth
self.linewidth = linewidth
self.double = double
def draw(self):
+ """
+ Output a plain text string based on the current object parameters. This
+ involves calculating the text width, breaking text at the maximum line
+ length, and then drawing a box around it all.
+ """
if self.double == True:
cornerchars = CORNERCHARS_DOUBLE
def backticks(l):
+ """
+ Surround every item in a list by backticks. Used for showing code in both
+ HTML and plain text formats (converted to <code> tags for HTML)
+ """
+
return ["`" + x + "`" for x in l]
"""
Return "1 noun" or "n nouns"
"""
+
if print_quantity:
if (quantity == 1):
return(str(quantity) + " " + noun)
"""
Return human-readable size from number of bytes
"""
+
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
if abs(num) < 1024.0:
return "%3.1f %s%s" % (num, unit, suffix)
"""
Replace variables in the title template provided in config
"""
+
r = Template(subject).safe_substitute(VARSUBST)
logger.debug("Returning subject line " + r)
return r
+
def opentag(tag, block=False, id=None, cl=None, style=None):
"""
Write HTML opening tag
"""
+
output = ""
if block:
output += '\n'
output += '\n'
return output
+
def closetag(tag, block=False):
"""
Write HTML closing tag
"""
+
if block:
return "\n</" + tag + ">\n"
else:
return "</" + tag + ">"
+
def tag(tag, block=False, content="", id=None, cl=None, style=None):
"""
Write HTML opening tag, content, and closing tag
"""
+
o = opentag(tag, block, id, cl, style)
c = closetag(tag, block)
return o + content + c
-#
-# utilities.py
-#
-# Commonly used general functions
-#
+"""
+Commonly used general functions.
-import re
+This module provides the following methods:
+ - `hostname`: get the current machine's hostname
+ - `getlocaldomain`: get the current machine's domain name
+ - `resolve`: attempt to convert a local/public IP to hostname
+ - `readlog`: read contents of a log file from disk
+"""
+
+from datetime import datetime, timedelta
+import inspect
+import logging
import os
+from pkg_resources import Requirement, resource_filename
+import re
import socket
-import inspect
from systemd import journal
-from datetime import datetime, timedelta
-import logging
-logger = logging.getLogger(__name__)
+from logparse import config
-from pkg_resources import Requirement, resource_filename
-from logparse import config
+logger = logging.getLogger(__name__)
+
def hostname(path): # get the hostname of current server
+ """
+ Get the hostname of the current machine using the file supplied in the
+ `hostname-path` config option.
+ """
+
hnfile = open(path, 'r')
hn = re.search('^(\w*)\n*', hnfile.read()).group(1)
return hn
+
def getlocaldomain(): # get the parent fqdn of current server
- domain = socket.getfqdn().split('.', 1) # Note: if socket.fetfqdn() returns localhost, make sure the first entry in /etc/hosts contains the fqdn
+ """
+ Get parent domain name (possibly FQDN) of the current machine. Note: if
+ `socket.fetfqdn()` returns localhost, make sure the first entry in the
+ hostname file includes the FQDN.
+ """
+
+ domain = socket.getfqdn().split('.', 1)
if len(domain) != 2:
- logger.warning('Could not get domain of this server, only hostname. Please consider updating /etc/hosts')
+ logger.warning("Could not get domain of this server, only hostname. Please consider updating the hostname file at {0}".format(config.prefs.get("logparse", "hostname-path")))
return 'localdomain'
else:
return domain[-1]
+
def resolve(ip, fqdn=None): # try to resolve an ip to hostname
- # Possible values for fqdn:
- # fqdn show full hostname and domain
- # fqdn-implicit show hostname and domain unless local
- # host-only only show hostname
- # ip never resolve anything
- # resolve-domains defined in individual sections of the config take priority over global config
+ """
+ Attempt to resolve an IP into a hostname or FQDN.
+ Possible values for fqdn:
+ - fqdn show full hostname and domain
+ - fqdn-implicit show hostname and domain unless local
+ - host-only only show hostname
+ - ip never resolve anything
+ Note resolve-domains settings defined in individual sections of the config
+ take priority over the global config (this is enforced in parser modules)
+ """
if not fqdn:
fqdn = config.prefs.get("logparse", "resolve-domains")
elif fqdn == 'host-only':
return(hn.split('.')[0])
else:
- logger.warning("invalid value for fqdn config")
+ logger.warning("Invalid value for FQDN config")
return(hn)
except socket.herror:
# cannot resolve ip
logger.warning("failed to resolve hostname for " + ip + ": " + str(err))
return(ip) # return ip if no hostname exists
-def readlog(path = None, mode = 'r'): # read file
+
+def readlog(path = None, mode = 'r'):
+ """
+ Read a logfile from disk and return string
+ """
+
if (path == None):
- logger.error("no path provided")
- return
+ logger.error("No path provided")
+ return 1
else:
if (os.path.isfile(path) is False):
logger.error("Log at {0} was requested but does not exist".format(path))