# -*- coding: utf-8 -*-
"""
This file contains global functions for formatting and printing data. This file
should be imported into individual log-parsing scripts located in the default
logparse.parsers module or in the user-supplied parsers directory. Data is
formatted in HTML or plaintext. Writing to disk and/or emailng data is left to
interface.py.
"""
import os
import re
import locale
from string import Template
from math import floor, ceil
from tabulate import tabulate
import textwrap
import logparse
from logparse import interface, util, mail, config
import logging
logger = None
logger = logging.getLogger(__name__)
locale.setlocale(locale.LC_ALL, '') # inherit system locale
DEG = u'\N{DEGREE SIGN}'
CEL = "C"
TIMEFMT = "%X"
DATEFMT = "%x"
CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
LINECHARS_DOUBLE = ['║', '═']
LINECHARS_SINGLE = ['│', '─']
JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
BULLET = "• "
INDENT = " "
SPLIT_CHARS = ['.', '(', ')', '[', ']', '&', r"/", "\\", ',', '-', '_']
global VARSUBST
def init_var():
"""
Initialise variable substitution templates (should be called before doing
any substitutions)
"""
global VARSUBST
css_path = config.prefs.get("html", "css")
if config.prefs.getboolean("html", "css-relpath"):
if interface.argparser.parse_args().no_write:
css_path = os.path.relpath(css_path, ".")
elif interface.argparser.parse_args().destination:
css_path = os.path.relpath(
css_path, interface.argparser.parse_args().destination())
elif config.prefs.get("logparse", "output"):
css_path = os.path.relpath(
css_path, config.prefs.get("logparse", "output"))
VARSUBST = {
"title": config.prefs.get("logparse", "title"),
"date": interface.start.strftime(DATEFMT),
"time": interface.start.strftime(TIMEFMT),
"hostname": util.hostname(config.prefs.get(
"logparse", "hostname-path")),
"version": logparse.__version__,
"css": css_path,
"period": util.LogPeriod("logparse").startdate.strftime(
TIMEFMT + " " + DATEFMT)
}
class Output:
"""
Base class for a data processor.
"""
def __init__(self):
self.content = ""
self.destination = ""
def append(self, content):
"""
Add a string
"""
self.content += content
def write(self, destination=""):
"""
Output contents into a file
"""
if destination == "":
destination = self.destination
if destination == "":
logger.warning("No destination path provided")
return 1
with open(destination, 'w') as f:
f.write(self.content)
logger.info("Written output to {}".format(destination))
def print_stdout(self, lines=False):
"""
Echo the contents to the console
"""
print()
if lines:
line = PlaintextLine(linewidth=
config.prefs.getint("plain", "linewidth"), double=True)
print(line.draw())
print(self.content)
if lines:
print(line.draw())
print()
class PlaintextOutput(Output):
"""
Processes & outputs data in a plaintext form which can be read with cat or
plaintext email.
"""
def __init__(self, linewidth=80):
self.content = ""
self.destination = ""
self.linewidth = linewidth;
def append_header(self, template=''):
"""
Print details with some primitive formatting
"""
box = PlaintextBox(content=
Template("$title $version on $hostname\n\n$time $date"
"\nParsing logs since $period")
.safe_substitute(VARSUBST),
vpadding=2, hpadding="\t\t", linewidth=self.linewidth)
self.append(box.draw() + "\n"*2)
def append_footer(self):
"""
Append a horizontal line and some details
"""
self.append(PlaintextLine(self.linewidth).draw())
self.append(Template("$hostname $time $date").safe_substitute(VARSUBST))
def append_section(self, section):
"""
Call the appropriate methods to format a section (provided by a parser).
This should be run by interface.py after every instance of parse_log().
"""
if section == None:
logger.warning("Received null section")
return
self.append(PlaintextBox(
content=section.title, double=False,
fullwidth=False, vpadding=0, hpadding=" ").draw())
if section.period and section.period.unique:
self.append("\n(since {0})".format(
section.period.startdate.strftime(DATEFMT + " " + TIMEFMT)))
self.append('\n'*2)
for data in section.data:
self.append(self._fmt_data(data.subtitle, data.items))
self.append('\n')
for table in section.tables:
self.append(table.draw_plain())
self.append("\n")
def _fmt_data(self, subtitle, data = None): # write title and data
"""
Format the properties of a data object into usable plaintext form with
a few fancy symbols/formatting tricks. Subtitle is required, data is
not. If only subtitle is supplied or subtitle + one data item, a single
line will be printed.
"""
if (subtitle == ""):
logger.warning("No subtitle provided.. skipping section")
return
logger.debug("Processing data {}".format(subtitle))
if (data == None or len(data) == 0):
# If no list items are provided, just print the subtitle
return subtitle + "\n"
elif (len(data) == 1):
# If only one item is provided, print it inline with subtitle
return self._wrap_datum("{}: {}".format(subtitle, data[0]),
bullet=False, indent=False) + "\n"
else:
# If many items are provided, print them all as a bulleted list
itemoutput = subtitle + ":\n"
for datum in data:
itemoutput += self._wrap_datum(datum) + "\n"
return itemoutput
def _wrap_datum(self, text, bullet=True, indent=True):
"""
Use cpython's textwrap module to limit line width to the value
specified in self.linewidth. This is much easier than doing it all
from scratch (which I tried to do originally). Note that line
continuations are automatically indented even if they don't have a
bullet. This is to make it clear which lines are continuations.
"""
wrapper = textwrap.TextWrapper(
initial_indent=(INDENT if indent else "") \
+ (BULLET if bullet else ""),
subsequent_indent=INDENT + (' '*len(BULLET) if bullet else ""),
width=self.linewidth,
replace_whitespace=True)
return wrapper.fill(text)
class HtmlOutput(Output):
"""
Process and output data in HTML format. All HTML formatting functions now
reside in this class to differentiate them from plain text.
"""
def __init__(self):
"""
Initialise variables (no parameters required for initialisation)
"""
self.content = ""
self.destination = ""
self.css = ""
self._embedded = ""
def embed_css(self, css):
"""
Convert stylesheet to inline tags
"""
if not self._embedded:
self._embedded = mail.mailprep(re.sub(
".*" + re.escape(VARSUBST['css']) + ".*\n", "", self.content),
css)
return self._embedded
def write_embedded(self, destination = ""):
"""
Write contents to file with inline CSS tags
"""
logger.debug("Writing HTML with embedded styles to " + destination)
if not self._embedded:
logger.warning("Call to write_embedded before embed_css - \
embedding stylesheets immediately")
self.embed_css(config.prefs.get("html", "css"))
if destination == "":
destination = self.destination
if destination == "":
logger.warning("No destination path provided")
return 1
with open(destination, 'w') as f:
f.write(self._embedded)
logger.info("Written output to {}".format(destination))
def append_header(self, template):
"""
Insert variables into header template file and append HTML tags
"""
self.headertemplate = template
headercontent = Template(open(template, 'r').read())
self.append(headercontent.safe_substitute(VARSUBST))
self.append(opentag('div', id='main'))
def append_footer(self):
"""
Close HTML tags that were opened in the template.
TODO: add footer template similar to header template.
"""
self.append(closetag('div') + closetag('body') + closetag('html'))
def append_section(self, section):
"""
Call the appropriate methods to generate HTML tags for a section
(provided by a parser). This should be run by interface.py after every
instance of parse_log().
"""
if section == None:
logger.warning("Received null section")
return
self.append(opentag('div', 1, section.title, 'section'))
self.append(self._gen_title(section.title))
if section.period and section.period.unique:
self.append(self._fmt_period(section.period))
for data in section.data:
self.append(self._fmt_data(data.subtitle, data.items, data.severity))
for table in section.tables:
self.append(table.draw_html())
self.append(closetag('div', 1))
def _gen_title(self, title):
"""
Format the title for a section
"""
if (title == '' or '\n' in title):
logger.error("Invalid title")
raise ValueError
logger.debug("Writing title for " + title)
return tag('h2', False, title)
def _fmt_data(self, subtitle, data=None, severity=0):
"""
Format the properties of a data object into usable HTML tags.
Subtitle is required, data is not. If only subtitle is supplied or
subtitle + one data item, a single line will be printed.
"""
if (subtitle == ""):
logger.warning("No subtitle provided.. skipping section")
return
if (data == None or len(data) == 0):
logger.debug("No data provided.. just printing subtitle")
return tag('p', False, subtitle, cl="severity-" + str(severity))
else:
logger.debug("Received data {}: {}".format(subtitle, data))
subtitle += ':'
if (len(data) == 1):
return tag('p', False, subtitle + ' ' + data[0],
cl="severity-" + str(severity))
else:
output = ""
output += tag('p', False, subtitle,
cl="severity-" + str(severity))
output += opentag('ul', 1)
coderegex = re.compile('`(.*)`')
for datum in data:
if datum == "" or datum == None:
continue
datum = coderegex.sub(r"\1", str(datum))
output += tag('li', False, datum)
output += closetag('ul', True)
return output
def _fmt_period(self, period):
output = ''
output += opentag('span', cl='period')
output += "since " + period.startdate.strftime(DATEFMT + " " + TIMEFMT)
output += closetag('span')
return output
def print_stdout_embedded(self, lines=False):
"""
Echo the version with embedded style tags to the console
"""
if self._embedded == "":
self.embed_css(config.prefs.get("html", "css"))
print()
if lines:
line = PlaintextLine(linewidth=
config.prefs.getint("plain", "linewidth"), double=True)
print(line.draw())
print(self._embedded)
if lines:
print(line.draw())
print()
class Section:
"""
Each parser should output a Section() which contains the title, returned
data, and applicable time period.
"""
def __init__(self, title, period=None):
self.title = title
self.data = []
self.tables = []
self.period = util.LogPeriod(self.title)
def append_data(self, data):
self.data.append(data)
def append_table(self, table):
self.tables.append(table)
class Data:
"""
Each section (parser) can have one or more Data() objects which are
essentially glorified lists with titles (`self.subtitle`).
"""
def __init__(self, subtitle="", items=[], severity=0):
"""
Initialise variables. No parameters are enforced upon initialisation,
but at least the subtitle is required for valid output. Severity refers
to the importance of the data (integer from 0 to 5). e.g. a failed
system should have severity 5 and will be formatted appropriately by
the Output object.
"""
self.subtitle = subtitle
self.items = items
self.severity = severity
def truncl(self, limit): # truncate list
"""
Truncate self.items to a specified value and state how many items are
hidden. Set limit to -1 to avoid truncating any items.
"""
if limit == -1:
return self
if (len(self.items) > limit):
more = len(self.items) - limit
if more == 1:
return 0
self.items = self.items[:limit]
self.items.append("+ {0} more".format(str(more)))
return self
def orderbyfreq(self):
"""
Order a list by frequency of each item, then remove duplicates and
append frequency in parentheses.
"""
unsorted = list(self.items)
self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(
set(unsorted), key = lambda x: -unsorted.count(x))]
return self
class Table(object):
"""
A wrapper for python-tabulate's Tabulate type.
"""
def __init__(self, double=False, borders=False, hpadding=" ",
maxwidth=80, headers=[]):
"""
Initialise variables. Note the keymap is used for a faster index map,
but is not currently used anywhere (may be removed in future).
"""
self.rows = [] # List of row objects
self.keymap = {} # For fast lookup of row by value of first column
self.double = double
self.borders = borders
self.align_cols = []
self.hpadding = hpadding
self.maxwidth = maxwidth
self.headers = headers
self._align_cols = []
def add_row(self, row):
"""
Append a row to the list and amend index mapping
"""
self.rows.append(row)
if len(row.columns) > 0:
self.keymap[row.columns[0]] = row
logger.debug("Added row with {0} columns".format(str(len(row.columns))))
def align_column(self, i, align):
"""
Set alignment for the 'i'th column (`align` should be 'l', 'c' or 'r')
"""
while len(self._align_cols) -1 < i:
self._align_cols.append("")
self._align_cols[i] = align
for row in self.rows:
row.columns[i].align = align
logger.debug("Column alignment is now {0}".format(self._align_cols))
def _gen_list(self):
"""
Used locally for organising rows and columns into a 2D list structure
"""
hierarchy = []
for row in self.rows:
row_data = []
for column in row.columns:
row_data.append(column.content)
hierarchy.append(row_data)
return hierarchy
def draw_html(self):
"""
Output HTML string (wrapper for tabulate)
"""
output = tabulate(self._gen_list(), self.headers, tablefmt="html",
colalign=tuple(self._align_cols))
return output
def draw_plain(self):
"""
Output plain text string (wrapper for tabulate)
"""
output = tabulate(self._gen_list(), self.headers,
tablefmt="fancy_grid" if self.borders
else "plain", colalign=tuple(self._align_cols))
return output + "\n"*2
class Row(object):
"""
Object representing a literal row in a 2D table with the individual cells
in the row represented by columns[].
"""
def __init__(self, columns=[], header=False):
"""
Initialise variables. The variable n is used locally to keep track of
the row width.
"""
self.columns = columns
self.header = header
self.n = len(self.columns)
def add_column(self, column):
"""
Append a single cell horizontally and increment the cell count
"""
self.columns.append(column)
self.n += 1
def rm_column(self, column):
"""
Remove the specified column object and decrement the cell count
"""
self.remove(column)
self.n -= 1
class Column(object):
"""
Object representing a single table cell. "Column" is somewhat of a misnomer
- one column object exists for each cell in the table. Columns are children
of rows.
"""
def __init__(self, content="", align="right"):
"""
Initialise variables. The align property sets the alignment of a single
cell ('l', 'c', or 'r').
"""
self.content = content
self.align = align
class PlaintextLine:
"""
Draw a horizontal line for plain text format, with optional padding/styling.
"""
def __init__(self, linewidth=80, double=True, vpadding=0, hpadding=""):
"""
Initialise variables
"""
self.linewidth = linewidth
self.double = double
self.vpadding = vpadding
self.hpadding = hpadding
def draw(self):
"""
Output a plain text string based on the current object parameters
"""
line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
return "\n" * self.vpadding + self.hpadding \
+ line * (self.linewidth - 2 * len(self.hpadding)) \
+ self.hpadding + "\n" * (self.vpadding + 1)
class PlaintextBox:
"""
Draw a rectangular box around text, with customisable padding/size/style
"""
def __init__(self, content="", double=True, fullwidth=True, linewidth=80,
hpadding="\t", vpadding=1):
"""
Initialise variables
"""
self.content = content
self.fullwidth = fullwidth
self.linewidth = linewidth
self.hpadding = hpadding
self.vpadding = vpadding
self.double = double
def draw(self):
"""
Output a plain text string based on the current object parameters. This
involves calculating the text width, breaking text at the maximum line
length, and then drawing a box around it all.
"""
if self.double == True:
cornerchars = CORNERCHARS_DOUBLE
linechars = LINECHARS_DOUBLE
else:
cornerchars = CORNERCHARS_SINGLE
linechars = LINECHARS_SINGLE
# Check hpadding has a definite width
self.hpadding = self.hpadding.replace("\t", " "*4)
# Calculate number of characters per line
contentlines = self.content.splitlines()
contentwidth = int((self.linewidth if self.linewidth > 0 else 80)
if self.content.splitlines()
else len(max(contentlines, key=len)))
logger.debug("Content width is {0}".format(str(contentwidth)))
logger.debug("Longest line is {0}".format(
len(max(contentlines, key=len))))
contentwidth += -2*(len(self.hpadding)+1)
if not self.fullwidth:
longestline = len(max(contentlines, key=len))
if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
contentwidth = longestline
# Split lines that are too long
for i, line in enumerate(contentlines):
if len(line) > contentwidth:
words = line.split()
if max(map(len, words)) > contentwidth:
continue
res, part, others = [], words[0], words[1:]
for word in others:
if len(' ') + len(word) > contentwidth - len(part):
res.append(part)
part = word
else:
part += ' ' + word
if part:
res.append(part)
contentlines[i] = res
# Flatten list
# Note list comprehension doesn't work here, so we must iterate
# through each item
newlines = []
for line in contentlines:
if isinstance(line, list):
for subline in line:
newlines.append(subline)
else:
newlines.append(line)
contentlines = newlines
# Add vertical padding
for _ in range(self.vpadding):
contentlines.insert(0, ' '*contentwidth)
contentlines.append(' '*contentwidth)
# Insert horizontal padding on lines that are too short
contentlines = [linechars[0] + self.hpadding + x
+ ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2)
if len(x) < contentwidth else 0)
+ self.hpadding + linechars[0] for x in contentlines]
contentlines.insert(0, cornerchars[3] + linechars[1]
* (contentwidth + len(self.hpadding)*2) + cornerchars[2])
contentlines.append(cornerchars[0] + linechars[1]
* (contentwidth + len(self.hpadding)*2) + cornerchars[1])
return ('\n').join(contentlines)
def backticks(l):
"""
Surround every item in a list by backticks. Used for showing code in both
HTML and plain text formats (converted to tags for HTML)
"""
return ["`" + x + "`" for x in l]
def plural(noun, quantity, print_quantity=True):
"""
Return "1 noun" or "n nouns"
"""
if (quantity == 1):
if print_quantity:
return(str(quantity) + " " + noun)
else:
return noun
else:
if noun.endswith("s"):
noun += "e"
if print_quantity:
return(str(quantity) + " " + noun + "s")
else:
return noun + "s"
def parsesize(num, suffix='B'):
"""
Return human-readable size from number of bytes
"""
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
if abs(num) < 1024.0:
return "%3.1f %s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
def fsubject(subject):
"""
Replace variables in the title template provided in config
"""
r = Template(subject).safe_substitute(VARSUBST)
logger.debug("Returning subject line " + r)
return r
def opentag(tag, block=False, id=None, cl=None, style=None):
"""
Write HTML opening tag
"""
output = ""
if block:
output += '\n'
output += '<' + tag
if id:
output += " id='" + id + "'"
if cl:
output += " class='" + cl + "'"
if style:
output += " style='"
output += " ".join("{0}: {1};".format(attr, value)
for attr, value in style.items())
output += "'"
output += '>'
if block:
output += '\n'
return output
def closetag(tag, block=False):
"""
Write HTML closing tag
"""
if block:
return "\n\n"
else:
return ""
def tag(tag, block=False, content="", id=None, cl=None, style=None):
"""
Write HTML opening tag, content, and closing tag
"""
o = opentag(tag, block, id, cl, style)
c = closetag(tag, block)
return o + content + c