-#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This file contains global functions for formatting and printing data. This file
-should be imported into individual log-parsing scripts located in logs/*. Data
-is formatted in HTML or plaintext. Writing to disk and/or emailng data is left
-to interface.py.
+should be imported into individual log-parsing scripts located in the default
+logparse.parsers module or in the user-supplied parsers directory. Data is
+formatted in HTML or plaintext. Writing to disk and/or emailng data is left to
+interface.py.
"""
import os
from string import Template
from math import floor, ceil
from tabulate import tabulate
+import textwrap
import logparse
from logparse import interface, util, mail, config
import logging
+logger = None
logger = logging.getLogger(__name__)
JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
BULLET = "• "
INDENT = " "
+SPLIT_CHARS = ['.', '(', ')', '[', ']', '&', r"/", "\\", ',', '-', '_']
global VARSUBST
if interface.argparser.parse_args().no_write:
css_path = os.path.relpath(css_path, ".")
elif interface.argparser.parse_args().destination:
- css_path = os.path.relpath(css_path, interface.argparser.parse_args().destination())
+ css_path = os.path.relpath(
+ css_path, interface.argparser.parse_args().destination())
elif config.prefs.get("logparse", "output"):
- css_path = os.path.relpath(css_path, config.prefs.get("logparse", "output"))
+ css_path = os.path.relpath(
+ css_path, config.prefs.get("logparse", "output"))
VARSUBST = {
"title": config.prefs.get("logparse", "title"),
"date": interface.start.strftime(DATEFMT),
"time": interface.start.strftime(TIMEFMT),
- "hostname": util.hostname(config.prefs.get("logparse", "hostname-path")),
+ "hostname": util.hostname(config.prefs.get(
+ "logparse", "hostname-path")),
"version": logparse.__version__,
- "css": css_path
+ "css": css_path,
+ "period": util.LogPeriod("logparse").startdate.strftime(
+ TIMEFMT + " " + DATEFMT)
}
print()
if lines:
- line = PlaintextLine(linewidth=config.prefs.getint("plain", "linewidth"), double=True)
+ line = PlaintextLine(linewidth=
+ config.prefs.getint("plain", "linewidth"), double=True)
print(line.draw())
print(self.content)
if lines:
class PlaintextOutput(Output):
"""
- Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
+ Processes & outputs data in a plaintext form which can be read with cat or
+ plaintext email.
"""
def __init__(self, linewidth=80):
"""
Print details with some primitive formatting
"""
- box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(VARSUBST), vpadding=2, hpadding="\t\t", linewidth=self.linewidth)
- line = PlaintextLine(self.linewidth)
- self.append(box.draw() + line.draw())
+ box = PlaintextBox(content=
+ Template("$title $version on $hostname\n\n$time $date"
+ "\nParsing logs since $period")
+ .safe_substitute(VARSUBST),
+ vpadding=2, hpadding="\t\t", linewidth=self.linewidth)
+ self.append(box.draw() + "\n"*2)
def append_footer(self):
"""
Append a horizontal line and some details
"""
- self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
+ self.append(PlaintextLine(self.linewidth).draw())
self.append(Template("$hostname $time $date").safe_substitute(VARSUBST))
def append_section(self, section):
This should be run by interface.py after every instance of parse_log().
"""
- self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
+ if section == None:
+ logger.warning("Received null section")
+ return
+ self.append(PlaintextBox(
+ content=section.title, double=False,
+ fullwidth=False, vpadding=0, hpadding=" ").draw())
+ if section.period and section.period.unique:
+ self.append("\n(since {0})".format(
+ section.period.startdate.strftime(DATEFMT + " " + TIMEFMT)))
self.append('\n'*2)
for data in section.data:
self.append(self._fmt_data(data.subtitle, data.items))
logger.warning("No subtitle provided.. skipping section")
return
+ logger.debug("Processing data {}".format(subtitle))
+
if (data == None or len(data) == 0):
- logger.debug("No data provided.. just printing subtitle")
- return subtitle + '\n'
+ # If no list items are provided, just print the subtitle
+ return subtitle + "\n"
+ elif (len(data) == 1):
+ # If only one item is provided, print it inline with subtitle
+ return self._wrap_datum("{}: {}".format(subtitle, data[0]),
+ bullet=False, indent=False) + "\n"
else:
- logger.debug("Received data " + str(data))
- subtitle += ':'
- if (len(data) == 1):
- return subtitle + ' ' + data[0] + '\n'
- else:
- itemoutput = subtitle + '\n'
- for datum in data:
- datum = BULLET + datum
- if len(datum) > self.linewidth - 3:
- words = datum.split()
- if max(map(len, words)) > self.linewidth - len(INDENT):
- continue
- res, part, others = [], words[0], words[1:]
- for word in others:
- if 1 + len(word) > self.linewidth - len(part):
- res.append(part)
- part = word
- else:
- part += ' ' + word
- if part:
- res.append(part)
- datum = ('\n ').join(res)
- datum = INDENT + datum
- itemoutput += datum + '\n'
- return itemoutput
+ # If many items are provided, print them all as a bulleted list
+ itemoutput = subtitle + ":\n"
+ for datum in data:
+ itemoutput += self._wrap_datum(datum) + "\n"
+ return itemoutput
+
+ def _wrap_datum(self, text, bullet=True, indent=True):
+ """
+ Use cpython's textwrap module to limit line width to the value
+ specified in self.linewidth. This is much easier than doing it all
+ from scratch (which I tried to do originally). Note that line
+ continuations are automatically indented even if they don't have a
+ bullet. This is to make it clear which lines are continuations.
+ """
+
+ wrapper = textwrap.TextWrapper(
+ initial_indent=(INDENT if indent else "") \
+ + (BULLET if bullet else ""),
+ subsequent_indent=INDENT + (' '*len(BULLET) if bullet else ""),
+ width=self.linewidth,
+ replace_whitespace=True)
+
+ return wrapper.fill(text)
+
class HtmlOutput(Output):
"""
"""
if not self._embedded:
- self._embedded = mail.mailprep(self.content, css)
+ self._embedded = mail.mailprep(re.sub(
+ ".*" + re.escape(VARSUBST['css']) + ".*\n", "", self.content),
+ css)
return self._embedded
+ def write_embedded(self, destination = ""):
+ """
+ Write contents to file with inline CSS tags
+ """
+
+ logger.debug("Writing HTML with embedded styles to " + destination)
+ if not self._embedded:
+ logger.warning("Call to write_embedded before embed_css - \
+ embedding stylesheets immediately")
+ self.embed_css(config.prefs.get("html", "css"))
+ if destination == "":
+ destination = self.destination
+ if destination == "":
+ logger.warning("No destination path provided")
+ return 1
+ with open(destination, 'w') as f:
+ f.write(self._embedded)
+ logger.info("Written output to {}".format(destination))
+
+
def append_header(self, template):
"""
Insert variables into header template file and append HTML tags
"""
+ self.headertemplate = template
headercontent = Template(open(template, 'r').read())
self.append(headercontent.safe_substitute(VARSUBST))
self.append(opentag('div', id='main'))
def append_section(self, section):
"""
- Call the appropriate methods to generate HTML tags for a section (provided by a parser).
- This should be run by interface.py after every instance of parse_log().
+ Call the appropriate methods to generate HTML tags for a section
+ (provided by a parser). This should be run by interface.py after every
+ instance of parse_log().
"""
+ if section == None:
+ logger.warning("Received null section")
+ return
self.append(opentag('div', 1, section.title, 'section'))
self.append(self._gen_title(section.title))
+ if section.period and section.period.unique:
+ self.append(self._fmt_period(section.period))
for data in section.data:
- self.append(self._fmt_data(data.subtitle, data.items))
+ self.append(self._fmt_data(data.subtitle, data.items, data.severity))
for table in section.tables:
self.append(table.draw_html())
self.append(closetag('div', 1))
logger.debug("Writing title for " + title)
return tag('h2', False, title)
- def _fmt_data(self, subtitle, data = None):
+ def _fmt_data(self, subtitle, data=None, severity=0):
"""
Format the properties of a data object into usable HTML tags.
- Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
+ Subtitle is required, data is not. If only subtitle is supplied or
+ subtitle + one data item, a single line will be printed.
"""
if (subtitle == ""):
if (data == None or len(data) == 0):
logger.debug("No data provided.. just printing subtitle")
- return tag('p', False, subtitle)
+ return tag('p', False, subtitle, cl="severity-" + str(severity))
else:
- logger.debug("Received data " + str(data))
+ logger.debug("Received data {}: {}".format(subtitle, data))
subtitle += ':'
if (len(data) == 1):
- return tag('p', False, subtitle + ' ' + data[0])
+ return tag('p', False, subtitle + ' ' + data[0],
+ cl="severity-" + str(severity))
else:
output = ""
- output += tag('p', False, subtitle)
+ output += tag('p', False, subtitle,
+ cl="severity-" + str(severity))
output += opentag('ul', 1)
coderegex = re.compile('`(.*)`')
for datum in data:
output += closetag('ul', True)
return output
+ def _fmt_period(self, period):
+ output = ''
+ output += opentag('span', cl='period')
+ output += "since " + period.startdate.strftime(DATEFMT + " " + TIMEFMT)
+ output += closetag('span')
+ return output
+
+ def print_stdout_embedded(self, lines=False):
+ """
+ Echo the version with embedded style tags to the console
+ """
+
+ if self._embedded == "":
+ self.embed_css(config.prefs.get("html", "css"))
+ print()
+ if lines:
+ line = PlaintextLine(linewidth=
+ config.prefs.getint("plain", "linewidth"), double=True)
+ print(line.draw())
+ print(self._embedded)
+ if lines:
+ print(line.draw())
+ print()
+
class Section:
"""
- Each parser should output a Section() which contains the title and returned data.
+ Each parser should output a Section() which contains the title, returned
+ data, and applicable time period.
"""
- def __init__(self, title):
+ def __init__(self, title, period=None):
self.title = title
self.data = []
self.tables = []
+ self.period = util.LogPeriod(self.title)
def append_data(self, data):
self.data.append(data)
class Data:
"""
Each section (parser) can have one or more Data() objects which are
- essentially glorified lists.
+ essentially glorified lists with titles (`self.subtitle`).
"""
- def __init__(self, subtitle="", items=[]):
+ def __init__(self, subtitle="", items=[], severity=0):
"""
Initialise variables. No parameters are enforced upon initialisation,
- but at least the subtitle is required for valid output.
+ but at least the subtitle is required for valid output. Severity refers
+ to the importance of the data (integer from 0 to 5). e.g. a failed
+ system should have severity 5 and will be formatted appropriately by
+ the Output object.
"""
self.subtitle = subtitle
self.items = items
+ self.severity = severity
def truncl(self, limit): # truncate list
"""
- Truncate self.items to a specified value and state how many items are hidden.
+ Truncate self.items to a specified value and state how many items are
+ hidden. Set limit to -1 to avoid truncating any items.
"""
+ if limit == -1:
+ return self
if (len(self.items) > limit):
more = len(self.items) - limit
if more == 1:
return 0
self.items = self.items[:limit]
self.items.append("+ {0} more".format(str(more)))
+ return self
def orderbyfreq(self):
"""
"""
unsorted = list(self.items)
- self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
+ self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(
+ set(unsorted), key = lambda x: -unsorted.count(x))]
+ return self
class Table(object):
A wrapper for python-tabulate's Tabulate type.
"""
- def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]):
+ def __init__(self, double=False, borders=False, hpadding=" ",
+ maxwidth=80, headers=[]):
"""
Initialise variables. Note the keymap is used for a faster index map,
but is not currently used anywhere (may be removed in future).
self._align_cols[i] = align
for row in self.rows:
row.columns[i].align = align
- logger.debug("Column alignment is now {0}".format(str(self._align_cols)))
+ logger.debug("Column alignment is now {0}".format(self._align_cols))
def _gen_list(self):
"""
Output HTML string (wrapper for tabulate)
"""
- output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols))
+ output = tabulate(self._gen_list(), self.headers, tablefmt="html",
+ colalign=tuple(self._align_cols))
return output
def draw_plain(self):
Output plain text string (wrapper for tabulate)
"""
- output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols))
+ output = tabulate(self._gen_list(), self.headers,
+ tablefmt="fancy_grid" if self.borders
+ else "plain", colalign=tuple(self._align_cols))
return output + "\n"*2
class Column(object):
"""
- Object representing a single table cell. This is somewhat of a misnomer -
- one column object exists for each cell in the table. Columns are children
+ Object representing a single table cell. "Column" is somewhat of a misnomer
+ - one column object exists for each cell in the table. Columns are children
of rows.
"""
Draw a horizontal line for plain text format, with optional padding/styling.
"""
- def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
+ def __init__(self, linewidth=80, double=True, vpadding=0, hpadding=""):
"""
Initialise variables
"""
"""
line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
- return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
+ return "\n" * self.vpadding + self.hpadding \
+ + line * (self.linewidth - 2 * len(self.hpadding)) \
+ + self.hpadding + "\n" * (self.vpadding + 1)
class PlaintextBox:
Draw a rectangular box around text, with customisable padding/size/style
"""
- def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
+ def __init__(self, content="", double=True, fullwidth=True, linewidth=80,
+ hpadding="\t", vpadding=1):
"""
Initialise variables
"""
# Calculate number of characters per line
contentlines = self.content.splitlines()
- contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
- logger.debug("Contentwidth is {0}".format(str(contentwidth)))
- logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
+ contentwidth = int((self.linewidth if self.linewidth > 0 else 80)
+ if self.content.splitlines()
+ else len(max(contentlines, key=len)))
+ logger.debug("Content width is {0}".format(str(contentwidth)))
+ logger.debug("Longest line is {0}".format(
+ len(max(contentlines, key=len))))
contentwidth += -2*(len(self.hpadding)+1)
if not self.fullwidth:
longestline = len(max(contentlines, key=len))
contentlines[i] = res
# Flatten list
- # Note list comprehension doesn't work here, so we must iterate through each item
+ # Note list comprehension doesn't work here, so we must iterate
+ # through each item
newlines = []
for line in contentlines:
if isinstance(line, list):
contentlines.append(' '*contentwidth)
# Insert horizontal padding on lines that are too short
- contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
- contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
- contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
+ contentlines = [linechars[0] + self.hpadding + x
+ + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2)
+ if len(x) < contentwidth else 0)
+ + self.hpadding + linechars[0] for x in contentlines]
+ contentlines.insert(0, cornerchars[3] + linechars[1]
+ * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
+ contentlines.append(cornerchars[0] + linechars[1]
+ * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
return ('\n').join(contentlines)
Return "1 noun" or "n nouns"
"""
- if print_quantity:
- if (quantity == 1):
+ if (quantity == 1):
+ if print_quantity:
return(str(quantity) + " " + noun)
else:
- return(str(quantity) + " " + noun + "s")
- else:
- if (quantity == 1):
return noun
+ else:
+ if noun.endswith("s"):
+ noun += "e"
+ if print_quantity:
+ return(str(quantity) + " " + noun + "s")
else:
return noun + "s"
output += " class='" + cl + "'"
if style:
output += " style='"
- output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items())
+ output += " ".join("{0}: {1};".format(attr, value)
+ for attr, value in style.items())
output += "'"
output += '>'
if block:
o = opentag(tag, block, id, cl, style)
c = closetag(tag, block)
return o + content + c
-