# # format.py # # This file contains global functions for formatting and printing data. This # file should be imported into individual log-parsing scripts located in # logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or # emailng data is left to __main__.py. # import os import re import locale from string import Template from math import floor, ceil from tabulate import tabulate import logparse from . import interface, util, config, mail import logging logger = logging.getLogger(__name__) locale.setlocale(locale.LC_ALL, '') # inherit system locale #DEG = "°".encode('unicode_escape') DEG = u'\N{DEGREE SIGN}' CEL = "C" TIMEFMT = "%X" DATEFMT = "%x" CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔'] CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌'] LINECHARS_DOUBLE = ['║', '═'] LINECHARS_SINGLE = ['│', '─'] JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬'] JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼'] BULLET = "• " INDENT = " " class Output: """ Base class for a data processor. """ def __init__(self): self.content = "" self.destination = "" def append(self, content): """ Add a string """ self.content += content def write(self, destination=""): """ Output contents into a file """ if destination == "": destination = self.destination if destination == "": logger.warning("No destination path provided") return 1 with open(destination, 'w') as f: f.write(self.content) logger.info("Written output to {}".format(destination)) class PlaintextOutput(Output): """ Processes & outputs data in a plaintext form which can be read with cat or plaintext email. """ def __init__(self, linewidth=80): self.content = "" self.destination = "" self.linewidth = linewidth; def append_header(self, template=''): """ Print details with some primitive formatting """ init_varfilter() box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth']) line = PlaintextLine(self.linewidth) self.append(box.draw() + line.draw()) def append_footer(self): """ Append a horizontal line and some details """ init_varfilter() self.append(PlaintextLine(self.linewidth, vpadding=1).draw()) self.append(Template("$hostname $time $date").safe_substitute(varsubst)) def append_section(self, section): """ Call the appropriate methods to format a section (provided by a parser). This should be run by interface.py after every instance of parse_log(). """ self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw()) self.append('\n'*2) for data in section.data: self.append(self._fmt_data(data.subtitle, data.items)) self.append('\n') for table in section.tables: self.append(table.draw_plain()) self.append("\n") def _fmt_data(self, subtitle, data = None): # write title and data """ Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks. Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed. """ if (subtitle == ""): logger.warning("No subtitle provided.. skipping section") return if (data == None or len(data) == 0): logger.debug("No data provided.. just printing subtitle") return subtitle + '\n' else: logger.debug("Received data " + str(data)) subtitle += ':' if (len(data) == 1): return subtitle + ' ' + data[0] + '\n' else: itemoutput = subtitle + '\n' for datum in data: datum = BULLET + datum if len(datum) > config.prefs['linewidth'] - 3: words = datum.split() if max(map(len, words)) > config.prefs['linewidth'] - len(INDENT): continue res, part, others = [], words[0], words[1:] for word in others: if 1 + len(word) > config.prefs['linewidth'] - len(part): res.append(part) part = word else: part += ' ' + word if part: res.append(part) datum = ('\n ').join(res) datum = INDENT + datum itemoutput += datum + '\n' return itemoutput class HtmlOutput(Output): """ Process and output data in HTML format. All HTML formatting functions now reside in this class to differentiate them from plaintext. """ def __init__(self): self.content = "" self.destination = "" self.css = "" def embed_css(self, css): """ Convert stylesheet to inline tags """ self.content = mail.mailprep(self.content, css) return self.content def append_header(self, template): """ Insert variables into header template file and append HTML tags """ init_varfilter() headercontent = Template(open(template, 'r').read()) self.append(headercontent.safe_substitute(varsubst)) self.append(opentag('div', id='main')) def append_footer(self): """ Close HTML tags that were opened in the template. TODO: add footer template similar to header template. """ self.append(closetag('div') + closetag('body') + closetag('html')) def append_section(self, section): """ Call the appropriate methods to generate HTML tags for a section (provided by a parser). This should be run by interface.py after every instance of parse_log(). """ self.append(opentag('div', 1, section.title, 'section')) self.append(self._gen_title(section.title)) for data in section.data: self.append(self._fmt_data(data.subtitle, data.items)) for table in section.tables: self.append(table.draw_html()) self.append(closetag('div', 1)) def _gen_title(self, title): """ Format the title for a section """ if (title == '' or '\n' in title): logger.error("Invalid title") raise ValueError logger.debug("Writing title for " + title) return tag('h2', False, title) def _fmt_data(self, subtitle, data = None): """ Format the properties of a data object into usable HTML tags. Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed. """ if (subtitle == ""): logger.warning("No subtitle provided.. skipping section") return if (data == None or len(data) == 0): logger.debug("No data provided.. just printing subtitle") return tag('p', False, subtitle) else: logger.debug("Received data " + str(data)) subtitle += ':' if (len(data) == 1): return tag('p', False, subtitle + ' ' + data[0]) else: output = "" output += tag('p', False, subtitle) output += opentag('ul', 1) coderegex = re.compile('`(.*)`') for datum in data: if datum == "" or datum == None: continue datum = coderegex.sub(r"\1", str(datum)) output += tag('li', False, datum) output += closetag('ul', True) return output class Section: """ Each parser should output a Section() which contains the title and returned data. """ def __init__(self, title): self.title = title self.data = [] self.tables = [] def append_data(self, data): self.data.append(data) def append_table(self, table): self.tables.append(table) class Data: """ Each section (parser) can have one or more Data() objects which are essentially glorified lists. """ def __init__(self, subtitle="", items=[]): self.subtitle = subtitle self.items = items def truncl(self, limit): # truncate list """ Truncate self.items to a specified value and state how many items are hidden. """ if (len(self.items) > limit): more = str(len(self.items) - limit) self.items = self.items[:limit] self.items.append("+ {0} more".format(more)) def orderbyfreq(self): """ Order a list by frequency of each item, then remove duplicates and append frequency in parentheses. """ unsorted = list(self.items) self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))] class Table(object): """ A wrapper for python-tabulate's Tabulate type. """ def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]): self.rows = [] # List of row objects self.keymap = {} # For fast lookup of row by value of first column self.double = double self.borders = borders self.align_cols = [] self.hpadding = hpadding self.maxwidth = maxwidth self.headers = headers self._align_cols = [] def add_row(self, row): self.rows.append(row) if len(row.columns) > 0: self.keymap[row.columns[0]] = row logger.debug("Added row with {0} columns".format(str(len(row.columns)))) def align_column(self, i, align): while len(self._align_cols) -1 < i: self._align_cols.append("") self._align_cols[i] = align for row in self.rows: row.columns[i].align = align logger.debug("Column alignment is now {0}".format(str(self._align_cols))) def _gen_list(self): hierarchy = [] for row in self.rows: row_data = [] for column in row.columns: row_data.append(column.content) hierarchy.append(row_data) return hierarchy def draw_html(self): output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols)) return output def draw_plain(self): output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols)) return output class Table0(object): """ A two-dimensional information display. This is a hacky implementation - Table() now relies on the Tabulate package which is much more reliable. """ def __init__(self, double=False, borders=True, hpadding="+", maxwidth=80): self.rows = [] # List of row objects self.keymap = {} # For fast lookup of row by value of first column self.double = double self.borders = borders self.align_cols = [] self.hpadding = hpadding self.maxwidth = maxwidth self._colwidths = [] def add_row(self, row): self.rows.append(row) for i, col in enumerate(row.columns): if len(self._colwidths) >= i + 1: self._colwidths[i] = max([self._colwidths[i], len(col.content)]) else: self._colwidths.append(len(col.content)) logger.debug("Added row with {0} columns. Column widths are now {1}.".format(str(len(row.columns)), str(self._colwidths))) if len(row.columns) > 0: self.keymap[row.columns[0]] = row def align_column(self, i, align): for row in self.rows: row.columns[i].align = align def draw_html(self): output = "" output += opentag("table", True, cl="data_table") for row in self.rows: if row.header: output += opentag("th", block=True, cl="header") else: output += opentag("tr", block=True) for column in row.columns: output += tag("td", content=column.content, style={"text-align": column.align} if column.align else {}) if row.header: output += closetag("th", True) else: output += closetag("tr", True) output += closetag("table", True) logger.debug("Built table with {0} rows and {1} columns".format(str(len(self.rows)), str(max([x.n for x in self.rows])))) return output def draw_plain(self): output = "" cols = [list(x) for x in zip(self.rows)] logger.debug("Cols are " + str(cols)) if self.double == True: cornerchars = CORNERCHARS_DOUBLE linechars = LINECHARS_DOUBLE jxnchars = JXNCHARS_DOUBLE else: cornerchars = CORNERCHARS_SINGLE linechars = LINECHARS_SINGLE jxnchars = JXNCHARS_SINGLE lengths = [] row_lengths = [] for row in self.rows: for i, col in enumerate(row.columns): if len(lengths) >= i + 1: lengths[i] = max([lengths[i], len(col.content)]) else: lengths.append(len(col.content)) logger.debug("Lengths are " + str(lengths)) for i, row in enumerate(self.rows): l = (len(INDENT) + len(self.hpadding)*2*len(row.columns) + ((1+len(row.columns)) if self.borders else 0) + sum([len(col.content) for col in row.columns])) if l > self.maxwidth: logger.debug("Line overflow for cell in row {0} of table".format(str(i))) words = row.columns[-1].content.split() if max(map(len, words)) > self.maxwidth: continue res, part, others = [], words[0], words[1:] for word in others: if l - len(word) < self.maxwidth: res.append(part) part = word else: part += ' ' + word if part: res.append(part) self._colwidths[-1] = max([len(f) for f in res] + [len(r.columns[-1].content) for r in self.rows if r != row]) if self.borders: row.columns[-1].content = res[0][:-1] + self.hpadding + " "*(self._colwidths[-1]-len(res[0])+1) + linechars[0] for fragment in res[1:]: row.columns[-1].content += "\n" + INDENT + "".join([(linechars[0] + self.hpadding + " "*x + self.hpadding) for x in lengths]) + linechars[0] + self.hpadding + fragment + " " * (max(self._colwidths) - len(fragment)) if self.borders: top = INDENT + cornerchars[3] + jxnchars[2].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[2] + "\n" bottom = INDENT + cornerchars[0] + jxnchars[3].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[1] + "\n" rowtext = INDENT + linechars[0] + linechars[0].join("{:>%d}" % l for l in self._colwidths) + linechars[0] + "\n" line = INDENT + jxnchars[0] + jxnchars[4].join(linechars[1] * (l+2) for l in self._colwidths) + jxnchars[1] + "\n" else: top = bottom = line = "" rowtext = " ".join("{:>%d}" % l for l in self._colwidths) + "\n" for i, row in enumerate(self.rows): logger.debug("Processing row {0} of {1}".format(str(i), str(len(self.rows)-1))) row_output = "" if i == 0: row_output += top row_output += (INDENT + linechars[0] if self.borders else "") for j, column in enumerate(row.columns): if column.align == "right": cell_output = self.hpadding + " "*(self._colwidths[j]-len(column.content)) + column.content + self.hpadding + (linechars[0] if self.borders else "") elif column.align == "left": cell_output = self.hpadding + column.content + " "*(self._colwidths[j]-len(column.content)) + self.hpadding + (linechars[0] if self.borders else "") elif column.align == "center": n_whitespace = (self._colwidths[j]-len(column.content))/2 cell_output = self.hpadding + " "*(floor(n_whitespace) if len(column.content) % 2 == 0 else ceil(n_whitespace)) + column.content + " "*(ceil(n_whitespace) if len(column.content) % 2 == 0 else floor(n_whitespace)) + self.hpadding + (linechars[0] if self.borders else "") else: logger.warning("Couldn't find alignment value for cell {0} of row {1} with content \"{2}\"".format(str(j), str(i), column.content())) continue row_output += cell_output if len(row_output) > self.maxwidth: logger.warning("Line overflow for row {0} of table".format(str(i))) output += row_output + "\n" if i == len(self.rows)-1: output += (bottom if self.borders else "") else: output += (line if self.borders else "") return output class Row(object): def __init__(self, columns=[], header=False): self.columns = columns self.header = header self.n = len(self.columns) def add_column(self, column): self.columns.append(column) self.n += 1 def rm_column(self, column): self.remove(column) self.n -= 1 class Column(object): def __init__(self, content="", align="right"): self.content = content self.align = align class PlaintextLine: """ Draw a horizontal line for plain text format, with optional padding/styling. """ def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""): self.linewidth = linewidth self.double = False self.vpadding = vpadding self.hpadding = hpadding def draw(self): line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1]) return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding class PlaintextBox: """ Draw a rectangular box around text, with customisable padding/size/style """ def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1): self.content = content self.fullwidth = fullwidth self.linewidth = linewidth self.hpadding = hpadding self.vpadding = vpadding self.double = double def draw(self): if self.double == True: cornerchars = CORNERCHARS_DOUBLE linechars = LINECHARS_DOUBLE else: cornerchars = CORNERCHARS_SINGLE linechars = LINECHARS_SINGLE # Check hpadding has a definite width self.hpadding = self.hpadding.replace("\t", " "*4) # Calculate number of characters per line contentlines = self.content.splitlines() contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len))) logger.debug("Contentwidth is {0}".format(str(contentwidth))) logger.debug("Longest line is {0}".format(len(max(contentlines, key=len)))) contentwidth += -2*(len(self.hpadding)+1) if not self.fullwidth: longestline = len(max(contentlines, key=len)) if longestline <= self.linewidth - 2*(len(self.hpadding)+1): contentwidth = longestline # Split lines that are too long for i, line in enumerate(contentlines): if len(line) > contentwidth: words = line.split() if max(map(len, words)) > contentwidth: continue res, part, others = [], words[0], words[1:] for word in others: if len(' ') + len(word) > contentwidth - len(part): res.append(part) part = word else: part += ' ' + word if part: res.append(part) contentlines[i] = res # Flatten list # Note list comprehension doesn't work here, so we must iterate through each item newlines = [] for line in contentlines: if isinstance(line, list): for subline in line: newlines.append(subline) else: newlines.append(line) contentlines = newlines # Add vertical padding for _ in range(self.vpadding): contentlines.insert(0, ' '*contentwidth) contentlines.append(' '*contentwidth) # Insert horizontal padding on lines that are too short contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines] contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2]) contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1]) return ('\n').join(contentlines) def init_varfilter(): global varfilter global varpattern global varsubst varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))} varfilter = dict((re.escape(k), v) for k, v in varfilter.items()) varpattern = re.compile("|".join(varfilter.keys())) varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))) def backticks(l): return ["`" + x + "`" for x in l] def plural(noun, quantity): """ Return "1 noun" or "n nouns" """ if (quantity == 1): return(str(quantity) + " " + noun) else: return(str(quantity) + " " + noun + "s") def parsesize(num, suffix='B'): """ Return human-readable size from number of bytes """ for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: if abs(num) < 1024.0: return "%3.1f %s%s" % (num, unit, suffix) num /= 1024.0 return "%.1f%s%s" % (num, 'Yi', suffix) def fsubject(template): """ Replace variables in the title template provided in config """ r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template) logger.debug("Returning subject line " + r) return r def opentag(tag, block=False, id=None, cl=None, style=None): """ Write HTML opening tag """ output = "" if block: output += '\n' output += '<' + tag if id: output += " id='" + id + "'" if cl: output += " class='" + cl + "'" if style: output += " style='" output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items()) output += "'" output += '>' if block: output += '\n' return output def closetag(tag, block=False): """ Write HTML closing tag """ if block: return "\n\n" else: return "" def tag(tag, block=False, content="", id=None, cl=None, style=None): """ Write HTML opening tag, content, and closing tag """ o = opentag(tag, block, id, cl, style) c = closetag(tag, block) return o + content + c