# # format.py # # This file contains global functions for formatting and printing data. This # file should be imported into individual log-parsing scripts located in # logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or # emailng data is left to __main__.py. # import os import re import locale from string import Template import logparse from . import interface, util, config, mail import logging logger = logging.getLogger(__name__) locale.setlocale(locale.LC_ALL, '') # inherit system locale #DEG = "°".encode('unicode_escape') DEG = u'\N{DEGREE SIGN}' CEL = "C" TIMEFMT = "%X" DATEFMT = "%x" CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔'] CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌'] LINECHARS_DOUBLE = ['║', '═'] LINECHARS_SINGLE = ['│', '─'] BULLET = "• " INDENT = " " class Output: """ Base class for a data processor. """ def __init__(self): self.content = "" self.destination = "" def append(self, content): """ Add a string """ self.content += content def write(self, destination=""): """ Output contents into a file """ if destination == "": destination = self.destination if destination == "": logger.warning("No destination path provided") return 1 with open(destination, 'w') as f: f.write(self.content) logger.info("Written output to {}".format(destination)) class PlaintextOutput(Output): """ Processes & outputs data in a plaintext form which can be read with cat or plaintext email. """ def __init__(self, linewidth=80): self.content = "" self.destination = "" self.linewidth = linewidth; def append_header(self, template=''): """ Print details with some primitive formatting """ init_varfilter() box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth']) line = PlaintextLine(self.linewidth) self.append(box.draw() + line.draw()) def append_footer(self): """ Append a horizontal line and some details """ init_varfilter() self.append(PlaintextLine(self.linewidth, vpadding=1).draw()) self.append(Template("$hostname $time $date").safe_substitute(varsubst)) def append_section(self, section): """ Call the appropriate methods to format a section (provided by a parser). This should be run by interface.py after every instance of parse_log(). """ self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw()) self.append('\n'*2) for data in section.data: self.append(self._fmt_data(data.subtitle, data.items)) self.append('\n') def _fmt_data(self, subtitle, data = None): # write title and data """ Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks. Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed. """ if (subtitle == ""): logger.warning("No subtitle provided.. skipping section") return if (data == None or len(data) == 0): logger.debug("No data provided.. just printing subtitle") return subtitle + '\n' else: logger.debug("Received data " + str(data)) subtitle += ':' if (len(data) == 1): return subtitle + ' ' + data[0] + '\n' else: itemoutput = subtitle + '\n' for datum in data: datum = BULLET + datum if len(datum) > config.prefs['linewidth'] - 3: words = datum.split() if max(map(len, words)) > config.prefs['linewidth'] - len(INDENT): continue res, part, others = [], words[0], words[1:] for word in others: if 1 + len(word) > config.prefs['linewidth'] - len(part): res.append(part) part = word else: part += ' ' + word if part: res.append(part) datum = ('\n ').join(res) datum = INDENT + datum itemoutput += datum + '\n' return itemoutput class HtmlOutput(Output): """ Process and output data in HTML format. All HTML formatting functions now reside in this class to differentiate them from plaintext. """ def __init__(self): self.content = "" self.destination = "" self.css = "" def embed_css(self, css): """ Convert stylesheet to inline tags """ self.content = mail.mailprep(self.content, css) return self.content def append_header(self, template): """ Insert variables into header template file and append HTML tags """ init_varfilter() headercontent = Template(open(template, 'r').read()) self.append(headercontent.safe_substitute(varsubst)) self.append(self.opentag('div', id='main')) def append_footer(self): """ Close HTML tags that were opened in the template. TODO: add footer template similar to header template. """ self.append(self.closetag('div') + self.closetag('body') + self.closetag('html')) def append_section(self, section): """ Call the appropriate methods to generate HTML tags for a section (provided by a parser). This should be run by interface.py after every instance of parse_log(). """ self.append(self.opentag('div', 1, section.title, 'section')) self.append(self._gen_title(section.title)) for data in section.data: self.append(self._fmt_data(data.subtitle, data.items)) self.append(self.closetag('div', 1)) def _gen_title(self, title): """ Format the title for a section """ if (title == '' or '\n' in title): logger.error("Invalid title") raise ValueError logger.debug("Writing title for " + title) return self.tag('h2', False, title) def _fmt_data(self, subtitle, data = None): """ Format the properties of a data object into usable HTML tags. Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed. """ if (subtitle == ""): logger.warning("No subtitle provided.. skipping section") return if (data == None or len(data) == 0): logger.debug("No data provided.. just printing subtitle") return self.tag('p', False, subtitle) else: logger.debug("Received data " + str(data)) subtitle += ':' if (len(data) == 1): return self.tag('p', False, subtitle + ' ' + data[0]) else: output = "" output += self.tag('p', False, subtitle) output += self.opentag('ul', 1) coderegex = re.compile('`(.*)`') for datum in data: if datum == "" or datum == None: continue datum = coderegex.sub(r"\1", str(datum)) output += self.tag('li', False, datum) output += self.closetag('ul', True) return output def opentag(self, tag, block=False, id=None, cl=None): """ Write HTML opening tag """ output = "" if (block): output += '\n' output += '<' + tag if (id != None): output += " id='" + id + "'" if (cl != None): output += " class='" + cl + "'" output += '>' if (block): output += '\n' return output def closetag(self, tag, block=False): """ Write HTML closing tag """ if block: return "\n\n" else: return "" def tag(self, tag, block=False, content=""): """ Write HTML opening tag, content, and closing tag """ o = self.opentag(tag, block) c = self.closetag(tag, block) return o + content + c class Section: """ Each parser should output a Section() which contains the title and returned data. """ def __init__(self, title): self.title = title self.data = [] def append_data(self, data): self.data.append(data) class Data: """ Each section (parser) can have one or more Data() objects which are essentially glorified lists. """ def __init__(self, subtitle="", items=[]): self.subtitle = subtitle self.items = items def truncl(self, limit): # truncate list """ Truncate self.items to a specified value and state how many items are hidden. """ if (len(self.items) > limit): more = str(len(self.items) - limit) self.items = self.items[:limit] self.items.append("+ {0} more".format(more)) def orderbyfreq(self): """ Order a list by frequency of each item, then remove duplicates and append frequency in parentheses. """ unsorted = list(self.items) self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))] class PlaintextLine: """ Draw a horizontal line for plain text format, with optional padding/styling. """ def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""): self.linewidth = linewidth self.double = False self.vpadding = vpadding self.hpadding = hpadding def draw(self): line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1]) return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding class PlaintextBox: """ Draw a rectangular box around text, with customisable padding/size/style """ def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1): self.content = content self.fullwidth = fullwidth self.linewidth = linewidth self.hpadding = hpadding self.vpadding = vpadding self.double = double def draw(self): if self.double == True: cornerchars = CORNERCHARS_DOUBLE linechars = LINECHARS_DOUBLE else: cornerchars = CORNERCHARS_SINGLE linechars = LINECHARS_SINGLE # Check hpadding has a definite width self.hpadding = self.hpadding.replace("\t", " "*4) # Calculate number of characters per line contentlines = self.content.splitlines() contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len))) logger.debug("Contentwidth is {0}".format(str(contentwidth))) logger.debug("Longest line is {0}".format(len(max(contentlines, key=len)))) contentwidth += -2*(len(self.hpadding)+1) if not self.fullwidth: longestline = len(max(contentlines, key=len)) if longestline <= self.linewidth - 2*(len(self.hpadding)+1): contentwidth = longestline # Split lines that are too long for i, line in enumerate(contentlines): if len(line) > contentwidth: words = line.split() if max(map(len, words)) > contentwidth: continue res, part, others = [], words[0], words[1:] for word in others: if len(' ') + len(word) > contentwidth - len(part): res.append(part) part = word else: part += ' ' + word if part: res.append(part) contentlines[i] = res # Flatten list # Note list comprehension doesn't work here, so we must iterate through each item newlines = [] for line in contentlines: if isinstance(line, list): for subline in line: newlines.append(subline) else: newlines.append(line) contentlines = newlines # Add vertical padding for _ in range(self.vpadding): contentlines.insert(0, ' '*contentwidth) contentlines.append(' '*contentwidth) # Insert horizontal padding on lines that are too short contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines] contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2]) contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1]) return ('\n').join(contentlines) def init_varfilter(): global varfilter global varpattern global varsubst varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))} varfilter = dict((re.escape(k), v) for k, v in varfilter.items()) varpattern = re.compile("|".join(varfilter.keys())) varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))) def backticks(l): return ["`" + x + "`" for x in l] def plural(noun, quantity): """ Return "1 noun" or "n nouns" """ if (quantity == 1): return(str(quantity) + " " + noun) else: return(str(quantity) + " " + noun + "s") def parsesize(num, suffix='B'): """ Return human-readable size from number of bytes """ for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: if abs(num) < 1024.0: return "%3.1f %s%s" % (num, unit, suffix) num /= 1024.0 return "%.1f%s%s" % (num, 'Yi', suffix) def fsubject(template): """ Replace variables in the title template provided in config """ r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template) logger.debug("Returning subject line " + r) return r