1#
2# format.py
3#
4# This file contains global functions for formatting and printing data. This
5# file should be imported into individual log-parsing scripts located in
6# logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or
7# emailng data is left to __main__.py.
8#
9
10import os
11import re
12import locale
13from string import Template
14from math import floor, ceil
15from tabulate import tabulate
16
17import logparse
18from . import interface, util, config, mail
19
20import logging
21logger = logging.getLogger(__name__)
22
23
24locale.setlocale(locale.LC_ALL, '') # inherit system locale
25#DEG = "°".encode('unicode_escape')
26DEG = u'\N{DEGREE SIGN}'
27CEL = "C"
28TIMEFMT = "%X"
29DATEFMT = "%x"
30CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
31CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
32LINECHARS_DOUBLE = ['║', '═']
33LINECHARS_SINGLE = ['│', '─']
34JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
35JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
36BULLET = "• "
37INDENT = " "
38
39
40class Output:
41 """
42 Base class for a data processor.
43 """
44
45 def __init__(self):
46 self.content = ""
47 self.destination = ""
48
49 def append(self, content):
50 """
51 Add a string
52 """
53 self.content += content
54
55 def write(self, destination=""):
56 """
57 Output contents into a file
58 """
59 if destination == "":
60 destination = self.destination
61 if destination == "":
62 logger.warning("No destination path provided")
63 return 1
64 with open(destination, 'w') as f:
65 f.write(self.content)
66 logger.info("Written output to {}".format(destination))
67
68
69class PlaintextOutput(Output):
70 """
71 Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
72 """
73
74 def __init__(self, linewidth=80):
75 self.content = ""
76 self.destination = ""
77 self.linewidth = linewidth;
78
79 def append_header(self, template=''):
80 """
81 Print details with some primitive formatting
82 """
83 init_varfilter()
84 box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth'])
85 line = PlaintextLine(self.linewidth)
86 self.append(box.draw() + line.draw())
87
88 def append_footer(self):
89 """
90 Append a horizontal line and some details
91 """
92 init_varfilter()
93 self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
94 self.append(Template("$hostname $time $date").safe_substitute(varsubst))
95
96 def append_section(self, section):
97 """
98 Call the appropriate methods to format a section (provided by a parser).
99 This should be run by interface.py after every instance of parse_log().
100 """
101 self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
102 self.append('\n'*2)
103 for data in section.data:
104 self.append(self._fmt_data(data.subtitle, data.items))
105 self.append('\n')
106 for table in section.tables:
107 self.append(table.draw_plain())
108 self.append("\n")
109
110 def _fmt_data(self, subtitle, data = None): # write title and data
111 """
112 Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks.
113 Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
114 """
115 if (subtitle == ""):
116 logger.warning("No subtitle provided.. skipping section")
117 return
118
119 if (data == None or len(data) == 0):
120 logger.debug("No data provided.. just printing subtitle")
121 return subtitle + '\n'
122 else:
123 logger.debug("Received data " + str(data))
124 subtitle += ':'
125 if (len(data) == 1):
126 return subtitle + ' ' + data[0] + '\n'
127 else:
128 itemoutput = subtitle + '\n'
129 for datum in data:
130 datum = BULLET + datum
131 if len(datum) > config.prefs['linewidth'] - 3:
132 words = datum.split()
133 if max(map(len, words)) > config.prefs['linewidth'] - len(INDENT):
134 continue
135 res, part, others = [], words[0], words[1:]
136 for word in others:
137 if 1 + len(word) > config.prefs['linewidth'] - len(part):
138 res.append(part)
139 part = word
140 else:
141 part += ' ' + word
142 if part:
143 res.append(part)
144 datum = ('\n ').join(res)
145 datum = INDENT + datum
146 itemoutput += datum + '\n'
147 return itemoutput
148
149class HtmlOutput(Output):
150 """
151 Process and output data in HTML format.
152 All HTML formatting functions now reside in this class to differentiate them from plaintext.
153 """
154
155 def __init__(self):
156 self.content = ""
157 self.destination = ""
158 self.css = ""
159
160 def embed_css(self, css):
161 """
162 Convert stylesheet to inline tags
163 """
164 self.content = mail.mailprep(self.content, css)
165 return self.content
166
167 def append_header(self, template):
168 """
169 Insert variables into header template file and append HTML tags
170 """
171 init_varfilter()
172 headercontent = Template(open(template, 'r').read())
173 self.append(headercontent.safe_substitute(varsubst))
174 self.append(opentag('div', id='main'))
175
176 def append_footer(self):
177 """
178 Close HTML tags that were opened in the template.
179 TODO: add footer template similar to header template.
180 """
181 self.append(closetag('div') + closetag('body') + closetag('html'))
182
183 def append_section(self, section):
184 """
185 Call the appropriate methods to generate HTML tags for a section (provided by a parser).
186 This should be run by interface.py after every instance of parse_log().
187 """
188 self.append(opentag('div', 1, section.title, 'section'))
189 self.append(self._gen_title(section.title))
190 for data in section.data:
191 self.append(self._fmt_data(data.subtitle, data.items))
192 for table in section.tables:
193 self.append(table.draw_html())
194 self.append(closetag('div', 1))
195
196 def _gen_title(self, title):
197 """
198 Format the title for a section
199 """
200 if (title == '' or '\n' in title):
201 logger.error("Invalid title")
202 raise ValueError
203 logger.debug("Writing title for " + title)
204 return tag('h2', False, title)
205
206 def _fmt_data(self, subtitle, data = None):
207 """
208 Format the properties of a data object into usable HTML tags.
209 Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
210 """
211 if (subtitle == ""):
212 logger.warning("No subtitle provided.. skipping section")
213 return
214
215 if (data == None or len(data) == 0):
216 logger.debug("No data provided.. just printing subtitle")
217 return tag('p', False, subtitle)
218 else:
219 logger.debug("Received data " + str(data))
220 subtitle += ':'
221 if (len(data) == 1):
222 return tag('p', False, subtitle + ' ' + data[0])
223 else:
224 output = ""
225 output += tag('p', False, subtitle)
226 output += opentag('ul', 1)
227 coderegex = re.compile('`(.*)`')
228 for datum in data:
229 if datum == "" or datum == None:
230 continue
231 datum = coderegex.sub(r"<code>\1</code>", str(datum))
232 output += tag('li', False, datum)
233 output += closetag('ul', True)
234 return output
235
236
237class Section:
238 """
239 Each parser should output a Section() which contains the title and returned data.
240 """
241
242 def __init__(self, title):
243 self.title = title
244 self.data = []
245 self.tables = []
246
247 def append_data(self, data):
248 self.data.append(data)
249
250 def append_table(self, table):
251 self.tables.append(table)
252
253
254class Data:
255 """
256 Each section (parser) can have one or more Data() objects which are essentially glorified lists.
257 """
258
259 def __init__(self, subtitle="", items=[]):
260 self.subtitle = subtitle
261 self.items = items
262
263 def truncl(self, limit): # truncate list
264 """
265 Truncate self.items to a specified value and state how many items are hidden.
266 """
267 if (len(self.items) > limit):
268 more = str(len(self.items) - limit)
269 self.items = self.items[:limit]
270 self.items.append("+ {0} more".format(more))
271
272 def orderbyfreq(self):
273 """
274 Order a list by frequency of each item, then remove duplicates and append frequency in parentheses.
275 """
276 unsorted = list(self.items)
277 self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
278
279class Table(object):
280 """
281 A wrapper for python-tabulate's Tabulate type.
282 """
283
284 def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]):
285 self.rows = [] # List of row objects
286 self.keymap = {} # For fast lookup of row by value of first column
287 self.double = double
288 self.borders = borders
289 self.align_cols = []
290 self.hpadding = hpadding
291 self.maxwidth = maxwidth
292 self.headers = headers
293 self._align_cols = []
294
295 def add_row(self, row):
296
297 self.rows.append(row)
298 if len(row.columns) > 0:
299 self.keymap[row.columns[0]] = row
300
301 logger.debug("Added row with {0} columns".format(str(len(row.columns))))
302
303 def align_column(self, i, align):
304 while len(self._align_cols) -1 < i:
305 self._align_cols.append("")
306 self._align_cols[i] = align
307 for row in self.rows:
308 row.columns[i].align = align
309 logger.debug("Column alignment is now {0}".format(str(self._align_cols)))
310
311 def _gen_list(self):
312 hierarchy = []
313 for row in self.rows:
314 row_data = []
315 for column in row.columns:
316 row_data.append(column.content)
317 hierarchy.append(row_data)
318 return hierarchy
319
320 def draw_html(self):
321 output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols))
322 return output
323
324 def draw_plain(self):
325 output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols))
326 return output
327
328
329class Table0(object):
330 """
331 A two-dimensional information display.
332 This is a hacky implementation - Table() now relies on the Tabulate package which is much more reliable.
333 """
334
335 def __init__(self, double=False, borders=True, hpadding="+", maxwidth=80):
336 self.rows = [] # List of row objects
337 self.keymap = {} # For fast lookup of row by value of first column
338 self.double = double
339 self.borders = borders
340 self.align_cols = []
341 self.hpadding = hpadding
342 self.maxwidth = maxwidth
343 self._colwidths = []
344
345 def add_row(self, row):
346 self.rows.append(row)
347 for i, col in enumerate(row.columns):
348 if len(self._colwidths) >= i + 1:
349 self._colwidths[i] = max([self._colwidths[i], len(col.content)])
350 else:
351 self._colwidths.append(len(col.content))
352 logger.debug("Added row with {0} columns. Column widths are now {1}.".format(str(len(row.columns)), str(self._colwidths)))
353 if len(row.columns) > 0:
354 self.keymap[row.columns[0]] = row
355
356 def align_column(self, i, align):
357 for row in self.rows:
358 row.columns[i].align = align
359
360
361 def draw_html(self):
362 output = ""
363 output += opentag("table", True, cl="data_table")
364 for row in self.rows:
365 if row.header:
366 output += opentag("th", block=True, cl="header")
367 else:
368 output += opentag("tr", block=True)
369 for column in row.columns:
370 output += tag("td", content=column.content, style={"text-align": column.align} if column.align else {})
371 if row.header:
372 output += closetag("th", True)
373 else:
374 output += closetag("tr", True)
375 output += closetag("table", True)
376 logger.debug("Built table with {0} rows and {1} columns".format(str(len(self.rows)), str(max([x.n for x in self.rows]))))
377 return output
378
379 def draw_plain(self):
380 output = ""
381 cols = [list(x) for x in zip(self.rows)]
382 logger.debug("Cols are " + str(cols))
383
384 if self.double == True:
385 cornerchars = CORNERCHARS_DOUBLE
386 linechars = LINECHARS_DOUBLE
387 jxnchars = JXNCHARS_DOUBLE
388 else:
389 cornerchars = CORNERCHARS_SINGLE
390 linechars = LINECHARS_SINGLE
391 jxnchars = JXNCHARS_SINGLE
392
393 lengths = []
394 row_lengths = []
395 for row in self.rows:
396 for i, col in enumerate(row.columns):
397 if len(lengths) >= i + 1:
398 lengths[i] = max([lengths[i], len(col.content)])
399 else:
400 lengths.append(len(col.content))
401
402 logger.debug("Lengths are " + str(lengths))
403
404 for i, row in enumerate(self.rows):
405 l = (len(INDENT) + len(self.hpadding)*2*len(row.columns) + ((1+len(row.columns)) if self.borders else 0) + sum([len(col.content) for col in row.columns]))
406 if l > self.maxwidth:
407 logger.debug("Line overflow for cell in row {0} of table".format(str(i)))
408 words = row.columns[-1].content.split()
409 if max(map(len, words)) > self.maxwidth:
410 continue
411 res, part, others = [], words[0], words[1:]
412 for word in others:
413 if l - len(word) < self.maxwidth:
414 res.append(part)
415 part = word
416 else:
417 part += ' ' + word
418 if part:
419 res.append(part)
420 self._colwidths[-1] = max([len(f) for f in res] + [len(r.columns[-1].content) for r in self.rows if r != row])
421 if self.borders:
422 row.columns[-1].content = res[0][:-1] + self.hpadding + " "*(self._colwidths[-1]-len(res[0])+1) + linechars[0]
423 for fragment in res[1:]:
424 row.columns[-1].content += "\n" + INDENT + "".join([(linechars[0] + self.hpadding + " "*x + self.hpadding) for x in lengths]) + linechars[0] + self.hpadding + fragment + " " * (max(self._colwidths) - len(fragment))
425
426 if self.borders:
427 top = INDENT + cornerchars[3] + jxnchars[2].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[2] + "\n"
428 bottom = INDENT + cornerchars[0] + jxnchars[3].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[1] + "\n"
429 rowtext = INDENT + linechars[0] + linechars[0].join("{:>%d}" % l for l in self._colwidths) + linechars[0] + "\n"
430 line = INDENT + jxnchars[0] + jxnchars[4].join(linechars[1] * (l+2) for l in self._colwidths) + jxnchars[1] + "\n"
431 else:
432 top = bottom = line = ""
433 rowtext = " ".join("{:>%d}" % l for l in self._colwidths) + "\n"
434
435 for i, row in enumerate(self.rows):
436 logger.debug("Processing row {0} of {1}".format(str(i), str(len(self.rows)-1)))
437 row_output = ""
438 if i == 0:
439 row_output += top
440 row_output += (INDENT + linechars[0] if self.borders else "")
441 for j, column in enumerate(row.columns):
442 if column.align == "right":
443 cell_output = self.hpadding + " "*(self._colwidths[j]-len(column.content)) + column.content + self.hpadding + (linechars[0] if self.borders else "")
444 elif column.align == "left":
445 cell_output = self.hpadding + column.content + " "*(self._colwidths[j]-len(column.content)) + self.hpadding + (linechars[0] if self.borders else "")
446 elif column.align == "center":
447 n_whitespace = (self._colwidths[j]-len(column.content))/2
448 cell_output = self.hpadding + " "*(floor(n_whitespace) if len(column.content) % 2 == 0 else ceil(n_whitespace)) + column.content + " "*(ceil(n_whitespace) if len(column.content) % 2 == 0 else floor(n_whitespace)) + self.hpadding + (linechars[0] if self.borders else "")
449 else:
450 logger.warning("Couldn't find alignment value for cell {0} of row {1} with content \"{2}\"".format(str(j), str(i), column.content()))
451 continue
452 row_output += cell_output
453 if len(row_output) > self.maxwidth:
454 logger.warning("Line overflow for row {0} of table".format(str(i)))
455
456 output += row_output + "\n"
457 if i == len(self.rows)-1:
458 output += (bottom if self.borders else "")
459 else:
460 output += (line if self.borders else "")
461
462 return output
463
464class Row(object):
465
466 def __init__(self, columns=[], header=False):
467 self.columns = columns
468 self.header = header
469 self.n = len(self.columns)
470
471 def add_column(self, column):
472 self.columns.append(column)
473 self.n += 1
474
475 def rm_column(self, column):
476 self.remove(column)
477 self.n -= 1
478
479class Column(object):
480
481 def __init__(self, content="", align="right"):
482 self.content = content
483 self.align = align
484
485class PlaintextLine:
486 """
487 Draw a horizontal line for plain text format, with optional padding/styling.
488 """
489
490 def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
491 self.linewidth = linewidth
492 self.double = False
493 self.vpadding = vpadding
494 self.hpadding = hpadding
495
496 def draw(self):
497 line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
498 return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
499
500
501class PlaintextBox:
502 """
503 Draw a rectangular box around text, with customisable padding/size/style
504 """
505
506 def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
507 self.content = content
508 self.fullwidth = fullwidth
509 self.linewidth = linewidth
510 self.hpadding = hpadding
511 self.vpadding = vpadding
512 self.double = double
513
514 def draw(self):
515
516 if self.double == True:
517 cornerchars = CORNERCHARS_DOUBLE
518 linechars = LINECHARS_DOUBLE
519 else:
520 cornerchars = CORNERCHARS_SINGLE
521 linechars = LINECHARS_SINGLE
522
523 # Check hpadding has a definite width
524 self.hpadding = self.hpadding.replace("\t", " "*4)
525
526 # Calculate number of characters per line
527 contentlines = self.content.splitlines()
528 contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
529 logger.debug("Contentwidth is {0}".format(str(contentwidth)))
530 logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
531 contentwidth += -2*(len(self.hpadding)+1)
532 if not self.fullwidth:
533 longestline = len(max(contentlines, key=len))
534 if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
535 contentwidth = longestline
536
537 # Split lines that are too long
538 for i, line in enumerate(contentlines):
539 if len(line) > contentwidth:
540 words = line.split()
541 if max(map(len, words)) > contentwidth:
542 continue
543 res, part, others = [], words[0], words[1:]
544 for word in others:
545 if len(' ') + len(word) > contentwidth - len(part):
546 res.append(part)
547 part = word
548 else:
549 part += ' ' + word
550 if part:
551 res.append(part)
552 contentlines[i] = res
553
554 # Flatten list
555 # Note list comprehension doesn't work here, so we must iterate through each item
556 newlines = []
557 for line in contentlines:
558 if isinstance(line, list):
559 for subline in line:
560 newlines.append(subline)
561 else:
562 newlines.append(line)
563 contentlines = newlines
564
565 # Add vertical padding
566 for _ in range(self.vpadding):
567 contentlines.insert(0, ' '*contentwidth)
568 contentlines.append(' '*contentwidth)
569
570 # Insert horizontal padding on lines that are too short
571 contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
572 contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
573 contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
574 return ('\n').join(contentlines)
575
576
577def init_varfilter():
578 global varfilter
579 global varpattern
580 global varsubst
581 varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))}
582 varfilter = dict((re.escape(k), v) for k, v in varfilter.items())
583 varpattern = re.compile("|".join(varfilter.keys()))
584 varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output'])))
585
586def backticks(l):
587 return ["`" + x + "`" for x in l]
588
589
590def plural(noun, quantity):
591 """
592 Return "1 noun" or "n nouns"
593 """
594 if (quantity == 1):
595 return(str(quantity) + " " + noun)
596 else:
597 return(str(quantity) + " " + noun + "s")
598
599
600def parsesize(num, suffix='B'):
601 """
602 Return human-readable size from number of bytes
603 """
604 for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
605 if abs(num) < 1024.0:
606 return "%3.1f %s%s" % (num, unit, suffix)
607 num /= 1024.0
608 return "%.1f%s%s" % (num, 'Yi', suffix)
609
610
611def fsubject(template):
612 """
613 Replace variables in the title template provided in config
614 """
615 r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template)
616 logger.debug("Returning subject line " + r)
617 return r
618
619def opentag(tag, block=False, id=None, cl=None, style=None):
620 """
621 Write HTML opening tag
622 """
623 output = ""
624 if block:
625 output += '\n'
626 output += '<' + tag
627 if id:
628 output += " id='" + id + "'"
629 if cl:
630 output += " class='" + cl + "'"
631 if style:
632 output += " style='"
633 output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items())
634 output += "'"
635 output += '>'
636 if block:
637 output += '\n'
638 return output
639
640def closetag(tag, block=False):
641 """
642 Write HTML closing tag
643 """
644 if block:
645 return "\n</" + tag + ">\n"
646 else:
647 return "</" + tag + ">"
648
649def tag(tag, block=False, content="", id=None, cl=None, style=None):
650 """
651 Write HTML opening tag, content, and closing tag
652 """
653 o = opentag(tag, block, id, cl, style)
654 c = closetag(tag, block)
655 return o + content + c
656