ab6266efc5233840d3e1369b21c03ff3aa5adec9
1#
2# format.py
3#
4# This file contains global functions for formatting and printing data. This
5# file should be imported into individual log-parsing scripts located in
6# logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or
7# emailng data is left to __main__.py.
8#
9
10import os
11import re
12import locale
13from string import Template
14from math import floor, ceil
15from tabulate import tabulate
16
17import logparse
18from . import interface, util, config, mail
19
20import logging
21logger = logging.getLogger(__name__)
22
23
24locale.setlocale(locale.LC_ALL, '') # inherit system locale
25#DEG = "°".encode('unicode_escape')
26DEG = u'\N{DEGREE SIGN}'
27CEL = "C"
28TIMEFMT = "%X"
29DATEFMT = "%x"
30CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
31CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
32LINECHARS_DOUBLE = ['║', '═']
33LINECHARS_SINGLE = ['│', '─']
34JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
35JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
36BULLET = "• "
37INDENT = " "
38
39
40class Output:
41 """
42 Base class for a data processor.
43 """
44
45 def __init__(self):
46 self.content = ""
47 self.destination = ""
48
49 def append(self, content):
50 """
51 Add a string
52 """
53 self.content += content
54
55 def write(self, destination=""):
56 """
57 Output contents into a file
58 """
59 if destination == "":
60 destination = self.destination
61 if destination == "":
62 logger.warning("No destination path provided")
63 return 1
64 with open(destination, 'w') as f:
65 f.write(self.content)
66 logger.info("Written output to {}".format(destination))
67
68
69class PlaintextOutput(Output):
70 """
71 Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
72 """
73
74 def __init__(self, linewidth=80):
75 self.content = ""
76 self.destination = ""
77 self.linewidth = linewidth;
78
79 def append_header(self, template=''):
80 """
81 Print details with some primitive formatting
82 """
83 init_varfilter()
84 box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth'])
85 line = PlaintextLine(self.linewidth)
86 self.append(box.draw() + line.draw())
87
88 def append_footer(self):
89 """
90 Append a horizontal line and some details
91 """
92 init_varfilter()
93 self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
94 self.append(Template("$hostname $time $date").safe_substitute(varsubst))
95
96 def append_section(self, section):
97 """
98 Call the appropriate methods to format a section (provided by a parser).
99 This should be run by interface.py after every instance of parse_log().
100 """
101 self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
102 self.append('\n'*2)
103 for data in section.data:
104 self.append(self._fmt_data(data.subtitle, data.items))
105 self.append('\n')
106 for table in section.tables:
107 self.append(table.draw_plain())
108 self.append("\n")
109
110 def _fmt_data(self, subtitle, data = None): # write title and data
111 """
112 Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks.
113 Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
114 """
115 if (subtitle == ""):
116 logger.warning("No subtitle provided.. skipping section")
117 return
118
119 if (data == None or len(data) == 0):
120 logger.debug("No data provided.. just printing subtitle")
121 return subtitle + '\n'
122 else:
123 logger.debug("Received data " + str(data))
124 subtitle += ':'
125 if (len(data) == 1):
126 return subtitle + ' ' + data[0] + '\n'
127 else:
128 itemoutput = subtitle + '\n'
129 for datum in data:
130 datum = BULLET + datum
131 if len(datum) > config.prefs['linewidth'] - 3:
132 words = datum.split()
133 if max(map(len, words)) > config.prefs['linewidth'] - len(INDENT):
134 continue
135 res, part, others = [], words[0], words[1:]
136 for word in others:
137 if 1 + len(word) > config.prefs['linewidth'] - len(part):
138 res.append(part)
139 part = word
140 else:
141 part += ' ' + word
142 if part:
143 res.append(part)
144 datum = ('\n ').join(res)
145 datum = INDENT + datum
146 itemoutput += datum + '\n'
147 return itemoutput
148
149class HtmlOutput(Output):
150 """
151 Process and output data in HTML format.
152 All HTML formatting functions now reside in this class to differentiate them from plaintext.
153 """
154
155 def __init__(self):
156 self.content = ""
157 self.destination = ""
158 self.css = ""
159
160 def embed_css(self, css):
161 """
162 Convert stylesheet to inline tags
163 """
164 self.content = mail.mailprep(self.content, css)
165 return self.content
166
167 def append_header(self, template):
168 """
169 Insert variables into header template file and append HTML tags
170 """
171 init_varfilter()
172 headercontent = Template(open(template, 'r').read())
173 self.append(headercontent.safe_substitute(varsubst))
174 self.append(opentag('div', id='main'))
175
176 def append_footer(self):
177 """
178 Close HTML tags that were opened in the template.
179 TODO: add footer template similar to header template.
180 """
181 self.append(closetag('div') + closetag('body') + closetag('html'))
182
183 def append_section(self, section):
184 """
185 Call the appropriate methods to generate HTML tags for a section (provided by a parser).
186 This should be run by interface.py after every instance of parse_log().
187 """
188 self.append(opentag('div', 1, section.title, 'section'))
189 self.append(self._gen_title(section.title))
190 for data in section.data:
191 self.append(self._fmt_data(data.subtitle, data.items))
192 for table in section.tables:
193 self.append(table.draw_html())
194 self.append(closetag('div', 1))
195
196 def _gen_title(self, title):
197 """
198 Format the title for a section
199 """
200 if (title == '' or '\n' in title):
201 logger.error("Invalid title")
202 raise ValueError
203 logger.debug("Writing title for " + title)
204 return tag('h2', False, title)
205
206 def _fmt_data(self, subtitle, data = None):
207 """
208 Format the properties of a data object into usable HTML tags.
209 Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
210 """
211 if (subtitle == ""):
212 logger.warning("No subtitle provided.. skipping section")
213 return
214
215 if (data == None or len(data) == 0):
216 logger.debug("No data provided.. just printing subtitle")
217 return tag('p', False, subtitle)
218 else:
219 logger.debug("Received data " + str(data))
220 subtitle += ':'
221 if (len(data) == 1):
222 return tag('p', False, subtitle + ' ' + data[0])
223 else:
224 output = ""
225 output += tag('p', False, subtitle)
226 output += opentag('ul', 1)
227 coderegex = re.compile('`(.*)`')
228 for datum in data:
229 if datum == "" or datum == None:
230 continue
231 datum = coderegex.sub(r"<code>\1</code>", str(datum))
232 output += tag('li', False, datum)
233 output += closetag('ul', True)
234 return output
235
236
237class Section:
238 """
239 Each parser should output a Section() which contains the title and returned data.
240 """
241
242 def __init__(self, title):
243 self.title = title
244 self.data = []
245 self.tables = []
246
247 def append_data(self, data):
248 self.data.append(data)
249
250 def append_table(self, table):
251 self.tables.append(table)
252
253
254class Data:
255 """
256 Each section (parser) can have one or more Data() objects which are essentially glorified lists.
257 """
258
259 def __init__(self, subtitle="", items=[]):
260 self.subtitle = subtitle
261 self.items = items
262
263 def truncl(self, limit): # truncate list
264 """
265 Truncate self.items to a specified value and state how many items are hidden.
266 """
267 if (len(self.items) > limit):
268 more = len(self.items) - limit
269 if more == 1:
270 return 0
271 self.items = self.items[:limit]
272 self.items.append("+ {0} more".format(str(more)))
273
274 def orderbyfreq(self):
275 """
276 Order a list by frequency of each item, then remove duplicates and append frequency in parentheses.
277 """
278 unsorted = list(self.items)
279 self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
280
281class Table(object):
282 """
283 A wrapper for python-tabulate's Tabulate type.
284 """
285
286 def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]):
287 self.rows = [] # List of row objects
288 self.keymap = {} # For fast lookup of row by value of first column
289 self.double = double
290 self.borders = borders
291 self.align_cols = []
292 self.hpadding = hpadding
293 self.maxwidth = maxwidth
294 self.headers = headers
295 self._align_cols = []
296
297 def add_row(self, row):
298
299 self.rows.append(row)
300 if len(row.columns) > 0:
301 self.keymap[row.columns[0]] = row
302
303 logger.debug("Added row with {0} columns".format(str(len(row.columns))))
304
305 def align_column(self, i, align):
306 while len(self._align_cols) -1 < i:
307 self._align_cols.append("")
308 self._align_cols[i] = align
309 for row in self.rows:
310 row.columns[i].align = align
311 logger.debug("Column alignment is now {0}".format(str(self._align_cols)))
312
313 def _gen_list(self):
314 hierarchy = []
315 for row in self.rows:
316 row_data = []
317 for column in row.columns:
318 row_data.append(column.content)
319 hierarchy.append(row_data)
320 return hierarchy
321
322 def draw_html(self):
323 output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols))
324 return output
325
326 def draw_plain(self):
327 output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols))
328 return output + "\n"*2
329
330
331class Table0(object):
332 """
333 A two-dimensional information display.
334 This is a hacky implementation - Table() now relies on the Tabulate package which is much more reliable.
335 """
336
337 def __init__(self, double=False, borders=True, hpadding="+", maxwidth=80):
338 self.rows = [] # List of row objects
339 self.keymap = {} # For fast lookup of row by value of first column
340 self.double = double
341 self.borders = borders
342 self.align_cols = []
343 self.hpadding = hpadding
344 self.maxwidth = maxwidth
345 self._colwidths = []
346
347 def add_row(self, row):
348 self.rows.append(row)
349 for i, col in enumerate(row.columns):
350 if len(self._colwidths) >= i + 1:
351 self._colwidths[i] = max([self._colwidths[i], len(col.content)])
352 else:
353 self._colwidths.append(len(col.content))
354 logger.debug("Added row with {0} columns. Column widths are now {1}.".format(str(len(row.columns)), str(self._colwidths)))
355 if len(row.columns) > 0:
356 self.keymap[row.columns[0]] = row
357
358 def align_column(self, i, align):
359 for row in self.rows:
360 row.columns[i].align = align
361
362
363 def draw_html(self):
364 output = ""
365 output += opentag("table", True, cl="data_table")
366 for row in self.rows:
367 if row.header:
368 output += opentag("th", block=True, cl="header")
369 else:
370 output += opentag("tr", block=True)
371 for column in row.columns:
372 output += tag("td", content=column.content, style={"text-align": column.align} if column.align else {})
373 if row.header:
374 output += closetag("th", True)
375 else:
376 output += closetag("tr", True)
377 output += closetag("table", True)
378 logger.debug("Built table with {0} rows and {1} columns".format(str(len(self.rows)), str(max([x.n for x in self.rows]))))
379 return output
380
381 def draw_plain(self):
382 output = ""
383 cols = [list(x) for x in zip(self.rows)]
384 logger.debug("Cols are " + str(cols))
385
386 if self.double == True:
387 cornerchars = CORNERCHARS_DOUBLE
388 linechars = LINECHARS_DOUBLE
389 jxnchars = JXNCHARS_DOUBLE
390 else:
391 cornerchars = CORNERCHARS_SINGLE
392 linechars = LINECHARS_SINGLE
393 jxnchars = JXNCHARS_SINGLE
394
395 lengths = []
396 row_lengths = []
397 for row in self.rows:
398 for i, col in enumerate(row.columns):
399 if len(lengths) >= i + 1:
400 lengths[i] = max([lengths[i], len(col.content)])
401 else:
402 lengths.append(len(col.content))
403
404 logger.debug("Lengths are " + str(lengths))
405
406 for i, row in enumerate(self.rows):
407 l = (len(INDENT) + len(self.hpadding)*2*len(row.columns) + ((1+len(row.columns)) if self.borders else 0) + sum([len(col.content) for col in row.columns]))
408 if l > self.maxwidth:
409 logger.debug("Line overflow for cell in row {0} of table".format(str(i)))
410 words = row.columns[-1].content.split()
411 if max(map(len, words)) > self.maxwidth:
412 continue
413 res, part, others = [], words[0], words[1:]
414 for word in others:
415 if l - len(word) < self.maxwidth:
416 res.append(part)
417 part = word
418 else:
419 part += ' ' + word
420 if part:
421 res.append(part)
422 self._colwidths[-1] = max([len(f) for f in res] + [len(r.columns[-1].content) for r in self.rows if r != row])
423 if self.borders:
424 row.columns[-1].content = res[0][:-1] + self.hpadding + " "*(self._colwidths[-1]-len(res[0])+1) + linechars[0]
425 for fragment in res[1:]:
426 row.columns[-1].content += "\n" + INDENT + "".join([(linechars[0] + self.hpadding + " "*x + self.hpadding) for x in lengths]) + linechars[0] + self.hpadding + fragment + " " * (max(self._colwidths) - len(fragment))
427
428 if self.borders:
429 top = INDENT + cornerchars[3] + jxnchars[2].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[2] + "\n"
430 bottom = INDENT + cornerchars[0] + jxnchars[3].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[1] + "\n"
431 rowtext = INDENT + linechars[0] + linechars[0].join("{:>%d}" % l for l in self._colwidths) + linechars[0] + "\n"
432 line = INDENT + jxnchars[0] + jxnchars[4].join(linechars[1] * (l+2) for l in self._colwidths) + jxnchars[1] + "\n"
433 else:
434 top = bottom = line = ""
435 rowtext = " ".join("{:>%d}" % l for l in self._colwidths) + "\n"
436
437 for i, row in enumerate(self.rows):
438 logger.debug("Processing row {0} of {1}".format(str(i), str(len(self.rows)-1)))
439 row_output = ""
440 if i == 0:
441 row_output += top
442 row_output += (INDENT + linechars[0] if self.borders else "")
443 for j, column in enumerate(row.columns):
444 if column.align == "right":
445 cell_output = self.hpadding + " "*(self._colwidths[j]-len(column.content)) + column.content + self.hpadding + (linechars[0] if self.borders else "")
446 elif column.align == "left":
447 cell_output = self.hpadding + column.content + " "*(self._colwidths[j]-len(column.content)) + self.hpadding + (linechars[0] if self.borders else "")
448 elif column.align == "center":
449 n_whitespace = (self._colwidths[j]-len(column.content))/2
450 cell_output = self.hpadding + " "*(floor(n_whitespace) if len(column.content) % 2 == 0 else ceil(n_whitespace)) + column.content + " "*(ceil(n_whitespace) if len(column.content) % 2 == 0 else floor(n_whitespace)) + self.hpadding + (linechars[0] if self.borders else "")
451 else:
452 logger.warning("Couldn't find alignment value for cell {0} of row {1} with content \"{2}\"".format(str(j), str(i), column.content()))
453 continue
454 row_output += cell_output
455 if len(row_output) > self.maxwidth:
456 logger.warning("Line overflow for row {0} of table".format(str(i)))
457
458 output += row_output + "\n"
459 if i == len(self.rows)-1:
460 output += (bottom if self.borders else "")
461 else:
462 output += (line if self.borders else "")
463
464 return output
465
466class Row(object):
467
468 def __init__(self, columns=[], header=False):
469 self.columns = columns
470 self.header = header
471 self.n = len(self.columns)
472
473 def add_column(self, column):
474 self.columns.append(column)
475 self.n += 1
476
477 def rm_column(self, column):
478 self.remove(column)
479 self.n -= 1
480
481class Column(object):
482
483 def __init__(self, content="", align="right"):
484 self.content = content
485 self.align = align
486
487class PlaintextLine:
488 """
489 Draw a horizontal line for plain text format, with optional padding/styling.
490 """
491
492 def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
493 self.linewidth = linewidth
494 self.double = False
495 self.vpadding = vpadding
496 self.hpadding = hpadding
497
498 def draw(self):
499 line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
500 return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
501
502
503class PlaintextBox:
504 """
505 Draw a rectangular box around text, with customisable padding/size/style
506 """
507
508 def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
509 self.content = content
510 self.fullwidth = fullwidth
511 self.linewidth = linewidth
512 self.hpadding = hpadding
513 self.vpadding = vpadding
514 self.double = double
515
516 def draw(self):
517
518 if self.double == True:
519 cornerchars = CORNERCHARS_DOUBLE
520 linechars = LINECHARS_DOUBLE
521 else:
522 cornerchars = CORNERCHARS_SINGLE
523 linechars = LINECHARS_SINGLE
524
525 # Check hpadding has a definite width
526 self.hpadding = self.hpadding.replace("\t", " "*4)
527
528 # Calculate number of characters per line
529 contentlines = self.content.splitlines()
530 contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
531 logger.debug("Contentwidth is {0}".format(str(contentwidth)))
532 logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
533 contentwidth += -2*(len(self.hpadding)+1)
534 if not self.fullwidth:
535 longestline = len(max(contentlines, key=len))
536 if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
537 contentwidth = longestline
538
539 # Split lines that are too long
540 for i, line in enumerate(contentlines):
541 if len(line) > contentwidth:
542 words = line.split()
543 if max(map(len, words)) > contentwidth:
544 continue
545 res, part, others = [], words[0], words[1:]
546 for word in others:
547 if len(' ') + len(word) > contentwidth - len(part):
548 res.append(part)
549 part = word
550 else:
551 part += ' ' + word
552 if part:
553 res.append(part)
554 contentlines[i] = res
555
556 # Flatten list
557 # Note list comprehension doesn't work here, so we must iterate through each item
558 newlines = []
559 for line in contentlines:
560 if isinstance(line, list):
561 for subline in line:
562 newlines.append(subline)
563 else:
564 newlines.append(line)
565 contentlines = newlines
566
567 # Add vertical padding
568 for _ in range(self.vpadding):
569 contentlines.insert(0, ' '*contentwidth)
570 contentlines.append(' '*contentwidth)
571
572 # Insert horizontal padding on lines that are too short
573 contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
574 contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
575 contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
576 return ('\n').join(contentlines)
577
578
579def init_varfilter():
580 global varfilter
581 global varpattern
582 global varsubst
583 varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))}
584 varfilter = dict((re.escape(k), v) for k, v in varfilter.items())
585 varpattern = re.compile("|".join(varfilter.keys()))
586 varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output'])))
587
588def backticks(l):
589 return ["`" + x + "`" for x in l]
590
591
592def plural(noun, quantity, print_quantity=True):
593 """
594 Return "1 noun" or "n nouns"
595 """
596 if print_quantity:
597 if (quantity == 1):
598 return(str(quantity) + " " + noun)
599 else:
600 return(str(quantity) + " " + noun + "s")
601 else:
602 if (quantity == 1):
603 return noun
604 else:
605 return noun + "s"
606
607
608def parsesize(num, suffix='B'):
609 """
610 Return human-readable size from number of bytes
611 """
612 for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
613 if abs(num) < 1024.0:
614 return "%3.1f %s%s" % (num, unit, suffix)
615 num /= 1024.0
616 return "%.1f%s%s" % (num, 'Yi', suffix)
617
618
619def fsubject(template):
620 """
621 Replace variables in the title template provided in config
622 """
623 r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template)
624 logger.debug("Returning subject line " + r)
625 return r
626
627def opentag(tag, block=False, id=None, cl=None, style=None):
628 """
629 Write HTML opening tag
630 """
631 output = ""
632 if block:
633 output += '\n'
634 output += '<' + tag
635 if id:
636 output += " id='" + id + "'"
637 if cl:
638 output += " class='" + cl + "'"
639 if style:
640 output += " style='"
641 output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items())
642 output += "'"
643 output += '>'
644 if block:
645 output += '\n'
646 return output
647
648def closetag(tag, block=False):
649 """
650 Write HTML closing tag
651 """
652 if block:
653 return "\n</" + tag + ">\n"
654 else:
655 return "</" + tag + ">"
656
657def tag(tag, block=False, content="", id=None, cl=None, style=None):
658 """
659 Write HTML opening tag, content, and closing tag
660 """
661 o = opentag(tag, block, id, cl, style)
662 c = closetag(tag, block)
663 return o + content + c
664