1#
2# format.py
3#
4# This file contains global functions for formatting and printing data. This
5# file should be imported into individual log-parsing scripts located in
6# logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or
7# emailng data is left to __main__.py.
8#
9
10import os
11import re
12import locale
13from string import Template
14from math import floor, ceil
15from tabulate import tabulate
16
17import logparse
18from logparse import interface, util, mail, config
19
20import logging
21logger = logging.getLogger(__name__)
22
23
24locale.setlocale(locale.LC_ALL, '') # inherit system locale
25#DEG = "°".encode('unicode_escape')
26DEG = u'\N{DEGREE SIGN}'
27CEL = "C"
28TIMEFMT = "%X"
29DATEFMT = "%x"
30CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
31CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
32LINECHARS_DOUBLE = ['║', '═']
33LINECHARS_SINGLE = ['│', '─']
34JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
35JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
36BULLET = "• "
37INDENT = " "
38
39global VARSUBST
40
41def init_var():
42 global VARSUBST
43 css_path = config.prefs.get("html", "css")
44 if config.prefs.getboolean("html", "css-relpath"):
45 if interface.argparser.parse_args().no_write:
46 css_path = os.path.relpath(css_path, ".")
47 elif interface.argparser.parse_args().destination:
48 css_path = os.path.relpath(css_path, interface.argparser.parse_args().destination())
49 elif config.prefs.get("logparse", "output"):
50 css_path = os.path.relpath(css_path, config.prefs.get("logparse", "output"))
51 VARSUBST = {
52 "title": config.prefs.get("logparse", "title"),
53 "date": interface.start.strftime(DATEFMT),
54 "time": interface.start.strftime(TIMEFMT),
55 "hostname": util.hostname(config.prefs.get("logparse", "hostname-path")),
56 "version": logparse.__version__,
57 "css": css_path
58 }
59
60
61class Output:
62 """
63 Base class for a data processor.
64 """
65
66 def __init__(self):
67 self.content = ""
68 self.destination = ""
69
70 def append(self, content):
71 """
72 Add a string
73 """
74 self.content += content
75
76 def write(self, destination=""):
77 """
78 Output contents into a file
79 """
80 if destination == "":
81 destination = self.destination
82 if destination == "":
83 logger.warning("No destination path provided")
84 return 1
85 with open(destination, 'w') as f:
86 f.write(self.content)
87 logger.info("Written output to {}".format(destination))
88
89 def print_stdout(self, lines=False):
90 """
91 Echo the contents to the console
92 """
93 print()
94 if lines:
95 line = PlaintextLine(linewidth=config.prefs.getint("plain", "linewidth"), double=True)
96 print(line.draw())
97 print(self.content)
98 if lines:
99 print(line.draw())
100 print()
101
102
103
104class PlaintextOutput(Output):
105 """
106 Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
107 """
108
109 def __init__(self, linewidth=80):
110 self.content = ""
111 self.destination = ""
112 self.linewidth = linewidth;
113
114 def append_header(self, template=''):
115 """
116 Print details with some primitive formatting
117 """
118 box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(VARSUBST), vpadding=2, hpadding="\t\t", linewidth=self.linewidth)
119 line = PlaintextLine(self.linewidth)
120 self.append(box.draw() + line.draw())
121
122 def append_footer(self):
123 """
124 Append a horizontal line and some details
125 """
126 self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
127 self.append(Template("$hostname $time $date").safe_substitute(VARSUBST))
128
129 def append_section(self, section):
130 """
131 Call the appropriate methods to format a section (provided by a parser).
132 This should be run by interface.py after every instance of parse_log().
133 """
134 self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
135 self.append('\n'*2)
136 for data in section.data:
137 self.append(self._fmt_data(data.subtitle, data.items))
138 self.append('\n')
139 for table in section.tables:
140 self.append(table.draw_plain())
141 self.append("\n")
142
143 def _fmt_data(self, subtitle, data = None): # write title and data
144 """
145 Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks.
146 Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
147 """
148 if (subtitle == ""):
149 logger.warning("No subtitle provided.. skipping section")
150 return
151
152 if (data == None or len(data) == 0):
153 logger.debug("No data provided.. just printing subtitle")
154 return subtitle + '\n'
155 else:
156 logger.debug("Received data " + str(data))
157 subtitle += ':'
158 if (len(data) == 1):
159 return subtitle + ' ' + data[0] + '\n'
160 else:
161 itemoutput = subtitle + '\n'
162 for datum in data:
163 datum = BULLET + datum
164 if len(datum) > self.linewidth - 3:
165 words = datum.split()
166 if max(map(len, words)) > self.linewidth - len(INDENT):
167 continue
168 res, part, others = [], words[0], words[1:]
169 for word in others:
170 if 1 + len(word) > self.linewidth - len(part):
171 res.append(part)
172 part = word
173 else:
174 part += ' ' + word
175 if part:
176 res.append(part)
177 datum = ('\n ').join(res)
178 datum = INDENT + datum
179 itemoutput += datum + '\n'
180 return itemoutput
181
182class HtmlOutput(Output):
183 """
184 Process and output data in HTML format.
185 All HTML formatting functions now reside in this class to differentiate them from plaintext.
186 """
187
188 def __init__(self):
189 self.content = ""
190 self.destination = ""
191 self.css = ""
192 self._embedded = ""
193
194 def embed_css(self, css):
195 """
196 Convert stylesheet to inline tags
197 """
198 if not self._embedded:
199 self._embedded = mail.mailprep(self.content, css)
200 return self._embedded
201
202 def append_header(self, template):
203 """
204 Insert variables into header template file and append HTML tags
205 """
206 headercontent = Template(open(template, 'r').read())
207 self.append(headercontent.safe_substitute(VARSUBST))
208 self.append(opentag('div', id='main'))
209
210 def append_footer(self):
211 """
212 Close HTML tags that were opened in the template.
213 TODO: add footer template similar to header template.
214 """
215 self.append(closetag('div') + closetag('body') + closetag('html'))
216
217 def append_section(self, section):
218 """
219 Call the appropriate methods to generate HTML tags for a section (provided by a parser).
220 This should be run by interface.py after every instance of parse_log().
221 """
222 self.append(opentag('div', 1, section.title, 'section'))
223 self.append(self._gen_title(section.title))
224 for data in section.data:
225 self.append(self._fmt_data(data.subtitle, data.items))
226 for table in section.tables:
227 self.append(table.draw_html())
228 self.append(closetag('div', 1))
229
230 def _gen_title(self, title):
231 """
232 Format the title for a section
233 """
234 if (title == '' or '\n' in title):
235 logger.error("Invalid title")
236 raise ValueError
237 logger.debug("Writing title for " + title)
238 return tag('h2', False, title)
239
240 def _fmt_data(self, subtitle, data = None):
241 """
242 Format the properties of a data object into usable HTML tags.
243 Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
244 """
245 if (subtitle == ""):
246 logger.warning("No subtitle provided.. skipping section")
247 return
248
249 if (data == None or len(data) == 0):
250 logger.debug("No data provided.. just printing subtitle")
251 return tag('p', False, subtitle)
252 else:
253 logger.debug("Received data " + str(data))
254 subtitle += ':'
255 if (len(data) == 1):
256 return tag('p', False, subtitle + ' ' + data[0])
257 else:
258 output = ""
259 output += tag('p', False, subtitle)
260 output += opentag('ul', 1)
261 coderegex = re.compile('`(.*)`')
262 for datum in data:
263 if datum == "" or datum == None:
264 continue
265 datum = coderegex.sub(r"<code>\1</code>", str(datum))
266 output += tag('li', False, datum)
267 output += closetag('ul', True)
268 return output
269
270
271class Section:
272 """
273 Each parser should output a Section() which contains the title and returned data.
274 """
275
276 def __init__(self, title):
277 self.title = title
278 self.data = []
279 self.tables = []
280
281 def append_data(self, data):
282 self.data.append(data)
283
284 def append_table(self, table):
285 self.tables.append(table)
286
287
288class Data:
289 """
290 Each section (parser) can have one or more Data() objects which are essentially glorified lists.
291 """
292
293 def __init__(self, subtitle="", items=[]):
294 self.subtitle = subtitle
295 self.items = items
296
297 def truncl(self, limit): # truncate list
298 """
299 Truncate self.items to a specified value and state how many items are hidden.
300 """
301 if (len(self.items) > limit):
302 more = len(self.items) - limit
303 if more == 1:
304 return 0
305 self.items = self.items[:limit]
306 self.items.append("+ {0} more".format(str(more)))
307
308 def orderbyfreq(self):
309 """
310 Order a list by frequency of each item, then remove duplicates and append frequency in parentheses.
311 """
312 unsorted = list(self.items)
313 self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
314
315class Table(object):
316 """
317 A wrapper for python-tabulate's Tabulate type.
318 """
319
320 def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]):
321 self.rows = [] # List of row objects
322 self.keymap = {} # For fast lookup of row by value of first column
323 self.double = double
324 self.borders = borders
325 self.align_cols = []
326 self.hpadding = hpadding
327 self.maxwidth = maxwidth
328 self.headers = headers
329 self._align_cols = []
330
331 def add_row(self, row):
332
333 self.rows.append(row)
334 if len(row.columns) > 0:
335 self.keymap[row.columns[0]] = row
336
337 logger.debug("Added row with {0} columns".format(str(len(row.columns))))
338
339 def align_column(self, i, align):
340 while len(self._align_cols) -1 < i:
341 self._align_cols.append("")
342 self._align_cols[i] = align
343 for row in self.rows:
344 row.columns[i].align = align
345 logger.debug("Column alignment is now {0}".format(str(self._align_cols)))
346
347 def _gen_list(self):
348 hierarchy = []
349 for row in self.rows:
350 row_data = []
351 for column in row.columns:
352 row_data.append(column.content)
353 hierarchy.append(row_data)
354 return hierarchy
355
356 def draw_html(self):
357 output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols))
358 return output
359
360 def draw_plain(self):
361 output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols))
362 return output + "\n"*2
363
364
365class Table0(object):
366 """
367 A two-dimensional information display.
368 This is a hacky implementation - Table() now relies on the Tabulate package which is much more reliable.
369 """
370
371 def __init__(self, double=False, borders=True, hpadding="+", maxwidth=80):
372 self.rows = [] # List of row objects
373 self.keymap = {} # For fast lookup of row by value of first column
374 self.double = double
375 self.borders = borders
376 self.align_cols = []
377 self.hpadding = hpadding
378 self.maxwidth = maxwidth
379 self._colwidths = []
380
381 def add_row(self, row):
382 self.rows.append(row)
383 for i, col in enumerate(row.columns):
384 if len(self._colwidths) >= i + 1:
385 self._colwidths[i] = max([self._colwidths[i], len(col.content)])
386 else:
387 self._colwidths.append(len(col.content))
388 logger.debug("Added row with {0} columns. Column widths are now {1}.".format(str(len(row.columns)), str(self._colwidths)))
389 if len(row.columns) > 0:
390 self.keymap[row.columns[0]] = row
391
392 def align_column(self, i, align):
393 for row in self.rows:
394 row.columns[i].align = align
395
396
397 def draw_html(self):
398 output = ""
399 output += opentag("table", True, cl="data_table")
400 for row in self.rows:
401 if row.header:
402 output += opentag("th", block=True, cl="header")
403 else:
404 output += opentag("tr", block=True)
405 for column in row.columns:
406 output += tag("td", content=column.content, style={"text-align": column.align} if column.align else {})
407 if row.header:
408 output += closetag("th", True)
409 else:
410 output += closetag("tr", True)
411 output += closetag("table", True)
412 logger.debug("Built table with {0} rows and {1} columns".format(str(len(self.rows)), str(max([x.n for x in self.rows]))))
413 return output
414
415 def draw_plain(self):
416 output = ""
417 cols = [list(x) for x in zip(self.rows)]
418 logger.debug("Cols are " + str(cols))
419
420 if self.double == True:
421 cornerchars = CORNERCHARS_DOUBLE
422 linechars = LINECHARS_DOUBLE
423 jxnchars = JXNCHARS_DOUBLE
424 else:
425 cornerchars = CORNERCHARS_SINGLE
426 linechars = LINECHARS_SINGLE
427 jxnchars = JXNCHARS_SINGLE
428
429 lengths = []
430 row_lengths = []
431 for row in self.rows:
432 for i, col in enumerate(row.columns):
433 if len(lengths) >= i + 1:
434 lengths[i] = max([lengths[i], len(col.content)])
435 else:
436 lengths.append(len(col.content))
437
438 logger.debug("Lengths are " + str(lengths))
439
440 for i, row in enumerate(self.rows):
441 l = (len(INDENT) + len(self.hpadding)*2*len(row.columns) + ((1+len(row.columns)) if self.borders else 0) + sum([len(col.content) for col in row.columns]))
442 if l > self.maxwidth:
443 logger.debug("Line overflow for cell in row {0} of table".format(str(i)))
444 words = row.columns[-1].content.split()
445 if max(map(len, words)) > self.maxwidth:
446 continue
447 res, part, others = [], words[0], words[1:]
448 for word in others:
449 if l - len(word) < self.maxwidth:
450 res.append(part)
451 part = word
452 else:
453 part += ' ' + word
454 if part:
455 res.append(part)
456 self._colwidths[-1] = max([len(f) for f in res] + [len(r.columns[-1].content) for r in self.rows if r != row])
457 if self.borders:
458 row.columns[-1].content = res[0][:-1] + self.hpadding + " "*(self._colwidths[-1]-len(res[0])+1) + linechars[0]
459 for fragment in res[1:]:
460 row.columns[-1].content += "\n" + INDENT + "".join([(linechars[0] + self.hpadding + " "*x + self.hpadding) for x in lengths]) + linechars[0] + self.hpadding + fragment + " " * (max(self._colwidths) - len(fragment))
461
462 if self.borders:
463 top = INDENT + cornerchars[3] + jxnchars[2].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[2] + "\n"
464 bottom = INDENT + cornerchars[0] + jxnchars[3].join(linechars[1] * (l+2) for l in self._colwidths) + cornerchars[1] + "\n"
465 rowtext = INDENT + linechars[0] + linechars[0].join("{:>%d}" % l for l in self._colwidths) + linechars[0] + "\n"
466 line = INDENT + jxnchars[0] + jxnchars[4].join(linechars[1] * (l+2) for l in self._colwidths) + jxnchars[1] + "\n"
467 else:
468 top = bottom = line = ""
469 rowtext = " ".join("{:>%d}" % l for l in self._colwidths) + "\n"
470
471 for i, row in enumerate(self.rows):
472 logger.debug("Processing row {0} of {1}".format(str(i), str(len(self.rows)-1)))
473 row_output = ""
474 if i == 0:
475 row_output += top
476 row_output += (INDENT + linechars[0] if self.borders else "")
477 for j, column in enumerate(row.columns):
478 if column.align == "right":
479 cell_output = self.hpadding + " "*(self._colwidths[j]-len(column.content)) + column.content + self.hpadding + (linechars[0] if self.borders else "")
480 elif column.align == "left":
481 cell_output = self.hpadding + column.content + " "*(self._colwidths[j]-len(column.content)) + self.hpadding + (linechars[0] if self.borders else "")
482 elif column.align == "center":
483 n_whitespace = (self._colwidths[j]-len(column.content))/2
484 cell_output = self.hpadding + " "*(floor(n_whitespace) if len(column.content) % 2 == 0 else ceil(n_whitespace)) + column.content + " "*(ceil(n_whitespace) if len(column.content) % 2 == 0 else floor(n_whitespace)) + self.hpadding + (linechars[0] if self.borders else "")
485 else:
486 logger.warning("Couldn't find alignment value for cell {0} of row {1} with content \"{2}\"".format(str(j), str(i), column.content()))
487 continue
488 row_output += cell_output
489 if len(row_output) > self.maxwidth:
490 logger.warning("Line overflow for row {0} of table".format(str(i)))
491
492 output += row_output + "\n"
493 if i == len(self.rows)-1:
494 output += (bottom if self.borders else "")
495 else:
496 output += (line if self.borders else "")
497
498 return output
499
500class Row(object):
501
502 def __init__(self, columns=[], header=False):
503 self.columns = columns
504 self.header = header
505 self.n = len(self.columns)
506
507 def add_column(self, column):
508 self.columns.append(column)
509 self.n += 1
510
511 def rm_column(self, column):
512 self.remove(column)
513 self.n -= 1
514
515class Column(object):
516
517 def __init__(self, content="", align="right"):
518 self.content = content
519 self.align = align
520
521class PlaintextLine:
522 """
523 Draw a horizontal line for plain text format, with optional padding/styling.
524 """
525
526 def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
527 self.linewidth = linewidth
528 self.double = double
529 self.vpadding = vpadding
530 self.hpadding = hpadding
531
532 def draw(self):
533 line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
534 return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
535
536
537class PlaintextBox:
538 """
539 Draw a rectangular box around text, with customisable padding/size/style
540 """
541
542 def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
543 self.content = content
544 self.fullwidth = fullwidth
545 self.linewidth = linewidth
546 self.hpadding = hpadding
547 self.vpadding = vpadding
548 self.double = double
549
550 def draw(self):
551
552 if self.double == True:
553 cornerchars = CORNERCHARS_DOUBLE
554 linechars = LINECHARS_DOUBLE
555 else:
556 cornerchars = CORNERCHARS_SINGLE
557 linechars = LINECHARS_SINGLE
558
559 # Check hpadding has a definite width
560 self.hpadding = self.hpadding.replace("\t", " "*4)
561
562 # Calculate number of characters per line
563 contentlines = self.content.splitlines()
564 contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
565 logger.debug("Contentwidth is {0}".format(str(contentwidth)))
566 logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
567 contentwidth += -2*(len(self.hpadding)+1)
568 if not self.fullwidth:
569 longestline = len(max(contentlines, key=len))
570 if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
571 contentwidth = longestline
572
573 # Split lines that are too long
574 for i, line in enumerate(contentlines):
575 if len(line) > contentwidth:
576 words = line.split()
577 if max(map(len, words)) > contentwidth:
578 continue
579 res, part, others = [], words[0], words[1:]
580 for word in others:
581 if len(' ') + len(word) > contentwidth - len(part):
582 res.append(part)
583 part = word
584 else:
585 part += ' ' + word
586 if part:
587 res.append(part)
588 contentlines[i] = res
589
590 # Flatten list
591 # Note list comprehension doesn't work here, so we must iterate through each item
592 newlines = []
593 for line in contentlines:
594 if isinstance(line, list):
595 for subline in line:
596 newlines.append(subline)
597 else:
598 newlines.append(line)
599 contentlines = newlines
600
601 # Add vertical padding
602 for _ in range(self.vpadding):
603 contentlines.insert(0, ' '*contentwidth)
604 contentlines.append(' '*contentwidth)
605
606 # Insert horizontal padding on lines that are too short
607 contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
608 contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
609 contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
610 return ('\n').join(contentlines)
611
612
613def backticks(l):
614 return ["`" + x + "`" for x in l]
615
616
617def plural(noun, quantity, print_quantity=True):
618 """
619 Return "1 noun" or "n nouns"
620 """
621 if print_quantity:
622 if (quantity == 1):
623 return(str(quantity) + " " + noun)
624 else:
625 return(str(quantity) + " " + noun + "s")
626 else:
627 if (quantity == 1):
628 return noun
629 else:
630 return noun + "s"
631
632
633def parsesize(num, suffix='B'):
634 """
635 Return human-readable size from number of bytes
636 """
637 for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
638 if abs(num) < 1024.0:
639 return "%3.1f %s%s" % (num, unit, suffix)
640 num /= 1024.0
641 return "%.1f%s%s" % (num, 'Yi', suffix)
642
643
644def fsubject(subject):
645 """
646 Replace variables in the title template provided in config
647 """
648 r = Template(subject).safe_substitute(VARSUBST)
649 logger.debug("Returning subject line " + r)
650 return r
651
652def opentag(tag, block=False, id=None, cl=None, style=None):
653 """
654 Write HTML opening tag
655 """
656 output = ""
657 if block:
658 output += '\n'
659 output += '<' + tag
660 if id:
661 output += " id='" + id + "'"
662 if cl:
663 output += " class='" + cl + "'"
664 if style:
665 output += " style='"
666 output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items())
667 output += "'"
668 output += '>'
669 if block:
670 output += '\n'
671 return output
672
673def closetag(tag, block=False):
674 """
675 Write HTML closing tag
676 """
677 if block:
678 return "\n</" + tag + ">\n"
679 else:
680 return "</" + tag + ">"
681
682def tag(tag, block=False, content="", id=None, cl=None, style=None):
683 """
684 Write HTML opening tag, content, and closing tag
685 """
686 o = opentag(tag, block, id, cl, style)
687 c = closetag(tag, block)
688 return o + content + c
689