1# -*- coding: utf-8 -*- 2 3""" 4This file contains global functions for formatting and printing data. This file 5should be imported into individual log-parsing scripts located in the default 6logparse.parsers module or in the user-supplied parsers directory. Data is 7formatted in HTML or plaintext. Writing to disk and/or emailng data is left to 8interface.py. 9""" 10 11import os 12import re 13import locale 14from string import Template 15from math import floor, ceil 16from tabulate import tabulate 17 18import logparse 19from logparse import interface, util, mail, config 20 21import logging 22logger =None 23logger = logging.getLogger(__name__) 24 25 26locale.setlocale(locale.LC_ALL,'')# inherit system locale 27 28 29DEG = u'\N{DEGREE SIGN}' 30CEL ="C" 31TIMEFMT ="%X" 32DATEFMT ="%x" 33CORNERCHARS_DOUBLE = ['╚','╝','╗','╔'] 34CORNERCHARS_SINGLE = ['└','┘','┐','┌'] 35LINECHARS_DOUBLE = ['║','═'] 36LINECHARS_SINGLE = ['│','─'] 37JXNCHARS_DOUBLE = ['╠','╣','╦','╩','╬'] 38JXNCHARS_SINGLE = ['├','┤','┬','┴','┼'] 39BULLET ="• " 40INDENT =" " 41 42 43global VARSUBST 44 45 46definit_var(): 47""" 48 Initialise variable substitution templates (should be called before doing 49 any substitutions) 50 """ 51 52global VARSUBST 53 css_path = config.prefs.get("html","css") 54if config.prefs.getboolean("html","css-relpath"): 55if interface.argparser.parse_args().no_write: 56 css_path = os.path.relpath(css_path,".") 57elif interface.argparser.parse_args().destination: 58 css_path = os.path.relpath( 59 css_path, interface.argparser.parse_args().destination()) 60elif config.prefs.get("logparse","output"): 61 css_path = os.path.relpath( 62 css_path, config.prefs.get("logparse","output")) 63 VARSUBST = { 64"title": config.prefs.get("logparse","title"), 65"date": interface.start.strftime(DATEFMT), 66"time": interface.start.strftime(TIMEFMT), 67"hostname": util.hostname(config.prefs.get( 68"logparse","hostname-path")), 69"version": logparse.__version__, 70"css": css_path 71} 72 73 74class Output: 75""" 76 Base class for a data processor. 77 """ 78 79def__init__(self): 80 self.content ="" 81 self.destination ="" 82 83defappend(self, content): 84""" 85 Add a string 86 """ 87 88 self.content += content 89 90defwrite(self, destination=""): 91""" 92 Output contents into a file 93 """ 94 95if destination =="": 96 destination = self.destination 97if destination =="": 98 logger.warning("No destination path provided") 99return1 100withopen(destination,'w')as f: 101 f.write(self.content) 102 logger.info("Written output to {}".format(destination)) 103 104defprint_stdout(self, lines=False): 105""" 106 Echo the contents to the console 107 """ 108 109print() 110if lines: 111 line =PlaintextLine(linewidth= 112 config.prefs.getint("plain","linewidth"), double=True) 113print(line.draw()) 114print(self.content) 115if lines: 116print(line.draw()) 117print() 118 119 120classPlaintextOutput(Output): 121""" 122 Processes & outputs data in a plaintext form which can be read with cat or 123 plaintext email. 124 """ 125 126def__init__(self, linewidth=80): 127 self.content ="" 128 self.destination ="" 129 self.linewidth = linewidth; 130 131defappend_header(self, template=''): 132""" 133 Print details with some primitive formatting 134 """ 135 box =PlaintextBox(content= 136Template("$title $version on $hostname\n\n$time $date") 137.safe_substitute(VARSUBST), 138 vpadding=2, hpadding="\t\t", linewidth=self.linewidth) 139 line =PlaintextLine(self.linewidth) 140 self.append(box.draw() + line.draw()) 141 142defappend_footer(self): 143""" 144 Append a horizontal line and some details 145 """ 146 self.append(PlaintextLine(self.linewidth, vpadding=1).draw()) 147 self.append(Template("$hostname $time $date").safe_substitute(VARSUBST)) 148 149defappend_section(self, section): 150""" 151 Call the appropriate methods to format a section (provided by a parser). 152 This should be run by interface.py after every instance of parse_log(). 153 """ 154 155if section ==None: 156 logger.warning("Received null section") 157return 158 self.append(PlaintextBox( 159 content=section.title, double=False, 160 fullwidth=False, vpadding=0, hpadding=" ").draw()) 161if section.period and section.period.unique: 162 self.append("\n(since{0})".format( 163 section.period.startdate.strftime(DATEFMT +" "+ TIMEFMT))) 164 self.append('\n'*2) 165for data in section.data: 166 self.append(self._fmt_data(data.subtitle, data.items)) 167 self.append('\n') 168for table in section.tables: 169 self.append(table.draw_plain()) 170 self.append("\n") 171 172def_fmt_data(self, subtitle, data =None):# write title and data 173""" 174 Format the properties of a data object into usable plaintext form with 175 a few fancy symbols/formatting tricks. Subtitle is required, data is 176 not. If only subtitle is supplied or subtitle + one data item, a single 177 line will be printed. 178 """ 179 180if(subtitle ==""): 181 logger.warning("No subtitle provided.. skipping section") 182return 183 184if(data ==None orlen(data) ==0): 185 logger.debug("No data provided.. just printing subtitle") 186return subtitle +'\n' 187else: 188 logger.debug("Received data "+str(data)) 189 subtitle +=':' 190if(len(data) ==1): 191return subtitle +' '+ data[0] +'\n' 192else: 193 itemoutput = subtitle +'\n' 194for datum in data: 195 datum = BULLET + datum 196iflen(datum) > self.linewidth -3: 197 words = datum.split() 198ifmax(map(len, words)) > self.linewidth -len(INDENT): 199continue 200 res, part, others = [], words[0], words[1:] 201for word in others: 202if1+len(word) > self.linewidth -len(part): 203 res.append(part) 204 part = word 205else: 206 part +=' '+ word 207if part: 208 res.append(part) 209 datum = ('\n').join(res) 210 datum = INDENT + datum 211 itemoutput += datum +'\n' 212return itemoutput 213 214classHtmlOutput(Output): 215""" 216 Process and output data in HTML format. All HTML formatting functions now 217 reside in this class to differentiate them from plain text. 218 """ 219 220def__init__(self): 221""" 222 Initialise variables (no parameters required for initialisation) 223 """ 224 225 self.content ="" 226 self.destination ="" 227 self.css ="" 228 self._embedded ="" 229 230defembed_css(self, css): 231""" 232 Convert stylesheet to inline tags 233 """ 234 235if not self._embedded: 236 self._embedded = mail.mailprep(re.sub( 237".*"+ re.escape(VARSUBST['css']) +".*\n","", self.content), 238 css) 239return self._embedded 240 241defwrite_embedded(self, destination =""): 242""" 243 Write contents to file with inline CSS tags 244 """ 245 246 logger.debug("Writing HTML with embedded styles to "+ destination) 247if not self._embedded: 248 logger.warning("Call to write_embedded before embed_css - \ 249 embedding stylesheets immediately") 250 self.embed_css(config.prefs.get("html","css")) 251if destination =="": 252 destination = self.destination 253if destination =="": 254 logger.warning("No destination path provided") 255return1 256withopen(destination,'w')as f: 257 f.write(self._embedded) 258 logger.info("Written output to {}".format(destination)) 259 260 261defappend_header(self, template): 262""" 263 Insert variables into header template file and append HTML tags 264 """ 265 266 self.headertemplate = template 267 headercontent =Template(open(template,'r').read()) 268 self.append(headercontent.safe_substitute(VARSUBST)) 269 self.append(opentag('div',id='main')) 270 271defappend_footer(self): 272""" 273 Close HTML tags that were opened in the template. 274 TODO: add footer template similar to header template. 275 """ 276 277 self.append(closetag('div') +closetag('body') +closetag('html')) 278 279defappend_section(self, section): 280""" 281 Call the appropriate methods to generate HTML tags for a section 282 (provided by a parser). This should be run by interface.py after every 283 instance of parse_log(). 284 """ 285 286if section ==None: 287 logger.warning("Received null section") 288return 289 self.append(opentag('div',1, section.title,'section')) 290 self.append(self._gen_title(section.title)) 291if section.period and section.period.unique: 292 self.append(self._fmt_period(section.period)) 293for data in section.data: 294 self.append(self._fmt_data(data.subtitle, data.items, data.severity)) 295for table in section.tables: 296 self.append(table.draw_html()) 297 self.append(closetag('div',1)) 298 299def_gen_title(self, title): 300""" 301 Format the title for a section 302 """ 303 304if(title ==''or'\n'in title): 305 logger.error("Invalid title") 306raiseValueError 307 logger.debug("Writing title for "+ title) 308returntag('h2',False, title) 309 310def_fmt_data(self, subtitle, data=None, severity=0): 311""" 312 Format the properties of a data object into usable HTML tags. 313 Subtitle is required, data is not. If only subtitle is supplied or 314 subtitle + one data item, a single line will be printed. 315 """ 316 317if(subtitle ==""): 318 logger.warning("No subtitle provided.. skipping section") 319return 320 321if(data ==None orlen(data) ==0): 322 logger.debug("No data provided.. just printing subtitle") 323returntag('p',False, subtitle, cl="severity-"+str(severity)) 324else: 325 logger.debug("Received data "+str(data)) 326 subtitle +=':' 327if(len(data) ==1): 328returntag('p',False, subtitle +' '+ data[0], 329 cl="severity-"+str(severity)) 330else: 331 output ="" 332 output +=tag('p',False, subtitle, 333 cl="severity-"+str(severity)) 334 output +=opentag('ul',1) 335 coderegex = re.compile('`(.*)`') 336for datum in data: 337if datum ==""or datum ==None: 338continue 339 datum = coderegex.sub(r"<code>\1</code>",str(datum)) 340 output +=tag('li',False, datum) 341 output +=closetag('ul',True) 342return output 343 344def_fmt_period(self, period): 345 output ='' 346 output +=opentag('span', cl='period') 347 output +="since "+ period.startdate.strftime(DATEFMT +" "+ TIMEFMT) 348 output +=closetag('span') 349return output 350 351defprint_stdout_embedded(self, lines=False): 352""" 353 Echo the version with embedded style tags to the console 354 """ 355 356if self._embedded =="": 357 self.embed_css(config.prefs.get("html","css")) 358print() 359if lines: 360 line =PlaintextLine(linewidth= 361 config.prefs.getint("plain","linewidth"), double=True) 362print(line.draw()) 363print(self._embedded) 364if lines: 365print(line.draw()) 366print() 367 368 369class Section: 370""" 371 Each parser should output a Section() which contains the title, returned 372 data, and applicable time period. 373 """ 374 375def__init__(self, title, period=None): 376 self.title = title 377 self.data = [] 378 self.tables = [] 379 self.period = util.LogPeriod(self.title) 380 381defappend_data(self, data): 382 self.data.append(data) 383 384defappend_table(self, table): 385 self.tables.append(table) 386 387 388class Data: 389""" 390 Each section (parser) can have one or more Data() objects which are 391 essentially glorified lists with titles (`self.subtitle`). 392 """ 393 394def__init__(self, subtitle="", items=[], severity=0): 395""" 396 Initialise variables. No parameters are enforced upon initialisation, 397 but at least the subtitle is required for valid output. Severity refers 398 to the importance of the data (integer from 0 to 5). e.g. a failed 399 system should have severity 5 and will be formatted appropriately by 400 the Output object. 401 """ 402 403 self.subtitle = subtitle 404 self.items = items 405 self.severity = severity 406 407deftruncl(self, limit):# truncate list 408""" 409 Truncate self.items to a specified value and state how many items are 410 hidden. 411 """ 412 413if(len(self.items) > limit): 414 more =len(self.items) - limit 415if more ==1: 416return0 417 self.items = self.items[:limit] 418 self.items.append("+{0}more".format(str(more))) 419return self 420 421deforderbyfreq(self): 422""" 423 Order a list by frequency of each item, then remove duplicates and 424 append frequency in parentheses. 425 """ 426 427 unsorted =list(self.items) 428 self.items = ["{0}({1})".format(y, unsorted.count(y))for y insorted( 429set(unsorted), key =lambda x: -unsorted.count(x))] 430return self 431 432 433classTable(object): 434""" 435 A wrapper for python-tabulate's Tabulate type. 436 """ 437 438def__init__(self, double=False, borders=False, hpadding=" ", 439 maxwidth=80, headers=[]): 440""" 441 Initialise variables. Note the keymap is used for a faster index map, 442 but is not currently used anywhere (may be removed in future). 443 """ 444 445 self.rows = []# List of row objects 446 self.keymap = {}# For fast lookup of row by value of first column 447 self.double = double 448 self.borders = borders 449 self.align_cols = [] 450 self.hpadding = hpadding 451 self.maxwidth = maxwidth 452 self.headers = headers 453 self._align_cols = [] 454 455defadd_row(self, row): 456""" 457 Append a row to the list and amend index mapping 458 """ 459 460 self.rows.append(row) 461iflen(row.columns) >0: 462 self.keymap[row.columns[0]] = row 463 464 logger.debug("Added row with{0}columns".format(str(len(row.columns)))) 465 466defalign_column(self, i, align): 467""" 468 Set alignment for the 'i'th column (`align` should be 'l', 'c' or 'r') 469 """ 470 471whilelen(self._align_cols) -1< i: 472 self._align_cols.append("") 473 self._align_cols[i] = align 474for row in self.rows: 475 row.columns[i].align = align 476 logger.debug("Column alignment is now{0}".format(self._align_cols)) 477 478def_gen_list(self): 479""" 480 Used locally for organising rows and columns into a 2D list structure 481 """ 482 483 hierarchy = [] 484for row in self.rows: 485 row_data = [] 486for column in row.columns: 487 row_data.append(column.content) 488 hierarchy.append(row_data) 489return hierarchy 490 491defdraw_html(self): 492""" 493 Output HTML string (wrapper for tabulate) 494 """ 495 496 output =tabulate(self._gen_list(), self.headers, tablefmt="html", 497 colalign=tuple(self._align_cols)) 498return output 499 500defdraw_plain(self): 501""" 502 Output plain text string (wrapper for tabulate) 503 """ 504 505 output =tabulate(self._gen_list(), self.headers, 506 tablefmt="fancy_grid"if self.borders 507else"plain", colalign=tuple(self._align_cols)) 508return output +"\n"*2 509 510 511classRow(object): 512""" 513 Object representing a literal row in a 2D table with the individual cells 514 in the row represented by columns[]. 515 """ 516 517def__init__(self, columns=[], header=False): 518""" 519 Initialise variables. The variable n is used locally to keep track of 520 the row width. 521 """ 522 523 self.columns = columns 524 self.header = header 525 self.n =len(self.columns) 526 527defadd_column(self, column): 528""" 529 Append a single cell horizontally and increment the cell count 530 """ 531 532 self.columns.append(column) 533 self.n +=1 534 535defrm_column(self, column): 536""" 537 Remove the specified column object and decrement the cell count 538 """ 539 540 self.remove(column) 541 self.n -=1 542 543 544classColumn(object): 545""" 546 Object representing a single table cell. This is somewhat of a misnomer - 547 one column object exists for each cell in the table. Columns are children 548 of rows. 549 """ 550 551def__init__(self, content="", align="right"): 552""" 553 Initialise variables. The align property sets the alignment of a single 554 cell ('l', 'c', or 'r'). 555 """ 556 557 self.content = content 558 self.align = align 559 560 561class PlaintextLine: 562""" 563 Draw a horizontal line for plain text format, with optional padding/styling. 564 """ 565 566def__init__(self, linewidth=80, double=True, vpadding=1, hpadding=""): 567""" 568 Initialise variables 569 """ 570 571 self.linewidth = linewidth 572 self.double = double 573 self.vpadding = vpadding 574 self.hpadding = hpadding 575 576defdraw(self): 577""" 578 Output a plain text string based on the current object parameters 579 """ 580 581 line = (LINECHARS_DOUBLE[1]if self.double else LINECHARS_SINGLE[1]) 582return"\n"* self.vpadding + self.hpadding \ 583+ line * (self.linewidth -2*len(self.hpadding)) \ 584+ self.hpadding +"\n"* self.vpadding 585 586 587class PlaintextBox: 588""" 589 Draw a rectangular box around text, with customisable padding/size/style 590 """ 591 592def__init__(self, content="", double=True, fullwidth=True, linewidth=80, 593 hpadding="\t", vpadding=1): 594""" 595 Initialise variables 596 """ 597 self.content = content 598 self.fullwidth = fullwidth 599 self.linewidth = linewidth 600 self.hpadding = hpadding 601 self.vpadding = vpadding 602 self.double = double 603 604defdraw(self): 605""" 606 Output a plain text string based on the current object parameters. This 607 involves calculating the text width, breaking text at the maximum line 608 length, and then drawing a box around it all. 609 """ 610 611if self.double ==True: 612 cornerchars = CORNERCHARS_DOUBLE 613 linechars = LINECHARS_DOUBLE 614else: 615 cornerchars = CORNERCHARS_SINGLE 616 linechars = LINECHARS_SINGLE 617 618# Check hpadding has a definite width 619 self.hpadding = self.hpadding.replace("\t"," "*4) 620 621# Calculate number of characters per line 622 contentlines = self.content.splitlines() 623 contentwidth =int((self.linewidth if self.linewidth >0else80) 624if self.content.splitlines() 625elselen(max(contentlines, key=len))) 626 logger.debug("Contentwidth is{0}".format(str(contentwidth))) 627 logger.debug("Longest line is{0}".format( 628len(max(contentlines, key=len)))) 629 contentwidth += -2*(len(self.hpadding)+1) 630if not self.fullwidth: 631 longestline =len(max(contentlines, key=len)) 632if longestline <= self.linewidth -2*(len(self.hpadding)+1): 633 contentwidth = longestline 634 635# Split lines that are too long 636for i, line inenumerate(contentlines): 637iflen(line) > contentwidth: 638 words = line.split() 639ifmax(map(len, words)) > contentwidth: 640continue 641 res, part, others = [], words[0], words[1:] 642for word in others: 643iflen(' ') +len(word) > contentwidth -len(part): 644 res.append(part) 645 part = word 646else: 647 part +=' '+ word 648if part: 649 res.append(part) 650 contentlines[i] = res 651 652# Flatten list 653# Note list comprehension doesn't work here, so we must iterate 654# through each item 655 newlines = [] 656for line in contentlines: 657ifisinstance(line,list): 658for subline in line: 659 newlines.append(subline) 660else: 661 newlines.append(line) 662 contentlines = newlines 663 664# Add vertical padding 665for _ inrange(self.vpadding): 666 contentlines.insert(0,' '*contentwidth) 667 contentlines.append(' '*contentwidth) 668 669# Insert horizontal padding on lines that are too short 670 contentlines = [linechars[0] + self.hpadding + x 671+' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) 672iflen(x) < contentwidth else0) 673+ self.hpadding + linechars[0]for x in contentlines] 674 contentlines.insert(0, cornerchars[3] + linechars[1] 675* (contentwidth +len(self.hpadding)*2) + cornerchars[2]) 676 contentlines.append(cornerchars[0] + linechars[1] 677* (contentwidth +len(self.hpadding)*2) + cornerchars[1]) 678return('\n').join(contentlines) 679 680 681defbackticks(l): 682""" 683 Surround every item in a list by backticks. Used for showing code in both 684 HTML and plain text formats (converted to <code> tags for HTML) 685 """ 686 687return["`"+ x +"`"for x in l] 688 689 690defplural(noun, quantity, print_quantity=True): 691""" 692 Return "1 noun" or "n nouns" 693 """ 694 695if(quantity ==1): 696if print_quantity: 697return(str(quantity) +" "+ noun) 698else: 699return noun 700else: 701if noun.endswith("s"): 702 noun +="e" 703if print_quantity: 704return(str(quantity) +" "+ noun +"s") 705else: 706return noun +"s" 707 708 709defparsesize(num, suffix='B'): 710""" 711 Return human-readable size from number of bytes 712 """ 713 714for unit in['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: 715ifabs(num) <1024.0: 716return"%3.1f%s%s"% (num, unit, suffix) 717 num /=1024.0 718return"%.1f%s%s"% (num,'Yi', suffix) 719 720 721deffsubject(subject): 722""" 723 Replace variables in the title template provided in config 724 """ 725 726 r =Template(subject).safe_substitute(VARSUBST) 727 logger.debug("Returning subject line "+ r) 728return r 729 730 731defopentag(tag, block=False,id=None, cl=None, style=None): 732""" 733 Write HTML opening tag 734 """ 735 736 output ="" 737if block: 738 output +='\n' 739 output +='<'+ tag 740ifid: 741 output +=" id='"+id+"'" 742if cl: 743 output +=" class='"+ cl +"'" 744if style: 745 output +=" style='" 746 output +=" ".join("{0}:{1};".format(attr, value) 747for attr, value in style.items()) 748 output +="'" 749 output +='>' 750if block: 751 output +='\n' 752return output 753 754 755defclosetag(tag, block=False): 756""" 757 Write HTML closing tag 758 """ 759 760if block: 761return"\n</"+ tag +">\n" 762else: 763return"</"+ tag +">" 764 765 766deftag(tag, block=False, content="",id=None, cl=None, style=None): 767""" 768 Write HTML opening tag, content, and closing tag 769 """ 770 771 o =opentag(tag, block,id, cl, style) 772 c =closetag(tag, block) 773return o + content + c