00ea9e45f663d71836bcc1a336d0eb0c4e3c2c08
   1#!/usr/bin/env python
   2# -*- coding: utf-8 -*-
   3
   4"""   
   5This file contains global functions for formatting and printing data. This file
   6should be imported into individual log-parsing scripts located in logs/*. Data
   7is formatted in HTML or plaintext. Writing to disk and/or emailng data is left
   8to interface.py.
   9"""
  10
  11import os
  12import re
  13import locale
  14from string import Template
  15from math import floor, ceil
  16from tabulate import tabulate
  17
  18import logparse
  19from logparse import interface, util, mail, config
  20
  21import logging
  22logger = logging.getLogger(__name__)
  23
  24
  25locale.setlocale(locale.LC_ALL, '') # inherit system locale
  26
  27
  28DEG = u'\N{DEGREE SIGN}'
  29CEL = "C"
  30TIMEFMT = "%X"
  31DATEFMT = "%x"
  32CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
  33CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
  34LINECHARS_DOUBLE = ['║', '═']
  35LINECHARS_SINGLE = ['│', '─']
  36JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
  37JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
  38BULLET = "• "
  39INDENT = "  "
  40
  41
  42global VARSUBST
  43
  44
  45def init_var():
  46    """
  47    Initialise variable substitution templates (should be called before doing
  48    any substitutions)
  49    """
  50
  51    global VARSUBST
  52    css_path = config.prefs.get("html", "css")
  53    if config.prefs.getboolean("html", "css-relpath"):
  54        if interface.argparser.parse_args().no_write:
  55            css_path = os.path.relpath(css_path, ".")
  56        elif interface.argparser.parse_args().destination:
  57            css_path = os.path.relpath(css_path, interface.argparser.parse_args().destination())
  58        elif config.prefs.get("logparse", "output"):
  59            css_path = os.path.relpath(css_path, config.prefs.get("logparse", "output"))
  60    VARSUBST = {
  61        "title": config.prefs.get("logparse", "title"),
  62        "date": interface.start.strftime(DATEFMT),
  63        "time": interface.start.strftime(TIMEFMT),
  64        "hostname": util.hostname(config.prefs.get("logparse", "hostname-path")),
  65        "version": logparse.__version__,
  66        "css": css_path
  67    }
  68
  69
  70class Output:
  71    """
  72    Base class for a data processor. 
  73    """
  74    
  75    def __init__(self):
  76        self.content = ""
  77        self.destination = ""
  78
  79    def append(self, content):
  80        """
  81        Add a string
  82        """
  83
  84        self.content += content
  85
  86    def write(self, destination=""):
  87        """
  88        Output contents into a file
  89        """
  90
  91        if destination == "":
  92            destination = self.destination
  93        if destination == "":
  94            logger.warning("No destination path provided")
  95            return 1
  96        with open(destination, 'w') as f:
  97            f.write(self.content)
  98            logger.info("Written output to {}".format(destination))
  99
 100    def print_stdout(self, lines=False):
 101        """
 102        Echo the contents to the console
 103        """
 104
 105        print()
 106        if lines:
 107            line = PlaintextLine(linewidth=config.prefs.getint("plain", "linewidth"), double=True)
 108            print(line.draw())
 109        print(self.content)
 110        if lines:
 111            print(line.draw())
 112        print()
 113
 114
 115class PlaintextOutput(Output):
 116    """
 117    Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
 118    """
 119
 120    def __init__(self, linewidth=80):
 121        self.content = ""
 122        self.destination = ""
 123        self.linewidth = linewidth;
 124
 125    def append_header(self, template=''):
 126        """
 127        Print details with some primitive formatting
 128        """
 129        box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(VARSUBST), vpadding=2, hpadding="\t\t", linewidth=self.linewidth)
 130        line = PlaintextLine(self.linewidth)
 131        self.append(box.draw() + line.draw())
 132
 133    def append_footer(self):
 134        """
 135        Append a horizontal line and some details
 136        """
 137        self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
 138        self.append(Template("$hostname $time $date").safe_substitute(VARSUBST))
 139
 140    def append_section(self, section):
 141        """
 142        Call the appropriate methods to format a section (provided by a parser).
 143        This should be run by interface.py after every instance of parse_log().
 144        """
 145
 146        self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
 147        if section.period and section.period.unique:
 148            self.append("\n(since {0})".format(section.period.startdate.strftime(DATEFMT + " " + TIMEFMT)))
 149        self.append('\n'*2)
 150        for data in section.data:
 151            self.append(self._fmt_data(data.subtitle, data.items))
 152            self.append('\n')
 153        for table in section.tables:
 154            self.append(table.draw_plain())
 155        self.append("\n")
 156
 157    def _fmt_data(self, subtitle, data = None):   # write title and data
 158        """
 159        Format the properties of a data object into usable plaintext form with
 160        a few fancy symbols/formatting tricks. Subtitle is required, data is
 161        not. If only subtitle is supplied or subtitle + one data item, a single
 162        line will be printed.
 163        """
 164
 165        if (subtitle == ""):
 166            logger.warning("No subtitle provided.. skipping section")
 167            return
 168
 169        if (data == None or len(data) == 0):
 170            logger.debug("No data provided.. just printing subtitle")
 171            return subtitle + '\n'
 172        else:
 173            logger.debug("Received data " + str(data))
 174            subtitle += ':'
 175            if (len(data) == 1):
 176                return subtitle + ' ' + data[0] + '\n'
 177            else:
 178                itemoutput = subtitle + '\n'
 179                for datum in data:
 180                    datum = BULLET + datum
 181                    if len(datum) > self.linewidth - 3:
 182                        words = datum.split()
 183                        if max(map(len, words)) > self.linewidth - len(INDENT):
 184                            continue
 185                        res, part, others = [], words[0], words[1:]
 186                        for word in others:
 187                            if 1 + len(word) > self.linewidth - len(part):
 188                                res.append(part)
 189                                part = word
 190                            else:
 191                                part += ' ' + word
 192                        if part:
 193                            res.append(part)
 194                        datum = ('\n    ').join(res)
 195                    datum = INDENT + datum
 196                    itemoutput += datum + '\n'
 197                return itemoutput
 198
 199class HtmlOutput(Output):
 200    """
 201    Process and output data in HTML format. All HTML formatting functions now
 202    reside in this class to differentiate them from plain text.
 203    """
 204
 205    def __init__(self):
 206        """
 207        Initialise variables (no parameters required for initialisation)
 208        """
 209
 210        self.content = ""
 211        self.destination = ""
 212        self.css = ""
 213        self._embedded = ""
 214
 215    def embed_css(self, css):
 216        """
 217        Convert stylesheet to inline tags
 218        """
 219
 220        if not self._embedded:
 221            self._embedded = mail.mailprep(re.sub(".*" + re.escape(VARSUBST['css']) + ".*\n", "", self.content), css)
 222        return self._embedded
 223
 224    def write_embedded(self, destination = ""):
 225        """
 226        Write contents to file with inline CSS tags
 227        """
 228
 229        logger.debug("Writing HTML with embedded styles to " + destination)
 230        if not self._embedded:
 231            logger.warning("Call to write_embedded before embed_css - \
 232                    embedding stylesheets immediately")
 233            self.embed_css(config.prefs.get("html", "css"))
 234        if destination == "":
 235            destination = self.destination
 236        if destination == "":
 237            logger.warning("No destination path provided")
 238            return 1
 239        with open(destination, 'w') as f:
 240            f.write(self._embedded)
 241            logger.info("Written output to {}".format(destination))
 242
 243
 244    def append_header(self, template):
 245        """
 246        Insert variables into header template file and append HTML tags
 247        """
 248
 249        self.headertemplate = template
 250        headercontent = Template(open(template, 'r').read())
 251        self.append(headercontent.safe_substitute(VARSUBST))
 252        self.append(opentag('div', id='main'))
 253
 254    def append_footer(self):
 255        """
 256        Close HTML tags that were opened in the template.
 257        TODO: add footer template similar to header template.
 258        """
 259
 260        self.append(closetag('div') + closetag('body') + closetag('html'))
 261
 262    def append_section(self, section):
 263        """
 264        Call the appropriate methods to generate HTML tags for a section (provided by a parser).
 265        This should be run by interface.py after every instance of parse_log().
 266        """
 267
 268        self.append(opentag('div', 1, section.title, 'section'))
 269        self.append(self._gen_title(section.title))
 270        if section.period and section.period.unique:
 271            self.append(self._fmt_period(section.period))
 272        for data in section.data:
 273            self.append(self._fmt_data(data.subtitle, data.items, data.severity))
 274        for table in section.tables:
 275            self.append(table.draw_html())
 276        self.append(closetag('div', 1))
 277
 278    def _gen_title(self, title):
 279        """
 280        Format the title for a section
 281        """
 282
 283        if (title == '' or '\n' in title):
 284            logger.error("Invalid title")
 285            raise ValueError 
 286        logger.debug("Writing title for " + title)
 287        return tag('h2', False, title)
 288
 289    def _fmt_data(self, subtitle, data=None, severity=0):
 290        """
 291        Format the properties of a data object into usable HTML tags.
 292        Subtitle is required, data is not. If only subtitle is supplied or
 293        subtitle + one data item, a single line will be printed.
 294        """
 295
 296        if (subtitle == ""):
 297            logger.warning("No subtitle provided.. skipping section")
 298            return
 299
 300        if (data == None or len(data) == 0):
 301            logger.debug("No data provided.. just printing subtitle")
 302            return tag('p', False, subtitle, cl="severity-" + str(severity))
 303        else:
 304            logger.debug("Received data " + str(data))
 305            subtitle += ':'
 306            if (len(data) == 1):
 307                return tag('p', False, subtitle + ' ' + data[0], cl="severity-" + str(severity))
 308            else:
 309                output = ""
 310                output += tag('p', False, subtitle, cl="severity-" + str(severity))
 311                output += opentag('ul', 1)
 312                coderegex = re.compile('`(.*)`')
 313                for datum in data:
 314                    if datum == "" or datum == None:
 315                        continue
 316                    datum = coderegex.sub(r"<code>\1</code>", str(datum))
 317                    output += tag('li', False, datum)
 318                output += closetag('ul', True)
 319                return output
 320
 321    def _fmt_period(self, period):
 322        output = ''
 323        output += opentag('span', cl='period')
 324        output += "since " + period.startdate.strftime(DATEFMT + " " + TIMEFMT)
 325        output += closetag('span')
 326        return output
 327
 328    def print_stdout_embedded(self, lines=False):
 329        """
 330        Echo the version with embedded style tags to the console
 331        """
 332
 333        if self._embedded == "":
 334            self.embed_css(config.prefs.get("html", "css"))
 335        print()
 336        if lines:
 337            line = PlaintextLine(linewidth=config.prefs.getint("plain", "linewidth"), double=True)
 338            print(line.draw())
 339        print(self._embedded)
 340        if lines:
 341            print(line.draw())
 342        print()
 343
 344
 345class Section:
 346    """
 347    Each parser should output a Section() which contains the title, returned
 348    data, and applicable time period.
 349    """
 350
 351    def __init__(self, title, period=None):
 352        self.title = title
 353        self.data = []
 354        self.tables = []
 355        self.period = util.LogPeriod(self.title)
 356
 357    def append_data(self, data):
 358        self.data.append(data)
 359
 360    def append_table(self, table):
 361        self.tables.append(table)
 362
 363
 364class Data:
 365    """
 366    Each section (parser) can have one or more Data() objects which are
 367    essentially glorified lists with titles (`self.subtitle`).
 368    """
 369    
 370    def __init__(self, subtitle="", items=[], severity=0):
 371        """
 372        Initialise variables. No parameters are enforced upon initialisation,
 373        but at least the subtitle is required for valid output. Severity refers
 374        to the importance of the data (integer from 0 to 5). e.g. a failed
 375        system should have severity 5 and will be formatted appropriately by
 376        the Output object.
 377        """
 378
 379        self.subtitle = subtitle
 380        self.items = items 
 381        self.severity = severity
 382
 383    def truncl(self, limit):      # truncate list
 384        """
 385        Truncate self.items to a specified value and state how many items are hidden.
 386        """
 387
 388        if (len(self.items) > limit):
 389            more = len(self.items) - limit
 390            if more == 1:
 391                return 0
 392            self.items = self.items[:limit]
 393            self.items.append("+ {0} more".format(str(more)))
 394        return self
 395
 396    def orderbyfreq(self):
 397        """
 398        Order a list by frequency of each item, then remove duplicates and
 399        append frequency in parentheses.
 400        """
 401
 402        unsorted = list(self.items)
 403        self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
 404        return self
 405
 406
 407class Table(object):
 408    """
 409    A wrapper for python-tabulate's Tabulate type.
 410    """
 411    
 412    def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]):
 413        """
 414        Initialise variables. Note the keymap is used for a faster index map,
 415        but is not currently used anywhere (may be removed in future).
 416        """
 417
 418        self.rows =  []     # List of row objects
 419        self.keymap = {}    # For fast lookup of row by value of first column 
 420        self.double = double
 421        self.borders = borders
 422        self.align_cols = []
 423        self.hpadding = hpadding
 424        self.maxwidth = maxwidth
 425        self.headers = headers
 426        self._align_cols = []
 427
 428    def add_row(self, row):
 429        """
 430        Append a row to the list and amend index mapping
 431        """
 432
 433        self.rows.append(row)
 434        if len(row.columns) > 0:
 435            self.keymap[row.columns[0]] = row
 436
 437        logger.debug("Added row with {0} columns".format(str(len(row.columns))))
 438
 439    def align_column(self, i, align):
 440        """
 441        Set alignment for the 'i'th column (`align` should be 'l', 'c' or 'r')
 442        """
 443
 444        while len(self._align_cols) -1 < i:
 445            self._align_cols.append("")
 446        self._align_cols[i] = align
 447        for row in self.rows:
 448            row.columns[i].align = align
 449        logger.debug("Column alignment is now {0}".format(str(self._align_cols)))
 450
 451    def _gen_list(self):
 452        """
 453        Used locally for organising rows and columns into a 2D list structure
 454        """
 455
 456        hierarchy = []
 457        for row in self.rows:
 458            row_data = []
 459            for column in row.columns:
 460                row_data.append(column.content)
 461            hierarchy.append(row_data)
 462        return hierarchy
 463
 464    def draw_html(self):
 465        """
 466        Output HTML string (wrapper for tabulate)
 467        """
 468
 469        output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols))
 470        return output
 471
 472    def draw_plain(self):
 473        """
 474        Output plain text string (wrapper for tabulate)
 475        """
 476
 477        output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols))
 478        return output + "\n"*2
 479
 480
 481class Row(object):
 482    """
 483    Object representing a literal row in a 2D table with the individual cells
 484    in the row represented by columns[].
 485    """
 486    
 487    def __init__(self, columns=[], header=False):
 488        """
 489        Initialise variables. The variable n is used locally to keep track of
 490        the row width.
 491        """
 492
 493        self.columns = columns
 494        self.header = header
 495        self.n = len(self.columns)
 496
 497    def add_column(self, column):
 498        """
 499        Append a single cell horizontally and increment the cell count
 500        """
 501
 502        self.columns.append(column)
 503        self.n += 1
 504
 505    def rm_column(self, column):
 506        """
 507        Remove the specified column object and decrement the cell count
 508        """
 509
 510        self.remove(column)
 511        self.n -= 1
 512
 513
 514class Column(object):
 515    """
 516    Object representing a single table cell. This is somewhat of a misnomer - 
 517    one column object exists for each cell in the table. Columns are children
 518    of rows.
 519    """
 520
 521    def __init__(self, content="", align="right"):
 522        """
 523        Initialise variables. The align property sets the alignment of a single
 524        cell ('l', 'c', or 'r').
 525        """
 526
 527        self.content = content
 528        self.align = align
 529
 530
 531class PlaintextLine:
 532    """
 533    Draw a horizontal line for plain text format, with optional padding/styling.
 534    """
 535
 536    def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
 537        """
 538        Initialise variables
 539        """
 540
 541        self.linewidth = linewidth
 542        self.double = double
 543        self.vpadding = vpadding
 544        self.hpadding = hpadding
 545
 546    def draw(self):
 547        """
 548        Output a plain text string based on the current object parameters
 549        """
 550
 551        line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
 552        return "\n" * self.vpadding + self.hpadding +  line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
 553
 554
 555class PlaintextBox:
 556    """
 557    Draw a rectangular box around text, with customisable padding/size/style
 558    """
 559
 560    def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
 561        """
 562        Initialise variables
 563        """
 564        self.content = content
 565        self.fullwidth = fullwidth
 566        self.linewidth = linewidth
 567        self.hpadding = hpadding 
 568        self.vpadding = vpadding
 569        self.double = double
 570
 571    def draw(self):
 572        """
 573        Output a plain text string based on the current object parameters. This
 574        involves calculating the text width, breaking text at the maximum line
 575        length, and then drawing a box around it all.
 576        """
 577
 578        if self.double == True:
 579            cornerchars = CORNERCHARS_DOUBLE
 580            linechars = LINECHARS_DOUBLE
 581        else:
 582            cornerchars = CORNERCHARS_SINGLE
 583            linechars = LINECHARS_SINGLE
 584
 585        # Check hpadding has a definite width
 586        self.hpadding = self.hpadding.replace("\t", " "*4)
 587
 588        # Calculate number of characters per line
 589        contentlines = self.content.splitlines()
 590        contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
 591        logger.debug("Contentwidth is {0}".format(str(contentwidth)))
 592        logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
 593        contentwidth += -2*(len(self.hpadding)+1)
 594        if not self.fullwidth:
 595            longestline = len(max(contentlines, key=len))
 596            if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
 597                contentwidth = longestline
 598
 599        # Split lines that are too long
 600        for i, line in enumerate(contentlines):
 601            if len(line) > contentwidth:
 602                words = line.split()
 603                if max(map(len, words)) > contentwidth:
 604                    continue
 605                res, part, others = [], words[0], words[1:]
 606                for word in others:
 607                    if len(' ') + len(word) > contentwidth - len(part):
 608                        res.append(part)
 609                        part = word
 610                    else:
 611                        part += ' ' + word
 612                if part:
 613                    res.append(part)
 614                contentlines[i] = res
 615
 616        # Flatten list
 617        #   Note list comprehension doesn't work here, so we must iterate through each item
 618        newlines = []
 619        for line in contentlines:
 620            if isinstance(line, list):
 621                for subline in line:
 622                    newlines.append(subline)
 623            else:
 624                newlines.append(line)
 625        contentlines = newlines
 626               
 627        # Add vertical padding
 628        for _ in range(self.vpadding):
 629            contentlines.insert(0, ' '*contentwidth)
 630            contentlines.append(' '*contentwidth)
 631
 632        # Insert horizontal padding on lines that are too short
 633        contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
 634        contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
 635        contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
 636        return ('\n').join(contentlines)
 637
 638
 639def backticks(l):
 640    """
 641    Surround every item in a list by backticks. Used for showing code in both
 642    HTML and plain text formats (converted to <code> tags for HTML)
 643    """
 644
 645    return ["`" + x + "`" for x in l]
 646
 647
 648def plural(noun, quantity, print_quantity=True):
 649    """
 650    Return "1 noun" or "n nouns"
 651    """
 652
 653    if (quantity == 1):
 654        if print_quantity:
 655            return(str(quantity) + " " + noun)
 656        else:
 657            return noun
 658    else:
 659        if noun.endswith("s"):
 660            noun += "e"
 661        if print_quantity:
 662            return(str(quantity) + " " + noun + "s")
 663        else:
 664            return noun + "s"
 665
 666
 667def parsesize(num, suffix='B'):
 668    """
 669    Return human-readable size from number of bytes
 670    """
 671
 672    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 673        if abs(num) < 1024.0:
 674            return "%3.1f %s%s" % (num, unit, suffix)
 675        num /= 1024.0
 676    return "%.1f%s%s" % (num, 'Yi', suffix)
 677
 678
 679def fsubject(subject):
 680    """
 681    Replace variables in the title template provided in config
 682    """
 683
 684    r = Template(subject).safe_substitute(VARSUBST)
 685    logger.debug("Returning subject line " + r)
 686    return r
 687
 688
 689def opentag(tag, block=False, id=None, cl=None, style=None):
 690    """
 691    Write HTML opening tag
 692    """
 693
 694    output = ""
 695    if block:
 696        output += '\n'
 697    output += '<' + tag
 698    if id:
 699        output += " id='" + id + "'"
 700    if cl:
 701        output += " class='" + cl + "'"
 702    if style:
 703        output += " style='"
 704        output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items())
 705        output += "'"
 706    output += '>'
 707    if block:
 708        output += '\n'
 709    return output
 710
 711
 712def closetag(tag, block=False):
 713    """
 714    Write HTML closing tag
 715    """
 716
 717    if block:
 718        return "\n</" + tag + ">\n"
 719    else:
 720        return "</" + tag + ">"
 721
 722
 723def tag(tag, block=False, content="", id=None, cl=None, style=None):
 724    """
 725    Write HTML opening tag, content, and closing tag
 726    """
 727
 728    o = opentag(tag, block, id, cl, style)
 729    c = closetag(tag, block)
 730    return o + content + c