8528e1b262413bc00dd477b62b7a8298ff7da47b
   1# -*- coding: utf-8 -*-
   2
   3"""   
   4This file contains global functions for formatting and printing data. This file
   5should be imported into individual log-parsing scripts located in logs/*. Data
   6is formatted in HTML or plaintext. Writing to disk and/or emailng data is left
   7to interface.py.
   8"""
   9
  10import os
  11import re
  12import locale
  13from string import Template
  14from math import floor, ceil
  15from tabulate import tabulate
  16
  17import logparse
  18from logparse import interface, util, mail, config
  19
  20import logging
  21logger = None
  22logger = logging.getLogger(__name__)
  23
  24
  25locale.setlocale(locale.LC_ALL, '') # inherit system locale
  26
  27
  28DEG = u'\N{DEGREE SIGN}'
  29CEL = "C"
  30TIMEFMT = "%X"
  31DATEFMT = "%x"
  32CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
  33CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
  34LINECHARS_DOUBLE = ['║', '═']
  35LINECHARS_SINGLE = ['│', '─']
  36JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
  37JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
  38BULLET = "• "
  39INDENT = "  "
  40
  41
  42global VARSUBST
  43
  44
  45def init_var():
  46    """
  47    Initialise variable substitution templates (should be called before doing
  48    any substitutions)
  49    """
  50
  51    global VARSUBST
  52    css_path = config.prefs.get("html", "css")
  53    if config.prefs.getboolean("html", "css-relpath"):
  54        if interface.argparser.parse_args().no_write:
  55            css_path = os.path.relpath(css_path, ".")
  56        elif interface.argparser.parse_args().destination:
  57            css_path = os.path.relpath(
  58                    css_path, interface.argparser.parse_args().destination())
  59        elif config.prefs.get("logparse", "output"):
  60            css_path = os.path.relpath(
  61                    css_path, config.prefs.get("logparse", "output"))
  62    VARSUBST = {
  63        "title": config.prefs.get("logparse", "title"),
  64        "date": interface.start.strftime(DATEFMT),
  65        "time": interface.start.strftime(TIMEFMT),
  66        "hostname": util.hostname(config.prefs.get(
  67            "logparse", "hostname-path")),
  68        "version": logparse.__version__,
  69        "css": css_path
  70    }
  71
  72
  73class Output:
  74    """
  75    Base class for a data processor. 
  76    """
  77    
  78    def __init__(self):
  79        self.content = ""
  80        self.destination = ""
  81
  82    def append(self, content):
  83        """
  84        Add a string
  85        """
  86
  87        self.content += content
  88
  89    def write(self, destination=""):
  90        """
  91        Output contents into a file
  92        """
  93
  94        if destination == "":
  95            destination = self.destination
  96        if destination == "":
  97            logger.warning("No destination path provided")
  98            return 1
  99        with open(destination, 'w') as f:
 100            f.write(self.content)
 101            logger.info("Written output to {}".format(destination))
 102
 103    def print_stdout(self, lines=False):
 104        """
 105        Echo the contents to the console
 106        """
 107
 108        print()
 109        if lines:
 110            line = PlaintextLine(linewidth=
 111                    config.prefs.getint("plain", "linewidth"), double=True)
 112            print(line.draw())
 113        print(self.content)
 114        if lines:
 115            print(line.draw())
 116        print()
 117
 118
 119class PlaintextOutput(Output):
 120    """
 121    Processes & outputs data in a plaintext form which can be read with cat or
 122    plaintext email.
 123    """
 124
 125    def __init__(self, linewidth=80):
 126        self.content = ""
 127        self.destination = ""
 128        self.linewidth = linewidth;
 129
 130    def append_header(self, template=''):
 131        """
 132        Print details with some primitive formatting
 133        """
 134        box = PlaintextBox(content=
 135                Template("$title $version on $hostname\n\n$time $date")
 136                .safe_substitute(VARSUBST),
 137                vpadding=2, hpadding="\t\t", linewidth=self.linewidth)
 138        line = PlaintextLine(self.linewidth)
 139        self.append(box.draw() + line.draw())
 140
 141    def append_footer(self):
 142        """
 143        Append a horizontal line and some details
 144        """
 145        self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
 146        self.append(Template("$hostname $time $date").safe_substitute(VARSUBST))
 147
 148    def append_section(self, section):
 149        """
 150        Call the appropriate methods to format a section (provided by a parser).
 151        This should be run by interface.py after every instance of parse_log().
 152        """
 153
 154        self.append(PlaintextBox(
 155            content=section.title, double=False,
 156            fullwidth=False, vpadding=0, hpadding=" ").draw())
 157        if section.period and section.period.unique:
 158            self.append("\n(since {0})".format(
 159                section.period.startdate.strftime(DATEFMT + " " + TIMEFMT)))
 160        self.append('\n'*2)
 161        for data in section.data:
 162            self.append(self._fmt_data(data.subtitle, data.items))
 163            self.append('\n')
 164        for table in section.tables:
 165            self.append(table.draw_plain())
 166        self.append("\n")
 167
 168    def _fmt_data(self, subtitle, data = None):   # write title and data
 169        """
 170        Format the properties of a data object into usable plaintext form with
 171        a few fancy symbols/formatting tricks. Subtitle is required, data is
 172        not. If only subtitle is supplied or subtitle + one data item, a single
 173        line will be printed.
 174        """
 175
 176        if (subtitle == ""):
 177            logger.warning("No subtitle provided.. skipping section")
 178            return
 179
 180        if (data == None or len(data) == 0):
 181            logger.debug("No data provided.. just printing subtitle")
 182            return subtitle + '\n'
 183        else:
 184            logger.debug("Received data " + str(data))
 185            subtitle += ':'
 186            if (len(data) == 1):
 187                return subtitle + ' ' + data[0] + '\n'
 188            else:
 189                itemoutput = subtitle + '\n'
 190                for datum in data:
 191                    datum = BULLET + datum
 192                    if len(datum) > self.linewidth - 3:
 193                        words = datum.split()
 194                        if max(map(len, words)) > self.linewidth - len(INDENT):
 195                            continue
 196                        res, part, others = [], words[0], words[1:]
 197                        for word in others:
 198                            if 1 + len(word) > self.linewidth - len(part):
 199                                res.append(part)
 200                                part = word
 201                            else:
 202                                part += ' ' + word
 203                        if part:
 204                            res.append(part)
 205                        datum = ('\n    ').join(res)
 206                    datum = INDENT + datum
 207                    itemoutput += datum + '\n'
 208                return itemoutput
 209
 210class HtmlOutput(Output):
 211    """
 212    Process and output data in HTML format. All HTML formatting functions now
 213    reside in this class to differentiate them from plain text.
 214    """
 215
 216    def __init__(self):
 217        """
 218        Initialise variables (no parameters required for initialisation)
 219        """
 220
 221        self.content = ""
 222        self.destination = ""
 223        self.css = ""
 224        self._embedded = ""
 225
 226    def embed_css(self, css):
 227        """
 228        Convert stylesheet to inline tags
 229        """
 230
 231        if not self._embedded:
 232            self._embedded = mail.mailprep(re.sub(
 233                ".*" + re.escape(VARSUBST['css']) + ".*\n", "", self.content),
 234                css)
 235        return self._embedded
 236
 237    def write_embedded(self, destination = ""):
 238        """
 239        Write contents to file with inline CSS tags
 240        """
 241
 242        logger.debug("Writing HTML with embedded styles to " + destination)
 243        if not self._embedded:
 244            logger.warning("Call to write_embedded before embed_css - \
 245                    embedding stylesheets immediately")
 246            self.embed_css(config.prefs.get("html", "css"))
 247        if destination == "":
 248            destination = self.destination
 249        if destination == "":
 250            logger.warning("No destination path provided")
 251            return 1
 252        with open(destination, 'w') as f:
 253            f.write(self._embedded)
 254            logger.info("Written output to {}".format(destination))
 255
 256
 257    def append_header(self, template):
 258        """
 259        Insert variables into header template file and append HTML tags
 260        """
 261
 262        self.headertemplate = template
 263        headercontent = Template(open(template, 'r').read())
 264        self.append(headercontent.safe_substitute(VARSUBST))
 265        self.append(opentag('div', id='main'))
 266
 267    def append_footer(self):
 268        """
 269        Close HTML tags that were opened in the template.
 270        TODO: add footer template similar to header template.
 271        """
 272
 273        self.append(closetag('div') + closetag('body') + closetag('html'))
 274
 275    def append_section(self, section):
 276        """
 277        Call the appropriate methods to generate HTML tags for a section
 278        (provided by a parser). This should be run by interface.py after every
 279        instance of parse_log().
 280        """
 281
 282        self.append(opentag('div', 1, section.title, 'section'))
 283        self.append(self._gen_title(section.title))
 284        if section.period and section.period.unique:
 285            self.append(self._fmt_period(section.period))
 286        for data in section.data:
 287            self.append(self._fmt_data(data.subtitle, data.items, data.severity))
 288        for table in section.tables:
 289            self.append(table.draw_html())
 290        self.append(closetag('div', 1))
 291
 292    def _gen_title(self, title):
 293        """
 294        Format the title for a section
 295        """
 296
 297        if (title == '' or '\n' in title):
 298            logger.error("Invalid title")
 299            raise ValueError 
 300        logger.debug("Writing title for " + title)
 301        return tag('h2', False, title)
 302
 303    def _fmt_data(self, subtitle, data=None, severity=0):
 304        """
 305        Format the properties of a data object into usable HTML tags.
 306        Subtitle is required, data is not. If only subtitle is supplied or
 307        subtitle + one data item, a single line will be printed.
 308        """
 309
 310        if (subtitle == ""):
 311            logger.warning("No subtitle provided.. skipping section")
 312            return
 313
 314        if (data == None or len(data) == 0):
 315            logger.debug("No data provided.. just printing subtitle")
 316            return tag('p', False, subtitle, cl="severity-" + str(severity))
 317        else:
 318            logger.debug("Received data " + str(data))
 319            subtitle += ':'
 320            if (len(data) == 1):
 321                return tag('p', False, subtitle + ' ' + data[0],
 322                        cl="severity-" + str(severity))
 323            else:
 324                output = ""
 325                output += tag('p', False, subtitle,
 326                        cl="severity-" + str(severity))
 327                output += opentag('ul', 1)
 328                coderegex = re.compile('`(.*)`')
 329                for datum in data:
 330                    if datum == "" or datum == None:
 331                        continue
 332                    datum = coderegex.sub(r"<code>\1</code>", str(datum))
 333                    output += tag('li', False, datum)
 334                output += closetag('ul', True)
 335                return output
 336
 337    def _fmt_period(self, period):
 338        output = ''
 339        output += opentag('span', cl='period')
 340        output += "since " + period.startdate.strftime(DATEFMT + " " + TIMEFMT)
 341        output += closetag('span')
 342        return output
 343
 344    def print_stdout_embedded(self, lines=False):
 345        """
 346        Echo the version with embedded style tags to the console
 347        """
 348
 349        if self._embedded == "":
 350            self.embed_css(config.prefs.get("html", "css"))
 351        print()
 352        if lines:
 353            line = PlaintextLine(linewidth=
 354                    config.prefs.getint("plain", "linewidth"), double=True)
 355            print(line.draw())
 356        print(self._embedded)
 357        if lines:
 358            print(line.draw())
 359        print()
 360
 361
 362class Section:
 363    """
 364    Each parser should output a Section() which contains the title, returned
 365    data, and applicable time period.
 366    """
 367
 368    def __init__(self, title, period=None):
 369        self.title = title
 370        self.data = []
 371        self.tables = []
 372        self.period = util.LogPeriod(self.title)
 373
 374    def append_data(self, data):
 375        self.data.append(data)
 376
 377    def append_table(self, table):
 378        self.tables.append(table)
 379
 380
 381class Data:
 382    """
 383    Each section (parser) can have one or more Data() objects which are
 384    essentially glorified lists with titles (`self.subtitle`).
 385    """
 386    
 387    def __init__(self, subtitle="", items=[], severity=0):
 388        """
 389        Initialise variables. No parameters are enforced upon initialisation,
 390        but at least the subtitle is required for valid output. Severity refers
 391        to the importance of the data (integer from 0 to 5). e.g. a failed
 392        system should have severity 5 and will be formatted appropriately by
 393        the Output object.
 394        """
 395
 396        self.subtitle = subtitle
 397        self.items = items 
 398        self.severity = severity
 399
 400    def truncl(self, limit):      # truncate list
 401        """
 402        Truncate self.items to a specified value and state how many items are
 403        hidden.
 404        """
 405
 406        if (len(self.items) > limit):
 407            more = len(self.items) - limit
 408            if more == 1:
 409                return 0
 410            self.items = self.items[:limit]
 411            self.items.append("+ {0} more".format(str(more)))
 412        return self
 413
 414    def orderbyfreq(self):
 415        """
 416        Order a list by frequency of each item, then remove duplicates and
 417        append frequency in parentheses.
 418        """
 419
 420        unsorted = list(self.items)
 421        self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(
 422            set(unsorted), key = lambda x: -unsorted.count(x))]
 423        return self
 424
 425
 426class Table(object):
 427    """
 428    A wrapper for python-tabulate's Tabulate type.
 429    """
 430    
 431    def __init__(self, double=False, borders=False, hpadding=" ",
 432            maxwidth=80, headers=[]):
 433        """
 434        Initialise variables. Note the keymap is used for a faster index map,
 435        but is not currently used anywhere (may be removed in future).
 436        """
 437
 438        self.rows =  []     # List of row objects
 439        self.keymap = {}    # For fast lookup of row by value of first column 
 440        self.double = double
 441        self.borders = borders
 442        self.align_cols = []
 443        self.hpadding = hpadding
 444        self.maxwidth = maxwidth
 445        self.headers = headers
 446        self._align_cols = []
 447
 448    def add_row(self, row):
 449        """
 450        Append a row to the list and amend index mapping
 451        """
 452
 453        self.rows.append(row)
 454        if len(row.columns) > 0:
 455            self.keymap[row.columns[0]] = row
 456
 457        logger.debug("Added row with {0} columns".format(str(len(row.columns))))
 458
 459    def align_column(self, i, align):
 460        """
 461        Set alignment for the 'i'th column (`align` should be 'l', 'c' or 'r')
 462        """
 463
 464        while len(self._align_cols) -1 < i:
 465            self._align_cols.append("")
 466        self._align_cols[i] = align
 467        for row in self.rows:
 468            row.columns[i].align = align
 469        logger.debug("Column alignment is now {0}".format(self._align_cols))
 470
 471    def _gen_list(self):
 472        """
 473        Used locally for organising rows and columns into a 2D list structure
 474        """
 475
 476        hierarchy = []
 477        for row in self.rows:
 478            row_data = []
 479            for column in row.columns:
 480                row_data.append(column.content)
 481            hierarchy.append(row_data)
 482        return hierarchy
 483
 484    def draw_html(self):
 485        """
 486        Output HTML string (wrapper for tabulate)
 487        """
 488
 489        output = tabulate(self._gen_list(), self.headers, tablefmt="html",
 490                colalign=tuple(self._align_cols))
 491        return output
 492
 493    def draw_plain(self):
 494        """
 495        Output plain text string (wrapper for tabulate)
 496        """
 497
 498        output = tabulate(self._gen_list(), self.headers,
 499                tablefmt="fancy_grid" if self.borders
 500                else "plain", colalign=tuple(self._align_cols))
 501        return output + "\n"*2
 502
 503
 504class Row(object):
 505    """
 506    Object representing a literal row in a 2D table with the individual cells
 507    in the row represented by columns[].
 508    """
 509    
 510    def __init__(self, columns=[], header=False):
 511        """
 512        Initialise variables. The variable n is used locally to keep track of
 513        the row width.
 514        """
 515
 516        self.columns = columns
 517        self.header = header
 518        self.n = len(self.columns)
 519
 520    def add_column(self, column):
 521        """
 522        Append a single cell horizontally and increment the cell count
 523        """
 524
 525        self.columns.append(column)
 526        self.n += 1
 527
 528    def rm_column(self, column):
 529        """
 530        Remove the specified column object and decrement the cell count
 531        """
 532
 533        self.remove(column)
 534        self.n -= 1
 535
 536
 537class Column(object):
 538    """
 539    Object representing a single table cell. This is somewhat of a misnomer - 
 540    one column object exists for each cell in the table. Columns are children
 541    of rows.
 542    """
 543
 544    def __init__(self, content="", align="right"):
 545        """
 546        Initialise variables. The align property sets the alignment of a single
 547        cell ('l', 'c', or 'r').
 548        """
 549
 550        self.content = content
 551        self.align = align
 552
 553
 554class PlaintextLine:
 555    """
 556    Draw a horizontal line for plain text format, with optional padding/styling.
 557    """
 558
 559    def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
 560        """
 561        Initialise variables
 562        """
 563
 564        self.linewidth = linewidth
 565        self.double = double
 566        self.vpadding = vpadding
 567        self.hpadding = hpadding
 568
 569    def draw(self):
 570        """
 571        Output a plain text string based on the current object parameters
 572        """
 573
 574        line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
 575        return "\n" * self.vpadding + self.hpadding \
 576                +  line * (self.linewidth - 2 * len(self.hpadding)) \
 577                + self.hpadding + "\n" * self.vpadding
 578
 579
 580class PlaintextBox:
 581    """
 582    Draw a rectangular box around text, with customisable padding/size/style
 583    """
 584
 585    def __init__(self, content="", double=True, fullwidth=True, linewidth=80,
 586            hpadding="\t", vpadding=1):
 587        """
 588        Initialise variables
 589        """
 590        self.content = content
 591        self.fullwidth = fullwidth
 592        self.linewidth = linewidth
 593        self.hpadding = hpadding 
 594        self.vpadding = vpadding
 595        self.double = double
 596
 597    def draw(self):
 598        """
 599        Output a plain text string based on the current object parameters. This
 600        involves calculating the text width, breaking text at the maximum line
 601        length, and then drawing a box around it all.
 602        """
 603
 604        if self.double == True:
 605            cornerchars = CORNERCHARS_DOUBLE
 606            linechars = LINECHARS_DOUBLE
 607        else:
 608            cornerchars = CORNERCHARS_SINGLE
 609            linechars = LINECHARS_SINGLE
 610
 611        # Check hpadding has a definite width
 612        self.hpadding = self.hpadding.replace("\t", " "*4)
 613
 614        # Calculate number of characters per line
 615        contentlines = self.content.splitlines()
 616        contentwidth = int((self.linewidth if self.linewidth > 0 else 80)
 617                if self.content.splitlines()
 618                else len(max(contentlines, key=len)))
 619        logger.debug("Contentwidth is {0}".format(str(contentwidth)))
 620        logger.debug("Longest line is {0}".format(
 621            len(max(contentlines, key=len))))
 622        contentwidth += -2*(len(self.hpadding)+1)
 623        if not self.fullwidth:
 624            longestline = len(max(contentlines, key=len))
 625            if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
 626                contentwidth = longestline
 627
 628        # Split lines that are too long
 629        for i, line in enumerate(contentlines):
 630            if len(line) > contentwidth:
 631                words = line.split()
 632                if max(map(len, words)) > contentwidth:
 633                    continue
 634                res, part, others = [], words[0], words[1:]
 635                for word in others:
 636                    if len(' ') + len(word) > contentwidth - len(part):
 637                        res.append(part)
 638                        part = word
 639                    else:
 640                        part += ' ' + word
 641                if part:
 642                    res.append(part)
 643                contentlines[i] = res
 644
 645        # Flatten list
 646        #   Note list comprehension doesn't work here, so we must iterate
 647        #   through each item
 648        newlines = []
 649        for line in contentlines:
 650            if isinstance(line, list):
 651                for subline in line:
 652                    newlines.append(subline)
 653            else:
 654                newlines.append(line)
 655        contentlines = newlines
 656               
 657        # Add vertical padding
 658        for _ in range(self.vpadding):
 659            contentlines.insert(0, ' '*contentwidth)
 660            contentlines.append(' '*contentwidth)
 661
 662        # Insert horizontal padding on lines that are too short
 663        contentlines = [linechars[0] + self.hpadding + x
 664                + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2)
 665                    if len(x) < contentwidth else 0)
 666                + self.hpadding + linechars[0] for x in contentlines]
 667        contentlines.insert(0, cornerchars[3] + linechars[1] 
 668                * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
 669        contentlines.append(cornerchars[0] + linechars[1]
 670                * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
 671        return ('\n').join(contentlines)
 672
 673
 674def backticks(l):
 675    """
 676    Surround every item in a list by backticks. Used for showing code in both
 677    HTML and plain text formats (converted to <code> tags for HTML)
 678    """
 679
 680    return ["`" + x + "`" for x in l]
 681
 682
 683def plural(noun, quantity, print_quantity=True):
 684    """
 685    Return "1 noun" or "n nouns"
 686    """
 687
 688    if (quantity == 1):
 689        if print_quantity:
 690            return(str(quantity) + " " + noun)
 691        else:
 692            return noun
 693    else:
 694        if noun.endswith("s"):
 695            noun += "e"
 696        if print_quantity:
 697            return(str(quantity) + " " + noun + "s")
 698        else:
 699            return noun + "s"
 700
 701
 702def parsesize(num, suffix='B'):
 703    """
 704    Return human-readable size from number of bytes
 705    """
 706
 707    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 708        if abs(num) < 1024.0:
 709            return "%3.1f %s%s" % (num, unit, suffix)
 710        num /= 1024.0
 711    return "%.1f%s%s" % (num, 'Yi', suffix)
 712
 713
 714def fsubject(subject):
 715    """
 716    Replace variables in the title template provided in config
 717    """
 718
 719    r = Template(subject).safe_substitute(VARSUBST)
 720    logger.debug("Returning subject line " + r)
 721    return r
 722
 723
 724def opentag(tag, block=False, id=None, cl=None, style=None):
 725    """
 726    Write HTML opening tag
 727    """
 728
 729    output = ""
 730    if block:
 731        output += '\n'
 732    output += '<' + tag
 733    if id:
 734        output += " id='" + id + "'"
 735    if cl:
 736        output += " class='" + cl + "'"
 737    if style:
 738        output += " style='"
 739        output += " ".join("{0}: {1};".format(attr, value)
 740                for attr, value in style.items())
 741        output += "'"
 742    output += '>'
 743    if block:
 744        output += '\n'
 745    return output
 746
 747
 748def closetag(tag, block=False):
 749    """
 750    Write HTML closing tag
 751    """
 752
 753    if block:
 754        return "\n</" + tag + ">\n"
 755    else:
 756        return "</" + tag + ">"
 757
 758
 759def tag(tag, block=False, content="", id=None, cl=None, style=None):
 760    """
 761    Write HTML opening tag, content, and closing tag
 762    """
 763
 764    o = opentag(tag, block, id, cl, style)
 765    c = closetag(tag, block)
 766    return o + content + c