624d66cf588a366ee375c5a19845b0f9e6f632a9
   1# -*- coding: utf-8 -*-
   2
   3"""   
   4This file contains global functions for formatting and printing data. This file
   5should be imported into individual log-parsing scripts located in the default
   6logparse.parsers module or in the user-supplied parsers directory. Data is
   7formatted in HTML or plaintext. Writing to disk and/or emailng data is left to
   8interface.py.
   9"""
  10
  11import os
  12import re
  13import locale
  14from string import Template
  15from math import floor, ceil
  16from tabulate import tabulate
  17
  18import logparse
  19from logparse import interface, util, mail, config
  20
  21import logging
  22logger = None
  23logger = logging.getLogger(__name__)
  24
  25
  26locale.setlocale(locale.LC_ALL, '') # inherit system locale
  27
  28
  29DEG = u'\N{DEGREE SIGN}'
  30CEL = "C"
  31TIMEFMT = "%X"
  32DATEFMT = "%x"
  33CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
  34CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
  35LINECHARS_DOUBLE = ['║', '═']
  36LINECHARS_SINGLE = ['│', '─']
  37JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
  38JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
  39BULLET = "• "
  40INDENT = "  "
  41
  42
  43global VARSUBST
  44
  45
  46def init_var():
  47    """
  48    Initialise variable substitution templates (should be called before doing
  49    any substitutions)
  50    """
  51
  52    global VARSUBST
  53    css_path = config.prefs.get("html", "css")
  54    if config.prefs.getboolean("html", "css-relpath"):
  55        if interface.argparser.parse_args().no_write:
  56            css_path = os.path.relpath(css_path, ".")
  57        elif interface.argparser.parse_args().destination:
  58            css_path = os.path.relpath(
  59                    css_path, interface.argparser.parse_args().destination())
  60        elif config.prefs.get("logparse", "output"):
  61            css_path = os.path.relpath(
  62                    css_path, config.prefs.get("logparse", "output"))
  63    VARSUBST = {
  64        "title": config.prefs.get("logparse", "title"),
  65        "date": interface.start.strftime(DATEFMT),
  66        "time": interface.start.strftime(TIMEFMT),
  67        "hostname": util.hostname(config.prefs.get(
  68            "logparse", "hostname-path")),
  69        "version": logparse.__version__,
  70        "css": css_path
  71    }
  72
  73
  74class Output:
  75    """
  76    Base class for a data processor. 
  77    """
  78    
  79    def __init__(self):
  80        self.content = ""
  81        self.destination = ""
  82
  83    def append(self, content):
  84        """
  85        Add a string
  86        """
  87
  88        self.content += content
  89
  90    def write(self, destination=""):
  91        """
  92        Output contents into a file
  93        """
  94
  95        if destination == "":
  96            destination = self.destination
  97        if destination == "":
  98            logger.warning("No destination path provided")
  99            return 1
 100        with open(destination, 'w') as f:
 101            f.write(self.content)
 102            logger.info("Written output to {}".format(destination))
 103
 104    def print_stdout(self, lines=False):
 105        """
 106        Echo the contents to the console
 107        """
 108
 109        print()
 110        if lines:
 111            line = PlaintextLine(linewidth=
 112                    config.prefs.getint("plain", "linewidth"), double=True)
 113            print(line.draw())
 114        print(self.content)
 115        if lines:
 116            print(line.draw())
 117        print()
 118
 119
 120class PlaintextOutput(Output):
 121    """
 122    Processes & outputs data in a plaintext form which can be read with cat or
 123    plaintext email.
 124    """
 125
 126    def __init__(self, linewidth=80):
 127        self.content = ""
 128        self.destination = ""
 129        self.linewidth = linewidth;
 130
 131    def append_header(self, template=''):
 132        """
 133        Print details with some primitive formatting
 134        """
 135        box = PlaintextBox(content=
 136                Template("$title $version on $hostname\n\n$time $date")
 137                .safe_substitute(VARSUBST),
 138                vpadding=2, hpadding="\t\t", linewidth=self.linewidth)
 139        line = PlaintextLine(self.linewidth)
 140        self.append(box.draw() + line.draw())
 141
 142    def append_footer(self):
 143        """
 144        Append a horizontal line and some details
 145        """
 146        self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
 147        self.append(Template("$hostname $time $date").safe_substitute(VARSUBST))
 148
 149    def append_section(self, section):
 150        """
 151        Call the appropriate methods to format a section (provided by a parser).
 152        This should be run by interface.py after every instance of parse_log().
 153        """
 154
 155        if section == None:
 156            logger.warning("Received null section")
 157            return
 158        self.append(PlaintextBox(
 159            content=section.title, double=False,
 160            fullwidth=False, vpadding=0, hpadding=" ").draw())
 161        if section.period and section.period.unique:
 162            self.append("\n(since {0})".format(
 163                section.period.startdate.strftime(DATEFMT + " " + TIMEFMT)))
 164        self.append('\n'*2)
 165        for data in section.data:
 166            self.append(self._fmt_data(data.subtitle, data.items))
 167            self.append('\n')
 168        for table in section.tables:
 169            self.append(table.draw_plain())
 170        self.append("\n")
 171
 172    def _fmt_data(self, subtitle, data = None):   # write title and data
 173        """
 174        Format the properties of a data object into usable plaintext form with
 175        a few fancy symbols/formatting tricks. Subtitle is required, data is
 176        not. If only subtitle is supplied or subtitle + one data item, a single
 177        line will be printed.
 178        """
 179
 180        if (subtitle == ""):
 181            logger.warning("No subtitle provided.. skipping section")
 182            return
 183
 184        if (data == None or len(data) == 0):
 185            logger.debug("No data provided.. just printing subtitle")
 186            return subtitle + '\n'
 187        else:
 188            logger.debug("Received data " + str(data))
 189            subtitle += ':'
 190            if (len(data) == 1):
 191                return subtitle + ' ' + data[0] + '\n'
 192            else:
 193                itemoutput = subtitle + '\n'
 194                for datum in data:
 195                    datum = BULLET + datum
 196                    if len(datum) > self.linewidth - 3:
 197                        words = datum.split()
 198                        if max(map(len, words)) > self.linewidth - len(INDENT):
 199                            continue
 200                        res, part, others = [], words[0], words[1:]
 201                        for word in others:
 202                            if 1 + len(word) > self.linewidth - len(part):
 203                                res.append(part)
 204                                part = word
 205                            else:
 206                                part += ' ' + word
 207                        if part:
 208                            res.append(part)
 209                        datum = ('\n    ').join(res)
 210                    datum = INDENT + datum
 211                    itemoutput += datum + '\n'
 212                return itemoutput
 213
 214class HtmlOutput(Output):
 215    """
 216    Process and output data in HTML format. All HTML formatting functions now
 217    reside in this class to differentiate them from plain text.
 218    """
 219
 220    def __init__(self):
 221        """
 222        Initialise variables (no parameters required for initialisation)
 223        """
 224
 225        self.content = ""
 226        self.destination = ""
 227        self.css = ""
 228        self._embedded = ""
 229
 230    def embed_css(self, css):
 231        """
 232        Convert stylesheet to inline tags
 233        """
 234
 235        if not self._embedded:
 236            self._embedded = mail.mailprep(re.sub(
 237                ".*" + re.escape(VARSUBST['css']) + ".*\n", "", self.content),
 238                css)
 239        return self._embedded
 240
 241    def write_embedded(self, destination = ""):
 242        """
 243        Write contents to file with inline CSS tags
 244        """
 245
 246        logger.debug("Writing HTML with embedded styles to " + destination)
 247        if not self._embedded:
 248            logger.warning("Call to write_embedded before embed_css - \
 249                    embedding stylesheets immediately")
 250            self.embed_css(config.prefs.get("html", "css"))
 251        if destination == "":
 252            destination = self.destination
 253        if destination == "":
 254            logger.warning("No destination path provided")
 255            return 1
 256        with open(destination, 'w') as f:
 257            f.write(self._embedded)
 258            logger.info("Written output to {}".format(destination))
 259
 260
 261    def append_header(self, template):
 262        """
 263        Insert variables into header template file and append HTML tags
 264        """
 265
 266        self.headertemplate = template
 267        headercontent = Template(open(template, 'r').read())
 268        self.append(headercontent.safe_substitute(VARSUBST))
 269        self.append(opentag('div', id='main'))
 270
 271    def append_footer(self):
 272        """
 273        Close HTML tags that were opened in the template.
 274        TODO: add footer template similar to header template.
 275        """
 276
 277        self.append(closetag('div') + closetag('body') + closetag('html'))
 278
 279    def append_section(self, section):
 280        """
 281        Call the appropriate methods to generate HTML tags for a section
 282        (provided by a parser). This should be run by interface.py after every
 283        instance of parse_log().
 284        """
 285
 286        if section == None:
 287            logger.warning("Received null section")
 288            return
 289        self.append(opentag('div', 1, section.title, 'section'))
 290        self.append(self._gen_title(section.title))
 291        if section.period and section.period.unique:
 292            self.append(self._fmt_period(section.period))
 293        for data in section.data:
 294            self.append(self._fmt_data(data.subtitle, data.items, data.severity))
 295        for table in section.tables:
 296            self.append(table.draw_html())
 297        self.append(closetag('div', 1))
 298
 299    def _gen_title(self, title):
 300        """
 301        Format the title for a section
 302        """
 303
 304        if (title == '' or '\n' in title):
 305            logger.error("Invalid title")
 306            raise ValueError 
 307        logger.debug("Writing title for " + title)
 308        return tag('h2', False, title)
 309
 310    def _fmt_data(self, subtitle, data=None, severity=0):
 311        """
 312        Format the properties of a data object into usable HTML tags.
 313        Subtitle is required, data is not. If only subtitle is supplied or
 314        subtitle + one data item, a single line will be printed.
 315        """
 316
 317        if (subtitle == ""):
 318            logger.warning("No subtitle provided.. skipping section")
 319            return
 320
 321        if (data == None or len(data) == 0):
 322            logger.debug("No data provided.. just printing subtitle")
 323            return tag('p', False, subtitle, cl="severity-" + str(severity))
 324        else:
 325            logger.debug("Received data " + str(data))
 326            subtitle += ':'
 327            if (len(data) == 1):
 328                return tag('p', False, subtitle + ' ' + data[0],
 329                        cl="severity-" + str(severity))
 330            else:
 331                output = ""
 332                output += tag('p', False, subtitle,
 333                        cl="severity-" + str(severity))
 334                output += opentag('ul', 1)
 335                coderegex = re.compile('`(.*)`')
 336                for datum in data:
 337                    if datum == "" or datum == None:
 338                        continue
 339                    datum = coderegex.sub(r"<code>\1</code>", str(datum))
 340                    output += tag('li', False, datum)
 341                output += closetag('ul', True)
 342                return output
 343
 344    def _fmt_period(self, period):
 345        output = ''
 346        output += opentag('span', cl='period')
 347        output += "since " + period.startdate.strftime(DATEFMT + " " + TIMEFMT)
 348        output += closetag('span')
 349        return output
 350
 351    def print_stdout_embedded(self, lines=False):
 352        """
 353        Echo the version with embedded style tags to the console
 354        """
 355
 356        if self._embedded == "":
 357            self.embed_css(config.prefs.get("html", "css"))
 358        print()
 359        if lines:
 360            line = PlaintextLine(linewidth=
 361                    config.prefs.getint("plain", "linewidth"), double=True)
 362            print(line.draw())
 363        print(self._embedded)
 364        if lines:
 365            print(line.draw())
 366        print()
 367
 368
 369class Section:
 370    """
 371    Each parser should output a Section() which contains the title, returned
 372    data, and applicable time period.
 373    """
 374
 375    def __init__(self, title, period=None):
 376        self.title = title
 377        self.data = []
 378        self.tables = []
 379        self.period = util.LogPeriod(self.title)
 380
 381    def append_data(self, data):
 382        self.data.append(data)
 383
 384    def append_table(self, table):
 385        self.tables.append(table)
 386
 387
 388class Data:
 389    """
 390    Each section (parser) can have one or more Data() objects which are
 391    essentially glorified lists with titles (`self.subtitle`).
 392    """
 393    
 394    def __init__(self, subtitle="", items=[], severity=0):
 395        """
 396        Initialise variables. No parameters are enforced upon initialisation,
 397        but at least the subtitle is required for valid output. Severity refers
 398        to the importance of the data (integer from 0 to 5). e.g. a failed
 399        system should have severity 5 and will be formatted appropriately by
 400        the Output object.
 401        """
 402
 403        self.subtitle = subtitle
 404        self.items = items 
 405        self.severity = severity
 406
 407    def truncl(self, limit):      # truncate list
 408        """
 409        Truncate self.items to a specified value and state how many items are
 410        hidden.
 411        """
 412
 413        if (len(self.items) > limit):
 414            more = len(self.items) - limit
 415            if more == 1:
 416                return 0
 417            self.items = self.items[:limit]
 418            self.items.append("+ {0} more".format(str(more)))
 419        return self
 420
 421    def orderbyfreq(self):
 422        """
 423        Order a list by frequency of each item, then remove duplicates and
 424        append frequency in parentheses.
 425        """
 426
 427        unsorted = list(self.items)
 428        self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(
 429            set(unsorted), key = lambda x: -unsorted.count(x))]
 430        return self
 431
 432
 433class Table(object):
 434    """
 435    A wrapper for python-tabulate's Tabulate type.
 436    """
 437    
 438    def __init__(self, double=False, borders=False, hpadding=" ",
 439            maxwidth=80, headers=[]):
 440        """
 441        Initialise variables. Note the keymap is used for a faster index map,
 442        but is not currently used anywhere (may be removed in future).
 443        """
 444
 445        self.rows =  []     # List of row objects
 446        self.keymap = {}    # For fast lookup of row by value of first column 
 447        self.double = double
 448        self.borders = borders
 449        self.align_cols = []
 450        self.hpadding = hpadding
 451        self.maxwidth = maxwidth
 452        self.headers = headers
 453        self._align_cols = []
 454
 455    def add_row(self, row):
 456        """
 457        Append a row to the list and amend index mapping
 458        """
 459
 460        self.rows.append(row)
 461        if len(row.columns) > 0:
 462            self.keymap[row.columns[0]] = row
 463
 464        logger.debug("Added row with {0} columns".format(str(len(row.columns))))
 465
 466    def align_column(self, i, align):
 467        """
 468        Set alignment for the 'i'th column (`align` should be 'l', 'c' or 'r')
 469        """
 470
 471        while len(self._align_cols) -1 < i:
 472            self._align_cols.append("")
 473        self._align_cols[i] = align
 474        for row in self.rows:
 475            row.columns[i].align = align
 476        logger.debug("Column alignment is now {0}".format(self._align_cols))
 477
 478    def _gen_list(self):
 479        """
 480        Used locally for organising rows and columns into a 2D list structure
 481        """
 482
 483        hierarchy = []
 484        for row in self.rows:
 485            row_data = []
 486            for column in row.columns:
 487                row_data.append(column.content)
 488            hierarchy.append(row_data)
 489        return hierarchy
 490
 491    def draw_html(self):
 492        """
 493        Output HTML string (wrapper for tabulate)
 494        """
 495
 496        output = tabulate(self._gen_list(), self.headers, tablefmt="html",
 497                colalign=tuple(self._align_cols))
 498        return output
 499
 500    def draw_plain(self):
 501        """
 502        Output plain text string (wrapper for tabulate)
 503        """
 504
 505        output = tabulate(self._gen_list(), self.headers,
 506                tablefmt="fancy_grid" if self.borders
 507                else "plain", colalign=tuple(self._align_cols))
 508        return output + "\n"*2
 509
 510
 511class Row(object):
 512    """
 513    Object representing a literal row in a 2D table with the individual cells
 514    in the row represented by columns[].
 515    """
 516    
 517    def __init__(self, columns=[], header=False):
 518        """
 519        Initialise variables. The variable n is used locally to keep track of
 520        the row width.
 521        """
 522
 523        self.columns = columns
 524        self.header = header
 525        self.n = len(self.columns)
 526
 527    def add_column(self, column):
 528        """
 529        Append a single cell horizontally and increment the cell count
 530        """
 531
 532        self.columns.append(column)
 533        self.n += 1
 534
 535    def rm_column(self, column):
 536        """
 537        Remove the specified column object and decrement the cell count
 538        """
 539
 540        self.remove(column)
 541        self.n -= 1
 542
 543
 544class Column(object):
 545    """
 546    Object representing a single table cell. This is somewhat of a misnomer - 
 547    one column object exists for each cell in the table. Columns are children
 548    of rows.
 549    """
 550
 551    def __init__(self, content="", align="right"):
 552        """
 553        Initialise variables. The align property sets the alignment of a single
 554        cell ('l', 'c', or 'r').
 555        """
 556
 557        self.content = content
 558        self.align = align
 559
 560
 561class PlaintextLine:
 562    """
 563    Draw a horizontal line for plain text format, with optional padding/styling.
 564    """
 565
 566    def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
 567        """
 568        Initialise variables
 569        """
 570
 571        self.linewidth = linewidth
 572        self.double = double
 573        self.vpadding = vpadding
 574        self.hpadding = hpadding
 575
 576    def draw(self):
 577        """
 578        Output a plain text string based on the current object parameters
 579        """
 580
 581        line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
 582        return "\n" * self.vpadding + self.hpadding \
 583                +  line * (self.linewidth - 2 * len(self.hpadding)) \
 584                + self.hpadding + "\n" * self.vpadding
 585
 586
 587class PlaintextBox:
 588    """
 589    Draw a rectangular box around text, with customisable padding/size/style
 590    """
 591
 592    def __init__(self, content="", double=True, fullwidth=True, linewidth=80,
 593            hpadding="\t", vpadding=1):
 594        """
 595        Initialise variables
 596        """
 597        self.content = content
 598        self.fullwidth = fullwidth
 599        self.linewidth = linewidth
 600        self.hpadding = hpadding 
 601        self.vpadding = vpadding
 602        self.double = double
 603
 604    def draw(self):
 605        """
 606        Output a plain text string based on the current object parameters. This
 607        involves calculating the text width, breaking text at the maximum line
 608        length, and then drawing a box around it all.
 609        """
 610
 611        if self.double == True:
 612            cornerchars = CORNERCHARS_DOUBLE
 613            linechars = LINECHARS_DOUBLE
 614        else:
 615            cornerchars = CORNERCHARS_SINGLE
 616            linechars = LINECHARS_SINGLE
 617
 618        # Check hpadding has a definite width
 619        self.hpadding = self.hpadding.replace("\t", " "*4)
 620
 621        # Calculate number of characters per line
 622        contentlines = self.content.splitlines()
 623        contentwidth = int((self.linewidth if self.linewidth > 0 else 80)
 624                if self.content.splitlines()
 625                else len(max(contentlines, key=len)))
 626        logger.debug("Contentwidth is {0}".format(str(contentwidth)))
 627        logger.debug("Longest line is {0}".format(
 628            len(max(contentlines, key=len))))
 629        contentwidth += -2*(len(self.hpadding)+1)
 630        if not self.fullwidth:
 631            longestline = len(max(contentlines, key=len))
 632            if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
 633                contentwidth = longestline
 634
 635        # Split lines that are too long
 636        for i, line in enumerate(contentlines):
 637            if len(line) > contentwidth:
 638                words = line.split()
 639                if max(map(len, words)) > contentwidth:
 640                    continue
 641                res, part, others = [], words[0], words[1:]
 642                for word in others:
 643                    if len(' ') + len(word) > contentwidth - len(part):
 644                        res.append(part)
 645                        part = word
 646                    else:
 647                        part += ' ' + word
 648                if part:
 649                    res.append(part)
 650                contentlines[i] = res
 651
 652        # Flatten list
 653        #   Note list comprehension doesn't work here, so we must iterate
 654        #   through each item
 655        newlines = []
 656        for line in contentlines:
 657            if isinstance(line, list):
 658                for subline in line:
 659                    newlines.append(subline)
 660            else:
 661                newlines.append(line)
 662        contentlines = newlines
 663               
 664        # Add vertical padding
 665        for _ in range(self.vpadding):
 666            contentlines.insert(0, ' '*contentwidth)
 667            contentlines.append(' '*contentwidth)
 668
 669        # Insert horizontal padding on lines that are too short
 670        contentlines = [linechars[0] + self.hpadding + x
 671                + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2)
 672                    if len(x) < contentwidth else 0)
 673                + self.hpadding + linechars[0] for x in contentlines]
 674        contentlines.insert(0, cornerchars[3] + linechars[1] 
 675                * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
 676        contentlines.append(cornerchars[0] + linechars[1]
 677                * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
 678        return ('\n').join(contentlines)
 679
 680
 681def backticks(l):
 682    """
 683    Surround every item in a list by backticks. Used for showing code in both
 684    HTML and plain text formats (converted to <code> tags for HTML)
 685    """
 686
 687    return ["`" + x + "`" for x in l]
 688
 689
 690def plural(noun, quantity, print_quantity=True):
 691    """
 692    Return "1 noun" or "n nouns"
 693    """
 694
 695    if (quantity == 1):
 696        if print_quantity:
 697            return(str(quantity) + " " + noun)
 698        else:
 699            return noun
 700    else:
 701        if noun.endswith("s"):
 702            noun += "e"
 703        if print_quantity:
 704            return(str(quantity) + " " + noun + "s")
 705        else:
 706            return noun + "s"
 707
 708
 709def parsesize(num, suffix='B'):
 710    """
 711    Return human-readable size from number of bytes
 712    """
 713
 714    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 715        if abs(num) < 1024.0:
 716            return "%3.1f %s%s" % (num, unit, suffix)
 717        num /= 1024.0
 718    return "%.1f%s%s" % (num, 'Yi', suffix)
 719
 720
 721def fsubject(subject):
 722    """
 723    Replace variables in the title template provided in config
 724    """
 725
 726    r = Template(subject).safe_substitute(VARSUBST)
 727    logger.debug("Returning subject line " + r)
 728    return r
 729
 730
 731def opentag(tag, block=False, id=None, cl=None, style=None):
 732    """
 733    Write HTML opening tag
 734    """
 735
 736    output = ""
 737    if block:
 738        output += '\n'
 739    output += '<' + tag
 740    if id:
 741        output += " id='" + id + "'"
 742    if cl:
 743        output += " class='" + cl + "'"
 744    if style:
 745        output += " style='"
 746        output += " ".join("{0}: {1};".format(attr, value)
 747                for attr, value in style.items())
 748        output += "'"
 749    output += '>'
 750    if block:
 751        output += '\n'
 752    return output
 753
 754
 755def closetag(tag, block=False):
 756    """
 757    Write HTML closing tag
 758    """
 759
 760    if block:
 761        return "\n</" + tag + ">\n"
 762    else:
 763        return "</" + tag + ">"
 764
 765
 766def tag(tag, block=False, content="", id=None, cl=None, style=None):
 767    """
 768    Write HTML opening tag, content, and closing tag
 769    """
 770
 771    o = opentag(tag, block, id, cl, style)
 772    c = closetag(tag, block)
 773    return o + content + c