acdba9fc5ab1d3efb871f14488c6ba1a18c4d6c4
   1#!/usr/bin/env python
   2# -*- coding: utf-8 -*-
   3
   4"""   
   5This file contains global functions for formatting and printing data. This file
   6should be imported into individual log-parsing scripts located in logs/*. Data
   7is formatted in HTML or plaintext. Writing to disk and/or emailng data is left
   8to interface.py.
   9"""
  10
  11import os
  12import re
  13import locale
  14from string import Template
  15from math import floor, ceil
  16from tabulate import tabulate
  17
  18import logparse
  19from logparse import interface, util, mail, config
  20
  21import logging
  22logger = logging.getLogger(__name__)
  23
  24
  25locale.setlocale(locale.LC_ALL, '') # inherit system locale
  26
  27
  28DEG = u'\N{DEGREE SIGN}'
  29CEL = "C"
  30TIMEFMT = "%X"
  31DATEFMT = "%x"
  32CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
  33CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
  34LINECHARS_DOUBLE = ['║', '═']
  35LINECHARS_SINGLE = ['│', '─']
  36JXNCHARS_DOUBLE = ['╠', '╣', '╦', '╩', '╬']
  37JXNCHARS_SINGLE = ['├', '┤', '┬', '┴', '┼']
  38BULLET = "• "
  39INDENT = "  "
  40
  41
  42global VARSUBST
  43
  44
  45def init_var():
  46    """
  47    Initialise variable substitution templates (should be called before doing
  48    any substitutions)
  49    """
  50
  51    global VARSUBST
  52    css_path = config.prefs.get("html", "css")
  53    if config.prefs.getboolean("html", "css-relpath"):
  54        if interface.argparser.parse_args().no_write:
  55            css_path = os.path.relpath(css_path, ".")
  56        elif interface.argparser.parse_args().destination:
  57            css_path = os.path.relpath(css_path, interface.argparser.parse_args().destination())
  58        elif config.prefs.get("logparse", "output"):
  59            css_path = os.path.relpath(css_path, config.prefs.get("logparse", "output"))
  60    VARSUBST = {
  61        "title": config.prefs.get("logparse", "title"),
  62        "date": interface.start.strftime(DATEFMT),
  63        "time": interface.start.strftime(TIMEFMT),
  64        "hostname": util.hostname(config.prefs.get("logparse", "hostname-path")),
  65        "version": logparse.__version__,
  66        "css": css_path
  67    }
  68
  69
  70class Output:
  71    """
  72    Base class for a data processor. 
  73    """
  74    
  75    def __init__(self):
  76        self.content = ""
  77        self.destination = ""
  78
  79    def append(self, content):
  80        """
  81        Add a string
  82        """
  83
  84        self.content += content
  85
  86    def write(self, destination=""):
  87        """
  88        Output contents into a file
  89        """
  90
  91        if destination == "":
  92            destination = self.destination
  93        if destination == "":
  94            logger.warning("No destination path provided")
  95            return 1
  96        with open(destination, 'w') as f:
  97            f.write(self.content)
  98            logger.info("Written output to {}".format(destination))
  99
 100    def print_stdout(self, lines=False):
 101        """
 102        Echo the contents to the console
 103        """
 104
 105        print()
 106        if lines:
 107            line = PlaintextLine(linewidth=config.prefs.getint("plain", "linewidth"), double=True)
 108            print(line.draw())
 109        print(self.content)
 110        if lines:
 111            print(line.draw())
 112        print()
 113
 114
 115class PlaintextOutput(Output):
 116    """
 117    Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
 118    """
 119
 120    def __init__(self, linewidth=80):
 121        self.content = ""
 122        self.destination = ""
 123        self.linewidth = linewidth;
 124
 125    def append_header(self, template=''):
 126        """
 127        Print details with some primitive formatting
 128        """
 129        box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(VARSUBST), vpadding=2, hpadding="\t\t", linewidth=self.linewidth)
 130        line = PlaintextLine(self.linewidth)
 131        self.append(box.draw() + line.draw())
 132
 133    def append_footer(self):
 134        """
 135        Append a horizontal line and some details
 136        """
 137        self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
 138        self.append(Template("$hostname $time $date").safe_substitute(VARSUBST))
 139
 140    def append_section(self, section):
 141        """
 142        Call the appropriate methods to format a section (provided by a parser).
 143        This should be run by interface.py after every instance of parse_log().
 144        """
 145
 146        self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
 147        self.append('\n'*2)
 148        for data in section.data:
 149            self.append(self._fmt_data(data.subtitle, data.items))
 150            self.append('\n')
 151        for table in section.tables:
 152            self.append(table.draw_plain())
 153        self.append("\n")
 154
 155    def _fmt_data(self, subtitle, data = None):   # write title and data
 156        """
 157        Format the properties of a data object into usable plaintext form with
 158        a few fancy symbols/formatting tricks. Subtitle is required, data is
 159        not. If only subtitle is supplied or subtitle + one data item, a single
 160        line will be printed.
 161        """
 162
 163        if (subtitle == ""):
 164            logger.warning("No subtitle provided.. skipping section")
 165            return
 166
 167        if (data == None or len(data) == 0):
 168            logger.debug("No data provided.. just printing subtitle")
 169            return subtitle + '\n'
 170        else:
 171            logger.debug("Received data " + str(data))
 172            subtitle += ':'
 173            if (len(data) == 1):
 174                return subtitle + ' ' + data[0] + '\n'
 175            else:
 176                itemoutput = subtitle + '\n'
 177                for datum in data:
 178                    datum = BULLET + datum
 179                    if len(datum) > self.linewidth - 3:
 180                        words = datum.split()
 181                        if max(map(len, words)) > self.linewidth - len(INDENT):
 182                            continue
 183                        res, part, others = [], words[0], words[1:]
 184                        for word in others:
 185                            if 1 + len(word) > self.linewidth - len(part):
 186                                res.append(part)
 187                                part = word
 188                            else:
 189                                part += ' ' + word
 190                        if part:
 191                            res.append(part)
 192                        datum = ('\n    ').join(res)
 193                    datum = INDENT + datum
 194                    itemoutput += datum + '\n'
 195                return itemoutput
 196
 197class HtmlOutput(Output):
 198    """
 199    Process and output data in HTML format. All HTML formatting functions now
 200    reside in this class to differentiate them from plain text.
 201    """
 202
 203    def __init__(self):
 204        """
 205        Initialise variables (no parameters required for initialisation)
 206        """
 207
 208        self.content = ""
 209        self.destination = ""
 210        self.css = ""
 211        self._embedded = ""
 212
 213    def embed_css(self, css):
 214        """
 215        Convert stylesheet to inline tags
 216        """
 217
 218        if not self._embedded:
 219            self._embedded = mail.mailprep(self.content, css)
 220        return self._embedded
 221
 222    def append_header(self, template):
 223        """
 224        Insert variables into header template file and append HTML tags
 225        """
 226
 227        headercontent = Template(open(template, 'r').read())
 228        self.append(headercontent.safe_substitute(VARSUBST))
 229        self.append(opentag('div', id='main'))
 230
 231    def append_footer(self):
 232        """
 233        Close HTML tags that were opened in the template.
 234        TODO: add footer template similar to header template.
 235        """
 236
 237        self.append(closetag('div') + closetag('body') + closetag('html'))
 238
 239    def append_section(self, section):
 240        """
 241        Call the appropriate methods to generate HTML tags for a section (provided by a parser).
 242        This should be run by interface.py after every instance of parse_log().
 243        """
 244
 245        self.append(opentag('div', 1, section.title, 'section'))
 246        self.append(self._gen_title(section.title))
 247        for data in section.data:
 248            self.append(self._fmt_data(data.subtitle, data.items))
 249        for table in section.tables:
 250            self.append(table.draw_html())
 251        self.append(closetag('div', 1))
 252
 253    def _gen_title(self, title):
 254        """
 255        Format the title for a section
 256        """
 257
 258        if (title == '' or '\n' in title):
 259            logger.error("Invalid title")
 260            raise ValueError 
 261        logger.debug("Writing title for " + title)
 262        return tag('h2', False, title)
 263
 264    def _fmt_data(self, subtitle, data = None):
 265        """
 266        Format the properties of a data object into usable HTML tags.
 267        Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
 268        """
 269
 270        if (subtitle == ""):
 271            logger.warning("No subtitle provided.. skipping section")
 272            return
 273
 274        if (data == None or len(data) == 0):
 275            logger.debug("No data provided.. just printing subtitle")
 276            return tag('p', False, subtitle)
 277        else:
 278            logger.debug("Received data " + str(data))
 279            subtitle += ':'
 280            if (len(data) == 1):
 281                return tag('p', False, subtitle + ' ' + data[0])
 282            else:
 283                output = ""
 284                output += tag('p', False, subtitle)
 285                output += opentag('ul', 1)
 286                coderegex = re.compile('`(.*)`')
 287                for datum in data:
 288                    if datum == "" or datum == None:
 289                        continue
 290                    datum = coderegex.sub(r"<code>\1</code>", str(datum))
 291                    output += tag('li', False, datum)
 292                output += closetag('ul', True)
 293                return output
 294
 295
 296class Section:
 297    """
 298    Each parser should output a Section() which contains the title and returned data.
 299    """
 300
 301    def __init__(self, title):
 302        self.title = title
 303        self.data = []
 304        self.tables = []
 305
 306    def append_data(self, data):
 307        self.data.append(data)
 308
 309    def append_table(self, table):
 310        self.tables.append(table)
 311
 312
 313class Data:
 314    """
 315    Each section (parser) can have one or more Data() objects which are
 316    essentially glorified lists.
 317    """
 318    
 319    def __init__(self, subtitle="", items=[]):
 320        """
 321        Initialise variables. No parameters are enforced upon initialisation,
 322        but at least the subtitle is required for valid output.
 323        """
 324
 325        self.subtitle = subtitle
 326        self.items = items 
 327
 328    def truncl(self, limit):      # truncate list
 329        """
 330        Truncate self.items to a specified value and state how many items are hidden.
 331        """
 332
 333        if (len(self.items) > limit):
 334            more = len(self.items) - limit
 335            if more == 1:
 336                return 0
 337            self.items = self.items[:limit]
 338            self.items.append("+ {0} more".format(str(more)))
 339
 340    def orderbyfreq(self):
 341        """
 342        Order a list by frequency of each item, then remove duplicates and
 343        append frequency in parentheses.
 344        """
 345
 346        unsorted = list(self.items)
 347        self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
 348
 349
 350class Table(object):
 351    """
 352    A wrapper for python-tabulate's Tabulate type.
 353    """
 354    
 355    def __init__(self, double=False, borders=False, hpadding=" ", maxwidth=80, headers=[]):
 356        """
 357        Initialise variables. Note the keymap is used for a faster index map,
 358        but is not currently used anywhere (may be removed in future).
 359        """
 360
 361        self.rows =  []     # List of row objects
 362        self.keymap = {}    # For fast lookup of row by value of first column 
 363        self.double = double
 364        self.borders = borders
 365        self.align_cols = []
 366        self.hpadding = hpadding
 367        self.maxwidth = maxwidth
 368        self.headers = headers
 369        self._align_cols = []
 370
 371    def add_row(self, row):
 372        """
 373        Append a row to the list and amend index mapping
 374        """
 375
 376        self.rows.append(row)
 377        if len(row.columns) > 0:
 378            self.keymap[row.columns[0]] = row
 379
 380        logger.debug("Added row with {0} columns".format(str(len(row.columns))))
 381
 382    def align_column(self, i, align):
 383        """
 384        Set alignment for the 'i'th column (`align` should be 'l', 'c' or 'r')
 385        """
 386
 387        while len(self._align_cols) -1 < i:
 388            self._align_cols.append("")
 389        self._align_cols[i] = align
 390        for row in self.rows:
 391            row.columns[i].align = align
 392        logger.debug("Column alignment is now {0}".format(str(self._align_cols)))
 393
 394    def _gen_list(self):
 395        """
 396        Used locally for organising rows and columns into a 2D list structure
 397        """
 398
 399        hierarchy = []
 400        for row in self.rows:
 401            row_data = []
 402            for column in row.columns:
 403                row_data.append(column.content)
 404            hierarchy.append(row_data)
 405        return hierarchy
 406
 407    def draw_html(self):
 408        """
 409        Output HTML string (wrapper for tabulate)
 410        """
 411
 412        output = tabulate(self._gen_list(), self.headers, tablefmt="html", colalign=tuple(self._align_cols))
 413        return output
 414
 415    def draw_plain(self):
 416        """
 417        Output plain text string (wrapper for tabulate)
 418        """
 419
 420        output = tabulate(self._gen_list(), self.headers, tablefmt="fancy_grid" if self.borders else "plain", colalign=tuple(self._align_cols))
 421        return output + "\n"*2
 422
 423
 424class Row(object):
 425    """
 426    Object representing a literal row in a 2D table with the individual cells
 427    in the row represented by columns[].
 428    """
 429    
 430    def __init__(self, columns=[], header=False):
 431        """
 432        Initialise variables. The variable n is used locally to keep track of
 433        the row width.
 434        """
 435
 436        self.columns = columns
 437        self.header = header
 438        self.n = len(self.columns)
 439
 440    def add_column(self, column):
 441        """
 442        Append a single cell horizontally and increment the cell count
 443        """
 444
 445        self.columns.append(column)
 446        self.n += 1
 447
 448    def rm_column(self, column):
 449        """
 450        Remove the specified column object and decrement the cell count
 451        """
 452
 453        self.remove(column)
 454        self.n -= 1
 455
 456
 457class Column(object):
 458    """
 459    Object representing a single table cell. This is somewhat of a misnomer - 
 460    one column object exists for each cell in the table. Columns are children
 461    of rows.
 462    """
 463
 464    def __init__(self, content="", align="right"):
 465        """
 466        Initialise variables. The align property sets the alignment of a single
 467        cell ('l', 'c', or 'r').
 468        """
 469
 470        self.content = content
 471        self.align = align
 472
 473
 474class PlaintextLine:
 475    """
 476    Draw a horizontal line for plain text format, with optional padding/styling.
 477    """
 478
 479    def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
 480        """
 481        Initialise variables
 482        """
 483
 484        self.linewidth = linewidth
 485        self.double = double
 486        self.vpadding = vpadding
 487        self.hpadding = hpadding
 488
 489    def draw(self):
 490        """
 491        Output a plain text string based on the current object parameters
 492        """
 493
 494        line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
 495        return "\n" * self.vpadding + self.hpadding +  line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
 496
 497
 498class PlaintextBox:
 499    """
 500    Draw a rectangular box around text, with customisable padding/size/style
 501    """
 502
 503    def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
 504        """
 505        Initialise variables
 506        """
 507        self.content = content
 508        self.fullwidth = fullwidth
 509        self.linewidth = linewidth
 510        self.hpadding = hpadding 
 511        self.vpadding = vpadding
 512        self.double = double
 513
 514    def draw(self):
 515        """
 516        Output a plain text string based on the current object parameters. This
 517        involves calculating the text width, breaking text at the maximum line
 518        length, and then drawing a box around it all.
 519        """
 520
 521        if self.double == True:
 522            cornerchars = CORNERCHARS_DOUBLE
 523            linechars = LINECHARS_DOUBLE
 524        else:
 525            cornerchars = CORNERCHARS_SINGLE
 526            linechars = LINECHARS_SINGLE
 527
 528        # Check hpadding has a definite width
 529        self.hpadding = self.hpadding.replace("\t", " "*4)
 530
 531        # Calculate number of characters per line
 532        contentlines = self.content.splitlines()
 533        contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
 534        logger.debug("Contentwidth is {0}".format(str(contentwidth)))
 535        logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
 536        contentwidth += -2*(len(self.hpadding)+1)
 537        if not self.fullwidth:
 538            longestline = len(max(contentlines, key=len))
 539            if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
 540                contentwidth = longestline
 541
 542        # Split lines that are too long
 543        for i, line in enumerate(contentlines):
 544            if len(line) > contentwidth:
 545                words = line.split()
 546                if max(map(len, words)) > contentwidth:
 547                    continue
 548                res, part, others = [], words[0], words[1:]
 549                for word in others:
 550                    if len(' ') + len(word) > contentwidth - len(part):
 551                        res.append(part)
 552                        part = word
 553                    else:
 554                        part += ' ' + word
 555                if part:
 556                    res.append(part)
 557                contentlines[i] = res
 558
 559        # Flatten list
 560        #   Note list comprehension doesn't work here, so we must iterate through each item
 561        newlines = []
 562        for line in contentlines:
 563            if isinstance(line, list):
 564                for subline in line:
 565                    newlines.append(subline)
 566            else:
 567                newlines.append(line)
 568        contentlines = newlines
 569               
 570        # Add vertical padding
 571        for _ in range(self.vpadding):
 572            contentlines.insert(0, ' '*contentwidth)
 573            contentlines.append(' '*contentwidth)
 574
 575        # Insert horizontal padding on lines that are too short
 576        contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
 577        contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
 578        contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
 579        return ('\n').join(contentlines)
 580
 581
 582def backticks(l):
 583    """
 584    Surround every item in a list by backticks. Used for showing code in both
 585    HTML and plain text formats (converted to <code> tags for HTML)
 586    """
 587
 588    return ["`" + x + "`" for x in l]
 589
 590
 591def plural(noun, quantity, print_quantity=True):
 592    """
 593    Return "1 noun" or "n nouns"
 594    """
 595
 596    if print_quantity:
 597        if (quantity == 1):
 598            return(str(quantity) + " " + noun)
 599        else:
 600            return(str(quantity) + " " + noun + "s")
 601    else:
 602        if (quantity == 1):
 603            return noun
 604        else:
 605            return noun + "s"
 606
 607
 608def parsesize(num, suffix='B'):
 609    """
 610    Return human-readable size from number of bytes
 611    """
 612
 613    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 614        if abs(num) < 1024.0:
 615            return "%3.1f %s%s" % (num, unit, suffix)
 616        num /= 1024.0
 617    return "%.1f%s%s" % (num, 'Yi', suffix)
 618
 619
 620def fsubject(subject):
 621    """
 622    Replace variables in the title template provided in config
 623    """
 624
 625    r = Template(subject).safe_substitute(VARSUBST)
 626    logger.debug("Returning subject line " + r)
 627    return r
 628
 629
 630def opentag(tag, block=False, id=None, cl=None, style=None):
 631    """
 632    Write HTML opening tag
 633    """
 634
 635    output = ""
 636    if block:
 637        output += '\n'
 638    output += '<' + tag
 639    if id:
 640        output += " id='" + id + "'"
 641    if cl:
 642        output += " class='" + cl + "'"
 643    if style:
 644        output += " style='"
 645        output += " ".join("{0}: {1};".format(attr, value) for attr, value in style.items())
 646        output += "'"
 647    output += '>'
 648    if block:
 649        output += '\n'
 650    return output
 651
 652
 653def closetag(tag, block=False):
 654    """
 655    Write HTML closing tag
 656    """
 657
 658    if block:
 659        return "\n</" + tag + ">\n"
 660    else:
 661        return "</" + tag + ">"
 662
 663
 664def tag(tag, block=False, content="", id=None, cl=None, style=None):
 665    """
 666    Write HTML opening tag, content, and closing tag
 667    """
 668
 669    o = opentag(tag, block, id, cl, style)
 670    c = closetag(tag, block)
 671    return o + content + c
 672