Andrew's git - logparse.git/blob - logparse/formatting.py

   1#
   2#   format.py
   3#   
   4#   This file contains global functions for formatting and printing data. This
   5#   file should be imported into individual log-parsing scripts located in
   6#   logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or
   7#   emailng data is left to __main__.py.
   8#
   9
  10import os
  11import re
  12import locale
  13from string import Template
  14
  15import logparse
  16from . import interface, util, config, mail
  17
  18import logging
  19logger = logging.getLogger(__name__)
  20
  21
  22locale.setlocale(locale.LC_ALL, '') # inherit system locale
  23#DEG = "&deg;".encode('unicode_escape')
  24DEG = u'\N{DEGREE SIGN}'
  25CEL = "C"
  26TIMEFMT = "%X"
  27DATEFMT = "%x"
  28CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
  29CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
  30LINECHARS_DOUBLE = ['║', '═']
  31LINECHARS_SINGLE = ['│', '─']
  32BULLET = "• "
  33INDENT = "  "
  34
  35
  36class Output:
  37    """
  38    Base class for a data processor. 
  39    """
  40    
  41    def __init__(self):
  42        self.content = ""
  43        self.destination = ""
  44
  45    def append(self, content):
  46        """
  47        Add a string
  48        """
  49        self.content += content
  50
  51    def write(self, destination=""):
  52        """
  53        Output contents into a file
  54        """
  55        if destination == "":
  56            destination = self.destination
  57        if destination == "":
  58            logger.warning("No destination path provided")
  59            return 1
  60        with open(destination, 'w') as f:
  61            f.write(self.content)
  62            logger.info("Written output to {}".format(destination))
  63
  64
  65class PlaintextOutput(Output):
  66    """
  67    Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
  68    """
  69
  70    def __init__(self, linewidth=80):
  71        self.content = ""
  72        self.destination = ""
  73        self.linewidth = linewidth;
  74
  75    def append_header(self, template=''):
  76        """
  77        Print details with some primitive formatting
  78        """
  79        init_varfilter()
  80        box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth'])
  81        line = PlaintextLine(self.linewidth)
  82        self.append(box.draw() + line.draw())
  83
  84    def append_footer(self):
  85        """
  86        Append a horizontal line and some details
  87        """
  88        init_varfilter()
  89        self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
  90        self.append(Template("$hostname $time $date").safe_substitute(varsubst))
  91
  92    def append_section(self, section):
  93        """
  94        Call the appropriate methods to format a section (provided by a parser).
  95        This should be run by interface.py after every instance of parse_log().
  96        """
  97        self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
  98        self.append('\n'*2)
  99        for data in section.data:
 100            self.append(self._fmt_data(data.subtitle, data.items))
 101            self.append('\n')
 102
 103    def _fmt_data(self, subtitle, data = None):   # write title and data
 104        """
 105        Format the properties of a data object into usable plaintext form with a few fancy symbols/formatting tricks.
 106        Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
 107        """
 108        if (subtitle == ""):
 109            logger.warning("No subtitle provided.. skipping section")
 110            return
 111
 112        if (data == None or len(data) == 0):
 113            logger.debug("No data provided.. just printing subtitle")
 114            return subtitle + '\n'
 115        else:
 116            logger.debug("Received data " + str(data))
 117            subtitle += ':'
 118            if (len(data) == 1):
 119                return subtitle + ' ' + data[0] + '\n'
 120            else:
 121                itemoutput = subtitle + '\n'
 122                for datum in data:
 123                    datum = BULLET + datum
 124                    if len(datum) > config.prefs['linewidth'] - 3:
 125                        words = datum.split()
 126                        if max(map(len, words)) > config.prefs['linewidth'] - len(INDENT):
 127                            continue
 128                        res, part, others = [], words[0], words[1:]
 129                        for word in others:
 130                            if 1 + len(word) > config.prefs['linewidth'] - len(part):
 131                                res.append(part)
 132                                part = word
 133                            else:
 134                                part += ' ' + word
 135                        if part:
 136                            res.append(part)
 137                        datum = ('\n    ').join(res)
 138                    datum = INDENT + datum
 139                    itemoutput += datum + '\n'
 140                return itemoutput
 141
 142
 143class HtmlOutput(Output):
 144    """
 145    Process and output data in HTML format.
 146    All HTML formatting functions now reside in this class to differentiate them from plaintext.
 147    """
 148
 149    def __init__(self):
 150        self.content = ""
 151        self.destination = ""
 152        self.css = ""
 153
 154    def embed_css(self, css):
 155        """
 156        Convert stylesheet to inline tags
 157        """
 158        self.content = mail.mailprep(self.content, css)
 159        return self.content
 160
 161    def append_header(self, template):
 162        """
 163        Insert variables into header template file and append HTML tags
 164        """
 165        init_varfilter()
 166        headercontent = Template(open(template, 'r').read())
 167        self.append(headercontent.safe_substitute(varsubst))
 168        self.append(self.opentag('div', id='main'))
 169
 170    def append_footer(self):
 171        """
 172        Close HTML tags that were opened in the template.
 173        TODO: add footer template similar to header template.
 174        """
 175        self.append(self.closetag('div') + self.closetag('body') + self.closetag('html'))
 176
 177    def append_section(self, section):
 178        """
 179        Call the appropriate methods to generate HTML tags for a section (provided by a parser).
 180        This should be run by interface.py after every instance of parse_log().
 181        """
 182        self.append(self.opentag('div', 1, section.title, 'section'))
 183        self.append(self._gen_title(section.title))
 184        for data in section.data:
 185            self.append(self._fmt_data(data.subtitle, data.items))
 186        self.append(self.closetag('div', 1))
 187
 188    def _gen_title(self, title):
 189        """
 190        Format the title for a section
 191        """
 192        if (title == '' or '\n' in title):
 193            logger.error("Invalid title")
 194            raise ValueError 
 195        logger.debug("Writing title for " + title)
 196        return self.tag('h2', False, title)
 197
 198    def _fmt_data(self, subtitle, data = None):
 199        """
 200        Format the properties of a data object into usable HTML tags.
 201        Subtitle is required, data is not. If only subtitle is supplied or subtitle + one data item, a single line will be printed.
 202        """
 203        if (subtitle == ""):
 204            logger.warning("No subtitle provided.. skipping section")
 205            return
 206
 207        if (data == None or len(data) == 0):
 208            logger.debug("No data provided.. just printing subtitle")
 209            return self.tag('p', False, subtitle)
 210        else:
 211            logger.debug("Received data " + str(data))
 212            subtitle += ':'
 213            if (len(data) == 1):
 214                return self.tag('p', False, subtitle + ' ' + data[0])
 215            else:
 216                output = ""
 217                output += self.tag('p', False, subtitle)
 218                output += self.opentag('ul', 1)
 219                coderegex = re.compile('`(.*)`')
 220                for datum in data:
 221                    if datum == "" or datum == None:
 222                        continue
 223                    datum = coderegex.sub(r"<code>\1</code>", str(datum))
 224                    output += self.tag('li', False, datum)
 225                output += self.closetag('ul', True)
 226                return output
 227
 228    def opentag(self, tag, block=False, id=None, cl=None):
 229        """
 230        Write HTML opening tag
 231        """
 232        output = ""
 233        if (block):
 234            output += '\n'
 235        output += '<' + tag
 236        if (id != None):
 237            output += " id='" + id + "'"
 238        if (cl != None):
 239            output += " class='" + cl + "'"
 240        output += '>'
 241        if (block):
 242            output += '\n'
 243        return output
 244
 245    def closetag(self, tag, block=False):
 246        """
 247        Write HTML closing tag
 248        """
 249        if block:
 250            return "\n</" + tag + ">\n"
 251        else:
 252            return "</" + tag + ">"
 253
 254    def tag(self, tag, block=False, content=""):
 255        """
 256        Write HTML opening tag, content, and closing tag
 257        """
 258        o = self.opentag(tag, block)
 259        c = self.closetag(tag, block)
 260        return o + content + c
 261
 262
 263class Section:
 264    """
 265    Each parser should output a Section() which contains the title and returned data.
 266    """
 267
 268    def __init__(self, title):
 269        self.title = title
 270        self.data = []
 271
 272    def append_data(self, data):
 273        self.data.append(data)
 274
 275
 276class Data:
 277    """
 278    Each section (parser) can have one or more Data() objects which are essentially glorified lists.
 279    """
 280    
 281    def __init__(self, subtitle="", items=[]):
 282        self.subtitle = subtitle
 283        self.items = items 
 284
 285    def truncl(self, limit):      # truncate list
 286        """
 287        Truncate self.items to a specified value and state how many items are hidden.
 288        """
 289        if (len(self.items) > limit):
 290            more = str(len(self.items) - limit)
 291            self.items = self.items[:limit]
 292            self.items.append("+ {0} more".format(more))
 293
 294    def orderbyfreq(self):
 295        """
 296        Order a list by frequency of each item, then remove duplicates and append frequency in parentheses.
 297        """
 298        unsorted = list(self.items)
 299        self.items = ["{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x))]
 300
 301
 302class PlaintextLine:
 303    """
 304    Draw a horizontal line for plain text format, with optional padding/styling.
 305    """
 306
 307    def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
 308        self.linewidth = linewidth
 309        self.double = False
 310        self.vpadding = vpadding
 311        self.hpadding = hpadding
 312
 313    def draw(self):
 314        line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
 315        return "\n" * self.vpadding + self.hpadding +  line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
 316
 317
 318class PlaintextBox:
 319    """
 320    Draw a rectangular box around text, with customisable padding/size/style
 321    """
 322
 323    def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
 324        self.content = content
 325        self.fullwidth = fullwidth
 326        self.linewidth = linewidth
 327        self.hpadding = hpadding 
 328        self.vpadding = vpadding
 329        self.double = double
 330
 331    def draw(self):
 332
 333        if self.double == True:
 334            cornerchars = CORNERCHARS_DOUBLE
 335            linechars = LINECHARS_DOUBLE
 336        else:
 337            cornerchars = CORNERCHARS_SINGLE
 338            linechars = LINECHARS_SINGLE
 339
 340        # Check hpadding has a definite width
 341        self.hpadding = self.hpadding.replace("\t", " "*4)
 342
 343        # Calculate number of characters per line
 344        contentlines = self.content.splitlines()
 345        contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
 346        logger.debug("Contentwidth is {0}".format(str(contentwidth)))
 347        logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
 348        contentwidth += -2*(len(self.hpadding)+1)
 349        if not self.fullwidth:
 350            longestline = len(max(contentlines, key=len))
 351            if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
 352                contentwidth = longestline
 353
 354        # Split lines that are too long
 355        for i, line in enumerate(contentlines):
 356            if len(line) > contentwidth:
 357                words = line.split()
 358                if max(map(len, words)) > contentwidth:
 359                    continue
 360                res, part, others = [], words[0], words[1:]
 361                for word in others:
 362                    if len(' ') + len(word) > contentwidth - len(part):
 363                        res.append(part)
 364                        part = word
 365                    else:
 366                        part += ' ' + word
 367                if part:
 368                    res.append(part)
 369                contentlines[i] = res
 370
 371        # Flatten list
 372        #   Note list comprehension doesn't work here, so we must iterate through each item
 373        newlines = []
 374        for line in contentlines:
 375            if isinstance(line, list):
 376                for subline in line:
 377                    newlines.append(subline)
 378            else:
 379                newlines.append(line)
 380        contentlines = newlines
 381               
 382        # Add vertical padding
 383        for _ in range(self.vpadding):
 384            contentlines.insert(0, ' '*contentwidth)
 385            contentlines.append(' '*contentwidth)
 386
 387        # Insert horizontal padding on lines that are too short
 388        contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
 389        contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
 390        contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
 391        return ('\n').join(contentlines)
 392
 393
 394def init_varfilter():
 395    global varfilter
 396    global varpattern
 397    global varsubst
 398    varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))}
 399    varfilter = dict((re.escape(k), v) for k, v in varfilter.items())
 400    varpattern = re.compile("|".join(varfilter.keys()))
 401    varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output'])))
 402
 403def backticks(l):
 404    return ["`" + x + "`" for x in l]
 405
 406
 407def plural(noun, quantity):
 408    """
 409    Return "1 noun" or "n nouns"
 410    """
 411    if (quantity == 1):
 412        return(str(quantity) + " " + noun)
 413    else:
 414        return(str(quantity) + " " + noun + "s")
 415
 416
 417def parsesize(num, suffix='B'):
 418    """
 419    Return human-readable size from number of bytes
 420    """
 421    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 422        if abs(num) < 1024.0:
 423            return "%3.1f %s%s" % (num, unit, suffix)
 424        num /= 1024.0
 425    return "%.1f%s%s" % (num, 'Yi', suffix)
 426
 427
 428def fsubject(template):
 429    """
 430    Replace variables in the title template provided in config
 431    """
 432    r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template)
 433    logger.debug("Returning subject line " + r)
 434    return r