Andrew's git - logparse.git/blob

b2670f81ab89d013f4c3d312108fd796bebb834d
   1#
   2#   format.py
   3#   
   4#   This file contains global functions for formatting and printing data. This
   5#   file should be imported into individual log-parsing scripts located in
   6#   logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or
   7#   emailng data is left to __main__.py.
   8#
   9
  10import os
  11import re
  12import locale
  13from string import Template
  14
  15import logparse
  16from . import interface, util, config
  17
  18import logging
  19logger = logging.getLogger(__name__)
  20
  21locale.setlocale(locale.LC_ALL, '') # inherit system locale
  22#DEG = "&deg;".encode('unicode_escape')
  23DEG = u'\N{DEGREE SIGN}'
  24CEL = "C"
  25TIMEFMT = "%X"
  26DATEFMT = "%x"
  27CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
  28CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
  29LINECHARS_DOUBLE = ['║', '═']
  30LINECHARS_SINGLE = ['│', '─']
  31
  32class Output:
  33    """
  34    Base class for a data processor. 
  35    """
  36    
  37    def __init__(self):
  38        self.content = ""
  39        self.destination = ""
  40
  41    def append(self, content):
  42        self.content += content
  43
  44    def write(self, destination=""):
  45        if destination == "":
  46            destination = self.destination
  47        if destination == "":
  48            logger.warning("No destination path provided")
  49            return 1
  50        with open(destination, 'w') as f:
  51            f.write(self.content)
  52            logger.info("Written output to {}".format(destination))
  53
  54
  55class PlaintextOutput(Output):
  56    """
  57    Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
  58    """
  59
  60    def __init__(self, linewidth=80):
  61        self.content = ""
  62        self.destination = ""
  63        self.linewidth = linewidth;
  64
  65    def append_header(self, template=''):
  66        init_varfilter()
  67        box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth'])
  68        line = PlaintextLine(self.linewidth)
  69        self.append(box.draw() + line.draw())
  70
  71    def append_footer(self):
  72        init_varfilter()
  73        self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
  74        self.append(Template("$hostname $time $date").safe_substitute(varsubst))
  75
  76    def append_section(self, section):
  77        self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
  78        self.append('\n'*2)
  79        for data in section.data:
  80            self.append(self._fmt_data(data.subtitle, data.items))
  81        self.append('\n')
  82
  83    def _fmt_data(self, subtitle, data = None):   # write title and data
  84        if (subtitle == ""):
  85            logger.warning("No subtitle provided.. skipping section")
  86            return
  87
  88        if (data == None or len(data) == 0):
  89            logger.debug("No data provided.. just printing subtitle")
  90            return subtitle + '\n'
  91        else:
  92            logger.debug("Received data " + str(data))
  93            subtitle += ':'
  94            if (len(data) == 1):
  95                return subtitle + ' ' + data[0] + '\n'
  96            else:
  97                itemoutput = subtitle + '\n'
  98                for datum in data:
  99                    datum = '• ' + datum
 100                    if len(datum) > config.prefs['linewidth']:
 101                        words = datum.split()
 102                        if max(map(len, words)) > config.prefs['linewidth']:
 103                            raise ValueError("Content width is too small")
 104                        res, part, others = [], words[0], words[1:]
 105                        for word in others:
 106                            if len(' ') + len(word) > config.prefs['linewidth'] - len(part):
 107                                res.append(part)
 108                                part = word
 109                            else:
 110                                part += ' ' + word
 111                        if part:
 112                            res.append(part)
 113                        datum = '\n'.join(res)
 114                    itemoutput += datum + '\n'
 115                return itemoutput
 116
 117
 118class HtmlOutput(Output):
 119    """
 120    Process and output data in HTML format.
 121    All HTML formatting functions now reside in this class to differentiate them from plaintext.
 122    """
 123
 124    def __init__(self):
 125        self.content = ""
 126        self.destination = ""
 127        self.css = ""
 128
 129    def embed_css(self, css):
 130        self.content = mail.mailprep(self.content, css)
 131
 132    def append_header(self, template):   # insert variables into header template file
 133        init_varfilter()
 134        headercontent = Template(open(template, 'r').read())
 135        self.append(headercontent.safe_substitute(varsubst))
 136        self.append(self.opentag('div', id='main'))
 137
 138    def append_footer(self):
 139        self.append(closetag('div') + closetag('body') + closetag('html'))
 140
 141    def append_section(self, section):
 142        self.append(self.opentag('div', 1, section.title, 'section'))
 143        self.append(self._gen_title(section.title))
 144        for data in section.data:
 145            self.append(self._fmt_data(data.subtitle, data.items))
 146        self.append(closetag('div', 1))
 147
 148    def _gen_title(self, title):  # write title for a section
 149        if (title == '' or '\n' in title):
 150            logger.error("Invalid title")
 151            raise ValueError 
 152        logger.debug("Writing title for " + title)
 153        return tag('h2', 0, title)
 154
 155    def _fmt_data(self, subtitle, data = None):   # write title and data
 156        if (subtitle == ""):
 157            logger.warning("No subtitle provided.. skipping section")
 158            return
 159
 160        if (data == None or len(data) == 0):
 161            logger.debug("No data provided.. just printing subtitle")
 162            return tag('p', 0, subtitle)
 163        else:
 164            logger.debug("Received data " + str(data))
 165            subtitle += ':'
 166            if (len(data) == 1):
 167                return tag('p', 0, subtitle + ' ' + data[0])
 168            else:
 169                output = ""
 170                output += tag('p', 0, subtitle)
 171                output += self.opentag('ul', 1)
 172                coderegex = re.compile('`(.*)`')
 173                for datum in data:
 174                    if datum == "" or datum == None:
 175                        continue
 176                    datum = coderegex.sub(r'<code>{\1}</code>', str(datum))
 177                    output += tag('li', 0, datum)
 178                output += closetag('ul', 1)
 179                return output
 180
 181    def opentag(self, tag, block = 0, id = None, cl = None):   # write html opening tag
 182        output = ""
 183        if (block):
 184            output += '\n'
 185        output += '<' + tag
 186        if (id != None):
 187            output += " id='" + id + "'"
 188        if (cl != None):
 189            output += " class='" + cl + "'"
 190        output += '>'
 191        if (block):
 192            output += '\n'
 193        return output
 194
 195    def closetag(self, tag, block = 0):  # write html closing tag
 196        if (block == 0):
 197            return "</" + tag + ">"
 198        else:
 199            return "\n</" + tag + ">\n"
 200
 201    def tag(self, tag, block = 0, content = ""):  # write html opening tag, content, and html closing tag
 202        o = self.opentag(tag, block)
 203        c = self.closetag(tag, block)
 204        return o + content + c
 205
 206
 207
 208class Section:
 209    """
 210    Each parser should output a Section() which contains the title and returned data.
 211    """
 212
 213    def __init__(self, title):
 214        self.title = title
 215        self.data = []
 216
 217    def add_data(self, data):
 218        self.data.append(data)
 219
 220class Data:
 221    """
 222    Each section (parser) can have one or more Data() objects which are essentially glorified lists.
 223    """
 224    
 225    def __init__(self, subtitle=None, items=[]):
 226        self.subtitle = subtitle
 227        self.items = items 
 228
 229    def truncl(self, limit):      # truncate list
 230        if (len(self.items) > limit):
 231            more = str(len(self.items) - limit)
 232            self.items = self.items[:limit]
 233            self.items..append("+ " + more + " more")
 234
 235    def orderbyfreq(self, l):     # order a list by the frequency of its elements and remove duplicates
 236        temp_l = l[:]
 237        l = list(set(l))
 238        l = [[i, temp_l.count(i)] for i in l]   # add count of each element
 239        l.sort(key=lambda x:temp_l.count(x[0])) # sort by count
 240        l = [i[0] + ' (' + str(i[1]) + ')' for i in l]  # put element and count into string
 241        l = l[::-1]     # reverse
 242        self.items = l 
 243
 244
 245class PlaintextLine:
 246    """
 247    Draw a horizontal line for plain text format, with optional padding/styling
 248    """
 249
 250    def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
 251        self.linewidth = linewidth
 252        self.double = False
 253        self.vpadding = vpadding
 254        self.hpadding = hpadding
 255
 256    def draw(self):
 257        line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
 258        return "\n" * self.vpadding + self.hpadding +  line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
 259
 260class PlaintextBox:
 261    """
 262    Draw a rectangular box around text, with customisable padding/size/style
 263    """
 264
 265    def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
 266        self.content = content
 267        self.fullwidth = fullwidth
 268        self.linewidth = linewidth
 269        self.hpadding = hpadding 
 270        self.vpadding = vpadding
 271        self.double = double
 272
 273    def draw(self):
 274
 275        if self.double == True:
 276            cornerchars = CORNERCHARS_DOUBLE
 277            linechars = LINECHARS_DOUBLE
 278        else:
 279            cornerchars = CORNERCHARS_SINGLE
 280            linechars = LINECHARS_SINGLE
 281
 282        # Check hpadding has a definite width
 283        self.hpadding = self.hpadding.replace("\t", " "*4)
 284
 285        # Calculate number of characters per line
 286        contentlines = self.content.splitlines()
 287        contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
 288        logger.debug("Contentwidth is {0}".format(str(contentwidth)))
 289        logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
 290        contentwidth += -2*(len(self.hpadding)+1)
 291        if not self.fullwidth:
 292            longestline = len(max(contentlines, key=len))
 293            if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
 294                contentwidth = longestline
 295
 296        # Split lines that are too long
 297        for i, line in enumerate(contentlines):
 298            if len(line) > contentwidth:
 299                words = line.split()
 300                if max(map(len, words)) > contentwidth:
 301                    raise ValueError("Content width is too small")
 302                res, part, others = [], words[0], words[1:]
 303                for word in others:
 304                    if len(' ') + len(word) > contentwidth - len(part):
 305                        res.append(part)
 306                        part = word
 307                    else:
 308                        part += ' ' + word
 309                if part:
 310                    res.append(part)
 311                contentlines[i] = res
 312
 313        # Flatten list
 314        #   Note list comprehension doesn't work here, so we must iterate through each item
 315        newlines = []
 316        for line in contentlines:
 317            if isinstance(line, list):
 318                for subline in line:
 319                    newlines.append(subline)
 320            else:
 321                newlines.append(line)
 322        contentlines = newlines
 323               
 324        # Add vertical padding
 325        for _ in range(self.vpadding):
 326            contentlines.insert(0, ' '*contentwidth)
 327            contentlines.append(' '*contentwidth)
 328
 329        # Insert horizontal padding on lines that are too short
 330        contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
 331        contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
 332        contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
 333        return ('\n').join(contentlines)
 334
 335def init_varfilter():
 336    global varfilter
 337    global varpattern
 338    global varsubst
 339    varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))}
 340    varfilter = dict((re.escape(k), v) for k, v in varfilter.items())
 341    varpattern = re.compile("|".join(varfilter.keys()))
 342    varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output'])))
 343
 344def writetitle(title):  # write title for a section
 345    if (title == '' or '\n' in title):
 346        logger.error("Invalid title")
 347        raise ValueError 
 348    logger.debug("Writing title for " + title)
 349    return tag('h2', 0, title)
 350
 351def addtag(l, tag):  # add prefix and suffix tags to each item in a list
 352    l2 = ['<' + tag + '>' + i + '</' + tag + '>' for i in l]
 353    return l2
 354
 355def backticks(l):
 356    return ["`" + x + "`" for x in l]
 357
 358def plural(noun, quantity): # return "1 noun" or "n nouns"
 359    if (quantity == 1):
 360        return(str(quantity) + " " + noun)
 361    else:
 362        return(str(quantity) + " " + noun + "s")
 363
 364def parsesize(num, suffix='B'):     # return human-readable size from number of bytes
 365    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 366        if abs(num) < 1024.0:
 367            return "%3.1f %s%s" % (num, unit, suffix)
 368        num /= 1024.0
 369    return "%.1f%s%s" % (num, 'Yi', suffix)
 370
 371def fsubject(template): # Replace variables in the title template provided in config
 372    r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template)
 373    logger.debug("Returning subject line " + r)
 374    return r