Andrew's git - logparse.git/blob

19eda9931596c074dc18511905dfe41ef09c9fd4
   1#
   2#   format.py
   3#   
   4#   This file contains global functions for formatting and printing data. This
   5#   file should be imported into individual log-parsing scripts located in
   6#   logs/*. Data is formatted in HTML or plaintext. Writing to disk and/or
   7#   emailng data is left to __main__.py.
   8#
   9
  10import os
  11import re
  12import locale
  13from string import Template
  14
  15import logparse
  16from . import interface, util, config
  17
  18import logging
  19logger = logging.getLogger(__name__)
  20
  21locale.setlocale(locale.LC_ALL, '') # inherit system locale
  22#DEG = "&deg;".encode('unicode_escape')
  23DEG = u'\N{DEGREE SIGN}'
  24CEL = "C"
  25TIMEFMT = "%X"
  26DATEFMT = "%x"
  27CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
  28CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
  29LINECHARS_DOUBLE = ['║', '═']
  30LINECHARS_SINGLE = ['│', '─']
  31INDENT = "  "
  32
  33class Output:
  34    """
  35    Base class for a data processor. 
  36    """
  37    
  38    def __init__(self):
  39        self.content = ""
  40        self.destination = ""
  41
  42    def append(self, content):
  43        self.content += content
  44
  45    def write(self, destination=""):
  46        if destination == "":
  47            destination = self.destination
  48        if destination == "":
  49            logger.warning("No destination path provided")
  50            return 1
  51        with open(destination, 'w') as f:
  52            f.write(self.content)
  53            logger.info("Written output to {}".format(destination))
  54
  55
  56class PlaintextOutput(Output):
  57    """
  58    Processes & outputs data in a plaintext form which can be read with cat or plaintext email.
  59    """
  60
  61    def __init__(self, linewidth=80):
  62        self.content = ""
  63        self.destination = ""
  64        self.linewidth = linewidth;
  65
  66    def append_header(self, template=''):
  67        init_varfilter()
  68        box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth'])
  69        line = PlaintextLine(self.linewidth)
  70        self.append(box.draw() + line.draw())
  71
  72    def append_footer(self):
  73        init_varfilter()
  74        self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
  75        self.append(Template("$hostname $time $date").safe_substitute(varsubst))
  76
  77    def append_section(self, section):
  78        self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
  79        self.append('\n'*2)
  80        for data in section.data:
  81            self.append(self._fmt_data(data.subtitle, data.items))
  82            self.append('\n')
  83
  84    def _fmt_data(self, subtitle, data = None):   # write title and data
  85        if (subtitle == ""):
  86            logger.warning("No subtitle provided.. skipping section")
  87            return
  88
  89        if (data == None or len(data) == 0):
  90            logger.debug("No data provided.. just printing subtitle")
  91            return subtitle + '\n'
  92        else:
  93            logger.debug("Received data " + str(data))
  94            subtitle += ':'
  95            if (len(data) == 1):
  96                return subtitle + ' ' + data[0] + '\n'
  97            else:
  98                itemoutput = subtitle + '\n'
  99                for datum in data:
 100                    datum = '• ' + datum
 101                    if len(datum) > config.prefs['linewidth'] - 3:
 102                        words = datum.split()
 103                        if max(map(len, words)) > config.prefs['linewidth'] - len(INDENT):
 104                            continue
 105                        res, part, others = [], words[0], words[1:]
 106                        for word in others:
 107                            if 1 + len(word) > config.prefs['linewidth'] - len(part):
 108                                res.append(part)
 109                                part = word
 110                            else:
 111                                part += ' ' + word
 112                        if part:
 113                            res.append(part)
 114                        datum = ('\n    ').join(res)
 115                    datum = INDENT + datum
 116                    itemoutput += datum + '\n'
 117                return itemoutput
 118
 119
 120class HtmlOutput(Output):
 121    """
 122    Process and output data in HTML format.
 123    All HTML formatting functions now reside in this class to differentiate them from plaintext.
 124    """
 125
 126    def __init__(self):
 127        self.content = ""
 128        self.destination = ""
 129        self.css = ""
 130
 131    def embed_css(self, css):
 132        self.content = mail.mailprep(self.content, css)
 133
 134    def append_header(self, template):   # insert variables into header template file
 135        init_varfilter()
 136        headercontent = Template(open(template, 'r').read())
 137        self.append(headercontent.safe_substitute(varsubst))
 138        self.append(self.opentag('div', id='main'))
 139
 140    def append_footer(self):
 141        self.append(closetag('div') + closetag('body') + closetag('html'))
 142
 143    def append_section(self, section):
 144        self.append(self.opentag('div', 1, section.title, 'section'))
 145        self.append(self._gen_title(section.title))
 146        for data in section.data:
 147            self.append(self._fmt_data(data.subtitle, data.items))
 148        self.append(closetag('div', 1))
 149
 150    def _gen_title(self, title):  # write title for a section
 151        if (title == '' or '\n' in title):
 152            logger.error("Invalid title")
 153            raise ValueError 
 154        logger.debug("Writing title for " + title)
 155        return tag('h2', 0, title)
 156
 157    def _fmt_data(self, subtitle, data = None):   # write title and data
 158        if (subtitle == ""):
 159            logger.warning("No subtitle provided.. skipping section")
 160            return
 161
 162        if (data == None or len(data) == 0):
 163            logger.debug("No data provided.. just printing subtitle")
 164            return tag('p', 0, subtitle)
 165        else:
 166            logger.debug("Received data " + str(data))
 167            subtitle += ':'
 168            if (len(data) == 1):
 169                return tag('p', 0, subtitle + ' ' + data[0])
 170            else:
 171                output = ""
 172                output += tag('p', 0, subtitle)
 173                output += self.opentag('ul', 1)
 174                coderegex = re.compile('`(.*)`')
 175                for datum in data:
 176                    if datum == "" or datum == None:
 177                        continue
 178                    datum = coderegex.sub(r'<code>{\1}</code>', str(datum))
 179                    output += tag('li', 0, datum)
 180                output += closetag('ul', 1)
 181                return output
 182
 183    def opentag(self, tag, block = 0, id = None, cl = None):   # write html opening tag
 184        output = ""
 185        if (block):
 186            output += '\n'
 187        output += '<' + tag
 188        if (id != None):
 189            output += " id='" + id + "'"
 190        if (cl != None):
 191            output += " class='" + cl + "'"
 192        output += '>'
 193        if (block):
 194            output += '\n'
 195        return output
 196
 197    def closetag(self, tag, block = 0):  # write html closing tag
 198        if (block == 0):
 199            return "</" + tag + ">"
 200        else:
 201            return "\n</" + tag + ">\n"
 202
 203    def tag(self, tag, block = 0, content = ""):  # write html opening tag, content, and html closing tag
 204        o = self.opentag(tag, block)
 205        c = self.closetag(tag, block)
 206        return o + content + c
 207
 208
 209
 210class Section:
 211    """
 212    Each parser should output a Section() which contains the title and returned data.
 213    """
 214
 215    def __init__(self, title):
 216        self.title = title
 217        self.data = []
 218
 219    def append_data(self, data):
 220        self.data.append(data)
 221
 222class Data:
 223    """
 224    Each section (parser) can have one or more Data() objects which are essentially glorified lists.
 225    """
 226    
 227    def __init__(self, subtitle="", items=[]):
 228        self.subtitle = subtitle
 229        self.items = items 
 230
 231    def truncl(self, limit):      # truncate list
 232        if (len(self.items) > limit):
 233            more = str(len(self.items) - limit)
 234            self.items = self.items[:limit]
 235            self.items.append("+ " + more + " more")
 236
 237    def orderbyfreq(self):     # order a list by the frequency of its elements and remove duplicates
 238#        temp = list(self.items)[:]
 239#        logger.debug(self.items)
 240#        self.items = list(set(self.items))
 241#        self.items = [[i, temp.count(i)] for i in self.items]   # add count of each element
 242#        self.items.sort(key=lambda x:temp.count(x[0])) # sort by count
 243#        self.items  = [i[0] + ' (' + str(i[1]) + ')' for i in self.items]  # put element and count into string
 244#        self.items = self.items[::-1]     # reverse
 245        unsorted = list(self.items)
 246        self.items = [ "{0} ({1})".format(y, unsorted.count(y)) for y in sorted(set(unsorted), key = lambda x: -unsorted.count(x)) ]
 247
 248
 249class PlaintextLine:
 250    """
 251    Draw a horizontal line for plain text format, with optional padding/styling
 252    """
 253
 254    def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
 255        self.linewidth = linewidth
 256        self.double = False
 257        self.vpadding = vpadding
 258        self.hpadding = hpadding
 259
 260    def draw(self):
 261        line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
 262        return "\n" * self.vpadding + self.hpadding +  line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
 263
 264class PlaintextBox:
 265    """
 266    Draw a rectangular box around text, with customisable padding/size/style
 267    """
 268
 269    def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
 270        self.content = content
 271        self.fullwidth = fullwidth
 272        self.linewidth = linewidth
 273        self.hpadding = hpadding 
 274        self.vpadding = vpadding
 275        self.double = double
 276
 277    def draw(self):
 278
 279        if self.double == True:
 280            cornerchars = CORNERCHARS_DOUBLE
 281            linechars = LINECHARS_DOUBLE
 282        else:
 283            cornerchars = CORNERCHARS_SINGLE
 284            linechars = LINECHARS_SINGLE
 285
 286        # Check hpadding has a definite width
 287        self.hpadding = self.hpadding.replace("\t", " "*4)
 288
 289        # Calculate number of characters per line
 290        contentlines = self.content.splitlines()
 291        contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
 292        logger.debug("Contentwidth is {0}".format(str(contentwidth)))
 293        logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
 294        contentwidth += -2*(len(self.hpadding)+1)
 295        if not self.fullwidth:
 296            longestline = len(max(contentlines, key=len))
 297            if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
 298                contentwidth = longestline
 299
 300        # Split lines that are too long
 301        for i, line in enumerate(contentlines):
 302            if len(line) > contentwidth:
 303                words = line.split()
 304                if max(map(len, words)) > contentwidth:
 305                    continue
 306                res, part, others = [], words[0], words[1:]
 307                for word in others:
 308                    if len(' ') + len(word) > contentwidth - len(part):
 309                        res.append(part)
 310                        part = word
 311                    else:
 312                        part += ' ' + word
 313                if part:
 314                    res.append(part)
 315                contentlines[i] = res
 316
 317        # Flatten list
 318        #   Note list comprehension doesn't work here, so we must iterate through each item
 319        newlines = []
 320        for line in contentlines:
 321            if isinstance(line, list):
 322                for subline in line:
 323                    newlines.append(subline)
 324            else:
 325                newlines.append(line)
 326        contentlines = newlines
 327               
 328        # Add vertical padding
 329        for _ in range(self.vpadding):
 330            contentlines.insert(0, ' '*contentwidth)
 331            contentlines.append(' '*contentwidth)
 332
 333        # Insert horizontal padding on lines that are too short
 334        contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
 335        contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
 336        contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
 337        return ('\n').join(contentlines)
 338
 339def init_varfilter():
 340    global varfilter
 341    global varpattern
 342    global varsubst
 343    varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))}
 344    varfilter = dict((re.escape(k), v) for k, v in varfilter.items())
 345    varpattern = re.compile("|".join(varfilter.keys()))
 346    varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output'])))
 347
 348def writetitle(title):  # write title for a section
 349    if (title == '' or '\n' in title):
 350        logger.error("Invalid title")
 351        raise ValueError 
 352    logger.debug("Writing title for " + title)
 353    return tag('h2', 0, title)
 354
 355def addtag(l, tag):  # add prefix and suffix tags to each item in a list
 356    l2 = ['<' + tag + '>' + i + '</' + tag + '>' for i in l]
 357    return l2
 358
 359def backticks(l):
 360    return ["`" + x + "`" for x in l]
 361
 362def plural(noun, quantity): # return "1 noun" or "n nouns"
 363    if (quantity == 1):
 364        return(str(quantity) + " " + noun)
 365    else:
 366        return(str(quantity) + " " + noun + "s")
 367
 368def parsesize(num, suffix='B'):     # return human-readable size from number of bytes
 369    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 370        if abs(num) < 1024.0:
 371            return "%3.1f %s%s" % (num, unit, suffix)
 372        num /= 1024.0
 373    return "%.1f%s%s" % (num, 'Yi', suffix)
 374
 375def fsubject(template): # Replace variables in the title template provided in config
 376    r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template)
 377    logger.debug("Returning subject line " + r)
 378    return r