Andrew's git - logparse.git/blob - logparse/formatting.py

   1#
   2#   format.py
   3#   
   4#   This file contains global functions for formatting and printing data. This
   5#   file should be imported into individual log-parsing scripts located in
   6#   logs/*. Data is all formatted in HTML. Writing to disk and/or emailng data
   7#   is left to __main__.py.
   8#
   9
  10import os
  11import re
  12import locale
  13from string import Template
  14
  15import logparse
  16from . import interface, util, config
  17
  18import logging
  19logger = logging.getLogger(__name__)
  20
  21locale.setlocale(locale.LC_ALL, '') # inherit system locale
  22#DEG = "&deg;".encode('unicode_escape')
  23DEG = u'\N{DEGREE SIGN}'
  24CEL = "C"
  25TIMEFMT = "%X"
  26DATEFMT = "%x"
  27CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔']
  28CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌']
  29LINECHARS_DOUBLE = ['║', '═']
  30LINECHARS_SINGLE = ['│', '─']
  31
  32class Output:
  33    
  34    def __init__(self):
  35        self.content = ""
  36        self.destination = ""
  37
  38    def append(self, content):
  39        self.content += content
  40
  41    def write(self, destination=""):
  42        if destination == "":
  43            destination = self.destination
  44        if destination == "":
  45            logger.warning("No destination path provided")
  46            return 1
  47        with open(destination, 'w') as f:
  48            f.write(self.content)
  49            logger.info("Written output to {}".format(destination))
  50
  51
  52class PlaintextOutput(Output):
  53
  54    def __init__(self, linewidth=80):
  55        self.content = ""
  56        self.destination = ""
  57        self.linewidth = linewidth;
  58
  59    def append_header(self, template=''):
  60        init_varfilter()
  61        box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth'])
  62        line = PlaintextLine(self.linewidth)
  63        self.append(box.draw() + line.draw())
  64
  65    def append_footer(self):
  66        init_varfilter()
  67        self.append(PlaintextLine(self.linewidth, vpadding=1).draw())
  68        self.append(Template("$hostname $time $date").safe_substitute(varsubst))
  69
  70    def append_section(self, section):
  71        self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw())
  72        self.append('\n'*2)
  73        for data in section.data:
  74            self.append(self._fmt_data(data.subtitle, data.items))
  75        self.append('\n')
  76
  77    def _fmt_data(self, subtitle, data = None):   # write title and data
  78        if (subtitle == ""):
  79            logger.warning("No subtitle provided.. skipping section")
  80            return
  81
  82        if (data == None or len(data) == 0):
  83            logger.debug("No data provided.. just printing subtitle")
  84            return subtitle + '\n'
  85        else:
  86            logger.debug("Received data " + str(data))
  87            subtitle += ':'
  88            if (len(data) == 1):
  89                return subtitle + ' ' + data[0] + '\n'
  90            else:
  91                itemoutput = subtitle + '\n'
  92                for datum in data:
  93                    datum = '• ' + datum
  94                    if len(datum) > config.prefs['linewidth']:
  95                        words = datum.split()
  96                        if max(map(len, words)) > config.prefs['linewidth']:
  97                            raise ValueError("Content width is too small")
  98                        res, part, others = [], words[0], words[1:]
  99                        for word in others:
 100                            if len(' ') + len(word) > config.prefs['linewidth'] - len(part):
 101                                res.append(part)
 102                                part = word
 103                            else:
 104                                part += ' ' + word
 105                        if part:
 106                            res.append(part)
 107                        datum = '\n'.join(res)
 108                    itemoutput += datum + '\n'
 109                return itemoutput
 110
 111
 112class HtmlOutput(Output):
 113
 114    def __init__(self):
 115        self.content = ""
 116        self.destination = ""
 117        self.css = ""
 118
 119    def embed_css(self, css):
 120        self.content = mail.mailprep(self.content, css)
 121
 122    def append_header(self, template):   # insert variables into header template file
 123        init_varfilter()
 124        headercontent = Template(open(template, 'r').read())
 125        self.append(headercontent.safe_substitute(varsubst))
 126        self.append(opentag('div', id='main'))
 127
 128    def append_footer(self):
 129        self.append(closetag('div') + closetag('body') + closetag('html'))
 130
 131    def append_section(self, section):
 132        self.append(opentag('div', 1, section.title, 'section'))
 133        self.append(self._gen_title(section.title))
 134        for data in section.data:
 135            self.append(self._fmt_data(data.subtitle, data.items))
 136        self.append(closetag('div', 1))
 137
 138    def _gen_title(self, title):  # write title for a section
 139        if (title == '' or '\n' in title):
 140            logger.error("Invalid title")
 141            raise ValueError 
 142        logger.debug("Writing title for " + title)
 143        return tag('h2', 0, title)
 144
 145    def _fmt_data(self, subtitle, data = None):   # write title and data
 146        if (subtitle == ""):
 147            logger.warning("No subtitle provided.. skipping section")
 148            return
 149
 150        if (data == None or len(data) == 0):
 151            logger.debug("No data provided.. just printing subtitle")
 152            return tag('p', 0, subtitle)
 153        else:
 154            logger.debug("Received data " + str(data))
 155            subtitle += ':'
 156            if (len(data) == 1):
 157                return tag('p', 0, subtitle + ' ' + data[0])
 158            else:
 159                output = ""
 160                output += tag('p', 0, subtitle)
 161                output += opentag('ul', 1)
 162                coderegex = re.compile('`(.*)`')
 163                for datum in data:
 164                    if datum == "" or datum == None:
 165                        continue
 166                    datum = coderegex.sub(r'<code>{\1}</code>', str(datum))
 167                    output += tag('li', 0, datum)
 168                output += closetag('ul', 1)
 169                return output
 170
 171
 172class Section:
 173
 174    def __init__(self, title):
 175        self.title = title
 176        self.data = []
 177
 178    def add_data(self, data):
 179        self.data.append(data)
 180
 181class Data:
 182    
 183    def __init__(self, subtitle, items=None):
 184        self.subtitle = subtitle
 185        self.items = items 
 186
 187
 188class PlaintextLine:
 189
 190    def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""):
 191        self.linewidth = linewidth
 192        self.double = False
 193        self.vpadding = vpadding
 194        self.hpadding = hpadding
 195
 196    def draw(self):
 197        line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1])
 198        return "\n" * self.vpadding + self.hpadding +  line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding
 199
 200class PlaintextBox:
 201
 202    def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1):
 203        self.content = content
 204        self.fullwidth = fullwidth
 205        self.linewidth = linewidth
 206        self.hpadding = hpadding 
 207        self.vpadding = vpadding
 208        self.double = double
 209
 210    def draw(self):
 211
 212        if self.double == True:
 213            cornerchars = CORNERCHARS_DOUBLE
 214            linechars = LINECHARS_DOUBLE
 215        else:
 216            cornerchars = CORNERCHARS_SINGLE
 217            linechars = LINECHARS_SINGLE
 218
 219        # Check hpadding has a definite width
 220        self.hpadding = self.hpadding.replace("\t", " "*4)
 221
 222        # Calculate number of characters per line
 223        contentlines = self.content.splitlines()
 224        contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len)))
 225        logger.debug("Contentwidth is {0}".format(str(contentwidth)))
 226        logger.debug("Longest line is {0}".format(len(max(contentlines, key=len))))
 227        contentwidth += -2*(len(self.hpadding)+1)
 228        if not self.fullwidth:
 229            longestline = len(max(contentlines, key=len))
 230            if longestline <= self.linewidth - 2*(len(self.hpadding)+1):
 231                contentwidth = longestline
 232
 233        # Split lines that are too long
 234        for i, line in enumerate(contentlines):
 235            if len(line) > contentwidth:
 236                words = line.split()
 237                if max(map(len, words)) > contentwidth:
 238                    raise ValueError("Content width is too small")
 239                res, part, others = [], words[0], words[1:]
 240                for word in others:
 241                    if len(' ') + len(word) > contentwidth - len(part):
 242                        res.append(part)
 243                        part = word
 244                    else:
 245                        part += ' ' + word
 246                if part:
 247                    res.append(part)
 248                contentlines[i] = res
 249
 250        # Flatten list
 251        #   Note list comprehension doesn't work here, so we must iterate through each item
 252        newlines = []
 253        for line in contentlines:
 254            if isinstance(line, list):
 255                for subline in line:
 256                    newlines.append(subline)
 257            else:
 258                newlines.append(line)
 259        contentlines = newlines
 260               
 261        # Add vertical padding
 262        for _ in range(self.vpadding):
 263            contentlines.insert(0, ' '*contentwidth)
 264            contentlines.append(' '*contentwidth)
 265
 266        # Insert horizontal padding on lines that are too short
 267        contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines]
 268        contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2])
 269        contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1])
 270        return ('\n').join(contentlines)
 271
 272def init_varfilter():
 273    global varfilter
 274    global varpattern
 275    global varsubst
 276    varfilter = {"$title$": config.prefs['title'], "$date$": interface.start.strftime(DATEFMT),"$time$": interface.start.strftime(TIMEFMT), "$hostname$": util.hostname(config.prefs['hostname-path']), "$version$": logparse.__version__, "$css$": os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output']))}
 277    varfilter = dict((re.escape(k), v) for k, v in varfilter.items())
 278    varpattern = re.compile("|".join(varfilter.keys()))
 279    varsubst = dict(title=config.prefs['title'], date=interface.start.strftime(DATEFMT), time=interface.start.strftime(TIMEFMT), hostname=util.hostname(config.prefs['hostname-path']), version=logparse.__version__, css=os.path.relpath(config.prefs['css'], os.path.dirname(config.prefs['output'])))
 280
 281def writetitle(title):  # write title for a section
 282    if (title == '' or '\n' in title):
 283        logger.error("Invalid title")
 284        raise ValueError 
 285    logger.debug("Writing title for " + title)
 286    return tag('h2', 0, title)
 287
 288def opentag(tag, block = 0, id = None, cl = None):   # write html opening tag
 289    output = ""
 290    if (block):
 291        output += '\n'
 292    output += '<' + tag
 293    if (id != None):
 294        output += " id='" + id + "'"
 295    if (cl != None):
 296        output += " class='" + cl + "'"
 297    output += '>'
 298    if (block):
 299        output += '\n'
 300    return output
 301
 302def closetag(tag, block = 0):  # write html closing tag
 303    if (block == 0):
 304        return "</" + tag + ">"
 305    else:
 306        return "\n</" + tag + ">\n"
 307
 308def tag(tag, block = 0, content = ""):  # write html opening tag, content, and html closing tag
 309    o = opentag(tag, block)
 310    c = closetag(tag, block)
 311    return o + content + c
 312
 313def orderbyfreq(l):     # order a list by the frequency of its elements and remove duplicates
 314    temp_l = l[:]
 315    l = list(set(l))
 316    l = [[i, temp_l.count(i)] for i in l]   # add count of each element
 317    l.sort(key=lambda x:temp_l.count(x[0])) # sort by count
 318    l = [i[0] + ' (' + str(i[1]) + ')' for i in l]  # put element and count into string
 319    l = l[::-1]     # reverse
 320    return l
 321
 322def addtag(l, tag):  # add prefix and suffix tags to each item in a list
 323    l2 = ['<' + tag + '>' + i + '</' + tag + '>' for i in l]
 324    return l2
 325
 326def truncl(input, limit):      # truncate list
 327    if (len(input) > limit):
 328        more = str(len(input) - limit)
 329        output = input[:limit]
 330        output.append("+ " + more + " more")
 331        return(output)
 332    else:
 333        return(input)
 334
 335def plural(noun, quantity): # return "1 noun" or "n nouns"
 336    if (quantity == 1):
 337        return(str(quantity) + " " + noun)
 338    else:
 339        return(str(quantity) + " " + noun + "s")
 340
 341def parsesize(num, suffix='B'):     # return human-readable size from number of bytes
 342    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
 343        if abs(num) < 1024.0:
 344            return "%3.1f %s%s" % (num, unit, suffix)
 345        num /= 1024.0
 346    return "%.1f%s%s" % (num, 'Yi', suffix)
 347
 348def fsubject(template): # Replace variables in the title template provided in config
 349    r = varpattern.sub(lambda m: varfilter[re.escape(m.group(0))], template)
 350    logger.debug("Returning subject line " + r)
 351    return r
 352
 353def writedata(subtitle, data = None):   # write title and data
 354    if (subtitle == ""):
 355        logger.warning("No subtitle provided.. skipping section")
 356        return
 357
 358    if (data == None or len(data) == 0):
 359        logger.debug("No data provided.. just printing subtitle")
 360        return tag('p', 0, subtitle)
 361    else:
 362        logger.debug("Received data " + str(data))
 363        subtitle += ':'
 364        if (len(data) == 1):
 365            return tag('p', 0, subtitle + ' ' + data[0])
 366        else:
 367            output = ""
 368            output += tag('p', 0, subtitle)
 369            output += opentag('ul', 1)
 370            for datum in data:
 371                output += tag('li', 0, datum)
 372            output += closetag('ul', 1)
 373            return output
 374