From ccd21602d306c921d1975af58c2339a5f95ff963 Mon Sep 17 00:00:00 2001 From: Andrew Lorimer Date: Wed, 28 Aug 2019 08:05:21 +1000 Subject: [PATCH] add plain text output capability --- logparse/config.py | 2 + logparse/formatting.py | 254 +++++++++++++++++++++++++++++++++++++-- logparse/interface.py | 37 +++--- logparse/parsers/cron.py | 15 +-- 4 files changed, 271 insertions(+), 37 deletions(-) diff --git a/logparse/config.py b/logparse/config.py index 3e61050..3dc2e46 100644 --- a/logparse/config.py +++ b/logparse/config.py @@ -41,7 +41,9 @@ defaults = Configuration({ 'output': '', 'header': '/etc/logparse/header.html', 'css': '/etc/logparse/main.css', + 'linewidth': 80, 'embed-styles': False, + 'plain': False, 'overwrite': False, 'title': logparse.__name__, 'maxlist': 10, diff --git a/logparse/formatting.py b/logparse/formatting.py index aa4b71f..d7a9b4d 100644 --- a/logparse/formatting.py +++ b/logparse/formatting.py @@ -24,6 +24,250 @@ DEG = u'\N{DEGREE SIGN}' CEL = "C" TIMEFMT = "%X" DATEFMT = "%x" +CORNERCHARS_DOUBLE = ['╚', '╝', '╗', '╔'] +CORNERCHARS_SINGLE = ['└', '┘', '┐', '┌'] +LINECHARS_DOUBLE = ['║', '═'] +LINECHARS_SINGLE = ['│', '─'] + +class Output: + + def __init__(self): + self.content = "" + self.destination = "" + + def append(self, content): + self.content += content + + def write(self, destination=""): + if destination == "": + destination = self.destination + if destination == "": + logger.warning("No destination path provided") + return 1 + with open(destination, 'w') as f: + f.write(self.content) + logger.info("Written output to {}".format(destination)) + + +class PlaintextOutput(Output): + + def __init__(self, linewidth=80): + self.content = "" + self.destination = "" + self.linewidth = linewidth; + + def append_header(self, template=''): + init_varfilter() + box = PlaintextBox(content=Template("$title $version on $hostname\n\n$time $date").safe_substitute(varsubst), vpadding=2, hpadding="\t\t", linewidth=config.prefs['linewidth']) + line = PlaintextLine(self.linewidth) + self.append(box.draw() + line.draw()) + + def append_footer(self): + init_varfilter() + self.append(PlaintextLine(self.linewidth, vpadding=1).draw()) + self.append(Template("$hostname $time $date").safe_substitute(varsubst)) + + def append_section(self, section): + self.append(PlaintextBox(content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw()) + self.append('\n'*2) + for data in section.data: + self.append(self._fmt_data(data.subtitle, data.items)) + self.append('\n') + + def _fmt_data(self, subtitle, data = None): # write title and data + if (subtitle == ""): + logger.warning("No subtitle provided.. skipping section") + return + + if (data == None or len(data) == 0): + logger.debug("No data provided.. just printing subtitle") + return subtitle + '\n' + else: + logger.debug("Received data " + str(data)) + subtitle += ':' + if (len(data) == 1): + return subtitle + ' ' + data[0] + '\n' + else: + itemoutput = subtitle + '\n' + for datum in data: + datum = '• ' + datum + if len(datum) > config.prefs['linewidth']: + words = datum.split() + if max(map(len, words)) > config.prefs['linewidth']: + raise ValueError("Content width is too small") + res, part, others = [], words[0], words[1:] + for word in others: + if len(' ') + len(word) > config.prefs['linewidth'] - len(part): + res.append(part) + part = word + else: + part += ' ' + word + if part: + res.append(part) + datum = '\n'.join(res) + itemoutput += datum + '\n' + return itemoutput + + +class HtmlOutput(Output): + + def __init__(self): + self.content = "" + self.destination = "" + self.css = "" + + def embed_css(self, css): + self.content = mail.mailprep(self.content, css) + + def append_header(self, template): # insert variables into header template file + init_varfilter() + headercontent = Template(open(template, 'r').read()) + self.append(headercontent.safe_substitute(varsubst)) + self.append(opentag('div', id='main')) + + def append_footer(self): + self.append(closetag('div') + closetag('body') + closetag('html')) + + def append_section(self, section): + self.append(opentag('div', 1, section.title, 'section')) + self.append(self._gen_title(section.title)) + for data in section.data: + self.append(self._fmt_data(data.subtitle, data.items)) + self.append(closetag('div', 1)) + + def _gen_title(self, title): # write title for a section + if (title == '' or '\n' in title): + logger.error("Invalid title") + raise ValueError + logger.debug("Writing title for " + title) + return tag('h2', 0, title) + + def _fmt_data(self, subtitle, data = None): # write title and data + if (subtitle == ""): + logger.warning("No subtitle provided.. skipping section") + return + + if (data == None or len(data) == 0): + logger.debug("No data provided.. just printing subtitle") + return tag('p', 0, subtitle) + else: + logger.debug("Received data " + str(data)) + subtitle += ':' + if (len(data) == 1): + return tag('p', 0, subtitle + ' ' + data[0]) + else: + output = "" + output += tag('p', 0, subtitle) + output += opentag('ul', 1) + coderegex = re.compile('`(.*)`') + for datum in data: + if datum == "" or datum == None: + continue + datum = coderegex.sub(r'{\1}', str(datum)) + output += tag('li', 0, datum) + output += closetag('ul', 1) + return output + + +class Section: + + def __init__(self, title): + self.title = title + self.data = [] + + def add_data(self, data): + self.data.append(data) + +class Data: + + def __init__(self, subtitle, items=None): + self.subtitle = subtitle + self.items = items + + +class PlaintextLine: + + def __init__(self, linewidth=80, double=True, vpadding=1, hpadding=""): + self.linewidth = linewidth + self.double = False + self.vpadding = vpadding + self.hpadding = hpadding + + def draw(self): + line = (LINECHARS_DOUBLE[1] if self.double else LINECHARS_SINGLE[1]) + return "\n" * self.vpadding + self.hpadding + line * (self.linewidth - 2 * len(self.hpadding)) + self.hpadding + "\n" * self.vpadding + +class PlaintextBox: + + def __init__(self, content="", double=True, fullwidth=True, linewidth=80, hpadding="\t", vpadding=1): + self.content = content + self.fullwidth = fullwidth + self.linewidth = linewidth + self.hpadding = hpadding + self.vpadding = vpadding + self.double = double + + def draw(self): + + if self.double == True: + cornerchars = CORNERCHARS_DOUBLE + linechars = LINECHARS_DOUBLE + else: + cornerchars = CORNERCHARS_SINGLE + linechars = LINECHARS_SINGLE + + # Check hpadding has a definite width + self.hpadding = self.hpadding.replace("\t", " "*4) + + # Calculate number of characters per line + contentlines = self.content.splitlines() + contentwidth = int((self.linewidth if self.linewidth > 0 else 80) if self.content.splitlines() else len(max(contentlines, key=len))) + logger.debug("Contentwidth is {0}".format(str(contentwidth))) + logger.debug("Longest line is {0}".format(len(max(contentlines, key=len)))) + contentwidth += -2*(len(self.hpadding)+1) + if not self.fullwidth: + longestline = len(max(contentlines, key=len)) + if longestline <= self.linewidth - 2*(len(self.hpadding)+1): + contentwidth = longestline + + # Split lines that are too long + for i, line in enumerate(contentlines): + if len(line) > contentwidth: + words = line.split() + if max(map(len, words)) > contentwidth: + raise ValueError("Content width is too small") + res, part, others = [], words[0], words[1:] + for word in others: + if len(' ') + len(word) > contentwidth - len(part): + res.append(part) + part = word + else: + part += ' ' + word + if part: + res.append(part) + contentlines[i] = res + + # Flatten list + # Note list comprehension doesn't work here, so we must iterate through each item + newlines = [] + for line in contentlines: + if isinstance(line, list): + for subline in line: + newlines.append(subline) + else: + newlines.append(line) + contentlines = newlines + + # Add vertical padding + for _ in range(self.vpadding): + contentlines.insert(0, ' '*contentwidth) + contentlines.append(' '*contentwidth) + + # Insert horizontal padding on lines that are too short + contentlines = [linechars[0] + self.hpadding + x + ' '*(self.linewidth-(len(x)+2*len(self.hpadding)+2) if len(x) < contentwidth else 0) + self.hpadding + linechars[0] for x in contentlines] + contentlines.insert(0, cornerchars[3] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[2]) + contentlines.append(cornerchars[0] + linechars[1] * (contentwidth + len(self.hpadding)*2) + cornerchars[1]) + return ('\n').join(contentlines) def init_varfilter(): global varfilter @@ -66,16 +310,6 @@ def tag(tag, block = 0, content = ""): # write html opening tag, content, and h c = closetag(tag, block) return o + content + c -def header(template): # return a parsed html header from file -# try: -# copyfile(config['css'], config['dest'] + '/' + os.path.basename(config['css'])) -# logger.debug("copied main.css") -# except Exception as e: -# logger.warning("could not copy main.css - " + str(e)) - init_varfilter() - headercontent = Template(open(template, 'r').read()) - return headercontent.safe_substitute(varsubst) - def orderbyfreq(l): # order a list by the frequency of its elements and remove duplicates temp_l = l[:] l = list(set(l)) diff --git a/logparse/interface.py b/logparse/interface.py index f9a76b2..a9bd2e1 100644 --- a/logparse/interface.py +++ b/logparse/interface.py @@ -46,6 +46,7 @@ def main(): argparser.add_argument('-l', '--logs', help='services to analyse', required=False) argparser.add_argument('-nl', '--ignore-logs', help='skip these services (takes precedence over -l)', required=False) argparser.add_argument('-es', '--embed-styles', help='make CSS rules inline rather than linking the file', required=False, default=False, action='store_true') + argparser.add_argument('-nh', '--plain', help='write/send plain text rather than HTML', required = False, default=False, action='store_true') # Load config if argparser.parse_args().config: @@ -74,11 +75,15 @@ def main(): logger.info("Beginning log analysis at {0} {1}".format(start.strftime(formatting.DATEFMT), start.strftime(formatting.TIMEFMT))) logger.debug("This is {0} version {1}, running on Python {2}".format(logparse.__name__, logparse.__version__, sys.version.replace('\n', ''))) - # Write HTML header + # Write header - global output_html - output_html = formatting.header(prefs['header']) - output_html += formatting.opentag('div', id='main') + global output + if argparser.parse_args().plain: + output = formatting.PlaintextOutput(linewidth=prefs['linewidth']) + else: + output = formatting.HtmlOutput() + + output.append_header(prefs['header']) # Find parsers @@ -117,32 +122,27 @@ def main(): logger.debug(str(parser_providers)) for parser in parser_providers: - output_html += parser.parse_log() + output.append_section(parser.parse_log()) # Write HTML footer - - output_html += formatting.closetag('div') + formatting.closetag('body') + formatting.closetag('html') + output.append_footer() if argparser.parse_args().printout: - print(output_html) + print(output) if argparser.parse_args().destination or prefs['output']: if argparser.parse_args().destination: dest_path = argparser.parse_args().destination else: dest_path = prefs['output'] logger.debug("Outputting to {0}".format(dest_path)) - if argparser.parse_args().embed_styles or prefs['embed-styles']: - output_html = mail.mailprep(output_html, prefs['css']) - if not os.path.isfile(dest_path) and not (argparser.parse_args().overwrite or config['overwrite']): - with open(dest_path, 'w') as f: - f.write(output_html) - logger.info("Written output to {}".format(dest_path)) + if (argparser.parse_args().embed_styles or prefs['embed-styles']) and not (argparser.parse_args.plain or prefs['plain']): + output.embed_css(prefs['css']) + if (not os.path.isfile(dest_path)) and not (argparser.parse_args().overwrite or config['overwrite']): + output.write(dest_path) else: logger.warning("Destination file already exists") if input("Would you like to overwrite {0}? (y/n) [n] ".format(dest_path)) == 'y': - with open(dest_path, 'w') as f: - f.write(output_html) - logger.debug("Written output to {}".format(dest_path)) + output.write(dest_path) else: logger.warning("No output written") @@ -151,7 +151,8 @@ def main(): to = argparser.parse_args().to else: to = prefs['mail']['to'] - mail.sendmail(mailbin=prefs['mail']['mailbin'], body=mail.mailprep(output_html, prefs['css']), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject'])) + if argparser.parse_args().plain or prefs['plain']: + mail.sendmail(mailbin=prefs['mail']['mailbin'], body=(output.embed_css(prefs['css']) if isinstance(output, formatting.HtmlOutput) else output.content), recipient=to, subject=formatting.fsubject(config.prefs['mail']['subject'])) # Print end message finish = datetime.now() diff --git a/logparse/parsers/cron.py b/logparse/parsers/cron.py index 2c7289b..3aba140 100644 --- a/logparse/parsers/cron.py +++ b/logparse/parsers/cron.py @@ -14,9 +14,8 @@ import logging logger = logging.getLogger(__name__) def parse_log(): - output = '' logger.debug("Starting cron section") - output += opentag('div', 1, 'cron', 'section') + section = Section("cron") matches = re.findall('.*CMD\s*\(\s*(?!.*cd)(.*)\)', readlog(config.prefs['logs']['cron'])) num = sum(1 for line in matches) commands = [] @@ -26,13 +25,11 @@ def parse_log(): #logger.debug("found cron command " + str(commands)) logger.info("Found " + str(num) + " cron jobs") subtitle = str(num) + " cron jobs run" - output += writetitle("cron") - output += writedata(subtitle) + section.add_data(Data(subtitle)) if (len(matches) > 0): - commands = addtag(commands, 'code') - commands = orderbyfreq(commands) + commands = ("`{0}`".format(x) for x in commands) + commands = orderbyfreq(list(commands)) commands = truncl(commands, config.prefs['maxcmd']) - output += writedata("top cron commands", [c for c in commands]) - output += closetag('div', 1) + section.add_data(Data("top cron commands", commands)) logger.info("Finished cron section") - return output + return section -- 2.47.0