From: Andrew Lorimer Date: Wed, 13 Mar 2019 07:37:23 +0000 (+1100) Subject: implement new resolve-domains config option, bugfixing for hostname resolver X-Git-Url: https://git.lorimer.id.au/logparse.git/diff_plain/769d14cb5af6773d6c18c07c662a65cf891d2bcf?ds=sidebyside implement new resolve-domains config option, bugfixing for hostname resolver --- diff --git a/logparse.py b/logparse.py index 896e764..638d02a 100755 --- a/logparse.py +++ b/logparse.py @@ -8,6 +8,7 @@ import yaml import ast import logging.handlers import types +import traceback # debugging only reload(sys) sys.setdefaultencoding('utf-8') # force utf-8 because anything else should die @@ -26,6 +27,7 @@ config = { 'title': 'logparse', 'maxlist': 10, 'maxcmd': 3, + 'resolve-domains': 'fqdn', 'mail': { 'to': '', 'from': '', @@ -37,6 +39,18 @@ config = { 'port': 7634, 'show-model': False, }, + 'apache': { + 'resolve-domains': '', + }, + 'sshd': { + 'resolve-domains': '', + }, + 'smbd': { + 'resolve-domains': '', + }, + 'httpd': { + 'resolve-domains': '', + }, 'du-paths': ['/', '/etc', '/home'], 'hostname-path': '/etc/hostname', 'logs': { @@ -84,18 +98,17 @@ def __main__(): debugfunc = parser.parse_args().function if debugfunc is not None: logger.debug("executing a single function: " + debugfunc) - try: - logger.debug((debugfunc + ': ' + eval(debugfunc))) - sys.exit() - except Exception as e: - sys.exit("debug function failed with error " + e) - logger.debug("finished executing debug function") + eval(debugfunc) + sys.exit() if not config['mail']['to']: logger.info("no recipient address provided, outputting to stdout") else: logger.info("email will be sent to " + config['mail']['to']) + global LOCALDOMAIN + LOCALDOMAIN = getlocaldomain() + global pathfilter global pathpattern pathfilter = {"auth": config['logs']['auth'], "cron": config['logs']['cron'], "sys": config['logs']['sys'], "postfix": config['logs']['postfix'], "smb": config['logs']['smb'], "zfs": config['logs']['zfs'], "alloc": config['logs']['alloc'], "httpd": config['logs']['httpd'], "header": config['header']} @@ -205,22 +218,50 @@ def hostname(): # get the hostname of current server hn = re.search('^(.*)\n*', hnfile.read()).group(1) return hn +def getlocaldomain(): # get the parent fqdn of current server + domain = socket.getfqdn().split('.', 1) + if len(domain) == 2: + logger.warning('Could not get domain of this server, only hostname. Please consider updating /etc/hosts') + return '' + else: + return domain[-1] + +def resolve(ip, fqdn = 'host-only'): # try to resolve an ip to hostname + # Possible values for fqdn: + # fqdn show full hostname and domain + # fqdn-implicit show hostname and domain unless local + # host-only only show hostname + # ip never resolve anything + # resolve-domains defined in individual sections of the config take priority over global config + + if not fqdn: + fqdn = config['resolve-domains'] + + if fqdn == 'ip': + return(ip) -def resolve(ip, fqdn = False): # try to resolve an ip to hostname try: socket.inet_aton(ip) # succeeds if text contains ip hn = socket.gethostbyaddr(ip)[0] # resolve ip to hostname - return(hn if fqdn else hn.split('.')[0]) - except OSError: - # already a hostname - logger.debug(ip + " is already a hostname") - return(ip) + if fqdn == 'fqdn-implicit' and hn.split('.', 1)[1] == LOCALDOMAIN: + return(hn.split('.')[0]) + elif fqdn == 'fqdn' or fqdn == 'fqdn-implicit': + return(hn) + elif fqdn == 'host-only': + return(hn.split('.')[0]) + else: + logger.warning("invalid value for fqdn config") + return(hn) except socket.herror: # cannot resolve ip logger.debug(ip + " cannot be found, might not exist anymore") return(ip) - except: - logger.debug("failed to resolve hostname for " + ip) + except (OSError, socket.error): # socket.error for Python 2 compatibility + # already a hostname + logger.debug(ip + " is already a hostname") + return(ip) + except Exception as err: + logger.warning("failed to resolve hostname for " + ip + ": " + str(err)) return(ip) # return ip if no hostname exists def plural(noun, quantity): # return "1 noun" or "n nouns" @@ -319,7 +360,7 @@ def sshd(): user = entry.group(1) ip = entry.group(2) - userhost = user + '@' + resolve(ip) + userhost = user + '@' + resolve(ip, fqdn=config['sshd']['resolve-domains']) exists = [i for i, item in enumerate(users) if re.search(userhost, item[0])] if (exists == []): users.append([userhost, 1]) @@ -462,7 +503,7 @@ def httpd(): for line in accesslog.split('\n'): fields = re.search('^(\S*) .*GET (\/.*) HTTP/\d\.\d\" 200 (\d*) \"(.*)\".*\((.*)\;', line) try: - ips.append(resolve(fields.group(1), fqdn=True)) + ips.append(resolve(fields.group(1), fqdn=config['httpd']['resolve-domains'])) files.append(fields.group(2)) useragents.append(fields.group(5)) data_b += int(fields.group(3)) @@ -471,6 +512,7 @@ def httpd(): pass else: logger.warning("error processing httpd access log: " + str(error)) + traceback.print_exc() logger.debug(str(data_b) + " bytes transferred") data_h = parsesize(data_b) writetitle("apache") @@ -555,8 +597,8 @@ def smbd(): # find the machine (ip or hostname) that this file represents ip = re.search('log\.(.*)', file).group(1) # get ip or hostname from file path (/var/log/samba/log.host) - host = resolve(ip) - if (host == ip): # if ip has disappeared, fall back to a hostname from logfile + host = resolve(ip, fqdn=config['smbd']['resolve-domains']) + if (host == ip and (config['smbd']['resolve-domains'] or config['resolve-domains']) != 'ip'): # if ip has disappeared, fall back to a hostname from logfile newhost = re.findall('.*\]\@\[(.*)\]', readlog(file)) if (len(set(newhost)) == 1): # all hosts in one file should be the same host = newhost[0].lower() @@ -627,17 +669,16 @@ def zfs(): logger.debug("starting zfs section") opentag('div', 1, 'zfs', 'section') zfslog = readlog('zfs') - logger.debug("got zfs logfile") pool = re.search('.*---\n(\w*)', zfslog).group(1) - scrub = re.search('.*scrub repaired (\d*) in \d*h\d*m with (\d*) errors on (\S*\s)(\S*)\s(\d+\s)', zfslog) + scrub = re.search('.*scrub repaired (\d*).* in .*\d*h\d*m with (\d*) errors on (\S*\s)(\S*)\s(\d+\s)', zfslog) iostat = re.search('.*---\n\w*\s*(\S*)\s*(\S*)\s', zfslog) scrubrepairs = scruberrors = scrubdate = None try: scrubrepairs = scrub.group(1) scruberrors = scrub.group(2) scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4) - except: - logger.debug("error getting scrub data") + except Exception as e: + logger.debug("error getting scrub data: " + str(e)) alloc = iostat.group(1) free = iostat.group(2) writetitle("zfs") @@ -691,7 +732,7 @@ def temp(): # Start it like this (bash): sudo hddtemp -d /dev/sda /dev/sdX... received = '' - sumtemp = 0 + sumtemp = 0.0 data = "" output = []