implement new resolve-domains config option, bugfixing for hostname resolver

diff --git a/logparse.py b/logparse.py
index 896e7640023a086e1c0f6fa697f8bbfbbe748de4..638d02a0a50d7f2dd209e3717e888c6a90fc7e37 100755 (executable)
--- a/logparse.py
+++ b/logparse.py
@@ -8,6 +8,7 @@ import yaml
 import ast
 import logging.handlers
 import types
 import ast
 import logging.handlers
 import types

+import traceback # debugging only

 
 reload(sys)
 sys.setdefaultencoding('utf-8')     # force utf-8 because anything else should die
 
 reload(sys)
 sys.setdefaultencoding('utf-8')     # force utf-8 because anything else should die


@@ -26,6 +27,7 @@ config = {
     'title': 'logparse',
     'maxlist': 10,
     'maxcmd': 3,
     'title': 'logparse',
     'maxlist': 10,
     'maxcmd': 3,

+    'resolve-domains': 'fqdn',

     'mail': {
         'to': '',
         'from': '',
     'mail': {
         'to': '',
         'from': '',


@@ -37,6 +39,18 @@ config = {
         'port': 7634,
         'show-model': False, 
     },
         'port': 7634,
         'show-model': False, 
     },

+    'apache': {
+        'resolve-domains': '',
+    },
+    'sshd': {
+        'resolve-domains': '',
+    },
+    'smbd': {
+        'resolve-domains': '',
+    },
+    'httpd': {
+        'resolve-domains': '',
+    },

     'du-paths': ['/', '/etc', '/home'],
     'hostname-path': '/etc/hostname',
     'logs': {
     'du-paths': ['/', '/etc', '/home'],
     'hostname-path': '/etc/hostname',
     'logs': {


@@ -84,18 +98,17 @@ def __main__():
     debugfunc = parser.parse_args().function
     if debugfunc is not None:
         logger.debug("executing a single function: " + debugfunc)
     debugfunc = parser.parse_args().function
     if debugfunc is not None:
         logger.debug("executing a single function: " + debugfunc)

-        try:
-            logger.debug((debugfunc + ': ' + eval(debugfunc)))
-            sys.exit()
-        except Exception as e:
-            sys.exit("debug function failed with error " + e)
-        logger.debug("finished executing debug function")
+        eval(debugfunc)
+        sys.exit()

         
     if not config['mail']['to']:
         logger.info("no recipient address provided, outputting to stdout")
     else:
         logger.info("email will be sent to " + config['mail']['to'])
 
         
     if not config['mail']['to']:
         logger.info("no recipient address provided, outputting to stdout")
     else:
         logger.info("email will be sent to " + config['mail']['to'])
 

+    global LOCALDOMAIN
+    LOCALDOMAIN = getlocaldomain()
+

     global pathfilter
     global pathpattern
     pathfilter = {"auth": config['logs']['auth'], "cron": config['logs']['cron'], "sys": config['logs']['sys'], "postfix": config['logs']['postfix'], "smb": config['logs']['smb'], "zfs": config['logs']['zfs'], "alloc": config['logs']['alloc'], "httpd": config['logs']['httpd'], "header": config['header']}
     global pathfilter
     global pathpattern
     pathfilter = {"auth": config['logs']['auth'], "cron": config['logs']['cron'], "sys": config['logs']['sys'], "postfix": config['logs']['postfix'], "smb": config['logs']['smb'], "zfs": config['logs']['zfs'], "alloc": config['logs']['alloc'], "httpd": config['logs']['httpd'], "header": config['header']}


@@ -205,22 +218,50 @@ def hostname(): # get the hostname of current server
     hn = re.search('^(.*)\n*', hnfile.read()).group(1)
     return hn
 
     hn = re.search('^(.*)\n*', hnfile.read()).group(1)
     return hn
 

+def getlocaldomain(): # get the parent fqdn of current server
+    domain = socket.getfqdn().split('.', 1)
+    if len(domain) == 2:
+        logger.warning('Could not get domain of this server, only hostname. Please consider updating /etc/hosts')
+        return ''
+    else:
+        return domain[-1]
+
+def resolve(ip, fqdn = 'host-only'):        # try to resolve an ip to hostname
+    # Possible values for fqdn:
+    #   fqdn            show full hostname and domain
+    #   fqdn-implicit   show hostname and domain unless local
+    #   host-only       only show hostname
+    #   ip              never resolve anything
+    # resolve-domains defined in individual sections of the config take priority over global config
+    
+    if not fqdn:
+        fqdn = config['resolve-domains']
+
+    if fqdn == 'ip':
+        return(ip)

 
 

-def resolve(ip, fqdn = False):        # try to resolve an ip to hostname

     try:
         socket.inet_aton(ip)  # succeeds if text contains ip
         hn = socket.gethostbyaddr(ip)[0] # resolve ip to hostname
     try:
         socket.inet_aton(ip)  # succeeds if text contains ip
         hn = socket.gethostbyaddr(ip)[0] # resolve ip to hostname

-        return(hn if fqdn else hn.split('.')[0])
-    except OSError:
-        # already a hostname
-        logger.debug(ip + " is already a hostname")
-        return(ip)
+        if fqdn == 'fqdn-implicit' and hn.split('.', 1)[1] == LOCALDOMAIN:
+            return(hn.split('.')[0])
+        elif fqdn == 'fqdn' or fqdn == 'fqdn-implicit':
+            return(hn)
+        elif fqdn == 'host-only':
+            return(hn.split('.')[0])
+        else:
+            logger.warning("invalid value for fqdn config")
+            return(hn)

     except socket.herror:
         # cannot resolve ip
         logger.debug(ip + " cannot be found, might not exist anymore")
         return(ip)
     except socket.herror:
         # cannot resolve ip
         logger.debug(ip + " cannot be found, might not exist anymore")
         return(ip)

-    except:
-        logger.debug("failed to resolve hostname for " + ip)
+    except (OSError, socket.error): # socket.error for Python 2 compatibility
+        # already a hostname
+        logger.debug(ip + " is already a hostname")
+        return(ip)
+    except Exception as err:
+        logger.warning("failed to resolve hostname for " + ip + ": " + str(err))

         return(ip)  # return ip if no hostname exists
 
 def plural(noun, quantity): # return "1 noun" or "n nouns"
         return(ip)  # return ip if no hostname exists
 
 def plural(noun, quantity): # return "1 noun" or "n nouns"


@@ -319,7 +360,7 @@ def sshd():
         user = entry.group(1)
         ip = entry.group(2)
 
         user = entry.group(1)
         ip = entry.group(2)
 

-        userhost = user + '@' + resolve(ip)
+        userhost = user + '@' + resolve(ip, fqdn=config['sshd']['resolve-domains'])

         exists = [i for i, item in enumerate(users) if re.search(userhost, item[0])]
         if (exists == []):
             users.append([userhost, 1])
         exists = [i for i, item in enumerate(users) if re.search(userhost, item[0])]
         if (exists == []):
             users.append([userhost, 1])


@@ -462,7 +503,7 @@ def httpd():
     for line in accesslog.split('\n'):
         fields = re.search('^(\S*) .*GET (\/.*) HTTP/\d\.\d\" 200 (\d*) \"(.*)\".*\((.*)\;', line)
         try:
     for line in accesslog.split('\n'):
         fields = re.search('^(\S*) .*GET (\/.*) HTTP/\d\.\d\" 200 (\d*) \"(.*)\".*\((.*)\;', line)
         try:

-            ips.append(resolve(fields.group(1), fqdn=True))
+            ips.append(resolve(fields.group(1), fqdn=config['httpd']['resolve-domains']))

             files.append(fields.group(2))
             useragents.append(fields.group(5))
             data_b += int(fields.group(3))
             files.append(fields.group(2))
             useragents.append(fields.group(5))
             data_b += int(fields.group(3))


@@ -471,6 +512,7 @@ def httpd():
                 pass
             else:
                 logger.warning("error processing httpd access log: " + str(error))
                 pass
             else:
                 logger.warning("error processing httpd access log: " + str(error))

+                traceback.print_exc()

     logger.debug(str(data_b) + " bytes transferred")
     data_h = parsesize(data_b)
     writetitle("apache")
     logger.debug(str(data_b) + " bytes transferred")
     data_h = parsesize(data_b)
     writetitle("apache")


@@ -555,8 +597,8 @@ def smbd():
 
         # find the machine (ip or hostname) that this file represents
         ip = re.search('log\.(.*)', file).group(1)    # get ip or hostname from file path (/var/log/samba/log.host)
 
         # find the machine (ip or hostname) that this file represents
         ip = re.search('log\.(.*)', file).group(1)    # get ip or hostname from file path (/var/log/samba/log.host)

-        host = resolve(ip)
-        if (host == ip):    # if ip has disappeared, fall back to a hostname from logfile
+        host = resolve(ip, fqdn=config['smbd']['resolve-domains'])
+        if (host == ip and (config['smbd']['resolve-domains'] or config['resolve-domains']) != 'ip'):    # if ip has disappeared, fall back to a hostname from logfile

             newhost = re.findall('.*\]\@\[(.*)\]', readlog(file))
             if (len(set(newhost)) == 1):    # all hosts in one file should be the same
                 host = newhost[0].lower()
             newhost = re.findall('.*\]\@\[(.*)\]', readlog(file))
             if (len(set(newhost)) == 1):    # all hosts in one file should be the same
                 host = newhost[0].lower()


@@ -627,17 +669,16 @@ def zfs():
     logger.debug("starting zfs section")
     opentag('div', 1, 'zfs', 'section')
     zfslog = readlog('zfs')
     logger.debug("starting zfs section")
     opentag('div', 1, 'zfs', 'section')
     zfslog = readlog('zfs')

-    logger.debug("got zfs logfile")

     pool = re.search('.*---\n(\w*)', zfslog).group(1)
     pool = re.search('.*---\n(\w*)', zfslog).group(1)

-    scrub = re.search('.*scrub repaired (\d*) in \d*h\d*m with (\d*) errors on (\S*\s)(\S*)\s(\d+\s)', zfslog)
+    scrub = re.search('.*scrub repaired (\d*).* in .*\d*h\d*m with (\d*) errors on (\S*\s)(\S*)\s(\d+\s)', zfslog)

     iostat = re.search('.*---\n\w*\s*(\S*)\s*(\S*)\s', zfslog)
     scrubrepairs = scruberrors = scrubdate = None
     try:
         scrubrepairs = scrub.group(1)
         scruberrors = scrub.group(2)
         scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4)
     iostat = re.search('.*---\n\w*\s*(\S*)\s*(\S*)\s', zfslog)
     scrubrepairs = scruberrors = scrubdate = None
     try:
         scrubrepairs = scrub.group(1)
         scruberrors = scrub.group(2)
         scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4)

-    except:
-        logger.debug("error getting scrub data")
+    except Exception as e:
+        logger.debug("error getting scrub data: " + str(e))

     alloc = iostat.group(1)
     free = iostat.group(2)
     writetitle("zfs")
     alloc = iostat.group(1)
     free = iostat.group(2)
     writetitle("zfs")


@@ -691,7 +732,7 @@ def temp():
     # Start it like this (bash):   sudo hddtemp -d /dev/sda /dev/sdX...
     
     received = ''
     # Start it like this (bash):   sudo hddtemp -d /dev/sda /dev/sdX...
     
     received = ''

-    sumtemp = 0 
+    sumtemp = 0.0 

     data = ""
     output = []
     
     data = ""
     output = []