-#
-# utilities.py
-#
-# Commonly used general functions
-#
+"""
+Commonly used general functions.
-import re
+This module provides the following methods:
+ - `hostname`: get the current machine's hostname
+ - `getlocaldomain`: get the current machine's domain name
+ - `resolve`: attempt to convert a local/public IP to hostname
+ - `readlog`: read contents of a log file from disk
+And the following classes:
+ - `LogPeriod`: period to search logs (wrapper for datetime.timedelata)
+"""
+
+from datetime import datetime, timedelta
+import copy
+from configparser import NoSectionError
+import ipaddress
+import logging
import os
+from pkg_resources import Requirement, resource_filename
+import re
import socket
-import inspect
+from sys import exit
+
+from logparse import config, formatting
+from logparse.timeparse import timeparse
+
-import logging
logger = logging.getLogger(__name__)
-from .config import prefs
-from pkg_resources import Requirement, resource_filename
def hostname(path): # get the hostname of current server
+ """
+ Get the hostname of the current machine using the file supplied in the
+ `hostname-path` config option.
+ """
+
hnfile = open(path, 'r')
hn = re.search('^(\w*)\n*', hnfile.read()).group(1)
return hn
+
def getlocaldomain(): # get the parent fqdn of current server
- domain = socket.getfqdn().split('.', 1) # Note: if socket.fetfqdn() returns localhost, make sure the first entry in /etc/hosts contains the fqdn
+ """
+ Get parent domain name (possibly FQDN) of the current machine. Note: if
+ `socket.fetfqdn()` returns localhost, make sure the first entry in the
+ hostname file includes the FQDN.
+ """
+
+ domain = socket.getfqdn().split('.', 1)
if len(domain) != 2:
- logger.warning('Could not get domain of this server, only hostname. Please consider updating /etc/hosts')
+ logger.warning("Could not get domain of this server, only hostname. "
+ "Please consider updating the hostname file at {0}".format(
+ config.prefs.get("logparse", "hostname-path")))
return 'localdomain'
else:
return domain[-1]
-def resolve(ip, fqdn = 'host-only'): # try to resolve an ip to hostname
- # Possible values for fqdn:
- # fqdn show full hostname and domain
- # fqdn-implicit show hostname and domain unless local
- # host-only only show hostname
- # ip never resolve anything
- # resolve-domains defined in individual sections of the config take priority over global config
+
+def resolve(ip, fqdn=None): # try to resolve an ip to hostname
+ """
+ Attempt to resolve an IP into a hostname or FQDN.
+ Possible values for fqdn:
+ - fqdn show full hostname and domain
+ - fqdn-implicit show hostname and domain unless local
+ - host-only only show hostname
+ - ip never resolve anything
+ Note resolve-domains settings defined in individual sections of the config
+ take priority over the global config (this is enforced in parser modules)
+ """
if not fqdn:
- fqdn = prefs['resolve-domains']
+ fqdn = config.prefs.get("logparse", "resolve-domains")
if fqdn == 'ip':
return(ip)
try:
- socket.inet_aton(ip) # succeeds if text contains ip
+ ip_obj = ipaddress.ip_address(ip)
+ except ValueError as err:
+ logger.debug("Invalid format: " + str(err))
+ return ip
+
+ try:
hn = socket.gethostbyaddr(ip)[0] # resolve ip to hostname
- if fqdn == 'fqdn-implicit' and hn.split('.', 1)[1] == getlocaldomain():
- return(hn.split('.')[0])
- elif fqdn == 'fqdn' or fqdn == 'fqdn-implicit':
- return(hn)
- elif fqdn == 'host-only':
- return(hn.split('.')[0])
- else:
- logger.warning("invalid value for fqdn config")
- return(hn)
except socket.herror:
# cannot resolve ip
- logger.debug(ip + " cannot be found, might not exist anymore")
- return(ip)
- except (OSError, socket.error): # socket.error for Python 2 compatibility
- # already a hostname
- logger.debug(ip + " is already a hostname")
+ # LOGGING DISABLED TO AVOID SPAM
+# logger.debug(ip + " cannot be found, might not exist anymore")
return(ip)
except Exception as err:
- logger.warning("failed to resolve hostname for " + ip + ": " + str(err))
+ logger.warning("Failed to resolve hostname for " + ip + ": " + str(err))
return(ip) # return ip if no hostname exists
-def readlog(path = None, mode = 'r'): # read file
+ if (fqdn == "host-only") or (fqdn == "fqdn-implicit" and ip_obj.is_private):
+ return hn.split('.')[0]
+ if fqdn == 'fqdn' or fqdn == 'fqdn-implicit':
+ return hn
+ return hn
+
+
+
+def readlog(path = None, mode = 'r'):
+ """
+ Read a logfile from disk and return string
+ """
+
if (path == None):
- logger.error("no path provided")
- return
+ logger.error("No path provided")
+ return 1
else:
if (os.path.isfile(path) is False):
- logger.error("Log at {0} was requested but does not exist".format(path))
+ logger.error("Log at {0} was requested but does not exist"
+ .format(path))
return ''
else:
try:
return open(path, mode).read()
except IOError or OSError as e:
- logger.warning("Error reading log at {0}: {1}".format(path, e.strerror))
+ logger.error("Error reading log at {0}: {1}"
+ .format(path, e.strerror))
return 1
+
+class LogPeriod:
+ """
+ Represents a time period for which logs should be parsed (this is given to
+ journald.seek_realtime). Uses timeparse.py by Will Roberts.
+ """
+
+ def __init__(self, section):
+ """
+ If no period is defined for the section config, it is inherited from
+ the global config. Failing that, the program will die.
+ """
+ try:
+ self.startdate = datetime.now() \
+ - timeparse(config.prefs.get(section.split("_")[0], "period"))
+ logger.debug("Parsing logs for {0} since {1}".format(
+ section, self.startdate.strftime(formatting.DATEFMT
+ + " " + formatting.TIMEFMT)))
+ self.unique = True
+ except (NoSectionError, TypeError):
+ logger.debug("No period defined in section {0} - inheriting "
+ "global period".format(section))
+ self.startdate = datetime.now() \
+ - timeparse(config.prefs.get("logparse", "period"))
+ self.unique = False
+ except Exception as e:
+ logger.error("Could not find valid time period for parser {0}"
+ " {1}".format(section, e))
+ return None
+
+ def compare(self, d, ephemeral=False) -> bool:
+ """
+ Compare the datetime `d` of a log record with the starting datetime of
+ this LogPeriod and return a boolean result representing whether the
+ log record is after the starting datetime. If either `self.startdate`
+ or `d` are UTC-naive, an appropriate correction is attempted. The
+ `ephemeral` boolean argument, if set to true, prevents permanent
+ modification of `self.startdate` to match the UTC-awareness of the
+ section's log records, in the case that `d` may represent an anomaly.
+ """
+ try:
+ # First attempt a direct comparison
+ return True if d > self.startdate else False
+
+ except TypeError as e:
+ # If the direct comparison fails, make sure that both objects have
+ # a UTC offset
+
+ if d.tzinfo is None \
+ or (d.tzinfo is not None
+ and not d.tzinfo.utcoffset(d)):
+ # d has no timezone info,
+ # OR d has timezone info but no offset
+
+ if self.startdate.tzinfo is not None \
+ and self.startdate.tzinfo.utcoffset(self.startdate):
+ # If d is naive and self.startdate is aware, assume that
+ # the timezone of d is the same as self.startdate by
+ # making self.startdate naive
+
+ logger.warning("{0} checking date {1}: {2}. Time "
+ "of log record is naive. Inheriting UTC "
+ "offset for logs from system date".format(
+ e.__class__.__name__, d, e))
+ if ephemeral:
+ # Compare with UTC-naive version of self.startdate
+ return True if d > self.startdate.replace(tzinfo=None) \
+ else False
+ else:
+ # Remove UTC awareness for self.startdate
+ self.startdate = self.startdate.replace(tzinfo=None)
+ return True if d > self.startdate else False
+
+ elif self.startdate.tzinfo is None \
+ or (self.startdate.tzinfo is not None
+ and not self.startdate.tzinfo.utcoffset(self.startdate)):
+ # d has timezoneinfo and offset, but self.startdate has either
+ # no timezone info, or timezone info and no offset. In this
+ # case, self.startdate inherits the offset of d.
+
+ logger.warning("{0} checking date {1}: {2}. Time of "
+ "start date is naive. Inheriting UTC offset "
+ "for date comparison from logs".format(
+ e.__class__.__name__, d, e))
+
+ if ephemeral:
+ # Compare with UTC-aware version of self.startdate
+ return True if d > self.startdate.astimezone(d.tzinfo) \
+ else False
+ else:
+ # Add UTC awareness for self.startdate
+ self.startdate = self.startdate.astimezone(d.tzinfo)
+ return True if d > self.startdate else False
+
+ else:
+ # Other errors return false (effectively ignore this record)
+ logger.error("{0} comparing date {1}: {2}".format(
+ e.__class__.__name__, d, e))
+ return False