From aa62c5645fb7ae0a1d1993b7073de8d1e5732f5c Mon Sep 17 00:00:00 2001 From: Andrew Lorimer Date: Sat, 21 Sep 2019 18:02:22 +1000 Subject: [PATCH] add parser-specific docs & rewrite sudo parser for journald --- doc/source/index.rst | 86 ++++++++++++++++++- logparse/config.py | 7 +- logparse/formatting.py | 13 ++- logparse/parsers/sudo.py | 6 ++ logparse/parsers/sudo_journald.py | 136 ++++++++++++++++++++++++++++++ 5 files changed, 242 insertions(+), 6 deletions(-) create mode 100644 logparse/parsers/sudo_journald.py diff --git a/doc/source/index.rst b/doc/source/index.rst index a3297d4..f824ae9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -51,9 +51,12 @@ The program is based on a model of independent **parsers** (consisting of Python - smbd - number of logins, list users & clients - sshd (DEPRECATED) - logins by user/hostname, attempted root logins, invalid users - sshd-journald - logins by user/hostname, attempted root logins, invalid users (requires libsystemd) -- sudo - number of sessions, list users and commands +- sudo (DEPRECATED)- number of sessions, list users and commands +- sudo-journald - number of sessions, list users and commands (requires libsystemd) - sysinfo - hostname, OS, OS version, platform, processors +- systemctl - system status, running/failed units (requires libsystemd) - temperature - instantaneous temperatures of motherboard, CPU, cores, disks +- ufw - blocked packets, port and IP data (requires libsystemd) - zfs - zpool scrub reports, disk usage .. _configuration: @@ -159,6 +162,87 @@ subject mailbin Path to the MTA binary (usually Postfix). Default: /usr/bin/mail +====================== +Default parser options +====================== + +Each parser has its own set of options in a section with the name of the parser. In the case of multiple versions of the same parser (e.g. sshd and sshd-journald), the configuration section goes by the base name (e.g. sshd). Options defined in individual parser sections override those defined in the global configuration. + +#### +cron +#### + +period + Maximum age of logs to analyse. Overrides global config. Only used in cron-journald at the moment. See :ref:`period` for more information. Default: empty + +.. _period: + +#### +sshd +#### + +period + Maximum age of logs to analyse. Overrides global config. Only used in sshd-journald at the moment. See :ref:`period` for more information. Default: empty +sshd-resolve-domains + DNS lookup configuration for sshd parsers only (overrides global config). Accepted values are `ip`, `fqdn`, `fqdn-implicit`, and `host-only`. See the global setting `resolve-domains` for more information. Default: empty + +#### +smbd +#### + +shares + Regular expression string for which Samba shares to include when parsing logs. To consider all shares, set this to `.*`. To exclude a certain share, use negative lookaround. Default: `^((?!IPC\$).)*$` +users + Regular expression string for which user@hostname values to include when parsing logs. This could be used to exclude logins from a trusted user or hostname. Default: `.*` +smbd-resolve-domains + DNS lookup configuration for smbd parsers only (overrides global config). Accepted values are `ip`, `fqdn`, `fqdn-implicit`, and `host-only`. See the global setting `resolve-domains` for more information. Default: empty +period + Maximum age of logs to analyse. Overrides global config. Only used in smbd-journald at the moment. See :ref:`period` for more information. Default: empty + +##### +httpd +##### + +httpd-resolve-domains + DNS lookup configuration for httpd parser only (overrides global config). Accepted values are `ip`, `fqdn`, `fqdn-implicit`, and `host-only`. See the global setting `resolve-domains` for more information. Default: empty +period + Maximum age of logs to analyse. Overrides global config. See :ref:`period` for more information. Default: empty + +### +ufw +### + +ufw-resolve-domains + DNS lookup configuration for ufw parser only (overrides global config). Accepted values are `ip`, `fqdn`, `fqdn-implicit`, and `host-only`. See the global setting `resolve-domains` for more information. Default: empty +period + Maximum age of logs to analyse. Overrides global config. See :ref:`period` for more information. Default: empty + +#### +sudo +#### + +period + Maximum age of logs to analyse. Overrides global config. See :ref:`period` for more information. Default: empty + +######### +systemctl +######### + +period + Maximum age of logs to analyse. Overrides global config. See :ref:`period` for more information. Default: empty +show-all + Whether to include services which are running but okay in the output. Default: true + + + +======================== +Log period configuration +======================== + +Some parsers support custom time periods to be searched for logs. This period is specified as a string in the configuration section of supported parsers, and is a timespan relative to the time when the parser is initialised. The time parsing functionality uses a modified version of `timeparse.py` originally written by Will Roberts under the MIT License. The following excerpt is taken from the documentation of `timeparse.py`: + +.. autofunction:: logparse.timeparse.strseconds + .. _variables: ===================== diff --git a/logparse/config.py b/logparse/config.py index e2aa830..bcdc8b2 100644 --- a/logparse/config.py +++ b/logparse/config.py @@ -35,7 +35,7 @@ defaults = { 'hostname-path': '/etc/hostname', 'parsers': '', 'ignore-parsers': '', - 'period': '1 day' + 'period': '1 minute' }, 'html': { 'header': '/etc/logparse/header.html', @@ -102,7 +102,10 @@ defaults = { 'period': '' }, 'sudo': { - 'period': '' + 'period': '', + 'list-users': True, + 'summary': True, + 'truncate-commands': True }, 'systemctl': { 'period': '', diff --git a/logparse/formatting.py b/logparse/formatting.py index 8528e1b..624d66c 100644 --- a/logparse/formatting.py +++ b/logparse/formatting.py @@ -2,9 +2,10 @@ """ This file contains global functions for formatting and printing data. This file -should be imported into individual log-parsing scripts located in logs/*. Data -is formatted in HTML or plaintext. Writing to disk and/or emailng data is left -to interface.py. +should be imported into individual log-parsing scripts located in the default +logparse.parsers module or in the user-supplied parsers directory. Data is +formatted in HTML or plaintext. Writing to disk and/or emailng data is left to +interface.py. """ import os @@ -151,6 +152,9 @@ class PlaintextOutput(Output): This should be run by interface.py after every instance of parse_log(). """ + if section == None: + logger.warning("Received null section") + return self.append(PlaintextBox( content=section.title, double=False, fullwidth=False, vpadding=0, hpadding=" ").draw()) @@ -279,6 +283,9 @@ class HtmlOutput(Output): instance of parse_log(). """ + if section == None: + logger.warning("Received null section") + return self.append(opentag('div', 1, section.title, 'section')) self.append(self._gen_title(section.title)) if section.period and section.period.unique: diff --git a/logparse/parsers/sudo.py b/logparse/parsers/sudo.py index d1c3b81..8a6dcd3 100644 --- a/logparse/parsers/sudo.py +++ b/logparse/parsers/sudo.py @@ -3,6 +3,10 @@ # # Get number of sudo sessions for each user # +# NOTE: This file is now deprecated in favour of the newer journald mechanism +# used in sudo-journald.py. This parser is still functional but is slower and +# has less features. Please switch over if possible. +# import re @@ -17,6 +21,8 @@ class Sudo(Parser): super().__init__() self.name = "sudo" self.info = "Get number of sudo sessions for each user" + self.deprecated = True + self.successor = "sudo_journald" def parse_log(self): logger.debug("Starting sudo section") diff --git a/logparse/parsers/sudo_journald.py b/logparse/parsers/sudo_journald.py new file mode 100644 index 0000000..ad82f27 --- /dev/null +++ b/logparse/parsers/sudo_journald.py @@ -0,0 +1,136 @@ +# +# sudo-journald.py +# +# Get number of sudo sessions for each user +# + +import re +from systemd import journal + +from logparse.formatting import * +from logparse.util import readlog +from logparse.config import prefs +from logparse.load_parsers import Parser + +class SudoCommand(): + + def __init__(self, msg): + self.init_user, self.pwd, self.su, self.command = re.search( + r"\s*(?P\w*) : TTY=.* ; PWD=(?P\S*) ;" + " USER=(?P\w*) ; COMMAND=(?P\S*)", msg)\ + .groupdict().values() + +class Sudo(Parser): + + def __init__(self): + super().__init__() + self.name = "sudo" + self.info = "Get number of sudo sessions for each user" + + def parse_log(self): + + logger.debug("Starting sudo section") + section = Section("sudo") + + if not (config.prefs.getboolean("sudo", "summary") + or config.prefs.getboolean("sudo", "list-users")): + logger.warning("Both summary and list-users configuration options " + "are set to false, so no output will be generated. " + "Skipping this parser.") + return None + + j = journal.Reader() + j.this_machine() + j.log_level(journal.LOG_INFO) + j.add_match(_COMM="sudo") + j.seek_realtime(section.period.startdate) + j.log_level(5) + + logger.debug("Searching for sudo commands") + + messages = [entry["MESSAGE"] for entry in j if "MESSAGE" in entry] + + commands_objects = [] # list of command objects + init_users = {} # keys are users, values are lists of commands + + logger.debug("Parsing sudo log messages") + + for msg in messages: + + if "COMMAND=" in msg: + try: + command_obj = SudoCommand(msg) + except Exception as e: + logger.warning("Malformed sudo log message: {0}. " + "Error message: {1}".format(msg, str(e))) + continue + if config.prefs.getboolean("sudo", "truncate-commands"): + command_obj.command = command_obj.command.split("/")[-1] + commands_objects.append(command_obj) + if not command_obj.init_user in init_users: + init_users[command_obj.init_user] = [] + init_users[command_obj.init_user].append( + command_obj.su + ": " + command_obj.command) + + logger.debug("Generating output") + + if len(commands_objects) == 0: + logger.warning("No sudo commands found") + return + + if config.prefs.getboolean("sudo", "summary"): + + summary_data = Data() + summary_data.subtitle = plural("sudo session", len(commands_objects)) + + if all(cmd.su == commands_objects[0].su for cmd in commands_objects): + # Only one superuser + if len(set(cmd.init_user for cmd in commands_objects)) > 1: + # Multiple initiating users + summary_data.subtitle += " for superuser " \ + + commands_objects[0].su + summary_data.items = ["{}: {}".format(cmd.init_user, cmd.command) + for cmd in commands_objects] + else: + # Only one initiating user + summary_data.subtitle += " opened by " \ + + commands_objects[0].init_user + " for " \ + + commands_objects[0].su + summary_data.items = [cmd.command + for cmd in commands_objects] + else: + # Multiple superusers + if len(set(cmd.init_user for cmd in commands_objects)) > 1: + # Multiple initiating users + summary_data.subtitle += " for " + plural("superuser", + len(set(cmd.su for cmd in commands_objects))) + summary_data.items = ["{}→{}: {}".format( + cmd.init_user, cmd.su, cmd.command) + for cmd in commands_objects] + else: + # Only one initiating user + summary_data.subtitle += " by " \ + + commands_objects[0].init_user \ + + " for " + plural("superuser", + len(set(cmd.su + for cmd in commands_objects))) + summary_data.items = ["{}: {}".format( + cmd.su, cmd.command) for cmd in commands_objects] + summary_data.orderbyfreq() + summary_data.truncl(config.prefs.getint("logparse", "maxcmd")) + section.append_data(summary_data) + + if config.prefs.getboolean("sudo", "list-users") \ + and len(set(cmd.init_user for cmd in commands_objects)) > 1: + for user, user_commands in init_users.items(): + user_data = Data() + user_data.subtitle = plural("sudo session", + len(user_commands)) + " for user " + user + user_data.items = user_commands + user_data.orderbyfreq() + user_data.truncl(config.prefs.getint("logparse", "maxcmd")) + section.append_data(user_data) + + logger.info("Finished sudo section") + + return section -- 2.43.2