fix bugs

diff --git a/logparse.py b/logparse.py
index cd1e632b4bfa3a748b43dcefebca17f19767eb17..b416218f8efb5255d30d4a12f89243b5f1b50252 100755 (executable)
--- a/logparse.py
+++ b/logparse.py
@@ -92,16 +92,21 @@ def writedata(subtitle, data = None):   # write title and data to tempfile
         loggger.warning("no subtitle provided.. skipping section")
         return
 
         loggger.warning("no subtitle provided.. skipping section")
         return
 

-    tag('p', 0, subtitle)

     if (data == None):
         logger.debug("no data provided.. just printing subtitle")
     if (data == None):
         logger.debug("no data provided.. just printing subtitle")

+        tag('p', 0, subtitle)

     else:
         logger.debug("received data " + str(data))
     else:
         logger.debug("received data " + str(data))

-        opentag('ul', 1)
-        for datum in data:
-            logger.debug("printing datum " + datum)
-            tag('li', 0, datum)
-        closetag('ul', 1)
+        subtitle += ':'
+        if (len(data) == 1):
+            tag('p', 0, subtitle + ' ' + data[0])
+        else:
+            tag('p', 0, subtitle)
+            opentag('ul', 1)
+            for datum in data:
+                logger.debug("printing datum " + datum)
+                tag('li', 0, datum)
+            closetag('ul', 1)

 
 def opentag(tag, block = 0, id = None, cl = None):   # write html opening tag
     if (block == 1):
 
 def opentag(tag, block = 0, id = None, cl = None):   # write html opening tag
     if (block == 1):


@@ -194,7 +199,10 @@ def getusage(path):     # Get disk usage statistics
 def orderbyfreq(l):     # order a list by the frequency of its elements and remove duplicates
     temp_l = l[:]
     l = list(set(l))
 def orderbyfreq(l):     # order a list by the frequency of its elements and remove duplicates
     temp_l = l[:]
     l = list(set(l))

-    l.sort(key=lambda x:temp_l.count(x))
+    l = [[i, temp_l.count(i)] for i in l]   # add count of each element
+    l.sort(key=lambda x:temp_l.count(x[0])) # sort by count
+    l = [i[0] + ' (' + str(i[1]) + ')' for i in l]  # put element and count into string
+    l = l[::-1]     # reverse

     return l
 
 def addtag(l, tag):  # add prefix and suffix tags to each item in a list
     return l
 
 def addtag(l, tag):  # add prefix and suffix tags to each item in a list


@@ -204,7 +212,7 @@ def addtag(l, tag):  # add prefix and suffix tags to each item in a list
 def truncl(input, limit):      # truncate list
     if (len(input) > limit):
         more = str(len(input) - limit)
 def truncl(input, limit):      # truncate list
     if (len(input) > limit):
         more = str(len(input) - limit)

-        output = input[-limit:]
+        output = input[:limit]

         output.append("+ " + more + " more")
         return(output)
     else:
         output.append("+ " + more + " more")
         return(output)
     else:


@@ -254,7 +262,6 @@ def sshd():
         subtitle += ' ' + users[0][0]
         writedata(subtitle)
     else:
         subtitle += ' ' + users[0][0]
         writedata(subtitle)
     else:

-        subtitle += ':'

         for user in users:
             data.append(user[0] + ' (' + str(user[1]) + ')')
             if len(data) > MAXLIST:     # if there are lots of users, truncate them
         for user in users:
             data.append(user[0] + ' (' + str(user[1]) + ')')
             if len(data) > MAXLIST:     # if there are lots of users, truncate them


@@ -288,11 +295,6 @@ def sudo():
     for cmd in cmatches:
         commands.append(cmd)
     logger.debug("found the following commands: " + str(commands))
     for cmd in cmatches:
         commands.append(cmd)
     logger.debug("found the following commands: " + str(commands))

-    # temp_cmd=commands[:]
-    # commands = list(set(commands))
-    # commands.sort(key=lambda x:temp_cmd.count(x))
-    commands = orderbyfreq(commands)
-    logger.debug("top 3 sudo commands: " + str(commands[-3:]))

 
     writetitle("sudo")
     subtitle = plural("sudo session", num) + " for"
 
     writetitle("sudo")
     subtitle = plural("sudo session", num) + " for"


@@ -301,16 +303,13 @@ def sudo():
         subtitle += ' ' + users[0][0]
         writedata(subtitle)
     else:
         subtitle += ' ' + users[0][0]
         writedata(subtitle)
     else:

-        subtitle += ':'

         for user in users:
             data.append(user[0] + ' (' + str(user[1]) + ')')
         for user in users:
             data.append(user[0] + ' (' + str(user[1]) + ')')

-            if len(data) > 3:
-                data.append('+ ' + str(len(users) - 2) + " more")
-                break

         logger.debug("found " + str(len(matches)) + " sudo sessions for users " + str(data))
         writedata(subtitle, data)
     if (len(commands) > 0):
         commands = addtag(commands, 'code')
         logger.debug("found " + str(len(matches)) + " sudo sessions for users " + str(data))
         writedata(subtitle, data)
     if (len(commands) > 0):
         commands = addtag(commands, 'code')

+        commands = orderbyfreq(commands)

         commands = truncl(commands, CMDNO)
         writedata("top sudo commands", [c for c in commands])
     closetag('div', 1)
         commands = truncl(commands, CMDNO)
         writedata("top sudo commands", [c for c in commands])
     closetag('div', 1)


@@ -335,8 +334,8 @@ def cron():
     writetitle("cron")
     writedata(subtitle)
     if (matches > 0):
     writetitle("cron")
     writedata(subtitle)
     if (matches > 0):

-        commands = orderbyfreq(commands)

         commands = addtag(commands, 'code')
         commands = addtag(commands, 'code')

+        commands = orderbyfreq(commands)

         commands = truncl(commands, CMDNO)
         writedata("top cron commands", [c for c in commands])
     closetag('div', 1)
         commands = truncl(commands, CMDNO)
         writedata("top cron commands", [c for c in commands])
     closetag('div', 1)


@@ -356,7 +355,7 @@ def nameget():
     for i in failed:
         l_f.append(i)
     logger.debug("the following downloads failed: " + str(l_f))
     for i in failed:
         l_f.append(i)
     logger.debug("the following downloads failed: " + str(l_f))

-    succ = re.findall('.*nameget.*downloaded.*', syslog)
+    succ = re.findall('.*nameget.*downloaded\s(.*)', syslog)

     n_s = sum(1 for i in succ)
     l_s = []
     for i in succ:
     n_s = sum(1 for i in succ)
     l_s = []
     for i in succ:


@@ -377,26 +376,58 @@ def httpd():
     logger.info("starting httpd section")
     opentag('div', 1, 'httpd', 'section')
     accesslog = readlog("httpd/access.log")
     logger.info("starting httpd section")
     opentag('div', 1, 'httpd', 'section')
     accesslog = readlog("httpd/access.log")

-    a = len(accesslog)
+    a = len(accesslog.split('\n'))

     errorlog = readlog("httpd/error.log")
     errorlog = readlog("httpd/error.log")

-    e = len(errorlog)
+    e = len(errorlog.split('\n'))

     data_b = 0
     data_b = 0

+    ips = []
+    files = []
+    useragents = []
+    errors = []
+    notfound = []
+    unprivileged = []

 
     for line in accesslog.split('\n'):
 
     for line in accesslog.split('\n'):

+        fields = re.search('^(\S*) .*GET (\/.*) HTTP/\d\.\d\" 200 (\d*) \"(.*)\".*\((.*)\;', line)

         try:
         try:

-            data_b += int(re.search('.*HTTP/\d\.\d\" 200 (\d*) ', line).group(1))
+            ips.append(fields.group(1))
+            files.append(fields.group(2))
+            useragents.append(fields.group(5))
+            logger.debug("transferred " + fields.group(3) + " bytes in this request")
+            data_b += int(fields.group(3))
+            logger.debug("data_b is now " + str(data_b))

         except Exception as error:
             if type(error) is AttributeError:
         except Exception as error:
             if type(error) is AttributeError:

-                pass
+                logger.debug("attributeerrror: " + str(error))

             else:
                 logger.warning("error processing httpd access log: " + str(error))
             else:
                 logger.warning("error processing httpd access log: " + str(error))

+    logger.debug(str(data_b) + " bytes transferred")

     data_h = parsesize(data_b)
     data_h = parsesize(data_b)

+    writetitle("apache")

 
     logger.debug("httpd has transferred " + str(data_b) + " bytes in response to " + str(a) + " requests with " + str(e) + " errors")
 
     logger.debug("httpd has transferred " + str(data_b) + " bytes in response to " + str(a) + " requests with " + str(e) + " errors")

+    if (a > 0):
+        logger.debug("found the following requests: " + str(files))
+        files = addtag(files, 'code')
+        files = orderbyfreq(files)
+        files = truncl(files, CMDNO)
+        writedata(str(a) + " requests", files)
+    if (ips != None):
+        logger.debug("found the following ips: " + str(ips))
+        ips = addtag(ips, 'code')
+        ips = orderbyfreq(ips)
+        n_ip = str(len(ips))
+        ips = truncl(ips, CMDNO)
+        writedata(n_ip + " unique clients", ips)
+    if (useragents != None):
+        logger.debug("found the following useragents: " + str(useragents))
+        useragents = addtag(useragents, 'code')
+        useragents = orderbyfreq(useragents)
+        n_ua = str(len(useragents))
+        useragents = truncl(useragents, CMDNO)
+        writedata(n_ua + " unique devices", useragents)

 
 

-    writetitle("apache")

     writedata(data_h + " transferred")
     writedata(data_h + " transferred")

-    writedata(str(a) + " requests")

     writedata(str(e) + " errors")
 
     closetag('div', 1)
     writedata(str(e) + " errors")
 
     closetag('div', 1)


@@ -430,7 +461,7 @@ def smbd():
     opentag('div', 1, 'smbd', 'section')
     files = glob.glob(SMBDDIR + "/log.*[!\.gz][!\.old]")    # find list of logfiles
     n_auths = 0         # total number of logins from all users
     opentag('div', 1, 'smbd', 'section')
     files = glob.glob(SMBDDIR + "/log.*[!\.gz][!\.old]")    # find list of logfiles
     n_auths = 0         # total number of logins from all users

-    sigma_auths = []    # contains users and their respective no. of logins
+    sigma_auths = []    # contains users

     output = ""
 
     for file in files:  # one log file for each client
     output = ""
 
     for file in files:  # one log file for each client


@@ -443,27 +474,23 @@ def smbd():
         matches = re.findall('.*sam authentication for user \[(.*)\] succeeded.*', readlog(file))
         for match in matches:
             userhost = match + "@" + host
         matches = re.findall('.*sam authentication for user \[(.*)\] succeeded.*', readlog(file))
         for match in matches:
             userhost = match + "@" + host

-            exists = [i for i, item in enumerate(sigma_auths) if re.search(userhost, item[0])]
-            if (exists == []):
-                sigma_auths.append([userhost, 1])
-            else:
-                sigma_auths[exists[0]][1] += 1
+            sigma_auths.append(userhost)
+            # exists = [i for i, item in enumerate(sigma_auths) if re.search(userhost, item[0])]
+            # if (exists == []):
+            #     sigma_auths.append([userhost, 1])
+            # else:
+            #     sigma_auths[exists[0]][1] += 1

             n_auths += 1
     writetitle("samba")
     subtitle = plural("login", n_auths) + " from"
             n_auths += 1
     writetitle("samba")
     subtitle = plural("login", n_auths) + " from"

-    data = []

     if (len(sigma_auths) == 1):             # if only one user, do not display no of logins for this user
         subtitle += ' ' + sigma_auths[0][0]
         writedata(subtitle)
     else:       # multiple users
     if (len(sigma_auths) == 1):             # if only one user, do not display no of logins for this user
         subtitle += ' ' + sigma_auths[0][0]
         writedata(subtitle)
     else:       # multiple users

-        subtitle += ':'
-        for x in sigma_auths:
-            data.append((str(x[0])) + " (" + str(x[1]) + ")")
-            if len(data) > MAXLIST:      # if many users, truncate them
-                data.append('+ ' + str(len(sigma_auths) - MAXLIST - 1) + " more")
-                break
+        sigma_auths = orderbyfreq(sigma_auths)
+        sigma_auths = truncl(sigma_auths, CMDNO)

         logger.debug("found " + str(n_auths) + " samba logins for users " + str(sigma_auths))
         logger.debug("found " + str(n_auths) + " samba logins for users " + str(sigma_auths))

-        writedata(subtitle, data)
+        writedata(subtitle, sigma_auths)

     closetag('div', 1)
     logger.info("finished smbd section")
 
     closetag('div', 1)
     logger.info("finished smbd section")
 


@@ -474,12 +501,29 @@ def smbd():
 def postfix():
     logger.debug("starting postfix section")
     opentag('div', 1, 'postfix', 'section')
 def postfix():
     logger.debug("starting postfix section")
     opentag('div', 1, 'postfix', 'section')

-    messages = re.findall('.*from\=<.*>, size\=(\d*),.*\n.*\n.*\: removed\n.*', readlog('postfix'))
-    size = sum([int(x) for x in messages])
+    messages = re.findall('.*from\=<(.*)>, size\=(\d*),.*\n.*to=<(.*)>', readlog('postfix'))
+    r = []
+    s = []
+    size = 0
+    for message in messages:
+        r.append(message[2])
+        s.append(message[0])
+        size += int(message[1])
+    # size = sum([int(x) for x in messages])

     size = parsesize(size)
     n = str(len(messages))
     writetitle("postfix")
     size = parsesize(size)
     n = str(len(messages))
     writetitle("postfix")

-    writedata(n + " messages sent")
+
+    if (len(r) > 0):
+        s = list(set(r))    # unique recipients
+        if (len(s) > 1):
+            r = orderbyfreq(r)
+            r = truncl(r, CMDNO)
+            writedata(n + " messages sent to", r)
+        else:
+            writedata(n + " messages sent to " + r[0])
+    else:
+        writedata(n + " messages sent")

     writedata("total of " + size)
     closetag('div', 1)
     logger.info("finished postfix section")
     writedata("total of " + size)
     closetag('div', 1)
     logger.info("finished postfix section")


@@ -496,14 +540,22 @@ def zfs():
     pool = re.search('.*---\n(\w*)', zfslog).group(1)
     scrub = re.search('.*scrub repaired (\d*) in \d*h\d*m with (\d*) errors on (\S*\s)(\S*)\s(\d+\s)', zfslog)
     iostat = re.search('.*---\n\w*\s*(\S*)\s*(\S*)\s', zfslog)
     pool = re.search('.*---\n(\w*)', zfslog).group(1)
     scrub = re.search('.*scrub repaired (\d*) in \d*h\d*m with (\d*) errors on (\S*\s)(\S*)\s(\d+\s)', zfslog)
     iostat = re.search('.*---\n\w*\s*(\S*)\s*(\S*)\s', zfslog)

-    scrubrepairs = scrub.group(1)
-    scruberrors = scrub.group(2)
-    scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4)
+    scrubrepairs = scruberrors = scrubdate = None
+    try:
+        scrubrepairs = scrub.group(1)
+        scruberrors = scrub.group(2)
+        scrubdate = scrub.group(3) + scrub.group(5) + scrub.group(4)
+    except:
+        logger.debug("error getting scrub data")

     alloc = iostat.group(1)
     free = iostat.group(2)
     writetitle("zfs")
     alloc = iostat.group(1)
     free = iostat.group(2)
     writetitle("zfs")

-    subtitle = "Scrub on " + scrubdate + ": "
-    data = [scrubrepairs + " repaired", scruberrors + " errors", alloc + " used", free + " free"]
+    if (scrubdate != None):
+        subtitle = "Scrub of " + pool + " on " + scrubdate
+        data = [scrubrepairs + " repaired", scruberrors + " errors", alloc + " used", free + " free"]
+    else:
+        subtitle = pool
+        data = [alloc + " used", free + " free"]

     writedata(subtitle, data)
     closetag('div', 1)
     logger.info("finished zfs section")
     writedata(subtitle, data)
     closetag('div', 1)
     logger.info("finished zfs section")