From 9db68b263dbc111754158f81073eeb2daaf36839 Mon Sep 17 00:00:00 2001 From: Brian Read Date: Mon, 17 Jun 2024 23:04:49 +0100 Subject: [PATCH] Properly align links at top of web page and add header text to page --- root/opt/mailstats/css/mailstats.css | 22 +-- .../opt/mailstats/templates/mailstats.html.pt | 2 + root/usr/bin/mailstats.py | 146 +++++++++++++++--- 3 files changed, 142 insertions(+), 28 deletions(-) diff --git a/root/opt/mailstats/css/mailstats.css b/root/opt/mailstats/css/mailstats.css index 388b7b8..c01c85f 100644 --- a/root/opt/mailstats/css/mailstats.css +++ b/root/opt/mailstats/css/mailstats.css @@ -33,19 +33,23 @@ tfoot tr { tbody tr:nth-child(odd) {background-color: #dfdfdf} +div.linksattop { + display: flex; + justify-content: space-between; +} + a.prevlink { - float: left; - width:33.33333%; - text-align:left; + text-align: left; } -div.divseeinbrowser { - float: right; + +div.divshowindex { + flex-grow: 1; + text-align: center; } + a.nextlink { - float: right; - width:33.33333%; - text-align:right; -}​ + text-align: right; +} .cssclass1 {background-color:#ffff99;} .cssclass2 {background-color:lightcoral;} diff --git a/root/opt/mailstats/templates/mailstats.html.pt b/root/opt/mailstats/templates/mailstats.html.pt index 1d1e129..3a46903 100644 --- a/root/opt/mailstats/templates/mailstats.html.pt +++ b/root/opt/mailstats/templates/mailstats.html.pt @@ -43,6 +43,8 @@

${title}

+
+
diff --git a/root/usr/bin/mailstats.py b/root/usr/bin/mailstats.py index c8ea283..ca52833 100644 --- a/root/usr/bin/mailstats.py +++ b/root/usr/bin/mailstats.py @@ -7,8 +7,10 @@ # Mailstats # # optional arguments: -# -h, --help show this help message and exit -# -d DATE, --date DATE Specify a valid date (yyyy-mm-dd) for the analysis +# -h, --help show this help message and exit +# -d DATE, --date DATE Specify a valid date (yyyy-mm-dd) for the analysis +# -ef EMAILFILE, --emailfile EMAILFILE +# Save an html file of the email sent (y/N) # # Re-written in python from Mailstats.pl (Perl) to conform to SME11 / Postfix / qpsmtpd log formats # and html output added @@ -16,6 +18,9 @@ # Todo # 2 Other stats # 3. Extra bits for sub tables +# 4. Percent char causes sort to fail - look at adding it in the template +# 5. Chase disparity in counts betweeen old mailstats and this +# 6. Count emails delivered over 25/587/465 # # Centos7: # yum install python3-chameleon --enablerepo=epel @@ -155,7 +160,6 @@ def is_private_ip(ip): ipaddress.ip_network('172.16.0.0/12'), ipaddress.ip_network('192.168.0.0/16'), ] - # Check if the IP address is within any of these ranges for private_range in private_ranges: if ip_addr in private_range: @@ -387,12 +391,9 @@ def insert_string_after(original:str, to_insert:str, after:str) -> str: :return: The new string with to_insert inserted after after. """ position = original.find(after) - #print(position) - if position == -1: - # 'after' string is not found in 'original' + print(f"insert_string_after:({after}) string is not found in original") return original - #print(f"{len(after)}") # Position of the insertion point insert_pos = position + len(after) @@ -575,6 +576,53 @@ def replace_between(text, start, end, replacement): # Using re.DOTALL to match any character including newline replaced_text = re.sub(pattern, replacement, text, flags=re.DOTALL) return replaced_text + +def get_heading(): + # + # Needs from anaytsis + # SATagLevel - done + # SARejectLevel - done + # warnnoreject - done + # totalexamined - done + # emailperhour - done + # spamavg - done + # rejectspamavg - done + # hamavg - done + # DMARCSendCount - done + # hamcount - done + # DMARCOkCount - deone + + # Clam Version/DB Count/Last DB update + clam_output = subprocess.getoutput("freshclam -V") + clam_info = f"Clam Version/DB Count/Last DB update: {clam_output}" + + # SpamAssassin Version + sa_output = subprocess.getoutput("spamassassin -V") + sa_info = f"SpamAssassin Version: {sa_output}" + + # Tag level and Reject level + tag_reject_info = f"Tag level: {SATagLevel}; Reject level: {SARejectLevel} {warnnoreject}" + + # SMTP connection stats + smtp_stats = f"All External SMTP connections accepted: {totalexternalsmtpsessions}\n"\ + f"All Internal SMTP connections accepted: {totalinternalsmtpsessions}\n"\ + f"Emails per hour: {emailperhour:.1f}/hr\n"\ + f"Average spam score (accepted): {spamavg or 0:.2f}\n"\ + f"Average spam score (rejected): {rejectspamavg or 0:.2f}\n"\ + f"Average ham score: {hamavg or 0:.2f}\n"\ + f"Number of DMARC reporting emails sent: {DMARCSendCount or 0} (not shown on table)" + + # DMARC approved emails + dmarc_info = "" + if hamcount != 0: + dmarc_ok_percentage = DMARCOkCount * 100 / hamcount + dmarc_info = f"Number of emails approved through DMARC: {DMARCOkCount or 0} ({dmarc_ok_percentage:.2f}% of Ham count)" + + # Accumulate all strings + header_str = "\n".join([clam_info, sa_info, tag_reject_info, smtp_stats, dmarc_info]) + # switch newlines to
+ header_str = header_str.replace("\n","
") + return header_str if __name__ == "__main__": try: @@ -620,6 +668,10 @@ if __name__ == "__main__": SARejectLevel = int(get_value(ConfigDB, "spamassassin", "RejectLevel","12")) #12 #$cdb->get('spamassassin')->prop('RejectLevel'); SATagLevel = int(get_value(ConfigDB, "spamassassin", "TagLevel","4")) #4 #$cdb->get('spamassassin')->prop('TagLevel'); + if SARejectLevel == 0: + warnnoreject = "(*Warning* 0 = no reject)" + else: + warnnoreject = "" EmailAddress = get_value(ConfigDB,"mailstats","Email","admin@"+DomainName) if '@' not in EmailAddress: @@ -699,10 +751,19 @@ if __name__ == "__main__": # Initial call to print the progress bar #unless none to show if sorted_len > 0: + spamavg = 0; + spamqueuedcount = 0 + hamcount = 0 + hamavg = 0 + rejectspamcount = 0 + rejectspamavg = 0 + DMARCSendCount = 0 + totalexamined = 0 if isThonny: print_progress_bar(0, sorted_len, prefix='Progress:', suffix='Complete', length=50) for timestamp, data in sorted_log_dict.items(): i += 1 + totalexamined += 1 if isThonny: print_progress_bar(i, sorted_len, prefix='Scanning for main table:', suffix='Complete', length=50) #print(f"{i*100/len}%") @@ -728,23 +789,42 @@ if __name__ == "__main__": if parsed_data['action'] == '(queue)': columnCounts_2d[hour][Ham] += 1 columnCounts_2d[ColTotals][Ham] += 1 - #spamassasin + # spamassassin not rejected + if parsed_data.get('spam-status') is not None and isinstance(parsed_data['spam-status'], str): + if parsed_data['spam-status'].lower().startswith('no'): + #Extract other parameters from this string + # example: No, score=-3.9 + spam_pattern = r'score=(-?\d+\.\d+) required=(-?\d+\.\d+)' + match = re.search(spam_pattern, parsed_data['spam-status']) + if match: + score = float(match.group(1)) + if score < SATagLevel: + # Accumulate allowed score (inc negatives?) + hamavg += score + hamcount += 1 + else: + spamavg += score + spamqueuedcount += 1 + #spamassasin rejects if parsed_data.get('spam-status') is not None and isinstance(parsed_data['spam-status'], str): if parsed_data['spam-status'].lower().startswith('yes'): #Extract other parameters from this string # example: Yes, score=10.3 required=4.0 autolearn=disable - spam_pattern = r'score=([\d.]+)\s+required=([\d.]+)' + spam_pattern = r'score=(-?\d+\.\d+) required=(-?\d+\.\d+)' match = re.search(spam_pattern, parsed_data['spam-status']) if match: score = float(match.group(1)) required = float(match.group(2)) #print(f"{parsed_data['spam-status']} / {score} {required}") + rejectspamavg += score + rejectspamcount += 1 if score >= SARejectLevel: columnCounts_2d[hour][DelSpam] += 1 columnCounts_2d[ColTotals][DelSpam] += 1 elif score >= required: columnCounts_2d[hour][QuedSpam] += 1 columnCounts_2d[ColTotals][QuedSpam] += 1 + #Local send elif DomainName in parsed_data['sendurl']: columnCounts_2d[hour][Local] += 1 @@ -778,8 +858,8 @@ if __name__ == "__main__": #my $logemail = $log_items[4]; if DMARCDomain in parsed_data['from-email']: #(index($DMARC_Report_emails,$logemail)>=0) or #$localsendtotal++; - #$DMARCSendCount++; - localflag = 1; + DMARCSendCount += 1 + #localflag = 1; else: # ignore incoming localhost spoofs if not 'msg denied before queued' in parsed_data['error-msg']: @@ -843,12 +923,24 @@ if __name__ == "__main__": columnCounts_2d[ColTotals][PERCENT] = '100%' columnCounts_2d[ColPercent][TOTALS] = '100%' + #other stats + emailperhour = (totalexamined / 24) + if not spamqueuedcount == 0: + spamavg = spamavg / spamqueuedcount + if not rejectspamcount == 0: + rejectspamavg = rejectspamavg / rejectspamcount + if not hamcount == 0: + hamavg = hamavg / hamcount + # Now scan for the other lines in the log of interest found_countries = defaultdict(int) geoip_pattern = re.compile(r".*check_badcountries: GeoIP Country: (.*)") dmarc_pattern = re.compile(r".*dmarc: pass") + helo_pattern = re.compile(r"Accepted connection.*?from (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) \/ ([\w.-]+)") total_countries = 0 DMARCOkCount = 0 + totalinternalsmtpsessions = 0 + totalexternalsmtpsessions = 0 i = 0 j = 0 @@ -860,22 +952,35 @@ if __name__ == "__main__": i += 1 if isThonny: print_progress_bar(i, log_len, prefix='Scanning for sub tables:', suffix='Complete', length=50) + + # Match initial connection message + match = helo_pattern.match(data[1]) + if match: + ip = match.group(1) + fqdn = match.group(2) + if is_private_ip(ip): + totalinternalsmtpsessions += 1 + else: + totalexternalsmtpsessions += 1 + continue + #Pull out Geoip countries for analysis table if "check_badcountries: GeoIP Country" in data: j += 1 - match = geoip_pattern.match(data[1]) - if match: - country = match.group(1) - found_countries[country] += 1 - total_countries += 1 - continue + match = geoip_pattern.match(data[1]) + if match: + country = match.group(1) + found_countries[country] += 1 + total_countries += 1 + continue + #Pull out DMARC approvals match = dmarc_pattern.match(data[1]) if match: DMARCOkCount += 1 continue - #print(f"J:{j} I:{i}") + #Now apply the results to the chameleon template - main table # Path to the template file template_path = template_dir+'mailstats.html.pt' @@ -894,7 +999,10 @@ if __name__ == "__main__": print(f"Chameleon render Exception {e}") total_html = rendered_html - #Now apply the results to the chameleon template - subservient tables + # Add in the header information + rendered_html = get_heading() + total_html = insert_string_after(total_html,rendered_html, "") + #add in the subservient tables.. #qpsmtd codes qpsmtpd_headers = ["Code",'Count','Percent','Reason'] qpsmtpd_title = 'Qpsmtpd codes league table:'