smecontribs/smeserver-mailstats
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Main table in html

Browse Source
This commit is contained in:
Brian Read 2024-05-29 16:46:58 +01:00
parent 02deabb6af
commit 731233cce1
3 changed files with 269 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,3 +5,4 @@
current.* current.*
*.xz *.xz
current current
*.html

22
mailstats.html.pt Normal file
View File

@ -0,0 +1,22 @@
<html>
<head>
<title>2D Array Display</title>
</head>
<body>
<h1>2D Array Contents</h1>
<table border="1">
<thead>
<tr>
<th>Hour</th>
<th tal:repeat="header column_headers" tal:content="header">Header</th>
</tr>
</thead>
<tbody>
<tr tal:repeat="row array_2d">
<td tal:content="repeat.row.index">Hour</td>
<td tal:repeat="cell row" tal:content="cell">Cell</td>
</tr>
</tbody>
</table>
</body>
</html>

246
root/usr/bin/mailstats.py
View File

@ -7,13 +7,57 @@
# Re-written in python from Mailstats.pl (Perl) to conform to SME11 / Postfix / qpsmtpd log formats # Re-written in python from Mailstats.pl (Perl) to conform to SME11 / Postfix / qpsmtpd log formats
# and html output added # and html output added
# #
# Todo
# 1. Make "yesterday" parameterised
#
import datetime import datetime
import sys import sys
from chameleon import PageTemplateFile,PageTemplate from chameleon import PageTemplateFile,PageTemplate
import pkg_resources import pkg_resources
import re
import ipaddress
Mailstats_version = '1.2' Mailstats_version = '1.2'
# Column numbering
Hour = 0
WebMail = 1
Local = 2
MailMan = 3
Relay = 4
DMARC = 5
Virus = 6
RBLDNS = 7
Geoip = 8
NonConf = 9
RejLoad = 10
DelSpam = 11
QuedSpam = 12
Ham = 13
TOTALS = 14
PERCENT = 15
ColTotals = 24
def is_private_ip(ip):
try:
# Convert string to an IPv4Address object
ip_addr = ipaddress.ip_address(ip)
except ValueError:
return False
# Define private IP ranges
private_ranges = [
ipaddress.ip_network('10.0.0.0/8'),
ipaddress.ip_network('172.16.0.0/12'),
ipaddress.ip_network('192.168.0.0/16'),
]
# Check if the IP address is within any of these ranges
for private_range in private_ranges:
if ip_addr in private_range:
return True
return False
def truncate_microseconds(timestamp): def truncate_microseconds(timestamp):
# Split timestamp into main part and microseconds # Split timestamp into main part and microseconds
main_part, microseconds = timestamp.split('.') main_part, microseconds = timestamp.split('.')
@ -83,22 +127,22 @@ def parse_data(data):
# and mapping: # and mapping:
try: try:
return_dict = { return_dict = {
'id': fields[0] if len(fields) > 0 else None, 'id': fields[0].strip() if len(fields) > 0 else None,
'action': fields[1] if len(fields) > 1 else None, 'action': fields[1].strip() if len(fields) > 1 else None,
'logterse': fields[2] if len(fields) > 2 else None, 'logterse': fields[2].strip() if len(fields) > 2 else None,
'ip': fields[3] if len(fields) > 3 else None, 'ip': fields[3].strip() if len(fields) > 3 else None,
'sendurl': fields[4] if len(fields) > 4 else None, 'sendurl': fields[4].strip() if len(fields) > 4 else None,
'sendurl1': fields[5] if len(fields) > 5 else None, 'sendurl1': fields[5].strip() if len(fields) > 5 else None,
'from-email': fields[6] if len(fields) > 6 else None, 'from-email': fields[6].strip() if len(fields) > 6 else None,
'error-reason': fields[6] if len(fields) > 6 else None, 'error-reason': fields[6].strip() if len(fields) > 6 else None,
'to-email': fields[7] if len(fields) > 7 else None, 'to-email': fields[7].strip() if len(fields) > 7 else None,
'error-plugin': fields[8] if len(fields) > 8 else None, 'error-plugin': fields[8].strip() if len(fields) > 8 else None,
'action1': fields[8] if len(fields) > 8 else None, 'action1': fields[8].strip() if len(fields) > 8 else None,
'error-number' : fields[9] if len(fields) > 9 else None, 'error-number' : fields[9].strip() if len(fields) > 9 else None,
'sender': fields[10] if len(fields) > 10 else None, 'sender': fields[10].strip() if len(fields) > 10 else None,
'error-msg' :fields[10] if len(fields) > 10 else None, 'error-msg' :fields[10].strip() if len(fields) > 10 else None,
'spam-status': fields[11] if len(fields) > 11 else None, 'spam-status': fields[11].strip() if len(fields) > 11 else None,
'error-result': fields[11] if len(fields) > 11 else None, 'error-result': fields[11].strip() if len(fields) > 11 else None,
# Add more fields as necessary # Add more fields as necessary
} }
except: except:
@ -106,6 +150,20 @@ def parse_data(data):
return_dict = {} return_dict = {}
return return_dict return return_dict
def count_entries_by_hour(log_entries):
hourly_counts = defaultdict(int)
for entry in log_entries:
# Extract hour from the timestamp
timestamp = entry['timestamp']
hour = datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d %H')
hourly_counts[hour] += 1
return hourly_counts
def initialize_2d_array(num_hours, column_headers_len):
num_hours += 1
return [[0] * column_headers_len for _ in range(num_hours)]
if __name__ == "__main__": if __name__ == "__main__":
try: try:
chameleon_version = pkg_resources.get_distribution("Chameleon").version chameleon_version = pkg_resources.get_distribution("Chameleon").version
@ -115,19 +173,169 @@ if __name__ == "__main__":
python_version = python_version[:8] python_version = python_version[:8]
current_datetime = datetime.datetime.now() current_datetime = datetime.datetime.now()
formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M") formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M")
yesterday = (datetime.datetime.now() - datetime.timedelta(days=1)).date()
formatted_yesterday = yesterday.strftime("%Y-%m-%d %H:%M")
hello_string = "Mailstats version:"+Mailstats_version+" Chameleon version:"+chameleon_version+" On Python:"+python_version+" at "+formatted_datetime hello_string = "Mailstats version:"+Mailstats_version+" Chameleon version:"+chameleon_version+" On Python:"+python_version+" at "+formatted_datetime
print(hello_string) print(hello_string)
num_hours = 24 # Represents hours from 0 to 23 - adds extra one for column totals
sorted_log_dict = read_and_filter_yesterday_log('/home/brianr/SME11Build/GITFiles/smecontribs/smeserver-mailstats/current.log') sorted_log_dict = read_and_filter_yesterday_log('/home/brianr/SME11Build/GITFiles/smecontribs/smeserver-mailstats/current.log')
#print(sorted_log_dict) columnHeaders = ['Count','WebMail','Local','MailMan','Relay','DMARC','Virus','RBL/DNS','Geoip.','Non.Conf.','Rej.Load','Del.Spam','Qued.Spam?',' Ham','TOTALS','PERCENT']
# dict for each colum identifying plugin that increments count
columnPlugin = [None] * 16
columnPlugin[Hour] = []
columnPlugin[WebMail] = []
columnPlugin[Local] = []
columnPlugin[MailMan] = []
columnPlugin[DMARC] = []
columnPlugin[Virus] = []
columnPlugin[RBLDNS] = []
columnPlugin[Geoip] = []
columnPlugin[NonConf] = []
columnPlugin[RejLoad] = []
columnPlugin[DelSpam] = []
columnPlugin[QuedSpam] = []
columnPlugin[Ham] = []
columnPlugin[TOTALS] = []
columnPlugin[PERCENT] = []
columnHeaders_len = len(columnHeaders)
columnCounts_2d = initialize_2d_array(num_hours, columnHeaders_len)
#From SMEServer DB
DomainName = 'bjsystems.co.uk' # $cdb->get('DomainName')->value;
RHSenabled = True #( $cdb->get('qpsmtpd')->prop('RHSBL') eq 'enabled' );
DNSenabled = True #( $cdb->get('qpsmtpd')->prop('DNSBL') eq 'enabled' );
SARejectLevel = 12 #$cdb->get('spamassassin')->prop('RejectLevel');
SATagLevel = 4 #$cdb->get('spamassassin')->prop('TagLevel');
FetchmailIP = '127.0.0.200'; #Apparent Ip address of fetchmail deliveries
WebmailIP = '127.0.0.1'; #Apparent Ip of Webmail sender
localhost = 'localhost'; #Apparent sender for webmail
FETCHMAIL = 'FETCHMAIL'; #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email
MAILMAN = "bounces"; #sender when mailman sending when orig is localhost
DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything)
DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval
i = 1 i = 1
for timestamp, data in sorted_log_dict.items(): for timestamp, data in sorted_log_dict.items():
if data['action'] == '(deny)': if data['action'] == '(deny)':
error = data['error-plugin'] error = data['error-plugin']
msg = data['error-msg'] msg = data['error-msg']
else: else:
error = "" error = ""
msg = "" msg = ""
print(f"{i}: {timestamp} IP = {data['ip']} Result:{data['action']} {error} {msg}" ) #print(f"{i}: {timestamp} IP = {data['ip']} Result:{data['action']} {error} {msg}" )
i = i + 1 i += 1
# Count of in which hour it falls
#hour = datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d %H')
# Parse the timestamp string into a datetime object
dt = datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S')
hour = dt.hour
# Increment Count in which headings it falls
#Hourly count and column total
columnCounts_2d[hour][Hour] += 1
columnCounts_2d[ColTotals][Hour] += 1
#Row Totals
columnCounts_2d[hour][TOTALS] += 1
#Total totals
columnCounts_2d[ColTotals][TOTALS] += 1
#Queued email
if data['action'] == '(queue)':
columnCounts_2d[hour][Ham] += 1
columnCounts_2d[ColTotals][Ham] += 1
#spamassasin
if data['spam-status'].lower().startswith('yes'):
#Extract other parameters from this string
# example: Yes, score=10.3 required=4.0 autolearn=disable
spam_pattern = r'score=([\d.]+)\s+required=([\d.]+)'
match = re.search(spam_pattern, data['spam-status'])
if match:
score = float(match.group(1))
required = float(match.group(2))
print(f"{data['spam-status']} / {score} {required}")
if score >= SARejectLevel:
columnCounts_2d[hour][DelSpam] += 1
columnCounts_2d[ColTotals][DelSpam] += 1
elif score >= required:
columnCounts_2d[hour][QuedSpam] += 1
columnCounts_2d[ColTotals][QuedSpam] += 1
#Local send
elif DomainName in data['sendurl']:
columnCounts_2d[hour][Local] += 1
columnCounts_2d[ColTotals][Local] += 1
#Relay or webmail
elif not is_private_ip(data['ip']) and is_private_ip(data['sendurl1']) and data['action1'] == 'queued':
#Relay
if data['action1'] == 'queued':
columnCounts_2d[hour][Relay] += 1
columnCounts_2d[ColTotals][Relay] += 1
elif WebmailIP in data['sendurl1'] and not is_private_ip(data['ip']):
#webmail
columnCounts_2d[hour][WebMail] += 1
columnCounts_2d[ColTotals][WebMail] += 1
elif localhost in data['sendurl']:
# but not if it comes from fetchmail
if not FETCHMAIL in data['sendurl1']:
# might still be from mailman here
if MAILMAN in data['sendurl1']:
#$mailmansendcount++;
#$localsendtotal++;
columnCounts_2d[hour][MailMan] += 1
columnCounts_2d[ColTotals][MailMan] += 1
#$counts{$abshour}{$CATMAILMAN}++;
#$localflag = 1;
else:
#Or sent to the DMARC server
#check for email address in $DMARC_Report_emails string
#my $logemail = $log_items[4];
if DMARCDomain in data['from-email']: #(index($DMARC_Report_emails,$logemail)>=0) or
#$localsendtotal++;
#$DMARCSendCount++;
localflag = 1;
else:
# ignore incoming localhost spoofs
if not 'msg denied before queued' in data['error-msg']:
#Webmail
#$localflag = 1;
#$WebMailsendtotal++;
columnCounts_2d[hour][WebMail] += 1
columnCounts_2d[ColTotals][WebMail] += 1
#$WebMailflag = 1;
else:
#$localflag = 1;
#$WebMailsendtotal++;
#$WebMailflag = 1;
columnCounts_2d[hour][WebMail] += 1
columnCounts_2d[ColTotals][WebMail] += 1
#Now increment the column which the plugin name indicates
#Now apply the results to the chameleon template
# Path to the template file
template_path = '/home/brianr/SME11Build/GITFiles/smecontribs/smeserver-mailstats/mailstats.html.pt'
# Load the template
with open(template_path, 'r') as template_file:
template_content = template_file.read()
# Create a Chameleon template instance
template = PageTemplate(template_content)
# Render the template with the 2D array data and column headers
rendered_html = template(array_2d=columnCounts_2d, column_headers=columnHeaders, reporting_date=formatted_yesterday)
# Write the rendered HTML to a file
output_path = '/home/brianr/SME11Build/GITFiles/smecontribs/smeserver-mailstats/mailstats_for_'+formatted_yesterday+'.html'
with open(output_path, 'w') as output_file:
output_file.write(rendered_html)
print(f"Rendered HTML saved to {output_path}")