Properly align links at top of web page and add header text to page

root/opt/mailstats/css/mailstats.css

@@ -33,19 +33,23 @@ tfoot tr {
 
 tbody tr:nth-child(odd) {background-color: #dfdfdf}
 
+div.linksattop {
+  display: flex;
+  justify-content: space-between;
+}
+
 a.prevlink {
-  float: left;
-  width:33.33333%;
-  text-align:left;
+  text-align: left;
 }
-div.divseeinbrowser {
-  float: right;
+
+div.divshowindex {
+  flex-grow: 1;
+  text-align: center;
 }
+
 a.nextlink {
- float: right;
- width:33.33333%;
- text-align:right;
-}​
+  text-align: right;
+}
 
 .cssclass1 {background-color:#ffff99;} 
 .cssclass2 {background-color:lightcoral;}

root/opt/mailstats/templates/mailstats.html.pt

@@ -43,6 +43,8 @@
     <!---Navigation here-->
     <br />
     <h2>${title}</h2>
+    <br />
+    <!---Add in header information here -->
     <br />
 	<table style="border-collapse:collapse;">
 		<thead>

root/usr/bin/mailstats.py

@@ -9,6 +9,8 @@
 # optional arguments:
 #   -h, --help            show this help message and exit
 #   -d DATE, --date DATE  Specify a valid date (yyyy-mm-dd) for the analysis
+#   -ef EMAILFILE, --emailfile EMAILFILE
+#                       Save an html file of the email sent (y/N)
 #
 # Re-written in python from Mailstats.pl (Perl) to conform to SME11 / Postfix / qpsmtpd log formats
 # and html output added
@@ -16,6 +18,9 @@
 # Todo
 # 2  Other stats
 # 3. Extra bits for sub tables
+# 4. Percent char causes sort to fail - look at adding it in the template
+# 5. Chase disparity in counts betweeen old mailstats and this
+# 6. Count emails delivered over 25/587/465
 #
 # Centos7:
 # yum install python3-chameleon --enablerepo=epel
@@ -155,7 +160,6 @@ def is_private_ip(ip):
 		ipaddress.ip_network('172.16.0.0/12'),
 		ipaddress.ip_network('192.168.0.0/16'),
 	]
-
 	# Check if the IP address is within any of these ranges
 	for private_range in private_ranges:
 		if ip_addr in private_range:
@@ -387,12 +391,9 @@ def insert_string_after(original:str, to_insert:str, after:str) -> str:
 	:return: The new string with to_insert inserted after after.
 	"""
 	position = original.find(after)
-	#print(position)
-	
 	if position == -1:
-		# 'after' string is not found in 'original'
+		print(f"insert_string_after:({after}) string is not found in original")
 		return original
-	#print(f"{len(after)}")
 	# Position of the insertion point
 	insert_pos = position + len(after)
 	
@@ -576,6 +577,53 @@ def replace_between(text, start, end, replacement):
     replaced_text = re.sub(pattern, replacement, text, flags=re.DOTALL)
     return replaced_text
     
+def get_heading():
+	#
+	# Needs from anaytsis
+		# SATagLevel - done
+		# SARejectLevel - done
+		# warnnoreject - done
+		# totalexamined - done
+		# emailperhour - done
+		# spamavg  - done
+		# rejectspamavg - done
+		# hamavg - done
+		# DMARCSendCount - done
+		# hamcount - done
+		# DMARCOkCount - deone
+
+	# Clam Version/DB Count/Last DB update
+	clam_output = subprocess.getoutput("freshclam -V")
+	clam_info = f"Clam Version/DB Count/Last DB update: {clam_output}"
+
+	# SpamAssassin Version
+	sa_output = subprocess.getoutput("spamassassin -V")
+	sa_info = f"SpamAssassin Version: {sa_output}"
+
+	# Tag level and Reject level
+	tag_reject_info = f"Tag level: {SATagLevel}; Reject level: {SARejectLevel} {warnnoreject}"
+
+	# SMTP connection stats
+	smtp_stats = f"All External SMTP connections accepted: {totalexternalsmtpsessions}\n"\
+				 f"All Internal SMTP connections accepted: {totalinternalsmtpsessions}\n"\
+				 f"Emails per hour: {emailperhour:.1f}/hr\n"\
+				 f"Average spam score (accepted): {spamavg or 0:.2f}\n"\
+				 f"Average spam score (rejected): {rejectspamavg or 0:.2f}\n"\
+				 f"Average ham score: {hamavg or 0:.2f}\n"\
+				 f"Number of DMARC reporting emails sent: {DMARCSendCount or 0} (not shown on table)"
+
+	# DMARC approved emails
+	dmarc_info = ""
+	if hamcount != 0:
+		dmarc_ok_percentage = DMARCOkCount * 100 / hamcount
+		dmarc_info = f"Number of emails approved through DMARC: {DMARCOkCount or 0} ({dmarc_ok_percentage:.2f}% of Ham count)"
+
+	# Accumulate all strings
+	header_str = "\n".join([clam_info, sa_info, tag_reject_info, smtp_stats, dmarc_info])
+	# switch newlines to <br />
+	header_str = header_str.replace("\n","<br />")
+	return header_str
+
 if __name__ == "__main__":
 	try:
 		chameleon_version = pkg_resources.get_distribution("Chameleon").version
@@ -620,6 +668,10 @@ if __name__ == "__main__":
 
 	SARejectLevel = int(get_value(ConfigDB, "spamassassin", "RejectLevel","12")) 	#12  #$cdb->get('spamassassin')->prop('RejectLevel');
 	SATagLevel = int(get_value(ConfigDB, "spamassassin", "TagLevel","4")) 			#4  #$cdb->get('spamassassin')->prop('TagLevel');
+	if SARejectLevel == 0:
+		warnnoreject = "(*Warning* 0 = no reject)"
+	else:
+		warnnoreject = ""
 
 	EmailAddress = get_value(ConfigDB,"mailstats","Email","admin@"+DomainName)
 	if '@' not in EmailAddress:
@@ -699,10 +751,19 @@ if __name__ == "__main__":
 	# Initial call to print the progress bar
 	#unless none to show
 	if sorted_len > 0:
+		spamavg = 0;
+		spamqueuedcount = 0
+		hamcount = 0
+		hamavg = 0
+		rejectspamcount = 0
+		rejectspamavg = 0
+		DMARCSendCount = 0
+		totalexamined = 0
 		if isThonny:
 			print_progress_bar(0, sorted_len, prefix='Progress:', suffix='Complete', length=50)
 		for timestamp, data in sorted_log_dict.items():
 			i += 1
+			totalexamined += 1
 			if isThonny:
 				print_progress_bar(i, sorted_len, prefix='Scanning for main table:', suffix='Complete', length=50)
 			#print(f"{i*100/len}%")
@@ -728,23 +789,42 @@ if __name__ == "__main__":
 			if parsed_data['action'] == '(queue)':
 				columnCounts_2d[hour][Ham] += 1
 				columnCounts_2d[ColTotals][Ham] += 1
-			#spamassasin
+				# spamassassin not rejected
+				if parsed_data.get('spam-status') is not None and isinstance(parsed_data['spam-status'], str):
+					if parsed_data['spam-status'].lower().startswith('no'):
+						#Extract other parameters from this string
+						# example: No, score=-3.9 
+						spam_pattern = r'score=(-?\d+\.\d+) required=(-?\d+\.\d+)'
+						match = re.search(spam_pattern, parsed_data['spam-status'])
+						if match:
+							score = float(match.group(1))
+							if score < SATagLevel:
+								# Accumulate allowed score (inc negatives?)
+								hamavg += score
+								hamcount += 1
+							else:
+								spamavg += score
+								spamqueuedcount += 1
+			#spamassasin rejects
 			if parsed_data.get('spam-status') is not None and isinstance(parsed_data['spam-status'], str):
 				if parsed_data['spam-status'].lower().startswith('yes'):
 					#Extract other parameters from this string
 					# example: Yes, score=10.3 required=4.0 autolearn=disable
-					spam_pattern = r'score=([\d.]+)\s+required=([\d.]+)'
+					spam_pattern = r'score=(-?\d+\.\d+) required=(-?\d+\.\d+)'
 					match = re.search(spam_pattern, parsed_data['spam-status'])
 					if match:
 						score = float(match.group(1))
 						required = float(match.group(2))
 						#print(f"{parsed_data['spam-status']} / {score} {required}")
+						rejectspamavg += score
+						rejectspamcount += 1
 						if score >= SARejectLevel:
 							columnCounts_2d[hour][DelSpam] += 1
 							columnCounts_2d[ColTotals][DelSpam] += 1
 						elif score >= required:
 							columnCounts_2d[hour][QuedSpam] += 1
 							columnCounts_2d[ColTotals][QuedSpam] += 1
+				
 				#Local send
 				elif DomainName in parsed_data['sendurl']:
 					columnCounts_2d[hour][Local] += 1
@@ -778,8 +858,8 @@ if __name__ == "__main__":
 						#my $logemail = $log_items[4];
 						if DMARCDomain in parsed_data['from-email']: #(index($DMARC_Report_emails,$logemail)>=0) or 
 							#$localsendtotal++;
-							#$DMARCSendCount++;
-							localflag = 1;
+							DMARCSendCount += 1
+							#localflag = 1;
 						else:
 							# ignore incoming localhost spoofs
 							if not 'msg denied before queued' in parsed_data['error-msg']:
@@ -843,12 +923,24 @@ if __name__ == "__main__":
 	columnCounts_2d[ColTotals][PERCENT] = '100%'
 	columnCounts_2d[ColPercent][TOTALS] = '100%'
 	
+	#other stats
+	emailperhour = (totalexamined / 24)
+	if not spamqueuedcount == 0:
+		spamavg = spamavg / spamqueuedcount
+	if not rejectspamcount == 0:
+		rejectspamavg = rejectspamavg / rejectspamcount
+	if not hamcount == 0:
+		hamavg = hamavg / hamcount
+
 	# Now scan for the other lines in the log of interest
 	found_countries = defaultdict(int)
 	geoip_pattern = re.compile(r".*check_badcountries: GeoIP Country: (.*)")
 	dmarc_pattern = re.compile(r".*dmarc: pass")
+	helo_pattern = re.compile(r"Accepted connection.*?from (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) \/ ([\w.-]+)")
 	total_countries = 0
 	DMARCOkCount = 0
+	totalinternalsmtpsessions = 0
+	totalexternalsmtpsessions = 0
 
 	i = 0
 	j = 0
@@ -860,6 +952,18 @@ if __name__ == "__main__":
 			i += 1
 			if isThonny:
 				print_progress_bar(i, log_len, prefix='Scanning for sub tables:', suffix='Complete', length=50)
+				
+			# Match initial connection message
+			match = helo_pattern.match(data[1])
+			if match:
+				ip  = match.group(1)
+				fqdn = match.group(2)
+				if is_private_ip(ip):
+					totalinternalsmtpsessions += 1
+				else:
+					totalexternalsmtpsessions += 1
+			continue
+					
 			#Pull out Geoip countries for analysis table
 			if "check_badcountries: GeoIP Country" in data:
 				j += 1
@@ -869,13 +973,14 @@ if __name__ == "__main__":
 					found_countries[country] += 1
 					total_countries += 1
 					continue
+					
 			#Pull out DMARC approvals
 			match = dmarc_pattern.match(data[1])
 			if match:
 				DMARCOkCount += 1
 				continue
 			
-	#print(f"J:{j} I:{i}")
+
 	#Now apply the results to the chameleon template - main table
 	# Path to the template file
 	template_path = template_dir+'mailstats.html.pt'
@@ -894,7 +999,10 @@ if __name__ == "__main__":
 		print(f"Chameleon render Exception {e}")
 		
 	total_html = rendered_html
-	#Now apply the results to the chameleon template - subservient tables
+	# Add in the header information
+	rendered_html = get_heading()
+	total_html =  insert_string_after(total_html,rendered_html, "<!---Add in header information here -->")
+	#add in the subservient tables..
 	#qpsmtd codes
 	qpsmtpd_headers = ["Code",'Count','Percent','Reason']
 	qpsmtpd_title = 'Qpsmtpd codes league table:'