Sort out switch to pymysql and add SME10 log convert and runmailstats for SME10

root/usr/bin/mailstats-convert-log-sme10-to-sme11.py

@@ -0,0 +1,97 @@
+import os
+import re
+from datetime import datetime, timedelta
+import glob  # Import the glob module
+
+def tai64n_to_datetime(tai64n):
+    """Convert TAI64N formatted timestamp to a datetime object."""
+    if len(tai64n) < 16:
+        raise ValueError(f"Invalid TAI64N timestamp length: {tai64n}")
+
+    high_bits = int(tai64n[:15], 16)
+    low_bits = int(tai64n[15:23], 16)
+
+    seconds_since_epoch = high_bits
+    nanoseconds = low_bits
+    
+    # Create datetime object
+    epoch = datetime(1970, 1, 1)
+    dt = epoch + timedelta(seconds=seconds_since_epoch)
+    dt += timedelta(microseconds=nanoseconds // 1000)
+
+    return dt
+
+def convert_log(file_paths, output_path):
+	host_name = "sme11"
+	total_files = 0
+	total_lines = 0
+
+	# Input file validation
+	for file_path in file_paths:
+		if not os.path.isfile(file_path):
+			print(f"Input file {file_path} does not exist.")
+			return
+	with open(output_path, 'w') as output_file:
+		for file_path in file_paths:
+			print(f"{file_path}")
+			# Determine the process name based on the file being read
+			if "sqpsmtpd" in file_path:
+				process_name = "sqpsmtpd-forkserver"
+			else:
+				process_name = "qpsmtpd-forkserver"
+			
+			with open(file_path, 'r', encoding='latin1') as log_file:
+				total_files += 1
+				try:
+					for line in log_file:
+						total_lines += 1
+						match = re.match(r'@(\w+) (\d+) \((.*?)\) (.*)', line.strip())
+						if match:
+							tai64n_timestamp, pid, context, message = match.groups()
+							try:
+								log_time = tai64n_to_datetime(tai64n_timestamp[1:])  # Ignore '@'
+								formatted_time = log_time.strftime('%b %d %H:%M:%S')
+								
+								# Replace "bjsystems.co.uk" with "thereadclan.me.uk" in the message
+								#message = message.replace("bjsystems.co.uk", "thereadclan.me.uk")
+								
+								# Correctly format the output line
+								formatted_line = f"{formatted_time} {host_name} {process_name}[{pid}]: {pid} ({context}) {message}\n"
+								output_file.write(formatted_line)
+							except Exception as e:
+								with open("error_log.txt", 'a') as error_file:
+									error_file.write(f"Could not convert timestamp {tai64n_timestamp}: {e}\n")
+								print(f"Error logged for timestamp {tai64n_timestamp}.")
+						else:
+							#does not mathc the logterse line, but still needed
+							match = re.match(r'@(\w+) (\d+) (.*)', line.strip())
+							if match:
+								tai64n_timestamp, pid, message = match.groups()
+								try:
+									log_time = tai64n_to_datetime(tai64n_timestamp[1:])  # Ignore '@'
+									formatted_time = log_time.strftime('%b %d %H:%M:%S')
+									# Replace "bjsystems.co.uk" with "thereadclan.me.uk" in the message
+									#message = message.replace("bjsystems.co.uk", "thereadclan.me.uk")
+									# Correctly format the output line
+									formatted_line = f"{formatted_time} {host_name} {process_name}[{pid}]: {pid} {message}\n"
+									output_file.write(formatted_line)
+								except Exception as e:
+									with open("error_log.txt", 'a') as error_file:
+										error_file.write(f"Could not convert timestamp {tai64n_timestamp}: {e}\n")
+									print(f"Error logged for timestamp {tai64n_timestamp}.")
+				except Exception as e:
+					print(f"Error reading file {file_path}: {e}")
+					continue
+	print(f"Processed {total_files} files and {total_lines} lines.")
+# Specify the input and output file paths
+# Use glob to expand file patterns
+input_log_files = (
+    glob.glob("/var/log/qpsmtpd/@*.s") +
+    ["/var/log/qpsmtpd/current", "/var/log/sqpsmtpd/current"] +
+    glob.glob("/var/log/sqpsmtpd/@*.s")  # Adjust the asterisk * as needed
+)
+output_log_file = "output_log.txt"  # Specify your desired output file path
+
+# Convert the log
+convert_log(input_log_files, output_log_file)
+print(f"Log conversion complete. Check the output at: {output_log_file}")

root/usr/bin/mailstats.py

@@ -375,12 +375,9 @@ def create_line_chart(data2d, xLabels, yLabels, save_path='line_chart.html'):
     graph_html = fig.to_html(full_html=False,include_plotlyjs='https://cdn.plot.ly/plotly-latest.min.js')
     return graph_html
 
-
+def save_summaries_to_db(cursor, conn, date_str, hour, parsed_data):
-
-    
-def save_summaries_to_db(date_str, hour, parsed_data):
-
     # Convert parsed_data to JSON string
+    global count_records_to_db
     json_data = json.dumps(parsed_data)
 
     # Insert the record
@@ -388,14 +385,15 @@ def save_summaries_to_db(date_str, hour, parsed_data):
     INSERT INTO SummaryLogs (Date, Hour, logData)
     VALUES (%s, %s, %s)
     """
-    
+   
     try:
         cursor.execute(insert_query, (date_str, hour, json_data))
         conn.commit()
-    except mysql.connector.Error as err:
+        count_records_to_db += 1
+    except pymysql.Error as err:
         print(f"DB Error {date_str} {hour} : {err}")
         conn.rollback()
-        
+                
 def is_running_under_thonny():
     # Check for the 'THONNY_USER_DIR' environment variable
     return 'THONNY_USER_DIR' in os.environ
@@ -839,7 +837,7 @@ def read_html_from_file(filepath):
 	# Need to add in here the contents of the css file at the end of the head section.
 	with open(filepath, 'r', encoding='utf-8') as file:
 		html_contents = file.read()
-	print("reading from html file")
+	print("Reading from html file")
 	# Get Filepath
 	css_path = os.path.dirname(filepath)+"/../css/mailstats.css"
 	# Read in CSS 
@@ -1115,52 +1113,60 @@ if __name__ == "__main__":
 	
 	BadCountries = get_value(ConfigDB,"qpsmtpd","BadCountries")
 	
+	count_records_to_db = 0;
+	
 	# Db save control
 	saveData = get_value(ConfigDB,"mailstats","SaveDataToMySQL","no") == 'yes' or forceDbSave
+
 	if saveData:
-		DBName = "mailstats";
+		DBName = "mailstats"
-		DBHost = get_value(ConfigDB,'mailstats','DBHost',"localhost")
+		DBHost = get_value(ConfigDB, 'mailstats', 'DBHost', "localhost")
-		DBPort = get_value(ConfigDB,'mailstats','DBPort',"3306")
+		DBPort = int(get_value(ConfigDB, 'mailstats', 'DBPort', "3306"))  # Ensure port is an integer
-		DBName = 'mailstats'
 		DBPassw = 'mailstats'
 		DBUser = 'mailstats'
 		UnixSocket = "/var/lib/mysql/mysql.sock" 
-		# see if the DB exists
+
-		# Try to Establish a database connection
+		# Try to establish a database connection
 		try:
-			conn = mysql.connector.connect(
+			conn = pymysql.connect(
 				host=DBHost,
 				user=DBUser,
 				password=DBPassw,
 				database=DBName,
 				port=DBPort,
-				unix_socket=UnixSocket
+				unix_socket=UnixSocket,
+				cursorclass=pymysql.cursors.DictCursor  # Optional: use DictCursor for dict output
 			)
 			cursor = conn.cursor()
-			# Create table if it doesn't exist
+			# Check if the table exists before creating it
-			cursor.execute("""
+			check_table_query = "SHOW TABLES LIKE 'SummaryLogs'"
-			CREATE TABLE IF NOT EXISTS SummaryLogs (
+			cursor.execute(check_table_query)
-				id INT AUTO_INCREMENT PRIMARY KEY,
+			table_exists = cursor.fetchone()
-				Date DATE,
+			if not table_exists:
-				Hour INT,
+				# Create table if it doesn't exist
-				logData TEXT
+				cursor.execute("""
-			)
+				CREATE TABLE IF NOT EXISTS SummaryLogs (
-			""")
+					id INT AUTO_INCREMENT PRIMARY KEY,
-			# and prune the DB here if needed.
+					Date DATE,
+					Hour INT,
+					logData TEXT
+				)
+				""")
 			# Delete existing records for the given date
 			try:
 				delete_query = """
 				DELETE FROM SummaryLogs
 				WHERE Date = %s
 				"""
-				cursor.execute(delete_query, (analysis_date,))   #Don't forget the syntactic sugar of the extra comma to make it a tuple!
+				cursor.execute(delete_query, (analysis_date,))  # Don't forget the extra comma for tuple
 				# Get the number of records deleted
 				rows_deleted = cursor.rowcount
 				if rows_deleted > 0:
 					print(f"Deleted {rows_deleted} rows for {analysis_date} ")
-			except mysql.connector.Error as e:
+			except pymysql.Error as e:
 				print(f"SQL Delete failed ({delete_query}) ({e}) ")
-		except mysql.connector.Error as e:
+
+		except pymysql.Error as e:
 			print(f"Unable to connect to {DBName} on {DBHost} port {DBPort} error ({e}) ")
 			saveData = False
 	
@@ -1269,12 +1275,12 @@ if __name__ == "__main__":
 			#else:
 			#Take out the mailstats email
 			if 'mailstats' in parsed_data['from-email'] and DomainName in parsed_data['from-email']:
-				print(f"{parsed_data}")
+				#print(f"{parsed_data}")
-				#continue
+				continue
 			# Save the data here if necessary
 			if saveData:
-				save_summaries_to_db(anaysis_date_obj.strftime('%Y-%m-%d'),hour,parsed_data)
+				save_summaries_to_db(cursor,conn,anaysis_date_obj.strftime('%Y-%m-%d'),hour,parsed_data)
-			
+
 			#Count the number of emails through each of qpsmtpd, uqpsmtpd and sqpsmtpd
 			# the forkserver column in the log indicates it.
 			if parsed_data['qpsmtpd'].startswith ('qpsmtpd'):
@@ -1632,22 +1638,22 @@ if __name__ == "__main__":
 	rendered_html = get_heading()
 	total_html =  insert_string_after(total_html,rendered_html, "<!---Add in header information here -->")
 
-	#add in the subservient tables..
+	#add in the subservient tables..(remeber they appear in the reverse order of below!)
+
+	#virus codes
+	virus_headers = ["Virus",'Count','Percent']
+	virus_title = 'Viruses found'
+	rendered_html = render_sub_table(virus_title,virus_headers,found_viruses)
+	# Add it to the total 
+	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
 
 	#qpsmtd codes
-	#print(f"{found_qpcodes}")
 	qpsmtpd_headers = ["Reason",'Count','Percent']
 	qpsmtpd_title = 'Qpsmtpd codes league table'
 	rendered_html = render_sub_table(qpsmtpd_title,qpsmtpd_headers,found_qpcodes)
 	# Add it to the total 
 	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
 	
-	#Geoip Country codes
-	geoip_headers  = ['Country','Count','Percent','Rejected?']
-	geoip_title = 'Geoip results'
-	rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list)
-	# Add it to the total 
-	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
 	
 	#Junk mails
 	junk_mail_count_headers = ['Username','Count', 'Percent']
@@ -1657,12 +1663,6 @@ if __name__ == "__main__":
 	# Add it to the total 
 	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
 
-	#virus codes
-	virus_headers = ["Virus",'Count','Percent']
-	virus_title = 'Viruses found'
-	rendered_html = render_sub_table(virus_title,virus_headers,found_viruses)
-	# Add it to the total 
-	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
 	
 	#Recipient counts
 	#print(f"{recipients_found}")
@@ -1672,6 +1672,14 @@ if __name__ == "__main__":
 	# Add it to the total 
 	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
 
+	#Geoip Country codes
+	geoip_headers  = ['Country','Count','Percent','Rejected?']
+	geoip_title = 'Geoip results'
+	rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list)
+	# Add it to the total 
+	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
+
+
 	
 	if saveData:
 		# Close the connection
@@ -1699,6 +1707,7 @@ if __name__ == "__main__":
 	else:
 		text_file_path = ""
 	
+	print(f"Written {count_records_to_db} records to DB")
 	
 	html_content = None
 	text_content = None

root/usr/bin/runmailstatsSME10.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#exec 1> >(logger -t $(basename $0)) 2>&1
+perl /usr/bin/mailstats.pl /var/log/qpsmtpd/\@* /var/log/qpsmtpd/current /var/log/sqpsmtpd/\@* /var/log/sqpsmtpd/current
+# and run new python one - start by copying and decoding log files
+yesterday_date=$(date -d "yesterday" +'%mm %d')
+#cd /var/log/qpsmtpd
+#cat \@*  current >/opt/mailstats/logs/current1 2>/dev/null
+#cd /var/log/sqpsmtpd
+#cat \@*  current >/opt/mailstats/logs/current2 2>/dev/null
+cd /opt/mailstats/logs
+#cat current1 current2  2>/dev/null |  /usr/local/bin/tai64nlocal | grep  "$yesterday_date" > current1.log 
+python3 /usr/bin/mailstats-convert-log-sme10-to-sme11.py
+yesterday_date=$(date -d "yesterday" +'%b %d')
+cat output_log.txt | grep "$yesterday_date" | sort >current.log
+ls -l
+python3 /usr/bin/mailstats.py
+echo "Done"