From 2d54c4f7f5844172d44f36a7f98404a187629f87 Mon Sep 17 00:00:00 2001 From: Brian Read Date: Mon, 13 Jan 2025 18:50:30 +0000 Subject: [PATCH] Sort out switch to pymysql and add SME10 log convert and runmailstats for SME10 --- .../mailstats-convert-log-sme10-to-sme11.py | 97 ++++++++++++++++ root/usr/bin/mailstats.py | 105 ++++++++++-------- root/usr/bin/runmailstatsSME10.sh | 17 +++ 3 files changed, 171 insertions(+), 48 deletions(-) create mode 100644 root/usr/bin/mailstats-convert-log-sme10-to-sme11.py create mode 100755 root/usr/bin/runmailstatsSME10.sh diff --git a/root/usr/bin/mailstats-convert-log-sme10-to-sme11.py b/root/usr/bin/mailstats-convert-log-sme10-to-sme11.py new file mode 100644 index 0000000..bffb11b --- /dev/null +++ b/root/usr/bin/mailstats-convert-log-sme10-to-sme11.py @@ -0,0 +1,97 @@ +import os +import re +from datetime import datetime, timedelta +import glob # Import the glob module + +def tai64n_to_datetime(tai64n): + """Convert TAI64N formatted timestamp to a datetime object.""" + if len(tai64n) < 16: + raise ValueError(f"Invalid TAI64N timestamp length: {tai64n}") + + high_bits = int(tai64n[:15], 16) + low_bits = int(tai64n[15:23], 16) + + seconds_since_epoch = high_bits + nanoseconds = low_bits + + # Create datetime object + epoch = datetime(1970, 1, 1) + dt = epoch + timedelta(seconds=seconds_since_epoch) + dt += timedelta(microseconds=nanoseconds // 1000) + + return dt + +def convert_log(file_paths, output_path): + host_name = "sme11" + total_files = 0 + total_lines = 0 + + # Input file validation + for file_path in file_paths: + if not os.path.isfile(file_path): + print(f"Input file {file_path} does not exist.") + return + with open(output_path, 'w') as output_file: + for file_path in file_paths: + print(f"{file_path}") + # Determine the process name based on the file being read + if "sqpsmtpd" in file_path: + process_name = "sqpsmtpd-forkserver" + else: + process_name = "qpsmtpd-forkserver" + + with open(file_path, 'r', encoding='latin1') as log_file: + total_files += 1 + try: + for line in log_file: + total_lines += 1 + match = re.match(r'@(\w+) (\d+) \((.*?)\) (.*)', line.strip()) + if match: + tai64n_timestamp, pid, context, message = match.groups() + try: + log_time = tai64n_to_datetime(tai64n_timestamp[1:]) # Ignore '@' + formatted_time = log_time.strftime('%b %d %H:%M:%S') + + # Replace "bjsystems.co.uk" with "thereadclan.me.uk" in the message + #message = message.replace("bjsystems.co.uk", "thereadclan.me.uk") + + # Correctly format the output line + formatted_line = f"{formatted_time} {host_name} {process_name}[{pid}]: {pid} ({context}) {message}\n" + output_file.write(formatted_line) + except Exception as e: + with open("error_log.txt", 'a') as error_file: + error_file.write(f"Could not convert timestamp {tai64n_timestamp}: {e}\n") + print(f"Error logged for timestamp {tai64n_timestamp}.") + else: + #does not mathc the logterse line, but still needed + match = re.match(r'@(\w+) (\d+) (.*)', line.strip()) + if match: + tai64n_timestamp, pid, message = match.groups() + try: + log_time = tai64n_to_datetime(tai64n_timestamp[1:]) # Ignore '@' + formatted_time = log_time.strftime('%b %d %H:%M:%S') + # Replace "bjsystems.co.uk" with "thereadclan.me.uk" in the message + #message = message.replace("bjsystems.co.uk", "thereadclan.me.uk") + # Correctly format the output line + formatted_line = f"{formatted_time} {host_name} {process_name}[{pid}]: {pid} {message}\n" + output_file.write(formatted_line) + except Exception as e: + with open("error_log.txt", 'a') as error_file: + error_file.write(f"Could not convert timestamp {tai64n_timestamp}: {e}\n") + print(f"Error logged for timestamp {tai64n_timestamp}.") + except Exception as e: + print(f"Error reading file {file_path}: {e}") + continue + print(f"Processed {total_files} files and {total_lines} lines.") +# Specify the input and output file paths +# Use glob to expand file patterns +input_log_files = ( + glob.glob("/var/log/qpsmtpd/@*.s") + + ["/var/log/qpsmtpd/current", "/var/log/sqpsmtpd/current"] + + glob.glob("/var/log/sqpsmtpd/@*.s") # Adjust the asterisk * as needed +) +output_log_file = "output_log.txt" # Specify your desired output file path + +# Convert the log +convert_log(input_log_files, output_log_file) +print(f"Log conversion complete. Check the output at: {output_log_file}") \ No newline at end of file diff --git a/root/usr/bin/mailstats.py b/root/usr/bin/mailstats.py index 0d1adc7..8f59dd1 100644 --- a/root/usr/bin/mailstats.py +++ b/root/usr/bin/mailstats.py @@ -375,12 +375,9 @@ def create_line_chart(data2d, xLabels, yLabels, save_path='line_chart.html'): graph_html = fig.to_html(full_html=False,include_plotlyjs='https://cdn.plot.ly/plotly-latest.min.js') return graph_html - - - -def save_summaries_to_db(date_str, hour, parsed_data): - +def save_summaries_to_db(cursor, conn, date_str, hour, parsed_data): # Convert parsed_data to JSON string + global count_records_to_db json_data = json.dumps(parsed_data) # Insert the record @@ -388,14 +385,15 @@ def save_summaries_to_db(date_str, hour, parsed_data): INSERT INTO SummaryLogs (Date, Hour, logData) VALUES (%s, %s, %s) """ - + try: cursor.execute(insert_query, (date_str, hour, json_data)) conn.commit() - except mysql.connector.Error as err: + count_records_to_db += 1 + except pymysql.Error as err: print(f"DB Error {date_str} {hour} : {err}") conn.rollback() - + def is_running_under_thonny(): # Check for the 'THONNY_USER_DIR' environment variable return 'THONNY_USER_DIR' in os.environ @@ -839,7 +837,7 @@ def read_html_from_file(filepath): # Need to add in here the contents of the css file at the end of the head section. with open(filepath, 'r', encoding='utf-8') as file: html_contents = file.read() - print("reading from html file") + print("Reading from html file") # Get Filepath css_path = os.path.dirname(filepath)+"/../css/mailstats.css" # Read in CSS @@ -1115,52 +1113,60 @@ if __name__ == "__main__": BadCountries = get_value(ConfigDB,"qpsmtpd","BadCountries") + count_records_to_db = 0; + # Db save control saveData = get_value(ConfigDB,"mailstats","SaveDataToMySQL","no") == 'yes' or forceDbSave + if saveData: - DBName = "mailstats"; - DBHost = get_value(ConfigDB,'mailstats','DBHost',"localhost") - DBPort = get_value(ConfigDB,'mailstats','DBPort',"3306") - DBName = 'mailstats' + DBName = "mailstats" + DBHost = get_value(ConfigDB, 'mailstats', 'DBHost', "localhost") + DBPort = int(get_value(ConfigDB, 'mailstats', 'DBPort', "3306")) # Ensure port is an integer DBPassw = 'mailstats' DBUser = 'mailstats' UnixSocket = "/var/lib/mysql/mysql.sock" - # see if the DB exists - # Try to Establish a database connection + + # Try to establish a database connection try: - conn = mysql.connector.connect( + conn = pymysql.connect( host=DBHost, user=DBUser, password=DBPassw, database=DBName, port=DBPort, - unix_socket=UnixSocket + unix_socket=UnixSocket, + cursorclass=pymysql.cursors.DictCursor # Optional: use DictCursor for dict output ) cursor = conn.cursor() - # Create table if it doesn't exist - cursor.execute(""" - CREATE TABLE IF NOT EXISTS SummaryLogs ( - id INT AUTO_INCREMENT PRIMARY KEY, - Date DATE, - Hour INT, - logData TEXT - ) - """) - # and prune the DB here if needed. + # Check if the table exists before creating it + check_table_query = "SHOW TABLES LIKE 'SummaryLogs'" + cursor.execute(check_table_query) + table_exists = cursor.fetchone() + if not table_exists: + # Create table if it doesn't exist + cursor.execute(""" + CREATE TABLE IF NOT EXISTS SummaryLogs ( + id INT AUTO_INCREMENT PRIMARY KEY, + Date DATE, + Hour INT, + logData TEXT + ) + """) # Delete existing records for the given date try: delete_query = """ DELETE FROM SummaryLogs WHERE Date = %s """ - cursor.execute(delete_query, (analysis_date,)) #Don't forget the syntactic sugar of the extra comma to make it a tuple! + cursor.execute(delete_query, (analysis_date,)) # Don't forget the extra comma for tuple # Get the number of records deleted rows_deleted = cursor.rowcount if rows_deleted > 0: print(f"Deleted {rows_deleted} rows for {analysis_date} ") - except mysql.connector.Error as e: + except pymysql.Error as e: print(f"SQL Delete failed ({delete_query}) ({e}) ") - except mysql.connector.Error as e: + + except pymysql.Error as e: print(f"Unable to connect to {DBName} on {DBHost} port {DBPort} error ({e}) ") saveData = False @@ -1269,12 +1275,12 @@ if __name__ == "__main__": #else: #Take out the mailstats email if 'mailstats' in parsed_data['from-email'] and DomainName in parsed_data['from-email']: - print(f"{parsed_data}") - #continue + #print(f"{parsed_data}") + continue # Save the data here if necessary if saveData: - save_summaries_to_db(anaysis_date_obj.strftime('%Y-%m-%d'),hour,parsed_data) - + save_summaries_to_db(cursor,conn,anaysis_date_obj.strftime('%Y-%m-%d'),hour,parsed_data) + #Count the number of emails through each of qpsmtpd, uqpsmtpd and sqpsmtpd # the forkserver column in the log indicates it. if parsed_data['qpsmtpd'].startswith ('qpsmtpd'): @@ -1632,22 +1638,22 @@ if __name__ == "__main__": rendered_html = get_heading() total_html = insert_string_after(total_html,rendered_html, "") - #add in the subservient tables.. + #add in the subservient tables..(remeber they appear in the reverse order of below!) + + #virus codes + virus_headers = ["Virus",'Count','Percent'] + virus_title = 'Viruses found' + rendered_html = render_sub_table(virus_title,virus_headers,found_viruses) + # Add it to the total + total_html = insert_string_after(total_html,rendered_html, "") #qpsmtd codes - #print(f"{found_qpcodes}") qpsmtpd_headers = ["Reason",'Count','Percent'] qpsmtpd_title = 'Qpsmtpd codes league table' rendered_html = render_sub_table(qpsmtpd_title,qpsmtpd_headers,found_qpcodes) # Add it to the total total_html = insert_string_after(total_html,rendered_html, "") - #Geoip Country codes - geoip_headers = ['Country','Count','Percent','Rejected?'] - geoip_title = 'Geoip results' - rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list) - # Add it to the total - total_html = insert_string_after(total_html,rendered_html, "") #Junk mails junk_mail_count_headers = ['Username','Count', 'Percent'] @@ -1657,12 +1663,6 @@ if __name__ == "__main__": # Add it to the total total_html = insert_string_after(total_html,rendered_html, "") - #virus codes - virus_headers = ["Virus",'Count','Percent'] - virus_title = 'Viruses found' - rendered_html = render_sub_table(virus_title,virus_headers,found_viruses) - # Add it to the total - total_html = insert_string_after(total_html,rendered_html, "") #Recipient counts #print(f"{recipients_found}") @@ -1672,6 +1672,14 @@ if __name__ == "__main__": # Add it to the total total_html = insert_string_after(total_html,rendered_html, "") + #Geoip Country codes + geoip_headers = ['Country','Count','Percent','Rejected?'] + geoip_title = 'Geoip results' + rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list) + # Add it to the total + total_html = insert_string_after(total_html,rendered_html, "") + + if saveData: # Close the connection @@ -1699,6 +1707,7 @@ if __name__ == "__main__": else: text_file_path = "" + print(f"Written {count_records_to_db} records to DB") html_content = None text_content = None diff --git a/root/usr/bin/runmailstatsSME10.sh b/root/usr/bin/runmailstatsSME10.sh new file mode 100755 index 0000000..48d6a8a --- /dev/null +++ b/root/usr/bin/runmailstatsSME10.sh @@ -0,0 +1,17 @@ +#!/bin/bash +#exec 1> >(logger -t $(basename $0)) 2>&1 +perl /usr/bin/mailstats.pl /var/log/qpsmtpd/\@* /var/log/qpsmtpd/current /var/log/sqpsmtpd/\@* /var/log/sqpsmtpd/current +# and run new python one - start by copying and decoding log files +yesterday_date=$(date -d "yesterday" +'%mm %d') +#cd /var/log/qpsmtpd +#cat \@* current >/opt/mailstats/logs/current1 2>/dev/null +#cd /var/log/sqpsmtpd +#cat \@* current >/opt/mailstats/logs/current2 2>/dev/null +cd /opt/mailstats/logs +#cat current1 current2 2>/dev/null | /usr/local/bin/tai64nlocal | grep "$yesterday_date" > current1.log +python3 /usr/bin/mailstats-convert-log-sme10-to-sme11.py +yesterday_date=$(date -d "yesterday" +'%b %d') +cat output_log.txt | grep "$yesterday_date" | sort >current.log +ls -l +python3 /usr/bin/mailstats.py +echo "Done" \ No newline at end of file