From 9ebe02b80e6e052295feb1e408a7db1e1470a554 Mon Sep 17 00:00:00 2001
From: Brian Read <brianr@koozali.org>
Date: Tue, 31 Dec 2024 13:34:23 +0000
Subject: [PATCH] More fixes for log format and make perl and python run from
 same datafile

---
 root/usr/bin/mailstats.pl    |  8 ++---
 root/usr/bin/mailstats.py    | 62 ++++++++++++++++++++----------------
 root/usr/bin/runmailstats.sh |  7 ++--
 3 files changed, 41 insertions(+), 36 deletions(-)
 mode change 100644 => 100755 root/usr/bin/runmailstats.sh

diff --git a/root/usr/bin/mailstats.pl b/root/usr/bin/mailstats.pl
index 76dbe5f..093fc76 100644
--- a/root/usr/bin/mailstats.pl
+++ b/root/usr/bin/mailstats.pl
@@ -535,14 +535,14 @@ LINE: while (<>) {
     #my $abshour = floor( $abstime / 3600 );    # Hours since the epoch
     
     # Create a timestamp for the previous hour
-	my $previous_hour_epoch = $current_epoch - 3600;  # Subtract 3600 seconds (1 hour)
+	my $previous_hour_epoch = $current_epoch; # - 3600;  # Subtract 3600 seconds (1 hour)
 
 	# Convert epoch time to local time
 	my ($sec, $min, $hour) = localtime($previous_hour_epoch);
 	#print $sec." ".$min." ".$hour."\n";
-	$hour = ($hour==23)?0:$hour;
-	my $abshour = $hour+1;
-	#print "Abs:".$abshour."\n";
+	#$hour = ($hour==23)?0:$hour;
+	my $abshour = $hour;
+	#print "Abs:".$abshour." ".strftime('%Y-%m-%dT%H:%M:%SZ',gmtime($previous_hour_epoch))."\n";
 
 
     my ($timestamp_part, $log_part) = split('`',$_,2);  #bjr 0.6.12
diff --git a/root/usr/bin/mailstats.py b/root/usr/bin/mailstats.py
index 9b9b72c..43b7307 100644
--- a/root/usr/bin/mailstats.py
+++ b/root/usr/bin/mailstats.py
@@ -565,7 +565,7 @@ def parse_data(data):
 	fields2 = parts[1].split('\t') if len(parts) > 1 else []
 	# then merge them
 	fields = fields1 + fields2
-#	if fields[4] == 'localhost':
+#	if fields[4] == ''local'host':
 #		i = 0
 #		print(f"len:{len(fields)}")
 #		for part in fields:
@@ -575,37 +575,39 @@ def parse_data(data):
 	# and mapping:
 	try:
 		return_dict = {
-			'id': fields[0].strip() if len(fields) > 0 else None,
-			'action': fields[1].strip() if len(fields) > 1 else None,
-			'logterse': fields[2].strip() if len(fields) > 2 else None,
-			'ip': fields[3].strip() if len(fields) > 3 else None,
-			'sendurl': fields[4].strip() if len(fields) > 4 else None,     #1
-			'sendurl1': fields[5].strip() if len(fields) > 5 else None,    #2
-			'from-email': fields[6].strip() if len(fields) > 6 else None,  #3
-			'error-reason': fields[6].strip() if len(fields) > 6 else None, #3
-			'to-email': fields[7].strip() if len(fields) > 7 else None,		#4
-			'error-plugin': fields[8].strip() if len(fields) > 8 else None,  #5
-			'action1': fields[8].strip() if len(fields) > 8 else None,       #5
-			'error-number' : fields[9].strip() if len(fields) > 9 else None, #6
-			'sender': fields[10].strip() if len(fields) > 10 else None,      #7
-			'error-msg' :fields[10].strip() if len(fields) > 10 else None,   #7
-			'spam-status': fields[11].strip() if len(fields) > 11 else None, #8 
-			'error-result': fields[11].strip() if len(fields) > 11 else None,#8
+			'sme': fields[0].strip() if len(fields) > 0 else None,
+			'qpsmtpd': fields[1].strip() if len(fields) > 1 else None,
+			'id': fields[2].strip() if len(fields) > 2 else None,
+			'action': fields[3].strip() if len(fields) > 3 else None,
+			'logterse': fields[4].strip() if len(fields) > 4 else None,
+			'ip': fields[5].strip() if len(fields) > 5 else None,
+			'sendurl': fields[6].strip() if len(fields) > 6 else None,     #1
+			'sendurl1': fields[7].strip() if len(fields) > 7 else None,    #2
+			'from-email': fields[8].strip() if len(fields) > 8 else None,  #3
+			'error-reason': fields[8].strip() if len(fields) > 8 else None, #3
+			'to-email': fields[9].strip() if len(fields) > 9 else None,		#4
+			'error-plugin': fields[10].strip() if len(fields) > 10 else None,  #5
+			'action1': fields[10].strip() if len(fields) > 10 else None,       #5
+			'error-number' : fields[11].strip() if len(fields) > 11 else None, #6
+			'sender': fields[12].strip() if len(fields) > 12 else None,      #7
+			'error-msg' :fields[12].strip() if len(fields) > 12 else None,   #7
+			'spam-status': fields[13].strip() if len(fields) > 13 else None, #8 
+			'error-result': fields[13].strip() if len(fields) > 13 else None,#8
 			# Add more fields as necessary
 		}
 	except:
-		#print(f"error:len:{len(fields)}")
+		print(f"error:len:{len(fields)}")
 		return_dict = {}  
 	return return_dict
 
-def count_entries_by_hour(log_entries):
-	hourly_counts = defaultdict(int)
-	for entry in log_entries:
-		# Extract hour from the timestamp
-		timestamp = entry['timestamp']
-		hour = datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d %H')
-		hourly_counts[hour] += 1
-	return hourly_counts
+# def count_entries_by_hour(log_entries):
+	# hourly_counts = defaultdict(int)
+	# for entry in log_entries:
+		# # Extract hour from the timestamp
+		# timestamp = entry['timestamp']
+		# hour = datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d %H')
+		# hourly_counts[hour] += 1
+	# return hourly_counts
 
 def initialize_2d_array(num_hours, column_headers_len,reporting_date):
 	num_hours += 1  # Adjust for the zeroth hour
@@ -947,7 +949,7 @@ if __name__ == "__main__":
 		datetime.strptime(analysis_date, '%Y-%m-%d')
 	except ValueError:
 		print("Specify a valid date (yyyy-mm-dd) for the analysis")
-		quit()
+		(quit)()
 
 	anaysis_date_obj = datetime.strptime(analysis_date, '%Y-%m-%d')
 	noemailfile = args.emailfile.lower()  == 'n'
@@ -1133,8 +1135,10 @@ if __name__ == "__main__":
 			# Parse the timestamp string into a datetime object
 			dt = timestamp
 			hour = dt.hour
+			#print(f"Abs:{hour} {timestamp}")
 			# parse the data
 			parsed_data = parse_data(data)
+			#print(f"{parsed_data}")
 			#Take out the mailstats email
 			if 'mailstats' in parsed_data['from-email'] and DomainName in parsed_data['from-email']:
 				continue
@@ -1150,9 +1154,11 @@ if __name__ == "__main__":
 			#Total totals
 			columnCounts_2d[ColTotals][TOTALS] += 1
 			
-			# first spot the fetchmail and local deliveries.
+			# first spot the fetchmail and 'local' deliveries.
 			#Local send
+			#print(f"{DomainName} {parsed_data['sendurl']}")
 			if DomainName in parsed_data['sendurl']:
+				#print(f"{DomainName} {parsed_data['sendurl']}")
 				columnCounts_2d[hour][Local] += 1
 				columnCounts_2d[ColTotals][Local] += 1
 			#Relay or webmail
diff --git a/root/usr/bin/runmailstats.sh b/root/usr/bin/runmailstats.sh
old mode 100644
new mode 100755
index ce695f0..3370883
--- a/root/usr/bin/runmailstats.sh
+++ b/root/usr/bin/runmailstats.sh
@@ -1,10 +1,7 @@
 #!/bin/bash
 #exec 1> >(logger -t $(basename $0)) 2>&1
-perl /usr/bin/mailstats.pl /var/log/qpsmtpd/*.log  /var/log/sqpsmtpd/*.log /var/log/uqpsmtpd/*.log \ 
-		/var/log/qpsmtpd/qpsmtpd.log-???????? /var/log/sqpsmtpd/qpsmtpd.log-???????? /var/log/qpsmtpd/uqpsmtpd.log-????????
-# and run new python one - start by copying and decoding log files
 yesterday_date=$(date -d "yesterday" +'%b %d')
-echo $yesterday_date
+echo "Processing for:"$yesterday_date
 cd /var/log/qpsmtpd
 cat *.log  qpsmtpd.log-???????? >/opt/mailstats/logs/current1 2>/dev/null
 cd /var/log/sqpsmtpd
@@ -14,5 +11,7 @@ cat *.log  uqpsmtpd.log-???????? >/opt/mailstats/logs/current3 2>/dev/null
 cd /opt/mailstats/logs
 cat current1 current2 current3  2>/dev/null | grep  "$yesterday_date" > current.log 
 ls -l
+perl /usr/bin/mailstats.pl /opt/mailstats/logs/current.log
+# and run new python one - start by copying and decoding log files
 python3 /usr/bin/mailstats.py
 echo "Done"
\ No newline at end of file