* Mon Sep 01 2025 Brian Read <brianr@koozali.org> 11.1-4.sme

- More fixes for Journal bytes instead of characters [SME: 13117]

.gitignore

@@ -2,14 +2,7 @@
 *.log
 *spec-20*
 *.tgz
-current.*
 *.xz
-current1
-current2
 *.html
 *.txt
-accounts
-configuration
-domains
-hosts
 *el8*

root/etc/e-smith/db/configuration/defaults/mailstats/status

@@ -0,0 +1 @@
+enabled

root/etc/e-smith/db/configuration/defaults/mailstats/type

@@ -0,0 +1 @@
+report

root/etc/e-smith/templates/etc/httpd/conf/httpd.conf/100mailstats

@@ -1,6 +1,6 @@
 {
 # mailstats
-  my $status = $mailstats{'Status'} || 'disabled';
+  my $status = $mailstats{'status'} || 'disabled';
 
   if ($status eq 'enabled') 
   {

root/usr/bin/mailstats.py

@@ -85,8 +85,8 @@ import argparse
 import tempfile
 #import mysql.connector
 import numpy as np
-import plotly.graph_objects as go
-import plotly.express as px
+#import plotly.graph_objects as go
+#import plotly.express as px
 import colorsys
 import pymysql
 import json
@@ -108,7 +108,7 @@ enable_graphs = True; #This could be a DB entry if required.
 try:
 	import matplotlib.pyplot as plt
 except ImportError:
-	logging.debug("Matplotlib is not installed  - no graphs")
+	logging.warning("Matplotlib is not installed  - no graphs")
 	enable_graphs = False;
 
 Mailstats_version = '1.2'
@@ -150,6 +150,10 @@ PERCENT = TOTALS + 1
 ColTotals = 24
 ColPercent = 25
 
+def strip_ansi_codes(text):
+    ansi_escape = re.compile(r'\x1b\[[0-9;]*m')
+    return ansi_escape.sub('', text)
+
 def replace_bracket_content(input_filename, output_filename):
     import re
 
@@ -238,12 +242,22 @@ def get_logs_from_Journalctl(date='yesterday'):
 		
 		# Retrieve logs within the time range
 		logs = []
+		log_count = 0
+		error_count = 0
 		for entry in j:
-			entry_timestamp = entry.get('__REALTIME_TIMESTAMP', None)
-			entry_microseconds = int(entry_timestamp.timestamp() * 1_000_000)
-			if entry_timestamp and since_microseconds <= entry_microseconds <= until_microseconds:
-				logs.append(entry)
-		
+			try:
+				entry_timestamp = entry.get('__REALTIME_TIMESTAMP', None)
+				entry_microseconds = int(entry_timestamp.timestamp() * 1_000_000)
+				if entry_timestamp and since_microseconds <= entry_microseconds <= until_microseconds:
+					log_count += 1
+					# takeout ASCII Escape sequences from the message
+					entry['MESSAGE'] = strip_ansi_codes(entry['MESSAGE'])
+					logs.append(entry)
+			except Exception as e:
+				logging.warning(f"Error - log line: {log_count} {entry['_PID']} {entry['SYSLOG_IDENTIFIER']} : {e}")
+				error_count += 1
+		if error_count:
+			logging.info(f"Had {error_count} errors on journal import  - probably non character bytes")
 		# Sort logs by __REALTIME_TIMESTAMP in ascending order
 		sorted_logs = sorted(logs, key=lambda x: x.get("__REALTIME_TIMESTAMP", 0))
 		
@@ -251,7 +265,7 @@ def get_logs_from_Journalctl(date='yesterday'):
 
 	except Exception as e:
 		logging.error(f"Unexpected error: {e}")
-		return {}
+	return {}
 
 
 def transform_to_dict(data, keys, iso_date):
@@ -588,32 +602,49 @@ def parse_data(data):
 	# and mapping:
 	try:
 		return_dict = {
-			'sme': fields[0].strip() if len(fields) > 0 else None,
-			'qpsmtpd': fields[1].strip() if len(fields) > 1 else None,
-			'id': fields[2].strip() if len(fields) > 2 else None,
-			'action': fields[3].strip() if len(fields) > 3 else None,       #5
-			'logterse': fields[4].strip() if len(fields) > 4 else None,
-			'ip': fields[5].strip() if len(fields) > 5 else None,
-			'sendurl': fields[6].strip() if len(fields) > 6 else None,     #1
-			'sendurl1': fields[7].strip() if len(fields) > 7 else None,    #2
-			'from-email': fields[8].strip() if len(fields) > 8 else None,  #3
-			'error-reason': fields[8].strip() if len(fields) > 9 else None, #3
-			'to-email': fields[9].strip() if len(fields) > 9 else None,		#4
-			'error-plugin': fields[10].strip() if len(fields) > 10 else None,  #5
-			'action1': fields[10].strip() if len(fields) > 10 else None,       #5
-			'error-number' : fields[11].strip() if len(fields) > 11 else None, #6
-			'sender': fields[12].strip() if len(fields) > 12 else None,      #7
-			'virus': fields[12].strip() if len(fields) > 12 else None,      #7
-			'error-msg' :fields[13].strip() if len(fields) > 13 else None,   #7
-			'spam-status': fields[13].strip() if len(fields) > 13 else None, #8 
-			'error-result': fields[14].strip() if len(fields) > 14 else None,#8
+			'sme': fields[0].strip() if len(fields) > 0 else "",
+			'qpsmtpd': fields[1].strip() if len(fields) > 1 else "",
+			'id': fields[2].strip() if len(fields) > 2 else "",
+			'action': fields[3].strip() if len(fields) > 3 else "",       #5
+			'logterse': fields[4].strip() if len(fields) > 4 else "",
+			'ip': fields[5].strip() if len(fields) > 5 else "",
+			'sendurl': fields[6].strip() if len(fields) > 6 else "",     #1
+			'sendurl1': fields[7].strip() if len(fields) > 7 else "",    #2
+			'from-email': fields[8].strip() if len(fields) > 8 else "",  #3
+			'error-reason': fields[8].strip() if len(fields) > 9 else "", #3
+			'to-email': fields[9].strip() if len(fields) > 9 else "",		#4
+			'error-plugin': fields[10].strip() if len(fields) > 10 else "",  #5
+			'action1': fields[10].strip() if len(fields) > 10 else "",       #5
+			'error-number' : fields[11].strip() if len(fields) > 11 else "", #6
+			'sender': fields[12].strip() if len(fields) > 12 else "",      #7
+			'virus': fields[12].strip() if len(fields) > 12 else "",      #7
+			'error-msg' :fields[13].strip() if len(fields) > 13 else "",   #7
+			'spam-status': fields[13].strip() if len(fields) > 13 else "", #8 
+			'error-result': fields[14].strip() if len(fields) > 14 else "",#8
 			# Add more fields as necessary
 		}
 	except:
 		logging.error(f"error:len:{len(fields)}")
-		return_dict = {}  
+		return_dict = create_empty_return()
 	return return_dict
 
+def safe_strip(lst, index):
+    if 0 <= index < len(lst):
+        value = lst[index]
+        if value is not None:
+            return value.strip()
+    return ""
+
+
+def create_empty_return():
+    # Return dictionary with all keys, values None
+    keys = [
+        'sme', 'qpsmtpd', 'id', 'action', 'logterse', 'ip', 'sendurl', 'sendurl1',
+        'from-email', 'error-reason', 'to-email', 'error-plugin', 'action1', 'error-number',
+        'sender', 'virus', 'error-msg', 'spam-status', 'error-result'
+    ]
+    return {key: "" for key in keys}
+
 # def count_entries_by_hour(log_entries):
 	# hourly_counts = defaultdict(int)
 	# for entry in log_entries:

smeserver-mailstats.spec

@@ -6,7 +6,7 @@ Summary: Daily mail statistics for SME Server
 %define name smeserver-mailstats
 Name: %{name}
 %define version 11.1
-%define release 2
+%define release 4
 Version: %{version}
 Release: %{release}%{?dist}
 License: GPL
@@ -35,11 +35,18 @@ A script that via cron.d e-mails mail statistics to admin on a daily basis.
 See http://www.contribs.org/bugzilla/show_bug.cgi?id=819
 
 %changelog
-* Sun Apr 06 2025 Brian Read <brianr@koozali.org> 11.2-2.sme
-- Add in SM2 panel [SME: ]
+* Mon Sep 01 2025 Brian Read <brianr@koozali.org> 11.1-4.sme
+- More fixes for Journal bytes instead of characters [SME: 13117]
 
-* Mon Dec 30 2024 Brian Read <brianr@koozali.org> 11.2-1.sme
-- Update mailstats.pl to accomodate change in log format for SME11 [SME: 12841]
+* Mon Sep 01 2025 Brian Read <brianr@koozali.org> 11.1-3.sme
+- Sort out ASCII escape codes in return from journalctl API  [SME: 13117]
+- Add in Status enabled t default for mailstats DB [SME: 13118]
+
+* Sun Apr 06 2025 Brian Read <brianr@koozali.org> 11.1-2.sme
+- First build on Koji - and Add in SM2 panel [SME: 13116]
+
+* Mon Dec 30 2024 Brian Read <brianr@koozali.org> 11.1-1.sme
+- Update mailstats.py to accomodate change in log format for SME11 [SME: 12841]
 
 * Fri Jun 07 2024 Brian Read <brianr@koozali.org> 1.1-18.sme
 - Pull in python re-write from SME11 dev [SME: ]
@@ -140,4 +147,4 @@ sed -i "s|__BUILD_DATE_TIME__|$now|" $RPM_BUILD_ROOT/usr/bin/mailstats.py
 /bin/rm -rf $RPM_BUILD_ROOT
 
 %files -f %{name}-%{version}-filelist
-%defattr(-,root,root)
+%defattr(-,root,root)