* Thu Sep 04 2025 Brian Read <brianr@koozali.org> 11.1-6.sme

- Add favicon to mailstats table, summary and detailed pages [SME: 13121]
- Bring DB config reading for mailstats itself inline with php summary and detailed logs - using /etc/mailstats/db.php [SME: 13121]
- Remove DB config fields from the SM2 config panel {sme: 13121]
- Arrange for password to be generated and mailstats user to be set with limited permissions [SME: 13121]

createlinks

@@ -6,8 +6,9 @@ $event = 'smeserver-mailstats-update';
 #see the /etc/systemd/system-preset/49-koozali.preset should be present for systemd integration on all you yum update event
 
 foreach my $file (qw(
-                /etc/systemd/system-preset/49-koozali.preset
-		/etc/e-smith/sql/init/99smeserver-mailstats.sql
+		/etc/systemd/system-preset/49-koozali.preset
+		/etc/mailstats/db.php
+		/etc/e-smith/sql/init/99mailstats
 		/etc/httpd/conf/httpd.conf
 ))
 {
@@ -20,7 +21,7 @@ event_link('systemd-reload', $event, '50');
 #event_link('action', $event, '30');
 #services we need to restart
 safe_symlink('restart', "root/etc/e-smith/events/$event/services2adjust/httpd-e-smith");
+safe_symlink("restart", "root/etc/e-smith/events/$event/services2adjust/mysql.init");;
 #and Server Mmanager panel link
 #panel_link('somefunction', 'manager');
-
-templates2events("/etc/e-smith/sql/init/99smeserver-mailstats.sql", "post-upgrade");
+#templates2events("/etc/e-smith/sql/init/99smeserver-mailstats.sql", "post-upgrade");

root/etc/e-smith/db/configuration/migrate/80DBPass

@@ -0,0 +1,16 @@
+{
+    use MIME::Base64 qw(encode_base64);
+
+    my $rec = $DB->get('mailstats') || $DB->new_record('mailstats', {type => 'report'});
+
+    my $pw = $rec->prop('DBPass');
+    return "" if $pw;
+
+    my $length = shift || 16; 
+
+    my @chars = ('A'..'Z', 'a'..'z', 0..9, qw(! @ $ % ^ & * ? _ - + =));
+    $pw = '';
+    $pw  .= $chars[rand @chars] for 1..$length;
+    $rec->set_prop('DBPass', $pw);
+    return ""
+}

root/etc/e-smith/templates/etc/e-smith/sql/init/99mailstats

@@ -0,0 +1,24 @@
+{
+    my $db = $mailstats{DBName} || 'mailstats';
+    my $user = $mailstats{DBUser} || 'mailstats_rw';
+    my $pass = $mailstats{DBPass} || 'changeme';
+    $OUT .= <<END
+#! /bin/sh
+    if [ -d /var/lib/mysql/mailstats ]; then
+      exit
+    fi
+    /usr/bin/mariadb <<EOF
+    CREATE DATABASE $db DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
+    USE $db;
+	CREATE TABLE IF NOT EXISTS SummaryLogs (
+		id INT AUTO_INCREMENT PRIMARY KEY,
+		Date DATE,
+		Hour INT,
+		logData TEXT
+	);
+	CREATE USER $user@localhost IDENTIFIED BY '$pass';
+	GRANT SELECT, INSERT, UPDATE, DELETE ON $db.* TO $user@localhost;
+	FLUSH PRIVILEGES;
+EOF
+END
+}

root/etc/e-smith/templates/etc/e-smith/sql/init/99smeserver-mailstats.sql

@@ -1,97 +0,0 @@
-CREATE DATABASE IF NOT EXISTS `mailstats`;
-
-USE `mailstats`;
-
-CREATE TABLE IF NOT EXISTS `ColumnStats` (
-  `ColumnStatsid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `timeid` int(11) NOT NULL default '0',
-  `descr` varchar(20) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`ColumnStatsid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `JunkMailStats` (
-  `JunkMailstatsid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
- `user` varchar(12) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) default NULL,
-  PRIMARY KEY  (`JunkMailstatsid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `SARules` (
-  `SARulesid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `rule` varchar(50) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `totalhits` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`SARulesid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `SAscores` (
-  `SAscoresid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `acceptedcount` bigint(20) NOT NULL default '0',
-  `rejectedcount` bigint(20) NOT NULL default '0',
-  `hamcount` bigint(20) NOT NULL default '0',
-  `acceptedscore` decimal(20,2) NOT NULL default '0.00',
-  `rejectedscore` decimal(20,2) NOT NULL default '0.00',
-  `hamscore` decimal(20,2) NOT NULL default '0.00',
-  `totalsmtp` bigint(20) NOT NULL default '0',
-  `totalrecip` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`SAscoresid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `VirusStats` (
-  `VirusStatsid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `descr` varchar(40) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`VirusStatsid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `date` (
-  `dateid` int(11) NOT NULL auto_increment,
-  `date` date NOT NULL default '0000-00-00',
-  PRIMARY KEY  (`dateid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-
-CREATE TABLE IF NOT EXISTS `domains` (
-  `domainsid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `domain` varchar(40) NOT NULL default '',
-  `type` varchar(10) NOT NULL default '',
-  `total` bigint(20) NOT NULL default '0',
-  `denied` bigint(20) NOT NULL default '0',
-  `xfererr` bigint(20) NOT NULL default '0',
-  `accept` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`domainsid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `qpsmtpdcodes` (
-  `qpsmtpdcodesid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `reason` varchar(40) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`qpsmtpdcodesid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `time` (
-  `timeid` int(11) NOT NULL auto_increment,
-  `time` time NOT NULL default '00:00:00',
-  PRIMARY KEY  (`timeid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-
-CREATE USER 'mailstats'@'localhost' IDENTIFIED BY 'mailstats';
-GRANT ALL PRIVILEGES ON mailstats.* TO 'mailstats'@'localhost';
-FLUSH PRIVILEGES;
-

root/etc/e-smith/templates/etc/mailstats/db.php/10DBDetails

@@ -0,0 +1,24 @@
+{
+    # Load SME::ConfigDB to read values from DB
+    my $cdb = esmith::ConfigDB->open() || die "Cannot open configuration DB\n";
+
+    # Get the fragment (report database definition)
+    my $report = $cdb->get('mailstats');
+
+    my $dbhost = $report->prop('DBHost') || 'localhost';
+    my $dbport = $report->prop('DBPort') || '3306';
+    my $dbuser = $report->prop('DBUser') || 'mailstats_rw';
+    # Assume password is stored in a property 'DBPass'
+    my $dbpass = $report->prop('DBPass') || 'changeme';
+    my $dbname = $report->key  || 'mailstats';
+
+    $OUT = <<"END";
+<?php
+return [
+  'host' => '$dbhost',
+  'user' => '$dbuser',
+  'pass' => '$dbpass',
+  'name' => '$dbname',
+];
+END
+}

root/opt/mailstats/html/ShowDetailedLogs.php

@@ -24,7 +24,9 @@ $dbname     = getenv('MAILSTATS_DB_NAME') ?: '';
 if ($username === '' || $password === '' || $dbname === '') {
     $cfgPath = '/etc/mailstats/db.php'; // optional fallback config file
     if (is_readable($cfgPath)) {
-        $cfg = include $cfgPath;
+		ob_start();
+			$cfg = include $cfgPath;
+		ob_end_clean();
         $servername = $cfg['host'] ?? $servername;
         $username   = $cfg['user'] ?? $username;
         $password   = $cfg['pass'] ?? $password;
@@ -227,6 +229,8 @@ $conn->close();
     <meta charset="UTF-8">
     <title>Log details for PID <?= e($pid) ?> (record <?= e($id) ?>)</title>
     <link rel="stylesheet" type="text/css" href="css/mailstats.css" />
+    <link rel="icon" type="image/x-icon" href="favicon.ico">
+
 </head>
 <body>
     <div class="mailstats-detail">

root/opt/mailstats/html/showSummaryLogs.php

@@ -26,7 +26,9 @@ $dbname     = getenv('MAILSTATS_DB_NAME') ?: '';
 if ($username === '' || $password === '' || $dbname === '') {
     $cfgPath = '/etc/mailstats/db.php';
     if (is_readable($cfgPath)) {
-        $cfg = include $cfgPath;
+		ob_start();
+			$cfg = include $cfgPath;
+		ob_end_clean();
         $servername = $cfg['host'] ?? $servername ?: 'localhost';
         $username   = $cfg['user'] ?? $username;
         $password   = $cfg['pass'] ?? $password;
@@ -200,6 +202,7 @@ function generateLogDataTable($logData) {
     <meta charset="UTF-8">
     <title>Summary Logs</title>
     <link rel="stylesheet" type="text/css" href="css/mailstats.css" />
+    <link rel="icon" type="image/x-icon" href="favicon.ico">
 </head>
 <body>
     <div class="mailstats-summary">

root/opt/mailstats/templates/mailstats.html.pt

@@ -3,6 +3,7 @@
 	<meta charset="utf-8">
 	<title>SMEServer Mailstats</title>
 	<link rel='stylesheet' type='text/css' href='css/mailstats.css' />
+    <link rel="icon" type="image/x-icon" href="favicon.ico">
 	<!-- Check links -->
 	<!--css here-->
 </head>

root/usr/bin/mailstats.py

@@ -111,7 +111,7 @@ except ImportError:
 	logging.warning("Matplotlib is not installed  - no graphs")
 	enable_graphs = False;
 
-Mailstats_version = '1.2'
+Mailstats_version = '1.3'
 build_date_time = "2024-06-18 12:03:40OURCE" 
 build_date_time = build_date_time[:19]  #Take out crap that sneaks in.
 
@@ -123,7 +123,6 @@ data_file_path = script_dir+'/../..'    #back to the top
 now = datetime.now()
 yesterday = now - timedelta(days=1)
 formatted_yesterday = yesterday.strftime("%Y-%m-%d")
-#html_page_path = data_file_path+"/home/e-smith/files/ibays/mesdb/html/mailstats/"
 html_page_dir = data_file_path+"/opt/mailstats/html/"
 template_dir = data_file_path+"/opt/mailstats/templates/"
 logs_dir = data_file_path+"/opt/mailstats/logs/"
@@ -454,23 +453,28 @@ def create_graph(data_dict, graph_type="line", output_file="graph.png",iso_date=
     # return data
 
 def save_summaries_to_db(cursor, conn, date_str, hour, parsed_data):
-    # Convert parsed_data to JSON string
     global count_records_to_db
     json_data = json.dumps(parsed_data)
-
-    # Insert the record
     insert_query = """
     INSERT INTO SummaryLogs (Date, Hour, logData)
     VALUES (%s, %s, %s)
     """
-   
     try:
+        # Check if the cursor is open (pymysql has no explicit is_closed; handle by try/except)
         cursor.execute(insert_query, (date_str, hour, json_data))
         conn.commit()
         count_records_to_db += 1
     except pymysql.Error as err:
-        logging.error(f"DB Error {date_str} {hour} : {err}")
+        # Handle cursor closed or other DB errors
+        if 'closed' in str(err).lower():
+            logging.error(f"DB Error {date_str} {hour} : Cursor is closed. Check connection handling.")
+        else:
+            logging.error(f"DB Error {date_str} {hour} : {err}")
         conn.rollback()
+    except Exception as ex:
+        logging.error(f"Unexpected DB Error {date_str} {hour} : {ex}")
+        conn.rollback()
+
                 
 def is_running_under_thonny():
     # Check for the 'THONNY_USER_DIR' environment variable
@@ -1251,6 +1255,41 @@ def format_duration(seconds: float) -> str:
     return str(timedelta(seconds=seconds))
 
 
+DB_CONFIG_PATH = '/etc/mailstats/db.php'
+
+def parse_php_config(path):
+    # Read file as text and extract key-value pairs using regex
+    try:
+        with open(path, 'r') as f:
+            content = f.read()
+            cfg = {}
+            for match in re.finditer(r"'(\w+)'\s*=>\s*'([^']*)'", content):
+                cfg[match.group(1)] = match.group(2)
+            return cfg
+    except Exception as e:
+        logging.error(f"Could not parse PHP config file: {e}")
+        return {}
+
+def load_db_config():
+    db_host = os.environ.get('MAILSTATS_DB_HOST', 'localhost')
+    db_user = os.environ.get('MAILSTATS_DB_USER', '')
+    db_pass = os.environ.get('MAILSTATS_DB_PASS', '')
+    db_name = os.environ.get('MAILSTATS_DB_NAME', '')
+    
+    if db_user == '' or db_pass == '' or db_name == '':
+        if os.path.isfile(DB_CONFIG_PATH) and os.access(DB_CONFIG_PATH, os.R_OK):
+            cfg = parse_php_config(DB_CONFIG_PATH)
+            db_host = cfg.get('host', db_host)
+            db_user = cfg.get('user', db_user)
+            db_pass = cfg.get('pass', db_pass)
+            db_name = cfg.get('name', db_name)
+    
+    if db_user == '' or db_pass == '' or db_name == '':
+        logging.error('DB credentials missing (env and config file).')
+        raise RuntimeError('DB credentials missing (env and config file)')
+    
+    return db_host, db_user, db_pass, db_name
+
 if __name__ == "__main__":
 	start_time = datetime.now()
 	try:
@@ -1334,18 +1373,17 @@ if __name__ == "__main__":
 	count_records_to_db = 0;
 	
 	# Db save control
-	saveData = get_value(ConfigDB,"mailstats","SaveDataToMySQL","no") == 'yes' or forceDbSave
+	saveData = get_value(ConfigDB,"mailstats","SaveDataToMySQL","yes") == 'yes' or forceDbSave
 	logging.debug(f"Save Mailstats to DB set:{saveData} ")
-
 	if saveData:
-		# Connect to MySQL DB for saving
-		DBName = "mailstats"
-		DBHost = get_value(ConfigDB, 'mailstats', 'DBHost', "localhost")
-		DBPort = int(get_value(ConfigDB, 'mailstats', 'DBPort', "3306"))  # Ensure port is an integer
-		DBPassw = 'mailstats'
-		DBUser = 'mailstats'
-		UnixSocket = "/var/lib/mysql/mysql.sock" 
-
+		# Database config retrieval
+		try:
+			DBHost, DBUser, DBPassw, DBName = load_db_config()
+			DBPort = 3306  # If you want configurability, load this from config too
+			UnixSocket = "/var/lib/mysql/mysql.sock"
+		except RuntimeError as err:
+			logging.error(f"Database config error: {err}")
+			saveData = False
 		# Try to establish a database connection
 		try:
 			conn = pymysql.connect(
@@ -1355,7 +1393,7 @@ if __name__ == "__main__":
 				database=DBName,
 				port=DBPort,
 				unix_socket=UnixSocket,
-				cursorclass=pymysql.cursors.DictCursor  # Optional: use DictCursor for dict output
+				cursorclass=pymysql.cursors.DictCursor
 			)
 			cursor = conn.cursor()
 			# Check if the table exists before creating it
@@ -1363,33 +1401,36 @@ if __name__ == "__main__":
 			cursor.execute(check_table_query)
 			table_exists = cursor.fetchone()
 			if not table_exists:
-				# Create table if it doesn't exist
 				cursor.execute("""
-				CREATE TABLE IF NOT EXISTS SummaryLogs (
-					id INT AUTO_INCREMENT PRIMARY KEY,
-					Date DATE,
-					Hour INT,
-					logData TEXT
-				)
+					CREATE TABLE IF NOT EXISTS SummaryLogs (
+						id INT AUTO_INCREMENT PRIMARY KEY,
+						Date DATE,
+						Hour INT,
+						logData TEXT
+					)
 				""")
+			
 			# Delete existing records for the given date
 			try:
 				delete_query = """
-				DELETE FROM SummaryLogs
-				WHERE Date = %s
+					DELETE FROM SummaryLogs
+					WHERE Date = %s
 				"""
-				cursor.execute(delete_query, (analysis_date,))  # Don't forget the extra comma for tuple
-				# Get the number of records deleted
+				cursor.execute(delete_query, (analysis_date,))
 				rows_deleted = cursor.rowcount
 				if rows_deleted > 0:
-					logging.debug(f"Deleted {rows_deleted} rows for {analysis_date} ")
+					logging.debug(f"Deleted {rows_deleted} rows for {analysis_date}")
 			except pymysql.Error as e:
-				logging.error(f"SQL Delete failed ({delete_query}) ({e}) ")
-
+				logging.error(f"SQL Delete failed ({delete_query}) ({e})")
+			
+			# Commit changes & close resources after all DB operations
+			conn.commit()
+			#cursor.close()
+			#conn.close()
 		except pymysql.Error as e:
-			logging.error(f"Unable to connect to {DBName} on {DBHost} port {DBPort} error ({e}) ")
+			logging.error(f"Unable to connect to {DBName} on {DBHost} port {DBPort} error ({e})")
 			saveData = False
-	
+
 	nolinks = not saveData 
 	# Needed to identify blacklist used to reject emails.
 	if get_value(ConfigDB,"qpsmtpd","RHSBL").lower() == 'enabled':

root/usr/bin/runallmailstats.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Extract the earliest date from the journalctl header for qpsmtpd service
+earliest_date=$(journalctl -u qpsmtpd | head -n 1 | sed -n 's/.*Logs begin at [A-Za-z]* \([0-9-]*\).*/\1/p')
+
+# Get yesterday's date
+yesterday=$(date -d 'yesterday' +%F)
+
+current_date="$earliest_date"
+
+# Loop from earliest date to yesterday
+while [[ "$current_date" < "$yesterday" || "$current_date" == "$yesterday" ]]; do
+  runmailstats.sh "$current_date"
+  current_date=$(date -I -d "$current_date + 1 day")
+done

root/usr/share/smanager/themes/default/templates/partials/_mst_CONFIG.html.ep

@@ -77,7 +77,7 @@
 	
 		<div class=dbwanted>
 		
-	
+<!--
 		<h2 class='subh2'><%=l('mst_Details_for_connection_to_database')%></h2>
 	
 		<p><span class=label>
@@ -108,7 +108,7 @@
 			% param 'DBPassword' => $mst_data->{DBPassword} unless param 'DBPassword';
 			%=password_field 'DBPassword', class => 'pass13 sme-password', autocomplete => 'off'
 		</span></p>
-		
+-->
 	
 		</div>
 		

smeserver-mailstats.spec

@@ -6,7 +6,7 @@ Summary: Daily mail statistics for SME Server
 %define name smeserver-mailstats
 Name: %{name}
 %define version 11.1
-%define release 5
+%define release 6
 Version: %{version}
 Release: %{release}%{?dist}
 License: GPL
@@ -90,6 +90,12 @@ usermod -aG systemd-journal www
 /sbin/ldconfig
 
 %changelog
+* Thu Sep 04 2025 Brian Read <brianr@koozali.org> 11.1-6.sme
+- Add favicon to mailstats table, summary and detailed pages [SME: 13121]
+- Bring DB config reading for mailstats itself inline with php summary and detailed logs - using /etc/mailstats/db.php [SME: 13121]
+- Remove DB config fields from the SM2 config panel {sme: 13121]
+- Arrange for password to be generated and mailstats user to be set with limited permissions [SME: 13121]
+
 * Tue Sep 02 2025 Brian Read <brianr@koozali.org> 11.1-5.sme
 - Speed up Journal access [SME: 13121]
 - Fix missing blacklist URL [SME: 13121]
@@ -181,4 +187,4 @@ usermod -aG systemd-journal www
 - Add Update event to createlinks Unexpected failure string in log file: auth::auth_cvm_unix_local see [SME 7089]
 
 * Sat May 26 2012 Brian J read <brianr@bjsystems.co.uk> 1.0-1.sme
-- Initial version
+- Initial version