* Fri Sep 12 2025 Brian Read <brianr@koozali.org> 11.1-10.sme

- Fix version and build date from spec file  [SME: 13121]

createlinks

@@ -6,8 +6,9 @@ $event = 'smeserver-mailstats-update';
 #see the /etc/systemd/system-preset/49-koozali.preset should be present for systemd integration on all you yum update event
 
 foreach my $file (qw(
-                /etc/systemd/system-preset/49-koozali.preset
+		/etc/systemd/system-preset/49-koozali.preset
-		/etc/e-smith/sql/init/99smeserver-mailstats.sql
+		/etc/mailstats/db.php
+		/etc/e-smith/sql/init/99mailstats
 		/etc/httpd/conf/httpd.conf
 ))
 {
@@ -20,7 +21,7 @@ event_link('systemd-reload', $event, '50');
 #event_link('action', $event, '30');
 #services we need to restart
 safe_symlink('restart', "root/etc/e-smith/events/$event/services2adjust/httpd-e-smith");
+safe_symlink("restart", "root/etc/e-smith/events/$event/services2adjust/mysql.init");;
 #and Server Mmanager panel link
 #panel_link('somefunction', 'manager');
-
+#templates2events("/etc/e-smith/sql/init/99smeserver-mailstats.sql", "post-upgrade");
-templates2events("/etc/e-smith/sql/init/99smeserver-mailstats.sql", "post-upgrade");

root/etc/e-smith/db/configuration/migrate/80DBPass

@@ -0,0 +1,16 @@
+{
+    use MIME::Base64 qw(encode_base64);
+
+    my $rec = $DB->get('mailstats') || $DB->new_record('mailstats', {type => 'report'});
+
+    my $pw = $rec->prop('DBPass');
+    return "" if $pw;
+
+    my $length = shift || 16; 
+
+    my @chars = ('A'..'Z', 'a'..'z', 0..9, qw(! @ $ % ^ & * ? _ - + =));
+    $pw = '';
+    $pw  .= $chars[rand @chars] for 1..$length;
+    $rec->set_prop('DBPass', $pw);
+    return ""
+}

root/etc/e-smith/templates/etc/e-smith/sql/init/99mailstats

@@ -0,0 +1,24 @@
+{
+    my $db = $mailstats{DBName} || 'mailstats';
+    my $user = $mailstats{DBUser} || 'mailstats_rw';
+    my $pass = $mailstats{DBPass} || 'changeme';
+    $OUT .= <<END
+#! /bin/sh
+    if [ -d /var/lib/mysql/mailstats ]; then
+      exit
+    fi
+    /usr/bin/mariadb <<EOF
+    CREATE DATABASE $db DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
+    USE $db;
+	CREATE TABLE IF NOT EXISTS SummaryLogs (
+		id INT AUTO_INCREMENT PRIMARY KEY,
+		Date DATE,
+		Hour INT,
+		logData TEXT
+	);
+	CREATE USER $user@localhost IDENTIFIED BY '$pass';
+	GRANT SELECT, INSERT, UPDATE, DELETE ON $db.* TO $user@localhost;
+	FLUSH PRIVILEGES;
+EOF
+END
+}

root/etc/e-smith/templates/etc/e-smith/sql/init/99smeserver-mailstats.sql

@@ -1,97 +0,0 @@
-CREATE DATABASE IF NOT EXISTS `mailstats`;
-
-USE `mailstats`;
-
-CREATE TABLE IF NOT EXISTS `ColumnStats` (
-  `ColumnStatsid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `timeid` int(11) NOT NULL default '0',
-  `descr` varchar(20) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`ColumnStatsid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `JunkMailStats` (
-  `JunkMailstatsid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
- `user` varchar(12) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) default NULL,
-  PRIMARY KEY  (`JunkMailstatsid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `SARules` (
-  `SARulesid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `rule` varchar(50) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `totalhits` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`SARulesid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `SAscores` (
-  `SAscoresid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `acceptedcount` bigint(20) NOT NULL default '0',
-  `rejectedcount` bigint(20) NOT NULL default '0',
-  `hamcount` bigint(20) NOT NULL default '0',
-  `acceptedscore` decimal(20,2) NOT NULL default '0.00',
-  `rejectedscore` decimal(20,2) NOT NULL default '0.00',
-  `hamscore` decimal(20,2) NOT NULL default '0.00',
-  `totalsmtp` bigint(20) NOT NULL default '0',
-  `totalrecip` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`SAscoresid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `VirusStats` (
-  `VirusStatsid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `descr` varchar(40) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`VirusStatsid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `date` (
-  `dateid` int(11) NOT NULL auto_increment,
-  `date` date NOT NULL default '0000-00-00',
-  PRIMARY KEY  (`dateid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-
-CREATE TABLE IF NOT EXISTS `domains` (
-  `domainsid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `domain` varchar(40) NOT NULL default '',
-  `type` varchar(10) NOT NULL default '',
-  `total` bigint(20) NOT NULL default '0',
-  `denied` bigint(20) NOT NULL default '0',
-  `xfererr` bigint(20) NOT NULL default '0',
-  `accept` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`domainsid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `qpsmtpdcodes` (
-  `qpsmtpdcodesid` int(11) NOT NULL auto_increment,
-  `dateid` int(11) NOT NULL default '0',
-  `reason` varchar(40) NOT NULL default '',
-  `count` bigint(20) NOT NULL default '0',
-  `servername` varchar(30) NOT NULL default '',
-  PRIMARY KEY  (`qpsmtpdcodesid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-CREATE TABLE IF NOT EXISTS `time` (
-  `timeid` int(11) NOT NULL auto_increment,
-  `time` time NOT NULL default '00:00:00',
-  PRIMARY KEY  (`timeid`)
-) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-
-
-CREATE USER 'mailstats'@'localhost' IDENTIFIED BY 'mailstats';
-GRANT ALL PRIVILEGES ON mailstats.* TO 'mailstats'@'localhost';
-FLUSH PRIVILEGES;
-

root/etc/e-smith/templates/etc/mailstats/db.php/10DBDetails

@@ -0,0 +1,24 @@
+{
+    # Load SME::ConfigDB to read values from DB
+    my $cdb = esmith::ConfigDB->open() || die "Cannot open configuration DB\n";
+
+    # Get the fragment (report database definition)
+    my $report = $cdb->get('mailstats');
+
+    my $dbhost = $report->prop('DBHost') || 'localhost';
+    my $dbport = $report->prop('DBPort') || '3306';
+    my $dbuser = $report->prop('DBUser') || 'mailstats_rw';
+    # Assume password is stored in a property 'DBPass'
+    my $dbpass = $report->prop('DBPass') || 'changeme';
+    my $dbname = $report->key  || 'mailstats';
+
+    $OUT = <<"END";
+<?php
+return [
+  'host' => '$dbhost',
+  'user' => '$dbuser',
+  'pass' => '$dbpass',
+  'name' => '$dbname',
+];
+END
+}

root/opt/mailstats/html/ShowDetailedLogs.php

@@ -24,7 +24,9 @@ $dbname     = getenv('MAILSTATS_DB_NAME') ?: '';
 if ($username === '' || $password === '' || $dbname === '') {
     $cfgPath = '/etc/mailstats/db.php'; // optional fallback config file
     if (is_readable($cfgPath)) {
-        $cfg = include $cfgPath;
+		ob_start();
+			$cfg = include $cfgPath;
+		ob_end_clean();
         $servername = $cfg['host'] ?? $servername;
         $username   = $cfg['user'] ?? $username;
         $password   = $cfg['pass'] ?? $password;
@@ -227,6 +229,8 @@ $conn->close();
     <meta charset="UTF-8">
     <title>Log details for PID <?= e($pid) ?> (record <?= e($id) ?>)</title>
     <link rel="stylesheet" type="text/css" href="css/mailstats.css" />
+    <link rel="icon" type="image/x-icon" href="favicon.ico">
+
 </head>
 <body>
     <div class="mailstats-detail">

root/opt/mailstats/html/showSummaryLogs.php

@@ -26,7 +26,9 @@ $dbname     = getenv('MAILSTATS_DB_NAME') ?: '';
 if ($username === '' || $password === '' || $dbname === '') {
     $cfgPath = '/etc/mailstats/db.php';
     if (is_readable($cfgPath)) {
-        $cfg = include $cfgPath;
+		ob_start();
+			$cfg = include $cfgPath;
+		ob_end_clean();
         $servername = $cfg['host'] ?? $servername ?: 'localhost';
         $username   = $cfg['user'] ?? $username;
         $password   = $cfg['pass'] ?? $password;
@@ -200,6 +202,7 @@ function generateLogDataTable($logData) {
     <meta charset="UTF-8">
     <title>Summary Logs</title>
     <link rel="stylesheet" type="text/css" href="css/mailstats.css" />
+    <link rel="icon" type="image/x-icon" href="favicon.ico">
 </head>
 <body>
     <div class="mailstats-summary">

root/opt/mailstats/templates/mailstats-sub-table.html.pt

@@ -1,7 +1,7 @@
 <div class="${classname}">
 <h2>${title}</h2>
 	<tal:block condition="threshold != 0">
-		<span class='greyed-out'>Display threshold set to ${threshold}%</span>
+		<span class='greyed-out'>${threshold}</span>
 	</tal:block>
 	<tal:block condition="threshold == 0">
 		<br>

root/opt/mailstats/templates/mailstats.html.pt

@@ -3,6 +3,7 @@
 	<meta charset="utf-8">
 	<title>SMEServer Mailstats</title>
 	<link rel='stylesheet' type='text/css' href='css/mailstats.css' />
+    <link rel="icon" type="image/x-icon" href="favicon.ico">
 	<!-- Check links -->
 	<!--css here-->
 </head>

root/usr/bin/mailstats.py

@@ -111,9 +111,11 @@ except ImportError:
 	logging.warning("Matplotlib is not installed  - no graphs")
 	enable_graphs = False;
 
-Mailstats_version = '1.2'
+Mailstats_version = '1.3'
 build_date_time = "2024-06-18 12:03:40OURCE" 
-build_date_time = build_date_time[:19]  #Take out crap that sneaks in.
+#Take out the crap that sneaks in...
+build_date_time = build_date_time[:19]  
+Mailstats_version = Mailstats_version[:6]
 
 #if build_date_time == "2024-06-18 12:03:40OURCE":
 #    build_date_time = "Unknown"
@@ -123,7 +125,6 @@ data_file_path = script_dir+'/../..'    #back to the top
 now = datetime.now()
 yesterday = now - timedelta(days=1)
 formatted_yesterday = yesterday.strftime("%Y-%m-%d")
-#html_page_path = data_file_path+"/home/e-smith/files/ibays/mesdb/html/mailstats/"
 html_page_dir = data_file_path+"/opt/mailstats/html/"
 template_dir = data_file_path+"/opt/mailstats/templates/"
 logs_dir = data_file_path+"/opt/mailstats/logs/"
@@ -454,23 +455,28 @@ def create_graph(data_dict, graph_type="line", output_file="graph.png",iso_date=
     # return data
 
 def save_summaries_to_db(cursor, conn, date_str, hour, parsed_data):
-    # Convert parsed_data to JSON string
     global count_records_to_db
     json_data = json.dumps(parsed_data)
-
-    # Insert the record
     insert_query = """
     INSERT INTO SummaryLogs (Date, Hour, logData)
     VALUES (%s, %s, %s)
     """
-   
     try:
+        # Check if the cursor is open (pymysql has no explicit is_closed; handle by try/except)
         cursor.execute(insert_query, (date_str, hour, json_data))
         conn.commit()
         count_records_to_db += 1
     except pymysql.Error as err:
-        logging.error(f"DB Error {date_str} {hour} : {err}")
+        # Handle cursor closed or other DB errors
+        if 'closed' in str(err).lower():
+            logging.error(f"DB Error {date_str} {hour} : Cursor is closed. Check connection handling.")
+        else:
+            logging.error(f"DB Error {date_str} {hour} : {err}")
         conn.rollback()
+    except Exception as ex:
+        logging.error(f"Unexpected DB Error {date_str} {hour} : {ex}")
+        conn.rollback()
+
                 
 def is_running_under_thonny():
     # Check for the 'THONNY_USER_DIR' environment variable
@@ -830,7 +836,123 @@ def split_timestamp_and_data(log_entry: str) -> list:
 		rest_of_line = log_entry  # If no match, return the whole line
 	return [timestamp, rest_of_line]
 
-def render_sub_table(table_title, table_headers, found_values, get_character=None, suppress_threshold=False):
+MIN_COUNT = 3               # Hide entries with count < 5
+MAX_TOTAL_ROWS = 10         # Total rows INCLUDING "Other"
+OTHER_TARGET_FRAC = 0.01    # Strictly less than 1%
+OTHER_LABEL = 'Other'
+SHOW_ALL = True            # Set True to show all entries >= MIN_COUNT, no "Other" row
+
+
+def select_rows_just_below(items, min_count=MIN_COUNT, 
+                            max_total_rows=MAX_TOTAL_ROWS, 
+                            other_target_frac=OTHER_TARGET_FRAC,
+                            other_label=OTHER_LABEL, show_all=SHOW_ALL):
+    """
+    Build rows with percentages of total (0..100).
+    - If show_all is True: show all entries with count >= min_count, no 'Other', ignore caps and 1% target.
+    - If show_all is False: pick as many top entries (count >= min_count) as needed so that
+      'Other' is strictly < other_target_frac (if possible), always include 'Other(n)',
+      and respect max_total_rows (including 'Other').
+
+    Output rows preserve original extra fields for selected entries.
+    The percent is written to field index 2 (replacing it if present, or appended if not).
+    """
+
+    # Normalize items to a list while preserving original rows
+    def to_rows(seq):
+        if isinstance(seq, dict):
+            # Convert dict to rows without extras
+            return [(k, v) for k, v in seq.items()]
+        rows_ = []
+        for it in seq:
+            if isinstance(it, (tuple, list)) and len(it) >= 2:
+                rows_.append(tuple(it))  # store as tuple
+            else:
+                raise TypeError("Each item must be a (key, count, ...) tuple/list or a dict mapping key->count.")
+        return rows_
+
+    def set_percent(row, pct_value):
+        # Return a tuple like the input row but with percent inserted at index 2 (0..100 number, rounded)
+        pct_value = round(pct_value, 2)
+        r = list(row)
+        if len(r) >= 3:
+            r[2] = pct_value
+        else:
+            r.append(pct_value)
+        return tuple(r)
+
+    rows_in = to_rows(items)
+
+    total = sum(r[1] for r in rows_in)
+    if total == 0:
+        return ([(f"{other_label}(0)", 0, 0.0)] if not show_all else []), 0, "No data."
+
+    # Filter by min_count and sort by count desc
+    eligible = [r for r in rows_in if r[1] >= min_count]
+    eligible.sort(key=lambda r: r[1], reverse=True)
+
+    if show_all:
+        # Show all eligible rows, no 'Other', ignore caps/target; compute percent (0..100) per row
+        rows_out = [set_percent(r, (r[1] / total) * 100.0) for r in eligible]
+        return rows_out, total, None
+
+    #logging.info(f"{show_all}")
+    # Leave room for the "Other" row
+    max_top_cap = max(0, max_total_rows - 1)
+
+    # Find smallest number of top rows so that Other is strictly < target
+    cum = 0
+    needed_top = None
+    for i, r in enumerate(eligible, start=1):
+        cum += r[1]
+        other_frac = (total - cum) / total
+        if other_frac < other_target_frac:
+            needed_top = i
+            break
+
+    notes = []
+
+    if needed_top is None:
+        # Even after including all eligible, Other >= target
+        final_top = min(len(eligible), max_top_cap)
+        #if final_top < len(eligible):
+            #notes.append(f"Row cap prevents adding enough rows to push Other below {other_target_frac*100:.2f}%.")
+        #else:
+            #notes.append(f"Cannot push Other below {other_target_frac*100:.2f}% with MIN_COUNT={min_count}.")
+    else:
+        # Apply cap
+        if needed_top > max_top_cap:
+            final_top = max_top_cap
+            #notes.append(
+            #    f"Row cap prevents reaching Other < {other_target_frac*100:.2f}%; "
+            #    f"need {needed_top} rows but only {max_top_cap} allowed before Other."
+            #)
+        else:
+            final_top = needed_top
+
+    top = eligible[:final_top]
+    shown_sum = sum(r[1] for r in top)
+    other_count = total - shown_sum
+    other_percent = (other_count / total) * 100.0
+
+    # Count how many rows are aggregated into Other: everything not in 'top'
+    other_rows_count = len(rows_in) - len(top)
+
+    # Build output: preserve extras; write percent at index 2 as a numeric percent 0..100
+    rows_out = [set_percent(r, (r[1] / total) * 100.0) for r in top]
+
+    # Build the Other row with percent; no extra fields beyond the percent
+    rows_out.append((f"{other_label}({other_rows_count})", other_count, round(other_percent, 2)))
+
+    #if other_percent >= other_target_frac * 100.0:
+        #notes.append(
+        #    f"Other is {other_percent:.2f}%, which is not strictly below {other_target_frac*100:.2f}% "
+        #    f"(MIN_COUNT={min_count}, MAX_TOTAL_ROWS={max_total_rows})."
+        #)
+
+    return rows_out, total, " ".join(notes) if notes else None
+    	
+def render_sub_table(table_title, table_headers, found_values, get_character=None, show_all=True):
 	#Check if any data provided
 	if len(found_values) != 0:
 		# Get the total
@@ -874,30 +996,36 @@ def render_sub_table(table_title, table_headers, found_values, get_character=Non
 				raise ValueError("found_values must be either a list of numbers or a list of dictionaries.")
 		else:
 			raise TypeError("found_values must be a dictionary or a list.")
+
+		# # Dynamic threshold calculation
+		# if not suppress_threshold:
+			# dynamic_threshold = max(1, 100 / (original_total**0.65)) if original_total > 0 else 0
+			# dynamic_threshold = round(dynamic_threshold,1)
+			# logging.debug(f"Threshold for {table_title} set to {dynamic_threshold}% ")
+		# else:
+			# dynamic_threshold=0
+		# absolute_floor = 10  # Minimum absolute value threshold
+
+		# # Filter results using early termination
+		# filtered_sub_result = []
+		# for row in sub_result:
+			# value = row[1]
+			# percentage = (value / original_total * 100) if original_total else 0
+			
+			# # Exit condition: below both thresholds
+			# if percentage < dynamic_threshold or value < absolute_floor:
+				# break
+				
+			# filtered_sub_result.append(row)
+		
+		# sub_result = filtered_sub_result  # Keep only significant rows
+
 		sub_result.sort(key=lambda x: float(x[1]), reverse=True)  # Sort by percentage in descending order
-
+		if not show_all:
-		# Dynamic threshold calculation
+			sub_result, total, note = select_rows_just_below(sub_result,show_all=False)
-		if not suppress_threshold:
-			dynamic_threshold = max(1, 100 / (original_total**0.5)) if original_total > 0 else 0
-			dynamic_threshold = round(dynamic_threshold,1)
-			logging.debug(f"Threshold for {table_title} set to {dynamic_threshold}% ")
 		else:
-			dynamic_threshold=0
+			note = "" #no threshold applied
-		absolute_floor = 50  # Minimum absolute value threshold
+			total = original_total
-
-		# Filter results using early termination
-		filtered_sub_result = []
-		for row in sub_result:
-			value = row[1]
-			percentage = (value / original_total * 100) if original_total else 0
-			
-			# Exit condition: below both thresholds
-			if percentage < dynamic_threshold and value < absolute_floor:
-				break
-				
-			filtered_sub_result.append(row)
-		
-		sub_result = filtered_sub_result  # Keep only significant rows
 		
 		sub_template_path = template_dir+'mailstats-sub-table.html.pt'
 		# Load the template
@@ -910,7 +1038,7 @@ def render_sub_table(table_title, table_headers, found_values, get_character=Non
 			try:
 				rendered_html = template(array_2d=sub_result, column_headers=table_headers, 
 										title=table_title, classname=get_first_word(table_title),
-										threshold=dynamic_threshold)
+										threshold=note)
 			except Exception as e:
 				raise ValueError(f"{table_title}: A chameleon controller render error occurred: {e}")
 		except Exception as e:
@@ -1251,6 +1379,41 @@ def format_duration(seconds: float) -> str:
     return str(timedelta(seconds=seconds))
 
 
+DB_CONFIG_PATH = '/etc/mailstats/db.php'
+
+def parse_php_config(path):
+    # Read file as text and extract key-value pairs using regex
+    try:
+        with open(path, 'r') as f:
+            content = f.read()
+            cfg = {}
+            for match in re.finditer(r"'(\w+)'\s*=>\s*'([^']*)'", content):
+                cfg[match.group(1)] = match.group(2)
+            return cfg
+    except Exception as e:
+        logging.error(f"Could not parse PHP config file: {e}")
+        return {}
+
+def load_db_config():
+    db_host = os.environ.get('MAILSTATS_DB_HOST', 'localhost')
+    db_user = os.environ.get('MAILSTATS_DB_USER', '')
+    db_pass = os.environ.get('MAILSTATS_DB_PASS', '')
+    db_name = os.environ.get('MAILSTATS_DB_NAME', '')
+    
+    if db_user == '' or db_pass == '' or db_name == '':
+        if os.path.isfile(DB_CONFIG_PATH) and os.access(DB_CONFIG_PATH, os.R_OK):
+            cfg = parse_php_config(DB_CONFIG_PATH)
+            db_host = cfg.get('host', db_host)
+            db_user = cfg.get('user', db_user)
+            db_pass = cfg.get('pass', db_pass)
+            db_name = cfg.get('name', db_name)
+    
+    if db_user == '' or db_pass == '' or db_name == '':
+        logging.error('DB credentials missing (env and config file).')
+        raise RuntimeError('DB credentials missing (env and config file)')
+    
+    return db_host, db_user, db_pass, db_name
+
 if __name__ == "__main__":
 	start_time = datetime.now()
 	try:
@@ -1334,18 +1497,17 @@ if __name__ == "__main__":
 	count_records_to_db = 0;
 	
 	# Db save control
-	saveData = get_value(ConfigDB,"mailstats","SaveDataToMySQL","no") == 'yes' or forceDbSave
+	saveData = get_value(ConfigDB,"mailstats","SaveDataToMySQL","yes") == 'yes' or forceDbSave
 	logging.debug(f"Save Mailstats to DB set:{saveData} ")
-
 	if saveData:
-		# Connect to MySQL DB for saving
+		# Database config retrieval
-		DBName = "mailstats"
+		try:
-		DBHost = get_value(ConfigDB, 'mailstats', 'DBHost', "localhost")
+			DBHost, DBUser, DBPassw, DBName = load_db_config()
-		DBPort = int(get_value(ConfigDB, 'mailstats', 'DBPort', "3306"))  # Ensure port is an integer
+			DBPort = 3306  # If you want configurability, load this from config too
-		DBPassw = 'mailstats'
+			UnixSocket = "/var/lib/mysql/mysql.sock"
-		DBUser = 'mailstats'
+		except RuntimeError as err:
-		UnixSocket = "/var/lib/mysql/mysql.sock" 
+			logging.error(f"Database config error: {err}")
-
+			saveData = False
 		# Try to establish a database connection
 		try:
 			conn = pymysql.connect(
@@ -1355,7 +1517,7 @@ if __name__ == "__main__":
 				database=DBName,
 				port=DBPort,
 				unix_socket=UnixSocket,
-				cursorclass=pymysql.cursors.DictCursor  # Optional: use DictCursor for dict output
+				cursorclass=pymysql.cursors.DictCursor
 			)
 			cursor = conn.cursor()
 			# Check if the table exists before creating it
@@ -1363,31 +1525,34 @@ if __name__ == "__main__":
 			cursor.execute(check_table_query)
 			table_exists = cursor.fetchone()
 			if not table_exists:
-				# Create table if it doesn't exist
 				cursor.execute("""
-				CREATE TABLE IF NOT EXISTS SummaryLogs (
+					CREATE TABLE IF NOT EXISTS SummaryLogs (
-					id INT AUTO_INCREMENT PRIMARY KEY,
+						id INT AUTO_INCREMENT PRIMARY KEY,
-					Date DATE,
+						Date DATE,
-					Hour INT,
+						Hour INT,
-					logData TEXT
+						logData TEXT
-				)
+					)
 				""")
+			
 			# Delete existing records for the given date
 			try:
 				delete_query = """
-				DELETE FROM SummaryLogs
+					DELETE FROM SummaryLogs
-				WHERE Date = %s
+					WHERE Date = %s
 				"""
-				cursor.execute(delete_query, (analysis_date,))  # Don't forget the extra comma for tuple
+				cursor.execute(delete_query, (analysis_date,))
-				# Get the number of records deleted
 				rows_deleted = cursor.rowcount
 				if rows_deleted > 0:
-					logging.debug(f"Deleted {rows_deleted} rows for {analysis_date} ")
+					logging.debug(f"Deleted {rows_deleted} rows for {analysis_date}")
 			except pymysql.Error as e:
-				logging.error(f"SQL Delete failed ({delete_query}) ({e}) ")
+				logging.error(f"SQL Delete failed ({delete_query}) ({e})")
 			
+			# Commit changes & close resources after all DB operations
+			conn.commit()
+			#cursor.close()
+			#conn.close()
 		except pymysql.Error as e:
-			logging.error(f"Unable to connect to {DBName} on {DBHost} port {DBPort} error ({e}) ")
+			logging.error(f"Unable to connect to {DBName} on {DBHost} port {DBPort} error ({e})")
 			saveData = False
 
 	nolinks = not saveData 
@@ -1632,7 +1797,10 @@ if __name__ == "__main__":
 					rejReason = match.group(1)
 					found_qpcodes[parsed_data['error-plugin']+"-"+rejReason] += 1
 				else:
-					found_qpcodes[parsed_data['action1']] += 1
+					if parsed_data['action1'] == "":
+						logging.warning(f"Found blank action1 {timestamp} {parsed_data['id']} {parsed_data['ip']} {parsed_data['sendurl']}")
+					else:
+						found_qpcodes[parsed_data['action1']] += 1
 
 			#Check for blacklist rejection
 			error_plugin = parsed_data['error-plugin'].strip()
@@ -1668,6 +1836,8 @@ if __name__ == "__main__":
 				else:
 					email = None
 			if email:
+				if '@' in email:
+					email = email.lower()
 				record = next((item for item in recipients_found if item['email'] == email), None)
 				if not record:
 					# If email is not in the array, we add it
@@ -1780,6 +1950,7 @@ if __name__ == "__main__":
 			try:
 				match = geoip_pattern.match(data['MESSAGE'])
 				if match:
+					logging.debug(f"Found bad country message {data['MESSAGE']} {match.group(1)} ")
 					j += 1
 					country = match.group(1)
 					found_countries[country] += 1
@@ -1887,7 +2058,7 @@ if __name__ == "__main__":
 	#virus codes
 	virus_headers = ["Virus",'Count','Percent']
 	virus_title = 'Viruses found'
-	virus_rendered_html = render_sub_table(virus_title,virus_headers,found_viruses,suppress_threshold=True)
+	virus_rendered_html = render_sub_table(virus_title,virus_headers,found_viruses)
 	# Add it to the total 
 	total_html = insert_string_after(total_html,virus_rendered_html, "<!---Add in sub tables here -->")
 
@@ -1903,7 +2074,7 @@ if __name__ == "__main__":
 	junk_mail_count_headers = ['Username','Count', 'Percent']
 	junk_mail_counts = scan_mail_users()
 	junk_mail_count_title = 'Junk mail counts'
-	junk_rendered_html = render_sub_table(junk_mail_count_title,junk_mail_count_headers,junk_mail_counts,suppress_threshold=True)
+	junk_rendered_html = render_sub_table(junk_mail_count_title,junk_mail_count_headers,junk_mail_counts)
 	# Add it to the total 
 	total_html = insert_string_after(total_html,junk_rendered_html, "<!---Add in sub tables here -->")
 
@@ -1911,21 +2082,21 @@ if __name__ == "__main__":
 	#Recipient counts
 	recipient_count_headers = ["Email",'Queued','Rejected','Spam tagged','Accepted Percent']
 	recipient_count_title = 'Incoming email recipients'
-	recipient_rendered_html = render_sub_table(recipient_count_title,recipient_count_headers,recipients_found,suppress_threshold=True)
+	recipient_rendered_html = render_sub_table(recipient_count_title,recipient_count_headers,recipients_found)
 	# Add it to the total 
 	total_html = insert_string_after(total_html,recipient_rendered_html, "<!---Add in sub tables here -->")
 
 	#Geoip Country codes
 	geoip_headers  = ['Country','Count','Percent','Rejected?']
 	geoip_title = 'Geoip results'
-	geoip_rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list)
+	geoip_rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list,show_all=False)
 	# Add it to the total 
 	total_html = insert_string_after(total_html,geoip_rendered_html, "<!---Add in sub tables here -->")
 	
 	#Blacklist counts
 	blacklist_headers = ['URL','Count','Percent']
 	blacklist_title = 'Blacklist used'
-	blacklist_rendered_html = render_sub_table(blacklist_title,blacklist_headers,blacklist_found,suppress_threshold=True)
+	blacklist_rendered_html = render_sub_table(blacklist_title,blacklist_headers,blacklist_found)
 	# Add it to the total 
 	total_html = insert_string_after(total_html,blacklist_rendered_html, "<!---Add in sub tables here -->")
 	

root/usr/bin/runallmailstats.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Extract the earliest date from the journalctl header for qpsmtpd service
+earliest_date=$(journalctl -u qpsmtpd | head -n 1 | sed -n 's/.*Logs begin at [A-Za-z]* \([0-9-]*\).*/\1/p')
+
+# Get yesterday's date
+yesterday=$(date -d 'yesterday' +%F)
+
+current_date="$earliest_date"
+
+# Loop from earliest date to yesterday
+while [[ "$current_date" < "$yesterday" || "$current_date" == "$yesterday" ]]; do
+  runmailstats.sh "$current_date"
+  current_date=$(date -I -d "$current_date + 1 day")
+done

root/usr/share/smanager/themes/default/templates/partials/_mst_CONFIG.html.ep

@@ -77,7 +77,7 @@
 	
 		<div class=dbwanted>
 		
-	
+<!--
 		<h2 class='subh2'><%=l('mst_Details_for_connection_to_database')%></h2>
 	
 		<p><span class=label>
@@ -108,7 +108,7 @@
 			% param 'DBPassword' => $mst_data->{DBPassword} unless param 'DBPassword';
 			%=password_field 'DBPassword', class => 'pass13 sme-password', autocomplete => 'off'
 		</span></p>
-		
+-->
 	
 		</div>
 		

smeserver-mailstats.spec

@@ -6,7 +6,8 @@ Summary: Daily mail statistics for SME Server
 %define name smeserver-mailstats
 Name: %{name}
 %define version 11.1
-%define release 5
+%define release 10
+%define full_version %{version}.%{release})
 Version: %{version}
 Release: %{release}%{?dist}
 License: GPL
@@ -49,21 +50,15 @@ perl createlinks
 (cd root ; /usr/bin/find . -depth -print | /bin/cpio -dump $RPM_BUILD_ROOT)
 chmod +x $RPM_BUILD_ROOT/usr/bin/runmailstats.sh
 
-#chmod 0640 $RPM_BUILD_ROOT/etc/mailstats/db.php
-#ls -l /builddir/build/BUILDROOT/smeserver-mailstats-11.1-5.el8.sme.x86_64/etc/mailstats/ 
-#chown root:102 $RPM_BUILD_ROOT/etc/mailstats/db.php
-
-# Define the placeholder and generate the current date and time
 now=$(date +"%Y-%m-%d %H:%M:%S")
-
+# Replace placeholders in the Python program using sed
-# Replace the placeholder in the Python program located at %{BUILDROOT}/usr/bin
+perl -pi -e 'if (!$done && s/^Mailstats_version *=.*/Mailstats_version = '\''%{full_version}'\'/') { $done = 1 }' $RPM_BUILD_ROOT/usr/bin/mailstats.py
-sed -i "s|__BUILD_DATE_TIME__|$now|" $RPM_BUILD_ROOT/usr/bin/mailstats.py
+perl -pi -e 'if (!$done && s/^build_date_time *=.*/build_date_time = "'"$now"'"/) { $done = 1 }' $RPM_BUILD_ROOT/usr/bin/mailstats.py
 
 /bin/rm -f %{name}-%{version}-filelist
 /sbin/e-smith/genfilelist --file '/etc/mailstats/db.php' 'attr(0640, root, apache)' $RPM_BUILD_ROOT | grep -v "\.pyc" | grep -v "\.pyo"  > %{name}-%{version}-filelist
 
 install -Dpm 0755 journalwrap %{buildroot}%{_bindir}/journalwrap
-#install -Dpm 0644 libjournalwrap.so %{buildroot}%{_libdir}/libjournalwrap.so
 
 
 %pre
@@ -84,12 +79,31 @@ install -Dpm 0755 journalwrap %{buildroot}%{_bindir}/journalwrap
 
 %post
 /sbin/ldconfig
-usermod -aG systemd-journal www
+#Remove www from systemd-journal group as is potential security risk
+gpasswd -d  www systemd-journal
+# and set setuid bit for c wrapper called from log detail web page
+chmod u+s /usr/bin/journalwrap
 
 %postun
 /sbin/ldconfig
 
 %changelog
+* Fri Sep 12 2025 Brian Read <brianr@koozali.org> 11.1-10.sme
+- Fix version and build date from spec file  [SME: 13121]
+
+* Fri Sep 12 2025 Brian Read <brianr@koozali.org> 11.1-8.sme
+- Remove www from systemd-journal group and setuid bit in journal wrapper [SME: 13121]
+
+* Fri Sep 12 2025 Brian Read <brianr@koozali.org> 11.1-7.sme
+- Truncate Geoip table and add other category [SME: 13121]
+- Cope with blank data in action1 [SME: 13121]
+
+* Thu Sep 04 2025 Brian Read <brianr@koozali.org> 11.1-6.sme
+- Add favicon to mailstats table, summary and detailed pages [SME: 13121]
+- Bring DB config reading for mailstats itself inline with php summary and detailed logs - using /etc/mailstats/db.php [SME: 13121]
+- Remove DB config fields from the SM2 config panel {sme: 13121]
+- Arrange for password to be generated and mailstats user to be set with limited permissions [SME: 13121]
+
 * Tue Sep 02 2025 Brian Read <brianr@koozali.org> 11.1-5.sme
 - Speed up Journal access [SME: 13121]
 - Fix missing blacklist URL [SME: 13121]