Sort out text version of table etc

root/usr/bin/mailstats.py

@@ -148,6 +148,24 @@ PERCENT = TOTALS + 1
 ColTotals = 24
 ColPercent = 25
 
+def replace_bracket_content(input_filename, output_filename):
+    import re
+
+    with open(input_filename, 'r', encoding='utf-8') as infile:
+        content = infile.read()
+
+    # Pattern to capture digits/spaces inside brackets
+    pattern = r'\[([\d\s]*)\]\(\./showSummaryLogs\.php\?date=\d{4}-\d{2}-\d{2}&hour=\d{1,2}\)'
+    
+    # Pad captured group to 10 characters
+    replaced_content = re.sub(pattern, lambda m: f"{m.group(1):8}", content)
+
+    with open(output_filename, 'w', encoding='utf-8') as outfile:
+        outfile.write(replaced_content)
+
+    return f"Replacements completed. Output written to {output_filename}"
+
+
 def get_logs_from_Journalctl(date='yesterday'):
 	# JSON-pretty output example from journalctl
 	# {
@@ -1129,7 +1147,7 @@ if __name__ == "__main__":
 	DomainName = get_value(ConfigDB, "DomainName", "type") 							#'bjsystems.co.uk' #  $cdb->get('DomainName')->value;
 	SystemName = get_value(ConfigDB, "SystemName", "type")
 	
-	hello_string = "Mailstats:"+Mailstats_version+' for '+SystemName+"."+DomainName+" for "+analysis_date+" logging.error(ed at:"+formatted_datetime
+	hello_string = "Mailstats:"+Mailstats_version+' for '+SystemName+"."+DomainName+" for "+analysis_date+" printed at:"+formatted_datetime
 	logging.info(hello_string)
 	version_string = "Chameleon:"+chameleon_version+" Python:"+python_version
 	if isThonny:
@@ -1150,7 +1168,7 @@ if __name__ == "__main__":
 	EmailAddress = get_value(ConfigDB,"mailstats","Email","admin@"+DomainName)
 	if '@' not in EmailAddress:
 		EmailAddress = EmailAddress+"@"+DomainName
-	EmailTextOrHTML = get_value(ConfigDB,"mailstats","EmailTextOrHTML","Both") 		#Text or Both or None
+	EmailTextorHTML = get_value(ConfigDB,"mailstats","TextorHTML","Both") 		#Text or Both or None
 	EmailHost = get_value(ConfigDB,"mailstats","EmailHost","localhost") 			#Default will be localhost
 	EmailPort = int(get_value(ConfigDB,"mailstats","EmailPort","25"))
 	EMailSMTPUser = get_value(ConfigDB,"mailstats","EmailUser")   					#None = default => no authenticatioon needed
@@ -1158,6 +1176,8 @@ if __name__ == "__main__":
 	
 	BadCountries = get_value(ConfigDB,"qpsmtpd","BadCountries")
 	
+	wanted_mailstats_email = get_value(ConfigDB,"mailstats","CountMailstatsEmail", "no")
+	
 	count_records_to_db = 0;
 	
 	# Db save control
@@ -1250,6 +1270,7 @@ if __name__ == "__main__":
 	logging.info(f"Found {len(summary_log_entries)} summary entries and skipped {skip_count} entries")
 	sorted_log_dict = sort_log_entries(summary_log_entries)
 	logging.info(f"Sorted {len(sorted_log_dict)} entries")
+	#print(f"{sorted_log_dict}")
 	#quit(1)
 
 	columnHeaders = ['Count','WebMail','Local','MailMan','Relay','DMARC','Virus','RBL/DNS','Geoip.','Non.Conf.','Karma','Rej.Load','Del.Spam','Qued.Spam?','  Ham','TOTALS','PERCENT']
@@ -1310,6 +1331,7 @@ if __name__ == "__main__":
 		if isThonny:
 			# Initial call to logging.error( the progress bar
 			print_progress_bar(0, sorted_len, prefix='Progress:', suffix='Complete', length=50)
+		count_ignored_mailstats = 0;
 		for timestamp, data in sorted_log_dict.items():
 			i += 1
 			totalexamined += 1
@@ -1321,8 +1343,10 @@ if __name__ == "__main__":
 			hour = dt.hour
 			# parse the data
 			parsed_data = parse_data(data)
-			#Take out the mailstats email
+			#Take out the mailstats email if necessay
+			if wanted_mailstats_email == 'no':
 				if 'mailstats' in parsed_data['from-email'] and DomainName in parsed_data['from-email']:
+					count_ignored_mailstats +=1
 					continue
 			# Save the data here if necessary
 			if saveData:
@@ -1525,6 +1549,8 @@ if __name__ == "__main__":
 			if isThonny:
 				logging.error() #seperate the [progress bar]
 				
+	if count_ignored_mailstats > 0:
+		logging.info(f"Ignored {count_ignored_mailstats} mailstats emails")
 	# Compute percentages
 	total_Count  = columnCounts_2d[ColTotals][TOTALS]
 	#Column of percentages
@@ -1671,7 +1697,7 @@ if __name__ == "__main__":
 	with open(template_path, 'r') as template_file:
 		template_content = template_file.read()
 	#Use the hello string to create a suitable heading for the web page
-	html_title = hello_string.replace("logging.error(ed at"," <span class='greyed-out'>logging.error(ed at")
+	html_title = hello_string.replace("Printed at"," <span class='greyed-out'>Printeded at")
 	html_title += "</span>"
 	
 	# Create a Chameleon template instance
@@ -1696,55 +1722,55 @@ if __name__ == "__main__":
 		
 	total_html = rendered_html
 	# Add in the header information
-	rendered_html = get_heading()
+	header_rendered_html = get_heading()
-	total_html =  insert_string_after(total_html,rendered_html, "<!---Add in header information here -->")
+	total_html =  insert_string_after(total_html,header_rendered_html, "<!---Add in header information here -->")
 
 	#add in the subservient tables..(remeber they appear in the reverse order of below!)
 
 	#virus codes
 	virus_headers = ["Virus",'Count','Percent']
 	virus_title = 'Viruses found'
-	rendered_html = render_sub_table(virus_title,virus_headers,found_viruses,suppress_threshold=True)
+	virus_rendered_html = render_sub_table(virus_title,virus_headers,found_viruses,suppress_threshold=True)
 	# Add it to the total 
-	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
+	total_html = insert_string_after(total_html,virus_rendered_html, "<!---Add in sub tables here -->")
 
 	#qpsmtd codes
 	qpsmtpd_headers = ["Reason",'Count','Percent']
 	qpsmtpd_title = 'Qpsmtpd codes league table'
-	rendered_html = render_sub_table(qpsmtpd_title,qpsmtpd_headers,found_qpcodes)
+	qpsmtpd_rendered_html = render_sub_table(qpsmtpd_title,qpsmtpd_headers,found_qpcodes)
 	# Add it to the total 
-	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
+	total_html = insert_string_after(total_html,qpsmtpd_rendered_html, "<!---Add in sub tables here -->")
 	
 	
 	#Junk mails
 	junk_mail_count_headers = ['Username','Count', 'Percent']
 	junk_mail_counts = scan_mail_users()
 	junk_mail_count_title = 'Junk mail counts'
-	rendered_html = render_sub_table(junk_mail_count_title,junk_mail_count_headers,junk_mail_counts,suppress_threshold=True)
+	junk_rendered_html = render_sub_table(junk_mail_count_title,junk_mail_count_headers,junk_mail_counts,suppress_threshold=True)
 	# Add it to the total 
-	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
+	total_html = insert_string_after(total_html,junk_rendered_html, "<!---Add in sub tables here -->")
 
 	
 	#Recipient counts
 	recipient_count_headers = ["Email",'Queued','Rejected','Spam tagged','Accepted Percent']
 	recipient_count_title = 'Incoming email recipients'
-	rendered_html = render_sub_table(recipient_count_title,recipient_count_headers,recipients_found,suppress_threshold=True)
+	recipient_rendered_html = render_sub_table(recipient_count_title,recipient_count_headers,recipients_found,suppress_threshold=True)
 	# Add it to the total 
-	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
+	total_html = insert_string_after(total_html,recipient_rendered_html, "<!---Add in sub tables here -->")
 
 	#Geoip Country codes
 	geoip_headers  = ['Country','Count','Percent','Rejected?']
 	geoip_title = 'Geoip results'
-	rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list)
+	geoip_rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list)
 	# Add it to the total 
-	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
+	total_html = insert_string_after(total_html,geoip_rendered_html, "<!---Add in sub tables here -->")
 	
 	#Blacklist counts
 	blacklist_headers = ['URL','Count','Percent']
 	blacklist_title = 'Blacklist used'
-	rendered_html = render_sub_table(blacklist_title,blacklist_headers,blacklist_found,suppress_threshold=True)
+	blacklist_rendered_html = render_sub_table(blacklist_title,blacklist_headers,blacklist_found,suppress_threshold=True)
 	# Add it to the total 
-	total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
+	total_html = insert_string_after(total_html,blacklist_rendered_html, "<!---Add in sub tables here -->")
 	
 	if saveData:
 		# Close the connection
@@ -1758,17 +1784,50 @@ if __name__ == "__main__":
 		output_file.write(total_html)
 	#and create a text version if the local version of html2text is suffiicent
 	if get_html2text_version() == '2019.9.26':
-		# Get a temporary file name
+		# Get temporary file
 		temp_file_name = tempfile.mktemp()
-		html_to_text(output_path+'.html',temp_file_name)
+		temp_file_name1 = tempfile.mktemp()
-		logging.info(f"Rendered HTML saved to {temp_file_name}")
+		# see if html has links in the table entries, if not then use the current html file, else generate one
+		if not nolinks:
+			# i.e. links in html
+			# Render the template with the 2D array data and column headers
+			try:
+				rendered_html = template(array_2d=columnCounts_2d, column_headers=columnHeaders, 
+											reporting_date=analysis_date, title=html_title, 
+											version=version_string,
+											nolinks=True,
+											PreviousDate=previous_date_str,
+											NextDate=next_date_str,
+											DomainName=DomainName,
+											SystemName=SystemName,
+											enable_graphs=enable_graphs
+											)
+			except Exception as e:
+				logging.error(f"Chameleon template Exception {e}")
+			# Need to add the sub tables
+			full_rendered_html = ''.join([
+				header_rendered_html,
+				rendered_html, 
+				blacklist_rendered_html,
+				geoip_rendered_html,
+				recipient_rendered_html,
+				junk_rendered_html,
+				qpsmtpd_rendered_html,
+				virus_rendered_html
+			])
+			with open(temp_file_name, 'w') as output_file:
+				output_file.write(full_rendered_html)
+		else:
+			temp_file_name = output_path+'.html'
+		html_to_text(temp_file_name,temp_file_name1)
+		logging.info(f"Rendered HTML saved to {temp_file_name1}")
 		# and save it if required
 		if not notextfile:
 			text_file_path = output_path+'.txt'
 			# and rename it
-			os.rename(temp_file_name, text_file_path)
+			os.rename(temp_file_name1, text_file_path)
 		else:
-			text_file_path = temp_file_name
+			text_file_path = temp_file_name1
 	else:
 		text_file_path = ""
 	
@@ -1777,8 +1836,8 @@ if __name__ == "__main__":
 	html_content = None
 	text_content = None
 	#Now see if Email required
-	if EmailTextOrHTML:
+	if EmailTextorHTML:
-		if EmailTextOrHTML == "HTML" or EmailTextOrHTML == "Both":
+		if EmailTextorHTML == "HTML" or EmailTextorHTML == "Both":
 			# Send html email (default))
 			filepath = html_page_dir+"mailstats_for_"+analysis_date+".html"
 			html_content = read_html_from_file(filepath)
@@ -1790,12 +1849,12 @@ if __name__ == "__main__":
 				email_file = html_page_dir + "Email_mailstats_for_"+analysis_date
 				with open(email_file+'.html', 'w') as output_file:
 					output_file.write(html_content)
-		if EmailTextOrHTML == "Text" or EmailTextOrHTML == "Both":
+		if EmailTextorHTML == "Text" or EmailTextorHTML == "Both":
 			#filepath = html_page_dir+"mailstats_for_"+analysis_date+".txt"
 			if not text_file_path == "":
 				text_content = read_text_from_file(text_file_path)
 			else:
-				text_content = "No text avaiable as html2text (was not "
+				text_content = "No text avaiable (as html2text was not installed) "
 		if EMailSMTPUser:
 			# Send authenticated
 			logging.info("Sending authenticated")

root/usr/share/smanager/lib/SrvMngr/I18N/Modules/Mailstats/mailstats_en.lex

@@ -6,16 +6,16 @@
 'mst_RBL_Servers_to_use' => 'RBL Servers to use',
 'mst_Port_number_for_email_server' => 'Port number for email server',
 'mst_User_name_for_DB_sending' => 'User name for DB sending',
-'mst_Table_of_email_status' => 'Table of email status',
+'mst_Table_of_email_status' => '',
 'mst_Score_to_fully_reject_emmail' => 'Score to fully reject email',
 'mst_User_Password_for_email_sending' => 'User Password for email sending',
 'mst_UBL_Servers_to_use' => 'UBL Servers to use',
 'mst_Would_you_like_to_save' => 'Would you like to save data in the DB?',
 'mst_Specify_if_you_would_like' => 'Specify if you would like to receive email in text or HTML form',
 'mst_Port_number_for_DB_server' => 'Port number for DB server',
-'mst_Mailstats' => 'mailshots',
+'mst_Mailstats' => 'Daily Email Status Table',
 'mst_Email_filtering_/_exclusion' => 'Email filtering / exclusion',
-'mst_Score_for_tagging_as_spam,' => 'Score for tagging as spam But queued',
+'mst_Score_for_tagging_as_spam,' => 'Score for tagging as spam but queued',
 'mst_Accumulated_country_codes_(editable)' => 'Accumulated country codes editable',
 'mst_Date_for_Stats_display' => 'Date for Stats display',
 'mst_Enable_URIBL_checking' => 'Enable URIBL checking',
@@ -27,11 +27,14 @@
 'mst_Spamassassin_scores_-_tag_and' => 'Spamassassin scores - tag and reject levels',
 'mst_SBL_Servers_to_use' => 'SBL Servers to use',
 'mst_Save' => 'Save',
-'mst_Descriptive_paragraph' => 'Descriptive paragraph',
+'mst_Descriptive_paragraph' => 'The Mailstats contrib analyzes your qpsmtpd log files and creates a webpage and sends a periodic email to the address you specify summarizing your server\'s email activity.
+ <br>You can use the <I>Configure Mailstats</I> button to set up mailstats and some associated qpsmtpd and spamassassin properties. ',
 'mst_APPLY' => 'Apply',
 'mst_Enable_DNSBL_checking' => 'Enable DNSBL checking',
 'mst_Host_name_for_email_server' => 'Host name for email server',
 'mst_Details_for_connection_to_database' => 'Details for connection to database for saving email status',
 'mst_TABLE_panel_action_was_successful' => 'TABLE panel action was successful',
-'mst_Configure_Mailstats' => 'Configure mailshots',
+'mst_Configure_Mailstats' => 'Configure Mailstats',
 'mst_User_name_for_email_sending' => 'User name for email sending',
+'mst_Full_Window' => 'Full Window',
+'mst_Use_Cursor_keys' => '(Use the cursor keys to select the date)'

root/usr/share/smanager/themes/default/public/css/mailstats.css

@@ -4,6 +4,39 @@ Generated by: SM2Gen version:0.9(20Jan2025) Chameleon version:4.5.4 On Python:3.
 object {
     border: 1px,solid, darkgrey;
 }
+
+.inline-buttons {
+    display: flex; /* Use flexbox to arrange items horizontally */
+    gap: 10px;     /* Optional: Add space between buttons */
+  }
+
+  .inline-buttons .link {
+    /* Additional styling can be added here if needed */
+  }
+  
+  
+  .inline-buttons .link {
+    display: inline-block; /* Keep links as inline-block for button shape */
+    padding: 7px 14px;     /* Adjusted padding to approximate 70% of the original */
+    margin: 0;             /* Remove margin */
+    background-color: #efefef; /* Light gray background color */
+    color: black;          /* Text color */
+    text-decoration: none; /* Remove underline */
+    border: 2px solid #bbb; /* Thin, light gray border */
+    border-radius: 3px;    /* Slightly rounded corners */
+    font-size: 11.2px;     /* Adjusted font size to approximate 70% of the original */
+    text-align: center;    /* Center the text */
+    cursor: pointer;       /* Pointer cursor on hover */  }
+
+  /* Hover and active effects for better interaction */
+  .inline-buttons .link:hover {
+    background-color: #d9d9d9; /* Darker shade on hover */
+  }
+
+  .inline-buttons .link:active {
+    background-color: #c0c0c0; /* Even darker shade on click */
+  }
+
 .Mailstats-panel {}
 .name {}
 .rout {}

root/usr/share/smanager/themes/default/public/js/mailstats.js

@@ -66,6 +66,7 @@ function initializeSelects() {
                         'mailstats_object' + selectId.replace('StatsDate_select', '');
         
         const mailstatsObject = document.getElementById(objectId);
+        const mailstatsfull = document.getElementById('mailstats-full-window');
         
         // Check if elements exist before proceeding
         if (!dateSelect) {
@@ -88,6 +89,7 @@ function initializeSelects() {
 					
 					// Update the data attribute with the full URL
 					mailstatsObject.setAttribute('data', fullUrl);
+					mailstatsfull.setAttribute('href', fullUrl);
 					
 					// Force a refresh of the object
 					// [refresh code here]

root/usr/share/smanager/themes/default/templates/partials/_mst_TABLE.html.ep

@@ -2,11 +2,7 @@
 %# Generated by SM2Gen version:0.9(20Jan2025) Chameleon version:4.5.4 On Python:3.12.3 at 2025-04-05 11:59:08 
 %#
 <div id="Mailstats-TABLE" class="partial Mailstats-TABLE">
-	<script>
+
-		window.onload = function() {
-		SelectInput();
-	};
-	</script>
 	% if (config->{debug} == 1) {
 		<pre>
 			%= dumper $mst_data
@@ -17,23 +13,21 @@
 		% param 'trt' => $mst_data->{trt} unless param 'trt';
 		%= hidden_field 'trt' => $mst_data->{trt}
 		%# Inputs etc in here.
-
+		<br>
 		<div class=inline-buttons>
-		
-	
 			<a href='mailstatsd?trt=CONFIG' class='link link1'>
 				%= l('mst_Configure_Mailstats')
 			</a>
-		%#= link_to l('mst_Configure_Mailstats'), 'mailstatsd?trt=CONFIG' , class=>'link link1'
+			<a id='mailstats-full-window' href='' class='link link1'>
-	
+				%= l('mst_Full_Window')
-	
+			</a>
 		</div>
 		
 	
 		<h2 class='subh1'><%=l('mst_Table_of_email_status')%></h2>
 	
 		<p class='paragraph para1'>
-			%=l('mst_Descriptive_paragraph')
+			%= $c->render_to_string(inline=>$c->l('mst_Descriptive_paragraph'))
 		</p>
 	
 		<p><span class=label>
@@ -42,8 +36,8 @@
 			% my @StatsDate_options = $c->get_mailstat_dates();
 			% param 'StatsDate' => $mst_data->{StatsDate} unless param 'StatsDate';
 			%= select_field 'StatsDate' => @StatsDate_options, class => 'input', id => 'StatsDate_select'
+			%=l('mst_Use_Cursor_keys')
 		<br></span>	</p>
-	
 		<object id = 'mailstats_object' data="<%='' %>" title="<%= $c->stash('title') %>"  type="text/html" height='600px' width='100%' ><%= $c->stash('title') %> not found</object>