Switch graphs to matplotlib which does not require internet access

root/etc/e-smith/templates/etc/httpd/conf/httpd.conf/100mailstats

@@ -10,6 +10,7 @@
 	$OUT .="\n";
 	$OUT .= qq(
 	# Alias for mailstats
+	Alias "/mailstats/js" "/opt/mailstats/js"
 	Alias "/mailstats/css" "/opt/mailstats/css"
 	Alias "/mailstats" "/opt/mailstats/html"
 
@@ -29,6 +30,11 @@
 		AllowOverride None
 		Require all granted
 	</Directory>
+
+	<Directory "/opt/mailstats/js">
+		AllowOverride None
+		Require all granted
+	</Directory>
 	);
 
   }
@@ -37,4 +43,3 @@
       $OUT .= "# mailstats is disabled";
     }
 }
-

root/opt/mailstats/css/mailstats.css

@@ -202,3 +202,9 @@ p.cssvalid,p.htmlvalid {float:left;margin-right:20px}
         flex: 0 1 100%; /* 100% width for 1 column */
     }
 }
+
+/* Taken from inline in the chameleon templates */
+.maindiv {width:100%;overflow-x:auto;font-size:1cqw}
+.traffictable {border-collapse:collapse;width:98%}
+.divseeinbrowser{text-align:center;}
+.bordercollapse{border-collapse:collapse;}

root/opt/mailstats/js/mailstats.js

@@ -0,0 +1,76 @@
+window.onload = function(){doNavs();} 
+//function openTab(event, tabId){
+//// Get all elements with class="tab-content" and hide them
+//const tabContents = document.querySelectorAll('.tab-content');
+//tabContents.forEach(content => content.classList.remove('tab-content-active'));
+
+//// Get all elements with class="tab" and remove the class "tab-active"
+//const tabs = document.querySelectorAll('.tab');
+//tabs.forEach(tab => tab.classList.remove('tab-active'));
+
+//// Show the current tab content, and add an "active" class to the clicked tab
+//document.getElementById(tabId).classList.add('tab-content-active');
+//event.target.classList.add('tab-active');}
+
+function LinkCheck(url){
+	var http = new XMLHttpRequest();
+	http.open('HEAD', url, false);
+	http.send();
+	return http.status!=404;
+}
+function doNavs() {
+	var aTags = document.getElementsByTagName('a'),
+		atl = aTags.length,i;
+	for (i = 0; i < atl; i++) {
+		if (aTags[i].innerText == "Previous") {
+			if (!LinkCheck(aTags[i].href)) { 
+				aTags[i].style.visibility = "hidden";
+			} else {
+				aTags[i].style.visibility = "visible";
+			}
+		} else if (aTags[i].innerText == "Next") {
+			if (!LinkCheck(aTags[i].href)) { 
+				aTags[i].style.visibility = "hidden";
+			} else {
+				aTags[i].style.visibility = "visible";
+			}
+		}
+	}
+}
+
+function openTab(evt, tabName) {
+// Declare all variables
+var i, tab_content, tab;
+
+// Get all elements with class="tab_content" and hide them
+tab_content = document.getElementsByClassName("tab-content");
+for (i = 0; i < tab_content.length; i++) {
+tab_content[i].style.display = "none";
+}
+
+// Get all elements with class="tab" and remove the class "active"
+tab = document.getElementsByClassName("tab");
+for (i = 0; i < tab.length; i++) {
+tab[i].className = tab[i].className.replace(" tab-active", "");
+}
+
+// Show the current tab, and add an "active" class to the link that opened the tab
+document.getElementById(tabName).style.display = "block";
+evt.currentTarget.className += " tab-active";
+
+// Store the active tab index
+sessionStorage.setItem('activeTab', evt.currentTarget.getAttribute("data-index"));
+}
+
+document.addEventListener("DOMContentLoaded", function() {
+    // Attach click event handler for all divs with the class "tab"
+    document.querySelectorAll(".tab").forEach(function(tab) {
+        tab.addEventListener("click", function(event) {
+            // Get the data-index attribute value
+            const tabIndex = this.getAttribute("data-index");
+
+            // Dynamically call openTab with the correct tab parameter
+            openTab(event, `tab${tabIndex}`);
+        });
+    });
+});

root/opt/mailstats/templates/mailstats-sub-table.html.pt

@@ -6,7 +6,7 @@
 	<tal:block condition="threshold == 0">
 		<br>
 	</tal:block>
-    <table style="border-collapse:collapse;">
+    <table class="bordercollapse">
         <thead>
             <tr>
                 <th tal:repeat="header column_headers">${header}</th>

root/opt/mailstats/templates/mailstats.html.pt

@@ -4,39 +4,10 @@
 	<title>SMEServer Mailstats</title>
 	<link rel='stylesheet' type='text/css' href='css/mailstats.css' />
 	<!-- Check links -->
-	<script>
-		function LinkCheck(url){
-			var http = new XMLHttpRequest();
-			http.open('HEAD', url, false);
-			http.send();
-			return http.status!=404;
-		}
-		function doNavs() {
-			var aTags = document.getElementsByTagName('a'),
-				atl = aTags.length,i;
-			for (i = 0; i < atl; i++) {
-				if (aTags[i].innerText == "Previous") {
-					if (!LinkCheck(aTags[i].href)) { 
-						aTags[i].style.visibility = "hidden";
-					} else {
-						aTags[i].style.visibility = "visible";
-					}
-				} else if (aTags[i].innerText == "Next") {
-					if (!LinkCheck(aTags[i].href)) { 
-						aTags[i].style.visibility = "hidden";
-					} else {
-						aTags[i].style.visibility = "visible";
-					}
-				}
-			}
-		}
-	</script>
-	<style>
 	<!--css here-->
-	</style>
 </head>
 <body>
-   	<div style="width:100%;overflow-x:auto;font-size:1cqw">
+   	<div class=maindiv>
 		<!---Navigation here-->
 		<div class='linksattop'>
 			<a class='prevlink' href='http://${SystemName}.${DomainName}/mailstats/mailstats_for_${PreviousDate}.html'>Previous</a>
@@ -54,15 +25,16 @@
     <br />
     <!--Tabs -->
     <div class="tab-container">
-    <div class="tab tab-active" data-index="0" onclick="openTab(event, 'tab1')">Table</div>
+		<div class="tab tab-active" data-index="0" >Table</div>
-    <div class="tab" data-index="1" onclick="openTab(event, 'tab2')">Stacked Bar Graph</div>
+		<div tal:condition="enable_graphs" class="tab" data-index="1">Line Graph</div>
-    <div class="tab" data-index="2" onclick="openTab(event, 'tab3')">Heat Map</div>
+		<div tal:condition="enable_graphs" class="tab" data-index="2">Stacked Bar Graph</div>
-    <div class="tab" data-index="3" onclick="openTab(event, 'tab4')">Line Graph</div>
+		<div tal:condition="enable_graphs" class="tab" data-index="3">Scatter Graph</div>
+		<div tal:condition="enable_graphs" class="tab" data-index="4">Pie Chart</div>
     </div>
 
-	<div id="tab1" class="tab-content tab-content-active">
+	<div id="tab0" class="tab-content tab-content-active">
 		<div class = "maintable">
-	<table style="border-collapse:collapse;width:98%">
+			<table class="traffictable">
 				<thead>
 					<tr>
 						<th>Date/Time</th>
@@ -101,80 +73,34 @@
 				</tbody>
 			</table>
 		</div>
-	</div>
-	<!-- Next Tab-->
-	<div id="tab2" class="tab-content">
-		<a href="stacked_bar_${reporting_date}.html">${structure: stacked_bar_graph}</a>
-	</div>
-	<!-- Next Tab-->
-	<div id="tab3" class="tab-content">
-		<a href="heatmap_${reporting_date}.html">${structure: heatmap}</a>
-	</div>
-	<!-- Next Tab-->
-	<div id="tab4" class="tab-content">
-		<a href="line_graph_${reporting_date}.html">${structure: line_graph}</a>
-	</div>
-	
 		<div class = "subtables">
 			<div class="table-container">
 			<!---Add in sub tables here -->
 			</div>
 		</div>
+	</div>
+	
+	<!-- Next Tab-->
+	<div tal:condition="enable_graphs" id="tab1" class="tab-content">
+		<img src="line_graph_${reporting_date}.png">
+	</div>
+	<!-- Next Tab-->
+	<div tal:condition="enable_graphs" id="tab2" class="tab-content">
+		<img src="bar_graph_${reporting_date}.png">
+	</div>
+	<!-- Next Tab-->
+	<div tal:condition="enable_graphs" id="tab3" class="tab-content">
+		<img src="scatter_graph_${reporting_date}.png">
+	</div>
+	<!-- Next Tab-->
+	<div tal:condition="enable_graphs" id="tab4" class="tab-content">
+		<img src="pie_chart_${reporting_date}.png">
+	</div>	
 	<br />
 		
     <footer class="footer">${version}</footer>
 
-	<script>
+	<script type='text/javascript' src='js/mailstats.js' ></script> 
-		window.onload = function(){doNavs();} 
-		//function openTab(event, tabId){
-		//// Get all elements with class="tab-content" and hide them
-		//const tabContents = document.querySelectorAll('.tab-content');
-		//tabContents.forEach(content => content.classList.remove('tab-content-active'));
-
-		//// Get all elements with class="tab" and remove the class "tab-active"
-		//const tabs = document.querySelectorAll('.tab');
-		//tabs.forEach(tab => tab.classList.remove('tab-active'));
-
-		//// Show the current tab content, and add an "active" class to the clicked tab
-		//document.getElementById(tabId).classList.add('tab-content-active');
-		//event.target.classList.add('tab-active');}
-		
-		function openTab(evt, tabName) {
-		// Declare all variables
-		var i, tab_content, tab;
-
-		// Get all elements with class="tab_content" and hide them
-		tab_content = document.getElementsByClassName("tab-content");
-		for (i = 0; i < tab_content.length; i++) {
-		tab_content[i].style.display = "none";
-		}
-
-		// Get all elements with class="tab" and remove the class "active"
-		tab = document.getElementsByClassName("tab");
-		for (i = 0; i < tab.length; i++) {
-		tab[i].className = tab[i].className.replace(" tab-active", "");
-		}
-
-		// Show the current tab, and add an "active" class to the link that opened the tab
-		document.getElementById(tabName).style.display = "block";
-		evt.currentTarget.className += " tab-active";
-
-		// Store the active tab index
-		sessionStorage.setItem('activeTab', evt.currentTarget.getAttribute("data-index"));
-		}
-
-		document.addEventListener("DOMContentLoaded", function() {
-		// Get the stored tab index from sessionStorage
-		let activeTab = sessionStorage.getItem('activeTab');
-
-		// If there's a stored tab, show that tab
-		if (activeTab !== null) {
-		document.querySelector(`[data-index="$${activeTab}"]`).click();
-		}
-		});
-
-		
-	</script> 
 
 <!--	
 	<p class="cssvalid">

root/usr/bin/mailstats.py

@@ -56,14 +56,15 @@
 # pip3 install numpy
 # pip3 install plotly
 # pip3 install pandas
+# pip3 install matplotlib
 #
 # Rocky8: (probably - not yet checked this)
 #
 # dnf install python3-chameleon --enablerepo=epel
 # dnf install html2text --enablerepo=epel
+# dnf install python3-matplotlib
 # pip3 install numpy
 # pip3 pymysql
-# pip3 install plotly
 # pip3 install pandas
 #
 #
@@ -89,6 +90,12 @@ import plotly.express as px
 import colorsys
 import pymysql
 import json
+enable_graphs = True;
+try:
+	import matplotlib.pyplot as plt
+except ImportError:
+	print("Matplotlib is not installed  - no graphs")
+	enable_graphs = False;
 
 Mailstats_version = '1.2'
 build_date_time = "2024-06-18 12:03:40OURCE" 
@@ -129,251 +136,145 @@ PERCENT = TOTALS + 1
 ColTotals = 24
 ColPercent = 25
 
-
+def transform_to_dict(data, keys, iso_date):
-def sanitize_and_filter_data_for_stacked_bar(data2d, xLabels, yLabels, exclude_columns_labels, exclude_rows_labels):
     """
-    Sanitize data by removing unwanted columns and rows, and converting to numeric values.
+    Transforms a 26x17 list of lists into a list of dictionaries with specified keys.
 
-    Parameters:
+    Args:
-    - data2d (list of lists): A 2D list containing the data.
+        data (list): A 26x17 list of lists.
-    - xLabels (list): Current labels for the x-axis.
+        keys (list): A 1D array specifying the keys for the dictionaries.
-    - yLabels (list): Current labels for the y-axis.
+        iso_date (str): A date in ISO format to prepend to each row number.
-    - exclude_columns_labels (list): Labels of columns to exclude from the data and x-axis.
-    - exclude_rows_labels (list): Labels of rows to exclude from the y-axis.
 
     Returns:
-    - numpy.ndarray: Sanitized 2D numpy array with numeric data.
+        list: A list of dictionaries with transformed data.
-    - list: Filtered x-axis labels.
-    - list: Filtered y-axis labels.
     """
-    def to_numeric(value):
+    # Validate input dimensions
-        try:
+    if len(data) != 26:
-            if isinstance(value, str):
+        raise ValueError("Input data must have 26 rows.")
-                # Remove any extra characters like '%' and convert to float
+    if len(keys) != len(data[0]):  # Account for the new column
-                return float(value.replace('%', '').strip())
+        raise ValueError(f"Keys must match the number of columns after transformation {len(keys)} {len(data[0])}")
+    
+    # Remove rows 25 and 26
+    filtered_data = data[:24]
+    
+    # and same for keys
+    modified_keys = keys[1:-2]
+
+    # Add new column with ISO date and row number
+    transformed_data = []
+    for i, row in enumerate(filtered_data):
+        new_column_value = f"{i}"  #f"{iso_date},{i}"
+        transformed_row = [new_column_value] + row[1:-2]  # Remove first and last two columns
+        transformed_data.append(transformed_row)
+
+    # Convert each row into a dictionary using supplied keys
+    result = [dict(zip(["Time"] + modified_keys, row)) for row in transformed_data]
+
+    return result
+
+
+def create_graph(data_dict, graph_type="line", output_file="graph.png",iso_date='1970-01-01'):
+	"""
+	Creates a graph from nested list data with hours as x-axis.
+
+	Args:
+		data_dict (list): List structure where:
+			- Each element is a list representing hour data
+			- First element is the hour (0-23)
+			- Remaining elements are counts for different types/categories
+		graph_type (str): Type of graph to create ("line", "bar", "scatter", "pie").
+		output_file (str): Path to save the image file.
+	"""
+	# Check if data is empty
+	if not data_dict:
+		raise ValueError("Input data cannot be empty")
+
+	# Extract hours (from the "NewColumn" key)
+	hours = [row["Time"] for row in data_dict]  # First column is the ISO date + row number
+
+	# Extract types (keys excluding "NewColumn")
+	types = [key for key in data_dict[0].keys() if key != "Time"]  # Dynamically get keys except "NewColumn"
+
+	# Extract counts for each type
+	counts = {typ: [row[typ] for row in data_dict] for typ in types}
+
+	plt.figure(figsize=(10, 6))  # Create a figure
+
+	# Generate different types of graphs based on the input parameter
+	if graph_type == "line":
+		for typ in types:
+			plt.plot(hours, counts[typ], label=typ, marker='o')
+		plt.title(f"Line Graph for {iso_date}")
+		plt.xlabel("Hours")
+		plt.ylabel("Counts")
+
+	elif graph_type == "bar":
+		bottom = [0] * len(hours)
+		for typ in types:
+			plt.bar(hours, counts[typ], bottom=bottom, label=typ)
+			bottom = [b + y for b, y in zip(bottom, counts[typ])]
+		plt.title(f"Bar Graph for {iso_date}")
+		plt.xlabel("Hours")
+		plt.ylabel("Counts")
+
+	elif graph_type == "scatter":
+		for typ in types:
+			plt.scatter(hours, counts[typ], label=typ)
+		plt.title(f"Scatter Plot for {iso_date}")
+		plt.xlabel("Hours")
+		plt.ylabel("Counts")
+
+	elif graph_type == "pie":
+		total_counts = {typ: sum(counts[typ]) for typ in types}
+		total_sum = sum(total_counts.values())
+		threshold_percent = 0.01 * total_sum
+
+		# Separate filtered counts and "Other" counts
+		filtered_counts = {}
+		other_total = 0
+
+		for typ, value in total_counts.items():
+			if value > 0 and value >= threshold_percent:
+				filtered_counts[typ] = value
 			else:
-                return float(value)
+				other_total += value
-        except ValueError:
-            return 0.0  # Default to 0 if conversion fails
 
-    # Filter out columns based on their labels
+		# Add "Other" category if there are values below the threshold
-    exclude_columns_indices = [xLabels.index(label) for label in exclude_columns_labels if label in xLabels]
+		if other_total > 0:
+			filtered_counts["Other"] = other_total
 
-    filtered_data2d = [
+		# Prepare data for the pie chart
-        [to_numeric(value) for idx, value in enumerate(row) if idx not in exclude_columns_indices]
+		labels = filtered_counts.keys()
-        for row in data2d
+		sizes = filtered_counts.values()
-    ]
 
-    filtered_xLabels = [label for idx, label in enumerate(xLabels) if idx not in exclude_columns_indices]
+		# Plot the pie chart
+		plt.pie(sizes, labels=labels, autopct='%1.1f%%', startangle=90)
+		plt.title(f"Pie Chart for {iso_date}")
 
-    # Filter out rows based on their labels
-    filtered_data2d = [row for label, row in zip(yLabels, filtered_data2d) if label not in exclude_rows_labels]
-    filtered_yLabels = [label for label in yLabels if label not in exclude_rows_labels]
-    
-    # Convert filtered data to numpy array
-    return np.array(filtered_data2d), filtered_xLabels, filtered_yLabels
-      
-def generate_distinct_colors(num_colors):
-    """Generate distinct colors using HSV color space."""
-    colors = []
-    for i in range(num_colors):
-        hue = i / num_colors
-        saturation = 0.7
-        value = 0.9
-        r, g, b = colorsys.hsv_to_rgb(hue, saturation, value)
-        colors.append(f'rgb({int(r * 255)},{int(g * 255)},{int(b * 255)})')
-    return colors
-
-def create_stacked_bar_graph(data2d, xLabels, yLabels, save_path='stacked_bar_graph.html'):
-    """
-    Creates and saves a stacked bar graph from given 2D numpy array data using Plotly.
-    
-    Parameters:
-    - data2d (list of lists or numpy.ndarray): A 2D list or numpy array containing the data.
-    - xLabels (list): A list of category labels for the x-axis.
-    - yLabels (list): A list of labels for the y-axis (e.g., hours).
-    - save_path (str): The path where the plot image will be saved.
-    """
-    # Identify columns to be removed based on their headers (label names) and indices (hours 24 and 25)
-    exclude_columns_labels = ["Count", "PERCENT","TOTALS"]
-    exclude_rows_labels = ["24:00", "25:00"]
-
-    # Ensure input yLabels correspond to the data
-    if len(yLabels) != len(data2d):
-        raise ValueError(f"The length of yLabels {len(yLabels)} must match the number of rows in the data {len(data2d)}.")
-    
-    # Sanitize and filter the data
-    sanitized_data, filtered_xLabels, filtered_yLabels = sanitize_and_filter_data_for_stacked_bar(data2d, xLabels, yLabels, exclude_columns_labels, exclude_rows_labels)
-    
-    # Ensure that the length of yLabels matches the number of rows (0 to n should be n+1 rows)
-    if len(filtered_yLabels) != sanitized_data.shape[0]:
-        raise ValueError(f"The length of filtered_yLabels {len(filtered_yLabels)} must match the number of rows in the data {sanitized_data.shape[0]}.")
-    
-    # Transpose the data so that hours are on the x-axis and categories are stacked in the y-axis
-    transposed_data = sanitized_data.T
-    
-    fig = go.Figure()
-    
-    # Get unique colors for each category
-    extended_colors = generate_distinct_colors(len(filtered_xLabels))
-    
-    for i, category in enumerate(filtered_xLabels):
-        fig.add_trace(go.Bar(
-            name=category,
-            x=filtered_yLabels,
-            y=transposed_data[i],
-            marker_color=extended_colors[i % len(extended_colors)]  # Cycle through the colors if there are more categories than colors
-
-        ))
-    
-    fig.update_layout(
-        barmode='stack',
-        title='Stacked Bar Graph by Hour',
-        xaxis=dict(title='Hour'),
-        yaxis=dict(title='Values'),
-        legend_title_text='Categories',
-        margin = {
-            'l': 50,  #left margin
-            'r': 120, #right margin
-            't': 50,  #top margin
-            'b': 50   #bottom margin
-        }
- 
-    )
-    
-    # Save the graph to an HTML file
-    fig.write_html(save_path)
-    # Write it to a var and return the string
-    graph_html = fig.to_html(full_html=False,include_plotlyjs='https://cdn.plot.ly/plotly-latest.min.js')
-    return graph_html
- 
-def sanitize_and_filter_data(data2d, exclude_labels, xLabels):
-    """
-    Sanitize data by removing unwanted columns and converting to numeric values.
-    
-    Parameters:
-    - data2d (list of lists): A 2D list containing the data.
-    - exclude_labels (list): Labels to exclude from the data and x-axis.
-    - xLabels (list): Current labels for the x-axis.
-    
-    Returns:
-    - numpy.ndarray: Sanitized 2D numpy array with numeric data.
-    - list: Filtered x-axis labels.
-    """
-    def to_numeric(value):
-        try:
-            if isinstance(value, str):
-                # Remove any extra characters like '%' and convert to float
-                return float(value.replace('%', '').strip())
 	else:
-                return float(value)
+		raise ValueError(f"Unsupported graph type: {graph_type}")
-        except ValueError:
-            return 0.0  # Default to 0 if conversion fails
 
-    # Create a boolean array for columns to keep (not in exclude_labels)
+	if graph_type != "pie":
-    columns_to_keep = [label not in exclude_labels for label in xLabels]
+		plt.xticks(hours)
+		plt.grid(alpha=0.3)
+		plt.legend()
 
-    # Filter out the columns both from the data and xLabels
+	# Save the graph to a file
-    filtered_data2d = []
+	plt.tight_layout()
-    for row in data2d:
+	plt.savefig(output_file)
-        filtered_row = [to_numeric(value) for keep, value in zip(columns_to_keep, row) if keep]
+	plt.close()
-        filtered_data2d.append(filtered_row)
 
-    filtered_xLabels = [label for label, keep in zip(xLabels, columns_to_keep) if keep]
+# def convert_to_numeric(data):
-
+    # """
-    return np.array(filtered_data2d), filtered_xLabels
+    # Converts all values in a nested list or dictionary to numeric types (int or float).
-
+    # """
-def create_heatmap(data2d, xLabels, yLabels, save_path='heatmap.html'):
+    # for i in range(len(data)):
-    """
+        # for j in range(1, len(data[i])):  # Skip the first column (hour)
-    Creates and saves a heatmap from given 2D numpy array data using Plotly.
+            # try:
-    Parameters:
+                # data[i][j] = float(data[i][j])  # Convert to float
-    - data2d (list of lists or numpy.ndarray): A 2D list or numpy array containing the data.
+            # except ValueError:
-    - xLabels (list): A list of category labels for the x-axis.
+                # raise ValueError(f"Non-numeric value found: {data[i][j]}")
-    - yLabels (list): A list of labels for the y-axis (e.g., hours).
+    # return data
-    - save_path (str): The path where the plot image will be saved.
-    """
-    excluded_columns = ["Count", "PERCENT", "TOTALS"]
-    # Remove rows 24 and 25 by slicing the data and labels
-    data2d = data2d[:24]
-    yLabels = yLabels[:24]  # Ensure yLabels also excludes those rows
-
-    # Sanitize and filter the data
-    sanitized_data, filtered_xLabels = sanitize_and_filter_data(data2d, excluded_columns, xLabels)
-
-    # Ensure that the length of yLabels matches the number of rows (0 to n should be n+1 rows)
-    if len(yLabels) != sanitized_data.shape[0]:
-        raise ValueError("The length of yLabels must match the number of rows in the data.")
-
-    # Create the heatmap
-    # Define a custom color scale where 0 is white
-    color_scale = [
-        [0, "lightgrey"],
-        [0.3, "blue"],
-        [0.6, 'green'],
-        [0.75,'yellow'],
-        [1,'red']
-    ]
-    fig = px.imshow(sanitized_data,
-                    labels=dict(x="Category", y="Hour", color="Count"),
-                    x=filtered_xLabels,
-                    y=yLabels,                  
-                    color_continuous_scale=color_scale)
-    
-    fig.update_layout(
-        title='Heatmap of Counts by Category per Hour',
-        xaxis_nticks=len(filtered_xLabels),
-        yaxis_nticks=len(yLabels),
-        margin=dict(l=0, r=0, t=30, b=0)
-
-    )
-    fig.update_xaxes(showticklabels=True, side='bottom', showline=True, linewidth=2, linecolor='black', mirror=True)
-    fig.update_yaxes(showticklabels=True, showline=True, linewidth=2, linecolor='black', mirror=True)
-   
-    fig.write_html(save_path)  
-    # Write it to a var and return the string
-    graph_html = fig.to_html(full_html=False,include_plotlyjs='https://cdn.plot.ly/plotly-latest.min.js')
-    return graph_html
-  
-   
-def create_line_chart(data2d, xLabels, yLabels, save_path='line_chart.html'):
-    fig = go.Figure()
-    
-    excluded_columns = ["Count", "PERCENT", "TOTALS"]
-    # Remove rows 24 and 25 by slicing the data and labels
-    data2d = data2d[:24]
-    yLabels = yLabels[:24]  # Ensure yLabels also excludes those rows
-
-    # Sanitize and filter the data
-    sanitized_data, filtered_xLabels = sanitize_and_filter_data(data2d, excluded_columns, xLabels)
-
-    # Ensure that the length of yLabels matches the number of rows (0 to n should be n+1 rows)
-    if len(yLabels) != sanitized_data.shape[0]:
-        raise ValueError("The length of yLabels must match the number of rows in the data.")
-
-    # Remove rows with all zero elements and the corresponding categories
-    nonzero_rows_indices = np.where(~np.all(sanitized_data == 0, axis=0))[0]  # find rows with non-zero elements
-    sanitized_data = sanitized_data[:, nonzero_rows_indices]
-    filtered_xLabels = [filtered_xLabels[i] for i in nonzero_rows_indices]  # update filtered_xLabels
-
-    for i, category in enumerate(filtered_xLabels):
-        fig.add_trace(go.Scatter(
-            mode='lines+markers',
-            name=category,
-            x= [f'{j:02d}:00' for j in range(sanitized_data.shape[0])],  
-            y=sanitized_data[:, i]
-        ))
-
-    fig.update_layout(
-        title='Line Chart of Counts by Category per Hour',
-        xaxis=dict(title='Hour'),
-        yaxis=dict(title='Count'),
-        legend_title_text='Category'
-    )
-    
-    fig.write_html(save_path)
-    # Write it to a var and return the string
-    graph_html = fig.to_html(full_html=False,include_plotlyjs='https://cdn.plot.ly/plotly-latest.min.js')
-    return graph_html
 
 def save_summaries_to_db(cursor, conn, date_str, hour, parsed_data):
     # Convert parsed_data to JSON string
@@ -529,6 +430,8 @@ def read_in_relevant_log_file(file_path,analysis_date=yesterday):
 					timestamp = timestamp.replace(year=yesterday_year)
 				except (ValueError, TypeError) as e:
 					print(f"Error {e} line {line_count} on timestamp extract {timestamp_str}:{entry[1]}")
+					ignore_record_count += 1
+					continue
 				#print(f"Stamps: {timestamp.date()} {analysis_date.date()}")
 				if timestamp.date() == analysis_date.date():
 					log_entries.append((timestamp, entry[1]))
@@ -571,7 +474,7 @@ def parse_data(data):
 #		for part in fields:
 #			print(f"{i}: {part}")
 #			i  = i +1
-#		quit()
+#		(quit)()
 	# and mapping:
 	try:
 		return_dict = {
@@ -861,7 +764,7 @@ def read_html_from_file(filepath):
 	# Read in CSS 
 	with open(css_path, 'r', encoding='utf-8') as file:
 		css_contents = file.read()
-	html_contents = insert_string_after(html_contents,"\n"+css_contents,"<!--css here-->")
+	html_contents = insert_string_after(html_contents,"\n<style>"+css_contents+"</style>","<!--css here-->")
 	return html_contents
 	
 def read_text_from_file(filepath):
@@ -1658,10 +1561,27 @@ if __name__ == "__main__":
 	previous_date_str = previous_date.strftime(day_format)
 
 	# Create graphs of data
-	yLabels = [f'{i:02d}:00' for i in range(len(columnCounts_2d))]
+	
-	stacked_Bar_html = create_stacked_bar_graph(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'stacked_bar_'+analysis_date+'.html')
+	# yLabels = [f'{i:02d}:00' for i in range(len(columnCounts_2d))]
-	heatmap_html = create_heatmap(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'heatmap_'+analysis_date+'.html')
+	# stacked_Bar_html = create_stacked_bar_graph(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'stacked_bar_'+analysis_date+'.html')
-	line_graph_html = create_line_chart(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'line_graph_'+analysis_date+'.html')
+	# heatmap_html = create_heatmap(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'heatmap_'+analysis_date+'.html')
+	# line_graph_html = create_line_chart(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'line_graph_'+analysis_date+'.html')
+	
+	columnCounts_2d_dict = transform_to_dict(columnCounts_2d,columnHeaders,analysis_date)
+
+	#Export as json for testing		
+	# with open("/opt/mailstats/html/colCounts_2d.json", "w") as json_file:
+		# json.dump(columnCounts_2d, json_file)
+	# with open("/opt/mailstats/html/colCounts_2d-dict", "w") as json_file:
+		# json.dump(columnCounts_2d_dict, json_file)
+	# with open("/opt/mailstats/html/keys.json", "w") as json_file:
+		# json.dump(columnHeaders, json_file)
+
+	if enable_graphs:
+		create_graph(columnCounts_2d_dict, "line", html_page_dir+"line_graph_"+analysis_date+".png",analysis_date)
+		create_graph(columnCounts_2d_dict, "bar", html_page_dir+"bar_graph_"+analysis_date+".png",analysis_date)
+		create_graph(columnCounts_2d_dict, "scatter", html_page_dir+"scatter_graph_"+analysis_date+".png",analysis_date)
+		create_graph(columnCounts_2d_dict, "pie", html_page_dir+"pie_chart_"+analysis_date+".png",analysis_date)
 
 	#Now apply the results to the chameleon template - main table
 	# Path to the template file
@@ -1682,13 +1602,11 @@ if __name__ == "__main__":
 										reporting_date=analysis_date, title=html_title, 
 										version=version_string,
 										nolinks=nolinks,
-										stacked_bar_graph=stacked_Bar_html,
-										heatmap=heatmap_html,
-										line_graph=line_graph_html,
 										PreviousDate=previous_date_str,
 										NextDate=next_date_str,
 										DomainName=DomainName,
-										SystemName=SystemName
+										SystemName=SystemName,
+										enable_graphs=enable_graphs
 										)
 		except Exception as e:
 			print(f"Chameleon template Exception {e}")
@@ -1785,7 +1703,7 @@ if __name__ == "__main__":
 			filepath = html_page_dir+"mailstats_for_"+analysis_date+".html"
 			html_content = read_html_from_file(filepath)
 			# Replace the Navigation by a "See in browser" prompt
-			replace_str = f"<div class='divseeinbrowser' style='text-align:center;'><a class='seeinbrowser' href='http://{SystemName}.{DomainName}/mailstats/mailstats_for_{analysis_date}.html'>See in browser</a></div>"
+			replace_str = f"<div class='divseeinbrowser'><a class='seeinbrowser' href='http://{SystemName}.{DomainName}/mailstats/mailstats_for_{analysis_date}.html'>See in browser</a></div>"
 			html_content = replace_between(html_content, "<div class='linksattop'>", ">Next</a></div>", replace_str)
 			if not noemailfile:
 				# Write out the email html to a web page

smeserver-mailstats.spec

@@ -24,6 +24,7 @@ Requires:       python36
 Requires:	html2text
 Requires:	python3-chameleon
 Requires:       python3-mysql
+Requires:       python3-matplotlib
 AutoReqProv: no
 
 %description
@@ -126,7 +127,6 @@ sed -i "s|__BUILD_DATE_TIME__|$now|" $RPM_BUILD_ROOT/usr/bin/mailstats.py
 %pre
 /usr/bin/pip3 install -q pymysql
 /usr/bin/pip3 install -q numpy
-/usr/bin/pip3 install -q plotly
 /usr/bin/pip3 install -q pandas
 
 %clean