smecontribs/smeserver-mailstats
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
c7cf518477
smeserver-mailstats/root/usr/bin/mailstats.py

1547 lines
56 KiB
Python
Raw Blame History

#
# Mailstats.py
#
#
# This script provides daily SpamFilter statistics.
#
# Mailstats
#
# usage: mailstats.py [-h] [-d DATE] [-ef EMAILFILE] [-tf TEXTFILE] [--version]
# [-db DBSAVE]
#
# Mailstats
#
# optional arguments:
# -h, --help show this help message and exit
# -d DATE, --date DATE Specify a valid date (yyyy-mm-dd) for the analysis
# -ef EMAILFILE, --emailfile EMAILFILE
# Save an html file of the email sent (y/N)
# -tf TEXTFILE, --textfile TEXTFILE
# Save a txt file of the html page (y/N)
# --version show program's version number and exit
# -db DBSAVE, --dbsave DBSAVE
# Force save of summary logs in DB (y/N)
#
#
# (June 2024 - bjr) Re-written in Python from Mailstats.pl (Perl) to conform to SME11 / Postfix / qpsmtpd log formats
# and html output added
#
# Todo:
# 2 Other stats
# 3. Extra bits for sub tables - DONE
# 4. Percent char causes sort to fail - look at adding it in the template - DONE
# 5. Chase disparity in counts betweeen old mailstats and this - Some of it DONE
# 6. Count emails delivered over ports 25/587/465 (SMTPS?)
# 7. Arrange that the spec file overwrites the date even if it has been overwritten before
# 8. Allow mailstats pages to be public or private (=> templating the fragment)) - DONE
# 9. Update format of the summarylogs page - DONE but still WIP
# 10. Add in links to summarylogs in web pages - DONE but still WIP
# 11. Move showSummaryLogs.php to individual directory "/opt/mailstats/php"
# 12. Make sure other directories not visible through apache
#
# Future:
# 1. Write summary line for each transaction to DB and link to it through cell in main table -DONE (write to DB))
# 2. Make DB password something more obscure.
# 3. Prune the DB according to parameter - delete corresponding page in opt/mailstats/html
# 4. Prune the html directory according to parameter
#
# Even more Future (if ever))
# 2. Link each summary line through DB to actual transaction lines
#
# Centos7:
# yum install python3-chameleon --enablerepo=epel
# yum install html2text --enablerepo=epel
# yum install mysql-connector-python --enablerepo=epel (not sure if this is required as well the pip3))
# pip3 install mysql-connector
# pip3 install numpy
# pip3 install plotly
# pip3 install pandas
#
# Rocky8: (probably - not yet checked this)
#
# dnf install python3-chameleon --enablerepo=epel
# dnf install html2text --enablerepo=epel
#
#
from datetime import datetime, timedelta
import sys
from chameleon import PageTemplateFile,PageTemplate
import pkg_resources
import re
import ipaddress
import subprocess
import os
from collections import defaultdict
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import codecs
import argparse
import tempfile
import mysql.connector
import numpy as np
import plotly.graph_objects as go
import plotly.express as px
import colorsys
Mailstats_version = '1.2'
build_date_time = "2024-06-18 12:03:40OURCE"
build_date_time = build_date_time[:19] #Take out crap that sneaks in.
#if build_date_time == "2024-06-18 12:03:40OURCE":
# build_date_time = "Unknown"
script_dir = os.path.dirname(os.path.abspath(__file__))
data_file_path = script_dir+'/../..' #back to the top
now = datetime.now()
yesterday = now - timedelta(days=1)
formatted_yesterday = yesterday.strftime("%Y-%m-%d")
#html_page_path = data_file_path+"/home/e-smith/files/ibays/mesdb/html/mailstats/"
html_page_dir = data_file_path+"/opt/mailstats/html/"
template_dir = data_file_path+"/opt/mailstats/templates/"
logs_dir = data_file_path+"/opt/mailstats/logs/"
# Column numbering (easy to renumber or add one in)
Hour = 0
WebMail = Hour + 1
Local = WebMail + 1
MailMan = Local + 1
Relay = MailMan + 1
DMARC = Relay + 1
Virus = DMARC + 1
RBLDNS = Virus + 1
Geoip = RBLDNS + 1
NonConf = Geoip + 1
RejLoad = NonConf + 1
Karma = RejLoad + 1
DelSpam = Karma + 1
QuedSpam = DelSpam + 1
Ham = QuedSpam + 1
TOTALS = Ham + 1
PERCENT = TOTALS + 1
ColTotals = 24
ColPercent = 25
import mysql.connector
import json
def sanitize_and_filter_data_for_stacked_bar(data2d, xLabels, yLabels, exclude_columns_labels, exclude_rows_labels):
"""
Sanitize data by removing unwanted columns and rows, and converting to numeric values.
Parameters:
- data2d (list of lists): A 2D list containing the data.
- xLabels (list): Current labels for the x-axis.
- yLabels (list): Current labels for the y-axis.
- exclude_columns_labels (list): Labels of columns to exclude from the data and x-axis.
- exclude_rows_labels (list): Labels of rows to exclude from the y-axis.
Returns:
- numpy.ndarray: Sanitized 2D numpy array with numeric data.
- list: Filtered x-axis labels.
- list: Filtered y-axis labels.
"""
def to_numeric(value):
try:
if isinstance(value, str):
# Remove any extra characters like '%' and convert to float
return float(value.replace('%', '').strip())
else:
return float(value)
except ValueError:
return 0.0 # Default to 0 if conversion fails
# Filter out columns based on their labels
exclude_columns_indices = [xLabels.index(label) for label in exclude_columns_labels if label in xLabels]
filtered_data2d = [
[to_numeric(value) for idx, value in enumerate(row) if idx not in exclude_columns_indices]
for row in data2d
]
filtered_xLabels = [label for idx, label in enumerate(xLabels) if idx not in exclude_columns_indices]
# Filter out rows based on their labels
filtered_data2d = [row for label, row in zip(yLabels, filtered_data2d) if label not in exclude_rows_labels]
filtered_yLabels = [label for label in yLabels if label not in exclude_rows_labels]
# Convert filtered data to numpy array
return np.array(filtered_data2d), filtered_xLabels, filtered_yLabels
def generate_distinct_colors(num_colors):
"""Generate distinct colors using HSV color space."""
colors = []
for i in range(num_colors):
hue = i / num_colors
saturation = 0.7
value = 0.9
r, g, b = colorsys.hsv_to_rgb(hue, saturation, value)
colors.append(f'rgb({int(r * 255)},{int(g * 255)},{int(b * 255)})')
return colors
def create_stacked_bar_graph(data2d, xLabels, yLabels, save_path='stacked_bar_graph.html'):
"""
Creates and saves a stacked bar graph from given 2D numpy array data using Plotly.
Parameters:
- data2d (list of lists or numpy.ndarray): A 2D list or numpy array containing the data.
- xLabels (list): A list of category labels for the x-axis.
- yLabels (list): A list of labels for the y-axis (e.g., hours).
- save_path (str): The path where the plot image will be saved.
"""
# Identify columns to be removed based on their headers (label names) and indices (hours 24 and 25)
exclude_columns_labels = ["Count", "PERCENT","TOTALS"]
exclude_rows_labels = ["24:00", "25:00"]
# Ensure input yLabels correspond to the data
if len(yLabels) != len(data2d):
raise ValueError(f"The length of yLabels {len(yLabels)} must match the number of rows in the data {len(data2d)}.")
# Sanitize and filter the data
sanitized_data, filtered_xLabels, filtered_yLabels = sanitize_and_filter_data_for_stacked_bar(data2d, xLabels, yLabels, exclude_columns_labels, exclude_rows_labels)
# Ensure that the length of yLabels matches the number of rows (0 to n should be n+1 rows)
if len(filtered_yLabels) != sanitized_data.shape[0]:
raise ValueError(f"The length of filtered_yLabels {len(filtered_yLabels)} must match the number of rows in the data {sanitized_data.shape[0]}.")
# Transpose the data so that hours are on the x-axis and categories are stacked in the y-axis
transposed_data = sanitized_data.T
fig = go.Figure()
# Get unique colors for each category
extended_colors = generate_distinct_colors(len(filtered_xLabels))
#print(len(filtered_xLabels))
#print(extended_colors)
#quit()
for i, category in enumerate(filtered_xLabels):
fig.add_trace(go.Bar(
name=category,
x=filtered_yLabels,
y=transposed_data[i],
marker_color=extended_colors[i % len(extended_colors)] # Cycle through the colors if there are more categories than colors
))
fig.update_layout(
barmode='stack',
title='Stacked Bar Graph by Hour',
xaxis=dict(title='Hour'),
yaxis=dict(title='Values'),
legend_title_text='Categories',
margin = {
'l': 50, #left margin
'r': 120, #right margin
't': 50, #top margin
'b': 50 #bottom margin
}
)
# Save the graph to an HTML file
fig.write_html(save_path)
# Write it to a var and return the string
graph_html = fig.to_html(full_html=False)
return graph_html
def sanitize_and_filter_data(data2d, exclude_labels, xLabels):
"""
Sanitize data by removing unwanted columns and converting to numeric values.
Parameters:
- data2d (list of lists): A 2D list containing the data.
- exclude_labels (list): Labels to exclude from the data and x-axis.
- xLabels (list): Current labels for the x-axis.
Returns:
- numpy.ndarray: Sanitized 2D numpy array with numeric data.
- list: Filtered x-axis labels.
"""
def to_numeric(value):
try:
if isinstance(value, str):
# Remove any extra characters like '%' and convert to float
return float(value.replace('%', '').strip())
else:
return float(value)
except ValueError:
return 0.0 # Default to 0 if conversion fails
# Create a boolean array for columns to keep (not in exclude_labels)
columns_to_keep = [label not in exclude_labels for label in xLabels]
# Filter out the columns both from the data and xLabels
filtered_data2d = []
for row in data2d:
filtered_row = [to_numeric(value) for keep, value in zip(columns_to_keep, row) if keep]
filtered_data2d.append(filtered_row)
filtered_xLabels = [label for label, keep in zip(xLabels, columns_to_keep) if keep]
return np.array(filtered_data2d), filtered_xLabels
def create_heatmap(data2d, xLabels, yLabels, save_path='heatmap.html'):
"""
Creates and saves a heatmap from given 2D numpy array data using Plotly.
Parameters:
- data2d (list of lists or numpy.ndarray): A 2D list or numpy array containing the data.
- xLabels (list): A list of category labels for the x-axis.
- yLabels (list): A list of labels for the y-axis (e.g., hours).
- save_path (str): The path where the plot image will be saved.
"""
excluded_columns = ["Count", "PERCENT", "TOTALS"]
# Remove rows 24 and 25 by slicing the data and labels
data2d = data2d[:24]
yLabels = yLabels[:24] # Ensure yLabels also excludes those rows
# Sanitize and filter the data
sanitized_data, filtered_xLabels = sanitize_and_filter_data(data2d, excluded_columns, xLabels)
# Ensure that the length of yLabels matches the number of rows (0 to n should be n+1 rows)
if len(yLabels) != sanitized_data.shape[0]:
raise ValueError("The length of yLabels must match the number of rows in the data.")
# Create the heatmap
# Define a custom color scale where 0 is white
color_scale = [
[0, "lightgrey"],
[0.3, "blue"],
[0.6, 'green'],
[0.75,'yellow'],
[1,'red']
]
fig = px.imshow(sanitized_data,
labels=dict(x="Category", y="Hour", color="Count"),
x=filtered_xLabels,
y=yLabels,
color_continuous_scale=color_scale)
fig.update_layout(
title='Heatmap of Counts by Category per Hour',
xaxis_nticks=len(filtered_xLabels),
yaxis_nticks=len(yLabels),
margin=dict(l=0, r=0, t=30, b=0)
)
fig.update_xaxes(showticklabels=True, side='bottom', showline=True, linewidth=2, linecolor='black', mirror=True)
fig.update_yaxes(showticklabels=True, showline=True, linewidth=2, linecolor='black', mirror=True)
fig.write_html(save_path)
# Write it to a var and return the string
graph_html = fig.to_html(full_html=False)
return graph_html
def create_line_chart(data2d, xLabels, yLabels, save_path='line_chart.html'):
fig = go.Figure()
excluded_columns = ["Count", "PERCENT", "TOTALS"]
# Remove rows 24 and 25 by slicing the data and labels
data2d = data2d[:24]
yLabels = yLabels[:24] # Ensure yLabels also excludes those rows
# Sanitize and filter the data
sanitized_data, filtered_xLabels = sanitize_and_filter_data(data2d, excluded_columns, xLabels)
# Ensure that the length of yLabels matches the number of rows (0 to n should be n+1 rows)
if len(yLabels) != sanitized_data.shape[0]:
raise ValueError("The length of yLabels must match the number of rows in the data.")
# Remove rows with all zero elements and the corresponding categories
nonzero_rows_indices = np.where(~np.all(sanitized_data == 0, axis=0))[0] # find rows with non-zero elements
sanitized_data = sanitized_data[:, nonzero_rows_indices]
filtered_xLabels = [filtered_xLabels[i] for i in nonzero_rows_indices] # update filtered_xLabels
for i, category in enumerate(filtered_xLabels):
fig.add_trace(go.Scatter(
mode='lines+markers',
name=category,
x= [f'{j:02d}:00' for j in range(sanitized_data.shape[0])],
y=sanitized_data[:, i]
))
fig.update_layout(
title='Line Chart of Counts by Category per Hour',
xaxis=dict(title='Hour'),
yaxis=dict(title='Count'),
legend_title_text='Category'
)
fig.write_html(save_path)
# Write it to a var and return the string
graph_html = fig.to_html(full_html=False)
return graph_html
def save_summaries_to_db(date_str, hour, parsed_data):
# Convert parsed_data to JSON string
json_data = json.dumps(parsed_data)
# Insert the record
insert_query = """
INSERT INTO SummaryLogs (Date, Hour, logData)
VALUES (%s, %s, %s)
"""
try:
cursor.execute(insert_query, (date_str, hour, json_data))
conn.commit()
except mysql.connector.Error as err:
print(f"DB Error {date_str} {hour} : {err}")
conn.rollback()
def is_running_under_thonny():
# Check for the 'THONNY_USER_DIR' environment variable
return 'THONNY_USER_DIR' in os.environ
# Routines to access the E-Smith dbs
def parse_entity_line(line):
"""
Parses a single line of key-value pairs.
:param line: Single line string to be parsed
:return: Dictionary with keys and values
"""
parts = line.split('|')
# First part contains the entity name and type in the format 'entity_name=type'
entity_part = parts.pop(0)
entity_name, entity_type = entity_part.split('=')
entity_dict = {'type': entity_type}
for i in range(0, len(parts)-1, 2):
key = parts[i]
value = parts[i+1]
entity_dict[key] = value
return entity_name, entity_dict
def parse_config(config_string):
"""
Parses a multi-line configuration string where each line is an entity with key-value pairs.
:param config_string: Multi-line string to be parsed
:return: Dictionary of dictionaries with entity names as keys
"""
config_dict = {}
lines = config_string.strip().split('\n')
for line in lines:
line = line.strip()
if line.startswith('#'): # Skip lines that start with '#'
continue
entity_name, entity_dict = parse_entity_line(line)
config_dict[entity_name] = entity_dict
return config_dict
def read_config_file(file_path):
"""
Reads a configuration file and parses its contents.
:param file_path: Path to the configuration file
:return: Parsed configuration dictionary
"""
with open(file_path, 'r') as file:
config_string = file.read()
return parse_config(config_string)
def get_value(config_dict, entity, key, default=None):
"""
Retrieves the value corresponding to the given key from a specific entity.
:param config_dict: Dictionary of dictionaries with parsed config
:param entity: Entity from which to retrieve the key's value
:param key: Key whose value needs to be retrieved
:param default: Default value to return if the entity or key does not exist
:return: Value corresponding to the key, or the default value if the entity or key does not exist
"""
return config_dict.get(entity, {}).get(key, default)
def is_private_ip(ip):
try:
# Convert string to an IPv4Address object
ip_addr = ipaddress.ip_address(ip)
except ValueError:
return False
# Define private IP ranges
private_ranges = [
ipaddress.ip_network('10.0.0.0/8'),
ipaddress.ip_network('172.16.0.0/12'),
ipaddress.ip_network('192.168.0.0/16'),
]
# Check if the IP address is within any of these ranges
for private_range in private_ranges:
if ip_addr in private_range:
return True
return False
def truncate_microseconds(timestamp):
# Split timestamp into main part and microseconds
try:
main_part, microseconds = timestamp.split('.')
# Truncate the last three digits of the microseconds
truncated_microseconds = microseconds[:-3]
# Combine the main part and truncated microseconds
truncated_timestamp = f"{main_part}.{truncated_microseconds}"
except Exception as e:
print(f"{e} {timestamp}")
raise ValueError
# Remove the microseconds completely if they exist
return truncated_timestamp.split('.')[0]
def read_in_relevant_log_file(file_path,analysis_date=yesterday):
# Read the file and split each line into a list - timestamp and the rest
log_entries = []
skip_record_count = 0
ignore_record_count = 0
with codecs.open(file_path, 'rb','utf-8', errors='replace') as file:
try:
for Line in file:
#extract time stamp
try:
entry = split_timestamp_and_data(Line)
# compare with anal date
timestamp_str = truncate_microseconds(entry[0])
except ValueError as e:
#print(f"ValueError {e} on timestamp create {timestamp_str}:{entry[0]} {entry[1]}")
skip_record_count += 1
continue
# Parse the timestamp string into a datetime object
# Ignoring extra microseconds
try:
timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S")
except ValueError as e:
print(f"ValueError {e} on timestamp extract {timestamp_str}:{entry[1]}")
#print(f"{timestamp.date()} {analysis_date.date()}")
#quit()
if timestamp.date() == analysis_date.date():
log_entries.append((timestamp, entry[1]))
else:
ignore_record_count += 1
except UnicodeDecodeError as e:
#print(f"{Line} {len(log_entries)} {e} ")
pass
return [log_entries,skip_record_count,ignore_record_count]
def filter_summary_records(log_entries):
# Return just the summary records
filtered_log_entries = []
skipped_entry_count = 0
for line in log_entries:
#print(line)
#quit()
if '`' in line[1]:
filtered_log_entries.append(line)
else:
skipped_entry_count += 1
return [filtered_log_entries,skipped_entry_count]
def sort_log_entries(log_entries):
# Sort the records, based on the timestamp
sorted_entries = sorted(log_entries, key=lambda x: x[0])
# and return a dictionary
sorted_dict = {entry[0]: entry[1] for entry in sorted_entries}
return sorted_dict
def parse_data(data):
# Split data string into parts and map to named fields.
# Adjust the field names and parsing logic according to your data format.
# Split at the backtick - before it fields split at space, after, fields split at tab
parts = data.split('`')
#print(f"{parts[0]}:{parts[1]}")
fields1 = parts[0].strip().split() if len(parts) > 0 else []
fields2 = parts[1].split('\t') if len(parts) > 1 else []
# then merge them
fields = fields1 + fields2
# if fields[4] == 'localhost':
# i = 0
# print(f"len:{len(fields)}")
# for part in fields:
# print(f"{i}: {part}")
# i = i +1
# quit()
# and mapping:
try:
return_dict = {
'id': fields[0].strip() if len(fields) > 0 else None,
'action': fields[1].strip() if len(fields) > 1 else None,
'logterse': fields[2].strip() if len(fields) > 2 else None,
'ip': fields[3].strip() if len(fields) > 3 else None,
'sendurl': fields[4].strip() if len(fields) > 4 else None, #1
'sendurl1': fields[5].strip() if len(fields) > 5 else None, #2
'from-email': fields[6].strip() if len(fields) > 6 else None, #3
'error-reason': fields[6].strip() if len(fields) > 6 else None, #3
'to-email': fields[7].strip() if len(fields) > 7 else None, #4
'error-plugin': fields[8].strip() if len(fields) > 8 else None, #5
'action1': fields[8].strip() if len(fields) > 8 else None, #5
'error-number' : fields[9].strip() if len(fields) > 9 else None, #6
'sender': fields[10].strip() if len(fields) > 10 else None, #7
'error-msg' :fields[10].strip() if len(fields) > 10 else None, #7
'spam-status': fields[11].strip() if len(fields) > 11 else None, #8
'error-result': fields[11].strip() if len(fields) > 11 else None,#8
# Add more fields as necessary
}
except:
#print(f"error:len:{len(fields)}")
return_dict = {}
#print(return_dict)
#quit()
return return_dict
def count_entries_by_hour(log_entries):
hourly_counts = defaultdict(int)
for entry in log_entries:
# Extract hour from the timestamp
timestamp = entry['timestamp']
hour = datetime.datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d %H')
hourly_counts[hour] += 1
return hourly_counts
def initialize_2d_array(num_hours, column_headers_len,reporting_date):
num_hours += 1 # Adjust for the zeroth hour
# Initialize the 2D list with zeroes
return [[0] * column_headers_len for _ in range(num_hours)]
def search_2d_list(target, data):
"""
Search for a target string in a 2D list of variable-length lists of strings.
:param target: str, the string to search for
:param data: list of lists of str, the 2D list to search
:return: int, the row number where the target string is found, or -1 if not found
"""
for row_idx, row in enumerate(data):
if target in row:
return row_idx
return -1 # Return -1 if not found
def check_html2text_installed():
try:
# Check if html2text is installed by running 'which html2text'
result = subprocess.run(
['which', 'html2text'],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
# If the command finds html2text, it will output the path
html2text_path = result.stdout.decode('utf-8').strip()
if not html2text_path:
raise FileNotFoundError
print(f"html2text is installed at: {html2text_path}")
return True
except subprocess.CalledProcessError:
print("html2text is not installed. Please install it using your package manager.", file=sys.stderr)
return False
def html_to_text(input_file, output_file):
if not check_html2text_installed():
sys.exit(1)
try:
# Run the html2text command with -b0 --pad-tables parameters
result = subprocess.run(
['html2text', '-b0', '--pad-tables', input_file],
check=True, # Raise a CalledProcessError on non-zero exit
stdout=subprocess.PIPE, # Capture stdout
stderr=subprocess.PIPE # Capture stderr
)
# Write the stdout from the command to the output file
with open(output_file, 'w', encoding='utf-8') as outfile:
outfile.write(result.stdout.decode('utf-8'))
print(f"Converted {input_file} to {output_file}")
except subprocess.CalledProcessError as e:
print(f"Error occurred: {e.stderr.decode('utf-8')}", file=sys.stderr)
sys.exit(e.returncode)
def get_html2text_version():
try:
result = subprocess.run(['html2text', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
# Ensure the result is treated as a string in Python 3.6+
return result.stdout.strip()
except subprocess.CalledProcessError as e:
print(f"Error occurred while checking html2text version: {e}", file=sys.stderr)
return None
def print_progress_bar(iteration, total, prefix='', suffix='', decimals=1, length=50, fill='█', print_end="\r"):
"""
Call in a loop to create a terminal progress bar
@params:
iteration - Required : current iteration (Int)
total - Required : total iterations (Int)
prefix - Optional : prefix string (Str)
suffix - Optional : suffix string (Str)
decimals - Optional : positive number of decimals in percent complete (Int)
length - Optional : character length of bar (Int)
fill - Optional : bar fill character (Str)
print_end - Optional : end character (e.g. "\r", "\r\n") (Str)
"""
if total == 0:
raise ValueError("Progress total is zero")
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
filled_length = int(length * iteration // total)
bar = fill * filled_length + '-' * (length - filled_length)
print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=print_end)
# Print New Line on Complete
if iteration == total:
print()
def insert_string_after(original:str, to_insert:str, after:str) -> str:
"""
Insert to_insert into original after the first occurrence of after.
:param original: The original string.
:param to_insert: The string to be inserted.
:param after: The set of characters after which the string will be inserted.
:return: The new string with to_insert inserted after after.
"""
position = original.find(after)
if position == -1:
print(f"insert_string_after:({after}) string is not found in original")
return original
# Position of the insertion point
insert_pos = position + len(after)
return original[:insert_pos] + to_insert + original[insert_pos:]
def split_timestamp_and_data(log_entry: str) -> list:
"""
Split a log entry into timestamp and the rest of the data.
:param log_entry: The log entry as a string.
:return: A list with two entries: [timestamp, rest_of_data].
"""
# The timestamp is always the first part, up to the first space after the milliseconds
parts = log_entry.split(' ', 2)
if len(parts) < 3:
raise ValueError(f"The log entry format is incorrect {parts}")
timestamp = ' '.join(parts[:2])
rest_of_data = parts[2]
return [timestamp, rest_of_data]
def render_sub_table(table_title,table_headers,found_values,get_character=None):
# Get the total
total_sum = sum(found_values.values())
# and add in list with second element the percentage
# Create a list of tuples with each tuple containing (key, value, percentage)
if get_character:
sub_result = [(key, value,
f"{round(value / total_sum * 100, 2)}",
f"{get_character(key)}") for key, value in found_values.items()
]
else:
sub_result = [(key, value,
f"{round(value / total_sum * 100, 2)}") for key, value in found_values.items()
]
sub_result.sort(key=lambda x: float(x[2]), reverse=True) # Sort by percentage in descending order
sub_template_path = template_dir+'mailstats-sub-table.html.pt'
# Load the template
with open(sub_template_path, 'r') as template_file:
template_content = template_file.read()
# Create a Chameleon template instance
try:
template = PageTemplate(template_content)
# Render the template with the 2D array data and column headers
try:
rendered_html = template(array_2d=sub_result, column_headers=table_headers, title=table_title)
except Exception as e:
raise ValueError(f"{table_title}: A chameleon controller render error occurred: {e}")
except Exception as e:
raise ValueError(f"{table_title}: A chameleon controller template error occurred: {e}")
return rendered_html
def get_character_in_reject_list(code):
if code in BadCountries:
return "*"
else:
return ""
def read_html_from_file(filepath):
"""
Reads HTML content from a given file.
Args:
filepath (str): Path to the HTML file.
Returns:
str: HTML content of the file.
"""
# Need to add in here the contents of the css file at the end of the head section.
with open(filepath, 'r', encoding='utf-8') as file:
html_contents = file.read()
print("reading from html file")
#print(len(html_contents))
# Get Filepath
css_path = os.path.dirname(filepath)+"/../css/mailstats.css"
#print(css_path)
# Read in CSS
with open(css_path, 'r', encoding='utf-8') as file:
css_contents = file.read()
#print(len(css_contents))
html_contents = insert_string_after(html_contents,"\n"+css_contents,"<!--css here-->")
#print(len(html_contents))
return html_contents
def read_text_from_file(filepath):
"""
Reads plain text content from a given file.
Args:
filepath (str): Path to the text file.
Returns:
str: Text content of the file.
"""
try:
with open(filepath, 'r', encoding='utf-8') as file:
return file.read()
except:
print(f"{filepath} not found")
return
def send_email(subject, from_email, to_email, smtp_server, smtp_port, HTML_content=None, Text_content=None, smtp_user=None, smtp_password=None):
"""
Sends an HTML email.
Args:
html_content (str): The HTML content to send in the email.
subject (str): The subject of the email.
from_email (str): The sender's email address.
to_email (str): The recipient's email address.
smtp_server (str): SMTP server address.
smtp_port (int): SMTP server port.
smtp_user (str, optional): SMTP server username. Default is None.
smtp_password (str, optional): SMTP server password. Default is None.
"""
#Example (which works!)
# send_email(
# subject="Your subject",
# from_email="mailstats@bjsystems.co.uk",
# to_email="brianr@bjsystems.co.uk",
# smtp_server="mail.bjsystems.co.uk",
# smtp_port=25
# HTML_content=html_content,
# Text_content=Text_content,
# )
# Set up the email
msg = MIMEMultipart('alternative')
msg['Subject'] = subject
msg['From'] = from_email
msg['To'] = to_email
if HTML_content:
part = MIMEText(HTML_content, 'html')
msg.attach(part)
if Text_content:
part = MIMEText(Text_content, 'plain')
msg.attach(part)
# Sending the email
with smtplib.SMTP(smtp_server, smtp_port) as server:
server.starttls() # Upgrade the connection to secure
if smtp_user and smtp_password:
server.login(smtp_user, smtp_password) # Authenticate only if credentials are provided
server.sendmail(from_email, to_email, msg.as_string())
def replace_between(text, start, end, replacement):
# Escaping start and end in case they contain special regex characters
pattern = re.escape(start) + '.*?' + re.escape(end)
# Using re.DOTALL to match any character including newline
replaced_text = re.sub(pattern, replacement, text, flags=re.DOTALL)
return replaced_text
def get_heading():
#
# Needs from anaytsis
# SATagLevel - done
# SARejectLevel - done
# warnnoreject - done
# totalexamined - done
# emailperhour - done
# spamavg - done
# rejectspamavg - done
# hamavg - done
# DMARCSendCount - done
# hamcount - done
# DMARCOkCount - deone
# Clam Version/DB Count/Last DB update
clam_output = subprocess.getoutput("freshclam -V")
clam_info = f"Clam Version/DB Count/Last DB update: {clam_output}"
# SpamAssassin Version
sa_output = subprocess.getoutput("spamassassin -V")
sa_info = f"SpamAssassin Version: {sa_output}"
# Tag level and Reject level
tag_reject_info = f"Tag level: {SATagLevel}; Reject level: {SARejectLevel} {warnnoreject}"
# SMTP connection stats
smtp_stats = f"External SMTP connections accepted: {totalexternalsmtpsessions}\n"\
f"Internal SMTP connections accepted: {totalinternalsmtpsessions}"
if len(connection_type_counts)>0:
for connect_type in connection_type_counts.keys():
smtp_stats = smtp_stats + f"\nCount of {connection_type} connections:{connection_type_counts[connect_type]}"
smtp_stats = smtp_stats + f"\nEmails per hour: {emailperhour:.1f}/hr\n"\
f"Average spam score (accepted): {spamavg or 0:.2f}\n"\
f"Average spam score (rejected): {rejectspamavg or 0:.2f}\n"\
f"Average ham score: {hamavg or 0:.2f}\n"\
f"Number of DMARC reporting emails sent: {DMARCSendCount or 0} (not shown on table)"
# DMARC approved emails
dmarc_info = ""
if hamcount != 0:
dmarc_ok_percentage = DMARCOkCount * 100 / hamcount
dmarc_info = f"Number of emails approved through DMARC: {DMARCOkCount or 0} ({dmarc_ok_percentage:.2f}% of Ham count)"
# Accumulate all strings
header_str = "\n".join([clam_info, sa_info, tag_reject_info, smtp_stats, dmarc_info])
# switch newlines to <br />
header_str = header_str.replace("\n","<br />")
return header_str
if __name__ == "__main__":
try:
chameleon_version = pkg_resources.get_distribution("Chameleon").version
except pkg_resources.DistributionNotFound:
chameleon_version = "Version information not available"
python_version = sys.version
python_version = python_version[:8]
current_datetime = datetime.now()
formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M")
# Command line parameters
parser = argparse.ArgumentParser(description="Mailstats")
parser.add_argument('-d', '--date', help='Specify a valid date (yyyy-mm-dd) for the analysis', default=formatted_yesterday)
parser.add_argument('-ef', '--emailfile', help='Save an html file of the email sent (y/N)', default='n')
parser.add_argument('-tf', '--textfile', help='Save a txt file of the html page (y/N)', default='n')
parser.add_argument('--version', action='version', version='%(prog)s '+Mailstats_version+" built on "+build_date_time)
parser.add_argument('-db', '--dbsave', help='Force save of summary logs in DB (y/N)', default='n')
args = parser.parse_args()
analysis_date = args.date
# and check its format is valid
try:
datetime.strptime(analysis_date, '%Y-%m-%d')
except ValueError:
print("Specify a valid date (yyyy-mm-dd) for the analysis")
quit()
anaysis_date_obj = datetime.strptime(analysis_date, '%Y-%m-%d')
noemailfile = args.emailfile.lower() == 'n'
notextfile = args.textfile.lower() == 'n'
isThonny = is_running_under_thonny()
forceDbSave = args.dbsave.lower() == 'y'
#E-Smith Config DBs
if isThonny:
db_dir = "/home/brianr/SME11Build/GITFiles/smecontribs/smeserver-mailstats/"
else:
db_dir = "/home/e-smith/db/"
#From SMEServer DB
ConfigDB = read_config_file(db_dir+"configuration")
DomainName = get_value(ConfigDB, "DomainName", "type") #'bjsystems.co.uk' # $cdb->get('DomainName')->value;
hello_string = "Mailstats:"+Mailstats_version+' for '+DomainName+" at "+formatted_datetime+" for "+analysis_date
print(hello_string)
version_string = "Chameleon:"+chameleon_version+" Python:"+python_version
if isThonny:
version_string = version_string + "...under Thonny"
print(version_string)
RHSenabled = get_value(ConfigDB, "qpsmtpd", "RHSBL","disabled") == "enabled" #True #( $cdb->get('qpsmtpd')->prop('RHSBL') eq 'enabled' );
DNSenabled = get_value(ConfigDB, "qpsmtpd", "DNSBL","disabled") == "enabled" #True #( $cdb->get('qpsmtpd')->prop('DNSBL') eq 'enabled' );
SARejectLevel = int(get_value(ConfigDB, "spamassassin", "RejectLevel","12")) #12 #$cdb->get('spamassassin')->prop('RejectLevel');
SATagLevel = int(get_value(ConfigDB, "spamassassin", "TagLevel","4")) #4 #$cdb->get('spamassassin')->prop('TagLevel');
if SARejectLevel == 0:
warnnoreject = "(*Warning* 0 = no reject)"
else:
warnnoreject = ""
EmailAddress = get_value(ConfigDB,"mailstats","Email","admin@"+DomainName)
if '@' not in EmailAddress:
EmailAddress = EmailAddress+"@"+DomainName
EmailTextOrHTML = get_value(ConfigDB,"mailstats","EmailTextOrHTML","Both") #Text or Both or None
EmailHost = get_value(ConfigDB,"mailstats","EmailHost","localhost") #Default will be localhost
EmailPort = int(get_value(ConfigDB,"mailstats","EmailPort","25"))
EMailSMTPUser = get_value(ConfigDB,"mailstats","EmailUser") #None = default => no authenticatioon needed
EMailSMTPPassword = get_value(ConfigDB,"mailstats","EmailPassword")
BadCountries = get_value(ConfigDB,"qpsmtpd","BadCountries")
# Db save control
saveData = get_value(ConfigDB,"mailstats","SaveDataToMySQL","no") == 'yes' or forceDbSave
if saveData:
DBName = "mailstats";
DBHost = get_value(ConfigDB,'mailstats','DBHost',"localhost")
DBPort = get_value(ConfigDB,'mailstats','DBPort',"3306")
DBName = 'mailstats'
DBPassw = 'mailstats'
DBUser = 'mailstats'
UnixSocket = "/var/lib/mysql/mysql.sock"
# see if the DB exists
# Try to Establish a database connection
try:
conn = mysql.connector.connect(
host=DBHost,
user=DBUser,
password=DBPassw,
database=DBName,
port=DBPort,
unix_socket=UnixSocket
)
cursor = conn.cursor()
# Create table if it doesn't exist
cursor.execute("""
CREATE TABLE IF NOT EXISTS SummaryLogs (
id INT AUTO_INCREMENT PRIMARY KEY,
Date DATE,
Hour INT,
logData TEXT
)
""")
# and prune the DB here if needed.
# Delete existing records for the given date
try:
delete_query = """
DELETE FROM SummaryLogs
WHERE Date = %s
"""
cursor.execute(delete_query, (analysis_date,)) #Don't forget the syntactic sugar of the extra comma to make it a tuple!
# Get the number of records deleted
rows_deleted = cursor.rowcount
print(rows_deleted)
#quit()
if rows_deleted > 0:
print(f"Deleted {rows_deleted} rows for {analysis_date} ")
except mysql.connector.Error as e:
print(f"SQL Delete failed ({delete_query}) ({e}) ")
except mysql.connector.Error as e:
print(f"Unable to connect to {DBName} on {DBHost} port {DBPort} error ({e}) ")
saveData = False
nolinks = not saveData
# Not sure we need these...
# if (ConfigDB,"qpsmtpd","RHSBL").lower() == 'enabled':
# RBLList = get_value(ConfigDB,"qpsmtpd","RBLList")
# else:
# RBLList = ""
# if (ConfigDB,"qpsmtpd","RBLList").lower() == 'enabled':
# SBLLIst = get_value(ConfigDB,"qpsmtpd","SBLLIst")
# else:
# RBLList = ""
# if (ConfigDB,"qpsmtpd","RBLList").lower() == 'enabled':
# UBLList = get_value(ConfigDB,"qpsmtpd","UBLLIst")
# else:
# RBLList = ""
FetchmailIP = '127.0.0.200'; #Apparent Ip address of fetchmail deliveries
WebmailIP = '127.0.0.1'; #Apparent Ip of Webmail sender
localhost = 'localhost'; #Apparent sender for webmail
FETCHMAIL = 'FETCHMAIL'; #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email
MAILMAN = "bounces"; #sender when mailman sending when orig is localhost
DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything)
DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval
num_hours = 25 # Represents hours from 0 to 23 - adds extra one for column totals and another for percentages
log_file = logs_dir+'current.log'
log_entries,skip_count,ignored_count = read_in_relevant_log_file(log_file,anaysis_date_obj)
# if len(log_entries) == 0:
# print(f"No records found in {log_file}")
# quit()
# else:
print(f"Found {len(log_entries)} entries in log for for {anaysis_date_obj.strftime('%Y-%m-%d')} Ignored: {ignored_count} skipped: {skip_count}")
summary_log_entries,skip_count = filter_summary_records(log_entries)
print(f"Found {len(summary_log_entries)} summary entries and skipped {skip_count} entries")
sorted_log_dict = sort_log_entries(summary_log_entries)
print(f"Sorted {len(sorted_log_dict)} entries")
columnHeaders = ['Count','WebMail','Local','MailMan','Relay','DMARC','Virus','RBL/DNS','Geoip.','Non.Conf.','Karma','Rej.Load','Del.Spam','Qued.Spam?',' Ham','TOTALS','PERCENT']
# dict for each colum identifying plugin that increments count
columnPlugin = [''] * 17
columnPlugin[Hour] = []
columnPlugin[WebMail] = []
columnPlugin[Local] = []
columnPlugin[MailMan] = []
columnPlugin[DMARC] = ['dmarc']
columnPlugin[Virus] = ['pattern_filter', 'virus::pattern_filter','virus::clamav']
columnPlugin[RBLDNS] = ['rhsbl', 'dnsbl','uribl']
columnPlugin[Geoip] = ['check_badcountries']
columnPlugin[NonConf] = ['check_earlytalker','check_relay','check_norelay', 'require_resolvable_fromhost'
,'check_basicheaders','check_badmailfrom','check_badrcptto_patterns'
,'check_badrcptto','check_spamhelo','check_goodrcptto extn','rcpt_ok'
,'check_goodrcptto','check_smtp_forward','count_unrecognized_commands','tls','auth::auth_cvm_unix_local'
,'auth::auth_imap', 'earlytalker','resolvable_fromhost','relay','headers','mailfrom','badrcptto','helo'
,'check_smtp_forward','sender_permitted_from']
columnPlugin[RejLoad] = ['loadcheck']
columnPlugin[DelSpam] = []
columnPlugin[QuedSpam] = []
columnPlugin[Ham] = []
columnPlugin[TOTALS] = []
columnPlugin[PERCENT] = []
columnPlugin[Karma] = ['karma']
columnHeaders_len = len(columnHeaders)
columnCounts_2d = initialize_2d_array(num_hours, columnHeaders_len,analysis_date)
virus_pattern = re.compile(r"Virus found: (.*)")
found_viruses = defaultdict(int)
found_qpcodes = defaultdict(int)
qpcodes_pattern = re.compile(r"(\(.*\)).*'")
i = 0;
sorted_len= len(sorted_log_dict)
#unless none to show
spamavg = 0;
spamqueuedcount = 0
hamcount = 0
hamavg = 0
rejectspamcount = 0
rejectspamavg = 0
DMARCSendCount = 0
totalexamined = 0
if sorted_len > 0:
if isThonny:
# Initial call to print the progress bar
print_progress_bar(0, sorted_len, prefix='Progress:', suffix='Complete', length=50)
for timestamp, data in sorted_log_dict.items():
i += 1
totalexamined += 1
if isThonny:
print_progress_bar(i, sorted_len, prefix='Scanning for main table:', suffix='Complete', length=50)
# Count of in which hour it falls
# Parse the timestamp string into a datetime object
dt = timestamp
hour = dt.hour
# parse the data
parsed_data = parse_data(data)
# Save the data here if necessary
if saveData:
save_summaries_to_db(anaysis_date_obj.strftime('%Y-%m-%d'),hour,parsed_data)
# Increment Count in which headings it falls
#Hourly count and column total
columnCounts_2d[hour][Hour] += 1
columnCounts_2d[ColTotals][Hour] += 1
#Row Totals
columnCounts_2d[hour][TOTALS] += 1
#Total totals
columnCounts_2d[ColTotals][TOTALS] += 1
# first spot the fetchmail and local deliveries.
#Local send
if DomainName in parsed_data['sendurl']:
columnCounts_2d[hour][Local] += 1
columnCounts_2d[ColTotals][Local] += 1
#Relay or webmail
elif not is_private_ip(parsed_data['ip']) and is_private_ip(parsed_data['sendurl1']) and parsed_data['action1'] == 'queued':
#Relay
columnCounts_2d[hour][Relay] += 1
columnCounts_2d[ColTotals][Relay] += 1
elif WebmailIP in parsed_data['sendurl1'] and not is_private_ip(parsed_data['ip']):
#webmail
columnCounts_2d[hour][WebMail] += 1
columnCounts_2d[ColTotals][WebMail] += 1
elif localhost in parsed_data['sendurl']:
# but not if it comes from fetchmail
if not FETCHMAIL in parsed_data['sendurl1']:
# might still be from mailman here
if MAILMAN in parsed_data['sendurl1']:
#$mailmansendcount++;
#$localsendtotal++;
columnCounts_2d[hour][MailMan] += 1
columnCounts_2d[ColTotals][MailMan] += 1
#$counts{$abshour}{$CATMAILMAN}++;
#$localflag = 1;
else:
#Or sent to the DMARC server
#check for email address in $DMARC_Report_emails string
#my $logemail = $log_items[4];
if DMARCDomain in parsed_data['from-email']: #(index($DMARC_Report_emails,$logemail)>=0) or
#$localsendtotal++;
DMARCSendCount += 1
#localflag = 1;
else:
# ignore incoming localhost spoofs
if not 'msg denied before queued' in parsed_data['error-msg']:
#Webmail
#$localflag = 1;
#$WebMailsendtotal++;
columnCounts_2d[hour][WebMail] += 1
columnCounts_2d[ColTotals][WebMail] += 1
#$WebMailflag = 1;
else:
#$localflag = 1;
#$WebMailsendtotal++;
#$WebMailflag = 1;
columnCounts_2d[hour][WebMail] += 1
columnCounts_2d[ColTotals][WebMail] += 1
#Queued email
if parsed_data['action'] == '(queue)':
columnCounts_2d[hour][Ham] += 1
columnCounts_2d[ColTotals][Ham] += 1
# spamassassin not rejected
if parsed_data.get('spam-status') is not None and isinstance(parsed_data['spam-status'], str):
if parsed_data['spam-status'].lower().startswith('no'):
#Extract other parameters from this string
# example: No, score=-3.9
spam_pattern = re.compile(r'score=(-?\d+\.\d+) required=(-?\d+\.\d+)')
match = re.search(spam_pattern, parsed_data['spam-status'])
if match:
score = float(match.group(1))
#print(score,SATagLevel)
if score < float(SATagLevel):
# Accumulate allowed score (inc negatives?)
hamavg += score
hamcount += 1
#spamassasin rejects
if parsed_data.get('spam-status') is not None and isinstance(parsed_data['spam-status'], str):
if parsed_data['spam-status'].lower().startswith('yes'):
#Extract other parameters from this string
# example: Yes, score=10.3 required=4.0 autolearn=disable
spam_pattern = re.compile(r'score=(-?\d+\.\d+) required=(-?\d+\.\d+)')
match = re.search(spam_pattern, parsed_data['spam-status'])
if match:
score = float(match.group(1))
required = float(match.group(2))
#print(f"{parsed_data['spam-status']} / {score} {required}")
if score >= SARejectLevel:
columnCounts_2d[hour][DelSpam] += 1
columnCounts_2d[ColTotals][DelSpam] += 1
rejectspamavg += score
rejectspamcount += 1
elif score >= required:
columnCounts_2d[hour][QuedSpam] += 1
columnCounts_2d[ColTotals][QuedSpam] += 1
spamavg += score
spamqueuedcount += 1
# Count the qpsmtpd codes
if parsed_data['error-plugin'].strip() == 'naughty':
#print(f"Found naughty {parsed_data['error-msg']}")
if parsed_data['error-msg'].startswith("(dnsbl)"):
#print("Found dnsbl")
columnCounts_2d[hour][RBLDNS]+= 1
columnCounts_2d[ColTotals][RBLDNS]+= 1
elif parsed_data['error-msg'].startswith("(karma)"):
columnCounts_2d[hour][KARMA] += 1
columnCounts_2d[ColTotals][KARMA]+= 1
elif parsed_data['error-msg'].startswith("(helo)"):
columnCounts_2d[hour][RBLDNS] += 1
columnCounts_2d[ColTotals][RBLDNS]+= 1
else:
match = qpcodes_pattern.match(parsed_data['action1'])
if match:
rejReason = match.group(1)
found_qpcodes[parsed_data['error-plugin']+"-"+rejReason] += 1
else:
found_qpcodes[parsed_data['action1']] += 1
#Now increment the column which the plugin name indicates
if parsed_data['action'] == '(deny)' and parsed_data['error-plugin']:
#print(f"Found plugin {parsed_data['error-plugin']}")
if parsed_data['error-plugin']:
row = search_2d_list(parsed_data['error-plugin'],columnPlugin)
#print(row,parsed_data['error-plugin'])
if not row == -1:
#print(f"Found row: {row}")
columnCounts_2d[hour][row] += 1
columnCounts_2d[ColTotals][row] += 1
# a few ad hoc extra extractons of data
if row == Virus:
match = virus_pattern.match(parsed_data['action1'])
if match:
found_viruses[match.group(1)] += 1
else:
found_viruses[parsed_data['action1']] += 1
else:
found_qpcodes[parsed_data['action1']] += 1
if isThonny:
print() #seperate the [progress bar]
# Compute percentages
total_Count = columnCounts_2d[ColTotals][TOTALS]
#Column of percentages
for row in range(ColTotals):
if total_Count == 0:
percentage_of_total = 0
else:
percentage_of_total = f"{round(round(columnCounts_2d[row][TOTALS] / total_Count,4) * 100,1)}%"
columnCounts_2d[row][PERCENT] = percentage_of_total
#Row of percentages
for col in range(TOTALS):
if total_Count == 0:
percentage_of_total = 0
else:
percentage_of_total = f"{round(round(columnCounts_2d[ColTotals][col] / total_Count,4) * 100,1)}%"
columnCounts_2d[ColPercent][col] = percentage_of_total
# and drop in the 100% to make it look correct!
columnCounts_2d[ColPercent][PERCENT] = '100%'
columnCounts_2d[ColTotals][PERCENT] = '100%'
columnCounts_2d[ColPercent][TOTALS] = '100%'
#other stats
emailperhour = (totalexamined / 24)
if not spamqueuedcount == 0:
spamavg = spamavg / spamqueuedcount
if not rejectspamcount == 0:
rejectspamavg = rejectspamavg / rejectspamcount
if not hamcount == 0:
hamavg = hamavg / hamcount
# Now scan for the other lines in the log of interest
found_countries = defaultdict(int)
geoip_pattern = re.compile(r".*check_badcountries: GeoIP Country: (.*)")
dmarc_pattern = re.compile(r".*dmarc: pass")
helo_pattern = re.compile(r".*Accepted connection.*?from (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) \/ ([\w.-]+)")
connect_type_pattern = re.compile(r".*connect via (.*)")
total_countries = 0
DMARCOkCount = 0
totalinternalsmtpsessions = 0
totalexternalsmtpsessions = 0
i = 0
j = 0
log_len = len(log_entries)
connection_type_counts = defaultdict(int)
if log_len > 0:
if isThonny:
print_progress_bar(0, log_len, prefix='Progress:', suffix='Complete', length=50)
for data in log_entries:
i += 1
if isThonny:
print_progress_bar(i, log_len, prefix='Scanning for sub tables:', suffix='Complete', length=50)
# Match initial connection message
match = helo_pattern.match(data[1])
if match:
ip = match.group(1)
fqdn = match.group(2)
if is_private_ip(ip):
totalinternalsmtpsessions += 1
else:
totalexternalsmtpsessions += 1
continue
#Pull out Geoip countries for analysis table
match = geoip_pattern.match(data[1])
if match:
j += 1
country = match.group(1)
found_countries[country] += 1
total_countries += 1
continue
#Pull out DMARC approvals
match = dmarc_pattern.match(data[1])
if match:
DMARCOkCount += 1
continue
#Pull out type of connection
match = connect_type_pattern.match(data[1])
if match:
connection_type = match.group(1)
connection_type_counts[connection_type] += 1
continue
#print(columnCounts_2d)
#quit()
#Compute next and previous dates
day_format = "%Y-%m-%d"
# Convert the time string to a datetime object
date_obj = datetime.strptime(analysis_date, day_format)
# Compute the next date by adding one day
next_date = date_obj + timedelta(days=1)
# Compute the previous date by subtracting one day
previous_date = date_obj - timedelta(days=1)
# Convert the datetime objects back to strings in the desired format
next_date_str = next_date.strftime(day_format)
previous_date_str = previous_date.strftime(day_format)
# Create graphs of data
yLabels = [f'{i:02d}:00' for i in range(len(columnCounts_2d))]
stacked_Bar_html = create_stacked_bar_graph(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'stacked_bar_'+analysis_date+'.html')
heatmap_html = create_heatmap(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'heatmap_'+analysis_date+'.html')
line_graph_html = create_line_chart(columnCounts_2d,columnHeaders,yLabels,html_page_dir+'line_graph_'+analysis_date+'.html')
#Now apply the results to the chameleon template - main table
# Path to the template file
template_path = template_dir+'mailstats.html.pt'
# Load the template
with open(template_path, 'r') as template_file:
template_content = template_file.read()
# Create a Chameleon template instance
try:
template = PageTemplate(template_content)
# Render the template with the 2D array data and column headers
try:
rendered_html = template(array_2d=columnCounts_2d, column_headers=columnHeaders,
reporting_date=analysis_date, title=hello_string,
version=version_string,
nolinks=nolinks,
stacked_bar_graph=stacked_Bar_html,
heatmap=heatmap_html,
line_graph=line_graph_html,
PreviousDate=previous_date_str,
NextDate=next_date_str,
DomainName=DomainName
)
except Exception as e:
print(f"Chameleon template Exception {e}")
except Exception as e:
print(f"Chameleon render Exception {e}")
total_html = rendered_html
# Add in the header information
rendered_html = get_heading()
total_html = insert_string_after(total_html,rendered_html, "<!---Add in header information here -->")
#add in the subservient tables..
#qpsmtd codes
qpsmtpd_headers = ["Reason",'Count','Percent']
qpsmtpd_title = 'Qpsmtpd codes league table:'
rendered_html = render_sub_table(qpsmtpd_title,qpsmtpd_headers,found_qpcodes)
# Add it to the total
total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
#Geoip Country codes
geoip_headers = ['Country','Count','Percent','Rejected?']
geoip_title = 'Geoip results:'
rendered_html = render_sub_table(geoip_title,geoip_headers,found_countries,get_character_in_reject_list)
# Add it to the total
total_html = insert_string_after(total_html,rendered_html, "<!---Add in sub tables here -->")
if saveData:
# Close the connection
cursor.close()
conn.close()
# #Add in navigation html - next/previous/see in browser
# navigation_str_html = "<div class='linksattop'>\
# <a class='prevlink' href='http://${DomainName}/mailstats/mailstats_for_${PreviousDate}.html'>Previous</a>\
# <div class='divshowindex'><a class='showindex' href='http://${DomainName}/mailstats/'>Index of files</a></div>\
# <a class='nextlink' href='http://${DomainName}/mailstats/mailstats_for_${NextDate}.html'>Next</a>\
# </div>"
# try:
# template = PageTemplate(navigation_str_html)
# try:
# Nav_str = template(PreviousDate=previous_date_str,NextDate=next_date_str,TodayDate=analysis_date,DomainName=DomainName)
# except Exception as e:
# print(f"Chameleon nav template Exception {e}")
# except Exception as e:
# print(f"Chameleon nav render Exception {e}")
# # And insert it
# total_html = insert_string_after(total_html,Nav_str, "<!---Navigation here-->")
# Write the rendered HTML to a file
output_path = html_page_dir+'mailstats_for_'+analysis_date
output_path = output_path.replace(' ','_')
with open(output_path+'.html', 'w') as output_file:
output_file.write(total_html)
#and create a text version if the local version of html2text is suffiicent
if get_html2text_version() == '2019.9.26':
# Get a temporary file name
temp_file_name = tempfile.mktemp()
html_to_text(output_path+'.html',temp_file_name)
print(f"Rendered HTML saved to {temp_file_name}")
# and save it if required
if not notextfile:
text_file_path = output_path+'.txt'
# and rename it
os.rename(temp_file_name, text_file_path)
else:
text_file_path = temp_file_name
else:
text_file_path = ""
html_content = None
text_content = None
#Now see if Email required
if EmailTextOrHTML:
if EmailTextOrHTML == "HTML" or EmailTextOrHTML == "Both":
# Send html email (default))
filepath = html_page_dir+"mailstats_for_"+analysis_date+".html"
html_content = read_html_from_file(filepath)
# Replace the Navigation by a "See in browser" prompt
replace_str = f"<div class='divseeinbrowser' style='text-align:center;'><a class='seeinbrowser' href='http://{DomainName}/mailstats/mailstats_for_{analysis_date}.html'>See in browser</a></div>"
html_content = replace_between(html_content, "<div class='linksattop'>", ">Next</a></div>", replace_str)
if not noemailfile:
# Write out the email html to a web page
email_file = html_page_dir + "Email_mailstats_for_"+analysis_date
with open(email_file+'.html', 'w') as output_file:
output_file.write(html_content)
if EmailTextOrHTML == "Text" or EmailTextOrHTML == "Both":
#filepath = html_page_dir+"mailstats_for_"+analysis_date+".txt"
if not text_file_path == "":
text_content = read_text_from_file(text_file_path)
else:
text_content = "No text avaiable as html2text (was not "
if EMailSMTPUser:
# Send authenticated
print("Sending authenticated")
send_email(
html_content=email_content,
subject="Mailstats for "+analysis_date,
from_email="mailstats@"+DomainName,
to_email=EmailAddress,
smtp_server=EmailHost,
smtp_port=EmailPort,
HTML_content=html_content,
Text_content=text_content,
smtp_user=EMailSMTPUser,
smtp_password=EMailSMTPPassword
)
else:
# No authentication
print(f"Sending non authenticated {EmailAddress} {EmailHost}")
try:
send_email(
subject="Mailstats for "+analysis_date,
from_email="mailstats@"+DomainName,
to_email=EmailAddress,
smtp_server=EmailHost,
smtp_port=EmailPort,
HTML_content=html_content,
Text_content=text_content
)
except Exception as e:
print(f"Email Exception {e}")
View Git Blame Copy Permalink