* Mon Dec 30 2024 Brian Read <brianr@koozali.org> 1.1-19.sme
- Update mailstats.pl to accomodate change in log format for SME11 [SME: ]
This commit is contained in:
parent
c1c4251361
commit
382489ecac
@ -33,6 +33,7 @@ use strict;
|
|||||||
# bjr - 18Oct20 - Alter use of lc to avoid uninitialised messages - bug 11044
|
# bjr - 18Oct20 - Alter use of lc to avoid uninitialised messages - bug 11044
|
||||||
# bjr - 02Apr21 - Fix up lc to try to avoif uninit messages - and alter warning status - bug 11519
|
# bjr - 02Apr21 - Fix up lc to try to avoif uninit messages - and alter warning status - bug 11519
|
||||||
# bjr - 15Feb23 - Add in auth::auth_imap after change to use dovecot as incoming authorisation Bugzilla 12327
|
# bjr - 15Feb23 - Add in auth::auth_imap after change to use dovecot as incoming authorisation Bugzilla 12327
|
||||||
|
# bjr - 29Dec24 - Convert to SME11 log date format
|
||||||
#
|
#
|
||||||
#############################################################################
|
#############################################################################
|
||||||
#
|
#
|
||||||
@ -70,6 +71,11 @@ use strict;
|
|||||||
#
|
#
|
||||||
|
|
||||||
# internal modules (part of core perl distribution)
|
# internal modules (part of core perl distribution)
|
||||||
|
#
|
||||||
|
# New format for SME11
|
||||||
|
#
|
||||||
|
#Dec 17 19:34:34 sme11 qpsmtpd-forkserver[441318]: 441318 (deny) logging::logterse: ` 192.168.1.4 pc-00004.thereadclan.me.uk bjsystems.co.uk <biodiversityadvanced@bjsystems.co.uk> check_goodrcptto 901 relaying denied smeserver@thereadclan.me.uk msg denied before queued
|
||||||
|
|
||||||
use strict;
|
use strict;
|
||||||
use warnings;
|
use warnings;
|
||||||
use Getopt::Long;
|
use Getopt::Long;
|
||||||
@ -105,7 +111,7 @@ if ($cdb->get('mailstats')){
|
|||||||
|
|
||||||
#Configuration section
|
#Configuration section
|
||||||
my %opt = (
|
my %opt = (
|
||||||
version => '0.7.16', # please update at each change.
|
version => '0.8.01', # please update at each change.
|
||||||
debug => 0, # guess what ?
|
debug => 0, # guess what ?
|
||||||
sendmail => '/usr/sbin/sendmail', # Path to sendmail stub
|
sendmail => '/usr/sbin/sendmail', # Path to sendmail stub
|
||||||
from => 'spamfilter-stats', # Who is the mail from
|
from => 'spamfilter-stats', # Who is the mail from
|
||||||
@ -370,17 +376,30 @@ my $makeHTMLpage = "no";
|
|||||||
|
|
||||||
|
|
||||||
# Init the hashes
|
# Init the hashes
|
||||||
my $nhour = floor( $start / 3600 );
|
my $nhour = 0;
|
||||||
|
#print "Hour:".$nhour."\n";
|
||||||
my $ncateg;
|
my $ncateg;
|
||||||
while ( $nhour < $end / 3600 ) {
|
while ( $nhour < 24 ) {
|
||||||
$counts{$nhour}=();
|
$counts{$nhour} = {}; # Initialize as a hash reference
|
||||||
$ncateg = 0;
|
#print "Hour:".$nhour."\n";
|
||||||
while ( $ncateg < @categs) {
|
my $ncateg = 0; # Reset $ncateg for each hour
|
||||||
$counts{$nhour}{$categs[$ncateg-1]} = 0;
|
|
||||||
$ncateg++
|
while ( $ncateg < @categs ) {
|
||||||
|
$counts{$nhour}{$categs[$ncateg]} = 0; # Corrected index
|
||||||
|
$ncateg++; # Increment $ncateg
|
||||||
}
|
}
|
||||||
$nhour++;
|
$nhour++; # Increment $nhour
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#while ( $nhour < $end / 3600 ) {
|
||||||
|
#$counts{$nhour}=();
|
||||||
|
#$ncateg = 0;
|
||||||
|
#while ( $ncateg < @categs) {
|
||||||
|
#$counts{$nhour}{$categs[$ncateg-1]} = 0;
|
||||||
|
#$ncateg++
|
||||||
|
#}
|
||||||
|
#$nhour++;
|
||||||
|
#}
|
||||||
# and grand totals, percent and display status from db entries, and column widths
|
# and grand totals, percent and display status from db entries, and column widths
|
||||||
$ncateg = 0;
|
$ncateg = 0;
|
||||||
my $colpadding = 0;
|
my $colpadding = 0;
|
||||||
@ -402,8 +421,17 @@ while ( $ncateg < @categs) {
|
|||||||
$ncateg++
|
$ncateg++
|
||||||
}
|
}
|
||||||
|
|
||||||
my $starttai = Time::TAI64::unixtai64n($start);
|
#foreach my $hour (sort keys %counts) {
|
||||||
my $endtai = Time::TAI64::unixtai64n($end);
|
#printf "Hour: %2d\n", $hour; # Right-aligned hour
|
||||||
|
#foreach my $categ (sort keys %{ $counts{$hour} }) {
|
||||||
|
#printf " %-12s: %d\n", $categ, $counts{$hour}{$categ}; # Left-aligned category
|
||||||
|
#}
|
||||||
|
#}
|
||||||
|
|
||||||
|
#die("die");
|
||||||
|
|
||||||
|
#my $starttai = Time::TAI64::unixtai64n($start);
|
||||||
|
#my $endtai = Time::TAI64::unixtai64n($end);
|
||||||
my $sum_SARules = 0;
|
my $sum_SARules = 0;
|
||||||
|
|
||||||
# we remove non valid files
|
# we remove non valid files
|
||||||
@ -424,13 +452,23 @@ my $count = -1; #for loop reduction in debugging mode
|
|||||||
my $CurrentMailId = "";
|
my $CurrentMailId = "";
|
||||||
|
|
||||||
LINE: while (<>) {
|
LINE: while (<>) {
|
||||||
|
chomp;
|
||||||
|
|
||||||
next LINE if !(my($tai,$log) = split(' ',$_,2));
|
# Use a regex to extract the date, time, and the rest of the log
|
||||||
|
if (/^(\w+\s+\d+\s+\d{2}:\d{2}:\d{2})\s+\S+\s+\S+\[(\d+)\]:\s+(.*)/) {
|
||||||
|
my $datetime = $1; # Get the date and time
|
||||||
|
my $log_id = $2; # Get the log ID
|
||||||
|
my $log_entry = $3; # The rest of the log line
|
||||||
|
|
||||||
|
# Convert datetime to epoch
|
||||||
#If date specified, only process lines matching date
|
my $current_epoch = convert_to_epoch($datetime);
|
||||||
next LINE if ( $tai lt $starttai );
|
|
||||||
next LINE if ( $tai gt $endtai );
|
# If date specified, only process lines matching date
|
||||||
|
#print $datetime." ".$current_epoch." ".$start." ".$end."\n";
|
||||||
|
next LINE if ($current_epoch < $start);
|
||||||
|
next LINE if ($current_epoch > $end);
|
||||||
|
|
||||||
|
#print $datetime."\n";
|
||||||
|
|
||||||
#Count lines and skip out if debugging
|
#Count lines and skip out if debugging
|
||||||
$count++;
|
$count++;
|
||||||
@ -493,20 +531,36 @@ LINE: while (<>) {
|
|||||||
#only select Logterse output
|
#only select Logterse output
|
||||||
next LINE unless m/logging::logterse:/;
|
next LINE unless m/logging::logterse:/;
|
||||||
|
|
||||||
my $abstime = Time::TAI64::tai2unix($tai);
|
#my $abstime = Time::TAI64::tai2unix($tai);
|
||||||
my $abshour = floor( $abstime / 3600 ); # Hours since the epoch
|
#my $abshour = floor( $abstime / 3600 ); # Hours since the epoch
|
||||||
|
|
||||||
|
# Create a timestamp for the previous hour
|
||||||
|
my $previous_hour_epoch = $current_epoch - 3600; # Subtract 3600 seconds (1 hour)
|
||||||
|
|
||||||
|
# Convert epoch time to local time
|
||||||
|
my ($sec, $min, $hour) = localtime($previous_hour_epoch);
|
||||||
|
#print $sec." ".$min." ".$hour."\n";
|
||||||
|
$hour = ($hour==23)?0:$hour;
|
||||||
|
my $abshour = $hour+1;
|
||||||
|
#print "Abs:".$abshour."\n";
|
||||||
|
|
||||||
|
|
||||||
my ($timestamp_part, $log_part) = split('`',$_,2); #bjr 0.6.12
|
my ($timestamp_part, $log_part) = split('`',$_,2); #bjr 0.6.12
|
||||||
my (@log_items) = split $FS, $log_part;
|
my (@log_items) = split $FS, $log_part;
|
||||||
|
#print "0".$log_items[0]."\n";
|
||||||
|
#print "1".$log_items[1]."\n";
|
||||||
|
#print "5".$log_items[5]."\n";
|
||||||
|
#print "7".$log_items[7]."\n";
|
||||||
|
|
||||||
my (@timestamp_items) = split(' ',$timestamp_part);
|
my (@timestamp_items) = split(' ',$timestamp_part);
|
||||||
|
#print $timestamp_items[5];
|
||||||
|
#print "\n";
|
||||||
|
|
||||||
my $result= "rejected"; #Tag as rejected unti we know otherwise
|
my $result= "rejected"; #Tag as rejected unti we know otherwise
|
||||||
# we store the more recent recipient domain, for domain statistics
|
# we store the more recent recipient domain, for domain statistics
|
||||||
# in fact, we only store the first recipient. Could be sort of headhache
|
# in fact, we only store the first recipient. Could be sort of headhache
|
||||||
# to obtain precise stats with many recipients on more than one domain !
|
# to obtain precise stats with many recipients on more than one domain !
|
||||||
my $proc = $timestamp_items[1] ; #numeric Id for the email
|
my $proc = $timestamp_items[5] ; #numeric Id for the email
|
||||||
my $emailnum = $proc; #proc gets modified later...
|
my $emailnum = $proc; #proc gets modified later...
|
||||||
|
|
||||||
if ($emailnum == 23244) {
|
if ($emailnum == 23244) {
|
||||||
@ -716,22 +770,24 @@ LINE: while (<>) {
|
|||||||
#extract the spam score
|
#extract the spam score
|
||||||
# Remove count for rejectred as it looks as if it might get through!!
|
# Remove count for rejectred as it looks as if it might get through!!
|
||||||
$result= "queued";
|
$result= "queued";
|
||||||
if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") {
|
if (defined($log_items[8])){
|
||||||
$score = trim($1);
|
if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") {
|
||||||
if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric
|
$score = trim($1);
|
||||||
{
|
if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric
|
||||||
if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;}
|
{
|
||||||
else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";}
|
if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;}
|
||||||
|
else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";}
|
||||||
|
} else {
|
||||||
|
print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n";
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n";
|
# no SA score - treat it as ham
|
||||||
|
$hamcount++;$counts{$abshour}{$CATHAM}++;
|
||||||
|
}
|
||||||
|
if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
|
||||||
|
$byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ;
|
||||||
|
$currentrcptdomain{ $proc } = '' ;
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
# no SA score - treat it as ham
|
|
||||||
$hamcount++;$counts{$abshour}{$CATHAM}++;
|
|
||||||
}
|
|
||||||
if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
|
|
||||||
$byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ;
|
|
||||||
$currentrcptdomain{ $proc } = '' ;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -822,12 +878,16 @@ LINE: while (<>) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
#exit if $emailnum == 15858;
|
#exit if $emailnum == 15858;
|
||||||
|
#print "Counts:*$abshour* ".$counts{$abshour}{$CATNONCONF}."\n";
|
||||||
|
|
||||||
|
;
|
||||||
|
|
||||||
|
} # end of if regexp
|
||||||
} #END OF MAIN LOOP
|
} #END OF MAIN LOOP
|
||||||
|
|
||||||
#total up grand total Columns
|
#total up grand total Columns
|
||||||
$nhour = floor( $start / 3600 );
|
$nhour = 0;
|
||||||
while ( $nhour < $end / 3600 ) {
|
while ( $nhour < 24 ) {
|
||||||
$ncateg = 0; #past the where it came from columns
|
$ncateg = 0; #past the where it came from columns
|
||||||
while ( $ncateg < @categs) {
|
while ( $ncateg < @categs) {
|
||||||
#total columns
|
#total columns
|
||||||
@ -846,8 +906,8 @@ while ( $nhour < $end / 3600 ) {
|
|||||||
|
|
||||||
|
|
||||||
#Compute row totals and row percentages
|
#Compute row totals and row percentages
|
||||||
$nhour = floor( $start / 3600 );
|
$nhour = 0; #floor( $start / 3600 );
|
||||||
while ( $nhour < $end / 3600 ) {
|
while ( $nhour < 24 ){ #}$end / 3600 ) {
|
||||||
$counts{$nhour}{$categs[@categs-1]} = $counts{$nhour}{$categs[@categs-2]}*100/$totalexamined if $totalexamined;
|
$counts{$nhour}{$categs[@categs-1]} = $counts{$nhour}{$categs[@categs-2]}*100/$totalexamined if $totalexamined;
|
||||||
$nhour++;
|
$nhour++;
|
||||||
|
|
||||||
@ -865,8 +925,8 @@ while ( $nhour < $end / 3600 ) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#compute sum of row percentages
|
#compute sum of row percentages
|
||||||
$nhour = floor( $start / 3600 );
|
$nhour = 0; #floor( $start / 3600 );
|
||||||
while ( $nhour < $end / 3600 ) {
|
while ( $nhour < 24 ){ #}$end / 3600 ) {
|
||||||
$counts{$GRANDTOTAL}{$categs[@categs-1]} += $counts{$nhour}{$categs[@categs-1]};
|
$counts{$GRANDTOTAL}{$categs[@categs-1]} += $counts{$nhour}{$categs[@categs-1]};
|
||||||
$nhour++;
|
$nhour++;
|
||||||
|
|
||||||
@ -995,15 +1055,15 @@ if ( !$disabled ) {
|
|||||||
my $Totals; #Corresponding totals
|
my $Totals; #Corresponding totals
|
||||||
my $Percent; # and column percentages
|
my $Percent; # and column percentages
|
||||||
|
|
||||||
my $hour = floor( $start / 3600 );
|
my $hour = 0; #floor( $start / 3600 );
|
||||||
$Line1 = '';
|
$Line1 = '';
|
||||||
$Line2 = '';
|
$Line2 = '';
|
||||||
$Titles = '';
|
$Titles = '';
|
||||||
$Values = '';
|
$Values = '';
|
||||||
$Totals = '';
|
$Totals = '';
|
||||||
$Percent = '';
|
$Percent = '';
|
||||||
while ( $hour < $end / 3600 ) {
|
while ( $hour < 24){ #}$end / 3600 ) {
|
||||||
if ($hour == floor( $start / 3600 )){
|
if ($hour == 0){ #}floor( $start / 3600 )){
|
||||||
#Do all the once only things
|
#Do all the once only things
|
||||||
$ncateg = 0;
|
$ncateg = 0;
|
||||||
while ( $ncateg < @categs) {
|
while ( $ncateg < @categs) {
|
||||||
@ -1289,13 +1349,16 @@ sub analysis_period {
|
|||||||
$sec=0;$min=0;$hour=$base;
|
$sec=0;$min=0;$hour=$base;
|
||||||
};
|
};
|
||||||
#$mday="05"; #$mday="03"; #$mday="16"; #Temp!!
|
#$mday="05"; #$mday="03"; #$mday="16"; #Temp!!
|
||||||
|
#print $sec." ".$min." ".$hour." ".$mday." ".$mon." ".$year." ".$wday." ".$yday."\n";
|
||||||
$time = timelocal($sec,$min,$hour,$mday,$mon,$year);
|
$time = timelocal($sec,$min,$hour,$mday,$mon,$year);
|
||||||
|
#print $time."\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
my $start = str2time( $startdate );
|
my $start = str2time( $startdate );
|
||||||
my $end = $enddate ? str2time( $enddate ) :
|
my $end = $enddate ? str2time( $enddate ) :
|
||||||
$startdate ? $start + $secsininterval : $time;
|
$startdate ? $start + $secsininterval : $time;
|
||||||
$start = $startdate ? $start : $end - $secsininterval;
|
$start = $startdate ? $start : $end - $secsininterval;
|
||||||
|
#print "Analysis".$start." ".$end;
|
||||||
return ( $start > $end ) ? ( $end, $start ) : ( $start, $end );
|
return ( $start > $end ) ? ( $end, $start ) : ( $start, $end );
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1667,7 +1730,7 @@ sub save_data
|
|||||||
"mailstats", "mailstats" )
|
"mailstats", "mailstats" )
|
||||||
or die "Cannot open mailstats db - has it beeen created?";
|
or die "Cannot open mailstats db - has it beeen created?";
|
||||||
|
|
||||||
my $hour = floor( $start / 3600 );
|
my $hour = 0; #floor( $start / 3600 );
|
||||||
my $reportdate = strftime( "%F", localtime( $hour * 3600 ) );
|
my $reportdate = strftime( "%F", localtime( $hour * 3600 ) );
|
||||||
my $dateid = get_dateid($dbh,$reportdate);
|
my $dateid = get_dateid($dbh,$reportdate);
|
||||||
my $reccount = 0; #count number of records written
|
my $reccount = 0; #count number of records written
|
||||||
@ -1740,9 +1803,9 @@ sub save_data
|
|||||||
}
|
}
|
||||||
# finally - the hourly breakdown
|
# finally - the hourly breakdown
|
||||||
# need to remember here that the date might change during the 24 hour span
|
# need to remember here that the date might change during the 24 hour span
|
||||||
my $nhour = floor( $start / 3600 );
|
my $nhour = 0; #floor( $start / 3600 );
|
||||||
my $ncateg;
|
my $ncateg;
|
||||||
while ( $nhour < $end / 3600 ) {
|
while ( $nhour < 24){ #}$end / 3600 ) {
|
||||||
#see if the time record has been created
|
#see if the time record has been created
|
||||||
# print strftime("%H",localtime( $nhour * 3600 ) ).":00:00\n";
|
# print strftime("%H",localtime( $nhour * 3600 ) ).":00:00\n";
|
||||||
my $sth =
|
my $sth =
|
||||||
@ -1891,12 +1954,34 @@ sub get_dateid
|
|||||||
#} else { return 0}
|
#} else { return 0}
|
||||||
#}
|
#}
|
||||||
|
|
||||||
|
sub convert_to_epoch {
|
||||||
|
|
||||||
|
my ($sec, $min, $hour1, $mday, $mon, $year) = localtime();
|
||||||
|
$year += 1900; # localtime returns year as years since 1900
|
||||||
|
|
||||||
|
my ($datetime) = @_;
|
||||||
|
my ($month, $day, $time) = split(' ', $datetime);
|
||||||
|
my ($hour, $minute, $second) = split(':', $time);
|
||||||
|
|
||||||
|
my $month_num = {
|
||||||
|
Jan => 0, Feb => 1, Mar => 2, Apr => 3,
|
||||||
|
May => 4, Jun => 5, Jul => 6, Aug => 7,
|
||||||
|
Sep => 8, Oct => 9, Nov => 10, Dec => 11,
|
||||||
|
}->{$month};
|
||||||
|
|
||||||
|
|
||||||
|
return timelocal($second, $minute, $hour, $day, $month_num, $year); # Adjust the year as necessary
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
sub test_for_private_ip {
|
sub test_for_private_ip {
|
||||||
use NetAddr::IP;
|
use NetAddr::IP;
|
||||||
$_ = shift;
|
$_ = shift;
|
||||||
|
# Remove leading whitespace
|
||||||
|
s/^\s+//;
|
||||||
return unless /(\d+\.\d+\.\d+\.\d+)/;
|
return unless /(\d+\.\d+\.\d+\.\d+)/;
|
||||||
my $ip = NetAddr::IP->new($1);
|
my $ip = NetAddr::IP->new($1);
|
||||||
return unless $ip;
|
return unless $ip;
|
||||||
return $ip->is_rfc1918();
|
return $ip->is_rfc1918();
|
||||||
}
|
}
|
@ -6,7 +6,7 @@ Summary: Daily mail statistics for SME Server
|
|||||||
%define name smeserver-mailstats
|
%define name smeserver-mailstats
|
||||||
Name: %{name}
|
Name: %{name}
|
||||||
%define version 1.1
|
%define version 1.1
|
||||||
%define release 18
|
%define release 19
|
||||||
Version: %{version}
|
Version: %{version}
|
||||||
Release: %{release}%{?dist}
|
Release: %{release}%{?dist}
|
||||||
License: GPL
|
License: GPL
|
||||||
@ -31,6 +31,9 @@ A script that via cron.d e-mails mail statistics to admin on a daily basis.
|
|||||||
See http://www.contribs.org/bugzilla/show_bug.cgi?id=819
|
See http://www.contribs.org/bugzilla/show_bug.cgi?id=819
|
||||||
|
|
||||||
%changelog
|
%changelog
|
||||||
|
* Mon Dec 30 2024 Brian Read <brianr@koozali.org> 1.1-19.sme
|
||||||
|
- Update mailstats.pl to accomodate change in log format for SME11 [SME: ]
|
||||||
|
|
||||||
* Fri Jun 07 2024 Brian Read <brianr@koozali.org> 1.1-18.sme
|
* Fri Jun 07 2024 Brian Read <brianr@koozali.org> 1.1-18.sme
|
||||||
- Pull in python re-write from SME11 dev [SME: ]
|
- Pull in python re-write from SME11 dev [SME: ]
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user