From 382489ecacbbec28b7dc863bae8276899f3c87a7 Mon Sep 17 00:00:00 2001 From: Brian Read Date: Mon, 30 Dec 2024 07:07:45 +0000 Subject: [PATCH] * Mon Dec 30 2024 Brian Read 1.1-19.sme - Update mailstats.pl to accomodate change in log format for SME11 [SME: ] --- root/usr/bin/mailstats.pl | 179 ++++++++++++++++++++++++++++---------- smeserver-mailstats.spec | 5 +- 2 files changed, 136 insertions(+), 48 deletions(-) diff --git a/root/usr/bin/mailstats.pl b/root/usr/bin/mailstats.pl index 420916b..0a60029 100644 --- a/root/usr/bin/mailstats.pl +++ b/root/usr/bin/mailstats.pl @@ -33,6 +33,7 @@ use strict; # bjr - 18Oct20 - Alter use of lc to avoid uninitialised messages - bug 11044 # bjr - 02Apr21 - Fix up lc to try to avoif uninit messages - and alter warning status - bug 11519 # bjr - 15Feb23 - Add in auth::auth_imap after change to use dovecot as incoming authorisation Bugzilla 12327 +# bjr - 29Dec24 - Convert to SME11 log date format # ############################################################################# # @@ -70,6 +71,11 @@ use strict; # # internal modules (part of core perl distribution) +# +# New format for SME11 +# +#Dec 17 19:34:34 sme11 qpsmtpd-forkserver[441318]: 441318 (deny) logging::logterse: ` 192.168.1.4 pc-00004.thereadclan.me.uk bjsystems.co.uk check_goodrcptto 901 relaying denied smeserver@thereadclan.me.uk msg denied before queued + use strict; use warnings; use Getopt::Long; @@ -105,7 +111,7 @@ if ($cdb->get('mailstats')){ #Configuration section my %opt = ( - version => '0.7.16', # please update at each change. + version => '0.8.01', # please update at each change. debug => 0, # guess what ? sendmail => '/usr/sbin/sendmail', # Path to sendmail stub from => 'spamfilter-stats', # Who is the mail from @@ -370,17 +376,30 @@ my $makeHTMLpage = "no"; # Init the hashes -my $nhour = floor( $start / 3600 ); +my $nhour = 0; +#print "Hour:".$nhour."\n"; my $ncateg; -while ( $nhour < $end / 3600 ) { - $counts{$nhour}=(); - $ncateg = 0; - while ( $ncateg < @categs) { - $counts{$nhour}{$categs[$ncateg-1]} = 0; - $ncateg++ +while ( $nhour < 24 ) { + $counts{$nhour} = {}; # Initialize as a hash reference + #print "Hour:".$nhour."\n"; + my $ncateg = 0; # Reset $ncateg for each hour + + while ( $ncateg < @categs ) { + $counts{$nhour}{$categs[$ncateg]} = 0; # Corrected index + $ncateg++; # Increment $ncateg } - $nhour++; + $nhour++; # Increment $nhour } + +#while ( $nhour < $end / 3600 ) { + #$counts{$nhour}=(); + #$ncateg = 0; + #while ( $ncateg < @categs) { + #$counts{$nhour}{$categs[$ncateg-1]} = 0; + #$ncateg++ + #} + #$nhour++; +#} # and grand totals, percent and display status from db entries, and column widths $ncateg = 0; my $colpadding = 0; @@ -402,8 +421,17 @@ while ( $ncateg < @categs) { $ncateg++ } -my $starttai = Time::TAI64::unixtai64n($start); -my $endtai = Time::TAI64::unixtai64n($end); +#foreach my $hour (sort keys %counts) { + #printf "Hour: %2d\n", $hour; # Right-aligned hour + #foreach my $categ (sort keys %{ $counts{$hour} }) { + #printf " %-12s: %d\n", $categ, $counts{$hour}{$categ}; # Left-aligned category + #} +#} + +#die("die"); + +#my $starttai = Time::TAI64::unixtai64n($start); +#my $endtai = Time::TAI64::unixtai64n($end); my $sum_SARules = 0; # we remove non valid files @@ -424,13 +452,23 @@ my $count = -1; #for loop reduction in debugging mode my $CurrentMailId = ""; LINE: while (<>) { + chomp; - next LINE if !(my($tai,$log) = split(' ',$_,2)); + # Use a regex to extract the date, time, and the rest of the log + if (/^(\w+\s+\d+\s+\d{2}:\d{2}:\d{2})\s+\S+\s+\S+\[(\d+)\]:\s+(.*)/) { + my $datetime = $1; # Get the date and time + my $log_id = $2; # Get the log ID + my $log_entry = $3; # The rest of the log line - - #If date specified, only process lines matching date - next LINE if ( $tai lt $starttai ); - next LINE if ( $tai gt $endtai ); + # Convert datetime to epoch + my $current_epoch = convert_to_epoch($datetime); + + # If date specified, only process lines matching date + #print $datetime." ".$current_epoch." ".$start." ".$end."\n"; + next LINE if ($current_epoch < $start); + next LINE if ($current_epoch > $end); + + #print $datetime."\n"; #Count lines and skip out if debugging $count++; @@ -493,20 +531,36 @@ LINE: while (<>) { #only select Logterse output next LINE unless m/logging::logterse:/; - my $abstime = Time::TAI64::tai2unix($tai); - my $abshour = floor( $abstime / 3600 ); # Hours since the epoch + #my $abstime = Time::TAI64::tai2unix($tai); + #my $abshour = floor( $abstime / 3600 ); # Hours since the epoch + + # Create a timestamp for the previous hour + my $previous_hour_epoch = $current_epoch - 3600; # Subtract 3600 seconds (1 hour) + + # Convert epoch time to local time + my ($sec, $min, $hour) = localtime($previous_hour_epoch); + #print $sec." ".$min." ".$hour."\n"; + $hour = ($hour==23)?0:$hour; + my $abshour = $hour+1; + #print "Abs:".$abshour."\n"; my ($timestamp_part, $log_part) = split('`',$_,2); #bjr 0.6.12 my (@log_items) = split $FS, $log_part; + #print "0".$log_items[0]."\n"; + #print "1".$log_items[1]."\n"; + #print "5".$log_items[5]."\n"; + #print "7".$log_items[7]."\n"; my (@timestamp_items) = split(' ',$timestamp_part); + #print $timestamp_items[5]; + #print "\n"; - my $result= "rejected"; #Tag as rejected unti we know otherwise + my $result= "rejected"; #Tag as rejected unti we know otherwise # we store the more recent recipient domain, for domain statistics # in fact, we only store the first recipient. Could be sort of headhache # to obtain precise stats with many recipients on more than one domain ! - my $proc = $timestamp_items[1] ; #numeric Id for the email + my $proc = $timestamp_items[5] ; #numeric Id for the email my $emailnum = $proc; #proc gets modified later... if ($emailnum == 23244) { @@ -716,22 +770,24 @@ LINE: while (<>) { #extract the spam score # Remove count for rejectred as it looks as if it might get through!! $result= "queued"; - if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") { - $score = trim($1); - if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric - { - if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;} - else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";} + if (defined($log_items[8])){ + if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") { + $score = trim($1); + if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric + { + if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;} + else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";} + } else { + print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n"; + } } else { - print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n"; + # no SA score - treat it as ham + $hamcount++;$counts{$abshour}{$CATHAM}++; + } + if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { + $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; + $currentrcptdomain{ $proc } = '' ; } - } else { - # no SA score - treat it as ham - $hamcount++;$counts{$abshour}{$CATHAM}++; - } - if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { - $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; - $currentrcptdomain{ $proc } = '' ; } } @@ -822,12 +878,16 @@ LINE: while (<>) { } } #exit if $emailnum == 15858; + #print "Counts:*$abshour* ".$counts{$abshour}{$CATNONCONF}."\n"; + +; + } # end of if regexp } #END OF MAIN LOOP #total up grand total Columns -$nhour = floor( $start / 3600 ); -while ( $nhour < $end / 3600 ) { +$nhour = 0; +while ( $nhour < 24 ) { $ncateg = 0; #past the where it came from columns while ( $ncateg < @categs) { #total columns @@ -846,8 +906,8 @@ while ( $nhour < $end / 3600 ) { #Compute row totals and row percentages -$nhour = floor( $start / 3600 ); -while ( $nhour < $end / 3600 ) { +$nhour = 0; #floor( $start / 3600 ); +while ( $nhour < 24 ){ #}$end / 3600 ) { $counts{$nhour}{$categs[@categs-1]} = $counts{$nhour}{$categs[@categs-2]}*100/$totalexamined if $totalexamined; $nhour++; @@ -865,8 +925,8 @@ while ( $nhour < $end / 3600 ) { } #compute sum of row percentages -$nhour = floor( $start / 3600 ); -while ( $nhour < $end / 3600 ) { +$nhour = 0; #floor( $start / 3600 ); +while ( $nhour < 24 ){ #}$end / 3600 ) { $counts{$GRANDTOTAL}{$categs[@categs-1]} += $counts{$nhour}{$categs[@categs-1]}; $nhour++; @@ -995,15 +1055,15 @@ if ( !$disabled ) { my $Totals; #Corresponding totals my $Percent; # and column percentages - my $hour = floor( $start / 3600 ); + my $hour = 0; #floor( $start / 3600 ); $Line1 = ''; $Line2 = ''; $Titles = ''; $Values = ''; $Totals = ''; $Percent = ''; - while ( $hour < $end / 3600 ) { - if ($hour == floor( $start / 3600 )){ + while ( $hour < 24){ #}$end / 3600 ) { + if ($hour == 0){ #}floor( $start / 3600 )){ #Do all the once only things $ncateg = 0; while ( $ncateg < @categs) { @@ -1289,13 +1349,16 @@ sub analysis_period { $sec=0;$min=0;$hour=$base; }; #$mday="05"; #$mday="03"; #$mday="16"; #Temp!! + #print $sec." ".$min." ".$hour." ".$mday." ".$mon." ".$year." ".$wday." ".$yday."\n"; $time = timelocal($sec,$min,$hour,$mday,$mon,$year); + #print $time."\n" } my $start = str2time( $startdate ); my $end = $enddate ? str2time( $enddate ) : $startdate ? $start + $secsininterval : $time; $start = $startdate ? $start : $end - $secsininterval; + #print "Analysis".$start." ".$end; return ( $start > $end ) ? ( $end, $start ) : ( $start, $end ); } @@ -1667,7 +1730,7 @@ sub save_data "mailstats", "mailstats" ) or die "Cannot open mailstats db - has it beeen created?"; - my $hour = floor( $start / 3600 ); + my $hour = 0; #floor( $start / 3600 ); my $reportdate = strftime( "%F", localtime( $hour * 3600 ) ); my $dateid = get_dateid($dbh,$reportdate); my $reccount = 0; #count number of records written @@ -1740,9 +1803,9 @@ sub save_data } # finally - the hourly breakdown # need to remember here that the date might change during the 24 hour span - my $nhour = floor( $start / 3600 ); + my $nhour = 0; #floor( $start / 3600 ); my $ncateg; - while ( $nhour < $end / 3600 ) { + while ( $nhour < 24){ #}$end / 3600 ) { #see if the time record has been created # print strftime("%H",localtime( $nhour * 3600 ) ).":00:00\n"; my $sth = @@ -1891,12 +1954,34 @@ sub get_dateid #} else { return 0} #} +sub convert_to_epoch { + + my ($sec, $min, $hour1, $mday, $mon, $year) = localtime(); + $year += 1900; # localtime returns year as years since 1900 + + my ($datetime) = @_; + my ($month, $day, $time) = split(' ', $datetime); + my ($hour, $minute, $second) = split(':', $time); + + my $month_num = { + Jan => 0, Feb => 1, Mar => 2, Apr => 3, + May => 4, Jun => 5, Jul => 6, Aug => 7, + Sep => 8, Oct => 9, Nov => 10, Dec => 11, + }->{$month}; + + + return timelocal($second, $minute, $hour, $day, $month_num, $year); # Adjust the year as necessary +} + + sub test_for_private_ip { use NetAddr::IP; $_ = shift; + # Remove leading whitespace + s/^\s+//; return unless /(\d+\.\d+\.\d+\.\d+)/; my $ip = NetAddr::IP->new($1); return unless $ip; return $ip->is_rfc1918(); -} +} \ No newline at end of file diff --git a/smeserver-mailstats.spec b/smeserver-mailstats.spec index f53b8ca..aebaf58 100644 --- a/smeserver-mailstats.spec +++ b/smeserver-mailstats.spec @@ -6,7 +6,7 @@ Summary: Daily mail statistics for SME Server %define name smeserver-mailstats Name: %{name} %define version 1.1 -%define release 18 +%define release 19 Version: %{version} Release: %{release}%{?dist} License: GPL @@ -31,6 +31,9 @@ A script that via cron.d e-mails mail statistics to admin on a daily basis. See http://www.contribs.org/bugzilla/show_bug.cgi?id=819 %changelog +* Mon Dec 30 2024 Brian Read 1.1-19.sme +- Update mailstats.pl to accomodate change in log format for SME11 [SME: ] + * Fri Jun 07 2024 Brian Read 1.1-18.sme - Pull in python re-write from SME11 dev [SME: ]