smeserver-sysmon/root/usr/sbin/sysmon

395 lines
19 KiB
Plaintext
Raw Normal View History

#!/usr/bin/perl
use warnings;
use strict;
use POSIX;
use RRDs;
use IPC::Open3;
use Symbol;
use Carp;
use esmith::config;
use esmith::db;
use sigtrap;
use File::Slurp qw/ read_file /;
use File::Find::Rule;
$ENV {'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/lib/sa';
$ENV {'LANG'} = 'C';
croak "ERROR: Already running!" if (-e "/var/lock/sysmon");
my $pingfactor;
my $return = `fping -C 1 localhost 2>&1`;
croak "ERROR: FPing must be installed setuid root or it will not work\n" if $return =~ m/only.+root/;
if ($return =~ m/bytes, ([0-9.]+)\sms\s+.*\n.*\n.*:\s+([0-9.]+)/) {
$pingfactor = 1000 * $2/$1;
} else {
$pingfactor = 1000;
}
my %conf;
tie %conf, 'esmith::config';
my $status = db_get_prop(\%conf, 'sysmon', 'status') || "disabled";
croak "ERROR: sysmon is not enabled" if $status ne 'enabled';
my $upsstatus = db_get_prop(\%conf, 'nut', 'status') || "disabled";
my $rrddir = '/var/lib/rrd';
unless (fork) {
open (FILELOCK,">/var/lock/sysmon");
print FILELOCK $$;
close FILELOCK;
&POSIX::setsid;
unlink("/var/log/sysmon.old") if -e "/var/log/sysmon.old";
rename("/var/log/sysmon","/var/log/sysmon.old") if -e "/var/log/sysmon";
open STDOUT,'>>/var/log/sysmon';
open STDERR,'>>/var/log/sysmon';
open STDIN, '</dev/null';
$SIG{QUIT} = sub { unlink("/var/lock/sysmon"); exit 0; };
$SIG{TERM} = sub { unlink("/var/lock/sysmon"); exit 0; };
while(1) {
mkdir($rrddir) unless (-e $rrddir);
my $days = 365;
my @files = File::Find::Rule->file("*.rrd")
->maxdepth(1)
->in($rrddir);
for my $file (@files){
if (-M $file > $days){
print "deleting $file\n";
unlink $file or warn $!;
}
}
my $lastupdate = RRDs::last("/var/lib/rrd/sysmon.rrd") || 0;
my $datahash = ();
my $badifaces = `/sbin/ifconfig -a | /bin/sed -ne '/^\\w\\+[0-9]/{;x;n;/inet addr/bd;x;s/^\\(\\w\\+[0-9]\\+\\).*/\\1/p;:d;}'` || 'none';
chomp($badifaces);
rename("/var/lib/rrd/sysmon.stats","/var/lib/rrd/sysmon.process");
if ( -x '/usr/lib/sa/sadc' ) {
`/usr/lib/sa/sadc 1 1 /var/lib/rrd/sysmon.process > /dev/null 2>&1; /usr/lib/sa/sadc 1 1 /var/lib/rrd/sysmon.stats > /dev/null 2>&1`;}
else
{
`/usr/lib64/sa/sadc 1 1 /var/lib/rrd/sysmon.process > /dev/null 2>&1; /usr/lib64/sa/sadc 1 1 /var/lib/rrd/sysmon.stats > /dev/null 2>&1`;}
my @sardata;
if ( -x '/usr/bin/sadf' ) { @sardata=`/usr/bin/sadf -U -- -qrbuS -n DEV /var/lib/rrd/sysmon.process 2> /dev/null`; }# sadf
else { @sardata = `/usr/bin/sar -hqruS -P ALL -n DEV -f /var/lib/rrd/sysmon.process 2> /dev/null`; }# old systat
unlink("/var/lib/rrd/sysmon.process");
if($#sardata > 0 && $sardata[0] =~ /\s+(\d+)\s+(\d{10})/ && $1 >= 45 && $1 <= 135 && $2 > $lastupdate) {
open(FILE,"</proc/uptime"); my @filedata = <FILE>; close(FILE);
if ($filedata[0] =~ /^([0-9,.]+)/) { $datahash->{uptime} = $1; };
my @gethd = `/bin/df --block-size=1 --portability --local | grep '^/dev'`;
foreach (@gethd) {
if(/^\/dev\/(?:mapper\/)?(\S+)\s+\d+\s+(\d+)\s+(\d+)\s+\d+\%/) {
$datahash->{hdtotal}->{$1} = $2 + $3;
$datahash->{hdfree}->{$1} = $3;
$datahash->{hdused}->{$1} = $2;
$datahash->{hdtotal}->{all} += $2 + $3;
$datahash->{hdfree}->{all} += $3;
$datahash->{hdused}->{all} += $2;
}
}
foreach(@sardata) {
if(/(all|cpu\d+)\s+\%(user|nice|system|iowait|idle)\s+([0-9,.]+)/) { $datahash->{$2}->{$1}=$3/100; }
if(/(runq-sz|plist-sz|ldavg-[15]+)\s+([0-9,.]+)/) { $datahash->{$1}=$2; }
if(/kb(memfree|memused|buffers|cached|swpfree|swpused)\s+([0-9,.]+)/) { $datahash->{$1}=$2*1024; }
if(/(\w+)\s+(rxkB|txkB|rxpck|txpck)\/s\s+([0-9,.]+)/) { $datahash->{$2}->{$1}=$3; }
}
foreach (keys %{$datahash->{idle}}) {
$datahash->{total}->{$_}=1-$datahash->{idle}->{$_};
}
$datahash->{memtotal} = $datahash->{memused} + $datahash->{memfree};
$datahash->{memused} -= $datahash->{cached} + $datahash->{buffers};
$datahash->{memfree} += $datahash->{cached} + $datahash->{buffers};
$datahash->{swptotal} = $datahash->{swpused} + $datahash->{swpfree};
RRDs::create("/var/lib/rrd/sysmon.rrd","-s","60",
"DS:total:GAUGE:120:0:150",
"DS:user:GAUGE:120:0:150",
"DS:nice:GAUGE:120:0:150",
"DS:system:GAUGE:120:0:150",
"DS:iowait:GAUGE:120:0:150",
"DS:idle:GAUGE:120:0:150",
"DS:runq-sz:GAUGE:120:0:100",
"DS:plist-sz:GAUGE:120:0:1000",
"DS:ldavg-1:GAUGE:120:0:100",
"DS:ldavg-5:GAUGE:120:0:100",
"DS:memtotal:GAUGE:120:0:U",
"DS:memfree:GAUGE:120:0:U",
"DS:memused:GAUGE:120:0:U",
"DS:buffers:GAUGE:120:0:U",
"DS:cached:GAUGE:120:0:U",
"DS:swptotal:GAUGE:120:0:U",
"DS:swpfree:GAUGE:120:0:U",
"DS:swpused:GAUGE:120:0:U",
"DS:hdtotal:GAUGE:120:0:U",
"DS:hdfree:GAUGE:120:0:U",
"DS:hdused:GAUGE:120:0:U",
"DS:uptime:GAUGE:120:0:U",
"RRA:MIN:0.5:1:1500",
"RRA:MIN:0.5:5:600",
"RRA:MIN:0.5:30:700",
"RRA:MIN:0.5:120:775",
"RRA:MIN:0.5:1440:797",
"RRA:AVERAGE:0.5:1:1500",
"RRA:AVERAGE:0.5:5:600",
"RRA:AVERAGE:0.5:30:700",
"RRA:AVERAGE:0.5:120:775",
"RRA:AVERAGE:0.5:1440:797",
"RRA:MAX:0.5:1:1500",
"RRA:MAX:0.5:5:600",
"RRA:MAX:0.5:30:700",
"RRA:MAX:0.5:120:775",
"RRA:MAX:0.5:1440:797") unless -e "/var/lib/rrd/sysmon.rrd";
RRDs::update("/var/lib/rrd/sysmon.rrd",
"N:$datahash->{total}->{all}:$datahash->{user}->{all}:$datahash->{nice}->{all}:" .
"$datahash->{system}->{all}:$datahash->{iowait}->{all}:$datahash->{idle}->{all}:" .
"$datahash->{'runq-sz'}:$datahash->{'plist-sz'}:$datahash->{'ldavg-1'}:" .
"$datahash->{'ldavg-5'}:$datahash->{memtotal}:$datahash->{memfree}:" .
"$datahash->{memused}:$datahash->{buffers}:$datahash->{cached}:$datahash->{swptotal}:" .
"$datahash->{swpfree}:$datahash->{swpused}:$datahash->{hdtotal}->{all}:$datahash->{hdfree}->{all}:" .
"$datahash->{hdused}->{all}:$datahash->{uptime}");
rename("/var/lib/rrd/sysmon.rrd","/var/lib/rrd/sysmon.bad") if(RRDs::error);
print RRDs::error . "\n" if (RRDs::error);
foreach my $cpu (keys %{$datahash->{idle}}) {
next if $cpu eq 'all';
RRDs::create("/var/lib/rrd/$cpu.rrd","-s","60",
"DS:total:GAUGE:120:0:150",
"DS:user:GAUGE:120:0:150",
"DS:nice:GAUGE:120:0:150",
"DS:system:GAUGE:120:0:150",
"DS:iowait:GAUGE:120:0:150",
"DS:idle:GAUGE:120:0:150",
"RRA:MIN:0.5:1:1500",
"RRA:MIN:0.5:5:600",
"RRA:MIN:0.5:30:700",
"RRA:MIN:0.5:120:775",
"RRA:MIN:0.5:1440:797",
"RRA:AVERAGE:0.5:1:1500",
"RRA:AVERAGE:0.5:5:600",
"RRA:AVERAGE:0.5:30:700",
"RRA:AVERAGE:0.5:120:775",
"RRA:AVERAGE:0.5:1440:797",
"RRA:MAX:0.5:1:1500",
"RRA:MAX:0.5:5:600",
"RRA:MAX:0.5:30:700",
"RRA:MAX:0.5:120:775",
"RRA:MAX:0.5:1440:797") unless -e "/var/lib/rrd/$cpu.rrd";
RRDs::update("/var/lib/rrd/$cpu.rrd",
"N:$datahash->{total}->{$cpu}:$datahash->{user}->{$cpu}:$datahash->{nice}->{$cpu}:" .
"$datahash->{system}->{$cpu}:$datahash->{iowait}->{$cpu}:$datahash->{idle}->{$cpu}");
rename("/var/lib/rrd/$cpu.rrd","/var/lib/rrd/$cpu.bad") if(RRDs::error);
print RRDs::error . "\n" if (RRDs::error);
}
foreach my $drive (keys %{$datahash->{hdtotal}}) {
next if $drive eq 'all';
RRDs::create("/var/lib/rrd/drive_$drive.rrd","-s","60",
"DS:hdtotal:GAUGE:120:0:U",
"DS:hdfree:GAUGE:120:0:U",
"DS:hdused:GAUGE:120:0:U",
"RRA:MIN:0.5:1:1500",
"RRA:MIN:0.5:5:600",
"RRA:MIN:0.5:30:700",
"RRA:MIN:0.5:120:775",
"RRA:MIN:0.5:1440:797",
"RRA:AVERAGE:0.5:1:1500",
"RRA:AVERAGE:0.5:5:600",
"RRA:AVERAGE:0.5:30:700",
"RRA:AVERAGE:0.5:120:775",
"RRA:AVERAGE:0.5:1440:797",
"RRA:MAX:0.5:1:1500",
"RRA:MAX:0.5:5:600",
"RRA:MAX:0.5:30:700",
"RRA:MAX:0.5:120:775",
"RRA:MAX:0.5:1440:797") unless -e "/var/lib/rrd/drive_$drive.rrd";
RRDs::update("/var/lib/rrd/drive_$drive.rrd",
"N:$datahash->{hdtotal}->{$drive}:$datahash->{hdfree}->{$drive}:$datahash->{hdused}->{$drive}");
rename("/var/lib/rrd/drive_$drive.rrd","/var/lib/rrd/drive_$drive.bad") if(RRDs::error);
print RRDs::error . "\n" if (RRDs::error);
}
foreach my $iface (keys %{$datahash->{rxkB}}) {
next if ($iface =~ /(lo|br\d+|imq\d+)$/);
next if ($datahash->{rxkB}->{$iface} == 0);
next if ($badifaces =~ /$iface/);
RRDs::create("/var/lib/rrd/iface_$iface.rrd","-s","60",
"DS:rxkB:" . ($iface =~ /^sw/ ? "COUNTER" : "GAUGE" ) . ":120:0:U",
"DS:txkB:" . ($iface =~ /^sw/ ? "COUNTER" : "GAUGE" ) . ":120:0:U",
"DS:rxpck:" . ($iface =~ /^sw/ ? "COUNTER" : "GAUGE" ) . ":120:0:U",
"DS:txpck:" . ($iface =~ /^sw/ ? "COUNTER" : "GAUGE" ) . ":120:0:U",
"RRA:MIN:0.5:1:1500",
"RRA:MIN:0.5:5:600",
"RRA:MIN:0.5:30:700",
"RRA:MIN:0.5:120:775",
"RRA:MIN:0.5:1440:797",
"RRA:AVERAGE:0.5:1:1500",
"RRA:AVERAGE:0.5:5:600",
"RRA:AVERAGE:0.5:30:700",
"RRA:AVERAGE:0.5:120:775",
"RRA:AVERAGE:0.5:1440:797",
"RRA:MAX:0.5:1:1500",
"RRA:MAX:0.5:5:600",
"RRA:MAX:0.5:30:700",
"RRA:MAX:0.5:120:775",
"RRA:MAX:0.5:1440:797") unless -e "/var/lib/rrd/iface_$iface.rrd";
RRDs::update("/var/lib/rrd/iface_$iface.rrd",
"N:$datahash->{rxkB}->{$iface}:$datahash->{txkB}->{$iface}:$datahash->{rxpck}->{$iface}:" .
"$datahash->{txpck}->{$iface}");
rename("/var/lib/rrd/iface_$iface.rrd","/var/lib/rrd/iface_$iface.bad") if(RRDs::error);
print RRDs::error . "\n" if (RRDs::error);
}
my $cnt=0;
if ( $upsstatus eq 'enabled') {
my @upsentries = ('i_voltage', 'u_load', 'b_charge', 'b_runtime', 'b_voltage', 'u_temperature') ;
#use File::Slurp qw/ read_file /;
my $upslines = read_file('/etc/ups/ups.conf');
my @upses = $upslines =~ m/\[(.*)\]\n/;
foreach(@upses) {
chomp;
my $upsname="$_\@localhost";
$cnt++;
my @upsdata = `/usr/bin/upsc $upsname 2> /dev/null`;
$upslines="ups $upsname: ";
foreach(@upsdata) {
if(/^([a-z])[^.]+\.(voltage|load|charge|runtime|temperature|humidity)\s*:\s*([0-9,.]+)/) {
my $element = "$1_$2";
$upslines .= "$element=$3, ";
$datahash->{ups}->{$cnt}->{$element}=$3;
if($element=~/time/i) { $datahash->{ups}->{$cnt}->{$element} /= 60; }
}
}
foreach my $element (@upsentries){
next if exists($datahash->{ups}->{$cnt}->{$element});
$datahash->{ups}->{$cnt}->{$element}='';
$upslines .= "$element=, ";
}
}
}
foreach my $ups (keys %{$datahash->{ups}}) {
RRDs::create("/var/lib/rrd/ups$ups.rrd","-s","60",
(map {"DS:$_:GAUGE:120:0:U"} sort keys %{$datahash->{ups}->{$ups}}),
"RRA:MIN:0.5:1:1500",
"RRA:MIN:0.5:5:600",
"RRA:MIN:0.5:30:700",
"RRA:MIN:0.5:120:775",
"RRA:MIN:0.5:1440:797",
"RRA:AVERAGE:0.5:1:1500",
"RRA:AVERAGE:0.5:5:600",
"RRA:AVERAGE:0.5:30:700",
"RRA:AVERAGE:0.5:120:775",
"RRA:AVERAGE:0.5:1440:797",
"RRA:MAX:0.5:1:1500",
"RRA:MAX:0.5:5:600",
"RRA:MAX:0.5:30:700",
"RRA:MAX:0.5:120:775",
"RRA:MAX:0.5:1440:797") unless -e "/var/lib/rrd/ups$ups.rrd";
RRDs::update("/var/lib/rrd/ups$ups.rrd",
"N:".(join ":", map { $datahash->{ups}->{$ups}->{$_} } sort keys %{$datahash->{ups}->{$ups}}));
rename("/var/lib/rrd/ups$ups.rrd","/var/lib/rrd/ups$ups.bad") if(RRDs::error);
print RRDs::error . "\n" if (RRDs::error);
}
if( -e '/usr/bin/sensors') {
my $check = `/sbin/lsmod 2> /dev/null`;
if( $check =~ /i2c_/ ) {
my @sensordata = `/usr/bin/sensors -u 2> /dev/null`;
foreach(@sensordata) {
#pre SME9 if(/(-?[0-9,.]+) \(((?:in|fan|temp|vid|vrm)[0-9]*)\)/) { $datahash->{sensors}->{$2} = $1; print "$2=$1, "}
if(/(in|fan|temp|vid|vrm)([0-9])_input: ?(-?[0-9,.]+)/) { $datahash->{sensors}->{"$1$2"} = $3;} #print "sensors:$1$2=$3\n"}
}
if(defined($datahash->{sensors})) {
RRDs::create("/var/lib/rrd/sensors.rrd","-s","60",
(map {"DS:$_:GAUGE:120:U:U"} sort keys %{$datahash->{sensors}}),
"RRA:MIN:0.5:1:1500",
"RRA:MIN:0.5:5:600",
"RRA:MIN:0.5:30:700",
"RRA:MIN:0.5:120:775",
"RRA:MIN:0.5:1440:797",
"RRA:AVERAGE:0.5:1:1500",
"RRA:AVERAGE:0.5:5:600",
"RRA:AVERAGE:0.5:30:700",
"RRA:AVERAGE:0.5:120:775",
"RRA:AVERAGE:0.5:1440:797",
"RRA:MAX:0.5:1:1500",
"RRA:MAX:0.5:5:600",
"RRA:MAX:0.5:30:700",
"RRA:MAX:0.5:120:775",
"RRA:MAX:0.5:1440:797") unless -e "/var/lib/rrd/sensors.rrd";
RRDs::update("/var/lib/rrd/sensors.rrd",
"N:".(join ":", map { $datahash->{sensors}->{$_} } sort keys %{$datahash->{sensors}}));
rename("/var/lib/rrd/sensors.rrd","/var/lib/rrd/sensors.bad") if(RRDs::error);
print RRDs::error . "\n" if (RRDs::error);
}
}
}
my $inh = gensym;
my $outh = gensym;
my $errh = gensym;
my $gateway = `/sbin/route -n | /bin/egrep '^0\.' | /bin/awk '{ print \$2 }' | /bin/egrep '^[1-9]'` ||
`/sbin/ifconfig | /bin/sed -ne '/P-t-P/p' | /bin/awk '{ print \$3 }' | /bin/cut -d: -f2` ||
'down';
chomp $gateway;
my $hosts = db_get_prop(\%conf, 'sysmon', 'hosts') || "gateway";
$hosts =~ s/\bgateway\b/$gateway/g;
$hosts =~ s/[,:; ]+/ /g;
my @pings = `fping -C 25 -q -B1 -t1000 -r1 -p 500 $hosts 2>&1`;
foreach (@pings) {
chomp;
my @times = split /\s+/;
my $ip = shift @times;
$ip =~ s/\b$gateway\b/gateway/g;
if (':' eq shift @times) {
@times = map {sprintf "%.10e", $_ / $pingfactor} sort {$a <=> $b} grep /^\d/, @times;
my $median = $times[int($#times/2)] || 'U';
my $loss = 25 - ($#times+1);
if ($loss >= 25) {
@times = map {'U'} 1..25;
} else {
splice @times, sprintf("%.0f", int(($#times+1)/2)),0,map {'U'} 1..$loss;
}
RRDs::create("/var/lib/rrd/host_$ip.rrd","-s","60",
"DS:loss:GAUGE:120:0:25",
"DS:median:GAUGE:120:0:180",
(map {"DS:ping${_}:GAUGE:120:0:180"} 1..25),
"RRA:MIN:0.5:1:1500",
"RRA:MIN:0.5:5:600",
"RRA:MIN:0.5:30:700",
"RRA:MIN:0.5:120:775",
"RRA:MIN:0.5:1440:797",
"RRA:AVERAGE:0.5:1:1500",
"RRA:AVERAGE:0.5:5:600",
"RRA:AVERAGE:0.5:30:700",
"RRA:AVERAGE:0.5:120:775",
"RRA:AVERAGE:0.5:1440:797",
"RRA:MAX:0.5:1:1500",
"RRA:MAX:0.5:5:600",
"RRA:MAX:0.5:30:700",
"RRA:MAX:0.5:120:775",
"RRA:MAX:0.5:1440:797") unless -e "/var/lib/rrd/host_$ip.rrd";
RRDs::update("/var/lib/rrd/host_$ip.rrd",
"N:$loss:$median:".(join ':', @times));
rename("/var/lib/rrd/host_$ip.rrd","/var/lib/rrd/host_$ip.bad") if(RRDs::error);
print RRDs::error . "\n" if (RRDs::error);
}
}
}
sleep(60-time%60);
}
}