Update to 2021-12-01 19:13

roles/g2cs/files/g2cs.pl

@@ -0,0 +1,183 @@
+#!/usr/bin/perl -w
+  
+use IO::Socket;
+use Getopt::Long;
+use File::Basename;
+use File::Path qw(make_path);
+use IO::Handle;
+
+my $maxlen = 16384;
+my $port = 514;
+my $maxlines = 10000;
+my $logdir = '/run/cs-gelf-server/';
+
+GetOptions(
+  "port=i" => \$port,
+  "maxlines=i" => \$maxlines,
+  "logdir=s" => \$logdir
+);
+
+if ($port !~ /^\d+$/ or $port < 1 or $port > 65535){
+  die "Invalid port $port\n";
+}
+if ($maxlines !~ /^\d+/ or $maxlines < 10){
+  die "Invalid max line specified\n";
+}
+if (not -d $logdir){
+  die "$logdir doesn't exists or is not a directory\n";
+}
+
+# Remove trailing / of the logdir, it's not nice in the logs when you have double /
+$logdir =~ s/\/$//;
+
+# Create files so crowdsec can open them before any lines are written
+foreach my $dir (qw(nginx httpd zimbra pveproxy)){
+  if (not -d $logdir . '/' . $dir){
+    make_path($logdir . '/' . $dir)
+  }
+}
+foreach my $file (qw(syslog.log nginx/access.log nginx/error.log httpd/access.log httpd/error.log zimbra/mailbox.log)){
+  open(FILE, '>', $logdir . '/' . $file);
+  print FILE '';
+  close FILE;
+}
+
+# List of syslog_identifier we're not intersted in
+my @ignored_syslog_id = qw(
+  c-icap
+  charon
+  unbound
+  sudo
+  zed
+  zimbramon
+  systemd
+  systemd-logind
+  CROND
+  ttrss_1
+  turnserver
+  syncoid
+  influxd
+);
+# List of log files we're not interested in
+my @ignored_log_files = qw(
+  /var/log/audit/audit.log
+  /var/log/squid/cache.log
+  /var/log/squid/access.log
+  /var/log/ufdbGuard/ufdbguardd.log
+  /opt/zimbra/log/gc.log
+  /var/log/samba/json/auth.log
+  /var/log/samba/json/dsdb.log
+  /var/log/samba/json/dsdb_password.log
+  /var/log/samba/json/dsdb_transaction.log
+);
+
+print "Start listening on UDP port $port\n";
+$sock = IO::Socket::INET->new(
+          LocalPort => $port,
+          Proto => 'udp'
+        ) or die("Socket: $@");
+
+my $buf;
+my $cnt = {};
+my $loghandles = {};
+
+while (1) {
+  $sock->recv($buf, $maxlen);
+  my ($port, $ipaddr) = sockaddr_in($sock->peername);
+  my $fields = {};
+
+  # We're not really interested in CEF headers. So let's extract
+  # the various fields
+  $buf =~ m/(?:(?:CEF:\d+\|)(?:[^=\\]+\|)+)(.*)/;
+  my $ext = $1;
+
+  # Taken from https://github.com/DavidJBianco/pycef
+  while ($ext =~ m/([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)/g) {
+    $fields->{$1} = $2;
+    # Unescape value string
+    $fields->{$1} =~ s/\\=/=/g;
+  }
+
+  # Skip lines we're not interested in early.
+  # So crowdsec will eat less CPU parsing useless stuff
+  if (
+       defined $fields->{syslog_identifier} and grep { $_ eq $fields->{syslog_identifier} } @ignored_syslog_id or
+       defined $fields->{log_file_path} and grep { $_ eq $fields->{log_file_path} } @ignored_log_files
+     ) {
+    next;
+  }
+
+  # We need a timestamp, a source and a msg at least
+  if (not defined $fields->{timestamp} or not defined $fields->{source} or not defined $fields->{msg}){
+    next;
+  }
+
+  my $msg;
+  # Default log will be syslog
+  my $logfile = $logdir . '/syslog.log';
+
+  # But for some services, we need special handling. Eg for web access logs
+  if (defined $fields->{event_dataset}){
+    if ($fields->{event_dataset} =~ m/^nginx\.(access|ingress_controller)/){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^nginx\.error/){
+      $logfile = $logdir . '/nginx/error.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^apache\.access/){
+      $logfile = $logdir . '/httpd/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^apache\.error/){
+      $logfile = $logdir . '/httpd/access.log';
+      $msg = $fields->{msg};
+    }
+  } elsif (defined $fields->{log_file_path}){
+    if ($fields->{log_file_path} eq '/var/log/pveproxy/access.log'){
+      $logfile = $logdir . '/pveproxy/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{log_file_path} eq '/opt/zimbra/log/nginx.access.log'){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{log_file_path} eq '/opt/zimbra/log/mailbox.log'){
+      $logfile = $logdir . '/zimbra/mailbox.log';
+      $msg = $fields->{msg};
+    }
+  } elsif (defined $fields->{application_name}){
+    if ($fields->{application_name} eq 'nginx'){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    }
+  }
+
+  # OK, no special handling (else $msg would be defined), so let's
+  # provide a syslog format
+  if (not defined $msg){
+    $msg .= $fields->{timestamp} . ' ' . $fields->{source} . ' ';
+    my $id = $fields->{syslog_identifier} || $fields->{program} || $fields->{application_name} || $fields->{process_name} || 'unknown';
+    # For older PfSense, which sent invalid syslog messages, we might extract
+    # the syslog identifier from the begining of the message
+    if ($id eq 'unknown' and $fields->{msg} =~ m/^(\w+(\[\d+\])?):\s(.*)/){
+      $id = $1;
+      $fields->{msg} = $3;
+    }
+    $msg .= $id;
+    # Try to append the pid of the process
+    if ($id ne 'kernel' and $id ne 'filterlog' and $id !~ m/\[\d+\]$/){
+      $msg .= '[';
+      $msg .= $fields->{process_pid} || $fields->{process_id} || $fields->{pid} || '0';
+      $msg .= ']';
+    }
+    $msg .= ': ' . $fields->{msg};
+  }
+
+  defined $loghandles->{$logfile} or open($loghandles->{$logfile}, ">>", $logfile);
+  # Truncate the file so it's not growing too large
+  # Crowdsec will read it in nearly real time anyway
+  if ($cnt->{$logfile}++ > $maxlines){
+    print "Truncating $logfile\n";
+    truncate $loghandles->{$logfile}, 0;
+    $cnt->{$logfile} = 0;
+  }
+  print { $loghandles->{$logfile} } $msg . "\n";
+  $loghandles->{$logfile}->flush;
+};