Update to 2021-12-01 19:13

roles/g2cs/README.md

@@ -0,0 +1,17 @@
+# G2CS
+
+This is a small daemon writtent in perl to allow a bridge between Graylog and Crowdsec.
+This idea is that if you collect your logs to a graylog instance, you can forward them all in a single stream from Graylog to CrowdSec, instead of collecting them all again on every hosts.
+
+So, this small g2cs daemon is a very simple perl utility which will listen on a port for a syslog stream. It should run a the server which will host your single crowdsec instance.
+
+On graylog, you have to install the syslog-output plugin, and configure it to output the streams you want to this daemon. You should choose UDP, the port on which g2cs binds, and the CEF format.
+
+When g2cs receive this stream of logs, it'll just make simple transformations so that your logs can be consumed by crowdsec :
+
+  * nginx logs go to nginx/
+  * httpd logs go to httpd/
+  * squid logs go to squid/
+  * Everything else goes to syslog.log
+
+Now, you can configure your acquisitions on crowdsec to just read these locations

roles/g2cs/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+
+# Port on which g2cs will listen
+g2cs_port: 3514
+
+# Where log files will be created. Thos files won't grow too large as g2cs truncates them after 10000 lines
+# so better to use a tmpfs
+g2cs_log_dir: /run/g2cs/logs
+
+# List of IP/CIDR for which g2cs port will be reachable
+g2cs_src_ip: []

roles/g2cs/files/g2cs.pl

@@ -0,0 +1,183 @@
+#!/usr/bin/perl -w
+  
+use IO::Socket;
+use Getopt::Long;
+use File::Basename;
+use File::Path qw(make_path);
+use IO::Handle;
+
+my $maxlen = 16384;
+my $port = 514;
+my $maxlines = 10000;
+my $logdir = '/run/cs-gelf-server/';
+
+GetOptions(
+  "port=i" => \$port,
+  "maxlines=i" => \$maxlines,
+  "logdir=s" => \$logdir
+);
+
+if ($port !~ /^\d+$/ or $port < 1 or $port > 65535){
+  die "Invalid port $port\n";
+}
+if ($maxlines !~ /^\d+/ or $maxlines < 10){
+  die "Invalid max line specified\n";
+}
+if (not -d $logdir){
+  die "$logdir doesn't exists or is not a directory\n";
+}
+
+# Remove trailing / of the logdir, it's not nice in the logs when you have double /
+$logdir =~ s/\/$//;
+
+# Create files so crowdsec can open them before any lines are written
+foreach my $dir (qw(nginx httpd zimbra pveproxy)){
+  if (not -d $logdir . '/' . $dir){
+    make_path($logdir . '/' . $dir)
+  }
+}
+foreach my $file (qw(syslog.log nginx/access.log nginx/error.log httpd/access.log httpd/error.log zimbra/mailbox.log)){
+  open(FILE, '>', $logdir . '/' . $file);
+  print FILE '';
+  close FILE;
+}
+
+# List of syslog_identifier we're not intersted in
+my @ignored_syslog_id = qw(
+  c-icap
+  charon
+  unbound
+  sudo
+  zed
+  zimbramon
+  systemd
+  systemd-logind
+  CROND
+  ttrss_1
+  turnserver
+  syncoid
+  influxd
+);
+# List of log files we're not interested in
+my @ignored_log_files = qw(
+  /var/log/audit/audit.log
+  /var/log/squid/cache.log
+  /var/log/squid/access.log
+  /var/log/ufdbGuard/ufdbguardd.log
+  /opt/zimbra/log/gc.log
+  /var/log/samba/json/auth.log
+  /var/log/samba/json/dsdb.log
+  /var/log/samba/json/dsdb_password.log
+  /var/log/samba/json/dsdb_transaction.log
+);
+
+print "Start listening on UDP port $port\n";
+$sock = IO::Socket::INET->new(
+          LocalPort => $port,
+          Proto => 'udp'
+        ) or die("Socket: $@");
+
+my $buf;
+my $cnt = {};
+my $loghandles = {};
+
+while (1) {
+  $sock->recv($buf, $maxlen);
+  my ($port, $ipaddr) = sockaddr_in($sock->peername);
+  my $fields = {};
+
+  # We're not really interested in CEF headers. So let's extract
+  # the various fields
+  $buf =~ m/(?:(?:CEF:\d+\|)(?:[^=\\]+\|)+)(.*)/;
+  my $ext = $1;
+
+  # Taken from https://github.com/DavidJBianco/pycef
+  while ($ext =~ m/([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)/g) {
+    $fields->{$1} = $2;
+    # Unescape value string
+    $fields->{$1} =~ s/\\=/=/g;
+  }
+
+  # Skip lines we're not interested in early.
+  # So crowdsec will eat less CPU parsing useless stuff
+  if (
+       defined $fields->{syslog_identifier} and grep { $_ eq $fields->{syslog_identifier} } @ignored_syslog_id or
+       defined $fields->{log_file_path} and grep { $_ eq $fields->{log_file_path} } @ignored_log_files
+     ) {
+    next;
+  }
+
+  # We need a timestamp, a source and a msg at least
+  if (not defined $fields->{timestamp} or not defined $fields->{source} or not defined $fields->{msg}){
+    next;
+  }
+
+  my $msg;
+  # Default log will be syslog
+  my $logfile = $logdir . '/syslog.log';
+
+  # But for some services, we need special handling. Eg for web access logs
+  if (defined $fields->{event_dataset}){
+    if ($fields->{event_dataset} =~ m/^nginx\.(access|ingress_controller)/){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^nginx\.error/){
+      $logfile = $logdir . '/nginx/error.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^apache\.access/){
+      $logfile = $logdir . '/httpd/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{event_dataset} =~ m/^apache\.error/){
+      $logfile = $logdir . '/httpd/access.log';
+      $msg = $fields->{msg};
+    }
+  } elsif (defined $fields->{log_file_path}){
+    if ($fields->{log_file_path} eq '/var/log/pveproxy/access.log'){
+      $logfile = $logdir . '/pveproxy/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{log_file_path} eq '/opt/zimbra/log/nginx.access.log'){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    } elsif ($fields->{log_file_path} eq '/opt/zimbra/log/mailbox.log'){
+      $logfile = $logdir . '/zimbra/mailbox.log';
+      $msg = $fields->{msg};
+    }
+  } elsif (defined $fields->{application_name}){
+    if ($fields->{application_name} eq 'nginx'){
+      $logfile = $logdir . '/nginx/access.log';
+      $msg = $fields->{msg};
+    }
+  }
+
+  # OK, no special handling (else $msg would be defined), so let's
+  # provide a syslog format
+  if (not defined $msg){
+    $msg .= $fields->{timestamp} . ' ' . $fields->{source} . ' ';
+    my $id = $fields->{syslog_identifier} || $fields->{program} || $fields->{application_name} || $fields->{process_name} || 'unknown';
+    # For older PfSense, which sent invalid syslog messages, we might extract
+    # the syslog identifier from the begining of the message
+    if ($id eq 'unknown' and $fields->{msg} =~ m/^(\w+(\[\d+\])?):\s(.*)/){
+      $id = $1;
+      $fields->{msg} = $3;
+    }
+    $msg .= $id;
+    # Try to append the pid of the process
+    if ($id ne 'kernel' and $id ne 'filterlog' and $id !~ m/\[\d+\]$/){
+      $msg .= '[';
+      $msg .= $fields->{process_pid} || $fields->{process_id} || $fields->{pid} || '0';
+      $msg .= ']';
+    }
+    $msg .= ': ' . $fields->{msg};
+  }
+
+  defined $loghandles->{$logfile} or open($loghandles->{$logfile}, ">>", $logfile);
+  # Truncate the file so it's not growing too large
+  # Crowdsec will read it in nearly real time anyway
+  if ($cnt->{$logfile}++ > $maxlines){
+    print "Truncating $logfile\n";
+    truncate $loghandles->{$logfile}, 0;
+    $cnt->{$logfile} = 0;
+  }
+  print { $loghandles->{$logfile} } $msg . "\n";
+  $loghandles->{$logfile}->flush;
+};

roles/g2cs/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+
+- name: restart g2cs
+  service: name=g2cs state=restarted

roles/g2cs/tasks/install.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Install dependencies
+  yum:
+    name:
+      - perl-IO
+      - perl-Getopt-Long
+  tags: cs
+
+- name: Install main script
+  copy: src=g2cs.pl dest=/usr/local/bin/g2cs mode=755
+  notify: restart g2cs
+  tags: cs
+
+- name: Deploy systemd unit
+  template: src=g2cs.service.j2 dest=/etc/systemd/system/g2cs.service
+  notify: restart g2cs
+  register: g2cs_unit
+  tags: cs
+
+- name: Reload systemd
+  systemd: daemon_reload=True
+  when: g2cs_unit.changed
+  tags: cs
+
+- name: Deploy tmpfiles.d config
+  copy:
+    content: |
+      d /run/g2cs 0755 g2cs g2cs - -
+      d /run/g2cs/logs 0700 g2cs g2cs - -
+    dest: /etc/tmpfiles.d/g2cs.conf
+  register: g2cs_tmpfiles
+  tags: cs
+
+- name: Create tmpfiles dir
+  command: systemd-tmpfiles --create
+  when: g2cs_tmpfiles.changed
+  tags: cs

roles/g2cs/tasks/iptables.yml

@@ -0,0 +1,8 @@
+---
+
+- name:  Handle g2cs port in the firewall
+  iptables_raw:
+    name: g2cs_port
+    state: "{{ (g2cs_src_ip | length > 0) | ternary('present','absent') }}"
+    rules: "-A INPUT -p udp --dport {{ g2cs_port }} -s {{ g2cs_src_ip | join(',') }} -j ACCEPT"
+  tags: firewall,cs

roles/g2cs/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+
+- include: user.yml
+- include: install.yml
+- include: iptables.yml
+  when: iptables_manage | default(True)
+- include: service.yml

roles/g2cs/tasks/service.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Start and enable the service
+  service: name=g2cs state=started enabled=True
+  tags: cs

roles/g2cs/tasks/user.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Create g2cs user account
+  user: name=g2cs system=True shell=/sbin/nologin
+  tags: cs

roles/g2cs/templates/g2cs.service.j2

@@ -0,0 +1,26 @@
+[Unit]
+Description=Graylog to Crowdsec syslog daemon
+After=syslog.target
+Before=crowdsec.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/g2cs --port={{ g2cs_port }} --logdir={{ g2cs_log_dir }}
+User=g2cs
+Group=g2cs
+Restart=always
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=full
+ProtectHome=yes
+NoNewPrivileges=yes
+SyslogIdentifier=g2cs
+
+# Allow binding on privileged ports
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+
+
+[Install]
+WantedBy=multi-user.target
+