Update to 2021-12-01 19:13

roles/crowdsec/defaults/main.yml

@@ -0,0 +1,97 @@
+---
+
+# Version to install
+cs_version: 1.1.1
+# URL of the archive
+cs_archive_url: https://github.com/crowdsecurity/crowdsec/releases/download/v{{ cs_version }}/crowdsec-release.tgz
+# Expected sha1 of the archive
+cs_archive_sha1: e128534e1fc5529441512451753ecb79c2cdcb85
+
+# Crowdsec usually should run as root to be able to access all your logs
+# but in some situations, when all your logs are readable by a less privileged user, you can run
+# crowdsec as another user account, for better security
+cs_user: root
+
+# Directory where data will be stored
+cs_root_dir: /opt/crowdsec
+
+# Can be sqlite or mysql
+cs_db_engine: sqlite
+# This is for mysql backend
+cs_db_server: "{{ mysql_server | default('localhost') }}"
+cs_db_port: 3306
+cs_db_name: crowdsec
+cs_db_user: crowdsec
+# If not defined, a random one will be generated and store in /etc/crowdsec/meta/ansible_dbpass
+# cs_db_pass: S3cr3t.
+
+# You can disable the Local API, if using a remote one for example
+cs_lapi_enabled: True
+# Set to true if Local API is enabled, and you intend to use it through a trusted reverse proxy
+cs_use_forwarded_headers: False
+# Port on which the Local API will listen
+cs_lapi_port: 8080
+# List of IP/CIDR allowed to access cs_lapi_port
+cs_lapi_src_ip: []
+
+# Address of the Local API server
+# The default config will make it standalone
+cs_lapi_url: http://localhost:{{ cs_lapi_port }}/
+cs_lapi_user: "{{ inventory_hostname }}"
+# On installation, ansible will register this host on the Local API
+# And will then validate the registration on the following server.
+# So set it to your own Local API server so ansible will delegate the task
+cs_lapi_server: "{{ inventory_hostname }}"
+
+# Use the central API, to share your banned IP, and received list of IP to ban
+# Requires cs_lapi_enabled to be true too
+cs_capi_enabled: False
+# You can either register manuelly and the the user/pass with those variable
+# Else, ansible will register and configure the credentials
+# cs_capi_user: 123456789
+# cs_capi_pass: azertyuiop
+
+# Port on which the prometheus metric endpoint will bind to
+cs_prometheus_port: 6060
+# List of IP/CIDR allowed to access the prometheus port
+cs_prometheus_src_ip: []
+
+# Default duration of a ban
+cs_trusted_countries:
+  - FR
+# Duration of bans for attacks from trusted countries
+cs_ban_trusted_duration: 15m
+# Default duration of a ban
+cs_ban_duration: 2h
+
+# List of parsers to install from the hub
+cs_parsers:
+  - crowdsecurity/syslog-logs
+  - crowdsecurity/geoip-enrich
+  - crowdsecurity/dateparse-enrich
+  - crowdsecurity/whitelists
+  - crowdsecurity/sshd-logs
+  - crowdsecurity/iptables-logs
+# List of scenarios to install from the hub
+cs_scenarios:
+  - crowdsecurity/ban-defcon-drop_range
+  - crowdsecurity/ssh-bf
+# List of postoverflows to install from the hub
+cs_postoverflows:
+  - crowdsecurity/cdn-whitelist
+  - crowdsecurity/rdns
+  - crowdsecurity/seo-bots-whitelist
+
+# If not set, crowdsec will look for yaml files in /etc/crowdsec/acquis/
+# The default will only read syslog using journalctl
+# If defined, only acquisition set by ansible will be used
+# cs_aquis:
+#   - journalctl_filter:
+#       - '_SYSTEMD_UNIT=sshd.service'
+#     labels:
+#       type: syslog
+#
+#   - filename:
+#       - /var/log/nginx/access.log
+#     labels:
+#       type: nginx

roles/crowdsec/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+
+- name: restart crowdsec
+  service: name=crowdsec state=restarted
+
+- name: reload crowdsec
+  service: name=crowdsec state=reloaded

roles/crowdsec/meta/main.yml

@@ -0,0 +1,6 @@
+---
+
+dependencies:
+  - role: mkdir
+  - role: mysql_server
+    when: cs_db_server in ['localhost','127.0.0.1']

roles/crowdsec/tasks/cleanup.yml

@@ -0,0 +1,8 @@
+---
+
+- name: Remove temp and obsolete files
+  file: path={{ item }} state=absent
+  loop:
+    - /tmp/crowdsec-release.tgz
+    - /tmp/crowdsec-v{{ cs_version }}
+  tags: cs

roles/crowdsec/tasks/conf.yml

@@ -0,0 +1,126 @@
+---
+
+- name: Deploy configuration
+  template: src={{ item }}.j2 dest=/etc/crowdsec/{{ item }}
+  loop:
+    - config.yaml
+    - acquis.yaml
+    - simulation.yaml
+    - profiles.yaml
+    - parsers/s02-enrich/trusted_ip.yaml
+    - dev.yaml
+  notify: reload crowdsec
+  tags: cs
+
+# Create the database
+- import_tasks: ../includes/webapps_create_mysql_db.yml
+  vars:
+    - db_name: "{{ cs_db_name }}"
+    - db_user: "{{ cs_db_user }}"
+    - db_server: "{{ cs_db_server }}"
+    - db_pass: "{{ cs_db_pass }}"
+  when:
+    - cs_db_engine == 'mysql'
+    - cs_lapi_enabled
+  tags: cs
+
+- when: cs_lapi_pass is not defined
+  block:
+    - name: Declare on the local API
+      command: cscli machines add {{ cs_lapi_user }} --auto --force --file /dev/stdout --output raw
+      register: cs_lapi_credentials
+      delegate_to: "{{ cs_lapi_server }}"
+    - set_fact: cs_lapi_credentials_yaml={{ cs_lapi_credentials.stdout | from_yaml }}
+    - copy: content={{ cs_lapi_credentials_yaml.password }} dest=/etc/crowdsec/meta/lapi_pass mode=600
+    - set_fact: cs_lapi_pass={{ cs_lapi_credentials_yaml.password }}
+  tags: cs
+
+- when:
+    - cs_lapi_enabled
+    - cs_capi_enabled
+    - cs_capi_user is not defined or cs_capi_pass is not defined
+  block:
+    - name: Register on the central API
+      command: cscli capi register -o raw -f /dev/stdout
+      register: cs_capi_credentials
+    - set_fact: cs_capi_credentials_yaml={{ cs_capi_credentials.stdout | from_yaml }}
+    - copy: content={{ cs_capi_credentials_yaml.login }} dest=/etc/crowdsec/meta/capi_user mode=600
+    - copy: content={{ cs_capi_credentials_yaml.password }} dest=/etc/crowdsec/meta/capi_pass mode=600
+    - set_fact: cs_capi_user={{ cs_capi_credentials_yaml.login }}
+    - set_fact: cs_capi_pass={{ cs_capi_credentials_yaml.password }}
+  tags: cs
+
+- name: Deploy credentials config
+  template: src={{ item }}_api_credentials.yaml.j2 dest=/etc/crowdsec/{{ item }}_api_credentials.yaml mode=600
+  loop:
+    - online
+    - local
+  notify: restart crowdsec
+  tags: cs
+
+- name: List installed parsers
+  shell: cscli parsers list -o json
+  register: cs_installed_parsers
+  changed_when: False
+  tags: cs
+
+- name: Install parsers
+  command: cscli parsers install {{ item }}
+  when: item not in cs_installed_parsers.stdout | from_json | map(attribute='name') | list
+  loop: "{{ cs_parsers }}"
+  notify: reload crowdsec
+  tags: cs
+
+- name: Upgrade parsers
+  command: cscli parsers upgrade {{ item }}
+  loop: "{{ cs_parsers }}"
+  when: cs_install_mode == 'upgrade'
+  notify: reload crowdsec
+  tags: cs
+
+- name: List installed scenarios
+  command: cscli scenarios list -o json
+  register: cs_installed_scenarios
+  changed_when: False
+  tags: cs
+
+- name: Install scenarios
+  command: cscli scenarios install {{ item }}
+  when: item not in cs_installed_scenarios.stdout | from_json | map(attribute='name') | list
+  loop: "{{ cs_scenarios }}"
+  notify: reload crowdsec
+  tags: cs
+
+- name: Upgrade scenarios
+  command: cscli scenarios upgrade {{ item }}
+  loop: "{{ cs_scenarios }}"
+  when: cs_install_mode == 'upgrade'
+  notify: reload crowdsec
+  tags: cs
+
+- name: List installed postoverflows
+  command: cscli postoverflows list -o json
+  register: cs_installed_postoverflows
+  changed_when: False
+  tags: cs
+
+- name: Install postoverflows
+  command: cscli postoverflows install {{ item }}
+  when: item not in cs_installed_postoverflows.stdout | from_json | map(attribute='name') | list
+  loop: "{{ cs_postoverflows }}"
+  notify: reload crowdsec
+  tags: cs
+
+- name: Upgrade postoverflows
+  command: cscli postoverflows upgrade {{ item }}
+  loop: "{{ cs_postoverflows }}"
+  when: cs_install_mode == 'upgrade'
+  notify: reload crowdsec
+  tags: cs
+
+- name: Set permissions on conf and data directories
+  file: path={{ item }} owner={{ cs_user }} group={{ cs_user }} recurse=True
+  loop:
+    - /etc/crowdsec
+    - "{{ cs_root_dir }}/data"
+  tags: cs

roles/crowdsec/tasks/directories.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Create required directories
+  file: path={{ item.dir }} state=directory owner={{ item.owner | default(omit) }} group={{ item.group | default(omit) }} mode={{ item.mode | default(omit) }}
+  loop:
+    - dir: /etc/crowdsec
+      mode: 755
+    - dir: "{{ cs_root_dir }}"
+    - dir: "{{ cs_root_dir }}/backup"
+      mode: 700
+    - dir: "{{ cs_root_dir }}/data"
+    - dir: /etc/crowdsec/parsers/s00-raw
+    - dir: /etc/crowdsec/parsers/s01-parse
+    - dir: /etc/crowdsec/parsers/s02-enrich
+    - dir: /etc/crowdsec/scenarios
+    - dir: /etc/crowdsec/postoverflows/s00-enrich
+    - dir: /etc/crowdsec/postoverflows/s01-whitelist
+    - dir: /etc/crowdsec/acquis
+    - dir: /etc/crowdsec/meta
+      mode: 700
+  tags: cs

roles/crowdsec/tasks/facts.yml

@@ -0,0 +1,84 @@
+---
+
+- name: Set initial facts
+  block:
+    - set_fact: cs_install_mode='none'
+    - set_fact: cs_current_version=''
+  tags: cs
+
+- name: Check if crowdsec is installed
+  stat: path=/usr/local/bin/crowdsec
+  register: cs_bin
+  tags: cs
+
+- name: Check installed version
+  shell: |
+    crowdsec -version 2>&1 | perl -ne 'm/version: v(\d+(\.\d+)*)/ && print $1'
+  register: cs_current_version
+  changed_when: False
+  when: cs_bin.stat.exists
+  tags: cs
+
+- name: Set install mode
+  set_fact: cs_install_mode='install'
+  when: not cs_bin.stat.exists
+  tags: cs
+
+- name: Set upgrade mode
+  set_fact: cs_install_mode='upgrade'
+  when:
+    - cs_bin.stat.exists
+    - cs_current_version.stdout != cs_version
+  tags: cs
+
+# Create a random db password if needed
+- block:
+    - import_tasks: ../includes/get_rand_pass.yml
+      vars:
+        - pass_file: "/etc/crowdsec/meta/ansible_db_pass"
+        - complex: False
+    - set_fact: cs_db_pass={{ rand_pass }}
+  when:
+    - cs_db_pass is not defined
+    - cs_lapi_enabled
+  tags: cs
+
+# Check if local API credentials are available in the meta dir
+- name: Check local API credential files
+  stat: path=/etc/crowdsec/meta/lapi_pass
+  register: cs_lapi_pass_file
+  tags: cs
+
+- name: Read the local API pass
+  block:
+    - slurp: src=/etc/crowdsec/meta/lapi_pass
+      register: cs_lapi_pass_meta
+    - set_fact: cs_lapi_pass={{ cs_lapi_pass_meta.content | b64decode | trim }}
+  when: cs_lapi_pass is not defined and cs_lapi_pass_file.stat.exists
+  tags: cs
+
+# Check if central API credentials are available in the meta dir
+- name: Check central API credential files
+  block: 
+    - stat: path=/etc/crowdsec/meta/capi_user
+      register: cs_capi_user_file
+    - stat: path=/etc/crowdsec/meta/capi_pass
+      register: cs_capi_pass_file
+  tags: cs
+
+- name: Read the central API user
+  block:
+    - slurp: src=/etc/crowdsec/meta/capi_user
+      register: cs_capi_user_meta
+    - set_fact: cs_capi_user={{ cs_capi_user_meta.content | b64decode | trim }}
+  when: cs_capi_user is not defined and cs_capi_user_file.stat.exists
+  tags: cs
+
+- name: Read the central API pass
+  block:
+    - slurp: src=/etc/crowdsec/meta/capi_pass
+      register: cs_capi_pass_meta
+    - set_fact: cs_capi_pass={{ cs_capi_pass_meta.content | b64decode | trim }}
+  when: cs_capi_pass is not defined and cs_capi_pass_file.stat.exists
+  tags: cs
+

roles/crowdsec/tasks/install.yml

@@ -0,0 +1,74 @@
+---
+
+- name: Install needed tools
+  package:
+    name:
+      - tar
+      - zstd
+  tags: cs
+
+- when: cs_install_mode != 'none'
+  block:
+    - name: Download crowdsec
+      get_url:
+        url: "{{ cs_archive_url }}"
+        dest: /tmp/
+        checksum: sha1:{{ cs_archive_sha1 }}
+
+    - name: Extract crowdsec
+      unarchive:
+        src: /tmp/crowdsec-release.tgz
+        dest: /tmp/
+        remote_src: True
+
+    - name: Install or upgrade crowdsec
+      command: ./wizard.sh --bin{{ cs_install_mode }} --force
+      args:
+        chdir: /tmp/crowdsec-v{{ cs_version }}/
+      notify: restart crowdsec
+
+  tags: cs
+
+- name: Update crowdsec hub
+  command: cscli hub update
+  changed_when: False
+  tags: cs
+
+- name: Create the systemd unit snippet dir
+  file: path=/etc/systemd/system/crowdsec.service.d state=directory
+  tags: cs
+
+- name: Make the service restart on failure
+  copy:
+    content: |
+      [Service]
+      Restart=on-failure
+      StartLimitInterval=0
+      RestartSec=30
+    dest: /etc/systemd/system/crowdsec.service.d/restart.conf
+  register: crodwsec_unit_restart
+  notify: restart crowdsec
+  tags: cs
+
+- name: Set user account which runs the service
+  copy:
+    content: |
+      [Service]
+      User={{ cs_user }}
+      Group={{ cs_user }}
+    dest: /etc/systemd/system/crowdsec.service.d/user.conf
+  register: crodwsec_unit_user
+  notify: restart crowdsec
+  tags: cs
+
+- name: Reload systemd
+  systemd: daemon_reload=True
+  when: crodwsec_unit_restart.changed or crodwsec_unit_user.changed
+  tags: cs
+
+- name: Install pre and post backup hooks
+  template: src={{ item }}-backup.j2 dest=/etc/backup/{{ item }}.d/crowdsec mode=700
+  loop:
+    - pre
+    - post
+  tags: cs

roles/crowdsec/tasks/iptables.yml

@@ -0,0 +1,15 @@
+---
+
+- name:  Handle crowdsec port in the firewall
+  iptables_raw:
+    name: "{{ item.name }}"
+    state: "{{ (item.src_ip | length > 0) | ternary('present','absent') }}"
+    rules: "-A INPUT -m state --state NEW -p tcp --dport {{ item.port }} -s {{ item.src_ip | join(',') }} -j ACCEPT"
+  loop:
+    - name: cs_lapi_port
+      port: "{{ cs_lapi_port }}"
+      src_ip: "{{ cs_lapi_src_ip }}"
+    - name: cs_prometheus_port
+      port: "{{ cs_prometheus_port }}"
+      src_ip: "{{ cs_prometheus_src_ip }}"
+  tags: firewall,cs

roles/crowdsec/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+
+- include: user.yml
+- include: directories.yml
+- include: facts.yml
+- include: install.yml
+- include: conf.yml
+- include: iptables.yml
+  when: iptables_manage | default(True)
+- include: services.yml
+- include: cleanup.yml

roles/crowdsec/tasks/services.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Start and enable the service
+  service: name=crowdsec state=started enabled=True
+  tags: cs

roles/crowdsec/tasks/user.yml

@@ -0,0 +1,6 @@
+---
+
+- name: Create crowdsec user
+  user: name={{ cs_user }} system=True shell=/sbin/nologin
+  when: cs_user != 'root'
+  tags: cs

roles/crowdsec/templates/acquis.yaml.j2

@@ -0,0 +1,6 @@
+{% if cs_acquis is defined and cs_acquis | length > 0%}
+{% for acquis in cs_acquis %}
+---
+{{ acquis | to_nice_yaml }}
+{% endfor %}
+{% endif %}

roles/crowdsec/templates/acquis/system.yaml.j2

@@ -0,0 +1,5 @@
+---
+journalctl_filter:
+  - ""
+labels:
+  type: syslog

roles/crowdsec/templates/config.yaml.j2

@@ -0,0 +1,65 @@
+common:
+  daemonize: true
+  pid_dir: /var/run/
+  log_media: stdout
+  log_level: info
+  working_dir: .
+
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: {{ cs_root_dir }}/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+
+crowdsec_service:
+{% if cs_acquis is defined %}
+  acquisition_path: /etc/crowdsec/acquis.yaml
+{% else %}
+  acquisition_dir: /etc/crowdsec/acquis/
+{% endif %}
+  parser_routines: 1
+
+cscli:
+  output: human
+  hub_branch: master
+
+db_config:
+  log_level: info
+{% if cs_db_engine == 'mysql' %}
+  type: mysql
+  user: {{ cs_db_user }}
+  password: {{ cs_db_pass | quote }}
+  db_name: {{ cs_db_name }}
+  host: {{ cs_db_server }}
+  port: {{ cs_db_port }}
+{% else %}
+  type: sqlite
+  db_path: {{ cs_root_dir }}/data/crowdsec.db
+{% endif %}
+  flush:
+    max_items: 100000
+    max_age: 730d
+
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+
+{% if cs_lapi_enabled %}
+  server:
+    log_level: info
+    listen_uri: 0.0.0.0:{{ cs_lapi_port }}
+    profiles_path: /etc/crowdsec/profiles.yaml
+{% if cs_capi_enabled %}
+    online_client: 
+      credentials_path: /etc/crowdsec/online_api_credentials.yaml
+{% endif %}
+{% endif %}
+
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: {{ (cs_prometheus_src_ip | length > 0) | ternary(ansible_all_ipv4_addresses[0],'127.0.0.1') }}
+  listen_port: {{ cs_prometheus_port }}
+

roles/crowdsec/templates/dev.yaml.j2

@@ -0,0 +1,39 @@
+common:
+  daemonize: false
+  log_media: stdout
+  log_level: info
+  working_dir: .
+
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: {{ cs_root_dir }}/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+
+crowdsec_service:
+  acquisition_path: /etc/crowdsec/acquis.yaml
+  parser_routines: 1
+
+cscli:
+  output: human
+  hub_branch: master
+
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: {{ cs_root_dir }}/data/dev.db
+  flush:
+    max_items: 1000
+    max_age: 30d
+
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+  server:
+    profiles_path: /etc/crowdsec/profiles.yaml
+
+prometheus:
+  enabled: false
+

roles/crowdsec/templates/local_api_credentials.yaml.j2

@@ -0,0 +1,3 @@
+url: {{ cs_lapi_enabled | ternary('http://127.0.0.1:' ~ cs_lapi_port,(cs_lapi_url is search('/$')) | ternary(cs_lapi_url, cs_lapi_url ~ '/')) }}
+login: {{ cs_lapi_user }}
+password: {{ cs_lapi_pass }}

roles/crowdsec/templates/online_api_credentials.yaml.j2

@@ -0,0 +1,7 @@
+url: https://api.crowdsec.net/
+{% if cs_capi_user is defined %}
+login: {{ cs_capi_user }}
+{% endif %}
+{% if cs_capi_pass is defined %}
+password: {{ cs_capi_pass }}
+{% endif %}

roles/crowdsec/templates/parsers/s02-enrich/trusted_ip.yaml.j2

@@ -0,0 +1,16 @@
+name: fws/trusted_ip
+description: "Whitelist events from trusted ip"
+whitelist:
+  reason: "trusted ip"
+  ip:
+{% for ip in trusted_ip | default([]) %}
+{% if ip is not search('/\d+$') %}
+    - "{{ ip }}"
+{% endif %}
+{% endfor %}
+  cidr:
+{% for ip in trusted_ip | default([]) %}
+{% if ip is search('/\d+$') %}
+    - "{{ ip }}"
+{% endif %}
+{% endfor %}

roles/crowdsec/templates/post-backup.j2

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+rm -f {{ cs_root_dir }}/backup/*

roles/crowdsec/templates/pre-backup.j2

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -eo pipefail
+
+{% if cs_lapi_enabled %}
+{% if cs_db_engine == 'mysql' %}
+/usr/bin/mysqldump \
+{% if cs_db_server not in ['localhost','127.0.0.1'] %}
+  --user={{ cs_db_user | quote }} \
+  --password={{ cs_db_pass | quote }} \
+  --host={{ cs_db_server | quote }} \
+  --port={{ cs_db_port | quote }} \
+{% endif %}
+  --quick --single-transaction \
+  --add-drop-table {{ cs_db_name | quote }} | zstd -c > {{ cs_root_dir }}/backup/{{ cs_db_name }}.sql.zst
+{% else %}
+sqlite3 {{ cs_root_dir }}/data/crowdsec.db .dump | zstd -c > {{ cs_root_dir }}/backup/crowdsec.sql.zst
+{% endif %}
+{% endif %}

roles/crowdsec/templates/profiles.yaml.j2

@@ -0,0 +1,33 @@
+{% if cs_trusted_countries | length > 0 %}
+name: trusted_countries_ip_remediation
+filters:
+  - Alert.Remediation == true && Alert.GetScope() == "Ip" && Alert.Source.Cn in ["{{ cs_trusted_countries | join('","') }}"]
+decisions:
+ - type: ban
+   duration: {{ cs_ban_trusted_duration }}
+on_success: break
+---
+name: trusted_countries_range_remediation
+filters:
+  - Alert.Remediation == true && Alert.GetScope() == "Range" && Alert.Source.Cn in ["{{ cs_trusted_countries | join('","') }}"]
+decisions:
+ - type: ban
+   duration: {{ cs_ban_trusted_duration }}
+on_success: break
+---
+{% endif %}
+name: default_ip_remediation
+filters:
+ - Alert.Remediation == true && Alert.GetScope() == "Ip"
+decisions:
+ - type: ban
+   duration: {{ cs_ban_duration }}
+on_success: break
+---
+name: default_range_remediation
+filters:
+ - Alert.Remediation == true && Alert.GetScope() == "Range"
+decisions:
+ - type: ban
+   duration: {{ cs_ban_duration }}
+on_success: break

roles/crowdsec/templates/simulation.yaml.j2

@@ -0,0 +1 @@
+simulation: off