Update to 2025-07-04 09:00

roles/httpd_common/tasks/main.yml

@@ -162,4 +162,7 @@
 
 - include_tasks: filebeat.yml
   tags: always
+
+- include_tasks: vector.yml
+  tags: always
 ...

roles/httpd_common/tasks/vector.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Deploy vector config
+  template: src=vector.yml.j2 dest=/etc/vector/conf.d/httpd.yml
+  tags: log,vector,web

roles/httpd_common/templates/vector.yml.j2

@@ -0,0 +1,20 @@
+---
+
+sources:
+  in_logs_httpd:
+    type: file
+    include: ["/var/log/httpd/access_log", "/var/log/httpd/error_log"]
+
+transforms:
+  format_logs_httpd:
+    type: remap
+    inputs: ["in_logs_httpd"]
+    source: |
+      if (.file == "/var/log/httpd/access_log"){
+        .http = parse_grok!(.message, "%{HOSTNAME:host} %{HTTPD_COMBINEDLOG}")
+      }
+      if (.file == "/var/log/httpd/error_log"){
+        .http = parse_apache_log!(.message, format:"error")
+      }
+      .service = "httpd"
+      .group   = "web"

roles/mkdir/tasks/main.yml

@@ -1,12 +1,5 @@
 ---
 
-- name: Create filebeat snippet direcories
-  file: path=/etc/filebeat/ansible_{{ item }}.d state=directory
-  loop:
-    - inputs
-    - modules
-  tags: log,mkdir
-
 - name: Create journald directory
   file: path=/var/log/journal state=directory
   notify: restart journald
@@ -38,3 +31,7 @@
 - name: Create bash_completion dir
   file: path=/etc/bash_completion.d state=directory
   tags: mkdir
+
+- name: Create vector config dir
+  file: path=/etc/vector/conf.d state=directory
+  tags: log,mkdir

roles/nginx/tasks/main.yml

@@ -33,3 +33,6 @@
 
 - include_tasks: filebeat.yml
   tags: always
+
+- include_tasks: vector.yml
+  tags: always

roles/nginx/tasks/vector.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Install vector configuration
+  template: src=vector.yml.j2 dest=/etc/vector/conf.d/nginx.yml
+  tags: log,web,vector

roles/nginx/templates/vector.yml.j2

@@ -0,0 +1,22 @@
+---
+
+sources:
+  in_logs_nginx:
+    type: file
+    include:
+      - /var/log/nginx/access.log
+      - /var/log/nginx/error.log
+
+transforms:
+  format_logs_nginx:
+    type: remap
+    inputs:
+      - in_logs_nginx
+    source: |
+      if (.file == "/var/log/nginx/access.log"){
+        .http = parse_grok!(.message, "%{HOSTNAME:host} %{HTTPD_COMBINEDLOG}")
+      } else if (.file == "/var/log/nginx/error.log"){
+        .http = parse_nginx_log!(.message, format:"error")
+      }
+      .service = "nginx"
+      .group   = "web"

roles/pve/tasks/main.yml

@@ -53,3 +53,6 @@
 
 - include_tasks: filebeat.yml
   tags: always
+
+- include_tasks: vector.yml
+  tags: always

roles/pve/tasks/vector.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Deploy vector configuration
+  template: src=vector.yml dest=/etc/vector/conf.d/pve.yml
+  tags: log,pve,vector

roles/pve/templates/vector.yml

@@ -0,0 +1,16 @@
+---
+
+sources:
+  in_logs_pve:
+    type: file
+    include:
+      - /var/log/pve-firewall.log
+      - /var/log/pveproxy/access.log
+      - /var/log/vzdump/*
+
+transforms:
+  format_logs_pve:
+    type: remap
+    inputs: ["in_logs_pve"]
+    source: |
+      # Nothing to do

roles/repo_vector/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+
+vector_major_version: 0

roles/repo_vector/tasks/Debian.yml

@@ -0,0 +1,20 @@
+---
+
+- name: Add Vector repo key
+  apt_key:
+    url: "{{ item }}"
+  environment:
+    https_proxy: "{{ system_proxy | default('') }}"
+  loop:
+    - https://keys.datadoghq.com/DATADOG_APT_KEY_CURRENT.public
+    - https://keys.datadoghq.com/DATADOG_APT_KEY_C0962C7D.public
+    - https://keys.datadoghq.com/DATADOG_APT_KEY_F14F620E.public
+  tags: repo,log,vector
+
+- name: Add Vector repo
+  apt_repository:
+    repo: deb https://apt.vector.dev/ stable vector-0
+    filename: vector
+  environment:
+    https_proxy: "{{ system_proxy | default('') }}"
+  tags: repo,log,vector

roles/repo_vector/tasks/RedHat.yml

@@ -0,0 +1,14 @@
+---
+
+- name: Configure vector repo
+  yum_repository:
+    name: vector
+    description: Vector
+    baseurl: https://yum.vector.dev/stable/vector-0/$basearch/
+    gpgcheck: True
+    gpgkey:
+      - https://keys.datadoghq.com/DATADOG_RPM_KEY_CURRENT.public
+      - https://keys.datadoghq.com/DATADOG_RPM_KEY_B01082D3.public
+      - https://keys.datadoghq.com/DATADOG_RPM_KEY_FD4BF915.public
+    priority: 1
+  tags: repo,vector,log

roles/repo_vector/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+
+- include_tasks: "{{ ansible_os_family }}.yml"
+  tags: always

roles/samba/tasks/main.yml

@@ -16,3 +16,5 @@
   tags: always
 - include_tasks: filebeat.yml
   tags: always
+- include_tasks: vector.yml
+  tags: always

roles/samba/tasks/vector.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Deploy vector configuration
+  template: src=vector.yml dest=/etc/vector/conf.d/samba.yml
+  tags: log,samba,vector

roles/samba/templates/vector.yml

@@ -0,0 +1,20 @@
+---
+  
+sources:
+  in_logs_samba:
+    type: file
+    include:
+      - /var/log/samba/json/auth.log
+      - /var/log/samba/json/dsdb.log
+      - /var/log/samba/json/dsdb_password.log
+      - /var/log/samba/json/dsdb_transaction.log
+
+transforms:
+  format_logs_samba:
+    type: remap
+    inputs: ["in_logs_samba"]
+    source: |
+      .message = string!(.message)
+      if (is_json(.message)) {
+        .samba = parse_json!(.message)
+      }

roles/squid/files/acl/software_various.domains

@@ -424,5 +424,6 @@ s3.eu-central-1.amazonaws.com
 # Vector.dev
 setup.vector.dev
 yum.vector.dev
+apt.vector.dev
 s3.amazonaws.com
 keys.datadoghq.com

roles/squid/files/ufdb.te

@@ -1,4 +1,4 @@
-module ufdb 1.3;
+module ufdb 1.4;
 
 require {
         type initrc_tmp_t;
@@ -6,6 +6,7 @@ require {
         type tmp_t;
         type squid_t;
         type unconfined_service_t;
+        type var_run_t;
         class sock_file write;
         class unix_stream_socket connectto;
 }
@@ -15,3 +16,4 @@ allow squid_t initrc_t:unix_stream_socket connectto;
 allow squid_t unconfined_service_t:unix_stream_socket connectto;
 allow squid_t initrc_tmp_t:sock_file write;
 allow squid_t tmp_t:sock_file write;
+allow squid_t var_run_t:sock_file write;

roles/squid/tasks/main.yml

@@ -226,3 +226,6 @@
 
 - include_tasks: filebeat.yml
   tags: always
+
+- include_tasks: vector.yml
+  tags: always

roles/squid/tasks/vector.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Deploy vector configuration
+  template: src=vector.yml dest=/etc/vector/conf.d/squid.yml
+  tags: log,proxy,vector

roles/squid/templates/vector.yml

@@ -0,0 +1,22 @@
+---
+
+sources:
+  in_logs_squid:
+    type: file
+    include:
+      - /var/log/squid/access.log
+      - /var/log/squid/cache.log
+      - /var/log/squid/ufdbgclient.log
+      - /var/log/ufdbguard/ufdbguardd.log
+
+transforms:
+  format_logs_squid:
+    type: remap
+    inputs: ["in_logs_squid"]
+    source: |
+      if (.file == "/var/log/squid/access.log"){
+        .squid = parse_grok!(
+                   .message,
+                   "%{HTTPDATE:timestamp}\\s+%{NUMBER:response_time} %{IPORHOST:src_ip} %{NOTSPACE:squid_request_status}/%{NUMBER:http_status_code} %{NUMBER:transfer_size} %{NOTSPACE:http_method} (%{URIPROTO:url_scheme}://)?(?<url_host>\\S+?)(:%{INT:url_port})?(/%{NOTSPACE:url_path})?\\s+%{NOTSPACE:client_identity}\\s+%{NOTSPACE:peer_code}/%{NOTSPACE:peerhost}\\s+%{NOTSPACE:content_type}"
+                 )
+      }

roles/vector/defaults/main.yml

@@ -0,0 +1,52 @@
+---
+
+vector_base_conf:
+  data_dir: /var/lib/vector
+  wildcard_matching: relaxed
+  sources:
+    in_logs_journald:
+      type: journald
+
+  transforms:
+    route_journald:
+      type: route
+      inputs: ["in_logs_journald"]
+      route:
+        dummy: exists(.dummy) && .dummy == "true"
+        iptables: exists(._TRANSPORT) && ._TRANSPORT == "kernel" && starts_with(string!(.message), "Firewall:")
+
+    parse_journald_dummy:
+      type: remap
+      inputs: ["route_journald.dummy"]
+      source: |
+        # Nothing to do
+
+    parse_journald_iptables:
+      type: remap
+      inputs: ["route_journald.iptables"]
+      source: |
+        msg = string!(.message)
+        msg = replace(msg, "Firewall: ", "")
+        .iptables = parse_key_value!(msg, whitespace:"strict")
+
+    format_logs_journald:
+      type: remap
+      inputs: ["route_journald._unmatched", "parse_journald_*"]
+      source: |
+        .group = "system"
+
+    format_logs_out:
+      type: remap
+      inputs: ['format_logs_*', 'route_*._unmatched']
+      source: |
+        # Nothing to do
+
+  sinks:
+    sink_blackhole:
+      type: blackhole
+      inputs:
+        - format_logs_out
+
+vector_extra_conf: {}
+vector_host_conf: {}
+vector_conf: "{{ vector_base_conf | combine(vector_extra_conf, recursive=true) | combine(vector_host_conf, recursive=true)}}"

roles/vector/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+
+- name: reload vector
+  service: name=vector state=reloaded
+
+- name: restart vector
+  service: name=vector state=restarted

roles/vector/meta/main.yml

@@ -0,0 +1,5 @@
+---
+
+dependencies:
+  - role: mkdir
+  - role: repo_vector

roles/vector/tasks/conf.yml

@@ -0,0 +1,10 @@
+---
+
+- name: Deploy config
+  template: src=vector.yml dest=/etc/vector/vector.yaml owner=root group=root mode=0600
+  notify: reload vector
+  tags: log,vector
+
+- name: Remove dummy conf if present
+  file: path=/etc/vector/conf.d/_dummy.yml state=absent
+  tags: log,vector

roles/vector/tasks/facts.yml

@@ -0,0 +1 @@
+---

roles/vector/tasks/install.yml

@@ -0,0 +1,16 @@
+---
+
+- name: Install vector
+  package: name=vector
+  tags: log,vector
+
+- name: Deploy custom systemd unit
+  template: src=vector.service.j2 dest=/etc/systemd/system/vector.service
+  register: vector_unit
+  notify: restart vector
+  tags: log,vector
+
+- name: Reload systemd
+  systemd: daemon_reload=true
+  when: vector_unit.changed
+  tags: log,vector

roles/vector/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+
+- include_tasks: facts.yml
+  tags: always
+
+- include_tasks: install.yml
+  tags: always
+
+- include_tasks: conf.yml
+  tags: always
+
+- include_tasks: services.yml
+  tags: always
+

roles/vector/tasks/services.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Start and enable vector
+  service: name=vector state=started enabled=true
+  tags: log,vector

roles/vector/templates/vector.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=Vector
+Documentation=https://vector.dev
+After=network-online.target
+Requires=network-online.target
+
+[Service]
+ExecStartPre=/usr/bin/vector --config /etc/vector/vector.yaml --config-dir /etc/vector/conf.d validate
+ExecStart=/usr/bin/vector --config /etc/vector/vector.yaml --config-dir /etc/vector/conf.d --watch-config
+ExecReload=/usr/bin/vector --config /etc/vector/vector.yaml --config-dir /etc/vector/conf.d validate --no-environment
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+EnvironmentFile=-/etc/default/vector
+# Since systemd 229, should be in [Unit] but in order to support systemd <229,
+# it is also supported to have it here.
+StartLimitInterval=10
+StartLimitBurst=5
+
+[Install]
+WantedBy=multi-user.target

roles/vector/templates/vector.yml

@@ -0,0 +1,2 @@
+---
+{{ vector_conf | to_nice_yaml(indent=2) }}

roles/zimbra/tasks/main.yml

@@ -214,3 +214,6 @@
 
 - include_tasks: filebeat.yml
   tags: always
+
+- include_tasks: vector.yml
+  tags: always

roles/zimbra/tasks/vector.yml

@@ -0,0 +1,5 @@
+---
+
+- name: Deploy vector configuration
+  template: src=vector.yml dest=/etc/vector/conf.d/zimbra.yml
+  tags: log,zcs,vector

roles/zimbra/templates/vector.yml

@@ -0,0 +1,14 @@
+---
+
+sources:
+  in_logs_zimbra:
+    type: file
+    include:
+      - /opt/zimbra/log/*.log
+
+transforms:
+  format_logs_zimbra:
+    type: remap
+    inputs: ["in_logs_zimbra"]
+    source: |
+      # Nothing to do