diff --git a/roles/httpd_common/tasks/main.yml b/roles/httpd_common/tasks/main.yml index cd344c7..a197cc9 100644 --- a/roles/httpd_common/tasks/main.yml +++ b/roles/httpd_common/tasks/main.yml @@ -162,4 +162,7 @@ - include_tasks: filebeat.yml tags: always + +- include_tasks: vector.yml + tags: always ... diff --git a/roles/httpd_common/tasks/vector.yml b/roles/httpd_common/tasks/vector.yml new file mode 100644 index 0000000..36de8ef --- /dev/null +++ b/roles/httpd_common/tasks/vector.yml @@ -0,0 +1,5 @@ +--- + +- name: Deploy vector config + template: src=vector.yml.j2 dest=/etc/vector/conf.d/httpd.yml + tags: log,vector,web diff --git a/roles/httpd_common/templates/vector.yml.j2 b/roles/httpd_common/templates/vector.yml.j2 new file mode 100644 index 0000000..ad54953 --- /dev/null +++ b/roles/httpd_common/templates/vector.yml.j2 @@ -0,0 +1,20 @@ +--- + +sources: + in_logs_httpd: + type: file + include: ["/var/log/httpd/access_log", "/var/log/httpd/error_log"] + +transforms: + format_logs_httpd: + type: remap + inputs: ["in_logs_httpd"] + source: | + if (.file == "/var/log/httpd/access_log"){ + .http = parse_grok!(.message, "%{HOSTNAME:host} %{HTTPD_COMBINEDLOG}") + } + if (.file == "/var/log/httpd/error_log"){ + .http = parse_apache_log!(.message, format:"error") + } + .service = "httpd" + .group = "web" diff --git a/roles/mkdir/tasks/main.yml b/roles/mkdir/tasks/main.yml index 0583f44..5626946 100644 --- a/roles/mkdir/tasks/main.yml +++ b/roles/mkdir/tasks/main.yml @@ -1,12 +1,5 @@ --- -- name: Create filebeat snippet direcories - file: path=/etc/filebeat/ansible_{{ item }}.d state=directory - loop: - - inputs - - modules - tags: log,mkdir - - name: Create journald directory file: path=/var/log/journal state=directory notify: restart journald @@ -38,3 +31,7 @@ - name: Create bash_completion dir file: path=/etc/bash_completion.d state=directory tags: mkdir + +- name: Create vector config dir + file: path=/etc/vector/conf.d state=directory + tags: log,mkdir diff --git a/roles/nginx/tasks/main.yml b/roles/nginx/tasks/main.yml index 1eb2d99..68e583a 100644 --- a/roles/nginx/tasks/main.yml +++ b/roles/nginx/tasks/main.yml @@ -33,3 +33,6 @@ - include_tasks: filebeat.yml tags: always + +- include_tasks: vector.yml + tags: always diff --git a/roles/nginx/tasks/vector.yml b/roles/nginx/tasks/vector.yml new file mode 100644 index 0000000..1981a84 --- /dev/null +++ b/roles/nginx/tasks/vector.yml @@ -0,0 +1,5 @@ +--- + +- name: Install vector configuration + template: src=vector.yml.j2 dest=/etc/vector/conf.d/nginx.yml + tags: log,web,vector diff --git a/roles/nginx/templates/vector.yml.j2 b/roles/nginx/templates/vector.yml.j2 new file mode 100644 index 0000000..165db16 --- /dev/null +++ b/roles/nginx/templates/vector.yml.j2 @@ -0,0 +1,22 @@ +--- + +sources: + in_logs_nginx: + type: file + include: + - /var/log/nginx/access.log + - /var/log/nginx/error.log + +transforms: + format_logs_nginx: + type: remap + inputs: + - in_logs_nginx + source: | + if (.file == "/var/log/nginx/access.log"){ + .http = parse_grok!(.message, "%{HOSTNAME:host} %{HTTPD_COMBINEDLOG}") + } else if (.file == "/var/log/nginx/error.log"){ + .http = parse_nginx_log!(.message, format:"error") + } + .service = "nginx" + .group = "web" diff --git a/roles/pve/tasks/main.yml b/roles/pve/tasks/main.yml index b15b1e5..a1bb102 100644 --- a/roles/pve/tasks/main.yml +++ b/roles/pve/tasks/main.yml @@ -53,3 +53,6 @@ - include_tasks: filebeat.yml tags: always + +- include_tasks: vector.yml + tags: always diff --git a/roles/pve/tasks/vector.yml b/roles/pve/tasks/vector.yml new file mode 100644 index 0000000..bd6f2d5 --- /dev/null +++ b/roles/pve/tasks/vector.yml @@ -0,0 +1,5 @@ +--- + +- name: Deploy vector configuration + template: src=vector.yml dest=/etc/vector/conf.d/pve.yml + tags: log,pve,vector diff --git a/roles/pve/templates/vector.yml b/roles/pve/templates/vector.yml new file mode 100644 index 0000000..a89f5aa --- /dev/null +++ b/roles/pve/templates/vector.yml @@ -0,0 +1,16 @@ +--- + +sources: + in_logs_pve: + type: file + include: + - /var/log/pve-firewall.log + - /var/log/pveproxy/access.log + - /var/log/vzdump/* + +transforms: + format_logs_pve: + type: remap + inputs: ["in_logs_pve"] + source: | + # Nothing to do diff --git a/roles/repo_vector/defaults/main.yml b/roles/repo_vector/defaults/main.yml new file mode 100644 index 0000000..2692d70 --- /dev/null +++ b/roles/repo_vector/defaults/main.yml @@ -0,0 +1,3 @@ +--- + +vector_major_version: 0 diff --git a/roles/repo_vector/tasks/Debian.yml b/roles/repo_vector/tasks/Debian.yml new file mode 100644 index 0000000..1126afe --- /dev/null +++ b/roles/repo_vector/tasks/Debian.yml @@ -0,0 +1,20 @@ +--- + +- name: Add Vector repo key + apt_key: + url: "{{ item }}" + environment: + https_proxy: "{{ system_proxy | default('') }}" + loop: + - https://keys.datadoghq.com/DATADOG_APT_KEY_CURRENT.public + - https://keys.datadoghq.com/DATADOG_APT_KEY_C0962C7D.public + - https://keys.datadoghq.com/DATADOG_APT_KEY_F14F620E.public + tags: repo,log,vector + +- name: Add Vector repo + apt_repository: + repo: deb https://apt.vector.dev/ stable vector-0 + filename: vector + environment: + https_proxy: "{{ system_proxy | default('') }}" + tags: repo,log,vector diff --git a/roles/repo_vector/tasks/RedHat.yml b/roles/repo_vector/tasks/RedHat.yml new file mode 100644 index 0000000..34f239c --- /dev/null +++ b/roles/repo_vector/tasks/RedHat.yml @@ -0,0 +1,14 @@ +--- + +- name: Configure vector repo + yum_repository: + name: vector + description: Vector + baseurl: https://yum.vector.dev/stable/vector-0/$basearch/ + gpgcheck: True + gpgkey: + - https://keys.datadoghq.com/DATADOG_RPM_KEY_CURRENT.public + - https://keys.datadoghq.com/DATADOG_RPM_KEY_B01082D3.public + - https://keys.datadoghq.com/DATADOG_RPM_KEY_FD4BF915.public + priority: 1 + tags: repo,vector,log diff --git a/roles/repo_vector/tasks/main.yml b/roles/repo_vector/tasks/main.yml new file mode 100644 index 0000000..8ce24eb --- /dev/null +++ b/roles/repo_vector/tasks/main.yml @@ -0,0 +1,4 @@ +--- + +- include_tasks: "{{ ansible_os_family }}.yml" + tags: always diff --git a/roles/samba/tasks/main.yml b/roles/samba/tasks/main.yml index df30354..aa243cc 100644 --- a/roles/samba/tasks/main.yml +++ b/roles/samba/tasks/main.yml @@ -16,3 +16,5 @@ tags: always - include_tasks: filebeat.yml tags: always +- include_tasks: vector.yml + tags: always diff --git a/roles/samba/tasks/vector.yml b/roles/samba/tasks/vector.yml new file mode 100644 index 0000000..533745d --- /dev/null +++ b/roles/samba/tasks/vector.yml @@ -0,0 +1,5 @@ +--- + +- name: Deploy vector configuration + template: src=vector.yml dest=/etc/vector/conf.d/samba.yml + tags: log,samba,vector diff --git a/roles/samba/templates/vector.yml b/roles/samba/templates/vector.yml new file mode 100644 index 0000000..6467c59 --- /dev/null +++ b/roles/samba/templates/vector.yml @@ -0,0 +1,20 @@ +--- + +sources: + in_logs_samba: + type: file + include: + - /var/log/samba/json/auth.log + - /var/log/samba/json/dsdb.log + - /var/log/samba/json/dsdb_password.log + - /var/log/samba/json/dsdb_transaction.log + +transforms: + format_logs_samba: + type: remap + inputs: ["in_logs_samba"] + source: | + .message = string!(.message) + if (is_json(.message)) { + .samba = parse_json!(.message) + } diff --git a/roles/squid/files/acl/software_various.domains b/roles/squid/files/acl/software_various.domains index c45c8b2..cd82a15 100644 --- a/roles/squid/files/acl/software_various.domains +++ b/roles/squid/files/acl/software_various.domains @@ -424,5 +424,6 @@ s3.eu-central-1.amazonaws.com # Vector.dev setup.vector.dev yum.vector.dev +apt.vector.dev s3.amazonaws.com keys.datadoghq.com diff --git a/roles/squid/files/ufdb.te b/roles/squid/files/ufdb.te index 5abaf98..210fd81 100644 --- a/roles/squid/files/ufdb.te +++ b/roles/squid/files/ufdb.te @@ -1,4 +1,4 @@ -module ufdb 1.3; +module ufdb 1.4; require { type initrc_tmp_t; @@ -6,6 +6,7 @@ require { type tmp_t; type squid_t; type unconfined_service_t; + type var_run_t; class sock_file write; class unix_stream_socket connectto; } @@ -15,3 +16,4 @@ allow squid_t initrc_t:unix_stream_socket connectto; allow squid_t unconfined_service_t:unix_stream_socket connectto; allow squid_t initrc_tmp_t:sock_file write; allow squid_t tmp_t:sock_file write; +allow squid_t var_run_t:sock_file write; diff --git a/roles/squid/tasks/main.yml b/roles/squid/tasks/main.yml index 2899b6f..6dd3508 100644 --- a/roles/squid/tasks/main.yml +++ b/roles/squid/tasks/main.yml @@ -226,3 +226,6 @@ - include_tasks: filebeat.yml tags: always + +- include_tasks: vector.yml + tags: always diff --git a/roles/squid/tasks/vector.yml b/roles/squid/tasks/vector.yml new file mode 100644 index 0000000..c99063e --- /dev/null +++ b/roles/squid/tasks/vector.yml @@ -0,0 +1,5 @@ +--- + +- name: Deploy vector configuration + template: src=vector.yml dest=/etc/vector/conf.d/squid.yml + tags: log,proxy,vector diff --git a/roles/squid/templates/vector.yml b/roles/squid/templates/vector.yml new file mode 100644 index 0000000..9f5ccbd --- /dev/null +++ b/roles/squid/templates/vector.yml @@ -0,0 +1,22 @@ +--- + +sources: + in_logs_squid: + type: file + include: + - /var/log/squid/access.log + - /var/log/squid/cache.log + - /var/log/squid/ufdbgclient.log + - /var/log/ufdbguard/ufdbguardd.log + +transforms: + format_logs_squid: + type: remap + inputs: ["in_logs_squid"] + source: | + if (.file == "/var/log/squid/access.log"){ + .squid = parse_grok!( + .message, + "%{HTTPDATE:timestamp}\\s+%{NUMBER:response_time} %{IPORHOST:src_ip} %{NOTSPACE:squid_request_status}/%{NUMBER:http_status_code} %{NUMBER:transfer_size} %{NOTSPACE:http_method} (%{URIPROTO:url_scheme}://)?(?\\S+?)(:%{INT:url_port})?(/%{NOTSPACE:url_path})?\\s+%{NOTSPACE:client_identity}\\s+%{NOTSPACE:peer_code}/%{NOTSPACE:peerhost}\\s+%{NOTSPACE:content_type}" + ) + } diff --git a/roles/vector/defaults/main.yml b/roles/vector/defaults/main.yml new file mode 100644 index 0000000..9fc43b3 --- /dev/null +++ b/roles/vector/defaults/main.yml @@ -0,0 +1,52 @@ +--- + +vector_base_conf: + data_dir: /var/lib/vector + wildcard_matching: relaxed + sources: + in_logs_journald: + type: journald + + transforms: + route_journald: + type: route + inputs: ["in_logs_journald"] + route: + dummy: exists(.dummy) && .dummy == "true" + iptables: exists(._TRANSPORT) && ._TRANSPORT == "kernel" && starts_with(string!(.message), "Firewall:") + + parse_journald_dummy: + type: remap + inputs: ["route_journald.dummy"] + source: | + # Nothing to do + + parse_journald_iptables: + type: remap + inputs: ["route_journald.iptables"] + source: | + msg = string!(.message) + msg = replace(msg, "Firewall: ", "") + .iptables = parse_key_value!(msg, whitespace:"strict") + + format_logs_journald: + type: remap + inputs: ["route_journald._unmatched", "parse_journald_*"] + source: | + .group = "system" + + format_logs_out: + type: remap + inputs: ['format_logs_*', 'route_*._unmatched'] + source: | + # Nothing to do + + sinks: + sink_blackhole: + type: blackhole + inputs: + - format_logs_out + +vector_extra_conf: {} +vector_host_conf: {} +vector_conf: "{{ vector_base_conf | combine(vector_extra_conf, recursive=true) | combine(vector_host_conf, recursive=true)}}" diff --git a/roles/vector/handlers/main.yml b/roles/vector/handlers/main.yml new file mode 100644 index 0000000..bda9155 --- /dev/null +++ b/roles/vector/handlers/main.yml @@ -0,0 +1,7 @@ +--- + +- name: reload vector + service: name=vector state=reloaded + +- name: restart vector + service: name=vector state=restarted diff --git a/roles/vector/meta/main.yml b/roles/vector/meta/main.yml new file mode 100644 index 0000000..ba769ba --- /dev/null +++ b/roles/vector/meta/main.yml @@ -0,0 +1,5 @@ +--- + +dependencies: + - role: mkdir + - role: repo_vector diff --git a/roles/vector/tasks/conf.yml b/roles/vector/tasks/conf.yml new file mode 100644 index 0000000..0ae3d09 --- /dev/null +++ b/roles/vector/tasks/conf.yml @@ -0,0 +1,10 @@ +--- + +- name: Deploy config + template: src=vector.yml dest=/etc/vector/vector.yaml owner=root group=root mode=0600 + notify: reload vector + tags: log,vector + +- name: Remove dummy conf if present + file: path=/etc/vector/conf.d/_dummy.yml state=absent + tags: log,vector diff --git a/roles/vector/tasks/facts.yml b/roles/vector/tasks/facts.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/roles/vector/tasks/facts.yml @@ -0,0 +1 @@ +--- diff --git a/roles/vector/tasks/install.yml b/roles/vector/tasks/install.yml new file mode 100644 index 0000000..7849ed1 --- /dev/null +++ b/roles/vector/tasks/install.yml @@ -0,0 +1,16 @@ +--- + +- name: Install vector + package: name=vector + tags: log,vector + +- name: Deploy custom systemd unit + template: src=vector.service.j2 dest=/etc/systemd/system/vector.service + register: vector_unit + notify: restart vector + tags: log,vector + +- name: Reload systemd + systemd: daemon_reload=true + when: vector_unit.changed + tags: log,vector diff --git a/roles/vector/tasks/main.yml b/roles/vector/tasks/main.yml new file mode 100644 index 0000000..6a888ea --- /dev/null +++ b/roles/vector/tasks/main.yml @@ -0,0 +1,14 @@ +--- + +- include_tasks: facts.yml + tags: always + +- include_tasks: install.yml + tags: always + +- include_tasks: conf.yml + tags: always + +- include_tasks: services.yml + tags: always + diff --git a/roles/vector/tasks/services.yml b/roles/vector/tasks/services.yml new file mode 100644 index 0000000..49c0266 --- /dev/null +++ b/roles/vector/tasks/services.yml @@ -0,0 +1,5 @@ +--- + +- name: Start and enable vector + service: name=vector state=started enabled=true + tags: log,vector diff --git a/roles/vector/templates/vector.service.j2 b/roles/vector/templates/vector.service.j2 new file mode 100644 index 0000000..0ae586f --- /dev/null +++ b/roles/vector/templates/vector.service.j2 @@ -0,0 +1,21 @@ +[Unit] +Description=Vector +Documentation=https://vector.dev +After=network-online.target +Requires=network-online.target + +[Service] +ExecStartPre=/usr/bin/vector --config /etc/vector/vector.yaml --config-dir /etc/vector/conf.d validate +ExecStart=/usr/bin/vector --config /etc/vector/vector.yaml --config-dir /etc/vector/conf.d --watch-config +ExecReload=/usr/bin/vector --config /etc/vector/vector.yaml --config-dir /etc/vector/conf.d validate --no-environment +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +AmbientCapabilities=CAP_NET_BIND_SERVICE +EnvironmentFile=-/etc/default/vector +# Since systemd 229, should be in [Unit] but in order to support systemd <229, +# it is also supported to have it here. +StartLimitInterval=10 +StartLimitBurst=5 + +[Install] +WantedBy=multi-user.target diff --git a/roles/vector/templates/vector.yml b/roles/vector/templates/vector.yml new file mode 100644 index 0000000..331f133 --- /dev/null +++ b/roles/vector/templates/vector.yml @@ -0,0 +1,2 @@ +--- +{{ vector_conf | to_nice_yaml(indent=2) }} diff --git a/roles/zimbra/tasks/main.yml b/roles/zimbra/tasks/main.yml index 513a827..0fbfc9e 100644 --- a/roles/zimbra/tasks/main.yml +++ b/roles/zimbra/tasks/main.yml @@ -214,3 +214,6 @@ - include_tasks: filebeat.yml tags: always + +- include_tasks: vector.yml + tags: always diff --git a/roles/zimbra/tasks/vector.yml b/roles/zimbra/tasks/vector.yml new file mode 100644 index 0000000..938dd10 --- /dev/null +++ b/roles/zimbra/tasks/vector.yml @@ -0,0 +1,5 @@ +--- + +- name: Deploy vector configuration + template: src=vector.yml dest=/etc/vector/conf.d/zimbra.yml + tags: log,zcs,vector diff --git a/roles/zimbra/templates/vector.yml b/roles/zimbra/templates/vector.yml new file mode 100644 index 0000000..645e39d --- /dev/null +++ b/roles/zimbra/templates/vector.yml @@ -0,0 +1,14 @@ +--- + +sources: + in_logs_zimbra: + type: file + include: + - /opt/zimbra/log/*.log + +transforms: + format_logs_zimbra: + type: remap + inputs: ["in_logs_zimbra"] + source: | + # Nothing to do