Update to 2023-07-08 16:00

This commit is contained in:
Daniel Berteaud
2023-07-08 16:00:11 +02:00
parent f16dd6e98d
commit d87c1bdc89
60 changed files with 708 additions and 439 deletions

View File

@@ -1,12 +1,5 @@
---
# Version of Nomad to install
nomad_version: 1.5.6
# URL of the archive
nomad_archive_url: https://releases.hashicorp.com/nomad/{{ nomad_version }}/nomad_{{ nomad_version }}_linux_amd64.zip
# Expected sha256 of the archive
nomad_archive_sha256: 65a7d5a4c6ade01a44292901b05a7c985b4459e0a079b5eb55f5d61474ee98e5
# List of plugins to install
nomad_plugins:
podman:
@@ -27,10 +20,6 @@ nomad_user: "{{ nomad_conf.client.enabled | ternary('root', 'nomad') }}"
# Used for example to grant access to cli cert with ACL
nomad_admin_groups: "{{ system_admin_groups | default([]) }}"
# If ACL are enabled, you need to set a management token for ansible
# to be able to manage Nomad (eg snapshot before upgrades)
# nomad_mgm_token: XXXXXXXXX
# List of nomad servers (not clients !)
nomad_servers: []
@@ -169,7 +158,7 @@ nomad_base_conf:
# Default scheduler config. Only used during cluster bootstrap
# If you want to change it after, you have to use the API
default_scheduler_config:
# can be binpack or spread. SPread makes more sens when running on premise
# can be binpack or spread. Spread makes more sense when running on premise
scheduler_algorithm: spread
memory_oversubscription_enabled: True
preemption_config:
@@ -247,24 +236,3 @@ nomad_extra_services: {}
nomad_host_services: {}
nomad_services: "{{ nomad_base_services | combine(nomad_extra_services, recursive=True) | combine(nomad_host_services, recursive=True) }}"
# When using vault to setup TLS for Nomad
nomad_base_vault_secrets:
vault_address: "{{ nomad_conf.vault.address | default('https://active.vault.service.consul:8200') }}"
# Token to use to issue certificates
# token: XXXXXXXXX
pki:
enabled: False
# The path of the PKI secret where cert will be issued
path: /pki/nomad
role: nomad-{{ nomad_conf.server.enabled | ternary('server', 'client') }}
# ttl: 6h # if not set, will use the default ttl of the role
consul_pki:
enabled: False
path: /pki/consul
root_path: /pki/root # This is used to fetch the top level root CA, as envoy can't validate the chain unless it has it complete up to the auto-signed root
role: nomad-client # Only nomad clients will use this
# ttl: 6h # if not set, will use the default ttl of the role
nomad_extra_vault_secrets: {}
nomad_host_vault_secrets: {}
nomad_vault_secrets: "{{ nomad_base_vault_secrets | combine(nomad_extra_vault_secrets, recursive=True) | combine(nomad_host_vault_secrets, recursive=True) }}"

View File

@@ -9,9 +9,5 @@
service: name=nomad state=reloaded
when: nomad_service_started is not defined or not nomad_service_started.changed
- name: restart consul-template-nomad
service: name=consul-template-nomad state=restarted
- name: systemd-tmpfiles
command: systemd-tmpfiles --create
- name: restart nomad-vault-agent
service: name=nomad-vault-agent state=restarted

View File

@@ -1,10 +1,10 @@
---
dependencies:
- role: nomad_bin
- role: repo_docker
when: nomad_conf.client.enabled and nomad_conf.client.task_drivers['containerd-driver'].enabled # with containerd, we need docker repo to install containerd.io
- role: docker
when: nomad_conf.client.enabled and nomad_conf.client.task_drivers.docker.enabled
- role: cni_plugins
when: nomad_conf.client.enabled
- role: consul_template # consul-template can be used for obtaining certs from vault

View File

@@ -1,14 +0,0 @@
---
- name: Compress previous version
command: tar cf {{ nomad_root_dir }}/archives/{{ nomad_current_version }}.tar.zst --use-compress-program=zstd ./
args:
chdir: "{{ nomad_root_dir }}/archives/{{ nomad_current_version }}"
environment:
ZSTD_CLEVEL: 10
tags: nomad
- name: Remove archive dir
file: path={{ nomad_root_dir }}/archives/{{ nomad_current_version }} state=absent
tags: nomad

View File

@@ -1,29 +0,0 @@
---
- name: Create the archive dir
file: path={{ nomad_root_dir }}/archives/{{ nomad_current_version }} state=directory
tags: nomad
- name: Snapshot nomad data
command: "{{ nomad_root_dir }}/bin/nomad operator snapshot save {{ nomad_root_dir }}/archives/{{ nomad_current_version }}/nomad.snap"
args:
creates: "{{ nomad_root_dir }}/archives/{{ nomad_current_version }}/nomad.snap"
when:
- nomad_conf.server.enabled
- nomad_sys_services.ansible_facts.services['nomad.service'] is defined
- nomad_sys_services.ansible_facts.services['nomad.service'].state == 'started'
environment:
NOMAD_TOKEN: "{{ nomad_mgm_token | default(omit) }}"
tags: nomad
- name: Backup previous version
synchronize:
src: "{{ nomad_root_dir }}/{{ item }}"
dest: "{{ nomad_root_dir }}/archives/{{ nomad_current_version }}/"
compress: False
delegate_to: "{{ inventory_hostname }}"
loop:
- bin
- plugins
tags: nomad

View File

@@ -3,8 +3,7 @@
- name: Remove tmp and obsolete files
file: path={{ item }} state=absent
loop:
- "{{ nomad_root_dir }}/tmp/nomad_{{ nomad_version }}_linux_amd64.zip"
- "{{ nomad_root_dir }}/tmp/nomad"
- "{{ nomad_root_dir }}/archives"
tags: nomad
- name: Remove temporary plugin files

View File

@@ -5,8 +5,7 @@
vars:
cert_path: "{{ nomad_conf.tls.cert_file }}"
cert_key_path: "{{ nomad_conf.tls.key_file }}"
cert_key_group: "{{ nomad_user }}"
cert_key_mode: 640
cert_key_mode: omit
tags: nomad
- name: Check if CA exists
@@ -26,8 +25,7 @@
vars:
cert_path: "{{ nomad_conf.consul.cert_file }}"
cert_key_path: "{{ nomad_conf.consul.key_file }}"
cert_key_group: "{{ nomad_user }}"
cert_key_mode: 640
cert_key_mode: omit
tags: nomad
- name: Check if CA exists
@@ -113,27 +111,67 @@
loop: "{{ nomad_backup_configs.stdout_lines }}"
tags: nomad
- name: Deploy consul-template config
template: src=consul-template.hcl.j2 dest={{ nomad_root_dir }}/consul-template/consul-template.hcl mode=600 owner=root group=root
notify: restart consul-template-nomad
when: nomad_vault_secrets.pki.enabled or nomad_vault_secrets.tokens.enabled
tags: nomad
- name: Deploy consul-template agent bundle template
template: src=agent_bundle.pem.tpl.j2 dest={{ nomad_root_dir }}/consul-template/agent_bundle.pem.tpl owner=root group=root
notify: restart consul-template-nomad
when: nomad_vault_secrets.pki.enabled
tags: nomad
- name: Deploy consul-template cli bundle template
template: src=cli_bundle.pem.tpl.j2 dest={{ nomad_root_dir }}/consul-template/cli_bundle.pem.tpl owner=root group=root
notify: restart consul-template-nomad
when: nomad_vault_secrets.pki.enabled and nomad_conf.server.enabled
tags: nomad
#- name: Deploy vault-agent config
# template: src=vault/agent.hcl.j2 dest={{ nomad_root_dir }}/vault/agent.hcl mode=640 owner=root group={{ nomad_user }}
# notify: restart nomad-vault-agent
# when: nomad_vault_agent.nomad_pki.enabled or nomad_vault_agent.consul_pki.enabled
# tags: nomad
#
#- name: Deploy agent cert bundle template
# template: src=vault/agent_bundle.pem.tpl.j2 dest={{ nomad_root_dir }}/vault/templates/agent_bundle.pem.tpl owner=root group=root
# notify: restart nomad-vault-agent
# when: nomad_vault_agent.nomad_pki.enabled or nomad_vault_agent.consul_pki.enabled
# tags: nomad
#
#- name: Deploy cli cert bundle template
# template: src=vault/cli_bundle.pem.tpl.j2 dest={{ nomad_root_dir }}/vault/templates/cli_bundle.pem.tpl owner=root group=root
# notify: restart nomad-vault-agent
# when: (nomad_vault_agent.nomad_pki.enabled or nomad_vault_agent.consul_pki.enabled) and nomad_conf.server.enabled
# tags: nomad
#
#- name: Deploy consul token template
# template: src=vault/consul_token.tpl.j2 dest={{ nomad_root_dir }}/vault/templates/consul_token.tpl owner=root group=root
# notify: restart nomad-vault-agent
# when: (nomad_vault_agent.consul_token.enabled) and nomad_conf.server.enabled
# tags: nomad
#
#- fail: msg="Only one of token or approle should be configured"
# when:
# - nomad_vault_agent.auth.approle is defined
# - nomad_vault_agent.auth.token is defined
# tags: nomad
#
#- name: Setup AppRole auth
# block:
# - copy: content={{ nomad_vault_agent.auth.approle.role_id }} dest={{ nomad_root_dir }}/vault/role_id owner=root group={{ nomad_user }} mode=640
# - copy: content={{ nomad_vault_agent.auth.approle.secret_id }} dest={{ nomad_root_dir }}/vault/secret_id owner=root group={{ nomad_user }} mode=640
# - file: path={{ nomad_root_dir }}/vault/token state=absent
# when:
# - nomad_vault_agent.auth is defined
# - nomad_vault_agent.auth.approle is defined
# - nomad_vault_agent.auth.approle.role_id is defined
# - nomad_vault_agent.auth.approle.secret_id is defined
# tags: nomad
#
#- name: Setup Token auth
# block:
# - copy: content={{ nomad_vault_agent.auth.token }} dest={{ nomad_root_dir }}/vault/token owner=root group={{ nomad_user }} mode=640
# - file: path={{ nomad_root_dir }}/vault/role_id state=absent
# - file: path={{ nomad_root_dir }}/vault/secret_id state=absent
# when:
# - nomad_vault_agent.auth is defined
# - nomad_vault_agent.auth.token is defined
# tags: nomad
- name: Set ACL on the TLS dir
shell: |
setfacl -R -k -b {{ nomad_root_dir }}/tls
setfacl -R -b -k {{ nomad_root_dir }}/tls
chown -R :{{ nomad_user }} {{ nomad_root_dir }}/tls
chmod 750 {{ nomad_root_dir }}/tls
chmod 640 {{ nomad_root_dir }}/tls/*
setfacl -m u:{{ nomad_user }}:rx {{ nomad_root_dir }}/tls
setfacl -m d:u:{{ nomad_user }}:r {{ nomad_root_dir }}/tls
setfacl -m u:{{ nomad_user }}:r {{ nomad_root_dir }}/tls/*
{% if nomad_admin_groups | length > 0 %}
setfacl -m {% for group in nomad_admin_groups %}g:{{ group }}:rx{{ ',' if not loop.last }}{% endfor %} {{ nomad_root_dir }}/tls
setfacl -m {% for group in nomad_admin_groups %}d:g:{{ group }}:r{{ ',' if not loop.last }}{% endfor %} {{ nomad_root_dir }}/tls
@@ -147,12 +185,6 @@
template: src=profile.sh.j2 dest=/etc/profile.d/nomad.sh
tags: nomad
- name: Deploy consul-template consul cert templates
template: src=consul_bundle.pem.tpl.j2 dest={{ nomad_root_dir }}/consul-template/consul_bundle.pem.tpl owner=root group=root
notify: restart consul-template-nomad
when: nomad_vault_secrets.consul_pki.enabled and nomad_conf.consul.ssl
tags: nomad
- name: Ensure the bridge module is loaded
modprobe: name=bridge state=present
when: nomad_conf.client.enabled and 'docker' in nomad_enabled_task_drivers

View File

@@ -7,10 +7,6 @@
owner: root
group: root
mode: 755
- dir: archives
owner: root
group: root
mode: 700
- dir: backup
owner: root
group: root
@@ -37,8 +33,11 @@
owner: root
mode: 755
- dir: tls
owner: root
mode: 755
- dir: consul-template
mode: 755
tags: nomad
- name: Create host_volume directories
file: path={{ item.path }} state=directory owner={{ item.owner | default(omit) }} group={{ item.group | default(omit) }} mode={{ item.mode | default(omit) }}
loop: "{{ nomad_conf.client.host_volumes | default([]) }}"
when: item.create | default(False)
tags: nomad

View File

@@ -1,28 +1,6 @@
---
# Load distribution specific variables
- include_vars: "{{ item }}"
with_first_found:
- "{{ role_path }}/vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
- "{{ role_path }}/vars/{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml"
- "{{ role_path }}/vars/{{ ansible_distribution }}.yml"
- "{{ role_path }}/vars/{{ ansible_os_family }}.yml"
tags: nomad
- set_fact: nomad_install_mode='none'
tags: nomad
- name: Detect if nomad is installed
stat: path=/usr/local/bin/nomad
register: nomad_bin
tags: nomad
- when: not nomad_bin.stat.exists
set_fact: nomad_install_mode='install'
tags: nomad
- when: nomad_bin.stat.exists
block:
- block:
- name: Detect installed version
shell: /usr/local/bin/nomad version | perl -ne '/Nomad v(\d+(\.\d+)*)\s.*/ && print "$1\n"'
changed_when: False
@@ -30,15 +8,11 @@
- set_fact: nomad_current_version={{ nomad_current_version.stdout }}
tags: nomad
- when: nomad_bin.stat.exists and nomad_current_version != nomad_version
set_fact: nomad_install_mode='upgrade'
tags: nomad
- name: Build a list of enabled task drivers
set_fact: nomad_enabled_task_drivers={{ nomad_conf.client.task_drivers | dict2items | selectattr('value.enabled', 'equalto', True) | map(attribute='key') }}
tags: nomad
- name: Check the state of the services
service_facts:
register: nomad_sys_services
- name: Check if the consul-template service still exists
stat: path=/etc/systemd/system/consul-template-nomad
register: nomad_ct_service
tags: nomad

View File

@@ -1,10 +1,5 @@
---
- name: Install needed tools
package:
name: "{{ nomad_packages }}"
tags: nomad
- name: Install task driver packages
package: name={{ nomad_task_driver_packages[item] }}
loop: "{{ nomad_enabled_task_drivers }}"
@@ -12,43 +7,20 @@
notify: restart nomad
tags: nomad
- when: nomad_install_mode == 'upgrade'
name: Clear plugin dir on upgrades
file: path={{ nomad_root_dir }}/plugins state={{ item }} owner={{ nomad_user }} group={{ nomad_user }}
loop:
- absent
- directory
- name: Deploy systemd service unit
template: src=nomad.service.j2 dest=/etc/systemd/system/nomad.service
register: nomad_unit
notify: restart nomad
tags: nomad
- when: nomad_install_mode != 'none'
- when: nomad_conf.client.enabled and nomad_unit.changed
block:
- name: Download nomad
get_url:
url: "{{ nomad_archive_url }}"
dest: "{{ nomad_root_dir }}/tmp"
checksum: sha256:{{ nomad_archive_sha256 }}
- name: Extract the archive
unarchive:
src: "{{ nomad_root_dir }}/tmp/nomad_{{ nomad_version }}_linux_amd64.zip"
dest: "{{ nomad_root_dir }}/tmp"
remote_src: True
- name: Install nomad binary
copy:
src: "{{ nomad_root_dir }}/tmp/nomad"
dest: "{{ nomad_root_dir }}/bin/nomad"
remote_src: True
mode: 755
notify: restart nomad
- name: Link in /usr/local/bin
file: src={{ nomad_root_dir }}/bin/nomad dest=/usr/local/bin/nomad state=link force=True
tags: nomad
- when: nomad_install_mode != 'none' and nomad_conf.client.enabled
block:
- name: Clear plugin dir on upgrades
file: path={{ nomad_root_dir }}/plugins state={{ item }} owner={{ nomad_user }} group={{ nomad_user }}
loop:
- absent
- directory
- name: Download plugins
get_url:
@@ -100,24 +72,11 @@
- name: Install bash completion support
copy:
content: |
complete -C {{ nomad_root_dir }}/bin/nomad nomad
complete -C /usr/local/bin/nomad nomad
dest: /etc/bash_completion.d/nomad
mode: 0644
tags: nomad
- name: Deploy systemd service unit
template: src=nomad.service.j2 dest=/etc/systemd/system/nomad.service
register: nomad_unit
notify: restart nomad
tags: nomad
- name: Install consul-template unit
template: src=consul-template-nomad.service.j2 dest=/etc/systemd/system/consul-template-nomad.service
register: nomad_consul_tpl_unit
notify: restart consul-template-nomad
when: nomad_vault_secrets.pki.enabled or nomad_vault_secrets.consul_pki.enabled
tags: nomad
- name: Install backup hooks
template: src={{ item }}-backup.j2 dest=/etc/backup/{{ item }}.d/nomad mode=755
loop:
@@ -125,14 +84,21 @@
- post
tags: nomad
# A tmpfs will be used to store tokens and other sensitive information
- name: Create tmpfile fragment
copy:
content: |
d /run/nomad 775 root {{ nomad_user }}
dest: /etc/tmpfiles.d/nomad.conf
notify: systemd-tmpfiles
register: nomad_tmpfiles
tags: nomad
- name: Create tmpfiles
command: systemd-tmpfiles --create
when: nomad_tmpfiles.changed
tags: nomad
# This script cleanups leftovers in iptables rules on boot
- name: Install iptables cleanup script
copy: src=iptables_cleanup.pl dest={{ nomad_root_dir }}/bin/iptables_cleanup.pl mode=755
tags: nomad
@@ -144,7 +110,7 @@
- name: Reload systemd
systemd: daemon_reload=True
when: nomad_unit.changed or nomad_ipt_cleanup_unit.changed or (nomad_consul_tpl_unit is defined and nomad_consul_tpl_unit.changed)
when: nomad_unit.changed or nomad_ipt_cleanup_unit.changed
tags: nomad
# This is needed for containers to be able to reach their own services through the host published port
@@ -152,8 +118,3 @@
copy: src=50-nomad-promisc dest=/etc/NetworkManager/dispatcher.d/50-nomad-promisc mode=755
tags: nomad
- name: Create host_volume directories
file: path={{ item.path }} state=directory owner={{ item.owner | default(omit) }} group={{ item.group | default(omit) }} mode={{ item.mode | default(omit) }}
loop: "{{ nomad_conf.client.host_volumes | default([]) }}"
when: item.create | default(False)
tags: nomad

View File

@@ -10,10 +10,6 @@
- include_tasks: facts.yml
tags: always
- include_tasks: archive_pre.yml
when: nomad_install_mode | default('none') == 'upgrade'
tags: always
- include_tasks: install.yml
tags: always
@@ -27,10 +23,6 @@
- include_tasks: services.yml
tags: always
- include_tasks: archive_post.yml
when: nomad_install_mode | default('none') == 'upgrade'
tags: always
- include_tasks: cleanup.yml
tags: always

View File

@@ -8,8 +8,9 @@
- name: Handle consul-template-nomad service
service:
name: consul-template-nomad
state: "{{ (nomad_vault_secrets.pki.enabled or nomad_vault_secrets.consul_pki.enabled or nomad_vault_secrets.tokens.enabled) | ternary('started', 'stopped') }}"
enabled: "{{ (nomad_vault_secrets.pki.enabled or nomad_vault_secrets.consul_pki.enabled or nomad_vault_secrets.tokens.enabled) | ternary(True, False) }}"
state: stopped
enabled: False
when: nomad_ct_service.stat.exists
tags: nomad
- name: Handle iptables-nomad-cleanup service
@@ -17,3 +18,11 @@
name: iptables-nomad-cleanup
enabled: "{{ (nomad_conf.client.enabled and iptables_manage | default(True)) | ternary(True, False) }}"
tags: nomad
#- name: Handle nomad-vault-agent service
# service:
# name: nomad-vault-agent
# state: "{{ (nomad_vault_agent.nomad_pki.enabled or nomad_vault_agent.consul_pki.enabled or nomad_vault_agent.consul_token.enabled) | ternary('started', 'stopped') }}"
# enabled: "{{ (nomad_vault_agent.nomad_pki.enabled or nomad_vault_agent.consul_pki.enabled or nomad_vault_agent.consul_token.enabled) | ternary(True, False) }}"
# tags: nomad

View File

@@ -5,10 +5,14 @@ Wants=network-online.target
After=network-online.target
After=vault.service
After=consul.service
After=nomad-vault-agent.service
ConditionFileNotEmpty={{ nomad_root_dir }}/etc/nomad.hcl
# Nomad version {{ nomad_current_version }}
[Service]
EnvironmentFile=-{{ nomad_root_dir }}/etc/nomad.env
EnvironmentFile=-/run/nomad/vault.env
EnvironmentFile=-/run/nomad/consul.env
{% if nomad_conf.consul.ssl and nomad_conf.client.enabled %}
Environment=CONSUL_HTTP_SSL=true
Environment=CONSUL_HTTP_ADDR=localhost:{{ (consul_services is defined) | ternary(consul_services.https.port, '8501') }}
@@ -18,7 +22,7 @@ Environment=CONSUL_CLIENT_KEY={{ nomad_conf.consul.key_file }}
{% endif %}
User={{ nomad_user }}
Group={{ nomad_user }}
ExecStart={{ nomad_root_dir }}/bin/nomad agent -config={{ nomad_root_dir }}/etc/
ExecStart=/usr/local/bin/nomad agent -config {{ nomad_root_dir }}/etc
ExecReload=/bin/kill --signal HUP $MAINPID
Restart=on-failure
LimitNOFILE=65536

View File

@@ -5,8 +5,6 @@ set -eo pipefail
{% if nomad_conf.tls.http %}
NOMAD_ADDR=https://localhost:{{ nomad_services.http.port }} \
NOMAD_CACERT={{ nomad_conf.tls.ca_file }} \
{% endif %}
{% if nomad_vault_secrets.pki.enabled %}
NOMAD_CLIENT_CERT={{ nomad_root_dir }}/tls/cli.crt \
NOMAD_CLIENT_KEY={{ nomad_root_dir }}/tls/cli.key \
{% endif %}

View File

@@ -1,11 +1,9 @@
{% if nomad_conf.tls.http and nomad_conf.server.enabled %}
export NOMAD_ADDR=https://localhost:{{ nomad_services.http.port }}
export NOMAD_CACERT={{ nomad_conf.tls.ca_file }}
{% if nomad_vault_secrets.pki.enabled %}
export NOMAD_CLIENT_CERT={{ nomad_root_dir }}/tls/cli.crt
export NOMAD_CLIENT_KEY={{ nomad_root_dir }}/tls/cli.key
export NOMAD_TLS_SERVER_NAME=server.{{ nomad_conf.region | default('global') }}.nomad
{% endif %}
{% else %}
# TLS not enabled or not running in server mode
{% endif %}