Update to 2022-09-02 18:00

roles/nomad/defaults/main.yml

@@ -1,11 +1,11 @@
 ---
 
 # Version of Nomad to install
-nomad_version: 1.3.4
+nomad_version: 1.3.5
 # URL of the archive
 nomad_archive_url: https://releases.hashicorp.com/nomad/{{ nomad_version }}/nomad_{{ nomad_version }}_linux_amd64.zip
 # Expected sha256 of the archive
-nomad_archive_sha256: c20d26b411c1df9c9b7a571e3ba6ee900772703211c565224612bfc199473be9
+nomad_archive_sha256: a4bf189e6a84c4bc7d6090529c87b32e6b4b09b47163514d33305aa867d7c4dc
 
 # List of plugins to install
 nomad_plugins:
@@ -23,6 +23,9 @@ nomad_root_dir: /opt/nomad
 # Servers can run under an unprivileged user, while clients should run as root (or with equivalent privileges)
 nomad_user: "{{ nomad_conf.client.enabled | ternary('root', 'nomad') }}"
 
+# List of Unix group which will be nomad admins
+nomad_admin_groups: "{{ system_admin_groups | default([]) }}"
+
 # If ACL are enabled, you need to set a management token for ansible
 # to be able to manage Nomad (eg snapshot before upgrades)
 # nomad_mgm_token: XXXXXXXXX
@@ -52,6 +55,18 @@ nomad_base_conf:
     # replication_token: ...
 
 
+  # TLS Settings
+  # See the nomad_vault configuration if you want to integrate with vault to obtain and renew the certificates
+  tls:
+    http: False
+    rpc: False
+    ca_file: "{{ nomad_root_dir }}/tls/ca.crt"
+    cert_file: "{{ nomad_root_dir }}/tls/nomad.crt"
+    key_file: "{{ nomad_root_dir }}/tls/nomad.key"
+    rpc_upgrade_mode: False
+    verify_https_client: False
+    verify_server_hostname: False
+
   # Client related settings
   # The default is to act as a client if the hostname is not listed in nomad servers
   client:
@@ -165,6 +180,14 @@ nomad_base_conf:
     # allow_unauthenticated: True
     # tags: []
 
+  # Vault integration
+  vault:
+    enabled: False
+    create_from_role: nomad-cluster
+    #address: https://vault.exaple.org:8200
+    #ca_path: /opt/nomad/tls/vault_ca.crt
+    #cert_file: /opt/nomad/tls/vault.crt
+    #key_file: /opt/nomad/tls/vault.key
 
 
 # You can override part of the default config without rewriting everything else
@@ -198,3 +221,19 @@ nomad_base_services:
 nomad_extra_services: {}
 nomad_host_services: {}
 nomad_services: "{{ nomad_base_services | combine(nomad_extra_services, recursive=True) | combine(nomad_host_services, recursive=True) }}"
+
+# When using vault to setup TLS for Nomad
+nomad_base_vault:
+  enabled: False
+  address: "{{ nomad_conf.vault.address | default(omit) }}"
+  # Token to use to issue certificates
+  # token: XXXXXXXXX
+  pki:
+    # The path of the PKI secret where cert will be issued
+    path: /pki/nomad
+    role: nomad-cluster
+    ttl: 24h
+
+nomad_extra_vault: {}
+nomad_host_vault: {}
+nomad_vault: "{{ nomad_base_vault | combine(nomad_extra_vault, recursive=True) | combine(nomad_host_vault, recursive=True) }}"

roles/nomad/handlers/main.yml

@@ -8,3 +8,6 @@
 - name: reload nomad
   service: name=nomad state=reloaded
   when: nomad_service_started is not defined or not nomad_service_started.changed
+
+- name: restart consul-template-nomad
+  service: name=consul-template-nomad state=restarted

roles/nomad/meta/main.yml

@@ -2,8 +2,9 @@
 
 dependencies:
   - role: repo_docker
-    when: nomad_conf.client.enabled and nomad_conf.client.task_drivers['containerd-driver'].enabled # with containerd, we just configure the repo and install containerd.io
+    when: nomad_conf.client.enabled and nomad_conf.client.task_drivers['containerd-driver'].enabled # with containerd, we need docker repo to install containerd.io
   - role: docker
     when: nomad_conf.client.enabled and nomad_conf.client.task_drivers.docker.enabled
   - role: cni_plugins
     when: nomad_conf.client.enabled
+  - role: consul_template # consul-template can be used for obtaining certs from vault

roles/nomad/tasks/conf.yml

@@ -1,5 +1,24 @@
 ---
 
+- name: Generate self-signed certificate
+  import_tasks: ../includes/create_selfsigned_cert.yml
+  vars:
+    cert_path: "{{ nomad_conf.tls.cert_file }}"
+    cert_key_path: "{{ nomad_conf.tls.key_file }}"
+    cert_key_group: "{{ nomad_user }}"
+    cert_key_mode: 640
+  tags: nomad
+
+- name: Check if CA exists
+  stat: path={{ nomad_conf.tls.ca_file }}
+  register: nomad_ca_file
+  tags: nomad
+
+- name: Copy cert as CA
+  copy: src={{ nomad_conf.tls.cert_file }} dest={{ nomad_conf.tls.ca_file }} remote_src=True
+  when: not nomad_ca_file.stat.exists
+  tags: nomad
+
 - name: Deploy nomad configuration
   block:
     - name: Deploy nomad configuration
@@ -73,3 +92,46 @@
   loop: "{{ nomad_backup_configs.stdout_lines }}"
   tags: nomad
 
+- when: nomad_vault.enabled
+  block:
+
+    - name: Deploy consul-template config
+      template: src=consul-template.hcl.j2 dest={{ nomad_root_dir }}/consul-template/consul-template.hcl
+      notify: restart consul-template-nomad
+
+    - name: Deploy consul-template agent cert template
+      template: src=agent_cert.tpl.j2 dest={{ nomad_root_dir }}/consul-template/{{ item.where }} owner=root group=root
+      loop:
+        - what: certificate
+          where: agent.crt.tpl
+        - what: private_key
+          where: agent.key.tpl
+        - what: issuing_ca
+          where: ca.crt.tpl
+      notify: restart consul-template-nomad
+
+    - name: Deploy consul-template cli cert template
+      template: src=cli_cert.tpl.j2 dest={{ nomad_root_dir }}/consul-template/{{ item.where }} owner=root group=root
+      loop:
+        - what: certificate
+          where: cli.crt.tpl
+        - what: private_key
+          where: cli.key.tpl
+      notify: restart consul-template-nomad
+
+  tags: nomad
+
+- name: Set ACL on the TLS dir
+  shell: |
+    setfacl -R -b -x {{ nomad_root_dir }}/tls
+    {% if nomad_admin_groups | length > 0 %}
+    setfacl -R -m {% for group in nomad_admin_groups %}g:{{ group }}:rX{{ ',' if not loop.last }}{% endfor %} {{ nomad_root_dir }}/tls
+    setfacl -R -m {% for group in nomad_admin_groups %}d:g:{{ group }}:rX{{ ',' if not loop.last }}{% endfor %} {{ nomad_root_dir }}/tls
+    {% endif %}
+  changed_when: False
+  failed_when: False # Do not fail if eg, the FS doesn't support ACL
+  tags: nomad
+
+- name: Deploy profile script
+  template: src=profile.sh.j2 dest=/etc/profile.d/nomad.sh
+  tags: nomad

roles/nomad/tasks/directories.yml

@@ -31,6 +31,10 @@
       group: "{{ nomad_user }}"
     - dir: etc
       owner: root
-      group: "{{ nomad_user }}"
-      mode: 750
+      mode: 755
+    - dir: tls
+      owner: root
+      mode: 755
+    - dir: consul-template
+      mode: 755
   tags: nomad

roles/nomad/tasks/install.yml

@@ -109,9 +109,22 @@
   notify: restart nomad
   tags: nomad
 
+- name: Install backup hooks
+  template: src={{ item }}-backup.j2 dest=/etc/backup/{{ item }}.d/nomad mode=755
+  loop:
+    - pre
+    - post
+  tags: nomad
+
+- name: Install consul-template unit
+  template: src=consul-template-nomad.service.j2 dest=/etc/systemd/system/consul-template-nomad.service
+  register: nomad_consul_tpl_unit
+  notify: restart consul-template-nomad
+  tags: nomad
+
 - name: Reload systemd
   systemd: daemon_reload=True
-  when: nomad_unit.changed
+  when: nomad_unit.changed or (nomad_consul_tpl_unit is defined and nomad_consul_tpl_unit.changed)
   tags: nomad
 
 - name: Install backup hooks

roles/nomad/tasks/services.yml

@@ -4,3 +4,7 @@
   service: name=nomad state=started enabled=True
   register: nomad_service_started
   tags: nomad
+
+- name: Handle consul-template-nomad service
+  service: name=consul-template-nomad state={{ nomad_vault.enabled | ternary('started', 'stopped') }} enabled={{ nomad_vault.enabled | ternary(True, False) }}
+  tags: nomad

roles/nomad/templates/agent.crt.tpl.j2

@@ -0,0 +1,3 @@
+{{ with secret "[[ nomad_vault.pki.path ]]/issue/[[ nomad_vault.pki.role ]]" "common_name=[[ (nomad_conf.server.enabled) | ternary('server', 'client') ]].[[ nomad_conf.region | default('global') ]].nomad" "ttl=[[ nomad_vault.pki.ttl ]]" "alt_names=localhost,[[ inventory_hostname ]],{% if nomad_conf.server.enabled and nomad_conf.client.enabled %}client.[[ nomad_conf.region | default('global') ]].nomad{% endif %}" "ip_sans=127.0.0.1,[[ ansible_default_ipv4.address ]]"}}
+{{ .Data.certificate }}
+{{ end }}

roles/nomad/templates/agent_cert.tpl.j2

@@ -0,0 +1,3 @@
+[[ with secret "{{ nomad_vault.pki.path }}/issue/{{ nomad_vault.pki.role }}" "common_name={{ (nomad_conf.server.enabled) | ternary('server', 'client') }}.{{ nomad_conf.region | default('global') }}.nomad" "ttl={{ nomad_vault.pki.ttl }}" "alt_names=localhost,{{ inventory_hostname }},{% if nomad_conf.server.enabled and nomad_conf.client.enabled %}client.{{ nomad_conf.region | default('global') }}.nomad{% endif %}" "ip_sans=127.0.0.1,{{ ansible_default_ipv4.address }}" ]]
+[[ .Data.{{ item.what }} ]]
+[[ end ]]

roles/nomad/templates/cli_cert.tpl.j2

@@ -0,0 +1,3 @@
+[[ with secret "{{ nomad_vault.pki.path }}/issue/{{ nomad_vault.pki.role }}" "ttl={{ nomad_vault.pki.ttl }}" ]]
+[[ .Data.{{ item.what }} ]]
+[[ end ]]

roles/nomad/templates/consul-template-nomad.service.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description="HashiCorp consul-template"
+Documentation=https://github.com/hashicorp/consul-template
+Requires=network-online.target
+After=network-online.target
+ConditionFileNotEmpty={{ nomad_root_dir }}/consul-template/consul-template.hcl
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/consul-template -config={{ nomad_root_dir }}/consul-template/consul-template.hcl
+ExecReload=/bin/kill --signal HUP $MAINPID
+KillSignal=SIGINT
+Restart=on-failure
+RestartSec=2
+
+[Install]
+WantedBy=multi-user.target
+

roles/nomad/templates/consul-template.hcl.j2

@@ -0,0 +1,54 @@
+vault {
+  address      = "{{ nomad_vault.address }}"
+  token        = "{{ nomad_vault.token }}"
+  unwrap_token = false
+}
+
+template {
+  source          = "{{ nomad_root_dir }}/consul-template/agent.crt.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ nomad_conf.tls.cert_file }}"
+  perms           = 0644
+  exec {
+    command = "systemctl reload nomad"
+  }
+}
+
+template {
+  source          = "{{ nomad_root_dir }}/consul-template/agent.key.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ nomad_conf.tls.key_file }}"
+  perms           = 0640
+  exec {
+    command = ["sh", "-c", "chgrp {{ nomad_user }} {{ nomad_conf.tls.key_file }} && systemctl reload nomad"]
+  }
+}
+
+template {
+  source          = "{{ nomad_root_dir }}/consul-template/ca.crt.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ nomad_conf.tls.ca_file }}"
+  perms           = 0644
+  exec {
+    command = "systemctl reload nomad"
+  }
+}
+
+template {
+  source          = "{{ nomad_root_dir }}/consul-template/cli.crt.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ nomad_root_dir }}/tls/cli.crt"
+}
+
+template {
+  source          = "{{ nomad_root_dir }}/consul-template/cli.key.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ nomad_root_dir }}/tls/cli.key"
+  perms           = 0640
+}
+

roles/nomad/templates/nomad.hcl.j2

@@ -181,3 +181,30 @@ consul {
 {% endfor %}
 {% endif %}
 }
+
+vault {
+{% for key in ['enabled', 'tls_skip_verify', 'allow_unauthenticated'] %}
+{% if nomad_conf.vault[key] is defined %}
+  {{ key }} = {{ nomad_conf.vault[key] | ternary('true', 'false') }}
+{% endif %}
+{% endfor %}
+{% for key in ['address', 'create_from_role', 'task_token_ttl', 'ca_file', 'ca_path', 'cert_file', 'key_file', 'namespace', 'tls_server_name', 'token'] %}
+{% if nomad_conf.vault[key] is defined %}
+  {{ key }} = "{{ nomad_conf.vault[key] }}"
+{% endif %}
+{% endfor %}
+}
+
+tls {
+{% for key in ['ca_file', 'cert_file', 'key_file', 'tls_min_version', 'tls_cipher_suites'] %}
+{% if nomad_conf.tls[key] is defined %}
+  {{ key }} = "{{ nomad_conf.tls[key] }}"
+{% endif %}
+{% endfor %}
+
+{% for key in ['http', 'rpc', 'rpc_upgrade_mode', 'tls_prefer_server_cipher_suites', 'verify_https_client', 'verify_server_hostname'] %}
+{% if nomad_conf.tls[key] is defined %}
+  {{ key }} = {{ nomad_conf.tls[key] | ternary('true', 'false') }}
+{% endif %}
+{% endfor %}
+}

roles/nomad/templates/pre-backup.j2

@@ -2,4 +2,12 @@
 
 set -eo pipefail
 
+{% if nomad_conf.tls.http %}
+NOMAD_ADDR=https://localhost:{{ nomad_services.http.port }} \
+NOMAD_CACERT={{ nomad_conf.tls.ca_file }} \
+{% endif %}
+{% if nomad_vault.enabled %}
+NOMAD_CLIENT_CERT={{ nomad_root_dir }}/tls/cli.crt \
+NOMAD_CLIENT_KEY={{ nomad_root_dir }}/tls/cli.key \
+{% endif %}
 {{ nomad_root_dir }}/bin/nomad operator snapshot save {{ nomad_root_dir }}/backup/nomad.snap

roles/nomad/templates/profile.sh.j2

@@ -0,0 +1,8 @@
+{% if nomad_conf.tls.http %}
+export NOMAD_ADDR=https://localhost:{{ nomad_services.http.port }}
+export NOMAD_CACERT={{ nomad_conf.tls.ca_file }}
+{% if nomad_vault.enabled %}
+export NOMAD_CLIENT_CERT={{ nomad_root_dir }}/tls/cli.crt
+export NOMAD_CLIENT_KEY={{ nomad_root_dir }}/tls/cli.key
+{% endif %}
+{% endif %}