Update to 2022-09-06 18:00

roles/consul/defaults/main.yml

@@ -88,6 +88,12 @@ consul_base_conf:
     enabled: False
     # The default_policy is also used for intentions in the service mesh
     default_policy: deny
+    enable_token_persistence: True
+    # You can set tokens used by the agent
+    # tokens:
+    #   default: ab47bc38-d97f-19af-93a5-17b528d154c9
+    #   agent: 5459979a-3f23-8b1f-ff8a-2478856e9216
+    tokens: {}
 
   tls:
     # No TLS will be stup unless this is set to True
@@ -110,18 +116,21 @@ consul_extra_conf: {}
 consul_host_conf: {}
 consul_conf: "{{ consul_base_conf | combine(consul_extra_conf, recursive=True) | combine(consul_host_conf, recursive=True) }}"
 
-# To get certificates from vault
-consul_base_vault_tls:
-  enabled: False
-  # address: https://active.vault.service.consul:8200
-  # token: XXXXXX
+# TLS certs and token retrival from vault
+consul_base_vault_secrets:
+  # vault_address: https://active.vault.service.consul:8200
+  # vault_token: XXXXXX
   pki:
+    enabled: False
     path: /pki/consul
     role: consul-{{ consul_conf.server | ternary('server', 'client') }}
-    ttl: 24h
-consul_extra_vault_tls: {}
-consul_host_vault_tls: {}
-consul_vault_tls: "{{ consul_base_vault_tls | combine(consul_extra_vault_tls, recursive=True) | combine(consul_host_vault_tls, recursive=True) }}"
+  tokens:
+    enabled: False
+    path: /consul
+    role: consul-agent
+consul_extra_vault_secrets: {}
+consul_host_vault_secrets: {}
+consul_vault_secrets: "{{ consul_base_vault_secrets | combine(consul_extra_vault_secrets, recursive=True) | combine(consul_host_vault_secrets, recursive=True) }}"
 
 # For example
 # consul_extra_conf:

roles/consul/tasks/conf.yml

@@ -94,42 +94,40 @@
   loop: "{{ consul_backup_configs.stdout_lines }}"
   tags: consul
 
-- when: consul_vault_tls.enabled
-  block:
-
-    - name: Deploy consul-template config
-      template: src=consul-template.hcl.j2 dest={{ consul_root_dir }}/consul-template/consul-template.hcl
-      notify: restart consul-template-consul
-
-    - name: Deploy consul-template agent cert template
-      template: src=agent_cert.tpl.j2 dest={{ consul_root_dir }}/consul-template/{{ item.where }} owner=root group=root
-      loop:
-        - what: certificate
-          where: agent.crt.tpl
-        - what: private_key
-          where: agent.key.tpl
-        - what: issuing_ca
-          where: ca.crt.tpl
-      notify: restart consul-template-consul
-
-    - name: Check if certificate exists
-      stat: path={{ consul_conf.tls.defaults.cert_file }}
-      register: consul_tls_cert_file
-
+- name: Deploy consul-template config
+  template: src=consul-template.hcl.j2 dest={{ consul_root_dir }}/consul-template/consul-template.hcl mode=600 owner=root group=root
+  notify: restart consul-template-consul
+  when: consul_vault_secrets.pki.enabled or consul_vault_secrets.tokens.enabled
   tags: consul
 
-- when: consul_vault_tls.enabled and consul_conf.server
-  block:
+- name: Deploy consul-template agent cert template
+  template: src=agent_cert.tpl.j2 dest={{ consul_root_dir }}/consul-template/{{ item.where }} owner=root group=root
+  loop:
+    - what: certificate
+      where: agent.crt.tpl
+    - what: private_key
+      where: agent.key.tpl
+    - what: issuing_ca
+      where: ca.crt.tpl
+  notify: restart consul-template-consul
+  when: consul_vault_secrets.pki.enabled
+  tags: consul
 
-    - name: Deploy consul-template cli cert template
-      template: src=cli_cert.tpl.j2 dest={{ consul_root_dir }}/consul-template/{{ item.where }} owner=root group=root
-      loop:
-        - what: certificate
-          where: cli.crt.tpl
-        - what: private_key
-          where: cli.key.tpl
-      notify: restart consul-template-consul
+- name: Deploy consul-template cli cert template
+  template: src=cli_cert.tpl.j2 dest={{ consul_root_dir }}/consul-template/{{ item.where }} owner=root group=root
+  loop:
+    - what: certificate
+      where: cli.crt.tpl
+    - what: private_key
+      where: cli.key.tpl
+  notify: restart consul-template-consul
+  when: consul_vault_secrets.pki.enabled and consul_conf.server
+  tags: consul
 
+- name: Deploy the consul-template agent token template
+  template: src=agent.token.tpl.j2 dest={{ consul_root_dir }}/consul-template/agent.token.tpl owner=root group=root
+  notify: restart consul-template-consul
+  when: consul_vault_secrets.tokens.enabled
   tags: consul
 
 - name: Set ACL on the TLS dir

roles/consul/tasks/services.yml

@@ -6,6 +6,9 @@
   tags: consul
 
 - name: Handle consul-template-consul service
-  service: name=consul-template-consul state={{ consul_vault_tls.enabled | ternary('started', 'stopped') }} enabled={{ consul_vault_tls.enabled | ternary(True, False) }}
+  service:
+    name: consul-template-consul
+    state: "{{ (consul_vault_secrets.pki.enabled or consul_vault_secrets.tokens.enabled) | ternary('started', 'stopped') }}"
+    enabled: "{{ (consul_vault_secrets.pki.enabled or consul_vault_secrets.tokens.enabled) | ternary(True, False) }}"
   tags: consul
 

roles/consul/templates/agent.token.tpl.j2

@@ -0,0 +1,3 @@
+[[ with secret "{{ consul_vault_secrets.tokens.path }}/creds/{{ consul_vault_secrets.tokens.role }}" ]]
+[[ .Data.token ]]
+[[ end ]]

roles/consul/templates/agent_cert.tpl.j2

@@ -1,14 +1,14 @@
 {% if consul_conf.server %}
-[[ with secret "{{ consul_vault_tls.pki.path }}/issue/{{ consul_vault_tls.pki.role }}" "common_name=server-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" "ttl={{ consul_vault_tls.pki.ttl }}" "alt_names=localhost,consul.service.{{ consul_conf.domain | default('consul') }},server.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" ]]
+[[ with secret "{{ consul_vault_secrets.pki.path }}/issue/{{ consul_vault_secrets.pki.role }}" "common_name=server-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" "alt_names=localhost,consul.service.{{ consul_conf.domain | default('consul') }},server.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" ]]
 [[ .Data.{{ item.what }} ]]
 [[ end ]]
 {% if item.what == 'certificate' %}
-[[ with secret "{{ consul_vault_tls.pki.path }}/cert/ca" ]]
+[[ with secret "{{ consul_vault_secrets.pki.path }}/cert/ca" ]]
 [[ .Data.certificate ]]
 [[ end ]]
 {% endif %}
 {% else %}
-[[ with secret "{{ consul_vault_tls.pki.path }}/cert/ca" ]]
+[[ with secret "{{ consul_vault_secrets.pki.path }}/cert/ca" ]]
 [[ .Data.certificate ]]
 [[ end ]]
 {% endif %}

roles/consul/templates/cli_cert.tpl.j2

@@ -1,3 +1,3 @@
-[[ with secret "{{ consul_vault_tls.pki.path }}/issue/{{ consul_vault_tls.pki.role }}" "ttl={{ consul_vault_tls.pki.ttl }}" "common_name=cli-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" ]]
+[[ with secret "{{ consul_vault_secrets.pki.path }}/issue/{{ consul_vault_secrets.pki.role }}" "common_name=cli-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" ]]
 [[ .Data.{{ item.what }} ]]
 [[ end ]]

roles/consul/templates/consul-template-consul.service.j2

@@ -8,6 +8,7 @@ ConditionFileNotEmpty={{ consul_root_dir }}/consul-template/consul-template.hcl
 [Service]
 Type=simple
 ExecStart=/usr/local/bin/consul-template -config={{ consul_root_dir }}/consul-template/consul-template.hcl
+SuccessExitStatus=12
 ExecReload=/bin/kill --signal HUP $MAINPID
 KillSignal=SIGINT
 Restart=on-failure

roles/consul/templates/consul-template.hcl.j2

@@ -1,6 +1,6 @@
 vault {
-  address      = "{{ consul_vault_tls.address }}"
-  token        = "{{ consul_vault_tls.token }}"
+  address      = "{{ consul_vault_secrets.vault_address }}"
+  token        = "{{ consul_vault_secrets.vault_token }}"
   unwrap_token = false
 }
 
@@ -11,10 +11,11 @@ template {
   destination     = "{{ consul_conf.tls.defaults.ca_file }}"
   perms           = 0644
   exec {
-    command = "systemctl reload consul"
+    command = "sh -c 'systemctl reload consul || true'"
   }
 }
 
+{% if consul_vault_secrets.pki.enabled %}
 {% if consul_conf.server %}
 template {
   source          = "{{ consul_root_dir }}/consul-template/agent.crt.tpl"
@@ -23,7 +24,7 @@ template {
   destination     = "{{ consul_conf.tls.defaults.cert_file }}"
   perms           = 0644
   exec {
-    command = "systemctl reload consul"
+    command = "sh -c 'systemctl reload consul || true'"
   }
 }
 
@@ -34,7 +35,7 @@ template {
   destination     = "{{ consul_conf.tls.defaults.key_file }}"
   perms           = 0640
   exec {
-    command = ["sh", "-c", "chgrp {{ consul_user }} {{ consul_conf.tls.defaults.key_file }} && systemctl reload consul"]
+    command = "sh -c 'chgrp {{ consul_user }} {{ consul_conf.tls.defaults.key_file }} && systemctl reload consul || true'"
   }
 }
 
@@ -53,3 +54,17 @@ template {
   perms           = 0640
 }
 {% endif %}
+{% endif %}
+
+{% if consul_vault_secrets.tokens.enabled %}
+template {
+  source          = "{{ consul_root_dir }}/consul-template/agent.token.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ consul_root_dir }}/tmp/agent.token"
+  perms           = 0600
+  exec {
+    command = "sh -c 'consul acl set-agent-token default $(grep -P \'^[^\s]\' {{ consul_root_dir }}/tmp/agent.token)'"
+  }
+}
+{% endif %}

roles/consul/templates/consul.hcl.j2

@@ -73,8 +73,19 @@ connect {
 {% endif %}
 
 acl {
-  enabled = {{ consul_conf.acl.enabled | ternary('true', 'false') }}
+{% for key in ['enabled', 'enable_token_persistence'] %}
+{% if consul_conf.acl[key] is defined %}
+  {{ key }} = {{ consul_conf.acl[key] | ternary('true', 'false') }}
+{% endif %}
+{% endfor %}
   default_policy = "{{ consul_conf.acl.default_policy }}"
+  tokens {
+{% for key in ['initial_management', 'default', 'agent', 'agent_recovery', 'replication'] %}
+{% if consul_conf.acl.tokens[key] is defined %}
+    {{ key }} = "{{ consul_conf.acl.tokens[key] }}"
+{% endif %}
+{% endfor %}
+  }
 }
 
 {% if consul_conf.tls.enabled %}
@@ -108,5 +119,4 @@ auto_encrypt {
   tls = true
 }
 {% endif %}
-
 {% endif %}

roles/consul/templates/profile.sh.j2

@@ -2,7 +2,7 @@
 export CONSUL_HTTP_ADDR=https://localhost:{{ consul_services.https.port }}
 export CONSUL_HTTP_SSL=true
 export CONSUL_CACERT={{ consul_conf.tls.defaults.ca_file }}
-{% if consul_vault_tls.enabled %}
+{% if consul_vault_secrets.pki.enabled %}
 export CONSUL_CLIENT_CERT={{ consul_root_dir }}/tls/cli.crt
 export CONSUL_CLIENT_KEY={{ consul_root_dir }}/tls/cli.key
 export CONSUL_TLS_SERVER_NAME=server.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}

roles/nomad/defaults/main.yml

@@ -57,7 +57,7 @@ nomad_base_conf:
 
 
   # TLS Settings
-  # See the nomad_vault_tls configuration if you want to integrate with vault to obtain and renew the certificates
+  # See the nomad_vault_secrets configuration if you want to integrate with vault to obtain and renew the certificates
   tls:
     http: False
     rpc: False
@@ -228,23 +228,23 @@ nomad_host_services: {}
 nomad_services: "{{ nomad_base_services | combine(nomad_extra_services, recursive=True) | combine(nomad_host_services, recursive=True) }}"
 
 # When using vault to setup TLS for Nomad
-nomad_base_vault_tls:
-  enabled: False
-  address: "{{ nomad_conf.vault.address | default(omit) }}"
+nomad_base_vault_secrets:
+  vault_address: "{{ nomad_conf.vault.address | default(omit) }}"
   # Token to use to issue certificates
   # token: XXXXXXXXX
   pki:
+    enabled: False
     # The path of the PKI secret where cert will be issued
     path: /pki/nomad
     role: nomad-{{ nomad_conf.server.enabled | ternary('server', 'client') }}
-    ttl: 24h
+    # ttl: 6h # if not set, will use the default ttl of the role
   consul_pki:
+    enabled: False
     path: /pki/consul
-    role: nomad-client # Only nomad clients will use the gRPC endpoint and requires a client cert for consul
-    ttl: 24h
-  root_pki:
-    path: /pki/root # This is used to fetch the top level root CA, as envoy can't validate the chain unless it has it complete up to the auto-signed root
+    root_path: /pki/root # This is used to fetch the top level root CA, as envoy can't validate the chain unless it has it complete up to the auto-signed root
+    role: nomad-client # Only nomad clients will use this
+    # ttl: 6h # if not set, will use the default ttl of the role
 
-nomad_extra_vault_tls: {}
-nomad_host_vault_tls: {}
-nomad_vault_tls: "{{ nomad_base_vault_tls | combine(nomad_extra_vault_tls, recursive=True) | combine(nomad_host_vault_tls, recursive=True) }}"
+nomad_extra_vault_secrets: {}
+nomad_host_vault_secrets: {}
+nomad_vault_secrets: "{{ nomad_base_vault_secrets | combine(nomad_extra_vault_secrets, recursive=True) | combine(nomad_host_vault_secrets, recursive=True) }}"

roles/nomad/tasks/conf.yml

@@ -113,38 +113,34 @@
   loop: "{{ nomad_backup_configs.stdout_lines }}"
   tags: nomad
 
-- when: nomad_vault_tls.enabled
-  block:
-
-    - name: Deploy consul-template config
-      template: src=consul-template.hcl.j2 dest={{ nomad_root_dir }}/consul-template/consul-template.hcl
-      notify: restart consul-template-nomad
-
-    - name: Deploy consul-template agent cert template
-      template: src=agent_cert.tpl.j2 dest={{ nomad_root_dir }}/consul-template/{{ item.where }} owner=root group=root
-      loop:
-        - what: certificate
-          where: agent.crt.tpl
-        - what: private_key
-          where: agent.key.tpl
-        - what: issuing_ca
-          where: ca.crt.tpl
-      notify: restart consul-template-nomad
-
+- name: Deploy consul-template config
+  template: src=consul-template.hcl.j2 dest={{ nomad_root_dir }}/consul-template/consul-template.hcl mode=600 owner=root group=root
+  notify: restart consul-template-nomad
+  when: nomad_vault_secrets.pki.enabled or nomad_vault_secrets.tokens.enabled
   tags: nomad
 
-- when: nomad_vault_tls.enabled and nomad_conf.server.enabled
-  block:
-
-    - name: Deploy consul-template cli cert template
-      template: src=cli_cert.tpl.j2 dest={{ nomad_root_dir }}/consul-template/{{ item.where }} owner=root group=root
-      loop:
-        - what: certificate
-          where: cli.crt.tpl
-        - what: private_key
-          where: cli.key.tpl
-      notify: restart consul-template-nomad
+- name: Deploy consul-template agent cert template
+  template: src=agent_cert.tpl.j2 dest={{ nomad_root_dir }}/consul-template/{{ item.where }} owner=root group=root
+  loop:
+    - what: certificate
+      where: agent.crt.tpl
+    - what: private_key
+      where: agent.key.tpl
+    - what: issuing_ca
+      where: ca.crt.tpl
+  notify: restart consul-template-nomad
+  when: nomad_vault_secrets.pki.enabled
+  tags: nomad
 
+- name: Deploy consul-template cli cert template
+  template: src=cli_cert.tpl.j2 dest={{ nomad_root_dir }}/consul-template/{{ item.where }} owner=root group=root
+  loop:
+    - what: certificate
+      where: cli.crt.tpl
+    - what: private_key
+      where: cli.key.tpl
+  notify: restart consul-template-nomad
+  when: nomad_vault_secrets.pki.enabled and nomad_conf.server.enabled
   tags: nomad
 
 - name: Set ACL on the TLS dir
@@ -172,6 +168,6 @@
     - what: issuing_ca
       where: consul_ca.crt.tpl
   notify: restart consul-template-nomad
-  when: nomad_conf.client.enabled and nomad_conf.consul.ssl
+  when: nomad_vault_secrets.consul_pki.enabled and nomad_conf.client.enabled and nomad_conf.consul.ssl
   tags: nomad
 

roles/nomad/tasks/install.yml

@@ -114,6 +114,7 @@
   template: src=consul-template-nomad.service.j2 dest=/etc/systemd/system/consul-template-nomad.service
   register: nomad_consul_tpl_unit
   notify: restart consul-template-nomad
+  when: nomad_vault_secrets.pki.enabled or nomad_vault_secrets.consul_pki.enabled
   tags: nomad
 
 - name: Reload systemd

roles/nomad/tasks/services.yml

@@ -6,5 +6,8 @@
   tags: nomad
 
 - name: Handle consul-template-nomad service
-  service: name=consul-template-nomad state={{ nomad_vault_tls.enabled | ternary('started', 'stopped') }} enabled={{ nomad_vault_tls.enabled | ternary(True, False) }}
+  service:
+    name: consul-template-nomad
+    state: "{{ (nomad_vault_secrets.pki.enabled or nomad_vault_secrets.consul_pki.enabled or nomad_vault_secrets.tokens.enabled) | ternary('started', 'stopped') }}"
+    enabled: "{{ (nomad_vault_secrets.pki.enabled or nomad_vault_secrets.consul_pki.enabled or nomad_vault_secrets.tokens.enabled) | ternary(True, False) }}"
   tags: nomad

roles/nomad/templates/agent_cert.tpl.j2

@@ -1,8 +1,8 @@
-[[ with secret "{{ nomad_vault_tls.pki.path }}/issue/{{ nomad_vault_tls.pki.role }}" "common_name={{ (nomad_conf.server.enabled) | ternary('server', 'client') }}-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ nomad_conf.region | default('global') }}.nomad" "ttl={{ nomad_vault_tls.pki.ttl }}" "alt_names=localhost,{{ (nomad_conf.server.enabled) | ternary('server', 'client') }}.{{ nomad_conf.region | default('global') }}.nomad{% if nomad_conf.server.enabled and nomad_conf.client.enabled %},client.{{ nomad_conf.region | default('global') }}.nomad{% endif %}{% if consul_conf is defined %},nomad{{ nomad_conf.server.enabled | ternary('', '-client') }}.service.{{ consul_conf.domain | default('consul') }}{% endif %}" ]]
+[[ with secret "{{ nomad_vault_secrets.pki.path }}/issue/{{ nomad_vault_secrets.pki.role }}" "common_name={{ (nomad_conf.server.enabled) | ternary('server', 'client') }}-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ nomad_conf.region | default('global') }}.nomad" "alt_names=localhost,{{ (nomad_conf.server.enabled) | ternary('server', 'client') }}.{{ nomad_conf.region | default('global') }}.nomad{% if nomad_conf.server.enabled and nomad_conf.client.enabled %},client.{{ nomad_conf.region | default('global') }}.nomad{% endif %}{% if consul_conf is defined %},nomad{{ nomad_conf.server.enabled | ternary('', '-client') }}.service.{{ consul_conf.domain | default('consul') }}{% endif %}"{% if nomad_vault_secrets.pki.ttl is defined %} "ttl={{ nomad_vault_secrets.pki.ttl }}"{% endif %} ]]
 [[ .Data.{{ item.what }} ]]
 [[ end ]]
 {% if item.what == 'certificate' %}
-[[ with secret "{{ nomad_vault_tls.pki.path }}/cert/ca" ]]
+[[ with secret "{{ nomad_vault_secrets.pki.path }}/cert/ca" ]]
 [[ .Data.certificate ]]
 [[ end ]]
 {% endif %}

roles/nomad/templates/cli_cert.tpl.j2

@@ -1,3 +1,3 @@
-[[ with secret "{{ nomad_vault_tls.pki.path }}/issue/{{ nomad_vault_tls.pki.role }}" "ttl={{ nomad_vault_tls.pki.ttl }}" "common_name=cli-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ nomad_conf.region | default('global') }}.nomad" ]]
+[[ with secret "{{ nomad_vault_secrets.pki.path }}/issue/{{ nomad_vault_secrets.pki.role }}" "common_name=cli-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ nomad_conf.region | default('global') }}.nomad" ]]
 [[ .Data.{{ item.what }} ]]
 [[ end ]]

roles/nomad/templates/consul-template-nomad.service.j2

@@ -3,11 +3,13 @@ Description="HashiCorp consul-template"
 Documentation=https://github.com/hashicorp/consul-template
 Requires=network-online.target
 After=network-online.target
+Before=nomad.service
 ConditionFileNotEmpty={{ nomad_root_dir }}/consul-template/consul-template.hcl
 
 [Service]
 Type=simple
 ExecStart=/usr/local/bin/consul-template -config={{ nomad_root_dir }}/consul-template/consul-template.hcl
+SuccessExitStatus=12
 ExecReload=/bin/kill --signal HUP $MAINPID
 KillSignal=SIGINT
 Restart=on-failure

roles/nomad/templates/consul-template.hcl.j2

@@ -1,9 +1,12 @@
 vault {
-  address      = "{{ nomad_vault_tls.address }}"
-  token        = "{{ nomad_vault_tls.token }}"
+  address      = "{{ nomad_vault_secrets.vault_address }}"
+  token        = "{{ nomad_vault_secrets.vault_token }}"
   unwrap_token = false
 }
 
+# Sleep 10 sec before sending nomad service a reload to prevent it from crashing as
+# Nomad doesn't support getting a reload while it's still initializing
+{% if nomad_vault_secrets.pki.enabled %}
 template {
   source          = "{{ nomad_root_dir }}/consul-template/agent.crt.tpl"
   left_delimiter  = "[["
@@ -11,7 +14,7 @@ template {
   destination     = "{{ nomad_conf.tls.cert_file }}"
   perms           = 0644
   exec {
-    command = "systemctl reload nomad"
+    command = "sh -c 'sleep 10 && systemctl reload nomad || true'"
   }
 }
 
@@ -22,7 +25,7 @@ template {
   destination     = "{{ nomad_conf.tls.key_file }}"
   perms           = 0640
   exec {
-    command = ["sh", "-c", "chgrp {{ nomad_user }} {{ nomad_conf.tls.key_file }} && systemctl reload nomad"]
+    command = ["sh", "-c", "chgrp {{ nomad_user }} {{ nomad_conf.tls.key_file }} && sleep 10 && systemctl reload nomad || true"]
   }
 }
 
@@ -33,7 +36,7 @@ template {
   destination     = "{{ nomad_conf.tls.ca_file }}"
   perms           = 0644
   exec {
-    command = "systemctl reload nomad"
+    command = "sh -c 'sleep 10 && systemctl reload nomad || true'"
   }
 }
 
@@ -53,13 +56,18 @@ template {
   perms           = 0640
 }
 {% endif %}
+{% endif %}
 
-{% if nomad_conf.client.enabled and nomad_conf.consul.ssl %}
+
+{% if nomad_vault_secrets.consul_pki.enabled and nomad_conf.client.enabled and nomad_conf.consul.ssl %}
 template {
   source          = "{{ nomad_root_dir }}/consul-template/consul.crt.tpl"
   left_delimiter  = "[["
   right_delimiter = "]]"
   destination     = "{{ nomad_conf.consul.cert_file }}"
+  exec {
+    command = "sh -c 'sleep 10 && systemctl reload nomad || true'"
+  }
 }
 
 template {
@@ -68,6 +76,9 @@ template {
   right_delimiter = "]]"
   destination     = "{{ nomad_conf.consul.key_file }}"
   perms           = 0640
+  exec {
+    command = "sh -c 'sleep 10 && systemctl reload nomad || true'"
+  }
 }
 
 template {
@@ -77,8 +88,8 @@ template {
   destination     = "{{ nomad_conf.consul.ca_file }}"
   perms           = 0644
   exec {
-    command = "systemctl reload nomad"
+    command = "sh -c 'sleep 10 && systemctl reload nomad || true'"
   }
 }
-
 {% endif %}
+

roles/nomad/templates/consul_cert.tpl.j2

@@ -1,8 +1,8 @@
-[[ with secret "{{ nomad_vault_tls.consul_pki.path }}/issue/{{ nomad_vault_tls.consul_pki.role }}" "common_name={{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ (consul_conf is defined and consul_conf.domain is defined) | ternary(consul_conf.domain, 'consul') }}" "ttl={{ nomad_vault_tls.consul_pki.ttl }}" ]]
+[[ with secret "{{ nomad_vault_secrets.consul_pki.path }}/issue/{{ nomad_vault_secrets.consul_pki.role }}" "common_name={{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ (consul_conf is defined and consul_conf.domain is defined) | ternary(consul_conf.domain, 'consul') }}"{% if nomad_vault_secrets.consul_pki.ttl is defined %} "ttl={{ nomad_vault_secrets.consul_pki.ttl }}"{% endif %} ]]
 [[ .Data.{{ item.what }} ]]
 [[ end ]]
 {% if item.what == 'issuing_ca' %}
-[[ with secret "{{ nomad_vault_tls.root_pki.path }}/cert/ca" ]]
+[[ with secret "{{ nomad_vault_secrets.consul_pki.root_path }}/cert/ca" ]]
 [[ .Data.certificate ]]
 [[ end ]]
 {% endif %}

roles/nomad/templates/nomad.hcl.j2

@@ -210,11 +210,20 @@ vault {
   {{ key }} = {{ nomad_conf.vault[key] | ternary('true', 'false') }}
 {% endif %}
 {% endfor %}
+{% if nomad_conf.server.enabled %}
 {% for key in ['address', 'create_from_role', 'task_token_ttl', 'ca_file', 'ca_path', 'cert_file', 'key_file', 'namespace', 'tls_server_name', 'token'] %}
 {% if nomad_conf.vault[key] is defined %}
   {{ key }} = "{{ nomad_conf.vault[key] }}"
 {% endif %}
 {% endfor %}
+{% elif nomad_conf.client.enabled and not nomad_conf.server.enabled %}
+{% for key in ['address', 'ca_file', 'ca_path', 'cert_file', 'key_file', 'namespace', 'tls_server_name'] %}
+{% if nomad_conf.vault[key] is defined %}
+  {{ key }} = "{{ nomad_conf.vault[key] }}"
+{% endif %}
+{% endfor %}
+
+{% endif %}
 }
 
 tls {

roles/nomad/templates/pre-backup.j2

@@ -6,7 +6,7 @@ set -eo pipefail
 NOMAD_ADDR=https://localhost:{{ nomad_services.http.port }} \
 NOMAD_CACERT={{ nomad_conf.tls.ca_file }} \
 {% endif %}
-{% if nomad_vault_tls.enabled %}
+{% if nomad_vault_secrets.pki.enabled %}
 NOMAD_CLIENT_CERT={{ nomad_root_dir }}/tls/cli.crt \
 NOMAD_CLIENT_KEY={{ nomad_root_dir }}/tls/cli.key \
 {% endif %}

roles/nomad/templates/profile.sh.j2

@@ -1,7 +1,7 @@
 {% if nomad_conf.tls.http and nomad_conf.server.enabled %}
 export NOMAD_ADDR=https://localhost:{{ nomad_services.http.port }}
 export NOMAD_CACERT={{ nomad_conf.tls.ca_file }}
-{% if nomad_vault_tls.enabled %}
+{% if nomad_vault_secrets.pki.enabled %}
 export NOMAD_CLIENT_CERT={{ nomad_root_dir }}/tls/cli.crt
 export NOMAD_CLIENT_KEY={{ nomad_root_dir }}/tls/cli.key
 export NOMAD_TLS_SERVER_NAME=server.{{ nomad_conf.region | default('global') }}.nomad

roles/vault/defaults/main.yml

@@ -126,3 +126,32 @@ vault_host_conf: {}
 # Merge all the conf
 vault_conf: "{{ vault_base_conf | combine(vault_extra_conf, recursive=True) | combine(vault_host_conf, recursive=True) }}"
 
+# This can be used to spawn a consul-template service which will obtain and renew client cert
+# to reach Nomad API, so the Nomad secret can be used securely
+vault_base_secrets:
+  # The vault API to query. Default is our own API
+  vault_address: "{{ vault_conf.api_addr }}"
+  # The vault token to use
+  vault_token: XXXXXXX
+  nomad:
+    enabled: False
+    # The Nomad API address
+    address: https://nomad.service.consul:4646
+    # The Nomad management token vault will use to issue tokens for users
+    token: XXXXXXX
+    pki:
+      # The path where the PKI used by Nomad is mounted. The PKI must be mounted and configured
+      path: /pki/nomad
+      # The role used to issue the certificate
+      role: nomad-user
+      # The TTL of the certificate issued for vault
+      ttl: 72h
+      # The common name of the certificate
+      cn: vault
+    secret:
+      # The path where the Nomad secret engine is mounted
+      # Note: the secret must be already mounted
+      path: nomad
+vault_extra_secrets: {}
+vault_host_secrets: {}
+vault_secrets: "{{ vault_base_secrets | combine(vault_extra_secrets, recursive=True) | combine(vault_host_secrets, recursive=True) }}"

roles/vault/handlers/main.yml

@@ -6,3 +6,6 @@
 
 - name: reload vault
   service: name=vault state=reloaded
+
+- name: restart consul-template-vault
+  service: name=consul-template-vault state=restarted

roles/vault/meta/main.yml

@@ -2,3 +2,4 @@
 
 dependencies:
   - role: mkdir
+  - role: consul_template

roles/vault/tasks/conf.yml

@@ -22,3 +22,20 @@
 - name: Setup logrotate
   template: src=logrotate.conf.j2 dest=/etc/logrotate.d/vault
   tags: vault
+
+- when: vault_secrets.nomad.enabled
+  block:
+
+    - name: Deploy the consul-template conf
+      template: src=consul-template.hcl.j2 dest={{ vault_root_dir }}/consul-template/consul-template.hcl
+      notify: restart consul-template-vault
+
+    - name: Deploy Nomad certificate bundle template for consul-template
+      template: src=nomad_client_bundle.json.tpl.j2 dest={{ vault_root_dir }}/consul-template/nomad_client_bundle.json.tpl
+      notify: restart consul-template-vault
+
+    - name: Deploy the update cert hook
+      template: src=update_nomad_cert.j2 dest={{ vault_root_dir }}/bin/update_nomad_cert mode=755
+      notify: restart consul-template-vault
+
+  tags: vault

roles/vault/tasks/directories.yml

@@ -44,4 +44,5 @@
       owner: root
       group: "{{ vault_user }}"
       mode: 750
+    - dir: consul-template
   tags: vault

roles/vault/tasks/install.yml

@@ -6,6 +6,7 @@
       - tar
       - zstd
       - unzip
+      - jq
   tags: vault
 
 - when: vault_install_mode != 'none'
@@ -48,9 +49,16 @@
   notify: restart vault
   tags: vault
 
+- name: Install consul-template unit
+  template: src=consul-template-vault.service.j2 dest=/etc/systemd/system/consul-template-vault.service
+  notify: restart consul-template-vault
+  when: vault_secrets.nomad.enabled
+  register: vault_secrets_nomad_unit
+  tags: vault
+
 - name: Reload systemd
   systemd: daemon_reload=True
-  when: vault_unit.changed
+  when: vault_unit.changed or (vault_secrets_nomad_unit is defined and vault_secrets_nomad_unit.changed)
   tags: vault
 
 - name: Install dehydrated hook
@@ -65,3 +73,4 @@
     dest: /etc/profile.d/vault.sh
     mode: 0755
   tags: vault
+

roles/vault/tasks/services.yml

@@ -4,3 +4,10 @@
   service: name=vault state=started enabled=True
   register: vault_service_started
   tags: vault
+
+- name: Handle consul-template-vault service
+  service:
+    name: consul-template-vault
+    state: "{{ vault_secrets.nomad.enabled | ternary('started', 'stopped') }}"
+    enabled: "{{ vault_secrets.nomad.enabled | ternary(True, False) }}"
+  tags: vault

roles/vault/templates/consul-template-vault.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description="HashiCorp consul-template"
+Documentation=https://github.com/hashicorp/consul-template
+Requires=network-online.target
+After=network-online.target
+After=vault.service
+ConditionFileNotEmpty={{ vault_root_dir }}/consul-template/consul-template.hcl
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/consul-template -config={{ vault_root_dir }}/consul-template/consul-template.hcl
+SuccessExitStatus=12
+ExecReload=/bin/kill --signal HUP $MAINPID
+KillSignal=SIGINT
+Restart=on-failure
+RestartSec=2
+
+[Install]
+WantedBy=multi-user.target
+

roles/vault/templates/consul-template.hcl.j2

@@ -0,0 +1,18 @@
+vault {
+  address      = "{{ vault_secrets.vault_address }}"
+  token        = "{{ vault_secrets.vault_token }}"
+  unwrap_token = false
+}
+
+{% if vault_secrets.nomad.enabled %}
+template {
+  source          = "{{ vault_root_dir }}/consul-template/nomad_client_bundle.json.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ vault_root_dir }}/tmp/nomad_client_bundle.json"
+  perms           = 0600
+  exec {
+    command = "{{ vault_root_dir }}/bin/update_nomad_cert {{ vault_secrets.nomad.token }} {{ vault_secrets.vault_token }}"
+  }
+}
+{% endif %}

roles/vault/templates/nomad_client_bundle.json.tpl.j2

@@ -0,0 +1,3 @@
+[[ with secret "{{ vault_secrets.nomad.pki.path }}/issue/{{ vault_secrets.nomad.pki.role }}" "ttl={{ vault_secrets.nomad.pki.ttl }}" "common_name={{ vault_secrets.nomad.pki.cn }}" ]]
+[[ .Data | toJSONPretty ]]
+[[ end ]]

roles/vault/templates/update_nomad_cert.j2

@@ -0,0 +1,29 @@
+#!/bin/sh
+  
+set -eo pipefail
+
+NOMAD_TOKEN=$1
+VAULT_TOKEN=$2
+
+NOMAD_CERT_BUNDLE={{ vault_root_dir }}/tmp/nomad_client_bundle.json
+VAULT_ADDR={{ vault_conf.api_addr }}
+
+if [ "$(vault status -format=json| jq .is_self)" != "true" ]; then
+  echo "We're not the active vault, exiting"
+elif [ "$(vault status -format=json | jq .sealed)" != "false" ]; then
+  echo "Vault is sealed, exiting"
+elif [ "$(vault status -format=json | jq .initialized)" != "true" ]; then
+  echo "Vault is not initialized yet, exiting"
+else
+  echo Updating Vault certificate to access Nomad API
+  VAULT_TOKEN=$VAULT_TOKEN \
+    vault write {{ vault_secrets.nomad.secret.path }}/config/access \
+      address="{{ vault_secrets.nomad.address }}" \
+      token="$NOMAD_TOKEN" \
+      ca_cert="$(cat $NOMAD_CERT_BUNDLE | jq -r .issuing_ca)" \
+      client_cert="$(cat $NOMAD_CERT_BUNDLE | jq -r .certificate)" \
+      client_key="$(cat $NOMAD_CERT_BUNDLE | jq -r .private_key)"
+fi
+
+echo Removing Nomad client certificate from the filesystem
+rm -f $NOMAD_CERT_BUNDLE