Update to 2022-09-04 18:00

roles/consul/defaults/main.yml

@@ -102,6 +102,9 @@ consul_base_conf:
     # TLS settings for interal RPC
     internal_rpc:
       verify_server_hostname: True
+    grpc:
+      # verify_incoming must be disabled on the gRPC endpoint for the envoy proxies when using Consul Connect !
+      verify_incoming: False
 
 consul_extra_conf: {}
 consul_host_conf: {}

roles/consul/templates/agent_cert.tpl.j2

@@ -1,5 +1,5 @@
 {% if consul_conf.server %}
-[[ with secret "{{ consul_vault_tls.pki.path }}/issue/{{ consul_vault_tls.pki.role }}" "common_name={{ consul_conf.server | ternary('server', 'client') }}-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" "ttl={{ consul_vault_tls.pki.ttl }}" "alt_names=localhost,{{ consul_conf.server | ternary('server', 'client') }}.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" ]]
+[[ with secret "{{ consul_vault_tls.pki.path }}/issue/{{ consul_vault_tls.pki.role }}" "common_name=server-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" "ttl={{ consul_vault_tls.pki.ttl }}" "alt_names=localhost,consul.service.{{ consul_conf.domain | default('consul') }},server.{{ consul_conf.datacenter | default('dc1') }}.{{ consul_conf.domain | default('consul') }}" ]]
 [[ .Data.{{ item.what }} ]]
 [[ end ]]
 {% else %}

roles/nomad/defaults/main.yml

@@ -24,6 +24,7 @@ nomad_root_dir: /opt/nomad
 nomad_user: "{{ nomad_conf.client.enabled | ternary('root', 'nomad') }}"
 
 # List of Unix group which will be nomad admins
+# Used for example to grant access to cli cert with ACL
 nomad_admin_groups: "{{ system_admin_groups | default([]) }}"
 
 # If ACL are enabled, you need to set a management token for ansible
@@ -179,6 +180,10 @@ nomad_base_conf:
     # address: http://localhost:8500
     # allow_unauthenticated: True
     # tags: []
+    ssl: "{{ (consul_conf is defined and consul_conf.tls is defined and consul_conf.tls.enabled is defined and consul_conf.tls.enabled) | ternary(True, False) }}"
+    ca_file: "{{ nomad_root_dir }}/tls/consul_ca.crt"
+    cert_file: "{{ nomad_root_dir }}/tls/consul.crt"
+    key_file: "{{ nomad_root_dir }}/tls/consul.key"
 
   # Vault integration
   vault:
@@ -233,6 +238,12 @@ nomad_base_vault_tls:
     path: /pki/nomad
     role: nomad-{{ nomad_conf.server.enabled | ternary('server', 'client') }}
     ttl: 24h
+  consul_pki:
+    path: /pki/consul
+    role: nomad-client # Only nomad clients will use the gRPC endpoint and requires a client cert for consul
+    ttl: 24h
+  root_pki:
+    path: /pki/root # This is used to fetch the top level root CA, as envoy can't validate the chain unless it has it complete up to the auto-signed root
 
 nomad_extra_vault_tls: {}
 nomad_host_vault_tls: {}

roles/nomad/tasks/conf.yml

@@ -19,6 +19,27 @@
   when: not nomad_ca_file.stat.exists
   tags: nomad
 
+- when: nomad_conf.consul.ca_file is defined
+  block:
+    - name: Generate self-signed certificate
+      import_tasks: ../includes/create_selfsigned_cert.yml
+      vars:
+        cert_path: "{{ nomad_conf.consul.cert_file }}"
+        cert_key_path: "{{ nomad_conf.consul.key_file }}"
+        cert_key_group: "{{ nomad_user }}"
+        cert_key_mode: 640
+      tags: nomad
+
+    - name: Check if CA exists
+      stat: path={{ nomad_conf.tls.ca_file }}
+      register: nomad_consul_ca_file
+  tags: nomad
+
+- name: Copy consul cert as consul CA
+  copy: src={{ nomad_conf.consul.cert_file }} dest={{ nomad_conf.consul.ca_file }} remote_src=True
+  when: nomad_conf.consul.ca_file is defined and not nomad_consul_ca_file.stat.exists
+  tags: nomad
+
 - name: Deploy nomad configuration
   block:
     - name: Deploy nomad configuration
@@ -140,3 +161,17 @@
 - name: Deploy profile script
   template: src=profile.sh.j2 dest=/etc/profile.d/nomad.sh
   tags: nomad
+
+- name: Deploy consul-template consul cert templates
+  template: src=consul_cert.tpl.j2 dest={{ nomad_root_dir }}/consul-template/{{ item.where }} owner=root group=root
+  loop:
+    - what: certificate
+      where: consul.crt.tpl
+    - what: private_key
+      where: consul.key.tpl
+    - what: issuing_ca
+      where: consul_ca.crt.tpl
+  notify: restart consul-template-nomad
+  when: nomad_conf.client.enabled and nomad_conf.consul.ssl
+  tags: nomad
+

roles/nomad/tasks/install.yml

@@ -6,6 +6,7 @@
       - tar
       - zstd
       - unzip
+      - acl
   tags: nomad
 
 - name: Install task driver packages

roles/nomad/templates/agent_cert.tpl.j2

@@ -1,3 +1,3 @@
-[[ with secret "{{ nomad_vault_tls.pki.path }}/issue/{{ nomad_vault_tls.pki.role }}" "common_name={{ (nomad_conf.server.enabled) | ternary('server', 'client') }}-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ nomad_conf.region | default('global') }}.nomad" "ttl={{ nomad_vault_tls.pki.ttl }}" "alt_names=localhost,{{ (nomad_conf.server.enabled) | ternary('server', 'client') }}.{{ nomad_conf.region | default('global') }}.nomad{% if nomad_conf.server.enabled and nomad_conf.client.enabled %},client.{{ nomad_conf.region | default('global') }}.nomad{% endif %}" ]]
+[[ with secret "{{ nomad_vault_tls.pki.path }}/issue/{{ nomad_vault_tls.pki.role }}" "common_name={{ (nomad_conf.server.enabled) | ternary('server', 'client') }}-{{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ nomad_conf.region | default('global') }}.nomad" "ttl={{ nomad_vault_tls.pki.ttl }}" "alt_names=localhost,{{ (nomad_conf.server.enabled) | ternary('server', 'client') }}.{{ nomad_conf.region | default('global') }}.nomad{% if nomad_conf.server.enabled and nomad_conf.client.enabled %},client.{{ nomad_conf.region | default('global') }}.nomad{% endif %}{% if consul_conf is defined %},nomad{{ nomad_conf.server.enabled | ternary('', '-client') }}.service.{{ consul_conf.domain | default('consul') }}{% endif %}" ]]
 [[ .Data.{{ item.what }} ]]
 [[ end ]]

roles/nomad/templates/consul-template.hcl.j2

@@ -35,7 +35,6 @@ template {
   exec {
     command = "systemctl reload nomad"
   }
-
 }
 
 {% if nomad_conf.server.enabled %}
@@ -54,3 +53,32 @@ template {
   perms           = 0640
 }
 {% endif %}
+
+{% if nomad_conf.client.enabled and nomad_conf.consul.ssl %}
+template {
+  source          = "{{ nomad_root_dir }}/consul-template/consul.crt.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ nomad_conf.consul.cert_file }}"
+}
+
+template {
+  source          = "{{ nomad_root_dir }}/consul-template/consul.key.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ nomad_conf.consul.key_file }}"
+  perms           = 0640
+}
+
+template {
+  source          = "{{ nomad_root_dir }}/consul-template/consul_ca.crt.tpl"
+  left_delimiter  = "[["
+  right_delimiter = "]]"
+  destination     = "{{ nomad_conf.consul.ca_file }}"
+  perms           = 0644
+  exec {
+    command = "systemctl reload nomad"
+  }
+}
+
+{% endif %}

roles/nomad/templates/consul_cert.tpl.j2

@@ -0,0 +1,8 @@
+[[ with secret "{{ nomad_vault_tls.consul_pki.path }}/issue/{{ nomad_vault_tls.consul_pki.role }}" "common_name={{ ansible_fqdn | regex_replace('\\.', '-') }}.{{ (consul_conf is defined and consul_conf.domain is defined) | ternary(consul_conf.domain, 'consul') }}" "ttl={{ nomad_vault_tls.consul_pki.ttl }}" ]]
+[[ .Data.{{ item.what }} ]]
+[[ end ]]
+{% if item.what == 'issuing_ca' %}
+[[ with secret "{{ nomad_vault_tls.root_pki.path }}/cert/ca" ]]
+[[ .Data.certificate ]]
+[[ end ]]
+{% endif %}

roles/nomad/templates/nomad.hcl.j2

@@ -167,18 +167,40 @@ telemetry {
 }
 
 consul {
-{% if nomad_conf.consul.address is defined %}
-  address = "{{ nomad_conf.consul.address }}"
+{% for key in ['address', 'auth', 'client_service_name', 'server_service_name', 'grpc_address', 'token'] %}
+{% if nomad_conf.consul[key] is defined %}
+  {{ key }} = "{{ nomad_conf.consul[key] }}"
 {% endif %}
-{% if nomad_conf.consul.allow_unauthenticated is defined %}
-  allow_unauthenticated = {{ nomad_conf.consul.allow_unauthenticated | ternary('true', 'false') }}
+{% endfor %}
+
+{% for key in ['allow_unauthenticated', 'auto_advertise', 'checks_use_advertise', 'server_auto_join'] %}
+{% if nomad_conf.consul[key] is defined %}
+  {{ key }} = {{ nomad_conf.consul[key] | ternary('true', 'false') }}
 {% endif %}
+{% endfor %}
+
 {% if nomad_conf.consul.tags is defined and nomad_conf.consul.tags is iterable %}
   tags = [
 {% for tag in nomad_conf.consul.tags %}
-    "{{ tag }}"
-  ]
+    "{{ tag }}",
 {% endfor %}
+  ]
+{% endif %}
+
+{% if nomad_conf.consul.ssl %}
+  ssl = true
+{% for key in ['ca_file', 'cert_file', 'key_file'] %}
+{% if nomad_conf.consul[key] is defined %}
+  {{ key }} = "{{ nomad_conf.consul[key] }}"
+{% endif %}
+{% endfor %}
+
+{% for key in ['verify_ssl', 'share_ssl'] %}
+{% if nomad_conf.consul[key] is defined %}
+  {{ key }} = {{ nomad_conf.consul[key] | ternary('true', 'false') }}
+{% endif %}
+{% endfor %}
+
 {% endif %}
 }
 

roles/nomad/templates/nomad.service.j2

@@ -7,6 +7,13 @@ ConditionFileNotEmpty={{ nomad_root_dir }}/etc/nomad.hcl
 
 [Service]
 EnvironmentFile=-{{ nomad_root_dir }}/etc/nomad.env
+{% if nomad_conf.consul.ssl and nomad_conf.client.enabled %}
+Environment=CONSUL_HTTP_SSL=true
+Environment=CONSUL_HTTP_ADDR=localhost:{{ (consul_services is defined) | ternary(consul_services.https.port, '8501') }}
+Environment=CONSUL_CACERT={{ nomad_conf.consul.ca_file }}
+Environment=CONSUL_CLIENT_CERT={{ nomad_conf.consul.cert_file }}
+Environment=CONSUL_CLIENT_KEY={{ nomad_conf.consul.key_file }}
+{% endif %}
 User={{ nomad_user }}
 Group={{ nomad_user }}
 ExecStart={{ nomad_root_dir }}/bin/nomad agent -config={{ nomad_root_dir }}/etc/