HashiCorp Nomad is a flexible workload orchestrator supporting containers, VMs, and standalone binaries. Nomad security depends on ACLs for authorization, TLS/mTLS for encryption, Vault integration for secrets management, and secure cluster communication. This guide covers securing Nomad clusters, ACL policies, TLS configuration, workload isolation, and secrets handling.
Enable ACLs in production:
# /etc/nomad.d/nomad.hcl
acl {
enabled = true
token_ttl = "30d"
policy_ttl = "30d"
}
Bootstrap ACL system:
# Bootstrap ACL system (run once)
nomad acl bootstrap
# Save the bootstrap token securely
# Output will show:
# Accessor ID: 00000000-0000-0000-0000-000000000000
# Secret ID: <save-this-securely>
Store bootstrap token securely:
# Save to secure location
echo "<bootstrap-token>" > /etc/nomad.d/bootstrap-token
chmod 600 /etc/nomad.d/bootstrap-token
# Or export to environment
export NOMAD_TOKEN="<bootstrap-token>"
Create read-only policy:
# read-only-policy.hcl
namespace "default" {
policy = "read"
}
namespace "monitoring" {
policy = "read"
}
agent {
policy = "read"
}
node {
policy = "read"
}
operator {
policy = "read"
}
Create deployer policy:
# deployer-policy.hcl
namespace "default" {
policy = "write"
capabilities = ["submit-job", "dispatch-job", "read-logs"]
}
namespace "production" {
policy = "read"
}
agent {
policy = "read"
}
node {
policy = "read"
}
keyring {
policy = "none"
}
operator {
policy = "none"
}
sentinel {
policy = "none"
}
variable {
path = "*"
capabilities = ["read"]
}
Apply ACL policies:
# Create policy
nomad acl policy create \
-description "Read-only access" \
-job "*" \
read-only-policy.hcl
# Create policy for deployer
nomad acl policy create \
-description "Deployer access to default namespace" \
-job "*" \
deployer-policy.hcl
Create token with policy:
# Create token with specific policy
nomad acl token create \
-name "deployer-token" \
-policy "deployer-policy" \
-expire "720h" # 30 days
# Create token for CI/CD
nomad acl token create \
-name "ci-cd-token" \
-policy "deployer-policy" \
-expire "24h" # Short-lived for CI/CD
Token best practices:
Create namespace-specific policies:
# team-alpha-policy.hcl
namespace "team-alpha" {
policy = "write"
capabilities = ["submit-job", "dispatch-job", "read-logs"]
}
namespace "shared" {
policy = "read"
}
agent {
policy = "none"
}
node {
policy = "none"
}
Create namespaces:
# Create namespace
nomad namespace create team-alpha
# Apply policy to namespace
nomad acl policy create \
-description "Team Alpha namespace access" \
-namespace "team-alpha" \
team-alpha-policy.hcl
Generate TLS certificates:
# Create CA
openssl genrsa -out nomad-ca.key 4096
openssl req -x509 -new -nodes -sha256 -key nomad-ca.key \
-days 3650 -out nomad-ca.pem \
-subj "/C=US/ST=State/L=City/O=Organization/CN=Nomad CA"
# Generate server certificate
openssl genrsa -out nomad-server.key 2048
openssl req -new -sha256 -key nomad-server.key \
-out nomad-server.csr \
-subj "/C=US/ST=State/L=City/O=Organization/CN=server.nomad.example.com"
# Create certificate extensions
cat > server-ext.cnf << EOF
subjectAltName = DNS:server.nomad.example.com,DNS:localhost,IP:127.0.0.1,IP:192.168.1.100
EOF
# Sign server certificate
openssl x509 -req -sha256 -days 365 \
-in nomad-server.csr -CA nomad-ca.pem -CAkey nomad-ca.key -CAcreateserial \
-out nomad-server.pem -extfile server-ext.cnf
# Generate client certificate
openssl genrsa -out nomad-client.key 2048
openssl req -new -sha256 -key nomad-client.key \
-out nomad-client.csr \
-subj "/C=US/ST=State/L=City/O=Organization/CN=client.nomad.example.com"
# Sign client certificate
openssl x509 -req -sha256 -days 365 \
-in nomad-client.csr -CA nomad-ca.pem -CAkey nomad-ca.key -CAcreateserial \
-out nomad-client.pem
Configure TLS on servers:
# /etc/nomad.d/server.hcl
tls {
http = true
rpc = true
ca_file = "/etc/nomad.d/tls/nomad-ca.pem"
cert_file = "/etc/nomad.d/tls/nomad-server.pem"
key_file = "/etc/nomad.d/tls/nomad-server.key"
verify_server_hostname = true
verify_https_client = true
}
Configure TLS on clients:
# /etc/nomad.d/client.hcl
tls {
http = true
rpc = true
ca_file = "/etc/nomad.d/tls/nomad-ca.pem"
cert_file = "/etc/nomad.d/tls/nomad-client.pem"
key_file = "/etc/nomad.d/tls/nomad-client.key"
verify_server_hostname = true
}
Configure auto-encrypt for clients:
# /etc/nomad.d/server.hcl
server {
enabled = true
# Enable auto-encrypt for client certificates
auto_encrypt {
allow_registration = true
}
}
# /etc/nomad.d/client.hcl
client {
enabled = true
auto_encrypt {
tls = true
}
}
Verify TLS configuration:
# Check TLS status
nomad operator debug -output=/tmp/nomad-debug
# Verify server certificate
openssl s_client -connect server.nomad.example.com:4646 -showcerts
# Verify client certificate
nomad node status -tls-skip-verify
Generate gossip encryption key:
# Generate key
nomad keygen
# Output: Base64-encoded key (save securely)
# Example: pVejSYRVGQzFBvXzR8+3jQ==
Configure gossip encryption:
# /etc/nomad.d/server.hcl
server {
enabled = true
encrypt = "pVejSYRVGQzFBvXzR8+3jQ=="
}
# /etc/nomad.d/client.hcl
client {
enabled = true
encrypt = "pVejSYRVGQzFBvXzR8+3jQ=="
}
Rotate gossip encryption key:
# Generate new key
NEW_KEY=$(nomad keygen)
# Install new key on all servers
nomad keyring install -key=$NEW_KEY -type=server
# Verify key installation
nomad keyring list -type=server
# Remove old key
nomad keyring remove -key=$OLD_KEY -type=server
Secure certificate permissions:
# Set ownership
chown -R nomad:nomad /etc/nomad.d/tls
# Set permissions
chmod 600 /etc/nomad.d/tls/*.key
chmod 644 /etc/nomad.d/tls/*.pem
# Verify
ls -la /etc/nomad.d/tls/
Automate certificate rotation:
#!/bin/bash
# /usr/local/bin/rotate-nomad-certs.sh
# Generate new certificates
# ... (certificate generation commands)
# Install new certificates
cp nomad-server.pem /etc/nomad.d/tls/
cp nomad-server.key /etc/nomad.d/tls/
# Reload Nomad
systemctl reload nomad
# Log rotation
logger "Nomad certificates rotated"
Enable Vault in Nomad:
# /etc/nomad.d/server.hcl
vault {
enabled = true
address = "https://vault.example.com:8200"
ca_file = "/etc/nomad.d/tls/vault-ca.pem"
cert_file = "/etc/nomad.d/tls/nomad-vault.pem"
key_file = "/etc/nomad.d/tls/nomad-vault.key"
tls_server_name = "vault.example.com"
}
Create Vault token for Nomad:
# Create Vault policy for Nomad
vault policy write nomad-server - <<EOF
path "sys/capabilities" {
capabilities = ["update"]
}
path "sys/capabilities-self" {
capabilities = ["update"]
}
path "auth/token/create" {
capabilities = ["update"]
}
path "auth/token/create/*" {
capabilities = ["update"]
}
path "auth/token/lookup/*" {
capabilities = ["read"]
}
path "auth/token/renew" {
capabilities = ["update"]
}
path "auth/token/renew-self" {
capabilities = ["update"]
}
path "auth/token/revoke-accessor" {
capabilities = ["update"]
}
path "auth/token/revoke-self" {
capabilities = ["update"]
}
path "identity/lookup-entity" {
capabilities = ["update"]
}
path "identity/lookup-group" {
capabilities = ["update"]
}
EOF
# Create token
vault token create -policy nomad-server -period 72h
Reference Vault secrets in job spec:
# job.hcl
job "webapp" {
datacenters = ["dc1"]
group "web" {
count = 3
task "web" {
driver = "docker"
config {
image = "webapp:1.0"
ports = ["http"]
}
# Vault template for secrets
template {
data = <<-EOT
DB_HOST={{ with secret "secret/data/database" }}{{ .Data.data.host }}{{ end }}
DB_USER={{ with secret "secret/data/database" }}{{ .Data.data.username }}{{ end }}
DB_PASS={{ with secret "secret/data/database" }}{{ .Data.data.password }}{{ end }}
EOT
destination = "secrets/db.env"
env = true
}
env {
DB_HOST = "${DB_HOST}"
DB_USER = "${DB_USER}"
DB_PASS = "${DB_PASS}"
}
}
}
}
Create Vault secrets:
# Enable KV secrets engine
vault secrets enable -path=secret kv-v2
# Store database credentials
vault kv put secret/data/database \
host=db.example.com \
username=webapp \
password=secure-password
Never hardcode secrets in job specs:
# BAD - Don't do this
job "webapp" {
task "web" {
env {
DB_PASSWORD = "hardcoded-password" # ⚠️ Security risk!
}
}
}
# GOOD - Use Vault
job "webapp" {
task "web" {
template {
data = <<-EOT
DB_PASSWORD={{ with secret "secret/data/database" }}{{ .Data.data.password }}{{ end }}
EOT
destination = "secrets/db.env"
env = true
}
}
}
Configure allowed drivers:
# /etc/nomad.d/client.hcl
client {
enabled = true
options = {
"driver.blacklist" = "java,exec" # Disable dangerous drivers
}
# Or whitelist specific drivers
options = {
"driver.whitelist" = "docker,exec"
}
}
Define allowed host volumes:
# /etc/nomad.d/client.hcl
client {
enabled = true
host_volume "logs" {
path = "/var/log/nomad"
read_only = true
}
host_volume "data" {
path = "/var/nomad/data"
read_only = false
}
}
Reference host volumes in jobs:
job "webapp" {
task "web" {
driver = "docker"
volume_mount {
volume = "logs"
destination = "/var/log/app"
read_only = true
}
}
volume "logs" {
type = "host"
read_only = true
source = "logs"
}
}
Configure node classes:
# /etc/nomad.d/client.hcl
client {
enabled = true
node_class = "production"
meta {
environment = "production"
trust_zone = "high"
}
}
Constraint jobs to node classes:
job "webapp" {
group "web" {
constraint {
attribute = "${node.class}"
operator = "="
value = "production"
}
constraint {
attribute = "${meta.trust_zone}"
operator = "="
value = "high"
}
}
}
Create Sentinel policy for job restrictions:
# restrict-privileged.sentinel
import "nomad"
# Deny privileged containers
main = rule {
all nomad.tasks as _, task {
task.config.privileged is not true
}
}
Apply Sentinel policy:
# Upload policy
sentinel apply restrict-privileged \
-description "Deny privileged containers" \
-enforcement-level advisory
# Apply to specific namespace
sentinel apply restrict-privileged \
-namespace production \
-enforcement-level hard-mandatory
Configure audit devices:
# /etc/nomad.d/server.hcl
audit {
type = "file"
path = "/var/log/nomad/audit.log"
}
audit {
type = "syslog"
facility = "LOCAL0"
}
View audit logs:
# View audit log
tail -f /var/log/nomad/audit.log
# Filter for specific events
grep "token.create" /var/log/nomad/audit.log
grep "job.register" /var/log/nomad/audit.log
Check cluster status:
# Check server status
nomad server members
# Check node status
nomad node status
# Check job status
nomad job status
# Check deployment status
nomad deployment status
Monitor for security events:
# Watch for failed authentications
grep "authentication failed" /var/log/nomad/nomad.log
# Watch for ACL denials
grep "permission denied" /var/log/nomad/nomad.log
# Watch for TLS errors
grep "TLS" /var/log/nomad/nomad.log
Configure Prometheus metrics:
# /etc/nomad.d/server.hcl
telemetry {
prometheus_metrics = true
disable_hostname = true
enable_hostname_label = true
}
Create security alerts:
# prometheus-rules.yaml
groups:
- name: nomad-security
rules:
- alert: NomadACLAuthFailure
expr: rate(nomad_client_acl_auth_failure[5m]) > 0.1
for: 5m
labels:
severity: warning
annotations:
summary: "High ACL authentication failure rate"
- alert: NomadJobSubmissionFailure
expr: rate(nomad_client_job_register_failure[5m]) > 0.1
for: 5m
labels:
severity: warning
annotations:
summary: "High job submission failure rate"
| Control | Status | Notes |
|---|---|---|
| ACLs enabled | ☐ | Production clusters only |
| TLS for HTTP/RPC | ☐ | All server/client communication |
| mTLS enabled | ☐ | Server-client authentication |
| Gossip encryption | ☐ | Serf/Raft channels |
| Vault integration | ☐ | Dynamic secrets |
| No plaintext secrets | ☐ | All secrets via Vault |
| Task drivers restricted | ☐ | Whitelist only needed |
| Host volumes defined | ☐ | Explicit allow list |
| Node isolation | ☐ | By environment/trust zone |
| Audit logging enabled | ☐ | File and/or syslog |
| Certificate rotation | ☐ | Automated process |
| Token rotation | ☐ | Regular schedule |
| Sentinel policies | ☐ | Job restrictions |
| Monitoring configured | ☐ | Security alerts |
Check Nomad security status:
# Check ACL status
nomad acl status
# Check ACL policies
nomad acl policy list
# Check ACL tokens
nomad acl token list
# Check TLS configuration
nomad operator debug -output=/tmp/nomad-debug
# Check gossip encryption
nomad keyring list -type=server
nomad keyring list -type=client
Audit security configuration:
# Check server configuration
grep -E "acl|tls|encrypt|vault" /etc/nomad.d/server.hcl
# Check client configuration
grep -E "acl|tls|encrypt|driver|volume" /etc/nomad.d/client.hcl
# Check certificate permissions
ls -la /etc/nomad.d/tls/
# Check token expiration
nomad acl token self