Refactor HAProxy configuration and certificate management
- Updated HAProxy configuration template to reflect changes for the Taurus Production Environment, including SSL settings and rate limiting for specific endpoints. - Introduced new playbooks for certificate distribution and validation with OCI Vault, ensuring certificates are correctly managed and renewed. - Added hooks for uploading renewed certificates to OCI Vault and validating their integrity. - Enhanced the HAProxy configuration playbook to ensure proper service management and verification of the HAProxy service. - Updated inventory variables for certificate management and ensured compatibility with the new structure.
This commit is contained in:
340
ansible/certbot/vault-validate.yml
Normal file
340
ansible/certbot/vault-validate.yml
Normal file
@@ -0,0 +1,340 @@
|
||||
---
|
||||
# -----------------------------------------------------------------------------
|
||||
# Vault Secret Validation Playbook
|
||||
# -----------------------------------------------------------------------------
|
||||
# Tests the full round-trip of OCI Vault secret read/write for all certbot
|
||||
# domains. Use this BEFORE running certbot to verify vault connectivity
|
||||
# and permissions without burning Let's Encrypt rate limits.
|
||||
#
|
||||
# Run from fornax:
|
||||
# ansible-playbook certbot/vault-validate.yml
|
||||
#
|
||||
# What it does:
|
||||
# 1. Verifies every expected vault secret exists and is readable
|
||||
# 2. Writes a unique test value to each secret
|
||||
# 3. Reads back and compares to confirm the write path works
|
||||
# 4. Restores the original content
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
- name: Validate OCI Vault certificate secrets
|
||||
hosts: localhost
|
||||
gather_facts: false
|
||||
environment:
|
||||
PATH: "/srv/ponos/.local/bin:/usr/local/bin:/usr/bin:/bin"
|
||||
HOME: "/srv/ponos"
|
||||
vars:
|
||||
# Must match the keys in taurus/terraform/bootes_certificates.tf
|
||||
cert_prefixes:
|
||||
- apollo-helu-ca
|
||||
- bootes-helu-ca
|
||||
- corvus-helu-ca
|
||||
- draco-helu-ca
|
||||
- iris-helu-ca
|
||||
- korax-helu-ca
|
||||
- nyx-helu-ca
|
||||
- orpheus-helu-ca
|
||||
- pan-helu-ca
|
||||
- perseus-helu-ca
|
||||
- wildcard-ouranos-helu-ca
|
||||
secret_suffixes:
|
||||
- fullchain
|
||||
- privkey
|
||||
|
||||
tasks:
|
||||
# -----------------------------------------------------------------
|
||||
# Phase 1: Verify all secrets exist and are readable
|
||||
# -----------------------------------------------------------------
|
||||
- name: Build list of all secret names
|
||||
ansible.builtin.set_fact:
|
||||
all_secret_names: "{{ cert_prefixes | product(secret_suffixes) | map('join', '-') | list }}"
|
||||
|
||||
- name: "Phase 1 — Read current value from vault"
|
||||
ansible.builtin.set_fact:
|
||||
"original_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}"
|
||||
loop: "{{ all_secret_names }}"
|
||||
register: phase1_read
|
||||
|
||||
- name: "Phase 1 — Confirm all secrets are readable"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- lookup('vars', 'original_' + item | replace('-', '_')) is defined
|
||||
- lookup('vars', 'original_' + item | replace('-', '_')) | length > 0
|
||||
fail_msg: "Secret '{{ item }}' is missing or empty in OCI Vault"
|
||||
success_msg: "✓ {{ item }}"
|
||||
loop: "{{ all_secret_names }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
|
||||
- name: "Phase 1 — Summary"
|
||||
ansible.builtin.debug:
|
||||
msg: "All {{ all_secret_names | length }} vault secrets exist and are readable."
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Phase 2: Look up secret OCIDs (needed for write operations)
|
||||
# -----------------------------------------------------------------
|
||||
- name: "Phase 2 — Look up secret OCID for each secret"
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- oci
|
||||
- vault
|
||||
- secret
|
||||
- list
|
||||
- --compartment-id
|
||||
- "{{ oci_govern_compartment_id }}"
|
||||
- --vault-id
|
||||
- "{{ oci_vault_id }}"
|
||||
- --name
|
||||
- "{{ item }}"
|
||||
- --lifecycle-state
|
||||
- ACTIVE
|
||||
- --query
|
||||
- "data[0].id"
|
||||
- --raw-output
|
||||
loop: "{{ all_secret_names }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
register: ocid_lookups
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 2 — Build secret OCID map"
|
||||
ansible.builtin.set_fact:
|
||||
secret_ocids: "{{ secret_ocids | default({}) | combine({item.item: item.stdout | trim}) }}"
|
||||
loop: "{{ ocid_lookups.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item }}"
|
||||
|
||||
- name: "Phase 2 — Verify all OCIDs resolved"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- secret_ocids[item] is defined
|
||||
- secret_ocids[item] | length > 0
|
||||
- secret_ocids[item] != "null"
|
||||
fail_msg: "Could not resolve OCID for secret '{{ item }}' — has terraform apply been run?"
|
||||
success_msg: "✓ {{ item }}"
|
||||
loop: "{{ all_secret_names }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
|
||||
- name: "Phase 2 — Summary"
|
||||
ansible.builtin.debug:
|
||||
msg: "Resolved OCIDs for all {{ all_secret_names | length }} secrets."
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Phase 3: Write a unique test value to each secret
|
||||
# -----------------------------------------------------------------
|
||||
- name: Generate a unique test marker
|
||||
ansible.builtin.set_fact:
|
||||
test_marker: "vault-validate-{{ lookup('pipe', 'date +%Y%m%dT%H%M%S') }}"
|
||||
|
||||
- name: "Phase 3 — Write test value to each secret"
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- oci
|
||||
- vault
|
||||
- secret
|
||||
- update-base64
|
||||
- --secret-id
|
||||
- "{{ secret_ocids[item] }}"
|
||||
- --secret-content-content
|
||||
- "{{ (test_marker + ':' + item) | b64encode }}"
|
||||
loop: "{{ all_secret_names }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
register: phase3_write
|
||||
changed_when: true
|
||||
|
||||
- name: "Phase 3 — Summary"
|
||||
ansible.builtin.debug:
|
||||
msg: "Wrote test values to {{ all_secret_names | length }} secrets."
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Phase 4: Read back and compare
|
||||
# -----------------------------------------------------------------
|
||||
- name: "Phase 4 — Read back each secret and verify"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- readback == expected
|
||||
fail_msg: >-
|
||||
MISMATCH on {{ item }}:
|
||||
expected '{{ expected }}'
|
||||
got '{{ readback }}'
|
||||
success_msg: "✓ {{ item }} round-trip OK"
|
||||
vars:
|
||||
expected: "{{ test_marker + ':' + item }}"
|
||||
readback: "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}"
|
||||
loop: "{{ all_secret_names }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
|
||||
- name: "Phase 4 — Summary"
|
||||
ansible.builtin.debug:
|
||||
msg: "All {{ all_secret_names | length }} secrets passed round-trip validation."
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Phase 5: Restore original content
|
||||
# -----------------------------------------------------------------
|
||||
- name: "Phase 5 — Restore original secret content"
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- oci
|
||||
- vault
|
||||
- secret
|
||||
- update-base64
|
||||
- --secret-id
|
||||
- "{{ secret_ocids[item] }}"
|
||||
- --secret-content-content
|
||||
- "{{ lookup('vars', 'original_' + item | replace('-', '_')) | b64encode }}"
|
||||
loop: "{{ all_secret_names }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
changed_when: true
|
||||
|
||||
- name: "Phase 5 — Summary"
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
Round-trip test passed — all {{ all_secret_names | length }} secrets verified.
|
||||
Original content restored.
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Phase 6: Validate certificate content in vault
|
||||
# -----------------------------------------------------------------
|
||||
- name: Create temporary directory for cert validation
|
||||
ansible.builtin.tempfile:
|
||||
state: directory
|
||||
prefix: vault-validate-
|
||||
register: validate_tmpdir
|
||||
|
||||
- name: "Phase 6 — Read fullchain secrets"
|
||||
ansible.builtin.set_fact:
|
||||
"fullchain_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item + '-fullchain', vault_id=oci_vault_id) }}"
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
|
||||
- name: "Phase 6 — Check fullchain is PEM formatted"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- lookup('vars', 'fullchain_' + item | replace('-', '_')) is search('-----BEGIN CERTIFICATE-----')
|
||||
- lookup('vars', 'fullchain_' + item | replace('-', '_')) is search('-----END CERTIFICATE-----')
|
||||
fail_msg: >-
|
||||
{{ item }}-fullchain does not contain a PEM certificate.
|
||||
Content starts with: {{ lookup('vars', 'fullchain_' + item | replace('-', '_'))[:60] }}
|
||||
success_msg: "✓ {{ item }}-fullchain is PEM formatted"
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
|
||||
- name: "Phase 6 — Write fullchain to temp files for openssl validation"
|
||||
ansible.builtin.copy:
|
||||
content: "{{ lookup('vars', 'fullchain_' + item | replace('-', '_')) }}"
|
||||
dest: "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem"
|
||||
mode: '0600'
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
|
||||
- name: "Phase 6 — Validate certificate is not expired"
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- openssl
|
||||
- x509
|
||||
- -in
|
||||
- "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem"
|
||||
- -checkend
|
||||
- "0"
|
||||
- -noout
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
register: cert_expiry_check
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 6 — Get certificate details"
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- openssl
|
||||
- x509
|
||||
- -in
|
||||
- "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem"
|
||||
- -noout
|
||||
- -subject
|
||||
- -enddate
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
register: cert_details
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 6 — Display certificate status"
|
||||
ansible.builtin.debug:
|
||||
msg: "✓ {{ item.item }}: {{ item.stdout }}"
|
||||
loop: "{{ cert_details.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item }}"
|
||||
|
||||
- name: "Phase 6 — Read privkey secrets"
|
||||
ansible.builtin.set_fact:
|
||||
"privkey_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item + '-privkey', vault_id=oci_vault_id) }}"
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
no_log: true
|
||||
|
||||
- name: "Phase 6 — Check privkey is PEM formatted"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- lookup('vars', 'privkey_' + item | replace('-', '_')) is search('-----BEGIN .*(PRIVATE KEY)-----')
|
||||
- lookup('vars', 'privkey_' + item | replace('-', '_')) is search('-----END .*(PRIVATE KEY)-----')
|
||||
fail_msg: "{{ item }}-privkey does not contain a PEM private key"
|
||||
success_msg: "✓ {{ item }}-privkey is PEM formatted"
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
|
||||
- name: "Phase 6 — Write privkey to temp files for modulus check"
|
||||
ansible.builtin.copy:
|
||||
content: "{{ lookup('vars', 'privkey_' + item | replace('-', '_')) }}"
|
||||
dest: "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem"
|
||||
mode: '0600'
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
no_log: true
|
||||
|
||||
- name: "Phase 6 — Verify private key matches certificate"
|
||||
ansible.builtin.shell: |
|
||||
set -euo pipefail
|
||||
cert_mod=$(openssl x509 -noout -modulus -in "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" | openssl md5)
|
||||
key_mod=$(openssl rsa -noout -modulus -in "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5 || \
|
||||
openssl ec -noout -text -in "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5)
|
||||
if [[ "${cert_mod}" != "${key_mod}" ]]; then
|
||||
echo "MISMATCH: cert=${cert_mod} key=${key_mod}"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: modulus match"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
loop: "{{ cert_prefixes }}"
|
||||
loop_control:
|
||||
label: "{{ item }}"
|
||||
register: modulus_check
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 6 — Display key match results"
|
||||
ansible.builtin.debug:
|
||||
msg: "✓ {{ item.item }}: cert/key pair verified"
|
||||
loop: "{{ modulus_check.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item }}"
|
||||
|
||||
- name: Clean up temporary directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ validate_tmpdir.path }}"
|
||||
state: absent
|
||||
|
||||
- name: Final summary
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
Validation complete:
|
||||
{{ all_secret_names | length }} vault secrets — read/write round-trip OK.
|
||||
{{ cert_prefixes | length }} certificates — valid PEM, not expired, key pairs match.
|
||||
Reference in New Issue
Block a user