--- # ----------------------------------------------------------------------------- # Vault Secret Validation Playbook # ----------------------------------------------------------------------------- # Tests the full round-trip of OCI Vault secret read/write for all certbot # domains. Use this BEFORE running certbot to verify vault connectivity # and permissions without burning Let's Encrypt rate limits. # # Run from fornax: # ansible-playbook certbot/vault-validate.yml # # What it does: # 1. Verifies every expected vault secret exists and is readable # 2. Writes a unique test value to each secret # 3. Reads back and compares to confirm the write path works # 4. Restores the original content # ----------------------------------------------------------------------------- - name: Validate OCI Vault certificate secrets hosts: localhost gather_facts: false environment: PATH: "/srv/ponos/.local/bin:/usr/local/bin:/usr/bin:/bin" HOME: "/srv/ponos" vars: # Must match the keys in taurus/terraform/bootes_certificates.tf cert_prefixes: - apollo-helu-ca - bootes-helu-ca - corvus-helu-ca - draco-helu-ca - iris-helu-ca - korax-helu-ca - nyx-helu-ca - orpheus-helu-ca - pan-helu-ca - perseus-helu-ca - wildcard-ouranos-helu-ca secret_suffixes: - fullchain - privkey tasks: # ----------------------------------------------------------------- # Phase 1: Verify all secrets exist and are readable # ----------------------------------------------------------------- - name: Build list of all secret names ansible.builtin.set_fact: all_secret_names: "{{ cert_prefixes | product(secret_suffixes) | map('join', '-') | list }}" - name: "Phase 1 — Read current value from vault" ansible.builtin.set_fact: "original_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}" loop: "{{ all_secret_names }}" register: phase1_read - name: "Phase 1 — Confirm all secrets are readable" ansible.builtin.assert: that: - lookup('vars', 'original_' + item | replace('-', '_')) is defined - lookup('vars', 'original_' + item | replace('-', '_')) | length > 0 fail_msg: "Secret '{{ item }}' is missing or empty in OCI Vault" success_msg: "✓ {{ item }}" loop: "{{ all_secret_names }}" loop_control: label: "{{ item }}" - name: "Phase 1 — Summary" ansible.builtin.debug: msg: "All {{ all_secret_names | length }} vault secrets exist and are readable." # ----------------------------------------------------------------- # Phase 2: Look up secret OCIDs (needed for write operations) # ----------------------------------------------------------------- - name: "Phase 2 — Look up secret OCID for each secret" ansible.builtin.command: argv: - oci - vault - secret - list - --compartment-id - "{{ oci_govern_compartment_id }}" - --vault-id - "{{ oci_vault_id }}" - --name - "{{ item }}" - --lifecycle-state - ACTIVE - --query - "data[0].id" - --raw-output loop: "{{ all_secret_names }}" loop_control: label: "{{ item }}" register: ocid_lookups changed_when: false - name: "Phase 2 — Build secret OCID map" ansible.builtin.set_fact: secret_ocids: "{{ secret_ocids | default({}) | combine({item.item: item.stdout | trim}) }}" loop: "{{ ocid_lookups.results }}" loop_control: label: "{{ item.item }}" - name: "Phase 2 — Verify all OCIDs resolved" ansible.builtin.assert: that: - secret_ocids[item] is defined - secret_ocids[item] | length > 0 - secret_ocids[item] != "null" fail_msg: "Could not resolve OCID for secret '{{ item }}' — has terraform apply been run?" success_msg: "✓ {{ item }}" loop: "{{ all_secret_names }}" loop_control: label: "{{ item }}" - name: "Phase 2 — Summary" ansible.builtin.debug: msg: "Resolved OCIDs for all {{ all_secret_names | length }} secrets." # ----------------------------------------------------------------- # Phase 3: Write a unique test value to each secret # ----------------------------------------------------------------- - name: Generate a unique test marker ansible.builtin.set_fact: test_marker: "vault-validate-{{ lookup('pipe', 'date +%Y%m%dT%H%M%S') }}" - name: "Phase 3 — Write test value to each secret" ansible.builtin.command: argv: - oci - vault - secret - update-base64 - --secret-id - "{{ secret_ocids[item] }}" - --secret-content-content - "{{ (test_marker + ':' + item) | b64encode }}" loop: "{{ all_secret_names }}" loop_control: label: "{{ item }}" register: phase3_write changed_when: true - name: "Phase 3 — Summary" ansible.builtin.debug: msg: "Wrote test values to {{ all_secret_names | length }} secrets." # ----------------------------------------------------------------- # Phase 4: Read back and compare # ----------------------------------------------------------------- - name: "Phase 4 — Read back each secret and verify" ansible.builtin.assert: that: - readback == expected fail_msg: >- MISMATCH on {{ item }}: expected '{{ expected }}' got '{{ readback }}' success_msg: "✓ {{ item }} round-trip OK" vars: expected: "{{ test_marker + ':' + item }}" readback: "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}" loop: "{{ all_secret_names }}" loop_control: label: "{{ item }}" - name: "Phase 4 — Summary" ansible.builtin.debug: msg: "All {{ all_secret_names | length }} secrets passed round-trip validation." # ----------------------------------------------------------------- # Phase 5: Restore original content # ----------------------------------------------------------------- - name: "Phase 5 — Restore original secret content" ansible.builtin.command: argv: - oci - vault - secret - update-base64 - --secret-id - "{{ secret_ocids[item] }}" - --secret-content-content - "{{ lookup('vars', 'original_' + item | replace('-', '_')) | b64encode }}" loop: "{{ all_secret_names }}" loop_control: label: "{{ item }}" changed_when: true - name: "Phase 5 — Summary" ansible.builtin.debug: msg: >- Round-trip test passed — all {{ all_secret_names | length }} secrets verified. Original content restored. # ----------------------------------------------------------------- # Phase 6: Validate certificate content in vault # ----------------------------------------------------------------- - name: Create temporary directory for cert validation ansible.builtin.tempfile: state: directory prefix: vault-validate- register: validate_tmpdir - name: "Phase 6 — Read fullchain secrets" ansible.builtin.set_fact: "fullchain_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item + '-fullchain', vault_id=oci_vault_id) }}" loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" - name: "Phase 6 — Check fullchain is PEM formatted" ansible.builtin.assert: that: - lookup('vars', 'fullchain_' + item | replace('-', '_')) is search('-----BEGIN CERTIFICATE-----') - lookup('vars', 'fullchain_' + item | replace('-', '_')) is search('-----END CERTIFICATE-----') fail_msg: >- {{ item }}-fullchain does not contain a PEM certificate. Content starts with: {{ lookup('vars', 'fullchain_' + item | replace('-', '_'))[:60] }} success_msg: "✓ {{ item }}-fullchain is PEM formatted" loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" - name: "Phase 6 — Write fullchain to temp files for openssl validation" ansible.builtin.copy: content: "{{ lookup('vars', 'fullchain_' + item | replace('-', '_')) }}" dest: "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" mode: '0600' loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" - name: "Phase 6 — Validate certificate is not expired" ansible.builtin.command: argv: - openssl - x509 - -in - "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" - -checkend - "0" - -noout loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" register: cert_expiry_check changed_when: false - name: "Phase 6 — Get certificate details" ansible.builtin.command: argv: - openssl - x509 - -in - "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" - -noout - -subject - -enddate loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" register: cert_details changed_when: false - name: "Phase 6 — Display certificate status" ansible.builtin.debug: msg: "✓ {{ item.item }}: {{ item.stdout }}" loop: "{{ cert_details.results }}" loop_control: label: "{{ item.item }}" - name: "Phase 6 — Read privkey secrets" ansible.builtin.set_fact: "privkey_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item + '-privkey', vault_id=oci_vault_id) }}" loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" no_log: true - name: "Phase 6 — Check privkey is PEM formatted" ansible.builtin.assert: that: - lookup('vars', 'privkey_' + item | replace('-', '_')) is search('-----BEGIN .*(PRIVATE KEY)-----') - lookup('vars', 'privkey_' + item | replace('-', '_')) is search('-----END .*(PRIVATE KEY)-----') fail_msg: "{{ item }}-privkey does not contain a PEM private key" success_msg: "✓ {{ item }}-privkey is PEM formatted" loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" - name: "Phase 6 — Write privkey to temp files for modulus check" ansible.builtin.copy: content: "{{ lookup('vars', 'privkey_' + item | replace('-', '_')) }}" dest: "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" mode: '0600' loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" no_log: true - name: "Phase 6 — Verify private key matches certificate" ansible.builtin.shell: | set -euo pipefail cert_mod=$(openssl x509 -noout -modulus -in "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" | openssl md5) key_mod=$(openssl rsa -noout -modulus -in "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5 || \ openssl ec -noout -text -in "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5) if [[ "${cert_mod}" != "${key_mod}" ]]; then echo "MISMATCH: cert=${cert_mod} key=${key_mod}" exit 1 fi echo "OK: modulus match" args: executable: /bin/bash loop: "{{ cert_prefixes }}" loop_control: label: "{{ item }}" register: modulus_check changed_when: false - name: "Phase 6 — Display key match results" ansible.builtin.debug: msg: "✓ {{ item.item }}: cert/key pair verified" loop: "{{ modulus_check.results }}" loop_control: label: "{{ item.item }}" - name: Clean up temporary directory ansible.builtin.file: path: "{{ validate_tmpdir.path }}" state: absent - name: Final summary ansible.builtin.debug: msg: >- Validation complete: {{ all_secret_names | length }} vault secrets — read/write round-trip OK. {{ cert_prefixes | length }} certificates — valid PEM, not expired, key pairs match.