Robert Helewka
0a053c1cd6
- Updated HAProxy configuration template to reflect changes for the Taurus Production Environment, including SSL settings and rate limiting for specific endpoints. - Introduced new playbooks for certificate distribution and validation with OCI Vault, ensuring certificates are correctly managed and renewed. - Added hooks for uploading renewed certificates to OCI Vault and validating their integrity. - Enhanced the HAProxy configuration playbook to ensure proper service management and verification of the HAProxy service. - Updated inventory variables for certificate management and ensured compatibility with the new structure.
341 lines
12 KiB
YAML
341 lines
12 KiB
YAML
---
|
|
# -----------------------------------------------------------------------------
|
|
# Vault Secret Validation Playbook
|
|
# -----------------------------------------------------------------------------
|
|
# Tests the full round-trip of OCI Vault secret read/write for all certbot
|
|
# domains. Use this BEFORE running certbot to verify vault connectivity
|
|
# and permissions without burning Let's Encrypt rate limits.
|
|
#
|
|
# Run from fornax:
|
|
# ansible-playbook certbot/vault-validate.yml
|
|
#
|
|
# What it does:
|
|
# 1. Verifies every expected vault secret exists and is readable
|
|
# 2. Writes a unique test value to each secret
|
|
# 3. Reads back and compares to confirm the write path works
|
|
# 4. Restores the original content
|
|
# -----------------------------------------------------------------------------
|
|
|
|
- name: Validate OCI Vault certificate secrets
|
|
hosts: localhost
|
|
gather_facts: false
|
|
environment:
|
|
PATH: "/srv/ponos/.local/bin:/usr/local/bin:/usr/bin:/bin"
|
|
HOME: "/srv/ponos"
|
|
vars:
|
|
# Must match the keys in taurus/terraform/bootes_certificates.tf
|
|
cert_prefixes:
|
|
- apollo-helu-ca
|
|
- bootes-helu-ca
|
|
- corvus-helu-ca
|
|
- draco-helu-ca
|
|
- iris-helu-ca
|
|
- korax-helu-ca
|
|
- nyx-helu-ca
|
|
- orpheus-helu-ca
|
|
- pan-helu-ca
|
|
- perseus-helu-ca
|
|
- wildcard-ouranos-helu-ca
|
|
secret_suffixes:
|
|
- fullchain
|
|
- privkey
|
|
|
|
tasks:
|
|
# -----------------------------------------------------------------
|
|
# Phase 1: Verify all secrets exist and are readable
|
|
# -----------------------------------------------------------------
|
|
- name: Build list of all secret names
|
|
ansible.builtin.set_fact:
|
|
all_secret_names: "{{ cert_prefixes | product(secret_suffixes) | map('join', '-') | list }}"
|
|
|
|
- name: "Phase 1 — Read current value from vault"
|
|
ansible.builtin.set_fact:
|
|
"original_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}"
|
|
loop: "{{ all_secret_names }}"
|
|
register: phase1_read
|
|
|
|
- name: "Phase 1 — Confirm all secrets are readable"
|
|
ansible.builtin.assert:
|
|
that:
|
|
- lookup('vars', 'original_' + item | replace('-', '_')) is defined
|
|
- lookup('vars', 'original_' + item | replace('-', '_')) | length > 0
|
|
fail_msg: "Secret '{{ item }}' is missing or empty in OCI Vault"
|
|
success_msg: "✓ {{ item }}"
|
|
loop: "{{ all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Phase 1 — Summary"
|
|
ansible.builtin.debug:
|
|
msg: "All {{ all_secret_names | length }} vault secrets exist and are readable."
|
|
|
|
# -----------------------------------------------------------------
|
|
# Phase 2: Look up secret OCIDs (needed for write operations)
|
|
# -----------------------------------------------------------------
|
|
- name: "Phase 2 — Look up secret OCID for each secret"
|
|
ansible.builtin.command:
|
|
argv:
|
|
- oci
|
|
- vault
|
|
- secret
|
|
- list
|
|
- --compartment-id
|
|
- "{{ oci_govern_compartment_id }}"
|
|
- --vault-id
|
|
- "{{ oci_vault_id }}"
|
|
- --name
|
|
- "{{ item }}"
|
|
- --lifecycle-state
|
|
- ACTIVE
|
|
- --query
|
|
- "data[0].id"
|
|
- --raw-output
|
|
loop: "{{ all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: ocid_lookups
|
|
changed_when: false
|
|
|
|
- name: "Phase 2 — Build secret OCID map"
|
|
ansible.builtin.set_fact:
|
|
secret_ocids: "{{ secret_ocids | default({}) | combine({item.item: item.stdout | trim}) }}"
|
|
loop: "{{ ocid_lookups.results }}"
|
|
loop_control:
|
|
label: "{{ item.item }}"
|
|
|
|
- name: "Phase 2 — Verify all OCIDs resolved"
|
|
ansible.builtin.assert:
|
|
that:
|
|
- secret_ocids[item] is defined
|
|
- secret_ocids[item] | length > 0
|
|
- secret_ocids[item] != "null"
|
|
fail_msg: "Could not resolve OCID for secret '{{ item }}' — has terraform apply been run?"
|
|
success_msg: "✓ {{ item }}"
|
|
loop: "{{ all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Phase 2 — Summary"
|
|
ansible.builtin.debug:
|
|
msg: "Resolved OCIDs for all {{ all_secret_names | length }} secrets."
|
|
|
|
# -----------------------------------------------------------------
|
|
# Phase 3: Write a unique test value to each secret
|
|
# -----------------------------------------------------------------
|
|
- name: Generate a unique test marker
|
|
ansible.builtin.set_fact:
|
|
test_marker: "vault-validate-{{ lookup('pipe', 'date +%Y%m%dT%H%M%S') }}"
|
|
|
|
- name: "Phase 3 — Write test value to each secret"
|
|
ansible.builtin.command:
|
|
argv:
|
|
- oci
|
|
- vault
|
|
- secret
|
|
- update-base64
|
|
- --secret-id
|
|
- "{{ secret_ocids[item] }}"
|
|
- --secret-content-content
|
|
- "{{ (test_marker + ':' + item) | b64encode }}"
|
|
loop: "{{ all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: phase3_write
|
|
changed_when: true
|
|
|
|
- name: "Phase 3 — Summary"
|
|
ansible.builtin.debug:
|
|
msg: "Wrote test values to {{ all_secret_names | length }} secrets."
|
|
|
|
# -----------------------------------------------------------------
|
|
# Phase 4: Read back and compare
|
|
# -----------------------------------------------------------------
|
|
- name: "Phase 4 — Read back each secret and verify"
|
|
ansible.builtin.assert:
|
|
that:
|
|
- readback == expected
|
|
fail_msg: >-
|
|
MISMATCH on {{ item }}:
|
|
expected '{{ expected }}'
|
|
got '{{ readback }}'
|
|
success_msg: "✓ {{ item }} round-trip OK"
|
|
vars:
|
|
expected: "{{ test_marker + ':' + item }}"
|
|
readback: "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}"
|
|
loop: "{{ all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Phase 4 — Summary"
|
|
ansible.builtin.debug:
|
|
msg: "All {{ all_secret_names | length }} secrets passed round-trip validation."
|
|
|
|
# -----------------------------------------------------------------
|
|
# Phase 5: Restore original content
|
|
# -----------------------------------------------------------------
|
|
- name: "Phase 5 — Restore original secret content"
|
|
ansible.builtin.command:
|
|
argv:
|
|
- oci
|
|
- vault
|
|
- secret
|
|
- update-base64
|
|
- --secret-id
|
|
- "{{ secret_ocids[item] }}"
|
|
- --secret-content-content
|
|
- "{{ lookup('vars', 'original_' + item | replace('-', '_')) | b64encode }}"
|
|
loop: "{{ all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
changed_when: true
|
|
|
|
- name: "Phase 5 — Summary"
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
Round-trip test passed — all {{ all_secret_names | length }} secrets verified.
|
|
Original content restored.
|
|
|
|
# -----------------------------------------------------------------
|
|
# Phase 6: Validate certificate content in vault
|
|
# -----------------------------------------------------------------
|
|
- name: Create temporary directory for cert validation
|
|
ansible.builtin.tempfile:
|
|
state: directory
|
|
prefix: vault-validate-
|
|
register: validate_tmpdir
|
|
|
|
- name: "Phase 6 — Read fullchain secrets"
|
|
ansible.builtin.set_fact:
|
|
"fullchain_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item + '-fullchain', vault_id=oci_vault_id) }}"
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Phase 6 — Check fullchain is PEM formatted"
|
|
ansible.builtin.assert:
|
|
that:
|
|
- lookup('vars', 'fullchain_' + item | replace('-', '_')) is search('-----BEGIN CERTIFICATE-----')
|
|
- lookup('vars', 'fullchain_' + item | replace('-', '_')) is search('-----END CERTIFICATE-----')
|
|
fail_msg: >-
|
|
{{ item }}-fullchain does not contain a PEM certificate.
|
|
Content starts with: {{ lookup('vars', 'fullchain_' + item | replace('-', '_'))[:60] }}
|
|
success_msg: "✓ {{ item }}-fullchain is PEM formatted"
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Phase 6 — Write fullchain to temp files for openssl validation"
|
|
ansible.builtin.copy:
|
|
content: "{{ lookup('vars', 'fullchain_' + item | replace('-', '_')) }}"
|
|
dest: "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem"
|
|
mode: '0600'
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Phase 6 — Validate certificate is not expired"
|
|
ansible.builtin.command:
|
|
argv:
|
|
- openssl
|
|
- x509
|
|
- -in
|
|
- "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem"
|
|
- -checkend
|
|
- "0"
|
|
- -noout
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: cert_expiry_check
|
|
changed_when: false
|
|
|
|
- name: "Phase 6 — Get certificate details"
|
|
ansible.builtin.command:
|
|
argv:
|
|
- openssl
|
|
- x509
|
|
- -in
|
|
- "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem"
|
|
- -noout
|
|
- -subject
|
|
- -enddate
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: cert_details
|
|
changed_when: false
|
|
|
|
- name: "Phase 6 — Display certificate status"
|
|
ansible.builtin.debug:
|
|
msg: "✓ {{ item.item }}: {{ item.stdout }}"
|
|
loop: "{{ cert_details.results }}"
|
|
loop_control:
|
|
label: "{{ item.item }}"
|
|
|
|
- name: "Phase 6 — Read privkey secrets"
|
|
ansible.builtin.set_fact:
|
|
"privkey_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item + '-privkey', vault_id=oci_vault_id) }}"
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
no_log: true
|
|
|
|
- name: "Phase 6 — Check privkey is PEM formatted"
|
|
ansible.builtin.assert:
|
|
that:
|
|
- lookup('vars', 'privkey_' + item | replace('-', '_')) is search('-----BEGIN .*(PRIVATE KEY)-----')
|
|
- lookup('vars', 'privkey_' + item | replace('-', '_')) is search('-----END .*(PRIVATE KEY)-----')
|
|
fail_msg: "{{ item }}-privkey does not contain a PEM private key"
|
|
success_msg: "✓ {{ item }}-privkey is PEM formatted"
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Phase 6 — Write privkey to temp files for modulus check"
|
|
ansible.builtin.copy:
|
|
content: "{{ lookup('vars', 'privkey_' + item | replace('-', '_')) }}"
|
|
dest: "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem"
|
|
mode: '0600'
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
no_log: true
|
|
|
|
- name: "Phase 6 — Verify private key matches certificate"
|
|
ansible.builtin.shell: |
|
|
set -euo pipefail
|
|
cert_mod=$(openssl x509 -noout -modulus -in "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" | openssl md5)
|
|
key_mod=$(openssl rsa -noout -modulus -in "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5 || \
|
|
openssl ec -noout -text -in "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5)
|
|
if [[ "${cert_mod}" != "${key_mod}" ]]; then
|
|
echo "MISMATCH: cert=${cert_mod} key=${key_mod}"
|
|
exit 1
|
|
fi
|
|
echo "OK: modulus match"
|
|
args:
|
|
executable: /bin/bash
|
|
loop: "{{ cert_prefixes }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: modulus_check
|
|
changed_when: false
|
|
|
|
- name: "Phase 6 — Display key match results"
|
|
ansible.builtin.debug:
|
|
msg: "✓ {{ item.item }}: cert/key pair verified"
|
|
loop: "{{ modulus_check.results }}"
|
|
loop_control:
|
|
label: "{{ item.item }}"
|
|
|
|
- name: Clean up temporary directory
|
|
ansible.builtin.file:
|
|
path: "{{ validate_tmpdir.path }}"
|
|
state: absent
|
|
|
|
- name: Final summary
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
Validation complete:
|
|
{{ all_secret_names | length }} vault secrets — read/write round-trip OK.
|
|
{{ cert_prefixes | length }} certificates — valid PEM, not expired, key pairs match.
|