From e472d83372dae761daf789ef322136f4970844c5 Mon Sep 17 00:00:00 2001 From: Robert Helewka Date: Tue, 17 Mar 2026 17:29:26 +0000 Subject: [PATCH] refactor: remove deprecated certificate management playbooks and hooks --- ansible/certbot/cert-distribute.yml | 87 ------ ansible/certbot/deploy.yml | 38 +-- ansible/certbot/renewal-hook.sh.j2 | 5 +- ansible/certbot/vault-certs.yml | 347 ------------------------ ansible/certbot/vault-upload-hook.sh.j2 | 115 -------- ansible/certbot/vault-validate.yml | 340 ----------------------- 6 files changed, 6 insertions(+), 926 deletions(-) delete mode 100644 ansible/certbot/cert-distribute.yml delete mode 100644 ansible/certbot/vault-certs.yml delete mode 100644 ansible/certbot/vault-upload-hook.sh.j2 delete mode 100644 ansible/certbot/vault-validate.yml diff --git a/ansible/certbot/cert-distribute.yml b/ansible/certbot/cert-distribute.yml deleted file mode 100644 index a9e878f..0000000 --- a/ansible/certbot/cert-distribute.yml +++ /dev/null @@ -1,87 +0,0 @@ ---- -# ----------------------------------------------------------------------------- -# Certificate Distribution Playbook -# ----------------------------------------------------------------------------- -# Pulls certificates from OCI Vault (uploaded by bootes certbot) and -# deploys them directly to target hosts for HAProxy/service TLS termination. -# -# Each target host defines its certificates in host_vars: -# certbot_distributed_certs: -# - cert_name: corvus.helu.ca -# cert_path: /etc/haproxy/certs/corvus.helu.ca.pem -# -# Run from fornax: -# ansible-playbook certbot/cert-distribute.yml -# -# Deployed as a weekly cron job on fornax. -# Can also be run manually after ad-hoc certificate renewals. -# ----------------------------------------------------------------------------- - -- name: Distribute certificates from OCI Vault to target hosts - hosts: ubuntu:debian - gather_facts: false - - handlers: - - name: reload haproxy - become: true - ansible.builtin.systemd: - name: haproxy - state: reloaded - when: "'haproxy' in services | default([])" - - tasks: - - name: Skip hosts without distributed certificates - ansible.builtin.meta: end_host - when: certbot_distributed_certs is not defined - - - name: Ensure cert directory exists - become: true - ansible.builtin.file: - path: "{{ certbot_distributed_certs[0].cert_path | dirname }}" - state: directory - owner: root - group: root - mode: '0755' - - - name: Deploy certificate from OCI Vault - become: true - ansible.builtin.copy: - content: | - {{ lookup('oci_secret', item.cert_name | replace('.', '-') + '-fullchain', vault_id=oci_vault_id) }} - {{ lookup('oci_secret', item.cert_name | replace('.', '-') + '-privkey', vault_id=oci_vault_id) }} - dest: "{{ item.cert_path }}" - owner: root - group: root - mode: '0640' - loop: "{{ certbot_distributed_certs }}" - loop_control: - label: "{{ item.cert_name }}" - no_log: true - notify: reload haproxy - - - name: Verify deployed certificates are valid PEM - become: true - ansible.builtin.command: - cmd: openssl x509 -noout -checkend 0 -in {{ item.cert_path }} - register: _cert_check - loop: "{{ certbot_distributed_certs }}" - loop_control: - label: "{{ item.cert_name }}" - changed_when: false - - - name: Show certificate expiry dates - become: true - ansible.builtin.command: - cmd: openssl x509 -noout -subject -enddate -in {{ item.cert_path }} - register: _cert_info - loop: "{{ certbot_distributed_certs }}" - loop_control: - label: "{{ item.cert_name }}" - changed_when: false - - - name: Log certificate status - ansible.builtin.debug: - msg: "{{ item.item.cert_name }}: {{ item.stdout }}" - loop: "{{ _cert_info.results }}" - loop_control: - label: "{{ item.item.cert_name }}" diff --git a/ansible/certbot/deploy.yml b/ansible/certbot/deploy.yml index dec536d..952d84e 100644 --- a/ansible/certbot/deploy.yml +++ b/ansible/certbot/deploy.yml @@ -138,16 +138,6 @@ state: present virtualenv: "{{ certbot_directory }}/.venv" - - name: Install OCI CLI in certbot venv (vault upload hosts) - become: true - become_user: "{{ certbot_user }}" - ansible.builtin.pip: - name: - - oci-cli - state: present - virtualenv: "{{ certbot_directory }}/.venv" - when: certbot_vault_upload | default(false) - # ------------------------------------------------------------------------- # Namecheap Credentials # ------------------------------------------------------------------------- @@ -179,7 +169,7 @@ # Renewal Hooks # ------------------------------------------------------------------------- - - name: Template renewal hook script (HAProxy reload) + - name: Template renewal hook script become: true ansible.builtin.template: src: renewal-hook.sh.j2 @@ -187,17 +177,6 @@ owner: "{{ certbot_user }}" group: "{{ certbot_group }}" mode: '0750' - when: not (certbot_vault_upload | default(false)) - - - name: Template vault upload hook script - become: true - ansible.builtin.template: - src: vault-upload-hook.sh.j2 - dest: "{{ certbot_directory }}/hooks/renewal-hook.sh" - owner: "{{ certbot_user }}" - group: "{{ certbot_group }}" - mode: '0750' - when: certbot_vault_upload | default(false) - name: Create Prometheus textfile directory become: true @@ -289,24 +268,13 @@ - name: Run renewal hook after certificate requests become: true ansible.builtin.command: "{{ certbot_directory }}/hooks/renewal-hook.sh" - environment: >- - {{ {'RENEWED_LINEAGE': certbot_directory + '/config/live/' + item.item.cert_name} - if certbot_vault_upload | default(false) else {} }} + environment: + RENEWED_LINEAGE: "{{ certbot_directory }}/config/live/{{ item.item.cert_name }}" loop: "{{ certbot_requests.results | default([]) }}" when: item.changed | default(false) loop_control: label: "{{ item.item.cert_name }}" - - name: Ensure vault is populated with current certificates - become: true - ansible.builtin.command: "{{ certbot_directory }}/hooks/renewal-hook.sh" - environment: - RENEWED_LINEAGE: "{{ certbot_directory }}/config/live/{{ item.cert_name }}" - loop: "{{ _certbot_certs }}" - when: certbot_vault_upload | default(false) - loop_control: - label: "{{ item.cert_name }}" - # ------------------------------------------------------------------------- # Systemd Timer for Auto-Renewal # ------------------------------------------------------------------------- diff --git a/ansible/certbot/renewal-hook.sh.j2 b/ansible/certbot/renewal-hook.sh.j2 index 0c0bb23..2df180d 100644 --- a/ansible/certbot/renewal-hook.sh.j2 +++ b/ansible/certbot/renewal-hook.sh.j2 @@ -10,8 +10,9 @@ set -euo pipefail -CERT_NAME="{{ certbot_cert_name }}" -CERT_DIR="{{ certbot_directory }}/config/live/${CERT_NAME}" +# RENEWED_LINEAGE is set by certbot --deploy-hook or passed explicitly by deploy.yml +CERT_DIR="${RENEWED_LINEAGE:?RENEWED_LINEAGE must be set}" +CERT_NAME=$(basename "${CERT_DIR}") HAPROXY_CERT="{{ haproxy_cert_path }}" HAPROXY_DIR="{{ haproxy_directory }}" diff --git a/ansible/certbot/vault-certs.yml b/ansible/certbot/vault-certs.yml deleted file mode 100644 index f2d586e..0000000 --- a/ansible/certbot/vault-certs.yml +++ /dev/null @@ -1,347 +0,0 @@ ---- -# ----------------------------------------------------------------------------- -# Vault Certificate Management Playbook -# ----------------------------------------------------------------------------- -# Checks certificate validity in OCI Vault and renews expired/expiring -# certificates via certbot on bootes. Designed for internal hosts that -# don't have public IPs — certs are stored in OCI Vault for distribution. -# -# Run from fornax: -# ansible-playbook certbot/vault-certs.yml -# -# Steps: -# 1. Validate vault secrets exist and are readable -# 2. Display public IP for Namecheap API whitelisting verification -# 3. Check certificate validity (PEM format + 30-day expiry window) -# 4. Request certificates for any that need renewal (conditional) -# 5. Post-validate renewed certificates in vault (conditional) -# 6. Verify renewal schedule is active -# -# Prerequisites: -# - certbot/deploy.yml has been run on bootes (certbot is installed) -# - certbot/vault-validate.yml has been run (vault R/W confirmed) -# - Namecheap API IP is whitelisted -# ----------------------------------------------------------------------------- - -- name: Manage Internal Host Certificates via OCI Vault - hosts: bootes.helu.ca - gather_facts: false - tags: [certbot, vault, certs] - - tasks: - # ------------------------------------------------------------------------- - # Derive Certificate List from Host Vars - # ------------------------------------------------------------------------- - - - name: Build certificate prefix list from host_vars - ansible.builtin.set_fact: - _cert_prefixes: "{{ certbot_certificates | map(attribute='cert_name') | map('replace', '.', '-') | list }}" - _secret_suffixes: [fullchain, privkey] - - - name: Build list of all vault secret names - ansible.builtin.set_fact: - _all_secret_names: "{{ _cert_prefixes | product(_secret_suffixes) | map('join', '-') | list }}" - - # ------------------------------------------------------------------------- - # Step 1: Validate Vault Secrets Exist - # ------------------------------------------------------------------------- - - - name: "Step 1 — Read vault secrets" - ansible.builtin.set_fact: - "_vault_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}" - loop: "{{ _all_secret_names }}" - loop_control: - label: "{{ item }}" - - - name: "Step 1 — Verify all secrets are readable and non-empty" - ansible.builtin.assert: - that: - - lookup('vars', '_vault_' + item | replace('-', '_')) is defined - - lookup('vars', '_vault_' + item | replace('-', '_')) | length > 0 - fail_msg: "Secret '{{ item }}' is missing or empty in OCI Vault" - success_msg: "{{ item }}" - loop: "{{ _all_secret_names }}" - loop_control: - label: "{{ item }}" - - - name: "Step 1 — Summary" - ansible.builtin.debug: - msg: "All {{ _all_secret_names | length }} vault secrets exist and are readable." - - # ------------------------------------------------------------------------- - # Step 2: IP Whitelisting Check - # ------------------------------------------------------------------------- - - - name: "Step 2 — Get public IP for Namecheap API" - ansible.builtin.uri: - url: https://ifconfig.me/ip - return_content: true - register: _public_ip_result - - - name: "Step 2 — Display public IP for verification" - ansible.builtin.debug: - msg: >- - Public IP: {{ _public_ip_result.content | trim }} - — Ensure whitelisted at https://ap.www.namecheap.com/settings/tools/apiaccess/ - - # ------------------------------------------------------------------------- - # Step 3: Certificate Validity Check - # ------------------------------------------------------------------------- - - - name: Create temporary directory for certificate validation - ansible.builtin.tempfile: - state: directory - prefix: vault-certs- - register: _validate_tmpdir - - - name: "Step 3 — Write fullchain PEMs to temp files" - ansible.builtin.copy: - content: "{{ lookup('vars', ('_vault_' + item | replace('.', '-') + '-fullchain') | replace('-', '_')) }}" - dest: "{{ _validate_tmpdir.path }}/{{ item }}.pem" - mode: '0600' - loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - - - name: "Step 3 — Check PEM format" - ansible.builtin.command: - cmd: openssl x509 -noout -in {{ _validate_tmpdir.path }}/{{ item }}.pem - loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - register: _pem_check - changed_when: false - failed_when: false - - - name: "Step 3 — Check certificate expiry (30-day window)" - ansible.builtin.command: - cmd: openssl x509 -checkend 2592000 -noout -in {{ _validate_tmpdir.path }}/{{ item }}.pem - loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - register: _expiry_check - changed_when: false - failed_when: false - - - name: "Step 3 — Get certificate details" - ansible.builtin.command: - cmd: openssl x509 -noout -subject -enddate -in {{ _validate_tmpdir.path }}/{{ item }}.pem - loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - register: _cert_details - changed_when: false - failed_when: false - - - name: "Step 3 — Build renewal status" - ansible.builtin.set_fact: - _certs_needing_renewal: >- - {{ _certs_needing_renewal | default([]) + - ([certbot_certificates[idx]] - if _pem_check.results[idx].rc != 0 or _expiry_check.results[idx].rc != 0 - else []) }} - loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}" - loop_control: - index_var: idx - label: "{{ item }}" - - - name: "Step 3 — Display certificate status" - ansible.builtin.debug: - msg: >- - {{ item.item }}: - {{ 'INVALID PEM' if item.rc != 0 else - (_cert_details.results[idx].stdout | default('unknown')) - + (' — NEEDS RENEWAL' if _expiry_check.results[idx].rc != 0 else ' — valid') }} - loop: "{{ _pem_check.results }}" - loop_control: - index_var: idx - label: "{{ item.item }}" - - - name: "Step 3 — Summary" - ansible.builtin.debug: - msg: >- - {{ (_certs_needing_renewal | default([]) | length == 0) - | ternary( - 'All ' + (certbot_certificates | length | string) + ' certificates are valid. Skipping renewal.', - (_certs_needing_renewal | default([]) | length | string) + ' certificate(s) need renewal: ' - + (_certs_needing_renewal | default([]) | map(attribute='cert_name') | join(', ')) - ) }} - - - name: Clean up validation temp directory - ansible.builtin.file: - path: "{{ _validate_tmpdir.path }}" - state: absent - - # ------------------------------------------------------------------------- - # Step 4: Request Certificates (conditional — only if needed) - # ------------------------------------------------------------------------- - - - name: "Step 4 — Request certificates that need renewal" - become: true - become_user: "{{ certbot_user }}" - ansible.builtin.shell: | - source {{ certbot_directory }}/.venv/bin/activate - certbot certonly \ - --non-interactive \ - --agree-tos \ - --email {{ certbot_email }} \ - --authenticator dns-namecheap \ - --dns-namecheap-credentials {{ certbot_directory }}/credentials/namecheap.ini \ - --dns-namecheap-propagation-seconds 120 \ - --config-dir {{ certbot_directory }}/config \ - --work-dir {{ certbot_directory }}/work \ - --logs-dir {{ certbot_directory }}/logs \ - --cert-name {{ item.cert_name }} \ - --force-renewal \ - {{ item.domains | map('regex_replace', '^', '-d ') | join(' ') }} - args: - executable: /bin/bash - loop: "{{ _certs_needing_renewal | default([]) }}" - loop_control: - label: "{{ item.cert_name }}" - register: _certbot_renewals - when: _certs_needing_renewal | default([]) | length > 0 - - - name: "Step 4 — Upload renewed certificates to vault" - become: true - ansible.builtin.command: "{{ certbot_directory }}/hooks/renewal-hook.sh" - environment: - RENEWED_LINEAGE: "{{ certbot_directory }}/config/live/{{ item.item.cert_name }}" - loop: "{{ _certbot_renewals.results | default([]) }}" - when: item.changed | default(false) - loop_control: - label: "{{ item.item.cert_name }}" - - - name: "Step 4 — Update certificate metrics" - become: true - ansible.builtin.command: "{{ certbot_directory }}/hooks/cert-metrics.sh" - changed_when: false - when: _certs_needing_renewal | default([]) | length > 0 - - # ------------------------------------------------------------------------- - # Step 5: Post-Validation (conditional — only after renewal) - # ------------------------------------------------------------------------- - - - name: "Step 5 — Re-read vault secrets after renewal" - ansible.builtin.set_fact: - "_post_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}" - loop: "{{ _all_secret_names }}" - loop_control: - label: "{{ item }}" - when: _certs_needing_renewal | default([]) | length > 0 - - - name: Create post-validation temp directory - ansible.builtin.tempfile: - state: directory - prefix: vault-post- - register: _post_tmpdir - when: _certs_needing_renewal | default([]) | length > 0 - - - name: "Step 5 — Write renewed fullchain PEMs" - ansible.builtin.copy: - content: "{{ lookup('vars', ('_post_' + item | replace('.', '-') + '-fullchain') | replace('-', '_')) }}" - dest: "{{ _post_tmpdir.path }}/{{ item }}-fullchain.pem" - mode: '0600' - loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - when: _certs_needing_renewal | default([]) | length > 0 - - - name: "Step 5 — Write renewed privkey PEMs" - ansible.builtin.copy: - content: "{{ lookup('vars', ('_post_' + item | replace('.', '-') + '-privkey') | replace('-', '_')) }}" - dest: "{{ _post_tmpdir.path }}/{{ item }}-privkey.pem" - mode: '0600' - loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - no_log: true - when: _certs_needing_renewal | default([]) | length > 0 - - - name: "Step 5 — Verify renewed certificates are valid and not expiring" - ansible.builtin.command: - cmd: openssl x509 -checkend 2592000 -noout -in {{ _post_tmpdir.path }}/{{ item }}-fullchain.pem - loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - changed_when: false - when: _certs_needing_renewal | default([]) | length > 0 - - - name: "Step 5 — Verify cert/key modulus match" - ansible.builtin.shell: | - set -euo pipefail - cert_mod=$(openssl x509 -noout -modulus -in "{{ _post_tmpdir.path }}/{{ item }}-fullchain.pem" | openssl md5) - key_mod=$(openssl rsa -noout -modulus -in "{{ _post_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5 || \ - openssl ec -noout -text -in "{{ _post_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5) - if [[ "${cert_mod}" != "${key_mod}" ]]; then - echo "MISMATCH: cert=${cert_mod} key=${key_mod}" >&2 - exit 1 - fi - echo "OK: modulus match" - args: - executable: /bin/bash - loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - changed_when: false - when: _certs_needing_renewal | default([]) | length > 0 - - - name: "Step 5 — Get renewed certificate expiry dates" - ansible.builtin.command: - cmd: openssl x509 -noout -subject -enddate -in {{ _post_tmpdir.path }}/{{ item }}-fullchain.pem - loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}" - loop_control: - label: "{{ item }}" - register: _post_cert_details - changed_when: false - when: _certs_needing_renewal | default([]) | length > 0 - - - name: "Step 5 — Display renewed certificate status" - ansible.builtin.debug: - msg: "{{ item.item }}: {{ item.stdout }}" - loop: "{{ _post_cert_details.results | default([]) }}" - loop_control: - label: "{{ item.item }}" - when: _certs_needing_renewal | default([]) | length > 0 - - - name: Clean up post-validation temp directory - ansible.builtin.file: - path: "{{ _post_tmpdir.path }}" - state: absent - when: _post_tmpdir.path is defined - - # ------------------------------------------------------------------------- - # Step 6: Schedule Verification (always runs) - # ------------------------------------------------------------------------- - - - name: "Step 6 — Verify certbot-renew.timer is enabled" - become: true - ansible.builtin.systemd: - name: certbot-renew.timer - enabled: true - state: started - register: _timer_status - - - name: "Step 6 — Get timer status" - become: true - ansible.builtin.command: - cmd: systemctl show certbot-renew.timer --property=ActiveState,NextElapseUSecRealtime,LastTriggerUSec - register: _timer_details - changed_when: false - - - name: "Step 6 — Display timer status" - ansible.builtin.debug: - msg: "certbot-renew.timer: {{ _timer_details.stdout_lines | join(', ') }}" - - - name: "Step 6 — Update certificate metrics" - become: true - ansible.builtin.command: "{{ certbot_directory }}/hooks/cert-metrics.sh" - changed_when: false - - - name: Final summary - ansible.builtin.debug: - msg: >- - Vault certificate check complete. - {{ (certbot_certificates | length) }} certificates checked. - {{ (_certs_needing_renewal | default([]) | length) }} renewed. - Renewal timer is {{ _timer_status.status.ActiveState | default('active') }}. diff --git a/ansible/certbot/vault-upload-hook.sh.j2 b/ansible/certbot/vault-upload-hook.sh.j2 deleted file mode 100644 index 0a3f9ce..0000000 --- a/ansible/certbot/vault-upload-hook.sh.j2 +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# Certbot post-renewal hook for OCI Vault upload -# Managed by Ansible - DO NOT EDIT MANUALLY -# -# This script uploads renewed certificates to OCI Vault so that -# fornax can distribute them to target hosts via Ansible. -# -# Uses Instance Principal authentication (no config file needed). -# Called by certbot --deploy-hook after each successful renewal. - -set -euo pipefail - -CERT_DIR="{{ certbot_directory }}/config/live" -LOG_PREFIX="[$(date '+%Y-%m-%d %H:%M:%S')] [vault-upload]" - -echo "${LOG_PREFIX} Starting vault upload hook" - -# RENEWED_LINEAGE is set by certbot to the path of the renewed cert -# e.g. /srv/certbot/config/live/bootes.helu.ca -if [[ -z "${RENEWED_LINEAGE:-}" ]]; then - echo "${LOG_PREFIX} ERROR: RENEWED_LINEAGE not set — not running under certbot?" - exit 1 -fi - -CERT_NAME=$(basename "${RENEWED_LINEAGE}") -FULLCHAIN="${RENEWED_LINEAGE}/fullchain.pem" -PRIVKEY="${RENEWED_LINEAGE}/privkey.pem" -OCI="{{ certbot_directory }}/.venv/bin/oci" -COMPARTMENT_ID="{{ oci_govern_compartment_id }}" -VAULT_ID="{{ oci_vault_id }}" - -# Convert dots to hyphens to match Terraform secret naming (e.g. pan.helu.ca → pan-helu-ca) -VAULT_PREFIX="${CERT_NAME//./-}" - -echo "${LOG_PREFIX} Processing certificate: ${CERT_NAME} (vault prefix: ${VAULT_PREFIX})" - -if [[ ! -f "${FULLCHAIN}" ]] || [[ ! -f "${PRIVKEY}" ]]; then - echo "${LOG_PREFIX} ERROR: Certificate files not found in ${RENEWED_LINEAGE}" - exit 1 -fi - -# Look up secret OCIDs by name (Terraform creates secrets named {domain-hyphens}-fullchain/-privkey) -lookup_secret_id() { - local secret_name="$1" - local result - if ! result=$(${OCI} vault secret list \ - --auth instance_principal \ - --compartment-id "${COMPARTMENT_ID}" \ - --vault-id "${VAULT_ID}" \ - --name "${secret_name}" \ - --lifecycle-state ACTIVE \ - --all \ - --query 'data[0].id' \ - --raw-output 2>&1); then - echo "${LOG_PREFIX} ERROR: OCI CLI failed looking up secret '${secret_name}': ${result}" >&2 - return 1 - fi - echo "${result}" -} - -FULLCHAIN_SECRET_ID=$(lookup_secret_id "${VAULT_PREFIX}-fullchain") || true -PRIVKEY_SECRET_ID=$(lookup_secret_id "${VAULT_PREFIX}-privkey") || true - -if [[ -z "${FULLCHAIN_SECRET_ID}" ]] || [[ "${FULLCHAIN_SECRET_ID}" == "null" ]] || \ - [[ -z "${PRIVKEY_SECRET_ID}" ]] || [[ "${PRIVKEY_SECRET_ID}" == "null" ]]; then - echo "${LOG_PREFIX} ERROR: Could not find vault secrets for ${VAULT_PREFIX} (fullchain=${FULLCHAIN_SECRET_ID:-missing}, privkey=${PRIVKEY_SECRET_ID:-missing})" - echo "${LOG_PREFIX} Ensure 'terraform apply' has been run on bootes_certificates.tf" - exit 1 -fi - -echo "${LOG_PREFIX} Found secret OCIDs for ${VAULT_PREFIX}" - -# Upload fullchain to OCI Vault -FULLCHAIN_B64=$(base64 -w 0 < "${FULLCHAIN}") -if ! upload_output=$(${OCI} vault secret update-base64 \ - --auth instance_principal \ - --secret-id "${FULLCHAIN_SECRET_ID}" \ - --secret-content-content "${FULLCHAIN_B64}" 2>&1); then - echo "${LOG_PREFIX} ERROR: Failed to upload fullchain for ${CERT_NAME}: ${upload_output}" - exit 1 -fi -echo "${LOG_PREFIX} Uploaded fullchain for ${CERT_NAME}" - -# Upload private key to OCI Vault -PRIVKEY_B64=$(base64 -w 0 < "${PRIVKEY}") -if ! upload_output=$(${OCI} vault secret update-base64 \ - --auth instance_principal \ - --secret-id "${PRIVKEY_SECRET_ID}" \ - --secret-content-content "${PRIVKEY_B64}" 2>&1); then - echo "${LOG_PREFIX} ERROR: Failed to upload privkey for ${CERT_NAME}: ${upload_output}" - exit 1 -fi -echo "${LOG_PREFIX} Uploaded privkey for ${CERT_NAME}" - -{% if certbot_local_cert_name is defined %} -# Also combine cert for local HAProxy if this is the local cert -if [[ "${CERT_NAME}" == "{{ certbot_local_cert_name }}" ]]; then - echo "${LOG_PREFIX} Combining local cert for HAProxy: ${CERT_NAME}" - HAPROXY_CERT="{{ haproxy_cert_path }}" - cat "${FULLCHAIN}" "${PRIVKEY}" > "${HAPROXY_CERT}.tmp" - mv "${HAPROXY_CERT}.tmp" "${HAPROXY_CERT}" - chown {{ certbot_user }}:{{ haproxy_group }} "${HAPROXY_CERT}" - chmod 640 "${HAPROXY_CERT}" - - if systemctl is-active --quiet haproxy; then - echo "${LOG_PREFIX} Reloading HAProxy..." - systemctl reload haproxy - fi -fi -{% endif %} - -# Update certificate metrics -{{ certbot_directory }}/hooks/cert-metrics.sh - -echo "${LOG_PREFIX} Vault upload hook completed successfully" diff --git a/ansible/certbot/vault-validate.yml b/ansible/certbot/vault-validate.yml deleted file mode 100644 index b7d0bf4..0000000 --- a/ansible/certbot/vault-validate.yml +++ /dev/null @@ -1,340 +0,0 @@ ---- -# ----------------------------------------------------------------------------- -# Vault Secret Validation Playbook -# ----------------------------------------------------------------------------- -# Tests the full round-trip of OCI Vault secret read/write for all certbot -# domains. Use this BEFORE running certbot to verify vault connectivity -# and permissions without burning Let's Encrypt rate limits. -# -# Run from fornax: -# ansible-playbook certbot/vault-validate.yml -# -# What it does: -# 1. Verifies every expected vault secret exists and is readable -# 2. Writes a unique test value to each secret -# 3. Reads back and compares to confirm the write path works -# 4. Restores the original content -# ----------------------------------------------------------------------------- - -- name: Validate OCI Vault certificate secrets - hosts: localhost - gather_facts: false - environment: - PATH: "/srv/ponos/.local/bin:/usr/local/bin:/usr/bin:/bin" - HOME: "/srv/ponos" - vars: - # Must match the keys in taurus/terraform/bootes_certificates.tf - cert_prefixes: - - apollo-helu-ca - - bootes-helu-ca - - corvus-helu-ca - - draco-helu-ca - - iris-helu-ca - - korax-helu-ca - - nyx-helu-ca - - orpheus-helu-ca - - pan-helu-ca - - perseus-helu-ca - - wildcard-ouranos-helu-ca - secret_suffixes: - - fullchain - - privkey - - tasks: - # ----------------------------------------------------------------- - # Phase 1: Verify all secrets exist and are readable - # ----------------------------------------------------------------- - - name: Build list of all secret names - ansible.builtin.set_fact: - all_secret_names: "{{ cert_prefixes | product(secret_suffixes) | map('join', '-') | list }}" - - - name: "Phase 1 — Read current value from vault" - ansible.builtin.set_fact: - "original_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}" - loop: "{{ all_secret_names }}" - register: phase1_read - - - name: "Phase 1 — Confirm all secrets are readable" - ansible.builtin.assert: - that: - - lookup('vars', 'original_' + item | replace('-', '_')) is defined - - lookup('vars', 'original_' + item | replace('-', '_')) | length > 0 - fail_msg: "Secret '{{ item }}' is missing or empty in OCI Vault" - success_msg: "✓ {{ item }}" - loop: "{{ all_secret_names }}" - loop_control: - label: "{{ item }}" - - - name: "Phase 1 — Summary" - ansible.builtin.debug: - msg: "All {{ all_secret_names | length }} vault secrets exist and are readable." - - # ----------------------------------------------------------------- - # Phase 2: Look up secret OCIDs (needed for write operations) - # ----------------------------------------------------------------- - - name: "Phase 2 — Look up secret OCID for each secret" - ansible.builtin.command: - argv: - - oci - - vault - - secret - - list - - --compartment-id - - "{{ oci_govern_compartment_id }}" - - --vault-id - - "{{ oci_vault_id }}" - - --name - - "{{ item }}" - - --lifecycle-state - - ACTIVE - - --query - - "data[0].id" - - --raw-output - loop: "{{ all_secret_names }}" - loop_control: - label: "{{ item }}" - register: ocid_lookups - changed_when: false - - - name: "Phase 2 — Build secret OCID map" - ansible.builtin.set_fact: - secret_ocids: "{{ secret_ocids | default({}) | combine({item.item: item.stdout | trim}) }}" - loop: "{{ ocid_lookups.results }}" - loop_control: - label: "{{ item.item }}" - - - name: "Phase 2 — Verify all OCIDs resolved" - ansible.builtin.assert: - that: - - secret_ocids[item] is defined - - secret_ocids[item] | length > 0 - - secret_ocids[item] != "null" - fail_msg: "Could not resolve OCID for secret '{{ item }}' — has terraform apply been run?" - success_msg: "✓ {{ item }}" - loop: "{{ all_secret_names }}" - loop_control: - label: "{{ item }}" - - - name: "Phase 2 — Summary" - ansible.builtin.debug: - msg: "Resolved OCIDs for all {{ all_secret_names | length }} secrets." - - # ----------------------------------------------------------------- - # Phase 3: Write a unique test value to each secret - # ----------------------------------------------------------------- - - name: Generate a unique test marker - ansible.builtin.set_fact: - test_marker: "vault-validate-{{ lookup('pipe', 'date +%Y%m%dT%H%M%S') }}" - - - name: "Phase 3 — Write test value to each secret" - ansible.builtin.command: - argv: - - oci - - vault - - secret - - update-base64 - - --secret-id - - "{{ secret_ocids[item] }}" - - --secret-content-content - - "{{ (test_marker + ':' + item) | b64encode }}" - loop: "{{ all_secret_names }}" - loop_control: - label: "{{ item }}" - register: phase3_write - changed_when: true - - - name: "Phase 3 — Summary" - ansible.builtin.debug: - msg: "Wrote test values to {{ all_secret_names | length }} secrets." - - # ----------------------------------------------------------------- - # Phase 4: Read back and compare - # ----------------------------------------------------------------- - - name: "Phase 4 — Read back each secret and verify" - ansible.builtin.assert: - that: - - readback == expected - fail_msg: >- - MISMATCH on {{ item }}: - expected '{{ expected }}' - got '{{ readback }}' - success_msg: "✓ {{ item }} round-trip OK" - vars: - expected: "{{ test_marker + ':' + item }}" - readback: "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}" - loop: "{{ all_secret_names }}" - loop_control: - label: "{{ item }}" - - - name: "Phase 4 — Summary" - ansible.builtin.debug: - msg: "All {{ all_secret_names | length }} secrets passed round-trip validation." - - # ----------------------------------------------------------------- - # Phase 5: Restore original content - # ----------------------------------------------------------------- - - name: "Phase 5 — Restore original secret content" - ansible.builtin.command: - argv: - - oci - - vault - - secret - - update-base64 - - --secret-id - - "{{ secret_ocids[item] }}" - - --secret-content-content - - "{{ lookup('vars', 'original_' + item | replace('-', '_')) | b64encode }}" - loop: "{{ all_secret_names }}" - loop_control: - label: "{{ item }}" - changed_when: true - - - name: "Phase 5 — Summary" - ansible.builtin.debug: - msg: >- - Round-trip test passed — all {{ all_secret_names | length }} secrets verified. - Original content restored. - - # ----------------------------------------------------------------- - # Phase 6: Validate certificate content in vault - # ----------------------------------------------------------------- - - name: Create temporary directory for cert validation - ansible.builtin.tempfile: - state: directory - prefix: vault-validate- - register: validate_tmpdir - - - name: "Phase 6 — Read fullchain secrets" - ansible.builtin.set_fact: - "fullchain_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item + '-fullchain', vault_id=oci_vault_id) }}" - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - - - name: "Phase 6 — Check fullchain is PEM formatted" - ansible.builtin.assert: - that: - - lookup('vars', 'fullchain_' + item | replace('-', '_')) is search('-----BEGIN CERTIFICATE-----') - - lookup('vars', 'fullchain_' + item | replace('-', '_')) is search('-----END CERTIFICATE-----') - fail_msg: >- - {{ item }}-fullchain does not contain a PEM certificate. - Content starts with: {{ lookup('vars', 'fullchain_' + item | replace('-', '_'))[:60] }} - success_msg: "✓ {{ item }}-fullchain is PEM formatted" - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - - - name: "Phase 6 — Write fullchain to temp files for openssl validation" - ansible.builtin.copy: - content: "{{ lookup('vars', 'fullchain_' + item | replace('-', '_')) }}" - dest: "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" - mode: '0600' - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - - - name: "Phase 6 — Validate certificate is not expired" - ansible.builtin.command: - argv: - - openssl - - x509 - - -in - - "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" - - -checkend - - "0" - - -noout - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - register: cert_expiry_check - changed_when: false - - - name: "Phase 6 — Get certificate details" - ansible.builtin.command: - argv: - - openssl - - x509 - - -in - - "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" - - -noout - - -subject - - -enddate - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - register: cert_details - changed_when: false - - - name: "Phase 6 — Display certificate status" - ansible.builtin.debug: - msg: "✓ {{ item.item }}: {{ item.stdout }}" - loop: "{{ cert_details.results }}" - loop_control: - label: "{{ item.item }}" - - - name: "Phase 6 — Read privkey secrets" - ansible.builtin.set_fact: - "privkey_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item + '-privkey', vault_id=oci_vault_id) }}" - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - no_log: true - - - name: "Phase 6 — Check privkey is PEM formatted" - ansible.builtin.assert: - that: - - lookup('vars', 'privkey_' + item | replace('-', '_')) is search('-----BEGIN .*(PRIVATE KEY)-----') - - lookup('vars', 'privkey_' + item | replace('-', '_')) is search('-----END .*(PRIVATE KEY)-----') - fail_msg: "{{ item }}-privkey does not contain a PEM private key" - success_msg: "✓ {{ item }}-privkey is PEM formatted" - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - - - name: "Phase 6 — Write privkey to temp files for modulus check" - ansible.builtin.copy: - content: "{{ lookup('vars', 'privkey_' + item | replace('-', '_')) }}" - dest: "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" - mode: '0600' - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - no_log: true - - - name: "Phase 6 — Verify private key matches certificate" - ansible.builtin.shell: | - set -euo pipefail - cert_mod=$(openssl x509 -noout -modulus -in "{{ validate_tmpdir.path }}/{{ item }}-fullchain.pem" | openssl md5) - key_mod=$(openssl rsa -noout -modulus -in "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5 || \ - openssl ec -noout -text -in "{{ validate_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5) - if [[ "${cert_mod}" != "${key_mod}" ]]; then - echo "MISMATCH: cert=${cert_mod} key=${key_mod}" - exit 1 - fi - echo "OK: modulus match" - args: - executable: /bin/bash - loop: "{{ cert_prefixes }}" - loop_control: - label: "{{ item }}" - register: modulus_check - changed_when: false - - - name: "Phase 6 — Display key match results" - ansible.builtin.debug: - msg: "✓ {{ item.item }}: cert/key pair verified" - loop: "{{ modulus_check.results }}" - loop_control: - label: "{{ item.item }}" - - - name: Clean up temporary directory - ansible.builtin.file: - path: "{{ validate_tmpdir.path }}" - state: absent - - - name: Final summary - ansible.builtin.debug: - msg: >- - Validation complete: - {{ all_secret_names | length }} vault secrets — read/write round-trip OK. - {{ cert_prefixes | length }} certificates — valid PEM, not expired, key pairs match.