Robert Helewka
0a053c1cd6
- Updated HAProxy configuration template to reflect changes for the Taurus Production Environment, including SSL settings and rate limiting for specific endpoints. - Introduced new playbooks for certificate distribution and validation with OCI Vault, ensuring certificates are correctly managed and renewed. - Added hooks for uploading renewed certificates to OCI Vault and validating their integrity. - Enhanced the HAProxy configuration playbook to ensure proper service management and verification of the HAProxy service. - Updated inventory variables for certificate management and ensured compatibility with the new structure.
348 lines
14 KiB
YAML
348 lines
14 KiB
YAML
---
|
|
# -----------------------------------------------------------------------------
|
|
# Vault Certificate Management Playbook
|
|
# -----------------------------------------------------------------------------
|
|
# Checks certificate validity in OCI Vault and renews expired/expiring
|
|
# certificates via certbot on bootes. Designed for internal hosts that
|
|
# don't have public IPs — certs are stored in OCI Vault for distribution.
|
|
#
|
|
# Run from fornax:
|
|
# ansible-playbook certbot/vault-certs.yml
|
|
#
|
|
# Steps:
|
|
# 1. Validate vault secrets exist and are readable
|
|
# 2. Display public IP for Namecheap API whitelisting verification
|
|
# 3. Check certificate validity (PEM format + 30-day expiry window)
|
|
# 4. Request certificates for any that need renewal (conditional)
|
|
# 5. Post-validate renewed certificates in vault (conditional)
|
|
# 6. Verify renewal schedule is active
|
|
#
|
|
# Prerequisites:
|
|
# - certbot/deploy.yml has been run on bootes (certbot is installed)
|
|
# - certbot/vault-validate.yml has been run (vault R/W confirmed)
|
|
# - Namecheap API IP is whitelisted
|
|
# -----------------------------------------------------------------------------
|
|
|
|
- name: Manage Internal Host Certificates via OCI Vault
|
|
hosts: bootes.helu.ca
|
|
gather_facts: false
|
|
tags: [certbot, vault, certs]
|
|
|
|
tasks:
|
|
# -------------------------------------------------------------------------
|
|
# Derive Certificate List from Host Vars
|
|
# -------------------------------------------------------------------------
|
|
|
|
- name: Build certificate prefix list from host_vars
|
|
ansible.builtin.set_fact:
|
|
_cert_prefixes: "{{ certbot_certificates | map(attribute='cert_name') | map('replace', '.', '-') | list }}"
|
|
_secret_suffixes: [fullchain, privkey]
|
|
|
|
- name: Build list of all vault secret names
|
|
ansible.builtin.set_fact:
|
|
_all_secret_names: "{{ _cert_prefixes | product(_secret_suffixes) | map('join', '-') | list }}"
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Step 1: Validate Vault Secrets Exist
|
|
# -------------------------------------------------------------------------
|
|
|
|
- name: "Step 1 — Read vault secrets"
|
|
ansible.builtin.set_fact:
|
|
"_vault_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}"
|
|
loop: "{{ _all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Step 1 — Verify all secrets are readable and non-empty"
|
|
ansible.builtin.assert:
|
|
that:
|
|
- lookup('vars', '_vault_' + item | replace('-', '_')) is defined
|
|
- lookup('vars', '_vault_' + item | replace('-', '_')) | length > 0
|
|
fail_msg: "Secret '{{ item }}' is missing or empty in OCI Vault"
|
|
success_msg: "{{ item }}"
|
|
loop: "{{ _all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Step 1 — Summary"
|
|
ansible.builtin.debug:
|
|
msg: "All {{ _all_secret_names | length }} vault secrets exist and are readable."
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Step 2: IP Whitelisting Check
|
|
# -------------------------------------------------------------------------
|
|
|
|
- name: "Step 2 — Get public IP for Namecheap API"
|
|
ansible.builtin.uri:
|
|
url: https://ifconfig.me/ip
|
|
return_content: true
|
|
register: _public_ip_result
|
|
|
|
- name: "Step 2 — Display public IP for verification"
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
Public IP: {{ _public_ip_result.content | trim }}
|
|
— Ensure whitelisted at https://ap.www.namecheap.com/settings/tools/apiaccess/
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Step 3: Certificate Validity Check
|
|
# -------------------------------------------------------------------------
|
|
|
|
- name: Create temporary directory for certificate validation
|
|
ansible.builtin.tempfile:
|
|
state: directory
|
|
prefix: vault-certs-
|
|
register: _validate_tmpdir
|
|
|
|
- name: "Step 3 — Write fullchain PEMs to temp files"
|
|
ansible.builtin.copy:
|
|
content: "{{ lookup('vars', ('_vault_' + item | replace('.', '-') + '-fullchain') | replace('-', '_')) }}"
|
|
dest: "{{ _validate_tmpdir.path }}/{{ item }}.pem"
|
|
mode: '0600'
|
|
loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
|
|
- name: "Step 3 — Check PEM format"
|
|
ansible.builtin.command:
|
|
cmd: openssl x509 -noout -in {{ _validate_tmpdir.path }}/{{ item }}.pem
|
|
loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: _pem_check
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: "Step 3 — Check certificate expiry (30-day window)"
|
|
ansible.builtin.command:
|
|
cmd: openssl x509 -checkend 2592000 -noout -in {{ _validate_tmpdir.path }}/{{ item }}.pem
|
|
loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: _expiry_check
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: "Step 3 — Get certificate details"
|
|
ansible.builtin.command:
|
|
cmd: openssl x509 -noout -subject -enddate -in {{ _validate_tmpdir.path }}/{{ item }}.pem
|
|
loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: _cert_details
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: "Step 3 — Build renewal status"
|
|
ansible.builtin.set_fact:
|
|
_certs_needing_renewal: >-
|
|
{{ _certs_needing_renewal | default([]) +
|
|
([certbot_certificates[idx]]
|
|
if _pem_check.results[idx].rc != 0 or _expiry_check.results[idx].rc != 0
|
|
else []) }}
|
|
loop: "{{ certbot_certificates | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
index_var: idx
|
|
label: "{{ item }}"
|
|
|
|
- name: "Step 3 — Display certificate status"
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
{{ item.item }}:
|
|
{{ 'INVALID PEM' if item.rc != 0 else
|
|
(_cert_details.results[idx].stdout | default('unknown'))
|
|
+ (' — NEEDS RENEWAL' if _expiry_check.results[idx].rc != 0 else ' — valid') }}
|
|
loop: "{{ _pem_check.results }}"
|
|
loop_control:
|
|
index_var: idx
|
|
label: "{{ item.item }}"
|
|
|
|
- name: "Step 3 — Summary"
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
{{ (_certs_needing_renewal | default([]) | length == 0)
|
|
| ternary(
|
|
'All ' + (certbot_certificates | length | string) + ' certificates are valid. Skipping renewal.',
|
|
(_certs_needing_renewal | default([]) | length | string) + ' certificate(s) need renewal: '
|
|
+ (_certs_needing_renewal | default([]) | map(attribute='cert_name') | join(', '))
|
|
) }}
|
|
|
|
- name: Clean up validation temp directory
|
|
ansible.builtin.file:
|
|
path: "{{ _validate_tmpdir.path }}"
|
|
state: absent
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Step 4: Request Certificates (conditional — only if needed)
|
|
# -------------------------------------------------------------------------
|
|
|
|
- name: "Step 4 — Request certificates that need renewal"
|
|
become: true
|
|
become_user: "{{ certbot_user }}"
|
|
ansible.builtin.shell: |
|
|
source {{ certbot_directory }}/.venv/bin/activate
|
|
certbot certonly \
|
|
--non-interactive \
|
|
--agree-tos \
|
|
--email {{ certbot_email }} \
|
|
--authenticator dns-namecheap \
|
|
--dns-namecheap-credentials {{ certbot_directory }}/credentials/namecheap.ini \
|
|
--dns-namecheap-propagation-seconds 120 \
|
|
--config-dir {{ certbot_directory }}/config \
|
|
--work-dir {{ certbot_directory }}/work \
|
|
--logs-dir {{ certbot_directory }}/logs \
|
|
--cert-name {{ item.cert_name }} \
|
|
--force-renewal \
|
|
{{ item.domains | map('regex_replace', '^', '-d ') | join(' ') }}
|
|
args:
|
|
executable: /bin/bash
|
|
loop: "{{ _certs_needing_renewal | default([]) }}"
|
|
loop_control:
|
|
label: "{{ item.cert_name }}"
|
|
register: _certbot_renewals
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: "Step 4 — Upload renewed certificates to vault"
|
|
become: true
|
|
ansible.builtin.command: "{{ certbot_directory }}/hooks/renewal-hook.sh"
|
|
environment:
|
|
RENEWED_LINEAGE: "{{ certbot_directory }}/config/live/{{ item.item.cert_name }}"
|
|
loop: "{{ _certbot_renewals.results | default([]) }}"
|
|
when: item.changed | default(false)
|
|
loop_control:
|
|
label: "{{ item.item.cert_name }}"
|
|
|
|
- name: "Step 4 — Update certificate metrics"
|
|
become: true
|
|
ansible.builtin.command: "{{ certbot_directory }}/hooks/cert-metrics.sh"
|
|
changed_when: false
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Step 5: Post-Validation (conditional — only after renewal)
|
|
# -------------------------------------------------------------------------
|
|
|
|
- name: "Step 5 — Re-read vault secrets after renewal"
|
|
ansible.builtin.set_fact:
|
|
"_post_{{ item | replace('-', '_') }}": "{{ lookup('oci_secret', item, vault_id=oci_vault_id) }}"
|
|
loop: "{{ _all_secret_names }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: Create post-validation temp directory
|
|
ansible.builtin.tempfile:
|
|
state: directory
|
|
prefix: vault-post-
|
|
register: _post_tmpdir
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: "Step 5 — Write renewed fullchain PEMs"
|
|
ansible.builtin.copy:
|
|
content: "{{ lookup('vars', ('_post_' + item | replace('.', '-') + '-fullchain') | replace('-', '_')) }}"
|
|
dest: "{{ _post_tmpdir.path }}/{{ item }}-fullchain.pem"
|
|
mode: '0600'
|
|
loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: "Step 5 — Write renewed privkey PEMs"
|
|
ansible.builtin.copy:
|
|
content: "{{ lookup('vars', ('_post_' + item | replace('.', '-') + '-privkey') | replace('-', '_')) }}"
|
|
dest: "{{ _post_tmpdir.path }}/{{ item }}-privkey.pem"
|
|
mode: '0600'
|
|
loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
no_log: true
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: "Step 5 — Verify renewed certificates are valid and not expiring"
|
|
ansible.builtin.command:
|
|
cmd: openssl x509 -checkend 2592000 -noout -in {{ _post_tmpdir.path }}/{{ item }}-fullchain.pem
|
|
loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
changed_when: false
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: "Step 5 — Verify cert/key modulus match"
|
|
ansible.builtin.shell: |
|
|
set -euo pipefail
|
|
cert_mod=$(openssl x509 -noout -modulus -in "{{ _post_tmpdir.path }}/{{ item }}-fullchain.pem" | openssl md5)
|
|
key_mod=$(openssl rsa -noout -modulus -in "{{ _post_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5 || \
|
|
openssl ec -noout -text -in "{{ _post_tmpdir.path }}/{{ item }}-privkey.pem" 2>/dev/null | openssl md5)
|
|
if [[ "${cert_mod}" != "${key_mod}" ]]; then
|
|
echo "MISMATCH: cert=${cert_mod} key=${key_mod}" >&2
|
|
exit 1
|
|
fi
|
|
echo "OK: modulus match"
|
|
args:
|
|
executable: /bin/bash
|
|
loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
changed_when: false
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: "Step 5 — Get renewed certificate expiry dates"
|
|
ansible.builtin.command:
|
|
cmd: openssl x509 -noout -subject -enddate -in {{ _post_tmpdir.path }}/{{ item }}-fullchain.pem
|
|
loop: "{{ _certs_needing_renewal | default([]) | map(attribute='cert_name') | list }}"
|
|
loop_control:
|
|
label: "{{ item }}"
|
|
register: _post_cert_details
|
|
changed_when: false
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: "Step 5 — Display renewed certificate status"
|
|
ansible.builtin.debug:
|
|
msg: "{{ item.item }}: {{ item.stdout }}"
|
|
loop: "{{ _post_cert_details.results | default([]) }}"
|
|
loop_control:
|
|
label: "{{ item.item }}"
|
|
when: _certs_needing_renewal | default([]) | length > 0
|
|
|
|
- name: Clean up post-validation temp directory
|
|
ansible.builtin.file:
|
|
path: "{{ _post_tmpdir.path }}"
|
|
state: absent
|
|
when: _post_tmpdir.path is defined
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Step 6: Schedule Verification (always runs)
|
|
# -------------------------------------------------------------------------
|
|
|
|
- name: "Step 6 — Verify certbot-renew.timer is enabled"
|
|
become: true
|
|
ansible.builtin.systemd:
|
|
name: certbot-renew.timer
|
|
enabled: true
|
|
state: started
|
|
register: _timer_status
|
|
|
|
- name: "Step 6 — Get timer status"
|
|
become: true
|
|
ansible.builtin.command:
|
|
cmd: systemctl show certbot-renew.timer --property=ActiveState,NextElapseUSecRealtime,LastTriggerUSec
|
|
register: _timer_details
|
|
changed_when: false
|
|
|
|
- name: "Step 6 — Display timer status"
|
|
ansible.builtin.debug:
|
|
msg: "certbot-renew.timer: {{ _timer_details.stdout_lines | join(', ') }}"
|
|
|
|
- name: "Step 6 — Update certificate metrics"
|
|
become: true
|
|
ansible.builtin.command: "{{ certbot_directory }}/hooks/cert-metrics.sh"
|
|
changed_when: false
|
|
|
|
- name: Final summary
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
Vault certificate check complete.
|
|
{{ (certbot_certificates | length) }} certificates checked.
|
|
{{ (_certs_needing_renewal | default([]) | length) }} renewed.
|
|
Renewal timer is {{ _timer_status.status.ActiveState | default('active') }}.
|