Compare commits

..

14 Commits

Author SHA1 Message Date
343b0e13d6 fix(certbot): harden renewal hook and fix permission errors
The renewal deploy-hook ran as the certbot user but lacked permissions to
write the combined PEM to /etc/haproxy/certs and to reload HAProxy,
causing silent failures that left a stale certificate in production until
expiry.

- Add certbot user to the haproxy group so it can write the combined PEM
- Grant certbot NOPASSWD sudo for `systemctl reload haproxy` only
- Make the Prometheus textfile directory group-owned by certbot (0775)
  so cert-metrics.sh can atomically update ssl_cert.prom
- Refactor renewal-hook.sh to always refresh cert metrics on exit via a
  trap, ensuring expiry alerts fire when the hook itself is broken
- Replace `set -e` with explicit error handling and structured logging
2026-06-17 09:58:46 -04:00
2f5a15eef5 chore(haproxy,terraform): harden haproxy stats and pin incus provider
- Add maxconn limit and HTTP timeouts to mitigate slowloris attacks
- Restrict stats endpoint to internal LAN and localhost only
- Hide HAProxy version on stats page
- Pin Incus Terraform provider to ~> 1.0 for stability
2026-06-09 22:52:23 -04:00
35061e3b6d Caliban: Update Rommie port 2026-06-07 08:14:55 -04:00
95682eca61 Caliban: configure Kernos mcp api key 2026-06-07 08:14:39 -04:00
711bbc093b Caliban: Update llama cpp ports 2026-06-07 08:14:18 -04:00
9bfa9a3617 feat(terraform): expand caliban port forwards and document port ranges
- Add proxy devices on caliban for SSH (25512), Postgres (25515),
  and three web ports (25516-25518) alongside existing RDP forward
- Remove HTTP/HTTPS proxy devices from prospero (now handled via
  HAProxy on titania)
- Document Incus port forwarding ranges (25510-25599) per host in
  ouranos.md and fix a typo
2026-06-07 06:40:42 -04:00
f2fb01ddd2 Titania: Add Hecate 2026-06-05 12:03:25 -04:00
c8ad7a0129 feat(terraform): add S3 storage bucket and credentials for Peitho 2026-06-01 13:47:18 -04:00
12b1db36f8 feat(haproxy): block internal observability endpoints from public traffic 2026-06-01 07:30:07 -04:00
77a82b4784 docs: update FreeCAD MCP README to document dual-service architecture 2026-05-31 10:13:43 -04:00
3893b91a55 feat(ansible): add CASE Field Systems MCP endpoint configuration
Configure FastAgent MCP server to connect to the CASE Field Systems
service over HTTP. Enables integration with LAN, SD Card, and
Provisioning workflows without authentication.

Uses dynamic Ansible variables for host and port to support
environment-specific deployments.
2026-05-30 10:19:24 -04:00
76a0e043e9 chore(ansible): add CASE agent configuration to kottos inventory
Introduce the CASE engineering agent by defining kottos_case_port
(24152) and updating the agents list comment. This extends the
systemd-managed pallas process configuration to include the CASE
runtime alongside existing Harper, Scotty, Research, and Tech
Research agents.
2026-05-30 09:44:07 -04:00
acf3419450 refactor(ansible): rename freecad_mcp env vars and rework deployment
- Drop `FREECAD_MCP_` prefix from env vars (use `FREECAD_*`)
- Update freecad_mcp port from 22032 to 22061
- Document that FreeCAD bridge is required for tool calls
- Replace kottos deployment with pallas deployment
2026-05-30 09:37:56 -04:00
bc431a3a2a refactor(alloy): remove athena syslog listener in favor of docker logs 2026-05-30 09:37:15 -04:00
37 changed files with 2382 additions and 956 deletions

View File

@@ -91,19 +91,11 @@ loki.source.syslog "angelia_logs" {
forward_to = [loki.write.default.receiver]
}
loki.source.syslog "athena_logs" {
listener {
address = "127.0.0.1:{{athena_syslog_port}}"
protocol = "tcp"
syslog_format = "{{ syslog_format }}"
labels = {
job = "athena",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
forward_to = [loki.write.default.receiver]
}
// Athena used to ship via syslog on {{athena_syslog_port}}; it logs to
// container stdout and is now picked up by the docker-socket block below
// (service="athena", component=app/mcp/nginx). The host_var is retained as a
// reserved port number but no listener binds to it — remove the var from the
// inventory when the rollout is verified.
loki.source.syslog "kairos_logs" {
listener {

View File

@@ -86,6 +86,19 @@
groups: "{{ certbot_group }}"
append: true
# The renewal deploy-hook runs as the certbot user and writes the combined
# PEM into the group-writable /etc/haproxy/certs (mode 0770, owned by the
# haproxy group). certbot must be a member of that group, otherwise the
# hook fails with "Permission denied" and HAProxy serves a stale cert until
# it expires.
- name: Add certbot user to the haproxy group
become: true
ansible.builtin.user:
name: "{{ certbot_user }}"
groups: "{{ haproxy_group }}"
append: true
when: "'haproxy' in services | default([])"
# -------------------------------------------------------------------------
# Directory Structure
# -------------------------------------------------------------------------
@@ -178,14 +191,32 @@
group: "{{ certbot_group }}"
mode: '0750'
# Group-owned by certbot and group-writable so cert-metrics.sh (run as the
# certbot user from the renewal hook) can atomically write ssl_cert.prom.
# node-exporter only needs to read these files, which 0775 still allows.
# The renewal hook reloads HAProxy after installing a new cert, but runs as
# the unprivileged certbot user. Grant exactly `systemctl reload haproxy`
# via sudo — nothing more. visudo validation prevents a malformed drop-in
# from locking out sudo.
- name: Allow certbot to reload HAProxy via sudo
become: true
ansible.builtin.copy:
dest: /etc/sudoers.d/certbot-haproxy-reload
content: "{{ certbot_user }} ALL=(root) NOPASSWD: /usr/bin/systemctl reload haproxy\n"
owner: root
group: root
mode: '0440'
validate: visudo -cf %s
when: "'haproxy' in services | default([])"
- name: Create Prometheus textfile directory
become: true
ansible.builtin.file:
path: "{{ prometheus_node_exporter_text_directory }}"
state: directory
owner: root
group: root
mode: '0755'
group: "{{ certbot_group }}"
mode: '0775'
- name: Template certificate metrics script
become: true

View File

@@ -8,7 +8,7 @@
# 3. Reloads HAProxy via systemd
# 4. Updates certificate metrics for Prometheus
set -euo pipefail
set -uo pipefail
# RENEWED_LINEAGE is set by certbot --deploy-hook or passed explicitly by deploy.yml
CERT_DIR="${RENEWED_LINEAGE:?RENEWED_LINEAGE must be set}"
@@ -16,37 +16,70 @@ CERT_NAME=$(basename "${CERT_DIR}")
HAPROXY_CERT="{{ haproxy_cert_path }}"
HAPROXY_DIR="{{ haproxy_directory }}"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting renewal hook for ${CERT_NAME}"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
fail() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2; }
# Always refresh Prometheus cert metrics on exit, even if installation below
# fails. The metrics drive the SSLCertificateExpired/ExpiringSoon alerts, so
# they must reflect reality precisely when the hook is broken — otherwise a
# failed renewal rots silently (which is exactly how the cert expired before).
# A non-zero exit is reported by certbot as a WARNING, surfacing the failure.
hook_status=0
finish() {
{{ certbot_directory }}/hooks/cert-metrics.sh || fail "cert-metrics.sh failed"
if [[ ${hook_status} -ne 0 ]]; then
fail "Renewal hook FAILED for ${CERT_NAME} — HAProxy is serving a STALE certificate"
fi
exit "${hook_status}"
}
trap finish EXIT
log "Starting renewal hook for ${CERT_NAME}"
# Check if certificate files exist
if [[ ! -f "${CERT_DIR}/fullchain.pem" ]] || [[ ! -f "${CERT_DIR}/privkey.pem" ]]; then
echo "ERROR: Certificate files not found in ${CERT_DIR}"
fail "Certificate files not found in ${CERT_DIR}"
hook_status=1
exit 1
fi
# Combine certificate and private key for HAProxy
# HAProxy requires both in a single PEM file
cat "${CERT_DIR}/fullchain.pem" "${CERT_DIR}/privkey.pem" > "${HAPROXY_CERT}.tmp"
# Combine certificate and private key for HAProxy (single PEM), writing to a
# temp file in the same directory and moving atomically so HAProxy never reads
# a partial file. A permission failure here is the documented failure mode.
if ! cat "${CERT_DIR}/fullchain.pem" "${CERT_DIR}/privkey.pem" > "${HAPROXY_CERT}.tmp"; then
fail "Could not write ${HAPROXY_CERT}.tmp — check ownership/permissions of $(dirname "${HAPROXY_CERT}")"
rm -f "${HAPROXY_CERT}.tmp"
hook_status=1
exit 1
fi
# Atomic move to avoid HAProxy reading partial file
mv "${HAPROXY_CERT}.tmp" "${HAPROXY_CERT}"
if ! mv "${HAPROXY_CERT}.tmp" "${HAPROXY_CERT}"; then
fail "Could not move combined PEM into place at ${HAPROXY_CERT}"
rm -f "${HAPROXY_CERT}.tmp"
hook_status=1
exit 1
fi
# Set permissions
chown {{ certbot_user }}:{{ haproxy_group }} "${HAPROXY_CERT}"
chmod 640 "${HAPROXY_CERT}"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Certificate combined and written to ${HAPROXY_CERT}"
log "Certificate combined and written to ${HAPROXY_CERT}"
# Reload HAProxy if running
# Reload HAProxy if running. The hook runs as the unprivileged certbot user,
# so the reload goes through sudo (a scoped sudoers rule grants exactly this
# command). sudo -n fails fast rather than blocking on a password prompt.
if systemctl is-active --quiet haproxy; then
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Reloading HAProxy..."
systemctl reload haproxy
echo "[$(date '+%Y-%m-%d %H:%M:%S')] HAProxy reloaded"
log "Reloading HAProxy..."
if sudo -n systemctl reload haproxy; then
log "HAProxy reloaded"
else
fail "HAProxy reload failed"
hook_status=1
exit 1
fi
else
echo "[$(date '+%Y-%m-%d %H:%M:%S')] HAProxy not running, skipping reload"
log "HAProxy not running, skipping reload"
fi
# Update certificate metrics
{{ certbot_directory }}/hooks/cert-metrics.sh
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Renewal hook completed successfully"
log "Renewal hook completed successfully"

469
ansible/comfyui/README.md Normal file
View File

@@ -0,0 +1,469 @@
<div align="center">
# ComfyUI
**The most powerful and modular AI engine for content creation.**
[![Website][website-shield]][website-url]
[![Dynamic JSON Badge][discord-shield]][discord-url]
[![Twitter][twitter-shield]][twitter-url]
[![Matrix][matrix-shield]][matrix-url]
<br>
[![][github-release-shield]][github-release-link]
[![][github-release-date-shield]][github-release-link]
[![][github-downloads-shield]][github-downloads-link]
[![][github-downloads-latest-shield]][github-downloads-link]
[matrix-shield]: https://img.shields.io/badge/Matrix-000000?style=flat&logo=matrix&logoColor=white
[matrix-url]: https://app.element.io/#/room/%23comfyui_space%3Amatrix.org
[website-shield]: https://img.shields.io/badge/ComfyOrg-4285F4?style=flat
[website-url]: https://www.comfy.org/
<!-- Workaround to display total user from https://github.com/badges/shields/issues/4500#issuecomment-2060079995 -->
[discord-shield]: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fdiscord.com%2Fapi%2Finvites%2Fcomfyorg%3Fwith_counts%3Dtrue&query=%24.approximate_member_count&logo=discord&logoColor=white&label=Discord&color=green&suffix=%20total
[discord-url]: https://discord.com/invite/comfyorg
[twitter-shield]: https://img.shields.io/twitter/follow/ComfyUI
[twitter-url]: https://x.com/ComfyUI
[github-release-shield]: https://img.shields.io/github/v/release/comfyanonymous/ComfyUI?style=flat&sort=semver
[github-release-link]: https://github.com/comfyanonymous/ComfyUI/releases
[github-release-date-shield]: https://img.shields.io/github/release-date/comfyanonymous/ComfyUI?style=flat
[github-downloads-shield]: https://img.shields.io/github/downloads/comfyanonymous/ComfyUI/total?style=flat
[github-downloads-latest-shield]: https://img.shields.io/github/downloads/comfyanonymous/ComfyUI/latest/total?style=flat&label=downloads%40latest
[github-downloads-link]: https://github.com/comfyanonymous/ComfyUI/releases
<img width="1590" height="795" alt="ComfyUI Screenshot" src="https://github.com/user-attachments/assets/36e065e0-bfae-4456-8c7f-8369d5ea48a2" />
<br>
</div>
ComfyUI is the AI creation engine for visual professionals who demand control over every model, every parameter, and every output. Its powerful and modular node graph interface empowers creatives to generate images, videos, 3D models, audio, and more...
- ComfyUI natively supports the latest open-source state of the art models.
- API nodes provide access to the best closed source models such as Nano Banana, Seedance, Hunyuan3D, etc.
- It is available on Windows, Linux, and macOS, locally with our [desktop application](https://www.comfy.org/download), our [portable install](#installing) or on our [cloud](https://www.comfy.org/cloud).
- The most sophisticated workflows can be exposed through a simple UI thanks to App Mode.
- It integrates seamlessly into production pipelines with our API endpoints.
## Get Started
### Local
#### [Desktop Application](https://www.comfy.org/download)
- The easiest way to get started.
- Available on Windows & macOS.
#### [Windows Portable Package](#installing)
- Get the latest commits and completely portable.
- Available on Windows.
#### [Manual Install](#manual-install-windows-linux)
Supports all operating systems and GPU types (NVIDIA, AMD, Intel, Apple Silicon, Ascend).
### Cloud
#### [Comfy Cloud](https://www.comfy.org/cloud)
- Our official paid cloud version for those who can't afford local hardware.
## Examples
See what ComfyUI can do with the [newer template workflows](https://comfy.org/workflows) or old [example workflows](https://comfyanonymous.github.io/ComfyUI_examples/).
## Features
- Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything.
- NOTE: There are many more models supported than the list below, if you want to see what is supported see our templates list inside ComfyUI.
- Image Models
- SD1.x, SD2.x ([unCLIP](https://comfyanonymous.github.io/ComfyUI_examples/unclip/))
- [SDXL](https://comfyanonymous.github.io/ComfyUI_examples/sdxl/), [SDXL Turbo](https://comfyanonymous.github.io/ComfyUI_examples/sdturbo/)
- [Stable Cascade](https://comfyanonymous.github.io/ComfyUI_examples/stable_cascade/)
- [SD3 and SD3.5](https://comfyanonymous.github.io/ComfyUI_examples/sd3/)
- Pixart Alpha and Sigma
- [AuraFlow](https://comfyanonymous.github.io/ComfyUI_examples/aura_flow/)
- [HunyuanDiT](https://comfyanonymous.github.io/ComfyUI_examples/hunyuan_dit/)
- [Flux](https://comfyanonymous.github.io/ComfyUI_examples/flux/)
- [Lumina Image 2.0](https://comfyanonymous.github.io/ComfyUI_examples/lumina2/)
- [HiDream](https://comfyanonymous.github.io/ComfyUI_examples/hidream/)
- [Qwen Image](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/)
- [Hunyuan Image 2.1](https://comfyanonymous.github.io/ComfyUI_examples/hunyuan_image/)
- [Flux 2](https://comfyanonymous.github.io/ComfyUI_examples/flux2/)
- [Z Image](https://comfyanonymous.github.io/ComfyUI_examples/z_image/)
- Ernie Image
- Image Editing Models
- [Omnigen 2](https://comfyanonymous.github.io/ComfyUI_examples/omnigen/)
- [Flux Kontext](https://comfyanonymous.github.io/ComfyUI_examples/flux/#flux-kontext-image-editing-model)
- [HiDream E1.1](https://comfyanonymous.github.io/ComfyUI_examples/hidream/#hidream-e11)
- [Qwen Image Edit](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/#edit-model)
- Video Models
- [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/)
- [Mochi](https://comfyanonymous.github.io/ComfyUI_examples/mochi/)
- [LTX-Video](https://comfyanonymous.github.io/ComfyUI_examples/ltxv/)
- [Hunyuan Video](https://comfyanonymous.github.io/ComfyUI_examples/hunyuan_video/)
- [Wan 2.1](https://comfyanonymous.github.io/ComfyUI_examples/wan/)
- [Wan 2.2](https://comfyanonymous.github.io/ComfyUI_examples/wan22/)
- [Hunyuan Video 1.5](https://docs.comfy.org/tutorials/video/hunyuan/hunyuan-video-1-5)
- Audio Models
- [Stable Audio](https://comfyanonymous.github.io/ComfyUI_examples/audio/)
- [ACE Step](https://comfyanonymous.github.io/ComfyUI_examples/audio/)
- 3D Models
- [Hunyuan3D 2.0](https://docs.comfy.org/tutorials/3d/hunyuan3D-2)
- Asynchronous Queue system
- Many optimizations: Only re-executes the parts of the workflow that changes between executions.
- Smart memory management: can automatically run large models on GPUs with as low as 1GB vram with smart offloading.
- Works even if you don't have a GPU with: ```--cpu``` (slow)
- Can load ckpt and safetensors: All in one checkpoints or standalone diffusion models, VAEs and CLIP models.
- Safe loading of ckpt, pt, pth, etc.. files.
- Embeddings/Textual inversion
- [Loras (regular, locon and loha)](https://comfyanonymous.github.io/ComfyUI_examples/lora/)
- [Hypernetworks](https://comfyanonymous.github.io/ComfyUI_examples/hypernetworks/)
- Loading full workflows (with seeds) from generated PNG, WebP and FLAC files.
- Saving/Loading workflows as Json files.
- Nodes interface can be used to create complex workflows like one for [Hires fix](https://comfyanonymous.github.io/ComfyUI_examples/2_pass_txt2img/) or much more advanced ones.
- [Area Composition](https://comfyanonymous.github.io/ComfyUI_examples/area_composition/)
- [Inpainting](https://comfyanonymous.github.io/ComfyUI_examples/inpaint/) with both regular and inpainting models.
- [ControlNet and T2I-Adapter](https://comfyanonymous.github.io/ComfyUI_examples/controlnet/)
- [Upscale Models (ESRGAN, ESRGAN variants, SwinIR, Swin2SR, etc...)](https://comfyanonymous.github.io/ComfyUI_examples/upscale_models/)
- [GLIGEN](https://comfyanonymous.github.io/ComfyUI_examples/gligen/)
- [Model Merging](https://comfyanonymous.github.io/ComfyUI_examples/model_merging/)
- [LCM models and Loras](https://comfyanonymous.github.io/ComfyUI_examples/lcm/)
- Latent previews with [TAESD](#how-to-show-high-quality-previews)
- Works fully offline: core will never download anything unless you want to.
- Optional API nodes to use paid models from external providers through the online [Comfy API](https://docs.comfy.org/tutorials/api-nodes/overview) disable with: `--disable-api-nodes`
- [Config file](extra_model_paths.yaml.example) to set the search paths for models.
Workflow examples can be found on the [Examples page](https://comfyanonymous.github.io/ComfyUI_examples/)
## Release Process
ComfyUI follows a weekly release cycle targeting Monday but this regularly changes because of model releases or large changes to the codebase. There are three interconnected repositories:
1. **[ComfyUI Core](https://github.com/comfyanonymous/ComfyUI)**
- Releases a new major stable version (e.g., v0.7.0) roughly every 2 weeks.
- Starting from v0.4.0 patch versions will be used for fixes backported onto the current stable release.
- Minor versions will be used for releases off the master branch.
- Patch versions may still be used for releases on the master branch in cases where a backport would not make sense.
- Commits outside of the stable release tags may be very unstable and break many custom nodes.
- Serves as the foundation for the desktop release
2. **[ComfyUI Desktop](https://github.com/Comfy-Org/desktop)**
- Builds a new release using the latest stable core version
3. **[ComfyUI Frontend](https://github.com/Comfy-Org/ComfyUI_frontend)**
- Every 2+ weeks frontend updates are merged into the core repository
- Features are frozen for the upcoming core release
- Development continues for the next release cycle
## Shortcuts
| Keybind | Explanation |
|------------------------------------|--------------------------------------------------------------------------------------------------------------------|
| `Ctrl` + `Enter` | Queue up current graph for generation |
| `Ctrl` + `Shift` + `Enter` | Queue up current graph as first for generation |
| `Ctrl` + `Alt` + `Enter` | Cancel current generation |
| `Ctrl` + `Z`/`Ctrl` + `Y` | Undo/Redo |
| `Ctrl` + `S` | Save workflow |
| `Ctrl` + `O` | Load workflow |
| `Ctrl` + `A` | Select all nodes |
| `Alt `+ `C` | Collapse/uncollapse selected nodes |
| `Ctrl` + `M` | Mute/unmute selected nodes |
| `Ctrl` + `B` | Bypass selected nodes (acts like the node was removed from the graph and the wires reconnected through) |
| `Delete`/`Backspace` | Delete selected nodes |
| `Ctrl` + `Backspace` | Delete the current graph |
| `Space` | Move the canvas around when held and moving the cursor |
| `Ctrl`/`Shift` + `Click` | Add clicked node to selection |
| `Ctrl` + `C`/`Ctrl` + `V` | Copy and paste selected nodes (without maintaining connections to outputs of unselected nodes) |
| `Ctrl` + `C`/`Ctrl` + `Shift` + `V` | Copy and paste selected nodes (maintaining connections from outputs of unselected nodes to inputs of pasted nodes) |
| `Shift` + `Drag` | Move multiple selected nodes at the same time |
| `Ctrl` + `D` | Load default graph |
| `Alt` + `+` | Canvas Zoom in |
| `Alt` + `-` | Canvas Zoom out |
| `Ctrl` + `Shift` + LMB + Vertical drag | Canvas Zoom in/out |
| `P` | Pin/Unpin selected nodes |
| `Ctrl` + `G` | Group selected nodes |
| `Q` | Toggle visibility of the queue |
| `H` | Toggle visibility of history |
| `R` | Refresh graph |
| `F` | Show/Hide menu |
| `.` | Fit view to selection (Whole graph when nothing is selected) |
| Double-Click LMB | Open node quick search palette |
| `Shift` + Drag | Move multiple wires at once |
| `Ctrl` + `Alt` + LMB | Disconnect all wires from clicked slot |
`Ctrl` can also be replaced with `Cmd` instead for macOS users
# Installing
## Windows Portable
There is a portable standalone build for Windows that should work for running on Nvidia GPUs or for running on your CPU only on the [releases page](https://github.com/comfyanonymous/ComfyUI/releases).
### [Direct link to download](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_nvidia.7z)
Simply download, extract with [7-Zip](https://7-zip.org) or with the windows explorer on recent windows versions and run. For smaller models you normally only need to put the checkpoints (the huge ckpt/safetensors files) in: ComfyUI\models\checkpoints but many of the larger models have multiple files. Make sure to follow the instructions to know which subfolder to put them in ComfyUI\models\
If you have trouble extracting it, right click the file -> properties -> unblock
The portable above currently comes with python 3.13 and pytorch cuda 13.0. Update your Nvidia drivers if it doesn't start.
#### All Official Portable Downloads:
[Portable for AMD GPUs](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_amd.7z)
[Portable for Intel GPUs](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_intel.7z)
[Portable for Nvidia GPUs](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_nvidia.7z) (supports 20 series and above).
[Portable for Nvidia GPUs with pytorch cuda 12.6 and python 3.12](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_nvidia_cu126.7z) (Supports Nvidia 10 series and older GPUs).
#### How do I share models between another UI and ComfyUI?
See the [Config file](extra_model_paths.yaml.example) to set the search paths for models. In the standalone windows build you can find this file in the ComfyUI directory. Rename this file to extra_model_paths.yaml and edit it with your favorite text editor.
## [comfy-cli](https://docs.comfy.org/comfy-cli/getting-started)
You can install and start ComfyUI using comfy-cli:
```bash
pip install comfy-cli
comfy install
```
## Manual Install (Windows, Linux)
Python 3.14 works but some custom nodes may have issues. The free threaded variant works but some dependencies will enable the GIL so it's not fully supported.
Python 3.13 is very well supported. If you have trouble with some custom node dependencies on 3.13 you can try 3.12
torch 2.4 and above is supported but some features and optimizations might only work on newer versions. We generally recommend using the latest major version of pytorch with the latest cuda version unless it is less than 2 weeks old.
### Instructions:
Git clone this repo.
Put your SD checkpoints (the huge ckpt/safetensors files) in: models/checkpoints
Put your VAE in: models/vae
### AMD GPUs (Linux)
AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.2```
This is the command to install the nightly with ROCm 7.2 which might have some performance improvements:
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm7.2```
### AMD GPUs (Experimental: Windows and Linux), RDNA 3, 3.5 and 4 only.
These have less hardware support than the builds above but they work on windows. You also need to install the pytorch version specific to your hardware.
RDNA 3 (RX 7000 series):
```pip install --pre torch torchvision torchaudio --index-url https://rocm.nightlies.amd.com/v2/gfx110X-all/```
RDNA 3.5 (Strix halo/Ryzen AI Max+ 365):
```pip install --pre torch torchvision torchaudio --index-url https://rocm.nightlies.amd.com/v2/gfx1151/```
RDNA 4 (RX 9000 series):
```pip install --pre torch torchvision torchaudio --index-url https://rocm.nightlies.amd.com/v2/gfx120X-all/```
### Intel GPUs (Windows and Linux)
Intel Arc GPU users can install native PyTorch with torch.xpu support using pip. More information can be found [here](https://pytorch.org/docs/main/notes/get_start_xpu.html)
1. To install PyTorch xpu, use the following command:
```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/xpu```
This is the command to install the Pytorch xpu nightly which might have some performance improvements:
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu```
### NVIDIA
Nvidia users should install stable pytorch using this command:
```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu130```
This is the command to install pytorch nightly instead which might have performance improvements.
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu132```
#### Troubleshooting
If you get the "Torch not compiled with CUDA enabled" error, uninstall torch with:
```pip uninstall torch```
And install it again with the command above.
### Dependencies
Install the dependencies by opening your terminal inside the ComfyUI folder and:
```pip install -r requirements.txt```
After this you should have everything installed and can proceed to running ComfyUI.
### Others:
#### Apple Mac silicon
You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS version.
1. Install pytorch nightly. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide (make sure to install the latest pytorch nightly).
1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux.
1. Install the ComfyUI [dependencies](#dependencies). If you have another Stable Diffusion UI [you might be able to reuse the dependencies](#i-already-have-another-ui-for-stable-diffusion-installed-do-i-really-have-to-install-all-of-these-dependencies).
1. Launch ComfyUI by running `python main.py`
> **Note**: Remember to add your models, VAE, LoRAs etc. to the corresponding Comfy folders, as discussed in [ComfyUI manual installation](#manual-install-windows-linux).
#### Ascend NPUs
For models compatible with Ascend Extension for PyTorch (torch_npu). To get started, ensure your environment meets the prerequisites outlined on the [installation](https://ascend.github.io/docs/sources/ascend/quick_install.html) page. Here's a step-by-step guide tailored to your platform and installation method:
1. Begin by installing the recommended or newer kernel version for Linux as specified in the Installation page of torch-npu, if necessary.
2. Proceed with the installation of Ascend Basekit, which includes the driver, firmware, and CANN, following the instructions provided for your specific platform.
3. Next, install the necessary packages for torch-npu by adhering to the platform-specific instructions on the [Installation](https://ascend.github.io/docs/sources/pytorch/install.html#pytorch) page.
4. Finally, adhere to the [ComfyUI manual installation](#manual-install-windows-linux) guide for Linux. Once all components are installed, you can run ComfyUI as described earlier.
#### Cambricon MLUs
For models compatible with Cambricon Extension for PyTorch (torch_mlu). Here's a step-by-step guide tailored to your platform and installation method:
1. Install the Cambricon CNToolkit by adhering to the platform-specific instructions on the [Installation](https://www.cambricon.com/docs/sdk_1.15.0/cntoolkit_3.7.2/cntoolkit_install_3.7.2/index.html)
2. Next, install the PyTorch(torch_mlu) following the instructions on the [Installation](https://www.cambricon.com/docs/sdk_1.15.0/cambricon_pytorch_1.17.0/user_guide_1.9/index.html)
3. Launch ComfyUI by running `python main.py`
#### Iluvatar Corex
For models compatible with Iluvatar Extension for PyTorch. Here's a step-by-step guide tailored to your platform and installation method:
1. Install the Iluvatar Corex Toolkit by adhering to the platform-specific instructions on the [Installation](https://support.iluvatar.com/#/DocumentCentre?id=1&nameCenter=2&productId=520117912052801536)
2. Launch ComfyUI by running `python main.py`
## [ComfyUI-Manager](https://github.com/Comfy-Org/ComfyUI-Manager/tree/manager-v4)
**ComfyUI-Manager** is an extension that allows you to easily install, update, and manage custom nodes for ComfyUI.
### Setup
1. Install the manager dependencies:
```bash
pip install -r manager_requirements.txt
```
2. Enable the manager with the `--enable-manager` flag when running ComfyUI:
```bash
python main.py --enable-manager
```
### Command Line Options
| Flag | Description |
|------|-------------|
| `--enable-manager` | Enable ComfyUI-Manager |
| `--enable-manager-legacy-ui` | Use the legacy manager UI instead of the new UI (implies `--enable-manager`) |
| `--disable-manager-ui` | Disable the manager UI and endpoints while keeping background features like security checks and scheduled installation completion (requires `--enable-manager`) |
# Running
```python main.py```
### For AMD cards not officially supported by ROCm
Try running it with this command if you have issues:
For 6700, 6600 and maybe other RDNA2 or older: ```HSA_OVERRIDE_GFX_VERSION=10.3.0 python main.py```
For AMD 7600 and maybe other RDNA3 cards: ```HSA_OVERRIDE_GFX_VERSION=11.0.0 python main.py```
### AMD ROCm Tips
You can enable experimental memory efficient attention on recent pytorch in ComfyUI on some AMD GPUs using this command, it should already be enabled by default on RDNA3. If this improves speed for you on latest pytorch on your GPU please report it so that I can enable it by default.
```TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 python main.py --use-pytorch-cross-attention```
You can also try setting this env variable `PYTORCH_TUNABLEOP_ENABLED=1` which might speed things up at the cost of a very slow initial run.
# Notes
Only parts of the graph that have an output with all the correct inputs will be executed.
Only parts of the graph that change from each execution to the next will be executed, if you submit the same graph twice only the first will be executed. If you change the last part of the graph only the part you changed and the part that depends on it will be executed.
Dragging a generated png on the webpage or loading one will give you the full workflow including seeds that were used to create it.
You can use () to change emphasis of a word or phrase like: (good code:1.2) or (bad code:0.8). The default emphasis for () is 1.1. To use () characters in your actual prompt escape them like \\( or \\).
You can use {day|night}, for wildcard/dynamic prompts. With this syntax "{wild|card|test}" will be randomly replaced by either "wild", "card" or "test" by the frontend every time you queue the prompt. To use {} characters in your actual prompt escape them like: \\{ or \\}.
Dynamic prompts also support C-style comments, like `// comment` or `/* comment */`.
To use a textual inversion concepts/embeddings in a text prompt put them in the models/embeddings directory and use them in the CLIPTextEncode node like this (you can omit the .pt extension):
```embedding:embedding_filename.pt```
## How to show high-quality previews?
Use ```--preview-method auto``` to enable previews.
The default installation includes a fast latent preview method that's low-resolution. To enable higher-quality previews with [TAESD](https://github.com/madebyollin/taesd), download the [taesd_decoder.pth, taesdxl_decoder.pth, taesd3_decoder.pth and taef1_decoder.pth](https://github.com/madebyollin/taesd/) and place them in the `models/vae_approx` folder. Once they're installed, restart ComfyUI and launch it with `--preview-method taesd` to enable high-quality previews.
## How to use TLS/SSL?
Generate a self-signed certificate (not appropriate for shared/production use) and key by running the command: `openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -sha256 -days 3650 -nodes -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=CommonNameOrHostname"`
Use `--tls-keyfile key.pem --tls-certfile cert.pem` to enable TLS/SSL, the app will now be accessible with `https://...` instead of `http://...`.
> Note: Windows users can use [alexisrolland/docker-openssl](https://github.com/alexisrolland/docker-openssl) or one of the [3rd party binary distributions](https://wiki.openssl.org/index.php/Binaries) to run the command example above.
<br/><br/>If you use a container, note that the volume mount `-v` can be a relative path so `... -v ".\:/openssl-certs" ...` would create the key & cert files in the current directory of your command prompt or powershell terminal.
## Support and dev channel
[Discord](https://comfy.org/discord): Try the #help or #feedback channels.
[Matrix space: #comfyui_space:matrix.org](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) (it's like discord but open source).
See also: [https://www.comfy.org/](https://www.comfy.org/)
> _psst — we're hiring!_ Help build ComfyUI: [comfy.org/careers](https://www.comfy.org/careers)
## Frontend Development
As of August 15, 2024, we have transitioned to a new frontend, which is now hosted in a separate repository: [ComfyUI Frontend](https://github.com/Comfy-Org/ComfyUI_frontend). The compiled JS files (from TS/Vue) are published to [pypi](https://pypi.org/project/comfyui-frontend-package) and installed as a dependency in ComfyUI.
### Reporting Issues and Requesting Features
For any bugs, issues, or feature requests related to the frontend, please use the [ComfyUI Frontend repository](https://github.com/Comfy-Org/ComfyUI_frontend). This will help us manage and address frontend-specific concerns more efficiently.
### Using the Latest Frontend
The new frontend is now the default for ComfyUI. However, please note:
1. The frontend in the main ComfyUI repository is updated fortnightly.
2. Daily releases are available in the separate frontend repository.
To use the most up-to-date frontend version:
1. For the latest daily release, launch ComfyUI with this command line argument:
```
--front-end-version Comfy-Org/ComfyUI_frontend@latest
```
2. For a specific version, replace `latest` with the desired version number:
```
--front-end-version Comfy-Org/ComfyUI_frontend@1.2.2
```
This approach allows you to easily switch between the stable fortnightly release and the cutting-edge daily updates, or even specific versions for testing purposes.
# QA
### Which GPU should I buy for this?
[See this page for some recommendations](https://github.com/comfyanonymous/ComfyUI/wiki/Which-GPU-should-I-buy-for-ComfyUI)

View File

@@ -4,18 +4,17 @@
# =============================================================================
# MCP Transport Configuration
# =============================================================================
FREECAD_MCP_TRANSPORT=http
FREECAD_MCP_HTTP_PORT={{ freecad_mcp_port }}
FREECAD_TRANSPORT=http
FREECAD_HTTP_PORT={{ freecad_mcp_port }}
# =============================================================================
# FreeCAD Connection Mode
# =============================================================================
FREECAD_MCP_MODE={{ freecad_mcp_mode | default('xmlrpc') }}
FREECAD_MCP_XMLRPC_HOST={{ freecad_mcp_xmlrpc_host | default('localhost') }}
FREECAD_MCP_XMLRPC_PORT={{ freecad_mcp_xmlrpc_port | default('9875') }}
FREECAD_MCP_TIMEOUT_MS={{ freecad_mcp_timeout_ms | default('30000') }}
FREECAD_MODE={{ freecad_mcp_mode | default('xmlrpc') }}
FREECAD_XMLRPC_PORT={{ freecad_mcp_xmlrpc_port | default('9875') }}
FREECAD_TIMEOUT_MS={{ freecad_mcp_timeout_ms | default('30000') }}
# =============================================================================
# Logging
# =============================================================================
FREECAD_MCP_LOG_LEVEL={{ freecad_mcp_log_level | default('INFO') }}
FREECAD_LOG_LEVEL={{ freecad_mcp_log_level | default('INFO') }}

View File

@@ -1,51 +1,104 @@
# FreeCAD Robust MCP Server — Ansible Deployment
Deploys the [FreeCAD Robust MCP Server](https://pypi.org/project/freecad-robust-mcp/)
to Caliban as a systemd service with HTTP transport, ready for MCP Switchboard
consumption.
to Caliban as **two** systemd services:
- **`freecad-mcp.service`** — the MCP server (HTTP/streamable-http transport on
`:22061`), pip-installed into a venv under `/srv/freecad-mcp`, run as the
hardened `harper` service user.
- **`freecad-mcp-bridge.service`** — FreeCAD itself running in **GUI** mode on
the XRDP desktop (display `:10`), exposing the XML-RPC bridge on
`localhost:9875`. Run as `robert` (the `principal_user`, who owns the X
session), from source staged as a tarball.
The MCP server connects to the bridge over `localhost:9875`; the bridge in turn
drives FreeCAD. The two halves rendezvous only on that local port.
## Architecture
```
┌─────────────────────────────────────────────────┐
│ caliban.incus │
│ │
│ ┌──────────────────────┐ │
│ │ freecad-mcp.service │ │
│ │ (streamable-http) │◄─── :22032 ──────────┤◄── MCP Switchboard
│ │ venv + PyPI package │ │ (oberon.incus)
│ └─────────────────────┘ │
│ xmlrpc :9875
┌──────────────────────┐
│ │ FreeCAD (future) │
│ │ XML-RPC server │
└──────────────────────┘
└─────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────
│ caliban.incus
│ ┌──────────────────────┐
│ │ freecad-mcp.service │
│ │ (streamable-http) │◄─── :22061 ────────────────────┤◄── MCP Client
│ │ venv + PyPI package │ (user: harper, hardened) │
│ └─────────────────────┘
│ xmlrpc localhost:9875
┌──────────────────────────────┐
│ freecad-mcp-bridge.service │
│ │ /usr/bin/freecad (GUI) DISPLAY=:10 (XRDP)
│ │ startup_bridge.py │ user: robert
│ XML-RPC :9875 / socket :9876│
└──────────────────────────────┘ │
└──────────────────────────────────────────────────────────┘
```
## Two services, two users (by design)
| Service | User | Transport / port | Hardened | Needs X |
| ---------------------------- | -------- | ----------------------- | -------- | ------- |
| `freecad-mcp.service` | `harper` | HTTP `:22061` | yes | no |
| `freecad-mcp-bridge.service` | `robert` | XML-RPC `:9875` (+ 9876) | no | yes (`:10`) |
The bridge runs as `robert` because it attaches to the standard XRDP display
`:10`, owned by `robert` with Xauthority `/home/robert/.Xauthority`. It cannot
be hardened like the server unit — it needs the user's X session and home.
## How the bridge starts (no `just`/`mise`/`uv` needed)
The bridge runs **inside FreeCAD's own Python interpreter** via
`/usr/bin/freecad <startup_bridge.py>`. The README "Option B"
(`just freecad::run-gui`) in the upstream repo is only a launcher wrapper that
locates FreeCAD and runs that same script — `just`, `mise`, and `uv` are not
required for the bridge.
The bridge scripts are **not** shipped in the pip wheel (it packages only
`src/freecad_mcp`). They live in the git repo under
`freecad/RobustMCPBridge/freecad_mcp_bridge/`, so the bridge is delivered
separately as a staged tarball (see Deployment below).
> **GUI vs headless:** We run GUI mode to keep the GUI-only tools (screenshots,
> object color, visibility, camera). `freecadcmd <blocking_bridge.py>` would run
> headless without those tools — not used here.
> **Python version:** FreeCAD 1.0.0 on Caliban uses the system Python (3.13),
> not a bundled 3.11. The upstream ABI-match warning applies only to *embedded*
> mode (importing `FreeCAD` into an external interpreter). We run scripts inside
> FreeCAD and the bridge is pure stdlib, so the version mismatch is a non-issue.
## Lazy connect: a green server healthcheck is not "FreeCAD reachable"
`freecad-mcp.service` starts and answers the MCP `initialize` handshake **without**
the bridge running — the XML-RPC connection to FreeCAD is only attempted on the
first CAD tool call. So the server playbook's `initialize` check proves
"transport up", **not** "FreeCAD reachable". The bridge playbook's validation
(below) is what proves the full chain.
## Prerequisites
- Caliban host in Ansible inventory (already exists in Ouranos)
- Python 3.11+ on Caliban (already present)
- Caliban host in the `freecad_mcp` inventory group (already configured).
- `python3` + `python3-venv` on Caliban (installed by the playbook).
- `freecad` package on Caliban (installed by the playbook).
- The XRDP display `:10` running, owned by `robert` (the standard Ouranos RDP
desktop — not configured here, it is always present).
## Deployment
### 1. Copy playbook files to Ouranos
Copy the contents of this directory into your Ouranos repo:
## Files in this role
```
ansible/freecad_mcp/
├── deploy.yml
├── .env.j2
── freecad-mcp.service.j2
├── deploy.yml # Two plays: MCP server + GUI bridge
├── stage.yml # Clones the fork + builds the bridge tarball
── .env.j2 # MCP server env (FREECAD_* vars)
├── freecad-mcp.service.j2 # MCP server unit (harper, hardened)
└── freecad-mcp-bridge.service.j2 # FreeCAD GUI bridge unit (robert, :10)
```
### 2. Add inventory group
## Inventory
Add to `ansible/inventory/hosts`:
`ansible/inventory/hosts` (already present):
```yaml
freecad_mcp:
@@ -53,70 +106,101 @@ freecad_mcp:
caliban.incus:
```
### 3. Add host variables
Add to `ansible/inventory/host_vars/caliban.incus.yml`:
Host vars in `ansible/inventory/host_vars/caliban.incus.yml`:
```yaml
# FreeCAD Robust MCP Server
freecad_mcp_user: harper
freecad_mcp_group: harper
freecad_mcp_directory: /srv/freecad-mcp
freecad_mcp_port: 22032
freecad_mcp_version: "0.5.0"
freecad_mcp_port: 22061
freecad_mcp_xmlrpc_port: 9875
freecad_mcp_socket_port: 9876
# FreeCAD MCP Bridge (GUI, runs as principal_user on the XRDP display)
freecad_mcp_bridge_directory: "/home/{{ principal_user }}/freecad-mcp-bridge"
freecad_mcp_bridge_display: ":10"
```
Update `services` list:
Group vars in `ansible/inventory/group_vars/all/vars.yml`:
```yaml
services:
- alloy
- caliban
- docker
- freecad_mcp
- kernos
freecad_mcp_version: 0.6.1 # PyPI version pin (server install)
freecad_mcp_git_ref: "main" # fork ref for BOTH the pip install and the staged bridge tarball
```
### 4. Run the playbook
## Deployment
The bridge source is delivered via the staging pattern: cloned on the Ansible
controller, packed with `git archive`, and unpacked on the host (no deploy keys
on Caliban). Stage first, then deploy:
```bash
cd ~/git/ouranos/ansible
source ~/env/ouranos/bin/activate
# 1. Build the bridge tarball on the controller (~/rel/freecad_mcp_bridge_<ref>.tar)
ansible-playbook freecad_mcp/stage.yml
# 2. Deploy the MCP server (idempotent) + the GUI bridge
ansible-playbook freecad_mcp/deploy.yml
```
`stage.yml` clones/pulls the fork into `~/gh/freecad-addon-robust-mcp-server` at
`freecad_mcp_git_ref` and `git archive`s it to
`~/rel/freecad_mcp_bridge_<ref>.tar`. `deploy.yml` unpacks that into
`~robert/freecad-mcp-bridge` and points the bridge unit at
`freecad/RobustMCPBridge/freecad_mcp_bridge/startup_bridge.py`.
## Upgrading
To upgrade to a new PyPI version, update `freecad_mcp_version` in host_vars
and re-run the playbook. The pip install task will detect the version change
and the handler will restart the service.
- **MCP server:** bump `freecad_mcp_version` (PyPI) and/or `freecad_mcp_git_ref`
in group vars, re-run `deploy.yml`. The pip task detects the change and the
handler restarts `freecad-mcp`.
- **Bridge:** re-run `stage.yml` (rebuilds the tarball from the latest fork
ref), then `deploy.yml`. The `unarchive` change notifies the
`restart freecad-mcp-bridge` handler.
## Validation
The playbook automatically validates the deployment by:
The playbooks validate automatically:
1. Waiting for the HTTP port to become available
2. Sending an MCP `initialize` JSON-RPC request to `/mcp`
3. Verifying a 200 response
- **Server play:** waits for `:22061`, sends an MCP `initialize` request to
`/mcp`, expects HTTP 200 (transport-level only — see lazy-connect note above).
- **Bridge play:** waits for `:9875`, then calls the bridge's XML-RPC `execute`
with `_result_ = bool(FreeCAD.GuiUp)` and asserts the result is `True`
proving FreeCAD is up **in GUI mode**, end to end.
You can also manually test:
Manual checks:
```bash
curl -X POST http://caliban.incus:22032/mcp \
# Transport up (no FreeCAD needed):
curl -X POST http://caliban.incus:22061/mcp \
-H "Content-Type: application/json" \
-d '{"jsonrpc":"2.0","method":"initialize","id":1,"params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"curl","version":"1.0.0"}}}'
# Bridge listening + in GUI mode:
ss -ltnp | grep 9875
python3 -c 'import xmlrpc.client as x; print(x.ServerProxy("http://localhost:9875", allow_none=True).execute("_result_ = bool(FreeCAD.GuiUp)"))'
```
## Service Management
```bash
# On Caliban
# MCP server
sudo systemctl status freecad-mcp
sudo systemctl restart freecad-mcp
sudo journalctl -u freecad-mcp -f
# FreeCAD GUI bridge
sudo systemctl status freecad-mcp-bridge
sudo systemctl restart freecad-mcp-bridge
sudo journalctl -u freecad-mcp-bridge -f
```
## Security
The systemd service runs with hardened settings:
The **MCP server** unit (`freecad-mcp.service`, user `harper`) is hardened:
| Setting | Value | Rationale |
|---------|-------|-----------|
@@ -126,5 +210,15 @@ The systemd service runs with hardened settings:
| `PrivateTmp` | `true` | Isolated /tmp namespace |
| `ReadWritePaths` | `/srv/freecad-mcp` | Only app directory is writable |
This is significantly more hardened than the Kernos service (which needs
broad filesystem access for shell commands).
The **bridge** unit (`freecad-mcp-bridge.service`, user `robert`) is **not**
hardened: FreeCAD GUI needs the user's X session, `.Xauthority`, and FreeCAD
config in the home directory. It binds XML-RPC/socket on `localhost` only.
## Known limitation
The bridge depends on the XRDP `:10` session (owned by `robert`). `Restart=on-failure`
recovers crashes, but **not** loss of the X display — if that session restarts,
restart `freecad-mcp-bridge` afterward. Auto-tying the two is a possible
follow-up.

View File

@@ -216,3 +216,102 @@
ansible.builtin.systemd:
name: freecad-mcp
state: restarted
# =============================================================================
# FreeCAD MCP Bridge (GUI) — runs FreeCAD on the XRDP desktop as principal_user,
# exposing the XML-RPC bridge on localhost:9875 that the MCP server connects to.
# =============================================================================
- name: Deploy FreeCAD MCP Bridge (GUI)
hosts: freecad_mcp
tasks:
- name: Ensure FreeCAD is installed
become: true
ansible.builtin.apt:
name: [freecad, tar]
state: present
update_cache: true
- name: Create FreeCAD MCP bridge directory
become: true
become_user: "{{ principal_user }}"
ansible.builtin.file:
path: "{{ freecad_mcp_bridge_directory }}"
state: directory
mode: '0755'
- name: Transfer and extract FreeCAD MCP bridge release
become: true
become_user: "{{ principal_user }}"
ansible.builtin.unarchive:
src: "~/rel/freecad_mcp_bridge_{{ freecad_mcp_git_ref }}.tar"
dest: "{{ freecad_mcp_bridge_directory }}"
notify: restart freecad-mcp-bridge
- name: Template FreeCAD MCP bridge systemd service
become: true
ansible.builtin.template:
src: freecad-mcp-bridge.service.j2
dest: /etc/systemd/system/freecad-mcp-bridge.service
owner: root
group: root
mode: '644'
notify:
- reload systemd
- restart freecad-mcp-bridge
- name: Enable and start freecad-mcp-bridge service
become: true
ansible.builtin.systemd:
name: freecad-mcp-bridge
enabled: true
state: started
daemon_reload: true
- name: Flush handlers to restart bridge before validation
ansible.builtin.meta: flush_handlers
- name: Wait for FreeCAD XML-RPC bridge to listen
ansible.builtin.wait_for:
port: "{{ freecad_mcp_xmlrpc_port | default(9875) }}"
host: localhost
delay: 5
timeout: 60
- name: Verify bridge is in GUI mode (FreeCAD.GuiUp via XML-RPC execute)
ansible.builtin.command:
argv:
- python3
- -c
- |
import sys, xmlrpc.client
proxy = xmlrpc.client.ServerProxy(
"http://localhost:{{ freecad_mcp_xmlrpc_port | default(9875) }}", allow_none=True)
resp = proxy.execute("_result_ = bool(FreeCAD.GuiUp)")
if not (resp.get("success") and resp.get("result") is True):
sys.exit("Bridge reachable but not in GUI mode: %r" % resp)
print("FreeCAD bridge GUI mode confirmed")
register: bridge_gui_check
retries: 5
delay: 5
until: bridge_gui_check.rc == 0
changed_when: false
- name: Display bridge info
ansible.builtin.debug:
msg: >-
FreeCAD MCP Bridge running in GUI mode on {{ inventory_hostname }},
XML-RPC localhost:{{ freecad_mcp_xmlrpc_port | default(9875) }}
handlers:
- name: reload systemd
become: true
ansible.builtin.systemd:
daemon_reload: true
- name: restart freecad-mcp-bridge
become: true
ansible.builtin.systemd:
name: freecad-mcp-bridge
state: restarted

View File

@@ -0,0 +1,21 @@
[Unit]
Description=FreeCAD MCP XML-RPC Bridge (GUI)
After=network.target
[Service]
Type=simple
User={{ principal_user }}
WorkingDirectory={{ freecad_mcp_bridge_directory }}
Environment=DISPLAY={{ freecad_mcp_bridge_display }}
Environment=XAUTHORITY=/home/{{ principal_user }}/.Xauthority
Environment=FREECAD_XMLRPC_PORT={{ freecad_mcp_xmlrpc_port | default('9875') }}
Environment=FREECAD_SOCKET_PORT={{ freecad_mcp_socket_port | default('9876') }}
ExecStart=/usr/bin/freecad {{ freecad_mcp_bridge_directory }}/freecad/RobustMCPBridge/freecad_mcp_bridge/startup_bridge.py
Restart=on-failure
RestartSec=10
StandardOutput=journal
StandardError=journal
SyslogIdentifier=freecad-mcp-bridge
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,46 @@
---
- name: Stage FreeCAD MCP bridge release tarball
hosts: localhost
gather_facts: false
vars:
freecad_mcp_archive: "{{rel_dir}}/freecad_mcp_bridge_{{freecad_mcp_git_ref}}.tar"
freecad_mcp_repo_url: "git@github.com:heluca/freecad-addon-robust-mcp-server.git"
freecad_mcp_repo_dir: "{{github_dir}}/freecad-addon-robust-mcp-server"
tasks:
- name: Ensure release directory exists
file:
path: "{{rel_dir}}"
state: directory
mode: '755'
- name: Ensure github directory exists
file:
path: "{{github_dir}}"
state: directory
mode: '755'
- name: Clone freecad-addon-robust-mcp-server repository if not present
ansible.builtin.git:
repo: "{{freecad_mcp_repo_url}}"
dest: "{{freecad_mcp_repo_dir}}"
version: "{{freecad_mcp_git_ref}}"
accept_hostkey: true
register: freecad_mcp_clone
- name: Fetch all remote branches and tags
ansible.builtin.command: git fetch --all
args:
chdir: "{{freecad_mcp_repo_dir}}"
when: freecad_mcp_clone is not changed
- name: Pull latest changes
ansible.builtin.command: git pull
args:
chdir: "{{freecad_mcp_repo_dir}}"
when: freecad_mcp_clone is not changed
- name: Create FreeCAD MCP bridge archive for specified release
ansible.builtin.command: git archive -o "{{freecad_mcp_archive}}" "{{freecad_mcp_git_ref}}"
args:
chdir: "{{freecad_mcp_repo_dir}}"

View File

@@ -18,6 +18,7 @@
- git-lfs
- curl
- memcached
- acl
state: present
update_cache: true
@@ -187,8 +188,8 @@
--config {{ gitea_config_file }}
--name "{{ gitea_oauth_name }}"
--provider openidConnect
--key "{{ gitea_oauth2_client_id }}"
--secret "{{ gitea_oauth2_client_secret }}"
--key "{{ gitea_oauth_client_id }}"
--secret "{{ gitea_oauth_client_secret }}"
--auto-discover-url "https://id.ouranos.helu.ca/.well-known/openid-configuration"
--scopes "{{ gitea_oauth_scopes }}"
--skip-local-2fa

View File

@@ -74,10 +74,14 @@
state: directory
mode: '0755'
# Mode 0770: the certbot renewal deploy-hook (running as the certbot user,
# a member of the haproxy group) must be able to create the temporary PEM
# file here. With 0750 the hook fails with "Permission denied" and HAProxy
# keeps serving a stale cert until it expires.
- name: Ensure /etc/haproxy/certs directory exists
ansible.builtin.file:
path: /etc/haproxy/certs
owner: "{{ haproxy_user | default('haproxy') }}"
group: "{{ haproxy_group | default('haproxy') }}"
state: directory
mode: '0750'
mode: '0770'

View File

@@ -9,6 +9,7 @@ global
log /dev/log local0
log /dev/log local1 notice
stats timeout 30s
maxconn 4096
# Ubuntu systemd service handles user/group and daemonization
# Default SSL material locations
@@ -30,16 +31,24 @@ defaults
timeout connect 5s
timeout client 50s
timeout server 50s
# Slowloris protection: cap time to receive the full request/keep-alive idle
timeout http-request 10s
timeout http-keep-alive 10s
# Stats page with Prometheus metrics
listen stats
bind *:{{ haproxy_stats_port }}
mode http
# Restrict to the Ouranos LAN + localhost (Alloy scrapes via localhost).
# Belt-and-suspenders alongside host-level firewalling.
acl from_internal src 10.10.0.0/16 127.0.0.0/8
http-request deny unless from_internal
stats enable
stats uri /metrics
stats refresh 15s
stats show-legends
stats show-node
stats hide-version
# Prometheus metrics endpoint
http-request use-service prometheus-exporter if { path /metrics }
@@ -88,6 +97,19 @@ frontend https_frontend
# Deny if auth endpoint rate exceeded
http-request deny deny_status 429 if host_id is_auth_endpoint { sc_http_req_rate(1,st_casdoor_auth) gt 20 }
# -------------------------------------------------------------------------
# Internal observability + probe endpoints
# -------------------------------------------------------------------------
# These must never be served through the public proxy. Real scrapes/probes
# reach app hosts directly on the LAN; anything arriving here is external.
# Defense-in-depth — app nginx also enforces this via a real-IP allowlist.
# 404 (not 403) so the edge doesn't advertise the path exists. Exact paths
# + trailing-slash forms only; never path_beg /mcp, which would break the
# real MCP endpoint. App-host-agnostic by design.
acl is_internal_obs path /metrics /nginx_status /mcp/live /mcp/ready /mcp/health
acl is_internal_obs path_beg /nginx_status/ /mcp/live/ /mcp/ready/ /mcp/health/
http-request deny deny_status 404 if is_internal_obs !{ src 10.10.0.0/16 }
{% for backend in haproxy_backends %}
{% if backend.subdomain %}
# ACL for {{ backend.subdomain }}.{{ haproxy_domain }} (matches with or without port)

View File

@@ -41,6 +41,7 @@ openwebui_rel: 0.8.3
pulseaudio_module_xrdp_rel: devel
searxng_oauth2_proxy_version: 7.6.0
# Git ref (branch, tag, or commit) - https://github.com/heluca/freecad-addon-robust-mcp-server
# Used for both the pip-installed MCP server and the staged GUI bridge tarball.
freecad_mcp_git_ref: "main"
# Docker image versions (third-party)

View File

@@ -1,476 +1,493 @@
$ANSIBLE_VAULT;1.1;AES256
63326636363139663932626130303461356133303435373330343137363439663136316231663533
6438303239333635353461326264363462386262363538660a366538663835646565633662326132
30636561623364326431396339303262663737393964383032346437623639363338376239383936
3263353265333639350a633063653430313932366631303139346236376164636433643139363731
34323237656564323738316335336637323037353235303335326539366234386539363639303364
33356566643437616437346162653366636434383466323764326439363534383862646239363539
39653765326363313464356635643962313931396133326433353334396435376334643031393133
32373830653462626632343237363064333236386132643630373735326661643836366661386539
33323031663738366630666134663838633564333536653763323037653864303033353963353434
31316364313462623737323163333761663534353536306633646331353437646431333563366438
66613261373362313935386664363066363839326662343963663365386664376431383064643334
65343336363465633265316537613064353964343836656130333139666563626634326333303231
31643134643966323132376631386437633764363265616565633031383962383464643163656433
35663737636538306432343362373738336635353437353431633961376161626234363835363038
63373332353262656338613533663035353233363365363362353939323832313130616163323466
36326166363532303835616132643132373134663031613462346335306133303761373938666535
63636134316337343532313061363736653937363866656434396236346637356339333463383037
30636632363039366632643936373535643436663163633862646461656233646139373036373437
36333662313463333863613830363166616630653030306661656133343363303339643538646664
38366562663764383364343237633765343733633738376335366132333831633461623730386665
31373535346366363761363630663437326261626363323663393734366631376564663065613464
32383837643766353764326361373962323538613534376362303737616439333130653437653437
61633134616437313330633665303532386234346436353666356633336666366135633261303264
62323938393832313965363739326438323261393934363637356639656566316634663561313630
61366666336137386331656335303362366538383161666264363035633531313331366661333235
38623038323531333266656163323935383636393466343335666230386637333964346563323666
30633065333831643833613733313339626138613831343536616237626136616330326234623332
39623530653364646331666439363636383865343363336137386636616161303934333634366562
30336435363039633439613965393331633638396136383764323563393738303735383033313436
35396132613239343930656638383034323237346138393938643830653963396134623535636563
39346339616462623435633364363934373132643139376335633863373039303731663837336664
32623639373438376663313665633139353865653936303662303164353363636532393036643132
33663630646664643063653639626439656435623439666330653636613165646637306661353239
38313565646537303465326330386133643239356230656638346461393265363964393839393539
35633634306565623562313932346136313262643636646466343765356534326565383462663839
64656437373032316334323361626666666635346430396534326363663964623735626132636630
38313864383331303161666465663431313462316466353539353437303265343036633637376662
63656535396466353535366366396433393964383536313431353063643264613431373731646136
32616237376664346435386137626166666563303463343763633734363466363931373735343130
65323532323136623631396664643361396233613939636630393165346333643561386237626531
34393335363331623765346634326364376635333130366331303830393166643331633732376432
61643361343530386564306665366563373532666336383539306437623232643962643837316537
31356464616336663734323862323934383730303039313062333166396562363765653339323665
61333633323633636565323338393438646533306333626232646233323862333437393132393233
30333637376231353232396331613631656238316665393863306135663065396633333733623139
65383033643264383865326161333837316232663236396661633963646331623765336563646165
30636530643039343164396138633232333736313762396139376135396235363739663566383661
66653361633065663838373965393939663766623834646261653038616262313239376632353965
64626233313633343235306564373065646161326532653638666135363065306437613263323866
63623535303830383062633934626338643265343036363534363332393361336361613064366234
30613761623464323261303139646131346366356236323239323866346635376535333030326564
35386333373233383336363664633539613734646430366164393631643464353436653765316266
38396336313734656361336232326365306238343365653163356166303035323633643262623161
35633732373337343837356162356430313365323536383430323032643639646462643262613431
36376639336238613734613435636362326562323665616463316161623039396530626663396662
34653338313363666530663461376533356532623135393738613330363339313966333339313530
65373466313234396239613032656561376261656462616666643734313232306435643136386434
30333661333066346634323331656563316632613236336631396662353435366235626533346136
38333232646461386636343831646339663565333266306665336166633565323337616333353663
66626630373864663038613463356564373034626131656466613261363034626332633931326261
34306462663737656134353634313062396164306530666336393932666634623638383239303234
64346237373666633034323539353032613739646662666437303633363131303534313539333530
61326263643962643030313737343530303137356464386339356363353164366338303361386364
39363335613962653263303631323539343634353534666664663233303031656238363932376431
30653265326537313261316263383864323534306363313139653063363865656265623339343262
30613337633761383438346566393330343264363736633235313632383665396636313734663331
65303434623833653637353639626662316132326135336532613864346266303230653233356634
61386334316437633337313430356234316363383334326563313432386366323635623465633339
62393830343733363639333565636361386231383936636465393531663734356266373266303063
62633763666661633434616538363965373630313233356363343335643135303334626566336464
38383838623438326164393366326362636339616334663536386563666231656332323538343866
39336331656336356238653138626331306564333739636161356133613862633236303438633662
61346331306162646664386339306338616433613138656533666431363565363365383063353834
39623237343865353861323531626366396639323937346166316138353939653836663962666131
33653866373663653764323662393034356638363661343336623938306465326539666232366632
65373065643461343964633037626464633333633030366435333664306637386637316539333536
38633264366631636436336631343364643633333464363038643234376131393862383130623735
32616563633363346564356138303566663139656233656633613864323939383336373231396138
37336430653235343635366338623563326431393130656634333165313336356264326338346533
34616261393334376639643933316563346165383735396264336534343534313038326563643230
35316434363432643864333264613565386661333130393332353439306635653832643165346166
33346432366433393737323133363834396362383464626233366136656362643835303036396236
33633963386231343664636534333062343964343633386534313631653138623531346662666561
30366439373433306536633064393134393264343565626466353230616533663134643064363537
30323562633461646234333363346630646130326565393930356632623462336237643235613935
39323065666466396633613661383461336162326136663338626161396439653162366262613235
37663531623239623036613738393964376139616330323132616236313762396361306235656165
38393764633234663564643365346561396431376666353338323130636239393331616539333736
64653831626638306237303633386464373139383935643261373935343234653465353263396637
66363165363161656333356435363832303533343261666534383036616535656664303863643065
66343866636664386364656130303236656365373065326239343261623533623731366435623461
64613834346362663236356461313735663936323730366566663362393962623565316433666238
33326131363335363965343162393161663634663861393137653739643633363062363065623032
65336661353237633531363665396464356563663638656665383564393566666466663661333636
32396331333938636233643136623066663235633064393532333661666161303431613133613563
31323032653565313664326365623165636162393265353131353133343837623532306162323632
35383831643062376232646231346261623864323065366666316634643432333264643432353138
39343566303064306266363665663033393439613263396339643761363430356637643433643938
63343363346266333866373036396635376638373737396261623862326163656664333361343265
31376466343736376230636164643961636538393131636362343434653765626263376462343436
64366164653662333338663138323532663635663832323565396465346362336233613066343262
30336431623036313839653632363264646263666161356561313531386136393937333934393830
66666338373564636430643037383638666665303631656263663364336430653338366565343636
65316636346136383936663830663733663462613339643962363531616438313239633733316531
64386161393638633961616334346163346466336364663830333563343430363163653933343565
66653562653966343332343634383531393134303962363734663763616433316566373934663531
30316431656163303066396166366432343065333634353038353134393233656164396236633430
39656230343030316436656535396336636139656336653166633065663133656332616631373935
38363561383865336634306435343230653637383435303433313236613831656539663563363761
66326230323061663538353032366465623733326436333332343863316339623835336130343565
36376531343361383861306565383539393861366436666231653638663465303534306264633635
63313039633531636433623330633638623937623737383764343731316566393439316630393934
36303936333931343461336531643938303062326633386539326365633263366533333661323632
32393238306433636136366261643635656166636437333934613062326264383132613936393331
30346132373461356565393362646537313830373364343531643262313361373636613631343631
36623531383638313439363535623333343935613763356364653335653638383766306238303362
32393839623838373936623166656430313136386161653365343836376362333630613864643263
32373764316366666462633336626562303136306165373735666131326566626336653934653362
35633531363039373063393237336430646362643765356234363535653138396134623232376435
33343165366634633663366631613531306432626439373935343061313031646537646263373231
65613438653065373261336639323037633538386665373630343235663731626264383364633566
66633934633064643936623330343233313932646238613664663761393536666438303730666266
30623130383632643162343365396637386438336639623964303936396432653039616162383564
62386236623139653366326133386139636463316438386662373961386532383266383234336561
35343161346539396236633931333935343736323633366363373135616636636363346232653530
30376361316636663135336438626437396466623230653765656236643839373663343533306535
32306534663462343136616363306361613233316637306233643530363539633138633635633565
33633235373339376234656661343561313030393665383038333831366439303630366537373338
30626561633737336436376265396434316362333133396333396365636264653764333262616237
65626330613664373139336337373638666439643130346566653338656662656432666437356139
61303138336134343662646665393665656537383033626133353235393135353163396563353634
33366338663365633035343564626632333235366336633961393937323135633262653961373130
61333562323734336631373033643266626261346437363436393031616261613966343130666265
33626363626332343130306435646538356363353865323664363238313637363338326261616232
33646135653638343333626366646236326261643637393465323239396132666665386537636361
34393034323735343636623663323965663430363630383561376336383133356666376636623261
63353164323138366635623365313665306536633566346637626630613834356665303433643262
34613433626138386534313833326438636432656233323731306334313836373434313936653063
64316638663734656334373762393434313239623234616235336433303733653263626532356437
35333764373565626432616639393565393766333939333165303865646438323832363638383336
64333262656664663037333964353563626339313361643366313161306335613935396430326236
65386538626663643662383363373162373034313633613461356364646332666561306464376164
61366666613938646139613738313439356337613436366365663262346366326532663061383966
36346135666564303539336336613463386362393232616536383465656332336434356132306437
65613666663235373537353334393336373730306631636165623361336536623165663062663139
34303162326139313163653265613066333261303035316432616530343966316339653638626364
62613736393063633937396363326631303230363665616534336432393437313665633838306531
61363337613539616631383036396638306237303962393430636234643163623339626462623834
65616331656239656362386262316436343130626633356362366533393330383135633832633634
66343865373337653637616238373132306564343563313562613239323261323235333233643030
61376465383535313462383337616539643035643830326637643836333534646266383161653066
66373832353631323237333730353238353332343430303834663330343665316562346630363033
30373064306630313031326239393731306630346463626364623635353463656361306263626663
32326264303737616433393838343665396232356136343139633536313036373464653561666633
31323763666464356363616634363465653534333333376263316138623263666266653133356438
37376535636565356337326263373161346335313964396630363262393835333266376465313362
34343234316463346336356533613633323862333263633862356632386235666663393563636663
65613030373963363665656336386132396136633330393439386466386562363034626533626336
31393639306632303566343332303265356532343736373539663063383339363064633432333835
35623235303138343162643761336565323230633566333733376532383962323137323430396164
34643539636365373833316264363665346337333031336262653262383162653664373233353661
62386463643165383037373366313037323162636264343763343063313066623162326538626530
65363236306637616438383638623334306561386537323061613137326664356332633330326636
61623636633235346266346661363533376438386637386462306562666563393139643732633161
31643630383139646631326362316465316463353136346562376634353664653133663632663133
39643466656266653165653361333231653339383164303564663936643334313035386534616137
61353231303630336237663732646138376339366461656338326632663361633665366463373531
61666532623032613634646164616334613237613662373561353466313638383239613139613032
32306335363935393065343062333437326662303765643363383532353436636163303830643330
37623334333837386231393766353138376532383136313831633337343234363366333261653233
62393030656533623438613230663062376565343039336537646265616233656664363565353739
65373365356339346336303338633462643732333034653130646436393430653736623437303262
65346437663366643863623063326565366631613138666464656239363439663061653061336632
38313964373838386265316663386237663863656364643633323538366134323332616134356131
63646462346337373735666138656665373438343366363532303661323262353165643734393462
65633832666135313265373033343732353535646165303939386639336133343930376536326331
62643737643938666639343163383333323862633064303432626165643038343034316264396431
63643362326139333163383634376364353530626663653362653366393539303331313233333233
65363463643539336362386663376338643966656336613038383430343237386365633334356566
30626335323731383062643366633161636632363761613830396631336230383664323732613462
31306134663634326362643331353063616264313233396366666532333530363234303462356239
63306431346433353639306665643365626362373533363532656337363438323333393062383963
39623337326433363532313132336536346365303061393364616437346130626366613634343036
61366136336137333861313938346337363833623337643332323965363235663263656435346563
62633532626462616634643036323665633762336636346362653133303339383338663030653532
63353330663062653832646530633065626431336166316562356134386231623435313838303764
63646562616638356561663636343639336138636666316634623538323363313831613838323131
36363431663363303864643230363631303132373036396338333036373637633564303834646366
64623530623332623031653731653931363262366432393032353839333637613761663661306233
62396665363139376366633730313465303939653339653861613136626534613561663765303232
33333330643533623333376236623730363435343265363635393731393765613364343762366239
34336338613035363062306233663466343965373733313436636264303163343563303835356237
65656335353634666133313161363264653430343738396465313538653466626130613730656239
63326563323532643133666561663631336339343432396163396533623762303034333534663366
63376661626637356130313439623839316134636537653430306133303761393130333234323838
62336137623032336263613133643333363739353262313130366238323539366561366266326261
64626137613030393863666138666666343037636437393464376433316461323266663131626131
38316437336566666466333034336134326235336636666236643837623466343530373331656538
37653634653462383062656334343538633236323265356531346334346565376538663936653131
35666664353837633437666334313532316464316135376665643835393464646636306638623634
30306131306137613963386439386232323636613337353437353630633235313661386664643864
38323134303036316163336564303330323663613739393665363736613562636663313064383366
62653862363865623634333236396161626635303361353030383539623963356163306436383533
30363436346663323662366632636136646331396130373232383737383235386638633231323836
36356338623162383065383730313538626130363866653963663266643230393134306263303138
39633664633366343265383932613964366565323532363266393465396664666231303261663833
32633564626132653861653937656238613036383463653363643633646666623732653330643636
39323864313763376336633766303730343961376239303232653436393934626533396461643939
65353838646364613162343933306432656565323566386466326433363366663463393262656335
61653936363661656166306165656534316337366636306263303638323034663030386532386332
33323630303434303963343433336638356432653561343638346438383535353263303136323234
65616335346461333562616337343466383135653638613761323039393231666639643133666433
38356135643664353738313239396338663165313563326661346533653962383030643334356665
38633964383364643135316361623561316464323035336431666332336563653265363030356531
32616535373135613239353934306431613536373438366230353333376462653630383664653839
36353364333034636332343632363839376130313231653436313433613636333532643638646134
34306632386164323163653333353764393961323164306161353163363130323736303862336239
65393235373135646263353237623532636363633339303765623562663038316633313561666366
38323862663232343362363564643533396465376337656362636236323239636531353836333536
31393131366465396535653332646662626464363963633033346131363039363430393332613236
36646165633164396632373561306664333366353431336138313938633463366130636635366562
37653363336161373035636661323235643838336637333033646438643566393462633666616439
65633439333763636661356332326336383739336438623338333736623264323233343738623935
63313834623038646430656230643835656536323764623734303939633562643636356439623435
38396165383964663231393335373835353963333339613837633833396439616261343335616233
32666432613634366136623863623337313061646138646336373134656231633738303836303039
65623337633837396337623431613635663734613066383939326430306639373733353464316138
38353234396633306334666662633764303038623066663936343737326265336266323733316337
38366666356136363237613435666531343133343438396138326562643735373634653664313037
30646165393066663930653337313564343635396561633464363461666435303836323262363836
38623262376436323265356535313162613066303836373135303836626162306635613462306330
32393032386662396264356537643231373331373235653863656462643333623861666366663930
33313962323738316538383131383838633333393836623963353162346133386431346638656233
36356436616139393630313661306130613936633666643366336636323639626433616331333665
31306236363962336334323339636465653163376238313266336539333739373765386137626131
39633939613365303064313231353932303033616166636462626134623134393863343864663536
61333137623435393865366130646634313366343966373063323530313732633538343932653437
39393539356361303465613762656337623333393066653235313633623939653062383565343262
34393438393331623864646237396533623730343363303763653535306563643838376439306232
37373133623863356231373966636462623832613463313231656233623839303739313936393261
30366464386538356533356136356330633466666165663864623738306238626531356139333661
64613632653539396530656361343436303238336661346438656566643339373365653166313436
30616562383066663634633434376330346538656230313338303731643536316237316333656636
39646136363461356463353230613962396261363035386333613239346532646464323063383638
37336565303163616238356431616432343735623364376264663735336239306439636331333338
65633137366565346636386330383539346662646532336533303664633832373831353931633937
61386561353265633537613064303337356362363261366630383038626535373035656430346530
39313733633837633230336135633433326238626536396337633161373865633361396663396362
31363931353731663939383462636365386337623065346664663531386235653461643565316531
32636332363834633334666465646563363663656462393139626365353137306536393262653534
63383636623735313139393038366261656539333935366330623233643865663935653731376664
31353762396661376465663034326237366530313436333163613539346262616661393361383534
32303462623964656162616463376233656239646536346335313530653336633236386532316435
66653130613837383264636566656438643131643732643835306466343335613162653366356633
33613364636661363337366666346535636137313264323535383833316165326336333735393132
39363835383665316666333866393037666166623963376538346666656437643730353765323264
65636335373361383530613866616132333231643736303237633936366664353132363665633962
66393937323264626363333465616565383361313633383739623235396236346265396664613436
62326663333834323939663832663235323564333866653133313730623061373663323866323936
37613738656565356437313039383562323634363839303633366632373630636634343636326666
62643333653935386264333430363331343761333634363636623235386438646164383664636462
64323733343562646230316663316338623563656238663735326433353561303437363332366533
39616439386539313339303230666232653238316233636231663535363733336330386562653561
30396663613537353031386536656461343439613863343062653039393238363333623138626263
66326661616363346538643432653936373033343862663361313239386538653464643263646666
63616566386239643139363063613330623133343436343366373239633664313137346639633534
39663236373635306332616362346339393764353761363864363934346462633532613165373839
61323931363238653839623361343761323762386233346465343230663634306262393164623433
65663965636433353333633632353737643364366630363938326664386437656264636631316537
30636561666264623834626539663838666331333737313262643536313235376363663866363736
38326264643634363866643963666632353961346464633731666561383732343066333666616630
33333830336264353539306433326566626633623461623665383038623161313135313761336666
39366238303735646263626231336230323934336465373534356665326136626137373033343265
35343830376134313637396237303734393839306661666537323038633163363133353938626664
32643234323631343534363533323830613636313062373539613839646466376538316639313562
39393731633739613139393338373330393761333633623166663666396161646236356238353231
37633561663631326562336538656137656566623061353261363431653533613739346138643732
61353936353137386165663239666336313730336235333438373432333830323331613162633633
64326234633830343632336163653734393633623137363335613830373536343637356130393635
63623666313930366664303464306533326334313865363734646662623630616239353734623366
61353037373564353264623533383938623366653863316664663730363234393433366233643638
31653236386665396639346364393837633035653531306139313831323161393438623431316638
32313335346530326136366662373834623735626339363435656464323265323361396337323536
34323939643337356237626230616335636564656162616636393339636133623766303039366537
66363832333232633037353165376563383566393731386335343666316362663530363339643563
36623139383933336133633536656636306261663134643035623066323433303937343137363032
36633963633734306663626330373266663565386131303365393636323438386166666662633233
36363032343232623631313037313063633739636637383665653063393066633163313461383664
66613537656335383465363931386437653361376133366266336364666634336261326231393764
38356562396538303934333631303736343335396164623131613238623761303639356330343866
61633537333332336430396563316639653337373831313236366135383633643832353839303738
36336530346438373135333630643134623664666535343764636162323630663432386535646337
32316338303663313566626564323664646237336363636434343330636638663366346438633839
38613238663363333130343238383237346630666530373630623732323635353138366561373335
32366537643639643735313662643538356233346137386332623862363736343261323939373638
31353732656230376137643635393063643835303135313733303338376531316436626365313665
37396131393231316238643634316566613963343364376361636439643562653636343731636135
30326265353032333834646532626539353235373265663236643037306235653136643537346166
33663262313662633338613434363664316662333165323231646664343836386133336136306532
64343634623837316136343066316139646532303266353166383434323233303161663666323835
34316438303763366434376534663333656635643661666635363064393038656461376335336565
38333961623736303564626230343164366433356564346465346365363636326630363438393431
30373334363066653539323531363136376162666262303232346238356132313963613336653762
38646132313038613038626238353862626233656639373738626362306634376338396432323435
65316166656166643230346565313438363435376162356431323337633636643831316133393534
34323565353031396661623337653562656437343730343064313532306232353437663437393464
39623863336563643762633765383465313431343666383033623430633136323161633730316430
30636362333962366661633334376531393632366433383136633865313665663864356135616465
35633437346339373565656238663730643762646439633734616336366166356466383864333737
65636637383662656365323837616661663638343733653762643366343230366662353036383438
38393939626539383664363936373563633136613335616531336235303562623037656233633963
65663436363666373864363864663331356238643530616332376364643061383235333161333730
30383062363830303836363639323938353630636534323866363863623364353435653733393132
36366130336330346161313836316262653833633835663434373432643564356662633665623736
39383736643536316234313764393436613162616538656635336435633535323063376430373733
63653539306636623331306136653134646462363432313337386164623835323338643364383137
31383632323866663766646536323133313939636432376133656338646261383165393634366434
64323235333163393835376132393139353938306532646664656233353433623534303665323863
35663039353061363131303464363761343966666563376131653734316661353138663061306364
35613932666634306639303535336561656361366130626237373333666233376434313637643731
62623234663533363037656538633261323636396334626463653835326466346366633634336165
38663232383963643965646364396230383465623261306536653965393532313732613561376332
66363737393464653535393336383764646532353564356439373764393935356136343634393561
64363137613963313433646233353335626134383335656265393636613963393465373234326230
62366137646335613532303638616262356630653861303464396263663337383338653766633331
35323361336332396236363838613038353339353634316535386533393332623730366235636131
63323264663436353634383866623235303166366561363965643639643764313134343939333635
64393361343964653332323735623037666161383262303439653039633764386234643763383632
62633535346538313866643032643234653539633230313562666330616232313331313764396432
36396530343232373330613932643866323831316638373338306336306362333537336636383562
36356531336163353562336236366366646565666339643130333637386231333366363638333030
38363133313165343165336364336537623564613663313235353436303930656361333134643035
31333366353865303766333233386230393062303139613163373131646136373437613966643531
36663438646563366438363939613334353134363962313139353263663932306635623334333964
62653631623336346238373934323730366266613335373564653331343739653630616564356633
65383331393363656363663634616239303335636166373233393937343836373234366135633739
35363833393062346261663130343166386336346265333562653138373439616434643163366639
31323931383130633061666333393035363163373233643837666334623630333937613532346632
61306164376535326661303235343561346332373533336139636634356166303935333733656639
31376430306638623662323231326633383766613530313234633030333464333230653232623039
63626639616536373136376135336534383862323630366332633234353533363838613065653538
35653031623036303538393934663236313732646332346435353164376134653433643263643135
65643139383361616630396234383238326264353138363039396331393238383362636165336337
62363966663137396564306166663662656431656364656564396437393535633531386463663061
33303837323364613936623665376465613466323836393136303964316339633332333036363337
32323035303236613061343463663936336238303066306132623435353765633563323934316539
31373666633666356332376566656233346366313334323231343237363662623538623738373434
31313839643631316237646364363032666365643163346163313737373131663665333461653831
35383366303936663764326539366137666163396233356466323831626633356536396636343839
65623964306565396631356264666638646430666234386166326534373233363735393762393332
32623765643662653731633061383661366263346631626262386665306430666238666665383039
38366631376437333939636566633731316339663031653331393462646437333730323437663532
32323538393331343865376531373734316137636464636537623934376332626666393130303734
66623934343137363535343232623435623764343964356663316237666361356162316231363666
61353430623361613133393563373131326463393837653830326235626631376236323132366339
62393066653333663965643362626465656463623933326433313238663533346136366638383530
39333865666534613530613838393965373833333334383261653264323534343866353332613434
61646239626338383632393631623863633166313634363638353432623166356434616638643335
31383434613433366238313564646136643430333765646337346135653764663733343835306533
38303931633963366561656663626163333635633235333733313738353464393435323235393833
33366234343162343931366462343337616666633837373537353032613837663661613536656461
65633139396631383538366532343764636338363963303137366438363339386532613861333337
62363033623832336562313936313935623138353236356665393932653564343636653563386538
38306137336537313164393331383332306366383735313264306332356161336430616433626166
36366465643432636532373965613937643165626433656464346234633965346566663263386564
33333262636562333934343562656432306437653431326465383864636237376239303563636135
63373333613032666231396433623761643234336238666262656338386133356435343662653030
32373331313938663337613538393063643237396362313264306533353139323739616164316464
31373765376364353065306439326139393137343137343333333366386435333764623764363861
32363131363063623762383463373038363334363266613464636436633432313431636633663638
64636337306164316366366434366264366535363134633366313533316363346162653730343561
36343234353962643736613365376431336231303136666231663461373664623039313465333164
32376330393334666564353463393638323439626164626435346637636365636232386532303533
65353163656163303932623162623662623862623139653563613033306537373562643336303436
61373735633966663239653835396664343031336461633863656230313265343165336662313137
38653765663832363632613365366434366461316161303535303236396163356531323437303262
62353832313438393935656633396662303364356266623131633335363136633033323866303362
34616634343533393732393437343363363333393563373361396166326630376438613731306261
39633932623636383532656166663737613362653437363836393363393363366635303839313139
34326236616666366336363330663166636465616163353233653136326166376663666638333931
62623063616666613236323939393962373634326534363361326462343233376436306637633763
36316331386439653039346466366139633231663565316239373430616639366235636163343635
63386665306566663963313765343065633034323030613136343637663333353633313234383864
65626336623131366439393833663635363635366637393135356462656331643131363032343938
66613462303734333762386364356563356137653731383431666161616461313831373633363961
61336339613433396161356635303830383865383536386239323237623533613737346239643634
63633738366262383438306461323563643530353464633132363637643663643138336164383936
32366433323630353939316333386533373032363733366635393265623833323331343661633266
66643665373566633462383264663637656236613236303364363561313230663164383133393038
64643464393031623837306431353963393832303266383333653339333238313738663361303665
32346633386566643730326431623233323666333962393438396664383663653236353930366131
63616265383538613637333833663732376362333366643666383337336666363265663737363464
38663932363233343565303230646666666538643039376231343537373636336431363939356635
32656337353661303231376565386332343039666236303139363462616634343136306637333238
33633337323435373862346237316532396534333665633931343633623266363338643931613862
34646466613431656134313232336132363739626138323262313034623962643330373639653633
34623864376439616539343330306666353830663564633165353564636534303135333964656464
61323530663437613831623931336538343863356439383233656137343065383237653930613230
33376635633632303331323338303733363764316130393338313536333538633066623163643431
65343362336233646664396166343730323537663835633266313236656137343064313861303865
33396239383739333265613336396362346365623532366466643630666435383731663263313661
38636637326237663437353337646264336532386539386634666139353161633066613031373030
39313737323061363838656338333763643165306165303334626533323463346434356332613830
39323134663063323439353662326437626238363862633534333635356464396436666331353266
36363231323534343035383231303664393238393930623861303165613930393238323066333739
39323133336131643139663537306637363637386530653462303233306439363730623263613037
31343630346331303561653663626432303832633665336632623464353766636263646463616139
63353835643031356434666236666464396165356566343861306564666432613963343431303661
65643937396362656439643932613263376239353464313237643965656630323432396534303061
30633661663834326638656131633033643861646161306266336331383836343262323066646231
37323836306239386137306633393864383133343135353431666366346131396362323331373331
61303330666238333431343933616561373335393362383132373534396563393933363033386230
62353234653036383664656539636538316363303666663565353465646233656263386261323131
35366662386436373661343366373664613230666436376131323763633665616666333963316538
66353161303636323033636135393566373435623465303239326333373261346532613263666266
39323562323132356365313063356130656532633332646533316633613763323163336631333132
37306363663661616535613631313562316230313331336530303630393733636231633335313365
65373530373761303836663563323231633835363232346335316336313036666637383531613363
63313362323034326662383033613761393035616262313430366532373036303535346430343130
36383539653838636237613164353433346466303330646538323232346436646263343562666434
61653438656464373165303032653433346666303663616263636434663231316433326630336236
32613965636665323064363739393439313738343933333566313432633734616531333930653363
34303566656232303863663062623636616434356362386365333066333662653333336566333361
66656566643430353536366134646266633762653338363162613365376232396639356637616132
62633936303534303564336162623639323134383431303930366435306563656437373866623365
32633434336536313033396233393831343361336633386633666434333138356363643831396638
32373331383661373465313335396537636161366236663363396139663963316339623231306162
63343337626463373733376139643537643564306633653439313464663539653764313037663364
65656261386631303364666137623436656330636433316136396564373135353535323630653466
62613937663835333435356530643764383437393963333039363966323536306630303432333361
39376263316130386631623161393438366233363838636361653131353230396634313466323466
33346536336630626335646539343438643038353766383062306634326338326631343162646462
35646538353563633939643135633264626664396137663938666134393334663730613630316433
62623030626461326465643864623537383066616365633061386263396265636132376436343332
30346234346432343839376332653062633165363531333563333262653239343866336132323566
65326466363233643663386131316164303366653937356432393931336564626130326433663830
35373663386363616432343761333361343132646233396437353939643066616632656463653065
35346638313830346362646538333637343566623566666137333762656539646332623535666366
32363530346465613933303936396561613766376261663162393234393130363135643737393532
33316661376434333765663663323762333861346237373430323332613935323462383462346238
63346637383132393362323233353037343363343532396134376331373535393137363563656631
31353830303132656436623635666663616561653732393764383137383736386232373764613833
32646165666366313862653939316533323066393161636164393765306633366662316134646630
37333232653936633131363933383835383531323031383230656639316364653738366137343135
38393032633637393030396162343738643439393731333232343034663338316130316332326636
63343834383363643038356235623933643266376431333232623765656238316462343539366365
33643234383134663533353166373166333334663963303738663337396364613663313238646466
65653166633631626566373037396533393032326532656634386464313266386366663233313564
65303561366137336162666461653562393738623432613763306631336564613232626262666535
34646239393936633534346437623231386535663134333263626131626339306538333764623830
39393039353239633335333538313062386262393735343237333234343363643436356465663765
63323066393663613236626636373135383435343664616438623865646264326363303965326139
61626265663439313361663362636665306438373966623639343363653466313464613430356530
61643538363366356236366537633134353932613831323132363339363231363464323661643262
62313865383762386338386136336239306364313539646566663330656439376365346265363030
39343763643538303938313232393362383031376233396437356239666634653966336436613861
33613233626262306362613039643738656361363235623735613630383261333937303139636339
35366662626233663238643236323762356334326338366330653063333630643865616338643936
35353362663937376238353533373564313563343335656430373661663935626165393565326263
65643539356432626661353832666633393837326437323637616235633435396435666632623536
36636330356433306436353032393362366232303061386335303636656536636130393961313365
63623235636335386166666262613435653537663835363336613431626366383331633538333534
61633235633730636263393238316232646435363964303839313564313135393235373531643938
35653762653437346433303931613230653161303135303335653834623763363338353465346430
30396333643663646537333233313539333866666539343533376162643239656131643036326437
37333835306336373062383539383632343737343233643233636231343338306331653064353537
31366130643264666438353565303463333833376433356131656166363565396437323531346262
62366539353431336334333935646364343033646661643963333936633139306230663330303539
32646439336463383862346137326366326666376638353735363633343539373037336364353731
32663537626265653535393231643665326535633131363233646561303730313037393034313461
38663039363363346333363661363261333266313036343935616634306336346538623636306438
36363239663163666435613939313164663530666439323738623334333131326465306133653935
34303030613736363065333761393332373533616438383834643661353664343635326135316465
63303437356638346263656466623066653333633165396431316662393439346237633432396666
366365626532376262353630333164613038
61303461373234626338303164373438363631653037303239393666636437633832303066626461
3233396130396437656562373763646165393231363464660a326364396463343861373236393733
62363134376266383866383933643966633332636562623536636536653563393263383066626337
6635643065643761360a343730636366623364633861653734343132363866323338343031613030
37306532306437656463326538623066343435623163643133383638396432623065376439366232
30313065626530356562336239373562313630613561653435323333623035653366323734663637
34626630353062323131643837353839323735393031643337313333396162623062653566646363
30666137613934626630323838353066616432343238653935646332376531396134333931306464
35353331663964373735623661643238623033353131356630376363353131623930366562313361
61636633393266373230636435613736333732323462353031646439316639396432393232613236
65613963623461373437326263626161323266373166363230653165613637656630663065303132
33366362373639343230373836633231656233343539393332336264643430346636366537643836
63343933353261363430333233623930326663313465393034356530393237636264626537303430
30323965636161653931643235636161396239643766613561636131343237343337366137326238
63393336306230353766386232396264393336636639666661303962626362636266303262663036
38393530636438313236633566313361393136346630376133396137316664636336326633383437
31653131663834663036313366643237376364316236313066316338663038343530616236396566
37643563366638393164396666616434313236376364383439343464366537386138363064666431
35623831653536313261373462376364306233346632626233376365323536313762663464373037
33613038303430313538313735353232353131653862383362613234323166323936613166323266
35393064623530316661353431613733643061393435383637653732656561613138653337353737
61623035646138313162336332613139316134613935353262653635336634383962633066653938
32346565633834646465393135393935353766616530366139303635623863633932666134366664
37326331383638376636313931393233636132336536306331396461616262663335333264363030
31626463346262616561616266313235346461623737353465636334623861393066373162396163
30303036653964313739373963636566383364646465386164636534363938633437636463663839
39396264623439613862346339636661643538343832326162353032313638633262626331623231
36346333353635333332376564353862313539333762663664666538633963383864623234396262
39343630313363656530656436663561623533343862373438356632323936303333666365653664
33313834343630326338306339643666383533366534616638646665663930626636653031343362
64613134393032306230353636353434356266343464653661386366333466393834313031616232
33663835393934623163303164626463393237363139303064636636663363356138383939393065
35363532616565633338383835306137666665376362346235323366653265333637633034663761
34383337363161353037356334313838386663303334393736306234353137633933353634333334
63323937666231333163373231663436366132383536303433623364656131323662373932313234
39633163666462313334326433346463323639363433656564623436366134653437386330313663
66626538326431393737663565656666393039366162623562623438366134646465346135366135
37373264333034653032346135383236366537353466346464393439613866613232323537643762
63353562366261346162323435323136666661643366326162306636386330623032656463623639
35383263343865393437376438383964396363613831666238623737376132633438346337363733
62656266373235356334383264633732633139646333623363633534393435643237663661666533
33646236343135323561396137363762303036353962326537653834363965373135303338353232
39613437326330333639396366656238623835306638393930666665366666666662366465633139
34653566376339313037313034326238363436303064313134633833343565333733316564343937
65393132313832633465393738653433303064353632613861373836343630653738343730623738
65373466306264613832326336366635323136346661386331353837623431373634643730623966
34396564366662306232616136636136323834326165636439623463633165363366326666306466
36616466313136346561616361383239653831323931356264366630646138663236333761306162
37323530643138306138656665306363383139643935623439646130633938343165643865343230
62623238336635656462663832333665643537333139346134313632646365633535643630653162
33616536653037396632396537326463316164306634653961343333353164306230323839336162
62336439633366616638376331386131373535333364326365373535623331633432336264646430
62613765343837393735326433626430666634333336366538313265613935303332646366633166
32633733626666373336636366656639636165343835386661636665313532666362363666383734
63363463303034386134376366656139376564323262373066656365386138613630393861373933
37623936643966313264323337613362633363613435303464366436343365623363336234653562
33643535393637313332653534633939303431356666326337666539376130613032316236633162
33353830626439623161353832366432643265323734373835323663643831626234653930623737
64663566356337326439353461376136373131366330653133666463653737353761653938653465
32633764663239323161323639643861623530626633313832373762353630333532313534653832
37313230623330643338353462633163346138323766356638373132316666323530396566323532
66383336313238626335363762313839333861346130383137363266626632653839363231636563
34303132343733663131383730393062396264626662623262353335663732633438373431346636
64353531636338323662653430343861653931396364616236653838646237643934306433373962
62303663383662643666363236383330643266336235316634346131343030646234633531653735
39646561373662343939363538393639313839643061393538346234363735653562626232656336
31633637633035306337303136383464373034313332656563343061316231333463626134396130
63313162303064613433663765333737626463353334643836313938333065303835326235616262
64633630346236306631336338623938346266646362383263366264653432393735616335376561
38383061663335356264383438643937336633613965376161663330366562643130643462323464
31343235313139313365316262643830323063343763633330356430666663346233643836386363
65646236663036326331356333373835636435383730623038346333343035613930363431653461
62323038313962643231336135316133373431353334623266656236636364353565363766656534
62373632313232336338633630626235323339373231326431626130356530363235333334363734
39623331396438653262343464383336383835383433656130666465306430653230616430313936
32616234653161323637356330616536636361623539363964633163636366666136323831313534
62656133383962316133646463393035353461643631666537386634303432343937643338636633
37333135363063613563663839373162323765303230653636303764393535346632393965663163
62343233346235666564306531323565323734633662663966353635646164383734303830323564
31353666356434356334646237373465336531306264393264646232323161656162326337303064
30666539623365633965386330386534383131663764663565336636343434383666666633323730
62653631393166656339313334666533373031383439393465356536333865626161623162616565
66323631313963323539343465336436313264363062353133306566313464336236643737306638
34653230363937363431653930336430613938373133373463613866613963333831353037643038
38323937363534623661353835653261303831376330323239303762613434376436323738613037
31353539613961613238393335616431643036653561383135323732393736626435653535323937
38373861303865323165363966666364393162346262663465353934616432303239656565623831
38626337323061303861616132323738363564613366366437356633373638373737393662636536
35646637386162323030366161666562653932336665393332373739663533323965363563336337
31663437363430343335656166393562326530616362333562373231366531373238373439633332
62316437336264343164643866333737626130333336353031353763376364313866353135656665
37326435343238326366376262653638346235666163633661666132323232303536376361303639
63656164653537333362346237383533336539663462363866323435306332636661663534363537
64623932373233326134383234613961316435626237363732383363383838353337353338353962
38626632646430373339383263643964303766356336623238356465643632306334343962643864
38333362663261616362366362663833623663376134373263643866333261623063393563356436
32653439636139353139636238333437306665313930346230366562306637356539383137373363
30623839386336366261393066373037323062323330613562633939643931353230376665623264
31353133363335323166646437616439323164383066386265366235653963326362656437343363
66663964333261396563336135356530306266313132633164643936343565666231343636376637
36303765366130363836373036633565313462663633663338656563623337306562393433373965
66306666393132393963313861393565623831343539376265653134373138613162386664356131
64306663383564353762313161323034366339326662303633353235393064383439343433306261
64373938636634643166313735346662323035633363323532646161383536616639393361393061
37336137623030616366393038663439396261646164306130663462386131666632306363356465
31353138306533613239653937646561363161633435303138343034646434316364343935393430
32633665393432643662373339626266633334316332353562363864633965323262393864633565
62393337376530356566633931353936303863393730396332396464353566633761336135363739
38653435363232666263643639326339613764393336633639353962613539636165376462616637
35653563303166626432323362643364373966346163386635653634333638663030336163633732
33616136393562386565326365333732363165366233363037646636373464336437303338616262
64333536383739633038666135626630643863663933666565323937343064376165303535303537
31623664373761353231336331343432333064656431393563623438633238303532303836656661
31336366393932333062306136346430653138626463656362646265333438376262366234383330
61303366313461653337323761613336643363306531333163313361616466363765636364366434
39323063363231343233333461333966643166643330303332353138636631363836323135646230
38346630616562613163396262343064386436353961633635353033383232326239616436356566
36666537363566333666333831323738366130386162323339633639343430313832336265376262
32303263343330616531376630643964653965386363393336313436633235616332373537316437
39336536656433646464633732643365613336313666616264373364393666663632373863366630
65363964313366656330316361383036386564363933346362316532656630333537393266343539
30633561363332653766333330363437643030636532373532623635326636353434653166393866
39343238613731353362323732343063633332656635333565373264653561663166353836643538
62353336646439313465363261383931393038366561643665643239633938336436326637336436
64646234366336323133646363303230326264633039316335366137656464326634323266633438
66396666366634623466383864653265623137613763343266646337343438633262336235376533
37376239306632356131353139336430303336653530303836316433336133666539333462343630
31353834363533343632383864323961626630313033613864323037373430373632383239613862
61313462663765393739323362383035326436626434333530396165653535373961323865326432
34353238656333633133616263373861303138643163343264646665613039626636323233303261
34626138333564633563666262343164343531626431363031626465343965313236353137663036
37356262333939346534303333316434306162666336343531356562383662366130336438393838
35626263346363396662373162396665376164353034656463393462346662326133613966636534
37313431623362396634336535336464383238633266643337663939313132333262313130373761
38623036326332663635396638326233303236366136643334356535353136303161333531356432
39366139356236346565363464313436643165386230646230343130303531633732663433353364
30363233646637326637663730653134383532336261376633633133316361353035626132363032
33636432633433303439663435366636323166343363343736323230363339373132333433666330
38656338316264383065623638643436353734646337633832326130383265326136346534613263
30656234343639323165323331633834613333333032333134363763633464333461643031623261
61393637336632653061373038333566313839633332613631313566633239636135326263633539
39636266666662626435393636343162333365653137376561396364373932393631363365623834
63306661373331656432663666356639326666363730306662316336366139353135336264343539
34393564373432626437363664316238333738626536653831333765623839313133626365646635
62373664613439643165656566363638326234363834323830373566343138386662626431303036
33663963373032356162326237343764396538666638326238386235626566343530336166366362
65363538363266623166343537643434306637623737373266653637326532616333363864653766
36333738653662623863333735663330663135613061386338663063303563323638363731313161
37326536633131393534393563346537323733373163353566373934316136613339623938346165
34383331633834636563383364633833313834626466326565396161643730666131376131636465
63363035386535653336373030666636333535393837323237633435366565653138373662303037
36363863336163626231353861343831333437643133306531633638346635363438336165383133
35626664303030303864366238656665306535623861633330653838313533303332353265643837
66616635343131356338303838323765326431326439393333336361303031373266656137623136
33633062366462633464303634663532313332333039323064323664366561323263663062336330
61323666373631336564346164313831366438363433383464323331353337336635643239636435
65666333356434393132343666653533623535656562353663343363386562616234626233396635
32613865646363373236613936643335653031316431303237633536613264663939636532353733
36333530313363386363313366343239663439643666376431623766666266386434313931323338
65386438376663313138633534633839666362393165663830393431303764316336343962616434
36346236356131623661626166316237643737353561626164643338656564313638633161396565
61633335633032376131336532626532383130313336653232356666616235623230313337306339
32383632376232643839383735336439393865393238333439313665623162383134383839393431
61643936373434333532366466663934303964643039626163613966326463323832313736663431
33353130643138393038373966663433306533326432306138653733346336643236663831346130
35623763323166393231653434383434373662613762306231643835393836323933323336383661
66636237653432326362313239393333613733343266336365653631326666383334643833663666
36383031643764316430653532316332633931663132646234376139646565646230613833386363
64396139663830643864386137643139313564666564636135333534653735316461623366356633
38386461653565623237346631323066323535646661393865613162333537393864303061313038
30613936373737306231623630613362333832663336333561633836393665343139306465343135
37343433393135393366663837343663653439366565663335653262343135626461323136373535
38366262336138393338616236393263356131333030613039373366373961373338663938386431
35306263613462613435646631343637663266333331396262313566613962316235386335353736
33333233373030393032386237316430623330353866613038373934393337343762393537383931
61643965333234363233333938313432396332663662316464366230313865633139613637646336
32366332643235393565396639373534646635613036653265373664393165336437393139303365
31643064656265333333626133383336663437366535653736616435663461303735653366356634
61663637353337396530343438633164633331663866363837326434353466626638386131356237
62313063303937396661626465613732316236626336353961376338363663653365656361346261
33613338613834346534366434623331643364646161646633366434303831356663303831333439
37636461613763613933613939633335643430323837656533306662303032346130353934353631
63343062323563363664313631396564633830626563336535383039326562666539383830393935
31663136313337343830663933343265366633356235626564656532663936353335383733653765
35323463646536326664623939363035313466376666386135646666366261306537646564646330
32663262333064383236393335663630386665346164346432396138396231653637326431383030
36316437313139366639323664376630666465396562393634356664353431356330656161306338
33303137353662313762353634343862393731353936646136336233366232336532346537666364
31336337313536386531653534393639306164316537656639626336633436323634396539306633
35663239636630313630396333343737663637663934313666636466323765616331306561366663
37373465643330346365303838383238383436316638646466363030376139656266656263623664
35636364356639336464636266353830336333623235636664616164383636646236646264306238
66643930303165626532626462313461373865666562663764396132363964343661376339653563
39303334303964666330343635663238386536316431316332663666623437646661316138353066
34376564366431313064306266623166623830333137343163613261326236373361613735653533
66643465376162303436323562613364313663663436313363313561306231366366333064653339
37613134343730386461613562306631393863666339303638653537663263333636363862623166
30653762393465323938623739303536393037626263353736346439393261616236323832376633
66626533366631616336363433626332633934643532303565326539663330323238376232303565
64383134623934386361323539643038653037633330653964353230623430313737333537353032
66343839623838373035373238656232636536343435333439653964383737373439656363653535
62316131303035313032333333613962386535666339393038393739636133366535633730343830
62326336383538623538643461356238343136343665343038373663623630653731353932313166
34336265656433656465633433363138356663313234376633376665326366363232633737353536
38653338643064306237386566316534343061653530323931323932633635303838623135336262
38646236306661343338663039666438653039306332396333356664333031383563323065333062
31393165333963326135383935326566383539666161393234303764383838316639366362383339
36386637346661343633356164313466653364663663336231636465323636646130623932326561
63323862343534643334653365653639353466656536373933363033383862393165613630646436
37666439313031633961636665333962303730643332323063326439356238343535623064303061
30393764306238363362356131366337396139303661616464363665353265646539663437363734
63396536326263326336383533656230656462313938623833613130306238343061303061636661
61323566346130623735323662636239356538366632636130663838383938613861343035333138
61376438643432323536363966353364643736663163356366663038626362396266353535313030
35623861376433316331666334313336633139306636333430336536363063613839306638613363
61633261343165386236643265333865623038306263613237656231323831633832646536363464
62323338343039303461663233626130393133643335636631383536633061376632653234353430
66303132653162323332636233326533396165303739376130313531623161316263643738653332
36303436386362386361346330636535626136373236333234653462656262363031383466313330
64346338393433636135616437623037343964383664663862386137316638353862323732646232
31333661626238633632623637626665353430393362653061383462666639326430386664633233
66663933643834613637613234633332636663356330313632356635356265316532346134373431
61356530656262323534626561636465386562646638663337313236336136386234623530656138
63366530353464393139636638343563383330306232376666386133316662373062616535656262
65663439333337373038663035633933323362666363663830666332613261643239613237666438
32376239393032333463306533663534616363636432326234383833653734376632316566383231
33633430616234333736373132653365653530373666316565376535303434393939656133363938
30653465623832373439376439366336613266386330333938613161633932376561616263623064
30376166626333666261626239623363663537636331393531386332653861326339376538393430
66366365333135336538376535346430313630366662656639363133393062623234623536353164
30333539623635626464333332313763333039346638376634343637313365393035333462356333
30356466303562316165356431613336326530346338366334373666333736373438663632613061
63303966323838636630323462343965626266666565626430626531373361366436333837353030
32326531356661666436363431303238613537633530383535376439653166643864303961313037
63663436646365663666363330656432353363356166626133353738653366346165643935326235
37613365653466646137336663313162333964323033663264653132636461363862633630653732
35333839663263643431373739663433393962626637616135336164313163613164333136333862
62326231626138383434363332306635626665656339633332333863306134396163373439343032
64623666343333363631663937646237316363373561633162346438636161633963303731386439
38353131373966663937313632326231623238333438303932346663306633303032353333396363
39393362373933383933636333376162303435386238346237356239633433656566353765356137
35633863646432326638653333313331343266646437643265333162303266323537366531336165
65393035623634323630343436653062316366616562633938356466333165616636613139656333
34313166383339393665313762316164323933393637326131623764326261376536363232316133
64373566326165633865316230666566653934366438376339636338623864643361666465613739
30643666643362646435666463376664323934343537373164616631356234313964316138633164
63646463363233653766666230656266343839386238373637336563616131326631313034623534
36383239316664336133323538396230346538643930383933343131656466373636346432646266
63303133396461306663643066326135343332643066616166396562333131623332636538316330
30646663356335313361653861396165653937343733316438376337306230383639613363343636
37353138323261663031366562316234306364643539356235396366303039326433303065353862
65626330366331663234313739636163646137666465633938393163366664623564373038633937
61346261646338616638663766396337303161383035316433306134633230353533373865393833
36353063613563393734306436646132646331353538363439623930326231306364613364346335
31313834393364373833623763613530646636346364313835643338373636653566326166333065
31663936343439316333646634643161323435333261383335303330613635306531636534336630
39323831643639303838356139316338313536306665396438366434636563333036343339386663
63396436363536396132303961366135636639326638643934663965376436633763663536636332
30303264333565303632333039326461333934616638393630376639646330613830323134323635
61366539343266366332646234373131383532633266636530663736356338316465323034306438
36663033633030303465326162653931333463373163383335343866366262363561373832306136
32333339626131613130646464656261343339306433643434653532653935366139333335323561
35383337393232333738626334343436376561663032653638303336333234333361343164323630
63386263376435313663353737666331336137363566323639616235363439323439653137393930
31643335633766636265336262663866353566623861326634316536663133313634386364353465
33646134313031336331356139356362336133396162623661643765663438336139306438303763
61626333356635303863336633383262633631616666336334346337343963393762626235353963
31316564393064656631356663666635656265653437343762373138376264373263353930343635
34336461303032356234636662633765363436303161323239393533356139653938653463316332
35383030323234356137643136363963353631663636383939633333333261323735653535393730
35616264396235326633336263313437613230626238623661316339323632323563363463653238
63303836613136356637353238343730616661656464663536366661653031366137313266353237
37633131636533326230623465653439303230643935313332666236653465333531366134643938
35313133633332313430386466303338303462306536343366303637666334316339353737373539
32373135393862396334386337653737353738323135353432353437633836663865386433396235
30653435323032303836666164623263633134346461343165346165313435626434656237363364
65346332643566323633623138623562363866663734373864383561356536323461383635363061
35326564646133666266326434313338646463643739346663396462353162663662333861663163
30363939343034383232636335353231333930656364653861623365626237636462316332356532
32313762306466333661306331356364636438353939643432626136623761326636636534623866
32366462366231316662303265316235323230626261646138636338346137663839626163353636
35343938333165313534623866633831363731383036616631336132323637373566306465306136
34653533313134633631303362303834656232623537383464643266663362653964616164323066
38613833313139383637366637623962626163653536393862666639346363623237373164656539
38323064613230643134656236623163343232326239333266313664643632396637633638653062
31333532666334396638393838633865633132356366626533643566623762623130313034613137
62656563613963626633613235393831393039313231353965626236623539313063626664666437
65326462326538646435313438643539333934323734343666386631643636383662623065383930
37663837316463366631353639303938666235363933343666316330613166313063336330313064
62333161636130306531636135306139376264396432623439326261383639326462316666343139
32343638303437303433623638666630656466663737616333666362353730646433306163663233
39393765633535343065663530346438376465386665643534326436383138303536363539643266
31316561363532626633653863323336393534623736613665336331396231613835643335333635
66363565643961356165373032653764366163363537636561393266653764623431363937393164
63326465623062396164343033666366346137353139363336316532656639313666633438343036
34323831336665656334343637343034356136306331356133306662396339653939303365326230
38363434663830323135306464333831333563333932393533383332653263313265653535353266
33616362356164616437353538316661333161636664626138316132383331656132343830393032
35303564396633393732393236346665343630346338663533313034366433653966386332666461
64333631376262393161383434373032323136633362386664326265326364316238363032333462
38346333323461643264363366643862303363663162363765613563323035633834386431373635
36303466326661306637306363623230623936343065376130393862393639363937386238373931
30653239326665613132333830353863626161316663633834626237396239626138656433303661
34643264343461346661313739646665363335313863663633343730323633623039623135343262
33353164393464363838323837633664363438333162666438616432346531373732613838343831
66376539623662343730326533373138323633636537636231346437393338343436666564373766
32336439653834393631363266366235316336613431383530376231373237643932376332306264
39663134656439306266373361323165353836396439383935623935336237623734623738306130
35666435663936616164626137376566613235363239396237353034303261666263393233623632
30363565353732363833633161613662623463366661396530366530346666393733396538643137
33396139343936666231353337636262333833386162373130306237366663373137313133323063
66613133636632353630363636373538336131663963663938393638393032303332636437616365
32326331353664313439363266626365396439613332616561323735353661663934343731326563
64336365623835363633306535376561616139373938663432633262306332393539353737623038
31303036356164313662396631633834363463636430316166633338303264393934383434656432
35386462633736633162313331346538633633376638353363666361633130346465373833353262
37643735333530346437313431366136316430626537336330303230616266626636383530313262
33623337616133356136383665623166393064633838363836633830383535326232303735646434
61306236383730633963343630613966326537616132393030623264643431333230373461323866
62393835616566616231616566386164356430656464666136306335343066643764393466663235
33386161353561396566376333363765643338393730373135313632313739383932623331386237
61333733356565656631633033343262326530653339313966626234383261643231333832623261
34336366663862313964303239303131653663373236366235363162343535306565383062373232
34633738663438353864303965626533633937306431366132613335386338633431653362376139
65626464663262393139303633623831656265633035373539303363366638333866373066376133
32353039393566623061643735346234616531623136313339336634653637643233623038373038
31303864623836333035653364366537303063303366346438666339643538366362363439366237
30336133383133343635353961636233366136663764346538366339333935353833643963353836
32623265396665613133613464333262656434393338336633626234646135316264653866623833
36653930313238353131636266623238313338306163356365346137656666653365333335393138
32393466616136323133633231363135333331333232656633373236356139643935643530366531
64363166393530306463643435313135303232653662383766383562303235303463616166333732
33663636363837663131656530353733633566366366313332346335336435333932393263613230
61623035316331626237356566643962613936383566333537613035323865363138393164346531
31313934313233633337326333356636636132356633656264323666306661386563323139626530
32396361376365623865373662386361653438363331636265626262656636303937323631336132
35396463656532396463666564323663653564313731363533626338313738613735393636643531
64353537343466633665306563636236313837653963633033383534656264323331396638613562
32393739353937656365626662393336633737633562393765313835663939343331643832383135
34653338623430336330343666613635303530333336313436393964333431386235383464623832
30313534306565326236653739353865646166653039323861623539643963643064346136343033
37353236646439386366393563363833333664353533373032343462376331623835363739393635
63613934313436326539383339653030646666363563653466623764313732393830343233383139
32313964383363366635303339323963633638373434306262353665646163663361663730333466
39353034333639656165383134396639316363383335303635633064343561616464643134323535
34356335616466316636396436386236613331653439316462663935653763316437366265373233
30356436666435373563623965333063663439356432383466626131663635386530326633336335
37356661336135653338326236666361373437393031346364303134333762363235346230326430
36643063636332333666653133663834333365346234383638323331356132326138666139313439
30373933343666663333636537313166396238303738313039666630326438623430636131666663
32656334323433326234323839663134376266653439366664393230393230656265346430333739
38366433323838343438363537636139336634396263656263323738643662346432663337633864
35383738393461613036383333633436306632303433343365323036383532633665373964313566
61653130336361613030613232623932623639346666353436626236303639373530373231613234
61336532623537346535656133376230363834373635393838656430333632386565633233386633
66636262633965636163633039303630346632386433633333356564323937616664383130663163
61643138373736303131666332623339623764336265393062356638633331633536353866323238
34363838366136373338326330643962393930356562346161313236663265656664353036323865
38363764306462346632326164343730333861663332636637383837646364333131346662663234
64613035643938643030373939643337303865346232643338323761306334333438663231316661
36653261636333656663373165346635623661616364633933646533343166383531326366323137
38623363376134323937636464616265376566373231643135363139333235393530623034636633
65366131313437623663643038653164316333366237643961636236306136316336653436356436
35343762626532613633386632326330376563323432303465373135616538653437383862643236
32333739306232613838363763646236623662613238316138666537646166303231333263653961
33643632363265323561396639363238323132656265666337363830393630343664613833376237
62396531353336653739636663373535616631623961366439656432626362386364636638316131
31316130326438316230623438376137323832643662363862663166363630373863313765303531
65316338643565643436656266326137663432613461653738373261386566346639353431613765
62653934386338383334636639376139393131393566643164376630333063383131366532326338
39383665356530393165663839303863346233613131313861313731356164643130353236393334
34356663336131383437656339333531323132393365633563353538353635356439623835383366
36363435636165626334613462393434663030666231343037366138396338623335353837643261
64366630663436316436336634383231363133353835633136643736613963313034316535376232
33383562663732376263653836623265633036313431393631363831313662303639626236653164
30646464323239613630663063653037613033396431396334363735376664646633343766316566
63633566333731396365353635316135326261316138303838626337303164613835383662356234
36363965643263396631393264363835643834356264303935336635323233353732663361313566
31393032316365623163393137623537633866333563393732323732306530373236306138313565
30376335356239373366383466383263393731663266626137396466336137636536383537623430
62646535303465633834333365363636663631633536366530323761666564366561306337643339
34613736393262306265363132643864306539663435396664323136313162343966633132343633
63613463363965616332653339626564616431626263393037323862663431323738656363653665
38386263326533626131616630653961306163373062653634666663376261353434643038396434
63383335306435623266666363323333356163303562323065313536353639343263313162366538
65313564653837653933343336626565356434666663313632346164363537376237643131663663
61346533613164643333323235336335393166613036626564653538313837653363613864663636
30323938663965643035313861316539613961326334323435363337613335653365386637396330
34313131393839316665626166333664663339646235666137363336356633323761393966373664
66333462323535366630616564336230653962646338643565343661343134326438386537623263
33373839373135626639373733323466636362323565353437663630323534306636313037363239
31626438306263623339613766666431343866323765333065303234646230643533376464646564
31306233396136633037393435366462303635313936336662616363393363383737653161663435
33623262313339666230643935376239656165633362373661623464653934383530323037646366
65333130373639636536616438393465383664306262623736623438346233623738653631666662
62643837333834326234616331303639626235326337383964633166656261633165336561666362
34636530323535343466393432626134613662663934366335383039613865316132313438303038
38666635343336303639353433366638323363376137616164333231653766616432646364316435
62343932346561316532663538363865353462316338326164643465313963323163393537646337
62313134386633316533663562653137663532636232303166393438623765626665616331653136
63346432656232663162653636373931383263346331343762363761666338343133353763666164
37663837326334666339303537653266303838613965396161383561336637646637666664343765
39313237646266643839323934386437643737393261343639373530623130343038356638613937
62363664663461326536353963393539633038313830666365643263383935366166316635323839
65313361373266613365636231366138346665326565643333633264646535643139366565383562
66303039636565303031373662353232376138383231623565343534623962633961383164306438
65323331623066383934663462333035656331336536393333396430333732326239643531393931
33316264636361613736353066643831386666643333346133396437666334633033393230633265
62613438306664326633656632303232323730313061306265656166343565336530333065613763
32653932663965366562393836633737613632343961346563623232366234323333623962363262
66373063643936386431323338323566633562366336333835636533313038386432646464663731
62633933396337663334643438386134383838326263326636393963323936396462663230643266
30666637303431373938656466343232653036643332356330616561393438353539373461356338
37613135633339316462323465303838643962633132396136666333366162313934383133623262
66343363666663316337643036383662353937656235633264663737636334333938666637383139
35616139643635653830316635393361666339323835613835653430396435653534306234306464
61323630663932666137373933353963306230363830353236376530326530386531386264633537
39656130653263353666623766643362613533303562633330373330656435643231363237636438
35326637303834616134643263386138346137633733356333313162356262666633383637663333
34303765336430613237393830376538333536333266303930373732386262333764363130323032
61343065343862393735376232316431383066326662306335326334373939623261363164303336
34613135323033656666666661353933626538333961653465646138346231643537376166636566
33336130633236383836363731616237323764333737313766636335376538636565613931613131
37383132353636323237396131383134633362623233336437626565396533303330373161613738
62353166353032643632623239353162353465316536343837613039323831616538323633613236
63313566353132346137306664383534666237303666356465383634616238643530633937343533
30663363633737353332633266326335336334323161346337616566633731363165333835616166
65346334646232353932373333623765613837336666653062363732643463386463356261623133
34316336313565646538626433363530313534636463646364353734616265393630333531326132
64336132653537373932313930373934333466363630363665303139363236666231343463363034
62313031633434623138383537333166623732306133306466613539396162303032646534616633
34663365663734363566353264313836343638633638373733383331383239656666386566613235
61653264383964303434363338326136396238626337653862653932323164663838353431623938
34396162393433333834366330383136386565383763326338316434316539656365356334396339
65333532376466653064643363613131663531343566393033356261663737623463303932396463
31356166653037363063656433323033633462323437363062386237373339636166646561346135
62356135386233316339666463323437656164306536373039323431373133626338613166366335
31353364623165366235326637323264383639643038623937333930363038616633333939383065
30333563623433666236303064333261643266393235623737303835643461386230643864666633
65396237313863316166636461663165333532376463306464653138393632393164363965393462
63363932653630323936616261386162356534623835313166633164666632336231333565336262
30646434653133373334356533643665376337383864616561303639616462396231393938396331
32336262633836303436303139626438366138343436376331373266343032323031623139663730
30356462343837643665343763613637663766376136393963346365613934306131343461363530
31383265393161313465633231363766653135356332353563386534323532333963373239626434
31336665353532636464653866386365386432633739313730306335326462626331313434353935
63346261363165633361626239626234313766353134303338636533396336633432323236636530
33633737393566663863663361366435646363393732643533333332383230383730663535613835
34646639663534353263376464316439313631616464306362303235653834323830633231386634
62363638336364333466653636343439353131663831656536343732336466646564323965373232
36383237363466373366373464656264653737396333323831343263646638613832396430643965
65633862636233373961646239363666346235643235356332626533303131313533353130656437
32303434346264376433363864323932656665383732333365383536323331316262663338616630
36313335633639623161346139323161633835333735663430393738373165306666326234303131
35333738626435366164613838333134393833363163373733313536616639633163636261343133
34326532306661316434303336363935653839313533303361366632613033353137303436363562
65636463666131626561326533343535616138313832663865646632353161616536303038626337
35656239326331623762643838303836333266366430313434306534623964633031346164306433
66303831353964663831303839666439323932326135323063653763633436636339343963383262
33646639303432646366646639356634356232323932353334333364303463656638326533653739
37363636376131326338663334643039666539376533326336383338653131306233373362313138
30363262666165656463333933336162323862613164313162323732633736613738373231353037
34636634663266326165343635316539313737313364663063303638306437666564616330303662
65623866353230616235636465363061323766383035356539636561666433623531306266333265
63393663616562376230383566613034386431376464366461316234373161356163396332656466
33316262393030656639306134643535396664356366616437313366623731653032386437396139
63643037666139343562613934613933646131336235653234623161383530356266303938653365
38366234393537393232663736396361663536386566643839373135323233643830326563373931
34616631376634623138333163343438326135383134373063306232346137636164646330333262
65393636376464393534393432386634373665643535656363663635646134313539393362393962
34363934376564373934346564323033613465333131386335663133633139383636313433636338
39653532653134363231663661663736376364653234393537333765376134323165386133643465
33633464376339666131613938313435396131626337303163363631663036353931646131646538
37303630373662396339386464613362616261396131323530346662656436636130313063643865
63396434333164363133363766306436386635653739636266383134346130613930343430323337
61313434303033393537643663343835653566653038386239613061316638386365633037376261
65396466646436333833366132643434373234303962356566363931353166383730313536383735
35323065643664386665653661653261613832366463363062393835313564366436646635636163
32373936616231356566323836373865313939653634373365363965663565663336343331373836
37646237393137613632393563356239633535656466343533353536613164656634373539633061
38626362633732626333336533313165636266623333393636343939666165373133346464373536
66383061313230383932306365643461653666353565356338626232313133656561316361653633
30303063343564626238373337306136373231303135383161303231343765313363663533393737
34383935623136646435306265663738383730633465306434356437376334386466316463393232
61343035306235326139386235346634616535376238643361333137663738303364316634386638
63383962303764303663323437366430623135303038623163646362323132613932363366633164
38613461323337373239663634333136643161653032326334656562313566646365663766646436
64326333303561653130656436303066383563333730633764366139623561323934306635663665
38336561646161363263626364313336663163316637313162383762386362343331313138613564
65623539656336326362323334336263346562643530303064346464643363376134666330653630
37316330323165373566353739663739333133643632363466346432633366663864633034316463
61343935663337373134

View File

@@ -23,12 +23,11 @@ alloy_log_level: "warn"
rommie_port: 20361
rommie_host: "0.0.0.0"
rommie_display: ":10"
rommie_allowed_hosts: "caliban.incus,rommie.ouranos.helu.ca"
rommie_model: Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
rommie_model_url: "http://nyx.helu.ca:22072"
rommie_model: Qwen3.6-27B-Q5_K_M
rommie_model_url: "http://nyx.helu.ca:29000"
rommie_provider: "openai"
rommie_ground_provider: "huggingface"
rommie_ground_url: "http://pan.helu.ca:22076"
rommie_ground_url: "http://pan.helu.ca:29000"
rommie_ground_model: "UI-TARS-7B-DPO-Q6_K_L.gguf"
rommie_grounding_width: 1024
rommie_grounding_height: 1024
@@ -41,6 +40,12 @@ freecad_mcp_user: harper
freecad_mcp_group: harper
freecad_mcp_directory: /srv/freecad-mcp
freecad_mcp_port: 22061
freecad_mcp_xmlrpc_port: 9875
freecad_mcp_socket_port: 9876
# FreeCAD MCP Bridge (GUI, runs as principal_user on the XRDP display)
freecad_mcp_bridge_directory: "/home/{{ principal_user }}/freecad-mcp-bridge"
freecad_mcp_bridge_display: ":10"
# JupyterLab Configuration
@@ -69,10 +74,11 @@ jupyterlab_oauth2_cookie_secret: "{{ vault_jupyterlab_oauth2_cookie_secret }}"
# Kernos MCP Shell Server Configuration
kernos_user: harper
kernos_group: harper
kernos_api_keys: "{{ vault_caliban_kernos_api_keys }}"
kernos_directory: /srv/kernos
kernos_port: 20261
kernos_host: "0.0.0.0"
kernos_log_level: INFO
kernos_log_format: json
kernos_environment: sandbox
kernos_allow_commands: "apt,awk,base64,bash,cat,chmod,cp,curl,cut,date,dd,df,dig,dmesg,du,echo,env,file,find,free,git,grep,gunzip,gzip,head,host,hostname,id,jq,kill,less,ln,ls,lsblk,lspci,lsusb,make,mkdir,mv,nc,node,nohup,npm,npx,ping,pip,pkill,pnpm,printenv,ps,pwd,python3,rm,rsync,run-captured,scp,sed,sleep,sort,source,ssh,ssh-keygen,ssh-keyscan,stat,sudo,tail,tar,tee,timeout,touch,tr,tree,uname,uniq,unzip,uptime,wc,wget,which,whoami,xargs,xz,zip"
kernos_allow_commands: "apt,awk,base64,bash,cat,chmod,cp,curl,cut,date,dd,df,dig,dmesg,docker,du,echo,env,file,find,free,git,grep,gunzip,gzip,head,host,hostname,id,ip,jq,kill,less,ln,ls,lsblk,lspci,lsusb,make,mkdir,mv,nc,node,nohup,npm,npx,ping,pip,pkill,pnpm,printenv,ps,pwd,python3,rm,rsync,run-captured,scp,sed,sleep,sort,source,ssh,ssh-keygen,ssh-keyscan,stat,sudo,tail,tar,tee,timeout,touch,tr,tree,uname,uniq,unzip,uptime,wc,wget,which,whoami,xargs,xz,zip"

View File

@@ -163,3 +163,11 @@ mnemosyne_app_metrics_host: caliban.incus
mnemosyne_app_metrics_port: 23181
mnemosyne_web_metrics_host: caliban.incus
mnemosyne_web_metrics_port: 23191
# Athena — two scrape targets (same shape as Mnemosyne):
# app: Django /metrics via nginx (django-prometheus)
# web: nginx-prometheus-exporter sidecar (nginx stub_status → Prometheus format)
athena_app_metrics_host: puck.incus
athena_app_metrics_port: 22481
athena_web_metrics_host: puck.incus
athena_web_metrics_port: 22491

View File

@@ -52,7 +52,7 @@ mnemosyne_web_metrics_port: 23191
# =============================================================================
# Kottos Configuration (Pallas FastAgent runtime)
# =============================================================================
# Engineering agents (Harper, Scotty, Research, Tech Research) running as a
# Engineering agents (Harper, Scotty, CASE, Research, Tech Research) running as a
# single systemd-managed ``pallas`` process. Logs land in journald via
# SyslogIdentifier=kottos, then Alloy's journal relabel block tags them as
# {service="pallas", project="kottos"} for Loki.
@@ -68,6 +68,7 @@ kottos_harper_port: 24101
kottos_scotty_port: 24102
kottos_research_port: 24150
kottos_tech_research_port: 24151
kottos_case_port: 24152
# Log level — INFO surfaces lifecycle + failures, DEBUG adds per-request
# detail and successful health probe lines. Ouranos Lab convention:
@@ -107,6 +108,12 @@ athena_directory: /srv/athena
athena_port: 22481
athena_domain: "ouranos.helu.ca"
# Prometheus scrape targets (see pplg/prometheus.yml.j2, athena job)
athena_app_metrics_host: "puck.incus"
athena_app_metrics_port: 22481
athena_web_metrics_host: "puck.incus"
athena_web_metrics_port: 22491
# Casdoor SSO Credentials (from vault)
athena_casdoor_client_id: "{{ vault_athena_oauth_client_id }}"
athena_casdoor_client_secret: "{{ vault_athena_oauth_client_secret }}"

View File

@@ -189,15 +189,21 @@ haproxy_backends:
- subdomain: "jupyterlab"
backend_host: "caliban.incus"
backend_port: 22071 # OAuth2-Proxy port
backend_port: 22071
health_path: "/ping"
timeout_server: 300s # WebSocket support
timeout_server: 300s
- subdomain: "hass"
backend_host: "oberon.incus"
backend_port: 8123
health_path: "/api/"
timeout_server: 300s # WebSocket support for HA frontend
timeout_server: 300s
- subdomain: "hecate"
backend_host: "caliban.incus"
backend_port: 20881
health_path: "/live"
timeout_server: 300s
- subdomain: "freecad-mcp"
backend_host: "caliban.incus"
@@ -205,9 +211,15 @@ haproxy_backends:
health_path: "/mcp"
timeout_server: 300s # SSE streaming support for MCP
- subdomain: "caliban"
backend_host: "caliban.incus"
backend_port: 20261
health_path: "/mcp"
timeout_server: 300s # SSE streaming support for MCP
- subdomain: "rommie"
backend_host: "caliban.incus"
backend_port: 22061
backend_port: 20361
health_path: "/mcp"
timeout_server: 300s # SSE streaming support for MCP

View File

@@ -1,24 +0,0 @@
# Kottos runtime environment — rendered by Ansible from inventory host_vars.
# ------------------------------------------------------------------------
# Loaded by systemd (EnvironmentFile=) and inherited by the pallas process.
# ``.env`` vars NOT set here come from pallas.server's defaults — tweak by
# adding the variable to host_vars and this template, not by editing the
# rendered file on the host.
# ── Logging ─────────────────────────────────────────────────────────────────
# Stdout JSON is the preferred sink for systemd+journald+Alloy deployments.
# Rotating file sink is disabled by pointing PALLAS_LOG_FILE at /dev/null so
# we don't write every record twice.
PALLAS_LOG_STDOUT=1
PALLAS_LOG_FILE=/dev/null
PALLAS_LOG_LEVEL={{ pallas_log_level | default('INFO') }}
# ── Config location ─────────────────────────────────────────────────────────
# PALLAS_AGENTS_CONFIG can be overridden to point at a non-default topology
# (e.g. staging scenarios). Default: agents.yaml next to the working dir.
PALLAS_AGENTS_CONFIG={{ kottos_directory }}/agents.yaml
# ── LLM provider / MCP server secrets ───────────────────────────────────────
# Secrets are rendered into fastagent.secrets.yaml rather than env vars so
# fast-agent's existing YAML-merge logic applies. This block stays empty
# intentionally — the template exists for future per-host tunables.

View File

@@ -1,43 +1,62 @@
# Kottos — Deployment Configuration (rendered by Ansible)
# ------------------------------------------------------------------
# Single source of truth for agent topology, ports, and registry
# metadata. Read by Pallas at startup. The kottos/agents.yaml
# committed in the kottos repo is the local-dev equivalent; Ansible
# overwrites it with this rendered version.
#
# Host + namespace + registry port come from inventory host_vars so
# Ouranos / Virgo / Taurus each get their own shape without template
# edits.
# Kottos — Deployment Configuration
# Single source of truth for agent topology, ports, and registry metadata.
# Read by Pallas at startup.
name: kottos
version: "1.0.0"
host: {{ kottos_agents_host | default(kottos_host) | default(inventory_hostname) }}
namespace: {{ kottos_namespace | default('ca.helu.kottos') }}
registry_port: {{ kottos_registry_port | default(24100) }}
host: {{ kottos_bind_host | default(inventory_hostname) }}
namespace: ca.helu.kottos
registry_port: {{ kottos_registry_port }}
agents:
harper:
module: agents.harper
port: {{ kottos_harper_port | default(24101) }}
port: 24101
title: Harper
description: "Scrappy engineer — rapid prototyping, hacking, and creative problem-solving"
depends_on: [research, tech_research]
{% if kottos_harper_model is defined %}
model: {{ kottos_harper_model }}
{% endif %}
scotty:
module: agents.scotty
port: {{ kottos_scotty_port | default(24102) }}
port: 24102
title: Scotty
description: "Systems administration expert — infrastructure diagnostics, security hardening, and keeping everything running"
depends_on: [tech_research]
{% if kottos_scotty_model is defined %}
model: {{ kottos_scotty_model }}
{% endif %}
research:
module: agents.research
port: {{ kottos_research_port | default(24150) }}
port: 24150
title: Research Agent
description: "Web search via Argos and knowledge graph via Neo4j"
{% if kottos_research_model is defined %}
model: {{ kottos_research_model }}
model_capabilities:
vision: {{ kottos_research_model_vision | default(true) }}
context_window: {{ kottos_research_model_context_window | default(16384) }}
max_output_tokens: {{ kottos_research_model_max_output_tokens | default(8192) }}
{% endif %}
tech_research:
module: agents.tech_research
port: {{ kottos_tech_research_port | default(24151) }}
port: 24151
title: Tech Research
description: "Technical investigation — library comparisons, API docs, framework patterns, code examples"
{% if kottos_tech_research_model is defined %}
model: {{ kottos_tech_research_model }}
{% endif %}
case:
module: agents.case
port: 24152
title: CASE
description: "Field systems agent — SD card imaging, LAN scanning, and storage operations on korax.helu.ca"
depends_on: []
{% if kottos_case_model is defined %}
model: {{ kottos_case_model }}
{% endif %}

View File

@@ -1,10 +1,17 @@
---
- name: Deploy Kottos (Pallas FastAgent runtime)
- name: Deploy Kottos AI Agent Platform
hosts: ubuntu
vars:
ansible_common_remote_group: "{{ kottos_group | default([]) }}"
allow_world_readable_tmpfiles: true
handlers:
- name: restart kottos
become: true
ansible.builtin.systemd:
name: kottos
state: restarted
tasks:
- name: Check if host has kottos service
ansible.builtin.set_fact:
@@ -14,51 +21,84 @@
ansible.builtin.meta: end_host
when: not has_kottos_service
- name: Install required packages
become: true
ansible.builtin.apt:
name:
- acl
- npm
- curl
state: present
update_cache: true
- name: Create Kottos group
become: true
ansible.builtin.group:
name: "{{ kottos_group }}"
state: present
- name: Create kottos user
- name: Create Kottos user
become: true
ansible.builtin.user:
name: "{{ kottos_user }}"
group: "{{ kottos_group }}"
home: "/home/{{ kottos_user }}"
home: "{{ kottos_directory }}"
shell: /bin/bash
system: false
create_home: true
system: true
create_home: false
- name: Add keeper_user to kottos group (optional — enables passwordless tailing)
- name: Add keeper_user to kottos group
become: true
ansible.builtin.user:
name: "{{ keeper_user }}"
groups: "{{ kottos_group }}"
append: true
when: keeper_user is defined
- name: Add kottos user to docker group
become: true
ansible.builtin.user:
name: "{{ kottos_user }}"
groups: docker
append: true
notify: restart kottos
- name: Reset connection to pick up new group membership
ansible.builtin.meta: reset_connection
- name: Create Kottos install directory
- name: Create Kottos directory
become: true
ansible.builtin.file:
path: "{{ kottos_directory }}"
owner: "{{ kottos_user }}"
group: "{{ kottos_group }}"
state: directory
mode: '0750'
mode: '750'
- name: Ensure base packages for Python + Docker MCP workflows
- name: Create vendored Pallas directory
become: true
ansible.builtin.file:
path: "{{ kottos_directory }}/vendor/pallas"
owner: "{{ kottos_user }}"
group: "{{ kottos_group }}"
state: directory
mode: '750'
- name: Ensure tar is installed for unarchive task
become: true
ansible.builtin.apt:
name:
- tar
- python3
- python3-venv
- python3-dev
- git
state: present
update_cache: true
- name: Ensure Python 3.13, venv, dev headers, and ACL are installed
become: true
ansible.builtin.apt:
name:
- python3.13
- python3.13-venv
- python3.13-dev
- acl
state: present
update_cache: true
@@ -69,43 +109,52 @@
dest: "{{ kottos_directory }}"
owner: "{{ kottos_user }}"
group: "{{ kottos_group }}"
mode: '0550'
mode: '550'
notify: restart kottos
- name: Ensure .venv directory ownership is correct
- name: Transfer and unarchive vendored Pallas source
become: true
ansible.builtin.file:
path: "{{ kottos_directory }}/.venv"
ansible.builtin.unarchive:
src: "~/rel/pallas_{{ pallas_rel }}.tar"
dest: "{{ kottos_directory }}/vendor/pallas"
owner: "{{ kottos_user }}"
group: "{{ kottos_group }}"
state: directory
recurse: true
when: ansible_facts['file'] is defined or true
mode: '550'
notify: restart kottos
- name: Create virtual environment for Kottos
- name: Rewrite pallas-mcp dependency to use vendored local path
become: true
ansible.builtin.replace:
path: "{{ kottos_directory }}/pyproject.toml"
regexp: '"pallas-mcp @ git\+ssh://[^"]+"'
replace: '"pallas-mcp @ file://{{ kottos_directory }}/vendor/pallas"'
notify: restart kottos
- name: Create virtual environment for Kottos (Python 3.13)
become: true
become_user: "{{ kottos_user }}"
ansible.builtin.command:
cmd: "python3 -m venv {{ kottos_directory }}/.venv/"
cmd: "python3.13 -m venv {{ kottos_directory }}/.venv/"
creates: "{{ kottos_directory }}/.venv/bin/activate"
- name: Install wheel in the virtualenv
- name: Install wheel and mcp-server-time in virtualenv
become: true
become_user: "{{ kottos_user }}"
ansible.builtin.pip:
name:
- wheel
- mcp-server-time
state: latest
virtualenv: "{{ kottos_directory }}/.venv"
- name: Install Kottos (pyproject.toml — pulls in pallas-mcp and fast-agent-mcp)
- name: Install Kottos (and its rewritten local pallas-mcp) in virtualenv
become: true
become_user: "{{ kottos_user }}"
ansible.builtin.pip:
chdir: "{{ kottos_directory }}/kottos"
chdir: "{{ kottos_directory }}"
name: .
virtualenv: "{{ kottos_directory }}/.venv"
virtualenv_command: python3 -m venv
virtualenv_command: python3.13 -m venv
notify: restart kottos
- name: Template agents.yaml
@@ -115,7 +164,7 @@
dest: "{{ kottos_directory }}/agents.yaml"
owner: "{{ kottos_user }}"
group: "{{ kottos_group }}"
mode: '0640'
mode: '640'
notify: restart kottos
- name: Template fastagent.config.yaml
@@ -125,38 +174,27 @@
dest: "{{ kottos_directory }}/fastagent.config.yaml"
owner: "{{ kottos_user }}"
group: "{{ kottos_group }}"
mode: '0640'
mode: '640'
notify: restart kottos
- name: Template fastagent.secrets.yaml (vault-rendered)
- name: Template fastagent.secrets.yaml
become: true
ansible.builtin.template:
src: fastagent.secrets.yaml.j2
dest: "{{ kottos_directory }}/fastagent.secrets.yaml"
owner: "{{ kottos_user }}"
group: "{{ kottos_group }}"
mode: '0600'
notify: restart kottos
no_log: true
- name: Template runtime .env (PALLAS_LOG_STDOUT etc.)
become: true
ansible.builtin.template:
src: .env.j2
dest: "{{ kottos_directory }}/.env"
owner: "{{ kottos_user }}"
group: "{{ kottos_group }}"
mode: '0640'
mode: '640'
notify: restart kottos
- name: Template systemd unit
- name: Template systemd service file
become: true
ansible.builtin.template:
src: kottos.service.j2
dest: /etc/systemd/system/kottos.service
owner: root
group: root
mode: '0644'
mode: '644'
notify: restart kottos
- name: Enable and start kottos service
@@ -167,26 +205,15 @@
state: started
daemon_reload: true
- name: Flush handlers before validation probes
- name: Flush handlers to restart service before validation
ansible.builtin.meta: flush_handlers
# ── Validation ──────────────────────────────────────────────────────────
# Registry is the only endpoint that responds with a deterministic JSON
# payload without requiring an MCP session, so we probe it. Agent ports
# are exercised by Daedalus's health-poll loop once registered.
- name: Validate Kottos registry responds
- name: Validate Kottos registry liveness
ansible.builtin.uri:
url: "http://localhost:{{ kottos_registry_port | default(24100) }}/.well-known/mcp/server.json"
url: "http://localhost:{{ kottos_registry_port }}/live"
status_code: 200
return_content: true
register: registry_check
register: kottos_live
retries: 10
delay: 3
until: registry_check.status == 200
handlers:
- name: restart kottos
become: true
ansible.builtin.systemd:
name: kottos
state: restarted
delay: 5
until: kottos_live.status == 200

View File

@@ -1,66 +1,70 @@
# Kottos — fast-agent configuration (rendered by Ansible)
# ------------------------------------------------------------------
# Committed-to-kottos copy is the local-dev equivalent; Ansible overwrites
# it with this rendered file on deploy. MCP server URLs are parametrised
# so the same template renders correctly for Ouranos (.incus) and Virgo
# (.virgo / .taurus) — each environment's host_vars supplies the base URLs.
# Kottos — Configuration
# LLM provider and MCP server settings.
# Secrets (api_key, tokens) live in fastagent.secrets.yaml (gitignored)
#
# This template is intended to be byte-identical between environments
# (Virgo dev, Taurus prod). All environment-specific values come from
# host_vars or group_vars/all/vars.yml. Do NOT introduce environment-
# specific literals here.
default_model: {{ kottos_default_model | default('openai.Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf') }}
# Default Model Definition
default_model: {{ kottos_default_model }}
# ── Model Capabilities ──────────────────────────────────────────────────────
# Declares capabilities for models not in fast-agent's ModelDatabase.
# vision: true adds image/jpeg, image/png, image/webp to the tokenizer list.
model_capabilities:
vision: {{ kottos_model_vision | default(true) | string | lower }}
context_window: {{ kottos_model_context_window | default(192000) }}
max_output_tokens: {{ kottos_model_max_output_tokens | default(16384) }}
vision: {{ kottos_model_vision }}
context_window: {{ kottos_model_context_window }}
max_output_tokens: {{ kottos_model_max_output_tokens }}
# ── LLM Providers ───────────────────────────────────────────────────────────
# LLM Providers
anthropic:
base_url: {{ kottos_anthropic_base_url }}
generic:
base_url: {{ kottos_generic_base_url }}
openai:
base_url: {{ kottos_openai_base_url | default('http://nyx.helu.ca:22079/v1') }}
base_url: {{ kottos_openai_base_url }}
# MCP Servers — alphabetical to match the dev sample (kottos/fastagent.config.yaml)
mcp:
servers:
# ── Web search via SearXNG (argos) ───────────────────────────────────────
argos:
transport: http
url: "{{ kottos_argos_url | default('http://miranda.incus:25534/mcp') }}"
# ── Knowledge graph — Neo4j ──────────────────────────────────────────────
neo4j_cypher:
transport: http
url: "{{ kottos_neo4j_cypher_url | default('http://circe.helu.ca:22034/mcp') }}"
# ── Shell + file operations — Kernos (Caliban) ───────────────────────────
kernos_scotty:
transport: http
url: "{{ kottos_kernos_scotty_url | default('http://caliban.incus:22062/mcp') }}"
load_on_start: false
# ── Agent S computer automation — Rommie on Caliban ──────────────────────
rommie:
transport: http
url: "{{ kottos_rommie_url | default('http://caliban.incus:20361/mcp') }}"
load_on_start: false
# ── Git repository management — Gitea MCP ────────────────────────────────
gitea:
transport: http
url: "{{ kottos_gitea_url | default('http://miranda.incus:25535/mcp') }}"
# ── Grafana observability ───────────────────────────────────────────────
grafana:
transport: http
url: "{{ kottos_grafana_url | default('http://miranda.incus:25533/mcp') }}"
# ── Shell + file operations — Kernos (Korax) ─────────────────────────────
## Andromeda Shell & File Operations — Kernos for Harper
### Auth header provided by fastagent.secrets.yaml (per-agent Kernos token)
andromeda:
transport: http
url: "{{ kottos_kernos_harper_url | default('http://caliban.helu.ca:20261/mcp') }}"
load_on_start: false
url: "{{ kottos_andromeda_mcp_url }}"
# ── GitHub MCP Server (local Docker, stdio) ──────────────────────────────
# GITHUB_PERSONAL_ACCESS_TOKEN provided by fastagent.secrets.yaml
## Argos Web Search & Page Fetch
### No Auth
argos:
transport: http
url: "{{ kottos_argos_mcp_url }}"
## Argus Shell & File Operations — Kernos for Scotty
### Auth header provided by fastagent.secrets.yaml (per-agent Kernos token)
argus:
transport: http
url: "{{ kottos_argus_mcp_url }}"
## CASE Field Systems — LAN, SD Card, Provisioning
### No Auth
case:
transport: http
url: "http://{{ kottos_case_host }}:{{ kottos_case_port }}"
## Context7 Library/framework documentation (local stdio)
context7:
command: "npx"
args: ["-y", "@upstash/context7-mcp"]
## Gitea Git Repository Management
### No client auth (server-side auth only)
gitea:
transport: http
url: "{{ kottos_gitea_mcp_url }}"
## GitHub MCP Server (local Docker, stdio)
### GITHUB_PERSONAL_ACCESS_TOKEN provided by fastagent.secrets.yaml
github:
command: "docker"
args:
@@ -71,38 +75,57 @@ mcp:
- "GITHUB_PERSONAL_ACCESS_TOKEN"
- "ghcr.io/github/github-mcp-server"
# ── Library/framework documentation — Context7 (local stdio) ─────────────
context7:
command: "npx"
args: ["-y", "@upstash/context7-mcp"]
## Grafana Observability
### No Auth
grafana:
transport: http
url: "{{ kottos_grafana_mcp_url }}"
# ── Current time and timezone (local stdio) ──────────────────────────────
time:
command: "mcp-server-time"
args: ["--local-timezone={{ kottos_timezone | default('America/Toronto') }}"]
## Korax Shell & File Operations — Kernos for CASE
### Auth header provided by fastagent.secrets.yaml (per-agent Kernos token)
korax:
transport: http
url: "{{ kottos_korax_mcp_url }}"
load_on_start: false
# ── Mnemosyne knowledge search — workspace-scoped ────────────────────────
# Auth is a long-lived team JWT supplied by fastagent.secrets.yaml
# (forward_inbound_auth=false — Mnemosyne validates the team JWT).
## Mnemosyne Knowledge Library — workspace-scoped
### Auth is a long-lived team JWT rendered into fastagent.secrets.yaml from
### the OCI Vault entry {env}-mnemosyne-kottos-token.
mnemosyne:
transport: http
url: "{{ kottos_mnemosyne_url | default('https://mnemosyne.ouranos.helu.ca/mcp/') }}"
url: "{{ kottos_mnemosyne_mcp_url }}"
# ── Kottos internal sub-agents ───────────────────────────────────────────
# These stay on localhost regardless of environment — Pallas serves the
# sub-agents on the same host as the top-level agents.
## Neo4j Cypher Memory Graph
neo4j_cypher:
transport: http
url: "{{ kottos_neo4j_mcp_url }}"
## Kottos internal sub-agents
### Research (Web, Knowledge)
research:
transport: http
url: "http://localhost:{{ kottos_research_port | default(24150) }}/mcp"
url: "{{ kottos_research_mcp_url }}"
## Rommie Agent S Computer Use Agent
rommie:
transport: http
url: "{{ kottos_rommie_mcp_url }}"
load_on_start: false
### Research (Web, Context7)
tech_research:
transport: http
url: "http://localhost:{{ kottos_tech_research_port | default(24151) }}/mcp"
url: "{{ kottos_tech_research_mcp_url }}"
## Current time and time calculator (local stdio)
time:
command: "{{ kottos_directory }}/.venv/bin/mcp-server-time"
args: ["--local-timezone={{ kottos_timezone | default('America/Toronto') }}"]
logger:
type: none
level: {{ kottos_fastagent_log_level | default('info') }}
progress_display: false
show_chat: false
show_tools: false
type: console
level: info
progress_display: true
show_chat: true
show_tools: true
truncate_tools: true

View File

@@ -1,27 +1,35 @@
# Kottos — fast-agent secrets (rendered by Ansible from the vault)
# ------------------------------------------------------------------
# Never commit the rendered file. Each value here pulls from a vault
# variable — if a vault variable is missing, Ansible will fail the
# template step with a clear error before the file is written.
#
# Same structure as fastagent.config.yaml; values merge with secrets
# taking precedence (fast-agent deep-merges the two).
# Kottos — Secrets
# Managed by Ansible. Values fetched from OCI Vault at deploy time.
# Merges with fastagent.config.yaml (secrets take precedence).
openai:
api_key: "{{ vault_kottos_openai_api_key }}"
api_key: "{{ kottos_openai_api_key }}"
anthropic:
api_key: "{{ kottos_anthropic_api_key }}"
mcp:
servers:
github:
env:
GITHUB_PERSONAL_ACCESS_TOKEN: "{{ vault_kottos_github_pat }}"
angelia:
# Per-agent Kernos MCP bearer tokens so Kernos can distinguish callers.
# Kottos itself does not consume these — they are surfaced to each agent
# module via fast-agent's server auth headers below.
argus:
headers:
Authorization: "Bearer {{ vault_kottos_angelia_bearer }}"
Authorization: "Bearer {{ scotty_kernos_mcp_token }}"
andromeda:
headers:
Authorization: "Bearer {{ harper_kernos_mcp_token }}"
korax:
headers:
Authorization: "Bearer {{ case_kernos_mcp_token }}"
# Long-lived team JWT minted in Daedalus admin UI.
# See kottos/README.md § "Mnemosyne memory" for the rotation procedure.
# Downstream MCP bearer tokens
arke:
headers:
Authorization: "Bearer {{ kottos_arke_mcp_token }}"
mnemosyne:
headers:
Authorization: "Bearer {{ vault_kottos_mnemosyne_jwt }}"
Authorization: "Bearer {{ mnemosyne_kottos_token }}"
github:
env:
GITHUB_PERSONAL_ACCESS_TOKEN: "{{ kottos_github_pa_token }}"

View File

@@ -1,6 +1,6 @@
[Unit]
Description=Kottos — Pallas FastAgent runtime ({{ kottos_host | default(inventory_hostname) }})
After=network.target
Description=Kottos AI Agent Platform
After=network-online.target
Wants=network-online.target
[Service]
@@ -8,26 +8,17 @@ Type=simple
User={{ kottos_user }}
Group={{ kottos_group }}
WorkingDirectory={{ kottos_directory }}
EnvironmentFile={{ kottos_directory }}/.env
Environment="PATH={{ kottos_directory }}/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
ExecStart={{ kottos_directory }}/.venv/bin/pallas
Restart=always
RestartSec=5
RestartSec=10
# Journal is the durable sink (Alloy picks up via loki.source.journal and
# relabels SyslogIdentifier=kottos into {service="pallas", project="kottos"}
# for Loki). Stdout from pallas is already JSON thanks to
# PALLAS_LOG_STDOUT=1 set in the .env file.
StandardOutput=journal
StandardError=journal
SyslogIdentifier=kottos
# Pallas needs to reach localhost sibling agents + upstream MCP servers
# and read its own .venv / agents.yaml / config files. No hardening flags
# that would block those paths.
NoNewPrivileges=false
ProtectSystem=false
ProtectHome=false
PrivateTmp=false
# Security hardening
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths={{ kottos_directory }}
[Install]
WantedBy=multi-user.target

34
ansible/kottos/remove.yml Normal file
View File

@@ -0,0 +1,34 @@
---
- name: Remove Kottos AI Agent Platform
hosts: ubuntu
become: true
tasks:
- name: Check if host has kottos service
ansible.builtin.set_fact:
has_kottos_service: "{{ 'kottos' in services | default([]) }}"
- name: Skip hosts without kottos service
ansible.builtin.meta: end_host
when: not has_kottos_service
- name: Stop and disable kottos service
ansible.builtin.systemd:
name: kottos
state: stopped
enabled: false
ignore_errors: true
- name: Remove systemd service file
ansible.builtin.file:
path: /etc/systemd/system/kottos.service
state: absent
- name: Reload systemd daemon
ansible.builtin.systemd:
daemon_reload: true
- name: Remove Kottos directory
ansible.builtin.file:
path: "{{ kottos_directory }}"
state: absent

View File

@@ -1,48 +1,84 @@
- name: Stage Kottos release tarball
---
- name: Stage Kottos and Pallas release tarballs
hosts: localhost
gather_facts: false
vars:
archive_path: "{{rel_dir}}/kottos_{{kottos_rel}}.tar"
kottos_archive_path: "{{ rel_dir }}/kottos_{{ kottos_rel }}.tar"
kottos_repo_url: "ssh://git@git.helu.ca:22022/r/kottos.git"
kottos_repo_dir: "{{repo_dir}}/kottos"
kottos_repo_dir: "{{ repo_dir }}/kottos"
pallas_archive_path: "{{ rel_dir }}/pallas_{{ pallas_rel }}.tar"
pallas_repo_url: "ssh://git@git.helu.ca:22022/r/pallas.git"
pallas_repo_dir: "{{ repo_dir }}/pallas"
tasks:
- name: Ensure release directory exists
file:
path: "{{rel_dir}}"
ansible.builtin.file:
path: "{{ rel_dir }}"
state: directory
mode: '755'
- name: Ensure repo directory exists
file:
path: "{{repo_dir}}"
ansible.builtin.file:
path: "{{ repo_dir }}"
state: directory
mode: '755'
# --- Kottos ------------------------------------------------------------
- name: Clone Kottos repository if not present
ansible.builtin.git:
repo: "{{kottos_repo_url}}"
dest: "{{kottos_repo_dir}}"
version: "{{kottos_rel}}"
repo: "{{ kottos_repo_url }}"
dest: "{{ kottos_repo_dir }}"
version: "{{ kottos_rel }}"
accept_hostkey: true
register: git_clone
register: kottos_clone
ignore_errors: true
- name: Fetch latest changes if already cloned
ansible.builtin.git:
repo: "{{kottos_repo_url}}"
dest: "{{kottos_repo_dir}}"
version: "{{kottos_rel}}"
update: true
force: true
- name: Fetch all remote branches and tags (kottos)
ansible.builtin.command: git fetch --all
args:
chdir: "{{ kottos_repo_dir }}"
when: kottos_clone is not changed
changed_when: false
- name: Create release archive
ansible.builtin.archive:
path: "{{kottos_repo_dir}}"
dest: "{{archive_path}}"
format: tar
exclude_path:
- "{{kottos_repo_dir}}/.git"
- "{{kottos_repo_dir}}/.venv"
- "{{kottos_repo_dir}}/__pycache__"
- "{{kottos_repo_dir}}/fastagent.secrets.yaml"
- name: Pull latest changes (kottos)
ansible.builtin.command: git pull
args:
chdir: "{{ kottos_repo_dir }}"
when: kottos_clone is not changed
changed_when: false
- name: Create Kottos archive for specified release
ansible.builtin.command: git archive -o "{{ kottos_archive_path }}" "{{ kottos_rel }}"
args:
chdir: "{{ kottos_repo_dir }}"
changed_when: true
# --- Pallas (kottos runtime dependency) --------------------------------
- name: Clone Pallas repository if not present
ansible.builtin.git:
repo: "{{ pallas_repo_url }}"
dest: "{{ pallas_repo_dir }}"
version: "{{ pallas_rel }}"
accept_hostkey: true
register: pallas_clone
ignore_errors: true
- name: Fetch all remote branches and tags (pallas)
ansible.builtin.command: git fetch --all
args:
chdir: "{{ pallas_repo_dir }}"
when: pallas_clone is not changed
changed_when: false
- name: Pull latest changes (pallas)
ansible.builtin.command: git pull
args:
chdir: "{{ pallas_repo_dir }}"
when: pallas_clone is not changed
changed_when: false
- name: Create Pallas archive for specified release
ansible.builtin.command: git archive -o "{{ pallas_archive_path }}" "{{ pallas_rel }}"
args:
chdir: "{{ pallas_repo_dir }}"
changed_when: true

View File

@@ -244,6 +244,23 @@ groups:
summary: "High log ingestion rate"
description: "Loki is receiving logs at {{ $value | humanize }}/s which may indicate excessive logging"
# ============================================================================
# Django Application Alerts (generic — any Django app exporting the counter)
# ============================================================================
# Apps emit django_superuser_logins_total from a user_logged_in signal when
# the authenticating user is a superuser. The job/component labels identify
# which app fired; forensic detail (user, IP) is in the matching Loki line.
- name: django_alerts
rules:
- alert: DjangoSuperuserLogin
expr: increase(django_superuser_logins_total[5m]) > 0
for: 0m
labels:
severity: warning
annotations:
summary: "Superuser login on {{ $labels.job }}"
description: "A superuser account just logged in to {{ $labels.job }} (component {{ $labels.component }}). This account is rarely used — confirm it was expected. Forensic detail (user, IP) in Loki: {service=\"{{ $labels.job }}\"} |= \"event=superuser_login\"."
# ============================================================================
# Daedalus Application Alerts
# ============================================================================

View File

@@ -68,6 +68,21 @@ scrape_configs:
labels:
component: web
# Athena — same shape as Mnemosyne: the Django container exposes /metrics
# (django-prometheus) proxied via nginx on the app port; a separate
# nginx-prometheus-exporter sidecar re-exposes the web container's
# stub_status in Prometheus format on the web-metrics port.
- job_name: 'athena'
metrics_path: '/metrics'
scrape_interval: 15s
static_configs:
- targets: ['{{ athena_app_metrics_host }}:{{ athena_app_metrics_port }}']
labels:
component: app
- targets: ['{{ athena_web_metrics_host }}:{{ athena_web_metrics_port }}']
labels:
component: web
# Pallas — each deployment is one scrape target (registry port).
# Pallas uses a single process-global registry, so per-agent /metrics
# endpoints serve the same snapshot; the `agent` dimension is carried

View File

@@ -29,7 +29,11 @@ ROMMIE_GROUNDING_HEIGHT={{ rommie_grounding_height | default(1024) }}
# ============================================================================
ROMMIE_HOST={{ rommie_host | default('0.0.0.0') }}
ROMMIE_PORT={{ rommie_port }}
ROMMIE_ALLOWED_HOSTS={{ rommie_allowed_hosts }}
# Idle MCP sessions are reaped after this many seconds (<=0 disables).
# Prevents unbounded StreamableHTTP transport accumulation from clients
# that drop their connection without sending an explicit DELETE.
ROMMIE_SESSION_IDLE_TIMEOUT={{ rommie_session_idle_timeout | default(1800) }}
# ============================================================================
# get_screenshot (parent-agent) output

289
docs/alloy.md Normal file
View File

@@ -0,0 +1,289 @@
# Alloy Log & Metric Collection
Grafana Alloy runs as a **native systemd service** (never in Docker) on every
Ouranos host with `alloy` in its `services` list. It collects logs and forwards
them to **Loki on Prospero** (`http://prospero.incus:3100/loki/api/v1/push`),
and scrapes host/container metrics that it **remote-writes** to **Prometheus on
Prospero** (`http://prospero.incus:9090/api/v1/write`).
## Overview
- **Default config:** [`ansible/alloy/config.alloy.j2`](../ansible/alloy/config.alloy.j2) — journal-only fallback for hosts without a dedicated config.
- **Per-host config:** [`ansible/alloy/<hostname_short>/config.alloy.j2`](../ansible/alloy/) — overrides the default when present.
- **Selection:** [`alloy/deploy.yml`](../ansible/alloy/deploy.yml) stat-checks `<hostname_short>/config.alloy.j2` on the controller; if it exists, that template is rendered, otherwise the default is used.
- **Log destination:** Loki on `prospero.incus:3100` via `loki.write "default"`.
- **Metric destination:** Prometheus on `prospero.incus:9090` via `prometheus.remote_write "default"`.
- **Environment:** every stream is labelled `environment="{{ deployment_environment }}"` (`ouranos`) and `hostname="{{ inventory_hostname }}"`.
- **Deploy:** `ansible-playbook alloy/deploy.yml` (optionally `--limit <host>`).
`deploy.yml` also adds the `alloy` user to the host's `docker` group when the
host has `docker` in its services — this is what lets Alloy read
`/var/run/docker.sock` for the Docker discovery and cAdvisor blocks below.
## Log Sources
Ouranos collects logs through three mechanisms. New Dockerised services should
use the **Docker socket discovery** path (preferred); the per-service syslog
listener is the older pattern, still in use on several hosts.
### 1. Systemd journal (native services)
Every host includes a `loki.source.journal` component capturing all systemd
unit output. By default journal entries are labelled `job="systemd"`; a
`loki.relabel` component can promote specific units to a richer label set (see
[Journal relabeling](#journal-relabeling-native-services)).
This is the correct path for **native systemd services** (binaries managed by a
`.service` unit) — they write to stdout/stderr, systemd captures it in the
journal, and Alloy forwards it. No syslog port or log file needed.
### 2. Docker socket discovery (preferred for containers)
> **Reference implementation:** [`ansible/alloy/puck/config.alloy.j2`](../ansible/alloy/puck/config.alloy.j2).
> Puck is currently the lead host for this pattern; other Docker hosts still use
> per-service syslog listeners and should migrate to this model over time.
A **single** pair of `discovery.docker` + `loki.source.docker` blocks collects
stdout from **every Compose project on the host**, current and future — no
per-service configuration. Container log streams are labelled from Docker's own
Compose metadata:
- `service` ← Compose **project** name (e.g. `athena`, `mnemosyne`, `daedalus`)
- `component` ← Compose **service** name (e.g. `app`, `mcp`, `nginx`, `worker`)
- `container` ← raw container name (for non-Compose `docker run` containers)
```alloy
discovery.docker "containers" {
host = "unix:///var/run/docker.sock"
refresh_interval = "30s"
}
discovery.relabel "containers" {
targets = discovery.docker.containers.targets
rule { // Compose project → service
source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
target_label = "service"
}
rule { // Compose service → component
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
target_label = "component"
}
rule { // container name (non-Compose)
source_labels = ["__meta_docker_container_name"]
regex = "/(.*)"
target_label = "container"
}
rule { // fall back to container name as service
source_labels = ["service", "container"]
separator = "@"
regex = "@(.+)"
target_label = "service"
}
}
loki.source.docker "containers" {
host = "unix:///var/run/docker.sock"
targets = discovery.relabel.containers.output
forward_to = [loki.write.default.receiver]
labels = {
hostname = "{{ inventory_hostname }}",
environment = "{{ deployment_environment }}",
}
}
```
**Why this is preferred over syslog listeners:**
- **Zero per-service wiring.** Adding a new Compose project requires no Alloy
change — it is discovered automatically and labelled by its project name.
- **No startup ordering hazard.** It scrapes Docker's default `json-file` log
driver, so containers never block on an Alloy listener being up (contrast the
syslog driver, below).
- **Consistent `{service, component}` schema** across apps, matching the
Prometheus `component` label used by multi-target scrape jobs (app vs web).
**Requirements:**
- The Compose project must use the default **`json-file`** log driver (i.e. it
must *not* set `logging: { driver: syslog }`). The app must log to **stdout**.
- The `alloy` user needs read access to `/var/run/docker.sock` (handled by
`deploy.yml` adding it to the `docker` group on Docker hosts).
- The `service` label is the **Compose project name**, which defaults to the
deploy directory's basename. Confirm it (`docker compose config``name:`)
when an alert or dashboard depends on a specific `service=` selector.
### 3. Docker syslog driver (legacy, per-service)
The older pattern: each container ships logs via Docker's `syslog` driver to a
dedicated Alloy `loki.source.syslog` listener on a localhost port, labelled with
a static `job`.
```alloy
loki.source.syslog "kairos_logs" {
listener {
address = "127.0.0.1:{{ kairos_syslog_port }}"
protocol = "tcp"
syslog_format = "{{ syslog_format }}" // rfc3164
labels = {
job = "kairos",
hostname = "{{ inventory_hostname }}",
environment = "{{ deployment_environment }}",
}
}
forward_to = [loki.write.default.receiver]
}
```
Container side, in the service's `docker-compose.yml.j2`:
```yaml
logging:
driver: syslog
options:
syslog-address: "tcp://127.0.0.1:{{ kairos_syslog_port }}"
syslog-format: "{{ syslog_format | default('rfc3164') }}"
```
Ports follow the `514XX` convention and live in the host's `host_vars`.
> ⚠️ **Ordering hazard.** The listener must exist before the container starts.
> If `docker compose up` runs while the Alloy listener is not bound, the
> container fails immediately with `failed to initialize logging driver: dial
> tcp 127.0.0.1:<port>: connect: connection refused`. Deploy/verify Alloy on the
> host *before* deploying a syslog-driver service. This hazard is the main
> reason new services should prefer the Docker-socket path instead.
> **Note — labels differ between the two Docker paths.** The syslog listener
> sets `job="<service>"` (no `service`/`component`). The Docker-socket block
> sets `service="<project>"` + `component="<compose service>"` (no `job`). When
> migrating a service off syslog, update any dashboards or alert annotations
> that filter on `{job="…"}` to use `{service="…"}`.
## Journal relabeling (native services)
By default all journal entries share `job="systemd"`, making per-service
filtering impossible. A `loki.relabel` component overrides labels based on the
systemd unit. The journal source forwards to the relabel component instead of
directly to `loki.write`.
```alloy
loki.source.journal "systemd_logs" {
forward_to = [loki.write.default.receiver]
relabel_rules = loki.relabel.journal_puck.rules
labels = {
hostname = "{{ inventory_hostname }}",
environment = "{{ deployment_environment }}",
}
}
loki.relabel "journal_puck" {
forward_to = []
rule { // Pallas runtime → service/project schema
source_labels = ["__journal_syslog_identifier"]
regex = "kottos"
target_label = "service"
replacement = "pallas"
}
rule { // default fallback
source_labels = ["__journal__systemd_unit"]
regex = ".+"
target_label = "job"
replacement = "systemd"
}
}
```
Rules run top-to-bottom; the first match per `target_label` wins, so the
generic `systemd` fallback stays **last**. Escape dots in unit regexes
(`alloy\\.service`). The `__journal_*` fields are hidden metadata — used for
relabeling, not shipped to Loki.
## Metrics
On Docker hosts the per-host config also scrapes host and container metrics and
**remote-writes** them to Prometheus (Alloy is the push agent; Prometheus does
not scrape these hosts directly):
- `prometheus.exporter.unix` — node metrics (Incus-safe collectors only).
- `prometheus.exporter.process``namedprocess_namegroup_*` per command.
- `prometheus.exporter.cadvisor``container_*` metrics via the Docker socket.
These feed `prometheus.scrape` (`job_name` = the host, e.g. `puck`) →
`prometheus.relabel` (adds `instance=<hostname>`) →
`prometheus.remote_write``prospero.incus:9090`.
> Application `/metrics` endpoints (e.g. django-prometheus, the
> nginx-prometheus-exporter sidecar) are **not** scraped by Alloy. Prometheus on
> Prospero scrapes those directly — see
> [`pplg/prometheus.yml.j2`](../ansible/pplg/prometheus.yml.j2).
## Current inventory
### Hosts using Docker socket discovery
| Host | Block | Notes |
|------|-------|-------|
| `puck` | `discovery.docker` + `loki.source.docker "containers"` | Reference implementation. Covers all Compose projects (athena, mnemosyne, daedalus, kairos, …) as `service`/`component`. |
### Hosts using per-service syslog listeners
| Host | Services (job labels) |
|------|-----------------------|
| `puck` | angelia, kairos, spelunker, jupyterlab *(transitional — see below)* |
| `miranda` | argos, neo4j-cypher, grafana_mcp, gitea-mcp, searxng |
| `oberon` | rabbitmq, smtp4dev |
| `rosalind` | gitea, hass, lobechat, jellyfin, searxng (+ apache log files) |
| `titania` | casdoor, haproxy |
| `ariel`, `umbriel` | neo4j |
### Transitional state on puck
`athena`, `mnemosyne`, and `daedalus` have **migrated off** their syslog
listeners to the Docker-socket block; their old `*_syslog_port` host_vars are
retained as reserved-but-unused and can be removed once each rollout is
verified. The remaining `puck` syslog listeners (angelia, kairos, spelunker,
jupyterlab) are candidates to migrate the same way.
## Querying in Grafana
```logql
# All Athena container logs (any component)
{service="athena"}
# Just the Athena MCP container
{service="athena", component="mcp"}
# Superuser-login forensic line behind the DjangoSuperuserLogin alert
{service="athena"} |= "event=superuser_login"
# A syslog-driver service (legacy label scheme)
{job="kairos"}
# Errors across everything on one host
{hostname="puck.incus"} |~ "(?i)error"
```
## Adding a new Dockerised service
**Preferred (Docker socket — no Alloy change needed):**
1. Ensure the service's Compose project uses the default `json-file` log driver
(do **not** set `logging: { driver: syslog }`) and the app logs to stdout.
2. Confirm the host's per-host Alloy config has the `discovery.docker` +
`loki.source.docker` blocks (currently `puck`). If not, add them once
(copy from [`puck/config.alloy.j2`](../ansible/alloy/puck/config.alloy.j2)).
3. Deploy the service. Verify in Grafana: `{service="<compose-project>"}`
returns entries, with `component=<compose-service>`.
**Legacy (syslog driver — only if the host has no Docker-socket block):**
1. Allocate a `514XX` syslog port in the host's `host_vars`.
2. Add a `loki.source.syslog` block to `ansible/alloy/<host>/config.alloy.j2`.
3. Add the `syslog` logging driver to the service's `docker-compose.yml.j2`.
4. **Deploy Alloy first**, then the service.
5. Verify: `{job="<label>", hostname="<host>"}` returns entries.
# Red Panda Seal of Approval 🐼

View File

@@ -9,8 +9,9 @@ This playbook deploys certbot with the Namecheap DNS plugin for DNS-01 validatio
| Installation | Python virtualenv in `/srv/certbot/.venv` |
| DNS Plugin | `certbot-dns-namecheap` |
| Validation | DNS-01 (supports wildcards) |
| Renewal | Systemd timer (twice daily) |
| Certificate Output | `/etc/haproxy/certs/{domain}.pem` |
| Renewal | Systemd timer (twice daily), runs as the `certbot` user |
| Certificate Output | Combined PEM at `haproxy_cert_path` (Titania: `/etc/haproxy/certs/ouranos.pem`) |
| HAProxy Reload | `systemctl reload haproxy` (native systemd, not Docker) |
| Metrics | Prometheus textfile collector |
## Deployments
@@ -69,12 +70,23 @@ services:
# ...
certbot_email: webmaster@helu.ca
certbot_cert_name: ouranos.helu.ca
certbot_domains:
- "*.ouranos.helu.ca"
- "ouranos.helu.ca"
certbot_certificates:
- cert_name: wildcard.ouranos.helu.ca
domains: ["*.ouranos.helu.ca", "ouranos.helu.ca"]
# Where the renewal hook writes the combined fullchain+privkey PEM for HAProxy
haproxy_cert_path: /etc/haproxy/certs/ouranos.pem
```
> The certbot lineage name is **`wildcard.ouranos.helu.ca`**, so the certbot
> config lives under `/srv/certbot/config/live/wildcard.ouranos.helu.ca/`. The
> combined PEM that HAProxy actually serves is a separate file at
> `haproxy_cert_path` (`ouranos.pem`) written by the renewal hook — do not
> confuse the two.
>
> The playbook also supports the single-cert form (`certbot_cert_name` +
> `certbot_domains`) for hosts with one certificate.
### 3. Deploy
```bash
@@ -91,9 +103,9 @@ ansible-playbook certbot/deploy.yml --limit titania.incus
| `/srv/certbot/credentials/namecheap.ini` | Namecheap API credentials (600 perms) |
| `/srv/certbot/hooks/renewal-hook.sh` | Post-renewal script |
| `/srv/certbot/hooks/cert-metrics.sh` | Prometheus metrics script |
| `/etc/haproxy/certs/ouranos.helu.ca.pem` | Combined cert for HAProxy (Titania) |
| `/etc/systemd/system/certbot-renew.service` | Renewal service unit |
| `/etc/systemd/system/certbot-renew.timer` | Twice-daily renewal timer |
| `/etc/haproxy/certs/ouranos.pem` | Combined cert for HAProxy (Titania), written by the renewal hook |
| `/etc/sudoers.d/certbot-haproxy-reload` | Scoped sudo rule letting certbot run `systemctl reload haproxy` |
| `/etc/systemd/system/certbot-renew.service` | Renewal service unit (runs as the `certbot` user) |
| `/etc/systemd/system/certbot-renew.timer` | Twice-daily renewal timer |
## Renewal Process
@@ -105,10 +117,36 @@ ansible-playbook certbot/deploy.yml --limit titania.incus
- Waits 120 seconds for propagation
- Validates and downloads new certificate
- Runs `renewal-hook.sh`
4. Renewal hook:
- Combines fullchain + privkey into HAProxy format
- Reloads HAProxy via `docker compose kill -s HUP haproxy`
- Updates Prometheus metrics
4. Renewal hook (`renewal-hook.sh`, run via certbot's `--deploy-hook`):
- Combines fullchain + privkey into the HAProxy PEM at `haproxy_cert_path`
- Reloads native HAProxy via `sudo -n systemctl reload haproxy`
- Always refreshes Prometheus metrics (even on failure — see below)
> **HAProxy on Titania runs natively under systemd, not in Docker.** The hook
> reloads it with `systemctl reload haproxy`. (Only Casdoor runs in Docker on
> Titania.)
### Permission model (why renewals can silently fail)
The renewal timer runs the hook as the unprivileged **`certbot`** user, so three
permissions must line up or the renewed cert never reaches HAProxy:
| Resource | Required state | Provided by |
|----------|----------------|-------------|
| `/etc/haproxy/certs` | `0770`, group `haproxy`; `certbot` is a member of `haproxy` | `haproxy/deploy.yml` (mode) + `certbot/deploy.yml` (group membership) |
| `systemctl reload haproxy` | allowed for `certbot` via sudo | `/etc/sudoers.d/certbot-haproxy-reload` |
| Prometheus textfile dir | group-writable by `certbot` | `certbot/deploy.yml` |
If any of these is wrong, the hook fails. **Certbot treats a deploy-hook failure
as a non-fatal WARNING and still reports "renewals succeeded"** — so a broken hook
will let the live cert renew while HAProxy keeps serving the *old* file until it
expires. To make this visible, the hook now:
- checks each step and exits non-zero with an explicit
`serving a STALE certificate` error (surfaced in the certbot/journal output), and
- refreshes the Prometheus cert metrics on *every* exit, so the
`SSLCertificateExpiringSoon` / `SSLCertificateExpired` alerts keep reflecting
reality even when installation fails.
## Prometheus Metrics
@@ -137,14 +175,29 @@ Example alert rule:
### View Certificate Status
```bash
# Check certificate expiry (Titania example)
openssl x509 -enddate -noout -in /etc/haproxy/certs/ouranos.helu.ca.pem
# Check expiry of the cert HAProxy actually serves (Titania)
sudo openssl x509 -enddate -noout -in /etc/haproxy/certs/ouranos.pem
# Confirm HAProxy is serving it on the wire
echo | openssl s_client -connect titania.incus:8443 \
-servername grafana.ouranos.helu.ca 2>/dev/null \
| openssl x509 -noout -enddate -issuer
# Check the underlying certbot lineage (may be newer than the served file
# if the deploy hook failed to install it)
sudo openssl x509 -enddate -noout \
-in /srv/certbot/config/live/wildcard.ouranos.helu.ca/fullchain.pem
# Check certbot certificates
sudo -u certbot /srv/certbot/.venv/bin/certbot certificates \
--config-dir /srv/certbot/config
```
> If the served file is older than the certbot lineage, the deploy hook is
> failing to install renewals. Check the hook output:
> `sudo grep -i hook /srv/certbot/logs/letsencrypt.log*` — look for
> `Permission denied`, `reload failed`, or `serving a STALE certificate`.
### Manual Renewal Test
```bash

View File

@@ -374,10 +374,10 @@ MinIO specifically expects certs at `~/.minio/certs/public.crt` and `~/.minio/ce
| Certbot location | On the host itself | OCI free host |
| Namecheap credentials | On the host | Only on OCI host |
| Cert delivery | Direct to HAProxy | Via OCI Vault → Ansible |
| Renewal hook | Docker HAProxy reload | OCI Vault upload |
| Renewal hook | Combine PEM + reload HAProxy | OCI Vault upload |
| Distribution | N/A (local only) | Ansible cron on controller |
| Environments served | Ouranos sandbox only | All environments |
| Service reload | `docker compose kill -s HUP` | `systemctl reload` per host_vars |
| Service reload | `systemctl reload haproxy` (native, via scoped sudo) | `systemctl reload` per host_vars |
Titania can remain self-contained (it's working) or migrate to this centralized model later.

View File

@@ -13,7 +13,61 @@ Infrastructure-as-Code project managing the **Ouranos Lab** — a development sa
> **DNS Domain**: Incus resolves containers via the `.incus` domain suffix (e.g., `oberon.incus`, `portia.incus`). IPv4 addresses are dynamically assigned — always use DNS names, never hardcode IPs.
---
## Project Numbers
- External Apps
- Well known: Postgresl, ssh, web, prometheus
- 220: External Apps (legacy)
- 290: External App 1
- 299: External App 9
- Django Projects:
- 221: Zelus
- 222: Angelia
- 224: Athena
- 225: Kairos
- 226: Icarlos
- 227: MCP Switchboard (227), Spelunker (228), Peitho (229), Mnemosyne (230)
- FastAgent Projects:
- 240: Pallas Iolaus
- 241: Pallas Kottos
- 242: Pallas Mentor
- FastAPI Projects:
- 200: Daedalus
- 201: Arke
- 202: Kernos
- 203: Rommie
- 204: Orpheus
- 205: Periplus
- 206: Nike
- 207: Stentor
- 208: Argos
- 209: Hecate
- 210: Rhema
- 211: Synesis
## Port Numbering
Well-known ports running as a service may be used: Postgresql 5432, Prometheus Metrics 9100.
However inside a docker project, the number plan needs to be followed to avoid port conflicts and confusion:
XXXYZ
XXX Project Number or 290-299 for external project (host specific)
Y Service: 0 reserved, 1-4 flexible, 5 database, 6 MCP, 7 API, 8 Web App, 9 Prometheus metrics
Z Instance: The running instance of this app on the same host, starting at 1. May also be used to handle exceptions.
255 Incus port forwarding: Ports in this range are forwarded from the Incus host to Incus containers (defined in Terraform), but HAProxy through Titania
| Range | Host | Purpose |
|-------|------|---------|
| 2551025519 | caliban | 25512→22 SSH, 25515→5432 Postgres, 25516→8006 web, 25517→8007 web, 25518→8008 web, 25519→3389 RDP |
| 2553025539 | miranda | MCP containers |
| 2554025544 | sycorax | Arke LLM proxy |
| 25554 | ariel | Neo4j |
| 25555 | umbriel | Neo4j (Mnemosyne) |
| 2556025569 | miranda | MCPO ports |
| 2557025589 | puck | 2557025588 app ports, 25589→3389 RDP |
| 2559025599 | oberon | App ports |
514ZZ is the syslog port. Docker containers send their syslog to an Alloy syslog collector port. ZZ is the application instance, they just need to be different on the same host and increment from 01.
## Uranian Host Architecture
@@ -40,12 +94,6 @@ This is the host that runs Python projects in the Ouranos sandbox.
It has an RDP server and is generally where application development happens.
Each project has a number that is used to determine port numbers.
- Docker engine
- JupyterLab (port 22071 via OAuth2-Proxy)
- Gitea Runner (CI/CD agent)
- Django Projects: Zelus (221), Angelia (222), Athena (224), Kairos (225), Icarlos (226), MCP Switchboard (227), Spelunker (228), Peitho (229), Mnemosyne (230)
- FastAgent Projects: Pallas (240)
- FastAPI Projects: Daedalus (200), Arke (201) Kernos (202), Rommie (203), Orpheus (204), Periplus (205), Nike (206), Stentor (207), Argos (208),
### caliban — Agent Automation
@@ -53,20 +101,19 @@ Autonomous computer agent learning through environmental interaction.
- Docker engine
- Agent S MCP Server (MATE desktop, AT-SPI automation)
- Kernos MCP Shell Server (port 22062)
- Rommie MCP Server (port 22061) — agent-to-agent GUI automation via Agent S
- FreeCAD Robust MCP Server (port 22063) — CAD automation via FreeCAD XML-RPC
- Kernos MCP Shell Server
- Rommie MCP Server — agent-to-agent GUI automation via Agent S
- FreeCAD Robust MCP Server — CAD automation via FreeCAD XML-RPC
- GPU passthrough
- RDP access (port 25521)
- RDP access
### oberon — Container Orchestration & Dockerized Shared Services
King of the Fairies orchestrating containers and managing MCP infrastructure.
- Docker engine
- MCP Switchboard (port 22781) — Django app routing MCP tool calls
- RabbitMQ message queue
- smtp4dev SMTP test server (port 22025)
- smtp4dev SMTP test server
### portia — Relational Database
@@ -78,10 +125,7 @@ Intelligent and resourceful — the reliability of relational databases.
### ariel — Graph Database
Air spirit — ethereal, interconnected nature mirroring graph relationships.
- Neo4j 5.26.0 (Docker)
- HTTP API: port 25554
- Bolt: port 7687 (reached as `ariel.incus:7687` on the internal network)
- Neo4j (Docker)
### umbriel — Graph Database (Mnemosyne)
@@ -91,20 +135,18 @@ instance so Mnemosyne's `Library`/`Collection`/`Item`/`Chunk`/`Concept` labels,
vector indexes, and schema migrations can't collide with another tenant's
graph on Ariel.
- Neo4j 5.26.0 (Docker)
- HTTP Browser: port 25555
- Bolt: port 7687 (reached as `umbriel.incus:7687` on the internal network)
- Neo4j (Docker)
### miranda — MCP Docker Host
Curious bridge between worlds — hosting MCP server containers.
- Docker engine (API exposed on port 2375 for MCP Switchboard)
- MCPO OpenAI-compatible MCP proxy 22071
- Argos MCP Server — web search via SearXNG (port 20861)
- Grafana MCP Server (port 22063)
- Neo4j MCP Server (port 22064)
- Gitea MCP Server (port 22062)
- Docker engine
- MCPO OpenAI-compatible MCP
- Argos MCP Server — web search via SearXNG
- Grafana MCP Server
- Neo4j MCP Server
- Gitea MCP Server
### prospero — Observability Stack
@@ -121,11 +163,9 @@ Master magician observing all events.
Witty and resourceful moon for PHP, Go, and Node.js runtimes.
- SearXNG privacy search (port 22083, behind OAuth2-Proxy)
- Gitea self-hosted Git (port 22082, SSH on 22022)
- LobeChat AI chat interface (port 22081)
- Nextcloud file sharing and collaboration (port 22083)
- AnythingLLM document AI workspace (port 22084)
- SearXNG privacy search
- Gitea self-hosted Git
- Nextcloud file sharing and collaboration
- Jellyfin media server (port 22086, NVIDIA transcoding, Casdoor SSO)
- Nextcloud data on dedicated Incus storage volume
- Open WebUI LLM interface (port 22088, PostgreSQL backend on Portia
@@ -135,7 +175,7 @@ Witty and resourceful moon for PHP, Go, and Node.js runtimes.
Original magical power wielding language magic.
- Arke LLM API Proxy (port 25540)
- Arke LLM API Proxy
- Multi-provider support (OpenAI, Anthropic, etc.)
- Session management with Memcached
- Database backend on Portia
@@ -144,7 +184,7 @@ Original magical power wielding language magic.
Queen of the Fairies managing access control and authentication.
- HAProxy 3.x with TLS termination (port 443)
- HAProxy 3.x with TLS termination
- Let's Encrypt wildcard certificate via certbot DNS-01 (Namecheap)
- HTTP to HTTPS redirect (port 80)
- Gitea SSH proxy (port 22022)
@@ -153,21 +193,6 @@ Queen of the Fairies managing access control and authentication.
---
## Port Numbering
Well-known ports running as a service may be used: Postgresql 5432, Prometheus Metrics 9100.
However inside a docker project, the number plan needs to be followed to avoid port conflicts and confusion:
XXXYZ
XXX Project Number or 220 for external project
Y Service: 0 reserved, 1-4 flexible, 5 database, 6 MCP, 7 API, 8 Web App, 9 Prometheus metrics
Z Instance: The running instance of this app on the same host, starting at 1. May also be used to handle exceptions.
255 Incus port forwarding: Ports in ths range are forwarded from the Incus host to Incus containers (defined in Terraform)
514ZZ is the syslog port. Docker containers send their syslog to an Alloy syslog collector port. ZZ is the application instance, they just need to be different on the same host and increment from 01.
---
## Application Conventions
@@ -256,36 +281,9 @@ Titania provides TLS termination and reverse proxy for all services.
- **HTTP**: port 80 (redirects to HTTPS)
- **Certificate**: Let's Encrypt wildcard via certbot DNS-01
### Route Table
### Subdomains
| Subdomain | Backend | Service |
|-----------|---------|---------|
| `ouranos.helu.ca` (root) | puck.incus:22281 | Angelia (Django) |
| `alertmanager.ouranos.helu.ca` | prospero.incus:443 (SSL) | AlertManager |
| `angelia.ouranos.helu.ca` | puck.incus:22281 | Angelia (Django) |
| `anythingllm.ouranos.helu.ca` | rosalind.incus:22084 | AnythingLLM |
| `arke.ouranos.helu.ca` | sycorax.incus:25540 | Arke LLM Proxy |
| `athena.ouranos.helu.ca` | puck.incus:22481 | Athena (Django) |
| `gitea.ouranos.helu.ca` | rosalind.incus:22082 | Gitea |
| `grafana.ouranos.helu.ca` | prospero.incus:443 (SSL) | Grafana |
| `hass.ouranos.helu.ca` | oberon.incus:8123 | Home Assistant |
| `id.ouranos.helu.ca` | titania.incus:22081 | Casdoor SSO |
| `jellyfin.ouranos.helu.ca` | rosalind.incus:22086 | Jellyfin |
| `icarlos.ouranos.helu.ca` | puck.incus:22681 | Icarlos (Django) |
| `jupyterlab.ouranos.helu.ca` | puck.incus:22071 | JupyterLab (OAuth2-Proxy) |
| `kairos.ouranos.helu.ca` | puck.incus:22581 | Kairos (Django) |
| `lobechat.ouranos.helu.ca` | rosalind.incus:22081 | LobeChat |
| `loki.ouranos.helu.ca` | prospero.incus:443 (SSL) | Loki |
| `mcp-switchboard.ouranos.helu.ca` | oberon.incus:22781 | MCP Switchboard |
| `nextcloud.ouranos.helu.ca` | rosalind.incus:22083 | Nextcloud |
| `openwebui.ouranos.helu.ca` | oberon.incus:22088 | Open WebUI |
| `peitho.ouranos.helu.ca` | puck.incus:22981 | Peitho (Django) |
| `periplus.ouranos.helu.ca` | puck.incus:20681 | Periplus (FastAPI + MCP via nginx) |
| `pgadmin.ouranos.helu.ca` | prospero.incus:443 (SSL) | PgAdmin 4 |
| `prometheus.ouranos.helu.ca` | prospero.incus:443 (SSL) | Prometheus |
| `searxng.ouranos.helu.ca` | oberon.incus:22073 | SearXNG (OAuth2-Proxy) |
| `smtp4dev.ouranos.helu.ca` | oberon.incus:22085 | smtp4dev |
| `spelunker.ouranos.helu.ca` | puck.incus:22881 | Spelunker (Django) |
Refer to the Ansible Titania host inventory (`inventory/host_vars/titania.incus.yml`) for current backend routing configuration.
---

View File

@@ -484,17 +484,35 @@ vault_casdoor_prometheus_access_key: "your-casdoor-access-key"
vault_casdoor_prometheus_access_secret: "your-casdoor-access-secret"
```
#### Certificate fetch fails
#### TLS cert expired / not renewing on `*.ouranos.helu.ca`
**Cause**: Titania not running or certbot hasn't provisioned the cert yet.
TLS for all PPLG subdomains is terminated by **Titania's native HAProxy** using
the Let's Encrypt wildcard cert managed by certbot on Titania (see
[certbot DNS-01 with Namecheap](cerbot.md)). PPLG itself holds no cert.
**Fix**: Ensure Titania is up and certbot has run:
**Most likely cause**: certbot renewed the lineage but the deploy hook failed to
install the new cert into HAProxy's served PEM (`/etc/haproxy/certs/ouranos.pem`),
so HAProxy keeps serving the old file until it expires. Certbot reports such hook
failures only as a WARNING, so the renewal looks successful.
**Diagnose** (on Titania):
```bash
ansible-playbook sandbox_up.yml
ansible-playbook certbot/deploy.yml
# Does the served file match the certbot lineage?
sudo openssl x509 -enddate -noout -in /etc/haproxy/certs/ouranos.pem
sudo openssl x509 -enddate -noout \
-in /srv/certbot/config/live/wildcard.ouranos.helu.ca/fullchain.pem
# Look for a failing hook
sudo grep -iE 'hook|Permission denied|reload failed|STALE' /srv/certbot/logs/letsencrypt.log*
```
The playbook falls back to a self-signed certificate if Titania is unavailable.
**Fix**: re-run the playbooks (in this order) and force a renewal to reinstall:
```bash
ansible-playbook haproxy/deploy.yml --limit titania.incus
ansible-playbook certbot/deploy.yml --limit titania.incus
```
See the certbot doc's [permission model](cerbot.md#permission-model-why-renewals-can-silently-fail)
for the `certbot`-user permissions the hook depends on.
#### OAuth2 redirect loops

View File

@@ -158,43 +158,68 @@ EOT
"security.nesting" = true
"raw.lxc" = "lxc.apparmor.profile=unconfined"
}
devices = [{
name = "caliban"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25519"
connect = "tcp:127.0.0.1:3389"
devices = [
{
name = "caliban_rdp"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25519"
connect = "tcp:127.0.0.1:3389"
}
},
{
name = "caliban_web3"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25518"
connect = "tcp:127.0.0.1:8008"
}
},
{
name = "caliban_web2"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25517"
connect = "tcp:127.0.0.1:8007"
}
},
{
name = "caliban_web1"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25516"
connect = "tcp:127.0.0.1:8006"
}
},
{
name = "caliban_postgres"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25515"
connect = "tcp:127.0.0.1:5432"
}
},
{
name = "caliban_ssh"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25512"
connect = "tcp:127.0.0.1:22"
}
},
{
name = "gpu"
type = "gpu"
properties = {}
}
},
{
name = "gpu"
type = "gpu"
properties = {}
}]
]
}
prospero = {
description = "Master magician observing events - PPLG observability stack with internal HAProxy"
role = "observability"
image = "noble"
config = {}
devices = [
{
name = "https_internal"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25510"
connect = "tcp:127.0.0.1:443"
}
},
{
name = "http_redirect"
type = "proxy"
properties = {
listen = "tcp:0.0.0.0:25511"
connect = "tcp:127.0.0.1:80"
}
}
]
devices = []
}
titania = {
description = "Proxy & SSO Services - Queen of the fairies managing access and authentication"

View File

@@ -164,3 +164,33 @@ output "mnemosyne_s3_credentials" {
}
sensitive = true
}
# S3 bucket for Peitho file storage (document versions + converted Office files)
resource "incus_storage_bucket" "peitho" {
name = "peitho"
pool = var.storage_pool
project = var.project_name
description = "Peitho document storage bucket"
depends_on = [incus_project.ouranos]
}
# Access key for Peitho S3 bucket
resource "incus_storage_bucket_key" "peitho_key" {
name = "peitho-access"
pool = incus_storage_bucket.peitho.pool
storage_bucket = incus_storage_bucket.peitho.name
project = var.project_name
role = "admin"
}
output "peitho_s3_credentials" {
description = "Peitho S3 bucket credentials - store in vault as vault_peitho_s3_*"
value = {
bucket = incus_storage_bucket.peitho.name
access_key = incus_storage_bucket_key.peitho_key.access_key
secret_key = incus_storage_bucket_key.peitho_key.secret_key
endpoint = "https://${incus_storage_bucket.peitho.location}"
}
sensitive = true
}

View File

@@ -4,6 +4,7 @@ terraform {
required_providers {
incus = {
source = "lxc/incus"
version = "~> 1.0"
}
}
}