feat(alloy): add journal relabeling and kottos integration on puck

Introduce structured journal relabel rules on puck to tag Pallas-managed
units with {service, project, component} labels matching the Mnemosyne
and Daedalus schema. Add kottos release variable and vault secrets
example entries for the new Pallas FastAgent runtime.

Remove the defunct mnemosyne syslog listener now that Mnemosyne ships
JSON logs via the docker-socket pipeline.

ansible/kottos/.env.j2

@@ -0,0 +1,24 @@
+# Kottos runtime environment — rendered by Ansible from inventory host_vars.
+# ------------------------------------------------------------------------
+# Loaded by systemd (EnvironmentFile=) and inherited by the pallas process.
+# ``.env`` vars NOT set here come from pallas.server's defaults — tweak by
+# adding the variable to host_vars and this template, not by editing the
+# rendered file on the host.
+
+# ── Logging ─────────────────────────────────────────────────────────────────
+# Stdout JSON is the preferred sink for systemd+journald+Alloy deployments.
+# Rotating file sink is disabled by pointing PALLAS_LOG_FILE at /dev/null so
+# we don't write every record twice.
+PALLAS_LOG_STDOUT=1
+PALLAS_LOG_FILE=/dev/null
+PALLAS_LOG_LEVEL={{ pallas_log_level | default('INFO') }}
+
+# ── Config location ─────────────────────────────────────────────────────────
+# PALLAS_AGENTS_CONFIG can be overridden to point at a non-default topology
+# (e.g. staging scenarios).  Default: agents.yaml next to the working dir.
+PALLAS_AGENTS_CONFIG={{ kottos_directory }}/agents.yaml
+
+# ── LLM provider / MCP server secrets ───────────────────────────────────────
+# Secrets are rendered into fastagent.secrets.yaml rather than env vars so
+# fast-agent's existing YAML-merge logic applies.  This block stays empty
+# intentionally — the template exists for future per-host tunables.

ansible/kottos/agents.yaml.j2

@@ -0,0 +1,43 @@
+# Kottos — Deployment Configuration (rendered by Ansible)
+# ------------------------------------------------------------------
+# Single source of truth for agent topology, ports, and registry
+# metadata.  Read by Pallas at startup.  The kottos/agents.yaml
+# committed in the kottos repo is the local-dev equivalent; Ansible
+# overwrites it with this rendered version.
+#
+# Host + namespace + registry port come from inventory host_vars so
+# Ouranos / Virgo / Taurus each get their own shape without template
+# edits.
+
+name: kottos
+version: "1.0.0"
+host: {{ kottos_agents_host | default(kottos_host) | default(inventory_hostname) }}
+namespace: {{ kottos_namespace | default('ca.helu.kottos') }}
+registry_port: {{ kottos_registry_port | default(24100) }}
+
+agents:
+  harper:
+    module: agents.harper
+    port: {{ kottos_harper_port | default(24101) }}
+    title: Harper
+    description: "Scrappy engineer — rapid prototyping, hacking, and creative problem-solving"
+    depends_on: [research, tech_research]
+
+  scotty:
+    module: agents.scotty
+    port: {{ kottos_scotty_port | default(24102) }}
+    title: Scotty
+    description: "Systems administration expert — infrastructure diagnostics, security hardening, and keeping everything running"
+    depends_on: [tech_research]
+
+  research:
+    module: agents.research
+    port: {{ kottos_research_port | default(24150) }}
+    title: Research Agent
+    description: "Web search via Argos and knowledge graph via Neo4j"
+
+  tech_research:
+    module: agents.tech_research
+    port: {{ kottos_tech_research_port | default(24151) }}
+    title: Tech Research
+    description: "Technical investigation — library comparisons, API docs, framework patterns, code examples"

ansible/kottos/deploy.yml

@@ -0,0 +1,192 @@
+---
+- name: Deploy Kottos (Pallas FastAgent runtime)
+  hosts: ubuntu
+  vars:
+    ansible_common_remote_group: "{{ kottos_group | default([]) }}"
+    allow_world_readable_tmpfiles: true
+
+  tasks:
+    - name: Check if host has kottos service
+      ansible.builtin.set_fact:
+        has_kottos_service: "{{ 'kottos' in services | default([]) }}"
+
+    - name: Skip hosts without kottos service
+      ansible.builtin.meta: end_host
+      when: not has_kottos_service
+
+    - name: Create Kottos group
+      become: true
+      ansible.builtin.group:
+        name: "{{ kottos_group }}"
+        state: present
+
+    - name: Create kottos user
+      become: true
+      ansible.builtin.user:
+        name: "{{ kottos_user }}"
+        group: "{{ kottos_group }}"
+        home: "/home/{{ kottos_user }}"
+        shell: /bin/bash
+        system: false
+        create_home: true
+
+    - name: Add keeper_user to kottos group (optional — enables passwordless tailing)
+      become: true
+      ansible.builtin.user:
+        name: "{{ keeper_user }}"
+        groups: "{{ kottos_group }}"
+        append: true
+      when: keeper_user is defined
+
+    - name: Reset connection to pick up new group membership
+      ansible.builtin.meta: reset_connection
+
+    - name: Create Kottos install directory
+      become: true
+      ansible.builtin.file:
+        path: "{{ kottos_directory }}"
+        owner: "{{ kottos_user }}"
+        group: "{{ kottos_group }}"
+        state: directory
+        mode: '0750'
+
+    - name: Ensure base packages for Python + Docker MCP workflows
+      become: true
+      ansible.builtin.apt:
+        name:
+          - tar
+          - python3
+          - python3-venv
+          - python3-dev
+          - git
+        state: present
+        update_cache: true
+
+    - name: Transfer and unarchive Kottos release
+      become: true
+      ansible.builtin.unarchive:
+        src: "~/rel/kottos_{{ kottos_rel }}.tar"
+        dest: "{{ kottos_directory }}"
+        owner: "{{ kottos_user }}"
+        group: "{{ kottos_group }}"
+        mode: '0550'
+      notify: restart kottos
+
+    - name: Ensure .venv directory ownership is correct
+      become: true
+      ansible.builtin.file:
+        path: "{{ kottos_directory }}/.venv"
+        owner: "{{ kottos_user }}"
+        group: "{{ kottos_group }}"
+        state: directory
+        recurse: true
+      when: ansible_facts['file'] is defined or true
+
+    - name: Create virtual environment for Kottos
+      become: true
+      become_user: "{{ kottos_user }}"
+      ansible.builtin.command:
+        cmd: "python3 -m venv {{ kottos_directory }}/.venv/"
+        creates: "{{ kottos_directory }}/.venv/bin/activate"
+
+    - name: Install wheel in the virtualenv
+      become: true
+      become_user: "{{ kottos_user }}"
+      ansible.builtin.pip:
+        name:
+          - wheel
+        state: latest
+        virtualenv: "{{ kottos_directory }}/.venv"
+
+    - name: Install Kottos (pyproject.toml — pulls in pallas-mcp and fast-agent-mcp)
+      become: true
+      become_user: "{{ kottos_user }}"
+      ansible.builtin.pip:
+        chdir: "{{ kottos_directory }}/kottos"
+        name: .
+        virtualenv: "{{ kottos_directory }}/.venv"
+        virtualenv_command: python3 -m venv
+      notify: restart kottos
+
+    - name: Template agents.yaml
+      become: true
+      ansible.builtin.template:
+        src: agents.yaml.j2
+        dest: "{{ kottos_directory }}/agents.yaml"
+        owner: "{{ kottos_user }}"
+        group: "{{ kottos_group }}"
+        mode: '0640'
+      notify: restart kottos
+
+    - name: Template fastagent.config.yaml
+      become: true
+      ansible.builtin.template:
+        src: fastagent.config.yaml.j2
+        dest: "{{ kottos_directory }}/fastagent.config.yaml"
+        owner: "{{ kottos_user }}"
+        group: "{{ kottos_group }}"
+        mode: '0640'
+      notify: restart kottos
+
+    - name: Template fastagent.secrets.yaml (vault-rendered)
+      become: true
+      ansible.builtin.template:
+        src: fastagent.secrets.yaml.j2
+        dest: "{{ kottos_directory }}/fastagent.secrets.yaml"
+        owner: "{{ kottos_user }}"
+        group: "{{ kottos_group }}"
+        mode: '0600'
+      notify: restart kottos
+      no_log: true
+
+    - name: Template runtime .env (PALLAS_LOG_STDOUT etc.)
+      become: true
+      ansible.builtin.template:
+        src: .env.j2
+        dest: "{{ kottos_directory }}/.env"
+        owner: "{{ kottos_user }}"
+        group: "{{ kottos_group }}"
+        mode: '0640'
+      notify: restart kottos
+
+    - name: Template systemd unit
+      become: true
+      ansible.builtin.template:
+        src: kottos.service.j2
+        dest: /etc/systemd/system/kottos.service
+        owner: root
+        group: root
+        mode: '0644'
+      notify: restart kottos
+
+    - name: Enable and start kottos service
+      become: true
+      ansible.builtin.systemd:
+        name: kottos
+        enabled: true
+        state: started
+        daemon_reload: true
+
+    - name: Flush handlers before validation probes
+      ansible.builtin.meta: flush_handlers
+
+    # ── Validation ──────────────────────────────────────────────────────────
+    # Registry is the only endpoint that responds with a deterministic JSON
+    # payload without requiring an MCP session, so we probe it.  Agent ports
+    # are exercised by Daedalus's health-poll loop once registered.
+    - name: Validate Kottos registry responds
+      ansible.builtin.uri:
+        url: "http://localhost:{{ kottos_registry_port | default(24100) }}/.well-known/mcp/server.json"
+        status_code: 200
+        return_content: true
+      register: registry_check
+      retries: 10
+      delay: 3
+      until: registry_check.status == 200
+
+  handlers:
+    - name: restart kottos
+      become: true
+      ansible.builtin.systemd:
+        name: kottos
+        state: restarted

ansible/kottos/fastagent.config.yaml.j2

@@ -0,0 +1,114 @@
+# Kottos — fast-agent configuration (rendered by Ansible)
+# ------------------------------------------------------------------
+# Committed-to-kottos copy is the local-dev equivalent; Ansible overwrites
+# it with this rendered file on deploy.  MCP server URLs are parametrised
+# so the same template renders correctly for Ouranos (.incus) and Virgo
+# (.virgo / .taurus) — each environment's host_vars supplies the base URLs.
+
+default_model: {{ kottos_default_model | default('openai.Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf') }}
+
+# ── Model Capabilities ──────────────────────────────────────────────────────
+# Declares capabilities for models not in fast-agent's ModelDatabase.
+# vision: true adds image/jpeg, image/png, image/webp to the tokenizer list.
+model_capabilities:
+  vision: {{ kottos_model_vision | default(true) | string | lower }}
+  context_window: {{ kottos_model_context_window | default(192000) }}
+  max_output_tokens: {{ kottos_model_max_output_tokens | default(16384) }}
+
+# ── LLM Providers ───────────────────────────────────────────────────────────
+openai:
+  base_url: {{ kottos_openai_base_url | default('http://nyx.helu.ca:22079/v1') }}
+
+mcp:
+  servers:
+    # ── Web search via SearXNG (argos) ───────────────────────────────────────
+    argos:
+      transport: http
+      url: "{{ kottos_argos_url | default('http://miranda.incus:25534/mcp') }}"
+
+    # ── Knowledge graph — Neo4j ──────────────────────────────────────────────
+    neo4j_cypher:
+      transport: http
+      url: "{{ kottos_neo4j_cypher_url | default('http://circe.helu.ca:22034/mcp') }}"
+
+    # ── Shell + file operations — Kernos (Caliban) ───────────────────────────
+    kernos_scotty:
+      transport: http
+      url: "{{ kottos_kernos_scotty_url | default('http://caliban.incus:22062/mcp') }}"
+      load_on_start: false
+
+    # ── Agent S computer automation — Rommie on Caliban ──────────────────────
+    rommie:
+      transport: http
+      url: "{{ kottos_rommie_url | default('http://caliban.incus:20361/mcp') }}"
+      load_on_start: false
+
+    # ── Git repository management — Gitea MCP ────────────────────────────────
+    gitea:
+      transport: http
+      url: "{{ kottos_gitea_url | default('http://miranda.incus:25535/mcp') }}"
+
+    # ── Grafana observability ───────────────────────────────────────────────
+    grafana:
+      transport: http
+      url: "{{ kottos_grafana_url | default('http://miranda.incus:25533/mcp') }}"
+
+    # ── Shell + file operations — Kernos (Korax) ─────────────────────────────
+    kernos_harper:
+      transport: http
+      url: "{{ kottos_kernos_harper_url | default('http://korax.helu.ca:20261/mcp') }}"
+      load_on_start: false
+
+    # ── Angelia messaging ───────────────────────────────────────────────────
+    # Auth header provided by fastagent.secrets.yaml (vault-rendered).
+    angelia:
+      transport: http
+      url: "{{ kottos_angelia_url | default('https://ouranos.helu.ca/mcp/') }}"
+
+    # ── GitHub MCP Server (local Docker, stdio) ──────────────────────────────
+    # GITHUB_PERSONAL_ACCESS_TOKEN provided by fastagent.secrets.yaml
+    github:
+      command: "docker"
+      args:
+        - "run"
+        - "-i"
+        - "--rm"
+        - "-e"
+        - "GITHUB_PERSONAL_ACCESS_TOKEN"
+        - "ghcr.io/github/github-mcp-server"
+
+    # ── Library/framework documentation — Context7 (local stdio) ─────────────
+    context7:
+      command: "npx"
+      args: ["-y", "@upstash/context7-mcp"]
+
+    # ── Current time and timezone (local stdio) ──────────────────────────────
+    time:
+      command: "mcp-server-time"
+      args: ["--local-timezone={{ kottos_timezone | default('America/Toronto') }}"]
+
+    # ── Mnemosyne knowledge search — workspace-scoped ────────────────────────
+    # Auth is a long-lived team JWT supplied by fastagent.secrets.yaml
+    # (forward_inbound_auth=false — Mnemosyne validates the team JWT).
+    mnemosyne:
+      transport: http
+      url: "{{ kottos_mnemosyne_url | default('https://mnemosyne.ouranos.helu.ca/mcp/') }}"
+
+    # ── Kottos internal sub-agents ───────────────────────────────────────────
+    # These stay on localhost regardless of environment — Pallas serves the
+    # sub-agents on the same host as the top-level agents.
+    research:
+      transport: http
+      url: "http://localhost:{{ kottos_research_port | default(24150) }}/mcp"
+
+    tech_research:
+      transport: http
+      url: "http://localhost:{{ kottos_tech_research_port | default(24151) }}/mcp"
+
+logger:
+  type: none
+  level: {{ kottos_fastagent_log_level | default('info') }}
+  progress_display: false
+  show_chat: false
+  show_tools: false
+  truncate_tools: true

ansible/kottos/fastagent.secrets.yaml.j2

@@ -0,0 +1,27 @@
+# Kottos — fast-agent secrets (rendered by Ansible from the vault)
+# ------------------------------------------------------------------
+# Never commit the rendered file.  Each value here pulls from a vault
+# variable — if a vault variable is missing, Ansible will fail the
+# template step with a clear error before the file is written.
+#
+# Same structure as fastagent.config.yaml; values merge with secrets
+# taking precedence (fast-agent deep-merges the two).
+
+openai:
+  api_key: "{{ vault_kottos_openai_api_key }}"
+
+mcp:
+  servers:
+    github:
+      env:
+        GITHUB_PERSONAL_ACCESS_TOKEN: "{{ vault_kottos_github_pat }}"
+
+    angelia:
+      headers:
+        Authorization: "Bearer {{ vault_kottos_angelia_bearer }}"
+
+    # Long-lived team JWT minted in Daedalus admin UI.
+    # See kottos/README.md § "Mnemosyne memory" for the rotation procedure.
+    mnemosyne:
+      headers:
+        Authorization: "Bearer {{ vault_kottos_mnemosyne_jwt }}"

ansible/kottos/kottos.service.j2

@@ -0,0 +1,33 @@
+[Unit]
+Description=Kottos — Pallas FastAgent runtime ({{ kottos_host | default(inventory_hostname) }})
+After=network.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ kottos_user }}
+Group={{ kottos_group }}
+WorkingDirectory={{ kottos_directory }}
+EnvironmentFile={{ kottos_directory }}/.env
+ExecStart={{ kottos_directory }}/.venv/bin/pallas
+Restart=always
+RestartSec=5
+
+# Journal is the durable sink (Alloy picks up via loki.source.journal and
+# relabels SyslogIdentifier=kottos into {service="pallas", project="kottos"}
+# for Loki).  Stdout from pallas is already JSON thanks to
+# PALLAS_LOG_STDOUT=1 set in the .env file.
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=kottos
+
+# Pallas needs to reach localhost sibling agents + upstream MCP servers
+# and read its own .venv / agents.yaml / config files.  No hardening flags
+# that would block those paths.
+NoNewPrivileges=false
+ProtectSystem=false
+ProtectHome=false
+PrivateTmp=false
+
+[Install]
+WantedBy=multi-user.target

ansible/kottos/stage.yml

@@ -0,0 +1,48 @@
+- name: Stage Kottos release tarball
+  hosts: localhost
+  gather_facts: false
+  vars:
+    archive_path: "{{rel_dir}}/kottos_{{kottos_rel}}.tar"
+    kottos_repo_url: "ssh://git@git.helu.ca:22022/r/kottos.git"
+    kottos_repo_dir: "{{repo_dir}}/kottos"
+
+  tasks:
+    - name: Ensure release directory exists
+      file:
+        path: "{{rel_dir}}"
+        state: directory
+        mode: '755'
+
+    - name: Ensure repo directory exists
+      file:
+        path: "{{repo_dir}}"
+        state: directory
+        mode: '755'
+
+    - name: Clone Kottos repository if not present
+      ansible.builtin.git:
+        repo: "{{kottos_repo_url}}"
+        dest: "{{kottos_repo_dir}}"
+        version: "{{kottos_rel}}"
+        accept_hostkey: true
+      register: git_clone
+      ignore_errors: true
+
+    - name: Fetch latest changes if already cloned
+      ansible.builtin.git:
+        repo: "{{kottos_repo_url}}"
+        dest: "{{kottos_repo_dir}}"
+        version: "{{kottos_rel}}"
+        update: true
+        force: true
+
+    - name: Create release archive
+      ansible.builtin.archive:
+        path: "{{kottos_repo_dir}}"
+        dest: "{{archive_path}}"
+        format: tar
+        exclude_path:
+          - "{{kottos_repo_dir}}/.git"
+          - "{{kottos_repo_dir}}/.venv"
+          - "{{kottos_repo_dir}}/__pycache__"
+          - "{{kottos_repo_dir}}/fastagent.secrets.yaml"