🐾 fix: resolve registry capabilities per agent, not once globally

_build_registry built one capabilities dict from the global default_model and
attached it to every entry, so any agent with an `agents.<name>.model` or
`model_capabilities` override was advertised under the wrong model — even
though server.py applies those overrides at startup. Resolve capabilities per
agent, mirroring _register_unknown_models.

Also stop emitting null context_window / max_output_tokens when
model_capabilities is absent: _register_one_model registers the model with
131072 / 16384, so the registry now advertises those effective values. The
defaults are hoisted to module constants in server.py so the two cannot drift.

Verified against mentor's config: capabilities go from
{"context_window": null, "max_output_tokens": null} to {131072, 16384}, model
name unchanged. No checked-in deployment uses per-agent overrides today, so no
currently-published model changes.

Adds tests/test_registry.py (9 tests, first coverage for registry.py) and
documents agents.<name>.model / model_capabilities, which the agents.yaml
field table omitted entirely.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-08-03 12:22:30 -04:00
parent 5af198911b
commit 73f6afdbf8
5 changed files with 258 additions and 26 deletions

202
tests/test_registry.py Normal file
View File

@@ -0,0 +1,202 @@
"""Tests for pallas.registry — per-agent model capability resolution.
The registry advertises the capabilities Pallas actually registered with
fast-agent, resolved *per agent*: an agent's own ``model`` /
``model_capabilities`` in agents.yaml override the global ``default_model`` /
``model_capabilities`` in fastagent.config.yaml, and the same effective
defaults as ``server._register_one_model`` apply when a field is omitted.
Regression cover for two bugs: a single capabilities dict was previously
built from ``default_model`` and attached to *every* agent entry (so an
agent with a ``model:`` override was advertised under the wrong model), and
``context_window`` / ``max_output_tokens`` were published as ``null`` when
``model_capabilities`` was absent even though the model had been registered
with 131072 / 16384.
``_build_registry`` takes the deployment config as an argument but reads
fastagent.config.yaml from the working directory on every call, so each test
chdirs into a clean temp workspace and writes only the config it needs.
"""
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from pallas import registry
from pallas.server import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS
# ── Helpers ──────────────────────────────────────────────────────────────────
@pytest.fixture
def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""Chdir into a clean temp workspace — _build_registry reads cwd."""
monkeypatch.chdir(tmp_path)
return tmp_path
def _write_fastagent(workspace: Path, **config) -> None:
(workspace / "fastagent.config.yaml").write_text(yaml.safe_dump(config))
def _deployment(**agents) -> dict:
"""Minimal agents.yaml-shaped config; each agent needs at least a port."""
return {
"name": "test-project",
"namespace": "ca.helu.test",
"host": "test-host",
"agents": {
name: {"module": f"agents.{name}", "port": 9000 + i, **overrides}
for i, (name, overrides) in enumerate(agents.items())
},
}
def _entries(config: dict) -> dict[str, dict]:
"""Build the registry and return ``{agent slug: server entry}``."""
servers = registry._build_registry(config)["servers"]
return {e["server"]["name"].rsplit("/", 1)[-1]: e["server"] for e in servers}
def _capabilities(config: dict, agent: str = "solo") -> dict | None:
return _entries(config)[agent].get("capabilities")
# ── Model resolution ─────────────────────────────────────────────────────────
def test_agent_model_overrides_default_model(workspace: Path) -> None:
"""An agents.yaml ``model:`` wins over the global default_model."""
_write_fastagent(workspace, default_model="openai.global-model")
caps = _capabilities(_deployment(solo={"model": "anthropic.agent-model"}))
assert caps is not None
assert caps["model"] == "agent-model"
def test_falls_back_to_default_model(workspace: Path) -> None:
"""With no per-agent model, the global default_model is advertised."""
_write_fastagent(workspace, default_model="anthropic.claude-opus-4-7")
caps = _capabilities(_deployment(solo={}))
assert caps is not None
# Provider prefix is stripped — clients get the bare model name.
assert caps["model"] == "claude-opus-4-7"
def test_model_without_provider_prefix_passes_through(workspace: Path) -> None:
"""A default_model with no ``provider.`` prefix is emitted verbatim."""
_write_fastagent(workspace, default_model="bare-model")
assert _capabilities(_deployment(solo={}))["model"] == "bare-model"
# ── Capability resolution ────────────────────────────────────────────────────
def test_agent_capabilities_override_global(workspace: Path) -> None:
"""A per-agent model_capabilities block replaces the global one."""
_write_fastagent(
workspace,
default_model="openai.global-model",
model_capabilities={"vision": False, "context_window": 200000},
)
caps = _capabilities(
_deployment(
solo={
"model": "anthropic.agent-model",
"model_capabilities": {"vision": True, "context_window": 400000},
}
)
)
assert caps["vision"] is True
assert caps["context_window"] == 400000
def test_effective_defaults_when_capabilities_absent(workspace: Path) -> None:
"""Absent model_capabilities publishes the values actually registered.
Regression: these were previously advertised as ``null`` while
``server._register_one_model`` registered 131072 / 16384.
"""
_write_fastagent(workspace, default_model="openai.some-model")
caps = _capabilities(_deployment(solo={}))
assert caps["context_window"] == DEFAULT_CONTEXT_WINDOW
assert caps["max_output_tokens"] == DEFAULT_MAX_OUTPUT_TOKENS
assert caps["vision"] is False
def test_global_capabilities_are_published(workspace: Path) -> None:
"""The ordinary case: one global model + capabilities for every agent."""
_write_fastagent(
workspace,
default_model="anthropic.claude-opus-4-7",
model_capabilities={
"vision": True,
"context_window": 200000,
"max_output_tokens": 50000,
},
)
caps = _capabilities(_deployment(solo={}))
assert caps == {
"model": "claude-opus-4-7",
"vision": True,
"context_window": 200000,
"max_output_tokens": 50000,
}
# ── Per-agent independence ───────────────────────────────────────────────────
def test_agents_resolve_independently(workspace: Path) -> None:
"""Each entry gets its own capabilities — the original shared-dict bug."""
_write_fastagent(
workspace,
default_model="openai.global-model",
model_capabilities={"vision": False, "context_window": 128000},
)
entries = _entries(
_deployment(
inherits={},
overrides={
"model": "anthropic.special-model",
"model_capabilities": {"vision": True, "context_window": 500000},
},
)
)
assert entries["inherits"]["capabilities"]["model"] == "global-model"
assert entries["inherits"]["capabilities"]["context_window"] == 128000
assert entries["overrides"]["capabilities"]["model"] == "special-model"
assert entries["overrides"]["capabilities"]["context_window"] == 500000
# ── Omission ─────────────────────────────────────────────────────────────────
def test_capabilities_omitted_without_any_model(workspace: Path) -> None:
"""No fastagent.config.yaml and no per-agent model → no capabilities key."""
entry = _entries(_deployment(solo={}))["solo"]
assert "capabilities" not in entry
def test_agent_model_published_without_fastagent_config(workspace: Path) -> None:
"""An agent's own model is advertised even with no fastagent.config.yaml."""
caps = _capabilities(_deployment(solo={"model": "anthropic.agent-model"}))
assert caps["model"] == "agent-model"
assert caps["context_window"] == DEFAULT_CONTEXT_WINDOW