Merge pull request '🐾 fix: resolve registry capabilities per agent, not once globally' (#6) from fix/registry-per-agent-capabilities into main
Reviewed-on: #6
This commit was merged in pull request #6.
This commit is contained in:
@@ -192,6 +192,8 @@ agents:
|
||||
| `agents.<name>.port` | yes | Port for this agent's StreamableHTTP MCP server |
|
||||
| `agents.<name>.title` | no | Display name in registry. Default: `name.title()` |
|
||||
| `agents.<name>.description` | no | Description in registry |
|
||||
| `agents.<name>.model` | no | `provider.model-name` override for this agent. Overrides `default_model`, is applied to every agent in the module at startup, and is what the registry advertises for this entry |
|
||||
| `agents.<name>.model_capabilities` | no | Per-agent `{vision, context_window, max_output_tokens}` block. Overrides the top-level `model_capabilities`; the same defaults apply to omitted fields |
|
||||
| `agents.<name>.depends_on` | no | List of agent names that must start and become ready before this agent |
|
||||
| `agents.<name>.max_iterations` | no | Hard cap on agentic-loop turns per `send_message`. Default: `15`. fast-agent returns a partial answer once exceeded |
|
||||
| `agents.<name>.loop_repeat_threshold` | no | Halt the loop after this many consecutive identical `(tool, args) → result` rounds. Default: `3`. `0` disables the guard |
|
||||
@@ -429,7 +431,9 @@ Built dynamically from `agents.yaml` + `fastagent.config.yaml`:
|
||||
|
||||
### Capabilities
|
||||
|
||||
If `model_capabilities` is defined in `fastagent.config.yaml`, each registry entry includes a `capabilities` object with model name, vision support, context window, and max output tokens. This allows clients to make informed decisions about what an agent can handle.
|
||||
Each registry entry includes a `capabilities` object — model name, vision support, context window, and max output tokens — whenever a model is known for that agent, i.e. `agents.<name>.model` is set or `default_model` is defined in `fastagent.config.yaml`.
|
||||
|
||||
Capabilities are resolved **per agent**: the agent's own `model` and `model_capabilities` take precedence over the global values, and omitted fields fall back to the same defaults Pallas uses to register the model (`vision: false`, `context_window: 131072`, `max_output_tokens: 16384`). The published values therefore match what was actually registered with fast-agent's `ModelDatabase`, rather than being null whenever `model_capabilities` was left out. Clients use them to make informed decisions about what an agent can handle.
|
||||
|
||||
---
|
||||
|
||||
@@ -673,12 +677,13 @@ Pallas registers models not in fast-agent's built-in `ModelDatabase` at startup,
|
||||
The process:
|
||||
|
||||
1. Read `default_model` and `model_capabilities` from config
|
||||
2. Extract the model name (portion after the provider prefix dot)
|
||||
3. Check if `ModelDatabase` already knows this model — if so, skip
|
||||
4. Register with `ModelDatabase.register_runtime_model_params()`:
|
||||
2. Also read every `agents.<name>.model` from `agents.yaml`, using that agent's own `model_capabilities` when it declares one
|
||||
3. Extract the model name (portion after the provider prefix dot)
|
||||
4. Check if `ModelDatabase` already knows this model — if so, skip
|
||||
5. Register with `ModelDatabase.register_runtime_model_params()`:
|
||||
- `vision: true` → multimodal tokenization (`QWEN_MULTIMODAL`)
|
||||
- `vision: false` → text-only tokenization (`TEXT_ONLY`)
|
||||
- `context_window` and `max_output_tokens` from config (with sensible defaults)
|
||||
- `context_window` and `max_output_tokens` from config, defaulting to `131072` / `16384` — the same values the registry advertises
|
||||
|
||||
This avoids the brittle pattern of inferring capabilities from model name substrings, which breaks for custom or fine-tuned models with non-standard names.
|
||||
|
||||
|
||||
@@ -120,7 +120,7 @@ No authentication. No query parameters.
|
||||
| `servers[].server.version` | string | no | Semver version string. |
|
||||
| `servers[].server.icons` | array | no | Array of `{ src, sizes }`. Daedalus uses the first entry. |
|
||||
| `servers[].server.remotes` | array | yes | Connection endpoints. Daedalus looks for `type: "streamable-http"` and uses its `url`. |
|
||||
| `servers[].server.capabilities` | object | no | Model capabilities. Contains `model` (string), `vision` (bool), `context_window` (int), `max_output_tokens` (int). Published when `model_capabilities` is configured in `fastagent.config.yaml`. |
|
||||
| `servers[].server.capabilities` | object | no | Model capabilities. Contains `model` (string), `vision` (bool), `context_window` (int), `max_output_tokens` (int). Published whenever a model is known for the agent (`agents.<name>.model` or `default_model`); resolved per agent, with omitted fields falling back to the defaults Pallas registered the model with. |
|
||||
| `servers[]._meta` | object | no | Registry metadata. Informational only — Daedalus does not act on it. |
|
||||
|
||||
#### Behaviour
|
||||
|
||||
@@ -22,11 +22,13 @@ import yaml
|
||||
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
|
||||
from starlette.applications import Starlette
|
||||
|
||||
from pallas.metrics import REGISTRY as _metrics_registry, set_agent_info
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse, PlainTextResponse, Response
|
||||
from starlette.routing import Route
|
||||
|
||||
from pallas.metrics import REGISTRY as _metrics_registry, set_agent_info
|
||||
from pallas.server import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -45,37 +47,53 @@ def _load_deployment_config() -> dict:
|
||||
return yaml.safe_load(f) or {}
|
||||
|
||||
|
||||
def _load_model_capabilities() -> dict:
|
||||
"""Read model info and capabilities from the active fastagent.config.yaml."""
|
||||
def _load_fastagent_defaults() -> tuple[str, dict]:
|
||||
"""Read ``(default_model, model_capabilities)`` from fastagent.config.yaml."""
|
||||
config_path = _config_root() / "fastagent.config.yaml"
|
||||
if not config_path.exists():
|
||||
return {}
|
||||
return "", {}
|
||||
|
||||
with open(config_path) as f:
|
||||
config = yaml.safe_load(f) or {}
|
||||
|
||||
default_model = config.get("default_model", "")
|
||||
capabilities = config.get("model_capabilities", {})
|
||||
|
||||
if not default_model and not capabilities:
|
||||
return {}
|
||||
|
||||
model_name = (
|
||||
default_model.split(".", 1)[-1] if "." in default_model else default_model
|
||||
return (
|
||||
config.get("default_model", "") or "",
|
||||
config.get("model_capabilities", {}) or {},
|
||||
)
|
||||
|
||||
|
||||
def _resolve_capabilities(
|
||||
agent: dict, default_model: str, default_capabilities: dict
|
||||
) -> dict | None:
|
||||
"""Resolve the capabilities Pallas actually registered for one agent.
|
||||
|
||||
Mirrors ``server._register_unknown_models``: an agent's own ``model``
|
||||
in agents.yaml wins over the global ``default_model``, its own
|
||||
``model_capabilities`` block wins over the global one, and the same
|
||||
defaults apply — so the registry advertises the effective values rather
|
||||
than nulls. Returns ``None`` when no model is known for this agent.
|
||||
"""
|
||||
model_spec = agent.get("model") or default_model
|
||||
if not model_spec:
|
||||
return None
|
||||
|
||||
capabilities = agent.get("model_capabilities") or default_capabilities
|
||||
model_name = model_spec.split(".", 1)[-1] if "." in model_spec else model_spec
|
||||
|
||||
return {
|
||||
"model": model_name or None,
|
||||
"model": model_name,
|
||||
"vision": capabilities.get("vision", False),
|
||||
"context_window": capabilities.get("context_window", None),
|
||||
"max_output_tokens": capabilities.get("max_output_tokens", None),
|
||||
"context_window": capabilities.get("context_window", DEFAULT_CONTEXT_WINDOW),
|
||||
"max_output_tokens": capabilities.get(
|
||||
"max_output_tokens", DEFAULT_MAX_OUTPUT_TOKENS
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _build_registry(config: dict) -> dict:
|
||||
"""Build the registry JSON from agents.yaml + fastagent.config.yaml."""
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
model_caps = _load_model_capabilities()
|
||||
default_model, default_capabilities = _load_fastagent_defaults()
|
||||
|
||||
host = config.get("host", "localhost")
|
||||
namespace = config.get("namespace", "")
|
||||
@@ -101,8 +119,9 @@ def _build_registry(config: dict) -> dict:
|
||||
}
|
||||
],
|
||||
}
|
||||
if model_caps:
|
||||
server_entry["capabilities"] = model_caps
|
||||
capabilities = _resolve_capabilities(agent, default_model, default_capabilities)
|
||||
if capabilities:
|
||||
server_entry["capabilities"] = capabilities
|
||||
|
||||
entries.append(
|
||||
{
|
||||
|
||||
@@ -24,6 +24,12 @@ from pallas.multimodal_server import MultimodalAgentMCPServer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Effective model capability defaults, applied when `model_capabilities` omits
|
||||
# them. registry.py mirrors these so the published registry advertises what
|
||||
# was actually registered with fast-agent's ModelDatabase.
|
||||
DEFAULT_CONTEXT_WINDOW = 131072
|
||||
DEFAULT_MAX_OUTPUT_TOKENS = 16384
|
||||
|
||||
|
||||
def _config_root() -> Path:
|
||||
"""Return the working directory where agents.yaml and fastagent configs live."""
|
||||
@@ -143,8 +149,8 @@ def _register_one_model(model_spec: str, capabilities: dict) -> None:
|
||||
return
|
||||
|
||||
is_vision = capabilities.get("vision", False)
|
||||
context_window = capabilities.get("context_window", 131072)
|
||||
max_output_tokens = capabilities.get("max_output_tokens", 16384)
|
||||
context_window = capabilities.get("context_window", DEFAULT_CONTEXT_WINDOW)
|
||||
max_output_tokens = capabilities.get("max_output_tokens", DEFAULT_MAX_OUTPUT_TOKENS)
|
||||
|
||||
if is_vision:
|
||||
tokenizes = list(ModelDatabase.QWEN_MULTIMODAL)
|
||||
|
||||
202
tests/test_registry.py
Normal file
202
tests/test_registry.py
Normal file
@@ -0,0 +1,202 @@
|
||||
"""Tests for pallas.registry — per-agent model capability resolution.
|
||||
|
||||
The registry advertises the capabilities Pallas actually registered with
|
||||
fast-agent, resolved *per agent*: an agent's own ``model`` /
|
||||
``model_capabilities`` in agents.yaml override the global ``default_model`` /
|
||||
``model_capabilities`` in fastagent.config.yaml, and the same effective
|
||||
defaults as ``server._register_one_model`` apply when a field is omitted.
|
||||
|
||||
Regression cover for two bugs: a single capabilities dict was previously
|
||||
built from ``default_model`` and attached to *every* agent entry (so an
|
||||
agent with a ``model:`` override was advertised under the wrong model), and
|
||||
``context_window`` / ``max_output_tokens`` were published as ``null`` when
|
||||
``model_capabilities`` was absent even though the model had been registered
|
||||
with 131072 / 16384.
|
||||
|
||||
``_build_registry`` takes the deployment config as an argument but reads
|
||||
fastagent.config.yaml from the working directory on every call, so each test
|
||||
chdirs into a clean temp workspace and writes only the config it needs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from pallas import registry
|
||||
from pallas.server import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS
|
||||
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
"""Chdir into a clean temp workspace — _build_registry reads cwd."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _write_fastagent(workspace: Path, **config) -> None:
|
||||
(workspace / "fastagent.config.yaml").write_text(yaml.safe_dump(config))
|
||||
|
||||
|
||||
def _deployment(**agents) -> dict:
|
||||
"""Minimal agents.yaml-shaped config; each agent needs at least a port."""
|
||||
return {
|
||||
"name": "test-project",
|
||||
"namespace": "ca.helu.test",
|
||||
"host": "test-host",
|
||||
"agents": {
|
||||
name: {"module": f"agents.{name}", "port": 9000 + i, **overrides}
|
||||
for i, (name, overrides) in enumerate(agents.items())
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _entries(config: dict) -> dict[str, dict]:
|
||||
"""Build the registry and return ``{agent slug: server entry}``."""
|
||||
servers = registry._build_registry(config)["servers"]
|
||||
return {e["server"]["name"].rsplit("/", 1)[-1]: e["server"] for e in servers}
|
||||
|
||||
|
||||
def _capabilities(config: dict, agent: str = "solo") -> dict | None:
|
||||
return _entries(config)[agent].get("capabilities")
|
||||
|
||||
|
||||
# ── Model resolution ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_agent_model_overrides_default_model(workspace: Path) -> None:
|
||||
"""An agents.yaml ``model:`` wins over the global default_model."""
|
||||
_write_fastagent(workspace, default_model="openai.global-model")
|
||||
|
||||
caps = _capabilities(_deployment(solo={"model": "anthropic.agent-model"}))
|
||||
|
||||
assert caps is not None
|
||||
assert caps["model"] == "agent-model"
|
||||
|
||||
|
||||
def test_falls_back_to_default_model(workspace: Path) -> None:
|
||||
"""With no per-agent model, the global default_model is advertised."""
|
||||
_write_fastagent(workspace, default_model="anthropic.claude-opus-4-7")
|
||||
|
||||
caps = _capabilities(_deployment(solo={}))
|
||||
|
||||
assert caps is not None
|
||||
# Provider prefix is stripped — clients get the bare model name.
|
||||
assert caps["model"] == "claude-opus-4-7"
|
||||
|
||||
|
||||
def test_model_without_provider_prefix_passes_through(workspace: Path) -> None:
|
||||
"""A default_model with no ``provider.`` prefix is emitted verbatim."""
|
||||
_write_fastagent(workspace, default_model="bare-model")
|
||||
|
||||
assert _capabilities(_deployment(solo={}))["model"] == "bare-model"
|
||||
|
||||
|
||||
# ── Capability resolution ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_agent_capabilities_override_global(workspace: Path) -> None:
|
||||
"""A per-agent model_capabilities block replaces the global one."""
|
||||
_write_fastagent(
|
||||
workspace,
|
||||
default_model="openai.global-model",
|
||||
model_capabilities={"vision": False, "context_window": 200000},
|
||||
)
|
||||
|
||||
caps = _capabilities(
|
||||
_deployment(
|
||||
solo={
|
||||
"model": "anthropic.agent-model",
|
||||
"model_capabilities": {"vision": True, "context_window": 400000},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert caps["vision"] is True
|
||||
assert caps["context_window"] == 400000
|
||||
|
||||
|
||||
def test_effective_defaults_when_capabilities_absent(workspace: Path) -> None:
|
||||
"""Absent model_capabilities publishes the values actually registered.
|
||||
|
||||
Regression: these were previously advertised as ``null`` while
|
||||
``server._register_one_model`` registered 131072 / 16384.
|
||||
"""
|
||||
_write_fastagent(workspace, default_model="openai.some-model")
|
||||
|
||||
caps = _capabilities(_deployment(solo={}))
|
||||
|
||||
assert caps["context_window"] == DEFAULT_CONTEXT_WINDOW
|
||||
assert caps["max_output_tokens"] == DEFAULT_MAX_OUTPUT_TOKENS
|
||||
assert caps["vision"] is False
|
||||
|
||||
|
||||
def test_global_capabilities_are_published(workspace: Path) -> None:
|
||||
"""The ordinary case: one global model + capabilities for every agent."""
|
||||
_write_fastagent(
|
||||
workspace,
|
||||
default_model="anthropic.claude-opus-4-7",
|
||||
model_capabilities={
|
||||
"vision": True,
|
||||
"context_window": 200000,
|
||||
"max_output_tokens": 50000,
|
||||
},
|
||||
)
|
||||
|
||||
caps = _capabilities(_deployment(solo={}))
|
||||
|
||||
assert caps == {
|
||||
"model": "claude-opus-4-7",
|
||||
"vision": True,
|
||||
"context_window": 200000,
|
||||
"max_output_tokens": 50000,
|
||||
}
|
||||
|
||||
|
||||
# ── Per-agent independence ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_agents_resolve_independently(workspace: Path) -> None:
|
||||
"""Each entry gets its own capabilities — the original shared-dict bug."""
|
||||
_write_fastagent(
|
||||
workspace,
|
||||
default_model="openai.global-model",
|
||||
model_capabilities={"vision": False, "context_window": 128000},
|
||||
)
|
||||
|
||||
entries = _entries(
|
||||
_deployment(
|
||||
inherits={},
|
||||
overrides={
|
||||
"model": "anthropic.special-model",
|
||||
"model_capabilities": {"vision": True, "context_window": 500000},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert entries["inherits"]["capabilities"]["model"] == "global-model"
|
||||
assert entries["inherits"]["capabilities"]["context_window"] == 128000
|
||||
assert entries["overrides"]["capabilities"]["model"] == "special-model"
|
||||
assert entries["overrides"]["capabilities"]["context_window"] == 500000
|
||||
|
||||
|
||||
# ── Omission ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_capabilities_omitted_without_any_model(workspace: Path) -> None:
|
||||
"""No fastagent.config.yaml and no per-agent model → no capabilities key."""
|
||||
entry = _entries(_deployment(solo={}))["solo"]
|
||||
|
||||
assert "capabilities" not in entry
|
||||
|
||||
|
||||
def test_agent_model_published_without_fastagent_config(workspace: Path) -> None:
|
||||
"""An agent's own model is advertised even with no fastagent.config.yaml."""
|
||||
caps = _capabilities(_deployment(solo={"model": "anthropic.agent-model"}))
|
||||
|
||||
assert caps["model"] == "agent-model"
|
||||
assert caps["context_window"] == DEFAULT_CONTEXT_WINDOW
|
||||
Reference in New Issue
Block a user