diff --git a/docs/pallas.md b/docs/pallas.md index e6c6855..0884936 100644 --- a/docs/pallas.md +++ b/docs/pallas.md @@ -192,6 +192,8 @@ agents: | `agents..port` | yes | Port for this agent's StreamableHTTP MCP server | | `agents..title` | no | Display name in registry. Default: `name.title()` | | `agents..description` | no | Description in registry | +| `agents..model` | no | `provider.model-name` override for this agent. Overrides `default_model`, is applied to every agent in the module at startup, and is what the registry advertises for this entry | +| `agents..model_capabilities` | no | Per-agent `{vision, context_window, max_output_tokens}` block. Overrides the top-level `model_capabilities`; the same defaults apply to omitted fields | | `agents..depends_on` | no | List of agent names that must start and become ready before this agent | | `agents..max_iterations` | no | Hard cap on agentic-loop turns per `send_message`. Default: `15`. fast-agent returns a partial answer once exceeded | | `agents..loop_repeat_threshold` | no | Halt the loop after this many consecutive identical `(tool, args) → result` rounds. Default: `3`. `0` disables the guard | @@ -429,7 +431,9 @@ Built dynamically from `agents.yaml` + `fastagent.config.yaml`: ### Capabilities -If `model_capabilities` is defined in `fastagent.config.yaml`, each registry entry includes a `capabilities` object with model name, vision support, context window, and max output tokens. This allows clients to make informed decisions about what an agent can handle. +Each registry entry includes a `capabilities` object — model name, vision support, context window, and max output tokens — whenever a model is known for that agent, i.e. `agents..model` is set or `default_model` is defined in `fastagent.config.yaml`. + +Capabilities are resolved **per agent**: the agent's own `model` and `model_capabilities` take precedence over the global values, and omitted fields fall back to the same defaults Pallas uses to register the model (`vision: false`, `context_window: 131072`, `max_output_tokens: 16384`). The published values therefore match what was actually registered with fast-agent's `ModelDatabase`, rather than being null whenever `model_capabilities` was left out. Clients use them to make informed decisions about what an agent can handle. --- @@ -673,12 +677,13 @@ Pallas registers models not in fast-agent's built-in `ModelDatabase` at startup, The process: 1. Read `default_model` and `model_capabilities` from config -2. Extract the model name (portion after the provider prefix dot) -3. Check if `ModelDatabase` already knows this model — if so, skip -4. Register with `ModelDatabase.register_runtime_model_params()`: +2. Also read every `agents..model` from `agents.yaml`, using that agent's own `model_capabilities` when it declares one +3. Extract the model name (portion after the provider prefix dot) +4. Check if `ModelDatabase` already knows this model — if so, skip +5. Register with `ModelDatabase.register_runtime_model_params()`: - `vision: true` → multimodal tokenization (`QWEN_MULTIMODAL`) - `vision: false` → text-only tokenization (`TEXT_ONLY`) - - `context_window` and `max_output_tokens` from config (with sensible defaults) + - `context_window` and `max_output_tokens` from config, defaulting to `131072` / `16384` — the same values the registry advertises This avoids the brittle pattern of inferring capabilities from model name substrings, which breaks for custom or fine-tuned models with non-standard names. diff --git a/docs/pallas_integration.md b/docs/pallas_integration.md index 7c6d4a0..5651ba4 100644 --- a/docs/pallas_integration.md +++ b/docs/pallas_integration.md @@ -120,7 +120,7 @@ No authentication. No query parameters. | `servers[].server.version` | string | no | Semver version string. | | `servers[].server.icons` | array | no | Array of `{ src, sizes }`. Daedalus uses the first entry. | | `servers[].server.remotes` | array | yes | Connection endpoints. Daedalus looks for `type: "streamable-http"` and uses its `url`. | -| `servers[].server.capabilities` | object | no | Model capabilities. Contains `model` (string), `vision` (bool), `context_window` (int), `max_output_tokens` (int). Published when `model_capabilities` is configured in `fastagent.config.yaml`. | +| `servers[].server.capabilities` | object | no | Model capabilities. Contains `model` (string), `vision` (bool), `context_window` (int), `max_output_tokens` (int). Published whenever a model is known for the agent (`agents..model` or `default_model`); resolved per agent, with omitted fields falling back to the defaults Pallas registered the model with. | | `servers[]._meta` | object | no | Registry metadata. Informational only — Daedalus does not act on it. | #### Behaviour diff --git a/pallas/registry.py b/pallas/registry.py index 30b7fad..b14b949 100644 --- a/pallas/registry.py +++ b/pallas/registry.py @@ -22,11 +22,13 @@ import yaml from prometheus_client import CONTENT_TYPE_LATEST, generate_latest from starlette.applications import Starlette -from pallas.metrics import REGISTRY as _metrics_registry, set_agent_info from starlette.requests import Request from starlette.responses import JSONResponse, PlainTextResponse, Response from starlette.routing import Route +from pallas.metrics import REGISTRY as _metrics_registry, set_agent_info +from pallas.server import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS + logger = logging.getLogger(__name__) @@ -45,37 +47,53 @@ def _load_deployment_config() -> dict: return yaml.safe_load(f) or {} -def _load_model_capabilities() -> dict: - """Read model info and capabilities from the active fastagent.config.yaml.""" +def _load_fastagent_defaults() -> tuple[str, dict]: + """Read ``(default_model, model_capabilities)`` from fastagent.config.yaml.""" config_path = _config_root() / "fastagent.config.yaml" if not config_path.exists(): - return {} + return "", {} with open(config_path) as f: config = yaml.safe_load(f) or {} - default_model = config.get("default_model", "") - capabilities = config.get("model_capabilities", {}) - - if not default_model and not capabilities: - return {} - - model_name = ( - default_model.split(".", 1)[-1] if "." in default_model else default_model + return ( + config.get("default_model", "") or "", + config.get("model_capabilities", {}) or {}, ) + +def _resolve_capabilities( + agent: dict, default_model: str, default_capabilities: dict +) -> dict | None: + """Resolve the capabilities Pallas actually registered for one agent. + + Mirrors ``server._register_unknown_models``: an agent's own ``model`` + in agents.yaml wins over the global ``default_model``, its own + ``model_capabilities`` block wins over the global one, and the same + defaults apply — so the registry advertises the effective values rather + than nulls. Returns ``None`` when no model is known for this agent. + """ + model_spec = agent.get("model") or default_model + if not model_spec: + return None + + capabilities = agent.get("model_capabilities") or default_capabilities + model_name = model_spec.split(".", 1)[-1] if "." in model_spec else model_spec + return { - "model": model_name or None, + "model": model_name, "vision": capabilities.get("vision", False), - "context_window": capabilities.get("context_window", None), - "max_output_tokens": capabilities.get("max_output_tokens", None), + "context_window": capabilities.get("context_window", DEFAULT_CONTEXT_WINDOW), + "max_output_tokens": capabilities.get( + "max_output_tokens", DEFAULT_MAX_OUTPUT_TOKENS + ), } def _build_registry(config: dict) -> dict: """Build the registry JSON from agents.yaml + fastagent.config.yaml.""" now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - model_caps = _load_model_capabilities() + default_model, default_capabilities = _load_fastagent_defaults() host = config.get("host", "localhost") namespace = config.get("namespace", "") @@ -101,8 +119,9 @@ def _build_registry(config: dict) -> dict: } ], } - if model_caps: - server_entry["capabilities"] = model_caps + capabilities = _resolve_capabilities(agent, default_model, default_capabilities) + if capabilities: + server_entry["capabilities"] = capabilities entries.append( { diff --git a/pallas/server.py b/pallas/server.py index 08855e4..0bc9ca5 100644 --- a/pallas/server.py +++ b/pallas/server.py @@ -24,6 +24,12 @@ from pallas.multimodal_server import MultimodalAgentMCPServer logger = logging.getLogger(__name__) +# Effective model capability defaults, applied when `model_capabilities` omits +# them. registry.py mirrors these so the published registry advertises what +# was actually registered with fast-agent's ModelDatabase. +DEFAULT_CONTEXT_WINDOW = 131072 +DEFAULT_MAX_OUTPUT_TOKENS = 16384 + def _config_root() -> Path: """Return the working directory where agents.yaml and fastagent configs live.""" @@ -143,8 +149,8 @@ def _register_one_model(model_spec: str, capabilities: dict) -> None: return is_vision = capabilities.get("vision", False) - context_window = capabilities.get("context_window", 131072) - max_output_tokens = capabilities.get("max_output_tokens", 16384) + context_window = capabilities.get("context_window", DEFAULT_CONTEXT_WINDOW) + max_output_tokens = capabilities.get("max_output_tokens", DEFAULT_MAX_OUTPUT_TOKENS) if is_vision: tokenizes = list(ModelDatabase.QWEN_MULTIMODAL) diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 0000000..d1a39d4 --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,202 @@ +"""Tests for pallas.registry — per-agent model capability resolution. + +The registry advertises the capabilities Pallas actually registered with +fast-agent, resolved *per agent*: an agent's own ``model`` / +``model_capabilities`` in agents.yaml override the global ``default_model`` / +``model_capabilities`` in fastagent.config.yaml, and the same effective +defaults as ``server._register_one_model`` apply when a field is omitted. + +Regression cover for two bugs: a single capabilities dict was previously +built from ``default_model`` and attached to *every* agent entry (so an +agent with a ``model:`` override was advertised under the wrong model), and +``context_window`` / ``max_output_tokens`` were published as ``null`` when +``model_capabilities`` was absent even though the model had been registered +with 131072 / 16384. + +``_build_registry`` takes the deployment config as an argument but reads +fastagent.config.yaml from the working directory on every call, so each test +chdirs into a clean temp workspace and writes only the config it needs. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from pallas import registry +from pallas.server import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +@pytest.fixture +def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Chdir into a clean temp workspace — _build_registry reads cwd.""" + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _write_fastagent(workspace: Path, **config) -> None: + (workspace / "fastagent.config.yaml").write_text(yaml.safe_dump(config)) + + +def _deployment(**agents) -> dict: + """Minimal agents.yaml-shaped config; each agent needs at least a port.""" + return { + "name": "test-project", + "namespace": "ca.helu.test", + "host": "test-host", + "agents": { + name: {"module": f"agents.{name}", "port": 9000 + i, **overrides} + for i, (name, overrides) in enumerate(agents.items()) + }, + } + + +def _entries(config: dict) -> dict[str, dict]: + """Build the registry and return ``{agent slug: server entry}``.""" + servers = registry._build_registry(config)["servers"] + return {e["server"]["name"].rsplit("/", 1)[-1]: e["server"] for e in servers} + + +def _capabilities(config: dict, agent: str = "solo") -> dict | None: + return _entries(config)[agent].get("capabilities") + + +# ── Model resolution ───────────────────────────────────────────────────────── + + +def test_agent_model_overrides_default_model(workspace: Path) -> None: + """An agents.yaml ``model:`` wins over the global default_model.""" + _write_fastagent(workspace, default_model="openai.global-model") + + caps = _capabilities(_deployment(solo={"model": "anthropic.agent-model"})) + + assert caps is not None + assert caps["model"] == "agent-model" + + +def test_falls_back_to_default_model(workspace: Path) -> None: + """With no per-agent model, the global default_model is advertised.""" + _write_fastagent(workspace, default_model="anthropic.claude-opus-4-7") + + caps = _capabilities(_deployment(solo={})) + + assert caps is not None + # Provider prefix is stripped — clients get the bare model name. + assert caps["model"] == "claude-opus-4-7" + + +def test_model_without_provider_prefix_passes_through(workspace: Path) -> None: + """A default_model with no ``provider.`` prefix is emitted verbatim.""" + _write_fastagent(workspace, default_model="bare-model") + + assert _capabilities(_deployment(solo={}))["model"] == "bare-model" + + +# ── Capability resolution ──────────────────────────────────────────────────── + + +def test_agent_capabilities_override_global(workspace: Path) -> None: + """A per-agent model_capabilities block replaces the global one.""" + _write_fastagent( + workspace, + default_model="openai.global-model", + model_capabilities={"vision": False, "context_window": 200000}, + ) + + caps = _capabilities( + _deployment( + solo={ + "model": "anthropic.agent-model", + "model_capabilities": {"vision": True, "context_window": 400000}, + } + ) + ) + + assert caps["vision"] is True + assert caps["context_window"] == 400000 + + +def test_effective_defaults_when_capabilities_absent(workspace: Path) -> None: + """Absent model_capabilities publishes the values actually registered. + + Regression: these were previously advertised as ``null`` while + ``server._register_one_model`` registered 131072 / 16384. + """ + _write_fastagent(workspace, default_model="openai.some-model") + + caps = _capabilities(_deployment(solo={})) + + assert caps["context_window"] == DEFAULT_CONTEXT_WINDOW + assert caps["max_output_tokens"] == DEFAULT_MAX_OUTPUT_TOKENS + assert caps["vision"] is False + + +def test_global_capabilities_are_published(workspace: Path) -> None: + """The ordinary case: one global model + capabilities for every agent.""" + _write_fastagent( + workspace, + default_model="anthropic.claude-opus-4-7", + model_capabilities={ + "vision": True, + "context_window": 200000, + "max_output_tokens": 50000, + }, + ) + + caps = _capabilities(_deployment(solo={})) + + assert caps == { + "model": "claude-opus-4-7", + "vision": True, + "context_window": 200000, + "max_output_tokens": 50000, + } + + +# ── Per-agent independence ─────────────────────────────────────────────────── + + +def test_agents_resolve_independently(workspace: Path) -> None: + """Each entry gets its own capabilities — the original shared-dict bug.""" + _write_fastagent( + workspace, + default_model="openai.global-model", + model_capabilities={"vision": False, "context_window": 128000}, + ) + + entries = _entries( + _deployment( + inherits={}, + overrides={ + "model": "anthropic.special-model", + "model_capabilities": {"vision": True, "context_window": 500000}, + }, + ) + ) + + assert entries["inherits"]["capabilities"]["model"] == "global-model" + assert entries["inherits"]["capabilities"]["context_window"] == 128000 + assert entries["overrides"]["capabilities"]["model"] == "special-model" + assert entries["overrides"]["capabilities"]["context_window"] == 500000 + + +# ── Omission ───────────────────────────────────────────────────────────────── + + +def test_capabilities_omitted_without_any_model(workspace: Path) -> None: + """No fastagent.config.yaml and no per-agent model → no capabilities key.""" + entry = _entries(_deployment(solo={}))["solo"] + + assert "capabilities" not in entry + + +def test_agent_model_published_without_fastagent_config(workspace: Path) -> None: + """An agent's own model is advertised even with no fastagent.config.yaml.""" + caps = _capabilities(_deployment(solo={"model": "anthropic.agent-model"})) + + assert caps["model"] == "agent-model" + assert caps["context_window"] == DEFAULT_CONTEXT_WINDOW