docs: update Mantle setup to reflect automatic shim detection
This commit is contained in:
145
pallas/mantle_shims.py
Normal file
145
pallas/mantle_shims.py
Normal file
@@ -0,0 +1,145 @@
|
||||
"""AWS Bedrock Mantle compatibility shims for fast-agent.
|
||||
|
||||
Mantle is AWS's Anthropic-Messages-API-compatible gateway, hosted at
|
||||
``https://bedrock-mantle.{region}.api.aws/anthropic``. Fast-agent talks to it
|
||||
via its built-in ``anthropic`` provider, but two layers of reshaping are needed
|
||||
before the wire traffic is valid:
|
||||
|
||||
1. **Model-name prefix.** Mantle requires the full ``anthropic.<name>`` wire
|
||||
id (e.g. ``anthropic.claude-opus-4-7``). Fast-agent's model-spec parser
|
||||
treats the ``anthropic.`` prefix as the provider hint and strips it off
|
||||
the wire name. We re-register the prefixed forms via
|
||||
``ModelDatabase._PROVIDER_WIRE_MODEL_NAMES`` so the right id goes out.
|
||||
|
||||
2. **``caller: null`` leakage on replayed ``tool_use`` blocks.** Anthropic
|
||||
SDK 0.100.x ``BetaToolUseBlock`` carries an optional ``caller`` field;
|
||||
the matching ``BetaToolUseBlockParam`` TypedDict declares it required.
|
||||
Fast-agent's multipart converter re-serialises assistant history with
|
||||
``exclude_none=False``, producing ``{"type": "tool_use", ..., "caller": null}``.
|
||||
``api.anthropic.com`` silently accepts that; Mantle rejects it as
|
||||
``tool_use.caller: Input should be a valid dictionary or object``,
|
||||
breaking the tool-use loop on the second turn. We strip ``caller`` from
|
||||
any ``tool_use`` dict emitted by the two static methods that feed
|
||||
replayed history back into the wire.
|
||||
|
||||
Upstream SDK tracker: https://github.com/anthropics/anthropic-sdk-python/issues/1454
|
||||
|
||||
Both shims are idempotent and may be installed at process startup before any
|
||||
fast-agent ``FastAgent`` instance is constructed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, cast
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Model ids known to be served on Mantle (keep in sync with AWS docs). ──
|
||||
# The key is fast-agent's internal model_name (provider prefix stripped),
|
||||
# the value is the wire id Mantle expects.
|
||||
MANTLE_WIRE_NAMES: dict[str, str] = {
|
||||
"claude-haiku-4-5": "anthropic.claude-haiku-4-5",
|
||||
"claude-opus-4-7": "anthropic.claude-opus-4-7",
|
||||
}
|
||||
|
||||
|
||||
def is_mantle_base_url(base_url: str | None) -> bool:
|
||||
"""Return True if the given anthropic base_url points at Mantle."""
|
||||
if not base_url:
|
||||
return False
|
||||
return "bedrock-mantle" in base_url
|
||||
|
||||
|
||||
# ── Shim 1: model-name prefix ────────────────────────────────────────────────
|
||||
|
||||
def install_wire_name_prefix() -> None:
|
||||
"""Register the prefixed wire ids for known Mantle-hosted Claude models."""
|
||||
from fast_agent.llm.model_database import ModelDatabase
|
||||
from fast_agent.llm.provider_types import Provider
|
||||
|
||||
for fa_name, wire_name in MANTLE_WIRE_NAMES.items():
|
||||
key = (Provider.ANTHROPIC, ModelDatabase.normalize_model_name(fa_name))
|
||||
ModelDatabase._PROVIDER_WIRE_MODEL_NAMES[key] = wire_name # noqa: SLF001
|
||||
|
||||
logger.info(
|
||||
"Mantle wire-name shim installed for models: %s",
|
||||
", ".join(sorted(MANTLE_WIRE_NAMES.keys())),
|
||||
)
|
||||
|
||||
|
||||
# ── Shim 2: strip `caller` from replayed tool_use blocks ─────────────────────
|
||||
|
||||
def _strip_tool_use_caller(blocks: list[Any]) -> list[Any]:
|
||||
"""Remove the stray ``caller`` field Anthropic SDK 0.100.x leaks into
|
||||
replayed ``tool_use`` blocks. Idempotent; only touches dicts whose
|
||||
``type == "tool_use"``.
|
||||
"""
|
||||
for block in blocks:
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use":
|
||||
block.pop("caller", None)
|
||||
return blocks
|
||||
|
||||
|
||||
_tool_use_patch_installed = False
|
||||
|
||||
|
||||
def install_tool_use_caller_strip() -> None:
|
||||
"""Monkeypatch ``AnthropicConverter`` to drop ``caller`` from replayed
|
||||
``tool_use`` blocks. Safe to call more than once; subsequent calls are
|
||||
no-ops.
|
||||
"""
|
||||
global _tool_use_patch_installed
|
||||
if _tool_use_patch_installed:
|
||||
return
|
||||
|
||||
from fast_agent.llm.provider.anthropic.multipart_converter_anthropic import (
|
||||
AnthropicConverter,
|
||||
)
|
||||
|
||||
original_deserialize = AnthropicConverter._deserialize_assistant_raw_blocks # noqa: SLF001
|
||||
|
||||
def patched_deserialize(
|
||||
channels: Mapping[str, Sequence[Any]],
|
||||
) -> list[Any]:
|
||||
result = original_deserialize(channels)
|
||||
return cast("list[Any]", _strip_tool_use_caller(list(result)))
|
||||
|
||||
AnthropicConverter._deserialize_assistant_raw_blocks = staticmethod( # noqa: SLF001
|
||||
patched_deserialize
|
||||
)
|
||||
|
||||
original_append = AnthropicConverter._append_server_tool_channel_blocks # noqa: SLF001
|
||||
|
||||
def patched_append(
|
||||
channels: Mapping[str, Sequence[Any]] | None,
|
||||
destination: list[Any],
|
||||
) -> None:
|
||||
original_append(channels, destination)
|
||||
_strip_tool_use_caller(destination)
|
||||
|
||||
AnthropicConverter._append_server_tool_channel_blocks = staticmethod( # noqa: SLF001
|
||||
patched_append
|
||||
)
|
||||
|
||||
_tool_use_patch_installed = True
|
||||
logger.info("Mantle tool_use.caller strip shim installed")
|
||||
|
||||
|
||||
# ── Orchestrator ─────────────────────────────────────────────────────────────
|
||||
|
||||
def install_all() -> None:
|
||||
"""Install all Mantle shims. Call once at process startup."""
|
||||
install_wire_name_prefix()
|
||||
install_tool_use_caller_strip()
|
||||
|
||||
|
||||
def maybe_install(anthropic_base_url: str | None) -> bool:
|
||||
"""Install shims only when ``anthropic_base_url`` is a Mantle endpoint.
|
||||
Returns True if the shims were installed.
|
||||
"""
|
||||
if not is_mantle_base_url(anthropic_base_url):
|
||||
return False
|
||||
install_all()
|
||||
return True
|
||||
100
pallas/server.py
100
pallas/server.py
@@ -123,84 +123,38 @@ def _preflight_mcp_servers(agent_name: str, servers: dict[str, dict]) -> None:
|
||||
# ── Model registration ────────────────────────────────────────────────────────
|
||||
|
||||
def _register_one_model(model_spec: str, capabilities: dict) -> None:
|
||||
"""Register a single model with fast-agent's ModelDatabase.
|
||||
"""Register a single unknown model with fast-agent's ModelDatabase.
|
||||
|
||||
Two cases:
|
||||
|
||||
1. **Unknown model** — if fast-agent has no built-in entry for this model,
|
||||
register a minimal ``ModelParameters`` with the declared capabilities.
|
||||
|
||||
2. **Mantle-hosted model** (``capabilities.mantle: true``) — regardless of
|
||||
whether the model has a built-in entry, install a provider-specific
|
||||
override for ``(Provider.ANTHROPIC, model_name)`` in
|
||||
``_PROVIDER_MODEL_OVERRIDES`` that strips the features the AWS Bedrock
|
||||
Mantle endpoint rejects:
|
||||
|
||||
- ``anthropic_required_betas`` (no ``anthropic-beta`` header)
|
||||
- ``reasoning`` / ``reasoning_effort_spec`` (no extended-thinking request)
|
||||
- ``anthropic_task_budget_supported``
|
||||
- ``anthropic_web_fetch_version`` / ``anthropic_web_search_version``
|
||||
- ``cache_ttl`` (prompt caching is not advertised as supported on
|
||||
Mantle for every model; disable the cache planner by default)
|
||||
|
||||
Without this override fast-agent sends beta headers and ``thinking``
|
||||
parameters that Mantle rejects with a misleading ``"model does not
|
||||
exist"`` 404.
|
||||
If fast-agent already has a built-in entry for this model we leave it
|
||||
alone. Otherwise we register a minimal ``ModelParameters`` using the
|
||||
declared capabilities so the model resolves cleanly at agent startup.
|
||||
"""
|
||||
from fast_agent.llm.model_database import ModelDatabase, ModelParameters
|
||||
from fast_agent.llm.provider_types import Provider
|
||||
|
||||
model_name = model_spec.split(".", 1)[-1] if "." in model_spec else model_spec
|
||||
|
||||
if ModelDatabase.get_model_params(model_name) is not None:
|
||||
return
|
||||
|
||||
is_vision = capabilities.get("vision", False)
|
||||
context_window = capabilities.get("context_window", 131072)
|
||||
max_output_tokens = capabilities.get("max_output_tokens", 16384)
|
||||
is_mantle = capabilities.get("mantle", False)
|
||||
|
||||
existing = ModelDatabase.get_model_params(model_name)
|
||||
|
||||
if existing is None:
|
||||
# Unknown model — register a fresh runtime entry.
|
||||
if is_vision:
|
||||
tokenizes = list(ModelDatabase.QWEN_MULTIMODAL)
|
||||
logger.info("Registered model '%s' with vision capabilities", model_name)
|
||||
else:
|
||||
tokenizes = list(ModelDatabase.TEXT_ONLY)
|
||||
logger.info("Registered model '%s' as text-only", model_name)
|
||||
|
||||
ModelDatabase.register_runtime_model_params(
|
||||
model_name,
|
||||
ModelParameters(
|
||||
context_window=context_window,
|
||||
max_output_tokens=max_output_tokens,
|
||||
tokenizes=tokenizes,
|
||||
),
|
||||
)
|
||||
base_params = ModelDatabase.get_model_params(model_name)
|
||||
if is_vision:
|
||||
tokenizes = list(ModelDatabase.QWEN_MULTIMODAL)
|
||||
logger.info("Registered model '%s' with vision capabilities", model_name)
|
||||
else:
|
||||
base_params = existing
|
||||
tokenizes = list(ModelDatabase.TEXT_ONLY)
|
||||
logger.info("Registered model '%s' as text-only", model_name)
|
||||
|
||||
if is_mantle and base_params is not None:
|
||||
# Clone the base params and strip Mantle-incompatible features.
|
||||
override = base_params.model_copy(
|
||||
update={
|
||||
"context_window": context_window,
|
||||
"max_output_tokens": max_output_tokens,
|
||||
"anthropic_required_betas": None,
|
||||
"reasoning": None,
|
||||
"reasoning_effort_spec": None,
|
||||
"anthropic_task_budget_supported": False,
|
||||
"anthropic_web_fetch_version": None,
|
||||
"anthropic_web_search_version": None,
|
||||
"cache_ttl": None,
|
||||
}
|
||||
)
|
||||
normalized = ModelDatabase.normalize_model_name(model_name)
|
||||
ModelDatabase._PROVIDER_MODEL_OVERRIDES[(Provider.ANTHROPIC, normalized)] = override
|
||||
logger.info(
|
||||
"Registered Mantle override for anthropic/'%s' (strips beta headers, thinking, web tools, caching)",
|
||||
model_name,
|
||||
)
|
||||
ModelDatabase.register_runtime_model_params(
|
||||
model_name,
|
||||
ModelParameters(
|
||||
context_window=context_window,
|
||||
max_output_tokens=max_output_tokens,
|
||||
tokenizes=tokenizes,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -212,7 +166,14 @@ def _register_unknown_models(deployment_config: dict) -> None:
|
||||
per model: if the agent carries its own ``model_capabilities`` block, those
|
||||
take effect; otherwise the top-level ``model_capabilities`` from
|
||||
``fastagent.config.yaml`` apply.
|
||||
|
||||
Also auto-detects an AWS Bedrock Mantle ``anthropic.base_url`` and installs
|
||||
the Mantle compatibility shims (wire-name prefix and ``tool_use.caller``
|
||||
strip) via :mod:`pallas.mantle_shims`. No config flag needed — Pallas
|
||||
reads the base_url and does the right thing.
|
||||
"""
|
||||
from pallas import mantle_shims
|
||||
|
||||
fastagent_config_path = _config_root() / "fastagent.config.yaml"
|
||||
if not fastagent_config_path.exists():
|
||||
return
|
||||
@@ -220,6 +181,13 @@ def _register_unknown_models(deployment_config: dict) -> None:
|
||||
with open(fastagent_config_path) as f:
|
||||
fa_config = yaml.safe_load(f) or {}
|
||||
|
||||
anthropic_base_url = fa_config.get("anthropic", {}).get("base_url", "")
|
||||
if mantle_shims.maybe_install(anthropic_base_url):
|
||||
logger.info(
|
||||
"Detected Bedrock Mantle endpoint (%s); installed fast-agent shims.",
|
||||
anthropic_base_url,
|
||||
)
|
||||
|
||||
default_model = fa_config.get("default_model", "")
|
||||
default_capabilities = fa_config.get("model_capabilities", {})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user