523 lines
18 KiB
Python
523 lines
18 KiB
Python
"""Tests for pallas.health — per-provider preflight dispatch.
|
|
|
|
Covers the matrix documented in ``pallas/pallas/health.py``:
|
|
|
|
- ``anthropic`` (direct, Mantle)
|
|
- ``openai``
|
|
- ``generic``
|
|
- ``bedrock`` (presence-only, no HTTP)
|
|
- unknown / malformed provider name
|
|
|
|
All HTTP is faked with ``httpx.MockTransport`` so nothing touches the network.
|
|
Tests use ``asyncio.run`` directly to match the existing convention in
|
|
``tests/test_mantle_shims.py`` (pallas has no pytest-asyncio dependency).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from pallas import health
|
|
|
|
|
|
def _run(coro):
|
|
return asyncio.run(coro)
|
|
|
|
|
|
# ── Helpers ──────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _patch_httpx(monkeypatch: pytest.MonkeyPatch, handler) -> None:
|
|
"""Replace ``health.httpx.AsyncClient`` so validate_llm_providers uses the mock."""
|
|
original_client = httpx.AsyncClient
|
|
|
|
def patched_client(*args, **kwargs):
|
|
kwargs["transport"] = httpx.MockTransport(handler)
|
|
return original_client(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(health.httpx, "AsyncClient", patched_client)
|
|
|
|
|
|
def _patch_httpx_raising(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Install a transport that raises on any request — used to prove that
|
|
bedrock / unknown paths make no HTTP call at all."""
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise AssertionError(
|
|
f"no HTTP call should be made, but got {request.method} {request.url}"
|
|
)
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
|
|
@pytest.fixture
|
|
def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
|
"""Chdir into a clean temp workspace and isolate env variables.
|
|
|
|
``validate_llm_providers`` reads ``fastagent.config.yaml`` /
|
|
``fastagent.secrets.yaml`` from cwd and also consults env vars for
|
|
fallback; each test starts with a clean slate.
|
|
"""
|
|
monkeypatch.chdir(tmp_path)
|
|
for var in (
|
|
"ANTHROPIC_API_KEY",
|
|
"ANTHROPIC_BASE_URL",
|
|
"OPENAI_API_KEY",
|
|
"OPENAI_BASE_URL",
|
|
"GENERIC_API_KEY",
|
|
"GENERIC_BASE_URL",
|
|
"AWS_BEARER_TOKEN_BEDROCK",
|
|
"AWS_ACCESS_KEY_ID",
|
|
"AWS_SECRET_ACCESS_KEY",
|
|
"AWS_PROFILE",
|
|
):
|
|
monkeypatch.delenv(var, raising=False)
|
|
return tmp_path
|
|
|
|
|
|
# ── _mantle_root_from_anthropic_base ────────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"base,expected",
|
|
[
|
|
(
|
|
"https://bedrock-mantle.us-east-1.api.aws/anthropic",
|
|
"https://bedrock-mantle.us-east-1.api.aws",
|
|
),
|
|
(
|
|
"https://bedrock-mantle.us-east-1.api.aws/anthropic/",
|
|
"https://bedrock-mantle.us-east-1.api.aws",
|
|
),
|
|
(
|
|
"https://bedrock-mantle.us-east-1.api.aws",
|
|
"https://bedrock-mantle.us-east-1.api.aws",
|
|
),
|
|
(
|
|
"https://example.com/proxy/anthropic",
|
|
"https://example.com/proxy",
|
|
),
|
|
],
|
|
)
|
|
def test_mantle_root_from_anthropic_base(base: str, expected: str) -> None:
|
|
assert health._mantle_root_from_anthropic_base(base) == expected
|
|
|
|
|
|
# ── _check_anthropic (direct + Mantle share this probe) ──────────────────────
|
|
|
|
|
|
def test_check_anthropic_success_direct() -> None:
|
|
captured: list[httpx.Request] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured.append(request)
|
|
return httpx.Response(200, json={"id": "claude-sonnet-4-5"})
|
|
|
|
async def go() -> str | None:
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
return await health._check_anthropic(
|
|
client,
|
|
"sk-ant-real",
|
|
"claude-sonnet-4-5",
|
|
"https://api.anthropic.com/v1",
|
|
)
|
|
|
|
assert _run(go()) is None
|
|
assert str(captured[0].url) == "https://api.anthropic.com/v1/models/claude-sonnet-4-5"
|
|
assert captured[0].headers["x-api-key"] == "sk-ant-real"
|
|
assert captured[0].headers["anthropic-version"] == "2023-06-01"
|
|
|
|
|
|
def test_check_anthropic_success_mantle_root() -> None:
|
|
captured: list[httpx.Request] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured.append(request)
|
|
return httpx.Response(200, json={"id": "anthropic.claude-opus-4-7"})
|
|
|
|
async def go() -> str | None:
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
return await health._check_anthropic(
|
|
client,
|
|
"sk-bedrock-fake",
|
|
"anthropic.claude-opus-4-7",
|
|
"https://bedrock-mantle.us-east-1.api.aws/v1",
|
|
)
|
|
|
|
assert _run(go()) is None
|
|
# Must hit the Mantle region root, not `/anthropic/v1/models/...`.
|
|
assert str(captured[0].url) == (
|
|
"https://bedrock-mantle.us-east-1.api.aws/v1"
|
|
"/models/anthropic.claude-opus-4-7"
|
|
)
|
|
|
|
|
|
def test_check_anthropic_401() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(401, json={"error": "invalid_api_key"})
|
|
|
|
async def go() -> str | None:
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
return await health._check_anthropic(
|
|
client,
|
|
"bad-key",
|
|
"claude-sonnet-4-5",
|
|
"https://api.anthropic.com/v1",
|
|
)
|
|
|
|
assert _run(go()) == "API request failed (401)"
|
|
|
|
|
|
def test_check_anthropic_404_model_missing() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(404, json={})
|
|
|
|
async def go() -> str | None:
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
|
return await health._check_anthropic(
|
|
client,
|
|
"key",
|
|
"claude-foo",
|
|
"https://api.anthropic.com/v1",
|
|
)
|
|
|
|
assert _run(go()) == "model 'claude-foo' not found"
|
|
|
|
|
|
# ── validate_llm_providers: anthropic direct ─────────────────────────────────
|
|
|
|
|
|
def test_validate_anthropic_direct_ok(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: anthropic.claude-sonnet-4-5\n"
|
|
)
|
|
(workspace / "fastagent.secrets.yaml").write_text(
|
|
'anthropic:\n api_key: "sk-ant-real"\n'
|
|
)
|
|
|
|
captured: list[httpx.Request] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured.append(request)
|
|
return httpx.Response(200, json={"id": "claude-sonnet-4-5"})
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
assert _run(health.validate_llm_providers(timeout=1.0)) == {
|
|
"anthropic": {"status": "ok", "model": "claude-sonnet-4-5"}
|
|
}
|
|
assert str(captured[0].url) == "https://api.anthropic.com/v1/models/claude-sonnet-4-5"
|
|
|
|
|
|
def test_validate_anthropic_missing_key(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: anthropic.claude-sonnet-4-5\n"
|
|
)
|
|
# No secrets file at all → ProviderKeyManager raises ProviderKeyError.
|
|
_patch_httpx_raising(monkeypatch)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["anthropic"]["status"] == "error"
|
|
assert "API key" in results["anthropic"]["message"]
|
|
|
|
|
|
# ── validate_llm_providers: anthropic via Mantle ─────────────────────────────
|
|
|
|
|
|
def test_validate_anthropic_mantle_uses_region_root(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: anthropic.claude-opus-4-7\n"
|
|
"anthropic:\n"
|
|
' base_url: "https://bedrock-mantle.us-east-1.api.aws/anthropic"\n'
|
|
)
|
|
(workspace / "fastagent.secrets.yaml").write_text(
|
|
'anthropic:\n api_key: "sk-bedrock-fake"\n'
|
|
)
|
|
|
|
captured: list[httpx.Request] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured.append(request)
|
|
return httpx.Response(200, json={"id": "anthropic.claude-opus-4-7"})
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
assert _run(health.validate_llm_providers(timeout=1.0)) == {
|
|
"anthropic": {"status": "ok", "model": "anthropic.claude-opus-4-7"}
|
|
}
|
|
# Must strip the `/anthropic` suffix AND apply the wire-name prefix.
|
|
assert str(captured[0].url) == (
|
|
"https://bedrock-mantle.us-east-1.api.aws/v1"
|
|
"/models/anthropic.claude-opus-4-7"
|
|
)
|
|
|
|
|
|
def test_validate_anthropic_mantle_401(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: anthropic.claude-opus-4-7\n"
|
|
"anthropic:\n"
|
|
' base_url: "https://bedrock-mantle.us-east-1.api.aws/anthropic"\n'
|
|
)
|
|
(workspace / "fastagent.secrets.yaml").write_text(
|
|
'anthropic:\n api_key: "sk-bogus"\n'
|
|
)
|
|
|
|
captured: list[httpx.Request] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured.append(request)
|
|
return httpx.Response(401, json={"error": "unauthorized"})
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results == {
|
|
"anthropic": {
|
|
"status": "error",
|
|
"model": "anthropic.claude-opus-4-7",
|
|
"message": "API request failed (401)",
|
|
}
|
|
}
|
|
assert "bedrock-mantle" in str(captured[0].url)
|
|
|
|
|
|
# ── validate_llm_providers: openai ───────────────────────────────────────────
|
|
|
|
|
|
def test_validate_openai_model_in_list(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: openai.gpt-4o-mini\n"
|
|
)
|
|
(workspace / "fastagent.secrets.yaml").write_text(
|
|
'openai:\n api_key: "sk-openai-real"\n'
|
|
)
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
json={"data": [{"id": "gpt-4o-mini"}, {"id": "gpt-4o"}]},
|
|
)
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["openai"]["status"] == "ok"
|
|
assert results["openai"]["model"] == "gpt-4o-mini"
|
|
|
|
|
|
def test_validate_openai_model_missing_from_list(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: openai.gpt-nonexistent\n"
|
|
)
|
|
(workspace / "fastagent.secrets.yaml").write_text(
|
|
'openai:\n api_key: "sk-openai-real"\n'
|
|
)
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, json={"data": [{"id": "gpt-4o-mini"}]})
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["openai"]["status"] == "error"
|
|
assert "gpt-nonexistent" in results["openai"]["message"]
|
|
assert "gpt-4o-mini" in results["openai"]["message"] # includes available list
|
|
|
|
|
|
# ── validate_llm_providers: generic ──────────────────────────────────────────
|
|
|
|
|
|
def test_validate_generic_ok_regardless_of_body(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""llama.cpp returns a non-OpenAI-shaped ``/v1/models`` payload; we only
|
|
care that the endpoint responds 200."""
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: generic.Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf\n"
|
|
"generic:\n"
|
|
' base_url: "http://nyx.helu.ca:22079/v1"\n'
|
|
)
|
|
# generic requires no api_key; no secrets file needed.
|
|
|
|
captured: list[httpx.Request] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
captured.append(request)
|
|
# Match llama.cpp's shape (Ollama-style `models` alongside OpenAI `data`).
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"models": [{"name": "Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf"}],
|
|
"object": "list",
|
|
"data": [{"id": "Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf"}],
|
|
},
|
|
)
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
assert _run(health.validate_llm_providers(timeout=1.0)) == {
|
|
"generic": {
|
|
"status": "ok",
|
|
"model": "Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf",
|
|
}
|
|
}
|
|
assert str(captured[0].url) == "http://nyx.helu.ca:22079/v1/models"
|
|
|
|
|
|
def test_validate_generic_unreachable(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: generic.Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf\n"
|
|
"generic:\n"
|
|
' base_url: "http://nyx.helu.ca:22079/v1"\n'
|
|
)
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ConnectError("connection refused")
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["generic"]["status"] == "error"
|
|
assert "unreachable" in results["generic"]["message"].lower()
|
|
|
|
|
|
def test_validate_generic_503(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: generic.Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf\n"
|
|
"generic:\n"
|
|
' base_url: "http://nyx.helu.ca:22079/v1"\n'
|
|
)
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(503)
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["generic"] == {
|
|
"status": "error",
|
|
"model": "Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf",
|
|
"message": "API request failed (503)",
|
|
}
|
|
|
|
|
|
# ── validate_llm_providers: bedrock (no HTTP) ────────────────────────────────
|
|
|
|
|
|
def test_validate_bedrock_ok_with_bearer(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: bedrock.anthropic.claude-sonnet-4-6\n"
|
|
)
|
|
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "abs-fake")
|
|
_patch_httpx_raising(monkeypatch) # any HTTP call is a test failure
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["bedrock"]["status"] == "ok"
|
|
assert results["bedrock"]["model"] == "anthropic.claude-sonnet-4-6"
|
|
|
|
|
|
def test_validate_bedrock_no_credentials(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: bedrock.anthropic.claude-sonnet-4-6\n"
|
|
)
|
|
# The real user has an ~/.aws/credentials file which would cause a false
|
|
# positive; redirect HOME so Path.home() / ".aws" does not exist.
|
|
monkeypatch.setenv("HOME", str(workspace))
|
|
_patch_httpx_raising(monkeypatch)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["bedrock"]["status"] == "error"
|
|
assert "AWS credentials" in results["bedrock"]["message"]
|
|
|
|
|
|
# ── validate_llm_providers: malformed / unknown provider ─────────────────────
|
|
|
|
|
|
def test_validate_default_model_missing_prefix(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text("default_model: just-a-name\n")
|
|
_patch_httpx_raising(monkeypatch)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["unknown"]["status"] == "error"
|
|
assert "provider prefix" in results["unknown"]["message"]
|
|
|
|
|
|
def test_validate_unknown_provider(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: imaginary.some-model\n"
|
|
)
|
|
_patch_httpx_raising(monkeypatch)
|
|
|
|
results = _run(health.validate_llm_providers(timeout=1.0))
|
|
assert results["imaginary"]["status"] == "error"
|
|
assert "unknown provider" in results["imaginary"]["message"]
|
|
|
|
|
|
# ── get_health() payload ─────────────────────────────────────────────────────
|
|
|
|
|
|
def test_get_health_reports_generic_ok(
|
|
workspace: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""End-to-end: after a successful generic preflight, get_health() should
|
|
return status=ok with no LLM error in the message. This is the exact
|
|
regression case that was showing up as ``LLM: generic: error`` in Daedalus.
|
|
"""
|
|
(workspace / "fastagent.config.yaml").write_text(
|
|
"default_model: generic.Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf\n"
|
|
"generic:\n"
|
|
' base_url: "http://nyx.helu.ca:22079/v1"\n'
|
|
)
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, json={"data": []})
|
|
|
|
_patch_httpx(monkeypatch, handler)
|
|
_run(health.validate_llm_providers(timeout=1.0))
|
|
|
|
# Simulate the MCP `get_health` tool by calling check_downstream_health
|
|
# with an empty server map and composing the message the same way
|
|
# register_health_tool does.
|
|
async def call() -> dict:
|
|
result = await health.check_downstream_health({}, timeout=1.0)
|
|
active = health._llm_status.get(health._active_provider)
|
|
if active is not None and active.get("status") != "ok":
|
|
result["status"] = "degraded"
|
|
existing = result.get("message", "")
|
|
msg = (
|
|
f"LLM: {health._active_provider}: "
|
|
f"{active.get('message', 'unknown error')}"
|
|
)
|
|
result["message"] = f"{existing}; {msg}" if existing else msg
|
|
return result
|
|
|
|
final = _run(call())
|
|
assert final["status"] == "ok"
|
|
assert "LLM" not in final.get("message", "")
|