feat: forward tool-result images to the MCP caller in send_message results
fast-agent's agent.send() returns only the final assistant text, so ImageContent produced by downstream tools during the agentic loop (playwright screenshots, rommie desktop captures) reached the agent's own vision model but never crossed the MCP boundary — Daedalus and lead agents saw text-only results. A per-request after_tool_call hook (pallas.image_passthrough, same composition pattern as assistant_stream / loop_guard) collects every ImageContent block from the turn's tool results; send_message then returns a FastMCP ToolResult of [final text, *images]. Turns with no images return the plain string — wire shape unchanged (the str-only output schema is dropped so the union return passes through cleanly; no consumer read structuredContent). Images cascade hop-by-hop up delegation chains with no extra wiring: verified live playwright → dolores → harper → MCP client, image intact at each hop. New per-agent agents.yaml knob max_result_images (default 8, keeps most recent, 0 disables) and pallas_result_images_total counter. Version 0.7.0. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
138
tests/test_image_passthrough.py
Normal file
138
tests/test_image_passthrough.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""Tests for ``pallas.image_passthrough``.
|
||||
|
||||
Drives the ``after_tool_call`` hook with handcrafted
|
||||
``PromptMessageExtended`` objects and asserts the collection, cap, and
|
||||
result-assembly behaviour. No fast-agent runtime is involved — the hook is
|
||||
a pure async function. Uses ``asyncio.run`` directly to match the
|
||||
convention in the other test modules (pallas has no pytest-asyncio
|
||||
dependency).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fast_agent.types import PromptMessageExtended
|
||||
from fastmcp.tools import ToolResult
|
||||
from mcp.types import CallToolResult, ImageContent, TextContent
|
||||
|
||||
from pallas.image_passthrough import (
|
||||
ImageCollector,
|
||||
build_result,
|
||||
install_for_request,
|
||||
)
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _image(tag: str) -> ImageContent:
|
||||
return ImageContent(type="image", data=f"b64-{tag}", mimeType="image/png")
|
||||
|
||||
|
||||
def _result_message(*blocks, call_id: str = "toolu_1") -> PromptMessageExtended:
|
||||
return PromptMessageExtended(
|
||||
role="user",
|
||||
content=[],
|
||||
tool_results={call_id: CallToolResult(content=list(blocks))},
|
||||
)
|
||||
|
||||
|
||||
def _collector(max_images: int = 8) -> ImageCollector:
|
||||
return ImageCollector(
|
||||
agent_name="dolores", conversation_id="c1", max_images=max_images
|
||||
)
|
||||
|
||||
|
||||
def test_collects_images_and_ignores_text():
|
||||
collector = _collector()
|
||||
hook = collector.as_after_tool_call_hook()
|
||||
|
||||
async def go():
|
||||
await hook(None, _result_message(TextContent(type="text", text="snapshot")))
|
||||
await hook(None, _result_message(_image("a")))
|
||||
await hook(
|
||||
None,
|
||||
_result_message(TextContent(type="text", text="took it"), _image("b")),
|
||||
)
|
||||
|
||||
_run(go())
|
||||
assert [i.data for i in collector.collected()] == ["b64-a", "b64-b"]
|
||||
|
||||
|
||||
def test_collects_across_multiple_results_in_one_message():
|
||||
collector = _collector()
|
||||
hook = collector.as_after_tool_call_hook()
|
||||
message = PromptMessageExtended(
|
||||
role="user",
|
||||
content=[],
|
||||
tool_results={
|
||||
"toolu_1": CallToolResult(content=[_image("a")]),
|
||||
"toolu_2": CallToolResult(content=[_image("b")]),
|
||||
},
|
||||
)
|
||||
|
||||
_run(hook(None, message))
|
||||
assert len(collector.collected()) == 2
|
||||
|
||||
|
||||
def test_cap_keeps_most_recent():
|
||||
collector = _collector(max_images=2)
|
||||
hook = collector.as_after_tool_call_hook()
|
||||
|
||||
async def go():
|
||||
for tag in ("a", "b", "c", "d"):
|
||||
await hook(None, _result_message(_image(tag)))
|
||||
|
||||
_run(go())
|
||||
assert [i.data for i in collector.collected()] == ["b64-c", "b64-d"]
|
||||
|
||||
|
||||
def test_empty_tool_results_are_harmless():
|
||||
collector = _collector()
|
||||
hook = collector.as_after_tool_call_hook()
|
||||
message = PromptMessageExtended(role="user", content=[], tool_results=None)
|
||||
|
||||
_run(hook(None, message))
|
||||
assert collector.collected() == []
|
||||
|
||||
|
||||
def test_build_result_plain_text_when_no_images():
|
||||
assert build_result("answer", []) == "answer"
|
||||
|
||||
|
||||
def test_build_result_wraps_text_and_images():
|
||||
result = build_result("answer", [_image("a"), _image("b")])
|
||||
assert isinstance(result, ToolResult)
|
||||
assert [b.type for b in result.content] == ["text", "image", "image"]
|
||||
assert result.content[0].text == "answer"
|
||||
assert result.content[1].data == "b64-a"
|
||||
|
||||
|
||||
def test_build_result_omits_empty_text_block():
|
||||
result = build_result("", [_image("a")])
|
||||
assert isinstance(result, ToolResult)
|
||||
assert [b.type for b in result.content] == ["image"]
|
||||
|
||||
|
||||
def test_install_disabled_with_nonpositive_max():
|
||||
agent = SimpleNamespace(tool_runner_hooks="sentinel")
|
||||
collector, restore = install_for_request(
|
||||
agent, agent_name="a", conversation_id=None, max_images=0
|
||||
)
|
||||
assert collector is None
|
||||
assert agent.tool_runner_hooks == "sentinel" # untouched
|
||||
restore() # no-op, must not raise
|
||||
|
||||
|
||||
def test_install_merges_and_restores():
|
||||
agent = SimpleNamespace(tool_runner_hooks=None)
|
||||
collector, restore = install_for_request(
|
||||
agent, agent_name="a", conversation_id=None
|
||||
)
|
||||
assert collector is not None
|
||||
assert agent.tool_runner_hooks is not None
|
||||
assert agent.tool_runner_hooks.after_tool_call is not None
|
||||
restore()
|
||||
assert agent.tool_runner_hooks is None
|
||||
Reference in New Issue
Block a user