feat: forward tool-result images to the MCP caller in send_message results

fast-agent's agent.send() returns only the final assistant text, so
ImageContent produced by downstream tools during the agentic loop
(playwright screenshots, rommie desktop captures) reached the agent's own
vision model but never crossed the MCP boundary — Daedalus and lead agents
saw text-only results.

A per-request after_tool_call hook (pallas.image_passthrough, same
composition pattern as assistant_stream / loop_guard) collects every
ImageContent block from the turn's tool results; send_message then returns
a FastMCP ToolResult of [final text, *images]. Turns with no images return
the plain string — wire shape unchanged (the str-only output schema is
dropped so the union return passes through cleanly; no consumer read
structuredContent). Images cascade hop-by-hop up delegation chains with no
extra wiring: verified live playwright → dolores → harper → MCP client,
image intact at each hop.

New per-agent agents.yaml knob max_result_images (default 8, keeps most
recent, 0 disables) and pallas_result_images_total counter. Version 0.7.0.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-01 22:47:25 -04:00
parent 54d639c13a
commit 1b05504207
7 changed files with 359 additions and 5 deletions

View File

@@ -0,0 +1,138 @@
"""Tests for ``pallas.image_passthrough``.
Drives the ``after_tool_call`` hook with handcrafted
``PromptMessageExtended`` objects and asserts the collection, cap, and
result-assembly behaviour. No fast-agent runtime is involved — the hook is
a pure async function. Uses ``asyncio.run`` directly to match the
convention in the other test modules (pallas has no pytest-asyncio
dependency).
"""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from fast_agent.types import PromptMessageExtended
from fastmcp.tools import ToolResult
from mcp.types import CallToolResult, ImageContent, TextContent
from pallas.image_passthrough import (
ImageCollector,
build_result,
install_for_request,
)
def _run(coro):
return asyncio.run(coro)
def _image(tag: str) -> ImageContent:
return ImageContent(type="image", data=f"b64-{tag}", mimeType="image/png")
def _result_message(*blocks, call_id: str = "toolu_1") -> PromptMessageExtended:
return PromptMessageExtended(
role="user",
content=[],
tool_results={call_id: CallToolResult(content=list(blocks))},
)
def _collector(max_images: int = 8) -> ImageCollector:
return ImageCollector(
agent_name="dolores", conversation_id="c1", max_images=max_images
)
def test_collects_images_and_ignores_text():
collector = _collector()
hook = collector.as_after_tool_call_hook()
async def go():
await hook(None, _result_message(TextContent(type="text", text="snapshot")))
await hook(None, _result_message(_image("a")))
await hook(
None,
_result_message(TextContent(type="text", text="took it"), _image("b")),
)
_run(go())
assert [i.data for i in collector.collected()] == ["b64-a", "b64-b"]
def test_collects_across_multiple_results_in_one_message():
collector = _collector()
hook = collector.as_after_tool_call_hook()
message = PromptMessageExtended(
role="user",
content=[],
tool_results={
"toolu_1": CallToolResult(content=[_image("a")]),
"toolu_2": CallToolResult(content=[_image("b")]),
},
)
_run(hook(None, message))
assert len(collector.collected()) == 2
def test_cap_keeps_most_recent():
collector = _collector(max_images=2)
hook = collector.as_after_tool_call_hook()
async def go():
for tag in ("a", "b", "c", "d"):
await hook(None, _result_message(_image(tag)))
_run(go())
assert [i.data for i in collector.collected()] == ["b64-c", "b64-d"]
def test_empty_tool_results_are_harmless():
collector = _collector()
hook = collector.as_after_tool_call_hook()
message = PromptMessageExtended(role="user", content=[], tool_results=None)
_run(hook(None, message))
assert collector.collected() == []
def test_build_result_plain_text_when_no_images():
assert build_result("answer", []) == "answer"
def test_build_result_wraps_text_and_images():
result = build_result("answer", [_image("a"), _image("b")])
assert isinstance(result, ToolResult)
assert [b.type for b in result.content] == ["text", "image", "image"]
assert result.content[0].text == "answer"
assert result.content[1].data == "b64-a"
def test_build_result_omits_empty_text_block():
result = build_result("", [_image("a")])
assert isinstance(result, ToolResult)
assert [b.type for b in result.content] == ["image"]
def test_install_disabled_with_nonpositive_max():
agent = SimpleNamespace(tool_runner_hooks="sentinel")
collector, restore = install_for_request(
agent, agent_name="a", conversation_id=None, max_images=0
)
assert collector is None
assert agent.tool_runner_hooks == "sentinel" # untouched
restore() # no-op, must not raise
def test_install_merges_and_restores():
agent = SimpleNamespace(tool_runner_hooks=None)
collector, restore = install_for_request(
agent, agent_name="a", conversation_id=None
)
assert collector is not None
assert agent.tool_runner_hooks is not None
assert agent.tool_runner_hooks.after_tool_call is not None
restore()
assert agent.tool_runner_hooks is None