fix(memory): flatten multimodal content before provider sync
Multimodal turns carry message content as a list of typed parts
({type: "text"|"image_url", ...}). _sync_external_memory_for_turn
passed that list straight into MemoryManager.sync_all, and providers
feed it to regexes — Honcho's sync_turn calls sanitize_context, where
re.sub raised 'expected string or bytes-like object, got list'. Every
turn with an attached image silently never synced.
Flatten to plain text at the boundary: text parts joined, images noted
as an [N image(s)] marker so the attachment isn't erased from recall.
Fixing here covers all providers instead of patching each plugin.
This commit is contained in:
@@ -979,6 +979,67 @@ class TestMemoryContextFencing:
|
||||
assert combined.index("weather") < fence_start
|
||||
|
||||
|
||||
class TestFlattenMessageContent:
|
||||
"""Multimodal message content (list of typed parts) must flatten to a
|
||||
plain string before reaching providers — a raw list crashes their regex
|
||||
sanitization with ``expected string or bytes-like object, got 'list'``."""
|
||||
|
||||
def test_string_passthrough(self):
|
||||
from agent.memory_manager import flatten_message_content
|
||||
assert flatten_message_content("hello") == "hello"
|
||||
|
||||
def test_none_is_empty(self):
|
||||
from agent.memory_manager import flatten_message_content
|
||||
assert flatten_message_content(None) == ""
|
||||
|
||||
def test_text_parts_joined(self):
|
||||
from agent.memory_manager import flatten_message_content
|
||||
content = [
|
||||
{"type": "text", "text": "first"},
|
||||
{"type": "text", "text": "second"},
|
||||
]
|
||||
assert flatten_message_content(content) == "first\nsecond"
|
||||
|
||||
def test_image_part_becomes_marker(self):
|
||||
from agent.memory_manager import flatten_message_content
|
||||
content = [
|
||||
{"type": "text", "text": "look at this"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,xyz"}},
|
||||
]
|
||||
assert flatten_message_content(content) == "[1 image] look at this"
|
||||
|
||||
def test_image_only_message(self):
|
||||
from agent.memory_manager import flatten_message_content
|
||||
content = [
|
||||
{"type": "image_url", "image_url": {"url": "data:..."}},
|
||||
{"type": "image_url", "image_url": {"url": "data:..."}},
|
||||
]
|
||||
assert flatten_message_content(content) == "[2 images]"
|
||||
|
||||
def test_unknown_parts_skipped(self):
|
||||
from agent.memory_manager import flatten_message_content
|
||||
content = [{"type": "audio", "data": "..."}, {"type": "text", "text": "ok"}, 42]
|
||||
assert flatten_message_content(content) == "ok"
|
||||
|
||||
def test_bare_strings_in_list(self):
|
||||
from agent.memory_manager import flatten_message_content
|
||||
assert flatten_message_content(["plain", "strings"]) == "plain\nstrings"
|
||||
|
||||
def test_scalar_fallback(self):
|
||||
from agent.memory_manager import flatten_message_content
|
||||
assert flatten_message_content(42) == "42"
|
||||
|
||||
def test_flattened_output_is_regex_safe(self):
|
||||
"""The original failure: sanitize_context(list) raised TypeError."""
|
||||
from agent.memory_manager import flatten_message_content, sanitize_context
|
||||
content = [
|
||||
{"type": "text", "text": "fix this bug"},
|
||||
{"type": "image_url", "image_url": {"url": "data:..."}},
|
||||
]
|
||||
# Must not raise.
|
||||
assert sanitize_context(flatten_message_content(content))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AIAgent.commit_memory_session — routes to MemoryManager.on_session_end
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -207,6 +207,57 @@ class TestSyncExternalMemoryForTurn:
|
||||
# sync_all still happened before the prefetch blew up.
|
||||
agent._memory_manager.sync_all.assert_called_once()
|
||||
|
||||
# --- Multimodal content flattening ----------------------------------
|
||||
|
||||
def test_multimodal_user_message_is_flattened(self):
|
||||
"""A turn with an attached image carries the user message as a
|
||||
list of typed parts. Providers feed the content to regexes
|
||||
(sanitize_context), so a raw list raised ``expected string or
|
||||
bytes-like object, got 'list'`` and the turn silently never
|
||||
synced. The boundary must flatten to text first."""
|
||||
agent = _bare_agent()
|
||||
agent._sync_external_memory_for_turn(
|
||||
original_user_message=[
|
||||
{"type": "text", "text": "what is in this screenshot?"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},
|
||||
],
|
||||
final_response="A terminal window showing a stack trace.",
|
||||
interrupted=False,
|
||||
)
|
||||
agent._memory_manager.sync_all.assert_called_once_with(
|
||||
"[1 image] what is in this screenshot?",
|
||||
"A terminal window showing a stack trace.",
|
||||
session_id="test_session_001",
|
||||
)
|
||||
agent._memory_manager.queue_prefetch_all.assert_called_once_with(
|
||||
"[1 image] what is in this screenshot?",
|
||||
session_id="test_session_001",
|
||||
)
|
||||
|
||||
def test_multimodal_response_is_flattened(self):
|
||||
agent = _bare_agent()
|
||||
agent._sync_external_memory_for_turn(
|
||||
original_user_message="describe it",
|
||||
final_response=[{"type": "text", "text": "a cat"}],
|
||||
interrupted=False,
|
||||
)
|
||||
agent._memory_manager.sync_all.assert_called_once_with(
|
||||
"describe it", "a cat",
|
||||
session_id="test_session_001",
|
||||
)
|
||||
|
||||
def test_multimodal_with_no_text_at_all_skips(self):
|
||||
"""Unknown-typed parts flatten to an empty string — don't sync a
|
||||
turn with no recoverable text."""
|
||||
agent = _bare_agent()
|
||||
agent._sync_external_memory_for_turn(
|
||||
original_user_message=[{"type": "audio", "data": "..."}],
|
||||
final_response="noted",
|
||||
interrupted=False,
|
||||
)
|
||||
agent._memory_manager.sync_all.assert_not_called()
|
||||
agent._memory_manager.queue_prefetch_all.assert_not_called()
|
||||
|
||||
# --- The specific matrix the reporter asked about ------------------
|
||||
|
||||
@pytest.mark.parametrize("interrupted,final,user,expect_sync", [
|
||||
|
||||
Reference in New Issue
Block a user