seb/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: uptick

Browse Source
This commit is contained in:
Brooklyn Nicholson
2026-05-02 03:19:39 -05:00
parent 420f68e4e2
commit db884f4646
240 changed files with 25206 additions and 3155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/run_agent/test_860_dedup.py
+2
View File
@@ -38,6 +38,8 @@ class TestFlushDeduplication:
skip_context_files=True,
skip_memory=True,
)
# Simulate lazy session creation (normally done by run_conversation)
agent._ensure_db_session()
return agent
def test_flush_writes_only_new_messages(self):
tests/run_agent/test_deepseek_reasoning_content_echo.py
+61 -19
View File
@@ -10,15 +10,21 @@ field, DeepSeek rejects the next request with HTTP 400::
Fix covers three paths:
1. ``_build_assistant_message`` — new tool-call messages without raw
reasoning_content get ``""`` pinned at creation time so nothing gets
reasoning_content get ``" "`` pinned at creation time so nothing gets
persisted poisoned.
2. ``_copy_reasoning_content_for_api`` — already-poisoned history replays
with ``reasoning_content=""`` injected defensively.
with ``reasoning_content=" "`` injected defensively.
3. Detection covers three signals: ``provider == "deepseek"``,
``"deepseek" in model``, and ``api.deepseek.com`` host match. The third
catches custom-provider setups pointing at DeepSeek.
Refs #15250 / #15353.
The placeholder is a single space (not empty string) because DeepSeek V4 Pro
tightened validation and rejects empty-string reasoning_content with a
400 ("The reasoning content in the thinking mode must be passed back to
the API"). A space satisfies non-empty checks everywhere without leaking
fabricated reasoning.
Refs #15250 / #15353 / #17341.
"""
from __future__ import annotations
@@ -105,8 +111,8 @@ class TestNeedsDeepSeekToolReasoning:
class TestCopyReasoningContentForApi:
"""_copy_reasoning_content_for_api pads reasoning_content for DeepSeek tool-calls."""
def test_deepseek_tool_call_poisoned_history_gets_empty_string(self) -> None:
"""Already-poisoned history (no reasoning_content, no reasoning) gets ''."""
def test_deepseek_tool_call_poisoned_history_gets_space_placeholder(self) -> None:
"""Already-poisoned history (no reasoning_content, no reasoning) gets ' '."""
agent = _make_agent(provider="deepseek", model="deepseek-v4-flash")
source = {
"role": "assistant",
@@ -115,7 +121,7 @@ class TestCopyReasoningContentForApi:
}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg.get("reasoning_content") == ""
assert api_msg.get("reasoning_content") == " "
def test_deepseek_assistant_no_tool_call_gets_padded(self) -> None:
"""DeepSeek thinking mode pads ALL assistant turns, even without tool_calls."""
@@ -123,7 +129,7 @@ class TestCopyReasoningContentForApi:
source = {"role": "assistant", "content": "hello"}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg.get("reasoning_content") == ""
assert api_msg.get("reasoning_content") == " "
def test_deepseek_explicit_reasoning_content_preserved(self) -> None:
"""When reasoning_content is already set, it's copied verbatim."""
@@ -137,6 +143,42 @@ class TestCopyReasoningContentForApi:
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg["reasoning_content"] == "<think>real chain of thought</think>"
def test_deepseek_stale_empty_placeholder_upgraded_to_space(self) -> None:
"""Sessions persisted before #17341 have ``reasoning_content=""`` pinned
at creation time. DeepSeek V4 Pro rejects "" with HTTP 400. When the
active provider enforces the thinking-mode echo, the replay path
upgrades "" → " " so stale history doesn't break the next turn.
"""
agent = _make_agent(provider="deepseek", model="deepseek-v4-pro")
source = {
"role": "assistant",
"content": "",
"reasoning_content": "",
"tool_calls": [{"id": "c1", "function": {"name": "terminal"}}],
}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg["reasoning_content"] == " "
def test_non_thinking_provider_preserves_empty_reasoning_content_verbatim(self) -> None:
"""The stale-placeholder upgrade ONLY fires when the active provider
enforces thinking-mode echo. On non-thinking providers, an empty
reasoning_content must still round-trip verbatim.
"""
agent = _make_agent(
provider="openrouter",
model="anthropic/claude-sonnet-4.6",
base_url="https://openrouter.ai/api/v1",
)
source = {
"role": "assistant",
"content": "hi",
"reasoning_content": "",
}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg["reasoning_content"] == ""
def test_deepseek_reasoning_field_promoted(self) -> None:
"""When only 'reasoning' is set, it gets promoted to reasoning_content."""
agent = _make_agent(provider="deepseek", model="deepseek-v4-flash")
@@ -155,7 +197,7 @@ class TestCopyReasoningContentForApi:
If the source turn has tool_calls AND a 'reasoning' field but NO
'reasoning_content' key, it's from a prior provider (the DeepSeek
build path pins reasoning_content at creation). Inject "" instead
build path pins reasoning_content at creation). Inject " " instead
of forwarding the prior provider's chain of thought.
"""
agent = _make_agent(provider="deepseek", model="deepseek-v4-flash")
@@ -167,7 +209,7 @@ class TestCopyReasoningContentForApi:
}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg["reasoning_content"] == ""
assert api_msg["reasoning_content"] == " "
def test_kimi_poisoned_cross_provider_history_padded(self) -> None:
"""Kimi path of #15748 — same rule as DeepSeek."""
@@ -180,7 +222,7 @@ class TestCopyReasoningContentForApi:
}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg["reasoning_content"] == ""
assert api_msg["reasoning_content"] == " "
def test_kimi_path_still_works(self) -> None:
"""Existing Kimi detection still pads reasoning_content."""
@@ -192,7 +234,7 @@ class TestCopyReasoningContentForApi:
}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg.get("reasoning_content") == ""
assert api_msg.get("reasoning_content") == " "
def test_kimi_moonshot_base_url(self) -> None:
agent = _make_agent(
@@ -205,7 +247,7 @@ class TestCopyReasoningContentForApi:
}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg.get("reasoning_content") == ""
assert api_msg.get("reasoning_content") == " "
def test_non_thinking_provider_not_padded(self) -> None:
"""Providers that don't require the echo are untouched."""
@@ -237,7 +279,7 @@ class TestCopyReasoningContentForApi:
}
api_msg: dict = {}
agent._copy_reasoning_content_for_api(source, api_msg)
assert api_msg.get("reasoning_content") == ""
assert api_msg.get("reasoning_content") == " "
def test_non_assistant_role_ignored(self) -> None:
"""User/tool messages are left alone."""
@@ -302,7 +344,7 @@ class TestBuildAssistantMessageDeepSeekReasoningContent:
assert msg["reasoning_content"] == "DeepSeek model_extra reasoning"
def test_deepseek_tool_call_without_raw_reasoning_content_gets_empty_string(self) -> None:
def test_deepseek_tool_call_without_raw_reasoning_content_gets_space_placeholder(self) -> None:
agent = _make_agent(provider="deepseek", model="deepseek-v4-flash")
assistant_message = SimpleNamespace(
content=None,
@@ -324,7 +366,7 @@ class TestBuildAssistantMessageDeepSeekReasoningContent:
msg = agent._build_assistant_message(assistant_message, "tool_calls")
assert msg["reasoning_content"] == ""
assert msg["reasoning_content"] == " "
assert msg["tool_calls"][0]["id"] == "call_1"
@@ -345,22 +387,22 @@ class TestBuildAssistantMessagePadsStrictProviders:
[
pytest.param(
"deepseek", "deepseek-v4-pro", "",
None, "",
None, " ",
id="deepseek-attr-none",
),
pytest.param(
"deepseek", "deepseek-v4-pro", "",
_ATTR_ABSENT, "",
_ATTR_ABSENT, " ",
id="deepseek-attr-absent",
),
pytest.param(
"kimi-coding", "kimi-k2.6", "",
None, "",
None, " ",
id="kimi-attr-none",
),
pytest.param(
"custom", "kimi-k2", "https://api.moonshot.ai/v1",
_ATTR_ABSENT, "",
_ATTR_ABSENT, " ",
id="moonshot-base-url",
),
pytest.param(
tests/run_agent/test_run_agent.py
+71 -4
View File
@@ -1465,8 +1465,8 @@ class TestBuildAssistantMessage:
This preserves ``_copy_reasoning_content_for_api``'s downstream
tiers at replay time — cross-provider leak guard (#15748),
promote-from-``reasoning``, and DeepSeek/Kimi ""-pad — which
would all be bypassed if we eagerly wrote ``reasoning_content=""``
promote-from-``reasoning``, and DeepSeek/Kimi " "-pad — which
would all be bypassed if we eagerly wrote ``reasoning_content=" "``
on every assistant turn regardless of provider.
"""
msg = _mock_assistant_msg(content="plain answer")
@@ -2181,6 +2181,73 @@ class TestHandleMaxIterations:
kwargs = agent.client.chat.completions.create.call_args.kwargs
assert "reasoning" not in kwargs.get("extra_body", {})
def test_codex_summary_sanitizes_orphan_tool_results(self, agent):
agent.api_mode = "codex_responses"
agent.provider = "openai-codex"
agent.base_url = "https://chatgpt.com/backend-api/codex"
agent._base_url_lower = agent.base_url.lower()
agent._base_url_hostname = "chatgpt.com"
agent.model = "gpt-5.5"
agent._cached_system_prompt = "You are helpful."
captured = {}
def fake_run_codex_stream(kwargs):
captured.update(kwargs)
return SimpleNamespace(
status="completed",
output=[
SimpleNamespace(
type="message",
status="completed",
content=[SimpleNamespace(type="output_text", text="Summary")],
)
],
)
messages = [
{"role": "user", "content": "do stuff"},
{
"role": "tool",
"tool_call_id": "call_orphan",
"content": "orphaned result from compressed history",
},
]
with patch.object(agent, "_run_codex_stream", side_effect=fake_run_codex_stream):
result = agent._handle_max_iterations(messages, 90)
assert result == "Summary"
input_items = captured["input"]
assert not any(
item.get("type") == "function_call_output"
and item.get("call_id") == "call_orphan"
for item in input_items
)
def test_api_sanitizer_matches_responses_call_id_when_id_differs(self, agent):
messages = [
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "fc_123",
"call_id": "call_123",
"response_item_id": "fc_123",
"type": "function",
"function": {"name": "web_search", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_123", "content": "result"},
]
sanitized = agent._sanitize_api_messages(messages)
assert [m.get("tool_call_id") for m in sanitized if m.get("role") == "tool"] == [
"call_123"
]
class TestRunConversation:
"""Tests for the main run_conversation method.
@@ -4550,7 +4617,7 @@ class TestReasoningReplayForStrictProviders:
agent.compression_enabled = False
agent.save_trajectories = False
def test_kimi_tool_replay_includes_empty_reasoning_content(self, agent):
def test_kimi_tool_replay_includes_space_reasoning_content(self, agent):
self._setup_agent(agent)
agent.base_url = "https://api.kimi.com/coding/v1"
agent._base_url_lower = agent.base_url.lower()
@@ -4587,7 +4654,7 @@ class TestReasoningReplayForStrictProviders:
assert replayed_assistant["role"] == "assistant"
assert replayed_assistant["tool_calls"][0]["function"]["name"] == "terminal"
assert "reasoning_content" in replayed_assistant
assert replayed_assistant["reasoning_content"] == ""
assert replayed_assistant["reasoning_content"] == " "
def test_explicit_reasoning_content_beats_normalized_reasoning_on_replay(self, agent):
self._setup_agent(agent)
tests/run_agent/test_tool_call_guardrail_runtime.py
+275
View File
@@ -0,0 +1,275 @@
"""Runtime tests for tool-call loop guardrails."""
import json
import uuid
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from run_agent import AIAgent
def _make_tool_defs(*names: str) -> list[dict]:
return [
{
"type": "function",
"function": {
"name": name,
"description": f"{name} tool",
"parameters": {"type": "object", "properties": {}},
},
}
for name in names
]
def _mock_tool_call(name="web_search", arguments="{}", call_id=None):
return SimpleNamespace(
id=call_id or f"call_{uuid.uuid4().hex[:8]}",
type="function",
function=SimpleNamespace(name=name, arguments=arguments),
)
def _mock_response(content="Hello", finish_reason="stop", tool_calls=None):
msg = SimpleNamespace(content=content, tool_calls=tool_calls)
choice = SimpleNamespace(message=msg, finish_reason=finish_reason)
return SimpleNamespace(choices=[choice], model="test/model", usage=None)
def _make_agent(*tool_names: str, max_iterations: int = 10, config: dict | None = None) -> AIAgent:
with (
patch("run_agent.get_tool_definitions", return_value=_make_tool_defs(*tool_names)),
patch("run_agent.check_toolset_requirements", return_value={}),
patch("hermes_cli.config.load_config", return_value=config or {}),
patch("run_agent.OpenAI"),
):
agent = AIAgent(
api_key="test-key-1234567890",
base_url="https://openrouter.ai/api/v1",
max_iterations=max_iterations,
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
agent.client = MagicMock()
agent._cached_system_prompt = "You are helpful."
agent._use_prompt_caching = False
agent.tool_delay = 0
agent.compression_enabled = False
agent.save_trajectories = False
return agent
def _seed_exact_failures(agent: AIAgent, tool_name: str, args: dict, count: int = 2) -> None:
for _ in range(count):
agent._tool_guardrails.after_call(
tool_name,
args,
json.dumps({"error": "boom"}),
failed=True,
)
def _hard_stop_config(**overrides) -> dict:
cfg = {
"tool_loop_guardrails": {
"warnings_enabled": True,
"hard_stop_enabled": True,
"hard_stop_after": {
"exact_failure": 2,
"same_tool_failure": 8,
"idempotent_no_progress": 5,
},
}
}
cfg["tool_loop_guardrails"].update(overrides)
return cfg
def test_default_sequential_path_warns_repeated_exact_failure_without_blocking_execution():
agent = _make_agent("web_search")
args = {"query": "same"}
_seed_exact_failures(agent, "web_search", args)
starts = []
progress = []
agent.tool_start_callback = lambda *a, **k: starts.append((a, k))
agent.tool_progress_callback = lambda *a, **k: progress.append((a, k))
tc = _mock_tool_call("web_search", json.dumps(args), "c-soft")
msg = SimpleNamespace(content="", tool_calls=[tc])
messages = []
with patch("run_agent.handle_function_call", return_value=json.dumps({"error": "boom"})) as mock_hfc:
agent._execute_tool_calls_sequential(msg, messages, "task-1")
mock_hfc.assert_called_once()
assert len(starts) == 1
assert any(event[0][0] == "tool.completed" for event in progress)
assert len(messages) == 1
assert messages[0]["role"] == "tool"
assert messages[0]["tool_call_id"] == "c-soft"
assert "repeated_exact_failure_warning" in messages[0]["content"]
assert "repeated_exact_failure_block" not in messages[0]["content"]
assert agent._tool_guardrail_halt_decision is None
def test_config_enabled_hard_stop_blocks_repeated_exact_failure_before_execution():
agent = _make_agent("web_search", config=_hard_stop_config())
args = {"query": "same"}
_seed_exact_failures(agent, "web_search", args)
starts = []
progress = []
agent.tool_start_callback = lambda *a, **k: starts.append((a, k))
agent.tool_progress_callback = lambda *a, **k: progress.append((a, k))
tc = _mock_tool_call("web_search", json.dumps(args), "c-block")
msg = SimpleNamespace(content="", tool_calls=[tc])
messages = []
with patch("run_agent.handle_function_call", return_value="SHOULD_NOT_RUN") as mock_hfc:
agent._execute_tool_calls_sequential(msg, messages, "task-1")
mock_hfc.assert_not_called()
assert starts == []
assert progress == []
assert len(messages) == 1
assert messages[0]["role"] == "tool"
assert messages[0]["tool_call_id"] == "c-block"
assert "repeated_exact_failure_block" in messages[0]["content"]
def test_sequential_after_call_appends_guidance_to_tool_result_without_extra_messages():
agent = _make_agent("web_search")
args = {"query": "same"}
_seed_exact_failures(agent, "web_search", args, count=1)
tc = _mock_tool_call("web_search", json.dumps(args), "c-warn")
msg = SimpleNamespace(content="", tool_calls=[tc])
messages = []
with patch("run_agent.handle_function_call", return_value=json.dumps({"error": "boom"})):
agent._execute_tool_calls_sequential(msg, messages, "task-1")
assert [m["role"] for m in messages] == ["tool"]
assert messages[0]["tool_call_id"] == "c-warn"
assert "Tool loop warning" in messages[0]["content"]
assert "repeated_exact_failure_warning" in messages[0]["content"]
def test_config_enabled_hard_stop_concurrent_path_does_not_submit_blocked_calls_and_preserves_result_order():
agent = _make_agent("web_search", config=_hard_stop_config())
blocked_args = {"query": "blocked"}
allowed_args = {"query": "allowed"}
_seed_exact_failures(agent, "web_search", blocked_args)
starts = []
progress_events = []
agent.tool_start_callback = lambda tool_call_id, name, args: starts.append((tool_call_id, name, args))
agent.tool_progress_callback = lambda event, name, preview, args, **kw: progress_events.append((event, name, args, kw))
calls = [
_mock_tool_call("web_search", json.dumps(blocked_args), "c-block"),
_mock_tool_call("web_search", json.dumps(allowed_args), "c-allow"),
]
msg = SimpleNamespace(content="", tool_calls=calls)
messages = []
executed = []
def fake_handle(name, args, task_id, **kwargs):
executed.append((name, args, kwargs["tool_call_id"]))
return json.dumps({"ok": args["query"]})
with patch("run_agent.handle_function_call", side_effect=fake_handle):
agent._execute_tool_calls_concurrent(msg, messages, "task-1")
assert executed == [("web_search", allowed_args, "c-allow")]
assert [m["tool_call_id"] for m in messages] == ["c-block", "c-allow"]
assert "repeated_exact_failure_block" in messages[0]["content"]
assert json.loads(messages[1]["content"]) == {"ok": "allowed"}
assert starts == [("c-allow", "web_search", allowed_args)]
started_events = [event for event in progress_events if event[0] == "tool.started"]
completed_events = [event for event in progress_events if event[0] == "tool.completed"]
assert started_events == [("tool.started", "web_search", allowed_args, {})]
assert len(completed_events) == 1
assert completed_events[0][1] == "web_search"
def test_plugin_pre_tool_block_wins_without_counting_as_toolguard_block():
agent = _make_agent("web_search")
args = {"query": "same"}
tc = _mock_tool_call("web_search", json.dumps(args), "c-plugin")
msg = SimpleNamespace(content="", tool_calls=[tc])
messages = []
with (
patch("hermes_cli.plugins.get_pre_tool_call_block_message", return_value="plugin policy"),
patch("run_agent.handle_function_call", return_value="SHOULD_NOT_RUN") as mock_hfc,
):
agent._execute_tool_calls_sequential(msg, messages, "task-1")
mock_hfc.assert_not_called()
assert "plugin policy" in messages[0]["content"]
assert agent._tool_guardrails.before_call("web_search", args).action == "allow"
def test_default_run_conversation_warns_without_guardrail_halt():
agent = _make_agent("web_search", max_iterations=10)
same_args = {"query": "same"}
responses = [
_mock_response(
content="",
finish_reason="tool_calls",
tool_calls=[_mock_tool_call("web_search", json.dumps(same_args), f"c{i}")],
)
for i in range(1, 4)
]
responses.append(_mock_response(content="done", finish_reason="stop", tool_calls=None))
agent.client.chat.completions.create.side_effect = responses
with (
patch("run_agent.handle_function_call", return_value=json.dumps({"error": "boom"})) as mock_hfc,
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
result = agent.run_conversation("search repeatedly")
assert mock_hfc.call_count == 3
assert result["turn_exit_reason"].startswith("text_response")
assert "guardrail" not in result
assert result["final_response"] == "done"
tool_contents = [m["content"] for m in result["messages"] if m.get("role") == "tool"]
assert any("repeated_exact_failure_warning" in content for content in tool_contents)
def test_config_enabled_hard_stop_run_conversation_returns_controlled_guardrail_halt_without_top_level_error():
agent = _make_agent("web_search", max_iterations=10, config=_hard_stop_config())
same_args = {"query": "same"}
responses = [
_mock_response(
content="",
finish_reason="tool_calls",
tool_calls=[_mock_tool_call("web_search", json.dumps(same_args), f"c{i}")],
)
for i in range(1, 10)
]
agent.client.chat.completions.create.side_effect = responses
with (
patch("run_agent.handle_function_call", return_value=json.dumps({"error": "boom"})) as mock_hfc,
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
result = agent.run_conversation("search repeatedly")
assert mock_hfc.call_count == 2
assert result["api_calls"] == 3
assert result["api_calls"] < agent.max_iterations
assert result["turn_exit_reason"] == "guardrail_halt"
assert "error" not in result
assert result["completed"] is True
assert "stopped retrying" in result["final_response"]
assert result["guardrail"]["code"] == "repeated_exact_failure_block"
assert result["guardrail"]["tool_name"] == "web_search"
assistant_tool_calls = [m for m in result["messages"] if m.get("role") == "assistant" and m.get("tool_calls")]
for assistant_msg in assistant_tool_calls:
call_ids = [tc["id"] for tc in assistant_msg["tool_calls"]]
following_results = [m for m in result["messages"] if m.get("role") == "tool" and m.get("tool_call_id") in call_ids]
assert len(following_results) == len(call_ids)