Merge branch 'main' of github.com:NousResearch/hermes-agent into bb/gui

# Conflicts:
#	tui_gateway/server.py
This commit is contained in:
Brooklyn Nicholson
2026-05-30 13:19:27 -05:00
157 changed files with 10059 additions and 831 deletions
+158
View File
@@ -491,6 +491,96 @@ class TestPreflightCompression:
for ev, msg in status_messages
)
def test_preflight_defers_when_recent_real_usage_fit(self, agent):
"""A noisy rough estimate should not re-compact a recently fitting request."""
agent.compression_enabled = True
agent.context_compressor.context_length = 200_000
agent.context_compressor.threshold_tokens = 100_000
agent.context_compressor.last_prompt_tokens = 58_000
agent.context_compressor.last_real_prompt_tokens = 58_000
agent.context_compressor.last_rough_tokens_when_real_prompt_fit = 113_000
big_history = []
for i in range(20):
big_history.append({"role": "user", "content": f"Message {i} padded"})
big_history.append({"role": "assistant", "content": f"Response {i} padded"})
ok_resp = _mock_response(
content="Used real fit",
finish_reason="stop",
usage={"prompt_tokens": 59_000, "completion_tokens": 100, "total_tokens": 59_100},
)
agent.client.chat.completions.create.side_effect = [ok_resp]
status_messages = []
agent.status_callback = lambda ev, msg: status_messages.append((ev, msg))
with (
patch("agent.conversation_loop.estimate_request_tokens_rough", return_value=114_000),
patch.object(agent, "_compress_context") as mock_compress,
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
result = agent.run_conversation("hello", conversation_history=big_history)
mock_compress.assert_not_called()
assert result["completed"] is True
assert result["final_response"] == "Used real fit"
assert not any(
ev == "lifecycle" and "Preflight compression" in msg
for ev, msg in status_messages
)
def test_preflight_compresses_when_rough_growth_after_fit_is_large(self, agent):
"""Large rough growth after a fitting request still triggers preflight."""
agent.compression_enabled = True
agent.context_compressor.context_length = 200_000
agent.context_compressor.threshold_tokens = 100_000
agent.context_compressor.last_prompt_tokens = 58_000
agent.context_compressor.last_real_prompt_tokens = 58_000
agent.context_compressor.last_rough_tokens_when_real_prompt_fit = 113_000
big_history = []
for i in range(20):
big_history.append({"role": "user", "content": f"Message {i} padded"})
big_history.append({"role": "assistant", "content": f"Response {i} padded"})
ok_resp = _mock_response(
content="Compressed after growth",
finish_reason="stop",
usage={"prompt_tokens": 50_000, "completion_tokens": 100, "total_tokens": 50_100},
)
agent.client.chat.completions.create.side_effect = [ok_resp]
# First rough estimate must clear the threshold so preflight fires
# (rough growth since the last fitting request is large, so the
# deferral path is NOT taken). Every estimate after compaction is
# sub-threshold. Use a callable side_effect rather than a fixed list
# so we don't have to predict how many times the loop re-estimates —
# the post-response real-token estimate is an extra call that a
# 2-element list would exhaust (StopIteration).
_rough_calls = {"n": 0}
def _rough_estimate(*_args, **_kwargs):
_rough_calls["n"] += 1
return 125_000 if _rough_calls["n"] == 1 else 40_000
with (
patch("agent.conversation_loop.estimate_request_tokens_rough", side_effect=_rough_estimate),
patch.object(agent, "_compress_context") as mock_compress,
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
mock_compress.return_value = (
[{"role": "user", "content": f"{SUMMARY_PREFIX}\nPrevious conversation"}],
"new system prompt",
)
result = agent.run_conversation("hello", conversation_history=big_history)
mock_compress.assert_called_once()
assert result["completed"] is True
def test_no_preflight_when_under_threshold(self, agent):
"""When history fits within context, no preflight compression needed."""
agent.compression_enabled = True
@@ -575,6 +665,74 @@ class TestPreflightCompression:
mock_compress.assert_not_called()
assert result["completed"] is True
def test_preflight_seeds_display_tokens_when_compression_aborts(self, agent):
"""Display must reflect the real context size even when compression no-ops.
Regression: the CLI status bar reads ``last_prompt_tokens``, which only
updated from a *successful* API response. When the loaded history was
oversized but compression failed to reduce it (e.g. the auxiliary
summary model timed out), the bar stayed stuck at the old, smaller
value while the preflight estimate reported a much larger number —
looking permanently out of sync.
"""
agent.compression_enabled = True
agent.context_compressor.context_length = 200_000
agent.context_compressor.threshold_tokens = 130_000
# Simulate a stale display value from an earlier, smaller turn.
agent.context_compressor.last_prompt_tokens = 74_400
big_history = []
for i in range(20):
big_history.append({"role": "user", "content": f"Message {i} padded text"})
big_history.append({"role": "assistant", "content": f"Response {i} padded text"})
ok_resp = _mock_response(content="After preflight", finish_reason="stop")
agent.client.chat.completions.create.side_effect = [ok_resp]
with (
patch("agent.conversation_loop.estimate_request_tokens_rough", return_value=144_669),
# Compression no-ops (returns input unchanged) — mirrors an aux
# summary-model timeout where the messages can't be reduced.
patch.object(agent, "_compress_context", side_effect=lambda msgs, *a, **k: (msgs, agent._cached_system_prompt)),
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
result = agent.run_conversation("hello", conversation_history=big_history)
assert result["completed"] is True
# The display token count was revised up to the fresh preflight estimate,
# not left at the stale 74_400.
assert agent.context_compressor.last_prompt_tokens == 144_669
def test_preflight_seed_only_revises_upward(self, agent):
"""A larger tracked value must not be clobbered by a smaller estimate."""
agent.compression_enabled = True
agent.context_compressor.context_length = 200_000
agent.context_compressor.threshold_tokens = 130_000
# A real, larger usage figure is already tracked.
agent.context_compressor.last_prompt_tokens = 160_000
big_history = []
for i in range(20):
big_history.append({"role": "user", "content": f"Message {i} padded text"})
big_history.append({"role": "assistant", "content": f"Response {i} padded text"})
ok_resp = _mock_response(content="After preflight", finish_reason="stop")
agent.client.chat.completions.create.side_effect = [ok_resp]
with (
patch("agent.conversation_loop.estimate_request_tokens_rough", return_value=144_669),
patch.object(agent, "_compress_context", side_effect=lambda msgs, *a, **k: (msgs, agent._cached_system_prompt)),
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
agent.run_conversation("hello", conversation_history=big_history)
# Smaller estimate must not overwrite the larger tracked value.
assert agent.context_compressor.last_prompt_tokens == 160_000
class TestToolResultPreflightCompression:
"""Compression should trigger when tool results push context past the threshold."""
+6 -1
View File
@@ -70,4 +70,9 @@ def test_tool_call_validation_accepts_dict_arguments(monkeypatch):
result = agent.run_conversation("read the file")
assert result["final_response"] == "done"
# The conversation hits max_iterations=3 (3 tool turns then forced summary).
# PR #34470 adds an explainer suffix to abnormal turn endings so users
# understand why the response is short instead of seeing a blank reply.
# The exact suffix wording is owned by conversation_loop; this test only
# cares that the model's actual text ('done') survives at the start.
assert result["final_response"].startswith("done")
+168 -5
View File
@@ -2543,6 +2543,122 @@ class TestConcurrentToolExecution:
assert json.loads(result) == {"error": "Blocked"}
assert agent._turns_since_memory == 5
def test_concurrent_blocked_write_skips_checkpoint(self, agent, monkeypatch):
"""Concurrent path: blocked write_file should not trigger checkpoint."""
tc1 = _mock_tool_call(name="write_file",
arguments='{"path":"test.txt","content":"hello"}',
call_id="c1")
tc2 = _mock_tool_call(name="read_file",
arguments='{"path":"other.py"}',
call_id="c2")
mock_msg = _mock_assistant_msg(content="", tool_calls=[tc1, tc2])
messages = []
monkeypatch.setattr(
"hermes_cli.plugins.get_pre_tool_call_block_message",
lambda *args, **kwargs: "Blocked" if args[0] == "write_file" else None,
)
agent._checkpoint_mgr.enabled = True
def fake_handle(name, args, task_id, **kwargs):
return f"result_{name}"
with patch("run_agent.handle_function_call", side_effect=fake_handle):
with patch.object(agent._checkpoint_mgr, "ensure_checkpoint") as cp_mock:
agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1")
cp_mock.assert_not_called()
def test_concurrent_blocked_patch_skips_checkpoint(self, agent, monkeypatch):
"""Concurrent path: blocked patch should not trigger checkpoint."""
tc1 = _mock_tool_call(name="patch",
arguments='{"path":"f.py","old":"a","new":"b"}',
call_id="c1")
tc2 = _mock_tool_call(name="read_file",
arguments='{"path":"other.py"}',
call_id="c2")
mock_msg = _mock_assistant_msg(content="", tool_calls=[tc1, tc2])
messages = []
monkeypatch.setattr(
"hermes_cli.plugins.get_pre_tool_call_block_message",
lambda *args, **kwargs: "Blocked" if args[0] == "patch" else None,
)
agent._checkpoint_mgr.enabled = True
def fake_handle(name, args, task_id, **kwargs):
return f"result_{name}"
with patch("run_agent.handle_function_call", side_effect=fake_handle):
with patch.object(agent._checkpoint_mgr, "ensure_checkpoint") as cp_mock:
agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1")
cp_mock.assert_not_called()
def test_concurrent_blocked_terminal_skips_checkpoint(self, agent, monkeypatch):
"""Concurrent path: blocked terminal should not trigger checkpoint."""
tc1 = _mock_tool_call(name="terminal",
arguments='{"command":"rm -rf /tmp/foo"}',
call_id="c1")
tc2 = _mock_tool_call(name="read_file",
arguments='{"path":"other.py"}',
call_id="c2")
mock_msg = _mock_assistant_msg(content="", tool_calls=[tc1, tc2])
messages = []
monkeypatch.setattr(
"hermes_cli.plugins.get_pre_tool_call_block_message",
lambda *args, **kwargs: "Blocked" if args[0] == "terminal" else None,
)
agent._checkpoint_mgr.enabled = True
def fake_handle(name, args, task_id, **kwargs):
return f"result_{name}"
with patch("run_agent.handle_function_call", side_effect=fake_handle):
with patch.object(agent._checkpoint_mgr, "ensure_checkpoint") as cp_mock:
with patch("agent.tool_executor._is_destructive_command", return_value=True):
agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1")
cp_mock.assert_not_called()
def test_concurrent_blocked_write_does_not_steal_slot_from_allowed_write(self, agent, monkeypatch):
"""When write_file is blocked, its dedup slot must not be consumed,
so a subsequent allowed write_file for the same path still checkpoints."""
tc1 = _mock_tool_call(name="write_file",
arguments='{"path":"dup.txt","content":"blocked"}',
call_id="c1")
tc2 = _mock_tool_call(name="write_file",
arguments='{"path":"dup.txt","content":"allowed"}',
call_id="c2")
mock_msg = _mock_assistant_msg(content="", tool_calls=[tc1, tc2])
messages = []
call_count = {"n": 0}
def block_first_only(*args, **kwargs):
call_count["n"] += 1
return "Blocked" if call_count["n"] == 1 else None
monkeypatch.setattr(
"hermes_cli.plugins.get_pre_tool_call_block_message",
block_first_only,
)
agent._checkpoint_mgr.enabled = True
def fake_handle(name, args, task_id, **kwargs):
return f"result_{name}"
with patch("run_agent.handle_function_call", side_effect=fake_handle):
with patch.object(agent._checkpoint_mgr, "ensure_checkpoint") as cp_mock:
agent._execute_tool_calls_concurrent(mock_msg, messages, "task-1")
# Second (allowed) write must checkpoint even though first was blocked.
cp_mock.assert_called_once()
class TestPathsOverlap:
"""Unit tests for the _paths_overlap helper."""
@@ -2756,6 +2872,40 @@ class TestHandleMaxIterations:
]
assert len(stub_ids) >= 1, f"No stub result for assistant tool_call: {stub_ids}"
def test_summary_strips_strict_schema_foreign_fields(self, agent):
"""Regression: the max-iterations summary request must NOT carry
Chat-Completions-schema-foreign keys — tool_name (SQLite FTS
bookkeeping), codex_* reasoning carriers, or internal _-prefixed
scaffolding. Strict gateways (Fireworks-backed OpenCode Go, Mistral,
Kimi) reject these with 'Extra inputs are not permitted, field:
messages[N].tool_name'. The transport's convert_messages() strips
them on the main loop; this hand-built summary path must mirror it."""
agent.client.chat.completions.create.return_value = _mock_response(content="Summary")
agent._cached_system_prompt = "You are helpful."
messages = [
{"role": "user", "content": "do stuff"},
{
"role": "assistant",
"tool_calls": [{"id": "call_1", "function": {"name": "execute_code", "arguments": "{}"}}],
"codex_reasoning_items": [{"id": "rs_1"}],
},
{"role": "tool", "tool_call_id": "call_1", "content": "result", "tool_name": "execute_code"},
{"role": "assistant", "content": "Done.", "_empty_recovery_synthetic": True},
]
result = agent._handle_max_iterations(messages, 60)
assert result == "Summary"
sent_msgs = agent.client.chat.completions.create.call_args.kwargs.get("messages", [])
for m in sent_msgs:
assert "tool_name" not in m, m
assert "codex_reasoning_items" not in m, m
assert "codex_message_items" not in m, m
assert not any(isinstance(k, str) and k.startswith("_") for k in m), m
# Internal history is untouched — the path copies each message.
assert messages[2]["tool_name"] == "execute_code"
assert messages[1]["codex_reasoning_items"] == [{"id": "rs_1"}]
def test_summary_omits_provider_preferences_for_non_openrouter(self, agent):
agent.base_url = "https://api.openai.com/v1"
agent._base_url_lower = agent.base_url.lower()
@@ -3046,7 +3196,11 @@ class TestRunConversation:
mock_compress.assert_not_called() # no compression triggered
assert result["completed"] is True
assert result["final_response"] == "(empty)"
# #34452: the bare "(empty)" sentinel is now replaced by a
# user-visible end-of-turn explanation so the failure isn't silent.
assert result["final_response"] != "(empty)"
assert "No reply:" in result["final_response"]
assert result["turn_exit_reason"] == "empty_response_exhausted"
assert result["api_calls"] == 6 # 1 original + 2 prefill + 3 retries
def test_reasoning_only_response_prefill_then_empty(self, agent):
@@ -3066,7 +3220,9 @@ class TestRunConversation:
):
result = agent.run_conversation("answer me")
assert result["completed"] is True
assert result["final_response"] == "(empty)"
# #34452: explanation replaces the bare "(empty)" sentinel.
assert result["final_response"] != "(empty)"
assert "No reply:" in result["final_response"]
assert result["api_calls"] == 6 # 1 original + 2 prefill + 3 retries
def test_reasoning_only_prefill_succeeds_on_continuation(self, agent):
@@ -3113,7 +3269,9 @@ class TestRunConversation:
):
result = agent.run_conversation("answer me")
assert result["completed"] is True
assert result["final_response"] == "(empty)"
# #34452: explanation replaces the bare "(empty)" sentinel.
assert result["final_response"] != "(empty)"
assert "No reply:" in result["final_response"]
assert result["api_calls"] == 4 # 1 original + 3 retries
def test_truly_empty_response_succeeds_on_nudge(self, agent):
@@ -3209,7 +3367,9 @@ class TestRunConversation:
):
result = agent.run_conversation("answer me")
assert result["completed"] is True
assert result["final_response"] == "(empty)"
# #34452: explanation replaces the bare "(empty)" sentinel.
assert result["final_response"] != "(empty)"
assert "No reply:" in result["final_response"]
def test_empty_response_emits_status_for_gateway(self, agent):
"""_emit_status is called during empty retries so gateway users see feedback."""
@@ -3235,7 +3395,10 @@ class TestRunConversation:
):
result = agent.run_conversation("answer me")
assert result["final_response"] == "(empty)"
# #34452: explanation replaces the bare "(empty)" sentinel, but the
# status emissions during retries are unchanged.
assert result["final_response"] != "(empty)"
assert "No reply:" in result["final_response"]
# Should have emitted retry statuses (3 retries) + final failure
retry_msgs = [m for m in status_messages if "retrying" in m.lower()]
assert len(retry_msgs) == 3, f"Expected 3 retry status messages, got {len(retry_msgs)}: {status_messages}"
@@ -0,0 +1,181 @@
"""Tests for the end-of-turn completion explainer (#34452).
When a turn ends abnormally after tools (empty content after retries, a
partial/truncated stream, exhausted retries, or an iteration/budget limit)
the user should get a single user-visible explanation of why the reply
stopped instead of a blank or fragmentary response box. Normal short
replies (e.g. ``Done.``) must stay quiet.
These tests exercise:
1. ``_format_turn_completion_explanation`` the pure reasonmessage map.
2. ``_turn_completion_explainer_enabled`` the env/config seam.
3. An end-to-end ``run_conversation`` turn that exhausts empty-response
retries and verifies the explanation reaches ``final_response``.
All assertions work under the mocked OpenAI SDK used elsewhere in this
suite (we patch ``run_agent.OpenAI`` and drive ``agent.client``), so they
pass identically in CI and locally.
"""
import os
import uuid
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from run_agent import AIAgent
# --------------------------------------------------------------------------
# Fixtures (mirrors tests/run_agent/test_tool_call_guardrail_runtime.py)
# --------------------------------------------------------------------------
def _mock_response(content="Hello", finish_reason="stop", tool_calls=None):
msg = SimpleNamespace(content=content, tool_calls=tool_calls)
choice = SimpleNamespace(message=msg, finish_reason=finish_reason)
return SimpleNamespace(choices=[choice], model="test/model", usage=None)
def _make_agent(max_iterations: int = 10, config: dict | None = None) -> AIAgent:
with (
patch("run_agent.get_tool_definitions", return_value=[]),
patch("run_agent.check_toolset_requirements", return_value={}),
patch("hermes_cli.config.load_config", return_value=config or {}),
patch("run_agent.OpenAI"),
):
agent = AIAgent(
api_key="test-key-1234567890",
base_url="https://openrouter.ai/api/v1",
max_iterations=max_iterations,
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
agent.client = MagicMock()
agent._cached_system_prompt = "You are helpful."
agent._use_prompt_caching = False
agent.tool_delay = 0
agent.compression_enabled = False
agent.save_trajectories = False
# No fallback chain so empty responses exhaust deterministically.
agent._fallback_chain = []
return agent
# --------------------------------------------------------------------------
# 1. Pure formatter
# --------------------------------------------------------------------------
def test_explanation_quiet_for_normal_text_response():
"""A healthy text_response exit must NOT produce any explanation."""
out = AIAgent._format_turn_completion_explanation(
"text_response(finish_reason=stop)"
)
assert out == ""
def test_explanation_quiet_for_empty_reason():
assert AIAgent._format_turn_completion_explanation("") == ""
assert AIAgent._format_turn_completion_explanation("unknown") == ""
# guardrail_halt surfaces its own message; explainer stays out of the way.
assert AIAgent._format_turn_completion_explanation("guardrail_halt") == ""
def test_explanation_for_empty_response_exhausted():
out = AIAgent._format_turn_completion_explanation("empty_response_exhausted")
assert out # non-empty
assert "empty content" in out
assert "continue" in out.lower()
def test_explanation_for_partial_stream_recovery():
out = AIAgent._format_turn_completion_explanation("partial_stream_recovery")
assert "partial" in out.lower()
assert "continue" in out.lower()
def test_explanation_for_max_iterations_reached_prefix_match():
"""``max_iterations_reached(...)`` carries a parenthetical suffix."""
out = AIAgent._format_turn_completion_explanation(
"max_iterations_reached(10/10)"
)
assert "iteration" in out.lower()
def test_explanation_for_all_retries_exhausted():
out = AIAgent._format_turn_completion_explanation(
"all_retries_exhausted_no_response"
)
assert "retries" in out.lower()
# --------------------------------------------------------------------------
# 2. Enable/disable seam
# --------------------------------------------------------------------------
def test_explainer_enabled_by_default():
agent = _make_agent()
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("HERMES_TURN_COMPLETION_EXPLAINER", None)
with patch("hermes_cli.config.load_config", return_value={}):
assert agent._turn_completion_explainer_enabled() is True
def test_explainer_disabled_via_env():
agent = _make_agent()
with patch.dict(
os.environ, {"HERMES_TURN_COMPLETION_EXPLAINER": "0"}, clear=False
):
assert agent._turn_completion_explainer_enabled() is False
def test_explainer_disabled_via_config():
agent = _make_agent()
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("HERMES_TURN_COMPLETION_EXPLAINER", None)
with patch(
"hermes_cli.config.load_config",
return_value={"display": {"turn_completion_explainer": False}},
):
assert agent._turn_completion_explainer_enabled() is False
# --------------------------------------------------------------------------
# 3. End-to-end: empty-response exhaustion surfaces the explanation
# --------------------------------------------------------------------------
def test_run_conversation_empty_exhausted_surfaces_explanation():
"""Four empty responses in a row should exhaust retries and the final
response should be the actionable explanation, not a bare '(empty)'."""
agent = _make_agent(max_iterations=10)
# 4 empty responses: retries 1..3 then the terminal on the 4th.
agent.client.chat.completions.create.side_effect = [
_mock_response(content="", finish_reason="stop") for _ in range(8)
]
with (
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
result = agent.run_conversation("do something")
assert result["turn_exit_reason"] == "empty_response_exhausted"
# The user must NOT be left with a bare sentinel; the explanation wins.
assert result["final_response"] != "(empty)"
assert result["final_response"].strip() != ""
assert "No reply:" in result["final_response"]
def test_run_conversation_normal_reply_stays_quiet():
"""A normal short reply like 'Done.' must NOT get an explainer footer."""
agent = _make_agent(max_iterations=10)
agent.client.chat.completions.create.side_effect = [
_mock_response(content="Done.", finish_reason="stop"),
]
with (
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
result = agent.run_conversation("do something")
assert result["turn_exit_reason"].startswith("text_response")
assert result["final_response"] == "Done."
assert "No reply:" not in result["final_response"]