Merge main into bb/gui.

Resolve merge conflicts while preserving bb/gui dashboard paths and STT provider support.
This commit is contained in:
Brooklyn Nicholson
2026-05-15 15:33:28 -05:00
415 changed files with 38391 additions and 20402 deletions
+47 -1
View File
@@ -1,6 +1,11 @@
"""Tests for acp_adapter.auth — provider detection."""
from acp_adapter.auth import has_provider, detect_provider
from acp_adapter.auth import (
TERMINAL_SETUP_AUTH_METHOD_ID,
build_auth_methods,
has_provider,
detect_provider,
)
class TestHasProvider:
@@ -54,3 +59,44 @@ class TestDetectProvider:
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _boom)
assert detect_provider() is None
def test_detect_provider_strips_and_lowercases_provider(self, monkeypatch):
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda: {"provider": " OpenRouter ", "api_key": " sk-or-test "},
)
assert detect_provider() == "openrouter"
class TestBuildAuthMethods:
def test_build_auth_methods_returns_provider_and_terminal_when_configured(self, monkeypatch):
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: "openrouter")
methods = build_auth_methods()
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in methods]
assert payloads[0]["id"] == "openrouter"
assert payloads[0]["name"] == "openrouter runtime credentials"
assert any(payload["id"] == TERMINAL_SETUP_AUTH_METHOD_ID for payload in payloads)
terminal = next(payload for payload in payloads if payload["id"] == TERMINAL_SETUP_AUTH_METHOD_ID)
assert terminal["type"] == "terminal"
assert terminal["args"] == ["--setup"]
def test_build_auth_methods_returns_terminal_setup_when_unconfigured(self, monkeypatch):
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: None)
methods = build_auth_methods()
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in methods]
assert payloads == [
{
"args": ["--setup"],
"description": (
"Open Hermes' interactive model/provider setup in a terminal. "
"Use this when Hermes has not been configured on this machine yet."
),
"id": TERMINAL_SETUP_AUTH_METHOD_ID,
"name": "Configure Hermes provider",
"type": "terminal",
}
]
+177 -1
View File
@@ -1,6 +1,9 @@
"""Tests for acp_adapter.entry startup wiring."""
import sys
import acp
import pytest
from acp_adapter import entry
@@ -15,6 +18,179 @@ def test_main_enables_unstable_protocol(monkeypatch):
monkeypatch.setattr(entry, "_load_env", lambda: None)
monkeypatch.setattr(acp, "run_agent", fake_run_agent)
entry.main()
entry.main([])
assert calls["kwargs"]["use_unstable_protocol"] is True
def test_main_version_prints_without_starting_server(monkeypatch, capsys):
monkeypatch.setattr(entry, "_setup_logging", lambda: (_ for _ in ()).throw(AssertionError("started server")))
entry.main(["--version"])
output = capsys.readouterr().out.strip()
assert output
assert "Starting hermes-agent ACP adapter" not in output
def test_main_check_prints_ok_without_starting_server(monkeypatch, capsys):
monkeypatch.setattr(entry, "_setup_logging", lambda: (_ for _ in ()).throw(AssertionError("started server")))
entry.main(["--check"])
assert capsys.readouterr().out.strip() == "Hermes ACP check OK"
def test_main_setup_runs_model_configuration(monkeypatch):
calls = {}
def fake_hermes_main():
calls["argv"] = sys.argv[:]
monkeypatch.setattr("hermes_cli.main.main", fake_hermes_main)
# Pretend stdin is not a TTY so the follow-up browser prompt is skipped.
# That keeps this test focused on the model-setup wiring; the
# browser-prompt path has its own test below.
monkeypatch.setattr("sys.stdin.isatty", lambda: False)
entry.main(["--setup"])
assert calls["argv"][1:] == ["model"]
def test_main_setup_offers_browser_install_when_tty(monkeypatch):
"""When stdin is a TTY and the user answers yes, model setup is followed
by a browser-tools bootstrap call."""
monkeypatch.setattr("hermes_cli.main.main", lambda: None)
monkeypatch.setattr("sys.stdin.isatty", lambda: True)
monkeypatch.setattr("builtins.input", lambda *_args, **_kwargs: "y")
bootstrap_calls = []
monkeypatch.setattr(
entry,
"_run_setup_browser",
lambda assume_yes=False: bootstrap_calls.append(assume_yes) or 0,
)
entry.main(["--setup"])
assert bootstrap_calls == [False]
def test_main_setup_skips_browser_prompt_on_no(monkeypatch):
monkeypatch.setattr("hermes_cli.main.main", lambda: None)
monkeypatch.setattr("sys.stdin.isatty", lambda: True)
monkeypatch.setattr("builtins.input", lambda *_args, **_kwargs: "")
called = []
monkeypatch.setattr(
entry,
"_run_setup_browser",
lambda assume_yes=False: called.append(assume_yes) or 0,
)
entry.main(["--setup"])
assert called == []
def test_main_setup_browser_invokes_bundled_script(monkeypatch):
"""`hermes-acp --setup-browser` must shell out to the bundled bootstrap
script — never reimplement the install logic inline."""
monkeypatch.setattr("platform.system", lambda: "Linux")
captured = {}
def fake_run(cmd, check=False):
captured["cmd"] = cmd
class _R:
returncode = 0
return _R()
monkeypatch.setattr("subprocess.run", fake_run)
entry.main(["--setup-browser"])
assert captured["cmd"][0] == "bash"
assert captured["cmd"][1].endswith("bootstrap_browser_tools.sh")
# --yes is NOT passed when the flag is absent.
assert "--yes" not in captured["cmd"]
def test_main_setup_browser_forwards_yes_flag(monkeypatch):
monkeypatch.setattr("platform.system", lambda: "Linux")
captured = {}
def fake_run(cmd, check=False):
captured["cmd"] = cmd
class _R:
returncode = 0
return _R()
monkeypatch.setattr("subprocess.run", fake_run)
entry.main(["--setup-browser", "--yes"])
assert "--yes" in captured["cmd"]
def test_main_setup_browser_uses_powershell_on_windows(monkeypatch):
monkeypatch.setattr("platform.system", lambda: "Windows")
captured = {}
def fake_run(cmd, check=False):
captured["cmd"] = cmd
class _R:
returncode = 0
return _R()
monkeypatch.setattr("subprocess.run", fake_run)
entry.main(["--setup-browser", "--yes"])
assert captured["cmd"][0] == "powershell.exe"
assert any(part.endswith("bootstrap_browser_tools.ps1") for part in captured["cmd"])
assert "-Yes" in captured["cmd"]
def test_main_setup_browser_propagates_failure(monkeypatch):
monkeypatch.setattr("platform.system", lambda: "Linux")
class _R:
returncode = 7
monkeypatch.setattr("subprocess.run", lambda cmd, check=False: _R())
with pytest.raises(SystemExit) as excinfo:
entry.main(["--setup-browser"])
assert excinfo.value.code == 7
def test_bootstrap_scripts_ship_with_package():
"""The package-data wiring (pyproject.toml) must include the bootstrap
scripts — otherwise `--setup-browser` 404s at runtime."""
from pathlib import Path
bootstrap_dir = Path(entry.__file__).resolve().parent / "bootstrap"
sh = bootstrap_dir / "bootstrap_browser_tools.sh"
ps1 = bootstrap_dir / "bootstrap_browser_tools.ps1"
assert sh.is_file(), f"missing bundled script: {sh}"
assert ps1.is_file(), f"missing bundled script: {ps1}"
sh_text = sh.read_text(encoding="utf-8")
ps1_text = ps1.read_text(encoding="utf-8")
# Sanity: scripts know how to find the Hermes-managed Node prefix.
assert "HERMES_HOME" in sh_text
assert "agent-browser" in sh_text
assert "HermesHome" in ps1_text
assert "agent-browser" in ps1_text
+127 -48
View File
@@ -1,89 +1,168 @@
"""Tests for acp_adapter.permissions — ACP approval bridging."""
"""Tests for acp_adapter.permissions."""
import asyncio
import inspect
from concurrent.futures import Future
from unittest.mock import MagicMock, patch
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from acp.schema import (
AllowedOutcome,
DeniedOutcome,
RequestPermissionResponse,
)
from acp_adapter.permissions import make_approval_callback
from tools.approval import prompt_dangerous_approval
def _make_response(outcome):
"""Helper to build a RequestPermissionResponse with the given outcome."""
return RequestPermissionResponse(outcome=outcome)
def _setup_callback(outcome, timeout=60.0):
"""
Create a callback wired to a mock request_permission coroutine
that resolves to the given outcome.
Returns:
(callback, mock_request_permission_fn)
"""
def _invoke_callback(
outcome,
*,
allow_permanent=True,
timeout=60.0,
use_prompt_path=False,
):
loop = MagicMock(spec=asyncio.AbstractEventLoop)
mock_rp = MagicMock(name="request_permission")
response = _make_response(outcome)
# Patch asyncio.run_coroutine_threadsafe so it returns a future
# that immediately yields the response.
request_permission = AsyncMock(name="request_permission")
future = MagicMock(spec=Future)
future.result.return_value = response
future.result.return_value = _make_response(outcome)
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", return_value=future):
cb = make_approval_callback(mock_rp, loop, session_id="s1", timeout=timeout)
result = cb("rm -rf /", "dangerous command")
scheduled = {}
return result
def _schedule(coro, passed_loop):
scheduled["coro"] = coro
scheduled["loop"] = passed_loop
return future
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=timeout)
if use_prompt_path:
result = prompt_dangerous_approval(
"rm -rf /",
"dangerous command",
allow_permanent=allow_permanent,
approval_callback=cb,
)
else:
result = cb(
"rm -rf /",
"dangerous command",
allow_permanent=allow_permanent,
)
scheduled["coro"].close()
_, kwargs = request_permission.call_args
return result, kwargs, scheduled, future, loop
class TestApprovalMapping:
def test_approval_allow_once_maps_correctly(self):
outcome = AllowedOutcome(option_id="allow_once", outcome="selected")
result = _setup_callback(outcome)
class TestApprovalBridge:
def test_bridge_schedules_request_on_the_given_loop(self):
result, kwargs, scheduled, _, loop = _invoke_callback(
AllowedOutcome(option_id="allow_once", outcome="selected"),
)
tool_call = kwargs["tool_call"]
option_ids = [option.option_id for option in kwargs["options"]]
assert result == "once"
assert scheduled["loop"] is loop
assert inspect.iscoroutine(scheduled["coro"])
assert kwargs["session_id"] == "s1"
assert tool_call.session_update == "tool_call_update"
assert tool_call.tool_call_id.startswith("perm-check-")
assert tool_call.kind == "execute"
assert tool_call.status == "pending"
assert tool_call.title == "dangerous command"
assert tool_call.raw_input == {
"command": "rm -rf /",
"description": "dangerous command",
}
assert option_ids == ["allow_once", "allow_session", "allow_always", "deny"]
def test_tool_call_ids_are_unique(self):
_, first_kwargs, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="allow_once", outcome="selected"),
)
_, second_kwargs, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="allow_once", outcome="selected"),
)
assert first_kwargs["tool_call"].tool_call_id != second_kwargs["tool_call"].tool_call_id
def test_prompt_path_keeps_session_option_when_permanent_disabled(self):
result, kwargs, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="allow_session", outcome="selected"),
allow_permanent=False,
use_prompt_path=True,
)
option_ids = [option.option_id for option in kwargs["options"]]
assert result == "session"
assert option_ids == ["allow_once", "allow_session", "deny"]
def test_allow_always_maps_correctly(self):
result, _, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="allow_always", outcome="selected"),
use_prompt_path=True,
)
def test_approval_allow_always_maps_correctly(self):
outcome = AllowedOutcome(option_id="allow_always", outcome="selected")
result = _setup_callback(outcome)
assert result == "always"
def test_approval_deny_maps_correctly(self):
outcome = DeniedOutcome(outcome="cancelled")
result = _setup_callback(outcome)
assert result == "deny"
def test_denied_and_unknown_outcomes_deny(self):
denied_result, _, _, _, _ = _invoke_callback(DeniedOutcome(outcome="cancelled"))
unknown_result, _, _, _, _ = _invoke_callback(
AllowedOutcome(option_id="unexpected", outcome="selected"),
)
def test_approval_timeout_returns_deny(self):
"""When the future times out, the callback should return 'deny'."""
assert denied_result == "deny"
assert unknown_result == "deny"
def test_timeout_returns_deny_and_cancels_future(self):
loop = MagicMock(spec=asyncio.AbstractEventLoop)
mock_rp = MagicMock(name="request_permission")
request_permission = AsyncMock(name="request_permission")
future = MagicMock(spec=Future)
future.result.side_effect = TimeoutError("timed out")
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", return_value=future):
cb = make_approval_callback(mock_rp, loop, session_id="s1", timeout=0.01)
result = cb("rm -rf /", "dangerous")
scheduled = {}
def _schedule(coro, passed_loop):
scheduled["coro"] = coro
scheduled["loop"] = passed_loop
return future
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=0.01)
result = cb("rm -rf /", "dangerous command")
scheduled["coro"].close()
assert result == "deny"
assert scheduled["loop"] is loop
assert future.cancel.call_count == 1
def test_approval_none_response_returns_deny(self):
"""When request_permission resolves to None, the callback should return 'deny'."""
def test_none_response_returns_deny(self):
"""When request_permission resolves to None, the callback returns 'deny'."""
loop = MagicMock(spec=asyncio.AbstractEventLoop)
mock_rp = MagicMock(name="request_permission")
request_permission = AsyncMock(name="request_permission")
future = MagicMock(spec=Future)
future.result.return_value = None
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", return_value=future):
cb = make_approval_callback(mock_rp, loop, session_id="s1", timeout=1.0)
scheduled = {}
def _schedule(coro, passed_loop):
scheduled["coro"] = coro
scheduled["loop"] = passed_loop
return future
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=1.0)
result = cb("echo hi", "demo")
scheduled["coro"].close()
assert result == "deny"
+90
View File
@@ -0,0 +1,90 @@
"""Tests for ACP Registry metadata shipped with Hermes."""
from __future__ import annotations
import json
import re
import tomllib
from pathlib import Path
import xml.etree.ElementTree as ET
ROOT = Path(__file__).resolve().parents[2]
MANIFEST = ROOT / "acp_registry" / "agent.json"
ICON = ROOT / "acp_registry" / "icon.svg"
FORBIDDEN_MANIFEST_KEYS = {"schema_version", "display_name"}
ALLOWED_DISTRIBUTIONS = {"binary", "npx", "uvx"}
def _manifest() -> dict:
return json.loads(MANIFEST.read_text(encoding="utf-8"))
def _pyproject_version() -> str:
data = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
return data["project"]["version"]
def test_agent_json_matches_official_registry_required_fields():
data = _manifest()
assert FORBIDDEN_MANIFEST_KEYS.isdisjoint(data)
assert data["id"] == "hermes-agent"
assert re.fullmatch(r"[a-z][a-z0-9-]*", data["id"])
assert data["name"] == "Hermes Agent"
assert data["description"]
assert data["repository"] == "https://github.com/NousResearch/hermes-agent"
assert data["website"].startswith("https://hermes-agent.nousresearch.com/")
assert data["authors"] == ["Nous Research"]
assert data["license"] == "MIT"
assert set(data["distribution"]) <= ALLOWED_DISTRIBUTIONS
def test_agent_json_uses_uvx_distribution_without_local_command_fields():
data = _manifest()
assert set(data["distribution"]) == {"uvx"}
uvx = data["distribution"]["uvx"]
# Schema allows {package, args, env}; we use {package, args}.
assert set(uvx) <= {"package", "args", "env"}
assert "package" in uvx
assert uvx["package"] == f"hermes-agent[acp]=={data['version']}"
assert uvx["args"] == ["hermes-acp"]
# Old command-shape fields must not leak back in.
assert "type" not in data["distribution"]
assert "command" not in data["distribution"]
def test_agent_json_version_matches_pyproject():
assert _manifest()["version"] == _pyproject_version()
def test_agent_json_pins_uvx_package_to_pyproject_version():
"""The registry CI rejects ``@latest`` and floating pins; the manifest must
always reference the exact PyPI version listed in pyproject.toml."""
assert _manifest()["distribution"]["uvx"]["package"] == (
f"hermes-agent[acp]=={_pyproject_version()}"
)
def test_icon_svg_is_16x16_current_color():
root = ET.fromstring(ICON.read_text(encoding="utf-8"))
assert root.attrib["viewBox"] == "0 0 16 16"
assert root.attrib["width"] == "16"
assert root.attrib["height"] == "16"
def test_icon_svg_has_no_hardcoded_colors_or_gradients():
text = ICON.read_text(encoding="utf-8")
assert "linearGradient" not in text
assert "radialGradient" not in text
assert "url(#" not in text
assert not re.search(r"#[0-9a-fA-F]{3,8}\b", text)
root = ET.fromstring(text)
for element in root.iter():
for attr in ("fill", "stroke"):
value = element.attrib.get(attr)
if value is not None:
assert value in {"currentColor", "none"}
+54
View File
@@ -33,6 +33,7 @@ from acp.schema import (
UsageUpdate,
UserMessageChunk,
)
from acp_adapter.auth import TERMINAL_SETUP_AUTH_METHOD_ID
from acp_adapter.server import HermesACPAgent, HERMES_VERSION
from acp_adapter.session import SessionManager
from hermes_state import SessionDB
@@ -92,6 +93,41 @@ class TestInitialize:
assert "list" in session_caps
assert "resume" in session_caps
@pytest.mark.asyncio
async def test_initialize_advertises_provider_and_terminal_auth_methods(self, agent, monkeypatch):
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: "openrouter")
monkeypatch.setattr("acp_adapter.server.detect_provider", lambda: "openrouter")
resp = await agent.initialize(protocol_version=1)
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in resp.auth_methods]
assert payloads[0]["id"] == "openrouter"
assert payloads[0]["name"] == "openrouter runtime credentials"
terminal = next(payload for payload in payloads if payload["id"] == TERMINAL_SETUP_AUTH_METHOD_ID)
assert terminal["type"] == "terminal"
assert terminal["args"] == ["--setup"]
@pytest.mark.asyncio
async def test_initialize_advertises_terminal_setup_auth_when_no_provider(self, agent, monkeypatch):
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: None)
monkeypatch.setattr("acp_adapter.server.detect_provider", lambda: None)
resp = await agent.initialize(protocol_version=1)
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in resp.auth_methods]
assert payloads == [
{
"args": ["--setup"],
"description": (
"Open Hermes' interactive model/provider setup in a terminal. "
"Use this when Hermes has not been configured on this machine yet."
),
"id": TERMINAL_SETUP_AUTH_METHOD_ID,
"name": "Configure Hermes provider",
"type": "terminal",
}
]
# ---------------------------------------------------------------------------
# authenticate
@@ -135,6 +171,24 @@ class TestAuthenticate:
resp = await agent.authenticate(method_id="openrouter")
assert resp is None
@pytest.mark.asyncio
async def test_authenticate_accepts_terminal_setup_after_provider_configured(self, agent, monkeypatch):
monkeypatch.setattr(
"acp_adapter.server.detect_provider",
lambda: "openrouter",
)
resp = await agent.authenticate(method_id=TERMINAL_SETUP_AUTH_METHOD_ID)
assert isinstance(resp, AuthenticateResponse)
@pytest.mark.asyncio
async def test_authenticate_rejects_terminal_setup_without_provider(self, agent, monkeypatch):
monkeypatch.setattr(
"acp_adapter.server.detect_provider",
lambda: None,
)
resp = await agent.authenticate(method_id=TERMINAL_SETUP_AUTH_METHOD_ID)
assert resp is None
# ---------------------------------------------------------------------------
# new_session / cancel / load / resume
+262
View File
@@ -0,0 +1,262 @@
"""Tests for cross-edit LSP delta filtering.
The delta-filter contract spans three pieces:
1. ``agent.lsp.manager._diag_key`` — strict equality key including
the diagnostic's position range. Two diagnostics with the same
content but different lines are NOT equal under this key (they
are genuinely different diagnostics).
2. ``agent.lsp.range_shift.build_line_shift`` — derives a function
mapping pre-edit line numbers to post-edit line numbers from a
pre/post text pair.
3. ``agent.lsp.manager.LSPService.get_diagnostics_sync(line_shift=…)``
— applies the shift to baseline diagnostics before computing the
set-difference, so pre-existing errors at shifted lines hash
equal to their post-edit counterparts and get filtered out.
These tests exercise the contract at the unit level; the E2E case
(real LSP server, real shift) is covered in test_service.py.
"""
from __future__ import annotations
from agent.lsp.client import _diagnostic_key
from agent.lsp.manager import _diag_key
from agent.lsp.range_shift import (
build_line_shift,
shift_baseline,
shift_diagnostic_range,
)
def _diag(*, line: int, message: str = "Undefined variable",
severity: int = 1, code: str = "reportUndefinedVariable",
source: str = "Pyright", end_line: int | None = None) -> dict:
if end_line is None:
end_line = line
return {
"severity": severity,
"code": code,
"source": source,
"message": message,
"range": {
"start": {"line": line, "character": 0},
"end": {"line": end_line, "character": 10},
},
}
# ----------------------------------------------------------------------
# _diag_key: strict equality (with range)
# ----------------------------------------------------------------------
def test_diag_key_treats_shifted_diagnostics_as_distinct():
"""Two diagnostics with the same message but at different lines hash
differently — they are genuinely different diagnostics. The shift
map is what makes them equal AFTER remapping; the key itself stays
strict."""
a = _diag(line=100)
b = _diag(line=200)
assert _diag_key(a) != _diag_key(b)
def test_diag_key_matches_client_key_for_shifted_baseline():
"""When a baseline diagnostic is remapped through a shift, its
_diag_key must match the corresponding post-edit diagnostic's key
at the same coordinates. This is the contract the delta filter
relies on."""
pre = _diag(line=200)
# Edit deletes 14 lines above line 200, so the same error now
# appears at line 186 post-edit.
shift = lambda L: L - 14 if L >= 14 else L
shifted = shift_diagnostic_range(pre, shift)
assert shifted is not None
post = _diag(line=186)
assert _diag_key(shifted) == _diag_key(post)
def test_diag_key_distinguishes_message():
a = _diag(line=100, message="foo")
b = _diag(line=100, message="bar")
assert _diag_key(a) != _diag_key(b)
def test_diag_key_distinguishes_severity():
a = _diag(line=100, severity=1)
b = _diag(line=100, severity=2)
assert _diag_key(a) != _diag_key(b)
def test_diag_key_distinguishes_source():
a = _diag(line=100, source="Pyright")
b = _diag(line=100, source="Ruff")
assert _diag_key(a) != _diag_key(b)
def test_diag_key_matches_client_key_byte_for_byte():
"""The manager-side and client-side keys must agree on diagnostic
identity — they're used by two layers that need to round-trip the
same diagnostics through dedup and delta filtering."""
d = _diag(line=42)
assert _diag_key(d) == _diagnostic_key(d)
# ----------------------------------------------------------------------
# build_line_shift
# ----------------------------------------------------------------------
def test_shift_identity_for_identical_content():
shift = build_line_shift("a\nb\nc\n", "a\nb\nc\n")
assert shift(0) == 0
assert shift(1) == 1
assert shift(2) == 2
def test_shift_pure_deletion_above_line():
"""Delete 2 lines at the top; everything below shifts up by 2."""
pre = "line0\nline1\nline2\nline3\nline4\n"
post = "line2\nline3\nline4\n" # deleted lines 0-1
shift = build_line_shift(pre, post)
# Pre lines 0,1 → deleted → None
assert shift(0) is None
assert shift(1) is None
# Pre line 2 → post line 0
assert shift(2) == 0
# Pre line 4 → post line 2
assert shift(4) == 2
def test_shift_pure_insertion_above_line():
"""Insert 3 lines at the top; everything below shifts down by 3."""
pre = "line0\nline1\nline2\n"
post = "new0\nnew1\nnew2\nline0\nline1\nline2\n"
shift = build_line_shift(pre, post)
# Pre lines unchanged in identity, shifted by 3
assert shift(0) == 3
assert shift(1) == 4
assert shift(2) == 5
def test_shift_replacement_in_middle():
"""Replace 2 lines in the middle with 1 line. Lines above
unchanged; lines below shift up by 1."""
pre = "a\nb\nc\nd\ne\n"
post = "a\nb\nX\ne\n" # replaced lines 2,3 (c,d) with X
shift = build_line_shift(pre, post)
assert shift(0) == 0 # a → a
assert shift(1) == 1 # b → b
assert shift(2) is None # c → deleted
assert shift(3) is None # d → deleted
assert shift(4) == 3 # e → post line 3
def test_shift_handles_empty_pre():
"""First write of a file: pre is empty, post has content. Nothing
to shift, so the function should be well-defined for empty pre."""
shift = build_line_shift("", "hello\nworld\n")
# Any pre line falls past the end of an empty pre — anchor at end of post
assert shift(0) == 1
def test_shift_handles_empty_post():
"""File deleted to empty. Every pre line returns None."""
shift = build_line_shift("line0\nline1\n", "")
assert shift(0) is None
assert shift(1) is None
# ----------------------------------------------------------------------
# shift_diagnostic_range
# ----------------------------------------------------------------------
def test_shift_diag_remaps_start_and_end():
pre = "a\nb\nc\nd\n"
post = "X\na\nb\nc\nd\n" # one line inserted at top
shift = build_line_shift(pre, post)
d = _diag(line=2, end_line=2)
remapped = shift_diagnostic_range(d, shift)
assert remapped is not None
assert remapped["range"]["start"]["line"] == 3
assert remapped["range"]["end"]["line"] == 3
def test_shift_diag_drops_diagnostic_in_deleted_region():
pre = "a\nb\nc\nd\n"
post = "a\nd\n" # deleted lines 1,2 (b,c)
shift = build_line_shift(pre, post)
d = _diag(line=1)
assert shift_diagnostic_range(d, shift) is None
def test_shift_diag_does_not_mutate_original():
pre = "a\nb\n"
post = "X\na\nb\n"
shift = build_line_shift(pre, post)
d = _diag(line=0)
original_line = d["range"]["start"]["line"]
_ = shift_diagnostic_range(d, shift)
assert d["range"]["start"]["line"] == original_line
def test_shift_baseline_drops_deleted_and_remaps_rest():
pre = "a\nb\nc\nd\ne\n"
post = "a\ne\n" # deleted b,c,d
shift = build_line_shift(pre, post)
baseline = [
_diag(line=0, message="err on a"),
_diag(line=1, message="err on b"), # → deleted
_diag(line=2, message="err on c"), # → deleted
_diag(line=4, message="err on e"),
]
out = shift_baseline(baseline, shift)
assert [d["message"] for d in out] == ["err on a", "err on e"]
assert out[0]["range"]["start"]["line"] == 0
assert out[1]["range"]["start"]["line"] == 1
# ----------------------------------------------------------------------
# End-to-end: simulate the delta-filter pipeline
# ----------------------------------------------------------------------
def test_pipeline_filters_shifted_baseline_under_strict_key():
"""The exact scenario the bug fix is for: an edit deletes lines,
every diagnostic below shifts, and the delta filter (strict key
+ shifted baseline) correctly identifies them as pre-existing."""
pre = "line0\nline1\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\n"
# Delete lines 2,3,4 — pre-existing errors at lines 7,8 should
# appear at lines 4,5 post-edit and be filtered out.
post = "line0\nline1\nline5\nline6\nline7\nline8\nline9\n"
shift = build_line_shift(pre, post)
baseline = [_diag(line=7, message="X"), _diag(line=8, message="Y")]
post_diags = [_diag(line=4, message="X"), _diag(line=5, message="Y")]
shifted_baseline = shift_baseline(baseline, shift)
seen = {_diag_key(d) for d in shifted_baseline}
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
# Both errors were pre-existing — filtered out.
assert new_diags == []
def test_pipeline_preserves_new_instance_at_different_line():
"""The case content-only keys would miss: the model introduces a
SECOND instance of the same error class at a new location. The
new instance must surface."""
pre = "good\ngood\ngood\n"
post = "good\nbad\ngood\nbad\n" # added 2 new error lines
shift = build_line_shift(pre, post)
baseline = [_diag(line=0, message="bad style")] # pre-existing
post_diags = [
_diag(line=0, message="bad style"), # pre-existing
_diag(line=1, message="bad style"), # NEW — different line
_diag(line=3, message="bad style"), # NEW — different line
]
shifted_baseline = shift_baseline(baseline, shift)
seen = {_diag_key(d) for d in shifted_baseline}
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
# Two genuinely new instances must be surfaced.
assert len(new_diags) == 2
assert {d["range"]["start"]["line"] for d in new_diags} == {1, 3}
+29
View File
@@ -130,6 +130,35 @@ def test_service_e2e_delta_filter(mock_pyright):
svc.shutdown()
def test_service_e2e_delta_filter_with_line_shift(mock_pyright):
"""End-to-end: an edit that shifts the diagnostic's line still
filters correctly when ``line_shift`` is supplied.
The mock LSP server emits a fixed error at line 0; for this test
we don't need to actually shift the server's output — we just
need to prove that supplying a line_shift through the API works
and doesn't break the existing delta path. The unit tests in
test_delta_key.py cover the shift semantics in detail.
"""
repo = mock_pyright
f = repo / "x.py"
f.write_text("print('hi')\n")
svc = LSPService(
enabled=True,
wait_mode="document",
wait_timeout=3.0,
install_strategy="manual",
)
try:
svc.snapshot_baseline(str(f))
# Identity shift — should behave exactly like no shift.
new_diags = svc.get_diagnostics_sync(str(f), line_shift=lambda L: L)
assert new_diags == []
finally:
svc.shutdown()
def test_service_status_includes_clients(mock_pyright):
repo = mock_pyright
f = repo / "x.py"
+16 -4
View File
@@ -12,12 +12,24 @@ Covers:
import json
import os
import time
from types import SimpleNamespace
from contextlib import contextmanager
from types import ModuleType, SimpleNamespace
from unittest.mock import MagicMock, patch, PropertyMock
import pytest
@contextmanager
def _mock_botocore_session(*, return_value=None, side_effect=None):
"""Patch botocore.session even when botocore is not installed."""
botocore_mod = ModuleType("botocore")
session_mod = ModuleType("botocore.session")
session_mod.get_session = MagicMock(return_value=return_value, side_effect=side_effect)
botocore_mod.session = session_mod
with patch.dict("sys.modules", {"botocore": botocore_mod, "botocore.session": session_mod}):
yield session_mod.get_session
# ---------------------------------------------------------------------------
# AWS credential detection
# ---------------------------------------------------------------------------
@@ -120,7 +132,7 @@ class TestResolveBedrocRegion:
from unittest.mock import patch, MagicMock
mock_session = MagicMock()
mock_session.get_config_variable.return_value = None
with patch("botocore.session.get_session", return_value=mock_session):
with _mock_botocore_session(return_value=mock_session):
assert resolve_bedrock_region({}) == "us-east-1"
def test_falls_back_to_botocore_profile_region(self):
@@ -128,13 +140,13 @@ class TestResolveBedrocRegion:
from unittest.mock import patch, MagicMock
mock_session = MagicMock()
mock_session.get_config_variable.return_value = "eu-central-1"
with patch("botocore.session.get_session", return_value=mock_session):
with _mock_botocore_session(return_value=mock_session):
assert resolve_bedrock_region({}) == "eu-central-1"
def test_botocore_failure_falls_back_to_us_east_1(self):
from agent.bedrock_adapter import resolve_bedrock_region
from unittest.mock import patch
with patch("botocore.session.get_session", side_effect=Exception("no botocore")):
with _mock_botocore_session(side_effect=Exception("no botocore")):
assert resolve_bedrock_region({}) == "us-east-1"
+15 -11
View File
@@ -253,20 +253,24 @@ class TestErrorClassifierBedrock:
# ---------------------------------------------------------------------------
class TestPackaging:
"""Verify bedrock optional dependency is declared."""
"""Verify Bedrock remains a declared lazy optional dependency."""
@staticmethod
def _optional_dependencies():
import tomllib
from pathlib import Path
content = (Path(__file__).parent.parent.parent / "pyproject.toml").read_text()
return tomllib.loads(content)["project"]["optional-dependencies"]
def test_bedrock_extra_exists(self):
import configparser
from pathlib import Path
# Read pyproject.toml to verify [bedrock] extra
toml_path = Path(__file__).parent.parent.parent / "pyproject.toml"
content = toml_path.read_text()
assert 'bedrock = ["boto3' in content
extras = self._optional_dependencies()
assert "bedrock" in extras
assert any(dep.startswith("boto3==") for dep in extras["bedrock"])
def test_bedrock_in_all_extra(self):
from pathlib import Path
content = (Path(__file__).parent.parent.parent / "pyproject.toml").read_text()
assert '"hermes-agent[bedrock]"' in content
def test_bedrock_is_not_eager_installed_by_all_extra(self):
extras = self._optional_dependencies()
assert "hermes-agent[bedrock]" not in extras["all"]
# ---------------------------------------------------------------------------
+98 -5
View File
@@ -991,9 +991,12 @@ class TestCompressWithClient:
mock_client.chat.completions.create.return_value = mock_response
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=2)
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=2)
# Last head message (index 2) is "user" → summary should be "assistant"
# NOTE: protect_first_n=2 preserves 2 non-system messages in addition to
# the system prompt (always implicitly protected), yielding head [system,
# user, user] with last head = user.
msgs = [
{"role": "system", "content": "system prompt"},
{"role": "user", "content": "msg 1"},
@@ -1059,11 +1062,13 @@ class TestCompressWithClient:
mock_response.choices[0].message.content = "summary text"
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=3)
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=3)
# Head: [system, user, assistant] → last head = assistant
# Tail: [user, assistant, user] → first tail = user
# summary_role="user" collides with tail, "assistant" collides with head → merge
# NOTE: protect_first_n=2 preserves 2 non-system messages in addition to
# the system prompt (always implicitly protected).
msgs = [
{"role": "system", "content": "system prompt"},
{"role": "user", "content": "msg 1"},
@@ -1097,7 +1102,7 @@ class TestCompressWithClient:
mock_response.choices[0].message.content = "summary text"
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=3)
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=3)
msgs = [
{"role": "system", "content": "system prompt"},
@@ -1133,13 +1138,15 @@ class TestCompressWithClient:
mock_response.choices[0].message.content = "summary text"
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=2)
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=1, protect_last_n=2)
# Head: [system, user] → last head = user
# Tail: [assistant, user, assistant] → first tail = assistant
# summary_role="assistant" collides with tail, "user" collides with head → merge
# NOTE: protect_first_n=1 preserves 1 non-system message in addition to
# the system prompt (always implicitly protected).
# With min_tail=3, tail = last 3 messages (indices 5-7).
# Need 8 messages: min_for_compress = 2+3+1 = 6, must have > 6.
# Need 8 messages: _min_for_compress = head(2) + 3 + 1 = 6, must have > 6.
msgs = [
{"role": "system", "content": "system prompt"},
{"role": "user", "content": "msg 1"},
@@ -1292,6 +1299,92 @@ class TestSummaryTargetRatio:
c = ContextCompressor(model="test", quiet_mode=True)
assert c.protect_last_n == 20
def test_default_protect_first_n_is_3(self):
"""Default protect_first_n is 3 (system + 3 extra non-system messages =
4 protected messages total when a system prompt is present). With the
new semantics, the constructor default is 3 — the system prompt is
always implicitly protected ON TOP OF protect_first_n non-system
messages.
"""
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
c = ContextCompressor(model="test", quiet_mode=True)
assert c.protect_first_n == 3
def test_protect_first_n_override(self):
"""protect_first_n=0 should be honoured — for users who rely on rolling
compaction and want NOTHING pinned at head except the system prompt
(always implicitly protected)."""
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=0)
assert c.protect_first_n == 0
def test_protect_first_n_0_preserves_only_system_prompt(self):
"""End-to-end: when protect_first_n=0, compression should treat only
the system prompt as head. All user/assistant messages between the
system prompt and the protected tail become summarization candidates.
This is the cleanest configuration for long-running rolling-compaction
sessions — no user/assistant turn gets pinned verbatim forever just
because it happened to be early in the session."""
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
c = ContextCompressor(
model="test",
quiet_mode=True,
protect_first_n=0,
protect_last_n=2,
)
msgs = (
[{"role": "system", "content": "System prompt"}]
+ [{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg {i}"}
for i in range(8)]
)
result = c.compress(msgs)
# System prompt (msg[0]) survives as head
assert result[0]["role"] == "system"
assert result[0]["content"].startswith("System prompt")
# The first user/assistant exchange (msg 0, msg 1) should NOT be pinned
# as head verbatim — those would have been summarized or absorbed.
# Under default protect_first_n=3, result[1..3] would be the literal
# "msg 0" / "msg 1" / "msg 2"; with protect_first_n=0 they aren't.
assert result[1].get("content") != "msg 0"
# Last 2 messages are tail-protected under protect_last_n=2
assert result[-1]["content"] == msgs[-1]["content"]
def test_protect_first_n_semantics_stable_without_system_prompt(self):
"""Regression: gateway /compress handler strips the system prompt
before calling compress(). protect_first_n must mean the same thing
in both paths — "N non-system head messages" — so configuring
protect_first_n=0 preserves NOTHING at the head regardless of whether
the system prompt is in the messages list.
Bug this covers: under the old semantics, protect_first_n counted
literally from messages[0]. In the gateway path (no system prompt)
that meant protect_first_n=1 would pin the first user turn of the
session forever — a user-reported complaint that a week-old
resolved question kept getting reinserted into every compaction
summary."""
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
c = ContextCompressor(
model="test",
quiet_mode=True,
protect_first_n=0,
protect_last_n=2,
)
# No system prompt — this is what the gateway passes to compress().
msgs = [
{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg {i}"}
for i in range(10)
]
head_size = c._protect_head_size(msgs)
# With no system prompt and protect_first_n=0 → head is empty.
# The first user message is NOT pinned as head.
assert head_size == 0
# And with protect_first_n=3 on the same no-system-prompt list →
# head size is 3 (the three earliest non-system messages).
c.protect_first_n = 3
assert c._protect_head_size(msgs) == 3
class TestTokenBudgetTailProtection:
"""Tests for token-budget-based tail protection (PR #6240).
@@ -27,10 +27,12 @@ def _messages_with_handoff(summary_body: str):
return [
{"role": "system", "content": "system prompt"},
{"role": "user", "content": f"{SUMMARY_PREFIX}\n{summary_body}"},
{"role": "assistant", "content": "handoff acknowledged after resume"},
{"role": "user", "content": "new user turn after resume"},
{"role": "assistant", "content": "new assistant work after resume"},
{"role": "user", "content": "more new work after resume"},
{"role": "assistant", "content": "latest tail response"},
{"role": "user", "content": "final active request stays in protected tail"},
]
+22
View File
@@ -1,6 +1,7 @@
"""Tests for agent/display.py — build_tool_preview() and inline diff previews."""
import os
import json
import pytest
from unittest.mock import MagicMock, patch
@@ -149,6 +150,27 @@ class TestCuteToolMessagePreviewLength:
assert path in line
assert "..." not in line
def test_write_file_lint_error_result_is_not_marked_failed(self):
result = json.dumps({
"bytes_written": 12,
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
})
line = get_cute_tool_message("write_file", {"path": "/tmp/a.py"}, 0.1, result=result)
assert "[error]" not in line
def test_patch_lsp_diagnostics_result_is_not_marked_failed(self):
result = json.dumps({
"success": True,
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
})
line = get_cute_tool_message("patch", {"path": "/tmp/a.py"}, 0.1, result=result)
assert "[error]" not in line
class TestEditDiffPreview:
def test_extract_edit_diff_for_patch(self):
+29
View File
@@ -913,6 +913,35 @@ class TestTranslateStreamEvent:
assert chunks[-1].choices[0].finish_reason == "tool_calls"
class TestMakeStreamChunk:
def test_reasoning_only_chunk_has_content_none(self):
from agent.gemini_cloudcode_adapter import _make_stream_chunk
chunk = _make_stream_chunk(model="m", reasoning="think")
delta = chunk.choices[0].delta
assert delta.content is None
assert delta.reasoning == "think"
def test_content_only_chunk_has_reasoning_none(self):
from agent.gemini_cloudcode_adapter import _make_stream_chunk
chunk = _make_stream_chunk(model="m", content="hello")
delta = chunk.choices[0].delta
assert delta.content == "hello"
assert delta.reasoning is None
assert delta.tool_calls is None
def test_finish_only_chunk_has_all_fields_none(self):
from agent.gemini_cloudcode_adapter import _make_stream_chunk
chunk = _make_stream_chunk(model="m", finish_reason="stop")
delta = chunk.choices[0].delta
assert delta.content is None
assert delta.reasoning is None
assert delta.tool_calls is None
assert chunk.choices[0].finish_reason == "stop"
class TestGeminiCloudCodeClient:
def test_client_exposes_openai_interface(self):
from agent.gemini_cloudcode_adapter import GeminiCloudCodeClient
+16
View File
@@ -7,6 +7,7 @@ from agent.tool_guardrails import (
ToolCallGuardrailController,
ToolCallSignature,
canonical_tool_args,
classify_tool_failure,
)
@@ -131,6 +132,21 @@ def test_success_resets_exact_signature_failure_streak():
assert controller.before_call("web_search", args).action == "allow"
def test_file_mutation_lint_error_result_is_not_a_tool_failure():
write_result = json.dumps({
"bytes_written": 12,
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
})
patch_result = json.dumps({
"success": True,
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
})
assert classify_tool_failure("write_file", write_result) == (False, "")
assert classify_tool_failure("patch", patch_result) == (False, "")
def test_same_tool_varying_args_warns_by_default_without_halting():
controller = ToolCallGuardrailController(
ToolCallGuardrailConfig(same_tool_failure_warn_after=2, same_tool_failure_halt_after=3)
@@ -0,0 +1,30 @@
"""Tests for shared tool result classification helpers."""
import json
from agent.tool_result_classification import file_mutation_result_landed
def test_write_file_with_nested_lint_error_counts_as_landed():
result = json.dumps({
"bytes_written": 12,
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
})
assert file_mutation_result_landed("write_file", result) is True
def test_patch_with_nested_lsp_diagnostics_counts_as_landed():
result = json.dumps({
"success": True,
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
})
assert file_mutation_result_landed("patch", result) is True
def test_top_level_file_mutation_error_does_not_count_as_landed():
result = json.dumps({"success": True, "error": "post-write verification failed"})
assert file_mutation_result_landed("patch", result) is False
+114
View File
@@ -0,0 +1,114 @@
"""Tests for agent/video_gen_registry.py — provider registration & active lookup."""
from __future__ import annotations
import pytest
from agent import video_gen_registry
from agent.video_gen_provider import VideoGenProvider
class _FakeProvider(VideoGenProvider):
def __init__(self, name: str, available: bool = True):
self._name = name
self._available = available
@property
def name(self) -> str:
return self._name
def is_available(self) -> bool:
return self._available
def generate(self, prompt, **kw):
return {"success": True, "video": f"{self._name}://{prompt}"}
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
class TestRegisterProvider:
def test_register_and_lookup(self):
provider = _FakeProvider("fake")
video_gen_registry.register_provider(provider)
assert video_gen_registry.get_provider("fake") is provider
def test_rejects_non_provider(self):
with pytest.raises(TypeError):
video_gen_registry.register_provider("not a provider") # type: ignore[arg-type]
def test_rejects_empty_name(self):
class Empty(VideoGenProvider):
@property
def name(self) -> str:
return ""
def generate(self, prompt, **kw):
return {}
with pytest.raises(ValueError):
video_gen_registry.register_provider(Empty())
def test_reregister_overwrites(self):
a = _FakeProvider("same")
b = _FakeProvider("same")
video_gen_registry.register_provider(a)
video_gen_registry.register_provider(b)
assert video_gen_registry.get_provider("same") is b
def test_list_is_sorted(self):
video_gen_registry.register_provider(_FakeProvider("zeta"))
video_gen_registry.register_provider(_FakeProvider("alpha"))
names = [p.name for p in video_gen_registry.list_providers()]
assert names == ["alpha", "zeta"]
class TestGetActiveProvider:
def test_single_provider_autoresolves(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
video_gen_registry.register_provider(_FakeProvider("solo"))
active = video_gen_registry.get_active_provider()
assert active is not None and active.name == "solo"
def test_no_provider_returns_none(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
assert video_gen_registry.get_active_provider() is None
def test_multi_without_config_returns_none(self, tmp_path, monkeypatch):
"""Unlike image_gen (which falls back to 'fal'), video_gen has no
legacy default when there are multiple providers and no config,
the registry returns None and the tool surfaces a helpful error.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
video_gen_registry.register_provider(_FakeProvider("xai"))
video_gen_registry.register_provider(_FakeProvider("fal"))
assert video_gen_registry.get_active_provider() is None
def test_config_selects_provider(self, tmp_path, monkeypatch):
import yaml
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"video_gen": {"provider": "fal"}})
)
video_gen_registry.register_provider(_FakeProvider("xai"))
video_gen_registry.register_provider(_FakeProvider("fal"))
active = video_gen_registry.get_active_provider()
assert active is not None and active.name == "fal"
def test_unknown_config_falls_back(self, tmp_path, monkeypatch):
"""If video_gen.provider names a provider that isn't registered,
the single-provider fallback still applies."""
import yaml
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"video_gen": {"provider": "ghost"}})
)
video_gen_registry.register_provider(_FakeProvider("only"))
active = video_gen_registry.get_active_provider()
assert active is not None and active.name == "only"
@@ -0,0 +1,243 @@
"""Tests for the optional codex app-server runtime gate.
These are unit tests for the api_mode rewriter and the wire-level transport
module. They do NOT require the `codex` CLI to be installed that's
covered by a separate live test gated on `codex --version`.
"""
from __future__ import annotations
import pytest
from hermes_cli.runtime_provider import (
_VALID_API_MODES,
_maybe_apply_codex_app_server_runtime,
)
class TestApiModeRegistration:
"""The new api_mode must be registered or downstream parsing rejects it."""
def test_codex_app_server_is_a_valid_api_mode(self) -> None:
assert "codex_app_server" in _VALID_API_MODES
def test_existing_api_modes_still_present(self) -> None:
# Regression guard: don't accidentally delete other api_modes when
# touching this set.
for mode in (
"chat_completions",
"codex_responses",
"anthropic_messages",
"bedrock_converse",
):
assert mode in _VALID_API_MODES
class TestMaybeApplyCodexAppServerRuntime:
"""The opt-in helper that rewrites api_mode → codex_app_server."""
@pytest.mark.parametrize(
"model_cfg",
[
None,
{},
{"openai_runtime": ""},
{"openai_runtime": "auto"},
{"openai_runtime": "AUTO"},
{"other_key": "codex_app_server"}, # wrong key
],
)
def test_default_off_for_openai(self, model_cfg) -> None:
"""Default behavior is preserved when the flag is unset/auto."""
got = _maybe_apply_codex_app_server_runtime(
provider="openai", api_mode="chat_completions", model_cfg=model_cfg
)
assert got == "chat_completions"
def test_opt_in_rewrites_openai(self) -> None:
got = _maybe_apply_codex_app_server_runtime(
provider="openai",
api_mode="chat_completions",
model_cfg={"openai_runtime": "codex_app_server"},
)
assert got == "codex_app_server"
def test_opt_in_rewrites_openai_codex(self) -> None:
got = _maybe_apply_codex_app_server_runtime(
provider="openai-codex",
api_mode="codex_responses",
model_cfg={"openai_runtime": "codex_app_server"},
)
assert got == "codex_app_server"
def test_case_insensitive(self) -> None:
got = _maybe_apply_codex_app_server_runtime(
provider="openai",
api_mode="chat_completions",
model_cfg={"openai_runtime": "Codex_App_Server"},
)
assert got == "codex_app_server"
@pytest.mark.parametrize(
"provider",
[
"anthropic",
"openrouter",
"xai",
"qwen-oauth",
"google-gemini-cli",
"opencode-zen",
"bedrock",
"",
],
)
def test_other_providers_never_rerouted(self, provider) -> None:
"""Non-OpenAI providers MUST NOT be rerouted even with the flag set —
codex's app-server can only run OpenAI/Codex auth flows."""
got = _maybe_apply_codex_app_server_runtime(
provider=provider,
api_mode="anthropic_messages",
model_cfg={"openai_runtime": "codex_app_server"},
)
assert got == "anthropic_messages", (
f"provider={provider!r} should not be rerouted to codex_app_server"
)
class TestCodexAppServerModule:
"""Module-surface tests for the JSON-RPC speaker. Don't require codex CLI."""
def test_module_imports(self) -> None:
from agent.transports import codex_app_server
assert codex_app_server.MIN_CODEX_VERSION >= (0, 1, 0)
assert callable(codex_app_server.parse_codex_version)
assert callable(codex_app_server.check_codex_binary)
def test_parse_codex_version_valid(self) -> None:
from agent.transports.codex_app_server import parse_codex_version
assert parse_codex_version("codex-cli 0.130.0") == (0, 130, 0)
assert parse_codex_version("codex-cli 1.2.3 (extra metadata)") == (1, 2, 3)
assert parse_codex_version("codex 99.0.1\n") == (99, 0, 1)
def test_parse_codex_version_invalid(self) -> None:
from agent.transports.codex_app_server import parse_codex_version
assert parse_codex_version("nope") is None
assert parse_codex_version("") is None
assert parse_codex_version(None) is None # type: ignore[arg-type]
def test_check_binary_handles_missing_executable(self) -> None:
from agent.transports.codex_app_server import check_codex_binary
ok, msg = check_codex_binary(codex_bin="/nonexistent/codex/binary/path")
assert ok is False
assert "not found" in msg.lower() or "no such" in msg.lower()
def test_codex_error_class_is_runtimeerror(self) -> None:
from agent.transports.codex_app_server import CodexAppServerError
err = CodexAppServerError(code=-32600, message="boom")
assert isinstance(err, RuntimeError)
assert "boom" in str(err)
assert "-32600" in str(err)
class TestSpawnEnvIsolation:
"""The codex spawn must NOT rewrite HOME — codex's shell tool spawns
subprocesses (gh, git, npm, aws, gcloud, ...) that need to find their
config in the real user $HOME. CODEX_HOME isolates codex's own state,
HOME stays unchanged.
OpenClaw hit this footgun (openclaw/openclaw#81562) — they were
rewriting HOME to a synthetic per-agent dir alongside CODEX_HOME,
and then `gh auth status` / git config / etc. all broke inside codex
shell calls. We avoid the same bug by only overlaying CODEX_HOME and
RUST_LOG on top of os.environ.copy().
"""
def test_spawn_env_preserves_HOME(self, monkeypatch):
"""The spawn env must contain the parent process's HOME unchanged.
Verifies via a subprocess-monkey-patch."""
import subprocess
from agent.transports import codex_app_server as cas
captured = {}
class FakePopen:
def __init__(self, cmd, *args, **kwargs):
captured["env"] = kwargs.get("env", {}).copy()
# Provide minimal Popen surface so __init__ doesn't crash
# on attribute access during construction.
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = 1
self.returncode = None
def poll(self):
return None
def terminate(self):
pass
def wait(self, timeout=None):
return 0
def kill(self):
pass
monkeypatch.setattr(subprocess, "Popen", FakePopen)
monkeypatch.setenv("HOME", "/users/alice")
client = cas.CodexAppServerClient(codex_bin="codex")
client._closed = True # so close() is a no-op
# The spawn env must have HOME=/users/alice unchanged
assert captured["env"].get("HOME") == "/users/alice", (
f"HOME got rewritten in codex spawn env: "
f"{captured['env'].get('HOME')!r}. Codex's shell tool's "
"subprocesses (gh, git, aws, npm) need the user's real HOME."
)
def test_spawn_env_sets_CODEX_HOME_when_provided(self, monkeypatch):
"""CODEX_HOME isolation must still work — that's the whole point
of the codex_home arg."""
import subprocess
from agent.transports import codex_app_server as cas
captured = {}
class FakePopen:
def __init__(self, cmd, *args, **kwargs):
captured["env"] = kwargs.get("env", {}).copy()
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = 1
self.returncode = None
def poll(self):
return None
def terminate(self):
pass
def wait(self, timeout=None):
return 0
def kill(self):
pass
monkeypatch.setattr(subprocess, "Popen", FakePopen)
monkeypatch.setenv("HOME", "/users/alice")
client = cas.CodexAppServerClient(
codex_bin="codex", codex_home="/tmp/profile/codex"
)
client._closed = True
assert captured["env"].get("CODEX_HOME") == "/tmp/profile/codex"
# And HOME still passes through unchanged
assert captured["env"].get("HOME") == "/users/alice"
@@ -0,0 +1,976 @@
"""Tests for CodexAppServerSession — drive turns through a mock client.
The session adapter has the most complex behavior of the three new modules:
notification draining, server-request handling (approvals), interrupt,
deadline timeouts. These tests pin all of that without spawning real codex.
"""
from __future__ import annotations
import threading
import time
from typing import Any, Optional
import pytest
from agent.transports.codex_app_server_session import (
CodexAppServerSession,
TurnResult,
_ServerRequestRouting,
_approval_choice_to_codex_decision,
)
class FakeClient:
"""Stand-in for CodexAppServerClient that records calls and lets the test
drive the notification / server-request streams synchronously."""
def __init__(self, *, codex_bin: str = "codex", codex_home=None) -> None:
self.codex_bin = codex_bin
self.codex_home = codex_home
self.requests: list[tuple[str, dict]] = []
self.notifications_responses: list[dict] = []
self.responses: list[tuple[Any, dict]] = []
self.error_responses: list[tuple[Any, int, str]] = []
self._initialized = False
self._closed = False
self._notifications: list[dict] = []
self._server_requests: list[dict] = []
self._request_handler = None # Optional[Callable[[str, dict], dict]]
# API matching CodexAppServerClient
def initialize(self, **kwargs):
self._initialized = True
return {"userAgent": "fake/0.0.0", "codexHome": "/tmp",
"platformOs": "linux", "platformFamily": "unix"}
def request(self, method: str, params: Optional[dict] = None, timeout: float = 30.0):
self.requests.append((method, params or {}))
if self._request_handler is not None:
return self._request_handler(method, params or {})
# Sensible defaults for protocol methods used by the session
if method == "thread/start":
return {"thread": {"id": "thread-fake-001"},
"activePermissionProfile": {"id": "workspace-write"}}
if method == "turn/start":
return {"turn": {"id": "turn-fake-001"}}
if method == "turn/interrupt":
return {}
return {}
def notify(self, method: str, params=None):
pass
def respond(self, request_id, result):
self.responses.append((request_id, result))
def respond_error(self, request_id, code, message, data=None):
self.error_responses.append((request_id, code, message))
def take_notification(self, timeout: float = 0.0):
if self._notifications:
return self._notifications.pop(0)
# Honor a tiny sleep so the loop doesn't hot-spin; the real client
# blocks on a queue. For tests we want determinism.
if timeout > 0:
time.sleep(min(timeout, 0.001))
return None
def take_server_request(self, timeout: float = 0.0):
if self._server_requests:
return self._server_requests.pop(0)
return None
def close(self):
self._closed = True
def is_alive(self) -> bool:
# Fake is "alive" until close() is called; tests that want a dead
# subprocess can patch this attribute or call close() directly.
return not self._closed
def stderr_tail(self, n: int = 20):
return list(getattr(self, "_stderr_tail", []))[-n:]
# Test helpers
def queue_notification(self, method: str, **params):
self._notifications.append({"method": method, "params": params})
def queue_server_request(self, method: str, request_id: Any = "srv-1", **params):
self._server_requests.append({"id": request_id, "method": method, "params": params})
def set_stderr_tail(self, lines):
"""Test helper: seed stderr_tail() output for OAuth-refresh classifier tests."""
self._stderr_tail = list(lines)
def make_session(client: FakeClient, **kwargs) -> CodexAppServerSession:
return CodexAppServerSession(
cwd="/tmp",
client_factory=lambda **kw: client,
**kwargs,
)
# ---- choice mapping ----
class TestApprovalChoiceMapping:
@pytest.mark.parametrize("choice,expected", [
("once", "accept"),
("session", "acceptForSession"),
("always", "acceptForSession"),
("deny", "decline"),
("anything-else", "decline"),
])
def test_mapping(self, choice, expected):
assert _approval_choice_to_codex_decision(choice) == expected
# ---- lifecycle ----
class TestLifecycle:
def test_ensure_started_is_idempotent(self):
client = FakeClient()
s = make_session(client)
tid_a = s.ensure_started()
tid_b = s.ensure_started()
assert tid_a == tid_b == "thread-fake-001"
# thread/start should be called exactly once
method_calls = [m for (m, _) in client.requests if m == "thread/start"]
assert len(method_calls) == 1
def test_thread_start_passes_cwd_only(self):
"""thread/start carries cwd. We intentionally do NOT pass `permissions`
on this codex version (experimentalApi-gated + requires matching
config.toml [permissions] table). Letting codex use its default
(read-only unless user configures otherwise) is the documented path."""
client = FakeClient()
s = make_session(client, permission_profile="workspace-write")
s.ensure_started()
method, params = next(r for r in client.requests if r[0] == "thread/start")
assert params["cwd"] == "/tmp"
assert "permissions" not in params # see session.ensure_started() comment
def test_close_idempotent(self):
client = FakeClient()
s = make_session(client)
s.ensure_started()
s.close()
s.close()
assert client._closed is True
# ---- turn loop ----
class TestRunTurn:
def test_simple_text_turn_returns_final_message(self):
client = FakeClient()
client.queue_notification("turn/started", threadId="t", turn={"id": "tu1"})
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1", "text": "hello world"},
threadId="t", turnId="tu1",
)
client.queue_notification(
"turn/completed",
threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
r = s.run_turn("hi", turn_timeout=2.0)
assert r.final_text == "hello world"
assert r.interrupted is False
assert r.error is None
assert any(m["role"] == "assistant" and m.get("content") == "hello world"
for m in r.projected_messages)
# turn_id propagated for downstream session-DB linkage
assert r.turn_id == "turn-fake-001"
def test_tool_iteration_counter_ticks(self):
client = FakeClient()
# Two completed exec items + one final agent message
for i, item_id in enumerate(("ex1", "ex2"), start=1):
client.queue_notification(
"item/completed",
item={
"type": "commandExecution", "id": item_id,
"command": f"cmd{i}", "cwd": "/tmp",
"status": "completed", "aggregatedOutput": "ok",
"exitCode": 0, "commandActions": [],
},
threadId="t", turnId="tu1",
)
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1", "text": "done"},
threadId="t", turnId="tu1",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
r = s.run_turn("do stuff", turn_timeout=2.0)
assert r.tool_iterations == 2
# Each tool item produces (assistant, tool) — 2*2 + final assistant = 5 msgs
assert len(r.projected_messages) == 5
def test_turn_start_failure_returns_error(self):
client = FakeClient()
from agent.transports.codex_app_server import CodexAppServerError
def boom(method, params):
if method == "turn/start":
raise CodexAppServerError(code=-32600, message="bad input")
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
client._request_handler = boom
s = make_session(client)
r = s.run_turn("hi", turn_timeout=2.0)
assert r.error is not None
assert "bad input" in r.error
assert r.final_text == ""
def test_turn_start_failure_attaches_redacted_stderr_tail(self):
"""When codex stderr has content (non-OAuth), the tail gets attached
to the user-facing error so config/provider problems are debuggable
instead of just 'Internal error'. Secrets in stderr are redacted
via agent.redact(force=True)."""
client = FakeClient()
client.set_stderr_tail([
"ERROR: provider auth failed",
"Authorization: Bearer sk-live-deadbeefdeadbeef",
"url=https://api.example.com/v1?token=querysecret12345",
])
from agent.transports.codex_app_server import CodexAppServerError
def boom(method, params):
if method == "turn/start":
raise CodexAppServerError(code=-32603, message="Internal error")
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
client._request_handler = boom
s = make_session(client)
r = s.run_turn("hi", turn_timeout=2.0)
assert r.error is not None
assert "turn/start failed" in r.error
assert "Internal error" in r.error
# Stderr tail attached
assert "codex stderr" in r.error
assert "provider auth failed" in r.error
# Secrets redacted
assert "sk-live-deadbeefdeadbeef" not in r.error
assert "querysecret12345" not in r.error
# Non-OAuth → should NOT retire (subprocess JSON-RPC is still healthy).
assert r.should_retire is False
def test_turn_start_timeout_attaches_redacted_stderr_tail(self):
"""A non-OAuth TimeoutError on turn/start surfaces with codex stderr
context attached and marks the session for retirement."""
client = FakeClient()
client.set_stderr_tail([
"WARN: provider request stalled",
"Authorization: Bearer sk-stalled-secret-abc123",
])
def stall(method, params):
if method == "turn/start":
raise TimeoutError("codex method 'turn/start' timed out after 10s")
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
client._request_handler = stall
s = make_session(client)
r = s.run_turn("hi", turn_timeout=2.0)
assert r.error is not None
assert "turn/start timed out" in r.error
assert "provider request stalled" in r.error
assert "sk-stalled-secret-abc123" not in r.error
assert r.should_retire is True
def test_startup_failure_returns_error_with_stderr(self):
"""Codex thread/start failures during ensure_started() used to bubble
up as uncaught exceptions. Now they return a TurnResult.error so
AIAgent surfaces a clean diagnostic instead of crashing the turn."""
client = FakeClient()
client.set_stderr_tail([
"FATAL: model_provider 'azure_foundry' not configured",
])
from agent.transports.codex_app_server import CodexAppServerError
def boom(method, params):
if method == "thread/start":
raise CodexAppServerError(code=-32603, message="Internal error")
return {}
client._request_handler = boom
s = make_session(client)
r = s.run_turn("hi", turn_timeout=2.0)
assert r.error is not None
assert "startup failed" in r.error
assert "model_provider 'azure_foundry' not configured" in r.error
assert r.should_retire is True
assert r.final_text == ""
def test_interrupt_during_turn_issues_turn_interrupt(self):
client = FakeClient()
# Don't queue turn/completed — the loop has to interrupt out
client.queue_notification(
"item/completed",
item={"type": "commandExecution", "id": "x", "command": "sleep 60",
"cwd": "/", "status": "inProgress",
"aggregatedOutput": None, "exitCode": None,
"commandActions": []},
threadId="t", turnId="tu1",
)
s = make_session(client)
s.ensure_started()
# Trip the interrupt before run_turn even consumes the notification.
# The loop will see interrupt set on its first iteration and bail.
s.request_interrupt()
r = s.run_turn("loop forever", turn_timeout=2.0)
assert r.interrupted is True
# turn/interrupt was requested with the right turnId
assert any(
method == "turn/interrupt" and params.get("turnId") == "turn-fake-001"
for (method, params) in client.requests
)
def test_deadline_exceeded_records_error(self):
client = FakeClient()
# No notifications and no completion → must hit deadline
s = make_session(client)
r = s.run_turn("never finishes", turn_timeout=0.05,
notification_poll_timeout=0.01)
assert r.interrupted is True
assert r.error and "timed out" in r.error
def test_failed_turn_records_error_from_turn_completed(self):
client = FakeClient()
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "failed",
"error": {"message": "model error"}},
)
s = make_session(client)
r = s.run_turn("x", turn_timeout=1.0)
assert r.error and "model error" in r.error
# ---- approval bridge ----
class TestServerRequestRouting:
def test_exec_approval_with_callback_approves_once(self):
client = FakeClient()
client.queue_server_request(
"item/commandExecution/requestApproval", request_id="req-1",
command="ls /tmp", cwd="/tmp",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
captured: dict = {}
def cb(command, description, *, allow_permanent=True):
captured["command"] = command
captured["description"] = description
return "once"
s = make_session(client, approval_callback=cb)
s.run_turn("hi", turn_timeout=1.0)
assert captured["command"] == "ls /tmp"
# The session must have responded to the server request with "accept"
assert ("req-1", {"decision": "accept"}) in client.responses
def test_exec_approval_no_callback_denies(self):
client = FakeClient()
client.queue_server_request("item/commandExecution/requestApproval", request_id="req-1",
command="rm -rf /", cwd="/")
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client) # no approval_callback wired
s.run_turn("hi", turn_timeout=1.0)
assert ("req-1", {"decision": "decline"}) in client.responses
def test_apply_patch_approval_session_maps_to_session_decision(self):
client = FakeClient()
client.queue_server_request(
"item/fileChange/requestApproval", request_id="req-2",
itemId="fc-1",
turnId="t1",
threadId="th",
startedAtMs=1234567890,
reason="create new file with hello() function",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
def cb(command, description, *, allow_permanent=True):
return "session"
s = make_session(client, approval_callback=cb)
s.run_turn("hi", turn_timeout=1.0)
assert ("req-2", {"decision": "acceptForSession"}) in client.responses
def test_unknown_server_request_replied_with_error(self):
client = FakeClient()
client.queue_server_request("totally/unknown", request_id="req-3")
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
s.run_turn("hi", turn_timeout=1.0)
assert any(
rid == "req-3" and code == -32601
for (rid, code, _msg) in client.error_responses
)
def test_mcp_elicitation_for_hermes_tools_auto_accepts(self):
"""When codex elicits on behalf of hermes-tools (our own callback),
accept automatically the user already opted in by enabling the
runtime."""
client = FakeClient()
client.queue_server_request(
"mcpServer/elicitation/request", request_id="elic-1",
threadId="t", turnId="tu1",
serverName="hermes-tools",
mode="form",
message="confirm",
requestedSchema={"type": "object", "properties": {}},
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
s.run_turn("hi", turn_timeout=1.0)
assert ("elic-1", {"action": "accept", "content": None, "_meta": None}) in client.responses
def test_mcp_elicitation_for_other_servers_declines(self):
"""For third-party MCP servers we decline by default so users
explicitly opt in through codex's own UI."""
client = FakeClient()
client.queue_server_request(
"mcpServer/elicitation/request", request_id="elic-2",
threadId="t", turnId="tu1",
serverName="some-third-party",
mode="url",
message="please log in",
url="https://example.com/oauth",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
s.run_turn("hi", turn_timeout=1.0)
assert ("elic-2", {"action": "decline", "content": None, "_meta": None}) in client.responses
def test_routing_auto_approve_bypass(self):
client = FakeClient()
client.queue_server_request("item/commandExecution/requestApproval", request_id="r1",
command="ls", cwd="/")
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
# No callback, but routing says auto-approve. Should approve.
s = make_session(client, request_routing=_ServerRequestRouting(
auto_approve_exec=True))
s.run_turn("hi", turn_timeout=1.0)
assert ("r1", {"decision": "accept"}) in client.responses
def test_callback_raises_falls_back_to_decline(self):
client = FakeClient()
client.queue_server_request("item/commandExecution/requestApproval", request_id="r1",
command="ls", cwd="/")
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
def boom(*a, **kw):
raise RuntimeError("ui crashed")
s = make_session(client, approval_callback=boom)
s.run_turn("hi", turn_timeout=1.0)
# Fail-closed: deny on callback exception
assert ("r1", {"decision": "decline"}) in client.responses
# ---- enriched approval prompts ----
class TestApprovalPromptEnrichment:
"""Quirk #4: apply_patch prompt should show what's changing.
Quirk #10: exec prompt should never show empty cwd."""
def test_exec_falls_back_to_session_cwd(self):
"""When codex omits cwd from the approval params, the prompt shows
the session cwd, not an empty string."""
client = FakeClient()
client.queue_server_request(
"item/commandExecution/requestApproval", request_id="r1",
command="ls", # no cwd
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
captured = {}
def cb(command, description, *, allow_permanent=True):
captured["description"] = description
return "once"
s = make_session(client, approval_callback=cb)
s.run_turn("hi", turn_timeout=1.0)
# Session cwd is /tmp by default in make_session()
assert "/tmp" in captured["description"]
assert "Codex requests exec in <unknown>" not in captured["description"]
def test_apply_patch_prompt_summarizes_pending_changes(self):
"""When the projector has cached the fileChange item from item/started,
the approval prompt surfaces the change summary."""
client = FakeClient()
# item/started fires first (carries the changes), then approval request
client.queue_notification(
"item/started",
item={"type": "fileChange", "id": "fc-1",
"changes": [
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
]},
threadId="t", turnId="tu1",
)
client.queue_server_request(
"item/fileChange/requestApproval", request_id="req-2",
itemId="fc-1", turnId="tu1", threadId="t",
startedAtMs=1234567890,
reason="add and update files",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
captured = {}
def cb(command, description, *, allow_permanent=True):
captured["command"] = command
captured["description"] = description
return "once"
s = make_session(client, approval_callback=cb)
s.run_turn("hi", turn_timeout=1.0)
# Both add and update kinds should be in the summary
assert "1 add" in captured["command"] or "1 add" in captured["description"]
assert "1 update" in captured["command"] or "1 update" in captured["description"]
# And at least one of the paths
joined = captured["command"] + " " + captured["description"]
assert "/tmp/new.py" in joined or "/tmp/old.py" in joined
def test_apply_patch_prompt_works_without_cached_summary(self):
"""When approval arrives before item/started (or without changes
info), prompt falls back to whatever codex provided."""
client = FakeClient()
client.queue_server_request(
"item/fileChange/requestApproval", request_id="req-2",
itemId="fc-orphan", turnId="tu1", threadId="t",
startedAtMs=1234567890,
reason="apply some changes",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
captured = {}
def cb(command, description, *, allow_permanent=True):
captured["command"] = command
return "once"
s = make_session(client, approval_callback=cb)
s.run_turn("hi", turn_timeout=1.0)
# Falls back to the reason
assert "apply some changes" in captured["command"]
# ---- openclaw beta.8 parity: retire/wedge/oauth/abort marker ----
class TestSessionRetirement:
"""Mirrors openclaw beta.8's resilience fixes:
- retire timed-out app-server clients (should_retire on deadline)
- post-tool completion watchdog (don't burn the full deadline after a
tool result if codex goes silent)
- <turn_aborted> raw marker as terminal (don't wait for turn/completed
that never comes)
- OAuth refresh failure classification (suggest `codex login` instead
of raw RPC error strings)
- dead subprocess detection between iterations
"""
def test_deadline_marks_session_for_retirement(self):
client = FakeClient()
s = make_session(client)
r = s.run_turn(
"never finishes",
turn_timeout=0.05,
notification_poll_timeout=0.01,
)
assert r.interrupted is True
assert r.error and "timed out" in r.error
assert r.should_retire is True, (
"Deadline exhaustion must signal retirement so the next turn "
"respawns codex instead of riding a wedged subprocess."
)
def test_completed_turn_does_not_retire(self):
client = FakeClient()
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1", "text": "hi"},
threadId="t", turnId="tu1",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
r = s.run_turn("hi", turn_timeout=1.0)
assert r.should_retire is False
def test_post_tool_quiet_watchdog_trips_and_retires(self):
client = FakeClient()
# One tool completion, then total silence — no further events,
# no turn/completed. With a tiny post_tool_quiet_timeout the
# watchdog must fire before the larger turn deadline.
client.queue_notification(
"item/completed",
item={
"type": "commandExecution", "id": "ex1",
"command": "echo hi", "cwd": "/tmp",
"status": "completed", "aggregatedOutput": "hi",
"exitCode": 0, "commandActions": [],
},
threadId="t", turnId="tu1",
)
s = make_session(client)
r = s.run_turn(
"tool then silence",
turn_timeout=5.0, # would be miserable to wait
notification_poll_timeout=0.02,
post_tool_quiet_timeout=0.15,
)
assert r.interrupted is True
assert r.should_retire is True
assert r.error and "silent" in r.error
# Confirm we issued turn/interrupt to free codex compute
assert any(method == "turn/interrupt" for (method, _) in client.requests)
def test_post_tool_watchdog_resets_on_further_activity(self):
"""A tool completion followed by an agent message should NOT trip
the watchdog further activity = codex still alive."""
client = FakeClient()
client.queue_notification(
"item/completed",
item={
"type": "commandExecution", "id": "ex1",
"command": "echo hi", "cwd": "/tmp",
"status": "completed", "aggregatedOutput": "hi",
"exitCode": 0, "commandActions": [],
},
threadId="t", turnId="tu1",
)
# Non-tool activity immediately after — resets watchdog.
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1", "text": "tool finished"},
threadId="t", turnId="tu1",
)
client.queue_notification(
"turn/completed", threadId="t",
turn={"id": "tu1", "status": "completed", "error": None},
)
s = make_session(client)
r = s.run_turn(
"tool then talk", turn_timeout=2.0,
notification_poll_timeout=0.01,
post_tool_quiet_timeout=0.05,
)
# Tool ran, then text reset the watchdog, then turn/completed.
# Should NOT be a retirement case.
assert r.tool_iterations == 1
assert r.final_text == "tool finished"
assert r.should_retire is False
assert r.interrupted is False
def test_turn_aborted_marker_in_text_is_terminal(self):
"""If codex emits `<turn_aborted>` in agent text and never sends
turn/completed, we still exit promptly instead of burning the
deadline."""
client = FakeClient()
client.queue_notification(
"item/completed",
item={
"type": "agentMessage", "id": "m1",
"text": "partial output... <turn_aborted>",
},
threadId="t", turnId="tu1",
)
# Deliberately NO turn/completed notification queued.
s = make_session(client)
r = s.run_turn(
"abort mid-turn", turn_timeout=2.0,
notification_poll_timeout=0.01,
)
assert r.interrupted is True
assert r.error and "turn_aborted" in r.error
# Should have exited fast — not waited for the full 2s deadline.
# (Can't measure wall clock reliably in CI; presence of the marker
# error string instead of a "timed out" message is the proxy.)
assert "timed out" not in r.error
def test_turn_aborted_self_closing_marker_also_terminal(self):
client = FakeClient()
client.queue_notification(
"item/completed",
item={"type": "agentMessage", "id": "m1",
"text": "<turn_aborted/>"},
threadId="t", turnId="tu1",
)
s = make_session(client)
r = s.run_turn("x", turn_timeout=2.0,
notification_poll_timeout=0.01)
assert r.interrupted is True
assert r.error and "turn_aborted" in r.error
def test_oauth_refresh_failure_on_turn_start_suggests_login(self):
from agent.transports.codex_app_server import CodexAppServerError
client = FakeClient()
def boom(method, params):
if method == "turn/start":
raise CodexAppServerError(
code=-32603,
message="auth refresh failed: invalid_grant",
)
return {"thread": {"id": "t"},
"activePermissionProfile": {"id": "x"}}
client._request_handler = boom
s = make_session(client)
r = s.run_turn("hi", turn_timeout=1.0)
assert r.error is not None
assert "codex login" in r.error
assert r.should_retire is True
def test_oauth_failure_from_stderr_on_turn_start_failure(self):
"""If the RPC error itself is opaque but stderr shows an auth
problem, we still classify it as a refresh failure."""
from agent.transports.codex_app_server import CodexAppServerError
client = FakeClient()
client.set_stderr_tail([
"[2026-05-14T10:00:00Z WARN codex_core::auth] token refresh failed",
"[2026-05-14T10:00:00Z ERROR codex_core] please log in again",
])
def boom(method, params):
if method == "turn/start":
raise CodexAppServerError(code=-32603, message="rpc broke")
return {"thread": {"id": "t"},
"activePermissionProfile": {"id": "x"}}
client._request_handler = boom
s = make_session(client)
r = s.run_turn("hi", turn_timeout=1.0)
assert r.error is not None
assert "codex login" in r.error
assert r.should_retire is True
def test_oauth_failure_in_turn_completed_error(self):
"""A failed turn/completed whose error mentions auth/refresh
triggers the re-auth hint + retirement."""
client = FakeClient()
client.queue_notification(
"turn/completed", threadId="t",
turn={
"id": "tu1", "status": "failed",
"error": {"message": "401 Unauthorized: please reauthenticate"},
},
)
s = make_session(client)
r = s.run_turn("x", turn_timeout=1.0,
notification_poll_timeout=0.01)
assert r.error is not None
assert "codex login" in r.error
assert r.should_retire is True
def test_generic_turn_failure_does_not_trigger_oauth_hint(self):
"""A boring model error must NOT rewrite the message into a fake
re-auth hint. Conservative classifier."""
client = FakeClient()
client.queue_notification(
"turn/completed", threadId="t",
turn={
"id": "tu1", "status": "failed",
"error": {"message": "rate limit exceeded"},
},
)
s = make_session(client)
r = s.run_turn("x", turn_timeout=1.0,
notification_poll_timeout=0.01)
assert r.error is not None
assert "codex login" not in r.error
assert "rate limit exceeded" in r.error
# Generic model failures don't retire — the session itself is fine
assert r.should_retire is False
def test_dead_subprocess_detected_between_iterations(self):
"""If codex dies (segfault, OOM, killed by its auth refresh
thread), the inter-iteration is_alive check breaks the loop
instead of waiting on a queue that will never fill."""
client = FakeClient()
s = make_session(client)
s.ensure_started()
# Simulate subprocess death by setting _closed (FakeClient's
# is_alive returns False when closed).
client._closed = True
client.set_stderr_tail([
"thread 'tokio-runtime-worker' panicked at 'oauth: invalid_grant'",
])
r = s.run_turn("x", turn_timeout=2.0,
notification_poll_timeout=0.01)
assert r.should_retire is True
# Stderr-derived auth hint takes precedence over generic message
assert r.error and "codex login" in r.error
# ---- thread/start cross-fill ----
class TestThreadStartCrossFill:
"""Mirrors openclaw beta.8's tolerance for thread.id/sessionId aliasing."""
def test_thread_id_under_thread_key(self):
client = FakeClient()
s = make_session(client)
tid = s.ensure_started()
assert tid == "thread-fake-001"
def test_thread_session_id_alias_under_thread_key(self):
client = FakeClient()
client._request_handler = lambda method, params: (
{"thread": {"sessionId": "alias-1"},
"activePermissionProfile": {"id": "x"}}
if method == "thread/start" else
{"turn": {"id": "tu1"}} if method == "turn/start" else {}
)
s = make_session(client)
tid = s.ensure_started()
assert tid == "alias-1"
def test_top_level_session_id_fallback(self):
client = FakeClient()
client._request_handler = lambda method, params: (
{"sessionId": "top-1"} if method == "thread/start" else
{"turn": {"id": "tu1"}} if method == "turn/start" else {}
)
s = make_session(client)
tid = s.ensure_started()
assert tid == "top-1"
def test_missing_thread_id_raises(self):
from agent.transports.codex_app_server import CodexAppServerError
client = FakeClient()
client._request_handler = lambda method, params: (
{"thread": {}, "activePermissionProfile": {"id": "x"}}
if method == "thread/start" else
{"turn": {"id": "tu1"}}
)
s = make_session(client)
with pytest.raises(CodexAppServerError, match="no thread id"):
s.ensure_started()
class TestHasTurnAbortedMarker:
"""Unit coverage for the marker matcher itself."""
def test_empty_string(self):
from agent.transports.codex_app_server_session import (
_has_turn_aborted_marker,
)
assert _has_turn_aborted_marker("") is False
assert _has_turn_aborted_marker(None) is False # type: ignore[arg-type]
def test_plain_text_no_marker(self):
from agent.transports.codex_app_server_session import (
_has_turn_aborted_marker,
)
assert _has_turn_aborted_marker("normal response with no markers") is False
def test_open_marker(self):
from agent.transports.codex_app_server_session import (
_has_turn_aborted_marker,
)
assert _has_turn_aborted_marker("blah <turn_aborted> blah") is True
def test_self_closing_marker(self):
from agent.transports.codex_app_server_session import (
_has_turn_aborted_marker,
)
assert _has_turn_aborted_marker("<turn_aborted/>") is True
class TestClassifyOAuthFailure:
"""Unit coverage for the OAuth classifier; conservative on purpose."""
def test_invalid_grant_classified(self):
from agent.transports.codex_app_server_session import (
_classify_oauth_failure,
)
hint = _classify_oauth_failure("error: invalid_grant returned by server")
assert hint is not None
assert "codex login" in hint
def test_token_refresh_classified(self):
from agent.transports.codex_app_server_session import (
_classify_oauth_failure,
)
hint = _classify_oauth_failure("token refresh failed: network error")
assert hint is not None
assert "codex login" in hint
def test_401_classified(self):
from agent.transports.codex_app_server_session import (
_classify_oauth_failure,
)
hint = _classify_oauth_failure("HTTP 401 Unauthorized")
assert hint is not None
def test_generic_error_not_classified(self):
from agent.transports.codex_app_server_session import (
_classify_oauth_failure,
)
assert _classify_oauth_failure("connection reset") is None
assert _classify_oauth_failure("model returned bad json") is None
assert _classify_oauth_failure("rate limit exceeded") is None
def test_empty_inputs(self):
from agent.transports.codex_app_server_session import (
_classify_oauth_failure,
)
assert _classify_oauth_failure() is None
assert _classify_oauth_failure("") is None
assert _classify_oauth_failure("", None) is None # type: ignore[arg-type]
def test_multi_string_search(self):
"""Hint can come from any of the provided strings."""
from agent.transports.codex_app_server_session import (
_classify_oauth_failure,
)
hint = _classify_oauth_failure(
"rpc returned -32603",
"[stderr] token has expired, run codex login",
)
assert hint is not None
@@ -0,0 +1,303 @@
"""Tests for CodexEventProjector — codex item/* events → Hermes messages list.
Drives projection against fixture notifications captured from codex 0.130.0
plus synthetic ones for item types we couldn't auth-test live."""
from __future__ import annotations
import json
import pytest
from agent.transports.codex_event_projector import (
CodexEventProjector,
ProjectionResult,
_deterministic_call_id,
_format_tool_args,
)
# --- Fixture: real `commandExecution` notification captured from codex 0.130.0
COMMAND_EXEC_COMPLETED = {
"method": "item/completed",
"params": {
"item": {
"type": "commandExecution",
"id": "f8a75c66-a89e-4fd7-8bcf-2d58e664fa9e",
"command": "/bin/bash -lc 'echo hello && ls /tmp | head -3'",
"cwd": "/tmp",
"processId": None,
"source": "userShell",
"status": "completed",
"commandActions": [
{"type": "listFiles", "command": "ls /tmp", "path": "tmp"}
],
"aggregatedOutput": "hello\naa_lang.json\n",
"exitCode": 0,
"durationMs": 10,
},
"threadId": "019e1a94-352b-71e1-b214-e5c67c9ec190",
"turnId": "019e1a94-3553-7940-8af3-4ca57142deb7",
"completedAtMs": 1778562381151,
},
}
class TestProjectionInvariants:
"""Universal invariants that must hold across all projection paths."""
def test_streaming_deltas_dont_materialize(self) -> None:
p = CodexEventProjector()
for delta_method in (
"item/commandExecution/outputDelta",
"item/agentMessage/delta",
"item/reasoning/delta",
):
r = p.project({"method": delta_method, "params": {"delta": "x"}})
assert r.messages == [], (
f"{delta_method} should NOT produce messages — only "
f"item/completed materializes"
)
assert r.is_tool_iteration is False
assert r.final_text is None
def test_turn_started_and_completed_are_silent(self) -> None:
p = CodexEventProjector()
for method in ("turn/started", "turn/completed", "thread/started"):
r = p.project({"method": method, "params": {}})
assert r.messages == []
def test_unknown_method_silent(self) -> None:
p = CodexEventProjector()
r = p.project({"method": "totally/unknown", "params": {}})
assert r.messages == []
class TestCommandExecutionProjection:
"""Real captured notification → assistant tool_call + tool result."""
def test_command_completed_produces_two_messages(self) -> None:
p = CodexEventProjector()
r = p.project(COMMAND_EXEC_COMPLETED)
assert len(r.messages) == 2
assert r.is_tool_iteration is True
def test_first_message_is_assistant_tool_call(self) -> None:
p = CodexEventProjector()
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
assistant = msgs[0]
assert assistant["role"] == "assistant"
assert assistant["content"] is None
assert len(assistant["tool_calls"]) == 1
tc = assistant["tool_calls"][0]
assert tc["type"] == "function"
assert tc["function"]["name"] == "exec_command"
args = json.loads(tc["function"]["arguments"])
assert "echo hello" in args["command"]
assert args["cwd"] == "/tmp"
def test_second_message_is_tool_result_correlating_by_id(self) -> None:
p = CodexEventProjector()
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
assistant, tool = msgs
assert tool["role"] == "tool"
assert tool["tool_call_id"] == assistant["tool_calls"][0]["id"]
assert "hello" in tool["content"]
def test_nonzero_exit_code_annotated_in_tool_result(self) -> None:
item = {**COMMAND_EXEC_COMPLETED["params"]["item"], "exitCode": 2,
"aggregatedOutput": "boom"}
notif = {
"method": "item/completed",
"params": {**COMMAND_EXEC_COMPLETED["params"], "item": item},
}
p = CodexEventProjector()
msgs = p.project(notif).messages
assert "[exit 2]" in msgs[1]["content"]
assert "boom" in msgs[1]["content"]
def test_deterministic_call_id_across_replay(self) -> None:
# Same item id → same call_id (prefix cache must stay valid).
p1 = CodexEventProjector()
p2 = CodexEventProjector()
a = p1.project(COMMAND_EXEC_COMPLETED).messages
b = p2.project(COMMAND_EXEC_COMPLETED).messages
assert a[0]["tool_calls"][0]["id"] == b[0]["tool_calls"][0]["id"]
class TestAgentMessageProjection:
"""assistant text → final_text + assistant message."""
def test_agent_message_projects_to_assistant(self) -> None:
p = CodexEventProjector()
r = p.project({
"method": "item/completed",
"params": {"item": {"type": "agentMessage", "id": "x",
"text": "hi there"}},
})
assert r.final_text == "hi there"
assert r.messages == [{"role": "assistant", "content": "hi there"}]
assert r.is_tool_iteration is False
def test_pending_reasoning_attaches_to_next_assistant_message(self) -> None:
p = CodexEventProjector()
# First a reasoning item lands
r1 = p.project({
"method": "item/completed",
"params": {"item": {"type": "reasoning", "id": "r1",
"summary": ["thinking..."],
"content": ["step 1", "step 2"]}},
})
assert r1.messages == [] # reasoning alone produces no message
# Then the assistant message
r2 = p.project({
"method": "item/completed",
"params": {"item": {"type": "agentMessage", "id": "a1",
"text": "ok"}},
})
assistant = r2.messages[0]
assert "reasoning" in assistant
assert "thinking" in assistant["reasoning"]
assert "step 1" in assistant["reasoning"]
def test_reasoning_consumed_after_attaching(self) -> None:
p = CodexEventProjector()
p.project({"method": "item/completed", "params": {"item": {
"type": "reasoning", "id": "r1", "summary": ["once"], "content": []}}})
first = p.project({"method": "item/completed", "params": {"item": {
"type": "agentMessage", "id": "a", "text": "first"}}}).messages[0]
second = p.project({"method": "item/completed", "params": {"item": {
"type": "agentMessage", "id": "b", "text": "second"}}}).messages[0]
assert "reasoning" in first
assert "reasoning" not in second
class TestFileChangeProjection:
def test_file_change_summary_no_inlined_content(self) -> None:
item = {
"type": "fileChange",
"id": "fc1",
"status": "applied",
"changes": [
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
],
}
p = CodexEventProjector()
msgs = p.project({"method": "item/completed",
"params": {"item": item}}).messages
assert len(msgs) == 2
tc = msgs[0]["tool_calls"][0]
assert tc["function"]["name"] == "apply_patch"
args = json.loads(tc["function"]["arguments"])
assert len(args["changes"]) == 2
assert all("kind" in c and "path" in c for c in args["changes"])
assert "applied" in msgs[1]["content"]
class TestMcpToolCallProjection:
def test_mcp_tool_call_namespaced(self) -> None:
item = {
"type": "mcpToolCall",
"id": "m1",
"server": "obsidian",
"tool": "search_notes",
"status": "completed",
"arguments": {"query": "hermes"},
"result": {"content": [{"text": "found"}]},
"error": None,
}
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert msgs[0]["tool_calls"][0]["function"]["name"] == "mcp.obsidian.search_notes"
assert "found" in msgs[1]["content"]
def test_mcp_error_surfaced(self) -> None:
item = {
"type": "mcpToolCall", "id": "m2",
"server": "x", "tool": "y", "status": "failed",
"arguments": {}, "result": None,
"error": {"code": -1, "message": "no"},
}
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert "error" in msgs[1]["content"]
class TestUserAndOpaqueProjection:
def test_user_message_text_fragments_only(self) -> None:
item = {
"type": "userMessage", "id": "u1",
"content": [
{"type": "text", "text": "hello"},
{"type": "image", "url": "http://x/y"},
{"type": "text", "text": "world"},
],
}
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert msgs[0]["role"] == "user"
assert "hello" in msgs[0]["content"]
assert "world" in msgs[0]["content"]
def test_opaque_item_recorded_without_fabricated_tool_calls(self) -> None:
item = {"type": "plan", "id": "p1", "text": "do the thing"}
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert len(msgs) == 1
assert msgs[0]["role"] == "assistant"
assert "plan" in msgs[0]["content"].lower()
assert "tool_calls" not in msgs[0]
class TestHelpers:
def test_deterministic_call_id_stable(self) -> None:
assert _deterministic_call_id("exec", "abc") == _deterministic_call_id("exec", "abc")
assert _deterministic_call_id("exec", "abc") != _deterministic_call_id("exec", "xyz")
def test_deterministic_call_id_handles_missing_id(self) -> None:
# Should not raise, should be stable for same item type
a = _deterministic_call_id("exec", "")
b = _deterministic_call_id("exec", "")
assert a == b
assert "exec" in a
def test_format_tool_args_sorted_keys(self) -> None:
# Sorted keys = deterministic across replays = prefix cache stays valid
a = _format_tool_args({"b": 1, "a": 2})
b = _format_tool_args({"a": 2, "b": 1})
assert a == b
class TestRoleAlternationInvariant:
"""The project must never emit two assistant messages back-to-back from
one item that breaks Hermes' message alternation invariant."""
@pytest.mark.parametrize(
"item",
[
{"type": "commandExecution", "id": "c1", "command": "x",
"cwd": "/", "status": "completed", "aggregatedOutput": "",
"exitCode": 0, "commandActions": []},
{"type": "fileChange", "id": "f1", "status": "applied",
"changes": []},
{"type": "mcpToolCall", "id": "m1", "server": "s", "tool": "t",
"status": "completed", "arguments": {}, "result": None,
"error": None},
{"type": "dynamicToolCall", "id": "d1", "tool": "x",
"arguments": {}, "status": "completed",
"contentItems": [], "success": True},
],
)
def test_tool_items_emit_assistant_then_tool(self, item) -> None:
msgs = CodexEventProjector().project(
{"method": "item/completed", "params": {"item": item}}
).messages
assert len(msgs) == 2
assert msgs[0]["role"] == "assistant"
assert msgs[1]["role"] == "tool"
assert msgs[1]["tool_call_id"] == msgs[0]["tool_calls"][0]["id"]
@@ -100,6 +100,44 @@ class TestCodexBuildKwargs:
)
assert "prompt_cache_key" not in kw
def test_xai_responses_sends_cache_key_via_extra_body(self, transport):
"""xAI's Responses API documents ``prompt_cache_key`` as the
body-level cache-routing key (the ``x-grok-conv-id`` header is
Chat-Completions-only). Passing it via ``extra_body`` is robust
against openai SDK builds whose ``Responses.stream()`` kwarg
signature ever drops the field the body field still serializes
and reaches xAI either way. The ``x-grok-conv-id`` header is kept
as a belt-and-braces fallback so cache routing survives even
when the body field would be stripped by an intermediate proxy.
Ref: https://docs.x.ai/developers/advanced-api-usage/prompt-caching/maximizing-cache-hits
"""
messages = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="grok-4.3", messages=messages, tools=[],
session_id="conv-xai-1",
is_xai_responses=True,
)
assert "prompt_cache_key" not in kw
assert kw.get("extra_body", {}).get("prompt_cache_key") == "conv-xai-1"
assert kw.get("extra_headers", {}).get("x-grok-conv-id") == "conv-xai-1"
def test_xai_responses_extra_body_preserves_caller_fields(self, transport):
"""When the caller already supplies ``extra_body`` (e.g. via
request_overrides), the xAI cache-key injection must merge into
the existing dict instead of overwriting it. Caller-supplied
``prompt_cache_key`` wins (setdefault semantics) so user overrides
aren't silently clobbered by the transport."""
messages = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="grok-4.3", messages=messages, tools=[],
session_id="conv-xai-1",
is_xai_responses=True,
request_overrides={"extra_body": {"prompt_cache_key": "caller-override", "other_field": 42}},
)
eb = kw.get("extra_body", {})
assert eb.get("prompt_cache_key") == "caller-override"
assert eb.get("other_field") == 42
def test_max_tokens(self, transport):
messages = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
@@ -0,0 +1,135 @@
"""Tests for the hermes-tools-as-MCP server module surface.
We don't run a live MCP session in unit tests — that requires the codex
subprocess + client + an event loop. These tests pin the static
contract: the module imports, the EXPOSED_TOOLS list is sane, and the
build helper assembles a server when the SDK is present.
"""
from __future__ import annotations
from unittest.mock import patch
import pytest
class TestModuleSurface:
def test_module_imports_clean(self):
from agent.transports import hermes_tools_mcp_server as m
assert callable(m.main)
assert callable(m._build_server)
assert isinstance(m.EXPOSED_TOOLS, tuple)
assert len(m.EXPOSED_TOOLS) > 0
def test_exposed_tools_are_safe_subset(self):
"""We MUST NOT expose tools codex already has, because codex'
own builtins are better-integrated with its sandbox + approvals.
Specifically: no terminal/shell, no read_file/write_file, no
patch those are codex's built-in tools."""
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
forbidden = {
"terminal", "shell", "read_file", "write_file", "patch",
"search_files", "process",
}
leaked = forbidden & set(EXPOSED_TOOLS)
assert not leaked, (
f"these tools must NOT be exposed via the codex callback "
f"because codex has built-in equivalents: {leaked}"
)
def test_expected_hermes_specific_tools_listed(self):
"""The Hermes-specific tools should be present so users on the
codex runtime keep access to them."""
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
for required in (
"web_search",
"web_extract",
"browser_navigate",
"vision_analyze",
"image_generate",
"skill_view",
):
assert required in EXPOSED_TOOLS, f"missing {required!r}"
def test_agent_loop_tools_not_exposed(self):
"""delegate_task / memory / session_search / todo require the
running AIAgent context to dispatch, so a stateless MCP callback
can't drive them. They must NOT be in EXPOSED_TOOLS."""
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
for agent_loop_tool in ("delegate_task", "memory", "session_search", "todo"):
assert agent_loop_tool not in EXPOSED_TOOLS, (
f"{agent_loop_tool!r} requires the agent loop context "
"and can't be reached through a stateless MCP callback"
)
def test_kanban_worker_tools_exposed(self):
"""Kanban workers run as `hermes chat -q` subprocesses; if they
come up on the codex_app_server runtime, the worker can do the
actual work via codex's shell but needs the kanban tools through
the MCP callback to report back to the kernel. Without these
tools available, the worker would hang at completion time."""
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
# Worker handoff tools — every dispatched worker uses at least
# one of {complete, block, comment} to close out its task.
for worker_tool in (
"kanban_complete",
"kanban_block",
"kanban_comment",
"kanban_heartbeat",
):
assert worker_tool in EXPOSED_TOOLS, (
f"{worker_tool!r} missing from codex callback — kanban "
"workers on codex_app_server runtime would hang"
)
def test_kanban_orchestrator_tools_exposed(self):
"""Orchestrator agents need to dispatch new tasks, query the
board, and unblock/link tasks. Exposed so an orchestrator on
codex_app_server can do its job."""
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
for orch_tool in (
"kanban_create",
"kanban_show",
"kanban_list",
"kanban_unblock",
"kanban_link",
):
assert orch_tool in EXPOSED_TOOLS, (
f"{orch_tool!r} missing from codex callback"
)
class TestMain:
def test_main_returns_2_when_mcp_unavailable(self, monkeypatch):
"""When the mcp package isn't installed, main() should exit
cleanly with code 2 and an install hint, not crash."""
import agent.transports.hermes_tools_mcp_server as m
def boom_build(*a, **kw):
raise ImportError("mcp not installed")
monkeypatch.setattr(m, "_build_server", boom_build)
rc = m.main(["--verbose"])
assert rc == 2
def test_main_handles_keyboard_interrupt(self, monkeypatch):
import agent.transports.hermes_tools_mcp_server as m
class FakeServer:
def run(self):
raise KeyboardInterrupt()
monkeypatch.setattr(m, "_build_server", lambda: FakeServer())
rc = m.main([])
assert rc == 0
def test_main_returns_1_on_runtime_error(self, monkeypatch):
import agent.transports.hermes_tools_mcp_server as m
class CrashingServer:
def run(self):
raise RuntimeError("boom")
monkeypatch.setattr(m, "_build_server", lambda: CrashingServer())
rc = m.main([])
assert rc == 1
+23 -15
View File
@@ -71,32 +71,40 @@ class TestForceFullRedraw:
"invalidate",
]
def test_resize_rebuilds_scrollback_before_prompt_toolkit_redraw(self, bare_cli, monkeypatch):
def test_resize_preserves_scrollback_and_resets_renderer(self, bare_cli, monkeypatch):
"""Resize recovery must NOT erase screen or scrollback.
The startup banner lives in normal terminal scrollback (printed
before prompt_toolkit owns the chrome). Clearing scrollback on
SIGWINCH removes it and ``_replay_output_history`` cannot
reconstruct it. The fix is to only reset the renderer cache and
let ``original_on_resize`` recalculate layout.
Additionally, ``_status_bar_suppressed_after_resize`` must be set
so the input rules and status bar hide until the next user input,
preventing duplicated-bar artifacts on column shrink (#19280).
"""
app = MagicMock()
out = app.renderer.output
events = []
out.reset_attributes.side_effect = lambda: events.append("reset_attrs")
out.erase_screen.side_effect = lambda: events.append("erase")
out.write_raw.side_effect = lambda text: events.append(("raw", text))
out.cursor_goto.side_effect = lambda *_: events.append("home")
out.flush.side_effect = lambda: events.append("flush")
app.renderer.reset.side_effect = lambda **_: events.append("renderer_reset")
monkeypatch.setattr(cli_mod, "_replay_output_history", lambda: events.append("replay"))
app.invalidate.side_effect = lambda: events.append("invalidate")
original_on_resize = lambda: events.append("original_resize")
# bare_cli skips __init__, so seed the attribute the way __init__ would.
bare_cli._status_bar_suppressed_after_resize = False
bare_cli._recover_after_resize(app, original_on_resize)
assert events == [
"reset_attrs",
"erase",
("raw", "\x1b[3J"),
"home",
"flush",
"renderer_reset",
"replay",
"invalidate",
"original_resize",
]
app.invalidate.assert_not_called()
# Must NOT clear the screen or scrollback — those destroy the banner.
app.renderer.output.erase_screen.assert_not_called()
app.renderer.output.write_raw.assert_not_called()
app.renderer.output.cursor_goto.assert_not_called()
# Status bar / input rules must be suppressed until the next prompt.
assert bare_cli._status_bar_suppressed_after_resize is True
def test_force_redraw_uses_full_screen_clear_without_scrollback_clear(self, bare_cli):
app = MagicMock()
+83
View File
@@ -319,6 +319,89 @@ class TestHistoryDisplay:
assert "Checking Running Hermes Agent" in output
assert "Use /resume <session id or title> to continue" in output
def test_sessions_command_no_args_lists_recent_sessions(self, capsys):
"""/sessions with no args prints the recent-sessions table (TUI parity).
Regression test: `sessions` was registered in the central command
registry and surfaced by /help and tab-completion, but the classic
CLI dispatcher had no elif branch for it, so the canonical name fell
through and printed `Unknown command: sessions`.
"""
cli = _make_cli()
cli.session_id = "current"
cli._session_db = MagicMock()
cli._session_db.list_sessions_rich.return_value = [
{
"id": "20260401_201329_d85961",
"title": "Checking Running Hermes Agent",
"preview": "check running gateways for hermes agent",
"last_active": 0,
},
]
# Drive it through the public dispatcher to also lock in the
# process_command wiring, not just the handler in isolation.
cli.process_command("/sessions")
output = capsys.readouterr().out
assert "Unknown command" not in output
assert "Recent sessions" in output
assert "Checking Running Hermes Agent" in output
assert "20260401_201329_d85961" in output
def test_sessions_list_subcommand_lists_recent_sessions(self, capsys):
"""/sessions list is an explicit alias for the no-arg list view."""
cli = _make_cli()
cli.session_id = "current"
cli._session_db = MagicMock()
cli._session_db.list_sessions_rich.return_value = [
{
"id": "20260401_201329_d85961",
"title": "Checking Running Hermes Agent",
"preview": "check running gateways for hermes agent",
"last_active": 0,
},
]
cli.process_command("/sessions list")
output = capsys.readouterr().out
assert "Unknown command" not in output
assert "Recent sessions" in output
assert "Checking Running Hermes Agent" in output
def test_sessions_with_target_delegates_to_resume(self):
"""/sessions <id_or_title> behaves identically to /resume <id_or_title>.
We intercept `_handle_resume_command` rather than the full resume
machinery (which would otherwise require simulating an entire session
switch). The contract under test is the dispatch wiring.
"""
cli = _make_cli()
with patch.object(cli, "_handle_resume_command") as mock_resume:
cli.process_command("/sessions Checking Running Hermes Agent")
mock_resume.assert_called_once_with(
"/resume Checking Running Hermes Agent"
)
def test_sessions_command_is_dispatched(self):
"""/sessions must hit _handle_sessions_command, not fall through.
Direct test that the process_command elif chain routes the canonical
name to the handler. Without this wiring, /sessions printed
`Unknown command: sessions` even though it was a registered command.
"""
cli = _make_cli()
cli._session_db = None # exercise the no-db path too
with patch.object(cli, "_handle_sessions_command") as mock_handler:
cli.process_command("/sessions")
mock_handler.assert_called_once()
called_with = mock_handler.call_args.args[0]
assert called_with.lower().startswith("/sessions")
class TestRootLevelProviderOverride:
"""Root-level provider/base_url in config.yaml must NOT override model.provider."""
+154
View File
@@ -0,0 +1,154 @@
"""Tests for the light-mode terminal detection + color remap in cli.py.
Covers the env-override path and the SkinConfig.get_color() wrapper that
the resize / light-mode salvage installs at module import time. We don't
try to fake an OSC 11 reply the env-override branch short-circuits
before the terminal query, which is the path most users hit.
"""
from __future__ import annotations
import importlib
import pytest
@pytest.fixture
def cli_mod(monkeypatch):
"""Import cli with the light-mode cache cleared each test."""
import cli as _cli
# The module-level _install_skin_light_mode_hook() and import-time
# _detect_light_mode() prime ran once at first import. We just reset
# the detection cache so the per-test env override takes effect.
monkeypatch.setattr(_cli, "_LIGHT_MODE_CACHE", None)
return _cli
class TestLightModeDetection:
def test_hermes_light_env_true_forces_light(self, cli_mod, monkeypatch):
monkeypatch.setenv("HERMES_LIGHT", "1")
assert cli_mod._detect_light_mode() is True
def test_hermes_light_env_false_forces_dark(self, cli_mod, monkeypatch):
monkeypatch.setenv("HERMES_LIGHT", "0")
# Also blank out other signals so nothing else flips it light.
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
monkeypatch.delenv("HERMES_TUI_THEME", raising=False)
monkeypatch.delenv("HERMES_TUI_BACKGROUND", raising=False)
monkeypatch.delenv("COLORFGBG", raising=False)
assert cli_mod._detect_light_mode() is False
def test_theme_hint_light(self, cli_mod, monkeypatch):
monkeypatch.delenv("HERMES_LIGHT", raising=False)
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
monkeypatch.setenv("HERMES_TUI_THEME", "light")
assert cli_mod._detect_light_mode() is True
def test_background_hex_hint_light(self, cli_mod, monkeypatch):
monkeypatch.delenv("HERMES_LIGHT", raising=False)
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
monkeypatch.delenv("HERMES_TUI_THEME", raising=False)
monkeypatch.setenv("HERMES_TUI_BACKGROUND", "#FFFFFF")
assert cli_mod._detect_light_mode() is True
def test_background_hex_hint_dark(self, cli_mod, monkeypatch):
monkeypatch.delenv("HERMES_LIGHT", raising=False)
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
monkeypatch.delenv("HERMES_TUI_THEME", raising=False)
monkeypatch.setenv("HERMES_TUI_BACKGROUND", "#1a1a2e")
monkeypatch.delenv("COLORFGBG", raising=False)
assert cli_mod._detect_light_mode() is False
def test_colorfgbg_light_bg_slot(self, cli_mod, monkeypatch):
monkeypatch.delenv("HERMES_LIGHT", raising=False)
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
monkeypatch.delenv("HERMES_TUI_THEME", raising=False)
monkeypatch.delenv("HERMES_TUI_BACKGROUND", raising=False)
monkeypatch.setenv("COLORFGBG", "0;15") # bg slot 15 = light
assert cli_mod._detect_light_mode() is True
def test_cache_is_sticky(self, cli_mod, monkeypatch):
monkeypatch.setenv("HERMES_LIGHT", "1")
assert cli_mod._detect_light_mode() is True
# Even if the env flips, the cached result wins until reset.
monkeypatch.setenv("HERMES_LIGHT", "0")
assert cli_mod._detect_light_mode() is True
class TestLightModeRemap:
def test_remap_no_op_in_dark_mode(self, cli_mod, monkeypatch):
monkeypatch.setenv("HERMES_LIGHT", "0")
# Cache is None from the fixture; first call sticks at False.
assert cli_mod._maybe_remap_for_light_mode("#FFF8DC") == "#FFF8DC"
def test_remap_known_dark_color(self, cli_mod, monkeypatch):
monkeypatch.setenv("HERMES_LIGHT", "1")
# Force the detect cache to True for this test.
cli_mod._LIGHT_MODE_CACHE = True
assert cli_mod._maybe_remap_for_light_mode("#FFF8DC") == "#1A1A1A"
assert cli_mod._maybe_remap_for_light_mode("#FFD700") == "#9A6B00"
def test_remap_case_insensitive(self, cli_mod, monkeypatch):
cli_mod._LIGHT_MODE_CACHE = True
# Lowercase input should still remap.
assert cli_mod._maybe_remap_for_light_mode("#fff8dc") == "#1A1A1A"
def test_remap_unknown_color_passthrough(self, cli_mod, monkeypatch):
cli_mod._LIGHT_MODE_CACHE = True
# A color not in the remap table is returned unchanged.
assert cli_mod._maybe_remap_for_light_mode("#ABCDEF") == "#ABCDEF"
def test_remap_skips_statusbar_paired_colors(self, cli_mod, monkeypatch):
"""Colors that live on a dark bg (status bar fg) MUST NOT be
remapped otherwise they go dark-on-dark and disappear.
Regression guard for the patch-11 fix (intentional table omission).
"""
cli_mod._LIGHT_MODE_CACHE = True
for fg in ("#C0C0C0", "#888888", "#555555", "#8B8682"):
assert cli_mod._maybe_remap_for_light_mode(fg) == fg, (
f"{fg} is a status-bar fg paired with dark bg; remapping it "
"would produce dark-on-dark"
)
class TestSkinConfigHook:
"""The salvage wraps SkinConfig.get_color at module import time so
every skin color read goes through the light-mode remap. Verify
the hook installed and functions correctly.
"""
def test_hook_installed(self, cli_mod):
from hermes_cli.skin_engine import SkinConfig
assert getattr(SkinConfig, "_hermes_light_mode_hook_installed", False) is True
def test_hook_is_idempotent(self, cli_mod):
# Calling the installer twice must not double-wrap (the marker
# attribute is the guard).
from hermes_cli.skin_engine import SkinConfig
before = SkinConfig.get_color
cli_mod._install_skin_light_mode_hook()
after = SkinConfig.get_color
assert before is after
def test_skin_color_remaps_through_wrapper_in_light_mode(self, cli_mod, monkeypatch):
from hermes_cli.skin_engine import SkinConfig
cli_mod._LIGHT_MODE_CACHE = True
skin = SkinConfig(
name="test",
colors={"banner_text": "#FFF8DC", "response_border": "#FFD700"},
)
# The wrapper kicks in at get_color, not at construction time.
assert skin.get_color("banner_text") == "#1A1A1A"
assert skin.get_color("response_border") == "#9A6B00"
def test_skin_color_passthrough_in_dark_mode(self, cli_mod, monkeypatch):
from hermes_cli.skin_engine import SkinConfig
cli_mod._LIGHT_MODE_CACHE = False
skin = SkinConfig(name="test", colors={"banner_text": "#FFF8DC"})
assert skin.get_color("banner_text") == "#FFF8DC"
+59 -2
View File
@@ -531,8 +531,8 @@ def test_model_flow_custom_saves_verified_v1_base_url(monkeypatch, capsys):
# After the probe detects a single model ("llm"), the flow asks
# "Use this model? [Y/n]:" — confirm with Enter, then context length,
# then display name.
answers = iter(["http://localhost:8000", "local-key", "", "", "", ""])
# then display name. The api_mode prompt also runs before model selection.
answers = iter(["http://localhost:8000", "local-key", "", "", "", "", ""])
monkeypatch.setattr("builtins.input", lambda _prompt="": next(answers))
monkeypatch.setattr("getpass.getpass", lambda _prompt="": next(answers))
@@ -546,6 +546,63 @@ def test_model_flow_custom_saves_verified_v1_base_url(monkeypatch, capsys):
assert saved_env["MODEL"] == "llm"
def test_model_flow_custom_persists_selected_api_mode(monkeypatch):
saved_cfg = {"model": {"default": "", "provider": "custom", "base_url": ""}}
captured_provider = {}
monkeypatch.setattr(
"hermes_cli.config.get_env_value",
lambda key: "" if key in {"OPENAI_BASE_URL", "OPENAI_API_KEY"} else "",
)
monkeypatch.setattr("hermes_cli.auth._save_model_choice", lambda model: None)
monkeypatch.setattr("hermes_cli.auth.deactivate_provider", lambda: None)
monkeypatch.setattr(
"hermes_cli.models.probe_api_models",
lambda api_key, base_url: {
"models": [],
"probed_url": f"{base_url.rstrip('/')}/models",
"resolved_base_url": None,
"suggested_base_url": None,
"used_fallback": False,
},
)
monkeypatch.setattr("hermes_cli.config.load_config", lambda: saved_cfg)
monkeypatch.setattr("hermes_cli.config.save_config", lambda cfg: saved_cfg.update(cfg))
monkeypatch.setattr(
"hermes_cli.main._save_custom_provider",
lambda base_url, api_key="", model="", context_length=None, name=None, api_mode=None: captured_provider.update(
{
"base_url": base_url,
"api_key": api_key,
"model": model,
"context_length": context_length,
"name": name,
"api_mode": api_mode,
}
),
)
answers = iter(
[
"https://codex.example.com/v1",
"3",
"chosen-model",
"",
"",
]
)
monkeypatch.setattr("builtins.input", lambda _prompt="": next(answers))
monkeypatch.setattr("getpass.getpass", lambda _prompt="": "test-key")
hermes_main._model_flow_custom({"model": {"provider": "custom"}})
assert saved_cfg["model"]["provider"] == "custom"
assert saved_cfg["model"]["base_url"] == "https://codex.example.com/v1"
assert saved_cfg["model"]["api_key"] == "test-key"
assert saved_cfg["model"]["api_mode"] == "codex_responses"
assert captured_provider["api_mode"] == "codex_responses"
def test_cmd_model_forwards_nous_login_tls_options(monkeypatch):
monkeypatch.setattr(hermes_main, "_require_tty", lambda *a: None)
monkeypatch.setattr(
+39
View File
@@ -332,6 +332,45 @@ class TestCLIStatusBar:
assert cli_obj._tui_input_rule_height("bottom", width=50) == 0
assert cli_obj._tui_input_rule_height("bottom", width=90) == 1
def test_input_rules_hide_after_resize_until_next_input(self):
"""When _status_bar_suppressed_after_resize is set, both rules hide.
See _recover_after_resize column shrink reflows already-rendered
bars into scrollback, so we hide the separators until the user
submits the next input, at which point the flag is cleared.
"""
cli_obj = _make_cli()
cli_obj._status_bar_suppressed_after_resize = True
assert cli_obj._tui_input_rule_height("top", width=90) == 0
assert cli_obj._tui_input_rule_height("bottom", width=90) == 0
cli_obj._status_bar_suppressed_after_resize = False
assert cli_obj._tui_input_rule_height("top", width=90) == 1
assert cli_obj._tui_input_rule_height("bottom", width=90) == 1
def test_scrollback_box_width_returns_viewport_width(self):
"""Decorative scrollback boxes use the full viewport width.
The previous clamp (max 56 cols) was reverted in favour of the
prompt_toolkit ``_output_screen_diff`` monkey-patch landed in
#26137, which keeps chrome out of scrollback at the source.
We accept that an aggressive column-shrink may visually reflow
already printed Panel borders that's a cosmetic artifact of
stamped scrollback history, not a live-render bug.
"""
from cli import HermesCLI
# Floor at 32 — narrow terminals still get something usable
# (avoids negative ``'─' * (w - 2)`` math).
assert HermesCLI._scrollback_box_width(20) == 32
assert HermesCLI._scrollback_box_width(32) == 32
# Above the floor, return the actual viewport width — no cap.
assert HermesCLI._scrollback_box_width(48) == 48
assert HermesCLI._scrollback_box_width(80) == 80
assert HermesCLI._scrollback_box_width(120) == 120
assert HermesCLI._scrollback_box_width(200) == 200
def test_agent_spacer_reclaimed_on_narrow_terminals(self):
cli_obj = _make_cli()
cli_obj._agent_running = True
+30 -3
View File
@@ -215,13 +215,15 @@ def test_cprint_swallows_prompt_toolkit_import_error(monkeypatch):
assert direct_prints == ["fallback2"]
def test_output_history_strips_ansi_and_keeps_recent_lines():
def test_output_history_preserves_ansi_and_keeps_recent_lines():
cli._configure_output_history(True, 10)
for idx in range(12):
cli._record_output_history(f"\x1b[31mline-{idx}\x1b[0m")
assert list(cli._OUTPUT_HISTORY) == [f"line-{idx}" for idx in range(2, 12)]
assert list(cli._OUTPUT_HISTORY) == [
f"\x1b[31mline-{idx}\x1b[0m" for idx in range(2, 12)
]
def test_replay_output_history_does_not_record_replayed_lines(monkeypatch):
@@ -258,10 +260,35 @@ def test_replay_output_history_rerenders_callable_entries(monkeypatch):
cli._replay_output_history()
assert widths_seen == ["called"]
assert printed == ["top border", "body"]
assert printed == ["top border\nbody"]
assert list(cli._OUTPUT_HISTORY) == [_render_current_width]
def test_replay_output_history_batches_rendered_lines_into_one_print(monkeypatch):
cli._configure_output_history(True, 10)
cli._record_output_history("first line")
cli._record_output_history("second line")
cli._record_output_history_entry(lambda: ["third line", "fourth line"])
printed = []
monkeypatch.setattr(cli, "_pt_print", lambda value: printed.append(value))
monkeypatch.setattr(cli, "_PT_ANSI", lambda text: text)
cli._replay_output_history()
assert printed == ["first line\nsecond line\nthird line\nfourth line"]
def test_chat_console_records_rich_ansi_for_resize_replay(monkeypatch):
cli._configure_output_history(True, 10)
monkeypatch.setattr(cli, "_pt_print", lambda *_args, **_kwargs: None)
cli.ChatConsole().print("[bold red]Hello[/]")
assert cli._OUTPUT_HISTORY
assert any("\x1b[" in line for line in cli._OUTPUT_HISTORY)
def test_suspend_output_history_blocks_recording():
cli._configure_output_history(True, 10)
+5 -4
View File
@@ -101,7 +101,6 @@ _CREDENTIAL_NAMES = frozenset({
"RETAINDB_API_KEY",
"HINDSIGHT_API_KEY",
"HINDSIGHT_LLM_API_KEY",
"TINKER_API_KEY",
"DAYTONA_API_KEY",
"TWILIO_AUTH_TOKEN",
"TELEGRAM_BOT_TOKEN",
@@ -476,12 +475,14 @@ def _reset_module_state():
except Exception:
pass
# --- agent.auxiliary_client — runtime main provider/model override ---
# Set per-turn by AIAgent.run_conversation; tests that import it must
# see a clean state so config.yaml fallback works as expected.
# --- agent.auxiliary_client — runtime main provider/model override and
# payment-error health cache. Both are process-global in production;
# reset them per test so one worker's fallback/402 test does not make
# later auxiliary-client tests skip otherwise-available providers.
try:
from agent import auxiliary_client as _aux_mod
_aux_mod.clear_runtime_main()
_aux_mod._reset_aux_unhealthy_cache()
except Exception:
pass
+87
View File
@@ -321,6 +321,93 @@ class TestPauseResumeJob:
assert resumed["paused_reason"] is None
class TestResolveJobRef:
"""Name-based job lookup for CLI/tool callers (PR #2627, @buntingszn)."""
def test_resolve_by_exact_id(self, tmp_cron_dir):
from cron.jobs import resolve_job_ref
job = create_job(prompt="A", schedule="1h", name="alpha")
assert resolve_job_ref(job["id"])["id"] == job["id"]
def test_resolve_by_name(self, tmp_cron_dir):
from cron.jobs import resolve_job_ref
job = create_job(prompt="A", schedule="1h", name="alpha")
assert resolve_job_ref("alpha")["id"] == job["id"]
def test_resolve_by_name_case_insensitive(self, tmp_cron_dir):
from cron.jobs import resolve_job_ref
job = create_job(prompt="A", schedule="1h", name="MyJob")
assert resolve_job_ref("myjob")["id"] == job["id"]
assert resolve_job_ref("MYJOB")["id"] == job["id"]
def test_resolve_returns_none_when_not_found(self, tmp_cron_dir):
from cron.jobs import resolve_job_ref
create_job(prompt="A", schedule="1h", name="alpha")
assert resolve_job_ref("does-not-exist") is None
assert resolve_job_ref("") is None
def test_resolve_id_wins_over_name(self, tmp_cron_dir):
"""If a job's name happens to equal another job's ID, ID match wins."""
from cron.jobs import resolve_job_ref
j1 = create_job(prompt="A", schedule="1h")
# Create a second job whose name is j1's ID
j2 = create_job(prompt="B", schedule="1h", name=j1["id"])
# Looking up j1["id"] must return j1, not the colliding-name job j2
assert resolve_job_ref(j1["id"])["id"] == j1["id"]
assert resolve_job_ref(j1["id"])["id"] != j2["id"]
def test_resolve_ambiguous_name_raises(self, tmp_cron_dir):
"""Two jobs sharing a name → refuse to pick, surface both IDs."""
from cron.jobs import AmbiguousJobReference, resolve_job_ref
j1 = create_job(prompt="A", schedule="1h", name="dup")
j2 = create_job(prompt="B", schedule="1h", name="dup")
with pytest.raises(AmbiguousJobReference) as exc_info:
resolve_job_ref("dup")
ids = {m["id"] for m in exc_info.value.matches}
assert ids == {j1["id"], j2["id"]}
# Error message mentions both IDs so the user can pick one
assert j1["id"] in str(exc_info.value)
assert j2["id"] in str(exc_info.value)
def test_trigger_by_name(self, tmp_cron_dir):
from cron.jobs import trigger_job
job = create_job(prompt="A", schedule="1h", name="alpha")
result = trigger_job("alpha")
assert result is not None
assert result["id"] == job["id"]
def test_pause_by_name(self, tmp_cron_dir):
job = create_job(prompt="A", schedule="1h", name="alpha")
result = pause_job("alpha", reason="manual")
assert result is not None
assert result["id"] == job["id"]
assert result["state"] == "paused"
def test_remove_by_name(self, tmp_cron_dir):
job = create_job(prompt="A", schedule="1h", name="alpha")
assert remove_job("alpha") is True
assert get_job(job["id"]) is None
def test_mutations_refuse_ambiguous_name(self, tmp_cron_dir):
"""pause/resume/trigger/remove must refuse to act on an ambiguous name."""
from cron.jobs import AmbiguousJobReference, trigger_job
create_job(prompt="A", schedule="1h", name="dup")
create_job(prompt="B", schedule="1h", name="dup")
for fn in (pause_job, resume_job, trigger_job):
with pytest.raises(AmbiguousJobReference):
fn("dup")
with pytest.raises(AmbiguousJobReference):
remove_job("dup")
class TestMarkJobRun:
def test_increments_completed(self, tmp_cron_dir):
job = create_job(prompt="Test", schedule="every 1h")
@@ -1,164 +0,0 @@
"""Security tests for Terminal-Bench 2 archive extraction."""
import base64
import importlib
import io
import sys
import tarfile
import types
import pytest
def _stub_module(name: str, **attrs):
module = types.ModuleType(name)
for key, value in attrs.items():
setattr(module, key, value)
return module
def _load_terminalbench_module(monkeypatch):
class _EvalHandlingEnum:
STOP_TRAIN = "stop_train"
class _APIServerConfig:
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
class _AgentResult:
pass
class _HermesAgentLoop:
pass
class _HermesAgentBaseEnv:
pass
class _HermesAgentEnvConfig:
pass
class _ToolContext:
pass
stub_modules = {
"atroposlib": _stub_module("atroposlib"),
"atroposlib.envs": _stub_module("atroposlib.envs"),
"atroposlib.envs.base": _stub_module(
"atroposlib.envs.base",
EvalHandlingEnum=_EvalHandlingEnum,
),
"atroposlib.envs.server_handling": _stub_module("atroposlib.envs.server_handling"),
"atroposlib.envs.server_handling.server_manager": _stub_module(
"atroposlib.envs.server_handling.server_manager",
APIServerConfig=_APIServerConfig,
),
"environments.agent_loop": _stub_module(
"environments.agent_loop",
AgentResult=_AgentResult,
HermesAgentLoop=_HermesAgentLoop,
),
"environments.hermes_base_env": _stub_module(
"environments.hermes_base_env",
HermesAgentBaseEnv=_HermesAgentBaseEnv,
HermesAgentEnvConfig=_HermesAgentEnvConfig,
),
"environments.tool_context": _stub_module(
"environments.tool_context",
ToolContext=_ToolContext,
),
"tools.terminal_tool": _stub_module(
"tools.terminal_tool",
register_task_env_overrides=lambda *args, **kwargs: None,
clear_task_env_overrides=lambda *args, **kwargs: None,
cleanup_vm=lambda *args, **kwargs: None,
),
}
stub_modules["atroposlib"].envs = stub_modules["atroposlib.envs"]
stub_modules["atroposlib.envs"].base = stub_modules["atroposlib.envs.base"]
stub_modules["atroposlib.envs"].server_handling = stub_modules["atroposlib.envs.server_handling"]
stub_modules["atroposlib.envs.server_handling"].server_manager = stub_modules[
"atroposlib.envs.server_handling.server_manager"
]
for name, module in stub_modules.items():
monkeypatch.setitem(sys.modules, name, module)
module_name = "environments.benchmarks.terminalbench_2.terminalbench2_env"
sys.modules.pop(module_name, None)
return importlib.import_module(module_name)
def _build_tar_b64(entries):
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
for entry in entries:
kind = entry["kind"]
info = tarfile.TarInfo(entry["name"])
if kind == "dir":
info.type = tarfile.DIRTYPE
tar.addfile(info)
continue
if kind == "file":
data = entry["data"].encode("utf-8")
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
continue
if kind == "symlink":
info.type = tarfile.SYMTYPE
info.linkname = entry["target"]
tar.addfile(info)
continue
raise ValueError(f"Unknown tar entry kind: {kind}")
return base64.b64encode(buf.getvalue()).decode("ascii")
def test_extract_base64_tar_allows_safe_files(tmp_path, monkeypatch):
module = _load_terminalbench_module(monkeypatch)
archive = _build_tar_b64(
[
{"kind": "dir", "name": "nested"},
{"kind": "file", "name": "nested/hello.txt", "data": "hello"},
]
)
target = tmp_path / "extract"
module._extract_base64_tar(archive, target)
assert (target / "nested" / "hello.txt").read_text(encoding="utf-8") == "hello"
def test_extract_base64_tar_rejects_path_traversal(tmp_path, monkeypatch):
module = _load_terminalbench_module(monkeypatch)
archive = _build_tar_b64(
[
{"kind": "file", "name": "../escape.txt", "data": "owned"},
]
)
target = tmp_path / "extract"
with pytest.raises(ValueError, match="Unsafe archive member path"):
module._extract_base64_tar(archive, target)
assert not (tmp_path / "escape.txt").exists()
def test_extract_base64_tar_rejects_symlinks(tmp_path, monkeypatch):
module = _load_terminalbench_module(monkeypatch)
archive = _build_tar_b64(
[
{"kind": "symlink", "name": "link", "target": "../../escape.txt"},
]
)
target = tmp_path / "extract"
with pytest.raises(ValueError, match="Unsupported archive member type"):
module._extract_base64_tar(archive, target)
assert not (target / "link").exists()
+8
View File
@@ -119,6 +119,14 @@ def _ensure_discord_mock() -> None:
self.title = title
self.description = description
self.color = color
self.fields = []
self.footer = None
def add_field(self, *, name=None, value=None, inline=False, **_):
self.fields.append({"name": name, "value": value, "inline": inline})
return self
def set_footer(self, *, text=None, icon_url=None, **_):
self.footer = {"text": text, "icon_url": icon_url}
return self
discord_mod.Embed = _FakeEmbed
# ui.View / ui.Select / ui.Button: real classes (not MagicMock) so
+62
View File
@@ -105,6 +105,29 @@ class TestResponseStore:
store = ResponseStore(max_size=10)
assert store.delete("resp_missing") is False
def test_delete_clears_conversation_mapping(self):
"""Deleting a response also removes conversation mappings that reference it."""
store = ResponseStore(max_size=10)
store.put("resp_1", {"output": "hello"})
store.set_conversation("chat-a", "resp_1")
assert store.get_conversation("chat-a") == "resp_1"
store.delete("resp_1")
assert store.get_conversation("chat-a") is None
def test_eviction_clears_conversation_mapping(self):
"""LRU eviction also removes conversation mappings for evicted responses."""
store = ResponseStore(max_size=2)
store.put("resp_1", {"output": "one"})
store.set_conversation("chat-a", "resp_1")
store.put("resp_2", {"output": "two"})
store.set_conversation("chat-b", "resp_2")
# Adding a 3rd should evict resp_1 and its conversation mapping
store.put("resp_3", {"output": "three"})
assert store.get("resp_1") is None
assert store.get_conversation("chat-a") is None
# resp_2 mapping should still be intact
assert store.get_conversation("chat-b") == "resp_2"
# ---------------------------------------------------------------------------
# _IdempotencyCache
@@ -2870,6 +2893,45 @@ class TestConversationParameter:
# Conversation mapping should NOT be set since store=false
assert adapter._response_store.get_conversation("ephemeral-chat") is None
@pytest.mark.asyncio
async def test_conversation_reuse_after_eviction_no_404(self, adapter):
"""After eviction clears a conversation mapping, reusing that name starts fresh (no 404)."""
adapter._response_store = ResponseStore(max_size=1)
app = _create_app(adapter)
async with TestClient(TestServer(app)) as cli:
with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
mock_run.return_value = (
{"final_response": "First", "messages": [], "api_calls": 1},
{"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
)
# Create conversation -> resp stored
resp1 = await cli.post("/v1/responses", json={
"input": "hello",
"conversation": "my-chat",
})
assert resp1.status == 200
# Evict by adding another response
mock_run.return_value = (
{"final_response": "Other", "messages": [], "api_calls": 1},
{"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
)
await cli.post("/v1/responses", json={"input": "other"})
# Conversation mapping should have been cleaned by eviction
assert adapter._response_store.get_conversation("my-chat") is None
# Reuse conversation name — should start fresh, not 404
mock_run.return_value = (
{"final_response": "Restarted", "messages": [], "api_calls": 1},
{"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
)
resp3 = await cli.post("/v1/responses", json={
"input": "hello again",
"conversation": "my-chat",
})
assert resp3.status == 200
# ---------------------------------------------------------------------------
# X-Hermes-Session-Id header (session continuity)
+57
View File
@@ -302,6 +302,43 @@ class TestLoadGatewayConfig:
assert config.thread_sessions_per_user is False
def test_bridges_discord_thread_require_mention_from_config_yaml(self, tmp_path, monkeypatch):
"""discord.thread_require_mention in config.yaml should reach the runtime env var."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
config_path = hermes_home / "config.yaml"
config_path.write_text(
"discord:\n"
" thread_require_mention: true\n",
encoding="utf-8",
)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.delenv("DISCORD_THREAD_REQUIRE_MENTION", raising=False)
load_gateway_config()
assert os.environ.get("DISCORD_THREAD_REQUIRE_MENTION") == "true"
def test_thread_require_mention_yaml_does_not_overwrite_env(self, tmp_path, monkeypatch):
"""Explicit env var should win over config.yaml (env > yaml precedence)."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
config_path = hermes_home / "config.yaml"
config_path.write_text(
"discord:\n"
" thread_require_mention: false\n",
encoding="utf-8",
)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.setenv("DISCORD_THREAD_REQUIRE_MENTION", "true") # user override
load_gateway_config()
# Env value preserved, not clobbered by yaml.
assert os.environ.get("DISCORD_THREAD_REQUIRE_MENTION") == "true"
def test_bridges_quoted_false_platform_enabled_from_config_yaml(self, tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
@@ -372,6 +409,26 @@ class TestLoadGatewayConfig:
"456": "Therapist mode",
}
def test_bridges_discord_history_backfill_settings_from_config_yaml(self, tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
config_path = hermes_home / "config.yaml"
config_path.write_text(
"discord:\n"
" history_backfill: true\n"
" history_backfill_limit: 17\n",
encoding="utf-8",
)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.delenv("DISCORD_HISTORY_BACKFILL", raising=False)
monkeypatch.delenv("DISCORD_HISTORY_BACKFILL_LIMIT", raising=False)
load_gateway_config()
assert os.getenv("DISCORD_HISTORY_BACKFILL") == "true"
assert os.getenv("DISCORD_HISTORY_BACKFILL_LIMIT") == "17"
def test_bridges_telegram_channel_prompts_from_config_yaml(self, tmp_path, monkeypatch):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
+76 -1
View File
@@ -10,6 +10,80 @@ import pytest
from gateway.config import Platform, PlatformConfig
class _FakeDingTalkModel:
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
class _FakeChatbotMessage(SimpleNamespace):
@classmethod
def from_dict(cls, data):
data = data or {}
return cls(
message_id=data.get("msgId") or data.get("messageId") or data.get("message_id") or "",
conversation_id=data.get("conversationId") or data.get("conversation_id") or "",
conversation_type=str(data.get("conversationType") or data.get("conversation_type") or "1"),
sender_id=data.get("senderId") or data.get("sender_id") or "",
sender_staff_id=data.get("senderStaffId") or data.get("sender_staff_id") or data.get("senderId") or "",
sender_nick=data.get("senderNick") or data.get("sender_nick") or "",
text=data.get("text") or "",
rich_text=data.get("richText") or data.get("rich_text"),
rich_text_content=data.get("richTextContent") or data.get("rich_text_content"),
session_webhook=data.get("sessionWebhook") or data.get("session_webhook") or "",
session_webhook_expired_time=data.get("sessionWebhookExpiredTime") or data.get("session_webhook_expired_time") or 0,
create_at=data.get("createAt") or data.get("create_at") or 0,
at_users=data.get("atUsers") or data.get("at_users") or [],
is_in_at_list=bool(data.get("isInAtList") or data.get("is_in_at_list")),
)
@pytest.fixture(autouse=True)
def _fake_dingtalk_optional_sdks(monkeypatch):
"""Keep DingTalk adapter tests hermetic when optional SDKs are absent."""
from gateway.platforms import dingtalk as dt
card_models = SimpleNamespace(**{
name: _FakeDingTalkModel
for name in (
"CreateCardRequest",
"CreateCardRequestCardData",
"CreateCardRequestImGroupOpenSpaceModel",
"CreateCardRequestImRobotOpenSpaceModel",
"CreateCardHeaders",
"DeliverCardRequest",
"DeliverCardRequestImGroupOpenDeliverModel",
"DeliverCardRequestImRobotOpenDeliverModel",
"DeliverCardHeaders",
"StreamingUpdateRequest",
"StreamingUpdateHeaders",
)
})
robot_models = SimpleNamespace(**{
name: _FakeDingTalkModel
for name in (
"RobotReplyEmotionRequestTextEmotion",
"RobotReplyEmotionRequest",
"RobotReplyEmotionHeaders",
"RobotRecallEmotionRequestTextEmotion",
"RobotRecallEmotionRequest",
"RobotRecallEmotionHeaders",
"RobotMessageFileDownloadRequest",
"RobotMessageFileDownloadHeaders",
)
})
monkeypatch.setattr(dt, "ChatbotMessage", _FakeChatbotMessage, raising=False)
monkeypatch.setattr(
dt,
"AckMessage",
SimpleNamespace(STATUS_OK=200, STATUS_SYSTEM_EXCEPTION=500),
raising=False,
)
monkeypatch.setattr(dt, "tea_util_models", SimpleNamespace(RuntimeOptions=_FakeDingTalkModel), raising=False)
monkeypatch.setattr(dt, "dingtalk_card_models", card_models, raising=False)
monkeypatch.setattr(dt, "dingtalk_robot_models", robot_models, raising=False)
# ---------------------------------------------------------------------------
# Requirements check
# ---------------------------------------------------------------------------
@@ -18,7 +92,8 @@ from gateway.config import Platform, PlatformConfig
class TestDingTalkRequirements:
def test_returns_false_when_sdk_missing(self, monkeypatch):
with patch.dict("sys.modules", {"dingtalk_stream": None}):
with patch.dict("sys.modules", {"dingtalk_stream": None}), \
patch("tools.lazy_deps.ensure", side_effect=ImportError("dingtalk_stream unavailable")):
monkeypatch.setattr(
"gateway.platforms.dingtalk.DINGTALK_STREAM_AVAILABLE", False
)
@@ -0,0 +1,408 @@
"""Tests for Discord clarify button rendering and resolution.
Mirrors test_telegram_clarify_buttons.py for the Discord ``send_clarify``
override and the ``ClarifyChoiceView`` callbacks. Discord uses ``discord.ui.View``
button callbacks (closures) rather than a string-prefixed callback_query
dispatcher like Telegram the auth + resolution path is the same:
· numeric choice resolve_gateway_clarify(clarify_id, choice_text)
· "Other" button mark_awaiting_text(clarify_id) so the text-intercept
captures the next user message in this session
· already-resolved or unauthorized ephemeral "this prompt..." reply
"""
import asyncio
import sys
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
# Repo root importable
_repo = str(Path(__file__).resolve().parents[2])
if _repo not in sys.path:
sys.path.insert(0, _repo)
# Triggers the shared discord mock from tests/gateway/conftest.py before
# importing the production module.
from gateway.platforms.discord import ( # noqa: E402
ClarifyChoiceView,
DiscordAdapter,
)
from gateway.config import PlatformConfig # noqa: E402
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_adapter(*, allowed_users=None, allowed_roles=None):
config = PlatformConfig(enabled=True, token="test-token", extra={})
adapter = DiscordAdapter(config)
adapter._client = MagicMock()
adapter._allowed_user_ids = set(allowed_users or [])
adapter._allowed_role_ids = set(allowed_roles or [])
return adapter
def _clear_clarify_state():
from tools import clarify_gateway as cm
with cm._lock:
cm._entries.clear()
cm._session_index.clear()
cm._notify_cbs.clear()
def _make_interaction(*, user_id="42", display_name="Tester", roles=None,
include_message=True):
"""Build a mock discord.Interaction with response.edit_message /
send_message / defer all coroutine-callable."""
user = SimpleNamespace(
id=user_id,
display_name=display_name,
roles=[SimpleNamespace(id=r) for r in (roles or [])],
)
response = SimpleNamespace(
edit_message=AsyncMock(),
send_message=AsyncMock(),
defer=AsyncMock(),
)
if include_message:
embed = MagicMock()
embed.color = None
embed.set_footer = MagicMock()
message = SimpleNamespace(embeds=[embed])
else:
message = None
return SimpleNamespace(user=user, response=response, message=message)
# ===========================================================================
# ClarifyChoiceView construction
# ===========================================================================
class TestClarifyChoiceViewConstruction:
"""The view should build numeric buttons plus an Other button."""
def test_renders_n_choice_buttons_plus_other(self):
view = ClarifyChoiceView(
choices=["apple", "banana", "cherry"],
clarify_id="cidX",
allowed_user_ids={"42"},
)
# 3 numeric + 1 "Other"
assert len(view.children) == 4
labels = [b.label for b in view.children]
assert labels[0].startswith("1. apple")
assert labels[1].startswith("2. banana")
assert labels[2].startswith("3. cherry")
assert "Other" in labels[3]
# custom_ids encode clarify_id + index/other
ids = [b.custom_id for b in view.children]
assert ids[0] == "clarify:cidX:0"
assert ids[1] == "clarify:cidX:1"
assert ids[2] == "clarify:cidX:2"
assert ids[3] == "clarify:cidX:other"
def test_caps_at_24_choices_plus_other(self):
choices = [f"choice-{i}" for i in range(50)]
view = ClarifyChoiceView(
choices=choices,
clarify_id="cidY",
allowed_user_ids=set(),
)
# Discord limit is 25 components; we cap choices at 24 + 1 Other = 25
assert len(view.children) == 25
assert "Other" in view.children[-1].label
def test_truncates_long_choice_label(self):
long_choice = "x" * 200
view = ClarifyChoiceView(
choices=[long_choice],
clarify_id="cidZ",
allowed_user_ids=set(),
)
# 75 chars + 3 ellipsis chars in the body, plus "1. " prefix
first_label = view.children[0].label
assert first_label.startswith("1. ")
assert first_label.endswith("...")
# Final label total <= 80 (Discord cap on button labels)
assert len(first_label) <= 80
# ===========================================================================
# Choice callback → resolve_gateway_clarify
# ===========================================================================
class TestClarifyChoiceResolve:
"""Clicking a numeric button should resolve the clarify entry."""
def setup_method(self):
_clear_clarify_state()
@pytest.mark.asyncio
async def test_choice_resolves_with_canonical_choice_text(self):
from tools import clarify_gateway as cm
cm.register("cidA", "sk-A", "Pick", ["red", "green", "blue"])
view = ClarifyChoiceView(
choices=["red", "green", "blue"],
clarify_id="cidA",
allowed_user_ids={"42"},
)
interaction = _make_interaction(user_id="42")
await view._resolve_choice(interaction, index=1, choice="green")
# Resolved through clarify primitive
with cm._lock:
entry = cm._entries.get("cidA")
assert entry is not None
assert entry.response == "green"
assert entry.event.is_set()
# Buttons disabled
assert all(b.disabled for b in view.children)
# Embed updated + edit_message called
interaction.response.edit_message.assert_called_once()
@pytest.mark.asyncio
async def test_choice_falls_back_to_label_text_when_entry_missing(self):
"""If the gateway entry vanished (race / stale view), the button's
own choice text is used as the response."""
from tools import clarify_gateway as cm
# Note: no cm.register() — entry intentionally absent
view = ClarifyChoiceView(
choices=["alpha"],
clarify_id="cidGone",
allowed_user_ids=set(),
)
interaction = _make_interaction()
# Doesn't raise; resolve_gateway_clarify returns False quietly
await view._resolve_choice(interaction, index=0, choice="alpha")
# Still marks the view resolved + disables buttons
assert view.resolved is True
assert all(b.disabled for b in view.children)
@pytest.mark.asyncio
async def test_already_resolved_sends_ephemeral_reply(self):
view = ClarifyChoiceView(
choices=["a", "b"],
clarify_id="cidB",
allowed_user_ids=set(),
)
view.resolved = True
interaction = _make_interaction()
await view._resolve_choice(interaction, index=0, choice="a")
interaction.response.send_message.assert_called_once()
kwargs = interaction.response.send_message.call_args.kwargs
assert kwargs.get("ephemeral") is True
# No resolve was called
interaction.response.edit_message.assert_not_called()
@pytest.mark.asyncio
async def test_unauthorized_user_rejected(self):
from tools import clarify_gateway as cm
cm.register("cidC", "sk-C", "Pick", ["x"])
# Allowlist set, user not in it
view = ClarifyChoiceView(
choices=["x"],
clarify_id="cidC",
allowed_user_ids={"99999"}, # not 42
)
interaction = _make_interaction(user_id="42")
await view._resolve_choice(interaction, index=0, choice="x")
# Ephemeral rejection, no resolution, no edit
interaction.response.send_message.assert_called_once()
kwargs = interaction.response.send_message.call_args.kwargs
assert kwargs.get("ephemeral") is True
interaction.response.edit_message.assert_not_called()
with cm._lock:
entry = cm._entries.get("cidC")
assert entry is not None
assert not entry.event.is_set()
# ===========================================================================
# "Other" button → mark_awaiting_text
# ===========================================================================
class TestClarifyOtherButton:
"""Clicking Other should flip the entry into text-capture mode."""
def setup_method(self):
_clear_clarify_state()
@pytest.mark.asyncio
async def test_other_flips_entry_to_awaiting_text(self):
from tools import clarify_gateway as cm
cm.register("cidD", "sk-D", "Pick", ["x", "y"])
view = ClarifyChoiceView(
choices=["x", "y"],
clarify_id="cidD",
allowed_user_ids=set(),
)
interaction = _make_interaction()
await view._on_other(interaction)
# Entry awaiting_text now
pending = cm.get_pending_for_session("sk-D")
assert pending is not None
assert pending.clarify_id == "cidD"
assert pending.awaiting_text is True
# Entry still pending (not resolved)
with cm._lock:
entry = cm._entries.get("cidD")
assert entry is not None
assert not entry.event.is_set()
# View locked + buttons disabled
assert view.resolved is True
assert all(b.disabled for b in view.children)
interaction.response.edit_message.assert_called_once()
@pytest.mark.asyncio
async def test_other_unauthorized_user_rejected(self):
from tools import clarify_gateway as cm
cm.register("cidE", "sk-E", "Pick", ["x"])
view = ClarifyChoiceView(
choices=["x"],
clarify_id="cidE",
allowed_user_ids={"99999"},
)
interaction = _make_interaction(user_id="42")
await view._on_other(interaction)
# Rejected; entry NOT awaiting text
interaction.response.send_message.assert_called_once()
pending = cm.get_pending_for_session("sk-E")
assert pending is None or pending.awaiting_text is False
# ===========================================================================
# DiscordAdapter.send_clarify integration
# ===========================================================================
class TestDiscordSendClarify:
"""Verify send_clarify renders an embed and (optionally) attaches the view."""
def setup_method(self):
_clear_clarify_state()
@pytest.mark.asyncio
async def test_multi_choice_attaches_view(self):
adapter = _make_adapter(allowed_users={"42"})
channel = MagicMock()
sent_msg = MagicMock()
sent_msg.id = 123456
channel.send = AsyncMock(return_value=sent_msg)
adapter._client.get_channel = MagicMock(return_value=channel)
result = await adapter.send_clarify(
chat_id="9001",
question="Pick a color",
choices=["red", "green", "blue"],
clarify_id="cidM",
session_key="sk-M",
)
assert result.success is True
assert result.message_id == "123456"
# Verify channel.send was called with embed + view kwargs
channel.send.assert_called_once()
kwargs = channel.send.call_args.kwargs
assert "embed" in kwargs
assert "view" in kwargs
assert isinstance(kwargs["view"], ClarifyChoiceView)
# 3 choice buttons + 1 Other
assert len(kwargs["view"].children) == 4
@pytest.mark.asyncio
async def test_open_ended_omits_view(self):
adapter = _make_adapter()
channel = MagicMock()
sent_msg = MagicMock()
sent_msg.id = 222
channel.send = AsyncMock(return_value=sent_msg)
adapter._client.get_channel = MagicMock(return_value=channel)
result = await adapter.send_clarify(
chat_id="9001",
question="What is your name?",
choices=None,
clarify_id="cidOE",
session_key="sk-OE",
)
assert result.success is True
channel.send.assert_called_once()
kwargs = channel.send.call_args.kwargs
# Open-ended path renders embed but no view (text-capture handles reply)
assert "embed" in kwargs
assert "view" not in kwargs
@pytest.mark.asyncio
async def test_routes_to_thread_when_metadata_thread_id_set(self):
adapter = _make_adapter()
channel = MagicMock()
sent_msg = MagicMock()
sent_msg.id = 333
channel.send = AsyncMock(return_value=sent_msg)
adapter._client.get_channel = MagicMock(return_value=channel)
await adapter.send_clarify(
chat_id="9001",
question="?",
choices=["a"],
clarify_id="cidT",
session_key="sk-T",
metadata={"thread_id": "7777"},
)
# Channel lookup should resolve to thread id, not chat_id
adapter._client.get_channel.assert_called_once_with(7777)
@pytest.mark.asyncio
async def test_not_connected_returns_failure(self):
adapter = _make_adapter()
adapter._client = None
result = await adapter.send_clarify(
chat_id="9001",
question="?",
choices=["a"],
clarify_id="cidNC",
session_key="sk-NC",
)
assert result.success is False
assert "Not connected" in (result.error or "")
@pytest.mark.asyncio
async def test_filters_empty_and_whitespace_choices(self):
adapter = _make_adapter()
channel = MagicMock()
sent_msg = MagicMock()
sent_msg.id = 444
channel.send = AsyncMock(return_value=sent_msg)
adapter._client.get_channel = MagicMock(return_value=channel)
await adapter.send_clarify(
chat_id="9001",
question="?",
choices=["", " ", "real-choice", None],
clarify_id="cidF",
session_key="sk-F",
)
kwargs = channel.send.call_args.kwargs
view = kwargs["view"]
# Only 1 real choice + 1 Other = 2 children
assert len(view.children) == 2
assert "real-choice" in view.children[0].label
+421
View File
@@ -62,6 +62,12 @@ class FakeTextChannel:
self.guild = SimpleNamespace(name=guild_name)
self.topic = None
def history(self, *, limit, before, after=None, oldest_first=None):
async def _iter():
return
yield
return _iter()
class FakeForumChannel:
def __init__(self, channel_id: int = 1, name: str = "support-forum", guild_name: str = "Hermes Server"):
@@ -81,6 +87,12 @@ class FakeThread:
self.guild = getattr(parent, "guild", None) or SimpleNamespace(name=guild_name)
self.topic = None
def history(self, *, limit, before, after=None, oldest_first=None):
async def _iter():
return
yield
return _iter()
@pytest.fixture
def adapter(monkeypatch):
@@ -88,6 +100,23 @@ def adapter(monkeypatch):
monkeypatch.setattr(discord_platform.discord, "Thread", FakeThread, raising=False)
monkeypatch.setattr(discord_platform.discord, "ForumChannel", FakeForumChannel, raising=False)
# Clear DISCORD_* env vars the test file exercises so tests don't leak
# process-env state from the contributor's shell into per-test behaviour.
# Individual tests still monkeypatch.setenv() for their own scenarios.
for _var in (
"DISCORD_REQUIRE_MENTION",
"DISCORD_THREAD_REQUIRE_MENTION",
"DISCORD_FREE_RESPONSE_CHANNELS",
"DISCORD_AUTO_THREAD",
"DISCORD_NO_THREAD_CHANNELS",
"DISCORD_ALLOWED_CHANNELS",
"DISCORD_IGNORED_CHANNELS",
"DISCORD_HISTORY_BACKFILL",
"DISCORD_HISTORY_BACKFILL_LIMIT",
"DISCORD_ALLOW_BOTS",
):
monkeypatch.delenv(_var, raising=False)
config = PlatformConfig(enabled=True, token="fake-token")
adapter = DiscordAdapter(config)
adapter._client = SimpleNamespace(user=SimpleNamespace(id=999))
@@ -111,6 +140,48 @@ def make_message(*, channel, content: str, mentions=None, msg_type=None):
)
def make_history_message(
*,
author,
content: str,
msg_id: int,
msg_type=None,
attachments=None,
):
return SimpleNamespace(
id=msg_id,
author=author,
content=content,
attachments=list(attachments or []),
type=msg_type if msg_type is not None else discord_platform.discord.MessageType.default,
)
class FakeHistoryChannel(FakeTextChannel):
def __init__(self, history_messages, **kwargs):
super().__init__(**kwargs)
self._history_messages = list(history_messages)
def history(self, *, limit, before, after=None, oldest_first=None):
before_id = int(getattr(before, "id", before))
after_id = int(getattr(after, "id", after)) if after is not None else None
if oldest_first is None:
oldest_first = after is not None
messages = [
message for message in self._history_messages
if int(message.id) < before_id
and (after_id is None or int(message.id) > after_id)
]
messages.sort(key=lambda message: int(message.id), reverse=not oldest_first)
async def _iter():
for message in messages[:limit]:
yield message
return _iter()
@pytest.mark.asyncio
async def test_discord_defaults_to_require_mention(adapter, monkeypatch):
"""Default behavior: require @mention in server channels."""
@@ -446,6 +517,37 @@ async def test_discord_voice_linked_channel_skips_mention_requirement_and_auto_t
assert event.source.chat_type == "group"
@pytest.mark.asyncio
async def test_discord_free_response_channel_skips_auto_thread(adapter, monkeypatch):
"""Free-response channels should reply inline, never spawn a new thread.
Without this, every message in a free-response channel would auto-create
a fresh thread (since the channel bypasses the @mention gate, every
message looks like a fresh trigger). That turns a "lightweight chat"
channel into a thread-spawning machine see the docs at
website/docs/user-guide/messaging/discord.md which already describe
this as the intended behavior.
"""
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
monkeypatch.setenv("DISCORD_FREE_RESPONSE_CHANNELS", "789")
monkeypatch.delenv("DISCORD_AUTO_THREAD", raising=False) # default true
adapter._auto_create_thread = AsyncMock()
message = make_message(
channel=FakeTextChannel(channel_id=789),
content="casual chat in free-response channel",
)
await adapter._handle_message(message)
adapter._auto_create_thread.assert_not_awaited()
adapter.handle_message.assert_awaited_once()
event = adapter.handle_message.await_args.args[0]
assert event.text == "casual chat in free-response channel"
assert event.source.chat_type == "group"
@pytest.mark.asyncio
@@ -463,3 +565,322 @@ async def test_discord_voice_linked_parent_thread_still_requires_mention(adapter
await adapter._handle_message(message)
adapter.handle_message.assert_not_awaited()
@pytest.mark.asyncio
async def test_discord_thread_default_keeps_responding_after_participation(adapter, monkeypatch):
"""Default behavior: once the bot is in a thread, it auto-responds without @mention."""
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
monkeypatch.delenv("DISCORD_THREAD_REQUIRE_MENTION", raising=False)
thread = FakeThread(channel_id=456, name="follow-up")
adapter._threads.mark("456") # bot has previously participated
message = make_message(channel=thread, content="follow-up without mention")
await adapter._handle_message(message)
adapter.handle_message.assert_awaited_once()
@pytest.mark.asyncio
async def test_discord_thread_require_mention_gates_followups(adapter, monkeypatch):
"""When thread_require_mention=true, even bot-participated threads need @mention."""
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
monkeypatch.setenv("DISCORD_THREAD_REQUIRE_MENTION", "true")
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
thread = FakeThread(channel_id=456, name="multi-bot thread")
adapter._threads.mark("456") # bot has previously participated
message = make_message(channel=thread, content="ambient chatter — not for me")
await adapter._handle_message(message)
adapter.handle_message.assert_not_awaited()
@pytest.mark.asyncio
async def test_discord_thread_require_mention_still_responds_when_mentioned(adapter, monkeypatch):
"""thread_require_mention=true still lets explicit @mentions through in threads."""
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
monkeypatch.setenv("DISCORD_THREAD_REQUIRE_MENTION", "true")
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
thread = FakeThread(channel_id=456, name="multi-bot thread")
adapter._threads.mark("456")
bot_user = adapter._client.user
message = make_message(
channel=thread,
content=f"<@{bot_user.id}> hey, this one's for you",
mentions=[bot_user],
)
await adapter._handle_message(message)
adapter.handle_message.assert_awaited_once()
@pytest.mark.asyncio
async def test_discord_thread_require_mention_via_config_extra(adapter, monkeypatch):
"""thread_require_mention can also be set via config.extra (yaml)."""
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
monkeypatch.delenv("DISCORD_THREAD_REQUIRE_MENTION", raising=False)
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
adapter.config.extra["thread_require_mention"] = True
thread = FakeThread(channel_id=456, name="multi-bot thread")
adapter._threads.mark("456")
message = make_message(channel=thread, content="ambient — should be ignored")
await adapter._handle_message(message)
adapter.handle_message.assert_not_awaited()
@pytest.mark.asyncio
async def test_fetch_channel_context_stops_at_self_message_and_reverses_to_chronological_order(adapter, monkeypatch):
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
adapter.config.extra["history_backfill_limit"] = 10
other_bot = SimpleNamespace(id=55, display_name="Gemini", name="Gemini", bot=True)
human = SimpleNamespace(id=56, display_name="Alice", name="Alice", bot=False)
old_human = SimpleNamespace(id=57, display_name="Bob", name="Bob", bot=False)
channel = FakeHistoryChannel(
[
make_history_message(author=human, content="latest human note", msg_id=4),
make_history_message(author=other_bot, content="latest bot note", msg_id=3),
make_history_message(author=adapter._client.user, content="our prior response", msg_id=2),
make_history_message(author=old_human, content="older than boundary", msg_id=1),
],
channel_id=123,
)
result = await adapter._fetch_channel_context(channel, before=make_message(channel=channel, content="trigger"))
assert result == (
"[Recent channel messages]\n"
"[Gemini [bot]] latest bot note\n"
"[Alice] latest human note"
)
@pytest.mark.asyncio
async def test_fetch_channel_context_skips_other_bots_when_allow_bots_none(adapter, monkeypatch):
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "none")
adapter.config.extra["history_backfill_limit"] = 10
other_bot = SimpleNamespace(id=55, display_name="Gemini", name="Gemini", bot=True)
human = SimpleNamespace(id=56, display_name="Alice", name="Alice", bot=False)
channel = FakeHistoryChannel(
[
make_history_message(author=human, content="human note", msg_id=3),
make_history_message(author=other_bot, content="bot note", msg_id=2),
],
channel_id=123,
)
result = await adapter._fetch_channel_context(channel, before=make_message(channel=channel, content="trigger"))
assert result == "[Recent channel messages]\n[Alice] human note"
@pytest.mark.asyncio
async def test_fetch_channel_context_uses_cache_to_narrow_window(adapter, monkeypatch):
"""When _last_self_message_id is cached, the fetch passes after= to skip old messages."""
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
adapter.config.extra["history_backfill_limit"] = 50
human = SimpleNamespace(id=56, display_name="Alice", name="Alice", bot=False)
# Record the after= arg passed to history()
recorded_after = {}
class CacheTrackingChannel(FakeHistoryChannel):
def history(self, *, limit, before, after=None, oldest_first=None):
recorded_after["value"] = after
return super().history(
limit=limit,
before=before,
after=after,
oldest_first=oldest_first,
)
channel = CacheTrackingChannel(
[make_history_message(author=human, content="hello", msg_id=200)],
channel_id=777,
)
# Seed the cache — bot's last message in this channel was ID 100
adapter._last_self_message_id["777"] = "100"
trigger = make_message(channel=channel, content="trigger")
trigger.id = 300 # trigger is newer than cache
result = await adapter._fetch_channel_context(channel, before=trigger)
assert result == "[Recent channel messages]\n[Alice] hello"
# Verify cache was used: after= should be set (not None)
assert recorded_after["value"] is not None
@pytest.mark.asyncio
async def test_fetch_channel_context_cache_uses_latest_window_when_after_set(adapter, monkeypatch):
"""Regression: discord.py defaults oldest_first=True when after= is provided.
The hot cache path passes both after= and before=. We still want the latest
messages before the trigger, not the earliest messages after our prior
response, otherwise tool traces can crowd out the final answer.
"""
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
adapter.config.extra["history_backfill_limit"] = 3
codex = SimpleNamespace(id=56, display_name="Codex", name="Codex", bot=True)
human = SimpleNamespace(id=57, display_name="Alice", name="Alice", bot=False)
channel = FakeHistoryChannel(
[
make_history_message(author=codex, content="old tool trace 1", msg_id=101),
make_history_message(author=codex, content="old tool trace 2", msg_id=102),
make_history_message(author=codex, content="old tool trace 3", msg_id=103),
make_history_message(author=codex, content="final analysis", msg_id=104),
make_history_message(author=human, content="latest follow-up", msg_id=105),
],
channel_id=777,
)
adapter._last_self_message_id["777"] = "100"
trigger = make_message(channel=channel, content="trigger")
trigger.id = 200
result = await adapter._fetch_channel_context(channel, before=trigger)
assert "[Codex [bot]] final analysis" in result
assert "[Alice] latest follow-up" in result
assert "old tool trace 1" not in result
assert "old tool trace 2" not in result
@pytest.mark.asyncio
async def test_fetch_channel_context_ignores_stale_cache(adapter, monkeypatch):
"""If cached ID is >= trigger ID (stale/future), fall back to cold-start scan."""
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
adapter.config.extra["history_backfill_limit"] = 50
human = SimpleNamespace(id=56, display_name="Alice", name="Alice", bot=False)
recorded_after = {}
class CacheTrackingChannel(FakeHistoryChannel):
def history(self, *, limit, before, after=None, oldest_first=None):
recorded_after["value"] = after
return super().history(
limit=limit,
before=before,
after=after,
oldest_first=oldest_first,
)
channel = CacheTrackingChannel(
[make_history_message(author=human, content="hello", msg_id=50)],
channel_id=777,
)
# Cache has a NEWER ID than the trigger — stale/invalid
adapter._last_self_message_id["777"] = "500"
trigger = make_message(channel=channel, content="trigger")
trigger.id = 300
result = await adapter._fetch_channel_context(channel, before=trigger)
assert result == "[Recent channel messages]\n[Alice] hello"
# Cache should have been ignored — after= should be None
assert recorded_after["value"] is None
@pytest.mark.asyncio
async def test_discord_shared_channel_backfill_prepends_context(adapter, monkeypatch):
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
monkeypatch.setenv("DISCORD_AUTO_THREAD", "false")
adapter.config.extra["group_sessions_per_user"] = False
adapter.config.extra["history_backfill"] = True
adapter._fetch_channel_context = AsyncMock(return_value="[Recent channel messages]\n[Alice] context")
bot_user = adapter._client.user
message = make_message(
channel=FakeTextChannel(channel_id=321),
content=f"<@{bot_user.id}> hello with mention",
mentions=[bot_user],
)
await adapter._handle_message(message)
adapter._fetch_channel_context.assert_awaited_once()
event = adapter.handle_message.await_args.args[0]
assert event.text == "hello with mention"
assert event.channel_context == "[Recent channel messages]\n[Alice] context"
@pytest.mark.asyncio
async def test_discord_per_user_channel_backfills_too(adapter, monkeypatch):
"""Per-user sessions also benefit from backfill: Alice's session is missing
other-channel-participants' context and her own pre-mention messages."""
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
monkeypatch.setenv("DISCORD_AUTO_THREAD", "false")
adapter.config.extra["group_sessions_per_user"] = True
adapter.config.extra["history_backfill"] = True
adapter._fetch_channel_context = AsyncMock(return_value="[Recent channel messages]\n[Alice] context")
bot_user = adapter._client.user
message = make_message(
channel=FakeTextChannel(channel_id=321),
content=f"<@{bot_user.id}> hello with mention",
mentions=[bot_user],
)
await adapter._handle_message(message)
adapter._fetch_channel_context.assert_awaited_once()
event = adapter.handle_message.await_args.args[0]
assert event.text == "hello with mention"
assert event.channel_context == "[Recent channel messages]\n[Alice] context"
@pytest.mark.asyncio
async def test_discord_dm_does_not_backfill(adapter, monkeypatch):
"""DMs skip backfill — every DM triggers the bot, so there's no mention gap."""
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
adapter.config.extra["history_backfill"] = True
adapter._fetch_channel_context = AsyncMock(return_value="[Recent channel messages]\n[Alice] context")
bot_user = adapter._client.user
dm_channel = SimpleNamespace(
id=999,
name=None,
guild=None,
topic=None,
)
# Make isinstance(channel, discord.DMChannel) return True
monkeypatch.setattr(
discord_platform.discord, "DMChannel", type(dm_channel), raising=False,
)
message = make_message(
channel=dm_channel,
content="hello in DM",
mentions=[],
)
await adapter._handle_message(message)
adapter._fetch_channel_context.assert_not_awaited()
if adapter.handle_message.await_args is not None:
event = adapter.handle_message.await_args.args[0]
assert event.channel_context is None
@@ -467,3 +467,59 @@ class TestCancellationHandlerDeliveryConfirmation:
final_response_sent = True
assert final_response_sent is True # the bug: partial promoted to final
class TestFinalContentDeliveredSuppression:
"""When stream consumer delivered the final content but the cosmetic
final edit (cursor removal) failed, the gateway must suppress the
fallback send to prevent duplicate messages.
Covers the scenario not handled by final_response_sent alone:
content reached the user via _send_or_edit, but the subsequent edit
that clears a typing cursor or streaming marker failed, leaving
final_response_sent=False even though the user already saw the text.
"""
def test_content_delivered_but_final_edit_failed_suppresses(self):
"""final_content_delivered=True + final_response_sent=False
must suppress (content already visible to user)."""
sc = SimpleNamespace(
already_sent=True,
final_response_sent=False,
final_content_delivered=True,
)
response = {"final_response": "Hello!", "response_previewed": False}
_streamed = bool(getattr(sc, "final_response_sent", False))
_previewed = bool(response.get("response_previewed"))
_content_delivered = bool(getattr(sc, "final_content_delivered", False))
_is_empty_sentinel = (
not response.get("final_response")
or response.get("final_response") == "(empty)"
)
if not _is_empty_sentinel and (_streamed or _previewed or _content_delivered):
response["already_sent"] = True
assert response.get("already_sent") is True
def test_intermediate_text_only_does_not_suppress(self):
"""already_sent=True from intermediate text + final_content_delivered=False
must NOT suppress (user still needs the real final answer)."""
sc = SimpleNamespace(
already_sent=True,
final_response_sent=False,
final_content_delivered=False,
)
response = {"final_response": "Real answer", "response_previewed": False}
_streamed = bool(getattr(sc, "final_response_sent", False))
_previewed = bool(response.get("response_previewed"))
_content_delivered = bool(getattr(sc, "final_content_delivered", False))
_is_empty_sentinel = (
not response.get("final_response")
or response.get("final_response") == "(empty)"
)
if not _is_empty_sentinel and (_streamed or _previewed or _content_delivered):
response["already_sent"] = True
assert "already_sent" not in response
+30 -1
View File
@@ -455,7 +455,36 @@ def test_admit_per_group_require_mention_overrides_global():
def test_hydrate_bot_identity_populates_self_ids_from_bot_v3_info(monkeypatch):
import asyncio
from gateway.platforms.feishu import FeishuAdapter
from gateway.platforms import feishu as feishu_mod
FeishuAdapter = feishu_mod.FeishuAdapter
class _FakeBaseRequestBuilder:
def __init__(self):
self._request = SimpleNamespace()
def http_method(self, value):
self._request.http_method = value
return self
def uri(self, value):
self._request.uri = value
return self
def token_types(self, value):
self._request.token_types = value
return self
def build(self):
return self._request
monkeypatch.setattr(
feishu_mod,
"BaseRequest",
SimpleNamespace(builder=lambda: _FakeBaseRequestBuilder()),
raising=False,
)
monkeypatch.setattr(feishu_mod, "HttpMethod", SimpleNamespace(GET="GET"), raising=False)
monkeypatch.setattr(feishu_mod, "AccessTokenType", SimpleNamespace(TENANT="TENANT"), raising=False)
adapter = object.__new__(FeishuAdapter)
adapter._bot_open_id = ""
+11 -5
View File
@@ -716,8 +716,10 @@ class TestMatrixModuleImport:
"sys.meta_path.insert(0, _Blocker())\n"
"for k in list(sys.modules):\n"
" if k.startswith('mautrix'): del sys.modules[k]\n"
"from unittest.mock import patch\n"
"from gateway.platforms.matrix import check_matrix_requirements\n"
"assert not check_matrix_requirements()\n"
"with patch('tools.lazy_deps.ensure', side_effect=ImportError('blocked')):\n"
" assert not check_matrix_requirements()\n"
"print('OK')\n"
)],
capture_output=True, text=True, timeout=10,
@@ -737,7 +739,8 @@ class TestMatrixRequirements:
import mautrix # noqa: F401
assert check_matrix_requirements() is True
except ImportError:
assert check_matrix_requirements() is False
with patch("tools.lazy_deps.ensure", side_effect=ImportError("mautrix unavailable")):
assert check_matrix_requirements() is False
def test_check_requirements_without_creds(self, monkeypatch):
monkeypatch.delenv("MATRIX_ACCESS_TOKEN", raising=False)
@@ -759,7 +762,8 @@ class TestMatrixRequirements:
monkeypatch.setenv("MATRIX_ENCRYPTION", "true")
from gateway.platforms import matrix as matrix_mod
with patch.object(matrix_mod, "_check_e2ee_deps", return_value=False):
with patch.object(matrix_mod, "_check_e2ee_deps", return_value=False), \
patch("tools.lazy_deps.ensure", side_effect=ImportError("mautrix unavailable")):
assert matrix_mod.check_matrix_requirements() is False
def test_check_requirements_encryption_false_no_e2ee_deps_ok(self, monkeypatch):
@@ -775,7 +779,8 @@ class TestMatrixRequirements:
import mautrix # noqa: F401
assert matrix_mod.check_matrix_requirements() is True
except ImportError:
assert matrix_mod.check_matrix_requirements() is False
with patch("tools.lazy_deps.ensure", side_effect=ImportError("mautrix unavailable")):
assert matrix_mod.check_matrix_requirements() is False
def test_check_requirements_encryption_true_with_e2ee_deps(self, monkeypatch):
"""MATRIX_ENCRYPTION=true should pass if E2EE deps are available."""
@@ -789,7 +794,8 @@ class TestMatrixRequirements:
import mautrix # noqa: F401
assert matrix_mod.check_matrix_requirements() is True
except ImportError:
assert matrix_mod.check_matrix_requirements() is False
with patch("tools.lazy_deps.ensure", side_effect=ImportError("mautrix unavailable")):
assert matrix_mod.check_matrix_requirements() is False
# ---------------------------------------------------------------------------
+314
View File
@@ -394,3 +394,317 @@ class TestPlatformsMerge:
assert "LabelTest" in label
finally:
_reg.unregister("labeltest")
# ── apply_yaml_config_fn (PlatformEntry field + load_gateway_config dispatch) ──
class TestApplyYamlConfigFnField:
"""The hook field itself — defaults, custom values, signature."""
def test_default_is_none(self):
entry = PlatformEntry(
name="test",
label="Test",
adapter_factory=lambda cfg: None,
check_fn=lambda: True,
)
assert entry.apply_yaml_config_fn is None
def test_accepts_callable(self):
def _hook(yaml_cfg, platform_cfg):
return None
entry = PlatformEntry(
name="test",
label="Test",
adapter_factory=lambda cfg: None,
check_fn=lambda: True,
apply_yaml_config_fn=_hook,
)
assert entry.apply_yaml_config_fn is _hook
# Sanity-check the signature contract.
assert entry.apply_yaml_config_fn({"x": 1}, {"y": 2}) is None
class TestApplyYamlConfigFnDispatch:
"""End-to-end dispatch through load_gateway_config().
Each test registers a temporary PlatformEntry, writes a config.yaml in
a tmp HERMES_HOME, calls load_gateway_config(), and asserts the hook
was invoked correctly. Cleanup unregisters the entry.
"""
def _write_config(self, tmp_path, content: str):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
(hermes_home / "config.yaml").write_text(content, encoding="utf-8")
return hermes_home
def _register_hook(self, name, hook_fn):
from gateway.platform_registry import platform_registry as _reg
entry = PlatformEntry(
name=name,
label=name.title(),
adapter_factory=lambda cfg: None,
check_fn=lambda: True,
source="plugin",
apply_yaml_config_fn=hook_fn,
)
_reg.register(entry)
return _reg
def test_hook_can_mutate_environ(self, tmp_path, monkeypatch):
"""A hook that mutates os.environ has its env vars set after load."""
env_var = "MYHOOKPLAT_FLAG"
monkeypatch.delenv(env_var, raising=False)
def _hook(yaml_cfg, platform_cfg):
if "flag" in platform_cfg and not os.getenv(env_var):
os.environ[env_var] = str(platform_cfg["flag"]).lower()
return None
reg = self._register_hook("myhookplat", _hook)
try:
home = self._write_config(
tmp_path, "myhookplat:\n flag: true\n",
)
monkeypatch.setenv("HERMES_HOME", str(home))
from gateway.config import load_gateway_config
load_gateway_config()
assert os.environ.get(env_var) == "true"
finally:
reg.unregister("myhookplat")
os.environ.pop(env_var, None)
def test_hook_returned_dict_merges_into_extra(self, tmp_path, monkeypatch):
"""A hook that returns a dict has it merged into PlatformConfig.extra."""
def _hook(yaml_cfg, platform_cfg):
return {"seeded_key": "seeded_value", "flag": platform_cfg.get("flag")}
reg = self._register_hook("myextraplat", _hook)
try:
home = self._write_config(
tmp_path, "myextraplat:\n flag: yes\n",
)
monkeypatch.setenv("HERMES_HOME", str(home))
from gateway.config import load_gateway_config
cfg = load_gateway_config()
plat = Platform("myextraplat")
assert plat in cfg.platforms
extra = cfg.platforms[plat].extra
assert extra.get("seeded_key") == "seeded_value"
# flag value carried through from yaml_cfg arg.
assert extra.get("flag") is True
finally:
reg.unregister("myextraplat")
def test_hook_receives_full_yaml_and_platform_subdict(
self, tmp_path, monkeypatch
):
"""Hook receives both the full yaml_cfg and its own platform sub-dict."""
captured: dict = {}
def _hook(yaml_cfg, platform_cfg):
captured["yaml_cfg"] = yaml_cfg
captured["platform_cfg"] = platform_cfg
return None
reg = self._register_hook("mycaptureplat", _hook)
try:
home = self._write_config(
tmp_path,
"top_level_key: 1\n"
"mycaptureplat:\n"
" inner_key: deep\n",
)
monkeypatch.setenv("HERMES_HOME", str(home))
from gateway.config import load_gateway_config
load_gateway_config()
assert captured["yaml_cfg"].get("top_level_key") == 1
assert captured["platform_cfg"] == {"inner_key": "deep"}
finally:
reg.unregister("mycaptureplat")
def test_hook_exception_swallowed(self, tmp_path, monkeypatch):
"""A misbehaving hook never aborts load_gateway_config()."""
def _bad_hook(yaml_cfg, platform_cfg):
raise RuntimeError("plugin author bug")
# Also register a well-behaved hook to ensure dispatch continues
# iterating after a bad one.
good_called = {"count": 0}
def _good_hook(yaml_cfg, platform_cfg):
good_called["count"] += 1
return None
from gateway.platform_registry import platform_registry as _reg
_reg.register(PlatformEntry(
name="mybadplat",
label="MyBad",
adapter_factory=lambda cfg: None,
check_fn=lambda: True,
source="plugin",
apply_yaml_config_fn=_bad_hook,
))
_reg.register(PlatformEntry(
name="mygoodplat",
label="MyGood",
adapter_factory=lambda cfg: None,
check_fn=lambda: True,
source="plugin",
apply_yaml_config_fn=_good_hook,
))
try:
home = self._write_config(
tmp_path,
"mybadplat:\n k: v\n"
"mygoodplat:\n k: v\n",
)
monkeypatch.setenv("HERMES_HOME", str(home))
# Must not raise.
from gateway.config import load_gateway_config
load_gateway_config()
assert good_called["count"] == 1
finally:
_reg.unregister("mybadplat")
_reg.unregister("mygoodplat")
def test_hook_skipped_when_platform_section_missing(
self, tmp_path, monkeypatch
):
"""Hook is NOT called when the platform's YAML section is absent."""
called = {"count": 0}
def _hook(yaml_cfg, platform_cfg):
called["count"] += 1
return None
reg = self._register_hook("myabsentplat", _hook)
try:
home = self._write_config(tmp_path, "telegram:\n k: v\n")
monkeypatch.setenv("HERMES_HOME", str(home))
from gateway.config import load_gateway_config
load_gateway_config()
assert called["count"] == 0
finally:
reg.unregister("myabsentplat")
def test_hook_skipped_when_platform_section_not_dict(
self, tmp_path, monkeypatch
):
"""Hook is NOT called when the platform's YAML section isn't a dict."""
called = {"count": 0}
def _hook(yaml_cfg, platform_cfg):
called["count"] += 1
return None
reg = self._register_hook("mybadshapeplat", _hook)
try:
home = self._write_config(
tmp_path, "mybadshapeplat: just-a-string\n",
)
monkeypatch.setenv("HERMES_HOME", str(home))
from gateway.config import load_gateway_config
load_gateway_config()
assert called["count"] == 0
finally:
reg.unregister("mybadshapeplat")
def test_env_var_takes_precedence_when_hook_uses_getenv_guard(
self, tmp_path, monkeypatch
):
"""The standard `not os.getenv(...)` guard preserves env > YAML."""
env_var = "MYPRECPLAT_FLAG"
monkeypatch.setenv(env_var, "preexisting")
def _hook(yaml_cfg, platform_cfg):
if "flag" in platform_cfg and not os.getenv(env_var):
os.environ[env_var] = str(platform_cfg["flag"]).lower()
return None
reg = self._register_hook("myprecplat", _hook)
try:
home = self._write_config(
tmp_path, "myprecplat:\n flag: yaml-value\n",
)
monkeypatch.setenv("HERMES_HOME", str(home))
from gateway.config import load_gateway_config
load_gateway_config()
# Pre-existing env var was NOT clobbered by the hook.
assert os.environ.get(env_var) == "preexisting"
finally:
reg.unregister("myprecplat")
os.environ.pop(env_var, None)
class TestPluginPlatformSharedKeyBridge:
"""Plugin-registered platforms get the same shared-key bridging as built-ins.
Without this, plugin authors using ``apply_yaml_config_fn`` would have to
re-implement bridging for every common key (``unauthorized_dm_behavior``,
``notice_delivery``, ``reply_prefix``, ``require_mention``, ``dm_policy``,
``allow_from``, etc.) defeating the hook's whole point of letting
plugins focus on their *platform-specific* keys.
"""
def _write_config(self, tmp_path, content: str):
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
(hermes_home / "config.yaml").write_text(content, encoding="utf-8")
return hermes_home
def test_shared_keys_bridged_for_plugin_platform(self, tmp_path, monkeypatch):
"""A plugin platform's ``require_mention``/``dm_policy``/etc. flow into
``PlatformConfig.extra`` without the plugin needing its own bridge."""
from gateway.platform_registry import platform_registry as _reg
_reg.register(PlatformEntry(
name="mysharedplat",
label="MySharedPlat",
adapter_factory=lambda cfg: None,
check_fn=lambda: True,
source="plugin",
))
try:
home = self._write_config(
tmp_path,
"mysharedplat:\n"
" require_mention: true\n"
" dm_policy: allow\n"
" reply_prefix: \"\"\n"
" allow_from: [\"alice\", \"bob\"]\n",
)
monkeypatch.setenv("HERMES_HOME", str(home))
from gateway.config import load_gateway_config, Platform
cfg = load_gateway_config()
plat = Platform("mysharedplat")
assert plat in cfg.platforms
extra = cfg.platforms[plat].extra
assert extra.get("require_mention") is True
assert extra.get("dm_policy") == "allow"
assert extra.get("reply_prefix") == ""
assert extra.get("allow_from") == ["alice", "bob"]
finally:
_reg.unregister("mysharedplat")
+3
View File
@@ -4,6 +4,7 @@ import asyncio
import json
import os
import sys
from types import SimpleNamespace
from unittest import mock
import pytest
@@ -578,6 +579,7 @@ class TestWaitForReconnection:
async def reconnect_after_delay():
await asyncio.sleep(0.3)
adapter._running = True
adapter._ws = SimpleNamespace(closed=False)
asyncio.get_event_loop().create_task(reconnect_after_delay())
@@ -603,6 +605,7 @@ class TestWaitForReconnection:
"""send() should not wait when already connected."""
adapter = self._make_adapter(app_id="a", client_secret="b")
adapter._running = True
adapter._ws = SimpleNamespace(closed=False)
adapter._http_client = mock.MagicMock()
async def fake_api_request(*args, **kwargs):
+71
View File
@@ -5,6 +5,7 @@ import pytest
from pathlib import Path
from unittest.mock import patch, MagicMock
from gateway.config import Platform, HomeChannel, GatewayConfig, PlatformConfig
from gateway.platforms.base import MessageEvent
from gateway.session import (
SessionSource,
SessionStore,
@@ -430,6 +431,76 @@ class TestBuildSessionContextPrompt:
assert "Multi-user thread" not in prompt
class TestSenderPrefixWithBackfill:
"""Regression: sender prefix must not wrap the backfill context block.
Tests exercise the real GatewayRunner._prepare_inbound_message_text()
method to ensure the [sender_name] prefix applies only to the trigger
message, not the channel_context backfill block.
"""
@pytest.fixture()
def runner(self):
from gateway.run import GatewayRunner
r = GatewayRunner.__new__(GatewayRunner)
r.config = GatewayConfig(group_sessions_per_user=False)
r.adapters = {}
r._model = "test-model"
r._base_url = ""
r._has_setup_skill = lambda: False
return r
@pytest.fixture()
def source(self):
return SessionSource(
platform=Platform.DISCORD,
chat_id="c1",
chat_type="group",
user_name="Alice",
)
@pytest.mark.asyncio
async def test_plain_message_gets_prefix(self, runner, source):
"""Normal message without backfill gets [sender] prefix."""
event = MessageEvent(text="hello world", source=source)
result = await runner._prepare_inbound_message_text(
event=event, source=source, history=[],
)
assert result == "[Alice] hello world"
@pytest.mark.asyncio
async def test_backfill_prefix_only_on_trigger(self, runner, source):
"""Backfill context must NOT get the sender prefix."""
event = MessageEvent(
text="hello world",
source=source,
channel_context="[Recent channel messages]\n[Bob] some context",
)
result = await runner._prepare_inbound_message_text(
event=event, source=source, history=[],
)
assert result.startswith("[Recent channel messages]")
assert "[Alice] [Recent channel messages]" not in result
assert "[New message]\n[Alice] hello world" in result
@pytest.mark.asyncio
async def test_backfill_preserves_context_block(self, runner, source):
"""The backfill block should pass through unchanged — no double-prefixing."""
context = "[Recent channel messages]\n[Bob] first\n[Charlie [bot]] second"
event = MessageEvent(
text="hey everyone", source=source, channel_context=context,
)
result = await runner._prepare_inbound_message_text(
event=event, source=source, history=[],
)
assert result.startswith(context)
assert "[Alice] hey everyone" in result
assert "[Alice] [Bob]" not in result
assert "[Alice] [Charlie" not in result
assert "[Alice] [Recent" not in result
class TestSessionStoreRewriteTranscript:
"""Regression: /retry and /undo must persist truncated history to disk."""
@@ -205,3 +205,78 @@ class TestResetPolicyNotify:
assert restored.notify == original.notify
assert restored.notify_exclude_platforms == original.notify_exclude_platforms
assert restored.mode == original.mode
# ---------------------------------------------------------------------------
# SessionEntry to_dict / from_dict roundtrip for auto-reset fields
# ---------------------------------------------------------------------------
class TestSessionEntryAutoResetRoundtrip:
def test_was_auto_reset_persists_across_roundtrip(self, tmp_path):
"""was_auto_reset=True survives to_dict() → from_dict() (gateway restart)."""
store = _make_store(
SessionResetPolicy(mode="idle", idle_minutes=1),
tmp_path,
)
source = _make_source()
entry = store.get_or_create_session(source)
entry.updated_at = datetime.now() - timedelta(minutes=5)
store._save()
entry2 = store.get_or_create_session(source)
assert entry2.was_auto_reset is True
assert entry2.auto_reset_reason == "idle"
assert entry2.session_id != entry.session_id
# Simulate gateway restart: reload from disk
store._loaded = False
store._entries.clear()
store._ensure_loaded()
reloaded = store._entries.get(entry2.session_key)
assert reloaded is not None
assert reloaded.was_auto_reset is True
assert reloaded.auto_reset_reason == "idle"
def test_reset_had_activity_persists_across_roundtrip(self, tmp_path):
"""reset_had_activity survives to_dict() → from_dict() (gateway restart)."""
store = _make_store(
SessionResetPolicy(mode="idle", idle_minutes=1),
tmp_path,
)
source = _make_source()
entry = store.get_or_create_session(source)
entry.total_tokens = 1000
entry.updated_at = datetime.now() - timedelta(minutes=5)
store._save()
entry2 = store.get_or_create_session(source)
assert entry2.reset_had_activity is True
store._loaded = False
store._entries.clear()
store._ensure_loaded()
reloaded = store._entries.get(entry2.session_key)
assert reloaded is not None
assert reloaded.reset_had_activity is True
def test_auto_reset_reason_none_roundtrip(self, tmp_path):
"""auto_reset_reason=None (no reset) survives roundtrip cleanly."""
store = _make_store(tmp_path=tmp_path)
source = _make_source()
entry = store.get_or_create_session(source)
assert entry.was_auto_reset is False
store._loaded = False
store._entries.clear()
store._ensure_loaded()
reloaded = store._entries.get(entry.session_key)
assert reloaded is not None
assert reloaded.was_auto_reset is False
assert reloaded.auto_reset_reason is None
assert reloaded.reset_had_activity is False
+347
View File
@@ -0,0 +1,347 @@
"""Tests for the SimpleX Chat platform-plugin adapter.
Loaded via the ``_plugin_adapter_loader`` helper so this lives under
``plugin_adapter_simplex`` in ``sys.modules`` and cannot collide with
sibling platform-plugin tests on the same xdist worker.
"""
from __future__ import annotations
import json
import os
from unittest.mock import AsyncMock, MagicMock
import pytest
from tests.gateway._plugin_adapter_loader import load_plugin_adapter
_simplex = load_plugin_adapter("simplex")
SimplexAdapter = _simplex.SimplexAdapter
check_requirements = _simplex.check_requirements
validate_config = _simplex.validate_config
is_connected = _simplex.is_connected
register = _simplex.register
_env_enablement = _simplex._env_enablement
_standalone_send = _simplex._standalone_send
_guess_extension = _simplex._guess_extension
_is_image_ext = _simplex._is_image_ext
_is_audio_ext = _simplex._is_audio_ext
_CORR_PREFIX = _simplex._CORR_PREFIX
# ---------------------------------------------------------------------------
# 1. Platform enum (plugin-discovered, not bundled)
# ---------------------------------------------------------------------------
def test_platform_enum_resolves_via_plugin_scan():
"""The plugin filesystem scan should expose Platform("simplex")."""
from gateway.config import Platform
p = Platform("simplex")
assert p.value == "simplex"
# Identity stability — repeated lookups return the same pseudo-member
assert Platform("simplex") is p
# ---------------------------------------------------------------------------
# 2. check_requirements / validate_config / is_connected
# ---------------------------------------------------------------------------
def test_check_requirements_needs_url(monkeypatch):
monkeypatch.delenv("SIMPLEX_WS_URL", raising=False)
assert check_requirements() is False
def test_check_requirements_true_when_configured(monkeypatch):
monkeypatch.setenv("SIMPLEX_WS_URL", "ws://127.0.0.1:5225")
# websockets is a dev dep in this repo via the test plugins; the
# check_requirements() gate also asserts the package imports.
websockets_present = True
try:
import websockets # noqa: F401
except ImportError:
websockets_present = False
assert check_requirements() is websockets_present
def test_validate_config_uses_env_or_extra():
from gateway.config import PlatformConfig
# Empty extra + no env → invalid
cfg = PlatformConfig(enabled=True)
assert validate_config(cfg) is False
# extra-only path → valid
cfg2 = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
assert validate_config(cfg2) is True
def test_is_connected_mirrors_validate(monkeypatch):
from gateway.config import PlatformConfig
monkeypatch.delenv("SIMPLEX_WS_URL", raising=False)
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://x"})
assert is_connected(cfg) is True
assert is_connected(PlatformConfig(enabled=True)) is False
# ---------------------------------------------------------------------------
# 3. _env_enablement seeds PlatformConfig.extra
# ---------------------------------------------------------------------------
def test_env_enablement_none_when_unset(monkeypatch):
monkeypatch.delenv("SIMPLEX_WS_URL", raising=False)
assert _env_enablement() is None
def test_env_enablement_seeds_ws_url(monkeypatch):
monkeypatch.setenv("SIMPLEX_WS_URL", "ws://127.0.0.1:5225")
monkeypatch.delenv("SIMPLEX_HOME_CHANNEL", raising=False)
seed = _env_enablement()
assert seed == {"ws_url": "ws://127.0.0.1:5225"}
def test_env_enablement_seeds_home_channel(monkeypatch):
monkeypatch.setenv("SIMPLEX_WS_URL", "ws://127.0.0.1:5225")
monkeypatch.setenv("SIMPLEX_HOME_CHANNEL", "42")
monkeypatch.setenv("SIMPLEX_HOME_CHANNEL_NAME", "Personal")
seed = _env_enablement()
assert seed["home_channel"] == {"chat_id": "42", "name": "Personal"}
def test_env_enablement_home_channel_defaults_name_to_id(monkeypatch):
monkeypatch.setenv("SIMPLEX_WS_URL", "ws://127.0.0.1:5225")
monkeypatch.setenv("SIMPLEX_HOME_CHANNEL", "42")
monkeypatch.delenv("SIMPLEX_HOME_CHANNEL_NAME", raising=False)
seed = _env_enablement()
assert seed["home_channel"] == {"chat_id": "42", "name": "42"}
# ---------------------------------------------------------------------------
# 4. Adapter init
# ---------------------------------------------------------------------------
def test_adapter_init_custom_url():
from gateway.config import PlatformConfig
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
adapter = SimplexAdapter(cfg)
assert adapter.ws_url == "ws://localhost:5225"
assert adapter._running is False
assert adapter._ws is None
def test_adapter_init_default_url():
from gateway.config import PlatformConfig
cfg = PlatformConfig(enabled=True)
adapter = SimplexAdapter(cfg)
assert adapter.ws_url == "ws://127.0.0.1:5225"
def test_adapter_platform_identity():
"""Adapter should expose Platform("simplex") identity."""
from gateway.config import Platform, PlatformConfig
cfg = PlatformConfig(enabled=True)
adapter = SimplexAdapter(cfg)
assert adapter.platform is Platform("simplex")
# ---------------------------------------------------------------------------
# 5. Helper functions (magic-byte detection)
# ---------------------------------------------------------------------------
def test_guess_extension_png():
assert _guess_extension(b"\x89PNG\r\n\x1a\n") == ".png"
def test_guess_extension_jpg():
assert _guess_extension(b"\xff\xd8\xff\xe0") == ".jpg"
def test_guess_extension_ogg():
assert _guess_extension(b"OggS\x00\x02") == ".ogg"
def test_guess_extension_unknown():
assert _guess_extension(b"\x00\x01\x02\x03") == ".bin"
def test_is_image_ext():
assert _is_image_ext(".png") is True
assert _is_image_ext(".webp") is True
assert _is_image_ext(".ogg") is False
def test_is_audio_ext():
assert _is_audio_ext(".ogg") is True
assert _is_audio_ext(".mp3") is True
assert _is_audio_ext(".pdf") is False
# ---------------------------------------------------------------------------
# 6. Correlation IDs
# ---------------------------------------------------------------------------
def test_corr_id_starts_with_prefix_and_tracks_pending():
from gateway.config import PlatformConfig
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
adapter = SimplexAdapter(cfg)
corr_id = adapter._make_corr_id()
assert corr_id.startswith(_CORR_PREFIX)
assert corr_id in adapter._pending_corr_ids
def test_corr_id_pending_set_self_trims():
from gateway.config import PlatformConfig
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
adapter = SimplexAdapter(cfg)
adapter._max_pending_corr = 4
for _ in range(10):
adapter._make_corr_id()
# After many additions, the pending set should be bounded by the trim
# logic — at most one trim window above the cap.
assert len(adapter._pending_corr_ids) <= adapter._max_pending_corr + 1
# ---------------------------------------------------------------------------
# 7. Outbound send (mocked WS)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_send_dm():
from gateway.config import PlatformConfig
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
adapter = SimplexAdapter(cfg)
mock_ws = AsyncMock()
adapter._ws = mock_ws
result = await adapter.send("contact-42", "Hello, SimpleX!")
mock_ws.send.assert_called_once()
payload = json.loads(mock_ws.send.call_args[0][0])
assert payload["cmd"] == "@[contact-42] Hello, SimpleX!"
assert payload["corrId"].startswith(_CORR_PREFIX)
assert result.success is True
@pytest.mark.asyncio
async def test_send_group():
from gateway.config import PlatformConfig
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
adapter = SimplexAdapter(cfg)
mock_ws = AsyncMock()
adapter._ws = mock_ws
result = await adapter.send("group:grp-99", "Hello, group!")
payload = json.loads(mock_ws.send.call_args[0][0])
assert payload["cmd"] == "#[grp-99] Hello, group!"
assert result.success is True
@pytest.mark.asyncio
async def test_send_when_ws_not_connected_does_not_crash():
from gateway.config import PlatformConfig
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
adapter = SimplexAdapter(cfg)
# No _ws assigned — _send_ws should drop quietly
result = await adapter.send("contact-42", "hi")
assert result.success is True # send() always returns success — fire-and-forget
# ---------------------------------------------------------------------------
# 8. Inbound: filter own-echo by corrId prefix
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_handle_event_filters_own_corr_id():
from gateway.config import PlatformConfig
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
adapter = SimplexAdapter(cfg)
# Pretend we sent a command with this corrId
own = adapter._make_corr_id()
handler_mock = AsyncMock()
adapter._handle_new_chat_item = handler_mock # type: ignore
await adapter._handle_event({"corrId": own, "type": "newChatItem"})
handler_mock.assert_not_called()
assert own not in adapter._pending_corr_ids # discarded
# ---------------------------------------------------------------------------
# 9. Standalone (out-of-process) send for cron
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_standalone_send_missing_websockets(monkeypatch):
"""When websockets is unimportable, return a clean error dict.
Implementation detail: the standalone path does ``import websockets``
inside the function body. We simulate the package being absent by
pulling it out of ``sys.modules`` and pointing the finder at None.
"""
import sys
saved_websockets = sys.modules.pop("websockets", None)
saved_meta = list(sys.meta_path)
class _Blocker:
@staticmethod
def find_spec(name, path=None, target=None):
if name == "websockets" or name.startswith("websockets."):
raise ImportError("websockets blocked for test")
return None
sys.meta_path.insert(0, _Blocker())
try:
pconfig = MagicMock()
pconfig.extra = {"ws_url": "ws://localhost:5225"}
result = await _standalone_send(pconfig, "contact-42", "hi")
assert isinstance(result, dict)
assert "error" in result
assert "websockets" in result["error"]
finally:
sys.meta_path[:] = saved_meta
if saved_websockets is not None:
sys.modules["websockets"] = saved_websockets
@pytest.mark.asyncio
async def test_standalone_send_missing_url(monkeypatch):
monkeypatch.delenv("SIMPLEX_WS_URL", raising=False)
pconfig = MagicMock()
pconfig.extra = {}
# We expect the URL fallback (extra+env both empty) to be empty string,
# producing an error. We also need websockets to be importable for the
# url-check branch to be reached, so skip when it's not.
try:
import websockets.client # noqa: F401
except ImportError:
pytest.skip("websockets not installed")
result = await _standalone_send(pconfig, "contact-42", "hi")
assert isinstance(result, dict)
# Either error about URL or a connection attempt failure — both are valid
# signals that the standalone path requires configuration.
assert "error" in result
# ---------------------------------------------------------------------------
# 10. register() — plugin-side metadata
# ---------------------------------------------------------------------------
def test_register_calls_register_platform():
ctx = MagicMock()
register(ctx)
ctx.register_platform.assert_called_once()
kwargs = ctx.register_platform.call_args.kwargs
assert kwargs["name"] == "simplex"
assert kwargs["label"] == "SimpleX Chat"
assert kwargs["required_env"] == ["SIMPLEX_WS_URL"]
assert kwargs["allowed_users_env"] == "SIMPLEX_ALLOWED_USERS"
assert kwargs["allow_all_env"] == "SIMPLEX_ALLOW_ALL_USERS"
assert kwargs["cron_deliver_env_var"] == "SIMPLEX_HOME_CHANNEL"
assert callable(kwargs["check_fn"])
assert callable(kwargs["validate_config"])
assert callable(kwargs["is_connected"])
assert callable(kwargs["env_enablement_fn"])
assert callable(kwargs["standalone_sender_fn"])
assert callable(kwargs["adapter_factory"])
assert callable(kwargs["setup_fn"])
# SimpleX uses opaque IDs only — no PII to redact.
assert kwargs["pii_safe"] is True
+88
View File
@@ -691,10 +691,98 @@ class TestSendVideo:
adapter._app.client.chat_postMessage.assert_called_once()
# ---------------------------------------------------------------------------
# TestBangPrefixCommands
# ---------------------------------------------------------------------------
class TestBangPrefixCommands:
"""``!cmd`` is rewritten to ``/cmd`` so commands work inside Slack threads.
Slack natively rejects slash commands invoked from a thread reply
("/queue is not supported in threads. Sorry!"). Typing ``!queue`` as a
plain text reply hits the message event pipeline instead, and the
adapter rewrites the leading ``!`` to ``/`` for any known gateway
command before downstream processing.
"""
def _make_event(self, text, thread_ts=None, channel_type="im", channel="D123"):
evt = {
"text": text,
"user": "U_USER",
"channel": channel,
"channel_type": channel_type,
"ts": "1234567890.000001",
}
if thread_ts:
evt["thread_ts"] = thread_ts
return evt
@pytest.mark.asyncio
async def test_bang_known_command_is_rewritten_to_slash(self, adapter):
"""``!queue`` → ``/queue`` and tagged as COMMAND."""
await adapter._handle_slack_message(self._make_event("!queue"))
adapter.handle_message.assert_called_once()
msg_event = adapter.handle_message.call_args[0][0]
assert msg_event.text.startswith("/queue")
assert msg_event.message_type == MessageType.COMMAND
@pytest.mark.asyncio
async def test_bang_command_with_args_preserved(self, adapter):
"""``!model gpt-5.4`` → ``/model gpt-5.4``."""
await adapter._handle_slack_message(self._make_event("!model gpt-5.4"))
msg_event = adapter.handle_message.call_args[0][0]
assert msg_event.text.startswith("/model gpt-5.4")
assert msg_event.message_type == MessageType.COMMAND
@pytest.mark.asyncio
async def test_bang_works_inside_thread(self, adapter):
"""The whole point: ``!stop`` inside a thread reply dispatches."""
evt = self._make_event("!stop", thread_ts="1111111111.000001")
await adapter._handle_slack_message(evt)
msg_event = adapter.handle_message.call_args[0][0]
assert msg_event.text.startswith("/stop")
assert msg_event.message_type == MessageType.COMMAND
# thread_id is preserved on the source so the reply lands in the
# same thread.
assert msg_event.source.thread_id == "1111111111.000001"
@pytest.mark.asyncio
async def test_bang_unknown_token_passes_through_unchanged(self, adapter):
"""``!nice work`` is just a casual message — must NOT be rewritten."""
await adapter._handle_slack_message(self._make_event("!nice work"))
msg_event = adapter.handle_message.call_args[0][0]
assert msg_event.text == "!nice work"
assert msg_event.message_type != MessageType.COMMAND
@pytest.mark.asyncio
async def test_bang_with_bot_suffix_resolves(self, adapter):
"""``!stop@hermes`` matches the get_command() ``@suffix`` stripping."""
await adapter._handle_slack_message(self._make_event("!stop@hermes"))
msg_event = adapter.handle_message.call_args[0][0]
assert msg_event.text.startswith("/stop@hermes")
assert msg_event.message_type == MessageType.COMMAND
@pytest.mark.asyncio
async def test_plain_slash_still_works(self, adapter):
"""Sanity check — ``/queue`` (top-level channel/DM) still dispatches."""
await adapter._handle_slack_message(self._make_event("/queue"))
msg_event = adapter.handle_message.call_args[0][0]
assert msg_event.text.startswith("/queue")
assert msg_event.message_type == MessageType.COMMAND
# ---------------------------------------------------------------------------
# TestIncomingDocumentHandling
# ---------------------------------------------------------------------------
class TestIncomingDocumentHandling:
def _make_event(self, files=None, text="hello", channel_type="im", blocks=None, attachments=None):
"""Build a mock Slack message event with file attachments."""
@@ -195,6 +195,29 @@ class TestTelegramExecApproval:
or kwargs.get("link_preview_options") is not None
)
@pytest.mark.asyncio
async def test_send_update_prompt_escapes_dynamic_prompt(self):
adapter = _make_adapter()
sent = {}
async def mock_send_message(**kwargs):
sent.update(kwargs)
return SimpleNamespace(message_id=55)
adapter._bot.send_message = AsyncMock(side_effect=mock_send_message)
result = await adapter.send_update_prompt(
chat_id="12345",
prompt="Fix [issue]_1 and verify *markdown*",
default="alpha_beta",
metadata={"thread_id": "999"},
)
assert result.success is True
assert "MARKDOWN_V2" in repr(sent["parse_mode"])
assert "Fix \\[issue\\]\\_1" in sent["text"]
assert "alpha\\_beta" in sent["text"]
@pytest.mark.asyncio
async def test_truncates_long_command(self):
adapter = _make_adapter()
@@ -210,9 +233,6 @@ class TestTelegramExecApproval:
kwargs = adapter._bot.send_message.call_args[1]
assert "..." in kwargs["text"]
assert len(kwargs["text"]) < 5000
# ===========================================================================
# _handle_callback_query — approval button clicks
# ===========================================================================
@@ -251,6 +271,34 @@ class TestTelegramApprovalCallback:
# State should be cleaned up
assert 1 not in adapter._approval_state
@pytest.mark.asyncio
async def test_approval_callback_escapes_dynamic_user_name(self):
adapter = _make_adapter()
adapter._approval_state[3] = "agent:main:telegram:group:12345:99"
query = AsyncMock()
query.data = "ea:once:3"
query.message = MagicMock()
query.message.chat_id = 12345
query.from_user = MagicMock()
query.from_user.first_name = "Alice_Bob"
query.answer = AsyncMock()
query.edit_message_text = AsyncMock()
update = MagicMock()
update.callback_query = query
context = MagicMock()
query.from_user.id = "12345"
with patch.dict(os.environ, {"TELEGRAM_ALLOWED_USERS": "*"}, clear=False):
with patch("tools.approval.resolve_gateway_approval", return_value=1):
await adapter._handle_callback_query(update, context)
edit_kwargs = query.edit_message_text.call_args[1]
assert "MARKDOWN_V2" in repr(edit_kwargs["parse_mode"])
assert "Alice\\_Bob" in edit_kwargs["text"]
assert "Approved once" in edit_kwargs["text"]
@pytest.mark.asyncio
async def test_deny_button(self):
adapter = _make_adapter()
+13
View File
@@ -210,6 +210,19 @@ class TestFormatMessageBoldItalic:
assert "*bold*" in result
assert "_italic_" in result
def test_reload_mcp_summary_escapes_dynamic_server_names(self, adapter):
content = (
"🔄 **MCP Servers Reloaded**\n"
"♻️ Reconnected: agent_one, tool[beta]\n"
" Added: alpha*prod\n"
"🔧 3 tool(s) available from 2 server(s)"
)
result = adapter.format_message(content)
assert "*MCP Servers Reloaded*" in result
assert "agent\\_one" in result
assert "tool\\[beta\\]" in result
assert "alpha\\*prod" in result
# =========================================================================
# format_message - headers
+103
View File
@@ -43,6 +43,109 @@ def _make_adapter():
class TestTelegramModelPicker:
@pytest.mark.asyncio
async def test_send_model_picker_escapes_dynamic_provider_label(self):
adapter = _make_adapter()
sent = {}
async def mock_send_message(**kwargs):
sent.update(kwargs)
return SimpleNamespace(message_id=101)
adapter._bot.send_message = AsyncMock(side_effect=mock_send_message)
result = await adapter.send_model_picker(
chat_id="12345",
providers=[
{"slug": "provider_one", "name": "Provider One", "total_models": 1, "is_current": True}
],
current_model="model_1",
current_provider="provider_one",
session_key="s",
on_model_selected=AsyncMock(),
metadata={"thread_id": "99999"},
)
assert result.success is True
assert "MARKDOWN_V2" in repr(sent["parse_mode"])
assert "provider\\_one" in sent["text"]
assert "`model_1`" in sent["text"]
@pytest.mark.asyncio
async def test_back_button_escapes_dynamic_provider_label(self):
adapter = _make_adapter()
adapter._model_picker_state["12345"] = {
"providers": [{"slug": "provider_one", "name": "Provider One", "total_models": 1, "is_current": True}],
"current_model": "model_1",
"current_provider": "provider_one",
"session_key": "s",
"on_model_selected": AsyncMock(),
"msg_id": 42,
}
query = AsyncMock()
query.data = "mb"
query.message = MagicMock()
query.message.chat_id = 12345
query.from_user = MagicMock()
query.answer = AsyncMock()
query.edit_message_text = AsyncMock()
update = MagicMock()
update.callback_query = query
context = MagicMock()
await adapter._handle_model_picker_callback(query, "mb", "12345")
edit_kwargs = query.edit_message_text.call_args[1]
assert "MARKDOWN_V2" in repr(edit_kwargs["parse_mode"])
assert "provider\\_one" in edit_kwargs["text"]
assert "`model_1`" in edit_kwargs["text"]
@pytest.mark.asyncio
async def test_model_selected_edits_message_on_success(self):
"""Regression: the mm: (model selected → switch) success path must
edit the picker message to show the confirmation and remove the
buttons. An earlier revision of this PR over-indented the
edit_message_text block so it lived inside the except branch and
only fired when the callback raised."""
adapter = _make_adapter()
callback = AsyncMock(return_value="Switched to `gpt-5`")
adapter._model_picker_state["12345"] = {
"providers": [
{"slug": "openai", "name": "OpenAI", "total_models": 1, "is_current": True}
],
"current_model": "model_1",
"current_provider": "openai",
"session_key": "s",
"on_model_selected": callback,
"selected_provider": "openai",
"model_list": ["gpt-5"],
"msg_id": 42,
}
query = AsyncMock()
query.data = "mm:0"
query.message = MagicMock()
query.message.chat_id = 12345
query.answer = AsyncMock()
query.edit_message_text = AsyncMock()
await adapter._handle_model_picker_callback(query, "mm:0", "12345")
# The callback was invoked with the selected model
callback.assert_awaited_once()
# edit_message_text MUST be called on the success path (this is the
# regression we're guarding).
query.edit_message_text.assert_awaited()
edit_kwargs = query.edit_message_text.call_args[1]
assert "MARKDOWN_V2" in repr(edit_kwargs["parse_mode"])
# The dynamic result text was routed through format_message
# (backtick code blocks survive escaping).
assert "`gpt-5`" in edit_kwargs["text"]
# State is cleaned up after a successful switch.
assert "12345" not in adapter._model_picker_state
@pytest.mark.asyncio
async def test_retries_without_thread_when_thread_not_found(self):
adapter = _make_adapter()
+59
View File
@@ -14,6 +14,8 @@ to ``_run_agent``'s return dict and uses it for the slice.
import pytest
from gateway.run import _preserve_queued_followup_history_offset
# ---------------------------------------------------------------------------
# Helpers - replicate the filtering logic from _run_agent
@@ -265,3 +267,60 @@ class TestTranscriptHistoryOffset:
assert len(fixed_new) == 2
assert fixed_new[0]["content"] == "Now search for dogs"
assert fixed_new[1]["content"] == "Dog results here."
def test_recursive_queued_followup_keeps_outer_history_offset(self):
"""Queued drain persistence must include every turn in the chain.
``_run_agent()`` recurses when a follow-up arrived while the current turn
was running. The recursive call naturally returns a later
``history_offset`` because it received the previous turn as part of its
input history. If the outer caller persists transcript rows using that
later offset, it only sees the *last* queued turn as new and drops the
earlier queued turn from the transcript.
"""
history_before_chain = [
{"role": "user", "content": "Earlier question"},
{"role": "assistant", "content": "Earlier answer"},
]
first_followup_turn = [
{"role": "user", "content": "First follow-up question"},
{"role": "assistant", "content": "First follow-up answer"},
]
second_followup_turn = [
{"role": "user", "content": "Second follow-up question"},
{"role": "assistant", "content": "Second follow-up answer"},
]
current_result = {
"history_offset": len(history_before_chain),
"messages": history_before_chain + first_followup_turn,
}
followup_result = {
"history_offset": len(history_before_chain + first_followup_turn),
"messages": (
history_before_chain
+ first_followup_turn
+ second_followup_turn
),
}
merged = _preserve_queued_followup_history_offset(
current_result,
followup_result,
)
assert merged["history_offset"] == len(history_before_chain)
persisted = merged["messages"][merged["history_offset"]:]
assert persisted == first_followup_turn + second_followup_turn
def test_recursive_queued_followup_preserves_smaller_existing_offset(self):
"""Do not widen the slice if the nested result is already conservative."""
current_result = {"history_offset": 4}
followup_result = {"history_offset": 3, "messages": []}
merged = _preserve_queued_followup_history_offset(
current_result,
followup_result,
)
assert merged["history_offset"] == 3
+39
View File
@@ -46,6 +46,10 @@ def _make_adapter():
adapter._message_queue = asyncio.Queue()
adapter._http_session = MagicMock()
adapter._mention_patterns = []
adapter._dm_policy = "open"
adapter._allow_from = set()
adapter._group_policy = "open"
adapter._group_allow_from = set()
return adapter
@@ -287,6 +291,41 @@ class TestSendChunking:
assert "Not connected" in result.error
# ---------------------------------------------------------------------------
# bridge event metadata
# ---------------------------------------------------------------------------
class TestBridgeEventMetadata:
"""WhatsApp bridge metadata is preserved for downstream consumers."""
@pytest.mark.asyncio
async def test_quoted_reply_metadata_is_preserved_in_raw_message(self):
adapter = _make_adapter()
data = {
"messageId": "incoming-msg",
"chatId": "15551234567@s.whatsapp.net",
"senderId": "15551234567@s.whatsapp.net",
"senderName": "Tester",
"chatName": "Tester",
"isGroup": False,
"body": "approved",
"hasMedia": False,
"mediaUrls": [],
"quotedMessageId": "outbound-msg",
"quotedParticipant": "99999999999@s.whatsapp.net",
"quotedRemoteJid": "15551234567@s.whatsapp.net",
"hasQuotedMessage": True,
}
event = await adapter._build_message_event(data)
assert event is not None
assert event.raw_message["quotedMessageId"] == "outbound-msg"
assert event.raw_message["quotedParticipant"] == "99999999999@s.whatsapp.net"
assert event.raw_message["quotedRemoteJid"] == "15551234567@s.whatsapp.net"
assert event.raw_message["hasQuotedMessage"] is True
# ---------------------------------------------------------------------------
# display_config tier classification
# ---------------------------------------------------------------------------
@@ -296,3 +296,78 @@ def test_config_bridges_whatsapp_allow_from(monkeypatch, tmp_path):
assert config.platforms[Platform.WHATSAPP].extra["allow_from"] == ["6281234567890@s.whatsapp.net"]
assert __import__("os").environ["WHATSAPP_DM_POLICY"] == "allowlist"
assert __import__("os").environ["WHATSAPP_ALLOWED_USERS"] == "6281234567890@s.whatsapp.net"
# --- Broadcast / status / newsletter pseudo-chats are always dropped ---
def test_status_broadcast_chats_are_always_dropped():
"""Felipe's gateway.log showed the agent replying to status@broadcast
(a contact's WhatsApp Story update). These pseudo-chats aren't real
conversations and the adapter must drop them regardless of dm_policy.
"""
from gateway.platforms.whatsapp import WhatsAppAdapter
# Even on the most permissive config — open DMs, no allowlist — Stories
# and Channel posts must not reach the agent.
adapter = _make_adapter(dm_policy="open")
# Classic Story update — what Felipe was seeing in production.
status_msg = _dm_message(
body="[video received]",
chatId="status@broadcast",
senderId="34612345678@s.whatsapp.net",
)
assert adapter._should_process_message(status_msg) is False
# Channel / Newsletter broadcast posts.
newsletter_msg = _dm_message(
body="check out our latest post",
chatId="120363999999999999@newsletter",
senderId="120363999999999999@newsletter",
)
assert adapter._should_process_message(newsletter_msg) is False
def test_broadcast_filter_runs_before_allowlist():
"""A status@broadcast message from an allowlisted sender still drops —
we never want to reply to Stories, even from authorized contacts.
"""
adapter = _make_adapter(
dm_policy="allowlist",
allow_from=["34612345678@s.whatsapp.net"],
)
msg = _dm_message(
body="[image received]",
chatId="status@broadcast",
senderId="34612345678@s.whatsapp.net",
)
assert adapter._should_process_message(msg) is False
def test_real_dm_still_processed_after_broadcast_filter():
"""Sanity check: the broadcast filter doesn't accidentally drop real DMs."""
adapter = _make_adapter(dm_policy="open")
msg = _dm_message(
body="hello",
chatId="34612345678@s.whatsapp.net",
senderId="34612345678@s.whatsapp.net",
)
assert adapter._should_process_message(msg) is True
def test_is_broadcast_chat_helper_recognizes_common_jids():
from gateway.platforms.whatsapp import WhatsAppAdapter
assert WhatsAppAdapter._is_broadcast_chat("status@broadcast") is True
assert WhatsAppAdapter._is_broadcast_chat("STATUS@BROADCAST") is True
assert WhatsAppAdapter._is_broadcast_chat(" status@broadcast ") is True
assert WhatsAppAdapter._is_broadcast_chat("120363999999999999@newsletter") is True
assert WhatsAppAdapter._is_broadcast_chat("1234@broadcast") is True # broadcast list
# Real chats must not match.
assert WhatsAppAdapter._is_broadcast_chat("34612345678@s.whatsapp.net") is False
assert WhatsAppAdapter._is_broadcast_chat("120363001234567890@g.us") is False
assert WhatsAppAdapter._is_broadcast_chat("") is False
assert WhatsAppAdapter._is_broadcast_chat(None) is False # type: ignore[arg-type]
+153
View File
@@ -1099,6 +1099,159 @@ class TestHuggingFaceModels:
assert _PROVIDER_LABELS["huggingface"] == "Hugging Face"
# =============================================================================
# NovitaAI provider tests (added by feat/add-novita-provider)
# =============================================================================
class TestNovitaProvider:
"""Tests for NovitaAI — an OpenAI-compatible multi-model aggregator."""
def test_novita_profile_loads(self):
from providers import get_provider_profile
profile = get_provider_profile("novita")
assert profile is not None
assert profile.name == "novita"
assert profile.display_name == "NovitaAI"
assert profile.base_url == "https://api.novita.ai/openai/v1"
assert "NOVITA_API_KEY" in profile.env_vars
def test_novita_aliases(self):
from providers import get_provider_profile
profile = get_provider_profile("novita")
assert "novita-ai" in profile.aliases
assert "novitaai" in profile.aliases
def test_novita_alias_resolves(self):
assert resolve_provider("novita-ai") == "novita"
assert resolve_provider("novitaai") == "novita"
def test_novita_in_provider_registry(self):
"""Auto-registration from ProviderProfile should expose Novita."""
assert "novita" in PROVIDER_REGISTRY
pconfig = PROVIDER_REGISTRY["novita"]
assert pconfig.auth_type == "api_key"
assert pconfig.id == "novita"
assert pconfig.inference_base_url == "https://api.novita.ai/openai/v1"
assert pconfig.api_key_env_vars == ("NOVITA_API_KEY",)
assert pconfig.base_url_env_var == "NOVITA_BASE_URL"
def test_novita_aliases_in_registry(self):
assert "novita-ai" in PROVIDER_REGISTRY
assert "novitaai" in PROVIDER_REGISTRY
def test_main_provider_models_has_novita(self):
from hermes_cli.main import _PROVIDER_MODELS
assert "novita" in _PROVIDER_MODELS
assert len(_PROVIDER_MODELS["novita"]) >= 1
def test_models_py_has_novita(self):
from hermes_cli.models import _PROVIDER_MODELS
assert "novita" in _PROVIDER_MODELS
assert len(_PROVIDER_MODELS["novita"]) >= 1
def test_novita_model_lists_match(self):
"""Model lists in main.py and models.py should be identical."""
from hermes_cli.main import _PROVIDER_MODELS as main_models
from hermes_cli.models import _PROVIDER_MODELS as models_models
assert main_models["novita"] == models_models["novita"]
def test_novita_models_use_org_name_format(self):
"""Novita models should use org/name format."""
from hermes_cli.models import _PROVIDER_MODELS
for model in _PROVIDER_MODELS["novita"]:
assert "/" in model, f"Novita model {model!r} missing org/ prefix"
def test_novita_aliases_in_models_py(self):
from hermes_cli.models import _PROVIDER_ALIASES
assert _PROVIDER_ALIASES.get("novita-ai") == "novita"
assert _PROVIDER_ALIASES.get("novitaai") == "novita"
def test_novita_label(self):
from hermes_cli.models import _PROVIDER_LABELS
assert "novita" in _PROVIDER_LABELS
assert _PROVIDER_LABELS["novita"] == "NovitaAI"
def test_novita_in_provider_prefixes(self):
from agent.model_metadata import _PROVIDER_PREFIXES
assert "novita" in _PROVIDER_PREFIXES
def test_novita_url_to_provider(self):
from agent.model_metadata import _URL_TO_PROVIDER
assert _URL_TO_PROVIDER.get("api.novita.ai") == "novita"
def test_context_size_in_context_length_keys(self):
"""Novita /v1/models uses 'context_size' as the context length key."""
from agent.model_metadata import _CONTEXT_LENGTH_KEYS
assert "context_size" in _CONTEXT_LENGTH_KEYS
def test_novita_pricing_unit_conversion(self):
"""Novita returns prices in 0.0001 USD per Mtok; divide by 10_000 * 1_000_000."""
from agent.model_metadata import _extract_pricing
# Sample shape from real Novita /v1/models response
payload = {
"id": "deepseek/deepseek-v3-0324",
"input_token_price_per_m": 2690, # = $0.269 / Mtok
"output_token_price_per_m": 4000, # = $0.400 / Mtok
}
result = _extract_pricing(payload)
# Resulting strings represent per-token prices in dollars.
assert "prompt" in result
assert "completion" in result
assert float(result["prompt"]) == 2690 / 10_000 / 1_000_000
assert float(result["completion"]) == 4000 / 10_000 / 1_000_000
def test_novita_pricing_cache(self, monkeypatch):
"""_fetch_novita_pricing should cache results in _pricing_cache."""
from hermes_cli import models as models_mod
monkeypatch.setenv("NOVITA_API_KEY", "sk-test-key")
monkeypatch.setenv("NOVITA_BASE_URL", "https://api.novita.ai/openai/v1")
models_mod._pricing_cache.pop("https://api.novita.ai/openai/v1", None)
call_count = {"n": 0}
fake_payload = {
"data": [
{
"id": "x/y",
"input_token_price_per_m": 1000,
"output_token_price_per_m": 2000,
}
]
}
class _FakeResp:
def __enter__(self):
return self
def __exit__(self, *args):
return False
def read(self):
import json as _json
return _json.dumps(fake_payload).encode()
def fake_urlopen(req, timeout=None):
call_count["n"] += 1
return _FakeResp()
monkeypatch.setattr(
models_mod.urllib.request, "urlopen", fake_urlopen
)
# First call hits the network.
first = models_mod._fetch_novita_pricing()
assert "x/y" in first
assert call_count["n"] == 1
# Second call returns cached result without re-hitting the network.
second = models_mod._fetch_novita_pricing()
assert second == first
assert call_count["n"] == 1
# force_refresh bypasses the cache.
models_mod._fetch_novita_pricing(force_refresh=True)
assert call_count["n"] == 2
# =============================================================================
# MiniMax OAuth provider tests (added by feat/minimax-oauth-provider)
# =============================================================================
File diff suppressed because it is too large Load Diff
+17 -2
View File
@@ -17,6 +17,8 @@ All Bedrock API calls are mocked — no real AWS credentials needed.
"""
import os
from contextlib import contextmanager
from types import ModuleType
from unittest.mock import MagicMock, patch
import pytest
@@ -26,6 +28,19 @@ import pytest
# Shared helpers / fixtures
# ---------------------------------------------------------------------------
@contextmanager
def _mock_botocore_session(*, return_value=None):
"""Patch botocore.session even when botocore is not installed."""
botocore_mod = ModuleType("botocore")
session_mod = ModuleType("botocore.session")
session_mod.get_session = MagicMock(return_value=return_value)
botocore_mod.session = session_mod
with patch.dict("sys.modules", {"botocore": botocore_mod, "botocore.session": session_mod}):
yield session_mod.get_session
_EU_MODELS = [
{"id": "eu.anthropic.claude-sonnet-4-6-20250514-v1:0", "name": "Claude Sonnet 4.6 (EU)", "provider": "inference-profile"},
{"id": "eu.anthropic.claude-haiku-4-5-20251015-v1:0", "name": "Claude Haiku 4.5 (EU)", "provider": "inference-profile"},
@@ -276,7 +291,7 @@ class TestBedrockRegionRouting:
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover), \
patch("botocore.session.get_session", return_value=mock_session):
_mock_botocore_session(return_value=mock_session):
providers = list_authenticated_providers(current_provider="bedrock")
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
@@ -310,7 +325,7 @@ class TestBedrockRegionRouting:
mock_session = MagicMock()
mock_session.get_config_variable.return_value = "eu-central-1"
with patch("botocore.session.get_session", return_value=mock_session):
with _mock_botocore_session(return_value=mock_session):
region = resolve_bedrock_region()
assert region == "us-west-2", "env var should override botocore profile"
@@ -0,0 +1,865 @@
"""Tests for the codex MCP plugin migration helper."""
from __future__ import annotations
from pathlib import Path
import pytest
from hermes_cli.codex_runtime_plugin_migration import (
MIGRATION_MARKER,
MIGRATION_END_MARKER,
MigrationReport,
_build_hermes_tools_mcp_entry,
_format_toml_value,
_looks_like_test_tempdir,
_strip_existing_managed_block,
_strip_unmanaged_plugin_tables,
_translate_one_server,
migrate,
render_codex_toml_section,
)
# ---- per-server translation ----
class TestTranslateOneServer:
def test_stdio_basic(self):
cfg, skipped = _translate_one_server("filesystem", {
"command": "npx",
"args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"],
"env": {"FOO": "bar"},
})
assert cfg == {
"command": "npx",
"args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"],
"env": {"FOO": "bar"},
}
assert skipped == []
def test_stdio_with_cwd(self):
cfg, _ = _translate_one_server("custom", {
"command": "/usr/bin/myserver",
"cwd": "/var/lib/mcp",
})
assert cfg["cwd"] == "/var/lib/mcp"
def test_http_basic(self):
cfg, skipped = _translate_one_server("api", {
"url": "https://x.example/mcp",
"headers": {"Authorization": "Bearer abc"},
})
assert cfg == {
"url": "https://x.example/mcp",
"http_headers": {"Authorization": "Bearer abc"},
}
assert skipped == []
def test_sse_falls_under_streamable_http_with_warning(self):
cfg, skipped = _translate_one_server("sse_server", {
"url": "http://localhost:8000/sse",
"transport": "sse",
})
assert cfg["url"] == "http://localhost:8000/sse"
assert any("sse" in s.lower() for s in skipped)
def test_timeouts_translate(self):
cfg, _ = _translate_one_server("x", {
"command": "y",
"timeout": 180,
"connect_timeout": 30,
})
assert cfg["tool_timeout_sec"] == 180.0
assert cfg["startup_timeout_sec"] == 30.0
def test_non_numeric_timeout_skipped(self):
cfg, skipped = _translate_one_server("x", {
"command": "y",
"timeout": "not-a-number",
})
assert "tool_timeout_sec" not in cfg
assert any("timeout" in s and "numeric" in s for s in skipped)
def test_disabled_server_emits_enabled_false(self):
cfg, _ = _translate_one_server("x", {
"command": "y",
"enabled": False,
})
assert cfg["enabled"] is False
def test_enabled_true_omitted(self):
cfg, _ = _translate_one_server("x", {"command": "y", "enabled": True})
assert "enabled" not in cfg # codex defaults to true
def test_command_and_url_prefers_stdio_warns(self):
cfg, skipped = _translate_one_server("x", {
"command": "y", "url": "http://z",
})
assert "command" in cfg
assert "url" not in cfg
assert any("url" in s for s in skipped)
def test_no_transport_returns_none(self):
cfg, skipped = _translate_one_server("broken", {"description": "x"})
assert cfg is None
assert "no command or url" in skipped[0]
def test_sampling_dropped_with_warning(self):
cfg, skipped = _translate_one_server("x", {
"command": "y",
"sampling": {"enabled": True, "model": "gemini-3-flash"},
})
assert "sampling" not in cfg
assert any("sampling" in s for s in skipped)
def test_unknown_keys_warned(self):
cfg, skipped = _translate_one_server("x", {
"command": "y",
"totally_made_up_key": "value",
})
assert "totally_made_up_key" not in cfg
assert any("totally_made_up_key" in s for s in skipped)
def test_non_dict_input(self):
cfg, skipped = _translate_one_server("x", "notadict") # type: ignore[arg-type]
assert cfg is None
# ---- TOML rendering ----
class TestTomlValueFormatter:
def test_string_quoted(self):
assert _format_toml_value("hello") == '"hello"'
def test_string_with_quotes_escaped(self):
assert _format_toml_value('a"b') == '"a\\"b"'
def test_bool(self):
assert _format_toml_value(True) == "true"
assert _format_toml_value(False) == "false"
def test_int(self):
assert _format_toml_value(42) == "42"
def test_float(self):
assert _format_toml_value(180.0) == "180.0"
def test_list_of_strings(self):
assert _format_toml_value(["a", "b"]) == '["a", "b"]'
def test_inline_table(self):
out = _format_toml_value({"FOO": "bar"})
assert out == '{ FOO = "bar" }'
def test_empty_inline_table(self):
assert _format_toml_value({}) == "{}"
def test_string_with_newline_escaped(self):
"""TOML basic strings don't allow literal newlines — a path or
env var containing a newline must use \\n. Otherwise codex would
refuse to load the config."""
out = _format_toml_value("line one\nline two")
assert "\n" not in out # no raw newline in output
assert "\\n" in out
def test_string_with_tab_escaped(self):
out = _format_toml_value("col1\tcol2")
assert "\t" not in out
assert "\\t" in out
def test_string_with_other_controls_escaped(self):
for raw, expected in [
("\r", "\\r"),
("\f", "\\f"),
("\b", "\\b"),
]:
out = _format_toml_value(f"x{raw}y")
assert raw not in out, f"{raw!r} should be escaped"
assert expected in out, f"{expected!r} should be in output"
def test_windows_path_escaped_correctly(self):
out = _format_toml_value(r"C:\Users\Alice\.codex")
# Each backslash should be doubled
assert out == r'"C:\\Users\\Alice\\.codex"'
def test_atomic_write_no_temp_leak_on_success(self, tmp_path):
"""The atomic-write path uses tempfile.mkstemp + rename. On
success the temp file should not be left behind."""
migrate({"mcp_servers": {"x": {"command": "y"}}},
codex_home=tmp_path,
discover_plugins=False,
expose_hermes_tools=False,
default_permission_profile=None)
# config.toml should exist
assert (tmp_path / "config.toml").exists()
# And no .config.toml.* temp files left behind
leftover = [p.name for p in tmp_path.iterdir()
if p.name.startswith(".config.toml.")]
assert leftover == [], f"temp file leaked after migration: {leftover}"
def test_atomic_write_cleanup_on_rename_failure(self, tmp_path, monkeypatch):
"""If rename fails partway through (out of disk, permissions,
crash), the temp file must be cleaned up. Otherwise repeated
failed migrations would pile up .config.toml.* files."""
from pathlib import Path as _Path
original_replace = _Path.replace
def failing_replace(self, target):
raise OSError("simulated disk full")
monkeypatch.setattr(_Path, "replace", failing_replace)
report = migrate(
{"mcp_servers": {"x": {"command": "y"}}},
codex_home=tmp_path,
discover_plugins=False,
expose_hermes_tools=False,
default_permission_profile=None,
)
# Error surfaced
assert any("simulated disk full" in e for e in report.errors)
# And no leaked temp file
leftover = [p.name for p in tmp_path.iterdir()
if p.name.startswith(".config.toml.")]
assert leftover == [], f"temp files leaked: {leftover}"
def test_unsupported_type_raises(self):
with pytest.raises(ValueError):
_format_toml_value(object())
class TestRenderToml:
def test_starts_with_marker(self):
out = render_codex_toml_section({})
assert out.startswith(MIGRATION_MARKER)
def test_empty_servers_emits_placeholder(self):
out = render_codex_toml_section({})
assert "no MCP servers" in out
def test_servers_sorted_alphabetically(self):
out = render_codex_toml_section({
"zoo": {"command": "z"},
"alpha": {"command": "a"},
"middle": {"command": "m"},
})
# Find the section header positions and confirm order
a_pos = out.find("[mcp_servers.alpha]")
m_pos = out.find("[mcp_servers.middle]")
z_pos = out.find("[mcp_servers.zoo]")
assert 0 < a_pos < m_pos < z_pos
def test_server_with_args_and_env(self):
out = render_codex_toml_section({
"fs": {
"command": "npx",
"args": ["-y", "filesystem"],
"env": {"PATH": "/usr/bin"},
}
})
assert "[mcp_servers.fs]" in out
assert 'command = "npx"' in out
assert 'args = ["-y", "filesystem"]' in out
# Env emitted as inline table
assert 'env = { PATH = "/usr/bin" }' in out
# ---- existing-block stripping ----
class TestStripExistingManagedBlock:
def test_no_managed_block_unchanged(self):
text = "[other]\nfoo = 1\n"
assert _strip_existing_managed_block(text) == text
def test_strips_managed_block_alone(self):
text = (
f"{MIGRATION_MARKER}\n"
"\n"
"[mcp_servers.fs]\n"
'command = "npx"\n'
)
assert _strip_existing_managed_block(text).strip() == ""
def test_preserves_user_content_above_managed_block(self):
text = (
"[model]\n"
'name = "gpt-5.5"\n'
"\n"
f"{MIGRATION_MARKER}\n"
"[mcp_servers.fs]\n"
'command = "x"\n'
)
out = _strip_existing_managed_block(text)
assert "[model]" in out
assert 'name = "gpt-5.5"' in out
assert "mcp_servers.fs" not in out
def test_preserves_unrelated_section_after_managed_block(self):
text = (
f"{MIGRATION_MARKER}\n"
"[mcp_servers.fs]\n"
'command = "x"\n'
"\n"
"[providers]\n"
'foo = "bar"\n'
)
out = _strip_existing_managed_block(text)
assert "mcp_servers.fs" not in out
assert "[providers]" in out
assert 'foo = "bar"' in out
# ---- end-to-end migrate(, expose_hermes_tools=False) ----
class TestMigrate:
def test_no_servers_no_plugins_no_perms_writes_placeholder(self, tmp_path):
report = migrate({}, codex_home=tmp_path,
discover_plugins=False,
default_permission_profile=None, expose_hermes_tools=False)
assert report.written
text = (tmp_path / "config.toml").read_text()
assert MIGRATION_MARKER in text
assert "no MCP servers" in text or "no MCP servers, plugins, or permissions" in text
def test_no_servers_still_writes_permissions_default(self, tmp_path):
"""Even with zero MCP servers, enabling the runtime should write the
default permissions profile so users don't get prompted on every
write attempt. This is the fix for quirk #2."""
report = migrate({}, codex_home=tmp_path, discover_plugins=False, expose_hermes_tools=False)
assert report.written
text = (tmp_path / "config.toml").read_text()
# Codex's schema: top-level `default_permissions` keying a built-in
# profile name (prefixed with ":"). NOT a [permissions] section
# (which is for *user-defined* profiles with structured fields).
assert 'default_permissions = ":workspace"' in text
assert report.wrote_permissions_default == ":workspace"
def test_explicit_none_permissions_skips_block(self, tmp_path):
report = migrate({"mcp_servers": {"x": {"command": "y"}}},
codex_home=tmp_path,
discover_plugins=False,
default_permission_profile=None, expose_hermes_tools=False)
text = (tmp_path / "config.toml").read_text()
assert "default_permissions" not in text
assert "[permissions]" not in text
assert report.wrote_permissions_default is None
def test_plugin_discovery_writes_plugin_blocks(self, tmp_path, monkeypatch):
"""Discovered curated plugins land as [plugins."<name>@<marketplace>"]
blocks. This is what OpenClaw calls 'migrate native codex plugins.'"""
from hermes_cli import codex_runtime_plugin_migration as crpm
def fake_query(codex_home=None, timeout=8.0):
return [
{"name": "google-calendar", "marketplace": "openai-curated",
"enabled": True},
{"name": "github", "marketplace": "openai-curated",
"enabled": True},
], None
monkeypatch.setattr(crpm, "_query_codex_plugins", fake_query)
report = migrate({}, codex_home=tmp_path, discover_plugins=True)
text = (tmp_path / "config.toml").read_text()
assert '[plugins."github@openai-curated"]' in text
assert '[plugins."google-calendar@openai-curated"]' in text
assert "enabled = true" in text
assert "google-calendar@openai-curated" in report.migrated_plugins
assert "github@openai-curated" in report.migrated_plugins
def test_plugin_discovery_skips_unavailable_plugins(self):
"""Plugins where codex reports availability != AVAILABLE should
be skipped they're broken/uninstallable on codex's side, so
migrating them would write config that fails at activation
time. Cf. openclaw#80815."""
from hermes_cli.codex_runtime_plugin_migration import _query_codex_plugins
from unittest.mock import patch
# Fake a plugin/list response where one plugin is unavailable
fake_response = {
"marketplaces": [{
"name": "openai-curated",
"plugins": [
{"name": "good-plugin", "installed": True,
"enabled": True, "availability": "AVAILABLE"},
{"name": "broken-plugin", "installed": True,
"enabled": True, "availability": "UNAVAILABLE"},
{"name": "auth-pending", "installed": True,
"enabled": True, "availability": "REQUIRES_AUTH"},
# Plugin without availability field — pass through
# (older codex versions or marketplaces that don't
# set it should still work).
{"name": "legacy-plugin", "installed": True,
"enabled": True},
]
}]
}
class FakeClient:
def __init__(self, **kw): pass
def initialize(self, **kw): pass
def request(self, method, params, timeout=None):
return fake_response
def close(self): pass
def __enter__(self): return self
def __exit__(self, *a): pass
with patch("agent.transports.codex_app_server.CodexAppServerClient",
FakeClient):
plugins, err = _query_codex_plugins()
assert err is None
names = [p["name"] for p in plugins]
assert "good-plugin" in names
assert "legacy-plugin" in names # no field → don't skip
assert "broken-plugin" not in names
assert "auth-pending" not in names
def test_plugin_discovery_failure_non_fatal(self, tmp_path, monkeypatch):
"""If codex isn't installed or RPC fails, MCP migration still
completes. The error surfaces in the report but doesn't abort."""
from hermes_cli import codex_runtime_plugin_migration as crpm
def fake_query_fails(codex_home=None, timeout=8.0):
return [], "codex CLI not available"
monkeypatch.setattr(crpm, "_query_codex_plugins", fake_query_fails)
report = migrate({"mcp_servers": {"x": {"command": "y"}}},
codex_home=tmp_path, discover_plugins=True, expose_hermes_tools=False)
assert report.written
assert report.migrated == ["x"]
assert report.plugin_query_error == "codex CLI not available"
assert report.migrated_plugins == []
def test_discover_plugins_false_skips_query(self, tmp_path, monkeypatch):
"""Tests and restricted environments can opt out of the subprocess
spawn entirely."""
from hermes_cli import codex_runtime_plugin_migration as crpm
called = {"yes": False}
def boom(*a, **kw):
called["yes"] = True
return [], None
monkeypatch.setattr(crpm, "_query_codex_plugins", boom)
migrate({"mcp_servers": {"x": {"command": "y"}}},
codex_home=tmp_path, discover_plugins=False, expose_hermes_tools=False)
assert called["yes"] is False
def test_dry_run_skips_plugin_query(self, tmp_path, monkeypatch):
"""Dry run should never spawn codex. Even with discover_plugins=True
the query is skipped because dry_run takes precedence."""
from hermes_cli import codex_runtime_plugin_migration as crpm
called = {"yes": False}
def boom(*a, **kw):
called["yes"] = True
return [], None
monkeypatch.setattr(crpm, "_query_codex_plugins", boom)
migrate({"mcp_servers": {"x": {"command": "y"}}},
codex_home=tmp_path, dry_run=True, discover_plugins=True, expose_hermes_tools=False)
assert called["yes"] is False
def test_re_run_replaces_plugin_block(self, tmp_path, monkeypatch):
"""Plugin blocks are managed and re-runs should replace them
cleanly same idempotency contract as MCP servers."""
from hermes_cli import codex_runtime_plugin_migration as crpm
# First run: only github
monkeypatch.setattr(crpm, "_query_codex_plugins",
lambda codex_home=None, timeout=8.0: (
[{"name": "github", "marketplace": "openai-curated", "enabled": True}],
None,
))
migrate({}, codex_home=tmp_path, discover_plugins=True,
default_permission_profile=None, expose_hermes_tools=False)
first = (tmp_path / "config.toml").read_text()
assert "github@openai-curated" in first
# Second run: only canva (github went away)
monkeypatch.setattr(crpm, "_query_codex_plugins",
lambda codex_home=None, timeout=8.0: (
[{"name": "canva", "marketplace": "openai-curated", "enabled": True}],
None,
))
migrate({}, codex_home=tmp_path, discover_plugins=True,
default_permission_profile=None, expose_hermes_tools=False)
second = (tmp_path / "config.toml").read_text()
assert "github@openai-curated" not in second
assert "canva@openai-curated" in second
def test_expose_hermes_tools_writes_callback_mcp_entry(self, tmp_path):
"""When expose_hermes_tools=True (production default), an
[mcp_servers.hermes-tools] entry is written so codex calls back
into Hermes for browser/web/delegate_task/vision/memory tools.
This is the fix for 'all other tools that codex doesn't provide
should be useable by hermes' — quirk #7."""
report = migrate({}, codex_home=tmp_path,
discover_plugins=False,
default_permission_profile=None,
expose_hermes_tools=True)
text = (tmp_path / "config.toml").read_text()
assert "[mcp_servers.hermes-tools]" in text
assert "hermes_tools_mcp_server" in text
# Must include startup + tool timeouts so codex doesn't give up
assert "startup_timeout_sec" in text
assert "tool_timeout_sec" in text
# And the entry is reported
assert "hermes-tools" in report.migrated
def test_expose_hermes_tools_disabled_skips_entry(self, tmp_path):
"""expose_hermes_tools=False suppresses the callback registration."""
migrate({}, codex_home=tmp_path,
discover_plugins=False,
default_permission_profile=None,
expose_hermes_tools=False)
text = (tmp_path / "config.toml").read_text()
assert "[mcp_servers.hermes-tools]" not in text
assert "hermes_tools_mcp_server" not in text
def test_dry_run_doesnt_write(self, tmp_path):
report = migrate({"mcp_servers": {"x": {"command": "y"}}},
codex_home=tmp_path, dry_run=True, expose_hermes_tools=False)
assert report.dry_run is True
assert not (tmp_path / "config.toml").exists()
assert "x" in report.migrated
def test_full_migration_round_trip(self, tmp_path):
hermes_cfg = {
"mcp_servers": {
"filesystem": {
"command": "npx",
"args": ["-y", "@modelcontextprotocol/server-filesystem"],
},
"github": {
"url": "https://api.github.com/mcp",
"headers": {"Authorization": "Bearer x"},
},
}
}
report = migrate(hermes_cfg, codex_home=tmp_path, expose_hermes_tools=False)
assert report.written
text = (tmp_path / "config.toml").read_text()
assert "[mcp_servers.filesystem]" in text
assert "[mcp_servers.github]" in text
assert 'command = "npx"' in text
assert 'url = "https://api.github.com/mcp"' in text
def test_idempotent_re_run_replaces_managed_block(self, tmp_path):
# First migration
migrate({"mcp_servers": {"a": {"command": "x"}}}, codex_home=tmp_path, expose_hermes_tools=False)
first_text = (tmp_path / "config.toml").read_text()
assert "[mcp_servers.a]" in first_text
# Second migration with different servers
migrate({"mcp_servers": {"b": {"command": "y"}}}, codex_home=tmp_path, expose_hermes_tools=False)
second_text = (tmp_path / "config.toml").read_text()
assert "[mcp_servers.a]" not in second_text
assert "[mcp_servers.b]" in second_text
def test_preserves_user_codex_config_above_marker(self, tmp_path):
target = tmp_path / "config.toml"
target.write_text(
"[model]\n"
'profile = "default"\n'
"\n"
"[providers.openai]\n"
'api_key = "sk-test"\n'
)
migrate({"mcp_servers": {"a": {"command": "x"}}}, codex_home=tmp_path, expose_hermes_tools=False)
new_text = target.read_text()
# User's codex config preserved
assert "[model]" in new_text
assert 'profile = "default"' in new_text
assert "[providers.openai]" in new_text
# And new MCP block inserted without breaking user tables
assert "[mcp_servers.a]" in new_text
assert MIGRATION_MARKER in new_text
def test_managed_root_keys_stay_top_level_when_config_ends_in_table(self, tmp_path):
"""TOML has no explicit 'leave current table' syntax. If Hermes appends
root keys like default_permissions after a user table such as [features],
Codex parses them as features.default_permissions and rejects the config.
The managed block must therefore be inserted before the first table."""
import tomllib
target = tmp_path / "config.toml"
target.write_text(
'model = "gpt-5.5"\n'
"\n"
"[features]\n"
"terminal_resize_reflow = true\n"
)
migrate({}, codex_home=tmp_path, discover_plugins=False, expose_hermes_tools=False)
new_text = target.read_text()
parsed = tomllib.loads(new_text)
assert parsed["default_permissions"] == ":workspace"
assert "default_permissions" not in parsed["features"]
assert new_text.index(MIGRATION_MARKER) < new_text.index("[features]")
def test_preserves_user_mcp_server_outside_managed_block(self, tmp_path):
"""Quirk #6: when a user adds their own MCP server entry directly
to ~/.codex/config.toml outside Hermes' managed block, re-running
migration must preserve it. Tested both above and below the
managed block."""
target = tmp_path / "config.toml"
target.write_text(
"[mcp_servers.user-above]\n"
'command = "/usr/bin/above-server"\n'
'args = ["--above"]\n'
)
# First migrate — adds managed block below user content
migrate({"mcp_servers": {"hermes-mcp": {"command": "npx"}}},
codex_home=tmp_path, discover_plugins=False,
expose_hermes_tools=False)
text = target.read_text()
assert "user-above" in text, "user MCP server above managed block got nuked"
assert 'command = "/usr/bin/above-server"' in text
# Append another user entry below the managed block
target.write_text(
text + "\n[mcp_servers.user-below]\ncommand = \"below-server\"\n"
)
# Re-migrate — both should survive
migrate({"mcp_servers": {"hermes-mcp": {"command": "npx"}}},
codex_home=tmp_path, discover_plugins=False,
expose_hermes_tools=False)
final = target.read_text()
assert "user-above" in final
assert "user-below" in final
# And our managed block is still there with the new content
assert "[mcp_servers.hermes-mcp]" in final
def test_skipped_keys_reported(self, tmp_path):
report = migrate({
"mcp_servers": {
"x": {
"command": "y",
"sampling": {"enabled": True}, # codex has no equivalent
}
}
}, codex_home=tmp_path, expose_hermes_tools=False)
assert "x" in report.skipped_keys_per_server
assert any("sampling" in s for s in report.skipped_keys_per_server["x"])
def test_invalid_mcp_servers_value(self, tmp_path):
report = migrate({"mcp_servers": "notadict"}, codex_home=tmp_path, expose_hermes_tools=False)
assert any("not a dict" in e for e in report.errors)
def test_server_without_transport_skipped_with_error(self, tmp_path):
report = migrate({
"mcp_servers": {"broken": {"description": "no command/url"}}
}, codex_home=tmp_path, expose_hermes_tools=False)
assert "broken" not in report.migrated
assert any("broken" in e for e in report.errors)
def test_summary_reports_migration_count(self, tmp_path):
report = migrate({
"mcp_servers": {"a": {"command": "x"}, "b": {"command": "y"}}
}, codex_home=tmp_path, expose_hermes_tools=False)
summary = report.summary()
assert "Migrated 2 MCP server(s)" in summary
assert "- a" in summary
assert "- b" in summary
# ---- Bug B: duplicate [plugins.X] tables ----
class TestStripUnmanagedPluginTables:
"""Regression tests for issue #26250 Bug B.
When codex itself writes ``[plugins."<name>@<marketplace>"]`` tables
(via the user running ``codex plugins enable`` directly), re-running
``hermes codex-runtime migrate`` would re-emit them inside the managed
block and the resulting duplicate-table-header would crash codex.
"""
def test_strips_plugin_tables_outside_managed_block(self):
text = (
'model = "gpt-5.5"\n'
"\n"
"[mcp_servers.user-thing]\n"
'command = "x"\n'
"\n"
'[plugins."tasks@openai-curated"]\n'
"enabled = true\n"
"\n"
'[plugins."web-search@openai-curated"]\n'
"enabled = true\n"
"\n"
"[features]\n"
"terminal_resize_reflow = true\n"
)
stripped = _strip_unmanaged_plugin_tables(text)
assert "[plugins." not in stripped
# Non-plugin content preserved
assert "[mcp_servers.user-thing]" in stripped
assert "[features]" in stripped
assert "terminal_resize_reflow = true" in stripped
def test_preserves_content_when_no_plugin_tables(self):
text = (
'model = "gpt-5.5"\n'
"\n"
"[mcp_servers.x]\n"
'command = "y"\n'
)
assert _strip_unmanaged_plugin_tables(text) == text
def test_multi_line_array_in_plugin_table_does_not_leak(self):
"""A multi-line TOML array inside a [plugins.X] table whose
continuation lines start with ``[`` (e.g. nested arrays) must NOT
prematurely exit the strip region otherwise array fragments
leak into top-level output and produce invalid TOML on the next
codex startup. Regression guard for #26260 review.
"""
text = (
'[plugins."tasks@openai-curated"]\n'
"allowed = [\n"
' "a",\n'
' ["nested"],\n'
"]\n"
"[features]\n"
"x = 1\n"
)
stripped = _strip_unmanaged_plugin_tables(text)
# Everything inside the plugin table — including the multi-line
# array's continuation lines starting with `[` — should be gone.
assert '["nested"]' not in stripped
assert "allowed" not in stripped
# Sibling user table survives intact.
assert "[features]" in stripped
assert "x = 1" in stripped
# Result is still valid TOML.
import tomllib
tomllib.loads(stripped)
def test_migrate_dedups_codex_owned_plugin_tables(self, tmp_path, monkeypatch):
"""End-to-end: codex's pre-existing [plugins.X] tables get replaced by
the managed block's re-emission rather than duplicated."""
target = tmp_path / "config.toml"
target.write_text(
"[mcp_servers.user-server]\n"
'command = "x"\n'
"\n"
'[plugins."tasks@openai-curated"]\n'
"enabled = true\n"
)
# Simulate codex's plugin/list reporting the same plugin tasks@openai-curated.
def fake_query(codex_home=None, timeout=8.0):
return (
[{"name": "tasks", "marketplace": "openai-curated", "enabled": True}],
None,
)
monkeypatch.setattr(
"hermes_cli.codex_runtime_plugin_migration._query_codex_plugins",
fake_query,
)
migrate({}, codex_home=tmp_path, discover_plugins=True, expose_hermes_tools=False)
new_text = target.read_text()
# Only ONE [plugins."tasks@openai-curated"] header should remain — inside
# the managed block — not the original outside-the-block copy.
assert new_text.count('[plugins."tasks@openai-curated"]') == 1
# And the surviving one is inside our managed section.
managed_start = new_text.index(MIGRATION_MARKER)
managed_end = new_text.index(MIGRATION_END_MARKER)
plugin_idx = new_text.index('[plugins."tasks@openai-curated"]')
assert managed_start < plugin_idx < managed_end
# File parses cleanly as TOML (the original duplicate-key error is gone).
import tomllib
tomllib.loads(new_text)
def test_migrate_preserves_plugin_tables_when_plugin_list_fails(self, tmp_path, monkeypatch):
"""If plugin/list RPC fails, we can't re-emit plugins authoritatively,
so we must NOT strip the user's existing [plugins.X] tables — that
would silently lose them."""
target = tmp_path / "config.toml"
target.write_text(
'[plugins."tasks@openai-curated"]\n'
"enabled = true\n"
)
def fake_query(codex_home=None, timeout=8.0):
return ([], "plugin/list query failed: codex not installed")
monkeypatch.setattr(
"hermes_cli.codex_runtime_plugin_migration._query_codex_plugins",
fake_query,
)
migrate({}, codex_home=tmp_path, discover_plugins=True, expose_hermes_tools=False)
new_text = target.read_text()
# User's plugin table preserved verbatim — we can't re-emit it.
assert '[plugins."tasks@openai-curated"]' in new_text
# ---- Bug C: HERMES_HOME tempdir leak into ~/.codex/config.toml ----
class TestHermesHomeLeakGuard:
"""Regression tests for issue #26250 Bug C.
Previously ``_build_hermes_tools_mcp_entry()`` read ``HERMES_HOME``
directly from ``os.environ``, so a pytest ``monkeypatch.setenv`` would
leak a transient tempdir path into the user's real ``~/.codex/config.toml``
once codex spawned the hermes-tools MCP subprocess.
"""
def test_tempdir_detector_recognizes_pytest_paths(self):
assert _looks_like_test_tempdir(
"/private/var/folders/abc/pytest-of-kshitij/pytest-137/popen-gw2/test_X/hermes_test"
)
assert _looks_like_test_tempdir(
"/tmp/pytest-of-user/pytest-12/test_X/hermes"
)
assert _looks_like_test_tempdir(
"/private/var/folders/zz/T/pytest-of-bob/pytest-1"
)
def test_tempdir_detector_accepts_real_hermes_home(self):
assert not _looks_like_test_tempdir("/Users/alice/.hermes")
assert not _looks_like_test_tempdir("/home/bob/.hermes")
assert not _looks_like_test_tempdir("/opt/hermes")
assert not _looks_like_test_tempdir("")
def test_pytest_tempdir_not_burned_into_mcp_env(self, monkeypatch):
"""The headline regression: even when HERMES_HOME points at a pytest
tempdir, _build_hermes_tools_mcp_entry() must NOT propagate it."""
monkeypatch.setenv(
"HERMES_HOME",
"/private/var/folders/xx/pytest-of-user/pytest-99/test_x/hermes_test",
)
entry = _build_hermes_tools_mcp_entry()
env = entry.get("env", {})
assert "HERMES_HOME" not in env, (
f"pytest-tempdir HERMES_HOME leaked into codex MCP entry: "
f"{env.get('HERMES_HOME')!r}"
)
def test_real_hermes_home_propagates(self, monkeypatch, tmp_path):
"""A legitimate HERMES_HOME (not a tempdir path) DOES propagate so the
MCP subprocess sees the same config as the parent CLI."""
# Use a path that looks real — under /Users or /home, not /var/folders.
# We can't easily create one in the test, so just use a stable path
# outside any tempdir-detector needle. The detector checks for tempdir
# markers, not for path existence.
real_path = "/Users/alice/.hermes"
monkeypatch.setenv("HERMES_HOME", real_path)
entry = _build_hermes_tools_mcp_entry()
env = entry.get("env", {})
assert env.get("HERMES_HOME") == real_path
def test_unset_hermes_home_omits_env_key(self, monkeypatch):
"""When HERMES_HOME is unset in the environment, the MCP entry MUST
NOT bake in a resolved-default path. The codex subprocess should
inherit whatever HERMES_HOME its launcher (systemd, gateway, shell)
sets at runtime, rather than being pinned to migrate-time defaults.
Regression guard for issue #26250 follow-up review."""
monkeypatch.delenv("HERMES_HOME", raising=False)
entry = _build_hermes_tools_mcp_entry()
env = entry.get("env", {})
assert "HERMES_HOME" not in env, (
f"HERMES_HOME should not be set when env var is unset, got: "
f"{env.get('HERMES_HOME')!r}"
)
@@ -0,0 +1,238 @@
"""Tests for the /codex-runtime slash-command shared logic.
These cover the pure-Python state machine; CLI and gateway handlers are
tested separately because they involve config persistence and prompt
formatting that's surface-specific."""
from __future__ import annotations
from unittest.mock import patch
import pytest
from hermes_cli import codex_runtime_switch as crs
class TestParseArgs:
@pytest.mark.parametrize("arg,expected", [
("", None),
(" ", None),
("auto", "auto"),
("codex_app_server", "codex_app_server"),
("on", "codex_app_server"),
("off", "auto"),
("codex", "codex_app_server"),
("default", "auto"),
("hermes", "auto"),
("ENABLE", "codex_app_server"), # case-insensitive
("DiSaBlE", "auto"),
])
def test_valid_args(self, arg, expected):
value, errors = crs.parse_args(arg)
assert errors == []
assert value == expected
def test_invalid_arg_returns_error(self):
value, errors = crs.parse_args("turbo")
assert value is None
assert errors and "Unknown runtime" in errors[0]
class TestGetCurrentRuntime:
def test_default_when_unset(self):
assert crs.get_current_runtime({}) == "auto"
assert crs.get_current_runtime({"model": {}}) == "auto"
assert crs.get_current_runtime({"model": {"openai_runtime": ""}}) == "auto"
def test_unrecognized_falls_back_to_auto(self):
assert crs.get_current_runtime(
{"model": {"openai_runtime": "garbage"}}
) == "auto"
def test_explicit_codex(self):
assert crs.get_current_runtime(
{"model": {"openai_runtime": "codex_app_server"}}
) == "codex_app_server"
def test_handles_non_dict_config(self):
assert crs.get_current_runtime(None) == "auto" # type: ignore[arg-type]
assert crs.get_current_runtime("notadict") == "auto" # type: ignore[arg-type]
assert crs.get_current_runtime({"model": "notadict"}) == "auto"
class TestSetRuntime:
def test_creates_model_section_if_missing(self):
cfg = {}
old = crs.set_runtime(cfg, "codex_app_server")
assert old == "auto"
assert cfg["model"]["openai_runtime"] == "codex_app_server"
def test_returns_previous_value(self):
cfg = {"model": {"openai_runtime": "codex_app_server"}}
old = crs.set_runtime(cfg, "auto")
assert old == "codex_app_server"
assert cfg["model"]["openai_runtime"] == "auto"
def test_invalid_value_raises(self):
with pytest.raises(ValueError):
crs.set_runtime({}, "garbage")
class TestApply:
def test_read_only_call_reports_state(self):
cfg = {"model": {"openai_runtime": "codex_app_server"}}
with patch.object(crs, "check_codex_binary_ok",
return_value=(True, "0.130.0")):
r = crs.apply(cfg, None)
assert r.success
assert r.new_value == "codex_app_server"
assert r.old_value == "codex_app_server"
assert "codex_app_server" in r.message
assert "0.130.0" in r.message
def test_no_change_when_already_set(self):
cfg = {"model": {"openai_runtime": "auto"}}
r = crs.apply(cfg, "auto")
assert r.success
assert r.message == "openai_runtime already set to auto"
def test_enable_blocked_when_codex_missing(self):
cfg = {}
with patch.object(crs, "check_codex_binary_ok",
return_value=(False, "codex not found")):
r = crs.apply(cfg, "codex_app_server")
assert r.success is False
assert "Cannot enable" in r.message
assert "npm i -g @openai/codex" in r.message
# Config NOT mutated on failure
assert cfg.get("model", {}).get("openai_runtime") in (None, "")
def test_enable_succeeds_when_codex_present(self):
cfg = {}
persisted = {}
def persist(c):
persisted.update(c)
# Patch migrate so this test doesn't reach into the user's real
# ~/.codex/config.toml. See issue #26250 Bug C — without this patch,
# crs.apply() invokes the real migrate() which writes to
# Path.home() / ".codex" using whatever HERMES_HOME the running pytest
# session has set, leaking pytest tempdir paths into the user's
# codex config.
with patch.object(crs, "check_codex_binary_ok",
return_value=(True, "0.130.0")), \
patch("hermes_cli.codex_runtime_plugin_migration.migrate"):
r = crs.apply(cfg, "codex_app_server", persist_callback=persist)
assert r.success
assert r.new_value == "codex_app_server"
assert r.old_value == "auto"
assert r.requires_new_session is True
assert "via MCP" in r.message # hermes-tools callback message
assert cfg["model"]["openai_runtime"] == "codex_app_server"
assert persisted["model"]["openai_runtime"] == "codex_app_server"
def test_disable_does_not_check_binary(self):
cfg = {"model": {"openai_runtime": "codex_app_server"}}
with patch.object(crs, "check_codex_binary_ok") as bin_check:
r = crs.apply(cfg, "auto")
assert r.success
# Binary check is irrelevant when disabling — should not be called
# with the codex_app_server enable-gate signature.
assert r.new_value == "auto"
assert r.old_value == "codex_app_server"
def test_persist_callback_failure_reported(self):
cfg = {}
def persist_boom(c):
raise IOError("disk full")
with patch.object(crs, "check_codex_binary_ok",
return_value=(True, "0.130.0")):
r = crs.apply(cfg, "codex_app_server", persist_callback=persist_boom)
assert r.success is False
assert "persist failed" in r.message
assert "disk full" in r.message
def test_enable_triggers_mcp_migration(self):
"""Enabling codex_app_server should auto-migrate Hermes mcp_servers
to ~/.codex/config.toml so the spawned subprocess sees them."""
cfg = {
"mcp_servers": {
"filesystem": {"command": "npx", "args": ["-y", "fs-server"]},
}
}
with patch.object(crs, "check_codex_binary_ok",
return_value=(True, "0.130.0")), \
patch("hermes_cli.codex_runtime_plugin_migration.migrate") as mig:
mig.return_value.migrated = ["filesystem", "hermes-tools"]
mig.return_value.migrated_plugins = []
mig.return_value.plugin_query_error = None
mig.return_value.wrote_permissions_default = ":workspace"
mig.return_value.errors = []
mig.return_value.target_path = "/fake/.codex/config.toml"
r = crs.apply(cfg, "codex_app_server")
assert r.success
assert mig.called # migration was triggered
# User MCP servers are reported (excluding internal hermes-tools)
assert "Migrated 1 MCP server" in r.message
assert "filesystem" in r.message
# Permissions default surfaces
assert "Default sandbox: :workspace" in r.message
# Hermes tool callback announcement
assert "via MCP" in r.message
def test_disable_does_not_trigger_migration(self):
"""Switching back to auto must not write to ~/.codex/."""
cfg = {
"model": {"openai_runtime": "codex_app_server"},
"mcp_servers": {"x": {"command": "y"}},
}
with patch("hermes_cli.codex_runtime_plugin_migration.migrate") as mig:
r = crs.apply(cfg, "auto")
assert r.success
assert not mig.called # disabling does not migrate
def test_migration_failure_does_not_block_enable(self):
"""If MCP migration raises, the runtime change still proceeds —
users can manually re-run migration later."""
cfg = {"mcp_servers": {"x": {"command": "y"}}}
with patch.object(crs, "check_codex_binary_ok",
return_value=(True, "0.130.0")), \
patch("hermes_cli.codex_runtime_plugin_migration.migrate",
side_effect=RuntimeError("disk full")):
r = crs.apply(cfg, "codex_app_server")
assert r.success # change still applied
assert r.new_value == "codex_app_server"
assert "MCP migration skipped" in r.message
assert "disk full" in r.message
def test_binary_check_cached_within_apply(self):
"""check_codex_binary_ok is invoked at most once per apply() call.
The enable path has three sites that need the version (state report,
enable gate, success message). Without caching, a single
/codex-runtime invocation spawns `codex --version` three times.
Regression guard against a refactor that drops the cache.
"""
cfg = {}
with patch.object(crs, "check_codex_binary_ok",
return_value=(True, "0.130.0")) as bin_check, \
patch("hermes_cli.codex_runtime_plugin_migration.migrate"):
r = crs.apply(cfg, "codex_app_server")
assert r.success
assert bin_check.call_count == 1, (
f"check_codex_binary_ok was called {bin_check.call_count} time(s); "
"should be cached and called exactly once per apply()"
)
def test_binary_check_cached_on_read_only_call(self):
"""Read-only call (new_value=None) calls the binary check exactly
once and reuses the result for the message."""
cfg = {"model": {"openai_runtime": "codex_app_server"}}
with patch.object(crs, "check_codex_binary_ok",
return_value=(True, "0.130.0")) as bin_check:
crs.apply(cfg, None)
assert bin_check.call_count == 1
+48
View File
@@ -140,6 +140,54 @@ class TestGenerateZsh:
# gateway has subcommands so a _cmds array must be generated
assert "gateway_cmds" in out
def test_registers_compdef_instead_of_invoking_completion_function(self):
out = generate_zsh(_make_parser())
assert 'compdef _hermes hermes' in out
assert '_hermes "$@"' not in out
def test_preserves_valid_zsh_arguments_alias_syntax(self):
out = generate_zsh(_make_parser())
assert "'(-)'{-h,--help}'[Show help and exit]'" in out
assert "'(-)'{-V,--version}'[Show version and exit]'" in out
assert "'(-)'{-p,--profile}'[Profile name]:profile:_hermes_profiles'" in out
assert "'(-h --help){-h,--help}[Show help and exit]'" not in out
assert '"(-h --help)"{-h,--help}"[Show help and exit]"' not in out
def test_valid_zsh_syntax(self):
if not shutil.which("zsh"):
pytest.skip("zsh not installed")
out = generate_zsh(_make_parser())
with tempfile.NamedTemporaryFile(mode="w", suffix=".zsh", delete=False) as f:
f.write(out)
path = f.name
try:
result = subprocess.run(["zsh", "-n", path], capture_output=True, text=True)
assert result.returncode == 0, result.stderr
finally:
os.unlink(path)
def test_zsh_eval_style_source_registers_after_compinit(self):
if not shutil.which("zsh"):
pytest.skip("zsh not installed")
out = generate_zsh(_make_parser())
with tempfile.NamedTemporaryFile(mode="w", suffix=".zsh", delete=False) as f:
f.write(out)
path = f.name
try:
result = subprocess.run(
[
"zsh",
"-fc",
f"autoload -Uz compinit && compinit -D; source {path}; [[ ${{_comps[hermes]}} == _hermes ]]",
],
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
assert result.stderr == ""
finally:
os.unlink(path)
# ---------------------------------------------------------------------------
# 4. Fish output
+193
View File
@@ -0,0 +1,193 @@
"""Tests for the load_env() process-level cache.
The cache exists to keep `hermes tools` "All Platforms" fast: every
`get_env_value()` lookup used to re-read and re-sanitise the entire
.env file, racking up hundreds of ms across one menu render. The
cache is keyed on (path, mtime, size); writers (save_env_value /
remove_env_value / sanitise_env_file) call invalidate_env_cache().
"""
from __future__ import annotations
import os
import tempfile
from pathlib import Path
from unittest.mock import patch
def _write_env(path: Path, contents: str) -> None:
path.write_text(contents, encoding="utf-8")
def test_load_env_caches_on_repeat_calls():
"""Repeated load_env() calls on the same file return the cached dict."""
from hermes_cli.config import invalidate_env_cache, load_env
invalidate_env_cache()
with tempfile.NamedTemporaryFile(
mode="w", suffix=".env", delete=False, encoding="utf-8"
) as f:
f.write("OPENAI_API_KEY=sk-first\n")
env_path = Path(f.name)
try:
with patch("hermes_cli.config.get_env_path", return_value=env_path):
first = load_env()
# Even if a writer outside our cache mutates the file, an
# mtime/size match means the cache still wins. We simulate that
# by writing identical bytes back — sanity check that the cache
# is keyed structurally, not on a counter.
second = load_env()
assert first == second
assert first.get("OPENAI_API_KEY") == "sk-first"
finally:
env_path.unlink(missing_ok=True)
invalidate_env_cache()
def test_load_env_invalidates_on_mtime_bump():
"""Editing the file (mtime changes) invalidates the cache."""
from hermes_cli.config import invalidate_env_cache, load_env
invalidate_env_cache()
with tempfile.NamedTemporaryFile(
mode="w", suffix=".env", delete=False, encoding="utf-8"
) as f:
f.write("OPENAI_API_KEY=sk-old\n")
env_path = Path(f.name)
try:
with patch("hermes_cli.config.get_env_path", return_value=env_path):
first = load_env()
assert first.get("OPENAI_API_KEY") == "sk-old"
# Rewrite file with new contents and bump mtime to make sure
# the FS records the change even on coarse-mtime filesystems.
_write_env(env_path, "OPENAI_API_KEY=sk-new\n")
future = env_path.stat().st_mtime + 5.0
os.utime(env_path, (future, future))
second = load_env()
assert second.get("OPENAI_API_KEY") == "sk-new", (
"load_env() returned stale value after file change"
)
finally:
env_path.unlink(missing_ok=True)
invalidate_env_cache()
def test_invalidate_env_cache_forces_reread():
"""invalidate_env_cache() forces the next load_env() to hit the disk.
This is the belt-and-braces knob for writers (save_env_value, etc.)
on filesystems where mtime resolution might miss a same-second write.
"""
from hermes_cli.config import invalidate_env_cache, load_env
invalidate_env_cache()
with tempfile.NamedTemporaryFile(
mode="w", suffix=".env", delete=False, encoding="utf-8"
) as f:
f.write("OPENAI_API_KEY=sk-old\n")
env_path = Path(f.name)
try:
with patch("hermes_cli.config.get_env_path", return_value=env_path):
assert load_env().get("OPENAI_API_KEY") == "sk-old"
# Rewrite WITHOUT bumping mtime — simulates same-second write.
mtime_before = env_path.stat().st_mtime
_write_env(env_path, "OPENAI_API_KEY=sk-new\n")
os.utime(env_path, (mtime_before, mtime_before))
# Without invalidation, cache hit might return stale.
invalidate_env_cache()
assert load_env().get("OPENAI_API_KEY") == "sk-new"
finally:
env_path.unlink(missing_ok=True)
invalidate_env_cache()
def test_save_env_value_invalidates_cache(tmp_path, monkeypatch):
"""save_env_value() invalidates the cache so subsequent reads see the update."""
from hermes_cli import config as config_mod
from hermes_cli.config import invalidate_env_cache, load_env, save_env_value
invalidate_env_cache()
env_path = tmp_path / ".env"
env_path.write_text("EXISTING_KEY=old\n", encoding="utf-8")
monkeypatch.setattr(config_mod, "get_env_path", lambda: env_path)
monkeypatch.setattr(config_mod, "ensure_hermes_home", lambda: None)
monkeypatch.setattr(config_mod, "_secure_file", lambda _p: None)
monkeypatch.setattr(config_mod, "is_managed", lambda: False)
try:
# Prime the cache.
first = load_env()
assert first.get("EXISTING_KEY") == "old"
save_env_value("NEW_KEY", "shiny")
# Same-second writes on coarse-mtime filesystems would normally
# let stale cache survive; invalidate_env_cache() inside the
# writer makes the next read see the new key.
result = load_env()
assert result.get("NEW_KEY") == "shiny"
assert result.get("EXISTING_KEY") == "old"
finally:
monkeypatch.delenv("NEW_KEY", raising=False)
invalidate_env_cache()
def test_remove_env_value_invalidates_cache(tmp_path, monkeypatch):
"""remove_env_value() invalidates the cache so the removed key disappears."""
from hermes_cli import config as config_mod
from hermes_cli.config import (
invalidate_env_cache,
load_env,
remove_env_value,
save_env_value,
)
invalidate_env_cache()
env_path = tmp_path / ".env"
monkeypatch.setattr(config_mod, "get_env_path", lambda: env_path)
monkeypatch.setattr(config_mod, "ensure_hermes_home", lambda: None)
monkeypatch.setattr(config_mod, "_secure_file", lambda _p: None)
monkeypatch.setattr(config_mod, "is_managed", lambda: False)
save_env_value("DOOMED_KEY", "value")
assert load_env().get("DOOMED_KEY") == "value"
try:
removed = remove_env_value("DOOMED_KEY")
assert removed is True
assert "DOOMED_KEY" not in load_env()
finally:
monkeypatch.delenv("DOOMED_KEY", raising=False)
invalidate_env_cache()
def test_load_env_handles_missing_file():
"""A nonexistent .env returns {} and caches the empty result."""
from hermes_cli.config import invalidate_env_cache, load_env
invalidate_env_cache()
nonexistent = Path(tempfile.gettempdir()) / "hermes-test-no-such-env-xyz123.env"
nonexistent.unlink(missing_ok=True)
try:
with patch("hermes_cli.config.get_env_path", return_value=nonexistent):
assert load_env() == {}
assert load_env() == {} # cached
finally:
invalidate_env_cache()
+224
View File
@@ -514,3 +514,227 @@ class TestJudgeParseFailureAutoPause:
reloaded = load_goal("parse-fail-sid-4")
assert reloaded is not None
assert reloaded.consecutive_parse_failures == 2
# ──────────────────────────────────────────────────────────────────────
# /subgoal — user-added criteria
# ──────────────────────────────────────────────────────────────────────
class TestGoalStateSubgoalsBackcompat:
def test_old_state_meta_row_loads_without_subgoals(self):
"""A goal serialized BEFORE the subgoals field existed must
round-trip with an empty list, not crash."""
import json
from hermes_cli.goals import GoalState
legacy = json.dumps({
"goal": "do a thing",
"status": "active",
"turns_used": 2,
"max_turns": 20,
"created_at": 1.0,
"last_turn_at": 2.0,
"consecutive_parse_failures": 0,
})
state = GoalState.from_json(legacy)
assert state.goal == "do a thing"
assert state.subgoals == []
def test_subgoals_round_trip(self):
from hermes_cli.goals import GoalState
state = GoalState(goal="g", subgoals=["a", "b", "c"])
rt = GoalState.from_json(state.to_json())
assert rt.subgoals == ["a", "b", "c"]
class TestGoalManagerSubgoals:
def test_add_subgoal(self, hermes_home):
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sub-add")
mgr.set("main goal")
text = mgr.add_subgoal(" use bullet points ")
assert text == "use bullet points"
assert mgr.state.subgoals == ["use bullet points"]
def test_add_subgoal_requires_active_goal(self, hermes_home):
import pytest
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sub-noactive")
with pytest.raises(RuntimeError):
mgr.add_subgoal("oops")
def test_add_empty_subgoal_rejected(self, hermes_home):
import pytest
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sub-empty")
mgr.set("g")
with pytest.raises(ValueError):
mgr.add_subgoal(" ")
def test_remove_subgoal(self, hermes_home):
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sub-remove")
mgr.set("g")
mgr.add_subgoal("first")
mgr.add_subgoal("second")
mgr.add_subgoal("third")
removed = mgr.remove_subgoal(2)
assert removed == "second"
assert mgr.state.subgoals == ["first", "third"]
def test_remove_subgoal_out_of_range(self, hermes_home):
import pytest
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sub-oob")
mgr.set("g")
mgr.add_subgoal("only")
with pytest.raises(IndexError):
mgr.remove_subgoal(5)
with pytest.raises(IndexError):
mgr.remove_subgoal(0)
def test_clear_subgoals(self, hermes_home):
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sub-clear")
mgr.set("g")
mgr.add_subgoal("a")
mgr.add_subgoal("b")
prev = mgr.clear_subgoals()
assert prev == 2
assert mgr.state.subgoals == []
def test_subgoals_persist_across_reloads(self, hermes_home):
"""Subgoals stored in SessionDB survive a fresh GoalManager."""
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sub-persist")
mgr.set("g")
mgr.add_subgoal("first")
mgr.add_subgoal("second")
mgr2 = GoalManager(session_id="sub-persist")
assert mgr2.state.subgoals == ["first", "second"]
class TestContinuationPromptWithSubgoals:
def test_empty_subgoals_uses_original_template(self, hermes_home):
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="cp-empty")
mgr.set("ship the feature")
prompt = mgr.next_continuation_prompt()
assert prompt is not None
assert "ship the feature" in prompt
assert "Additional criteria" not in prompt
def test_with_subgoals_includes_them(self, hermes_home):
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="cp-with")
mgr.set("ship the feature")
mgr.add_subgoal("write tests")
mgr.add_subgoal("update docs")
prompt = mgr.next_continuation_prompt()
assert prompt is not None
assert "ship the feature" in prompt
assert "Additional criteria" in prompt
assert "1. write tests" in prompt
assert "2. update docs" in prompt
class TestJudgeGoalWithSubgoals:
def test_judge_uses_subgoals_template_when_provided(self, hermes_home):
"""judge_goal switches templates when subgoals is non-empty.
We don't actually call the model — we patch the aux client to
capture the prompt that would be sent.
"""
from unittest.mock import patch, MagicMock
from hermes_cli import goals
captured = {}
class _FakeMsg:
content = '{"done": true, "reason": "all done"}'
class _FakeChoice:
message = _FakeMsg()
class _FakeResp:
choices = [_FakeChoice()]
class _FakeClient:
class chat:
class completions:
@staticmethod
def create(**kwargs):
captured.update(kwargs)
return _FakeResp()
with patch.object(goals, "get_text_auxiliary_client",
return_value=(_FakeClient, "fake-model"), create=True), \
patch.object(goals, "get_auxiliary_extra_body",
return_value=None, create=True), \
patch("agent.auxiliary_client.get_text_auxiliary_client",
return_value=(_FakeClient, "fake-model")), \
patch("agent.auxiliary_client.get_auxiliary_extra_body",
return_value=None):
verdict, reason, parse_failed = goals.judge_goal(
"ship the feature",
"ok shipped",
subgoals=["write tests", "update docs"],
)
# The aux client was called with a prompt that includes the subgoals.
sent_messages = captured.get("messages") or []
user_msg = next((m["content"] for m in sent_messages if m["role"] == "user"), "")
assert "Additional criteria" in user_msg
assert "1. write tests" in user_msg
assert "2. update docs" in user_msg
assert "every additional criterion" in user_msg
assert verdict == "done"
def test_judge_uses_original_template_when_no_subgoals(self, hermes_home):
from unittest.mock import patch
from hermes_cli import goals
captured = {}
class _FakeMsg:
content = '{"done": true, "reason": "ok"}'
class _FakeChoice:
message = _FakeMsg()
class _FakeResp:
choices = [_FakeChoice()]
class _FakeClient:
class chat:
class completions:
@staticmethod
def create(**kwargs):
captured.update(kwargs)
return _FakeResp()
with patch("agent.auxiliary_client.get_text_auxiliary_client",
return_value=(_FakeClient, "fake-model")), \
patch("agent.auxiliary_client.get_auxiliary_extra_body",
return_value=None):
goals.judge_goal("ship it", "done", subgoals=None)
sent_messages = captured.get("messages") or []
user_msg = next((m["content"] for m in sent_messages if m["role"] == "user"), "")
assert "Additional criteria" not in user_msg
assert "ship it" in user_msg
class TestStatusLineSubgoalCount:
def test_status_line_no_subgoals(self, hermes_home):
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sl-empty")
mgr.set("ship it")
line = mgr.status_line()
assert "ship it" in line
assert "subgoal" not in line.lower()
def test_status_line_with_subgoals(self, hermes_home):
from hermes_cli.goals import GoalManager
mgr = GoalManager(session_id="sl-with")
mgr.set("ship it")
mgr.add_subgoal("a")
mgr.add_subgoal("b")
line = mgr.status_line()
assert "2 subgoals" in line
+27
View File
@@ -103,6 +103,33 @@ class TestPluginPickerInjection:
visible = tools_config._visible_providers(browser, {})
assert all(p.get("image_gen_plugin_name") is None for p in visible)
def test_post_setup_propagated_when_declared(self, monkeypatch):
from hermes_cli import tools_config
image_gen_registry.register_provider(_FakeProvider(
"xai_img",
schema={
"name": "xAI Grok Imagine",
"badge": "paid",
"tag": "grok image",
"env_vars": [],
"post_setup": "xai_grok",
},
))
rows = tools_config._plugin_image_gen_providers()
match = next(r for r in rows if r.get("image_gen_plugin_name") == "xai_img")
assert match["post_setup"] == "xai_grok"
def test_post_setup_omitted_when_not_declared(self, monkeypatch):
from hermes_cli import tools_config
image_gen_registry.register_provider(_FakeProvider("plain_img"))
rows = tools_config._plugin_image_gen_providers()
match = next(r for r in rows if r.get("image_gen_plugin_name") == "plain_img")
assert "post_setup" not in match
class TestPluginCatalog:
def test_plugin_catalog_returns_models(self):
+378
View File
@@ -0,0 +1,378 @@
"""Behavior tests for hermes_cli.inventory.
Locks the invariants the three migrated consumers (web_server.py
/api/model/options, tui_gateway model.options, tui_gateway model.save_key)
depend on:
- load_picker_context() reproduces the inline 17-LOC config-slice exactly.
- with_overrides() is truthy-only (empty agent attrs must not clobber).
- build_models_payload() returns a stable {providers, model, provider}
shape and delegates curation to list_authenticated_providers (does not
call provider_model_ids per row).
- canonical_order keys on slug membership, not is_user_defined section
3 of list_authenticated_providers sets is_user_defined=True for
canonical slugs in the providers: dict, and that flag must NOT demote
them to the tail.
- picker_hints adds authenticated/auth_type/key_env/warning per row,
matching the TUI ModelPickerDialog shape.
"""
from __future__ import annotations
from unittest.mock import patch
import pytest
from hermes_cli.inventory import (
ConfigContext,
build_models_payload,
load_picker_context,
)
# ─── load_picker_context ───────────────────────────────────────────────
def _cfg(model=None, providers=None, custom_providers=None) -> dict:
return {
"model": model if model is not None else {},
"providers": providers if providers is not None else {},
"custom_providers": custom_providers if custom_providers is not None else [],
}
def test_load_picker_context_full_dict():
cfg = _cfg(
model={
"default": "anthropic/claude-sonnet-4.6",
"provider": "openrouter",
"base_url": "https://openrouter.ai/api/v1",
},
providers={"openrouter": {}},
custom_providers=[{"name": "Ollama", "base_url": "http://localhost:11434/v1"}],
)
with patch("hermes_cli.config.load_config", return_value=cfg):
ctx = load_picker_context()
assert ctx.current_model == "anthropic/claude-sonnet-4.6"
assert ctx.current_provider == "openrouter"
assert ctx.current_base_url == "https://openrouter.ai/api/v1"
assert "openrouter" in ctx.user_providers
# custom_providers comes from get_compatible_custom_providers, which
# merges legacy list + v12+ keyed providers — both present here means
# at least one row.
assert isinstance(ctx.custom_providers, list)
def test_load_picker_context_falls_back_to_name_when_default_missing():
cfg = _cfg(model={"name": "gpt-5.4", "provider": "openai"})
with patch("hermes_cli.config.load_config", return_value=cfg):
ctx = load_picker_context()
assert ctx.current_model == "gpt-5.4"
assert ctx.current_provider == "openai"
def test_load_picker_context_string_model_legacy_shape():
"""config.model can be a bare string in older configs."""
cfg = {"model": "some-model", "providers": {}, "custom_providers": []}
with patch("hermes_cli.config.load_config", return_value=cfg):
ctx = load_picker_context()
assert ctx.current_model == "some-model"
assert ctx.current_provider == ""
assert ctx.current_base_url == ""
def test_load_picker_context_empty_config():
cfg = _cfg()
with patch("hermes_cli.config.load_config", return_value=cfg):
ctx = load_picker_context()
assert ctx.current_provider == ""
assert ctx.current_model == ""
assert ctx.current_base_url == ""
assert ctx.user_providers == {}
assert ctx.custom_providers == []
# ─── with_overrides ────────────────────────────────────────────────────
def _empty_ctx(provider="orig", model="orig-model", base_url="orig-url"):
return ConfigContext(
current_provider=provider,
current_model=model,
current_base_url=base_url,
user_providers={},
custom_providers=[],
)
def test_with_overrides_truthy_only_strings():
"""Empty strings must NOT clobber disk config — TUI calls this with
empty getattr(agent, 'provider', '') when no agent is spawned yet."""
ctx = _empty_ctx()
overlaid = ctx.with_overrides(
current_provider="",
current_model="",
current_base_url="",
)
assert overlaid.current_provider == "orig"
assert overlaid.current_model == "orig-model"
assert overlaid.current_base_url == "orig-url"
def test_with_overrides_truthy_value_replaces():
ctx = _empty_ctx()
overlaid = ctx.with_overrides(current_provider="anthropic")
assert overlaid.current_provider == "anthropic"
assert overlaid.current_model == "orig-model" # untouched
def test_with_overrides_no_args_returns_self_or_equivalent():
ctx = _empty_ctx()
assert ctx.with_overrides() == ctx
# ─── build_models_payload ──────────────────────────────────────────────
def _list_auth_returning(rows: list[dict]):
"""Patch list_authenticated_providers to return a fixed row list."""
return patch(
"hermes_cli.model_switch.list_authenticated_providers",
return_value=rows,
)
def test_build_models_payload_returns_expected_shape():
rows = [
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
"total_models": 1, "is_current": True, "is_user_defined": False,
"source": "built-in"},
]
ctx = _empty_ctx(provider="openrouter", model="m1", base_url="")
with _list_auth_returning(rows):
payload = build_models_payload(ctx)
assert set(payload.keys()) == {"providers", "model", "provider"}
assert payload["model"] == "m1"
assert payload["provider"] == "openrouter"
assert payload["providers"] == rows
def test_build_models_payload_does_not_call_provider_model_ids():
"""Curated lists must come from list_authenticated_providers, not
provider_model_ids that would pull TTS/embeddings/etc.
"""
rows = [{"slug": "nous", "name": "Nous", "models": ["hermes-4-405b"],
"total_models": 1, "is_current": False, "is_user_defined": False,
"source": "built-in"}]
ctx = _empty_ctx()
with _list_auth_returning(rows), \
patch("hermes_cli.models.provider_model_ids") as mock_pm:
build_models_payload(ctx)
mock_pm.assert_not_called()
def test_include_unconfigured_appends_canonical_skeletons():
"""include_unconfigured=True adds CANONICAL_PROVIDERS rows that
list_authenticated_providers didn't emit. Skeleton rows have empty
models and source='canonical'."""
rows = [
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
"total_models": 1, "is_current": True, "is_user_defined": False,
"source": "built-in"},
]
ctx = _empty_ctx(provider="openrouter")
with _list_auth_returning(rows):
payload = build_models_payload(ctx, include_unconfigured=True)
# All canonical providers other than openrouter should appear as
# skeleton rows.
from hermes_cli.models import CANONICAL_PROVIDERS
seen_slugs = {r["slug"] for r in payload["providers"]}
for entry in CANONICAL_PROVIDERS:
assert entry.slug in seen_slugs, f"missing {entry.slug}"
# Skeletons have empty models and source='canonical'.
skeletons = [r for r in payload["providers"]
if r.get("source") == "canonical"]
assert all(r["models"] == [] for r in skeletons)
assert all(r["total_models"] == 0 for r in skeletons)
def test_include_unconfigured_skips_already_present_slugs():
"""If list_authenticated_providers already returned a row for a
canonical slug, include_unconfigured must NOT duplicate it."""
rows = [
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
"total_models": 1, "is_current": True, "is_user_defined": False,
"source": "built-in"},
]
ctx = _empty_ctx()
with _list_auth_returning(rows):
payload = build_models_payload(ctx, include_unconfigured=True)
or_rows = [r for r in payload["providers"] if r["slug"] == "openrouter"]
assert len(or_rows) == 1
assert or_rows[0]["models"] == ["m1"] # the authenticated row, not skeleton
# ─── picker_hints ──────────────────────────────────────────────────────
def test_picker_hints_marks_authed_rows_authenticated():
rows = [
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
"total_models": 1, "is_current": True, "is_user_defined": False,
"source": "built-in"},
]
ctx = _empty_ctx()
with _list_auth_returning(rows):
payload = build_models_payload(ctx, picker_hints=True)
assert payload["providers"][0]["authenticated"] is True
def test_picker_hints_adds_warning_to_skeleton_rows():
"""Skeleton rows (unconfigured canonical providers) must carry the
setup hint the picker UI displays."""
rows = []
ctx = _empty_ctx()
with _list_auth_returning(rows):
payload = build_models_payload(
ctx, include_unconfigured=True, picker_hints=True,
)
skeleton_rows = [r for r in payload["providers"]
if r.get("source") == "canonical"]
assert skeleton_rows, "test setup: expected at least one skeleton row"
for row in skeleton_rows:
assert row["authenticated"] is False
assert "auth_type" in row
assert "warning" in row
# api_key providers get "paste X to activate" / others get the
# hermes model fallback.
assert (
row["warning"].startswith("paste ")
or row["warning"].startswith("run `hermes model`")
)
def test_picker_hints_api_key_warning_format():
"""For api_key providers with a defined env var, the warning must
point to that env var."""
rows = []
ctx = _empty_ctx()
with _list_auth_returning(rows):
payload = build_models_payload(
ctx, include_unconfigured=True, picker_hints=True,
)
# anthropic uses api_key + ANTHROPIC_API_KEY.
anthropic = next(
r for r in payload["providers"] if r["slug"] == "anthropic"
)
assert "ANTHROPIC_API_KEY" in anthropic["warning"]
assert anthropic["warning"].startswith("paste ")
# ─── canonical_order ───────────────────────────────────────────────────
def test_canonical_order_uses_slug_not_is_user_defined_flag():
"""Section 3 of list_authenticated_providers sets is_user_defined=True
for canonical slugs that appear in the providers: config dict.
canonical_order MUST key on slug membership, not the flag otherwise
canonical providers configured via the keyed schema get demoted to
the tail.
"""
from hermes_cli.models import CANONICAL_PROVIDERS
canonical_slug = CANONICAL_PROVIDERS[2].slug # any canonical
rows = [
# A truly-custom row (correct: is_user_defined=True)
{"slug": "custom:Ollama", "name": "Ollama", "models": [],
"total_models": 0, "is_current": False, "is_user_defined": True,
"source": "user-config"},
# A canonical row that the substrate flagged as user-defined
# because the user configured it via providers: dict.
{"slug": canonical_slug, "name": "x", "models": ["m1"],
"total_models": 1, "is_current": False, "is_user_defined": True,
"source": "built-in"},
]
ctx = _empty_ctx()
with _list_auth_returning(rows):
payload = build_models_payload(ctx, canonical_order=True)
slugs = [r["slug"] for r in payload["providers"]]
# Canonical-slug row must come BEFORE truly-custom rows, regardless
# of is_user_defined.
canonical_idx = slugs.index(canonical_slug)
custom_idx = slugs.index("custom:Ollama")
assert canonical_idx < custom_idx, (
f"canonical {canonical_slug} demoted to tail "
f"(canonical_idx={canonical_idx} > custom_idx={custom_idx})"
)
def test_canonical_order_with_unconfigured_preserves_full_universe():
"""Combined picker call: include_unconfigured + picker_hints +
canonical_order is the production TUI shape. Verify the result
has CANONICAL_PROVIDERS in declaration order, hints applied,
custom rows trailing.
"""
from hermes_cli.models import CANONICAL_PROVIDERS
rows = [
{"slug": "custom:Ollama", "name": "Ollama", "models": [],
"total_models": 0, "is_current": False, "is_user_defined": True,
"source": "user-config"},
]
ctx = _empty_ctx()
with _list_auth_returning(rows):
payload = build_models_payload(
ctx,
include_unconfigured=True,
picker_hints=True,
canonical_order=True,
)
slugs = [r["slug"] for r in payload["providers"]]
# First row: first canonical provider in declaration order.
assert slugs[0] == CANONICAL_PROVIDERS[0].slug
# Custom row trails canonical universe.
assert slugs.index("custom:Ollama") >= len(CANONICAL_PROVIDERS)
# ─── Integration: end-to-end through real load_picker_context ──────────
def test_end_to_end_with_real_context_no_credentials_leak(monkeypatch):
"""Full pipeline: real load_picker_context + real
list_authenticated_providers. Verify no credential string ever
appears in the returned payload, even with picker_hints=True."""
canary = "sk-canary-XYZ-must-not-appear"
monkeypatch.setenv("OPENROUTER_API_KEY", canary)
monkeypatch.setenv("ANTHROPIC_API_KEY", canary)
cfg = _cfg(model={"provider": "openrouter"})
with patch("hermes_cli.config.load_config", return_value=cfg):
ctx = load_picker_context()
payload = build_models_payload(
ctx, include_unconfigured=True, picker_hints=True,
)
import json as _json
assert canary not in _json.dumps(payload)
def test_payload_shape_compatible_with_modelpickerdialog_frontend():
"""Frontend (web/src/components/ModelPickerDialog.tsx) reads:
name, slug, models, total_models, is_current, warning, authenticated.
Verify every authenticated/skeleton row exposes those keys.
"""
rows = [
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
"total_models": 1, "is_current": True, "is_user_defined": False,
"source": "built-in"},
]
ctx = _empty_ctx()
with _list_auth_returning(rows):
payload = build_models_payload(
ctx, include_unconfigured=True, picker_hints=True,
)
required_keys = {"name", "slug", "models", "total_models", "is_current",
"authenticated"}
for row in payload["providers"]:
missing = required_keys - row.keys()
assert not missing, f"row {row['slug']} missing keys: {missing}"
@@ -177,6 +177,40 @@ class TestProviderPersistsAfterModelSave:
assert model.get("api_mode") == "codex_responses"
assert config["agent"]["reasoning_effort"] == "high"
def test_named_custom_provider_preserves_explicit_api_mode(self, config_home):
"""Named custom providers should re-activate with their saved api_mode."""
import yaml
from hermes_cli.main import _model_flow_named_custom
provider_info = {
"name": "Packy",
"base_url": "https://packy.example.com/v1",
"api_key": "sk-test",
"model": "gpt-5.4",
"api_mode": "codex_responses",
}
# Patch fetch_api_models so the named custom flow returns one model;
# patch simple_term_menu to force the input() fallback; patch input to
# auto-select the first model from the fallback prompt.
from unittest.mock import MagicMock
fake_menu_module = MagicMock()
fake_menu_module.TerminalMenu.side_effect = OSError("no tty in test")
with patch("hermes_cli.auth._save_model_choice"), \
patch("hermes_cli.auth.deactivate_provider"), \
patch("hermes_cli.models.fetch_api_models", return_value=["gpt-5.4"]), \
patch.dict("sys.modules", {"simple_term_menu": fake_menu_module}), \
patch("builtins.input", return_value="1"):
_model_flow_named_custom({}, provider_info)
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
model = config.get("model")
assert isinstance(model, dict)
assert model.get("provider") == "custom"
assert model.get("base_url") == "https://packy.example.com/v1"
assert model.get("api_mode") == "codex_responses"
def test_copilot_acp_provider_saved_when_selected(self, config_home):
"""_model_flow_copilot_acp should persist provider/base_url/model together."""
from hermes_cli.main import _model_flow_copilot_acp
@@ -0,0 +1,144 @@
"""Tests for the get_nous_auth_status() process-level cache.
The cache avoids re-validating Nous credentials on every menu paint
`hermes tools` "All Platforms" used to fire ~31 OAuth refresh POSTs
against portal.nousresearch.com during one render. The cache is keyed
on auth.json mtime so login/logout flows invalidate naturally; tests
and other writers can also call invalidate_nous_auth_status_cache().
"""
from __future__ import annotations
import json
import os
from unittest.mock import patch
def _seed_auth_file(tmp_path):
"""Drop a placeholder auth.json into the test HERMES_HOME.
The exact content doesn't matter for cache-key purposes — only that
the file exists and we can mutate it to bump mtime.
"""
auth = tmp_path / "auth.json"
auth.write_text(json.dumps({"providers": {}}), encoding="utf-8")
return auth
def test_get_nous_auth_status_caches_consecutive_calls(tmp_path, monkeypatch):
"""A second call within the TTL skips re-computing the snapshot."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_seed_auth_file(tmp_path)
from hermes_cli import auth as auth_mod
auth_mod.invalidate_nous_auth_status_cache()
call_count = {"n": 0}
def fake_compute():
call_count["n"] += 1
return {"logged_in": False, "source": "auth_store", "call": call_count["n"]}
with patch.object(auth_mod, "_compute_nous_auth_status", side_effect=fake_compute):
first = auth_mod.get_nous_auth_status()
second = auth_mod.get_nous_auth_status()
third = auth_mod.get_nous_auth_status()
assert call_count["n"] == 1, (
f"_compute_nous_auth_status was called {call_count['n']}×"
"cache is not deduplicating within TTL."
)
# Each call returns a copy so callers can't mutate the cached dict.
assert first == second == third
first["mutated"] = True
assert "mutated" not in auth_mod.get_nous_auth_status()
auth_mod.invalidate_nous_auth_status_cache()
def test_get_nous_auth_status_invalidates_on_auth_file_mtime(tmp_path, monkeypatch):
"""Touching auth.json (login/logout) forces a re-compute."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
auth_path = _seed_auth_file(tmp_path)
from hermes_cli import auth as auth_mod
auth_mod.invalidate_nous_auth_status_cache()
call_count = {"n": 0}
def fake_compute():
call_count["n"] += 1
return {"logged_in": False, "source": "auth_store", "call": call_count["n"]}
with patch.object(auth_mod, "_compute_nous_auth_status", side_effect=fake_compute):
auth_mod.get_nous_auth_status()
# Bump mtime forward so coarse-resolution filesystems still record
# a change.
future = auth_path.stat().st_mtime + 5.0
os.utime(auth_path, (future, future))
auth_mod.get_nous_auth_status()
assert call_count["n"] == 2, (
"auth.json mtime change should invalidate the cache, but only "
f"{call_count['n']} compute call(s) happened."
)
auth_mod.invalidate_nous_auth_status_cache()
def test_invalidate_nous_auth_status_cache_forces_recompute(tmp_path, monkeypatch):
"""Explicit invalidate forces the next call to re-compute."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_seed_auth_file(tmp_path)
from hermes_cli import auth as auth_mod
auth_mod.invalidate_nous_auth_status_cache()
call_count = {"n": 0}
def fake_compute():
call_count["n"] += 1
return {"logged_in": False, "source": "auth_store"}
with patch.object(auth_mod, "_compute_nous_auth_status", side_effect=fake_compute):
auth_mod.get_nous_auth_status()
auth_mod.invalidate_nous_auth_status_cache()
auth_mod.get_nous_auth_status()
assert call_count["n"] == 2
auth_mod.invalidate_nous_auth_status_cache()
def test_get_nous_auth_status_caches_failure_path(tmp_path, monkeypatch):
"""Logged-out snapshots are cached too — that's where the cost was.
Teknium's case: ~31 cache misses per `hermes tools` "All Platforms"
menu paint, all returning logged_in=False after a failed refresh POST.
The whole point of the cache is to memoise that failure path too.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_seed_auth_file(tmp_path)
from hermes_cli import auth as auth_mod
auth_mod.invalidate_nous_auth_status_cache()
call_count = {"n": 0}
def fake_compute():
call_count["n"] += 1
return {"logged_in": False, "source": "auth_store", "error": "refresh failed"}
with patch.object(auth_mod, "_compute_nous_auth_status", side_effect=fake_compute):
for _ in range(10):
auth_mod.get_nous_auth_status()
assert call_count["n"] == 1, (
f"Logged-out snapshots must cache; got {call_count['n']} computes for 10 calls."
)
auth_mod.invalidate_nous_auth_status_cache()
+89
View File
@@ -538,6 +538,95 @@ class TestPreToolCallBlocking:
assert get_pre_tool_call_block_message("terminal", {}) == "first blocker"
class TestThreadToolWhitelist:
"""Tests for the thread-local tool whitelist used by background review forks."""
def test_allowed_tool_passes_through_to_hooks(self, monkeypatch):
from hermes_cli.plugins import (
set_thread_tool_whitelist,
clear_thread_tool_whitelist,
)
monkeypatch.setattr(
"hermes_cli.plugins.invoke_hook",
lambda hook_name, **kwargs: [],
)
set_thread_tool_whitelist({"memory", "skill_manage"})
try:
assert get_pre_tool_call_block_message("memory", {}) is None
finally:
clear_thread_tool_whitelist()
def test_disallowed_tool_blocked_with_message(self, monkeypatch):
from hermes_cli.plugins import (
set_thread_tool_whitelist,
clear_thread_tool_whitelist,
)
monkeypatch.setattr(
"hermes_cli.plugins.invoke_hook",
lambda hook_name, **kwargs: [],
)
set_thread_tool_whitelist(
{"memory"}, deny_msg_fmt="denied: {tool_name}"
)
try:
msg = get_pre_tool_call_block_message("terminal", {})
assert msg == "denied: terminal"
finally:
clear_thread_tool_whitelist()
def test_clear_restores_unrestricted_behavior(self, monkeypatch):
from hermes_cli.plugins import (
set_thread_tool_whitelist,
clear_thread_tool_whitelist,
)
monkeypatch.setattr(
"hermes_cli.plugins.invoke_hook",
lambda hook_name, **kwargs: [],
)
set_thread_tool_whitelist({"memory"})
clear_thread_tool_whitelist()
# After clearing, any tool should pass through to plugin hooks (which
# return [] here, so result is None).
assert get_pre_tool_call_block_message("terminal", {}) is None
def test_whitelist_is_thread_local(self, monkeypatch):
"""Setting a whitelist in one thread must NOT leak into another."""
import threading
from hermes_cli.plugins import (
set_thread_tool_whitelist,
clear_thread_tool_whitelist,
)
monkeypatch.setattr(
"hermes_cli.plugins.invoke_hook",
lambda hook_name, **kwargs: [],
)
# Main thread: install a restrictive whitelist.
set_thread_tool_whitelist({"memory"})
try:
assert get_pre_tool_call_block_message("terminal", {}) is not None
# Worker thread: should NOT inherit main thread's whitelist.
result = {}
def worker():
result["msg"] = get_pre_tool_call_block_message("terminal", {})
t = threading.Thread(target=worker)
t.start()
t.join()
assert result["msg"] is None, (
"thread-local whitelist leaked across threads"
)
finally:
clear_thread_tool_whitelist()
# ── TestPluginContext ──────────────────────────────────────────────────────
-28
View File
@@ -29,8 +29,6 @@ from hermes_cli.profiles import (
rename_profile,
export_profile,
import_profile,
generate_bash_completion,
generate_zsh_completion,
_get_profiles_root,
_get_default_hermes_home,
seed_profile_skills,
@@ -1013,32 +1011,6 @@ class TestProfileIsolation:
assert (beta_dir / "skills").is_dir()
# ===================================================================
# TestCompletion
# ===================================================================
class TestCompletion:
"""Tests for bash/zsh completion generators."""
def test_bash_completion_contains_complete(self):
script = generate_bash_completion()
assert len(script) > 0
assert "complete" in script
def test_zsh_completion_contains_compdef(self):
script = generate_zsh_completion()
assert len(script) > 0
assert "compdef" in script
def test_bash_completion_has_hermes_profiles_function(self):
script = generate_bash_completion()
assert "_hermes_profiles" in script
def test_zsh_completion_has_hermes_function(self):
script = generate_zsh_completion()
assert "_hermes" in script
# ===================================================================
# TestGetProfilesRoot / TestGetDefaultHermesHome (internal helpers)
# ===================================================================
+512
View File
@@ -0,0 +1,512 @@
"""Tests for the `hermes proxy` subcommand and its upstream adapters."""
from __future__ import annotations
import asyncio
import json
import os
import threading
from pathlib import Path
from typing import Any, Dict
from unittest.mock import MagicMock, patch
import pytest
from hermes_cli.proxy.adapters import ADAPTERS, get_adapter
from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential
from hermes_cli.proxy.adapters.nous_portal import NousPortalAdapter
# ---------------------------------------------------------------------------
# Adapter registry
# ---------------------------------------------------------------------------
def test_registry_lists_nous():
assert "nous" in ADAPTERS
def test_get_adapter_returns_instance():
adapter = get_adapter("nous")
assert isinstance(adapter, NousPortalAdapter)
assert isinstance(adapter, UpstreamAdapter)
def test_get_adapter_case_insensitive():
assert isinstance(get_adapter("NOUS"), NousPortalAdapter)
assert isinstance(get_adapter(" Nous "), NousPortalAdapter)
def test_get_adapter_unknown_provider_raises():
with pytest.raises(ValueError, match="anthropic"):
get_adapter("anthropic") # not yet implemented
# ---------------------------------------------------------------------------
# NousPortalAdapter
# ---------------------------------------------------------------------------
def _write_auth_store(hermes_home: Path, nous_state: Dict[str, Any]) -> Path:
"""Write an auth.json with the given nous state into a hermetic HERMES_HOME."""
auth_path = hermes_home / "auth.json"
auth_path.write_text(json.dumps({
"version": 1,
"providers": {"nous": nous_state},
}))
return auth_path
def test_nous_adapter_metadata():
adapter = NousPortalAdapter()
assert adapter.name == "nous"
assert adapter.display_name == "Nous Portal"
assert "/chat/completions" in adapter.allowed_paths
assert "/embeddings" in adapter.allowed_paths
assert "/completions" in adapter.allowed_paths
assert "/models" in adapter.allowed_paths
def test_nous_adapter_not_authenticated_when_no_auth_file(tmp_path, monkeypatch):
# HERMES_HOME is already set by conftest, but make doubly sure
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
adapter = NousPortalAdapter()
assert not adapter.is_authenticated()
def test_nous_adapter_not_authenticated_when_provider_missing(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / "auth.json").write_text(json.dumps({
"version": 1,
"providers": {},
}))
assert not NousPortalAdapter().is_authenticated()
def test_nous_adapter_authenticated_with_agent_key(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_write_auth_store(tmp_path, {
"agent_key": "ov-test-key",
"agent_key_expires_at": "2099-01-01T00:00:00Z",
"inference_base_url": "https://inference-api.nousresearch.com/v1",
})
assert NousPortalAdapter().is_authenticated()
def test_nous_adapter_authenticated_with_refresh_token_only(tmp_path, monkeypatch):
"""If access_token+refresh_token exist but no agent_key yet, we can still mint."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_write_auth_store(tmp_path, {
"access_token": "access-tok",
"refresh_token": "refresh-tok",
})
assert NousPortalAdapter().is_authenticated()
def test_nous_adapter_get_credential_refreshes_and_persists(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_write_auth_store(tmp_path, {
"access_token": "access-tok",
"refresh_token": "refresh-tok",
"client_id": "hermes-cli",
"portal_base_url": "https://portal.nousresearch.com",
"inference_base_url": "https://inference-api.nousresearch.com/v1",
})
refreshed_state = {
"access_token": "access-tok",
"refresh_token": "refresh-tok",
"client_id": "hermes-cli",
"portal_base_url": "https://portal.nousresearch.com",
"inference_base_url": "https://inference-api.nousresearch.com/v1",
"agent_key": "minted-bearer",
"agent_key_expires_at": "2099-01-01T00:00:00Z",
}
with patch(
"hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state",
return_value=refreshed_state,
) as mock_refresh:
adapter = NousPortalAdapter()
cred = adapter.get_credential()
mock_refresh.assert_called_once()
assert cred.bearer == "minted-bearer"
assert cred.base_url == "https://inference-api.nousresearch.com/v1"
assert cred.expires_at == "2099-01-01T00:00:00Z"
assert cred.token_type == "Bearer"
# Verify state was persisted back
stored = json.loads((tmp_path / "auth.json").read_text())
assert stored["providers"]["nous"]["agent_key"] == "minted-bearer"
def test_nous_adapter_get_credential_raises_when_not_logged_in(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
adapter = NousPortalAdapter()
with pytest.raises(RuntimeError, match="hermes login nous"):
adapter.get_credential()
def test_nous_adapter_get_credential_raises_on_refresh_failure(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_write_auth_store(tmp_path, {
"access_token": "access-tok",
"refresh_token": "refresh-tok",
})
with patch(
"hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state",
side_effect=RuntimeError("Refresh session has been revoked"),
):
adapter = NousPortalAdapter()
with pytest.raises(RuntimeError, match="Refresh session has been revoked"):
adapter.get_credential()
def test_nous_adapter_get_credential_raises_when_no_agent_key_returned(tmp_path, monkeypatch):
"""If the refresh helper succeeds but produces no agent_key, we surface a clear error."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_write_auth_store(tmp_path, {
"access_token": "access-tok",
"refresh_token": "refresh-tok",
})
with patch(
"hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state",
return_value={"access_token": "a", "refresh_token": "r"},
):
adapter = NousPortalAdapter()
with pytest.raises(RuntimeError, match="did not return a usable agent_key"):
adapter.get_credential()
def test_nous_adapter_concurrent_refresh_serialized(tmp_path, monkeypatch):
"""Two parallel get_credential() calls must serialize through the lock."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_write_auth_store(tmp_path, {
"access_token": "a", "refresh_token": "r",
})
call_log: list = []
in_flight = threading.Event()
overlap_detected = threading.Event()
counter = [0]
counter_lock = threading.Lock()
def serializing_refresh(state, **kwargs):
# If another thread is already inside refresh, the lock is broken.
if in_flight.is_set():
overlap_detected.set()
in_flight.set()
try:
call_log.append(threading.current_thread().ident)
# Simulate refresh latency so any race window is exposed.
import time
time.sleep(0.05)
with counter_lock:
counter[0] += 1
idx = counter[0]
return {
**state,
"agent_key": f"key-{idx}",
"agent_key_expires_at": "2099-01-01T00:00:00Z",
"inference_base_url": "https://inference-api.nousresearch.com/v1",
}
finally:
in_flight.clear()
adapter = NousPortalAdapter()
results: list = []
errors: list = []
def worker():
try:
results.append(adapter.get_credential().bearer)
except Exception as exc: # pragma: no cover - shouldn't happen
errors.append(exc)
with patch(
"hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state",
side_effect=serializing_refresh,
):
threads = [threading.Thread(target=worker) for _ in range(3)]
for t in threads:
t.start()
for t in threads:
t.join()
assert not errors, f"workers errored: {errors}"
assert len(results) == 3
assert len(call_log) == 3
assert not overlap_detected.is_set(), "refresh calls overlapped — lock is broken"
assert all(r.startswith("key-") for r in results)
# ---------------------------------------------------------------------------
# Server: path filtering + forwarding
#
# We run the proxy AND a fake upstream as real aiohttp servers on ephemeral
# ports. Avoids pytest-aiohttp's fixtures (extra dependency for one test file).
# ---------------------------------------------------------------------------
aiohttp = pytest.importorskip("aiohttp")
from aiohttp import web # noqa: E402
from hermes_cli.proxy.server import create_app # noqa: E402
class FakeAdapter(UpstreamAdapter):
"""A test adapter that returns a fixed credential without touching disk."""
def __init__(self, base_url: str, bearer: str = "test-bearer",
allowed=None, raise_on_credential=False):
self._base_url = base_url
self._bearer = bearer
self._allowed = frozenset(allowed or ["/chat/completions"])
self._raise = raise_on_credential
self.calls = 0
@property
def name(self): return "fake"
@property
def display_name(self): return "Fake Provider"
@property
def allowed_paths(self): return self._allowed
def is_authenticated(self): return True
def get_credential(self):
self.calls += 1
if self._raise:
raise RuntimeError("simulated auth failure")
return UpstreamCredential(
bearer=self._bearer, base_url=self._base_url,
expires_at="2099-01-01T00:00:00Z",
)
async def _start_runner(app: "web.Application"):
"""Spin up an aiohttp app on an ephemeral localhost port. Returns (runner, base_url)."""
runner = web.AppRunner(app, access_log=None)
await runner.setup()
site = web.TCPSite(runner, host="127.0.0.1", port=0)
await site.start()
sockets = list(site._server.sockets) # type: ignore[union-attr]
port = sockets[0].getsockname()[1]
return runner, f"http://127.0.0.1:{port}"
def _build_fake_upstream(captured: Dict[str, Any]) -> "web.Application":
async def echo(request):
body = await request.read()
captured["requests"].append({
"method": request.method,
"path": request.path,
"auth": request.headers.get("Authorization"),
"body": body.decode("utf-8") if body else "",
})
return web.json_response({"echoed": True, "path": request.path})
async def sse(request):
resp = web.StreamResponse(
status=200, headers={"Content-Type": "text/event-stream"},
)
await resp.prepare(request)
for chunk in [b"data: hello\n\n", b"data: world\n\n", b"data: [DONE]\n\n"]:
await resp.write(chunk)
await resp.write_eof()
return resp
app = web.Application()
app.router.add_route("*", "/v1/chat/completions", echo)
app.router.add_route("*", "/v1/embeddings", echo)
app.router.add_route("*", "/v1/sse", sse)
return app
def test_server_forwards_chat_completions():
async def run():
captured: Dict[str, Any] = {"requests": []}
upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured))
adapter = FakeAdapter(f"{upstream_base}/v1", bearer="real-portal-key")
proxy_runner, proxy_base = await _start_runner(create_app(adapter))
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{proxy_base}/v1/chat/completions",
json={"model": "Hermes-4-70B",
"messages": [{"role": "user", "content": "hi"}]},
headers={"Authorization": "Bearer client-dummy-key"},
) as resp:
assert resp.status == 200
data = await resp.json()
assert data["echoed"] is True
assert len(captured["requests"]) == 1
req = captured["requests"][0]
assert req["auth"] == "Bearer real-portal-key"
assert "Hermes-4-70B" in req["body"]
finally:
await proxy_runner.cleanup()
await upstream_runner.cleanup()
asyncio.run(run())
def test_server_rejects_disallowed_path():
async def run():
adapter = FakeAdapter("http://unused.example/v1", allowed=["/chat/completions"])
runner, base = await _start_runner(create_app(adapter))
try:
async with aiohttp.ClientSession() as session:
async with session.get(f"{base}/v1/random/endpoint") as resp:
assert resp.status == 404
body = await resp.json()
assert body["error"]["type"] == "path_not_allowed"
assert "/chat/completions" in body["error"]["message"]
finally:
await runner.cleanup()
asyncio.run(run())
def test_server_returns_401_when_adapter_fails():
async def run():
adapter = FakeAdapter("http://unused.example/v1", raise_on_credential=True)
runner, base = await _start_runner(create_app(adapter))
try:
async with aiohttp.ClientSession() as session:
async with session.post(f"{base}/v1/chat/completions", json={}) as resp:
assert resp.status == 401
body = await resp.json()
assert body["error"]["type"] == "upstream_auth_failed"
assert "simulated auth failure" in body["error"]["message"]
finally:
await runner.cleanup()
asyncio.run(run())
def test_server_health_endpoint():
async def run():
adapter = FakeAdapter("http://unused.example/v1")
runner, base = await _start_runner(create_app(adapter))
try:
async with aiohttp.ClientSession() as session:
async with session.get(f"{base}/health") as resp:
assert resp.status == 200
body = await resp.json()
assert body["status"] == "ok"
assert body["upstream"] == "Fake Provider"
assert body["authenticated"] is True
finally:
await runner.cleanup()
asyncio.run(run())
def test_server_streams_sse():
async def run():
captured: Dict[str, Any] = {"requests": []}
upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured))
adapter = FakeAdapter(f"{upstream_base}/v1", allowed=["/sse"])
proxy_runner, proxy_base = await _start_runner(create_app(adapter))
try:
async with aiohttp.ClientSession() as session:
async with session.get(f"{proxy_base}/v1/sse") as resp:
assert resp.status == 200
chunks = []
async for chunk in resp.content.iter_any():
chunks.append(chunk)
full = b"".join(chunks)
assert b"data: hello" in full
assert b"data: [DONE]" in full
finally:
await proxy_runner.cleanup()
await upstream_runner.cleanup()
asyncio.run(run())
def test_server_strips_client_auth_header():
"""The client's Authorization header MUST NOT reach the upstream."""
async def run():
captured: Dict[str, Any] = {"requests": []}
upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured))
adapter = FakeAdapter(f"{upstream_base}/v1", bearer="ours")
proxy_runner, proxy_base = await _start_runner(create_app(adapter))
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{proxy_base}/v1/chat/completions",
json={},
headers={"Authorization": "Bearer SHOULD_NOT_LEAK"},
) as resp:
await resp.read()
assert captured["requests"][0]["auth"] == "Bearer ours"
assert "SHOULD_NOT_LEAK" not in captured["requests"][0]["auth"]
finally:
await proxy_runner.cleanup()
await upstream_runner.cleanup()
asyncio.run(run())
# ---------------------------------------------------------------------------
# CLI handlers
# ---------------------------------------------------------------------------
def test_cmd_proxy_status_runs(capsys, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from hermes_cli.proxy.cli import cmd_proxy_status
args = MagicMock()
rc = cmd_proxy_status(args)
assert rc == 0
out = capsys.readouterr().out
assert "nous" in out
assert "Nous Portal" in out
assert "not logged in" in out
def test_cmd_proxy_providers_runs(capsys):
from hermes_cli.proxy.cli import cmd_proxy_list_providers
args = MagicMock()
rc = cmd_proxy_list_providers(args)
assert rc == 0
out = capsys.readouterr().out
assert "nous" in out
assert "Nous Portal" in out
def test_cmd_proxy_start_refuses_unknown_provider(capsys):
from hermes_cli.proxy.cli import cmd_proxy_start
args = MagicMock()
args.provider = "no-such-provider"
args.host = None
args.port = None
rc = cmd_proxy_start(args)
assert rc == 2
err = capsys.readouterr().err
assert "no-such-provider" in err
def test_cmd_proxy_start_refuses_when_unauthenticated(capsys, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from hermes_cli.proxy.cli import cmd_proxy_start
args = MagicMock()
args.provider = "nous"
args.host = None
args.port = None
rc = cmd_proxy_start(args)
assert rc == 2
err = capsys.readouterr().err
assert "hermes login nous" in err
@@ -39,8 +39,6 @@ class TestExplicitAllowlist:
"OPENROUTER_API_KEY",
"OPENAI_API_KEY",
"ANTHROPIC_API_KEY",
"WANDB_API_KEY",
"TINKER_API_KEY",
"HONCHO_API_KEY",
"FIRECRAWL_API_KEY",
"BROWSERBASE_API_KEY",
-42
View File
@@ -573,48 +573,6 @@ def test_vercel_setup_prefills_project_and_team_from_link_file(tmp_path, monkeyp
assert defaults[" Vercel team ID"] == "linked-team"
def test_offer_launch_chat_relaunches_via_bin(monkeypatch):
from hermes_cli import setup as setup_mod
from hermes_cli import relaunch as relaunch_mod
monkeypatch.setattr(setup_mod, "prompt_yes_no", lambda *_args, **_kwargs: True)
monkeypatch.setattr(relaunch_mod, "resolve_hermes_bin", lambda: "/usr/local/bin/hermes")
exec_calls = []
def fake_execvp(path, argv):
exec_calls.append((path, argv))
raise SystemExit(0)
monkeypatch.setattr(relaunch_mod.os, "execvp", fake_execvp)
with pytest.raises(SystemExit):
setup_mod._offer_launch_chat()
assert exec_calls == [("/usr/local/bin/hermes", ["/usr/local/bin/hermes", "chat"])]
def test_offer_launch_chat_falls_back_to_module(monkeypatch):
from hermes_cli import setup as setup_mod
from hermes_cli import relaunch as relaunch_mod
monkeypatch.setattr(setup_mod, "prompt_yes_no", lambda *_args, **_kwargs: True)
monkeypatch.setattr(relaunch_mod, "resolve_hermes_bin", lambda: None)
exec_calls = []
def fake_execvp(path, argv):
exec_calls.append((path, argv))
raise SystemExit(0)
monkeypatch.setattr(relaunch_mod.os, "execvp", fake_execvp)
with pytest.raises(SystemExit):
setup_mod._offer_launch_chat()
assert exec_calls == [(sys.executable, [sys.executable, "-m", "hermes_cli.main", "chat"])]
def test_setup_slack_saves_home_channel(monkeypatch):
"""_setup_slack() saves SLACK_HOME_CHANNEL when the user provides one."""
saved = {}
@@ -18,4 +18,3 @@ def test_setup_hermes_script_has_termux_path():
assert ".[termux]" in content
assert "constraints-termux.txt" in content
assert "$PREFIX/bin" in content
assert "Skipping tinker-atropos on Termux" in content
@@ -262,7 +262,6 @@ class TestSetupWizardOpenclawIntegration:
patch.object(setup_mod, "setup_tools"),
patch.object(setup_mod, "save_config"),
patch.object(setup_mod, "_print_setup_summary"),
patch.object(setup_mod, "_offer_launch_chat"),
):
setup_mod.run_setup_wizard(args)
@@ -294,7 +293,6 @@ class TestSetupWizardOpenclawIntegration:
patch.object(setup_mod, "setup_tools"),
patch.object(setup_mod, "save_config"),
patch.object(setup_mod, "_print_setup_summary"),
patch.object(setup_mod, "_offer_launch_chat"),
):
setup_mod.run_setup_wizard(args)
@@ -327,7 +325,6 @@ class TestSetupWizardOpenclawIntegration:
patch.object(setup_mod, "setup_tools"),
patch.object(setup_mod, "save_config"),
patch.object(setup_mod, "_print_setup_summary"),
patch.object(setup_mod, "_offer_launch_chat"),
):
setup_mod.run_setup_wizard(args)
@@ -63,7 +63,6 @@ def _enter_existing_install_patches(stack, **extra):
("hermes_cli.setup.get_env_value", {"return_value": None}),
("hermes_cli.auth.get_active_provider", {"return_value": "openrouter"}),
("hermes_cli.setup._print_setup_summary", {}),
("hermes_cli.setup._offer_launch_chat", {}),
("hermes_cli.setup._offer_openclaw_migration", {"return_value": False}),
]:
stack.enter_context(patch(target, **kwargs))
+31
View File
@@ -199,6 +199,37 @@ class TestUserSkins:
# Should inherit defaults for unspecified colors
assert skin.get_color("banner_border") == "#CD7F32" # from default
def test_load_user_skin_invalid_section_types_fall_back_to_defaults(self, tmp_path, monkeypatch):
from hermes_cli.skin_engine import load_skin
skins_dir = tmp_path / "skins"
skins_dir.mkdir()
import yaml
(skins_dir / "broken.yaml").write_text(
yaml.dump(
{
"name": "broken",
"colors": ["not", "a", "mapping"],
"spinner": "invalid",
"branding": ["also", "invalid"],
"tool_emojis": ["invalid"],
"tool_prefix": "!",
}
),
encoding="utf-8",
)
monkeypatch.setattr("hermes_cli.skin_engine._skins_dir", lambda: skins_dir)
skin = load_skin("broken")
assert skin.name == "broken"
assert skin.get_color("banner_title") == "#FFD700"
assert skin.get_branding("agent_name") == "Hermes Agent"
assert skin.spinner.get("waiting_faces", []) == []
assert skin.tool_emojis == {}
assert skin.tool_prefix == "!"
def test_list_skins_includes_user_skins(self, tmp_path, monkeypatch):
from hermes_cli.skin_engine import list_skins
skins_dir = tmp_path / "skins"
+6
View File
@@ -83,6 +83,12 @@ def test_get_platform_tools_default_telegram_includes_messaging():
assert "messaging" in enabled
def test_get_platform_tools_default_whatsapp_includes_web():
enabled = _get_platform_tools({}, "whatsapp")
assert "web" in enabled
def test_get_platform_tools_homeassistant_platform_keeps_homeassistant_toolset():
enabled = _get_platform_tools({}, "homeassistant")
@@ -305,6 +305,7 @@ def _setup_update_mocks(monkeypatch, tmp_path):
monkeypatch.setattr(hermes_config, "get_missing_config_fields", lambda: [])
monkeypatch.setattr(hermes_config, "check_config_version", lambda: (5, 5))
monkeypatch.setattr(hermes_config, "migrate_config", lambda **kw: {"env_added": [], "config_added": []})
monkeypatch.setattr(hermes_main, "_refresh_active_lazy_features", lambda: None)
def test_cmd_update_retries_optional_extras_individually_when_all_fails(monkeypatch, tmp_path, capsys):
+237
View File
@@ -0,0 +1,237 @@
"""Tests for plugin video_gen providers in the tools picker.
Covers the reconfigure path that previously failed to write
``video_gen.provider`` when a user picked an xAI/etc. plugin backend
through Reconfigure tool Video Generation. The first-time configure
path already handled it; the reconfigure path forgot to mirror it.
"""
from __future__ import annotations
from typing import Any, Dict, List, Optional
import pytest
from agent import video_gen_registry
from agent.video_gen_provider import VideoGenProvider
class _FakeVideoProvider(VideoGenProvider):
def __init__(
self,
name: str,
available: bool = True,
schema: Optional[Dict[str, Any]] = None,
models: Optional[List[Dict[str, Any]]] = None,
):
self._name = name
self._available = available
self._schema = schema or {
"name": name.title(),
"badge": "test",
"tag": f"{name} test tag",
"env_vars": [{"key": f"{name.upper()}_API_KEY", "prompt": f"{name} key"}],
}
self._models = models or [
{
"id": f"{name}-video-v1",
"display": f"{name} v1",
"speed": "~10s",
"strengths": "test",
"price": "$",
},
]
@property
def name(self) -> str:
return self._name
def is_available(self) -> bool:
return self._available
def list_models(self):
return list(self._models)
def default_model(self):
return self._models[0]["id"] if self._models else None
def get_setup_schema(self):
return dict(self._schema)
def generate(self, prompt, **kw):
return {"success": True, "video": f"{self._name}://{prompt}"}
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
class TestReconfigureWritesProvider:
"""Regression tests for the video_gen reconfigure path.
Before the fix, _reconfigure_provider() handled image_gen_plugin_name
in both the no-env-vars branch and the post-env-vars branch but
missed video_gen_plugin_name in both. Picking xAI via Reconfigure
tool Video Generation silently no-op'd: the env var was already
set, the env-var loop ran (Enter to keep), and the function fell
through without ever writing config["video_gen"]["provider"].
"""
def test_reconfigure_with_env_vars_already_set_writes_provider(
self, monkeypatch, tmp_path
):
"""Env vars present and user accepts current value → still writes
video_gen.provider via the post-env-vars branch."""
from hermes_cli import tools_config
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
video_gen_registry.register_provider(_FakeVideoProvider("xai_fake"))
# Picker prompts replaced — no TTY in tests.
monkeypatch.setattr(tools_config, "_prompt_choice", lambda *a, **kw: 0)
# User presses Enter to keep the existing key.
monkeypatch.setattr(tools_config, "_prompt", lambda *a, **kw: "")
# Pretend the env var is already set so the reconfigure path
# hits the "Kept current" branch.
monkeypatch.setattr(
tools_config,
"get_env_value",
lambda key: "sk-fake" if key == "XAI_FAKE_API_KEY" else "",
)
config: dict = {}
provider_row = {
"name": "xAI",
"env_vars": [{"key": "XAI_FAKE_API_KEY", "prompt": "xAI key"}],
"video_gen_plugin_name": "xai_fake",
}
tools_config._reconfigure_provider(provider_row, config)
assert config["video_gen"]["provider"] == "xai_fake"
assert config["video_gen"]["model"] == "xai_fake-video-v1"
assert config["video_gen"]["use_gateway"] is False
def test_reconfigure_with_no_env_vars_writes_provider(
self, monkeypatch, tmp_path
):
"""No env vars at all (managed-style plugin) → writes
video_gen.provider via the no-env-vars early-return branch."""
from hermes_cli import tools_config
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
video_gen_registry.register_provider(_FakeVideoProvider(
"noenv_video",
schema={
"name": "NoEnvVideo",
"badge": "free",
"tag": "",
"env_vars": [],
},
))
monkeypatch.setattr(tools_config, "_prompt_choice", lambda *a, **kw: 0)
config: dict = {}
provider_row = {
"name": "NoEnvVideo",
"env_vars": [],
"video_gen_plugin_name": "noenv_video",
}
tools_config._reconfigure_provider(provider_row, config)
assert config["video_gen"]["provider"] == "noenv_video"
assert config["video_gen"]["model"] == "noenv_video-video-v1"
assert config["video_gen"]["use_gateway"] is False
class TestPluginVideoProvidersRow:
"""Tests for _plugin_video_gen_providers row contents."""
def test_post_setup_propagated_when_declared(self, monkeypatch):
from hermes_cli import tools_config
video_gen_registry.register_provider(_FakeVideoProvider(
"xai_video",
schema={
"name": "xAI Grok Imagine",
"badge": "paid",
"tag": "grok video",
"env_vars": [],
"post_setup": "xai_grok",
},
))
rows = tools_config._plugin_video_gen_providers()
match = next(r for r in rows if r.get("video_gen_plugin_name") == "xai_video")
assert match["post_setup"] == "xai_grok"
def test_post_setup_omitted_when_not_declared(self, monkeypatch):
from hermes_cli import tools_config
video_gen_registry.register_provider(_FakeVideoProvider("plain_video"))
rows = tools_config._plugin_video_gen_providers()
match = next(r for r in rows if r.get("video_gen_plugin_name") == "plain_video")
assert "post_setup" not in match
class TestVideoPluginProviderActive:
"""Tests for _is_provider_active recognizing video_gen_plugin_name."""
def test_active_when_video_gen_provider_matches(self):
from hermes_cli import tools_config
config = {"video_gen": {"provider": "xai"}}
row = {"name": "xAI Grok Imagine", "video_gen_plugin_name": "xai"}
assert tools_config._is_provider_active(row, config) is True
def test_inactive_when_video_gen_provider_differs(self):
from hermes_cli import tools_config
config = {"video_gen": {"provider": "fal"}}
row = {"name": "xAI Grok Imagine", "video_gen_plugin_name": "xai"}
assert tools_config._is_provider_active(row, config) is False
def test_inactive_when_video_gen_section_missing(self):
from hermes_cli import tools_config
row = {"name": "xAI Grok Imagine", "video_gen_plugin_name": "xai"}
assert tools_config._is_provider_active(row, {}) is False
def test_detect_active_index_picks_video_plugin_match(self, monkeypatch):
"""When xAI is the configured video_gen provider, the picker should
default to the xAI row even if FAL_KEY happens to be set in env.
Regression: previously _detect_active_provider_index() saw
_is_provider_active(xai) return False (no video_gen branch),
skipped xAI (empty env_vars), and matched the FAL row via the
env-var fallback so the picker visually defaulted to FAL even
though the user picked xAI. The xAI row uses empty env_vars
because authentication is handled via xAI Grok OAuth (post_setup
hook).
"""
from hermes_cli import tools_config
monkeypatch.setattr(
tools_config,
"get_env_value",
lambda key: "fal-key" if key == "FAL_KEY" else "",
)
config = {"video_gen": {"provider": "xai"}}
providers = [
{"name": "xAI Grok Imagine", "env_vars": [], "video_gen_plugin_name": "xai"},
{
"name": "FAL.ai",
"env_vars": [{"key": "FAL_KEY", "prompt": "FAL"}],
"video_gen_plugin_name": "fal",
},
]
assert tools_config._detect_active_provider_index(providers, config) == 0
+40 -9
View File
@@ -6,6 +6,8 @@ import os
from pathlib import Path
from unittest.mock import patch, MagicMock
from hermes_cli.profiles import _get_default_hermes_home
import pytest
from plugins.memory.honcho.client import (
@@ -349,18 +351,25 @@ class TestResolveConfigPath:
result = resolve_config_path()
assert result == local_cfg
def test_falls_back_to_global_when_no_local(self, tmp_path):
hermes_home = tmp_path / "hermes"
hermes_home.mkdir()
# No honcho.json in HERMES_HOME — also isolate ~/.hermes so
# the default-profile fallback doesn't hit the real filesystem.
def test_falls_back_to_default_profile_when_no_local(self, tmp_path, monkeypatch):
# Profile mode: HERMES_HOME points at ~/.hermes/profiles/<name>, so
# _get_default_hermes_home() must resolve back to ~/.hermes — that's
# the bug the HOME-anchored helper fixes (vs. blindly using Path.home()).
fake_home = tmp_path / "fakehome"
fake_home.mkdir()
default_home = fake_home / ".hermes"
profile_home = default_home / "profiles" / "work"
profile_home.mkdir(parents=True)
default_cfg = default_home / "honcho.json"
default_cfg.write_text('{"apiKey": "default-key"}')
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}), \
patch.object(Path, "home", return_value=fake_home):
result = resolve_config_path()
assert result == fake_home / ".honcho" / "config.json"
monkeypatch.setattr(Path, "home", lambda: fake_home)
monkeypatch.setenv("HERMES_HOME", str(profile_home))
result = resolve_config_path()
assert _get_default_hermes_home() == default_home
assert result == default_cfg
def test_falls_back_to_global_without_hermes_home_env(self, tmp_path):
fake_home = tmp_path / "fakehome"
@@ -383,6 +392,28 @@ class TestResolveConfigPath:
assert resolve_global_config_path() == fake_home / ".honcho" / "config.json"
assert resolve_config_path() == fake_home / ".honcho" / "config.json"
def test_from_global_config_uses_default_profile_fallback(self, tmp_path, monkeypatch):
# Profile mode: from_global_config() reads the default-profile honcho.json
# via the HOME-anchored helper, not Path.home() / ".hermes".
fake_home = tmp_path / "fakehome"
fake_home.mkdir()
default_home = fake_home / ".hermes"
profile_home = default_home / "profiles" / "work"
profile_home.mkdir(parents=True)
default_cfg = default_home / "honcho.json"
default_cfg.write_text(json.dumps({
"apiKey": "default-key",
"workspace": "default-ws",
}))
monkeypatch.setattr(Path, "home", lambda: fake_home)
monkeypatch.setenv("HERMES_HOME", str(profile_home))
config = HonchoClientConfig.from_global_config()
assert config.api_key == "default-key"
assert config.workspace_id == "default-ws"
def test_from_global_config_uses_local_path(self, tmp_path):
hermes_home = tmp_path / "hermes"
hermes_home.mkdir()
+6 -3
View File
@@ -72,10 +72,13 @@ class TestXAIImageGenProvider:
provider = XAIImageGenProvider()
schema = provider.get_setup_schema()
assert schema["name"] == "xAI (Grok)"
assert schema["name"] == "xAI Grok Imagine (image)"
assert schema["badge"] == "paid"
assert len(schema["env_vars"]) == 1
assert schema["env_vars"][0]["key"] == "XAI_API_KEY"
# Auth resolution is delegated to the shared "xai_grok" post_setup
# hook so the picker doesn't blindly prompt for XAI_API_KEY when the
# user is already signed in via xAI Grok OAuth.
assert schema["env_vars"] == []
assert schema["post_setup"] == "xai_grok"
# ---------------------------------------------------------------------------
@@ -1,4 +1,5 @@
import json
import zipfile
from types import SimpleNamespace
from unittest.mock import MagicMock
@@ -156,6 +157,43 @@ def test_tool_add_resource_uploads_existing_local_directory_and_cleans_zip(tmp_p
assert result["root_uri"] == "viking://resources/docs"
def test_tool_add_resource_directory_zip_skips_symlink_escape(tmp_path):
secret = tmp_path / "outside-secret.txt"
secret.write_text("do not upload\n", encoding="utf-8")
docs = tmp_path / "docs"
docs.mkdir()
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
link = docs / "leak.txt"
try:
link.symlink_to(secret)
except OSError as exc:
pytest.skip(f"symlinks unavailable in test environment: {exc}")
provider = OpenVikingMemoryProvider()
provider._client = MagicMock()
archive_entries = {}
def inspect_upload(path):
with zipfile.ZipFile(path) as archive:
archive_entries["names"] = archive.namelist()
archive_entries["payloads"] = {
name: archive.read(name)
for name in archive.namelist()
}
return "upload_docs.zip"
provider._client.upload_temp_file.side_effect = inspect_upload
provider._client.post.return_value = {
"status": "ok",
"result": {"root_uri": "viking://resources/docs"},
}
json.loads(provider._tool_add_resource({"url": str(docs)}))
assert archive_entries["names"] == ["guide.md"]
assert b"do not upload" not in b"".join(archive_entries["payloads"].values())
def test_tool_add_resource_cleans_local_directory_zip_when_add_fails(tmp_path):
docs = tmp_path / "docs"
docs.mkdir()
+537 -1
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import importlib
import logging
import sys
from pathlib import Path
@@ -164,7 +165,542 @@ class TestHooksInert:
# Each hook should just return; no exceptions.
mod.on_pre_llm_call(task_id="t", session_id="s", messages=[{"role": "user", "content": "hi"}])
mod.on_pre_llm_request(task_id="t", session_id="s", api_call_count=1, messages=[])
mod.on_pre_llm_request(task_id="t", session_id="s", api_call_count=1, request_messages=[])
mod.on_post_llm_call(task_id="t", session_id="s", api_call_count=1)
mod.on_pre_tool_call(tool_name="read_file", args={}, task_id="t", session_id="s")
mod.on_post_tool_call(tool_name="read_file", args={}, result="ok", task_id="t", session_id="s")
# ---------------------------------------------------------------------------
# Placeholder-credential guard (#23823).
#
# Regression coverage for the silent-failure bug: when an operator leaves
# HERMES_LANGFUSE_PUBLIC_KEY / SECRET_KEY at a template value like
# "placeholder", "test-key", or "your-langfuse-key", the SDK accepts the
# credentials at construction time (it does no server-side validation
# eagerly) but drops every trace at flush time, with no signal in the
# Hermes logs. The fix in `_get_langfuse()` validates the documented
# `pk-lf-` / `sk-lf-` prefix Langfuse always issues, surfaces a one-shot
# warning naming the offending env var(s), and short-circuits via the
# same `_INIT_FAILED` path used for missing credentials so subsequent
# hook invocations don't re-log.
# ---------------------------------------------------------------------------
class _FakeLangfuse:
"""Stand-in for the real :class:`langfuse.Langfuse` so tests don't
need the optional ``langfuse`` SDK installed. The plugin's runtime
gate refuses to proceed past ``if Langfuse is None`` when the SDK
is missing, which would short-circuit before the placeholder check
can fire. Patching ``plugin.Langfuse`` with this class lets the
placeholder validator exercise its full code path."""
instances: list["_FakeLangfuse"] = []
def __init__(self, **kwargs):
self.kwargs = kwargs
_FakeLangfuse.instances.append(self)
class TestPlaceholderKeyDetection:
LOGGER_NAME = "plugins.observability.langfuse"
def _fresh_plugin(self, monkeypatch=None):
mod_name = "plugins.observability.langfuse"
sys.modules.pop(mod_name, None)
mod = importlib.import_module(mod_name)
if monkeypatch is not None:
# Pretend the SDK is installed so `_get_langfuse()` actually
# reaches the placeholder check. Real SDK calls are never
# made because the placeholder/missing-credentials paths
# return before constructing a client.
_FakeLangfuse.instances.clear()
monkeypatch.setattr(mod, "Langfuse", _FakeLangfuse, raising=False)
return mod
@staticmethod
def _clear_env(monkeypatch):
for k in (
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
):
monkeypatch.delenv(k, raising=False)
# -- helper unit tests (no SDK stub needed: these don't go through
# _get_langfuse, they exercise the pure-Python helpers directly) ------
def test_redact_key_preview_empty(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._redact_key_preview("") == "<empty>"
def test_redact_key_preview_short_value_echoed(self, monkeypatch):
"""Short placeholder strings are echoed in full so the operator
can see exactly which template they forgot to replace."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._redact_key_preview("placeholder") == "'placeholder'"
assert plugin._redact_key_preview("test-key") == "'test-key'"
def test_redact_key_preview_long_value_truncated(self, monkeypatch):
"""If an operator pasted a real secret into the wrong env var the
preview must NOT echo it in full only the leading 6 chars."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
result = plugin._redact_key_preview("sk-lf-abcdefghijklmnop")
assert "abcdefghij" not in result
assert result.startswith("'sk-lf-")
assert result.endswith("...'")
def test_validate_langfuse_key_accepts_documented_prefix(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._validate_langfuse_key(
"HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz"
) is None
assert plugin._validate_langfuse_key(
"HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz"
) is None
def test_validate_langfuse_key_rejects_wrong_prefix(self, monkeypatch):
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
msg = plugin._validate_langfuse_key(
"HERMES_LANGFUSE_PUBLIC_KEY", "placeholder"
)
assert msg is not None
assert "HERMES_LANGFUSE_PUBLIC_KEY" in msg
assert "pk-lf-" in msg
def test_validate_langfuse_key_unknown_name_passes(self, monkeypatch):
"""Defensive: an env var with no registered prefix is trusted."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin()
assert plugin._validate_langfuse_key("HERMES_LANGFUSE_BASE_URL", "anything") is None
# -- end-to-end _get_langfuse() behaviour --------------------------------
# These tests pass `monkeypatch` to _fresh_plugin() so the helper can
# stub out `Langfuse` (the optional SDK). Without that, every call
# short-circuits at `if Langfuse is None` before reaching the
# placeholder validator — masking the very behaviour we're testing.
def test_placeholder_public_key_warns_and_skips(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
text = caplog.text
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
assert "'placeholder'" in text
assert "pk-lf-" in text
# The valid secret value must NOT appear (the var NAME does, in
# the "or unset ..." hint, but the value preview shouldn't).
assert "'sk-lf-" not in text
# Never constructed the SDK client — short-circuited before that.
assert _FakeLangfuse.instances == []
def test_placeholder_secret_key_warns_and_skips(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "test-key")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
text = caplog.text
assert "HERMES_LANGFUSE_SECRET_KEY" in text
assert "'test-key'" in text
assert "sk-lf-" in text
# The valid public value must NOT appear.
assert "'pk-lf-" not in text
assert _FakeLangfuse.instances == []
def test_both_placeholders_one_warning_with_both_keys(self, monkeypatch, caplog):
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert len(warnings) == 1, (
f"Expected a single combined warning; got {len(warnings)}:\n"
+ "\n".join(r.getMessage() for r in warnings)
)
text = warnings[0].getMessage()
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
assert "HERMES_LANGFUSE_SECRET_KEY" in text
def test_repeated_calls_do_not_re_warn(self, monkeypatch, caplog):
"""The cached ``_INIT_FAILED`` sentinel must short-circuit
subsequent calls so each hook invocation isn't a fresh log
line otherwise a busy gateway will spam the operator's
terminal."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
for _ in range(15):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert len(warnings) == 1, (
f"Warning fired {len(warnings)} times across 15 calls; "
"expected 1 (cached via _INIT_FAILED)"
)
@pytest.mark.parametrize("placeholder", [
"placeholder",
"test-key",
"your-langfuse-key",
"change-me",
"xxx",
"dummy-key-here",
"<your-key>",
"REPLACE_ME",
])
def test_common_placeholders_detected(self, monkeypatch, caplog, placeholder):
"""A grab-bag of values that real-world ``.env.example`` templates
use as stand-ins. Any of them in either key must trip the guard."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", placeholder)
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
def test_legacy_LANGFUSE_PUBLIC_KEY_also_validated(self, monkeypatch, caplog):
"""The plugin reads both the canonical HERMES_-prefixed env var and
the legacy bare ``LANGFUSE_PUBLIC_KEY``. The validator must run on
whichever value ``_get_langfuse()`` actually consumed."""
self._clear_env(monkeypatch)
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
# Warning names the canonical user-facing env var (the bare
# LANGFUSE_PUBLIC_KEY is a backwards-compat alias for the
# HERMES_-prefixed one — operators set the HERMES_-prefixed one).
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
assert "'placeholder'" in caplog.text
def test_missing_credentials_still_skip_silently(self, monkeypatch, caplog):
"""Missing-creds is the documented opt-out path (operator hasn't
configured the plugin yet) it must remain SILENT. Regression
guard against the placeholder validator accidentally running on
empty values and re-introducing log noise for unconfigured
installs."""
self._clear_env(monkeypatch)
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert warnings == []
def test_sdk_not_installed_still_skips_silently(self, monkeypatch, caplog):
"""If the langfuse SDK isn't installed at all, the placeholder
check should never run there's nothing the operator can do
about a credential mismatch when the package is missing, and
re-warning here would dilute the actually-actionable SDK-missing
signal upstream. The ``Langfuse is None`` guard at the top of
``_get_langfuse`` already handles this; this test pins that
behaviour."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
# NO monkeypatch on Langfuse here — falls back to whatever the
# plugin imported at module load (None if SDK absent).
plugin = self._fresh_plugin()
monkeypatch.setattr(plugin, "Langfuse", None, raising=False)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
assert plugin._get_langfuse() is None
warnings = [r for r in caplog.records if r.levelname == "WARNING"
and r.name == self.LOGGER_NAME]
assert warnings == []
def test_valid_prefixes_do_not_trigger_placeholder_warning(self, monkeypatch, caplog):
"""Real Langfuse keys (``pk-lf-…`` / ``sk-lf-…``) must pass the
guard and proceed to SDK init. We stub the SDK constructor with
a recording fake so the assertion can confirm BOTH that the
placeholder warning didn't fire AND that the client was actually
constructed the latter is the success signal the bug report
wanted."""
self._clear_env(monkeypatch)
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
plugin = self._fresh_plugin(monkeypatch)
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
client = plugin._get_langfuse()
assert isinstance(client, _FakeLangfuse)
assert client.kwargs["public_key"] == "pk-lf-real-public-xyz"
assert client.kwargs["secret_key"] == "sk-lf-real-secret-xyz"
assert "placeholders" not in caplog.text.lower(), (
f"Valid Langfuse keys tripped the placeholder guard: {caplog.text!r}"
)
class TestRequestMessageCoercion:
def test_prefers_request_messages_then_messages_then_history_then_user_message(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
assert mod._coerce_request_messages(
request_messages=[{"role": "system", "content": "s"}],
messages=[{"role": "user", "content": "m"}],
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "system", "content": "s"}]
assert mod._coerce_request_messages(
messages=[{"role": "user", "content": "m"}],
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "user", "content": "m"}]
assert mod._coerce_request_messages(
conversation_history=[{"role": "user", "content": "h"}],
user_message="u",
) == [{"role": "user", "content": "h"}]
assert mod._coerce_request_messages(user_message="u") == [{"role": "user", "content": "u"}]
class TestToolCallOutputBackfill:
def test_post_tool_call_backfills_matching_turn_tool_call_output(self, monkeypatch):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
observation = object()
state = mod.TraceState(trace_id="trace-1", root_ctx=None, root_span=None)
state.tools["call-1"] = observation
state.turn_tool_calls.append({
"id": "call-1",
"type": "function",
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
"function": {
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
},
})
task_key = mod._trace_key("task-1", "session-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end_observation(obs, *, output=None, metadata=None, usage_details=None, cost_details=None):
ended["observation"] = obs
ended["output"] = output
ended["metadata"] = metadata
monkeypatch.setattr(mod, "_end_observation", fake_end_observation)
mod.on_post_tool_call(
tool_name="web_extract",
args={"urls": ["https://example.com"]},
result='{"results": [{"url": "https://example.com", "content": "Example Domain"}]}',
task_id="task-1",
session_id="session-1",
tool_call_id="call-1",
)
assert ended["observation"] is observation
assert state.turn_tool_calls[0]["output"] == ended["output"]
assert state.turn_tool_calls[0]["function"]["output"] == ended["output"]
assert state.turn_tool_calls[0]["output"] == {
"results": [{"url": "https://example.com", "content": "Example Domain"}]
}
def test_serialize_messages_keeps_tool_name_and_call_id(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
messages = [{
"role": "tool",
"name": "web_extract",
"tool_call_id": "call-1",
"content": '{"ok": true}',
}]
assert mod._serialize_messages(messages) == [{
"role": "tool",
"name": "web_extract",
"tool_call_id": "call-1",
"content": {"ok": True},
}]
def test_serialize_tool_calls_emits_openai_style_function_shape(self):
sys.modules.pop("plugins.observability.langfuse", None)
mod = importlib.import_module("plugins.observability.langfuse")
class _Fn:
name = "web_extract"
arguments = '{"urls": ["https://example.com"]}'
class _ToolCall:
id = "call-1"
type = "function"
function = _Fn()
assert mod._serialize_tool_calls([_ToolCall()]) == [{
"id": "call-1",
"type": "function",
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
"function": {
"name": "web_extract",
"arguments": '{"urls": ["https://example.com"]}',
},
}]
class TestToolObservationKeying:
"""Tests for pre/post tool_call observation matching when tool_call_id is absent."""
def _make_mod(self):
sys.modules.pop("plugins.observability.langfuse", None)
return importlib.import_module("plugins.observability.langfuse")
def test_empty_tool_call_id_single_tool_sets_output(self, monkeypatch):
mod = self._make_mod()
obs = object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name.setdefault("my_tool", []).append(obs)
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end(o, *, output=None, metadata=None, **kw):
ended["obs"] = o
ended["output"] = output
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="my_tool",
args={},
result='{"ok": true}',
task_id="task-1",
session_id="sess-1",
tool_call_id="",
)
assert ended["obs"] is obs
assert ended["output"] == {"ok": True}
assert state.pending_tools_by_name.get("my_tool") is None
def test_empty_tool_call_id_observations_are_fifo_within_tool_name(self, monkeypatch):
"""Two queued observations are consumed in FIFO order so the first
post hook gets the first observation's output, not the second.
Sequential-on-one-thread coverage; the real concurrent case is
guarded by ``_STATE_LOCK`` around every read-modify-write on
``pending_tools_by_name`` and is exercised in
``test_threaded_post_calls_preserve_fifo_under_lock`` below.
"""
mod = self._make_mod()
obs_a, obs_b = object(), object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name["web_extract"] = [obs_a, obs_b]
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
calls = []
def fake_end(o, *, output=None, metadata=None, **kw):
calls.append((o, output))
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"val": "a"}',
task_id="task-1", session_id="sess-1", tool_call_id="",
)
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"val": "b"}',
task_id="task-1", session_id="sess-1", tool_call_id="",
)
assert calls[0] == (obs_a, {"val": "a"})
assert calls[1] == (obs_b, {"val": "b"})
assert state.pending_tools_by_name.get("web_extract") is None
def test_threaded_post_calls_preserve_fifo_under_lock(self, monkeypatch):
"""The actual concurrency contract: when 8 threads race to drain
the pending queue, no observation is consumed twice and none is
lost. Validates ``_STATE_LOCK`` discipline, not Python list
semantics."""
import threading
mod = self._make_mod()
n = 8
observations = [object() for _ in range(n)]
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.pending_tools_by_name["web_extract"] = list(observations)
task_key = mod._trace_key("task-thr", "sess-thr")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
recorded: list = []
lock = threading.Lock()
def fake_end(o, *, output=None, metadata=None, **kw):
with lock:
recorded.append(o)
monkeypatch.setattr(mod, "_end_observation", fake_end)
barrier = threading.Barrier(n)
def worker():
barrier.wait()
mod.on_post_tool_call(
tool_name="web_extract", args={}, result='{"ok": true}',
task_id="task-thr", session_id="sess-thr", tool_call_id="",
)
threads = [threading.Thread(target=worker) for _ in range(n)]
for t in threads:
t.start()
for t in threads:
t.join()
# Every observation was consumed exactly once; queue is empty.
assert len(recorded) == n
assert set(map(id, recorded)) == set(map(id, observations))
assert state.pending_tools_by_name.get("web_extract") is None
def test_explicit_tool_call_id_uses_tools_dict(self, monkeypatch):
"""When tool_call_id is present, pending_tools_by_name is not touched."""
mod = self._make_mod()
obs = object()
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
state.tools["call-99"] = obs
task_key = mod._trace_key("task-1", "sess-1")
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
ended = {}
def fake_end(o, *, output=None, metadata=None, **kw):
ended["obs"] = o
ended["output"] = output
monkeypatch.setattr(mod, "_end_observation", fake_end)
mod.on_post_tool_call(
tool_name="my_tool", args={}, result='{"status": "done"}',
task_id="task-1", session_id="sess-1", tool_call_id="call-99",
)
assert ended["obs"] is obs
assert ended["output"] == {"status": "done"}
assert not state.tools
+1
View File
@@ -0,0 +1 @@
"""Make tests/plugins/video_gen a package."""
+314
View File
@@ -0,0 +1,314 @@
"""Tests for the FAL video gen plugin — family routing, payload shape."""
from __future__ import annotations
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
def test_fal_provider_registers():
from plugins.video_gen.fal import FALVideoGenProvider, DEFAULT_MODEL
provider = FALVideoGenProvider()
video_gen_registry.register_provider(provider)
assert video_gen_registry.get_provider("fal") is provider
assert provider.display_name == "FAL"
# DEFAULT_MODEL is the cheap-tier default
assert provider.default_model() == DEFAULT_MODEL
assert DEFAULT_MODEL in {"pixverse-v6", "ltx-2.3"}
def test_fal_family_catalog():
"""Each family declares both endpoints. The catalog covers the
cheap + premium tiers Teknium listed."""
from plugins.video_gen.fal import FAL_FAMILIES
expected = {
# cheap
"ltx-2.3", "pixverse-v6",
# premium
"veo3.1", "seedance-2.0", "kling-v3-4k", "happy-horse",
}
assert expected.issubset(set(FAL_FAMILIES.keys())), (
f"missing families: {expected - set(FAL_FAMILIES.keys())}"
)
for fid, meta in FAL_FAMILIES.items():
assert meta.get("text_endpoint"), f"{fid} missing text_endpoint"
assert meta.get("image_endpoint"), f"{fid} missing image_endpoint"
assert meta["text_endpoint"] != meta["image_endpoint"]
assert meta.get("tier") in {"cheap", "premium"}, (
f"{fid} has invalid tier"
)
def test_kling_4k_uses_start_image_url():
"""Kling v3 4K's image-to-video endpoint expects start_image_url,
not image_url. The family must declare image_param_key='start_image_url'."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["kling-v3-4k"]
assert meta.get("image_param_key") == "start_image_url"
payload = _build_payload(
meta,
prompt="x",
image_url="https://example.com/i.png",
duration=5,
aspect_ratio="16:9",
resolution="720p",
negative_prompt=None,
audio=None,
seed=None,
)
assert payload.get("start_image_url") == "https://example.com/i.png"
assert "image_url" not in payload
def test_fal_list_models_advertises_both_modalities():
from plugins.video_gen.fal import FALVideoGenProvider
models = FALVideoGenProvider().list_models()
for m in models:
assert set(m["modalities"]) == {"text", "image"}, (
f"{m['id']} doesn't advertise both modalities — every family "
f"should have t2v + i2v"
)
def test_fal_unavailable_without_key(monkeypatch):
from plugins.video_gen.fal import FALVideoGenProvider
monkeypatch.delenv("FAL_KEY", raising=False)
assert FALVideoGenProvider().is_available() is False
def test_fal_generate_requires_fal_key(monkeypatch):
from plugins.video_gen.fal import FALVideoGenProvider
monkeypatch.delenv("FAL_KEY", raising=False)
result = FALVideoGenProvider().generate("a happy dog")
assert result["success"] is False
assert result["error_type"] == "auth_required"
class TestFamilyRouting:
"""The headline behavior: image_url presence picks the endpoint."""
@pytest.fixture
def with_fake_fal(self, monkeypatch):
"""Stub fal_client.subscribe to capture which endpoint we hit."""
import sys
import types
captured = {"endpoint": None, "arguments": None}
fake = types.ModuleType("fal_client")
def _subscribe(endpoint, arguments=None, with_logs=False):
captured["endpoint"] = endpoint
captured["arguments"] = arguments
return {"video": {"url": "https://fake/out.mp4"}}
fake.subscribe = _subscribe # type: ignore
monkeypatch.setitem(sys.modules, "fal_client", fake)
# Reset the lazy global so it picks up our stub
from plugins.video_gen import fal as fal_plugin
fal_plugin._fal_client = None
monkeypatch.setenv("FAL_KEY", "test")
return captured
def test_text_to_video_routes_to_text_endpoint(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"a dog running",
model="pixverse-v6",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/text-to-video"
assert result["modality"] == "text"
assert with_fake_fal["arguments"]["prompt"] == "a dog running"
assert "image_url" not in with_fake_fal["arguments"]
def test_image_to_video_routes_to_image_endpoint(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"animate this dog",
model="pixverse-v6",
image_url="https://example.com/dog.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/image-to-video"
assert result["modality"] == "image"
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
def test_default_family_text_routing(self, with_fake_fal):
"""No model arg → DEFAULT_MODEL → text-to-video endpoint."""
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate("a dog")
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_default_family_image_routing(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate(
"animate this",
image_url="https://example.com/i.png",
)
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["image_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_unknown_family_falls_back_to_default(self, with_fake_fal):
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
result = FALVideoGenProvider().generate(
"x",
model="not-a-real-family",
)
assert result["success"] is True
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
assert with_fake_fal["endpoint"] == expected_endpoint
def test_premium_seedance_routing(self, with_fake_fal):
"""Sanity check the premium-tier seedance routes correctly."""
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"a dog",
model="seedance-2.0",
image_url="https://example.com/dog.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "bytedance/seedance-2.0/image-to-video"
# Seedance uses regular image_url (not start_image_url)
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
def test_kling_4k_remaps_image_param(self, with_fake_fal):
"""Kling v3 4K image-to-video receives start_image_url, not image_url."""
from plugins.video_gen.fal import FALVideoGenProvider
result = FALVideoGenProvider().generate(
"x",
model="kling-v3-4k",
image_url="https://example.com/frame.png",
)
assert result["success"] is True
assert with_fake_fal["endpoint"] == "fal-ai/kling-video/v3/4k/image-to-video"
assert with_fake_fal["arguments"].get("start_image_url") == "https://example.com/frame.png"
assert "image_url" not in with_fake_fal["arguments"]
class TestPayloadBuilder:
def test_drops_unsupported_keys(self):
"""Veo enum-clamps duration, supports aspect+resolution+audio+neg."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["veo3.1"]
p = _build_payload(
meta,
prompt="x",
image_url=None,
duration=12, # not in enum (4,6,8) — snap to 8
aspect_ratio="16:9",
resolution="720p",
negative_prompt="ugly",
audio=True,
seed=42,
)
assert p["prompt"] == "x"
assert p["duration"] == "8" # FAL queue API uses strings
assert p["aspect_ratio"] == "16:9"
assert p["resolution"] == "720p"
assert p["generate_audio"] is True
assert p["negative_prompt"] == "ugly"
assert p["seed"] == 42
def test_pixverse_range_clamps_correctly(self):
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["pixverse-v6"]
p = _build_payload(
meta,
prompt="x",
image_url="https://i.png",
duration=99, # over max → 15
aspect_ratio="16:9",
resolution="540p",
negative_prompt=None,
audio=None,
seed=None,
)
assert p["duration"] == "15"
def test_kling_4k_clamps_below_min(self):
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["kling-v3-4k"]
p = _build_payload(
meta,
prompt="x",
image_url="https://i.png",
duration=1, # below min (3) → 3
aspect_ratio="16:9",
resolution="720p",
negative_prompt=None,
audio=None,
seed=None,
)
assert p["duration"] == "3"
def test_ltx_omits_duration_aspect_resolution(self):
"""LTX 2.3 doesn't declare duration/aspect/resolution enums —
the payload should NOT include those keys (let FAL default)."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["ltx-2.3"]
p = _build_payload(
meta,
prompt="x",
image_url=None,
duration=8,
aspect_ratio="16:9",
resolution="720p",
negative_prompt="ugly",
audio=True,
seed=None,
)
assert "duration" not in p
assert "aspect_ratio" not in p
assert "resolution" not in p
# But audio + negative are advertised
assert p["generate_audio"] is True
assert p["negative_prompt"] == "ugly"
def test_happy_horse_minimal_payload(self):
"""Happy Horse has sparse docs — payload should be minimal."""
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
meta = FAL_FAMILIES["happy-horse"]
p = _build_payload(
meta,
prompt="a horse galloping",
image_url=None,
duration=8,
aspect_ratio="16:9",
resolution="720p",
negative_prompt="watermark",
audio=True,
seed=None,
)
# Only prompt — no payload bloat for fields we can't verify
assert p == {"prompt": "a horse galloping"}
+113
View File
@@ -0,0 +1,113 @@
"""Smoke tests for the xAI video gen plugin — load & register surface."""
from __future__ import annotations
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
def test_xai_provider_registers():
from plugins.video_gen.xai import XAIVideoGenProvider
provider = XAIVideoGenProvider()
video_gen_registry.register_provider(provider)
assert video_gen_registry.get_provider("xai") is provider
assert provider.display_name == "xAI"
assert provider.default_model() == "grok-imagine-video"
def test_xai_capabilities_text_and_image_only():
"""xAI was previously advertised with edit/extend operations. The
simplified surface only exposes text-to-video and image-to-video
confirm those are the only modalities advertised."""
from plugins.video_gen.xai import XAIVideoGenProvider
caps = XAIVideoGenProvider().capabilities()
assert caps["modalities"] == ["text", "image"]
# No 'operations' key in the simplified surface
assert "operations" not in caps
assert caps["max_reference_images"] == 7
def test_xai_unavailable_without_key(monkeypatch):
from plugins.video_gen.xai import XAIVideoGenProvider
monkeypatch.delenv("XAI_API_KEY", raising=False)
assert XAIVideoGenProvider().is_available() is False
def test_xai_generate_requires_xai_key(monkeypatch):
from plugins.video_gen.xai import XAIVideoGenProvider
monkeypatch.delenv("XAI_API_KEY", raising=False)
result = XAIVideoGenProvider().generate("a happy dog")
assert result["success"] is False
assert result["error_type"] == "auth_required"
def test_xai_available_with_oauth_only(monkeypatch):
"""The plugin must honour xAI Grok OAuth credentials, not just
XAI_API_KEY. Otherwise the agent's tool-availability check filters
``video_generate`` out of the toolbelt and the agent silently falls
back to whatever skill advertises video generation (e.g. comfyui).
"""
import plugins.video_gen.xai as xai_plugin
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(
"tools.xai_http.resolve_xai_http_credentials",
lambda: {
"provider": "xai-oauth",
"api_key": "oauth-bearer-token",
"base_url": "https://api.x.ai/v1",
},
)
assert xai_plugin.XAIVideoGenProvider().is_available() is True
def test_xai_resolved_credentials_threaded_through_request(monkeypatch):
"""OAuth-resolved creds must reach the HTTP layer — bug class where
``is_available()`` says yes but the request still hits with no key.
"""
import plugins.video_gen.xai as xai_plugin
monkeypatch.delenv("XAI_API_KEY", raising=False)
monkeypatch.setattr(
"tools.xai_http.resolve_xai_http_credentials",
lambda: {
"provider": "xai-oauth",
"api_key": "oauth-bearer-token",
"base_url": "https://api.x.ai/v1",
},
)
api_key, base_url = xai_plugin._resolve_xai_credentials()
assert api_key == "oauth-bearer-token"
assert base_url == "https://api.x.ai/v1"
headers = xai_plugin._xai_headers(api_key)
assert headers["Authorization"] == "Bearer oauth-bearer-token"
def test_xai_no_operation_kwarg():
"""The ABC's generate() signature no longer accepts 'operation'.
Passing it through **kwargs should be ignored (forward-compat)."""
from plugins.video_gen.xai import XAIVideoGenProvider
# We're not actually hitting the network — just verify the call
# doesn't TypeError on the unexpected kwarg.
# Will fail with auth_required (no XAI_API_KEY), but should NOT
# fail with TypeError.
result = XAIVideoGenProvider().generate("x", operation="generate")
assert result["success"] is False
# auth_required, NOT some signature error
assert result["error_type"] in ("auth_required", "api_error")
@@ -0,0 +1,191 @@
"""Integration tests for the xAI video gen plugin's simplified surface.
xAI exposes only text-to-video and image-to-video through the unified
``video_generate`` tool. We assert the endpoint hit and the payload shape
because routing is the part most likely to break silently.
"""
from __future__ import annotations
import asyncio
import json
from typing import Any, Dict, List, Optional
import pytest
from agent import video_gen_registry
@pytest.fixture(autouse=True)
def _reset_registry():
video_gen_registry._reset_for_tests()
yield
video_gen_registry._reset_for_tests()
class _FakeResponse:
def __init__(self, status: int = 200, payload: Optional[Dict[str, Any]] = None):
self.status_code = status
self._payload = payload or {}
self.text = json.dumps(self._payload)
def raise_for_status(self):
if self.status_code >= 400:
import httpx
raise httpx.HTTPStatusError("err", request=None, response=self) # type: ignore
def json(self):
return self._payload
class _FakeAsyncClient:
def __init__(self):
self.posts: List[Dict[str, Any]] = []
async def __aenter__(self):
return self
async def __aexit__(self, *args):
return None
async def post(self, url, headers=None, json=None, timeout=None):
self.posts.append({"url": url, "json": json})
return _FakeResponse(200, {"request_id": "req-123"})
async def get(self, url, headers=None, timeout=None):
return _FakeResponse(200, {
"status": "done",
"video": {"url": "https://xai-cdn/out.mp4", "duration": 8},
"model": "grok-imagine-video",
})
@pytest.fixture
def xai_provider(monkeypatch):
monkeypatch.setenv("XAI_API_KEY", "test-key")
import plugins.video_gen.xai as xai_plugin
captured: Dict[str, _FakeAsyncClient] = {}
def _client_factory():
captured["client"] = _FakeAsyncClient()
return captured["client"]
monkeypatch.setattr(xai_plugin.httpx, "AsyncClient", _client_factory)
async def _no_sleep(*a, **k):
return None
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
provider = xai_plugin.XAIVideoGenProvider()
return provider, captured
def _last_post(captured) -> Dict[str, Any]:
return captured["client"].posts[-1]
class TestXAIEndpoint:
"""xAI uses one endpoint — ``/videos/generations`` — for both modes."""
def test_text_to_video_hits_generations(self, xai_provider):
provider, captured = xai_provider
result = provider.generate("a dog on a skateboard")
assert result["success"] is True
assert _last_post(captured)["url"].endswith("/videos/generations")
assert result["modality"] == "text"
def test_image_to_video_hits_generations(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"animate this",
image_url="https://example.com/cat.png",
)
assert result["success"] is True
assert _last_post(captured)["url"].endswith("/videos/generations")
assert result["modality"] == "image"
class TestXAIPayload:
def test_text_payload_has_no_image_field(self, xai_provider):
provider, captured = xai_provider
provider.generate("a dog at sunset")
payload = _last_post(captured)["json"]
assert payload["prompt"] == "a dog at sunset"
assert "image" not in payload
assert "reference_images" not in payload
def test_image_payload_has_image_field(self, xai_provider):
provider, captured = xai_provider
provider.generate("animate this", image_url="https://example.com/cat.png")
payload = _last_post(captured)["json"]
assert payload["image"] == {"url": "https://example.com/cat.png"}
def test_reference_images_payload(self, xai_provider):
provider, captured = xai_provider
provider.generate(
"keep this character",
reference_image_urls=[
"https://example.com/a.png",
"https://example.com/b.png",
],
)
payload = _last_post(captured)["json"]
assert payload["reference_images"] == [
{"url": "https://example.com/a.png"},
{"url": "https://example.com/b.png"},
]
class TestXAIValidation:
def test_missing_prompt_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate("")
assert result["success"] is False
assert result["error_type"] == "missing_prompt"
# Never hit the network
assert "client" not in captured or not captured["client"].posts
def test_image_plus_refs_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"x",
image_url="https://example.com/i.png",
reference_image_urls=["https://example.com/r.png"],
)
assert result["success"] is False
assert result["error_type"] == "conflicting_inputs"
assert "client" not in captured or not captured["client"].posts
def test_too_many_references_rejects(self, xai_provider):
provider, captured = xai_provider
result = provider.generate(
"x",
reference_image_urls=[f"https://example.com/r{i}.png" for i in range(8)],
)
assert result["success"] is False
assert result["error_type"] == "too_many_references"
class TestXAIClamping:
def test_duration_clamped_to_15(self, xai_provider):
provider, captured = xai_provider
provider.generate("x", duration=30)
assert _last_post(captured)["json"]["duration"] == 15
def test_duration_clamped_when_refs_present(self, xai_provider):
provider, captured = xai_provider
provider.generate(
"x",
duration=15,
reference_image_urls=["https://example.com/r.png"],
)
# refs present caps to 10
assert _last_post(captured)["json"]["duration"] == 10
def test_invalid_aspect_ratio_soft_clamps(self, xai_provider):
provider, captured = xai_provider
provider.generate("x", aspect_ratio="21:9")
assert _last_post(captured)["json"]["aspect_ratio"] == "16:9"
View File
@@ -0,0 +1,475 @@
"""Plugin-side tests for the web search provider migration (PR #25182).
Covers:
- All seven bundled plugins (brave-free, ddgs, searxng, exa, parallel,
tavily, firecrawl) instantiate and self-report the expected
capabilities + ABC-derived defaults.
- Each plugin's ``is_available()`` correctly reflects env-var presence.
- The web_search_registry resolves an active provider in the documented
scenarios (explicit config wins ignoring availability, fallback walks
legacy preference filtered by availability, unknown name falls back).
- Plugin response shapes match the legacy bit-for-bit contract.
Per the dev skill: these tests use *real* imports from the plugin
modules no mocking of provider classes themselves so the test
catches drift in the ABC interface, the registry, and the plugin
glue layer simultaneously.
"""
from __future__ import annotations
import asyncio
import inspect
import os
import sys
from typing import Any, Dict, List
import pytest
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Strip every web-provider env var so is_available() returns False."""
for k in (
"BRAVE_SEARCH_API_KEY",
"SEARXNG_URL",
"TAVILY_API_KEY",
"TAVILY_BASE_URL",
"EXA_API_KEY",
"PARALLEL_API_KEY",
"PARALLEL_SEARCH_MODE",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
"FIRECRAWL_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_USER_TOKEN",
):
monkeypatch.delenv(k, raising=False)
def _ensure_plugins_loaded() -> None:
"""Idempotently load plugins so the registry is populated."""
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered()
# ---------------------------------------------------------------------------
# Per-plugin discovery + capability flags
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""Each test starts with a clean web-provider env."""
_clear_web_env(monkeypatch)
class TestBundledPluginsRegister:
"""All seven bundled web plugins discover and register correctly."""
def test_all_seven_plugins_present_in_registry(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import list_providers
names = sorted(p.name for p in list_providers())
assert names == [
"brave-free",
"ddgs",
"exa",
"firecrawl",
"parallel",
"searxng",
"tavily",
]
@pytest.mark.parametrize(
"plugin_name,expected_search,expected_extract,expected_crawl",
[
("brave-free", True, False, False),
("ddgs", True, False, False),
("searxng", True, False, False),
("exa", True, True, False),
("parallel", True, True, False),
("tavily", True, True, True),
# firecrawl: search + extract + crawl. Crawl was originally
# disabled in the migration (fell through to a legacy inline
# path); the follow-up commit enabled it natively.
("firecrawl", True, True, True),
],
)
def test_capability_flags_match_spec(
self,
plugin_name: str,
expected_search: bool,
expected_extract: bool,
expected_crawl: bool,
) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None, f"plugin {plugin_name!r} not registered"
assert provider.supports_search() is expected_search
assert provider.supports_extract() is expected_extract
assert provider.supports_crawl() is expected_crawl
@pytest.mark.parametrize(
"plugin_name",
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl"],
)
def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
assert provider.name == plugin_name
assert provider.display_name # any non-empty string
@pytest.mark.parametrize(
"plugin_name",
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl"],
)
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
"""``get_setup_schema()`` returns a dict the picker can consume."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
provider = get_provider(plugin_name)
assert provider is not None
schema = provider.get_setup_schema()
assert isinstance(schema, dict)
assert "name" in schema
assert "env_vars" in schema
# ---------------------------------------------------------------------------
# is_available() behavior
# ---------------------------------------------------------------------------
class TestIsAvailable:
"""Each plugin's ``is_available()`` returns False without env config."""
def test_brave_free_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("brave-free")
assert p is not None
assert p.is_available() is False # no BRAVE_SEARCH_API_KEY
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "real")
assert p.is_available() is True
def test_searxng_requires_url(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("searxng")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080")
assert p.is_available() is True
def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("TAVILY_API_KEY", "real")
assert p.is_available() is True
def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("EXA_API_KEY", "real")
assert p.is_available() is True
def test_parallel_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("parallel")
assert p is not None
assert p.is_available() is False
monkeypatch.setenv("PARALLEL_API_KEY", "real")
assert p.is_available() is True
def test_firecrawl_requires_either_key_or_url(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert p.is_available() is False
# Either FIRECRAWL_API_KEY or FIRECRAWL_API_URL lights it up.
monkeypatch.setenv("FIRECRAWL_API_KEY", "real")
assert p.is_available() is True
monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
monkeypatch.setenv("FIRECRAWL_API_URL", "http://localhost:3002")
assert p.is_available() is True
def test_ddgs_always_available_when_package_importable(self) -> None:
"""DDGS is the always-on fallback — no API key required.
It may report unavailable if the ``ddgs`` package itself isn't
installed in the env (legitimate the plugin's post_setup hook
triggers pip install on first selection). We only assert that
is_available() doesn't raise.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("ddgs")
assert p is not None
# Truthy or falsy, just must not raise.
_ = bool(p.is_available())
# ---------------------------------------------------------------------------
# Registry resolution semantics (Option B — conservative smart fallback)
# ---------------------------------------------------------------------------
class TestRegistryResolution:
"""``_resolve()`` follows explicit-config + availability-filtered fallback."""
def test_explicit_configured_provider_returned_even_when_unavailable(
self,
) -> None:
"""Explicit ``web.search_backend`` wins regardless of is_available().
Without availability filtering on the explicit path, the dispatcher
would silently switch backends; with this check the dispatcher
surfaces a precise "FOO_API_KEY is not set" error instead.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve, get_provider
# No BRAVE_SEARCH_API_KEY (fixture cleared it).
result = _resolve("brave-free", capability="search")
assert result is not None
assert result.name == "brave-free"
# Confirm it's the unavailable one — dispatcher will surface
# a typed credential-missing error to the caller.
assert result.is_available() is False
def test_unknown_configured_name_falls_back_to_available_provider(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Typo / uninstalled plugin → walk legacy preference, pick available."""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
monkeypatch.setenv("EXA_API_KEY", "real")
result = _resolve("not-a-real-provider", capability="search")
# Either ddgs (no-key fallback) or exa (the only available
# premium provider) — both are valid. The point is the unknown
# name shouldn't return None when SOMETHING is available.
assert result is not None
assert result.is_available() is True
def test_explicit_search_only_provider_for_extract_falls_back(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Asking for extract via a search-only backend → fall back.
``brave-free`` is search-only (``supports_extract() is False``).
When the registry resolves it for an extract capability, the
explicit-config branch rejects it as capability-incompatible
and the fallback walk picks an extract-capable provider.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
monkeypatch.setenv("EXA_API_KEY", "real")
result = _resolve("brave-free", capability="extract")
# Should land on exa (only extract-capable available provider).
assert result is not None
assert result.supports_extract() is True
assert result.is_available() is True
def test_no_config_no_credentials_returns_none(
self,
) -> None:
"""No backend configured AND no available providers → typically None.
``ddgs`` is the no-credential fallback; if its ``ddgs`` Python
package is installed in the test env, ddgs will be picked.
Otherwise the resolver returns None. Either outcome is correct.
"""
_ensure_plugins_loaded()
from agent.web_search_registry import _resolve
result = _resolve(None, capability="search")
if result is not None:
# The only no-credential provider is ddgs; anything else
# means an env var leaked in.
assert result.is_available() is True
# ---------------------------------------------------------------------------
# Sync-vs-async extract detection
# ---------------------------------------------------------------------------
class TestAsyncExtractDispatch:
"""The dispatcher detects async vs sync extract methods correctly."""
def test_parallel_extract_is_async(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("parallel")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is True
def test_firecrawl_extract_is_async(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is True
def test_exa_extract_is_sync(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is False
def test_tavily_extract_is_sync(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
assert inspect.iscoroutinefunction(p.extract) is False
# ---------------------------------------------------------------------------
# Error response shape (preserved bit-for-bit from legacy)
# ---------------------------------------------------------------------------
class TestErrorResponseShapes:
"""When credentials are missing, plugins return typed errors, not raises."""
def test_brave_free_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("brave-free")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_searxng_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("searxng")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_exa_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("exa")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_tavily_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
result = p.search("test", limit=5)
assert isinstance(result, dict)
assert result.get("success") is False
assert "error" in result
def test_parallel_extract_returns_per_url_errors_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("parallel")
assert p is not None
result = asyncio.run(p.extract(["https://example.com"]))
assert isinstance(result, list)
assert len(result) == 1
assert "error" in result[0]
assert result[0]["url"] == "https://example.com"
def test_firecrawl_extract_returns_per_url_errors_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
# firecrawl extract returns [] when the website-policy gate rejects
# the URL, or a per-URL error dict when the gate passes but the
# firecrawl client fails. Use a URL the policy allows to make sure
# we hit the credential-missing path.
result = asyncio.run(p.extract(["https://example.com"]))
assert isinstance(result, list)
if result: # if anything came back, it should be an error entry
assert "error" in result[0]
def test_tavily_crawl_returns_error_dict_when_unconfigured(self) -> None:
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("tavily")
assert p is not None
result = p.crawl("https://example.com")
assert isinstance(result, dict)
assert "results" in result
assert isinstance(result["results"], list)
if result["results"]:
assert "error" in result["results"][0]
def test_firecrawl_crawl_returns_error_dict_when_unconfigured(self) -> None:
"""firecrawl crawl is async (wraps SDK in to_thread); error must be
surfaced via the per-page result shape, not raised."""
_ensure_plugins_loaded()
from agent.web_search_registry import get_provider
p = get_provider("firecrawl")
assert p is not None
assert inspect.iscoroutinefunction(p.crawl)
result = asyncio.run(p.crawl("https://example.com"))
assert isinstance(result, dict)
assert "results" in result
assert isinstance(result["results"], list)
# Without FIRECRAWL_API_KEY, the plugin's _get_firecrawl_client()
# raises ValueError which is caught and returned as a per-page error.
assert len(result["results"]) >= 1
assert "error" in result["results"][0]
assert result["results"][0]["url"] == "https://example.com"
+3 -3
View File
@@ -46,14 +46,14 @@ def test_bundled_plugins_discovered():
assert (child / "plugin.yaml").exists(), f"{child.name} missing plugin.yaml"
def test_all_33_profiles_register():
"""After discovery, the registry must contain exactly 33 distinct profiles."""
def test_all_34_profiles_register():
"""After discovery, the registry must contain exactly 34 distinct profiles."""
_clear_provider_caches()
from providers import list_providers
profiles = list_providers()
names = sorted(p.name for p in profiles)
assert len(names) == 33, f"Expected 33 profiles, got {len(names)}: {names}"
assert len(names) == 34, f"Expected 34 profiles, got {len(names)}: {names}"
# Spot-check representative providers from different categories
for required in (
+26
View File
@@ -415,6 +415,32 @@ class TestHTTP413Compression:
class TestPreflightCompression:
"""Preflight compression should compress history before the first API call."""
def test_compress_context_emits_lifecycle_status_before_work(self, agent):
"""Direct context compression should tell gateway users why the turn paused."""
events = []
agent.status_callback = lambda ev, msg: events.append((ev, msg))
def _fake_compress(messages, current_tokens=None, focus_topic=None):
events.append(("compress", "started"))
return [{"role": "user", "content": f"{SUMMARY_PREFIX}\nPrevious conversation"}]
with (
patch.object(agent.context_compressor, "compress", side_effect=_fake_compress),
patch.object(agent, "_build_system_prompt", return_value="new system prompt"),
patch("run_agent.estimate_request_tokens_rough", return_value=42),
):
compressed, new_system_prompt = agent._compress_context(
[{"role": "user", "content": "hello"}],
"system prompt",
approx_tokens=1234,
)
assert compressed == [{"role": "user", "content": f"{SUMMARY_PREFIX}\nPrevious conversation"}]
assert new_system_prompt == "new system prompt"
assert events[0][0] == "lifecycle"
assert "Compacting context" in events[0][1]
assert events[1] == ("compress", "started")
def test_preflight_compresses_oversized_history(self, agent):
"""When loaded history exceeds the model's context threshold, compress before API call."""
agent.compression_enabled = True
-505
View File
@@ -1,505 +0,0 @@
"""
Tests for environments/agent_loop.py HermesAgentLoop.
Tests the multi-turn agent engine using mocked servers, without needing
real API keys or running servers.
"""
import asyncio
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional
from unittest.mock import MagicMock
import pytest
# Ensure repo root is importable
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
try:
from environments.agent_loop import (
AgentResult,
HermesAgentLoop,
ToolError,
_extract_reasoning_from_message,
resize_tool_pool,
)
except ImportError:
pytest.skip("atroposlib not installed", allow_module_level=True)
# ─── Mock server infrastructure ─────────────────────────────────────────
@dataclass
class MockFunction:
name: str
arguments: str
@dataclass
class MockToolCall:
id: str
function: MockFunction
type: str = "function"
@dataclass
class MockMessage:
content: Optional[str]
role: str = "assistant"
tool_calls: Optional[List[MockToolCall]] = None
reasoning_content: Optional[str] = None
reasoning: Optional[str] = None
reasoning_details: Optional[list] = None
@dataclass
class MockChoice:
message: MockMessage
finish_reason: str = "stop"
index: int = 0
@dataclass
class MockChatCompletion:
choices: List[MockChoice]
id: str = "chatcmpl-mock"
model: str = "mock-model"
class MockServer:
"""
Mock server that returns pre-configured responses in sequence.
Mimics the chat_completion() interface.
"""
def __init__(self, responses: List[MockChatCompletion]):
self.responses = responses
self.call_count = 0
self.call_history: List[Dict[str, Any]] = []
async def chat_completion(self, **kwargs) -> MockChatCompletion:
self.call_history.append(kwargs)
if self.call_count >= len(self.responses):
# Return a simple text response if we run out
return MockChatCompletion(
choices=[MockChoice(message=MockMessage(content="Done."))]
)
resp = self.responses[self.call_count]
self.call_count += 1
return resp
def make_text_response(content: str) -> MockChatCompletion:
"""Create a simple text-only response (no tool calls)."""
return MockChatCompletion(
choices=[MockChoice(message=MockMessage(content=content))]
)
def make_tool_response(
tool_name: str,
arguments: dict,
content: str = "",
tool_call_id: str = "call_001",
) -> MockChatCompletion:
"""Create a response with a single tool call."""
return MockChatCompletion(
choices=[
MockChoice(
message=MockMessage(
content=content,
tool_calls=[
MockToolCall(
id=tool_call_id,
function=MockFunction(
name=tool_name,
arguments=json.dumps(arguments),
),
)
],
),
finish_reason="tool_calls",
)
]
)
# ─── Tests ───────────────────────────────────────────────────────────────
class TestAgentResult:
def test_defaults(self):
result = AgentResult(messages=[])
assert result.messages == []
assert result.managed_state is None
assert result.turns_used == 0
assert result.finished_naturally is False
assert result.reasoning_per_turn == []
assert result.tool_errors == []
class TestExtractReasoning:
def test_reasoning_content_field(self):
msg = MockMessage(content="hello", reasoning_content="I think...")
assert _extract_reasoning_from_message(msg) == "I think..."
def test_reasoning_field(self):
msg = MockMessage(content="hello", reasoning="Let me consider...")
assert _extract_reasoning_from_message(msg) == "Let me consider..."
def test_reasoning_details(self):
detail = MagicMock()
detail.text = "Detail reasoning"
msg = MockMessage(content="hello", reasoning_details=[detail])
assert _extract_reasoning_from_message(msg) == "Detail reasoning"
def test_reasoning_details_dict_format(self):
msg = MockMessage(
content="hello",
reasoning_details=[{"text": "Dict reasoning"}],
)
assert _extract_reasoning_from_message(msg) == "Dict reasoning"
def test_no_reasoning(self):
msg = MockMessage(content="hello")
assert _extract_reasoning_from_message(msg) is None
def test_reasoning_content_takes_priority(self):
msg = MockMessage(
content="hello",
reasoning_content="First",
reasoning="Second",
)
assert _extract_reasoning_from_message(msg) == "First"
class TestHermesAgentLoop:
"""Test the agent loop with mock servers."""
@pytest.fixture
def basic_tools(self):
"""Minimal tool schema for testing."""
return [
{
"type": "function",
"function": {
"name": "terminal",
"description": "Run a command",
"parameters": {
"type": "object",
"properties": {
"command": {
"type": "string",
"description": "Command to run",
}
},
"required": ["command"],
},
},
},
{
"type": "function",
"function": {
"name": "read_file",
"description": "Read a file",
"parameters": {
"type": "object",
"properties": {
"path": {"type": "string"},
},
"required": ["path"],
},
},
},
]
@pytest.fixture
def valid_names(self):
return {"terminal", "read_file", "todo"}
@pytest.mark.asyncio
async def test_simple_text_response(self, basic_tools, valid_names):
"""Model responds with text only, no tool calls."""
server = MockServer([make_text_response("Hello! How can I help?")])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "Hi"}]
result = await agent.run(messages)
assert result.finished_naturally is True
assert result.turns_used == 1
assert len(result.messages) >= 2 # user + assistant
assert result.messages[-1]["role"] == "assistant"
assert result.messages[-1]["content"] == "Hello! How can I help?"
@pytest.mark.asyncio
async def test_tool_call_then_text(self, basic_tools, valid_names):
"""Model calls a tool, then responds with text."""
server = MockServer([
make_tool_response("todo", {"todos": [{"id": "1", "content": "test", "status": "pending"}]}),
make_text_response("I created a todo for you."),
])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "Create a todo"}]
result = await agent.run(messages)
assert result.finished_naturally is True
assert result.turns_used == 2
# Should have: user, assistant (tool_call), tool (result), assistant (text)
roles = [m["role"] for m in result.messages]
assert roles == ["user", "assistant", "tool", "assistant"]
@pytest.mark.asyncio
async def test_max_turns_reached(self, basic_tools, valid_names):
"""Model keeps calling tools until max_turns is hit."""
# Create responses that always call a tool
responses = [
make_tool_response("todo", {"todos": [{"id": str(i), "content": f"task {i}", "status": "pending"}]}, tool_call_id=f"call_{i}")
for i in range(10)
]
server = MockServer(responses)
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=3,
)
messages = [{"role": "user", "content": "Keep going"}]
result = await agent.run(messages)
assert result.finished_naturally is False
assert result.turns_used == 3
@pytest.mark.asyncio
async def test_unknown_tool_name(self, basic_tools, valid_names):
"""Model calls a tool not in valid_tool_names."""
server = MockServer([
make_tool_response("nonexistent_tool", {"arg": "val"}),
make_text_response("OK, that didn't work."),
])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "Call something weird"}]
result = await agent.run(messages)
# Should record a tool error
assert len(result.tool_errors) >= 1
assert result.tool_errors[0].tool_name == "nonexistent_tool"
@pytest.mark.asyncio
async def test_empty_response(self, basic_tools, valid_names):
"""Server returns empty response."""
server = MockServer([MockChatCompletion(choices=[])])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "Hi"}]
result = await agent.run(messages)
assert result.finished_naturally is False
assert result.turns_used == 1
@pytest.mark.asyncio
async def test_api_error_handling(self, basic_tools, valid_names):
"""Server raises an exception."""
class FailingServer:
async def chat_completion(self, **kwargs):
raise ConnectionError("Server unreachable")
agent = HermesAgentLoop(
server=FailingServer(),
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "Hi"}]
result = await agent.run(messages)
assert result.finished_naturally is False
assert result.turns_used == 1
@pytest.mark.asyncio
async def test_tools_passed_to_server(self, basic_tools, valid_names):
"""Verify tools are passed in the chat_completion kwargs."""
server = MockServer([make_text_response("OK")])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "Hi"}]
await agent.run(messages)
assert len(server.call_history) == 1
assert "tools" in server.call_history[0]
assert server.call_history[0]["tools"] == basic_tools
@pytest.mark.asyncio
async def test_extra_body_forwarded(self, basic_tools, valid_names):
"""extra_body should be forwarded to server."""
extra = {"provider": {"ignore": ["DeepInfra"]}}
server = MockServer([make_text_response("OK")])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
extra_body=extra,
)
messages = [{"role": "user", "content": "Hi"}]
await agent.run(messages)
assert server.call_history[0].get("extra_body") == extra
@pytest.mark.asyncio
async def test_managed_state_returned(self, basic_tools, valid_names):
"""If server has get_state(), result should include managed_state."""
server = MockServer([make_text_response("OK")])
server.get_state = lambda: {"nodes": [{"test": True}]}
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "Hi"}]
result = await agent.run(messages)
assert result.managed_state is not None
assert "nodes" in result.managed_state
@pytest.mark.asyncio
async def test_no_managed_state_without_get_state(self, basic_tools, valid_names):
"""Regular server without get_state() should return None managed_state."""
server = MockServer([make_text_response("OK")])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "Hi"}]
result = await agent.run(messages)
assert result.managed_state is None
@pytest.mark.asyncio
async def test_memory_tool_blocked(self, basic_tools):
"""Memory tool should return error in RL environments."""
valid = {"terminal", "read_file", "todo", "memory"}
server = MockServer([
make_tool_response("memory", {"action": "add", "target": "user", "content": "test"}),
make_text_response("Done"),
])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid,
max_turns=10,
)
messages = [{"role": "user", "content": "Remember this"}]
result = await agent.run(messages)
# Find the tool response
tool_msgs = [m for m in result.messages if m["role"] == "tool"]
assert len(tool_msgs) >= 1
tool_result = json.loads(tool_msgs[0]["content"])
assert "error" in tool_result
assert "not available" in tool_result["error"].lower()
@pytest.mark.asyncio
async def test_session_search_blocked(self, basic_tools):
"""session_search should return error in RL environments."""
valid = {"terminal", "read_file", "todo", "session_search"}
server = MockServer([
make_tool_response("session_search", {"query": "test"}),
make_text_response("Done"),
])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid,
max_turns=10,
)
messages = [{"role": "user", "content": "Search sessions"}]
result = await agent.run(messages)
tool_msgs = [m for m in result.messages if m["role"] == "tool"]
assert len(tool_msgs) >= 1
tool_result = json.loads(tool_msgs[0]["content"])
assert "error" in tool_result
@pytest.mark.asyncio
async def test_reasoning_content_preserved(self, basic_tools, valid_names):
"""Reasoning content should be extracted and preserved."""
resp = MockChatCompletion(
choices=[
MockChoice(
message=MockMessage(
content="The answer is 42.",
reasoning_content="Let me think about this step by step...",
)
)
]
)
server = MockServer([resp])
agent = HermesAgentLoop(
server=server,
tool_schemas=basic_tools,
valid_tool_names=valid_names,
max_turns=10,
)
messages = [{"role": "user", "content": "What is the meaning of life?"}]
result = await agent.run(messages)
assert len(result.reasoning_per_turn) == 1
assert result.reasoning_per_turn[0] == "Let me think about this step by step..."
class TestResizeToolPool:
def test_resize_works(self):
"""resize_tool_pool should not raise."""
resize_tool_pool(16) # Small pool for testing
resize_tool_pool(128) # Restore default
def test_resize_shuts_down_previous_executor(self, monkeypatch):
"""Replacing the global tool executor should shut down the old pool."""
import environments.agent_loop as agent_loop_module
old_executor = MagicMock()
new_executor = MagicMock()
monkeypatch.setattr(agent_loop_module, "_tool_executor", old_executor)
monkeypatch.setattr(
agent_loop_module.concurrent.futures,
"ThreadPoolExecutor",
MagicMock(return_value=new_executor),
)
resize_tool_pool(16)
old_executor.shutdown.assert_called_once_with(wait=False)
assert agent_loop_module._tool_executor is new_executor
@@ -1,552 +0,0 @@
"""Integration tests for HermesAgentLoop tool calling.
Tests the full agent loop with real LLM calls via OpenRouter.
Uses stepfun/step-3.5-flash:free by default (zero cost), falls back
to anthropic/claude-sonnet-4 if the free model is unavailable.
These tests verify:
1. Single tool call: model calls a tool, gets result, responds
2. Multi-tool call: model calls multiple tools in one turn
3. Multi-turn: model calls tools across multiple turns
4. Unknown tool rejection: model calling a non-existent tool gets an error
5. Max turns: loop stops when max_turns is reached
6. No tools: model responds without calling any tools
7. Tool error handling: tool execution errors are captured
Run:
pytest tests/test_agent_loop_tool_calling.py -v
pytest tests/test_agent_loop_tool_calling.py -v -k "single" # run one test
"""
import asyncio
import json
import os
import sys
from pathlib import Path
from typing import Any, Dict, List, Set
from unittest.mock import patch
import pytest
# pytestmark removed — tests skip gracefully via OPENROUTER_API_KEY check on line 59
# Ensure repo root is importable
_repo_root = Path(__file__).resolve().parent.parent.parent
if str(_repo_root) not in sys.path:
sys.path.insert(0, str(_repo_root))
try:
from environments.agent_loop import AgentResult, HermesAgentLoop
from atroposlib.envs.server_handling.openai_server import OpenAIServer # noqa: F401
except ImportError:
pytest.skip("atroposlib not installed", allow_module_level=True)
# =========================================================================
# Test infrastructure
# =========================================================================
# Models to try, in order of preference (free first)
_MODELS = [
"stepfun/step-3.5-flash:free",
"google/gemini-2.0-flash-001",
"anthropic/claude-sonnet-4",
]
def _get_api_key():
key = os.getenv("OPENROUTER_API_KEY", "")
if not key:
pytest.skip("OPENROUTER_API_KEY not set")
return key
def _make_server(model: str = None):
"""Create an OpenAI server for testing."""
from atroposlib.envs.server_handling.openai_server import OpenAIServer
from atroposlib.envs.server_handling.server_manager import APIServerConfig
config = APIServerConfig(
base_url="https://openrouter.ai/api/v1",
model_name=model or _MODELS[0],
server_type="openai",
api_key=_get_api_key(),
health_check=False,
)
return OpenAIServer(config)
async def _try_models(test_fn):
"""Try running a test with each model until one works."""
last_error = None
for model in _MODELS:
try:
server = _make_server(model)
return await test_fn(server, model)
except Exception as e:
last_error = e
if "rate" in str(e).lower() or "limit" in str(e).lower():
continue # Rate limited, try next model
raise # Real error
pytest.skip(f"All models failed. Last error: {last_error}")
# =========================================================================
# Fake tools for testing
# =========================================================================
# Simple calculator tool
CALC_TOOL = {
"type": "function",
"function": {
"name": "calculate",
"description": "Calculate a math expression. Returns the numeric result.",
"parameters": {
"type": "object",
"properties": {
"expression": {
"type": "string",
"description": "Math expression to evaluate, e.g. '2 + 3'"
}
},
"required": ["expression"],
},
},
}
# Weather lookup tool
WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a city. Returns temperature and conditions.",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string",
"description": "City name, e.g. 'Tokyo'"
}
},
"required": ["city"],
},
},
}
# Lookup tool (always succeeds)
LOOKUP_TOOL = {
"type": "function",
"function": {
"name": "lookup",
"description": "Look up a fact. Returns a short answer string.",
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "What to look up"
}
},
"required": ["query"],
},
},
}
# Error tool (always fails)
ERROR_TOOL = {
"type": "function",
"function": {
"name": "failing_tool",
"description": "A tool that always fails with an error.",
"parameters": {
"type": "object",
"properties": {
"input": {"type": "string"}
},
"required": ["input"],
},
},
}
def _fake_tool_handler(tool_name: str, args: Dict[str, Any], **kwargs) -> str:
"""Handle fake tool calls for testing."""
if tool_name == "calculate":
expr = args.get("expression", "0")
try:
# Safe eval for simple math
result = eval(expr, {"__builtins__": {}}, {})
return json.dumps({"result": result})
except Exception as e:
return json.dumps({"error": str(e)})
elif tool_name == "get_weather":
city = args.get("city", "Unknown")
# Return canned weather
return json.dumps({
"city": city,
"temperature": 22,
"conditions": "sunny",
"humidity": 45,
})
elif tool_name == "lookup":
query = args.get("query", "")
return json.dumps({"answer": f"The answer to '{query}' is 42."})
elif tool_name == "failing_tool":
raise RuntimeError("This tool always fails!")
return json.dumps({"error": f"Unknown tool: {tool_name}"})
# =========================================================================
# Tests
# =========================================================================
@pytest.mark.asyncio
async def test_single_tool_call():
"""Model should call a single tool, get the result, and respond."""
async def _run(server, model):
agent = HermesAgentLoop(
server=server,
tool_schemas=[WEATHER_TOOL],
valid_tool_names={"get_weather"},
max_turns=5,
temperature=0.0,
max_tokens=500,
)
messages = [
{"role": "user", "content": "What's the weather in Tokyo? Use the get_weather tool."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
assert isinstance(result, AgentResult)
assert result.turns_used >= 2, f"Expected at least 2 turns (tool call + response), got {result.turns_used}"
# Verify a tool call happened
tool_calls_found = False
for msg in result.messages:
if msg.get("role") == "assistant" and msg.get("tool_calls"):
for tc in msg["tool_calls"]:
if tc["function"]["name"] == "get_weather":
tool_calls_found = True
args = json.loads(tc["function"]["arguments"])
assert "city" in args
assert tool_calls_found, "Model should have called get_weather"
# Verify tool result is in conversation
tool_results = [m for m in result.messages if m.get("role") == "tool"]
assert len(tool_results) >= 1, "Should have at least one tool result"
# Verify the final response references the weather
final_msg = result.messages[-1]
assert final_msg["role"] == "assistant"
assert final_msg["content"], "Final response should have content"
return result
await _try_models(_run)
@pytest.mark.asyncio
async def test_multi_tool_single_turn():
"""Model should call multiple tools in a single turn."""
async def _run(server, model):
agent = HermesAgentLoop(
server=server,
tool_schemas=[WEATHER_TOOL, CALC_TOOL],
valid_tool_names={"get_weather", "calculate"},
max_turns=5,
temperature=0.0,
max_tokens=500,
)
messages = [
{"role": "user", "content": (
"I need two things at once: "
"1) What's the weather in Paris? Use get_weather. "
"2) What is 15 * 7? Use calculate. "
"Call BOTH tools in a single response."
)},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# Count distinct tools called
tools_called = set()
for msg in result.messages:
if msg.get("role") == "assistant" and msg.get("tool_calls"):
for tc in msg["tool_calls"]:
tools_called.add(tc["function"]["name"])
# At minimum, both tools should have been called (maybe in different turns)
assert "get_weather" in tools_called, f"get_weather not called. Called: {tools_called}"
assert "calculate" in tools_called, f"calculate not called. Called: {tools_called}"
return result
await _try_models(_run)
@pytest.mark.asyncio
async def test_multi_turn_conversation():
"""Agent should handle multiple turns of tool calls."""
async def _run(server, model):
agent = HermesAgentLoop(
server=server,
tool_schemas=[LOOKUP_TOOL, CALC_TOOL],
valid_tool_names={"lookup", "calculate"},
max_turns=10,
temperature=0.0,
max_tokens=500,
)
messages = [
{"role": "user", "content": (
"First, use the lookup tool to look up 'meaning of life'. "
"Then use calculate to compute 6 * 7. "
"Do these in separate tool calls, one at a time."
)},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# Should have used both tools
tools_called = set()
for msg in result.messages:
if msg.get("role") == "assistant" and msg.get("tool_calls"):
for tc in msg["tool_calls"]:
tools_called.add(tc["function"]["name"])
assert "lookup" in tools_called, f"lookup not called. Called: {tools_called}"
assert "calculate" in tools_called, f"calculate not called. Called: {tools_called}"
# Should finish naturally
assert result.finished_naturally, "Should finish naturally after answering"
return result
await _try_models(_run)
@pytest.mark.asyncio
async def test_unknown_tool_rejected():
"""If the model calls a tool not in valid_tool_names, it gets an error."""
async def _run(server, model):
# Only allow "calculate" but give schema for both
agent = HermesAgentLoop(
server=server,
tool_schemas=[CALC_TOOL, WEATHER_TOOL],
valid_tool_names={"calculate"}, # weather NOT allowed
max_turns=5,
temperature=0.0,
max_tokens=500,
)
messages = [
{"role": "user", "content": "What's the weather in London? Use get_weather."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# Check if get_weather was called and rejected
if result.tool_errors:
weather_errors = [e for e in result.tool_errors if e.tool_name == "get_weather"]
assert len(weather_errors) > 0, "get_weather should have been rejected"
assert "Unknown tool" in weather_errors[0].error
return result
await _try_models(_run)
@pytest.mark.asyncio
async def test_max_turns_limit():
"""Agent should stop after max_turns even if model keeps calling tools."""
async def _run(server, model):
agent = HermesAgentLoop(
server=server,
tool_schemas=[LOOKUP_TOOL],
valid_tool_names={"lookup"},
max_turns=2, # Very low limit
temperature=0.0,
max_tokens=500,
)
messages = [
{"role": "user", "content": (
"Keep looking up facts. Look up 'fact 1', then 'fact 2', "
"then 'fact 3', then 'fact 4'. Do them one at a time."
)},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
assert result.turns_used <= 2, f"Should stop at max_turns=2, used {result.turns_used}"
assert not result.finished_naturally, "Should NOT finish naturally (hit max_turns)"
return result
await _try_models(_run)
@pytest.mark.asyncio
async def test_no_tools_direct_response():
"""When no tools are useful, model should respond directly."""
async def _run(server, model):
agent = HermesAgentLoop(
server=server,
tool_schemas=[WEATHER_TOOL],
valid_tool_names={"get_weather"},
max_turns=5,
temperature=0.0,
max_tokens=200,
)
messages = [
{"role": "user", "content": "What is 2 + 2? Just answer directly, no tools needed."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
assert result.finished_naturally, "Should finish naturally with a direct response"
assert result.turns_used == 1, f"Should take exactly 1 turn for a direct answer, took {result.turns_used}"
final = result.messages[-1]
assert final["role"] == "assistant"
assert final["content"], "Should have text content"
assert "4" in final["content"], "Should contain the answer '4'"
return result
await _try_models(_run)
@pytest.mark.asyncio
async def test_tool_error_handling():
"""Tool execution errors should be captured and reported to the model."""
async def _run(server, model):
agent = HermesAgentLoop(
server=server,
tool_schemas=[ERROR_TOOL],
valid_tool_names={"failing_tool"},
max_turns=5,
temperature=0.0,
max_tokens=500,
)
messages = [
{"role": "user", "content": "Please call the failing_tool with input 'test'."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# The tool error should be recorded
assert len(result.tool_errors) >= 1, "Should have at least one tool error"
assert "RuntimeError" in result.tool_errors[0].error or "always fails" in result.tool_errors[0].error
# The error should be in the conversation as a tool result
tool_results = [m for m in result.messages if m.get("role") == "tool"]
assert len(tool_results) >= 1
error_result = json.loads(tool_results[0]["content"])
assert "error" in error_result
return result
await _try_models(_run)
@pytest.mark.asyncio
async def test_agent_result_structure():
"""Verify the AgentResult has all expected fields populated."""
async def _run(server, model):
agent = HermesAgentLoop(
server=server,
tool_schemas=[CALC_TOOL],
valid_tool_names={"calculate"},
max_turns=5,
temperature=0.0,
max_tokens=300,
)
messages = [
{"role": "user", "content": "What is 3 + 4? Use the calculate tool."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# Structural checks
assert isinstance(result, AgentResult)
assert isinstance(result.messages, list)
assert len(result.messages) >= 3, "Should have user + assistant(tool) + tool_result + assistant(final)"
assert isinstance(result.turns_used, int)
assert result.turns_used > 0
assert isinstance(result.finished_naturally, bool)
assert isinstance(result.tool_errors, list)
assert isinstance(result.reasoning_per_turn, list)
# Messages should follow OpenAI format
for msg in result.messages:
assert "role" in msg, f"Message missing 'role': {msg}"
assert msg["role"] in ("system", "user", "assistant", "tool"), f"Invalid role: {msg['role']}"
return result
await _try_models(_run)
@pytest.mark.asyncio
async def test_conversation_history_preserved():
"""The full conversation history should be in result.messages."""
async def _run(server, model):
agent = HermesAgentLoop(
server=server,
tool_schemas=[WEATHER_TOOL],
valid_tool_names={"get_weather"},
max_turns=5,
temperature=0.0,
max_tokens=500,
)
messages = [
{"role": "system", "content": "You are a helpful weather assistant."},
{"role": "user", "content": "What's the weather in Berlin? Use get_weather."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# System message should be preserved
assert result.messages[0]["role"] == "system"
assert "weather assistant" in result.messages[0]["content"]
# User message should be preserved
assert result.messages[1]["role"] == "user"
assert "Berlin" in result.messages[1]["content"]
# Should have assistant + tool + assistant sequence
roles = [m["role"] for m in result.messages]
assert "tool" in roles, "Should have tool results in conversation"
return result
await _try_models(_run)
-359
View File
@@ -1,359 +0,0 @@
"""Integration tests for HermesAgentLoop with a local vLLM server.
Tests the full Phase 2 flow: ManagedServer + tool calling with a real
vLLM backend, producing actual token IDs and logprobs for RL training.
Requires a running vLLM server. Start one from the atropos directory:
python -m example_trainer.vllm_api_server \
--model Qwen/Qwen3-4B-Thinking-2507 \
--port 9001 \
--gpu-memory-utilization 0.8 \
--max-model-len=32000
Tests are automatically skipped if the server is not reachable.
Run:
pytest tests/test_agent_loop_vllm.py -v
pytest tests/test_agent_loop_vllm.py -v -k "single"
"""
import asyncio
import json
import os
import sys
from pathlib import Path
from typing import Any, Dict
from unittest.mock import patch
import pytest
import requests
# Ensure repo root is importable
_repo_root = Path(__file__).resolve().parent.parent.parent
if str(_repo_root) not in sys.path:
sys.path.insert(0, str(_repo_root))
try:
from environments.agent_loop import AgentResult, HermesAgentLoop
except ImportError:
pytest.skip("atroposlib not installed", allow_module_level=True)
# =========================================================================
# Configuration
# =========================================================================
VLLM_HOST = "localhost"
VLLM_PORT = 9001
VLLM_BASE_URL = f"http://{VLLM_HOST}:{VLLM_PORT}"
VLLM_MODEL = "Qwen/Qwen3-4B-Thinking-2507"
def _vllm_is_running() -> bool:
"""Check if the vLLM server is reachable."""
try:
r = requests.get(f"{VLLM_BASE_URL}/health", timeout=3)
return r.status_code == 200
except Exception:
return False
# Skip all tests in this module if vLLM is not running
pytestmark = pytest.mark.skipif(
not _vllm_is_running(),
reason=(
f"vLLM server not reachable at {VLLM_BASE_URL}. "
"Start it with: python -m example_trainer.vllm_api_server "
f"--model {VLLM_MODEL} --port {VLLM_PORT} "
"--gpu-memory-utilization 0.8 --max-model-len=32000"
),
)
# =========================================================================
# Server setup
# =========================================================================
def _make_server_manager():
"""Create a ServerManager pointing to the local vLLM server."""
from atroposlib.envs.server_handling.server_manager import (
ServerManager,
APIServerConfig,
)
config = APIServerConfig(
base_url=VLLM_BASE_URL,
model_name=VLLM_MODEL,
server_type="vllm",
health_check=False,
)
sm = ServerManager([config], tool_parser="hermes")
sm.servers[0].server_healthy = True
return sm
def _get_tokenizer():
"""Load the tokenizer for the model."""
from transformers import AutoTokenizer
return AutoTokenizer.from_pretrained(VLLM_MODEL)
# =========================================================================
# Fake tools
# =========================================================================
WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a city. Returns temperature and conditions.",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string",
"description": "City name, e.g. 'Tokyo'",
}
},
"required": ["city"],
},
},
}
CALC_TOOL = {
"type": "function",
"function": {
"name": "calculate",
"description": "Calculate a math expression. Returns the numeric result.",
"parameters": {
"type": "object",
"properties": {
"expression": {
"type": "string",
"description": "Math expression, e.g. '2 + 3'",
}
},
"required": ["expression"],
},
},
}
def _fake_tool_handler(tool_name: str, args: Dict[str, Any], **kwargs) -> str:
"""Handle fake tool calls for testing."""
if tool_name == "get_weather":
city = args.get("city", "Unknown")
return json.dumps({
"city": city,
"temperature": 22,
"conditions": "sunny",
"humidity": 45,
})
elif tool_name == "calculate":
expr = args.get("expression", "0")
try:
result = eval(expr, {"__builtins__": {}}, {})
return json.dumps({"result": result})
except Exception as e:
return json.dumps({"error": str(e)})
return json.dumps({"error": f"Unknown tool: {tool_name}"})
# =========================================================================
# Tests
# =========================================================================
@pytest.mark.asyncio
async def test_vllm_single_tool_call():
"""vLLM model calls a tool, gets result, responds — full Phase 2 flow."""
sm = _make_server_manager()
tokenizer = _get_tokenizer()
async with sm.managed_server(tokenizer=tokenizer) as managed:
agent = HermesAgentLoop(
server=managed,
tool_schemas=[WEATHER_TOOL],
valid_tool_names={"get_weather"},
max_turns=5,
temperature=0.6,
max_tokens=1000,
)
messages = [
{"role": "user", "content": "What's the weather in Tokyo? Use the get_weather tool."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
assert isinstance(result, AgentResult)
assert result.turns_used >= 2, f"Expected at least 2 turns, got {result.turns_used}"
# Verify tool call happened
tool_calls_found = False
for msg in result.messages:
if msg.get("role") == "assistant" and msg.get("tool_calls"):
for tc in msg["tool_calls"]:
if tc["function"]["name"] == "get_weather":
tool_calls_found = True
args = json.loads(tc["function"]["arguments"])
assert "city" in args
assert tool_calls_found, "Model should have called get_weather"
# Verify tool results in conversation
tool_results = [m for m in result.messages if m.get("role") == "tool"]
assert len(tool_results) >= 1
@pytest.mark.asyncio
async def test_vllm_multi_tool_calls():
"""vLLM model calls multiple tools across turns."""
sm = _make_server_manager()
tokenizer = _get_tokenizer()
async with sm.managed_server(tokenizer=tokenizer) as managed:
agent = HermesAgentLoop(
server=managed,
tool_schemas=[WEATHER_TOOL, CALC_TOOL],
valid_tool_names={"get_weather", "calculate"},
max_turns=10,
temperature=0.6,
max_tokens=1000,
)
messages = [
{"role": "user", "content": (
"I need two things: "
"1) What's the weather in Paris? Use get_weather. "
"2) What is 15 * 7? Use calculate."
)},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# Both tools should be called
tools_called = set()
for msg in result.messages:
if msg.get("role") == "assistant" and msg.get("tool_calls"):
for tc in msg["tool_calls"]:
tools_called.add(tc["function"]["name"])
assert "get_weather" in tools_called, f"get_weather not called. Called: {tools_called}"
assert "calculate" in tools_called, f"calculate not called. Called: {tools_called}"
@pytest.mark.asyncio
async def test_vllm_managed_server_produces_nodes():
"""ManagedServer should produce SequenceNodes with tokens and logprobs."""
sm = _make_server_manager()
tokenizer = _get_tokenizer()
async with sm.managed_server(tokenizer=tokenizer) as managed:
agent = HermesAgentLoop(
server=managed,
tool_schemas=[WEATHER_TOOL],
valid_tool_names={"get_weather"},
max_turns=5,
temperature=0.6,
max_tokens=1000,
)
messages = [
{"role": "user", "content": "What's the weather in Berlin? Use get_weather."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# Get the managed state — should have SequenceNodes
state = managed.get_state()
assert state is not None, "ManagedServer should return state"
nodes = state.get("nodes", [])
assert len(nodes) >= 1, f"Should have at least 1 node, got {len(nodes)}"
node = nodes[0]
assert hasattr(node, "tokens"), "Node should have tokens"
assert hasattr(node, "logprobs"), "Node should have logprobs"
assert len(node.tokens) > 0, "Tokens should not be empty"
assert len(node.logprobs) > 0, "Logprobs should not be empty"
assert len(node.tokens) == len(node.logprobs), (
f"Tokens ({len(node.tokens)}) and logprobs ({len(node.logprobs)}) should have same length"
)
@pytest.mark.asyncio
async def test_vllm_no_tools_direct_response():
"""vLLM model should respond directly when no tools are needed."""
sm = _make_server_manager()
tokenizer = _get_tokenizer()
async with sm.managed_server(tokenizer=tokenizer) as managed:
agent = HermesAgentLoop(
server=managed,
tool_schemas=[WEATHER_TOOL],
valid_tool_names={"get_weather"},
max_turns=5,
temperature=0.6,
max_tokens=500,
)
messages = [
{"role": "user", "content": "What is 2 + 2? Answer directly, no tools."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
assert result.finished_naturally, "Should finish naturally"
assert result.turns_used == 1, f"Should take 1 turn, took {result.turns_used}"
final = result.messages[-1]
assert final["role"] == "assistant"
assert final["content"], "Should have content"
@pytest.mark.asyncio
async def test_vllm_thinking_content_extracted():
"""Qwen3-Thinking model should produce reasoning content."""
sm = _make_server_manager()
tokenizer = _get_tokenizer()
async with sm.managed_server(
tokenizer=tokenizer,
preserve_think_blocks=True,
) as managed:
agent = HermesAgentLoop(
server=managed,
tool_schemas=[CALC_TOOL],
valid_tool_names={"calculate"},
max_turns=5,
temperature=0.6,
max_tokens=1000,
)
messages = [
{"role": "user", "content": "What is 123 * 456? Use the calculate tool."},
]
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
result = await agent.run(messages)
# Qwen3-Thinking should generate <think> blocks
# Check if any content contains thinking markers
has_thinking = False
for msg in result.messages:
content = msg.get("content", "") or ""
if "<think>" in content or "</think>" in content:
has_thinking = True
break
# Also check reasoning_per_turn
has_reasoning = any(r for r in result.reasoning_per_turn if r)
# At least one of these should be true for a thinking model
assert has_thinking or has_reasoning, (
"Qwen3-Thinking should produce <think> blocks or reasoning content"
)
@@ -59,7 +59,7 @@ class TestTruncatedAnthropicResponseNormalization:
nr = get_transport("anthropic_messages").normalize_response(response)
# The continuation block checks these two attributes:
# assistant_message.content → appended to truncated_response_prefix
# assistant_message.content → appended to truncated_response_parts
# assistant_message.tool_calls → guards the text-retry branch
assert nr.content is not None
assert "partial response" in nr.content
@@ -20,6 +20,9 @@ def _bare_agent() -> AIAgent:
agent._memory_store = object()
agent._memory_enabled = True
agent._user_profile_enabled = False
agent._cached_system_prompt = "test-cached-system-prompt"
import datetime as _dt
agent.session_start = _dt.datetime(2026, 1, 1, 12, 0, 0)
agent._MEMORY_REVIEW_PROMPT = "review memory"
agent._SKILL_REVIEW_PROMPT = "review skills"
agent._COMBINED_REVIEW_PROMPT = "review both"
@@ -0,0 +1,185 @@
"""Tests that the background review fork inherits the parent's cached system prompt.
Regression coverage for issue #25322 (and PR #17276's first root cause): the
background review's outbound HTTP request must carry the same system bytes as
the parent's so Anthropic/OpenRouter's exact-prefix cache key matches.
Without this, every review rebuilds the system prompt from scratch fresh
``_hermes_now()`` timestamp, fresh ``session_id``, and a different skills
prompt under the (former) narrow toolset and the prefix-cache miss costs
roughly the full uncached system-prompt cost per nudge (~26% end-to-end on
Sonnet 4.5 per the contributor's measurement).
"""
from unittest.mock import patch
def _make_agent_stub(agent_cls):
"""Create a minimal AIAgent-like object with just enough state for _spawn_background_review."""
agent = object.__new__(agent_cls)
agent.model = "test-model"
agent.platform = "test"
agent.provider = "openai"
agent.session_id = "sess-123"
agent.quiet_mode = True
agent._memory_store = None
agent._memory_enabled = True
agent._user_profile_enabled = False
agent._memory_nudge_interval = 5
agent._skill_nudge_interval = 5
agent.background_review_callback = None
agent.status_callback = None
agent._cached_system_prompt = (
"PARENT-SYSTEM-PROMPT-BYTES — must be inherited verbatim "
"for prefix-cache parity"
)
import datetime as _dt
agent.session_start = _dt.datetime(2026, 1, 1, 12, 0, 0)
agent._MEMORY_REVIEW_PROMPT = "review memory"
agent._SKILL_REVIEW_PROMPT = "review skills"
agent._COMBINED_REVIEW_PROMPT = "review both"
return agent
class _SyncThread:
"""Drop-in replacement for threading.Thread that runs the target inline."""
def __init__(self, *, target=None, daemon=None, name=None):
self._target = target
def start(self):
if self._target:
self._target()
class _ReviewAgentRecorder:
"""Stand-in for the review-fork AIAgent that records the prompt assignment."""
def __init__(self, *args, **kwargs):
self._cached_system_prompt = None
self._memory_write_origin = None
self._memory_write_context = None
self._memory_store = None
self._memory_enabled = None
self._user_profile_enabled = None
self._memory_nudge_interval = None
self._skill_nudge_interval = None
self.suppress_status_output = None
def run_conversation(self, *args, **kwargs):
raise RuntimeError("stop after recording state — don't actually call the API")
def shutdown_memory_provider(self):
pass
def close(self):
pass
def test_review_fork_inherits_parent_cached_system_prompt():
"""The review fork's _cached_system_prompt must equal the parent's byte-for-byte.
Anthropic's prefix cache keys on exact bytes; any divergence (timestamp
minute tick, fresh session_id, narrower skills_prompt) shifts the key
and forces a full re-cache. Inheriting the parent's cached prompt is
the cheap, mechanical fix.
"""
import run_agent
agent = _make_agent_stub(run_agent.AIAgent)
captured = {}
parent_prompt = agent._cached_system_prompt
# Hook the assignment site: record what gets put on the review agent.
real_recorder_init = _ReviewAgentRecorder.__init__
def _recorder_init(self, *args, **kwargs):
real_recorder_init(self, *args, **kwargs)
# The actual production code assigns _cached_system_prompt AFTER __init__,
# so we need to capture it on attribute set. Use a property-style sentinel
# via __setattr__ on this instance.
with patch.object(run_agent, "AIAgent", _ReviewAgentRecorder), \
patch("threading.Thread", _SyncThread):
# Wrap the recorder's __setattr__ so we can see the _cached_system_prompt
# write that _spawn_background_review performs after construction.
orig_setattr = _ReviewAgentRecorder.__setattr__
def _spy_setattr(self, name, value):
if name == "_cached_system_prompt":
captured["written_prompt"] = value
orig_setattr(self, name, value)
with patch.object(_ReviewAgentRecorder, "__setattr__", _spy_setattr):
agent._spawn_background_review(
messages_snapshot=[],
review_memory=True,
review_skills=False,
)
assert "written_prompt" in captured, (
"_spawn_background_review never assigned _cached_system_prompt on the review agent"
)
assert captured["written_prompt"] == parent_prompt, (
f"Review fork's _cached_system_prompt diverged from parent's. "
f"Got {captured['written_prompt']!r}, expected {parent_prompt!r}. "
"This breaks Anthropic/OpenRouter prefix-cache parity (#25322)."
)
def test_review_fork_pins_session_start_and_session_id():
"""Defensive complement to cached-system-prompt inheritance.
Even though ``_cached_system_prompt`` inheritance short-circuits the
normal rebuild path, pinning ``session_start`` and ``session_id`` to
the parent's guarantees byte-identical output from any code path that
re-renders parts of the system prompt (compression, plugin hooks).
"""
import run_agent
agent = _make_agent_stub(run_agent.AIAgent)
captured = {}
class _Recorder:
def __init__(self, *args, **kwargs):
self._cached_system_prompt = None
self._memory_write_origin = None
self._memory_write_context = None
self._memory_store = None
self._memory_enabled = None
self._user_profile_enabled = None
self._memory_nudge_interval = None
self._skill_nudge_interval = None
self.suppress_status_output = None
self.session_start = None
self.session_id = None
def run_conversation(self, *args, **kwargs):
captured["session_start"] = self.session_start
captured["session_id"] = self.session_id
raise RuntimeError("stop after recording")
def shutdown_memory_provider(self):
pass
def close(self):
pass
with patch.object(run_agent, "AIAgent", _Recorder), \
patch("threading.Thread", _SyncThread):
agent._spawn_background_review(
messages_snapshot=[],
review_memory=True,
review_skills=False,
)
assert captured.get("session_start") == agent.session_start, (
"Review fork did not inherit parent's session_start — "
"system-prompt rebuild paths would diverge."
)
assert captured.get("session_id") == agent.session_id, (
"Review fork did not inherit parent's session_id — "
"system-prompt rebuild paths would diverge."
)
@@ -1,8 +1,16 @@
"""Tests that the background review agent is restricted to memory+skills toolsets.
"""Tests that the background review agent restricts tools at runtime, not at schema time.
Regression coverage for issue #15204: the background skill-review agent
inherited the full default toolset, allowing it to perform non-skill side
effects (terminal, send_message, delegate_task, etc.).
Regression coverage for issue #15204 (the background skill-review agent must
not perform non-skill side effects like terminal, send_message, delegate_task)
combined with issue #25322 / PR #17276 (the review fork must hit the parent's
Anthropic/OpenRouter prefix cache).
Reconciling the two: the fork now inherits the parent's full ``tools`` schema
so the cache-key matches, and enforces the memory+skills restriction at
runtime via a thread-local whitelist on the existing
``get_pre_tool_call_block_message`` gate. Safety is preserved mechanically
(any non-whitelisted dispatch is blocked) without the schema-level narrowing
that caused the prefix-cache miss.
"""
import threading
@@ -24,6 +32,9 @@ def _make_agent_stub(agent_cls):
agent._skill_nudge_interval = 5
agent.background_review_callback = None
agent.status_callback = None
agent._cached_system_prompt = None
import datetime as _dt
agent.session_start = _dt.datetime(2026, 1, 1, 12, 0, 0)
agent._MEMORY_REVIEW_PROMPT = "review memory"
agent._SKILL_REVIEW_PROMPT = "review skills"
agent._COMBINED_REVIEW_PROMPT = "review both"
@@ -41,15 +52,20 @@ class _SyncThread:
self._target()
def test_background_review_agent_uses_restricted_toolsets():
"""The review agent must only have access to 'memory' and 'skills' toolsets."""
def test_background_review_does_not_narrow_toolset_schema():
"""The review fork must NOT pass enabled_toolsets to AIAgent.
Narrowing the schema diverges the ``tools`` cache key from the parent's,
which sits above ``system`` in Anthropic's cache hierarchy and forces a
full prefix-cache miss on every review (see #25322, PR #17276).
"""
import run_agent
agent = _make_agent_stub(run_agent.AIAgent)
captured = {}
def _capture_init(self, *args, **kwargs):
captured["enabled_toolsets"] = kwargs.get("enabled_toolsets")
captured["enabled_toolsets"] = kwargs.get("enabled_toolsets", "UNSET")
raise RuntimeError("stop after capturing init args")
with patch.object(run_agent.AIAgent, "__init__", _capture_init), \
@@ -61,11 +77,71 @@ def test_background_review_agent_uses_restricted_toolsets():
)
assert "enabled_toolsets" in captured, "AIAgent.__init__ was not called"
assert sorted(captured["enabled_toolsets"]) == ["memory", "skills"]
# The kwarg must be absent — letting AIAgent inherit the default full
# toolset so the schema bytes match the parent's.
assert captured["enabled_toolsets"] == "UNSET", (
f"Review fork narrowed the toolset schema (got {captured['enabled_toolsets']!r}), "
"which breaks prefix-cache parity with the parent."
)
def test_background_review_installs_thread_local_whitelist():
"""The review fork must install a memory/skills-only thread-local whitelist.
The schema-level toolset narrowing was lifted (for prefix-cache parity),
so #15204's safety contract now relies on the runtime whitelist gate to
deny terminal/send_message/delegate_task at dispatch time. Verify the
whitelist is set with exactly the memory+skills tool names.
"""
import run_agent
from hermes_cli import plugins as _plugins
captured = {}
def _capture_whitelist(whitelist, deny_msg_fmt=None):
captured["whitelist"] = set(whitelist)
captured["deny_msg_fmt"] = deny_msg_fmt
# Stop here — we just want to see what gets installed.
raise RuntimeError("stop after capturing whitelist")
agent = _make_agent_stub(run_agent.AIAgent)
def _no_init(self, *args, **kwargs):
# Don't crash AIAgent.__init__; let execution flow reach
# set_thread_tool_whitelist.
return None
with patch.object(run_agent.AIAgent, "__init__", _no_init), \
patch.object(_plugins, "set_thread_tool_whitelist", _capture_whitelist), \
patch("threading.Thread", _SyncThread):
agent._spawn_background_review(
messages_snapshot=[],
review_memory=True,
review_skills=False,
)
assert "whitelist" in captured, "set_thread_tool_whitelist was not called"
whitelist = captured["whitelist"]
# memory + skills tools must be allowed
assert "memory" in whitelist
assert "skill_manage" in whitelist
assert "skill_view" in whitelist
assert "skills_list" in whitelist
# dangerous tools must NOT be in the whitelist
assert "terminal" not in whitelist
assert "send_message" not in whitelist
assert "delegate_task" not in whitelist
assert "web_search" not in whitelist
assert "execute_code" not in whitelist
def test_background_review_agent_tools_are_limited():
"""Verify the resolved memory+skills toolsets only contain memory and skill tools."""
"""Verify the resolved memory+skills toolsets only contain memory and skill tools.
Sanity check on the source of truth for what the runtime whitelist is
derived from if a future PR adds e.g. `terminal` to the `memory`
toolset, the review-fork safety contract silently breaks.
"""
from toolsets import resolve_multiple_toolsets
expected_tools = set(resolve_multiple_toolsets(["memory", "skills"]))
@@ -0,0 +1,418 @@
"""Integration test for the codex_app_server runtime path through AIAgent.
Verifies that:
- api_mode='codex_app_server' is accepted on AIAgent construction
- run_conversation() takes the early-return path and never enters the
chat completions loop
- Projected messages from a fake Codex session land in the messages list
- tool_iterations from the codex session tick the skill nudge counter
- Memory nudge counter ticks once per turn
- The returned dict has the same shape as the chat_completions path
"""
from __future__ import annotations
from unittest.mock import patch
import pytest
import run_agent
from agent.transports.codex_app_server_session import CodexAppServerSession, TurnResult
@pytest.fixture
def fake_session(monkeypatch):
"""Replace CodexAppServerSession with a stub that returns a fixed
TurnResult, so we can drive AIAgent without spawning real codex."""
def fake_run_turn(self, user_input: str, **kwargs):
return TurnResult(
final_text=f"echo: {user_input}",
projected_messages=[
{"role": "assistant", "content": None,
"tool_calls": [{"id": "exec_1", "type": "function",
"function": {"name": "exec_command",
"arguments": "{}"}}]},
{"role": "tool", "tool_call_id": "exec_1", "content": "ok"},
{"role": "assistant", "content": f"echo: {user_input}"},
],
tool_iterations=1,
interrupted=False,
error=None,
turn_id="turn-stub-1",
thread_id="thread-stub-1",
)
monkeypatch.setattr(CodexAppServerSession, "run_turn", fake_run_turn)
monkeypatch.setattr(
CodexAppServerSession, "ensure_started", lambda self: "thread-stub-1"
)
def _make_codex_agent():
"""Construct an AIAgent in codex_app_server mode without contacting any
real provider. We pass api_mode explicitly so the constructor takes the
fast path for direct credentials."""
return run_agent.AIAgent(
api_key="stub",
base_url="https://stub.invalid",
provider="openai",
api_mode="codex_app_server",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
class TestApiModeAccepted:
def test_api_mode_is_codex_app_server(self):
agent = _make_codex_agent()
assert agent.api_mode == "codex_app_server"
class TestRunConversationCodexPath:
def test_run_conversation_returns_codex_shape(self, fake_session):
agent = _make_codex_agent()
# No background review fork during tests
with patch.object(agent, "_spawn_background_review", return_value=None):
result = agent.run_conversation("hello there")
assert result["final_response"] == "echo: hello there"
assert result["completed"] is True
assert result["partial"] is False
assert result["error"] is None
assert result["api_calls"] == 1
assert result["codex_thread_id"] == "thread-stub-1"
assert result["codex_turn_id"] == "turn-stub-1"
def test_projected_messages_are_spliced(self, fake_session):
agent = _make_codex_agent()
with patch.object(agent, "_spawn_background_review", return_value=None):
result = agent.run_conversation("hello")
msgs = result["messages"]
# User message + 3 projected (assistant tool_call + tool + assistant text)
assert len(msgs) >= 4
assert msgs[0]["role"] == "user"
assert msgs[0]["content"] == "hello"
# Last assistant message has the final text
final = [m for m in msgs if m.get("role") == "assistant"
and m.get("content") == "echo: hello"]
assert final, f"expected final assistant message in {msgs}"
def test_nudge_counters_tick(self, fake_session):
"""The skill nudge counter must accumulate tool_iterations across
turns. The memory nudge counter is gated on memory being configured
(which we skip via skip_memory=True), so we don't assert on it here —
a separate test below covers that path explicitly."""
agent = _make_codex_agent()
agent._iters_since_skill = 0
agent._user_turn_count = 0
with patch.object(agent, "_spawn_background_review", return_value=None):
agent.run_conversation("first")
assert agent._iters_since_skill == 1 # one tool_iteration in fake turn
# _user_turn_count is incremented by run_conversation pre-loop, not
# by the codex helper — confirms we delegate that to the standard flow.
assert agent._user_turn_count == 1
with patch.object(agent, "_spawn_background_review", return_value=None):
agent.run_conversation("second")
assert agent._iters_since_skill == 2
assert agent._user_turn_count == 2
def test_user_message_not_duplicated(self, fake_session):
"""Regression guard: the user message must appear exactly once in
the messages list. The standard run_conversation pre-loop appends
it, and the codex helper must NOT append again."""
agent = _make_codex_agent()
with patch.object(agent, "_spawn_background_review", return_value=None):
result = agent.run_conversation("ping unique 12345")
user_count = sum(
1 for m in result["messages"]
if m.get("role") == "user" and m.get("content") == "ping unique 12345"
)
assert user_count == 1, f"user message appeared {user_count}× in {result['messages']}"
def test_background_review_NOT_invoked_below_threshold(self, fake_session):
"""A single turn shouldn't trigger background review — counters
haven't reached the nudge interval (default 10)."""
agent = _make_codex_agent()
agent._memory_nudge_interval = 10
agent._skill_nudge_interval = 10
agent._iters_since_skill = 0
with patch.object(agent, "_spawn_background_review",
return_value=None) as spawn:
agent.run_conversation("ping")
# Below threshold → review should NOT fire (was a real bug:
# the helper was calling _spawn_background_review() with no
# args after every turn, which would crash with TypeError).
assert not spawn.called
def test_background_review_skill_trigger_fires_above_threshold(
self, monkeypatch
):
"""When tool iterations cross the skill nudge interval, the
background review fires with review_skills=True and the right
messages_snapshot signature."""
from agent.transports.codex_app_server_session import (
CodexAppServerSession, TurnResult,
)
# Make the fake session report 10 tool iterations in one turn
# (matching the default skill threshold).
def fake_run_turn(self, user_input: str, **kwargs):
return TurnResult(
final_text=f"echo: {user_input}",
projected_messages=[
{"role": "assistant", "content": f"echo: {user_input}"},
],
tool_iterations=10,
turn_id="t1", thread_id="th1",
)
monkeypatch.setattr(CodexAppServerSession, "run_turn", fake_run_turn)
monkeypatch.setattr(
CodexAppServerSession, "ensure_started", lambda self: "th1"
)
agent = _make_codex_agent()
agent._skill_nudge_interval = 10
agent._iters_since_skill = 0
# Make valid_tool_names include 'skill_manage' so the gate passes
agent.valid_tool_names = set(getattr(agent, "valid_tool_names", set()))
agent.valid_tool_names.add("skill_manage")
with patch.object(agent, "_spawn_background_review",
return_value=None) as spawn:
agent.run_conversation("do tool work")
assert spawn.called, "skill threshold tripped but review didn't fire"
# Verify the call signature matches what _spawn_background_review
# actually expects — this is the regression guard for the original
# bug where the codex path called it with no args at all.
call = spawn.call_args
assert "messages_snapshot" in call.kwargs
assert isinstance(call.kwargs["messages_snapshot"], list)
assert call.kwargs["review_skills"] is True
# Counter should be reset after the review fires
assert agent._iters_since_skill == 0
def test_background_review_signature_never_breaks(self, fake_session):
"""Even when no trigger fires, the helper must never call
_spawn_background_review with the wrong signature. Run a turn,
then run another turn after manually tripping the skill counter
and confirm the call shape is the kwargs-only form the function
actually accepts."""
agent = _make_codex_agent()
agent._skill_nudge_interval = 1 # very low so any iter trips it
agent._iters_since_skill = 0
agent.valid_tool_names = set(getattr(agent, "valid_tool_names", set()))
agent.valid_tool_names.add("skill_manage")
with patch.object(agent, "_spawn_background_review",
return_value=None) as spawn:
agent.run_conversation("first")
# The fake session reports tool_iterations=1, which trips
# _skill_nudge_interval=1. So review should fire.
assert spawn.called
# Critical invariant: positional args must be empty, all real
# args must be kwargs (matching _spawn_background_review's
# actual signature).
call = spawn.call_args
assert call.args == (), (
f"expected no positional args, got {call.args!r}"
"would crash _spawn_background_review at runtime"
)
assert "messages_snapshot" in call.kwargs
def test_chat_completions_loop_is_not_entered(self, fake_session):
"""The early-return must bypass the regular API call loop entirely.
We confirm by patching the SDK call and asserting it's never invoked."""
agent = _make_codex_agent()
# The chat_completions loop calls self.client.chat.completions.create(...)
# If our early-return works, that path is dead.
with patch.object(agent, "client") as client_mock, patch.object(
agent, "_spawn_background_review", return_value=None
):
agent.run_conversation("hi")
assert not client_mock.chat.completions.create.called
class TestReviewForkApiModeDowngrade:
"""When the parent agent runs on codex_app_server, the background
review fork must downgrade to codex_responses otherwise the fork
can't dispatch agent-loop tools (memory, skill_manage) which is the
whole point of the review."""
def test_codex_app_server_parent_downgrades_review_fork(self):
"""Live test against the real _spawn_background_review code path:
verify the review_agent gets api_mode=codex_responses when the
parent is codex_app_server."""
from unittest.mock import MagicMock, patch as _patch
agent = _make_codex_agent()
# Pretend memory + skills are configured so the review fork
# reaches the AIAgent constructor.
agent._memory_store = MagicMock()
agent._memory_enabled = True
agent._user_profile_enabled = True
# Mock _current_main_runtime to return the parent's codex_app_server
# state so we can confirm the helper detects + downgrades it.
agent._current_main_runtime = lambda: {
"api_mode": "codex_app_server",
"base_url": "https://chatgpt.com/backend-api/codex",
"api_key": "stub-token",
}
# Capture what AIAgent gets constructed with inside the helper.
captured = {}
def _capture_init(self, **kwargs):
captured.update(kwargs)
# Set bare attributes the rest of the spawn function reads
# so it can finish without exploding.
self.api_mode = kwargs.get("api_mode")
self.provider = kwargs.get("provider")
self.model = kwargs.get("model")
self._memory_write_origin = None
self._memory_write_context = None
self._memory_store = None
self._memory_enabled = False
self._user_profile_enabled = False
self._memory_nudge_interval = 0
self._skill_nudge_interval = 0
self.suppress_status_output = False
self._session_messages = []
def _no_op_run_conv(*a, **kw):
return {"final_response": "", "messages": []}
self.run_conversation = _no_op_run_conv
def _no_op_close(*a, **kw):
return None
self.close = _no_op_close
with _patch("run_agent.AIAgent.__init__", _capture_init):
agent._spawn_background_review(
messages_snapshot=[{"role": "user", "content": "x"}],
review_memory=True,
review_skills=False,
)
# Wait for the spawned thread to actually execute
import time
for _ in range(30):
if "api_mode" in captured:
break
time.sleep(0.1)
assert captured.get("api_mode") == "codex_responses", (
f"review fork should be downgraded to codex_responses when "
f"parent is codex_app_server; got {captured.get('api_mode')!r}"
)
class TestErrorHandling:
def test_session_exception_returns_partial_with_error(self, monkeypatch):
def boom_run_turn(self, user_input, **kwargs):
raise RuntimeError("subprocess died")
monkeypatch.setattr(CodexAppServerSession, "ensure_started",
lambda self: "t1")
monkeypatch.setattr(CodexAppServerSession, "run_turn", boom_run_turn)
agent = _make_codex_agent()
with patch.object(agent, "_spawn_background_review", return_value=None):
result = agent.run_conversation("hi")
assert result["completed"] is False
assert result["partial"] is True
assert "subprocess died" in result["error"]
assert "codex-runtime auto" in result["final_response"]
def test_interrupted_turn_marked_partial(self, monkeypatch):
def interrupted_turn(self, user_input, **kwargs):
return TurnResult(
final_text="",
projected_messages=[],
tool_iterations=0,
interrupted=True,
error="user interrupted",
turn_id="t",
thread_id="th",
)
monkeypatch.setattr(CodexAppServerSession, "ensure_started",
lambda self: "th")
monkeypatch.setattr(CodexAppServerSession, "run_turn", interrupted_turn)
agent = _make_codex_agent()
with patch.object(agent, "_spawn_background_review", return_value=None):
result = agent.run_conversation("hi")
assert result["completed"] is False
assert result["partial"] is True
assert result["error"] == "user interrupted"
class TestSessionRetirementOnRunAgent:
"""run_agent.py side: when run_turn returns should_retire=True, the
AIAgent must close + null _codex_session so the next turn respawns."""
def test_should_retire_drops_session(self, monkeypatch):
closes = {"count": 0}
def fake_run_turn(self, user_input, **kwargs):
return TurnResult(
final_text="",
projected_messages=[],
tool_iterations=0,
interrupted=True,
error="turn timed out after 600.0s",
turn_id="tu1",
thread_id="th1",
should_retire=True,
)
def fake_close(self):
closes["count"] += 1
monkeypatch.setattr(CodexAppServerSession, "ensure_started",
lambda self: "th1")
monkeypatch.setattr(CodexAppServerSession, "run_turn", fake_run_turn)
monkeypatch.setattr(CodexAppServerSession, "close", fake_close)
agent = _make_codex_agent()
with patch.object(agent, "_spawn_background_review", return_value=None):
result = agent.run_conversation("hi")
# The session was closed and cleared
assert closes["count"] == 1
assert getattr(agent, "_codex_session", "MISSING") is None
# Partial result was still returned (caller still sees the error)
assert result["partial"] is True
assert result["error"] == "turn timed out after 600.0s"
def test_normal_turn_keeps_session(self, fake_session):
"""fake_session fixture returns should_retire=False (default).
The session must stay attached for the next turn to reuse."""
agent = _make_codex_agent()
with patch.object(agent, "_spawn_background_review", return_value=None):
agent.run_conversation("hi")
# Session was lazily created and still attached.
assert getattr(agent, "_codex_session", None) is not None
def test_exception_path_also_drops_session(self, monkeypatch):
"""Even if run_turn raises (not just sets should_retire), we must
drop the session a thrown exception is the strongest possible
signal the process is dead."""
closes = {"count": 0}
def boom_run_turn(self, user_input, **kwargs):
raise RuntimeError("codex segfaulted")
def fake_close(self):
closes["count"] += 1
monkeypatch.setattr(CodexAppServerSession, "ensure_started",
lambda self: "th1")
monkeypatch.setattr(CodexAppServerSession, "run_turn", boom_run_turn)
monkeypatch.setattr(CodexAppServerSession, "close", fake_close)
agent = _make_codex_agent()
with patch.object(agent, "_spawn_background_review", return_value=None):
result = agent.run_conversation("hi")
assert closes["count"] == 1
assert agent._codex_session is None
assert result["completed"] is False
assert "codex segfaulted" in result["error"]
@@ -16,6 +16,16 @@ from run_agent import AIAgent
from agent.context_compressor import ContextCompressor
@pytest.fixture(autouse=True)
def _stable_aux_provider_config():
"""Keep feasibility tests independent from the developer's config.yaml."""
with patch(
"agent.auxiliary_client._resolve_task_provider_model",
return_value=("auto", None, None, None, None),
):
yield
def _make_agent(
*,
compression_enabled: bool = True,
@@ -41,6 +51,7 @@ def _make_agent(
agent.tool_progress_callback = None
agent._compression_warning = None
agent._aux_compression_context_length_config = None
agent._custom_providers = []
agent.tools = []
compressor = MagicMock(spec=ContextCompressor)
@@ -182,6 +193,7 @@ def test_feasibility_check_passes_config_context_length(mock_get_client, mock_ct
api_key="sk-custom",
config_context_length=1_000_000,
provider="openrouter",
custom_providers=[],
)
@@ -205,6 +217,7 @@ def test_feasibility_check_ignores_invalid_context_length(mock_get_client, mock_
api_key="sk-test",
config_context_length=None,
provider="openrouter",
custom_providers=[],
)
@@ -258,6 +271,7 @@ def test_init_feasibility_check_uses_aux_context_override_from_config():
api_key="sk-custom",
config_context_length=1_000_000,
provider="",
custom_providers=[],
)
@@ -166,6 +166,56 @@ class TestRecordFileMutationResult:
)
assert agent._turn_failed_file_mutations == {}
def test_write_file_with_lint_error_counts_as_landed(self):
agent = _bare_agent()
agent._record_file_mutation_result(
"write_file",
{"path": "/tmp/a.py", "content": "bad"},
json.dumps({"error": "write failed"}),
is_error=True,
)
assert "/tmp/a.py" in agent._turn_failed_file_mutations
result = json.dumps({
"bytes_written": 24,
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
})
agent._record_file_mutation_result(
"write_file",
{"path": "/tmp/a.py", "content": "def nope(:\n"},
result,
is_error=True,
)
assert agent._turn_failed_file_mutations == {}
def test_patch_with_lsp_diagnostics_counts_as_landed(self):
agent = _bare_agent()
agent._record_file_mutation_result(
"patch",
{"mode": "replace", "path": "/tmp/a.py", "old_string": "x", "new_string": "y"},
json.dumps({"error": "Could not find old_string"}),
is_error=True,
)
assert "/tmp/a.py" in agent._turn_failed_file_mutations
result = json.dumps({
"success": True,
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
"files_modified": ["/tmp/a.py"],
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
})
agent._record_file_mutation_result(
"patch",
{"mode": "replace", "path": "/tmp/a.py", "old_string": "x", "new_string": "y"},
result,
is_error=True,
)
assert agent._turn_failed_file_mutations == {}
def test_repeated_failure_keeps_first_error(self):
agent = _bare_agent()
agent._record_file_mutation_result(
+3 -2
View File
@@ -2524,8 +2524,9 @@ class TestRunConversation:
assert [call["api_call_count"] for call in pre_request_calls] == [1, 2]
assert [call["api_call_count"] for call in post_request_calls] == [1, 2]
assert all(call["session_id"] == agent.session_id for call in pre_request_calls)
assert all("message_count" in c and "messages" not in c for c in pre_request_calls)
assert all("usage" in c and "response" not in c for c in post_request_calls)
assert all("message_count" in c and isinstance(c.get("request_messages"), list) for c in pre_request_calls)
assert any(msg.get("role") == "user" and msg.get("content") == "search something" for msg in pre_request_calls[0]["request_messages"])
assert all("usage" in c and "response" in c and "assistant_message" in c for c in post_request_calls)
def test_content_with_tool_calls_stays_silent_for_non_cli_quiet_mode(self, agent):
self._setup_agent(agent)
@@ -578,6 +578,197 @@ def test_run_conversation_codex_refreshes_after_401_and_retries(monkeypatch):
assert result["final_response"] == "Recovered after refresh"
def _build_xai_oauth_agent(monkeypatch):
_patch_agent_bootstrap(monkeypatch)
agent = run_agent.AIAgent(
model="grok-4.3",
provider="xai-oauth",
api_mode="codex_responses",
base_url="https://api.x.ai/v1",
api_key="xai-oauth-token",
quiet_mode=True,
max_iterations=4,
skip_context_files=True,
skip_memory=True,
)
agent._cleanup_task_resources = lambda task_id: None
agent._persist_session = lambda messages, history=None: None
agent._save_trajectory = lambda messages, user_message, completed: None
agent._save_session_log = lambda messages: None
return agent
def test_build_api_kwargs_xai_oauth_sends_cache_key_via_extra_body(monkeypatch):
"""xai-oauth + codex_responses must route prompt caching via the
``prompt_cache_key`` body field on /v1/responses (xAI's documented
Responses-API cache key see docs.x.ai prompt-caching/maximizing-
cache-hits).
We pass it through ``extra_body`` rather than as a top-level kwarg so
the body field is serialized into JSON regardless of whether the
installed openai SDK build still accepts ``prompt_cache_key`` on
``Responses.stream()``. Older or trimmed SDK builds drop it from the
signature and would otherwise raise ``TypeError`` before the request
reaches api.x.ai. The ``x-grok-conv-id`` header is retained as a
belt-and-braces fallback for clients/proxies that route on headers."""
agent = _build_xai_oauth_agent(monkeypatch)
kwargs = agent._build_api_kwargs(
[
{"role": "system", "content": "You are Hermes."},
{"role": "user", "content": "Ping"},
]
)
assert kwargs.get("model") == "grok-4.3"
# Top-level kwarg must NOT be set — that's the openai SDK
# incompatibility this whole indirection exists to dodge.
assert "prompt_cache_key" not in kwargs
extra_body = kwargs.get("extra_body") or {}
assert extra_body.get("prompt_cache_key"), (
"xAI prompt-cache routing must travel via extra_body.prompt_cache_key "
"for /v1/responses — body field is the documented surface."
)
headers = kwargs.get("extra_headers") or {}
assert "x-grok-conv-id" in headers, (
"x-grok-conv-id header kept as belt-and-braces fallback for clients "
"that route on headers."
)
def test_run_conversation_xai_oauth_refreshes_after_401_and_retries(monkeypatch):
"""xai-oauth speaks the Responses API just like codex. When the access
token is rejected mid-call (401), the same proactive refresh-and-retry
handler that fires for openai-codex must also fire for xai-oauth the
bug it caught: the gating condition checked only ``provider == "openai-codex"``,
so xai-oauth 401s leaked straight to non-retryable abort path with no
chance to swap in a freshly refreshed access token."""
agent = _build_xai_oauth_agent(monkeypatch)
calls = {"api": 0, "refresh": 0}
class _UnauthorizedError(RuntimeError):
def __init__(self):
super().__init__("Error code: 401 - unauthorized")
self.status_code = 401
def _fake_api_call(api_kwargs):
calls["api"] += 1
if calls["api"] == 1:
raise _UnauthorizedError()
return _codex_message_response("Recovered after xAI refresh")
def _fake_refresh(*, force=True):
calls["refresh"] += 1
assert force is True
return True
monkeypatch.setattr(agent, "_interruptible_api_call", _fake_api_call)
monkeypatch.setattr(agent, "_try_refresh_codex_client_credentials", _fake_refresh)
result = agent.run_conversation("Say OK")
assert calls["api"] == 2
assert calls["refresh"] == 1
assert result["completed"] is True
assert result["final_response"] == "Recovered after xAI refresh"
def test_try_refresh_codex_client_credentials_handles_xai_oauth(monkeypatch):
"""``_try_refresh_codex_client_credentials`` must rebuild the OpenAI
client with freshly resolved xAI OAuth credentials when the active
provider is xai-oauth. The function name is shared between codex and
xai-oauth (both speak codex_responses) covering both cases prevents
silent regressions where the function gets gated to a single provider."""
agent = _build_xai_oauth_agent(monkeypatch)
closed = {"value": False}
rebuilt = {"kwargs": None}
class _ExistingClient:
def close(self):
closed["value"] = True
class _RebuiltClient:
pass
def _fake_openai(**kwargs):
rebuilt["kwargs"] = kwargs
return _RebuiltClient()
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
# The pre-refresh guard reads the singleton with refresh_if_expiring=False
# to verify that the agent's active key still matches; the actual
# refresh later passes force_refresh=True. Both calls must succeed.
return {
"api_key": "fresh-xai-token" if force_refresh else agent.api_key,
"base_url": "https://api.x.ai/v1",
}
monkeypatch.setattr(
"hermes_cli.auth.resolve_xai_oauth_runtime_credentials",
_fake_resolve,
)
monkeypatch.setattr(run_agent, "OpenAI", _fake_openai)
agent.client = _ExistingClient()
ok = agent._try_refresh_codex_client_credentials(force=True)
assert ok is True
assert closed["value"] is True
assert rebuilt["kwargs"]["api_key"] == "fresh-xai-token"
assert rebuilt["kwargs"]["base_url"] == "https://api.x.ai/v1"
assert isinstance(agent.client, _RebuiltClient)
assert agent.api_key == "fresh-xai-token"
def test_try_refresh_codex_client_credentials_skips_xai_oauth_when_singleton_differs(monkeypatch):
"""An xai-oauth agent constructed with a non-singleton credential
(e.g. a manual pool entry whose tokens belong to a different account
than the loopback_pkce singleton, or an explicit ``api_key=`` arg)
MUST NOT silently adopt the singleton's tokens on a 401 reactive
refresh. Otherwise a 401 mid-conversation would re-route the rest
of the conversation onto a different account, with no user feedback.
The credential pool's reactive recovery is the right channel for
pool-managed credentials; this fallback path is for the singleton-
only case and must short-circuit when the active key differs."""
agent = _build_xai_oauth_agent(monkeypatch)
# Agent is using "xai-oauth-token" (per the builder); singleton holds
# a *different* account's token. No force_refresh should fire.
refresh_calls = {"count": 0}
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
if force_refresh:
refresh_calls["count"] += 1
return {
"api_key": "singleton-account-token",
"base_url": "https://api.x.ai/v1",
}
# The pre-refresh guard read — return the singleton's view of the
# singleton's token, which is NOT what the agent is currently using.
return {
"api_key": "singleton-account-token",
"base_url": "https://api.x.ai/v1",
}
monkeypatch.setattr(
"hermes_cli.auth.resolve_xai_oauth_runtime_credentials",
_fake_resolve,
)
pre_refresh_key = agent.api_key
ok = agent._try_refresh_codex_client_credentials(force=True)
assert ok is False, (
"must not refresh when the active credential isn't the singleton; "
"otherwise the conversation silently swaps accounts mid-flight."
)
assert refresh_calls["count"] == 0, (
"force_refresh must not run — that would mutate the singleton's "
"tokens on disk and consume its single-use refresh_token for an "
"agent that wasn't even using the singleton."
)
assert agent.api_key == pre_refresh_key
def test_run_conversation_copilot_refreshes_after_401_and_retries(monkeypatch):
agent = _build_copilot_agent(monkeypatch)
calls = {"api": 0, "refresh": 0}
@@ -624,12 +815,18 @@ def test_try_refresh_codex_client_credentials_rebuilds_client(monkeypatch):
rebuilt["kwargs"] = kwargs
return _RebuiltClient()
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
# Pre-refresh guard reads the singleton (refresh_if_expiring=False).
# It must report the agent's current api_key so the equality check
# passes; only then does the actual force_refresh run.
return {
"api_key": "new-codex-token" if force_refresh else agent.api_key,
"base_url": "https://chatgpt.com/backend-api/codex",
}
monkeypatch.setattr(
"hermes_cli.auth.resolve_codex_runtime_credentials",
lambda force_refresh=True: {
"api_key": "new-codex-token",
"base_url": "https://chatgpt.com/backend-api/codex",
},
_fake_resolve,
)
monkeypatch.setattr(run_agent, "OpenAI", _fake_openai)
@@ -23,7 +23,7 @@ class TestStreamingAssemblyRepair:
These tests verify the REPAIR FUNCTION itself works correctly for the
cases that arise during streaming assembly. Integration tests that
exercise the full streaming path are in test_agent_loop_tool_calling.py.
exercise the full streaming path are in run_agent.py's streaming tests.
"""
# -- Truncation cases (most common streaming failure) --

Some files were not shown because too many files have changed in this diff Show More