Merge main into bb/gui.
Resolve merge conflicts while preserving bb/gui dashboard paths and STT provider support.
This commit is contained in:
+47
-1
@@ -1,6 +1,11 @@
|
||||
"""Tests for acp_adapter.auth — provider detection."""
|
||||
|
||||
from acp_adapter.auth import has_provider, detect_provider
|
||||
from acp_adapter.auth import (
|
||||
TERMINAL_SETUP_AUTH_METHOD_ID,
|
||||
build_auth_methods,
|
||||
has_provider,
|
||||
detect_provider,
|
||||
)
|
||||
|
||||
|
||||
class TestHasProvider:
|
||||
@@ -54,3 +59,44 @@ class TestDetectProvider:
|
||||
|
||||
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _boom)
|
||||
assert detect_provider() is None
|
||||
|
||||
def test_detect_provider_strips_and_lowercases_provider(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
||||
lambda: {"provider": " OpenRouter ", "api_key": " sk-or-test "},
|
||||
)
|
||||
assert detect_provider() == "openrouter"
|
||||
|
||||
|
||||
class TestBuildAuthMethods:
|
||||
def test_build_auth_methods_returns_provider_and_terminal_when_configured(self, monkeypatch):
|
||||
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: "openrouter")
|
||||
|
||||
methods = build_auth_methods()
|
||||
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in methods]
|
||||
|
||||
assert payloads[0]["id"] == "openrouter"
|
||||
assert payloads[0]["name"] == "openrouter runtime credentials"
|
||||
assert any(payload["id"] == TERMINAL_SETUP_AUTH_METHOD_ID for payload in payloads)
|
||||
terminal = next(payload for payload in payloads if payload["id"] == TERMINAL_SETUP_AUTH_METHOD_ID)
|
||||
assert terminal["type"] == "terminal"
|
||||
assert terminal["args"] == ["--setup"]
|
||||
|
||||
def test_build_auth_methods_returns_terminal_setup_when_unconfigured(self, monkeypatch):
|
||||
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: None)
|
||||
|
||||
methods = build_auth_methods()
|
||||
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in methods]
|
||||
|
||||
assert payloads == [
|
||||
{
|
||||
"args": ["--setup"],
|
||||
"description": (
|
||||
"Open Hermes' interactive model/provider setup in a terminal. "
|
||||
"Use this when Hermes has not been configured on this machine yet."
|
||||
),
|
||||
"id": TERMINAL_SETUP_AUTH_METHOD_ID,
|
||||
"name": "Configure Hermes provider",
|
||||
"type": "terminal",
|
||||
}
|
||||
]
|
||||
|
||||
+177
-1
@@ -1,6 +1,9 @@
|
||||
"""Tests for acp_adapter.entry startup wiring."""
|
||||
|
||||
import sys
|
||||
|
||||
import acp
|
||||
import pytest
|
||||
|
||||
from acp_adapter import entry
|
||||
|
||||
@@ -15,6 +18,179 @@ def test_main_enables_unstable_protocol(monkeypatch):
|
||||
monkeypatch.setattr(entry, "_load_env", lambda: None)
|
||||
monkeypatch.setattr(acp, "run_agent", fake_run_agent)
|
||||
|
||||
entry.main()
|
||||
entry.main([])
|
||||
|
||||
assert calls["kwargs"]["use_unstable_protocol"] is True
|
||||
|
||||
|
||||
def test_main_version_prints_without_starting_server(monkeypatch, capsys):
|
||||
monkeypatch.setattr(entry, "_setup_logging", lambda: (_ for _ in ()).throw(AssertionError("started server")))
|
||||
|
||||
entry.main(["--version"])
|
||||
|
||||
output = capsys.readouterr().out.strip()
|
||||
assert output
|
||||
assert "Starting hermes-agent ACP adapter" not in output
|
||||
|
||||
|
||||
def test_main_check_prints_ok_without_starting_server(monkeypatch, capsys):
|
||||
monkeypatch.setattr(entry, "_setup_logging", lambda: (_ for _ in ()).throw(AssertionError("started server")))
|
||||
|
||||
entry.main(["--check"])
|
||||
|
||||
assert capsys.readouterr().out.strip() == "Hermes ACP check OK"
|
||||
|
||||
|
||||
def test_main_setup_runs_model_configuration(monkeypatch):
|
||||
calls = {}
|
||||
|
||||
def fake_hermes_main():
|
||||
calls["argv"] = sys.argv[:]
|
||||
|
||||
monkeypatch.setattr("hermes_cli.main.main", fake_hermes_main)
|
||||
# Pretend stdin is not a TTY so the follow-up browser prompt is skipped.
|
||||
# That keeps this test focused on the model-setup wiring; the
|
||||
# browser-prompt path has its own test below.
|
||||
monkeypatch.setattr("sys.stdin.isatty", lambda: False)
|
||||
|
||||
entry.main(["--setup"])
|
||||
|
||||
assert calls["argv"][1:] == ["model"]
|
||||
|
||||
|
||||
def test_main_setup_offers_browser_install_when_tty(monkeypatch):
|
||||
"""When stdin is a TTY and the user answers yes, model setup is followed
|
||||
by a browser-tools bootstrap call."""
|
||||
monkeypatch.setattr("hermes_cli.main.main", lambda: None)
|
||||
monkeypatch.setattr("sys.stdin.isatty", lambda: True)
|
||||
monkeypatch.setattr("builtins.input", lambda *_args, **_kwargs: "y")
|
||||
|
||||
bootstrap_calls = []
|
||||
monkeypatch.setattr(
|
||||
entry,
|
||||
"_run_setup_browser",
|
||||
lambda assume_yes=False: bootstrap_calls.append(assume_yes) or 0,
|
||||
)
|
||||
|
||||
entry.main(["--setup"])
|
||||
|
||||
assert bootstrap_calls == [False]
|
||||
|
||||
|
||||
def test_main_setup_skips_browser_prompt_on_no(monkeypatch):
|
||||
monkeypatch.setattr("hermes_cli.main.main", lambda: None)
|
||||
monkeypatch.setattr("sys.stdin.isatty", lambda: True)
|
||||
monkeypatch.setattr("builtins.input", lambda *_args, **_kwargs: "")
|
||||
|
||||
called = []
|
||||
monkeypatch.setattr(
|
||||
entry,
|
||||
"_run_setup_browser",
|
||||
lambda assume_yes=False: called.append(assume_yes) or 0,
|
||||
)
|
||||
|
||||
entry.main(["--setup"])
|
||||
|
||||
assert called == []
|
||||
|
||||
|
||||
def test_main_setup_browser_invokes_bundled_script(monkeypatch):
|
||||
"""`hermes-acp --setup-browser` must shell out to the bundled bootstrap
|
||||
script — never reimplement the install logic inline."""
|
||||
monkeypatch.setattr("platform.system", lambda: "Linux")
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_run(cmd, check=False):
|
||||
captured["cmd"] = cmd
|
||||
|
||||
class _R:
|
||||
returncode = 0
|
||||
|
||||
return _R()
|
||||
|
||||
monkeypatch.setattr("subprocess.run", fake_run)
|
||||
|
||||
entry.main(["--setup-browser"])
|
||||
|
||||
assert captured["cmd"][0] == "bash"
|
||||
assert captured["cmd"][1].endswith("bootstrap_browser_tools.sh")
|
||||
# --yes is NOT passed when the flag is absent.
|
||||
assert "--yes" not in captured["cmd"]
|
||||
|
||||
|
||||
def test_main_setup_browser_forwards_yes_flag(monkeypatch):
|
||||
monkeypatch.setattr("platform.system", lambda: "Linux")
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_run(cmd, check=False):
|
||||
captured["cmd"] = cmd
|
||||
|
||||
class _R:
|
||||
returncode = 0
|
||||
|
||||
return _R()
|
||||
|
||||
monkeypatch.setattr("subprocess.run", fake_run)
|
||||
|
||||
entry.main(["--setup-browser", "--yes"])
|
||||
|
||||
assert "--yes" in captured["cmd"]
|
||||
|
||||
|
||||
def test_main_setup_browser_uses_powershell_on_windows(monkeypatch):
|
||||
monkeypatch.setattr("platform.system", lambda: "Windows")
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_run(cmd, check=False):
|
||||
captured["cmd"] = cmd
|
||||
|
||||
class _R:
|
||||
returncode = 0
|
||||
|
||||
return _R()
|
||||
|
||||
monkeypatch.setattr("subprocess.run", fake_run)
|
||||
|
||||
entry.main(["--setup-browser", "--yes"])
|
||||
|
||||
assert captured["cmd"][0] == "powershell.exe"
|
||||
assert any(part.endswith("bootstrap_browser_tools.ps1") for part in captured["cmd"])
|
||||
assert "-Yes" in captured["cmd"]
|
||||
|
||||
|
||||
def test_main_setup_browser_propagates_failure(monkeypatch):
|
||||
monkeypatch.setattr("platform.system", lambda: "Linux")
|
||||
|
||||
class _R:
|
||||
returncode = 7
|
||||
|
||||
monkeypatch.setattr("subprocess.run", lambda cmd, check=False: _R())
|
||||
|
||||
with pytest.raises(SystemExit) as excinfo:
|
||||
entry.main(["--setup-browser"])
|
||||
assert excinfo.value.code == 7
|
||||
|
||||
|
||||
def test_bootstrap_scripts_ship_with_package():
|
||||
"""The package-data wiring (pyproject.toml) must include the bootstrap
|
||||
scripts — otherwise `--setup-browser` 404s at runtime."""
|
||||
from pathlib import Path
|
||||
|
||||
bootstrap_dir = Path(entry.__file__).resolve().parent / "bootstrap"
|
||||
sh = bootstrap_dir / "bootstrap_browser_tools.sh"
|
||||
ps1 = bootstrap_dir / "bootstrap_browser_tools.ps1"
|
||||
|
||||
assert sh.is_file(), f"missing bundled script: {sh}"
|
||||
assert ps1.is_file(), f"missing bundled script: {ps1}"
|
||||
|
||||
sh_text = sh.read_text(encoding="utf-8")
|
||||
ps1_text = ps1.read_text(encoding="utf-8")
|
||||
|
||||
# Sanity: scripts know how to find the Hermes-managed Node prefix.
|
||||
assert "HERMES_HOME" in sh_text
|
||||
assert "agent-browser" in sh_text
|
||||
assert "HermesHome" in ps1_text
|
||||
assert "agent-browser" in ps1_text
|
||||
|
||||
+127
-48
@@ -1,89 +1,168 @@
|
||||
"""Tests for acp_adapter.permissions — ACP approval bridging."""
|
||||
"""Tests for acp_adapter.permissions."""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
from concurrent.futures import Future
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from acp.schema import (
|
||||
AllowedOutcome,
|
||||
DeniedOutcome,
|
||||
RequestPermissionResponse,
|
||||
)
|
||||
|
||||
from acp_adapter.permissions import make_approval_callback
|
||||
from tools.approval import prompt_dangerous_approval
|
||||
|
||||
|
||||
def _make_response(outcome):
|
||||
"""Helper to build a RequestPermissionResponse with the given outcome."""
|
||||
return RequestPermissionResponse(outcome=outcome)
|
||||
|
||||
|
||||
def _setup_callback(outcome, timeout=60.0):
|
||||
"""
|
||||
Create a callback wired to a mock request_permission coroutine
|
||||
that resolves to the given outcome.
|
||||
|
||||
Returns:
|
||||
(callback, mock_request_permission_fn)
|
||||
"""
|
||||
def _invoke_callback(
|
||||
outcome,
|
||||
*,
|
||||
allow_permanent=True,
|
||||
timeout=60.0,
|
||||
use_prompt_path=False,
|
||||
):
|
||||
loop = MagicMock(spec=asyncio.AbstractEventLoop)
|
||||
mock_rp = MagicMock(name="request_permission")
|
||||
|
||||
response = _make_response(outcome)
|
||||
|
||||
# Patch asyncio.run_coroutine_threadsafe so it returns a future
|
||||
# that immediately yields the response.
|
||||
request_permission = AsyncMock(name="request_permission")
|
||||
future = MagicMock(spec=Future)
|
||||
future.result.return_value = response
|
||||
future.result.return_value = _make_response(outcome)
|
||||
|
||||
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", return_value=future):
|
||||
cb = make_approval_callback(mock_rp, loop, session_id="s1", timeout=timeout)
|
||||
result = cb("rm -rf /", "dangerous command")
|
||||
scheduled = {}
|
||||
|
||||
return result
|
||||
def _schedule(coro, passed_loop):
|
||||
scheduled["coro"] = coro
|
||||
scheduled["loop"] = passed_loop
|
||||
return future
|
||||
|
||||
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
|
||||
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=timeout)
|
||||
if use_prompt_path:
|
||||
result = prompt_dangerous_approval(
|
||||
"rm -rf /",
|
||||
"dangerous command",
|
||||
allow_permanent=allow_permanent,
|
||||
approval_callback=cb,
|
||||
)
|
||||
else:
|
||||
result = cb(
|
||||
"rm -rf /",
|
||||
"dangerous command",
|
||||
allow_permanent=allow_permanent,
|
||||
)
|
||||
|
||||
scheduled["coro"].close()
|
||||
_, kwargs = request_permission.call_args
|
||||
return result, kwargs, scheduled, future, loop
|
||||
|
||||
|
||||
class TestApprovalMapping:
|
||||
def test_approval_allow_once_maps_correctly(self):
|
||||
outcome = AllowedOutcome(option_id="allow_once", outcome="selected")
|
||||
result = _setup_callback(outcome)
|
||||
class TestApprovalBridge:
|
||||
def test_bridge_schedules_request_on_the_given_loop(self):
|
||||
result, kwargs, scheduled, _, loop = _invoke_callback(
|
||||
AllowedOutcome(option_id="allow_once", outcome="selected"),
|
||||
)
|
||||
|
||||
tool_call = kwargs["tool_call"]
|
||||
option_ids = [option.option_id for option in kwargs["options"]]
|
||||
|
||||
assert result == "once"
|
||||
assert scheduled["loop"] is loop
|
||||
assert inspect.iscoroutine(scheduled["coro"])
|
||||
assert kwargs["session_id"] == "s1"
|
||||
assert tool_call.session_update == "tool_call_update"
|
||||
assert tool_call.tool_call_id.startswith("perm-check-")
|
||||
assert tool_call.kind == "execute"
|
||||
assert tool_call.status == "pending"
|
||||
assert tool_call.title == "dangerous command"
|
||||
assert tool_call.raw_input == {
|
||||
"command": "rm -rf /",
|
||||
"description": "dangerous command",
|
||||
}
|
||||
assert option_ids == ["allow_once", "allow_session", "allow_always", "deny"]
|
||||
|
||||
def test_tool_call_ids_are_unique(self):
|
||||
_, first_kwargs, _, _, _ = _invoke_callback(
|
||||
AllowedOutcome(option_id="allow_once", outcome="selected"),
|
||||
)
|
||||
_, second_kwargs, _, _, _ = _invoke_callback(
|
||||
AllowedOutcome(option_id="allow_once", outcome="selected"),
|
||||
)
|
||||
|
||||
assert first_kwargs["tool_call"].tool_call_id != second_kwargs["tool_call"].tool_call_id
|
||||
|
||||
def test_prompt_path_keeps_session_option_when_permanent_disabled(self):
|
||||
result, kwargs, _, _, _ = _invoke_callback(
|
||||
AllowedOutcome(option_id="allow_session", outcome="selected"),
|
||||
allow_permanent=False,
|
||||
use_prompt_path=True,
|
||||
)
|
||||
|
||||
option_ids = [option.option_id for option in kwargs["options"]]
|
||||
|
||||
assert result == "session"
|
||||
assert option_ids == ["allow_once", "allow_session", "deny"]
|
||||
|
||||
def test_allow_always_maps_correctly(self):
|
||||
result, _, _, _, _ = _invoke_callback(
|
||||
AllowedOutcome(option_id="allow_always", outcome="selected"),
|
||||
use_prompt_path=True,
|
||||
)
|
||||
|
||||
def test_approval_allow_always_maps_correctly(self):
|
||||
outcome = AllowedOutcome(option_id="allow_always", outcome="selected")
|
||||
result = _setup_callback(outcome)
|
||||
assert result == "always"
|
||||
|
||||
def test_approval_deny_maps_correctly(self):
|
||||
outcome = DeniedOutcome(outcome="cancelled")
|
||||
result = _setup_callback(outcome)
|
||||
assert result == "deny"
|
||||
def test_denied_and_unknown_outcomes_deny(self):
|
||||
denied_result, _, _, _, _ = _invoke_callback(DeniedOutcome(outcome="cancelled"))
|
||||
unknown_result, _, _, _, _ = _invoke_callback(
|
||||
AllowedOutcome(option_id="unexpected", outcome="selected"),
|
||||
)
|
||||
|
||||
def test_approval_timeout_returns_deny(self):
|
||||
"""When the future times out, the callback should return 'deny'."""
|
||||
assert denied_result == "deny"
|
||||
assert unknown_result == "deny"
|
||||
|
||||
def test_timeout_returns_deny_and_cancels_future(self):
|
||||
loop = MagicMock(spec=asyncio.AbstractEventLoop)
|
||||
mock_rp = MagicMock(name="request_permission")
|
||||
|
||||
request_permission = AsyncMock(name="request_permission")
|
||||
future = MagicMock(spec=Future)
|
||||
future.result.side_effect = TimeoutError("timed out")
|
||||
|
||||
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", return_value=future):
|
||||
cb = make_approval_callback(mock_rp, loop, session_id="s1", timeout=0.01)
|
||||
result = cb("rm -rf /", "dangerous")
|
||||
scheduled = {}
|
||||
|
||||
def _schedule(coro, passed_loop):
|
||||
scheduled["coro"] = coro
|
||||
scheduled["loop"] = passed_loop
|
||||
return future
|
||||
|
||||
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
|
||||
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=0.01)
|
||||
result = cb("rm -rf /", "dangerous command")
|
||||
|
||||
scheduled["coro"].close()
|
||||
|
||||
assert result == "deny"
|
||||
assert scheduled["loop"] is loop
|
||||
assert future.cancel.call_count == 1
|
||||
|
||||
def test_approval_none_response_returns_deny(self):
|
||||
"""When request_permission resolves to None, the callback should return 'deny'."""
|
||||
def test_none_response_returns_deny(self):
|
||||
"""When request_permission resolves to None, the callback returns 'deny'."""
|
||||
loop = MagicMock(spec=asyncio.AbstractEventLoop)
|
||||
mock_rp = MagicMock(name="request_permission")
|
||||
|
||||
request_permission = AsyncMock(name="request_permission")
|
||||
future = MagicMock(spec=Future)
|
||||
future.result.return_value = None
|
||||
|
||||
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", return_value=future):
|
||||
cb = make_approval_callback(mock_rp, loop, session_id="s1", timeout=1.0)
|
||||
scheduled = {}
|
||||
|
||||
def _schedule(coro, passed_loop):
|
||||
scheduled["coro"] = coro
|
||||
scheduled["loop"] = passed_loop
|
||||
return future
|
||||
|
||||
with patch("acp_adapter.permissions.asyncio.run_coroutine_threadsafe", side_effect=_schedule):
|
||||
cb = make_approval_callback(request_permission, loop, session_id="s1", timeout=1.0)
|
||||
result = cb("echo hi", "demo")
|
||||
|
||||
scheduled["coro"].close()
|
||||
|
||||
assert result == "deny"
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Tests for ACP Registry metadata shipped with Hermes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
MANIFEST = ROOT / "acp_registry" / "agent.json"
|
||||
ICON = ROOT / "acp_registry" / "icon.svg"
|
||||
FORBIDDEN_MANIFEST_KEYS = {"schema_version", "display_name"}
|
||||
ALLOWED_DISTRIBUTIONS = {"binary", "npx", "uvx"}
|
||||
|
||||
|
||||
def _manifest() -> dict:
|
||||
return json.loads(MANIFEST.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _pyproject_version() -> str:
|
||||
data = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
|
||||
return data["project"]["version"]
|
||||
|
||||
|
||||
def test_agent_json_matches_official_registry_required_fields():
|
||||
data = _manifest()
|
||||
|
||||
assert FORBIDDEN_MANIFEST_KEYS.isdisjoint(data)
|
||||
assert data["id"] == "hermes-agent"
|
||||
assert re.fullmatch(r"[a-z][a-z0-9-]*", data["id"])
|
||||
assert data["name"] == "Hermes Agent"
|
||||
assert data["description"]
|
||||
assert data["repository"] == "https://github.com/NousResearch/hermes-agent"
|
||||
assert data["website"].startswith("https://hermes-agent.nousresearch.com/")
|
||||
assert data["authors"] == ["Nous Research"]
|
||||
assert data["license"] == "MIT"
|
||||
assert set(data["distribution"]) <= ALLOWED_DISTRIBUTIONS
|
||||
|
||||
|
||||
def test_agent_json_uses_uvx_distribution_without_local_command_fields():
|
||||
data = _manifest()
|
||||
|
||||
assert set(data["distribution"]) == {"uvx"}
|
||||
uvx = data["distribution"]["uvx"]
|
||||
# Schema allows {package, args, env}; we use {package, args}.
|
||||
assert set(uvx) <= {"package", "args", "env"}
|
||||
assert "package" in uvx
|
||||
assert uvx["package"] == f"hermes-agent[acp]=={data['version']}"
|
||||
assert uvx["args"] == ["hermes-acp"]
|
||||
# Old command-shape fields must not leak back in.
|
||||
assert "type" not in data["distribution"]
|
||||
assert "command" not in data["distribution"]
|
||||
|
||||
|
||||
def test_agent_json_version_matches_pyproject():
|
||||
assert _manifest()["version"] == _pyproject_version()
|
||||
|
||||
|
||||
def test_agent_json_pins_uvx_package_to_pyproject_version():
|
||||
"""The registry CI rejects ``@latest`` and floating pins; the manifest must
|
||||
always reference the exact PyPI version listed in pyproject.toml."""
|
||||
assert _manifest()["distribution"]["uvx"]["package"] == (
|
||||
f"hermes-agent[acp]=={_pyproject_version()}"
|
||||
)
|
||||
|
||||
|
||||
def test_icon_svg_is_16x16_current_color():
|
||||
root = ET.fromstring(ICON.read_text(encoding="utf-8"))
|
||||
|
||||
assert root.attrib["viewBox"] == "0 0 16 16"
|
||||
assert root.attrib["width"] == "16"
|
||||
assert root.attrib["height"] == "16"
|
||||
|
||||
|
||||
def test_icon_svg_has_no_hardcoded_colors_or_gradients():
|
||||
text = ICON.read_text(encoding="utf-8")
|
||||
|
||||
assert "linearGradient" not in text
|
||||
assert "radialGradient" not in text
|
||||
assert "url(#" not in text
|
||||
assert not re.search(r"#[0-9a-fA-F]{3,8}\b", text)
|
||||
|
||||
root = ET.fromstring(text)
|
||||
for element in root.iter():
|
||||
for attr in ("fill", "stroke"):
|
||||
value = element.attrib.get(attr)
|
||||
if value is not None:
|
||||
assert value in {"currentColor", "none"}
|
||||
@@ -33,6 +33,7 @@ from acp.schema import (
|
||||
UsageUpdate,
|
||||
UserMessageChunk,
|
||||
)
|
||||
from acp_adapter.auth import TERMINAL_SETUP_AUTH_METHOD_ID
|
||||
from acp_adapter.server import HermesACPAgent, HERMES_VERSION
|
||||
from acp_adapter.session import SessionManager
|
||||
from hermes_state import SessionDB
|
||||
@@ -92,6 +93,41 @@ class TestInitialize:
|
||||
assert "list" in session_caps
|
||||
assert "resume" in session_caps
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_advertises_provider_and_terminal_auth_methods(self, agent, monkeypatch):
|
||||
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: "openrouter")
|
||||
monkeypatch.setattr("acp_adapter.server.detect_provider", lambda: "openrouter")
|
||||
|
||||
resp = await agent.initialize(protocol_version=1)
|
||||
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in resp.auth_methods]
|
||||
|
||||
assert payloads[0]["id"] == "openrouter"
|
||||
assert payloads[0]["name"] == "openrouter runtime credentials"
|
||||
terminal = next(payload for payload in payloads if payload["id"] == TERMINAL_SETUP_AUTH_METHOD_ID)
|
||||
assert terminal["type"] == "terminal"
|
||||
assert terminal["args"] == ["--setup"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_advertises_terminal_setup_auth_when_no_provider(self, agent, monkeypatch):
|
||||
monkeypatch.setattr("acp_adapter.auth.detect_provider", lambda: None)
|
||||
monkeypatch.setattr("acp_adapter.server.detect_provider", lambda: None)
|
||||
|
||||
resp = await agent.initialize(protocol_version=1)
|
||||
payloads = [method.model_dump(by_alias=True, exclude_none=True) for method in resp.auth_methods]
|
||||
|
||||
assert payloads == [
|
||||
{
|
||||
"args": ["--setup"],
|
||||
"description": (
|
||||
"Open Hermes' interactive model/provider setup in a terminal. "
|
||||
"Use this when Hermes has not been configured on this machine yet."
|
||||
),
|
||||
"id": TERMINAL_SETUP_AUTH_METHOD_ID,
|
||||
"name": "Configure Hermes provider",
|
||||
"type": "terminal",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# authenticate
|
||||
@@ -135,6 +171,24 @@ class TestAuthenticate:
|
||||
resp = await agent.authenticate(method_id="openrouter")
|
||||
assert resp is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticate_accepts_terminal_setup_after_provider_configured(self, agent, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"acp_adapter.server.detect_provider",
|
||||
lambda: "openrouter",
|
||||
)
|
||||
resp = await agent.authenticate(method_id=TERMINAL_SETUP_AUTH_METHOD_ID)
|
||||
assert isinstance(resp, AuthenticateResponse)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticate_rejects_terminal_setup_without_provider(self, agent, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"acp_adapter.server.detect_provider",
|
||||
lambda: None,
|
||||
)
|
||||
resp = await agent.authenticate(method_id=TERMINAL_SETUP_AUTH_METHOD_ID)
|
||||
assert resp is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# new_session / cancel / load / resume
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Tests for cross-edit LSP delta filtering.
|
||||
|
||||
The delta-filter contract spans three pieces:
|
||||
|
||||
1. ``agent.lsp.manager._diag_key`` — strict equality key including
|
||||
the diagnostic's position range. Two diagnostics with the same
|
||||
content but different lines are NOT equal under this key (they
|
||||
are genuinely different diagnostics).
|
||||
2. ``agent.lsp.range_shift.build_line_shift`` — derives a function
|
||||
mapping pre-edit line numbers to post-edit line numbers from a
|
||||
pre/post text pair.
|
||||
3. ``agent.lsp.manager.LSPService.get_diagnostics_sync(line_shift=…)``
|
||||
— applies the shift to baseline diagnostics before computing the
|
||||
set-difference, so pre-existing errors at shifted lines hash
|
||||
equal to their post-edit counterparts and get filtered out.
|
||||
|
||||
These tests exercise the contract at the unit level; the E2E case
|
||||
(real LSP server, real shift) is covered in test_service.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from agent.lsp.client import _diagnostic_key
|
||||
from agent.lsp.manager import _diag_key
|
||||
from agent.lsp.range_shift import (
|
||||
build_line_shift,
|
||||
shift_baseline,
|
||||
shift_diagnostic_range,
|
||||
)
|
||||
|
||||
|
||||
def _diag(*, line: int, message: str = "Undefined variable",
|
||||
severity: int = 1, code: str = "reportUndefinedVariable",
|
||||
source: str = "Pyright", end_line: int | None = None) -> dict:
|
||||
if end_line is None:
|
||||
end_line = line
|
||||
return {
|
||||
"severity": severity,
|
||||
"code": code,
|
||||
"source": source,
|
||||
"message": message,
|
||||
"range": {
|
||||
"start": {"line": line, "character": 0},
|
||||
"end": {"line": end_line, "character": 10},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# _diag_key: strict equality (with range)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_diag_key_treats_shifted_diagnostics_as_distinct():
|
||||
"""Two diagnostics with the same message but at different lines hash
|
||||
differently — they are genuinely different diagnostics. The shift
|
||||
map is what makes them equal AFTER remapping; the key itself stays
|
||||
strict."""
|
||||
a = _diag(line=100)
|
||||
b = _diag(line=200)
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_matches_client_key_for_shifted_baseline():
|
||||
"""When a baseline diagnostic is remapped through a shift, its
|
||||
_diag_key must match the corresponding post-edit diagnostic's key
|
||||
at the same coordinates. This is the contract the delta filter
|
||||
relies on."""
|
||||
pre = _diag(line=200)
|
||||
# Edit deletes 14 lines above line 200, so the same error now
|
||||
# appears at line 186 post-edit.
|
||||
shift = lambda L: L - 14 if L >= 14 else L
|
||||
shifted = shift_diagnostic_range(pre, shift)
|
||||
assert shifted is not None
|
||||
post = _diag(line=186)
|
||||
assert _diag_key(shifted) == _diag_key(post)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_message():
|
||||
a = _diag(line=100, message="foo")
|
||||
b = _diag(line=100, message="bar")
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_severity():
|
||||
a = _diag(line=100, severity=1)
|
||||
b = _diag(line=100, severity=2)
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_source():
|
||||
a = _diag(line=100, source="Pyright")
|
||||
b = _diag(line=100, source="Ruff")
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_matches_client_key_byte_for_byte():
|
||||
"""The manager-side and client-side keys must agree on diagnostic
|
||||
identity — they're used by two layers that need to round-trip the
|
||||
same diagnostics through dedup and delta filtering."""
|
||||
d = _diag(line=42)
|
||||
assert _diag_key(d) == _diagnostic_key(d)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# build_line_shift
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_shift_identity_for_identical_content():
|
||||
shift = build_line_shift("a\nb\nc\n", "a\nb\nc\n")
|
||||
assert shift(0) == 0
|
||||
assert shift(1) == 1
|
||||
assert shift(2) == 2
|
||||
|
||||
|
||||
def test_shift_pure_deletion_above_line():
|
||||
"""Delete 2 lines at the top; everything below shifts up by 2."""
|
||||
pre = "line0\nline1\nline2\nline3\nline4\n"
|
||||
post = "line2\nline3\nline4\n" # deleted lines 0-1
|
||||
shift = build_line_shift(pre, post)
|
||||
# Pre lines 0,1 → deleted → None
|
||||
assert shift(0) is None
|
||||
assert shift(1) is None
|
||||
# Pre line 2 → post line 0
|
||||
assert shift(2) == 0
|
||||
# Pre line 4 → post line 2
|
||||
assert shift(4) == 2
|
||||
|
||||
|
||||
def test_shift_pure_insertion_above_line():
|
||||
"""Insert 3 lines at the top; everything below shifts down by 3."""
|
||||
pre = "line0\nline1\nline2\n"
|
||||
post = "new0\nnew1\nnew2\nline0\nline1\nline2\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
# Pre lines unchanged in identity, shifted by 3
|
||||
assert shift(0) == 3
|
||||
assert shift(1) == 4
|
||||
assert shift(2) == 5
|
||||
|
||||
|
||||
def test_shift_replacement_in_middle():
|
||||
"""Replace 2 lines in the middle with 1 line. Lines above
|
||||
unchanged; lines below shift up by 1."""
|
||||
pre = "a\nb\nc\nd\ne\n"
|
||||
post = "a\nb\nX\ne\n" # replaced lines 2,3 (c,d) with X
|
||||
shift = build_line_shift(pre, post)
|
||||
assert shift(0) == 0 # a → a
|
||||
assert shift(1) == 1 # b → b
|
||||
assert shift(2) is None # c → deleted
|
||||
assert shift(3) is None # d → deleted
|
||||
assert shift(4) == 3 # e → post line 3
|
||||
|
||||
|
||||
def test_shift_handles_empty_pre():
|
||||
"""First write of a file: pre is empty, post has content. Nothing
|
||||
to shift, so the function should be well-defined for empty pre."""
|
||||
shift = build_line_shift("", "hello\nworld\n")
|
||||
# Any pre line falls past the end of an empty pre — anchor at end of post
|
||||
assert shift(0) == 1
|
||||
|
||||
|
||||
def test_shift_handles_empty_post():
|
||||
"""File deleted to empty. Every pre line returns None."""
|
||||
shift = build_line_shift("line0\nline1\n", "")
|
||||
assert shift(0) is None
|
||||
assert shift(1) is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# shift_diagnostic_range
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_shift_diag_remaps_start_and_end():
|
||||
pre = "a\nb\nc\nd\n"
|
||||
post = "X\na\nb\nc\nd\n" # one line inserted at top
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=2, end_line=2)
|
||||
remapped = shift_diagnostic_range(d, shift)
|
||||
assert remapped is not None
|
||||
assert remapped["range"]["start"]["line"] == 3
|
||||
assert remapped["range"]["end"]["line"] == 3
|
||||
|
||||
|
||||
def test_shift_diag_drops_diagnostic_in_deleted_region():
|
||||
pre = "a\nb\nc\nd\n"
|
||||
post = "a\nd\n" # deleted lines 1,2 (b,c)
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=1)
|
||||
assert shift_diagnostic_range(d, shift) is None
|
||||
|
||||
|
||||
def test_shift_diag_does_not_mutate_original():
|
||||
pre = "a\nb\n"
|
||||
post = "X\na\nb\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=0)
|
||||
original_line = d["range"]["start"]["line"]
|
||||
_ = shift_diagnostic_range(d, shift)
|
||||
assert d["range"]["start"]["line"] == original_line
|
||||
|
||||
|
||||
def test_shift_baseline_drops_deleted_and_remaps_rest():
|
||||
pre = "a\nb\nc\nd\ne\n"
|
||||
post = "a\ne\n" # deleted b,c,d
|
||||
shift = build_line_shift(pre, post)
|
||||
baseline = [
|
||||
_diag(line=0, message="err on a"),
|
||||
_diag(line=1, message="err on b"), # → deleted
|
||||
_diag(line=2, message="err on c"), # → deleted
|
||||
_diag(line=4, message="err on e"),
|
||||
]
|
||||
out = shift_baseline(baseline, shift)
|
||||
assert [d["message"] for d in out] == ["err on a", "err on e"]
|
||||
assert out[0]["range"]["start"]["line"] == 0
|
||||
assert out[1]["range"]["start"]["line"] == 1
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# End-to-end: simulate the delta-filter pipeline
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_pipeline_filters_shifted_baseline_under_strict_key():
|
||||
"""The exact scenario the bug fix is for: an edit deletes lines,
|
||||
every diagnostic below shifts, and the delta filter (strict key
|
||||
+ shifted baseline) correctly identifies them as pre-existing."""
|
||||
pre = "line0\nline1\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\n"
|
||||
# Delete lines 2,3,4 — pre-existing errors at lines 7,8 should
|
||||
# appear at lines 4,5 post-edit and be filtered out.
|
||||
post = "line0\nline1\nline5\nline6\nline7\nline8\nline9\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
|
||||
baseline = [_diag(line=7, message="X"), _diag(line=8, message="Y")]
|
||||
post_diags = [_diag(line=4, message="X"), _diag(line=5, message="Y")]
|
||||
|
||||
shifted_baseline = shift_baseline(baseline, shift)
|
||||
seen = {_diag_key(d) for d in shifted_baseline}
|
||||
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
|
||||
|
||||
# Both errors were pre-existing — filtered out.
|
||||
assert new_diags == []
|
||||
|
||||
|
||||
def test_pipeline_preserves_new_instance_at_different_line():
|
||||
"""The case content-only keys would miss: the model introduces a
|
||||
SECOND instance of the same error class at a new location. The
|
||||
new instance must surface."""
|
||||
pre = "good\ngood\ngood\n"
|
||||
post = "good\nbad\ngood\nbad\n" # added 2 new error lines
|
||||
shift = build_line_shift(pre, post)
|
||||
|
||||
baseline = [_diag(line=0, message="bad style")] # pre-existing
|
||||
post_diags = [
|
||||
_diag(line=0, message="bad style"), # pre-existing
|
||||
_diag(line=1, message="bad style"), # NEW — different line
|
||||
_diag(line=3, message="bad style"), # NEW — different line
|
||||
]
|
||||
|
||||
shifted_baseline = shift_baseline(baseline, shift)
|
||||
seen = {_diag_key(d) for d in shifted_baseline}
|
||||
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
|
||||
|
||||
# Two genuinely new instances must be surfaced.
|
||||
assert len(new_diags) == 2
|
||||
assert {d["range"]["start"]["line"] for d in new_diags} == {1, 3}
|
||||
@@ -130,6 +130,35 @@ def test_service_e2e_delta_filter(mock_pyright):
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_service_e2e_delta_filter_with_line_shift(mock_pyright):
|
||||
"""End-to-end: an edit that shifts the diagnostic's line still
|
||||
filters correctly when ``line_shift`` is supplied.
|
||||
|
||||
The mock LSP server emits a fixed error at line 0; for this test
|
||||
we don't need to actually shift the server's output — we just
|
||||
need to prove that supplying a line_shift through the API works
|
||||
and doesn't break the existing delta path. The unit tests in
|
||||
test_delta_key.py cover the shift semantics in detail.
|
||||
"""
|
||||
repo = mock_pyright
|
||||
f = repo / "x.py"
|
||||
f.write_text("print('hi')\n")
|
||||
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=3.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
svc.snapshot_baseline(str(f))
|
||||
# Identity shift — should behave exactly like no shift.
|
||||
new_diags = svc.get_diagnostics_sync(str(f), line_shift=lambda L: L)
|
||||
assert new_diags == []
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_service_status_includes_clients(mock_pyright):
|
||||
repo = mock_pyright
|
||||
f = repo / "x.py"
|
||||
|
||||
@@ -12,12 +12,24 @@ Covers:
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from contextlib import contextmanager
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch, PropertyMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _mock_botocore_session(*, return_value=None, side_effect=None):
|
||||
"""Patch botocore.session even when botocore is not installed."""
|
||||
botocore_mod = ModuleType("botocore")
|
||||
session_mod = ModuleType("botocore.session")
|
||||
session_mod.get_session = MagicMock(return_value=return_value, side_effect=side_effect)
|
||||
botocore_mod.session = session_mod
|
||||
with patch.dict("sys.modules", {"botocore": botocore_mod, "botocore.session": session_mod}):
|
||||
yield session_mod.get_session
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AWS credential detection
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -120,7 +132,7 @@ class TestResolveBedrocRegion:
|
||||
from unittest.mock import patch, MagicMock
|
||||
mock_session = MagicMock()
|
||||
mock_session.get_config_variable.return_value = None
|
||||
with patch("botocore.session.get_session", return_value=mock_session):
|
||||
with _mock_botocore_session(return_value=mock_session):
|
||||
assert resolve_bedrock_region({}) == "us-east-1"
|
||||
|
||||
def test_falls_back_to_botocore_profile_region(self):
|
||||
@@ -128,13 +140,13 @@ class TestResolveBedrocRegion:
|
||||
from unittest.mock import patch, MagicMock
|
||||
mock_session = MagicMock()
|
||||
mock_session.get_config_variable.return_value = "eu-central-1"
|
||||
with patch("botocore.session.get_session", return_value=mock_session):
|
||||
with _mock_botocore_session(return_value=mock_session):
|
||||
assert resolve_bedrock_region({}) == "eu-central-1"
|
||||
|
||||
def test_botocore_failure_falls_back_to_us_east_1(self):
|
||||
from agent.bedrock_adapter import resolve_bedrock_region
|
||||
from unittest.mock import patch
|
||||
with patch("botocore.session.get_session", side_effect=Exception("no botocore")):
|
||||
with _mock_botocore_session(side_effect=Exception("no botocore")):
|
||||
assert resolve_bedrock_region({}) == "us-east-1"
|
||||
|
||||
|
||||
|
||||
@@ -253,20 +253,24 @@ class TestErrorClassifierBedrock:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPackaging:
|
||||
"""Verify bedrock optional dependency is declared."""
|
||||
"""Verify Bedrock remains a declared lazy optional dependency."""
|
||||
|
||||
@staticmethod
|
||||
def _optional_dependencies():
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
content = (Path(__file__).parent.parent.parent / "pyproject.toml").read_text()
|
||||
return tomllib.loads(content)["project"]["optional-dependencies"]
|
||||
|
||||
def test_bedrock_extra_exists(self):
|
||||
import configparser
|
||||
from pathlib import Path
|
||||
# Read pyproject.toml to verify [bedrock] extra
|
||||
toml_path = Path(__file__).parent.parent.parent / "pyproject.toml"
|
||||
content = toml_path.read_text()
|
||||
assert 'bedrock = ["boto3' in content
|
||||
extras = self._optional_dependencies()
|
||||
assert "bedrock" in extras
|
||||
assert any(dep.startswith("boto3==") for dep in extras["bedrock"])
|
||||
|
||||
def test_bedrock_in_all_extra(self):
|
||||
from pathlib import Path
|
||||
content = (Path(__file__).parent.parent.parent / "pyproject.toml").read_text()
|
||||
assert '"hermes-agent[bedrock]"' in content
|
||||
def test_bedrock_is_not_eager_installed_by_all_extra(self):
|
||||
extras = self._optional_dependencies()
|
||||
assert "hermes-agent[bedrock]" not in extras["all"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -991,9 +991,12 @@ class TestCompressWithClient:
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=2)
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=2)
|
||||
|
||||
# Last head message (index 2) is "user" → summary should be "assistant"
|
||||
# NOTE: protect_first_n=2 preserves 2 non-system messages in addition to
|
||||
# the system prompt (always implicitly protected), yielding head [system,
|
||||
# user, user] with last head = user.
|
||||
msgs = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": "msg 1"},
|
||||
@@ -1059,11 +1062,13 @@ class TestCompressWithClient:
|
||||
mock_response.choices[0].message.content = "summary text"
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=3)
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=3)
|
||||
|
||||
# Head: [system, user, assistant] → last head = assistant
|
||||
# Tail: [user, assistant, user] → first tail = user
|
||||
# summary_role="user" collides with tail, "assistant" collides with head → merge
|
||||
# NOTE: protect_first_n=2 preserves 2 non-system messages in addition to
|
||||
# the system prompt (always implicitly protected).
|
||||
msgs = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": "msg 1"},
|
||||
@@ -1097,7 +1102,7 @@ class TestCompressWithClient:
|
||||
mock_response.choices[0].message.content = "summary text"
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=3)
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=3)
|
||||
|
||||
msgs = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
@@ -1133,13 +1138,15 @@ class TestCompressWithClient:
|
||||
mock_response.choices[0].message.content = "summary text"
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=2)
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=1, protect_last_n=2)
|
||||
|
||||
# Head: [system, user] → last head = user
|
||||
# Tail: [assistant, user, assistant] → first tail = assistant
|
||||
# summary_role="assistant" collides with tail, "user" collides with head → merge
|
||||
# NOTE: protect_first_n=1 preserves 1 non-system message in addition to
|
||||
# the system prompt (always implicitly protected).
|
||||
# With min_tail=3, tail = last 3 messages (indices 5-7).
|
||||
# Need 8 messages: min_for_compress = 2+3+1 = 6, must have > 6.
|
||||
# Need 8 messages: _min_for_compress = head(2) + 3 + 1 = 6, must have > 6.
|
||||
msgs = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": "msg 1"},
|
||||
@@ -1292,6 +1299,92 @@ class TestSummaryTargetRatio:
|
||||
c = ContextCompressor(model="test", quiet_mode=True)
|
||||
assert c.protect_last_n == 20
|
||||
|
||||
def test_default_protect_first_n_is_3(self):
|
||||
"""Default protect_first_n is 3 (system + 3 extra non-system messages =
|
||||
4 protected messages total when a system prompt is present). With the
|
||||
new semantics, the constructor default is 3 — the system prompt is
|
||||
always implicitly protected ON TOP OF protect_first_n non-system
|
||||
messages.
|
||||
"""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True)
|
||||
assert c.protect_first_n == 3
|
||||
|
||||
def test_protect_first_n_override(self):
|
||||
"""protect_first_n=0 should be honoured — for users who rely on rolling
|
||||
compaction and want NOTHING pinned at head except the system prompt
|
||||
(always implicitly protected)."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=0)
|
||||
assert c.protect_first_n == 0
|
||||
|
||||
def test_protect_first_n_0_preserves_only_system_prompt(self):
|
||||
"""End-to-end: when protect_first_n=0, compression should treat only
|
||||
the system prompt as head. All user/assistant messages between the
|
||||
system prompt and the protected tail become summarization candidates.
|
||||
|
||||
This is the cleanest configuration for long-running rolling-compaction
|
||||
sessions — no user/assistant turn gets pinned verbatim forever just
|
||||
because it happened to be early in the session."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(
|
||||
model="test",
|
||||
quiet_mode=True,
|
||||
protect_first_n=0,
|
||||
protect_last_n=2,
|
||||
)
|
||||
msgs = (
|
||||
[{"role": "system", "content": "System prompt"}]
|
||||
+ [{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg {i}"}
|
||||
for i in range(8)]
|
||||
)
|
||||
result = c.compress(msgs)
|
||||
# System prompt (msg[0]) survives as head
|
||||
assert result[0]["role"] == "system"
|
||||
assert result[0]["content"].startswith("System prompt")
|
||||
# The first user/assistant exchange (msg 0, msg 1) should NOT be pinned
|
||||
# as head verbatim — those would have been summarized or absorbed.
|
||||
# Under default protect_first_n=3, result[1..3] would be the literal
|
||||
# "msg 0" / "msg 1" / "msg 2"; with protect_first_n=0 they aren't.
|
||||
assert result[1].get("content") != "msg 0"
|
||||
# Last 2 messages are tail-protected under protect_last_n=2
|
||||
assert result[-1]["content"] == msgs[-1]["content"]
|
||||
|
||||
def test_protect_first_n_semantics_stable_without_system_prompt(self):
|
||||
"""Regression: gateway /compress handler strips the system prompt
|
||||
before calling compress(). protect_first_n must mean the same thing
|
||||
in both paths — "N non-system head messages" — so configuring
|
||||
protect_first_n=0 preserves NOTHING at the head regardless of whether
|
||||
the system prompt is in the messages list.
|
||||
|
||||
Bug this covers: under the old semantics, protect_first_n counted
|
||||
literally from messages[0]. In the gateway path (no system prompt)
|
||||
that meant protect_first_n=1 would pin the first user turn of the
|
||||
session forever — a user-reported complaint that a week-old
|
||||
resolved question kept getting reinserted into every compaction
|
||||
summary."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(
|
||||
model="test",
|
||||
quiet_mode=True,
|
||||
protect_first_n=0,
|
||||
protect_last_n=2,
|
||||
)
|
||||
# No system prompt — this is what the gateway passes to compress().
|
||||
msgs = [
|
||||
{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg {i}"}
|
||||
for i in range(10)
|
||||
]
|
||||
head_size = c._protect_head_size(msgs)
|
||||
# With no system prompt and protect_first_n=0 → head is empty.
|
||||
# The first user message is NOT pinned as head.
|
||||
assert head_size == 0
|
||||
|
||||
# And with protect_first_n=3 on the same no-system-prompt list →
|
||||
# head size is 3 (the three earliest non-system messages).
|
||||
c.protect_first_n = 3
|
||||
assert c._protect_head_size(msgs) == 3
|
||||
|
||||
|
||||
class TestTokenBudgetTailProtection:
|
||||
"""Tests for token-budget-based tail protection (PR #6240).
|
||||
|
||||
@@ -27,10 +27,12 @@ def _messages_with_handoff(summary_body: str):
|
||||
return [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": f"{SUMMARY_PREFIX}\n{summary_body}"},
|
||||
{"role": "assistant", "content": "handoff acknowledged after resume"},
|
||||
{"role": "user", "content": "new user turn after resume"},
|
||||
{"role": "assistant", "content": "new assistant work after resume"},
|
||||
{"role": "user", "content": "more new work after resume"},
|
||||
{"role": "assistant", "content": "latest tail response"},
|
||||
{"role": "user", "content": "final active request stays in protected tail"},
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Tests for agent/display.py — build_tool_preview() and inline diff previews."""
|
||||
|
||||
import os
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -149,6 +150,27 @@ class TestCuteToolMessagePreviewLength:
|
||||
assert path in line
|
||||
assert "..." not in line
|
||||
|
||||
def test_write_file_lint_error_result_is_not_marked_failed(self):
|
||||
result = json.dumps({
|
||||
"bytes_written": 12,
|
||||
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
|
||||
})
|
||||
|
||||
line = get_cute_tool_message("write_file", {"path": "/tmp/a.py"}, 0.1, result=result)
|
||||
|
||||
assert "[error]" not in line
|
||||
|
||||
def test_patch_lsp_diagnostics_result_is_not_marked_failed(self):
|
||||
result = json.dumps({
|
||||
"success": True,
|
||||
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
|
||||
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
|
||||
})
|
||||
|
||||
line = get_cute_tool_message("patch", {"path": "/tmp/a.py"}, 0.1, result=result)
|
||||
|
||||
assert "[error]" not in line
|
||||
|
||||
|
||||
class TestEditDiffPreview:
|
||||
def test_extract_edit_diff_for_patch(self):
|
||||
|
||||
@@ -913,6 +913,35 @@ class TestTranslateStreamEvent:
|
||||
assert chunks[-1].choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
class TestMakeStreamChunk:
|
||||
def test_reasoning_only_chunk_has_content_none(self):
|
||||
from agent.gemini_cloudcode_adapter import _make_stream_chunk
|
||||
|
||||
chunk = _make_stream_chunk(model="m", reasoning="think")
|
||||
delta = chunk.choices[0].delta
|
||||
assert delta.content is None
|
||||
assert delta.reasoning == "think"
|
||||
|
||||
def test_content_only_chunk_has_reasoning_none(self):
|
||||
from agent.gemini_cloudcode_adapter import _make_stream_chunk
|
||||
|
||||
chunk = _make_stream_chunk(model="m", content="hello")
|
||||
delta = chunk.choices[0].delta
|
||||
assert delta.content == "hello"
|
||||
assert delta.reasoning is None
|
||||
assert delta.tool_calls is None
|
||||
|
||||
def test_finish_only_chunk_has_all_fields_none(self):
|
||||
from agent.gemini_cloudcode_adapter import _make_stream_chunk
|
||||
|
||||
chunk = _make_stream_chunk(model="m", finish_reason="stop")
|
||||
delta = chunk.choices[0].delta
|
||||
assert delta.content is None
|
||||
assert delta.reasoning is None
|
||||
assert delta.tool_calls is None
|
||||
assert chunk.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
class TestGeminiCloudCodeClient:
|
||||
def test_client_exposes_openai_interface(self):
|
||||
from agent.gemini_cloudcode_adapter import GeminiCloudCodeClient
|
||||
|
||||
@@ -7,6 +7,7 @@ from agent.tool_guardrails import (
|
||||
ToolCallGuardrailController,
|
||||
ToolCallSignature,
|
||||
canonical_tool_args,
|
||||
classify_tool_failure,
|
||||
)
|
||||
|
||||
|
||||
@@ -131,6 +132,21 @@ def test_success_resets_exact_signature_failure_streak():
|
||||
assert controller.before_call("web_search", args).action == "allow"
|
||||
|
||||
|
||||
def test_file_mutation_lint_error_result_is_not_a_tool_failure():
|
||||
write_result = json.dumps({
|
||||
"bytes_written": 12,
|
||||
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
|
||||
})
|
||||
patch_result = json.dumps({
|
||||
"success": True,
|
||||
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
|
||||
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
|
||||
})
|
||||
|
||||
assert classify_tool_failure("write_file", write_result) == (False, "")
|
||||
assert classify_tool_failure("patch", patch_result) == (False, "")
|
||||
|
||||
|
||||
def test_same_tool_varying_args_warns_by_default_without_halting():
|
||||
controller = ToolCallGuardrailController(
|
||||
ToolCallGuardrailConfig(same_tool_failure_warn_after=2, same_tool_failure_halt_after=3)
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Tests for shared tool result classification helpers."""
|
||||
|
||||
import json
|
||||
|
||||
from agent.tool_result_classification import file_mutation_result_landed
|
||||
|
||||
|
||||
def test_write_file_with_nested_lint_error_counts_as_landed():
|
||||
result = json.dumps({
|
||||
"bytes_written": 12,
|
||||
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
|
||||
})
|
||||
|
||||
assert file_mutation_result_landed("write_file", result) is True
|
||||
|
||||
|
||||
def test_patch_with_nested_lsp_diagnostics_counts_as_landed():
|
||||
result = json.dumps({
|
||||
"success": True,
|
||||
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
|
||||
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
|
||||
})
|
||||
|
||||
assert file_mutation_result_landed("patch", result) is True
|
||||
|
||||
|
||||
def test_top_level_file_mutation_error_does_not_count_as_landed():
|
||||
result = json.dumps({"success": True, "error": "post-write verification failed"})
|
||||
|
||||
assert file_mutation_result_landed("patch", result) is False
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Tests for agent/video_gen_registry.py — provider registration & active lookup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
from agent.video_gen_provider import VideoGenProvider
|
||||
|
||||
|
||||
class _FakeProvider(VideoGenProvider):
|
||||
def __init__(self, name: str, available: bool = True):
|
||||
self._name = name
|
||||
self._available = available
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
def generate(self, prompt, **kw):
|
||||
return {"success": True, "video": f"{self._name}://{prompt}"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
class TestRegisterProvider:
|
||||
def test_register_and_lookup(self):
|
||||
provider = _FakeProvider("fake")
|
||||
video_gen_registry.register_provider(provider)
|
||||
assert video_gen_registry.get_provider("fake") is provider
|
||||
|
||||
def test_rejects_non_provider(self):
|
||||
with pytest.raises(TypeError):
|
||||
video_gen_registry.register_provider("not a provider") # type: ignore[arg-type]
|
||||
|
||||
def test_rejects_empty_name(self):
|
||||
class Empty(VideoGenProvider):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return ""
|
||||
|
||||
def generate(self, prompt, **kw):
|
||||
return {}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
video_gen_registry.register_provider(Empty())
|
||||
|
||||
def test_reregister_overwrites(self):
|
||||
a = _FakeProvider("same")
|
||||
b = _FakeProvider("same")
|
||||
video_gen_registry.register_provider(a)
|
||||
video_gen_registry.register_provider(b)
|
||||
assert video_gen_registry.get_provider("same") is b
|
||||
|
||||
def test_list_is_sorted(self):
|
||||
video_gen_registry.register_provider(_FakeProvider("zeta"))
|
||||
video_gen_registry.register_provider(_FakeProvider("alpha"))
|
||||
names = [p.name for p in video_gen_registry.list_providers()]
|
||||
assert names == ["alpha", "zeta"]
|
||||
|
||||
|
||||
class TestGetActiveProvider:
|
||||
def test_single_provider_autoresolves(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
video_gen_registry.register_provider(_FakeProvider("solo"))
|
||||
active = video_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "solo"
|
||||
|
||||
def test_no_provider_returns_none(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
assert video_gen_registry.get_active_provider() is None
|
||||
|
||||
def test_multi_without_config_returns_none(self, tmp_path, monkeypatch):
|
||||
"""Unlike image_gen (which falls back to 'fal'), video_gen has no
|
||||
legacy default — when there are multiple providers and no config,
|
||||
the registry returns None and the tool surfaces a helpful error.
|
||||
"""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
video_gen_registry.register_provider(_FakeProvider("xai"))
|
||||
video_gen_registry.register_provider(_FakeProvider("fal"))
|
||||
assert video_gen_registry.get_active_provider() is None
|
||||
|
||||
def test_config_selects_provider(self, tmp_path, monkeypatch):
|
||||
import yaml
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"video_gen": {"provider": "fal"}})
|
||||
)
|
||||
video_gen_registry.register_provider(_FakeProvider("xai"))
|
||||
video_gen_registry.register_provider(_FakeProvider("fal"))
|
||||
active = video_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "fal"
|
||||
|
||||
def test_unknown_config_falls_back(self, tmp_path, monkeypatch):
|
||||
"""If video_gen.provider names a provider that isn't registered,
|
||||
the single-provider fallback still applies."""
|
||||
import yaml
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"video_gen": {"provider": "ghost"}})
|
||||
)
|
||||
video_gen_registry.register_provider(_FakeProvider("only"))
|
||||
active = video_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "only"
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Tests for the optional codex app-server runtime gate.
|
||||
|
||||
These are unit tests for the api_mode rewriter and the wire-level transport
|
||||
module. They do NOT require the `codex` CLI to be installed — that's
|
||||
covered by a separate live test gated on `codex --version`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.runtime_provider import (
|
||||
_VALID_API_MODES,
|
||||
_maybe_apply_codex_app_server_runtime,
|
||||
)
|
||||
|
||||
|
||||
class TestApiModeRegistration:
|
||||
"""The new api_mode must be registered or downstream parsing rejects it."""
|
||||
|
||||
def test_codex_app_server_is_a_valid_api_mode(self) -> None:
|
||||
assert "codex_app_server" in _VALID_API_MODES
|
||||
|
||||
def test_existing_api_modes_still_present(self) -> None:
|
||||
# Regression guard: don't accidentally delete other api_modes when
|
||||
# touching this set.
|
||||
for mode in (
|
||||
"chat_completions",
|
||||
"codex_responses",
|
||||
"anthropic_messages",
|
||||
"bedrock_converse",
|
||||
):
|
||||
assert mode in _VALID_API_MODES
|
||||
|
||||
|
||||
class TestMaybeApplyCodexAppServerRuntime:
|
||||
"""The opt-in helper that rewrites api_mode → codex_app_server."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_cfg",
|
||||
[
|
||||
None,
|
||||
{},
|
||||
{"openai_runtime": ""},
|
||||
{"openai_runtime": "auto"},
|
||||
{"openai_runtime": "AUTO"},
|
||||
{"other_key": "codex_app_server"}, # wrong key
|
||||
],
|
||||
)
|
||||
def test_default_off_for_openai(self, model_cfg) -> None:
|
||||
"""Default behavior is preserved when the flag is unset/auto."""
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai", api_mode="chat_completions", model_cfg=model_cfg
|
||||
)
|
||||
assert got == "chat_completions"
|
||||
|
||||
def test_opt_in_rewrites_openai(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai",
|
||||
api_mode="chat_completions",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
def test_opt_in_rewrites_openai_codex(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai-codex",
|
||||
api_mode="codex_responses",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
def test_case_insensitive(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai",
|
||||
api_mode="chat_completions",
|
||||
model_cfg={"openai_runtime": "Codex_App_Server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider",
|
||||
[
|
||||
"anthropic",
|
||||
"openrouter",
|
||||
"xai",
|
||||
"qwen-oauth",
|
||||
"google-gemini-cli",
|
||||
"opencode-zen",
|
||||
"bedrock",
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_other_providers_never_rerouted(self, provider) -> None:
|
||||
"""Non-OpenAI providers MUST NOT be rerouted even with the flag set —
|
||||
codex's app-server can only run OpenAI/Codex auth flows."""
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider=provider,
|
||||
api_mode="anthropic_messages",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "anthropic_messages", (
|
||||
f"provider={provider!r} should not be rerouted to codex_app_server"
|
||||
)
|
||||
|
||||
|
||||
class TestCodexAppServerModule:
|
||||
"""Module-surface tests for the JSON-RPC speaker. Don't require codex CLI."""
|
||||
|
||||
def test_module_imports(self) -> None:
|
||||
from agent.transports import codex_app_server
|
||||
|
||||
assert codex_app_server.MIN_CODEX_VERSION >= (0, 1, 0)
|
||||
assert callable(codex_app_server.parse_codex_version)
|
||||
assert callable(codex_app_server.check_codex_binary)
|
||||
|
||||
def test_parse_codex_version_valid(self) -> None:
|
||||
from agent.transports.codex_app_server import parse_codex_version
|
||||
|
||||
assert parse_codex_version("codex-cli 0.130.0") == (0, 130, 0)
|
||||
assert parse_codex_version("codex-cli 1.2.3 (extra metadata)") == (1, 2, 3)
|
||||
assert parse_codex_version("codex 99.0.1\n") == (99, 0, 1)
|
||||
|
||||
def test_parse_codex_version_invalid(self) -> None:
|
||||
from agent.transports.codex_app_server import parse_codex_version
|
||||
|
||||
assert parse_codex_version("nope") is None
|
||||
assert parse_codex_version("") is None
|
||||
assert parse_codex_version(None) is None # type: ignore[arg-type]
|
||||
|
||||
def test_check_binary_handles_missing_executable(self) -> None:
|
||||
from agent.transports.codex_app_server import check_codex_binary
|
||||
|
||||
ok, msg = check_codex_binary(codex_bin="/nonexistent/codex/binary/path")
|
||||
assert ok is False
|
||||
assert "not found" in msg.lower() or "no such" in msg.lower()
|
||||
|
||||
def test_codex_error_class_is_runtimeerror(self) -> None:
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
err = CodexAppServerError(code=-32600, message="boom")
|
||||
assert isinstance(err, RuntimeError)
|
||||
assert "boom" in str(err)
|
||||
assert "-32600" in str(err)
|
||||
|
||||
|
||||
class TestSpawnEnvIsolation:
|
||||
"""The codex spawn must NOT rewrite HOME — codex's shell tool spawns
|
||||
subprocesses (gh, git, npm, aws, gcloud, ...) that need to find their
|
||||
config in the real user $HOME. CODEX_HOME isolates codex's own state,
|
||||
HOME stays unchanged.
|
||||
|
||||
OpenClaw hit this footgun (openclaw/openclaw#81562) — they were
|
||||
rewriting HOME to a synthetic per-agent dir alongside CODEX_HOME,
|
||||
and then `gh auth status` / git config / etc. all broke inside codex
|
||||
shell calls. We avoid the same bug by only overlaying CODEX_HOME and
|
||||
RUST_LOG on top of os.environ.copy().
|
||||
"""
|
||||
|
||||
def test_spawn_env_preserves_HOME(self, monkeypatch):
|
||||
"""The spawn env must contain the parent process's HOME unchanged.
|
||||
Verifies via a subprocess-monkey-patch."""
|
||||
import subprocess
|
||||
from agent.transports import codex_app_server as cas
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakePopen:
|
||||
def __init__(self, cmd, *args, **kwargs):
|
||||
captured["env"] = kwargs.get("env", {}).copy()
|
||||
# Provide minimal Popen surface so __init__ doesn't crash
|
||||
# on attribute access during construction.
|
||||
self.stdin = None
|
||||
self.stdout = None
|
||||
self.stderr = None
|
||||
self.pid = 1
|
||||
self.returncode = None
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", FakePopen)
|
||||
monkeypatch.setenv("HOME", "/users/alice")
|
||||
|
||||
client = cas.CodexAppServerClient(codex_bin="codex")
|
||||
client._closed = True # so close() is a no-op
|
||||
|
||||
# The spawn env must have HOME=/users/alice unchanged
|
||||
assert captured["env"].get("HOME") == "/users/alice", (
|
||||
f"HOME got rewritten in codex spawn env: "
|
||||
f"{captured['env'].get('HOME')!r}. Codex's shell tool's "
|
||||
"subprocesses (gh, git, aws, npm) need the user's real HOME."
|
||||
)
|
||||
|
||||
def test_spawn_env_sets_CODEX_HOME_when_provided(self, monkeypatch):
|
||||
"""CODEX_HOME isolation must still work — that's the whole point
|
||||
of the codex_home arg."""
|
||||
import subprocess
|
||||
from agent.transports import codex_app_server as cas
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakePopen:
|
||||
def __init__(self, cmd, *args, **kwargs):
|
||||
captured["env"] = kwargs.get("env", {}).copy()
|
||||
self.stdin = None
|
||||
self.stdout = None
|
||||
self.stderr = None
|
||||
self.pid = 1
|
||||
self.returncode = None
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", FakePopen)
|
||||
monkeypatch.setenv("HOME", "/users/alice")
|
||||
|
||||
client = cas.CodexAppServerClient(
|
||||
codex_bin="codex", codex_home="/tmp/profile/codex"
|
||||
)
|
||||
client._closed = True
|
||||
|
||||
assert captured["env"].get("CODEX_HOME") == "/tmp/profile/codex"
|
||||
# And HOME still passes through unchanged
|
||||
assert captured["env"].get("HOME") == "/users/alice"
|
||||
@@ -0,0 +1,976 @@
|
||||
"""Tests for CodexAppServerSession — drive turns through a mock client.
|
||||
|
||||
The session adapter has the most complex behavior of the three new modules:
|
||||
notification draining, server-request handling (approvals), interrupt,
|
||||
deadline timeouts. These tests pin all of that without spawning real codex.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.transports.codex_app_server_session import (
|
||||
CodexAppServerSession,
|
||||
TurnResult,
|
||||
_ServerRequestRouting,
|
||||
_approval_choice_to_codex_decision,
|
||||
)
|
||||
|
||||
|
||||
class FakeClient:
|
||||
"""Stand-in for CodexAppServerClient that records calls and lets the test
|
||||
drive the notification / server-request streams synchronously."""
|
||||
|
||||
def __init__(self, *, codex_bin: str = "codex", codex_home=None) -> None:
|
||||
self.codex_bin = codex_bin
|
||||
self.codex_home = codex_home
|
||||
self.requests: list[tuple[str, dict]] = []
|
||||
self.notifications_responses: list[dict] = []
|
||||
self.responses: list[tuple[Any, dict]] = []
|
||||
self.error_responses: list[tuple[Any, int, str]] = []
|
||||
self._initialized = False
|
||||
self._closed = False
|
||||
self._notifications: list[dict] = []
|
||||
self._server_requests: list[dict] = []
|
||||
self._request_handler = None # Optional[Callable[[str, dict], dict]]
|
||||
|
||||
# API matching CodexAppServerClient
|
||||
def initialize(self, **kwargs):
|
||||
self._initialized = True
|
||||
return {"userAgent": "fake/0.0.0", "codexHome": "/tmp",
|
||||
"platformOs": "linux", "platformFamily": "unix"}
|
||||
|
||||
def request(self, method: str, params: Optional[dict] = None, timeout: float = 30.0):
|
||||
self.requests.append((method, params or {}))
|
||||
if self._request_handler is not None:
|
||||
return self._request_handler(method, params or {})
|
||||
# Sensible defaults for protocol methods used by the session
|
||||
if method == "thread/start":
|
||||
return {"thread": {"id": "thread-fake-001"},
|
||||
"activePermissionProfile": {"id": "workspace-write"}}
|
||||
if method == "turn/start":
|
||||
return {"turn": {"id": "turn-fake-001"}}
|
||||
if method == "turn/interrupt":
|
||||
return {}
|
||||
return {}
|
||||
|
||||
def notify(self, method: str, params=None):
|
||||
pass
|
||||
|
||||
def respond(self, request_id, result):
|
||||
self.responses.append((request_id, result))
|
||||
|
||||
def respond_error(self, request_id, code, message, data=None):
|
||||
self.error_responses.append((request_id, code, message))
|
||||
|
||||
def take_notification(self, timeout: float = 0.0):
|
||||
if self._notifications:
|
||||
return self._notifications.pop(0)
|
||||
# Honor a tiny sleep so the loop doesn't hot-spin; the real client
|
||||
# blocks on a queue. For tests we want determinism.
|
||||
if timeout > 0:
|
||||
time.sleep(min(timeout, 0.001))
|
||||
return None
|
||||
|
||||
def take_server_request(self, timeout: float = 0.0):
|
||||
if self._server_requests:
|
||||
return self._server_requests.pop(0)
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
self._closed = True
|
||||
|
||||
def is_alive(self) -> bool:
|
||||
# Fake is "alive" until close() is called; tests that want a dead
|
||||
# subprocess can patch this attribute or call close() directly.
|
||||
return not self._closed
|
||||
|
||||
def stderr_tail(self, n: int = 20):
|
||||
return list(getattr(self, "_stderr_tail", []))[-n:]
|
||||
|
||||
# Test helpers
|
||||
def queue_notification(self, method: str, **params):
|
||||
self._notifications.append({"method": method, "params": params})
|
||||
|
||||
def queue_server_request(self, method: str, request_id: Any = "srv-1", **params):
|
||||
self._server_requests.append({"id": request_id, "method": method, "params": params})
|
||||
|
||||
def set_stderr_tail(self, lines):
|
||||
"""Test helper: seed stderr_tail() output for OAuth-refresh classifier tests."""
|
||||
self._stderr_tail = list(lines)
|
||||
|
||||
|
||||
def make_session(client: FakeClient, **kwargs) -> CodexAppServerSession:
|
||||
return CodexAppServerSession(
|
||||
cwd="/tmp",
|
||||
client_factory=lambda **kw: client,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ---- choice mapping ----
|
||||
|
||||
class TestApprovalChoiceMapping:
|
||||
@pytest.mark.parametrize("choice,expected", [
|
||||
("once", "accept"),
|
||||
("session", "acceptForSession"),
|
||||
("always", "acceptForSession"),
|
||||
("deny", "decline"),
|
||||
("anything-else", "decline"),
|
||||
])
|
||||
def test_mapping(self, choice, expected):
|
||||
assert _approval_choice_to_codex_decision(choice) == expected
|
||||
|
||||
|
||||
# ---- lifecycle ----
|
||||
|
||||
class TestLifecycle:
|
||||
def test_ensure_started_is_idempotent(self):
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
tid_a = s.ensure_started()
|
||||
tid_b = s.ensure_started()
|
||||
assert tid_a == tid_b == "thread-fake-001"
|
||||
# thread/start should be called exactly once
|
||||
method_calls = [m for (m, _) in client.requests if m == "thread/start"]
|
||||
assert len(method_calls) == 1
|
||||
|
||||
def test_thread_start_passes_cwd_only(self):
|
||||
"""thread/start carries cwd. We intentionally do NOT pass `permissions`
|
||||
on this codex version (experimentalApi-gated + requires matching
|
||||
config.toml [permissions] table). Letting codex use its default
|
||||
(read-only unless user configures otherwise) is the documented path."""
|
||||
client = FakeClient()
|
||||
s = make_session(client, permission_profile="workspace-write")
|
||||
s.ensure_started()
|
||||
method, params = next(r for r in client.requests if r[0] == "thread/start")
|
||||
assert params["cwd"] == "/tmp"
|
||||
assert "permissions" not in params # see session.ensure_started() comment
|
||||
|
||||
def test_close_idempotent(self):
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
s.ensure_started()
|
||||
s.close()
|
||||
s.close()
|
||||
assert client._closed is True
|
||||
|
||||
|
||||
# ---- turn loop ----
|
||||
|
||||
class TestRunTurn:
|
||||
def test_simple_text_turn_returns_final_message(self):
|
||||
client = FakeClient()
|
||||
client.queue_notification("turn/started", threadId="t", turn={"id": "tu1"})
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1", "text": "hello world"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed",
|
||||
threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.final_text == "hello world"
|
||||
assert r.interrupted is False
|
||||
assert r.error is None
|
||||
assert any(m["role"] == "assistant" and m.get("content") == "hello world"
|
||||
for m in r.projected_messages)
|
||||
# turn_id propagated for downstream session-DB linkage
|
||||
assert r.turn_id == "turn-fake-001"
|
||||
|
||||
def test_tool_iteration_counter_ticks(self):
|
||||
client = FakeClient()
|
||||
# Two completed exec items + one final agent message
|
||||
for i, item_id in enumerate(("ex1", "ex2"), start=1):
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={
|
||||
"type": "commandExecution", "id": item_id,
|
||||
"command": f"cmd{i}", "cwd": "/tmp",
|
||||
"status": "completed", "aggregatedOutput": "ok",
|
||||
"exitCode": 0, "commandActions": [],
|
||||
},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1", "text": "done"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("do stuff", turn_timeout=2.0)
|
||||
assert r.tool_iterations == 2
|
||||
# Each tool item produces (assistant, tool) — 2*2 + final assistant = 5 msgs
|
||||
assert len(r.projected_messages) == 5
|
||||
|
||||
def test_turn_start_failure_returns_error(self):
|
||||
client = FakeClient()
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
def boom(method, params):
|
||||
if method == "turn/start":
|
||||
raise CodexAppServerError(code=-32600, message="bad input")
|
||||
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.error is not None
|
||||
assert "bad input" in r.error
|
||||
assert r.final_text == ""
|
||||
|
||||
def test_turn_start_failure_attaches_redacted_stderr_tail(self):
|
||||
"""When codex stderr has content (non-OAuth), the tail gets attached
|
||||
to the user-facing error so config/provider problems are debuggable
|
||||
instead of just 'Internal error'. Secrets in stderr are redacted
|
||||
via agent.redact(force=True)."""
|
||||
client = FakeClient()
|
||||
client.set_stderr_tail([
|
||||
"ERROR: provider auth failed",
|
||||
"Authorization: Bearer sk-live-deadbeefdeadbeef",
|
||||
"url=https://api.example.com/v1?token=querysecret12345",
|
||||
])
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
def boom(method, params):
|
||||
if method == "turn/start":
|
||||
raise CodexAppServerError(code=-32603, message="Internal error")
|
||||
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.error is not None
|
||||
assert "turn/start failed" in r.error
|
||||
assert "Internal error" in r.error
|
||||
# Stderr tail attached
|
||||
assert "codex stderr" in r.error
|
||||
assert "provider auth failed" in r.error
|
||||
# Secrets redacted
|
||||
assert "sk-live-deadbeefdeadbeef" not in r.error
|
||||
assert "querysecret12345" not in r.error
|
||||
# Non-OAuth → should NOT retire (subprocess JSON-RPC is still healthy).
|
||||
assert r.should_retire is False
|
||||
|
||||
def test_turn_start_timeout_attaches_redacted_stderr_tail(self):
|
||||
"""A non-OAuth TimeoutError on turn/start surfaces with codex stderr
|
||||
context attached and marks the session for retirement."""
|
||||
client = FakeClient()
|
||||
client.set_stderr_tail([
|
||||
"WARN: provider request stalled",
|
||||
"Authorization: Bearer sk-stalled-secret-abc123",
|
||||
])
|
||||
|
||||
def stall(method, params):
|
||||
if method == "turn/start":
|
||||
raise TimeoutError("codex method 'turn/start' timed out after 10s")
|
||||
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = stall
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.error is not None
|
||||
assert "turn/start timed out" in r.error
|
||||
assert "provider request stalled" in r.error
|
||||
assert "sk-stalled-secret-abc123" not in r.error
|
||||
assert r.should_retire is True
|
||||
|
||||
def test_startup_failure_returns_error_with_stderr(self):
|
||||
"""Codex thread/start failures during ensure_started() used to bubble
|
||||
up as uncaught exceptions. Now they return a TurnResult.error so
|
||||
AIAgent surfaces a clean diagnostic instead of crashing the turn."""
|
||||
client = FakeClient()
|
||||
client.set_stderr_tail([
|
||||
"FATAL: model_provider 'azure_foundry' not configured",
|
||||
])
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
def boom(method, params):
|
||||
if method == "thread/start":
|
||||
raise CodexAppServerError(code=-32603, message="Internal error")
|
||||
return {}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.error is not None
|
||||
assert "startup failed" in r.error
|
||||
assert "model_provider 'azure_foundry' not configured" in r.error
|
||||
assert r.should_retire is True
|
||||
assert r.final_text == ""
|
||||
|
||||
def test_interrupt_during_turn_issues_turn_interrupt(self):
|
||||
client = FakeClient()
|
||||
# Don't queue turn/completed — the loop has to interrupt out
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "commandExecution", "id": "x", "command": "sleep 60",
|
||||
"cwd": "/", "status": "inProgress",
|
||||
"aggregatedOutput": None, "exitCode": None,
|
||||
"commandActions": []},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
s = make_session(client)
|
||||
s.ensure_started()
|
||||
# Trip the interrupt before run_turn even consumes the notification.
|
||||
# The loop will see interrupt set on its first iteration and bail.
|
||||
s.request_interrupt()
|
||||
r = s.run_turn("loop forever", turn_timeout=2.0)
|
||||
assert r.interrupted is True
|
||||
# turn/interrupt was requested with the right turnId
|
||||
assert any(
|
||||
method == "turn/interrupt" and params.get("turnId") == "turn-fake-001"
|
||||
for (method, params) in client.requests
|
||||
)
|
||||
|
||||
def test_deadline_exceeded_records_error(self):
|
||||
client = FakeClient()
|
||||
# No notifications and no completion → must hit deadline
|
||||
s = make_session(client)
|
||||
r = s.run_turn("never finishes", turn_timeout=0.05,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.interrupted is True
|
||||
assert r.error and "timed out" in r.error
|
||||
|
||||
def test_failed_turn_records_error_from_turn_completed(self):
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "failed",
|
||||
"error": {"message": "model error"}},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("x", turn_timeout=1.0)
|
||||
assert r.error and "model error" in r.error
|
||||
|
||||
|
||||
# ---- approval bridge ----
|
||||
|
||||
class TestServerRequestRouting:
|
||||
def test_exec_approval_with_callback_approves_once(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"item/commandExecution/requestApproval", request_id="req-1",
|
||||
command="ls /tmp", cwd="/tmp",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
captured["command"] = command
|
||||
captured["description"] = description
|
||||
return "once"
|
||||
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert captured["command"] == "ls /tmp"
|
||||
# The session must have responded to the server request with "accept"
|
||||
assert ("req-1", {"decision": "accept"}) in client.responses
|
||||
|
||||
def test_exec_approval_no_callback_denies(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request("item/commandExecution/requestApproval", request_id="req-1",
|
||||
command="rm -rf /", cwd="/")
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client) # no approval_callback wired
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("req-1", {"decision": "decline"}) in client.responses
|
||||
|
||||
def test_apply_patch_approval_session_maps_to_session_decision(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"item/fileChange/requestApproval", request_id="req-2",
|
||||
itemId="fc-1",
|
||||
turnId="t1",
|
||||
threadId="th",
|
||||
startedAtMs=1234567890,
|
||||
reason="create new file with hello() function",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
return "session"
|
||||
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("req-2", {"decision": "acceptForSession"}) in client.responses
|
||||
|
||||
def test_unknown_server_request_replied_with_error(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request("totally/unknown", request_id="req-3")
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert any(
|
||||
rid == "req-3" and code == -32601
|
||||
for (rid, code, _msg) in client.error_responses
|
||||
)
|
||||
|
||||
def test_mcp_elicitation_for_hermes_tools_auto_accepts(self):
|
||||
"""When codex elicits on behalf of hermes-tools (our own callback),
|
||||
accept automatically — the user already opted in by enabling the
|
||||
runtime."""
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"mcpServer/elicitation/request", request_id="elic-1",
|
||||
threadId="t", turnId="tu1",
|
||||
serverName="hermes-tools",
|
||||
mode="form",
|
||||
message="confirm",
|
||||
requestedSchema={"type": "object", "properties": {}},
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("elic-1", {"action": "accept", "content": None, "_meta": None}) in client.responses
|
||||
|
||||
def test_mcp_elicitation_for_other_servers_declines(self):
|
||||
"""For third-party MCP servers we decline by default so users
|
||||
explicitly opt in through codex's own UI."""
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"mcpServer/elicitation/request", request_id="elic-2",
|
||||
threadId="t", turnId="tu1",
|
||||
serverName="some-third-party",
|
||||
mode="url",
|
||||
message="please log in",
|
||||
url="https://example.com/oauth",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("elic-2", {"action": "decline", "content": None, "_meta": None}) in client.responses
|
||||
|
||||
def test_routing_auto_approve_bypass(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request("item/commandExecution/requestApproval", request_id="r1",
|
||||
command="ls", cwd="/")
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
# No callback, but routing says auto-approve. Should approve.
|
||||
s = make_session(client, request_routing=_ServerRequestRouting(
|
||||
auto_approve_exec=True))
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("r1", {"decision": "accept"}) in client.responses
|
||||
|
||||
def test_callback_raises_falls_back_to_decline(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request("item/commandExecution/requestApproval", request_id="r1",
|
||||
command="ls", cwd="/")
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
|
||||
def boom(*a, **kw):
|
||||
raise RuntimeError("ui crashed")
|
||||
|
||||
s = make_session(client, approval_callback=boom)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
# Fail-closed: deny on callback exception
|
||||
assert ("r1", {"decision": "decline"}) in client.responses
|
||||
|
||||
|
||||
# ---- enriched approval prompts ----
|
||||
|
||||
class TestApprovalPromptEnrichment:
|
||||
"""Quirk #4: apply_patch prompt should show what's changing.
|
||||
Quirk #10: exec prompt should never show empty cwd."""
|
||||
|
||||
def test_exec_falls_back_to_session_cwd(self):
|
||||
"""When codex omits cwd from the approval params, the prompt shows
|
||||
the session cwd, not an empty string."""
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"item/commandExecution/requestApproval", request_id="r1",
|
||||
command="ls", # no cwd
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
captured = {}
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
captured["description"] = description
|
||||
return "once"
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
# Session cwd is /tmp by default in make_session()
|
||||
assert "/tmp" in captured["description"]
|
||||
assert "Codex requests exec in <unknown>" not in captured["description"]
|
||||
|
||||
def test_apply_patch_prompt_summarizes_pending_changes(self):
|
||||
"""When the projector has cached the fileChange item from item/started,
|
||||
the approval prompt surfaces the change summary."""
|
||||
client = FakeClient()
|
||||
# item/started fires first (carries the changes), then approval request
|
||||
client.queue_notification(
|
||||
"item/started",
|
||||
item={"type": "fileChange", "id": "fc-1",
|
||||
"changes": [
|
||||
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
|
||||
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
|
||||
]},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_server_request(
|
||||
"item/fileChange/requestApproval", request_id="req-2",
|
||||
itemId="fc-1", turnId="tu1", threadId="t",
|
||||
startedAtMs=1234567890,
|
||||
reason="add and update files",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
captured = {}
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
captured["command"] = command
|
||||
captured["description"] = description
|
||||
return "once"
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
# Both add and update kinds should be in the summary
|
||||
assert "1 add" in captured["command"] or "1 add" in captured["description"]
|
||||
assert "1 update" in captured["command"] or "1 update" in captured["description"]
|
||||
# And at least one of the paths
|
||||
joined = captured["command"] + " " + captured["description"]
|
||||
assert "/tmp/new.py" in joined or "/tmp/old.py" in joined
|
||||
|
||||
def test_apply_patch_prompt_works_without_cached_summary(self):
|
||||
"""When approval arrives before item/started (or without changes
|
||||
info), prompt falls back to whatever codex provided."""
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"item/fileChange/requestApproval", request_id="req-2",
|
||||
itemId="fc-orphan", turnId="tu1", threadId="t",
|
||||
startedAtMs=1234567890,
|
||||
reason="apply some changes",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
captured = {}
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
captured["command"] = command
|
||||
return "once"
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
# Falls back to the reason
|
||||
assert "apply some changes" in captured["command"]
|
||||
|
||||
|
||||
# ---- openclaw beta.8 parity: retire/wedge/oauth/abort marker ----
|
||||
|
||||
class TestSessionRetirement:
|
||||
"""Mirrors openclaw beta.8's resilience fixes:
|
||||
- retire timed-out app-server clients (should_retire on deadline)
|
||||
- post-tool completion watchdog (don't burn the full deadline after a
|
||||
tool result if codex goes silent)
|
||||
- <turn_aborted> raw marker as terminal (don't wait for turn/completed
|
||||
that never comes)
|
||||
- OAuth refresh failure classification (suggest `codex login` instead
|
||||
of raw RPC error strings)
|
||||
- dead subprocess detection between iterations
|
||||
"""
|
||||
|
||||
def test_deadline_marks_session_for_retirement(self):
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
r = s.run_turn(
|
||||
"never finishes",
|
||||
turn_timeout=0.05,
|
||||
notification_poll_timeout=0.01,
|
||||
)
|
||||
assert r.interrupted is True
|
||||
assert r.error and "timed out" in r.error
|
||||
assert r.should_retire is True, (
|
||||
"Deadline exhaustion must signal retirement so the next turn "
|
||||
"respawns codex instead of riding a wedged subprocess."
|
||||
)
|
||||
|
||||
def test_completed_turn_does_not_retire(self):
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1", "text": "hi"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=1.0)
|
||||
assert r.should_retire is False
|
||||
|
||||
def test_post_tool_quiet_watchdog_trips_and_retires(self):
|
||||
client = FakeClient()
|
||||
# One tool completion, then total silence — no further events,
|
||||
# no turn/completed. With a tiny post_tool_quiet_timeout the
|
||||
# watchdog must fire before the larger turn deadline.
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={
|
||||
"type": "commandExecution", "id": "ex1",
|
||||
"command": "echo hi", "cwd": "/tmp",
|
||||
"status": "completed", "aggregatedOutput": "hi",
|
||||
"exitCode": 0, "commandActions": [],
|
||||
},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn(
|
||||
"tool then silence",
|
||||
turn_timeout=5.0, # would be miserable to wait
|
||||
notification_poll_timeout=0.02,
|
||||
post_tool_quiet_timeout=0.15,
|
||||
)
|
||||
assert r.interrupted is True
|
||||
assert r.should_retire is True
|
||||
assert r.error and "silent" in r.error
|
||||
# Confirm we issued turn/interrupt to free codex compute
|
||||
assert any(method == "turn/interrupt" for (method, _) in client.requests)
|
||||
|
||||
def test_post_tool_watchdog_resets_on_further_activity(self):
|
||||
"""A tool completion followed by an agent message should NOT trip
|
||||
the watchdog — further activity = codex still alive."""
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={
|
||||
"type": "commandExecution", "id": "ex1",
|
||||
"command": "echo hi", "cwd": "/tmp",
|
||||
"status": "completed", "aggregatedOutput": "hi",
|
||||
"exitCode": 0, "commandActions": [],
|
||||
},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
# Non-tool activity immediately after — resets watchdog.
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1", "text": "tool finished"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn(
|
||||
"tool then talk", turn_timeout=2.0,
|
||||
notification_poll_timeout=0.01,
|
||||
post_tool_quiet_timeout=0.05,
|
||||
)
|
||||
# Tool ran, then text reset the watchdog, then turn/completed.
|
||||
# Should NOT be a retirement case.
|
||||
assert r.tool_iterations == 1
|
||||
assert r.final_text == "tool finished"
|
||||
assert r.should_retire is False
|
||||
assert r.interrupted is False
|
||||
|
||||
def test_turn_aborted_marker_in_text_is_terminal(self):
|
||||
"""If codex emits `<turn_aborted>` in agent text and never sends
|
||||
turn/completed, we still exit promptly instead of burning the
|
||||
deadline."""
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={
|
||||
"type": "agentMessage", "id": "m1",
|
||||
"text": "partial output... <turn_aborted>",
|
||||
},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
# Deliberately NO turn/completed notification queued.
|
||||
s = make_session(client)
|
||||
r = s.run_turn(
|
||||
"abort mid-turn", turn_timeout=2.0,
|
||||
notification_poll_timeout=0.01,
|
||||
)
|
||||
assert r.interrupted is True
|
||||
assert r.error and "turn_aborted" in r.error
|
||||
# Should have exited fast — not waited for the full 2s deadline.
|
||||
# (Can't measure wall clock reliably in CI; presence of the marker
|
||||
# error string instead of a "timed out" message is the proxy.)
|
||||
assert "timed out" not in r.error
|
||||
|
||||
def test_turn_aborted_self_closing_marker_also_terminal(self):
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1",
|
||||
"text": "<turn_aborted/>"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("x", turn_timeout=2.0,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.interrupted is True
|
||||
assert r.error and "turn_aborted" in r.error
|
||||
|
||||
def test_oauth_refresh_failure_on_turn_start_suggests_login(self):
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
client = FakeClient()
|
||||
|
||||
def boom(method, params):
|
||||
if method == "turn/start":
|
||||
raise CodexAppServerError(
|
||||
code=-32603,
|
||||
message="auth refresh failed: invalid_grant",
|
||||
)
|
||||
return {"thread": {"id": "t"},
|
||||
"activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=1.0)
|
||||
assert r.error is not None
|
||||
assert "codex login" in r.error
|
||||
assert r.should_retire is True
|
||||
|
||||
def test_oauth_failure_from_stderr_on_turn_start_failure(self):
|
||||
"""If the RPC error itself is opaque but stderr shows an auth
|
||||
problem, we still classify it as a refresh failure."""
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
client = FakeClient()
|
||||
client.set_stderr_tail([
|
||||
"[2026-05-14T10:00:00Z WARN codex_core::auth] token refresh failed",
|
||||
"[2026-05-14T10:00:00Z ERROR codex_core] please log in again",
|
||||
])
|
||||
|
||||
def boom(method, params):
|
||||
if method == "turn/start":
|
||||
raise CodexAppServerError(code=-32603, message="rpc broke")
|
||||
return {"thread": {"id": "t"},
|
||||
"activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=1.0)
|
||||
assert r.error is not None
|
||||
assert "codex login" in r.error
|
||||
assert r.should_retire is True
|
||||
|
||||
def test_oauth_failure_in_turn_completed_error(self):
|
||||
"""A failed turn/completed whose error mentions auth/refresh
|
||||
triggers the re-auth hint + retirement."""
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={
|
||||
"id": "tu1", "status": "failed",
|
||||
"error": {"message": "401 Unauthorized: please reauthenticate"},
|
||||
},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("x", turn_timeout=1.0,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.error is not None
|
||||
assert "codex login" in r.error
|
||||
assert r.should_retire is True
|
||||
|
||||
def test_generic_turn_failure_does_not_trigger_oauth_hint(self):
|
||||
"""A boring model error must NOT rewrite the message into a fake
|
||||
re-auth hint. Conservative classifier."""
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={
|
||||
"id": "tu1", "status": "failed",
|
||||
"error": {"message": "rate limit exceeded"},
|
||||
},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("x", turn_timeout=1.0,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.error is not None
|
||||
assert "codex login" not in r.error
|
||||
assert "rate limit exceeded" in r.error
|
||||
# Generic model failures don't retire — the session itself is fine
|
||||
assert r.should_retire is False
|
||||
|
||||
def test_dead_subprocess_detected_between_iterations(self):
|
||||
"""If codex dies (segfault, OOM, killed by its auth refresh
|
||||
thread), the inter-iteration is_alive check breaks the loop
|
||||
instead of waiting on a queue that will never fill."""
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
s.ensure_started()
|
||||
# Simulate subprocess death by setting _closed (FakeClient's
|
||||
# is_alive returns False when closed).
|
||||
client._closed = True
|
||||
client.set_stderr_tail([
|
||||
"thread 'tokio-runtime-worker' panicked at 'oauth: invalid_grant'",
|
||||
])
|
||||
r = s.run_turn("x", turn_timeout=2.0,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.should_retire is True
|
||||
# Stderr-derived auth hint takes precedence over generic message
|
||||
assert r.error and "codex login" in r.error
|
||||
|
||||
|
||||
# ---- thread/start cross-fill ----
|
||||
|
||||
class TestThreadStartCrossFill:
|
||||
"""Mirrors openclaw beta.8's tolerance for thread.id/sessionId aliasing."""
|
||||
|
||||
def test_thread_id_under_thread_key(self):
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
tid = s.ensure_started()
|
||||
assert tid == "thread-fake-001"
|
||||
|
||||
def test_thread_session_id_alias_under_thread_key(self):
|
||||
client = FakeClient()
|
||||
client._request_handler = lambda method, params: (
|
||||
{"thread": {"sessionId": "alias-1"},
|
||||
"activePermissionProfile": {"id": "x"}}
|
||||
if method == "thread/start" else
|
||||
{"turn": {"id": "tu1"}} if method == "turn/start" else {}
|
||||
)
|
||||
s = make_session(client)
|
||||
tid = s.ensure_started()
|
||||
assert tid == "alias-1"
|
||||
|
||||
def test_top_level_session_id_fallback(self):
|
||||
client = FakeClient()
|
||||
client._request_handler = lambda method, params: (
|
||||
{"sessionId": "top-1"} if method == "thread/start" else
|
||||
{"turn": {"id": "tu1"}} if method == "turn/start" else {}
|
||||
)
|
||||
s = make_session(client)
|
||||
tid = s.ensure_started()
|
||||
assert tid == "top-1"
|
||||
|
||||
def test_missing_thread_id_raises(self):
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
client = FakeClient()
|
||||
client._request_handler = lambda method, params: (
|
||||
{"thread": {}, "activePermissionProfile": {"id": "x"}}
|
||||
if method == "thread/start" else
|
||||
{"turn": {"id": "tu1"}}
|
||||
)
|
||||
s = make_session(client)
|
||||
with pytest.raises(CodexAppServerError, match="no thread id"):
|
||||
s.ensure_started()
|
||||
|
||||
|
||||
class TestHasTurnAbortedMarker:
|
||||
"""Unit coverage for the marker matcher itself."""
|
||||
|
||||
def test_empty_string(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_has_turn_aborted_marker,
|
||||
)
|
||||
assert _has_turn_aborted_marker("") is False
|
||||
assert _has_turn_aborted_marker(None) is False # type: ignore[arg-type]
|
||||
|
||||
def test_plain_text_no_marker(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_has_turn_aborted_marker,
|
||||
)
|
||||
assert _has_turn_aborted_marker("normal response with no markers") is False
|
||||
|
||||
def test_open_marker(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_has_turn_aborted_marker,
|
||||
)
|
||||
assert _has_turn_aborted_marker("blah <turn_aborted> blah") is True
|
||||
|
||||
def test_self_closing_marker(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_has_turn_aborted_marker,
|
||||
)
|
||||
assert _has_turn_aborted_marker("<turn_aborted/>") is True
|
||||
|
||||
|
||||
class TestClassifyOAuthFailure:
|
||||
"""Unit coverage for the OAuth classifier; conservative on purpose."""
|
||||
|
||||
def test_invalid_grant_classified(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
hint = _classify_oauth_failure("error: invalid_grant returned by server")
|
||||
assert hint is not None
|
||||
assert "codex login" in hint
|
||||
|
||||
def test_token_refresh_classified(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
hint = _classify_oauth_failure("token refresh failed: network error")
|
||||
assert hint is not None
|
||||
assert "codex login" in hint
|
||||
|
||||
def test_401_classified(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
hint = _classify_oauth_failure("HTTP 401 Unauthorized")
|
||||
assert hint is not None
|
||||
|
||||
def test_generic_error_not_classified(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
assert _classify_oauth_failure("connection reset") is None
|
||||
assert _classify_oauth_failure("model returned bad json") is None
|
||||
assert _classify_oauth_failure("rate limit exceeded") is None
|
||||
|
||||
def test_empty_inputs(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
assert _classify_oauth_failure() is None
|
||||
assert _classify_oauth_failure("") is None
|
||||
assert _classify_oauth_failure("", None) is None # type: ignore[arg-type]
|
||||
|
||||
def test_multi_string_search(self):
|
||||
"""Hint can come from any of the provided strings."""
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
hint = _classify_oauth_failure(
|
||||
"rpc returned -32603",
|
||||
"[stderr] token has expired, run codex login",
|
||||
)
|
||||
assert hint is not None
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Tests for CodexEventProjector — codex item/* events → Hermes messages list.
|
||||
|
||||
Drives projection against fixture notifications captured from codex 0.130.0
|
||||
plus synthetic ones for item types we couldn't auth-test live."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.transports.codex_event_projector import (
|
||||
CodexEventProjector,
|
||||
ProjectionResult,
|
||||
_deterministic_call_id,
|
||||
_format_tool_args,
|
||||
)
|
||||
|
||||
|
||||
# --- Fixture: real `commandExecution` notification captured from codex 0.130.0
|
||||
COMMAND_EXEC_COMPLETED = {
|
||||
"method": "item/completed",
|
||||
"params": {
|
||||
"item": {
|
||||
"type": "commandExecution",
|
||||
"id": "f8a75c66-a89e-4fd7-8bcf-2d58e664fa9e",
|
||||
"command": "/bin/bash -lc 'echo hello && ls /tmp | head -3'",
|
||||
"cwd": "/tmp",
|
||||
"processId": None,
|
||||
"source": "userShell",
|
||||
"status": "completed",
|
||||
"commandActions": [
|
||||
{"type": "listFiles", "command": "ls /tmp", "path": "tmp"}
|
||||
],
|
||||
"aggregatedOutput": "hello\naa_lang.json\n",
|
||||
"exitCode": 0,
|
||||
"durationMs": 10,
|
||||
},
|
||||
"threadId": "019e1a94-352b-71e1-b214-e5c67c9ec190",
|
||||
"turnId": "019e1a94-3553-7940-8af3-4ca57142deb7",
|
||||
"completedAtMs": 1778562381151,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestProjectionInvariants:
|
||||
"""Universal invariants that must hold across all projection paths."""
|
||||
|
||||
def test_streaming_deltas_dont_materialize(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
for delta_method in (
|
||||
"item/commandExecution/outputDelta",
|
||||
"item/agentMessage/delta",
|
||||
"item/reasoning/delta",
|
||||
):
|
||||
r = p.project({"method": delta_method, "params": {"delta": "x"}})
|
||||
assert r.messages == [], (
|
||||
f"{delta_method} should NOT produce messages — only "
|
||||
f"item/completed materializes"
|
||||
)
|
||||
assert r.is_tool_iteration is False
|
||||
assert r.final_text is None
|
||||
|
||||
def test_turn_started_and_completed_are_silent(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
for method in ("turn/started", "turn/completed", "thread/started"):
|
||||
r = p.project({"method": method, "params": {}})
|
||||
assert r.messages == []
|
||||
|
||||
def test_unknown_method_silent(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project({"method": "totally/unknown", "params": {}})
|
||||
assert r.messages == []
|
||||
|
||||
|
||||
class TestCommandExecutionProjection:
|
||||
"""Real captured notification → assistant tool_call + tool result."""
|
||||
|
||||
def test_command_completed_produces_two_messages(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project(COMMAND_EXEC_COMPLETED)
|
||||
assert len(r.messages) == 2
|
||||
assert r.is_tool_iteration is True
|
||||
|
||||
def test_first_message_is_assistant_tool_call(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assistant = msgs[0]
|
||||
assert assistant["role"] == "assistant"
|
||||
assert assistant["content"] is None
|
||||
assert len(assistant["tool_calls"]) == 1
|
||||
tc = assistant["tool_calls"][0]
|
||||
assert tc["type"] == "function"
|
||||
assert tc["function"]["name"] == "exec_command"
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert "echo hello" in args["command"]
|
||||
assert args["cwd"] == "/tmp"
|
||||
|
||||
def test_second_message_is_tool_result_correlating_by_id(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assistant, tool = msgs
|
||||
assert tool["role"] == "tool"
|
||||
assert tool["tool_call_id"] == assistant["tool_calls"][0]["id"]
|
||||
assert "hello" in tool["content"]
|
||||
|
||||
def test_nonzero_exit_code_annotated_in_tool_result(self) -> None:
|
||||
item = {**COMMAND_EXEC_COMPLETED["params"]["item"], "exitCode": 2,
|
||||
"aggregatedOutput": "boom"}
|
||||
notif = {
|
||||
"method": "item/completed",
|
||||
"params": {**COMMAND_EXEC_COMPLETED["params"], "item": item},
|
||||
}
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(notif).messages
|
||||
assert "[exit 2]" in msgs[1]["content"]
|
||||
assert "boom" in msgs[1]["content"]
|
||||
|
||||
def test_deterministic_call_id_across_replay(self) -> None:
|
||||
# Same item id → same call_id (prefix cache must stay valid).
|
||||
p1 = CodexEventProjector()
|
||||
p2 = CodexEventProjector()
|
||||
a = p1.project(COMMAND_EXEC_COMPLETED).messages
|
||||
b = p2.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assert a[0]["tool_calls"][0]["id"] == b[0]["tool_calls"][0]["id"]
|
||||
|
||||
|
||||
class TestAgentMessageProjection:
|
||||
"""assistant text → final_text + assistant message."""
|
||||
|
||||
def test_agent_message_projects_to_assistant(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "agentMessage", "id": "x",
|
||||
"text": "hi there"}},
|
||||
})
|
||||
assert r.final_text == "hi there"
|
||||
assert r.messages == [{"role": "assistant", "content": "hi there"}]
|
||||
assert r.is_tool_iteration is False
|
||||
|
||||
def test_pending_reasoning_attaches_to_next_assistant_message(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
# First a reasoning item lands
|
||||
r1 = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "reasoning", "id": "r1",
|
||||
"summary": ["thinking..."],
|
||||
"content": ["step 1", "step 2"]}},
|
||||
})
|
||||
assert r1.messages == [] # reasoning alone produces no message
|
||||
# Then the assistant message
|
||||
r2 = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "agentMessage", "id": "a1",
|
||||
"text": "ok"}},
|
||||
})
|
||||
assistant = r2.messages[0]
|
||||
assert "reasoning" in assistant
|
||||
assert "thinking" in assistant["reasoning"]
|
||||
assert "step 1" in assistant["reasoning"]
|
||||
|
||||
def test_reasoning_consumed_after_attaching(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "reasoning", "id": "r1", "summary": ["once"], "content": []}}})
|
||||
first = p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "agentMessage", "id": "a", "text": "first"}}}).messages[0]
|
||||
second = p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "agentMessage", "id": "b", "text": "second"}}}).messages[0]
|
||||
assert "reasoning" in first
|
||||
assert "reasoning" not in second
|
||||
|
||||
|
||||
class TestFileChangeProjection:
|
||||
def test_file_change_summary_no_inlined_content(self) -> None:
|
||||
item = {
|
||||
"type": "fileChange",
|
||||
"id": "fc1",
|
||||
"status": "applied",
|
||||
"changes": [
|
||||
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
|
||||
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
|
||||
],
|
||||
}
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project({"method": "item/completed",
|
||||
"params": {"item": item}}).messages
|
||||
assert len(msgs) == 2
|
||||
tc = msgs[0]["tool_calls"][0]
|
||||
assert tc["function"]["name"] == "apply_patch"
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert len(args["changes"]) == 2
|
||||
assert all("kind" in c and "path" in c for c in args["changes"])
|
||||
assert "applied" in msgs[1]["content"]
|
||||
|
||||
|
||||
class TestMcpToolCallProjection:
|
||||
def test_mcp_tool_call_namespaced(self) -> None:
|
||||
item = {
|
||||
"type": "mcpToolCall",
|
||||
"id": "m1",
|
||||
"server": "obsidian",
|
||||
"tool": "search_notes",
|
||||
"status": "completed",
|
||||
"arguments": {"query": "hermes"},
|
||||
"result": {"content": [{"text": "found"}]},
|
||||
"error": None,
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert msgs[0]["tool_calls"][0]["function"]["name"] == "mcp.obsidian.search_notes"
|
||||
assert "found" in msgs[1]["content"]
|
||||
|
||||
def test_mcp_error_surfaced(self) -> None:
|
||||
item = {
|
||||
"type": "mcpToolCall", "id": "m2",
|
||||
"server": "x", "tool": "y", "status": "failed",
|
||||
"arguments": {}, "result": None,
|
||||
"error": {"code": -1, "message": "no"},
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert "error" in msgs[1]["content"]
|
||||
|
||||
|
||||
class TestUserAndOpaqueProjection:
|
||||
def test_user_message_text_fragments_only(self) -> None:
|
||||
item = {
|
||||
"type": "userMessage", "id": "u1",
|
||||
"content": [
|
||||
{"type": "text", "text": "hello"},
|
||||
{"type": "image", "url": "http://x/y"},
|
||||
{"type": "text", "text": "world"},
|
||||
],
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert msgs[0]["role"] == "user"
|
||||
assert "hello" in msgs[0]["content"]
|
||||
assert "world" in msgs[0]["content"]
|
||||
|
||||
def test_opaque_item_recorded_without_fabricated_tool_calls(self) -> None:
|
||||
item = {"type": "plan", "id": "p1", "text": "do the thing"}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0]["role"] == "assistant"
|
||||
assert "plan" in msgs[0]["content"].lower()
|
||||
assert "tool_calls" not in msgs[0]
|
||||
|
||||
|
||||
class TestHelpers:
|
||||
def test_deterministic_call_id_stable(self) -> None:
|
||||
assert _deterministic_call_id("exec", "abc") == _deterministic_call_id("exec", "abc")
|
||||
assert _deterministic_call_id("exec", "abc") != _deterministic_call_id("exec", "xyz")
|
||||
|
||||
def test_deterministic_call_id_handles_missing_id(self) -> None:
|
||||
# Should not raise, should be stable for same item type
|
||||
a = _deterministic_call_id("exec", "")
|
||||
b = _deterministic_call_id("exec", "")
|
||||
assert a == b
|
||||
assert "exec" in a
|
||||
|
||||
def test_format_tool_args_sorted_keys(self) -> None:
|
||||
# Sorted keys = deterministic across replays = prefix cache stays valid
|
||||
a = _format_tool_args({"b": 1, "a": 2})
|
||||
b = _format_tool_args({"a": 2, "b": 1})
|
||||
assert a == b
|
||||
|
||||
|
||||
class TestRoleAlternationInvariant:
|
||||
"""The project must never emit two assistant messages back-to-back from
|
||||
one item — that breaks Hermes' message alternation invariant."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"item",
|
||||
[
|
||||
{"type": "commandExecution", "id": "c1", "command": "x",
|
||||
"cwd": "/", "status": "completed", "aggregatedOutput": "",
|
||||
"exitCode": 0, "commandActions": []},
|
||||
{"type": "fileChange", "id": "f1", "status": "applied",
|
||||
"changes": []},
|
||||
{"type": "mcpToolCall", "id": "m1", "server": "s", "tool": "t",
|
||||
"status": "completed", "arguments": {}, "result": None,
|
||||
"error": None},
|
||||
{"type": "dynamicToolCall", "id": "d1", "tool": "x",
|
||||
"arguments": {}, "status": "completed",
|
||||
"contentItems": [], "success": True},
|
||||
],
|
||||
)
|
||||
def test_tool_items_emit_assistant_then_tool(self, item) -> None:
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert len(msgs) == 2
|
||||
assert msgs[0]["role"] == "assistant"
|
||||
assert msgs[1]["role"] == "tool"
|
||||
assert msgs[1]["tool_call_id"] == msgs[0]["tool_calls"][0]["id"]
|
||||
@@ -100,6 +100,44 @@ class TestCodexBuildKwargs:
|
||||
)
|
||||
assert "prompt_cache_key" not in kw
|
||||
|
||||
def test_xai_responses_sends_cache_key_via_extra_body(self, transport):
|
||||
"""xAI's Responses API documents ``prompt_cache_key`` as the
|
||||
body-level cache-routing key (the ``x-grok-conv-id`` header is
|
||||
Chat-Completions-only). Passing it via ``extra_body`` is robust
|
||||
against openai SDK builds whose ``Responses.stream()`` kwarg
|
||||
signature ever drops the field — the body field still serializes
|
||||
and reaches xAI either way. The ``x-grok-conv-id`` header is kept
|
||||
as a belt-and-braces fallback so cache routing survives even
|
||||
when the body field would be stripped by an intermediate proxy.
|
||||
Ref: https://docs.x.ai/developers/advanced-api-usage/prompt-caching/maximizing-cache-hits
|
||||
"""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
session_id="conv-xai-1",
|
||||
is_xai_responses=True,
|
||||
)
|
||||
assert "prompt_cache_key" not in kw
|
||||
assert kw.get("extra_body", {}).get("prompt_cache_key") == "conv-xai-1"
|
||||
assert kw.get("extra_headers", {}).get("x-grok-conv-id") == "conv-xai-1"
|
||||
|
||||
def test_xai_responses_extra_body_preserves_caller_fields(self, transport):
|
||||
"""When the caller already supplies ``extra_body`` (e.g. via
|
||||
request_overrides), the xAI cache-key injection must merge into
|
||||
the existing dict instead of overwriting it. Caller-supplied
|
||||
``prompt_cache_key`` wins (setdefault semantics) so user overrides
|
||||
aren't silently clobbered by the transport."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
session_id="conv-xai-1",
|
||||
is_xai_responses=True,
|
||||
request_overrides={"extra_body": {"prompt_cache_key": "caller-override", "other_field": 42}},
|
||||
)
|
||||
eb = kw.get("extra_body", {})
|
||||
assert eb.get("prompt_cache_key") == "caller-override"
|
||||
assert eb.get("other_field") == 42
|
||||
|
||||
def test_max_tokens(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Tests for the hermes-tools-as-MCP server module surface.
|
||||
|
||||
We don't run a live MCP session in unit tests — that requires the codex
|
||||
subprocess + client + an event loop. These tests pin the static
|
||||
contract: the module imports, the EXPOSED_TOOLS list is sane, and the
|
||||
build helper assembles a server when the SDK is present.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestModuleSurface:
|
||||
def test_module_imports_clean(self):
|
||||
from agent.transports import hermes_tools_mcp_server as m
|
||||
assert callable(m.main)
|
||||
assert callable(m._build_server)
|
||||
assert isinstance(m.EXPOSED_TOOLS, tuple)
|
||||
assert len(m.EXPOSED_TOOLS) > 0
|
||||
|
||||
def test_exposed_tools_are_safe_subset(self):
|
||||
"""We MUST NOT expose tools codex already has, because codex'
|
||||
own builtins are better-integrated with its sandbox + approvals.
|
||||
Specifically: no terminal/shell, no read_file/write_file, no
|
||||
patch — those are codex's built-in tools."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
forbidden = {
|
||||
"terminal", "shell", "read_file", "write_file", "patch",
|
||||
"search_files", "process",
|
||||
}
|
||||
leaked = forbidden & set(EXPOSED_TOOLS)
|
||||
assert not leaked, (
|
||||
f"these tools must NOT be exposed via the codex callback "
|
||||
f"because codex has built-in equivalents: {leaked}"
|
||||
)
|
||||
|
||||
def test_expected_hermes_specific_tools_listed(self):
|
||||
"""The Hermes-specific tools should be present so users on the
|
||||
codex runtime keep access to them."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for required in (
|
||||
"web_search",
|
||||
"web_extract",
|
||||
"browser_navigate",
|
||||
"vision_analyze",
|
||||
"image_generate",
|
||||
"skill_view",
|
||||
):
|
||||
assert required in EXPOSED_TOOLS, f"missing {required!r}"
|
||||
|
||||
def test_agent_loop_tools_not_exposed(self):
|
||||
"""delegate_task / memory / session_search / todo require the
|
||||
running AIAgent context to dispatch, so a stateless MCP callback
|
||||
can't drive them. They must NOT be in EXPOSED_TOOLS."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for agent_loop_tool in ("delegate_task", "memory", "session_search", "todo"):
|
||||
assert agent_loop_tool not in EXPOSED_TOOLS, (
|
||||
f"{agent_loop_tool!r} requires the agent loop context "
|
||||
"and can't be reached through a stateless MCP callback"
|
||||
)
|
||||
|
||||
def test_kanban_worker_tools_exposed(self):
|
||||
"""Kanban workers run as `hermes chat -q` subprocesses; if they
|
||||
come up on the codex_app_server runtime, the worker can do the
|
||||
actual work via codex's shell but needs the kanban tools through
|
||||
the MCP callback to report back to the kernel. Without these
|
||||
tools available, the worker would hang at completion time."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
# Worker handoff tools — every dispatched worker uses at least
|
||||
# one of {complete, block, comment} to close out its task.
|
||||
for worker_tool in (
|
||||
"kanban_complete",
|
||||
"kanban_block",
|
||||
"kanban_comment",
|
||||
"kanban_heartbeat",
|
||||
):
|
||||
assert worker_tool in EXPOSED_TOOLS, (
|
||||
f"{worker_tool!r} missing from codex callback — kanban "
|
||||
"workers on codex_app_server runtime would hang"
|
||||
)
|
||||
|
||||
def test_kanban_orchestrator_tools_exposed(self):
|
||||
"""Orchestrator agents need to dispatch new tasks, query the
|
||||
board, and unblock/link tasks. Exposed so an orchestrator on
|
||||
codex_app_server can do its job."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for orch_tool in (
|
||||
"kanban_create",
|
||||
"kanban_show",
|
||||
"kanban_list",
|
||||
"kanban_unblock",
|
||||
"kanban_link",
|
||||
):
|
||||
assert orch_tool in EXPOSED_TOOLS, (
|
||||
f"{orch_tool!r} missing from codex callback"
|
||||
)
|
||||
|
||||
|
||||
class TestMain:
|
||||
def test_main_returns_2_when_mcp_unavailable(self, monkeypatch):
|
||||
"""When the mcp package isn't installed, main() should exit
|
||||
cleanly with code 2 and an install hint, not crash."""
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
def boom_build(*a, **kw):
|
||||
raise ImportError("mcp not installed")
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", boom_build)
|
||||
rc = m.main(["--verbose"])
|
||||
assert rc == 2
|
||||
|
||||
def test_main_handles_keyboard_interrupt(self, monkeypatch):
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
class FakeServer:
|
||||
def run(self):
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", lambda: FakeServer())
|
||||
rc = m.main([])
|
||||
assert rc == 0
|
||||
|
||||
def test_main_returns_1_on_runtime_error(self, monkeypatch):
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
class CrashingServer:
|
||||
def run(self):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", lambda: CrashingServer())
|
||||
rc = m.main([])
|
||||
assert rc == 1
|
||||
@@ -71,32 +71,40 @@ class TestForceFullRedraw:
|
||||
"invalidate",
|
||||
]
|
||||
|
||||
def test_resize_rebuilds_scrollback_before_prompt_toolkit_redraw(self, bare_cli, monkeypatch):
|
||||
def test_resize_preserves_scrollback_and_resets_renderer(self, bare_cli, monkeypatch):
|
||||
"""Resize recovery must NOT erase screen or scrollback.
|
||||
|
||||
The startup banner lives in normal terminal scrollback (printed
|
||||
before prompt_toolkit owns the chrome). Clearing scrollback on
|
||||
SIGWINCH removes it and ``_replay_output_history`` cannot
|
||||
reconstruct it. The fix is to only reset the renderer cache and
|
||||
let ``original_on_resize`` recalculate layout.
|
||||
|
||||
Additionally, ``_status_bar_suppressed_after_resize`` must be set
|
||||
so the input rules and status bar hide until the next user input,
|
||||
preventing duplicated-bar artifacts on column shrink (#19280).
|
||||
"""
|
||||
app = MagicMock()
|
||||
out = app.renderer.output
|
||||
events = []
|
||||
out.reset_attributes.side_effect = lambda: events.append("reset_attrs")
|
||||
out.erase_screen.side_effect = lambda: events.append("erase")
|
||||
out.write_raw.side_effect = lambda text: events.append(("raw", text))
|
||||
out.cursor_goto.side_effect = lambda *_: events.append("home")
|
||||
out.flush.side_effect = lambda: events.append("flush")
|
||||
app.renderer.reset.side_effect = lambda **_: events.append("renderer_reset")
|
||||
monkeypatch.setattr(cli_mod, "_replay_output_history", lambda: events.append("replay"))
|
||||
app.invalidate.side_effect = lambda: events.append("invalidate")
|
||||
original_on_resize = lambda: events.append("original_resize")
|
||||
|
||||
# bare_cli skips __init__, so seed the attribute the way __init__ would.
|
||||
bare_cli._status_bar_suppressed_after_resize = False
|
||||
bare_cli._recover_after_resize(app, original_on_resize)
|
||||
|
||||
assert events == [
|
||||
"reset_attrs",
|
||||
"erase",
|
||||
("raw", "\x1b[3J"),
|
||||
"home",
|
||||
"flush",
|
||||
"renderer_reset",
|
||||
"replay",
|
||||
"invalidate",
|
||||
"original_resize",
|
||||
]
|
||||
app.invalidate.assert_not_called()
|
||||
# Must NOT clear the screen or scrollback — those destroy the banner.
|
||||
app.renderer.output.erase_screen.assert_not_called()
|
||||
app.renderer.output.write_raw.assert_not_called()
|
||||
app.renderer.output.cursor_goto.assert_not_called()
|
||||
# Status bar / input rules must be suppressed until the next prompt.
|
||||
assert bare_cli._status_bar_suppressed_after_resize is True
|
||||
|
||||
def test_force_redraw_uses_full_screen_clear_without_scrollback_clear(self, bare_cli):
|
||||
app = MagicMock()
|
||||
|
||||
@@ -319,6 +319,89 @@ class TestHistoryDisplay:
|
||||
assert "Checking Running Hermes Agent" in output
|
||||
assert "Use /resume <session id or title> to continue" in output
|
||||
|
||||
def test_sessions_command_no_args_lists_recent_sessions(self, capsys):
|
||||
"""/sessions with no args prints the recent-sessions table (TUI parity).
|
||||
|
||||
Regression test: `sessions` was registered in the central command
|
||||
registry and surfaced by /help and tab-completion, but the classic
|
||||
CLI dispatcher had no elif branch for it, so the canonical name fell
|
||||
through and printed `Unknown command: sessions`.
|
||||
"""
|
||||
cli = _make_cli()
|
||||
cli.session_id = "current"
|
||||
cli._session_db = MagicMock()
|
||||
cli._session_db.list_sessions_rich.return_value = [
|
||||
{
|
||||
"id": "20260401_201329_d85961",
|
||||
"title": "Checking Running Hermes Agent",
|
||||
"preview": "check running gateways for hermes agent",
|
||||
"last_active": 0,
|
||||
},
|
||||
]
|
||||
|
||||
# Drive it through the public dispatcher to also lock in the
|
||||
# process_command wiring, not just the handler in isolation.
|
||||
cli.process_command("/sessions")
|
||||
output = capsys.readouterr().out
|
||||
|
||||
assert "Unknown command" not in output
|
||||
assert "Recent sessions" in output
|
||||
assert "Checking Running Hermes Agent" in output
|
||||
assert "20260401_201329_d85961" in output
|
||||
|
||||
def test_sessions_list_subcommand_lists_recent_sessions(self, capsys):
|
||||
"""/sessions list is an explicit alias for the no-arg list view."""
|
||||
cli = _make_cli()
|
||||
cli.session_id = "current"
|
||||
cli._session_db = MagicMock()
|
||||
cli._session_db.list_sessions_rich.return_value = [
|
||||
{
|
||||
"id": "20260401_201329_d85961",
|
||||
"title": "Checking Running Hermes Agent",
|
||||
"preview": "check running gateways for hermes agent",
|
||||
"last_active": 0,
|
||||
},
|
||||
]
|
||||
|
||||
cli.process_command("/sessions list")
|
||||
output = capsys.readouterr().out
|
||||
|
||||
assert "Unknown command" not in output
|
||||
assert "Recent sessions" in output
|
||||
assert "Checking Running Hermes Agent" in output
|
||||
|
||||
def test_sessions_with_target_delegates_to_resume(self):
|
||||
"""/sessions <id_or_title> behaves identically to /resume <id_or_title>.
|
||||
|
||||
We intercept `_handle_resume_command` rather than the full resume
|
||||
machinery (which would otherwise require simulating an entire session
|
||||
switch). The contract under test is the dispatch wiring.
|
||||
"""
|
||||
cli = _make_cli()
|
||||
with patch.object(cli, "_handle_resume_command") as mock_resume:
|
||||
cli.process_command("/sessions Checking Running Hermes Agent")
|
||||
|
||||
mock_resume.assert_called_once_with(
|
||||
"/resume Checking Running Hermes Agent"
|
||||
)
|
||||
|
||||
def test_sessions_command_is_dispatched(self):
|
||||
"""/sessions must hit _handle_sessions_command, not fall through.
|
||||
|
||||
Direct test that the process_command elif chain routes the canonical
|
||||
name to the handler. Without this wiring, /sessions printed
|
||||
`Unknown command: sessions` even though it was a registered command.
|
||||
"""
|
||||
cli = _make_cli()
|
||||
cli._session_db = None # exercise the no-db path too
|
||||
|
||||
with patch.object(cli, "_handle_sessions_command") as mock_handler:
|
||||
cli.process_command("/sessions")
|
||||
|
||||
mock_handler.assert_called_once()
|
||||
called_with = mock_handler.call_args.args[0]
|
||||
assert called_with.lower().startswith("/sessions")
|
||||
|
||||
|
||||
class TestRootLevelProviderOverride:
|
||||
"""Root-level provider/base_url in config.yaml must NOT override model.provider."""
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Tests for the light-mode terminal detection + color remap in cli.py.
|
||||
|
||||
Covers the env-override path and the SkinConfig.get_color() wrapper that
|
||||
the resize / light-mode salvage installs at module import time. We don't
|
||||
try to fake an OSC 11 reply — the env-override branch short-circuits
|
||||
before the terminal query, which is the path most users hit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cli_mod(monkeypatch):
|
||||
"""Import cli with the light-mode cache cleared each test."""
|
||||
import cli as _cli
|
||||
|
||||
# The module-level _install_skin_light_mode_hook() and import-time
|
||||
# _detect_light_mode() prime ran once at first import. We just reset
|
||||
# the detection cache so the per-test env override takes effect.
|
||||
monkeypatch.setattr(_cli, "_LIGHT_MODE_CACHE", None)
|
||||
return _cli
|
||||
|
||||
|
||||
class TestLightModeDetection:
|
||||
def test_hermes_light_env_true_forces_light(self, cli_mod, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_LIGHT", "1")
|
||||
assert cli_mod._detect_light_mode() is True
|
||||
|
||||
def test_hermes_light_env_false_forces_dark(self, cli_mod, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_LIGHT", "0")
|
||||
# Also blank out other signals so nothing else flips it light.
|
||||
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_THEME", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_BACKGROUND", raising=False)
|
||||
monkeypatch.delenv("COLORFGBG", raising=False)
|
||||
assert cli_mod._detect_light_mode() is False
|
||||
|
||||
def test_theme_hint_light(self, cli_mod, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_LIGHT", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
|
||||
monkeypatch.setenv("HERMES_TUI_THEME", "light")
|
||||
assert cli_mod._detect_light_mode() is True
|
||||
|
||||
def test_background_hex_hint_light(self, cli_mod, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_LIGHT", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_THEME", raising=False)
|
||||
monkeypatch.setenv("HERMES_TUI_BACKGROUND", "#FFFFFF")
|
||||
assert cli_mod._detect_light_mode() is True
|
||||
|
||||
def test_background_hex_hint_dark(self, cli_mod, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_LIGHT", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_THEME", raising=False)
|
||||
monkeypatch.setenv("HERMES_TUI_BACKGROUND", "#1a1a2e")
|
||||
monkeypatch.delenv("COLORFGBG", raising=False)
|
||||
assert cli_mod._detect_light_mode() is False
|
||||
|
||||
def test_colorfgbg_light_bg_slot(self, cli_mod, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_LIGHT", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_LIGHT", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_THEME", raising=False)
|
||||
monkeypatch.delenv("HERMES_TUI_BACKGROUND", raising=False)
|
||||
monkeypatch.setenv("COLORFGBG", "0;15") # bg slot 15 = light
|
||||
assert cli_mod._detect_light_mode() is True
|
||||
|
||||
def test_cache_is_sticky(self, cli_mod, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_LIGHT", "1")
|
||||
assert cli_mod._detect_light_mode() is True
|
||||
# Even if the env flips, the cached result wins until reset.
|
||||
monkeypatch.setenv("HERMES_LIGHT", "0")
|
||||
assert cli_mod._detect_light_mode() is True
|
||||
|
||||
|
||||
class TestLightModeRemap:
|
||||
def test_remap_no_op_in_dark_mode(self, cli_mod, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_LIGHT", "0")
|
||||
# Cache is None from the fixture; first call sticks at False.
|
||||
assert cli_mod._maybe_remap_for_light_mode("#FFF8DC") == "#FFF8DC"
|
||||
|
||||
def test_remap_known_dark_color(self, cli_mod, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_LIGHT", "1")
|
||||
# Force the detect cache to True for this test.
|
||||
cli_mod._LIGHT_MODE_CACHE = True
|
||||
assert cli_mod._maybe_remap_for_light_mode("#FFF8DC") == "#1A1A1A"
|
||||
assert cli_mod._maybe_remap_for_light_mode("#FFD700") == "#9A6B00"
|
||||
|
||||
def test_remap_case_insensitive(self, cli_mod, monkeypatch):
|
||||
cli_mod._LIGHT_MODE_CACHE = True
|
||||
# Lowercase input should still remap.
|
||||
assert cli_mod._maybe_remap_for_light_mode("#fff8dc") == "#1A1A1A"
|
||||
|
||||
def test_remap_unknown_color_passthrough(self, cli_mod, monkeypatch):
|
||||
cli_mod._LIGHT_MODE_CACHE = True
|
||||
# A color not in the remap table is returned unchanged.
|
||||
assert cli_mod._maybe_remap_for_light_mode("#ABCDEF") == "#ABCDEF"
|
||||
|
||||
def test_remap_skips_statusbar_paired_colors(self, cli_mod, monkeypatch):
|
||||
"""Colors that live on a dark bg (status bar fg) MUST NOT be
|
||||
remapped — otherwise they go dark-on-dark and disappear.
|
||||
|
||||
Regression guard for the patch-11 fix (intentional table omission).
|
||||
"""
|
||||
cli_mod._LIGHT_MODE_CACHE = True
|
||||
for fg in ("#C0C0C0", "#888888", "#555555", "#8B8682"):
|
||||
assert cli_mod._maybe_remap_for_light_mode(fg) == fg, (
|
||||
f"{fg} is a status-bar fg paired with dark bg; remapping it "
|
||||
"would produce dark-on-dark"
|
||||
)
|
||||
|
||||
|
||||
class TestSkinConfigHook:
|
||||
"""The salvage wraps SkinConfig.get_color at module import time so
|
||||
every skin color read goes through the light-mode remap. Verify
|
||||
the hook installed and functions correctly.
|
||||
"""
|
||||
|
||||
def test_hook_installed(self, cli_mod):
|
||||
from hermes_cli.skin_engine import SkinConfig
|
||||
|
||||
assert getattr(SkinConfig, "_hermes_light_mode_hook_installed", False) is True
|
||||
|
||||
def test_hook_is_idempotent(self, cli_mod):
|
||||
# Calling the installer twice must not double-wrap (the marker
|
||||
# attribute is the guard).
|
||||
from hermes_cli.skin_engine import SkinConfig
|
||||
|
||||
before = SkinConfig.get_color
|
||||
cli_mod._install_skin_light_mode_hook()
|
||||
after = SkinConfig.get_color
|
||||
assert before is after
|
||||
|
||||
def test_skin_color_remaps_through_wrapper_in_light_mode(self, cli_mod, monkeypatch):
|
||||
from hermes_cli.skin_engine import SkinConfig
|
||||
|
||||
cli_mod._LIGHT_MODE_CACHE = True
|
||||
skin = SkinConfig(
|
||||
name="test",
|
||||
colors={"banner_text": "#FFF8DC", "response_border": "#FFD700"},
|
||||
)
|
||||
# The wrapper kicks in at get_color, not at construction time.
|
||||
assert skin.get_color("banner_text") == "#1A1A1A"
|
||||
assert skin.get_color("response_border") == "#9A6B00"
|
||||
|
||||
def test_skin_color_passthrough_in_dark_mode(self, cli_mod, monkeypatch):
|
||||
from hermes_cli.skin_engine import SkinConfig
|
||||
|
||||
cli_mod._LIGHT_MODE_CACHE = False
|
||||
skin = SkinConfig(name="test", colors={"banner_text": "#FFF8DC"})
|
||||
assert skin.get_color("banner_text") == "#FFF8DC"
|
||||
@@ -531,8 +531,8 @@ def test_model_flow_custom_saves_verified_v1_base_url(monkeypatch, capsys):
|
||||
|
||||
# After the probe detects a single model ("llm"), the flow asks
|
||||
# "Use this model? [Y/n]:" — confirm with Enter, then context length,
|
||||
# then display name.
|
||||
answers = iter(["http://localhost:8000", "local-key", "", "", "", ""])
|
||||
# then display name. The api_mode prompt also runs before model selection.
|
||||
answers = iter(["http://localhost:8000", "local-key", "", "", "", "", ""])
|
||||
monkeypatch.setattr("builtins.input", lambda _prompt="": next(answers))
|
||||
monkeypatch.setattr("getpass.getpass", lambda _prompt="": next(answers))
|
||||
|
||||
@@ -546,6 +546,63 @@ def test_model_flow_custom_saves_verified_v1_base_url(monkeypatch, capsys):
|
||||
assert saved_env["MODEL"] == "llm"
|
||||
|
||||
|
||||
def test_model_flow_custom_persists_selected_api_mode(monkeypatch):
|
||||
saved_cfg = {"model": {"default": "", "provider": "custom", "base_url": ""}}
|
||||
captured_provider = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.config.get_env_value",
|
||||
lambda key: "" if key in {"OPENAI_BASE_URL", "OPENAI_API_KEY"} else "",
|
||||
)
|
||||
monkeypatch.setattr("hermes_cli.auth._save_model_choice", lambda model: None)
|
||||
monkeypatch.setattr("hermes_cli.auth.deactivate_provider", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.models.probe_api_models",
|
||||
lambda api_key, base_url: {
|
||||
"models": [],
|
||||
"probed_url": f"{base_url.rstrip('/')}/models",
|
||||
"resolved_base_url": None,
|
||||
"suggested_base_url": None,
|
||||
"used_fallback": False,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr("hermes_cli.config.load_config", lambda: saved_cfg)
|
||||
monkeypatch.setattr("hermes_cli.config.save_config", lambda cfg: saved_cfg.update(cfg))
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.main._save_custom_provider",
|
||||
lambda base_url, api_key="", model="", context_length=None, name=None, api_mode=None: captured_provider.update(
|
||||
{
|
||||
"base_url": base_url,
|
||||
"api_key": api_key,
|
||||
"model": model,
|
||||
"context_length": context_length,
|
||||
"name": name,
|
||||
"api_mode": api_mode,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
answers = iter(
|
||||
[
|
||||
"https://codex.example.com/v1",
|
||||
"3",
|
||||
"chosen-model",
|
||||
"",
|
||||
"",
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr("builtins.input", lambda _prompt="": next(answers))
|
||||
monkeypatch.setattr("getpass.getpass", lambda _prompt="": "test-key")
|
||||
|
||||
hermes_main._model_flow_custom({"model": {"provider": "custom"}})
|
||||
|
||||
assert saved_cfg["model"]["provider"] == "custom"
|
||||
assert saved_cfg["model"]["base_url"] == "https://codex.example.com/v1"
|
||||
assert saved_cfg["model"]["api_key"] == "test-key"
|
||||
assert saved_cfg["model"]["api_mode"] == "codex_responses"
|
||||
assert captured_provider["api_mode"] == "codex_responses"
|
||||
|
||||
|
||||
def test_cmd_model_forwards_nous_login_tls_options(monkeypatch):
|
||||
monkeypatch.setattr(hermes_main, "_require_tty", lambda *a: None)
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -332,6 +332,45 @@ class TestCLIStatusBar:
|
||||
assert cli_obj._tui_input_rule_height("bottom", width=50) == 0
|
||||
assert cli_obj._tui_input_rule_height("bottom", width=90) == 1
|
||||
|
||||
def test_input_rules_hide_after_resize_until_next_input(self):
|
||||
"""When _status_bar_suppressed_after_resize is set, both rules hide.
|
||||
|
||||
See _recover_after_resize — column shrink reflows already-rendered
|
||||
bars into scrollback, so we hide the separators until the user
|
||||
submits the next input, at which point the flag is cleared.
|
||||
"""
|
||||
cli_obj = _make_cli()
|
||||
cli_obj._status_bar_suppressed_after_resize = True
|
||||
|
||||
assert cli_obj._tui_input_rule_height("top", width=90) == 0
|
||||
assert cli_obj._tui_input_rule_height("bottom", width=90) == 0
|
||||
|
||||
cli_obj._status_bar_suppressed_after_resize = False
|
||||
assert cli_obj._tui_input_rule_height("top", width=90) == 1
|
||||
assert cli_obj._tui_input_rule_height("bottom", width=90) == 1
|
||||
|
||||
def test_scrollback_box_width_returns_viewport_width(self):
|
||||
"""Decorative scrollback boxes use the full viewport width.
|
||||
|
||||
The previous clamp (max 56 cols) was reverted in favour of the
|
||||
prompt_toolkit ``_output_screen_diff`` monkey-patch landed in
|
||||
#26137, which keeps chrome out of scrollback at the source.
|
||||
We accept that an aggressive column-shrink may visually reflow
|
||||
already printed Panel borders — that's a cosmetic artifact of
|
||||
stamped scrollback history, not a live-render bug.
|
||||
"""
|
||||
from cli import HermesCLI
|
||||
|
||||
# Floor at 32 — narrow terminals still get something usable
|
||||
# (avoids negative ``'─' * (w - 2)`` math).
|
||||
assert HermesCLI._scrollback_box_width(20) == 32
|
||||
assert HermesCLI._scrollback_box_width(32) == 32
|
||||
# Above the floor, return the actual viewport width — no cap.
|
||||
assert HermesCLI._scrollback_box_width(48) == 48
|
||||
assert HermesCLI._scrollback_box_width(80) == 80
|
||||
assert HermesCLI._scrollback_box_width(120) == 120
|
||||
assert HermesCLI._scrollback_box_width(200) == 200
|
||||
|
||||
def test_agent_spacer_reclaimed_on_narrow_terminals(self):
|
||||
cli_obj = _make_cli()
|
||||
cli_obj._agent_running = True
|
||||
|
||||
@@ -215,13 +215,15 @@ def test_cprint_swallows_prompt_toolkit_import_error(monkeypatch):
|
||||
assert direct_prints == ["fallback2"]
|
||||
|
||||
|
||||
def test_output_history_strips_ansi_and_keeps_recent_lines():
|
||||
def test_output_history_preserves_ansi_and_keeps_recent_lines():
|
||||
cli._configure_output_history(True, 10)
|
||||
|
||||
for idx in range(12):
|
||||
cli._record_output_history(f"\x1b[31mline-{idx}\x1b[0m")
|
||||
|
||||
assert list(cli._OUTPUT_HISTORY) == [f"line-{idx}" for idx in range(2, 12)]
|
||||
assert list(cli._OUTPUT_HISTORY) == [
|
||||
f"\x1b[31mline-{idx}\x1b[0m" for idx in range(2, 12)
|
||||
]
|
||||
|
||||
|
||||
def test_replay_output_history_does_not_record_replayed_lines(monkeypatch):
|
||||
@@ -258,10 +260,35 @@ def test_replay_output_history_rerenders_callable_entries(monkeypatch):
|
||||
cli._replay_output_history()
|
||||
|
||||
assert widths_seen == ["called"]
|
||||
assert printed == ["top border", "body"]
|
||||
assert printed == ["top border\nbody"]
|
||||
assert list(cli._OUTPUT_HISTORY) == [_render_current_width]
|
||||
|
||||
|
||||
def test_replay_output_history_batches_rendered_lines_into_one_print(monkeypatch):
|
||||
cli._configure_output_history(True, 10)
|
||||
cli._record_output_history("first line")
|
||||
cli._record_output_history("second line")
|
||||
cli._record_output_history_entry(lambda: ["third line", "fourth line"])
|
||||
printed = []
|
||||
|
||||
monkeypatch.setattr(cli, "_pt_print", lambda value: printed.append(value))
|
||||
monkeypatch.setattr(cli, "_PT_ANSI", lambda text: text)
|
||||
|
||||
cli._replay_output_history()
|
||||
|
||||
assert printed == ["first line\nsecond line\nthird line\nfourth line"]
|
||||
|
||||
|
||||
def test_chat_console_records_rich_ansi_for_resize_replay(monkeypatch):
|
||||
cli._configure_output_history(True, 10)
|
||||
monkeypatch.setattr(cli, "_pt_print", lambda *_args, **_kwargs: None)
|
||||
|
||||
cli.ChatConsole().print("[bold red]Hello[/]")
|
||||
|
||||
assert cli._OUTPUT_HISTORY
|
||||
assert any("\x1b[" in line for line in cli._OUTPUT_HISTORY)
|
||||
|
||||
|
||||
def test_suspend_output_history_blocks_recording():
|
||||
cli._configure_output_history(True, 10)
|
||||
|
||||
|
||||
+5
-4
@@ -101,7 +101,6 @@ _CREDENTIAL_NAMES = frozenset({
|
||||
"RETAINDB_API_KEY",
|
||||
"HINDSIGHT_API_KEY",
|
||||
"HINDSIGHT_LLM_API_KEY",
|
||||
"TINKER_API_KEY",
|
||||
"DAYTONA_API_KEY",
|
||||
"TWILIO_AUTH_TOKEN",
|
||||
"TELEGRAM_BOT_TOKEN",
|
||||
@@ -476,12 +475,14 @@ def _reset_module_state():
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# --- agent.auxiliary_client — runtime main provider/model override ---
|
||||
# Set per-turn by AIAgent.run_conversation; tests that import it must
|
||||
# see a clean state so config.yaml fallback works as expected.
|
||||
# --- agent.auxiliary_client — runtime main provider/model override and
|
||||
# payment-error health cache. Both are process-global in production;
|
||||
# reset them per test so one worker's fallback/402 test does not make
|
||||
# later auxiliary-client tests skip otherwise-available providers.
|
||||
try:
|
||||
from agent import auxiliary_client as _aux_mod
|
||||
_aux_mod.clear_runtime_main()
|
||||
_aux_mod._reset_aux_unhealthy_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@@ -321,6 +321,93 @@ class TestPauseResumeJob:
|
||||
assert resumed["paused_reason"] is None
|
||||
|
||||
|
||||
class TestResolveJobRef:
|
||||
"""Name-based job lookup for CLI/tool callers (PR #2627, @buntingszn)."""
|
||||
|
||||
def test_resolve_by_exact_id(self, tmp_cron_dir):
|
||||
from cron.jobs import resolve_job_ref
|
||||
|
||||
job = create_job(prompt="A", schedule="1h", name="alpha")
|
||||
assert resolve_job_ref(job["id"])["id"] == job["id"]
|
||||
|
||||
def test_resolve_by_name(self, tmp_cron_dir):
|
||||
from cron.jobs import resolve_job_ref
|
||||
|
||||
job = create_job(prompt="A", schedule="1h", name="alpha")
|
||||
assert resolve_job_ref("alpha")["id"] == job["id"]
|
||||
|
||||
def test_resolve_by_name_case_insensitive(self, tmp_cron_dir):
|
||||
from cron.jobs import resolve_job_ref
|
||||
|
||||
job = create_job(prompt="A", schedule="1h", name="MyJob")
|
||||
assert resolve_job_ref("myjob")["id"] == job["id"]
|
||||
assert resolve_job_ref("MYJOB")["id"] == job["id"]
|
||||
|
||||
def test_resolve_returns_none_when_not_found(self, tmp_cron_dir):
|
||||
from cron.jobs import resolve_job_ref
|
||||
|
||||
create_job(prompt="A", schedule="1h", name="alpha")
|
||||
assert resolve_job_ref("does-not-exist") is None
|
||||
assert resolve_job_ref("") is None
|
||||
|
||||
def test_resolve_id_wins_over_name(self, tmp_cron_dir):
|
||||
"""If a job's name happens to equal another job's ID, ID match wins."""
|
||||
from cron.jobs import resolve_job_ref
|
||||
|
||||
j1 = create_job(prompt="A", schedule="1h")
|
||||
# Create a second job whose name is j1's ID
|
||||
j2 = create_job(prompt="B", schedule="1h", name=j1["id"])
|
||||
# Looking up j1["id"] must return j1, not the colliding-name job j2
|
||||
assert resolve_job_ref(j1["id"])["id"] == j1["id"]
|
||||
assert resolve_job_ref(j1["id"])["id"] != j2["id"]
|
||||
|
||||
def test_resolve_ambiguous_name_raises(self, tmp_cron_dir):
|
||||
"""Two jobs sharing a name → refuse to pick, surface both IDs."""
|
||||
from cron.jobs import AmbiguousJobReference, resolve_job_ref
|
||||
|
||||
j1 = create_job(prompt="A", schedule="1h", name="dup")
|
||||
j2 = create_job(prompt="B", schedule="1h", name="dup")
|
||||
with pytest.raises(AmbiguousJobReference) as exc_info:
|
||||
resolve_job_ref("dup")
|
||||
ids = {m["id"] for m in exc_info.value.matches}
|
||||
assert ids == {j1["id"], j2["id"]}
|
||||
# Error message mentions both IDs so the user can pick one
|
||||
assert j1["id"] in str(exc_info.value)
|
||||
assert j2["id"] in str(exc_info.value)
|
||||
|
||||
def test_trigger_by_name(self, tmp_cron_dir):
|
||||
from cron.jobs import trigger_job
|
||||
|
||||
job = create_job(prompt="A", schedule="1h", name="alpha")
|
||||
result = trigger_job("alpha")
|
||||
assert result is not None
|
||||
assert result["id"] == job["id"]
|
||||
|
||||
def test_pause_by_name(self, tmp_cron_dir):
|
||||
job = create_job(prompt="A", schedule="1h", name="alpha")
|
||||
result = pause_job("alpha", reason="manual")
|
||||
assert result is not None
|
||||
assert result["id"] == job["id"]
|
||||
assert result["state"] == "paused"
|
||||
|
||||
def test_remove_by_name(self, tmp_cron_dir):
|
||||
job = create_job(prompt="A", schedule="1h", name="alpha")
|
||||
assert remove_job("alpha") is True
|
||||
assert get_job(job["id"]) is None
|
||||
|
||||
def test_mutations_refuse_ambiguous_name(self, tmp_cron_dir):
|
||||
"""pause/resume/trigger/remove must refuse to act on an ambiguous name."""
|
||||
from cron.jobs import AmbiguousJobReference, trigger_job
|
||||
|
||||
create_job(prompt="A", schedule="1h", name="dup")
|
||||
create_job(prompt="B", schedule="1h", name="dup")
|
||||
for fn in (pause_job, resume_job, trigger_job):
|
||||
with pytest.raises(AmbiguousJobReference):
|
||||
fn("dup")
|
||||
with pytest.raises(AmbiguousJobReference):
|
||||
remove_job("dup")
|
||||
|
||||
|
||||
class TestMarkJobRun:
|
||||
def test_increments_completed(self, tmp_cron_dir):
|
||||
job = create_job(prompt="Test", schedule="every 1h")
|
||||
|
||||
@@ -1,164 +0,0 @@
|
||||
"""Security tests for Terminal-Bench 2 archive extraction."""
|
||||
|
||||
import base64
|
||||
import importlib
|
||||
import io
|
||||
import sys
|
||||
import tarfile
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _stub_module(name: str, **attrs):
|
||||
module = types.ModuleType(name)
|
||||
for key, value in attrs.items():
|
||||
setattr(module, key, value)
|
||||
return module
|
||||
|
||||
|
||||
def _load_terminalbench_module(monkeypatch):
|
||||
class _EvalHandlingEnum:
|
||||
STOP_TRAIN = "stop_train"
|
||||
|
||||
class _APIServerConfig:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
|
||||
class _AgentResult:
|
||||
pass
|
||||
|
||||
class _HermesAgentLoop:
|
||||
pass
|
||||
|
||||
class _HermesAgentBaseEnv:
|
||||
pass
|
||||
|
||||
class _HermesAgentEnvConfig:
|
||||
pass
|
||||
|
||||
class _ToolContext:
|
||||
pass
|
||||
|
||||
stub_modules = {
|
||||
"atroposlib": _stub_module("atroposlib"),
|
||||
"atroposlib.envs": _stub_module("atroposlib.envs"),
|
||||
"atroposlib.envs.base": _stub_module(
|
||||
"atroposlib.envs.base",
|
||||
EvalHandlingEnum=_EvalHandlingEnum,
|
||||
),
|
||||
"atroposlib.envs.server_handling": _stub_module("atroposlib.envs.server_handling"),
|
||||
"atroposlib.envs.server_handling.server_manager": _stub_module(
|
||||
"atroposlib.envs.server_handling.server_manager",
|
||||
APIServerConfig=_APIServerConfig,
|
||||
),
|
||||
"environments.agent_loop": _stub_module(
|
||||
"environments.agent_loop",
|
||||
AgentResult=_AgentResult,
|
||||
HermesAgentLoop=_HermesAgentLoop,
|
||||
),
|
||||
"environments.hermes_base_env": _stub_module(
|
||||
"environments.hermes_base_env",
|
||||
HermesAgentBaseEnv=_HermesAgentBaseEnv,
|
||||
HermesAgentEnvConfig=_HermesAgentEnvConfig,
|
||||
),
|
||||
"environments.tool_context": _stub_module(
|
||||
"environments.tool_context",
|
||||
ToolContext=_ToolContext,
|
||||
),
|
||||
"tools.terminal_tool": _stub_module(
|
||||
"tools.terminal_tool",
|
||||
register_task_env_overrides=lambda *args, **kwargs: None,
|
||||
clear_task_env_overrides=lambda *args, **kwargs: None,
|
||||
cleanup_vm=lambda *args, **kwargs: None,
|
||||
),
|
||||
}
|
||||
|
||||
stub_modules["atroposlib"].envs = stub_modules["atroposlib.envs"]
|
||||
stub_modules["atroposlib.envs"].base = stub_modules["atroposlib.envs.base"]
|
||||
stub_modules["atroposlib.envs"].server_handling = stub_modules["atroposlib.envs.server_handling"]
|
||||
stub_modules["atroposlib.envs.server_handling"].server_manager = stub_modules[
|
||||
"atroposlib.envs.server_handling.server_manager"
|
||||
]
|
||||
|
||||
for name, module in stub_modules.items():
|
||||
monkeypatch.setitem(sys.modules, name, module)
|
||||
|
||||
module_name = "environments.benchmarks.terminalbench_2.terminalbench2_env"
|
||||
sys.modules.pop(module_name, None)
|
||||
return importlib.import_module(module_name)
|
||||
|
||||
|
||||
def _build_tar_b64(entries):
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
||||
for entry in entries:
|
||||
kind = entry["kind"]
|
||||
info = tarfile.TarInfo(entry["name"])
|
||||
|
||||
if kind == "dir":
|
||||
info.type = tarfile.DIRTYPE
|
||||
tar.addfile(info)
|
||||
continue
|
||||
|
||||
if kind == "file":
|
||||
data = entry["data"].encode("utf-8")
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
continue
|
||||
|
||||
if kind == "symlink":
|
||||
info.type = tarfile.SYMTYPE
|
||||
info.linkname = entry["target"]
|
||||
tar.addfile(info)
|
||||
continue
|
||||
|
||||
raise ValueError(f"Unknown tar entry kind: {kind}")
|
||||
|
||||
return base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
def test_extract_base64_tar_allows_safe_files(tmp_path, monkeypatch):
|
||||
module = _load_terminalbench_module(monkeypatch)
|
||||
archive = _build_tar_b64(
|
||||
[
|
||||
{"kind": "dir", "name": "nested"},
|
||||
{"kind": "file", "name": "nested/hello.txt", "data": "hello"},
|
||||
]
|
||||
)
|
||||
|
||||
target = tmp_path / "extract"
|
||||
module._extract_base64_tar(archive, target)
|
||||
|
||||
assert (target / "nested" / "hello.txt").read_text(encoding="utf-8") == "hello"
|
||||
|
||||
|
||||
def test_extract_base64_tar_rejects_path_traversal(tmp_path, monkeypatch):
|
||||
module = _load_terminalbench_module(monkeypatch)
|
||||
archive = _build_tar_b64(
|
||||
[
|
||||
{"kind": "file", "name": "../escape.txt", "data": "owned"},
|
||||
]
|
||||
)
|
||||
|
||||
target = tmp_path / "extract"
|
||||
with pytest.raises(ValueError, match="Unsafe archive member path"):
|
||||
module._extract_base64_tar(archive, target)
|
||||
|
||||
assert not (tmp_path / "escape.txt").exists()
|
||||
|
||||
|
||||
def test_extract_base64_tar_rejects_symlinks(tmp_path, monkeypatch):
|
||||
module = _load_terminalbench_module(monkeypatch)
|
||||
archive = _build_tar_b64(
|
||||
[
|
||||
{"kind": "symlink", "name": "link", "target": "../../escape.txt"},
|
||||
]
|
||||
)
|
||||
|
||||
target = tmp_path / "extract"
|
||||
with pytest.raises(ValueError, match="Unsupported archive member type"):
|
||||
module._extract_base64_tar(archive, target)
|
||||
|
||||
assert not (target / "link").exists()
|
||||
@@ -119,6 +119,14 @@ def _ensure_discord_mock() -> None:
|
||||
self.title = title
|
||||
self.description = description
|
||||
self.color = color
|
||||
self.fields = []
|
||||
self.footer = None
|
||||
def add_field(self, *, name=None, value=None, inline=False, **_):
|
||||
self.fields.append({"name": name, "value": value, "inline": inline})
|
||||
return self
|
||||
def set_footer(self, *, text=None, icon_url=None, **_):
|
||||
self.footer = {"text": text, "icon_url": icon_url}
|
||||
return self
|
||||
discord_mod.Embed = _FakeEmbed
|
||||
|
||||
# ui.View / ui.Select / ui.Button: real classes (not MagicMock) so
|
||||
|
||||
@@ -105,6 +105,29 @@ class TestResponseStore:
|
||||
store = ResponseStore(max_size=10)
|
||||
assert store.delete("resp_missing") is False
|
||||
|
||||
def test_delete_clears_conversation_mapping(self):
|
||||
"""Deleting a response also removes conversation mappings that reference it."""
|
||||
store = ResponseStore(max_size=10)
|
||||
store.put("resp_1", {"output": "hello"})
|
||||
store.set_conversation("chat-a", "resp_1")
|
||||
assert store.get_conversation("chat-a") == "resp_1"
|
||||
store.delete("resp_1")
|
||||
assert store.get_conversation("chat-a") is None
|
||||
|
||||
def test_eviction_clears_conversation_mapping(self):
|
||||
"""LRU eviction also removes conversation mappings for evicted responses."""
|
||||
store = ResponseStore(max_size=2)
|
||||
store.put("resp_1", {"output": "one"})
|
||||
store.set_conversation("chat-a", "resp_1")
|
||||
store.put("resp_2", {"output": "two"})
|
||||
store.set_conversation("chat-b", "resp_2")
|
||||
# Adding a 3rd should evict resp_1 and its conversation mapping
|
||||
store.put("resp_3", {"output": "three"})
|
||||
assert store.get("resp_1") is None
|
||||
assert store.get_conversation("chat-a") is None
|
||||
# resp_2 mapping should still be intact
|
||||
assert store.get_conversation("chat-b") == "resp_2"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _IdempotencyCache
|
||||
@@ -2870,6 +2893,45 @@ class TestConversationParameter:
|
||||
# Conversation mapping should NOT be set since store=false
|
||||
assert adapter._response_store.get_conversation("ephemeral-chat") is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conversation_reuse_after_eviction_no_404(self, adapter):
|
||||
"""After eviction clears a conversation mapping, reusing that name starts fresh (no 404)."""
|
||||
adapter._response_store = ResponseStore(max_size=1)
|
||||
app = _create_app(adapter)
|
||||
async with TestClient(TestServer(app)) as cli:
|
||||
with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
|
||||
mock_run.return_value = (
|
||||
{"final_response": "First", "messages": [], "api_calls": 1},
|
||||
{"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
)
|
||||
# Create conversation -> resp stored
|
||||
resp1 = await cli.post("/v1/responses", json={
|
||||
"input": "hello",
|
||||
"conversation": "my-chat",
|
||||
})
|
||||
assert resp1.status == 200
|
||||
|
||||
# Evict by adding another response
|
||||
mock_run.return_value = (
|
||||
{"final_response": "Other", "messages": [], "api_calls": 1},
|
||||
{"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
)
|
||||
await cli.post("/v1/responses", json={"input": "other"})
|
||||
|
||||
# Conversation mapping should have been cleaned by eviction
|
||||
assert adapter._response_store.get_conversation("my-chat") is None
|
||||
|
||||
# Reuse conversation name — should start fresh, not 404
|
||||
mock_run.return_value = (
|
||||
{"final_response": "Restarted", "messages": [], "api_calls": 1},
|
||||
{"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
)
|
||||
resp3 = await cli.post("/v1/responses", json={
|
||||
"input": "hello again",
|
||||
"conversation": "my-chat",
|
||||
})
|
||||
assert resp3.status == 200
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# X-Hermes-Session-Id header (session continuity)
|
||||
|
||||
@@ -302,6 +302,43 @@ class TestLoadGatewayConfig:
|
||||
|
||||
assert config.thread_sessions_per_user is False
|
||||
|
||||
def test_bridges_discord_thread_require_mention_from_config_yaml(self, tmp_path, monkeypatch):
|
||||
"""discord.thread_require_mention in config.yaml should reach the runtime env var."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
config_path = hermes_home / "config.yaml"
|
||||
config_path.write_text(
|
||||
"discord:\n"
|
||||
" thread_require_mention: true\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
monkeypatch.delenv("DISCORD_THREAD_REQUIRE_MENTION", raising=False)
|
||||
|
||||
load_gateway_config()
|
||||
|
||||
assert os.environ.get("DISCORD_THREAD_REQUIRE_MENTION") == "true"
|
||||
|
||||
def test_thread_require_mention_yaml_does_not_overwrite_env(self, tmp_path, monkeypatch):
|
||||
"""Explicit env var should win over config.yaml (env > yaml precedence)."""
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
config_path = hermes_home / "config.yaml"
|
||||
config_path.write_text(
|
||||
"discord:\n"
|
||||
" thread_require_mention: false\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
monkeypatch.setenv("DISCORD_THREAD_REQUIRE_MENTION", "true") # user override
|
||||
|
||||
load_gateway_config()
|
||||
|
||||
# Env value preserved, not clobbered by yaml.
|
||||
assert os.environ.get("DISCORD_THREAD_REQUIRE_MENTION") == "true"
|
||||
|
||||
def test_bridges_quoted_false_platform_enabled_from_config_yaml(self, tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
@@ -372,6 +409,26 @@ class TestLoadGatewayConfig:
|
||||
"456": "Therapist mode",
|
||||
}
|
||||
|
||||
def test_bridges_discord_history_backfill_settings_from_config_yaml(self, tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
config_path = hermes_home / "config.yaml"
|
||||
config_path.write_text(
|
||||
"discord:\n"
|
||||
" history_backfill: true\n"
|
||||
" history_backfill_limit: 17\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
monkeypatch.delenv("DISCORD_HISTORY_BACKFILL", raising=False)
|
||||
monkeypatch.delenv("DISCORD_HISTORY_BACKFILL_LIMIT", raising=False)
|
||||
|
||||
load_gateway_config()
|
||||
|
||||
assert os.getenv("DISCORD_HISTORY_BACKFILL") == "true"
|
||||
assert os.getenv("DISCORD_HISTORY_BACKFILL_LIMIT") == "17"
|
||||
|
||||
def test_bridges_telegram_channel_prompts_from_config_yaml(self, tmp_path, monkeypatch):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
|
||||
@@ -10,6 +10,80 @@ import pytest
|
||||
from gateway.config import Platform, PlatformConfig
|
||||
|
||||
|
||||
class _FakeDingTalkModel:
|
||||
def __init__(self, **kwargs):
|
||||
self.__dict__.update(kwargs)
|
||||
|
||||
|
||||
class _FakeChatbotMessage(SimpleNamespace):
|
||||
@classmethod
|
||||
def from_dict(cls, data):
|
||||
data = data or {}
|
||||
return cls(
|
||||
message_id=data.get("msgId") or data.get("messageId") or data.get("message_id") or "",
|
||||
conversation_id=data.get("conversationId") or data.get("conversation_id") or "",
|
||||
conversation_type=str(data.get("conversationType") or data.get("conversation_type") or "1"),
|
||||
sender_id=data.get("senderId") or data.get("sender_id") or "",
|
||||
sender_staff_id=data.get("senderStaffId") or data.get("sender_staff_id") or data.get("senderId") or "",
|
||||
sender_nick=data.get("senderNick") or data.get("sender_nick") or "",
|
||||
text=data.get("text") or "",
|
||||
rich_text=data.get("richText") or data.get("rich_text"),
|
||||
rich_text_content=data.get("richTextContent") or data.get("rich_text_content"),
|
||||
session_webhook=data.get("sessionWebhook") or data.get("session_webhook") or "",
|
||||
session_webhook_expired_time=data.get("sessionWebhookExpiredTime") or data.get("session_webhook_expired_time") or 0,
|
||||
create_at=data.get("createAt") or data.get("create_at") or 0,
|
||||
at_users=data.get("atUsers") or data.get("at_users") or [],
|
||||
is_in_at_list=bool(data.get("isInAtList") or data.get("is_in_at_list")),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fake_dingtalk_optional_sdks(monkeypatch):
|
||||
"""Keep DingTalk adapter tests hermetic when optional SDKs are absent."""
|
||||
from gateway.platforms import dingtalk as dt
|
||||
|
||||
card_models = SimpleNamespace(**{
|
||||
name: _FakeDingTalkModel
|
||||
for name in (
|
||||
"CreateCardRequest",
|
||||
"CreateCardRequestCardData",
|
||||
"CreateCardRequestImGroupOpenSpaceModel",
|
||||
"CreateCardRequestImRobotOpenSpaceModel",
|
||||
"CreateCardHeaders",
|
||||
"DeliverCardRequest",
|
||||
"DeliverCardRequestImGroupOpenDeliverModel",
|
||||
"DeliverCardRequestImRobotOpenDeliverModel",
|
||||
"DeliverCardHeaders",
|
||||
"StreamingUpdateRequest",
|
||||
"StreamingUpdateHeaders",
|
||||
)
|
||||
})
|
||||
robot_models = SimpleNamespace(**{
|
||||
name: _FakeDingTalkModel
|
||||
for name in (
|
||||
"RobotReplyEmotionRequestTextEmotion",
|
||||
"RobotReplyEmotionRequest",
|
||||
"RobotReplyEmotionHeaders",
|
||||
"RobotRecallEmotionRequestTextEmotion",
|
||||
"RobotRecallEmotionRequest",
|
||||
"RobotRecallEmotionHeaders",
|
||||
"RobotMessageFileDownloadRequest",
|
||||
"RobotMessageFileDownloadHeaders",
|
||||
)
|
||||
})
|
||||
|
||||
monkeypatch.setattr(dt, "ChatbotMessage", _FakeChatbotMessage, raising=False)
|
||||
monkeypatch.setattr(
|
||||
dt,
|
||||
"AckMessage",
|
||||
SimpleNamespace(STATUS_OK=200, STATUS_SYSTEM_EXCEPTION=500),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(dt, "tea_util_models", SimpleNamespace(RuntimeOptions=_FakeDingTalkModel), raising=False)
|
||||
monkeypatch.setattr(dt, "dingtalk_card_models", card_models, raising=False)
|
||||
monkeypatch.setattr(dt, "dingtalk_robot_models", robot_models, raising=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Requirements check
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -18,7 +92,8 @@ from gateway.config import Platform, PlatformConfig
|
||||
class TestDingTalkRequirements:
|
||||
|
||||
def test_returns_false_when_sdk_missing(self, monkeypatch):
|
||||
with patch.dict("sys.modules", {"dingtalk_stream": None}):
|
||||
with patch.dict("sys.modules", {"dingtalk_stream": None}), \
|
||||
patch("tools.lazy_deps.ensure", side_effect=ImportError("dingtalk_stream unavailable")):
|
||||
monkeypatch.setattr(
|
||||
"gateway.platforms.dingtalk.DINGTALK_STREAM_AVAILABLE", False
|
||||
)
|
||||
|
||||
@@ -0,0 +1,408 @@
|
||||
"""Tests for Discord clarify button rendering and resolution.
|
||||
|
||||
Mirrors test_telegram_clarify_buttons.py for the Discord ``send_clarify``
|
||||
override and the ``ClarifyChoiceView`` callbacks. Discord uses ``discord.ui.View``
|
||||
button callbacks (closures) rather than a string-prefixed callback_query
|
||||
dispatcher like Telegram — the auth + resolution path is the same:
|
||||
|
||||
· numeric choice → resolve_gateway_clarify(clarify_id, choice_text)
|
||||
· "Other" button → mark_awaiting_text(clarify_id) so the text-intercept
|
||||
captures the next user message in this session
|
||||
· already-resolved or unauthorized → ephemeral "this prompt..." reply
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Repo root importable
|
||||
_repo = str(Path(__file__).resolve().parents[2])
|
||||
if _repo not in sys.path:
|
||||
sys.path.insert(0, _repo)
|
||||
|
||||
# Triggers the shared discord mock from tests/gateway/conftest.py before
|
||||
# importing the production module.
|
||||
from gateway.platforms.discord import ( # noqa: E402
|
||||
ClarifyChoiceView,
|
||||
DiscordAdapter,
|
||||
)
|
||||
from gateway.config import PlatformConfig # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_adapter(*, allowed_users=None, allowed_roles=None):
|
||||
config = PlatformConfig(enabled=True, token="test-token", extra={})
|
||||
adapter = DiscordAdapter(config)
|
||||
adapter._client = MagicMock()
|
||||
adapter._allowed_user_ids = set(allowed_users or [])
|
||||
adapter._allowed_role_ids = set(allowed_roles or [])
|
||||
return adapter
|
||||
|
||||
|
||||
def _clear_clarify_state():
|
||||
from tools import clarify_gateway as cm
|
||||
with cm._lock:
|
||||
cm._entries.clear()
|
||||
cm._session_index.clear()
|
||||
cm._notify_cbs.clear()
|
||||
|
||||
|
||||
def _make_interaction(*, user_id="42", display_name="Tester", roles=None,
|
||||
include_message=True):
|
||||
"""Build a mock discord.Interaction with response.edit_message /
|
||||
send_message / defer all coroutine-callable."""
|
||||
user = SimpleNamespace(
|
||||
id=user_id,
|
||||
display_name=display_name,
|
||||
roles=[SimpleNamespace(id=r) for r in (roles or [])],
|
||||
)
|
||||
response = SimpleNamespace(
|
||||
edit_message=AsyncMock(),
|
||||
send_message=AsyncMock(),
|
||||
defer=AsyncMock(),
|
||||
)
|
||||
if include_message:
|
||||
embed = MagicMock()
|
||||
embed.color = None
|
||||
embed.set_footer = MagicMock()
|
||||
message = SimpleNamespace(embeds=[embed])
|
||||
else:
|
||||
message = None
|
||||
return SimpleNamespace(user=user, response=response, message=message)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# ClarifyChoiceView construction
|
||||
# ===========================================================================
|
||||
|
||||
class TestClarifyChoiceViewConstruction:
|
||||
"""The view should build numeric buttons plus an Other button."""
|
||||
|
||||
def test_renders_n_choice_buttons_plus_other(self):
|
||||
view = ClarifyChoiceView(
|
||||
choices=["apple", "banana", "cherry"],
|
||||
clarify_id="cidX",
|
||||
allowed_user_ids={"42"},
|
||||
)
|
||||
# 3 numeric + 1 "Other"
|
||||
assert len(view.children) == 4
|
||||
labels = [b.label for b in view.children]
|
||||
assert labels[0].startswith("1. apple")
|
||||
assert labels[1].startswith("2. banana")
|
||||
assert labels[2].startswith("3. cherry")
|
||||
assert "Other" in labels[3]
|
||||
# custom_ids encode clarify_id + index/other
|
||||
ids = [b.custom_id for b in view.children]
|
||||
assert ids[0] == "clarify:cidX:0"
|
||||
assert ids[1] == "clarify:cidX:1"
|
||||
assert ids[2] == "clarify:cidX:2"
|
||||
assert ids[3] == "clarify:cidX:other"
|
||||
|
||||
def test_caps_at_24_choices_plus_other(self):
|
||||
choices = [f"choice-{i}" for i in range(50)]
|
||||
view = ClarifyChoiceView(
|
||||
choices=choices,
|
||||
clarify_id="cidY",
|
||||
allowed_user_ids=set(),
|
||||
)
|
||||
# Discord limit is 25 components; we cap choices at 24 + 1 Other = 25
|
||||
assert len(view.children) == 25
|
||||
assert "Other" in view.children[-1].label
|
||||
|
||||
def test_truncates_long_choice_label(self):
|
||||
long_choice = "x" * 200
|
||||
view = ClarifyChoiceView(
|
||||
choices=[long_choice],
|
||||
clarify_id="cidZ",
|
||||
allowed_user_ids=set(),
|
||||
)
|
||||
# 75 chars + 3 ellipsis chars in the body, plus "1. " prefix
|
||||
first_label = view.children[0].label
|
||||
assert first_label.startswith("1. ")
|
||||
assert first_label.endswith("...")
|
||||
# Final label total <= 80 (Discord cap on button labels)
|
||||
assert len(first_label) <= 80
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Choice callback → resolve_gateway_clarify
|
||||
# ===========================================================================
|
||||
|
||||
class TestClarifyChoiceResolve:
|
||||
"""Clicking a numeric button should resolve the clarify entry."""
|
||||
|
||||
def setup_method(self):
|
||||
_clear_clarify_state()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_choice_resolves_with_canonical_choice_text(self):
|
||||
from tools import clarify_gateway as cm
|
||||
cm.register("cidA", "sk-A", "Pick", ["red", "green", "blue"])
|
||||
|
||||
view = ClarifyChoiceView(
|
||||
choices=["red", "green", "blue"],
|
||||
clarify_id="cidA",
|
||||
allowed_user_ids={"42"},
|
||||
)
|
||||
|
||||
interaction = _make_interaction(user_id="42")
|
||||
await view._resolve_choice(interaction, index=1, choice="green")
|
||||
|
||||
# Resolved through clarify primitive
|
||||
with cm._lock:
|
||||
entry = cm._entries.get("cidA")
|
||||
assert entry is not None
|
||||
assert entry.response == "green"
|
||||
assert entry.event.is_set()
|
||||
# Buttons disabled
|
||||
assert all(b.disabled for b in view.children)
|
||||
# Embed updated + edit_message called
|
||||
interaction.response.edit_message.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_choice_falls_back_to_label_text_when_entry_missing(self):
|
||||
"""If the gateway entry vanished (race / stale view), the button's
|
||||
own choice text is used as the response."""
|
||||
from tools import clarify_gateway as cm
|
||||
# Note: no cm.register() — entry intentionally absent
|
||||
|
||||
view = ClarifyChoiceView(
|
||||
choices=["alpha"],
|
||||
clarify_id="cidGone",
|
||||
allowed_user_ids=set(),
|
||||
)
|
||||
interaction = _make_interaction()
|
||||
# Doesn't raise; resolve_gateway_clarify returns False quietly
|
||||
await view._resolve_choice(interaction, index=0, choice="alpha")
|
||||
# Still marks the view resolved + disables buttons
|
||||
assert view.resolved is True
|
||||
assert all(b.disabled for b in view.children)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_already_resolved_sends_ephemeral_reply(self):
|
||||
view = ClarifyChoiceView(
|
||||
choices=["a", "b"],
|
||||
clarify_id="cidB",
|
||||
allowed_user_ids=set(),
|
||||
)
|
||||
view.resolved = True
|
||||
|
||||
interaction = _make_interaction()
|
||||
await view._resolve_choice(interaction, index=0, choice="a")
|
||||
|
||||
interaction.response.send_message.assert_called_once()
|
||||
kwargs = interaction.response.send_message.call_args.kwargs
|
||||
assert kwargs.get("ephemeral") is True
|
||||
# No resolve was called
|
||||
interaction.response.edit_message.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unauthorized_user_rejected(self):
|
||||
from tools import clarify_gateway as cm
|
||||
cm.register("cidC", "sk-C", "Pick", ["x"])
|
||||
|
||||
# Allowlist set, user not in it
|
||||
view = ClarifyChoiceView(
|
||||
choices=["x"],
|
||||
clarify_id="cidC",
|
||||
allowed_user_ids={"99999"}, # not 42
|
||||
)
|
||||
|
||||
interaction = _make_interaction(user_id="42")
|
||||
await view._resolve_choice(interaction, index=0, choice="x")
|
||||
|
||||
# Ephemeral rejection, no resolution, no edit
|
||||
interaction.response.send_message.assert_called_once()
|
||||
kwargs = interaction.response.send_message.call_args.kwargs
|
||||
assert kwargs.get("ephemeral") is True
|
||||
interaction.response.edit_message.assert_not_called()
|
||||
with cm._lock:
|
||||
entry = cm._entries.get("cidC")
|
||||
assert entry is not None
|
||||
assert not entry.event.is_set()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# "Other" button → mark_awaiting_text
|
||||
# ===========================================================================
|
||||
|
||||
class TestClarifyOtherButton:
|
||||
"""Clicking Other should flip the entry into text-capture mode."""
|
||||
|
||||
def setup_method(self):
|
||||
_clear_clarify_state()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_other_flips_entry_to_awaiting_text(self):
|
||||
from tools import clarify_gateway as cm
|
||||
cm.register("cidD", "sk-D", "Pick", ["x", "y"])
|
||||
|
||||
view = ClarifyChoiceView(
|
||||
choices=["x", "y"],
|
||||
clarify_id="cidD",
|
||||
allowed_user_ids=set(),
|
||||
)
|
||||
|
||||
interaction = _make_interaction()
|
||||
await view._on_other(interaction)
|
||||
|
||||
# Entry awaiting_text now
|
||||
pending = cm.get_pending_for_session("sk-D")
|
||||
assert pending is not None
|
||||
assert pending.clarify_id == "cidD"
|
||||
assert pending.awaiting_text is True
|
||||
# Entry still pending (not resolved)
|
||||
with cm._lock:
|
||||
entry = cm._entries.get("cidD")
|
||||
assert entry is not None
|
||||
assert not entry.event.is_set()
|
||||
# View locked + buttons disabled
|
||||
assert view.resolved is True
|
||||
assert all(b.disabled for b in view.children)
|
||||
interaction.response.edit_message.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_other_unauthorized_user_rejected(self):
|
||||
from tools import clarify_gateway as cm
|
||||
cm.register("cidE", "sk-E", "Pick", ["x"])
|
||||
|
||||
view = ClarifyChoiceView(
|
||||
choices=["x"],
|
||||
clarify_id="cidE",
|
||||
allowed_user_ids={"99999"},
|
||||
)
|
||||
|
||||
interaction = _make_interaction(user_id="42")
|
||||
await view._on_other(interaction)
|
||||
|
||||
# Rejected; entry NOT awaiting text
|
||||
interaction.response.send_message.assert_called_once()
|
||||
pending = cm.get_pending_for_session("sk-E")
|
||||
assert pending is None or pending.awaiting_text is False
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# DiscordAdapter.send_clarify integration
|
||||
# ===========================================================================
|
||||
|
||||
class TestDiscordSendClarify:
|
||||
"""Verify send_clarify renders an embed and (optionally) attaches the view."""
|
||||
|
||||
def setup_method(self):
|
||||
_clear_clarify_state()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_choice_attaches_view(self):
|
||||
adapter = _make_adapter(allowed_users={"42"})
|
||||
channel = MagicMock()
|
||||
sent_msg = MagicMock()
|
||||
sent_msg.id = 123456
|
||||
channel.send = AsyncMock(return_value=sent_msg)
|
||||
adapter._client.get_channel = MagicMock(return_value=channel)
|
||||
|
||||
result = await adapter.send_clarify(
|
||||
chat_id="9001",
|
||||
question="Pick a color",
|
||||
choices=["red", "green", "blue"],
|
||||
clarify_id="cidM",
|
||||
session_key="sk-M",
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.message_id == "123456"
|
||||
# Verify channel.send was called with embed + view kwargs
|
||||
channel.send.assert_called_once()
|
||||
kwargs = channel.send.call_args.kwargs
|
||||
assert "embed" in kwargs
|
||||
assert "view" in kwargs
|
||||
assert isinstance(kwargs["view"], ClarifyChoiceView)
|
||||
# 3 choice buttons + 1 Other
|
||||
assert len(kwargs["view"].children) == 4
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_ended_omits_view(self):
|
||||
adapter = _make_adapter()
|
||||
channel = MagicMock()
|
||||
sent_msg = MagicMock()
|
||||
sent_msg.id = 222
|
||||
channel.send = AsyncMock(return_value=sent_msg)
|
||||
adapter._client.get_channel = MagicMock(return_value=channel)
|
||||
|
||||
result = await adapter.send_clarify(
|
||||
chat_id="9001",
|
||||
question="What is your name?",
|
||||
choices=None,
|
||||
clarify_id="cidOE",
|
||||
session_key="sk-OE",
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
channel.send.assert_called_once()
|
||||
kwargs = channel.send.call_args.kwargs
|
||||
# Open-ended path renders embed but no view (text-capture handles reply)
|
||||
assert "embed" in kwargs
|
||||
assert "view" not in kwargs
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routes_to_thread_when_metadata_thread_id_set(self):
|
||||
adapter = _make_adapter()
|
||||
channel = MagicMock()
|
||||
sent_msg = MagicMock()
|
||||
sent_msg.id = 333
|
||||
channel.send = AsyncMock(return_value=sent_msg)
|
||||
adapter._client.get_channel = MagicMock(return_value=channel)
|
||||
|
||||
await adapter.send_clarify(
|
||||
chat_id="9001",
|
||||
question="?",
|
||||
choices=["a"],
|
||||
clarify_id="cidT",
|
||||
session_key="sk-T",
|
||||
metadata={"thread_id": "7777"},
|
||||
)
|
||||
|
||||
# Channel lookup should resolve to thread id, not chat_id
|
||||
adapter._client.get_channel.assert_called_once_with(7777)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_connected_returns_failure(self):
|
||||
adapter = _make_adapter()
|
||||
adapter._client = None
|
||||
result = await adapter.send_clarify(
|
||||
chat_id="9001",
|
||||
question="?",
|
||||
choices=["a"],
|
||||
clarify_id="cidNC",
|
||||
session_key="sk-NC",
|
||||
)
|
||||
assert result.success is False
|
||||
assert "Not connected" in (result.error or "")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_filters_empty_and_whitespace_choices(self):
|
||||
adapter = _make_adapter()
|
||||
channel = MagicMock()
|
||||
sent_msg = MagicMock()
|
||||
sent_msg.id = 444
|
||||
channel.send = AsyncMock(return_value=sent_msg)
|
||||
adapter._client.get_channel = MagicMock(return_value=channel)
|
||||
|
||||
await adapter.send_clarify(
|
||||
chat_id="9001",
|
||||
question="?",
|
||||
choices=["", " ", "real-choice", None],
|
||||
clarify_id="cidF",
|
||||
session_key="sk-F",
|
||||
)
|
||||
kwargs = channel.send.call_args.kwargs
|
||||
view = kwargs["view"]
|
||||
# Only 1 real choice + 1 Other = 2 children
|
||||
assert len(view.children) == 2
|
||||
assert "real-choice" in view.children[0].label
|
||||
@@ -62,6 +62,12 @@ class FakeTextChannel:
|
||||
self.guild = SimpleNamespace(name=guild_name)
|
||||
self.topic = None
|
||||
|
||||
def history(self, *, limit, before, after=None, oldest_first=None):
|
||||
async def _iter():
|
||||
return
|
||||
yield
|
||||
return _iter()
|
||||
|
||||
|
||||
class FakeForumChannel:
|
||||
def __init__(self, channel_id: int = 1, name: str = "support-forum", guild_name: str = "Hermes Server"):
|
||||
@@ -81,6 +87,12 @@ class FakeThread:
|
||||
self.guild = getattr(parent, "guild", None) or SimpleNamespace(name=guild_name)
|
||||
self.topic = None
|
||||
|
||||
def history(self, *, limit, before, after=None, oldest_first=None):
|
||||
async def _iter():
|
||||
return
|
||||
yield
|
||||
return _iter()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def adapter(monkeypatch):
|
||||
@@ -88,6 +100,23 @@ def adapter(monkeypatch):
|
||||
monkeypatch.setattr(discord_platform.discord, "Thread", FakeThread, raising=False)
|
||||
monkeypatch.setattr(discord_platform.discord, "ForumChannel", FakeForumChannel, raising=False)
|
||||
|
||||
# Clear DISCORD_* env vars the test file exercises so tests don't leak
|
||||
# process-env state from the contributor's shell into per-test behaviour.
|
||||
# Individual tests still monkeypatch.setenv() for their own scenarios.
|
||||
for _var in (
|
||||
"DISCORD_REQUIRE_MENTION",
|
||||
"DISCORD_THREAD_REQUIRE_MENTION",
|
||||
"DISCORD_FREE_RESPONSE_CHANNELS",
|
||||
"DISCORD_AUTO_THREAD",
|
||||
"DISCORD_NO_THREAD_CHANNELS",
|
||||
"DISCORD_ALLOWED_CHANNELS",
|
||||
"DISCORD_IGNORED_CHANNELS",
|
||||
"DISCORD_HISTORY_BACKFILL",
|
||||
"DISCORD_HISTORY_BACKFILL_LIMIT",
|
||||
"DISCORD_ALLOW_BOTS",
|
||||
):
|
||||
monkeypatch.delenv(_var, raising=False)
|
||||
|
||||
config = PlatformConfig(enabled=True, token="fake-token")
|
||||
adapter = DiscordAdapter(config)
|
||||
adapter._client = SimpleNamespace(user=SimpleNamespace(id=999))
|
||||
@@ -111,6 +140,48 @@ def make_message(*, channel, content: str, mentions=None, msg_type=None):
|
||||
)
|
||||
|
||||
|
||||
def make_history_message(
|
||||
*,
|
||||
author,
|
||||
content: str,
|
||||
msg_id: int,
|
||||
msg_type=None,
|
||||
attachments=None,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
id=msg_id,
|
||||
author=author,
|
||||
content=content,
|
||||
attachments=list(attachments or []),
|
||||
type=msg_type if msg_type is not None else discord_platform.discord.MessageType.default,
|
||||
)
|
||||
|
||||
|
||||
class FakeHistoryChannel(FakeTextChannel):
|
||||
def __init__(self, history_messages, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._history_messages = list(history_messages)
|
||||
|
||||
def history(self, *, limit, before, after=None, oldest_first=None):
|
||||
before_id = int(getattr(before, "id", before))
|
||||
after_id = int(getattr(after, "id", after)) if after is not None else None
|
||||
if oldest_first is None:
|
||||
oldest_first = after is not None
|
||||
|
||||
messages = [
|
||||
message for message in self._history_messages
|
||||
if int(message.id) < before_id
|
||||
and (after_id is None or int(message.id) > after_id)
|
||||
]
|
||||
messages.sort(key=lambda message: int(message.id), reverse=not oldest_first)
|
||||
|
||||
async def _iter():
|
||||
for message in messages[:limit]:
|
||||
yield message
|
||||
|
||||
return _iter()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_defaults_to_require_mention(adapter, monkeypatch):
|
||||
"""Default behavior: require @mention in server channels."""
|
||||
@@ -446,6 +517,37 @@ async def test_discord_voice_linked_channel_skips_mention_requirement_and_auto_t
|
||||
assert event.source.chat_type == "group"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_free_response_channel_skips_auto_thread(adapter, monkeypatch):
|
||||
"""Free-response channels should reply inline, never spawn a new thread.
|
||||
|
||||
Without this, every message in a free-response channel would auto-create
|
||||
a fresh thread (since the channel bypasses the @mention gate, every
|
||||
message looks like a fresh trigger). That turns a "lightweight chat"
|
||||
channel into a thread-spawning machine — see the docs at
|
||||
website/docs/user-guide/messaging/discord.md which already describe
|
||||
this as the intended behavior.
|
||||
"""
|
||||
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.setenv("DISCORD_FREE_RESPONSE_CHANNELS", "789")
|
||||
monkeypatch.delenv("DISCORD_AUTO_THREAD", raising=False) # default true
|
||||
|
||||
adapter._auto_create_thread = AsyncMock()
|
||||
|
||||
message = make_message(
|
||||
channel=FakeTextChannel(channel_id=789),
|
||||
content="casual chat in free-response channel",
|
||||
)
|
||||
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter._auto_create_thread.assert_not_awaited()
|
||||
adapter.handle_message.assert_awaited_once()
|
||||
event = adapter.handle_message.await_args.args[0]
|
||||
assert event.text == "casual chat in free-response channel"
|
||||
assert event.source.chat_type == "group"
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -463,3 +565,322 @@ async def test_discord_voice_linked_parent_thread_still_requires_mention(adapter
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter.handle_message.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_thread_default_keeps_responding_after_participation(adapter, monkeypatch):
|
||||
"""Default behavior: once the bot is in a thread, it auto-responds without @mention."""
|
||||
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
|
||||
monkeypatch.delenv("DISCORD_THREAD_REQUIRE_MENTION", raising=False)
|
||||
|
||||
thread = FakeThread(channel_id=456, name="follow-up")
|
||||
adapter._threads.mark("456") # bot has previously participated
|
||||
|
||||
message = make_message(channel=thread, content="follow-up without mention")
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter.handle_message.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_thread_require_mention_gates_followups(adapter, monkeypatch):
|
||||
"""When thread_require_mention=true, even bot-participated threads need @mention."""
|
||||
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.setenv("DISCORD_THREAD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
|
||||
|
||||
thread = FakeThread(channel_id=456, name="multi-bot thread")
|
||||
adapter._threads.mark("456") # bot has previously participated
|
||||
|
||||
message = make_message(channel=thread, content="ambient chatter — not for me")
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter.handle_message.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_thread_require_mention_still_responds_when_mentioned(adapter, monkeypatch):
|
||||
"""thread_require_mention=true still lets explicit @mentions through in threads."""
|
||||
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.setenv("DISCORD_THREAD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
|
||||
|
||||
thread = FakeThread(channel_id=456, name="multi-bot thread")
|
||||
adapter._threads.mark("456")
|
||||
bot_user = adapter._client.user
|
||||
|
||||
message = make_message(
|
||||
channel=thread,
|
||||
content=f"<@{bot_user.id}> hey, this one's for you",
|
||||
mentions=[bot_user],
|
||||
)
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter.handle_message.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_thread_require_mention_via_config_extra(adapter, monkeypatch):
|
||||
"""thread_require_mention can also be set via config.extra (yaml)."""
|
||||
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.delenv("DISCORD_THREAD_REQUIRE_MENTION", raising=False)
|
||||
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
|
||||
adapter.config.extra["thread_require_mention"] = True
|
||||
|
||||
thread = FakeThread(channel_id=456, name="multi-bot thread")
|
||||
adapter._threads.mark("456")
|
||||
|
||||
message = make_message(channel=thread, content="ambient — should be ignored")
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter.handle_message.assert_not_awaited()
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_channel_context_stops_at_self_message_and_reverses_to_chronological_order(adapter, monkeypatch):
|
||||
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
|
||||
adapter.config.extra["history_backfill_limit"] = 10
|
||||
|
||||
other_bot = SimpleNamespace(id=55, display_name="Gemini", name="Gemini", bot=True)
|
||||
human = SimpleNamespace(id=56, display_name="Alice", name="Alice", bot=False)
|
||||
old_human = SimpleNamespace(id=57, display_name="Bob", name="Bob", bot=False)
|
||||
|
||||
channel = FakeHistoryChannel(
|
||||
[
|
||||
make_history_message(author=human, content="latest human note", msg_id=4),
|
||||
make_history_message(author=other_bot, content="latest bot note", msg_id=3),
|
||||
make_history_message(author=adapter._client.user, content="our prior response", msg_id=2),
|
||||
make_history_message(author=old_human, content="older than boundary", msg_id=1),
|
||||
],
|
||||
channel_id=123,
|
||||
)
|
||||
|
||||
result = await adapter._fetch_channel_context(channel, before=make_message(channel=channel, content="trigger"))
|
||||
|
||||
assert result == (
|
||||
"[Recent channel messages]\n"
|
||||
"[Gemini [bot]] latest bot note\n"
|
||||
"[Alice] latest human note"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_channel_context_skips_other_bots_when_allow_bots_none(adapter, monkeypatch):
|
||||
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "none")
|
||||
adapter.config.extra["history_backfill_limit"] = 10
|
||||
|
||||
other_bot = SimpleNamespace(id=55, display_name="Gemini", name="Gemini", bot=True)
|
||||
human = SimpleNamespace(id=56, display_name="Alice", name="Alice", bot=False)
|
||||
|
||||
channel = FakeHistoryChannel(
|
||||
[
|
||||
make_history_message(author=human, content="human note", msg_id=3),
|
||||
make_history_message(author=other_bot, content="bot note", msg_id=2),
|
||||
],
|
||||
channel_id=123,
|
||||
)
|
||||
|
||||
result = await adapter._fetch_channel_context(channel, before=make_message(channel=channel, content="trigger"))
|
||||
|
||||
assert result == "[Recent channel messages]\n[Alice] human note"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_channel_context_uses_cache_to_narrow_window(adapter, monkeypatch):
|
||||
"""When _last_self_message_id is cached, the fetch passes after= to skip old messages."""
|
||||
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
|
||||
adapter.config.extra["history_backfill_limit"] = 50
|
||||
|
||||
human = SimpleNamespace(id=56, display_name="Alice", name="Alice", bot=False)
|
||||
|
||||
# Record the after= arg passed to history()
|
||||
recorded_after = {}
|
||||
|
||||
class CacheTrackingChannel(FakeHistoryChannel):
|
||||
def history(self, *, limit, before, after=None, oldest_first=None):
|
||||
recorded_after["value"] = after
|
||||
return super().history(
|
||||
limit=limit,
|
||||
before=before,
|
||||
after=after,
|
||||
oldest_first=oldest_first,
|
||||
)
|
||||
|
||||
channel = CacheTrackingChannel(
|
||||
[make_history_message(author=human, content="hello", msg_id=200)],
|
||||
channel_id=777,
|
||||
)
|
||||
|
||||
# Seed the cache — bot's last message in this channel was ID 100
|
||||
adapter._last_self_message_id["777"] = "100"
|
||||
|
||||
trigger = make_message(channel=channel, content="trigger")
|
||||
trigger.id = 300 # trigger is newer than cache
|
||||
|
||||
result = await adapter._fetch_channel_context(channel, before=trigger)
|
||||
|
||||
assert result == "[Recent channel messages]\n[Alice] hello"
|
||||
# Verify cache was used: after= should be set (not None)
|
||||
assert recorded_after["value"] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_channel_context_cache_uses_latest_window_when_after_set(adapter, monkeypatch):
|
||||
"""Regression: discord.py defaults oldest_first=True when after= is provided.
|
||||
|
||||
The hot cache path passes both after= and before=. We still want the latest
|
||||
messages before the trigger, not the earliest messages after our prior
|
||||
response, otherwise tool traces can crowd out the final answer.
|
||||
"""
|
||||
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
|
||||
adapter.config.extra["history_backfill_limit"] = 3
|
||||
|
||||
codex = SimpleNamespace(id=56, display_name="Codex", name="Codex", bot=True)
|
||||
human = SimpleNamespace(id=57, display_name="Alice", name="Alice", bot=False)
|
||||
|
||||
channel = FakeHistoryChannel(
|
||||
[
|
||||
make_history_message(author=codex, content="old tool trace 1", msg_id=101),
|
||||
make_history_message(author=codex, content="old tool trace 2", msg_id=102),
|
||||
make_history_message(author=codex, content="old tool trace 3", msg_id=103),
|
||||
make_history_message(author=codex, content="final analysis", msg_id=104),
|
||||
make_history_message(author=human, content="latest follow-up", msg_id=105),
|
||||
],
|
||||
channel_id=777,
|
||||
)
|
||||
adapter._last_self_message_id["777"] = "100"
|
||||
|
||||
trigger = make_message(channel=channel, content="trigger")
|
||||
trigger.id = 200
|
||||
|
||||
result = await adapter._fetch_channel_context(channel, before=trigger)
|
||||
|
||||
assert "[Codex [bot]] final analysis" in result
|
||||
assert "[Alice] latest follow-up" in result
|
||||
assert "old tool trace 1" not in result
|
||||
assert "old tool trace 2" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_channel_context_ignores_stale_cache(adapter, monkeypatch):
|
||||
"""If cached ID is >= trigger ID (stale/future), fall back to cold-start scan."""
|
||||
monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all")
|
||||
adapter.config.extra["history_backfill_limit"] = 50
|
||||
|
||||
human = SimpleNamespace(id=56, display_name="Alice", name="Alice", bot=False)
|
||||
|
||||
recorded_after = {}
|
||||
|
||||
class CacheTrackingChannel(FakeHistoryChannel):
|
||||
def history(self, *, limit, before, after=None, oldest_first=None):
|
||||
recorded_after["value"] = after
|
||||
return super().history(
|
||||
limit=limit,
|
||||
before=before,
|
||||
after=after,
|
||||
oldest_first=oldest_first,
|
||||
)
|
||||
|
||||
channel = CacheTrackingChannel(
|
||||
[make_history_message(author=human, content="hello", msg_id=50)],
|
||||
channel_id=777,
|
||||
)
|
||||
|
||||
# Cache has a NEWER ID than the trigger — stale/invalid
|
||||
adapter._last_self_message_id["777"] = "500"
|
||||
|
||||
trigger = make_message(channel=channel, content="trigger")
|
||||
trigger.id = 300
|
||||
|
||||
result = await adapter._fetch_channel_context(channel, before=trigger)
|
||||
|
||||
assert result == "[Recent channel messages]\n[Alice] hello"
|
||||
# Cache should have been ignored — after= should be None
|
||||
assert recorded_after["value"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_shared_channel_backfill_prepends_context(adapter, monkeypatch):
|
||||
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
|
||||
monkeypatch.setenv("DISCORD_AUTO_THREAD", "false")
|
||||
adapter.config.extra["group_sessions_per_user"] = False
|
||||
adapter.config.extra["history_backfill"] = True
|
||||
adapter._fetch_channel_context = AsyncMock(return_value="[Recent channel messages]\n[Alice] context")
|
||||
|
||||
bot_user = adapter._client.user
|
||||
message = make_message(
|
||||
channel=FakeTextChannel(channel_id=321),
|
||||
content=f"<@{bot_user.id}> hello with mention",
|
||||
mentions=[bot_user],
|
||||
)
|
||||
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter._fetch_channel_context.assert_awaited_once()
|
||||
event = adapter.handle_message.await_args.args[0]
|
||||
assert event.text == "hello with mention"
|
||||
assert event.channel_context == "[Recent channel messages]\n[Alice] context"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_per_user_channel_backfills_too(adapter, monkeypatch):
|
||||
"""Per-user sessions also benefit from backfill: Alice's session is missing
|
||||
other-channel-participants' context and her own pre-mention messages."""
|
||||
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
|
||||
monkeypatch.delenv("DISCORD_FREE_RESPONSE_CHANNELS", raising=False)
|
||||
monkeypatch.setenv("DISCORD_AUTO_THREAD", "false")
|
||||
adapter.config.extra["group_sessions_per_user"] = True
|
||||
adapter.config.extra["history_backfill"] = True
|
||||
adapter._fetch_channel_context = AsyncMock(return_value="[Recent channel messages]\n[Alice] context")
|
||||
|
||||
bot_user = adapter._client.user
|
||||
message = make_message(
|
||||
channel=FakeTextChannel(channel_id=321),
|
||||
content=f"<@{bot_user.id}> hello with mention",
|
||||
mentions=[bot_user],
|
||||
)
|
||||
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter._fetch_channel_context.assert_awaited_once()
|
||||
event = adapter.handle_message.await_args.args[0]
|
||||
assert event.text == "hello with mention"
|
||||
assert event.channel_context == "[Recent channel messages]\n[Alice] context"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discord_dm_does_not_backfill(adapter, monkeypatch):
|
||||
"""DMs skip backfill — every DM triggers the bot, so there's no mention gap."""
|
||||
monkeypatch.setenv("DISCORD_REQUIRE_MENTION", "true")
|
||||
adapter.config.extra["history_backfill"] = True
|
||||
adapter._fetch_channel_context = AsyncMock(return_value="[Recent channel messages]\n[Alice] context")
|
||||
|
||||
bot_user = adapter._client.user
|
||||
dm_channel = SimpleNamespace(
|
||||
id=999,
|
||||
name=None,
|
||||
guild=None,
|
||||
topic=None,
|
||||
)
|
||||
# Make isinstance(channel, discord.DMChannel) return True
|
||||
monkeypatch.setattr(
|
||||
discord_platform.discord, "DMChannel", type(dm_channel), raising=False,
|
||||
)
|
||||
|
||||
message = make_message(
|
||||
channel=dm_channel,
|
||||
content="hello in DM",
|
||||
mentions=[],
|
||||
)
|
||||
|
||||
await adapter._handle_message(message)
|
||||
|
||||
adapter._fetch_channel_context.assert_not_awaited()
|
||||
if adapter.handle_message.await_args is not None:
|
||||
event = adapter.handle_message.await_args.args[0]
|
||||
assert event.channel_context is None
|
||||
|
||||
|
||||
|
||||
@@ -467,3 +467,59 @@ class TestCancellationHandlerDeliveryConfirmation:
|
||||
final_response_sent = True
|
||||
|
||||
assert final_response_sent is True # the bug: partial promoted to final
|
||||
|
||||
|
||||
class TestFinalContentDeliveredSuppression:
|
||||
"""When stream consumer delivered the final content but the cosmetic
|
||||
final edit (cursor removal) failed, the gateway must suppress the
|
||||
fallback send to prevent duplicate messages.
|
||||
|
||||
Covers the scenario not handled by final_response_sent alone:
|
||||
content reached the user via _send_or_edit, but the subsequent edit
|
||||
that clears a typing cursor or streaming marker failed, leaving
|
||||
final_response_sent=False even though the user already saw the text.
|
||||
"""
|
||||
|
||||
def test_content_delivered_but_final_edit_failed_suppresses(self):
|
||||
"""final_content_delivered=True + final_response_sent=False
|
||||
must suppress (content already visible to user)."""
|
||||
sc = SimpleNamespace(
|
||||
already_sent=True,
|
||||
final_response_sent=False,
|
||||
final_content_delivered=True,
|
||||
)
|
||||
response = {"final_response": "Hello!", "response_previewed": False}
|
||||
|
||||
_streamed = bool(getattr(sc, "final_response_sent", False))
|
||||
_previewed = bool(response.get("response_previewed"))
|
||||
_content_delivered = bool(getattr(sc, "final_content_delivered", False))
|
||||
_is_empty_sentinel = (
|
||||
not response.get("final_response")
|
||||
or response.get("final_response") == "(empty)"
|
||||
)
|
||||
if not _is_empty_sentinel and (_streamed or _previewed or _content_delivered):
|
||||
response["already_sent"] = True
|
||||
|
||||
assert response.get("already_sent") is True
|
||||
|
||||
def test_intermediate_text_only_does_not_suppress(self):
|
||||
"""already_sent=True from intermediate text + final_content_delivered=False
|
||||
must NOT suppress (user still needs the real final answer)."""
|
||||
sc = SimpleNamespace(
|
||||
already_sent=True,
|
||||
final_response_sent=False,
|
||||
final_content_delivered=False,
|
||||
)
|
||||
response = {"final_response": "Real answer", "response_previewed": False}
|
||||
|
||||
_streamed = bool(getattr(sc, "final_response_sent", False))
|
||||
_previewed = bool(response.get("response_previewed"))
|
||||
_content_delivered = bool(getattr(sc, "final_content_delivered", False))
|
||||
_is_empty_sentinel = (
|
||||
not response.get("final_response")
|
||||
or response.get("final_response") == "(empty)"
|
||||
)
|
||||
if not _is_empty_sentinel and (_streamed or _previewed or _content_delivered):
|
||||
response["already_sent"] = True
|
||||
|
||||
assert "already_sent" not in response
|
||||
|
||||
@@ -455,7 +455,36 @@ def test_admit_per_group_require_mention_overrides_global():
|
||||
def test_hydrate_bot_identity_populates_self_ids_from_bot_v3_info(monkeypatch):
|
||||
import asyncio
|
||||
|
||||
from gateway.platforms.feishu import FeishuAdapter
|
||||
from gateway.platforms import feishu as feishu_mod
|
||||
FeishuAdapter = feishu_mod.FeishuAdapter
|
||||
|
||||
class _FakeBaseRequestBuilder:
|
||||
def __init__(self):
|
||||
self._request = SimpleNamespace()
|
||||
|
||||
def http_method(self, value):
|
||||
self._request.http_method = value
|
||||
return self
|
||||
|
||||
def uri(self, value):
|
||||
self._request.uri = value
|
||||
return self
|
||||
|
||||
def token_types(self, value):
|
||||
self._request.token_types = value
|
||||
return self
|
||||
|
||||
def build(self):
|
||||
return self._request
|
||||
|
||||
monkeypatch.setattr(
|
||||
feishu_mod,
|
||||
"BaseRequest",
|
||||
SimpleNamespace(builder=lambda: _FakeBaseRequestBuilder()),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(feishu_mod, "HttpMethod", SimpleNamespace(GET="GET"), raising=False)
|
||||
monkeypatch.setattr(feishu_mod, "AccessTokenType", SimpleNamespace(TENANT="TENANT"), raising=False)
|
||||
|
||||
adapter = object.__new__(FeishuAdapter)
|
||||
adapter._bot_open_id = ""
|
||||
|
||||
@@ -716,8 +716,10 @@ class TestMatrixModuleImport:
|
||||
"sys.meta_path.insert(0, _Blocker())\n"
|
||||
"for k in list(sys.modules):\n"
|
||||
" if k.startswith('mautrix'): del sys.modules[k]\n"
|
||||
"from unittest.mock import patch\n"
|
||||
"from gateway.platforms.matrix import check_matrix_requirements\n"
|
||||
"assert not check_matrix_requirements()\n"
|
||||
"with patch('tools.lazy_deps.ensure', side_effect=ImportError('blocked')):\n"
|
||||
" assert not check_matrix_requirements()\n"
|
||||
"print('OK')\n"
|
||||
)],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
@@ -737,7 +739,8 @@ class TestMatrixRequirements:
|
||||
import mautrix # noqa: F401
|
||||
assert check_matrix_requirements() is True
|
||||
except ImportError:
|
||||
assert check_matrix_requirements() is False
|
||||
with patch("tools.lazy_deps.ensure", side_effect=ImportError("mautrix unavailable")):
|
||||
assert check_matrix_requirements() is False
|
||||
|
||||
def test_check_requirements_without_creds(self, monkeypatch):
|
||||
monkeypatch.delenv("MATRIX_ACCESS_TOKEN", raising=False)
|
||||
@@ -759,7 +762,8 @@ class TestMatrixRequirements:
|
||||
monkeypatch.setenv("MATRIX_ENCRYPTION", "true")
|
||||
|
||||
from gateway.platforms import matrix as matrix_mod
|
||||
with patch.object(matrix_mod, "_check_e2ee_deps", return_value=False):
|
||||
with patch.object(matrix_mod, "_check_e2ee_deps", return_value=False), \
|
||||
patch("tools.lazy_deps.ensure", side_effect=ImportError("mautrix unavailable")):
|
||||
assert matrix_mod.check_matrix_requirements() is False
|
||||
|
||||
def test_check_requirements_encryption_false_no_e2ee_deps_ok(self, monkeypatch):
|
||||
@@ -775,7 +779,8 @@ class TestMatrixRequirements:
|
||||
import mautrix # noqa: F401
|
||||
assert matrix_mod.check_matrix_requirements() is True
|
||||
except ImportError:
|
||||
assert matrix_mod.check_matrix_requirements() is False
|
||||
with patch("tools.lazy_deps.ensure", side_effect=ImportError("mautrix unavailable")):
|
||||
assert matrix_mod.check_matrix_requirements() is False
|
||||
|
||||
def test_check_requirements_encryption_true_with_e2ee_deps(self, monkeypatch):
|
||||
"""MATRIX_ENCRYPTION=true should pass if E2EE deps are available."""
|
||||
@@ -789,7 +794,8 @@ class TestMatrixRequirements:
|
||||
import mautrix # noqa: F401
|
||||
assert matrix_mod.check_matrix_requirements() is True
|
||||
except ImportError:
|
||||
assert matrix_mod.check_matrix_requirements() is False
|
||||
with patch("tools.lazy_deps.ensure", side_effect=ImportError("mautrix unavailable")):
|
||||
assert matrix_mod.check_matrix_requirements() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -394,3 +394,317 @@ class TestPlatformsMerge:
|
||||
assert "LabelTest" in label
|
||||
finally:
|
||||
_reg.unregister("labeltest")
|
||||
|
||||
|
||||
# ── apply_yaml_config_fn (PlatformEntry field + load_gateway_config dispatch) ──
|
||||
|
||||
|
||||
class TestApplyYamlConfigFnField:
|
||||
"""The hook field itself — defaults, custom values, signature."""
|
||||
|
||||
def test_default_is_none(self):
|
||||
entry = PlatformEntry(
|
||||
name="test",
|
||||
label="Test",
|
||||
adapter_factory=lambda cfg: None,
|
||||
check_fn=lambda: True,
|
||||
)
|
||||
assert entry.apply_yaml_config_fn is None
|
||||
|
||||
def test_accepts_callable(self):
|
||||
def _hook(yaml_cfg, platform_cfg):
|
||||
return None
|
||||
|
||||
entry = PlatformEntry(
|
||||
name="test",
|
||||
label="Test",
|
||||
adapter_factory=lambda cfg: None,
|
||||
check_fn=lambda: True,
|
||||
apply_yaml_config_fn=_hook,
|
||||
)
|
||||
assert entry.apply_yaml_config_fn is _hook
|
||||
# Sanity-check the signature contract.
|
||||
assert entry.apply_yaml_config_fn({"x": 1}, {"y": 2}) is None
|
||||
|
||||
|
||||
class TestApplyYamlConfigFnDispatch:
|
||||
"""End-to-end dispatch through load_gateway_config().
|
||||
|
||||
Each test registers a temporary PlatformEntry, writes a config.yaml in
|
||||
a tmp HERMES_HOME, calls load_gateway_config(), and asserts the hook
|
||||
was invoked correctly. Cleanup unregisters the entry.
|
||||
"""
|
||||
|
||||
def _write_config(self, tmp_path, content: str):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text(content, encoding="utf-8")
|
||||
return hermes_home
|
||||
|
||||
def _register_hook(self, name, hook_fn):
|
||||
from gateway.platform_registry import platform_registry as _reg
|
||||
|
||||
entry = PlatformEntry(
|
||||
name=name,
|
||||
label=name.title(),
|
||||
adapter_factory=lambda cfg: None,
|
||||
check_fn=lambda: True,
|
||||
source="plugin",
|
||||
apply_yaml_config_fn=hook_fn,
|
||||
)
|
||||
_reg.register(entry)
|
||||
return _reg
|
||||
|
||||
def test_hook_can_mutate_environ(self, tmp_path, monkeypatch):
|
||||
"""A hook that mutates os.environ has its env vars set after load."""
|
||||
env_var = "MYHOOKPLAT_FLAG"
|
||||
monkeypatch.delenv(env_var, raising=False)
|
||||
|
||||
def _hook(yaml_cfg, platform_cfg):
|
||||
if "flag" in platform_cfg and not os.getenv(env_var):
|
||||
os.environ[env_var] = str(platform_cfg["flag"]).lower()
|
||||
return None
|
||||
|
||||
reg = self._register_hook("myhookplat", _hook)
|
||||
try:
|
||||
home = self._write_config(
|
||||
tmp_path, "myhookplat:\n flag: true\n",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
from gateway.config import load_gateway_config
|
||||
load_gateway_config()
|
||||
|
||||
assert os.environ.get(env_var) == "true"
|
||||
finally:
|
||||
reg.unregister("myhookplat")
|
||||
os.environ.pop(env_var, None)
|
||||
|
||||
def test_hook_returned_dict_merges_into_extra(self, tmp_path, monkeypatch):
|
||||
"""A hook that returns a dict has it merged into PlatformConfig.extra."""
|
||||
|
||||
def _hook(yaml_cfg, platform_cfg):
|
||||
return {"seeded_key": "seeded_value", "flag": platform_cfg.get("flag")}
|
||||
|
||||
reg = self._register_hook("myextraplat", _hook)
|
||||
try:
|
||||
home = self._write_config(
|
||||
tmp_path, "myextraplat:\n flag: yes\n",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
from gateway.config import load_gateway_config
|
||||
cfg = load_gateway_config()
|
||||
|
||||
plat = Platform("myextraplat")
|
||||
assert plat in cfg.platforms
|
||||
extra = cfg.platforms[plat].extra
|
||||
assert extra.get("seeded_key") == "seeded_value"
|
||||
# flag value carried through from yaml_cfg arg.
|
||||
assert extra.get("flag") is True
|
||||
finally:
|
||||
reg.unregister("myextraplat")
|
||||
|
||||
def test_hook_receives_full_yaml_and_platform_subdict(
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
"""Hook receives both the full yaml_cfg and its own platform sub-dict."""
|
||||
captured: dict = {}
|
||||
|
||||
def _hook(yaml_cfg, platform_cfg):
|
||||
captured["yaml_cfg"] = yaml_cfg
|
||||
captured["platform_cfg"] = platform_cfg
|
||||
return None
|
||||
|
||||
reg = self._register_hook("mycaptureplat", _hook)
|
||||
try:
|
||||
home = self._write_config(
|
||||
tmp_path,
|
||||
"top_level_key: 1\n"
|
||||
"mycaptureplat:\n"
|
||||
" inner_key: deep\n",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
from gateway.config import load_gateway_config
|
||||
load_gateway_config()
|
||||
|
||||
assert captured["yaml_cfg"].get("top_level_key") == 1
|
||||
assert captured["platform_cfg"] == {"inner_key": "deep"}
|
||||
finally:
|
||||
reg.unregister("mycaptureplat")
|
||||
|
||||
def test_hook_exception_swallowed(self, tmp_path, monkeypatch):
|
||||
"""A misbehaving hook never aborts load_gateway_config()."""
|
||||
|
||||
def _bad_hook(yaml_cfg, platform_cfg):
|
||||
raise RuntimeError("plugin author bug")
|
||||
|
||||
# Also register a well-behaved hook to ensure dispatch continues
|
||||
# iterating after a bad one.
|
||||
good_called = {"count": 0}
|
||||
|
||||
def _good_hook(yaml_cfg, platform_cfg):
|
||||
good_called["count"] += 1
|
||||
return None
|
||||
|
||||
from gateway.platform_registry import platform_registry as _reg
|
||||
_reg.register(PlatformEntry(
|
||||
name="mybadplat",
|
||||
label="MyBad",
|
||||
adapter_factory=lambda cfg: None,
|
||||
check_fn=lambda: True,
|
||||
source="plugin",
|
||||
apply_yaml_config_fn=_bad_hook,
|
||||
))
|
||||
_reg.register(PlatformEntry(
|
||||
name="mygoodplat",
|
||||
label="MyGood",
|
||||
adapter_factory=lambda cfg: None,
|
||||
check_fn=lambda: True,
|
||||
source="plugin",
|
||||
apply_yaml_config_fn=_good_hook,
|
||||
))
|
||||
try:
|
||||
home = self._write_config(
|
||||
tmp_path,
|
||||
"mybadplat:\n k: v\n"
|
||||
"mygoodplat:\n k: v\n",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
# Must not raise.
|
||||
from gateway.config import load_gateway_config
|
||||
load_gateway_config()
|
||||
|
||||
assert good_called["count"] == 1
|
||||
finally:
|
||||
_reg.unregister("mybadplat")
|
||||
_reg.unregister("mygoodplat")
|
||||
|
||||
def test_hook_skipped_when_platform_section_missing(
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
"""Hook is NOT called when the platform's YAML section is absent."""
|
||||
called = {"count": 0}
|
||||
|
||||
def _hook(yaml_cfg, platform_cfg):
|
||||
called["count"] += 1
|
||||
return None
|
||||
|
||||
reg = self._register_hook("myabsentplat", _hook)
|
||||
try:
|
||||
home = self._write_config(tmp_path, "telegram:\n k: v\n")
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
from gateway.config import load_gateway_config
|
||||
load_gateway_config()
|
||||
|
||||
assert called["count"] == 0
|
||||
finally:
|
||||
reg.unregister("myabsentplat")
|
||||
|
||||
def test_hook_skipped_when_platform_section_not_dict(
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
"""Hook is NOT called when the platform's YAML section isn't a dict."""
|
||||
called = {"count": 0}
|
||||
|
||||
def _hook(yaml_cfg, platform_cfg):
|
||||
called["count"] += 1
|
||||
return None
|
||||
|
||||
reg = self._register_hook("mybadshapeplat", _hook)
|
||||
try:
|
||||
home = self._write_config(
|
||||
tmp_path, "mybadshapeplat: just-a-string\n",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
from gateway.config import load_gateway_config
|
||||
load_gateway_config()
|
||||
|
||||
assert called["count"] == 0
|
||||
finally:
|
||||
reg.unregister("mybadshapeplat")
|
||||
|
||||
def test_env_var_takes_precedence_when_hook_uses_getenv_guard(
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
"""The standard `not os.getenv(...)` guard preserves env > YAML."""
|
||||
env_var = "MYPRECPLAT_FLAG"
|
||||
monkeypatch.setenv(env_var, "preexisting")
|
||||
|
||||
def _hook(yaml_cfg, platform_cfg):
|
||||
if "flag" in platform_cfg and not os.getenv(env_var):
|
||||
os.environ[env_var] = str(platform_cfg["flag"]).lower()
|
||||
return None
|
||||
|
||||
reg = self._register_hook("myprecplat", _hook)
|
||||
try:
|
||||
home = self._write_config(
|
||||
tmp_path, "myprecplat:\n flag: yaml-value\n",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
from gateway.config import load_gateway_config
|
||||
load_gateway_config()
|
||||
|
||||
# Pre-existing env var was NOT clobbered by the hook.
|
||||
assert os.environ.get(env_var) == "preexisting"
|
||||
finally:
|
||||
reg.unregister("myprecplat")
|
||||
os.environ.pop(env_var, None)
|
||||
|
||||
|
||||
class TestPluginPlatformSharedKeyBridge:
|
||||
"""Plugin-registered platforms get the same shared-key bridging as built-ins.
|
||||
|
||||
Without this, plugin authors using ``apply_yaml_config_fn`` would have to
|
||||
re-implement bridging for every common key (``unauthorized_dm_behavior``,
|
||||
``notice_delivery``, ``reply_prefix``, ``require_mention``, ``dm_policy``,
|
||||
``allow_from``, etc.) — defeating the hook's whole point of letting
|
||||
plugins focus on their *platform-specific* keys.
|
||||
"""
|
||||
|
||||
def _write_config(self, tmp_path, content: str):
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text(content, encoding="utf-8")
|
||||
return hermes_home
|
||||
|
||||
def test_shared_keys_bridged_for_plugin_platform(self, tmp_path, monkeypatch):
|
||||
"""A plugin platform's ``require_mention``/``dm_policy``/etc. flow into
|
||||
``PlatformConfig.extra`` without the plugin needing its own bridge."""
|
||||
from gateway.platform_registry import platform_registry as _reg
|
||||
|
||||
_reg.register(PlatformEntry(
|
||||
name="mysharedplat",
|
||||
label="MySharedPlat",
|
||||
adapter_factory=lambda cfg: None,
|
||||
check_fn=lambda: True,
|
||||
source="plugin",
|
||||
))
|
||||
try:
|
||||
home = self._write_config(
|
||||
tmp_path,
|
||||
"mysharedplat:\n"
|
||||
" require_mention: true\n"
|
||||
" dm_policy: allow\n"
|
||||
" reply_prefix: \"→ \"\n"
|
||||
" allow_from: [\"alice\", \"bob\"]\n",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
|
||||
from gateway.config import load_gateway_config, Platform
|
||||
cfg = load_gateway_config()
|
||||
|
||||
plat = Platform("mysharedplat")
|
||||
assert plat in cfg.platforms
|
||||
extra = cfg.platforms[plat].extra
|
||||
assert extra.get("require_mention") is True
|
||||
assert extra.get("dm_policy") == "allow"
|
||||
assert extra.get("reply_prefix") == "→ "
|
||||
assert extra.get("allow_from") == ["alice", "bob"]
|
||||
finally:
|
||||
_reg.unregister("mysharedplat")
|
||||
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
@@ -578,6 +579,7 @@ class TestWaitForReconnection:
|
||||
async def reconnect_after_delay():
|
||||
await asyncio.sleep(0.3)
|
||||
adapter._running = True
|
||||
adapter._ws = SimpleNamespace(closed=False)
|
||||
|
||||
asyncio.get_event_loop().create_task(reconnect_after_delay())
|
||||
|
||||
@@ -603,6 +605,7 @@ class TestWaitForReconnection:
|
||||
"""send() should not wait when already connected."""
|
||||
adapter = self._make_adapter(app_id="a", client_secret="b")
|
||||
adapter._running = True
|
||||
adapter._ws = SimpleNamespace(closed=False)
|
||||
adapter._http_client = mock.MagicMock()
|
||||
|
||||
async def fake_api_request(*args, **kwargs):
|
||||
|
||||
@@ -5,6 +5,7 @@ import pytest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
from gateway.config import Platform, HomeChannel, GatewayConfig, PlatformConfig
|
||||
from gateway.platforms.base import MessageEvent
|
||||
from gateway.session import (
|
||||
SessionSource,
|
||||
SessionStore,
|
||||
@@ -430,6 +431,76 @@ class TestBuildSessionContextPrompt:
|
||||
assert "Multi-user thread" not in prompt
|
||||
|
||||
|
||||
class TestSenderPrefixWithBackfill:
|
||||
"""Regression: sender prefix must not wrap the backfill context block.
|
||||
|
||||
Tests exercise the real GatewayRunner._prepare_inbound_message_text()
|
||||
method to ensure the [sender_name] prefix applies only to the trigger
|
||||
message, not the channel_context backfill block.
|
||||
"""
|
||||
|
||||
@pytest.fixture()
|
||||
def runner(self):
|
||||
from gateway.run import GatewayRunner
|
||||
|
||||
r = GatewayRunner.__new__(GatewayRunner)
|
||||
r.config = GatewayConfig(group_sessions_per_user=False)
|
||||
r.adapters = {}
|
||||
r._model = "test-model"
|
||||
r._base_url = ""
|
||||
r._has_setup_skill = lambda: False
|
||||
return r
|
||||
|
||||
@pytest.fixture()
|
||||
def source(self):
|
||||
return SessionSource(
|
||||
platform=Platform.DISCORD,
|
||||
chat_id="c1",
|
||||
chat_type="group",
|
||||
user_name="Alice",
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_message_gets_prefix(self, runner, source):
|
||||
"""Normal message without backfill gets [sender] prefix."""
|
||||
event = MessageEvent(text="hello world", source=source)
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event, source=source, history=[],
|
||||
)
|
||||
assert result == "[Alice] hello world"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_prefix_only_on_trigger(self, runner, source):
|
||||
"""Backfill context must NOT get the sender prefix."""
|
||||
event = MessageEvent(
|
||||
text="hello world",
|
||||
source=source,
|
||||
channel_context="[Recent channel messages]\n[Bob] some context",
|
||||
)
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event, source=source, history=[],
|
||||
)
|
||||
assert result.startswith("[Recent channel messages]")
|
||||
assert "[Alice] [Recent channel messages]" not in result
|
||||
assert "[New message]\n[Alice] hello world" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_preserves_context_block(self, runner, source):
|
||||
"""The backfill block should pass through unchanged — no double-prefixing."""
|
||||
context = "[Recent channel messages]\n[Bob] first\n[Charlie [bot]] second"
|
||||
event = MessageEvent(
|
||||
text="hey everyone", source=source, channel_context=context,
|
||||
)
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event, source=source, history=[],
|
||||
)
|
||||
assert result.startswith(context)
|
||||
assert "[Alice] hey everyone" in result
|
||||
assert "[Alice] [Bob]" not in result
|
||||
assert "[Alice] [Charlie" not in result
|
||||
assert "[Alice] [Recent" not in result
|
||||
|
||||
|
||||
class TestSessionStoreRewriteTranscript:
|
||||
"""Regression: /retry and /undo must persist truncated history to disk."""
|
||||
|
||||
|
||||
@@ -205,3 +205,78 @@ class TestResetPolicyNotify:
|
||||
assert restored.notify == original.notify
|
||||
assert restored.notify_exclude_platforms == original.notify_exclude_platforms
|
||||
assert restored.mode == original.mode
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SessionEntry to_dict / from_dict roundtrip for auto-reset fields
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSessionEntryAutoResetRoundtrip:
|
||||
def test_was_auto_reset_persists_across_roundtrip(self, tmp_path):
|
||||
"""was_auto_reset=True survives to_dict() → from_dict() (gateway restart)."""
|
||||
store = _make_store(
|
||||
SessionResetPolicy(mode="idle", idle_minutes=1),
|
||||
tmp_path,
|
||||
)
|
||||
source = _make_source()
|
||||
|
||||
entry = store.get_or_create_session(source)
|
||||
entry.updated_at = datetime.now() - timedelta(minutes=5)
|
||||
store._save()
|
||||
|
||||
entry2 = store.get_or_create_session(source)
|
||||
assert entry2.was_auto_reset is True
|
||||
assert entry2.auto_reset_reason == "idle"
|
||||
assert entry2.session_id != entry.session_id
|
||||
|
||||
# Simulate gateway restart: reload from disk
|
||||
store._loaded = False
|
||||
store._entries.clear()
|
||||
store._ensure_loaded()
|
||||
|
||||
reloaded = store._entries.get(entry2.session_key)
|
||||
assert reloaded is not None
|
||||
assert reloaded.was_auto_reset is True
|
||||
assert reloaded.auto_reset_reason == "idle"
|
||||
|
||||
def test_reset_had_activity_persists_across_roundtrip(self, tmp_path):
|
||||
"""reset_had_activity survives to_dict() → from_dict() (gateway restart)."""
|
||||
store = _make_store(
|
||||
SessionResetPolicy(mode="idle", idle_minutes=1),
|
||||
tmp_path,
|
||||
)
|
||||
source = _make_source()
|
||||
|
||||
entry = store.get_or_create_session(source)
|
||||
entry.total_tokens = 1000
|
||||
entry.updated_at = datetime.now() - timedelta(minutes=5)
|
||||
store._save()
|
||||
|
||||
entry2 = store.get_or_create_session(source)
|
||||
assert entry2.reset_had_activity is True
|
||||
|
||||
store._loaded = False
|
||||
store._entries.clear()
|
||||
store._ensure_loaded()
|
||||
|
||||
reloaded = store._entries.get(entry2.session_key)
|
||||
assert reloaded is not None
|
||||
assert reloaded.reset_had_activity is True
|
||||
|
||||
def test_auto_reset_reason_none_roundtrip(self, tmp_path):
|
||||
"""auto_reset_reason=None (no reset) survives roundtrip cleanly."""
|
||||
store = _make_store(tmp_path=tmp_path)
|
||||
source = _make_source()
|
||||
|
||||
entry = store.get_or_create_session(source)
|
||||
assert entry.was_auto_reset is False
|
||||
|
||||
store._loaded = False
|
||||
store._entries.clear()
|
||||
store._ensure_loaded()
|
||||
|
||||
reloaded = store._entries.get(entry.session_key)
|
||||
assert reloaded is not None
|
||||
assert reloaded.was_auto_reset is False
|
||||
assert reloaded.auto_reset_reason is None
|
||||
assert reloaded.reset_had_activity is False
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
"""Tests for the SimpleX Chat platform-plugin adapter.
|
||||
|
||||
Loaded via the ``_plugin_adapter_loader`` helper so this lives under
|
||||
``plugin_adapter_simplex`` in ``sys.modules`` and cannot collide with
|
||||
sibling platform-plugin tests on the same xdist worker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.gateway._plugin_adapter_loader import load_plugin_adapter
|
||||
|
||||
_simplex = load_plugin_adapter("simplex")
|
||||
|
||||
SimplexAdapter = _simplex.SimplexAdapter
|
||||
check_requirements = _simplex.check_requirements
|
||||
validate_config = _simplex.validate_config
|
||||
is_connected = _simplex.is_connected
|
||||
register = _simplex.register
|
||||
_env_enablement = _simplex._env_enablement
|
||||
_standalone_send = _simplex._standalone_send
|
||||
_guess_extension = _simplex._guess_extension
|
||||
_is_image_ext = _simplex._is_image_ext
|
||||
_is_audio_ext = _simplex._is_audio_ext
|
||||
_CORR_PREFIX = _simplex._CORR_PREFIX
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Platform enum (plugin-discovered, not bundled)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_platform_enum_resolves_via_plugin_scan():
|
||||
"""The plugin filesystem scan should expose Platform("simplex")."""
|
||||
from gateway.config import Platform
|
||||
p = Platform("simplex")
|
||||
assert p.value == "simplex"
|
||||
# Identity stability — repeated lookups return the same pseudo-member
|
||||
assert Platform("simplex") is p
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. check_requirements / validate_config / is_connected
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_check_requirements_needs_url(monkeypatch):
|
||||
monkeypatch.delenv("SIMPLEX_WS_URL", raising=False)
|
||||
assert check_requirements() is False
|
||||
|
||||
|
||||
def test_check_requirements_true_when_configured(monkeypatch):
|
||||
monkeypatch.setenv("SIMPLEX_WS_URL", "ws://127.0.0.1:5225")
|
||||
# websockets is a dev dep in this repo via the test plugins; the
|
||||
# check_requirements() gate also asserts the package imports.
|
||||
websockets_present = True
|
||||
try:
|
||||
import websockets # noqa: F401
|
||||
except ImportError:
|
||||
websockets_present = False
|
||||
assert check_requirements() is websockets_present
|
||||
|
||||
|
||||
def test_validate_config_uses_env_or_extra():
|
||||
from gateway.config import PlatformConfig
|
||||
# Empty extra + no env → invalid
|
||||
cfg = PlatformConfig(enabled=True)
|
||||
assert validate_config(cfg) is False
|
||||
# extra-only path → valid
|
||||
cfg2 = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
|
||||
assert validate_config(cfg2) is True
|
||||
|
||||
|
||||
def test_is_connected_mirrors_validate(monkeypatch):
|
||||
from gateway.config import PlatformConfig
|
||||
monkeypatch.delenv("SIMPLEX_WS_URL", raising=False)
|
||||
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://x"})
|
||||
assert is_connected(cfg) is True
|
||||
assert is_connected(PlatformConfig(enabled=True)) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. _env_enablement seeds PlatformConfig.extra
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_env_enablement_none_when_unset(monkeypatch):
|
||||
monkeypatch.delenv("SIMPLEX_WS_URL", raising=False)
|
||||
assert _env_enablement() is None
|
||||
|
||||
|
||||
def test_env_enablement_seeds_ws_url(monkeypatch):
|
||||
monkeypatch.setenv("SIMPLEX_WS_URL", "ws://127.0.0.1:5225")
|
||||
monkeypatch.delenv("SIMPLEX_HOME_CHANNEL", raising=False)
|
||||
seed = _env_enablement()
|
||||
assert seed == {"ws_url": "ws://127.0.0.1:5225"}
|
||||
|
||||
|
||||
def test_env_enablement_seeds_home_channel(monkeypatch):
|
||||
monkeypatch.setenv("SIMPLEX_WS_URL", "ws://127.0.0.1:5225")
|
||||
monkeypatch.setenv("SIMPLEX_HOME_CHANNEL", "42")
|
||||
monkeypatch.setenv("SIMPLEX_HOME_CHANNEL_NAME", "Personal")
|
||||
seed = _env_enablement()
|
||||
assert seed["home_channel"] == {"chat_id": "42", "name": "Personal"}
|
||||
|
||||
|
||||
def test_env_enablement_home_channel_defaults_name_to_id(monkeypatch):
|
||||
monkeypatch.setenv("SIMPLEX_WS_URL", "ws://127.0.0.1:5225")
|
||||
monkeypatch.setenv("SIMPLEX_HOME_CHANNEL", "42")
|
||||
monkeypatch.delenv("SIMPLEX_HOME_CHANNEL_NAME", raising=False)
|
||||
seed = _env_enablement()
|
||||
assert seed["home_channel"] == {"chat_id": "42", "name": "42"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Adapter init
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_adapter_init_custom_url():
|
||||
from gateway.config import PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
|
||||
adapter = SimplexAdapter(cfg)
|
||||
assert adapter.ws_url == "ws://localhost:5225"
|
||||
assert adapter._running is False
|
||||
assert adapter._ws is None
|
||||
|
||||
|
||||
def test_adapter_init_default_url():
|
||||
from gateway.config import PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True)
|
||||
adapter = SimplexAdapter(cfg)
|
||||
assert adapter.ws_url == "ws://127.0.0.1:5225"
|
||||
|
||||
|
||||
def test_adapter_platform_identity():
|
||||
"""Adapter should expose Platform("simplex") identity."""
|
||||
from gateway.config import Platform, PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True)
|
||||
adapter = SimplexAdapter(cfg)
|
||||
assert adapter.platform is Platform("simplex")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. Helper functions (magic-byte detection)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_guess_extension_png():
|
||||
assert _guess_extension(b"\x89PNG\r\n\x1a\n") == ".png"
|
||||
|
||||
|
||||
def test_guess_extension_jpg():
|
||||
assert _guess_extension(b"\xff\xd8\xff\xe0") == ".jpg"
|
||||
|
||||
|
||||
def test_guess_extension_ogg():
|
||||
assert _guess_extension(b"OggS\x00\x02") == ".ogg"
|
||||
|
||||
|
||||
def test_guess_extension_unknown():
|
||||
assert _guess_extension(b"\x00\x01\x02\x03") == ".bin"
|
||||
|
||||
|
||||
def test_is_image_ext():
|
||||
assert _is_image_ext(".png") is True
|
||||
assert _is_image_ext(".webp") is True
|
||||
assert _is_image_ext(".ogg") is False
|
||||
|
||||
|
||||
def test_is_audio_ext():
|
||||
assert _is_audio_ext(".ogg") is True
|
||||
assert _is_audio_ext(".mp3") is True
|
||||
assert _is_audio_ext(".pdf") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. Correlation IDs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_corr_id_starts_with_prefix_and_tracks_pending():
|
||||
from gateway.config import PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
|
||||
adapter = SimplexAdapter(cfg)
|
||||
corr_id = adapter._make_corr_id()
|
||||
assert corr_id.startswith(_CORR_PREFIX)
|
||||
assert corr_id in adapter._pending_corr_ids
|
||||
|
||||
|
||||
def test_corr_id_pending_set_self_trims():
|
||||
from gateway.config import PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
|
||||
adapter = SimplexAdapter(cfg)
|
||||
adapter._max_pending_corr = 4
|
||||
for _ in range(10):
|
||||
adapter._make_corr_id()
|
||||
# After many additions, the pending set should be bounded by the trim
|
||||
# logic — at most one trim window above the cap.
|
||||
assert len(adapter._pending_corr_ids) <= adapter._max_pending_corr + 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. Outbound send (mocked WS)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_dm():
|
||||
from gateway.config import PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
|
||||
adapter = SimplexAdapter(cfg)
|
||||
|
||||
mock_ws = AsyncMock()
|
||||
adapter._ws = mock_ws
|
||||
|
||||
result = await adapter.send("contact-42", "Hello, SimpleX!")
|
||||
mock_ws.send.assert_called_once()
|
||||
payload = json.loads(mock_ws.send.call_args[0][0])
|
||||
assert payload["cmd"] == "@[contact-42] Hello, SimpleX!"
|
||||
assert payload["corrId"].startswith(_CORR_PREFIX)
|
||||
assert result.success is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_group():
|
||||
from gateway.config import PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
|
||||
adapter = SimplexAdapter(cfg)
|
||||
|
||||
mock_ws = AsyncMock()
|
||||
adapter._ws = mock_ws
|
||||
|
||||
result = await adapter.send("group:grp-99", "Hello, group!")
|
||||
payload = json.loads(mock_ws.send.call_args[0][0])
|
||||
assert payload["cmd"] == "#[grp-99] Hello, group!"
|
||||
assert result.success is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_when_ws_not_connected_does_not_crash():
|
||||
from gateway.config import PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
|
||||
adapter = SimplexAdapter(cfg)
|
||||
# No _ws assigned — _send_ws should drop quietly
|
||||
result = await adapter.send("contact-42", "hi")
|
||||
assert result.success is True # send() always returns success — fire-and-forget
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 8. Inbound: filter own-echo by corrId prefix
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_event_filters_own_corr_id():
|
||||
from gateway.config import PlatformConfig
|
||||
cfg = PlatformConfig(enabled=True, extra={"ws_url": "ws://localhost:5225"})
|
||||
adapter = SimplexAdapter(cfg)
|
||||
# Pretend we sent a command with this corrId
|
||||
own = adapter._make_corr_id()
|
||||
handler_mock = AsyncMock()
|
||||
adapter._handle_new_chat_item = handler_mock # type: ignore
|
||||
|
||||
await adapter._handle_event({"corrId": own, "type": "newChatItem"})
|
||||
handler_mock.assert_not_called()
|
||||
assert own not in adapter._pending_corr_ids # discarded
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 9. Standalone (out-of-process) send for cron
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_standalone_send_missing_websockets(monkeypatch):
|
||||
"""When websockets is unimportable, return a clean error dict.
|
||||
|
||||
Implementation detail: the standalone path does ``import websockets``
|
||||
inside the function body. We simulate the package being absent by
|
||||
pulling it out of ``sys.modules`` and pointing the finder at None.
|
||||
"""
|
||||
import sys
|
||||
saved_websockets = sys.modules.pop("websockets", None)
|
||||
saved_meta = list(sys.meta_path)
|
||||
|
||||
class _Blocker:
|
||||
@staticmethod
|
||||
def find_spec(name, path=None, target=None):
|
||||
if name == "websockets" or name.startswith("websockets."):
|
||||
raise ImportError("websockets blocked for test")
|
||||
return None
|
||||
|
||||
sys.meta_path.insert(0, _Blocker())
|
||||
try:
|
||||
pconfig = MagicMock()
|
||||
pconfig.extra = {"ws_url": "ws://localhost:5225"}
|
||||
result = await _standalone_send(pconfig, "contact-42", "hi")
|
||||
assert isinstance(result, dict)
|
||||
assert "error" in result
|
||||
assert "websockets" in result["error"]
|
||||
finally:
|
||||
sys.meta_path[:] = saved_meta
|
||||
if saved_websockets is not None:
|
||||
sys.modules["websockets"] = saved_websockets
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_standalone_send_missing_url(monkeypatch):
|
||||
monkeypatch.delenv("SIMPLEX_WS_URL", raising=False)
|
||||
pconfig = MagicMock()
|
||||
pconfig.extra = {}
|
||||
# We expect the URL fallback (extra+env both empty) to be empty string,
|
||||
# producing an error. We also need websockets to be importable for the
|
||||
# url-check branch to be reached, so skip when it's not.
|
||||
try:
|
||||
import websockets.client # noqa: F401
|
||||
except ImportError:
|
||||
pytest.skip("websockets not installed")
|
||||
|
||||
result = await _standalone_send(pconfig, "contact-42", "hi")
|
||||
assert isinstance(result, dict)
|
||||
# Either error about URL or a connection attempt failure — both are valid
|
||||
# signals that the standalone path requires configuration.
|
||||
assert "error" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10. register() — plugin-side metadata
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_register_calls_register_platform():
|
||||
ctx = MagicMock()
|
||||
register(ctx)
|
||||
ctx.register_platform.assert_called_once()
|
||||
kwargs = ctx.register_platform.call_args.kwargs
|
||||
assert kwargs["name"] == "simplex"
|
||||
assert kwargs["label"] == "SimpleX Chat"
|
||||
assert kwargs["required_env"] == ["SIMPLEX_WS_URL"]
|
||||
assert kwargs["allowed_users_env"] == "SIMPLEX_ALLOWED_USERS"
|
||||
assert kwargs["allow_all_env"] == "SIMPLEX_ALLOW_ALL_USERS"
|
||||
assert kwargs["cron_deliver_env_var"] == "SIMPLEX_HOME_CHANNEL"
|
||||
assert callable(kwargs["check_fn"])
|
||||
assert callable(kwargs["validate_config"])
|
||||
assert callable(kwargs["is_connected"])
|
||||
assert callable(kwargs["env_enablement_fn"])
|
||||
assert callable(kwargs["standalone_sender_fn"])
|
||||
assert callable(kwargs["adapter_factory"])
|
||||
assert callable(kwargs["setup_fn"])
|
||||
# SimpleX uses opaque IDs only — no PII to redact.
|
||||
assert kwargs["pii_safe"] is True
|
||||
@@ -691,10 +691,98 @@ class TestSendVideo:
|
||||
adapter._app.client.chat_postMessage.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestBangPrefixCommands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBangPrefixCommands:
|
||||
"""``!cmd`` is rewritten to ``/cmd`` so commands work inside Slack threads.
|
||||
|
||||
Slack natively rejects slash commands invoked from a thread reply
|
||||
("/queue is not supported in threads. Sorry!"). Typing ``!queue`` as a
|
||||
plain text reply hits the message event pipeline instead, and the
|
||||
adapter rewrites the leading ``!`` to ``/`` for any known gateway
|
||||
command before downstream processing.
|
||||
"""
|
||||
|
||||
def _make_event(self, text, thread_ts=None, channel_type="im", channel="D123"):
|
||||
evt = {
|
||||
"text": text,
|
||||
"user": "U_USER",
|
||||
"channel": channel,
|
||||
"channel_type": channel_type,
|
||||
"ts": "1234567890.000001",
|
||||
}
|
||||
if thread_ts:
|
||||
evt["thread_ts"] = thread_ts
|
||||
return evt
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_known_command_is_rewritten_to_slash(self, adapter):
|
||||
"""``!queue`` → ``/queue`` and tagged as COMMAND."""
|
||||
await adapter._handle_slack_message(self._make_event("!queue"))
|
||||
|
||||
adapter.handle_message.assert_called_once()
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text.startswith("/queue")
|
||||
assert msg_event.message_type == MessageType.COMMAND
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_command_with_args_preserved(self, adapter):
|
||||
"""``!model gpt-5.4`` → ``/model gpt-5.4``."""
|
||||
await adapter._handle_slack_message(self._make_event("!model gpt-5.4"))
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text.startswith("/model gpt-5.4")
|
||||
assert msg_event.message_type == MessageType.COMMAND
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_works_inside_thread(self, adapter):
|
||||
"""The whole point: ``!stop`` inside a thread reply dispatches."""
|
||||
evt = self._make_event("!stop", thread_ts="1111111111.000001")
|
||||
await adapter._handle_slack_message(evt)
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text.startswith("/stop")
|
||||
assert msg_event.message_type == MessageType.COMMAND
|
||||
# thread_id is preserved on the source so the reply lands in the
|
||||
# same thread.
|
||||
assert msg_event.source.thread_id == "1111111111.000001"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_unknown_token_passes_through_unchanged(self, adapter):
|
||||
"""``!nice work`` is just a casual message — must NOT be rewritten."""
|
||||
await adapter._handle_slack_message(self._make_event("!nice work"))
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text == "!nice work"
|
||||
assert msg_event.message_type != MessageType.COMMAND
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bang_with_bot_suffix_resolves(self, adapter):
|
||||
"""``!stop@hermes`` matches the get_command() ``@suffix`` stripping."""
|
||||
await adapter._handle_slack_message(self._make_event("!stop@hermes"))
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text.startswith("/stop@hermes")
|
||||
assert msg_event.message_type == MessageType.COMMAND
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_slash_still_works(self, adapter):
|
||||
"""Sanity check — ``/queue`` (top-level channel/DM) still dispatches."""
|
||||
await adapter._handle_slack_message(self._make_event("/queue"))
|
||||
|
||||
msg_event = adapter.handle_message.call_args[0][0]
|
||||
assert msg_event.text.startswith("/queue")
|
||||
assert msg_event.message_type == MessageType.COMMAND
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestIncomingDocumentHandling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIncomingDocumentHandling:
|
||||
def _make_event(self, files=None, text="hello", channel_type="im", blocks=None, attachments=None):
|
||||
"""Build a mock Slack message event with file attachments."""
|
||||
|
||||
@@ -195,6 +195,29 @@ class TestTelegramExecApproval:
|
||||
or kwargs.get("link_preview_options") is not None
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_update_prompt_escapes_dynamic_prompt(self):
|
||||
adapter = _make_adapter()
|
||||
sent = {}
|
||||
|
||||
async def mock_send_message(**kwargs):
|
||||
sent.update(kwargs)
|
||||
return SimpleNamespace(message_id=55)
|
||||
|
||||
adapter._bot.send_message = AsyncMock(side_effect=mock_send_message)
|
||||
|
||||
result = await adapter.send_update_prompt(
|
||||
chat_id="12345",
|
||||
prompt="Fix [issue]_1 and verify *markdown*",
|
||||
default="alpha_beta",
|
||||
metadata={"thread_id": "999"},
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert "MARKDOWN_V2" in repr(sent["parse_mode"])
|
||||
assert "Fix \\[issue\\]\\_1" in sent["text"]
|
||||
assert "alpha\\_beta" in sent["text"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_truncates_long_command(self):
|
||||
adapter = _make_adapter()
|
||||
@@ -210,9 +233,6 @@ class TestTelegramExecApproval:
|
||||
kwargs = adapter._bot.send_message.call_args[1]
|
||||
assert "..." in kwargs["text"]
|
||||
assert len(kwargs["text"]) < 5000
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# _handle_callback_query — approval button clicks
|
||||
# ===========================================================================
|
||||
|
||||
@@ -251,6 +271,34 @@ class TestTelegramApprovalCallback:
|
||||
# State should be cleaned up
|
||||
assert 1 not in adapter._approval_state
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approval_callback_escapes_dynamic_user_name(self):
|
||||
adapter = _make_adapter()
|
||||
adapter._approval_state[3] = "agent:main:telegram:group:12345:99"
|
||||
|
||||
query = AsyncMock()
|
||||
query.data = "ea:once:3"
|
||||
query.message = MagicMock()
|
||||
query.message.chat_id = 12345
|
||||
query.from_user = MagicMock()
|
||||
query.from_user.first_name = "Alice_Bob"
|
||||
query.answer = AsyncMock()
|
||||
query.edit_message_text = AsyncMock()
|
||||
|
||||
update = MagicMock()
|
||||
update.callback_query = query
|
||||
context = MagicMock()
|
||||
query.from_user.id = "12345"
|
||||
|
||||
with patch.dict(os.environ, {"TELEGRAM_ALLOWED_USERS": "*"}, clear=False):
|
||||
with patch("tools.approval.resolve_gateway_approval", return_value=1):
|
||||
await adapter._handle_callback_query(update, context)
|
||||
|
||||
edit_kwargs = query.edit_message_text.call_args[1]
|
||||
assert "MARKDOWN_V2" in repr(edit_kwargs["parse_mode"])
|
||||
assert "Alice\\_Bob" in edit_kwargs["text"]
|
||||
assert "Approved once" in edit_kwargs["text"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deny_button(self):
|
||||
adapter = _make_adapter()
|
||||
|
||||
@@ -210,6 +210,19 @@ class TestFormatMessageBoldItalic:
|
||||
assert "*bold*" in result
|
||||
assert "_italic_" in result
|
||||
|
||||
def test_reload_mcp_summary_escapes_dynamic_server_names(self, adapter):
|
||||
content = (
|
||||
"🔄 **MCP Servers Reloaded**\n"
|
||||
"♻️ Reconnected: agent_one, tool[beta]\n"
|
||||
"➕ Added: alpha*prod\n"
|
||||
"🔧 3 tool(s) available from 2 server(s)"
|
||||
)
|
||||
result = adapter.format_message(content)
|
||||
assert "*MCP Servers Reloaded*" in result
|
||||
assert "agent\\_one" in result
|
||||
assert "tool\\[beta\\]" in result
|
||||
assert "alpha\\*prod" in result
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# format_message - headers
|
||||
|
||||
@@ -43,6 +43,109 @@ def _make_adapter():
|
||||
|
||||
|
||||
class TestTelegramModelPicker:
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_model_picker_escapes_dynamic_provider_label(self):
|
||||
adapter = _make_adapter()
|
||||
sent = {}
|
||||
|
||||
async def mock_send_message(**kwargs):
|
||||
sent.update(kwargs)
|
||||
return SimpleNamespace(message_id=101)
|
||||
|
||||
adapter._bot.send_message = AsyncMock(side_effect=mock_send_message)
|
||||
|
||||
result = await adapter.send_model_picker(
|
||||
chat_id="12345",
|
||||
providers=[
|
||||
{"slug": "provider_one", "name": "Provider One", "total_models": 1, "is_current": True}
|
||||
],
|
||||
current_model="model_1",
|
||||
current_provider="provider_one",
|
||||
session_key="s",
|
||||
on_model_selected=AsyncMock(),
|
||||
metadata={"thread_id": "99999"},
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert "MARKDOWN_V2" in repr(sent["parse_mode"])
|
||||
assert "provider\\_one" in sent["text"]
|
||||
assert "`model_1`" in sent["text"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_back_button_escapes_dynamic_provider_label(self):
|
||||
adapter = _make_adapter()
|
||||
adapter._model_picker_state["12345"] = {
|
||||
"providers": [{"slug": "provider_one", "name": "Provider One", "total_models": 1, "is_current": True}],
|
||||
"current_model": "model_1",
|
||||
"current_provider": "provider_one",
|
||||
"session_key": "s",
|
||||
"on_model_selected": AsyncMock(),
|
||||
"msg_id": 42,
|
||||
}
|
||||
|
||||
query = AsyncMock()
|
||||
query.data = "mb"
|
||||
query.message = MagicMock()
|
||||
query.message.chat_id = 12345
|
||||
query.from_user = MagicMock()
|
||||
query.answer = AsyncMock()
|
||||
query.edit_message_text = AsyncMock()
|
||||
|
||||
update = MagicMock()
|
||||
update.callback_query = query
|
||||
context = MagicMock()
|
||||
|
||||
await adapter._handle_model_picker_callback(query, "mb", "12345")
|
||||
|
||||
edit_kwargs = query.edit_message_text.call_args[1]
|
||||
assert "MARKDOWN_V2" in repr(edit_kwargs["parse_mode"])
|
||||
assert "provider\\_one" in edit_kwargs["text"]
|
||||
assert "`model_1`" in edit_kwargs["text"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_selected_edits_message_on_success(self):
|
||||
"""Regression: the mm: (model selected → switch) success path must
|
||||
edit the picker message to show the confirmation and remove the
|
||||
buttons. An earlier revision of this PR over-indented the
|
||||
edit_message_text block so it lived inside the except branch and
|
||||
only fired when the callback raised."""
|
||||
adapter = _make_adapter()
|
||||
callback = AsyncMock(return_value="Switched to `gpt-5`")
|
||||
adapter._model_picker_state["12345"] = {
|
||||
"providers": [
|
||||
{"slug": "openai", "name": "OpenAI", "total_models": 1, "is_current": True}
|
||||
],
|
||||
"current_model": "model_1",
|
||||
"current_provider": "openai",
|
||||
"session_key": "s",
|
||||
"on_model_selected": callback,
|
||||
"selected_provider": "openai",
|
||||
"model_list": ["gpt-5"],
|
||||
"msg_id": 42,
|
||||
}
|
||||
|
||||
query = AsyncMock()
|
||||
query.data = "mm:0"
|
||||
query.message = MagicMock()
|
||||
query.message.chat_id = 12345
|
||||
query.answer = AsyncMock()
|
||||
query.edit_message_text = AsyncMock()
|
||||
|
||||
await adapter._handle_model_picker_callback(query, "mm:0", "12345")
|
||||
|
||||
# The callback was invoked with the selected model
|
||||
callback.assert_awaited_once()
|
||||
# edit_message_text MUST be called on the success path (this is the
|
||||
# regression we're guarding).
|
||||
query.edit_message_text.assert_awaited()
|
||||
edit_kwargs = query.edit_message_text.call_args[1]
|
||||
assert "MARKDOWN_V2" in repr(edit_kwargs["parse_mode"])
|
||||
# The dynamic result text was routed through format_message
|
||||
# (backtick code blocks survive escaping).
|
||||
assert "`gpt-5`" in edit_kwargs["text"]
|
||||
# State is cleaned up after a successful switch.
|
||||
assert "12345" not in adapter._model_picker_state
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retries_without_thread_when_thread_not_found(self):
|
||||
adapter = _make_adapter()
|
||||
|
||||
@@ -14,6 +14,8 @@ to ``_run_agent``'s return dict and uses it for the slice.
|
||||
|
||||
import pytest
|
||||
|
||||
from gateway.run import _preserve_queued_followup_history_offset
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers - replicate the filtering logic from _run_agent
|
||||
@@ -265,3 +267,60 @@ class TestTranscriptHistoryOffset:
|
||||
assert len(fixed_new) == 2
|
||||
assert fixed_new[0]["content"] == "Now search for dogs"
|
||||
assert fixed_new[1]["content"] == "Dog results here."
|
||||
|
||||
def test_recursive_queued_followup_keeps_outer_history_offset(self):
|
||||
"""Queued drain persistence must include every turn in the chain.
|
||||
|
||||
``_run_agent()`` recurses when a follow-up arrived while the current turn
|
||||
was running. The recursive call naturally returns a later
|
||||
``history_offset`` because it received the previous turn as part of its
|
||||
input history. If the outer caller persists transcript rows using that
|
||||
later offset, it only sees the *last* queued turn as new and drops the
|
||||
earlier queued turn from the transcript.
|
||||
"""
|
||||
history_before_chain = [
|
||||
{"role": "user", "content": "Earlier question"},
|
||||
{"role": "assistant", "content": "Earlier answer"},
|
||||
]
|
||||
first_followup_turn = [
|
||||
{"role": "user", "content": "First follow-up question"},
|
||||
{"role": "assistant", "content": "First follow-up answer"},
|
||||
]
|
||||
second_followup_turn = [
|
||||
{"role": "user", "content": "Second follow-up question"},
|
||||
{"role": "assistant", "content": "Second follow-up answer"},
|
||||
]
|
||||
|
||||
current_result = {
|
||||
"history_offset": len(history_before_chain),
|
||||
"messages": history_before_chain + first_followup_turn,
|
||||
}
|
||||
followup_result = {
|
||||
"history_offset": len(history_before_chain + first_followup_turn),
|
||||
"messages": (
|
||||
history_before_chain
|
||||
+ first_followup_turn
|
||||
+ second_followup_turn
|
||||
),
|
||||
}
|
||||
|
||||
merged = _preserve_queued_followup_history_offset(
|
||||
current_result,
|
||||
followup_result,
|
||||
)
|
||||
assert merged["history_offset"] == len(history_before_chain)
|
||||
|
||||
persisted = merged["messages"][merged["history_offset"]:]
|
||||
assert persisted == first_followup_turn + second_followup_turn
|
||||
|
||||
def test_recursive_queued_followup_preserves_smaller_existing_offset(self):
|
||||
"""Do not widen the slice if the nested result is already conservative."""
|
||||
current_result = {"history_offset": 4}
|
||||
followup_result = {"history_offset": 3, "messages": []}
|
||||
|
||||
merged = _preserve_queued_followup_history_offset(
|
||||
current_result,
|
||||
followup_result,
|
||||
)
|
||||
|
||||
assert merged["history_offset"] == 3
|
||||
|
||||
@@ -46,6 +46,10 @@ def _make_adapter():
|
||||
adapter._message_queue = asyncio.Queue()
|
||||
adapter._http_session = MagicMock()
|
||||
adapter._mention_patterns = []
|
||||
adapter._dm_policy = "open"
|
||||
adapter._allow_from = set()
|
||||
adapter._group_policy = "open"
|
||||
adapter._group_allow_from = set()
|
||||
return adapter
|
||||
|
||||
|
||||
@@ -287,6 +291,41 @@ class TestSendChunking:
|
||||
assert "Not connected" in result.error
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# bridge event metadata
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBridgeEventMetadata:
|
||||
"""WhatsApp bridge metadata is preserved for downstream consumers."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quoted_reply_metadata_is_preserved_in_raw_message(self):
|
||||
adapter = _make_adapter()
|
||||
data = {
|
||||
"messageId": "incoming-msg",
|
||||
"chatId": "15551234567@s.whatsapp.net",
|
||||
"senderId": "15551234567@s.whatsapp.net",
|
||||
"senderName": "Tester",
|
||||
"chatName": "Tester",
|
||||
"isGroup": False,
|
||||
"body": "approved",
|
||||
"hasMedia": False,
|
||||
"mediaUrls": [],
|
||||
"quotedMessageId": "outbound-msg",
|
||||
"quotedParticipant": "99999999999@s.whatsapp.net",
|
||||
"quotedRemoteJid": "15551234567@s.whatsapp.net",
|
||||
"hasQuotedMessage": True,
|
||||
}
|
||||
|
||||
event = await adapter._build_message_event(data)
|
||||
|
||||
assert event is not None
|
||||
assert event.raw_message["quotedMessageId"] == "outbound-msg"
|
||||
assert event.raw_message["quotedParticipant"] == "99999999999@s.whatsapp.net"
|
||||
assert event.raw_message["quotedRemoteJid"] == "15551234567@s.whatsapp.net"
|
||||
assert event.raw_message["hasQuotedMessage"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# display_config tier classification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -296,3 +296,78 @@ def test_config_bridges_whatsapp_allow_from(monkeypatch, tmp_path):
|
||||
assert config.platforms[Platform.WHATSAPP].extra["allow_from"] == ["6281234567890@s.whatsapp.net"]
|
||||
assert __import__("os").environ["WHATSAPP_DM_POLICY"] == "allowlist"
|
||||
assert __import__("os").environ["WHATSAPP_ALLOWED_USERS"] == "6281234567890@s.whatsapp.net"
|
||||
|
||||
|
||||
# --- Broadcast / status / newsletter pseudo-chats are always dropped ---
|
||||
|
||||
|
||||
def test_status_broadcast_chats_are_always_dropped():
|
||||
"""Felipe's gateway.log showed the agent replying to status@broadcast
|
||||
(a contact's WhatsApp Story update). These pseudo-chats aren't real
|
||||
conversations and the adapter must drop them regardless of dm_policy.
|
||||
"""
|
||||
from gateway.platforms.whatsapp import WhatsAppAdapter
|
||||
|
||||
# Even on the most permissive config — open DMs, no allowlist — Stories
|
||||
# and Channel posts must not reach the agent.
|
||||
adapter = _make_adapter(dm_policy="open")
|
||||
|
||||
# Classic Story update — what Felipe was seeing in production.
|
||||
status_msg = _dm_message(
|
||||
body="[video received]",
|
||||
chatId="status@broadcast",
|
||||
senderId="34612345678@s.whatsapp.net",
|
||||
)
|
||||
assert adapter._should_process_message(status_msg) is False
|
||||
|
||||
# Channel / Newsletter broadcast posts.
|
||||
newsletter_msg = _dm_message(
|
||||
body="check out our latest post",
|
||||
chatId="120363999999999999@newsletter",
|
||||
senderId="120363999999999999@newsletter",
|
||||
)
|
||||
assert adapter._should_process_message(newsletter_msg) is False
|
||||
|
||||
|
||||
def test_broadcast_filter_runs_before_allowlist():
|
||||
"""A status@broadcast message from an allowlisted sender still drops —
|
||||
we never want to reply to Stories, even from authorized contacts.
|
||||
"""
|
||||
adapter = _make_adapter(
|
||||
dm_policy="allowlist",
|
||||
allow_from=["34612345678@s.whatsapp.net"],
|
||||
)
|
||||
|
||||
msg = _dm_message(
|
||||
body="[image received]",
|
||||
chatId="status@broadcast",
|
||||
senderId="34612345678@s.whatsapp.net",
|
||||
)
|
||||
assert adapter._should_process_message(msg) is False
|
||||
|
||||
|
||||
def test_real_dm_still_processed_after_broadcast_filter():
|
||||
"""Sanity check: the broadcast filter doesn't accidentally drop real DMs."""
|
||||
adapter = _make_adapter(dm_policy="open")
|
||||
|
||||
msg = _dm_message(
|
||||
body="hello",
|
||||
chatId="34612345678@s.whatsapp.net",
|
||||
senderId="34612345678@s.whatsapp.net",
|
||||
)
|
||||
assert adapter._should_process_message(msg) is True
|
||||
|
||||
|
||||
def test_is_broadcast_chat_helper_recognizes_common_jids():
|
||||
from gateway.platforms.whatsapp import WhatsAppAdapter
|
||||
|
||||
assert WhatsAppAdapter._is_broadcast_chat("status@broadcast") is True
|
||||
assert WhatsAppAdapter._is_broadcast_chat("STATUS@BROADCAST") is True
|
||||
assert WhatsAppAdapter._is_broadcast_chat(" status@broadcast ") is True
|
||||
assert WhatsAppAdapter._is_broadcast_chat("120363999999999999@newsletter") is True
|
||||
assert WhatsAppAdapter._is_broadcast_chat("1234@broadcast") is True # broadcast list
|
||||
# Real chats must not match.
|
||||
assert WhatsAppAdapter._is_broadcast_chat("34612345678@s.whatsapp.net") is False
|
||||
assert WhatsAppAdapter._is_broadcast_chat("120363001234567890@g.us") is False
|
||||
assert WhatsAppAdapter._is_broadcast_chat("") is False
|
||||
assert WhatsAppAdapter._is_broadcast_chat(None) is False # type: ignore[arg-type]
|
||||
|
||||
@@ -1099,6 +1099,159 @@ class TestHuggingFaceModels:
|
||||
assert _PROVIDER_LABELS["huggingface"] == "Hugging Face"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# NovitaAI provider tests (added by feat/add-novita-provider)
|
||||
# =============================================================================
|
||||
|
||||
class TestNovitaProvider:
|
||||
"""Tests for NovitaAI — an OpenAI-compatible multi-model aggregator."""
|
||||
|
||||
def test_novita_profile_loads(self):
|
||||
from providers import get_provider_profile
|
||||
profile = get_provider_profile("novita")
|
||||
assert profile is not None
|
||||
assert profile.name == "novita"
|
||||
assert profile.display_name == "NovitaAI"
|
||||
assert profile.base_url == "https://api.novita.ai/openai/v1"
|
||||
assert "NOVITA_API_KEY" in profile.env_vars
|
||||
|
||||
def test_novita_aliases(self):
|
||||
from providers import get_provider_profile
|
||||
profile = get_provider_profile("novita")
|
||||
assert "novita-ai" in profile.aliases
|
||||
assert "novitaai" in profile.aliases
|
||||
|
||||
def test_novita_alias_resolves(self):
|
||||
assert resolve_provider("novita-ai") == "novita"
|
||||
assert resolve_provider("novitaai") == "novita"
|
||||
|
||||
def test_novita_in_provider_registry(self):
|
||||
"""Auto-registration from ProviderProfile should expose Novita."""
|
||||
assert "novita" in PROVIDER_REGISTRY
|
||||
pconfig = PROVIDER_REGISTRY["novita"]
|
||||
assert pconfig.auth_type == "api_key"
|
||||
assert pconfig.id == "novita"
|
||||
assert pconfig.inference_base_url == "https://api.novita.ai/openai/v1"
|
||||
assert pconfig.api_key_env_vars == ("NOVITA_API_KEY",)
|
||||
assert pconfig.base_url_env_var == "NOVITA_BASE_URL"
|
||||
|
||||
def test_novita_aliases_in_registry(self):
|
||||
assert "novita-ai" in PROVIDER_REGISTRY
|
||||
assert "novitaai" in PROVIDER_REGISTRY
|
||||
|
||||
def test_main_provider_models_has_novita(self):
|
||||
from hermes_cli.main import _PROVIDER_MODELS
|
||||
assert "novita" in _PROVIDER_MODELS
|
||||
assert len(_PROVIDER_MODELS["novita"]) >= 1
|
||||
|
||||
def test_models_py_has_novita(self):
|
||||
from hermes_cli.models import _PROVIDER_MODELS
|
||||
assert "novita" in _PROVIDER_MODELS
|
||||
assert len(_PROVIDER_MODELS["novita"]) >= 1
|
||||
|
||||
def test_novita_model_lists_match(self):
|
||||
"""Model lists in main.py and models.py should be identical."""
|
||||
from hermes_cli.main import _PROVIDER_MODELS as main_models
|
||||
from hermes_cli.models import _PROVIDER_MODELS as models_models
|
||||
assert main_models["novita"] == models_models["novita"]
|
||||
|
||||
def test_novita_models_use_org_name_format(self):
|
||||
"""Novita models should use org/name format."""
|
||||
from hermes_cli.models import _PROVIDER_MODELS
|
||||
for model in _PROVIDER_MODELS["novita"]:
|
||||
assert "/" in model, f"Novita model {model!r} missing org/ prefix"
|
||||
|
||||
def test_novita_aliases_in_models_py(self):
|
||||
from hermes_cli.models import _PROVIDER_ALIASES
|
||||
assert _PROVIDER_ALIASES.get("novita-ai") == "novita"
|
||||
assert _PROVIDER_ALIASES.get("novitaai") == "novita"
|
||||
|
||||
def test_novita_label(self):
|
||||
from hermes_cli.models import _PROVIDER_LABELS
|
||||
assert "novita" in _PROVIDER_LABELS
|
||||
assert _PROVIDER_LABELS["novita"] == "NovitaAI"
|
||||
|
||||
def test_novita_in_provider_prefixes(self):
|
||||
from agent.model_metadata import _PROVIDER_PREFIXES
|
||||
assert "novita" in _PROVIDER_PREFIXES
|
||||
|
||||
def test_novita_url_to_provider(self):
|
||||
from agent.model_metadata import _URL_TO_PROVIDER
|
||||
assert _URL_TO_PROVIDER.get("api.novita.ai") == "novita"
|
||||
|
||||
def test_context_size_in_context_length_keys(self):
|
||||
"""Novita /v1/models uses 'context_size' as the context length key."""
|
||||
from agent.model_metadata import _CONTEXT_LENGTH_KEYS
|
||||
assert "context_size" in _CONTEXT_LENGTH_KEYS
|
||||
|
||||
def test_novita_pricing_unit_conversion(self):
|
||||
"""Novita returns prices in 0.0001 USD per Mtok; divide by 10_000 * 1_000_000."""
|
||||
from agent.model_metadata import _extract_pricing
|
||||
# Sample shape from real Novita /v1/models response
|
||||
payload = {
|
||||
"id": "deepseek/deepseek-v3-0324",
|
||||
"input_token_price_per_m": 2690, # = $0.269 / Mtok
|
||||
"output_token_price_per_m": 4000, # = $0.400 / Mtok
|
||||
}
|
||||
result = _extract_pricing(payload)
|
||||
# Resulting strings represent per-token prices in dollars.
|
||||
assert "prompt" in result
|
||||
assert "completion" in result
|
||||
assert float(result["prompt"]) == 2690 / 10_000 / 1_000_000
|
||||
assert float(result["completion"]) == 4000 / 10_000 / 1_000_000
|
||||
|
||||
def test_novita_pricing_cache(self, monkeypatch):
|
||||
"""_fetch_novita_pricing should cache results in _pricing_cache."""
|
||||
from hermes_cli import models as models_mod
|
||||
monkeypatch.setenv("NOVITA_API_KEY", "sk-test-key")
|
||||
monkeypatch.setenv("NOVITA_BASE_URL", "https://api.novita.ai/openai/v1")
|
||||
models_mod._pricing_cache.pop("https://api.novita.ai/openai/v1", None)
|
||||
|
||||
call_count = {"n": 0}
|
||||
fake_payload = {
|
||||
"data": [
|
||||
{
|
||||
"id": "x/y",
|
||||
"input_token_price_per_m": 1000,
|
||||
"output_token_price_per_m": 2000,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
class _FakeResp:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
import json as _json
|
||||
return _json.dumps(fake_payload).encode()
|
||||
|
||||
def fake_urlopen(req, timeout=None):
|
||||
call_count["n"] += 1
|
||||
return _FakeResp()
|
||||
|
||||
monkeypatch.setattr(
|
||||
models_mod.urllib.request, "urlopen", fake_urlopen
|
||||
)
|
||||
|
||||
# First call hits the network.
|
||||
first = models_mod._fetch_novita_pricing()
|
||||
assert "x/y" in first
|
||||
assert call_count["n"] == 1
|
||||
|
||||
# Second call returns cached result without re-hitting the network.
|
||||
second = models_mod._fetch_novita_pricing()
|
||||
assert second == first
|
||||
assert call_count["n"] == 1
|
||||
|
||||
# force_refresh bypasses the cache.
|
||||
models_mod._fetch_novita_pricing(force_refresh=True)
|
||||
assert call_count["n"] == 2
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MiniMax OAuth provider tests (added by feat/minimax-oauth-provider)
|
||||
# =============================================================================
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,6 +17,8 @@ All Bedrock API calls are mocked — no real AWS credentials needed.
|
||||
"""
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from types import ModuleType
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -26,6 +28,19 @@ import pytest
|
||||
# Shared helpers / fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _mock_botocore_session(*, return_value=None):
|
||||
"""Patch botocore.session even when botocore is not installed."""
|
||||
botocore_mod = ModuleType("botocore")
|
||||
session_mod = ModuleType("botocore.session")
|
||||
session_mod.get_session = MagicMock(return_value=return_value)
|
||||
botocore_mod.session = session_mod
|
||||
with patch.dict("sys.modules", {"botocore": botocore_mod, "botocore.session": session_mod}):
|
||||
yield session_mod.get_session
|
||||
|
||||
|
||||
_EU_MODELS = [
|
||||
{"id": "eu.anthropic.claude-sonnet-4-6-20250514-v1:0", "name": "Claude Sonnet 4.6 (EU)", "provider": "inference-profile"},
|
||||
{"id": "eu.anthropic.claude-haiku-4-5-20251015-v1:0", "name": "Claude Haiku 4.5 (EU)", "provider": "inference-profile"},
|
||||
@@ -276,7 +291,7 @@ class TestBedrockRegionRouting:
|
||||
|
||||
with patch("agent.bedrock_adapter.has_aws_credentials", return_value=True), \
|
||||
patch("agent.bedrock_adapter.discover_bedrock_models", side_effect=_mock_discover), \
|
||||
patch("botocore.session.get_session", return_value=mock_session):
|
||||
_mock_botocore_session(return_value=mock_session):
|
||||
providers = list_authenticated_providers(current_provider="bedrock")
|
||||
|
||||
bedrock = next((p for p in providers if p["slug"] == "bedrock"), None)
|
||||
@@ -310,7 +325,7 @@ class TestBedrockRegionRouting:
|
||||
mock_session = MagicMock()
|
||||
mock_session.get_config_variable.return_value = "eu-central-1"
|
||||
|
||||
with patch("botocore.session.get_session", return_value=mock_session):
|
||||
with _mock_botocore_session(return_value=mock_session):
|
||||
region = resolve_bedrock_region()
|
||||
|
||||
assert region == "us-west-2", "env var should override botocore profile"
|
||||
|
||||
@@ -0,0 +1,865 @@
|
||||
"""Tests for the codex MCP plugin migration helper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.codex_runtime_plugin_migration import (
|
||||
MIGRATION_MARKER,
|
||||
MIGRATION_END_MARKER,
|
||||
MigrationReport,
|
||||
_build_hermes_tools_mcp_entry,
|
||||
_format_toml_value,
|
||||
_looks_like_test_tempdir,
|
||||
_strip_existing_managed_block,
|
||||
_strip_unmanaged_plugin_tables,
|
||||
_translate_one_server,
|
||||
migrate,
|
||||
render_codex_toml_section,
|
||||
)
|
||||
|
||||
|
||||
# ---- per-server translation ----
|
||||
|
||||
class TestTranslateOneServer:
|
||||
def test_stdio_basic(self):
|
||||
cfg, skipped = _translate_one_server("filesystem", {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"],
|
||||
"env": {"FOO": "bar"},
|
||||
})
|
||||
assert cfg == {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"],
|
||||
"env": {"FOO": "bar"},
|
||||
}
|
||||
assert skipped == []
|
||||
|
||||
def test_stdio_with_cwd(self):
|
||||
cfg, _ = _translate_one_server("custom", {
|
||||
"command": "/usr/bin/myserver",
|
||||
"cwd": "/var/lib/mcp",
|
||||
})
|
||||
assert cfg["cwd"] == "/var/lib/mcp"
|
||||
|
||||
def test_http_basic(self):
|
||||
cfg, skipped = _translate_one_server("api", {
|
||||
"url": "https://x.example/mcp",
|
||||
"headers": {"Authorization": "Bearer abc"},
|
||||
})
|
||||
assert cfg == {
|
||||
"url": "https://x.example/mcp",
|
||||
"http_headers": {"Authorization": "Bearer abc"},
|
||||
}
|
||||
assert skipped == []
|
||||
|
||||
def test_sse_falls_under_streamable_http_with_warning(self):
|
||||
cfg, skipped = _translate_one_server("sse_server", {
|
||||
"url": "http://localhost:8000/sse",
|
||||
"transport": "sse",
|
||||
})
|
||||
assert cfg["url"] == "http://localhost:8000/sse"
|
||||
assert any("sse" in s.lower() for s in skipped)
|
||||
|
||||
def test_timeouts_translate(self):
|
||||
cfg, _ = _translate_one_server("x", {
|
||||
"command": "y",
|
||||
"timeout": 180,
|
||||
"connect_timeout": 30,
|
||||
})
|
||||
assert cfg["tool_timeout_sec"] == 180.0
|
||||
assert cfg["startup_timeout_sec"] == 30.0
|
||||
|
||||
def test_non_numeric_timeout_skipped(self):
|
||||
cfg, skipped = _translate_one_server("x", {
|
||||
"command": "y",
|
||||
"timeout": "not-a-number",
|
||||
})
|
||||
assert "tool_timeout_sec" not in cfg
|
||||
assert any("timeout" in s and "numeric" in s for s in skipped)
|
||||
|
||||
def test_disabled_server_emits_enabled_false(self):
|
||||
cfg, _ = _translate_one_server("x", {
|
||||
"command": "y",
|
||||
"enabled": False,
|
||||
})
|
||||
assert cfg["enabled"] is False
|
||||
|
||||
def test_enabled_true_omitted(self):
|
||||
cfg, _ = _translate_one_server("x", {"command": "y", "enabled": True})
|
||||
assert "enabled" not in cfg # codex defaults to true
|
||||
|
||||
def test_command_and_url_prefers_stdio_warns(self):
|
||||
cfg, skipped = _translate_one_server("x", {
|
||||
"command": "y", "url": "http://z",
|
||||
})
|
||||
assert "command" in cfg
|
||||
assert "url" not in cfg
|
||||
assert any("url" in s for s in skipped)
|
||||
|
||||
def test_no_transport_returns_none(self):
|
||||
cfg, skipped = _translate_one_server("broken", {"description": "x"})
|
||||
assert cfg is None
|
||||
assert "no command or url" in skipped[0]
|
||||
|
||||
def test_sampling_dropped_with_warning(self):
|
||||
cfg, skipped = _translate_one_server("x", {
|
||||
"command": "y",
|
||||
"sampling": {"enabled": True, "model": "gemini-3-flash"},
|
||||
})
|
||||
assert "sampling" not in cfg
|
||||
assert any("sampling" in s for s in skipped)
|
||||
|
||||
def test_unknown_keys_warned(self):
|
||||
cfg, skipped = _translate_one_server("x", {
|
||||
"command": "y",
|
||||
"totally_made_up_key": "value",
|
||||
})
|
||||
assert "totally_made_up_key" not in cfg
|
||||
assert any("totally_made_up_key" in s for s in skipped)
|
||||
|
||||
def test_non_dict_input(self):
|
||||
cfg, skipped = _translate_one_server("x", "notadict") # type: ignore[arg-type]
|
||||
assert cfg is None
|
||||
|
||||
|
||||
# ---- TOML rendering ----
|
||||
|
||||
class TestTomlValueFormatter:
|
||||
def test_string_quoted(self):
|
||||
assert _format_toml_value("hello") == '"hello"'
|
||||
|
||||
def test_string_with_quotes_escaped(self):
|
||||
assert _format_toml_value('a"b') == '"a\\"b"'
|
||||
|
||||
def test_bool(self):
|
||||
assert _format_toml_value(True) == "true"
|
||||
assert _format_toml_value(False) == "false"
|
||||
|
||||
def test_int(self):
|
||||
assert _format_toml_value(42) == "42"
|
||||
|
||||
def test_float(self):
|
||||
assert _format_toml_value(180.0) == "180.0"
|
||||
|
||||
def test_list_of_strings(self):
|
||||
assert _format_toml_value(["a", "b"]) == '["a", "b"]'
|
||||
|
||||
def test_inline_table(self):
|
||||
out = _format_toml_value({"FOO": "bar"})
|
||||
assert out == '{ FOO = "bar" }'
|
||||
|
||||
def test_empty_inline_table(self):
|
||||
assert _format_toml_value({}) == "{}"
|
||||
|
||||
def test_string_with_newline_escaped(self):
|
||||
"""TOML basic strings don't allow literal newlines — a path or
|
||||
env var containing a newline must use \\n. Otherwise codex would
|
||||
refuse to load the config."""
|
||||
out = _format_toml_value("line one\nline two")
|
||||
assert "\n" not in out # no raw newline in output
|
||||
assert "\\n" in out
|
||||
|
||||
def test_string_with_tab_escaped(self):
|
||||
out = _format_toml_value("col1\tcol2")
|
||||
assert "\t" not in out
|
||||
assert "\\t" in out
|
||||
|
||||
def test_string_with_other_controls_escaped(self):
|
||||
for raw, expected in [
|
||||
("\r", "\\r"),
|
||||
("\f", "\\f"),
|
||||
("\b", "\\b"),
|
||||
]:
|
||||
out = _format_toml_value(f"x{raw}y")
|
||||
assert raw not in out, f"{raw!r} should be escaped"
|
||||
assert expected in out, f"{expected!r} should be in output"
|
||||
|
||||
def test_windows_path_escaped_correctly(self):
|
||||
out = _format_toml_value(r"C:\Users\Alice\.codex")
|
||||
# Each backslash should be doubled
|
||||
assert out == r'"C:\\Users\\Alice\\.codex"'
|
||||
|
||||
def test_atomic_write_no_temp_leak_on_success(self, tmp_path):
|
||||
"""The atomic-write path uses tempfile.mkstemp + rename. On
|
||||
success the temp file should not be left behind."""
|
||||
migrate({"mcp_servers": {"x": {"command": "y"}}},
|
||||
codex_home=tmp_path,
|
||||
discover_plugins=False,
|
||||
expose_hermes_tools=False,
|
||||
default_permission_profile=None)
|
||||
# config.toml should exist
|
||||
assert (tmp_path / "config.toml").exists()
|
||||
# And no .config.toml.* temp files left behind
|
||||
leftover = [p.name for p in tmp_path.iterdir()
|
||||
if p.name.startswith(".config.toml.")]
|
||||
assert leftover == [], f"temp file leaked after migration: {leftover}"
|
||||
|
||||
def test_atomic_write_cleanup_on_rename_failure(self, tmp_path, monkeypatch):
|
||||
"""If rename fails partway through (out of disk, permissions,
|
||||
crash), the temp file must be cleaned up. Otherwise repeated
|
||||
failed migrations would pile up .config.toml.* files."""
|
||||
from pathlib import Path as _Path
|
||||
original_replace = _Path.replace
|
||||
|
||||
def failing_replace(self, target):
|
||||
raise OSError("simulated disk full")
|
||||
|
||||
monkeypatch.setattr(_Path, "replace", failing_replace)
|
||||
report = migrate(
|
||||
{"mcp_servers": {"x": {"command": "y"}}},
|
||||
codex_home=tmp_path,
|
||||
discover_plugins=False,
|
||||
expose_hermes_tools=False,
|
||||
default_permission_profile=None,
|
||||
)
|
||||
# Error surfaced
|
||||
assert any("simulated disk full" in e for e in report.errors)
|
||||
# And no leaked temp file
|
||||
leftover = [p.name for p in tmp_path.iterdir()
|
||||
if p.name.startswith(".config.toml.")]
|
||||
assert leftover == [], f"temp files leaked: {leftover}"
|
||||
|
||||
def test_unsupported_type_raises(self):
|
||||
with pytest.raises(ValueError):
|
||||
_format_toml_value(object())
|
||||
|
||||
|
||||
class TestRenderToml:
|
||||
def test_starts_with_marker(self):
|
||||
out = render_codex_toml_section({})
|
||||
assert out.startswith(MIGRATION_MARKER)
|
||||
|
||||
def test_empty_servers_emits_placeholder(self):
|
||||
out = render_codex_toml_section({})
|
||||
assert "no MCP servers" in out
|
||||
|
||||
def test_servers_sorted_alphabetically(self):
|
||||
out = render_codex_toml_section({
|
||||
"zoo": {"command": "z"},
|
||||
"alpha": {"command": "a"},
|
||||
"middle": {"command": "m"},
|
||||
})
|
||||
# Find the section header positions and confirm order
|
||||
a_pos = out.find("[mcp_servers.alpha]")
|
||||
m_pos = out.find("[mcp_servers.middle]")
|
||||
z_pos = out.find("[mcp_servers.zoo]")
|
||||
assert 0 < a_pos < m_pos < z_pos
|
||||
|
||||
def test_server_with_args_and_env(self):
|
||||
out = render_codex_toml_section({
|
||||
"fs": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "filesystem"],
|
||||
"env": {"PATH": "/usr/bin"},
|
||||
}
|
||||
})
|
||||
assert "[mcp_servers.fs]" in out
|
||||
assert 'command = "npx"' in out
|
||||
assert 'args = ["-y", "filesystem"]' in out
|
||||
# Env emitted as inline table
|
||||
assert 'env = { PATH = "/usr/bin" }' in out
|
||||
|
||||
|
||||
# ---- existing-block stripping ----
|
||||
|
||||
class TestStripExistingManagedBlock:
|
||||
def test_no_managed_block_unchanged(self):
|
||||
text = "[other]\nfoo = 1\n"
|
||||
assert _strip_existing_managed_block(text) == text
|
||||
|
||||
def test_strips_managed_block_alone(self):
|
||||
text = (
|
||||
f"{MIGRATION_MARKER}\n"
|
||||
"\n"
|
||||
"[mcp_servers.fs]\n"
|
||||
'command = "npx"\n'
|
||||
)
|
||||
assert _strip_existing_managed_block(text).strip() == ""
|
||||
|
||||
def test_preserves_user_content_above_managed_block(self):
|
||||
text = (
|
||||
"[model]\n"
|
||||
'name = "gpt-5.5"\n'
|
||||
"\n"
|
||||
f"{MIGRATION_MARKER}\n"
|
||||
"[mcp_servers.fs]\n"
|
||||
'command = "x"\n'
|
||||
)
|
||||
out = _strip_existing_managed_block(text)
|
||||
assert "[model]" in out
|
||||
assert 'name = "gpt-5.5"' in out
|
||||
assert "mcp_servers.fs" not in out
|
||||
|
||||
def test_preserves_unrelated_section_after_managed_block(self):
|
||||
text = (
|
||||
f"{MIGRATION_MARKER}\n"
|
||||
"[mcp_servers.fs]\n"
|
||||
'command = "x"\n'
|
||||
"\n"
|
||||
"[providers]\n"
|
||||
'foo = "bar"\n'
|
||||
)
|
||||
out = _strip_existing_managed_block(text)
|
||||
assert "mcp_servers.fs" not in out
|
||||
assert "[providers]" in out
|
||||
assert 'foo = "bar"' in out
|
||||
|
||||
|
||||
# ---- end-to-end migrate(, expose_hermes_tools=False) ----
|
||||
|
||||
class TestMigrate:
|
||||
def test_no_servers_no_plugins_no_perms_writes_placeholder(self, tmp_path):
|
||||
report = migrate({}, codex_home=tmp_path,
|
||||
discover_plugins=False,
|
||||
default_permission_profile=None, expose_hermes_tools=False)
|
||||
assert report.written
|
||||
text = (tmp_path / "config.toml").read_text()
|
||||
assert MIGRATION_MARKER in text
|
||||
assert "no MCP servers" in text or "no MCP servers, plugins, or permissions" in text
|
||||
|
||||
def test_no_servers_still_writes_permissions_default(self, tmp_path):
|
||||
"""Even with zero MCP servers, enabling the runtime should write the
|
||||
default permissions profile so users don't get prompted on every
|
||||
write attempt. This is the fix for quirk #2."""
|
||||
report = migrate({}, codex_home=tmp_path, discover_plugins=False, expose_hermes_tools=False)
|
||||
assert report.written
|
||||
text = (tmp_path / "config.toml").read_text()
|
||||
# Codex's schema: top-level `default_permissions` keying a built-in
|
||||
# profile name (prefixed with ":"). NOT a [permissions] section
|
||||
# (which is for *user-defined* profiles with structured fields).
|
||||
assert 'default_permissions = ":workspace"' in text
|
||||
assert report.wrote_permissions_default == ":workspace"
|
||||
|
||||
def test_explicit_none_permissions_skips_block(self, tmp_path):
|
||||
report = migrate({"mcp_servers": {"x": {"command": "y"}}},
|
||||
codex_home=tmp_path,
|
||||
discover_plugins=False,
|
||||
default_permission_profile=None, expose_hermes_tools=False)
|
||||
text = (tmp_path / "config.toml").read_text()
|
||||
assert "default_permissions" not in text
|
||||
assert "[permissions]" not in text
|
||||
assert report.wrote_permissions_default is None
|
||||
|
||||
def test_plugin_discovery_writes_plugin_blocks(self, tmp_path, monkeypatch):
|
||||
"""Discovered curated plugins land as [plugins."<name>@<marketplace>"]
|
||||
blocks. This is what OpenClaw calls 'migrate native codex plugins.'"""
|
||||
from hermes_cli import codex_runtime_plugin_migration as crpm
|
||||
|
||||
def fake_query(codex_home=None, timeout=8.0):
|
||||
return [
|
||||
{"name": "google-calendar", "marketplace": "openai-curated",
|
||||
"enabled": True},
|
||||
{"name": "github", "marketplace": "openai-curated",
|
||||
"enabled": True},
|
||||
], None
|
||||
monkeypatch.setattr(crpm, "_query_codex_plugins", fake_query)
|
||||
|
||||
report = migrate({}, codex_home=tmp_path, discover_plugins=True)
|
||||
text = (tmp_path / "config.toml").read_text()
|
||||
assert '[plugins."github@openai-curated"]' in text
|
||||
assert '[plugins."google-calendar@openai-curated"]' in text
|
||||
assert "enabled = true" in text
|
||||
assert "google-calendar@openai-curated" in report.migrated_plugins
|
||||
assert "github@openai-curated" in report.migrated_plugins
|
||||
|
||||
def test_plugin_discovery_skips_unavailable_plugins(self):
|
||||
"""Plugins where codex reports availability != AVAILABLE should
|
||||
be skipped — they're broken/uninstallable on codex's side, so
|
||||
migrating them would write config that fails at activation
|
||||
time. Cf. openclaw#80815."""
|
||||
from hermes_cli.codex_runtime_plugin_migration import _query_codex_plugins
|
||||
from unittest.mock import patch
|
||||
|
||||
# Fake a plugin/list response where one plugin is unavailable
|
||||
fake_response = {
|
||||
"marketplaces": [{
|
||||
"name": "openai-curated",
|
||||
"plugins": [
|
||||
{"name": "good-plugin", "installed": True,
|
||||
"enabled": True, "availability": "AVAILABLE"},
|
||||
{"name": "broken-plugin", "installed": True,
|
||||
"enabled": True, "availability": "UNAVAILABLE"},
|
||||
{"name": "auth-pending", "installed": True,
|
||||
"enabled": True, "availability": "REQUIRES_AUTH"},
|
||||
# Plugin without availability field — pass through
|
||||
# (older codex versions or marketplaces that don't
|
||||
# set it should still work).
|
||||
{"name": "legacy-plugin", "installed": True,
|
||||
"enabled": True},
|
||||
]
|
||||
}]
|
||||
}
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, **kw): pass
|
||||
def initialize(self, **kw): pass
|
||||
def request(self, method, params, timeout=None):
|
||||
return fake_response
|
||||
def close(self): pass
|
||||
def __enter__(self): return self
|
||||
def __exit__(self, *a): pass
|
||||
|
||||
with patch("agent.transports.codex_app_server.CodexAppServerClient",
|
||||
FakeClient):
|
||||
plugins, err = _query_codex_plugins()
|
||||
|
||||
assert err is None
|
||||
names = [p["name"] for p in plugins]
|
||||
assert "good-plugin" in names
|
||||
assert "legacy-plugin" in names # no field → don't skip
|
||||
assert "broken-plugin" not in names
|
||||
assert "auth-pending" not in names
|
||||
|
||||
def test_plugin_discovery_failure_non_fatal(self, tmp_path, monkeypatch):
|
||||
"""If codex isn't installed or RPC fails, MCP migration still
|
||||
completes. The error surfaces in the report but doesn't abort."""
|
||||
from hermes_cli import codex_runtime_plugin_migration as crpm
|
||||
|
||||
def fake_query_fails(codex_home=None, timeout=8.0):
|
||||
return [], "codex CLI not available"
|
||||
monkeypatch.setattr(crpm, "_query_codex_plugins", fake_query_fails)
|
||||
|
||||
report = migrate({"mcp_servers": {"x": {"command": "y"}}},
|
||||
codex_home=tmp_path, discover_plugins=True, expose_hermes_tools=False)
|
||||
assert report.written
|
||||
assert report.migrated == ["x"]
|
||||
assert report.plugin_query_error == "codex CLI not available"
|
||||
assert report.migrated_plugins == []
|
||||
|
||||
def test_discover_plugins_false_skips_query(self, tmp_path, monkeypatch):
|
||||
"""Tests and restricted environments can opt out of the subprocess
|
||||
spawn entirely."""
|
||||
from hermes_cli import codex_runtime_plugin_migration as crpm
|
||||
|
||||
called = {"yes": False}
|
||||
def boom(*a, **kw):
|
||||
called["yes"] = True
|
||||
return [], None
|
||||
monkeypatch.setattr(crpm, "_query_codex_plugins", boom)
|
||||
|
||||
migrate({"mcp_servers": {"x": {"command": "y"}}},
|
||||
codex_home=tmp_path, discover_plugins=False, expose_hermes_tools=False)
|
||||
assert called["yes"] is False
|
||||
|
||||
def test_dry_run_skips_plugin_query(self, tmp_path, monkeypatch):
|
||||
"""Dry run should never spawn codex. Even with discover_plugins=True
|
||||
the query is skipped because dry_run takes precedence."""
|
||||
from hermes_cli import codex_runtime_plugin_migration as crpm
|
||||
|
||||
called = {"yes": False}
|
||||
def boom(*a, **kw):
|
||||
called["yes"] = True
|
||||
return [], None
|
||||
monkeypatch.setattr(crpm, "_query_codex_plugins", boom)
|
||||
|
||||
migrate({"mcp_servers": {"x": {"command": "y"}}},
|
||||
codex_home=tmp_path, dry_run=True, discover_plugins=True, expose_hermes_tools=False)
|
||||
assert called["yes"] is False
|
||||
|
||||
def test_re_run_replaces_plugin_block(self, tmp_path, monkeypatch):
|
||||
"""Plugin blocks are managed and re-runs should replace them
|
||||
cleanly — same idempotency contract as MCP servers."""
|
||||
from hermes_cli import codex_runtime_plugin_migration as crpm
|
||||
|
||||
# First run: only github
|
||||
monkeypatch.setattr(crpm, "_query_codex_plugins",
|
||||
lambda codex_home=None, timeout=8.0: (
|
||||
[{"name": "github", "marketplace": "openai-curated", "enabled": True}],
|
||||
None,
|
||||
))
|
||||
migrate({}, codex_home=tmp_path, discover_plugins=True,
|
||||
default_permission_profile=None, expose_hermes_tools=False)
|
||||
first = (tmp_path / "config.toml").read_text()
|
||||
assert "github@openai-curated" in first
|
||||
|
||||
# Second run: only canva (github went away)
|
||||
monkeypatch.setattr(crpm, "_query_codex_plugins",
|
||||
lambda codex_home=None, timeout=8.0: (
|
||||
[{"name": "canva", "marketplace": "openai-curated", "enabled": True}],
|
||||
None,
|
||||
))
|
||||
migrate({}, codex_home=tmp_path, discover_plugins=True,
|
||||
default_permission_profile=None, expose_hermes_tools=False)
|
||||
second = (tmp_path / "config.toml").read_text()
|
||||
assert "github@openai-curated" not in second
|
||||
assert "canva@openai-curated" in second
|
||||
|
||||
def test_expose_hermes_tools_writes_callback_mcp_entry(self, tmp_path):
|
||||
"""When expose_hermes_tools=True (production default), an
|
||||
[mcp_servers.hermes-tools] entry is written so codex calls back
|
||||
into Hermes for browser/web/delegate_task/vision/memory tools.
|
||||
|
||||
This is the fix for 'all other tools that codex doesn't provide
|
||||
should be useable by hermes' — quirk #7."""
|
||||
report = migrate({}, codex_home=tmp_path,
|
||||
discover_plugins=False,
|
||||
default_permission_profile=None,
|
||||
expose_hermes_tools=True)
|
||||
text = (tmp_path / "config.toml").read_text()
|
||||
assert "[mcp_servers.hermes-tools]" in text
|
||||
assert "hermes_tools_mcp_server" in text
|
||||
# Must include startup + tool timeouts so codex doesn't give up
|
||||
assert "startup_timeout_sec" in text
|
||||
assert "tool_timeout_sec" in text
|
||||
# And the entry is reported
|
||||
assert "hermes-tools" in report.migrated
|
||||
|
||||
def test_expose_hermes_tools_disabled_skips_entry(self, tmp_path):
|
||||
"""expose_hermes_tools=False suppresses the callback registration."""
|
||||
migrate({}, codex_home=tmp_path,
|
||||
discover_plugins=False,
|
||||
default_permission_profile=None,
|
||||
expose_hermes_tools=False)
|
||||
text = (tmp_path / "config.toml").read_text()
|
||||
assert "[mcp_servers.hermes-tools]" not in text
|
||||
assert "hermes_tools_mcp_server" not in text
|
||||
|
||||
def test_dry_run_doesnt_write(self, tmp_path):
|
||||
report = migrate({"mcp_servers": {"x": {"command": "y"}}},
|
||||
codex_home=tmp_path, dry_run=True, expose_hermes_tools=False)
|
||||
assert report.dry_run is True
|
||||
assert not (tmp_path / "config.toml").exists()
|
||||
assert "x" in report.migrated
|
||||
|
||||
def test_full_migration_round_trip(self, tmp_path):
|
||||
hermes_cfg = {
|
||||
"mcp_servers": {
|
||||
"filesystem": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@modelcontextprotocol/server-filesystem"],
|
||||
},
|
||||
"github": {
|
||||
"url": "https://api.github.com/mcp",
|
||||
"headers": {"Authorization": "Bearer x"},
|
||||
},
|
||||
}
|
||||
}
|
||||
report = migrate(hermes_cfg, codex_home=tmp_path, expose_hermes_tools=False)
|
||||
assert report.written
|
||||
text = (tmp_path / "config.toml").read_text()
|
||||
assert "[mcp_servers.filesystem]" in text
|
||||
assert "[mcp_servers.github]" in text
|
||||
assert 'command = "npx"' in text
|
||||
assert 'url = "https://api.github.com/mcp"' in text
|
||||
|
||||
def test_idempotent_re_run_replaces_managed_block(self, tmp_path):
|
||||
# First migration
|
||||
migrate({"mcp_servers": {"a": {"command": "x"}}}, codex_home=tmp_path, expose_hermes_tools=False)
|
||||
first_text = (tmp_path / "config.toml").read_text()
|
||||
assert "[mcp_servers.a]" in first_text
|
||||
# Second migration with different servers
|
||||
migrate({"mcp_servers": {"b": {"command": "y"}}}, codex_home=tmp_path, expose_hermes_tools=False)
|
||||
second_text = (tmp_path / "config.toml").read_text()
|
||||
assert "[mcp_servers.a]" not in second_text
|
||||
assert "[mcp_servers.b]" in second_text
|
||||
|
||||
def test_preserves_user_codex_config_above_marker(self, tmp_path):
|
||||
target = tmp_path / "config.toml"
|
||||
target.write_text(
|
||||
"[model]\n"
|
||||
'profile = "default"\n'
|
||||
"\n"
|
||||
"[providers.openai]\n"
|
||||
'api_key = "sk-test"\n'
|
||||
)
|
||||
migrate({"mcp_servers": {"a": {"command": "x"}}}, codex_home=tmp_path, expose_hermes_tools=False)
|
||||
new_text = target.read_text()
|
||||
# User's codex config preserved
|
||||
assert "[model]" in new_text
|
||||
assert 'profile = "default"' in new_text
|
||||
assert "[providers.openai]" in new_text
|
||||
# And new MCP block inserted without breaking user tables
|
||||
assert "[mcp_servers.a]" in new_text
|
||||
assert MIGRATION_MARKER in new_text
|
||||
|
||||
def test_managed_root_keys_stay_top_level_when_config_ends_in_table(self, tmp_path):
|
||||
"""TOML has no explicit 'leave current table' syntax. If Hermes appends
|
||||
root keys like default_permissions after a user table such as [features],
|
||||
Codex parses them as features.default_permissions and rejects the config.
|
||||
The managed block must therefore be inserted before the first table."""
|
||||
import tomllib
|
||||
|
||||
target = tmp_path / "config.toml"
|
||||
target.write_text(
|
||||
'model = "gpt-5.5"\n'
|
||||
"\n"
|
||||
"[features]\n"
|
||||
"terminal_resize_reflow = true\n"
|
||||
)
|
||||
migrate({}, codex_home=tmp_path, discover_plugins=False, expose_hermes_tools=False)
|
||||
new_text = target.read_text()
|
||||
parsed = tomllib.loads(new_text)
|
||||
assert parsed["default_permissions"] == ":workspace"
|
||||
assert "default_permissions" not in parsed["features"]
|
||||
assert new_text.index(MIGRATION_MARKER) < new_text.index("[features]")
|
||||
|
||||
def test_preserves_user_mcp_server_outside_managed_block(self, tmp_path):
|
||||
"""Quirk #6: when a user adds their own MCP server entry directly
|
||||
to ~/.codex/config.toml outside Hermes' managed block, re-running
|
||||
migration must preserve it. Tested both above and below the
|
||||
managed block."""
|
||||
target = tmp_path / "config.toml"
|
||||
target.write_text(
|
||||
"[mcp_servers.user-above]\n"
|
||||
'command = "/usr/bin/above-server"\n'
|
||||
'args = ["--above"]\n'
|
||||
)
|
||||
# First migrate — adds managed block below user content
|
||||
migrate({"mcp_servers": {"hermes-mcp": {"command": "npx"}}},
|
||||
codex_home=tmp_path, discover_plugins=False,
|
||||
expose_hermes_tools=False)
|
||||
text = target.read_text()
|
||||
assert "user-above" in text, "user MCP server above managed block got nuked"
|
||||
assert 'command = "/usr/bin/above-server"' in text
|
||||
|
||||
# Append another user entry below the managed block
|
||||
target.write_text(
|
||||
text + "\n[mcp_servers.user-below]\ncommand = \"below-server\"\n"
|
||||
)
|
||||
# Re-migrate — both should survive
|
||||
migrate({"mcp_servers": {"hermes-mcp": {"command": "npx"}}},
|
||||
codex_home=tmp_path, discover_plugins=False,
|
||||
expose_hermes_tools=False)
|
||||
final = target.read_text()
|
||||
assert "user-above" in final
|
||||
assert "user-below" in final
|
||||
# And our managed block is still there with the new content
|
||||
assert "[mcp_servers.hermes-mcp]" in final
|
||||
|
||||
def test_skipped_keys_reported(self, tmp_path):
|
||||
report = migrate({
|
||||
"mcp_servers": {
|
||||
"x": {
|
||||
"command": "y",
|
||||
"sampling": {"enabled": True}, # codex has no equivalent
|
||||
}
|
||||
}
|
||||
}, codex_home=tmp_path, expose_hermes_tools=False)
|
||||
assert "x" in report.skipped_keys_per_server
|
||||
assert any("sampling" in s for s in report.skipped_keys_per_server["x"])
|
||||
|
||||
def test_invalid_mcp_servers_value(self, tmp_path):
|
||||
report = migrate({"mcp_servers": "notadict"}, codex_home=tmp_path, expose_hermes_tools=False)
|
||||
assert any("not a dict" in e for e in report.errors)
|
||||
|
||||
def test_server_without_transport_skipped_with_error(self, tmp_path):
|
||||
report = migrate({
|
||||
"mcp_servers": {"broken": {"description": "no command/url"}}
|
||||
}, codex_home=tmp_path, expose_hermes_tools=False)
|
||||
assert "broken" not in report.migrated
|
||||
assert any("broken" in e for e in report.errors)
|
||||
|
||||
def test_summary_reports_migration_count(self, tmp_path):
|
||||
report = migrate({
|
||||
"mcp_servers": {"a": {"command": "x"}, "b": {"command": "y"}}
|
||||
}, codex_home=tmp_path, expose_hermes_tools=False)
|
||||
summary = report.summary()
|
||||
assert "Migrated 2 MCP server(s)" in summary
|
||||
assert "- a" in summary
|
||||
assert "- b" in summary
|
||||
|
||||
|
||||
# ---- Bug B: duplicate [plugins.X] tables ----
|
||||
|
||||
|
||||
class TestStripUnmanagedPluginTables:
|
||||
"""Regression tests for issue #26250 Bug B.
|
||||
|
||||
When codex itself writes ``[plugins."<name>@<marketplace>"]`` tables
|
||||
(via the user running ``codex plugins enable`` directly), re-running
|
||||
``hermes codex-runtime migrate`` would re-emit them inside the managed
|
||||
block and the resulting duplicate-table-header would crash codex.
|
||||
"""
|
||||
|
||||
def test_strips_plugin_tables_outside_managed_block(self):
|
||||
text = (
|
||||
'model = "gpt-5.5"\n'
|
||||
"\n"
|
||||
"[mcp_servers.user-thing]\n"
|
||||
'command = "x"\n'
|
||||
"\n"
|
||||
'[plugins."tasks@openai-curated"]\n'
|
||||
"enabled = true\n"
|
||||
"\n"
|
||||
'[plugins."web-search@openai-curated"]\n'
|
||||
"enabled = true\n"
|
||||
"\n"
|
||||
"[features]\n"
|
||||
"terminal_resize_reflow = true\n"
|
||||
)
|
||||
stripped = _strip_unmanaged_plugin_tables(text)
|
||||
assert "[plugins." not in stripped
|
||||
# Non-plugin content preserved
|
||||
assert "[mcp_servers.user-thing]" in stripped
|
||||
assert "[features]" in stripped
|
||||
assert "terminal_resize_reflow = true" in stripped
|
||||
|
||||
def test_preserves_content_when_no_plugin_tables(self):
|
||||
text = (
|
||||
'model = "gpt-5.5"\n'
|
||||
"\n"
|
||||
"[mcp_servers.x]\n"
|
||||
'command = "y"\n'
|
||||
)
|
||||
assert _strip_unmanaged_plugin_tables(text) == text
|
||||
|
||||
def test_multi_line_array_in_plugin_table_does_not_leak(self):
|
||||
"""A multi-line TOML array inside a [plugins.X] table whose
|
||||
continuation lines start with ``[`` (e.g. nested arrays) must NOT
|
||||
prematurely exit the strip region — otherwise array fragments
|
||||
leak into top-level output and produce invalid TOML on the next
|
||||
codex startup. Regression guard for #26260 review.
|
||||
"""
|
||||
text = (
|
||||
'[plugins."tasks@openai-curated"]\n'
|
||||
"allowed = [\n"
|
||||
' "a",\n'
|
||||
' ["nested"],\n'
|
||||
"]\n"
|
||||
"[features]\n"
|
||||
"x = 1\n"
|
||||
)
|
||||
stripped = _strip_unmanaged_plugin_tables(text)
|
||||
# Everything inside the plugin table — including the multi-line
|
||||
# array's continuation lines starting with `[` — should be gone.
|
||||
assert '["nested"]' not in stripped
|
||||
assert "allowed" not in stripped
|
||||
# Sibling user table survives intact.
|
||||
assert "[features]" in stripped
|
||||
assert "x = 1" in stripped
|
||||
# Result is still valid TOML.
|
||||
import tomllib
|
||||
tomllib.loads(stripped)
|
||||
|
||||
def test_migrate_dedups_codex_owned_plugin_tables(self, tmp_path, monkeypatch):
|
||||
"""End-to-end: codex's pre-existing [plugins.X] tables get replaced by
|
||||
the managed block's re-emission rather than duplicated."""
|
||||
target = tmp_path / "config.toml"
|
||||
target.write_text(
|
||||
"[mcp_servers.user-server]\n"
|
||||
'command = "x"\n'
|
||||
"\n"
|
||||
'[plugins."tasks@openai-curated"]\n'
|
||||
"enabled = true\n"
|
||||
)
|
||||
|
||||
# Simulate codex's plugin/list reporting the same plugin tasks@openai-curated.
|
||||
def fake_query(codex_home=None, timeout=8.0):
|
||||
return (
|
||||
[{"name": "tasks", "marketplace": "openai-curated", "enabled": True}],
|
||||
None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.codex_runtime_plugin_migration._query_codex_plugins",
|
||||
fake_query,
|
||||
)
|
||||
migrate({}, codex_home=tmp_path, discover_plugins=True, expose_hermes_tools=False)
|
||||
new_text = target.read_text()
|
||||
# Only ONE [plugins."tasks@openai-curated"] header should remain — inside
|
||||
# the managed block — not the original outside-the-block copy.
|
||||
assert new_text.count('[plugins."tasks@openai-curated"]') == 1
|
||||
# And the surviving one is inside our managed section.
|
||||
managed_start = new_text.index(MIGRATION_MARKER)
|
||||
managed_end = new_text.index(MIGRATION_END_MARKER)
|
||||
plugin_idx = new_text.index('[plugins."tasks@openai-curated"]')
|
||||
assert managed_start < plugin_idx < managed_end
|
||||
# File parses cleanly as TOML (the original duplicate-key error is gone).
|
||||
import tomllib
|
||||
tomllib.loads(new_text)
|
||||
|
||||
def test_migrate_preserves_plugin_tables_when_plugin_list_fails(self, tmp_path, monkeypatch):
|
||||
"""If plugin/list RPC fails, we can't re-emit plugins authoritatively,
|
||||
so we must NOT strip the user's existing [plugins.X] tables — that
|
||||
would silently lose them."""
|
||||
target = tmp_path / "config.toml"
|
||||
target.write_text(
|
||||
'[plugins."tasks@openai-curated"]\n'
|
||||
"enabled = true\n"
|
||||
)
|
||||
|
||||
def fake_query(codex_home=None, timeout=8.0):
|
||||
return ([], "plugin/list query failed: codex not installed")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.codex_runtime_plugin_migration._query_codex_plugins",
|
||||
fake_query,
|
||||
)
|
||||
migrate({}, codex_home=tmp_path, discover_plugins=True, expose_hermes_tools=False)
|
||||
new_text = target.read_text()
|
||||
# User's plugin table preserved verbatim — we can't re-emit it.
|
||||
assert '[plugins."tasks@openai-curated"]' in new_text
|
||||
|
||||
|
||||
# ---- Bug C: HERMES_HOME tempdir leak into ~/.codex/config.toml ----
|
||||
|
||||
|
||||
class TestHermesHomeLeakGuard:
|
||||
"""Regression tests for issue #26250 Bug C.
|
||||
|
||||
Previously ``_build_hermes_tools_mcp_entry()`` read ``HERMES_HOME``
|
||||
directly from ``os.environ``, so a pytest ``monkeypatch.setenv`` would
|
||||
leak a transient tempdir path into the user's real ``~/.codex/config.toml``
|
||||
once codex spawned the hermes-tools MCP subprocess.
|
||||
"""
|
||||
|
||||
def test_tempdir_detector_recognizes_pytest_paths(self):
|
||||
assert _looks_like_test_tempdir(
|
||||
"/private/var/folders/abc/pytest-of-kshitij/pytest-137/popen-gw2/test_X/hermes_test"
|
||||
)
|
||||
assert _looks_like_test_tempdir(
|
||||
"/tmp/pytest-of-user/pytest-12/test_X/hermes"
|
||||
)
|
||||
assert _looks_like_test_tempdir(
|
||||
"/private/var/folders/zz/T/pytest-of-bob/pytest-1"
|
||||
)
|
||||
|
||||
def test_tempdir_detector_accepts_real_hermes_home(self):
|
||||
assert not _looks_like_test_tempdir("/Users/alice/.hermes")
|
||||
assert not _looks_like_test_tempdir("/home/bob/.hermes")
|
||||
assert not _looks_like_test_tempdir("/opt/hermes")
|
||||
assert not _looks_like_test_tempdir("")
|
||||
|
||||
def test_pytest_tempdir_not_burned_into_mcp_env(self, monkeypatch):
|
||||
"""The headline regression: even when HERMES_HOME points at a pytest
|
||||
tempdir, _build_hermes_tools_mcp_entry() must NOT propagate it."""
|
||||
monkeypatch.setenv(
|
||||
"HERMES_HOME",
|
||||
"/private/var/folders/xx/pytest-of-user/pytest-99/test_x/hermes_test",
|
||||
)
|
||||
entry = _build_hermes_tools_mcp_entry()
|
||||
env = entry.get("env", {})
|
||||
assert "HERMES_HOME" not in env, (
|
||||
f"pytest-tempdir HERMES_HOME leaked into codex MCP entry: "
|
||||
f"{env.get('HERMES_HOME')!r}"
|
||||
)
|
||||
|
||||
def test_real_hermes_home_propagates(self, monkeypatch, tmp_path):
|
||||
"""A legitimate HERMES_HOME (not a tempdir path) DOES propagate so the
|
||||
MCP subprocess sees the same config as the parent CLI."""
|
||||
# Use a path that looks real — under /Users or /home, not /var/folders.
|
||||
# We can't easily create one in the test, so just use a stable path
|
||||
# outside any tempdir-detector needle. The detector checks for tempdir
|
||||
# markers, not for path existence.
|
||||
real_path = "/Users/alice/.hermes"
|
||||
monkeypatch.setenv("HERMES_HOME", real_path)
|
||||
entry = _build_hermes_tools_mcp_entry()
|
||||
env = entry.get("env", {})
|
||||
assert env.get("HERMES_HOME") == real_path
|
||||
|
||||
def test_unset_hermes_home_omits_env_key(self, monkeypatch):
|
||||
"""When HERMES_HOME is unset in the environment, the MCP entry MUST
|
||||
NOT bake in a resolved-default path. The codex subprocess should
|
||||
inherit whatever HERMES_HOME its launcher (systemd, gateway, shell)
|
||||
sets at runtime, rather than being pinned to migrate-time defaults.
|
||||
Regression guard for issue #26250 follow-up review."""
|
||||
monkeypatch.delenv("HERMES_HOME", raising=False)
|
||||
entry = _build_hermes_tools_mcp_entry()
|
||||
env = entry.get("env", {})
|
||||
assert "HERMES_HOME" not in env, (
|
||||
f"HERMES_HOME should not be set when env var is unset, got: "
|
||||
f"{env.get('HERMES_HOME')!r}"
|
||||
)
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Tests for the /codex-runtime slash-command shared logic.
|
||||
|
||||
These cover the pure-Python state machine; CLI and gateway handlers are
|
||||
tested separately because they involve config persistence and prompt
|
||||
formatting that's surface-specific."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli import codex_runtime_switch as crs
|
||||
|
||||
|
||||
class TestParseArgs:
|
||||
@pytest.mark.parametrize("arg,expected", [
|
||||
("", None),
|
||||
(" ", None),
|
||||
("auto", "auto"),
|
||||
("codex_app_server", "codex_app_server"),
|
||||
("on", "codex_app_server"),
|
||||
("off", "auto"),
|
||||
("codex", "codex_app_server"),
|
||||
("default", "auto"),
|
||||
("hermes", "auto"),
|
||||
("ENABLE", "codex_app_server"), # case-insensitive
|
||||
("DiSaBlE", "auto"),
|
||||
])
|
||||
def test_valid_args(self, arg, expected):
|
||||
value, errors = crs.parse_args(arg)
|
||||
assert errors == []
|
||||
assert value == expected
|
||||
|
||||
def test_invalid_arg_returns_error(self):
|
||||
value, errors = crs.parse_args("turbo")
|
||||
assert value is None
|
||||
assert errors and "Unknown runtime" in errors[0]
|
||||
|
||||
|
||||
class TestGetCurrentRuntime:
|
||||
def test_default_when_unset(self):
|
||||
assert crs.get_current_runtime({}) == "auto"
|
||||
assert crs.get_current_runtime({"model": {}}) == "auto"
|
||||
assert crs.get_current_runtime({"model": {"openai_runtime": ""}}) == "auto"
|
||||
|
||||
def test_unrecognized_falls_back_to_auto(self):
|
||||
assert crs.get_current_runtime(
|
||||
{"model": {"openai_runtime": "garbage"}}
|
||||
) == "auto"
|
||||
|
||||
def test_explicit_codex(self):
|
||||
assert crs.get_current_runtime(
|
||||
{"model": {"openai_runtime": "codex_app_server"}}
|
||||
) == "codex_app_server"
|
||||
|
||||
def test_handles_non_dict_config(self):
|
||||
assert crs.get_current_runtime(None) == "auto" # type: ignore[arg-type]
|
||||
assert crs.get_current_runtime("notadict") == "auto" # type: ignore[arg-type]
|
||||
assert crs.get_current_runtime({"model": "notadict"}) == "auto"
|
||||
|
||||
|
||||
class TestSetRuntime:
|
||||
def test_creates_model_section_if_missing(self):
|
||||
cfg = {}
|
||||
old = crs.set_runtime(cfg, "codex_app_server")
|
||||
assert old == "auto"
|
||||
assert cfg["model"]["openai_runtime"] == "codex_app_server"
|
||||
|
||||
def test_returns_previous_value(self):
|
||||
cfg = {"model": {"openai_runtime": "codex_app_server"}}
|
||||
old = crs.set_runtime(cfg, "auto")
|
||||
assert old == "codex_app_server"
|
||||
assert cfg["model"]["openai_runtime"] == "auto"
|
||||
|
||||
def test_invalid_value_raises(self):
|
||||
with pytest.raises(ValueError):
|
||||
crs.set_runtime({}, "garbage")
|
||||
|
||||
|
||||
class TestApply:
|
||||
def test_read_only_call_reports_state(self):
|
||||
cfg = {"model": {"openai_runtime": "codex_app_server"}}
|
||||
with patch.object(crs, "check_codex_binary_ok",
|
||||
return_value=(True, "0.130.0")):
|
||||
r = crs.apply(cfg, None)
|
||||
assert r.success
|
||||
assert r.new_value == "codex_app_server"
|
||||
assert r.old_value == "codex_app_server"
|
||||
assert "codex_app_server" in r.message
|
||||
assert "0.130.0" in r.message
|
||||
|
||||
def test_no_change_when_already_set(self):
|
||||
cfg = {"model": {"openai_runtime": "auto"}}
|
||||
r = crs.apply(cfg, "auto")
|
||||
assert r.success
|
||||
assert r.message == "openai_runtime already set to auto"
|
||||
|
||||
def test_enable_blocked_when_codex_missing(self):
|
||||
cfg = {}
|
||||
with patch.object(crs, "check_codex_binary_ok",
|
||||
return_value=(False, "codex not found")):
|
||||
r = crs.apply(cfg, "codex_app_server")
|
||||
assert r.success is False
|
||||
assert "Cannot enable" in r.message
|
||||
assert "npm i -g @openai/codex" in r.message
|
||||
# Config NOT mutated on failure
|
||||
assert cfg.get("model", {}).get("openai_runtime") in (None, "")
|
||||
|
||||
def test_enable_succeeds_when_codex_present(self):
|
||||
cfg = {}
|
||||
persisted = {}
|
||||
|
||||
def persist(c):
|
||||
persisted.update(c)
|
||||
|
||||
# Patch migrate so this test doesn't reach into the user's real
|
||||
# ~/.codex/config.toml. See issue #26250 Bug C — without this patch,
|
||||
# crs.apply() invokes the real migrate() which writes to
|
||||
# Path.home() / ".codex" using whatever HERMES_HOME the running pytest
|
||||
# session has set, leaking pytest tempdir paths into the user's
|
||||
# codex config.
|
||||
with patch.object(crs, "check_codex_binary_ok",
|
||||
return_value=(True, "0.130.0")), \
|
||||
patch("hermes_cli.codex_runtime_plugin_migration.migrate"):
|
||||
r = crs.apply(cfg, "codex_app_server", persist_callback=persist)
|
||||
assert r.success
|
||||
assert r.new_value == "codex_app_server"
|
||||
assert r.old_value == "auto"
|
||||
assert r.requires_new_session is True
|
||||
assert "via MCP" in r.message # hermes-tools callback message
|
||||
assert cfg["model"]["openai_runtime"] == "codex_app_server"
|
||||
assert persisted["model"]["openai_runtime"] == "codex_app_server"
|
||||
|
||||
def test_disable_does_not_check_binary(self):
|
||||
cfg = {"model": {"openai_runtime": "codex_app_server"}}
|
||||
with patch.object(crs, "check_codex_binary_ok") as bin_check:
|
||||
r = crs.apply(cfg, "auto")
|
||||
assert r.success
|
||||
# Binary check is irrelevant when disabling — should not be called
|
||||
# with the codex_app_server enable-gate signature.
|
||||
assert r.new_value == "auto"
|
||||
assert r.old_value == "codex_app_server"
|
||||
|
||||
def test_persist_callback_failure_reported(self):
|
||||
cfg = {}
|
||||
|
||||
def persist_boom(c):
|
||||
raise IOError("disk full")
|
||||
|
||||
with patch.object(crs, "check_codex_binary_ok",
|
||||
return_value=(True, "0.130.0")):
|
||||
r = crs.apply(cfg, "codex_app_server", persist_callback=persist_boom)
|
||||
assert r.success is False
|
||||
assert "persist failed" in r.message
|
||||
assert "disk full" in r.message
|
||||
|
||||
def test_enable_triggers_mcp_migration(self):
|
||||
"""Enabling codex_app_server should auto-migrate Hermes mcp_servers
|
||||
to ~/.codex/config.toml so the spawned subprocess sees them."""
|
||||
cfg = {
|
||||
"mcp_servers": {
|
||||
"filesystem": {"command": "npx", "args": ["-y", "fs-server"]},
|
||||
}
|
||||
}
|
||||
|
||||
with patch.object(crs, "check_codex_binary_ok",
|
||||
return_value=(True, "0.130.0")), \
|
||||
patch("hermes_cli.codex_runtime_plugin_migration.migrate") as mig:
|
||||
mig.return_value.migrated = ["filesystem", "hermes-tools"]
|
||||
mig.return_value.migrated_plugins = []
|
||||
mig.return_value.plugin_query_error = None
|
||||
mig.return_value.wrote_permissions_default = ":workspace"
|
||||
mig.return_value.errors = []
|
||||
mig.return_value.target_path = "/fake/.codex/config.toml"
|
||||
r = crs.apply(cfg, "codex_app_server")
|
||||
assert r.success
|
||||
assert mig.called # migration was triggered
|
||||
# User MCP servers are reported (excluding internal hermes-tools)
|
||||
assert "Migrated 1 MCP server" in r.message
|
||||
assert "filesystem" in r.message
|
||||
# Permissions default surfaces
|
||||
assert "Default sandbox: :workspace" in r.message
|
||||
# Hermes tool callback announcement
|
||||
assert "via MCP" in r.message
|
||||
|
||||
def test_disable_does_not_trigger_migration(self):
|
||||
"""Switching back to auto must not write to ~/.codex/."""
|
||||
cfg = {
|
||||
"model": {"openai_runtime": "codex_app_server"},
|
||||
"mcp_servers": {"x": {"command": "y"}},
|
||||
}
|
||||
with patch("hermes_cli.codex_runtime_plugin_migration.migrate") as mig:
|
||||
r = crs.apply(cfg, "auto")
|
||||
assert r.success
|
||||
assert not mig.called # disabling does not migrate
|
||||
|
||||
def test_migration_failure_does_not_block_enable(self):
|
||||
"""If MCP migration raises, the runtime change still proceeds —
|
||||
users can manually re-run migration later."""
|
||||
cfg = {"mcp_servers": {"x": {"command": "y"}}}
|
||||
with patch.object(crs, "check_codex_binary_ok",
|
||||
return_value=(True, "0.130.0")), \
|
||||
patch("hermes_cli.codex_runtime_plugin_migration.migrate",
|
||||
side_effect=RuntimeError("disk full")):
|
||||
r = crs.apply(cfg, "codex_app_server")
|
||||
assert r.success # change still applied
|
||||
assert r.new_value == "codex_app_server"
|
||||
assert "MCP migration skipped" in r.message
|
||||
assert "disk full" in r.message
|
||||
|
||||
def test_binary_check_cached_within_apply(self):
|
||||
"""check_codex_binary_ok is invoked at most once per apply() call.
|
||||
|
||||
The enable path has three sites that need the version (state report,
|
||||
enable gate, success message). Without caching, a single
|
||||
/codex-runtime invocation spawns `codex --version` three times.
|
||||
Regression guard against a refactor that drops the cache.
|
||||
"""
|
||||
cfg = {}
|
||||
with patch.object(crs, "check_codex_binary_ok",
|
||||
return_value=(True, "0.130.0")) as bin_check, \
|
||||
patch("hermes_cli.codex_runtime_plugin_migration.migrate"):
|
||||
r = crs.apply(cfg, "codex_app_server")
|
||||
assert r.success
|
||||
assert bin_check.call_count == 1, (
|
||||
f"check_codex_binary_ok was called {bin_check.call_count} time(s); "
|
||||
"should be cached and called exactly once per apply()"
|
||||
)
|
||||
|
||||
def test_binary_check_cached_on_read_only_call(self):
|
||||
"""Read-only call (new_value=None) calls the binary check exactly
|
||||
once and reuses the result for the message."""
|
||||
cfg = {"model": {"openai_runtime": "codex_app_server"}}
|
||||
with patch.object(crs, "check_codex_binary_ok",
|
||||
return_value=(True, "0.130.0")) as bin_check:
|
||||
crs.apply(cfg, None)
|
||||
assert bin_check.call_count == 1
|
||||
@@ -140,6 +140,54 @@ class TestGenerateZsh:
|
||||
# gateway has subcommands so a _cmds array must be generated
|
||||
assert "gateway_cmds" in out
|
||||
|
||||
def test_registers_compdef_instead_of_invoking_completion_function(self):
|
||||
out = generate_zsh(_make_parser())
|
||||
assert 'compdef _hermes hermes' in out
|
||||
assert '_hermes "$@"' not in out
|
||||
|
||||
def test_preserves_valid_zsh_arguments_alias_syntax(self):
|
||||
out = generate_zsh(_make_parser())
|
||||
assert "'(-)'{-h,--help}'[Show help and exit]'" in out
|
||||
assert "'(-)'{-V,--version}'[Show version and exit]'" in out
|
||||
assert "'(-)'{-p,--profile}'[Profile name]:profile:_hermes_profiles'" in out
|
||||
assert "'(-h --help){-h,--help}[Show help and exit]'" not in out
|
||||
assert '"(-h --help)"{-h,--help}"[Show help and exit]"' not in out
|
||||
|
||||
def test_valid_zsh_syntax(self):
|
||||
if not shutil.which("zsh"):
|
||||
pytest.skip("zsh not installed")
|
||||
out = generate_zsh(_make_parser())
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".zsh", delete=False) as f:
|
||||
f.write(out)
|
||||
path = f.name
|
||||
try:
|
||||
result = subprocess.run(["zsh", "-n", path], capture_output=True, text=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_zsh_eval_style_source_registers_after_compinit(self):
|
||||
if not shutil.which("zsh"):
|
||||
pytest.skip("zsh not installed")
|
||||
out = generate_zsh(_make_parser())
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".zsh", delete=False) as f:
|
||||
f.write(out)
|
||||
path = f.name
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"zsh",
|
||||
"-fc",
|
||||
f"autoload -Uz compinit && compinit -D; source {path}; [[ ${{_comps[hermes]}} == _hermes ]]",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert result.stderr == ""
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Fish output
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Tests for the load_env() process-level cache.
|
||||
|
||||
The cache exists to keep `hermes tools` → "All Platforms" fast: every
|
||||
`get_env_value()` lookup used to re-read and re-sanitise the entire
|
||||
.env file, racking up hundreds of ms across one menu render. The
|
||||
cache is keyed on (path, mtime, size); writers (save_env_value /
|
||||
remove_env_value / sanitise_env_file) call invalidate_env_cache().
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
def _write_env(path: Path, contents: str) -> None:
|
||||
path.write_text(contents, encoding="utf-8")
|
||||
|
||||
|
||||
def test_load_env_caches_on_repeat_calls():
|
||||
"""Repeated load_env() calls on the same file return the cached dict."""
|
||||
from hermes_cli.config import invalidate_env_cache, load_env
|
||||
|
||||
invalidate_env_cache()
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".env", delete=False, encoding="utf-8"
|
||||
) as f:
|
||||
f.write("OPENAI_API_KEY=sk-first\n")
|
||||
env_path = Path(f.name)
|
||||
|
||||
try:
|
||||
with patch("hermes_cli.config.get_env_path", return_value=env_path):
|
||||
first = load_env()
|
||||
# Even if a writer outside our cache mutates the file, an
|
||||
# mtime/size match means the cache still wins. We simulate that
|
||||
# by writing identical bytes back — sanity check that the cache
|
||||
# is keyed structurally, not on a counter.
|
||||
second = load_env()
|
||||
|
||||
assert first == second
|
||||
assert first.get("OPENAI_API_KEY") == "sk-first"
|
||||
finally:
|
||||
env_path.unlink(missing_ok=True)
|
||||
invalidate_env_cache()
|
||||
|
||||
|
||||
def test_load_env_invalidates_on_mtime_bump():
|
||||
"""Editing the file (mtime changes) invalidates the cache."""
|
||||
from hermes_cli.config import invalidate_env_cache, load_env
|
||||
|
||||
invalidate_env_cache()
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".env", delete=False, encoding="utf-8"
|
||||
) as f:
|
||||
f.write("OPENAI_API_KEY=sk-old\n")
|
||||
env_path = Path(f.name)
|
||||
|
||||
try:
|
||||
with patch("hermes_cli.config.get_env_path", return_value=env_path):
|
||||
first = load_env()
|
||||
assert first.get("OPENAI_API_KEY") == "sk-old"
|
||||
|
||||
# Rewrite file with new contents and bump mtime to make sure
|
||||
# the FS records the change even on coarse-mtime filesystems.
|
||||
_write_env(env_path, "OPENAI_API_KEY=sk-new\n")
|
||||
future = env_path.stat().st_mtime + 5.0
|
||||
os.utime(env_path, (future, future))
|
||||
|
||||
second = load_env()
|
||||
assert second.get("OPENAI_API_KEY") == "sk-new", (
|
||||
"load_env() returned stale value after file change"
|
||||
)
|
||||
finally:
|
||||
env_path.unlink(missing_ok=True)
|
||||
invalidate_env_cache()
|
||||
|
||||
|
||||
def test_invalidate_env_cache_forces_reread():
|
||||
"""invalidate_env_cache() forces the next load_env() to hit the disk.
|
||||
|
||||
This is the belt-and-braces knob for writers (save_env_value, etc.)
|
||||
on filesystems where mtime resolution might miss a same-second write.
|
||||
"""
|
||||
from hermes_cli.config import invalidate_env_cache, load_env
|
||||
|
||||
invalidate_env_cache()
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".env", delete=False, encoding="utf-8"
|
||||
) as f:
|
||||
f.write("OPENAI_API_KEY=sk-old\n")
|
||||
env_path = Path(f.name)
|
||||
|
||||
try:
|
||||
with patch("hermes_cli.config.get_env_path", return_value=env_path):
|
||||
assert load_env().get("OPENAI_API_KEY") == "sk-old"
|
||||
|
||||
# Rewrite WITHOUT bumping mtime — simulates same-second write.
|
||||
mtime_before = env_path.stat().st_mtime
|
||||
_write_env(env_path, "OPENAI_API_KEY=sk-new\n")
|
||||
os.utime(env_path, (mtime_before, mtime_before))
|
||||
|
||||
# Without invalidation, cache hit might return stale.
|
||||
invalidate_env_cache()
|
||||
|
||||
assert load_env().get("OPENAI_API_KEY") == "sk-new"
|
||||
finally:
|
||||
env_path.unlink(missing_ok=True)
|
||||
invalidate_env_cache()
|
||||
|
||||
|
||||
def test_save_env_value_invalidates_cache(tmp_path, monkeypatch):
|
||||
"""save_env_value() invalidates the cache so subsequent reads see the update."""
|
||||
from hermes_cli import config as config_mod
|
||||
from hermes_cli.config import invalidate_env_cache, load_env, save_env_value
|
||||
|
||||
invalidate_env_cache()
|
||||
|
||||
env_path = tmp_path / ".env"
|
||||
env_path.write_text("EXISTING_KEY=old\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(config_mod, "get_env_path", lambda: env_path)
|
||||
monkeypatch.setattr(config_mod, "ensure_hermes_home", lambda: None)
|
||||
monkeypatch.setattr(config_mod, "_secure_file", lambda _p: None)
|
||||
monkeypatch.setattr(config_mod, "is_managed", lambda: False)
|
||||
|
||||
try:
|
||||
# Prime the cache.
|
||||
first = load_env()
|
||||
assert first.get("EXISTING_KEY") == "old"
|
||||
|
||||
save_env_value("NEW_KEY", "shiny")
|
||||
|
||||
# Same-second writes on coarse-mtime filesystems would normally
|
||||
# let stale cache survive; invalidate_env_cache() inside the
|
||||
# writer makes the next read see the new key.
|
||||
result = load_env()
|
||||
assert result.get("NEW_KEY") == "shiny"
|
||||
assert result.get("EXISTING_KEY") == "old"
|
||||
finally:
|
||||
monkeypatch.delenv("NEW_KEY", raising=False)
|
||||
invalidate_env_cache()
|
||||
|
||||
|
||||
def test_remove_env_value_invalidates_cache(tmp_path, monkeypatch):
|
||||
"""remove_env_value() invalidates the cache so the removed key disappears."""
|
||||
from hermes_cli import config as config_mod
|
||||
from hermes_cli.config import (
|
||||
invalidate_env_cache,
|
||||
load_env,
|
||||
remove_env_value,
|
||||
save_env_value,
|
||||
)
|
||||
|
||||
invalidate_env_cache()
|
||||
|
||||
env_path = tmp_path / ".env"
|
||||
monkeypatch.setattr(config_mod, "get_env_path", lambda: env_path)
|
||||
monkeypatch.setattr(config_mod, "ensure_hermes_home", lambda: None)
|
||||
monkeypatch.setattr(config_mod, "_secure_file", lambda _p: None)
|
||||
monkeypatch.setattr(config_mod, "is_managed", lambda: False)
|
||||
|
||||
save_env_value("DOOMED_KEY", "value")
|
||||
assert load_env().get("DOOMED_KEY") == "value"
|
||||
|
||||
try:
|
||||
removed = remove_env_value("DOOMED_KEY")
|
||||
assert removed is True
|
||||
assert "DOOMED_KEY" not in load_env()
|
||||
finally:
|
||||
monkeypatch.delenv("DOOMED_KEY", raising=False)
|
||||
invalidate_env_cache()
|
||||
|
||||
|
||||
def test_load_env_handles_missing_file():
|
||||
"""A nonexistent .env returns {} and caches the empty result."""
|
||||
from hermes_cli.config import invalidate_env_cache, load_env
|
||||
|
||||
invalidate_env_cache()
|
||||
|
||||
nonexistent = Path(tempfile.gettempdir()) / "hermes-test-no-such-env-xyz123.env"
|
||||
nonexistent.unlink(missing_ok=True)
|
||||
|
||||
try:
|
||||
with patch("hermes_cli.config.get_env_path", return_value=nonexistent):
|
||||
assert load_env() == {}
|
||||
assert load_env() == {} # cached
|
||||
finally:
|
||||
invalidate_env_cache()
|
||||
@@ -514,3 +514,227 @@ class TestJudgeParseFailureAutoPause:
|
||||
reloaded = load_goal("parse-fail-sid-4")
|
||||
assert reloaded is not None
|
||||
assert reloaded.consecutive_parse_failures == 2
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# /subgoal — user-added criteria
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGoalStateSubgoalsBackcompat:
|
||||
def test_old_state_meta_row_loads_without_subgoals(self):
|
||||
"""A goal serialized BEFORE the subgoals field existed must
|
||||
round-trip with an empty list, not crash."""
|
||||
import json
|
||||
from hermes_cli.goals import GoalState
|
||||
|
||||
legacy = json.dumps({
|
||||
"goal": "do a thing",
|
||||
"status": "active",
|
||||
"turns_used": 2,
|
||||
"max_turns": 20,
|
||||
"created_at": 1.0,
|
||||
"last_turn_at": 2.0,
|
||||
"consecutive_parse_failures": 0,
|
||||
})
|
||||
state = GoalState.from_json(legacy)
|
||||
assert state.goal == "do a thing"
|
||||
assert state.subgoals == []
|
||||
|
||||
def test_subgoals_round_trip(self):
|
||||
from hermes_cli.goals import GoalState
|
||||
state = GoalState(goal="g", subgoals=["a", "b", "c"])
|
||||
rt = GoalState.from_json(state.to_json())
|
||||
assert rt.subgoals == ["a", "b", "c"]
|
||||
|
||||
|
||||
class TestGoalManagerSubgoals:
|
||||
def test_add_subgoal(self, hermes_home):
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sub-add")
|
||||
mgr.set("main goal")
|
||||
text = mgr.add_subgoal(" use bullet points ")
|
||||
assert text == "use bullet points"
|
||||
assert mgr.state.subgoals == ["use bullet points"]
|
||||
|
||||
def test_add_subgoal_requires_active_goal(self, hermes_home):
|
||||
import pytest
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sub-noactive")
|
||||
with pytest.raises(RuntimeError):
|
||||
mgr.add_subgoal("oops")
|
||||
|
||||
def test_add_empty_subgoal_rejected(self, hermes_home):
|
||||
import pytest
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sub-empty")
|
||||
mgr.set("g")
|
||||
with pytest.raises(ValueError):
|
||||
mgr.add_subgoal(" ")
|
||||
|
||||
def test_remove_subgoal(self, hermes_home):
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sub-remove")
|
||||
mgr.set("g")
|
||||
mgr.add_subgoal("first")
|
||||
mgr.add_subgoal("second")
|
||||
mgr.add_subgoal("third")
|
||||
removed = mgr.remove_subgoal(2)
|
||||
assert removed == "second"
|
||||
assert mgr.state.subgoals == ["first", "third"]
|
||||
|
||||
def test_remove_subgoal_out_of_range(self, hermes_home):
|
||||
import pytest
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sub-oob")
|
||||
mgr.set("g")
|
||||
mgr.add_subgoal("only")
|
||||
with pytest.raises(IndexError):
|
||||
mgr.remove_subgoal(5)
|
||||
with pytest.raises(IndexError):
|
||||
mgr.remove_subgoal(0)
|
||||
|
||||
def test_clear_subgoals(self, hermes_home):
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sub-clear")
|
||||
mgr.set("g")
|
||||
mgr.add_subgoal("a")
|
||||
mgr.add_subgoal("b")
|
||||
prev = mgr.clear_subgoals()
|
||||
assert prev == 2
|
||||
assert mgr.state.subgoals == []
|
||||
|
||||
def test_subgoals_persist_across_reloads(self, hermes_home):
|
||||
"""Subgoals stored in SessionDB survive a fresh GoalManager."""
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sub-persist")
|
||||
mgr.set("g")
|
||||
mgr.add_subgoal("first")
|
||||
mgr.add_subgoal("second")
|
||||
|
||||
mgr2 = GoalManager(session_id="sub-persist")
|
||||
assert mgr2.state.subgoals == ["first", "second"]
|
||||
|
||||
|
||||
class TestContinuationPromptWithSubgoals:
|
||||
def test_empty_subgoals_uses_original_template(self, hermes_home):
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="cp-empty")
|
||||
mgr.set("ship the feature")
|
||||
prompt = mgr.next_continuation_prompt()
|
||||
assert prompt is not None
|
||||
assert "ship the feature" in prompt
|
||||
assert "Additional criteria" not in prompt
|
||||
|
||||
def test_with_subgoals_includes_them(self, hermes_home):
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="cp-with")
|
||||
mgr.set("ship the feature")
|
||||
mgr.add_subgoal("write tests")
|
||||
mgr.add_subgoal("update docs")
|
||||
prompt = mgr.next_continuation_prompt()
|
||||
assert prompt is not None
|
||||
assert "ship the feature" in prompt
|
||||
assert "Additional criteria" in prompt
|
||||
assert "1. write tests" in prompt
|
||||
assert "2. update docs" in prompt
|
||||
|
||||
|
||||
class TestJudgeGoalWithSubgoals:
|
||||
def test_judge_uses_subgoals_template_when_provided(self, hermes_home):
|
||||
"""judge_goal switches templates when subgoals is non-empty.
|
||||
|
||||
We don't actually call the model — we patch the aux client to
|
||||
capture the prompt that would be sent.
|
||||
"""
|
||||
from unittest.mock import patch, MagicMock
|
||||
from hermes_cli import goals
|
||||
|
||||
captured = {}
|
||||
|
||||
class _FakeMsg:
|
||||
content = '{"done": true, "reason": "all done"}'
|
||||
class _FakeChoice:
|
||||
message = _FakeMsg()
|
||||
class _FakeResp:
|
||||
choices = [_FakeChoice()]
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return _FakeResp()
|
||||
|
||||
with patch.object(goals, "get_text_auxiliary_client",
|
||||
return_value=(_FakeClient, "fake-model"), create=True), \
|
||||
patch.object(goals, "get_auxiliary_extra_body",
|
||||
return_value=None, create=True), \
|
||||
patch("agent.auxiliary_client.get_text_auxiliary_client",
|
||||
return_value=(_FakeClient, "fake-model")), \
|
||||
patch("agent.auxiliary_client.get_auxiliary_extra_body",
|
||||
return_value=None):
|
||||
verdict, reason, parse_failed = goals.judge_goal(
|
||||
"ship the feature",
|
||||
"ok shipped",
|
||||
subgoals=["write tests", "update docs"],
|
||||
)
|
||||
|
||||
# The aux client was called with a prompt that includes the subgoals.
|
||||
sent_messages = captured.get("messages") or []
|
||||
user_msg = next((m["content"] for m in sent_messages if m["role"] == "user"), "")
|
||||
assert "Additional criteria" in user_msg
|
||||
assert "1. write tests" in user_msg
|
||||
assert "2. update docs" in user_msg
|
||||
assert "every additional criterion" in user_msg
|
||||
assert verdict == "done"
|
||||
|
||||
def test_judge_uses_original_template_when_no_subgoals(self, hermes_home):
|
||||
from unittest.mock import patch
|
||||
from hermes_cli import goals
|
||||
|
||||
captured = {}
|
||||
|
||||
class _FakeMsg:
|
||||
content = '{"done": true, "reason": "ok"}'
|
||||
class _FakeChoice:
|
||||
message = _FakeMsg()
|
||||
class _FakeResp:
|
||||
choices = [_FakeChoice()]
|
||||
class _FakeClient:
|
||||
class chat:
|
||||
class completions:
|
||||
@staticmethod
|
||||
def create(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return _FakeResp()
|
||||
|
||||
with patch("agent.auxiliary_client.get_text_auxiliary_client",
|
||||
return_value=(_FakeClient, "fake-model")), \
|
||||
patch("agent.auxiliary_client.get_auxiliary_extra_body",
|
||||
return_value=None):
|
||||
goals.judge_goal("ship it", "done", subgoals=None)
|
||||
|
||||
sent_messages = captured.get("messages") or []
|
||||
user_msg = next((m["content"] for m in sent_messages if m["role"] == "user"), "")
|
||||
assert "Additional criteria" not in user_msg
|
||||
assert "ship it" in user_msg
|
||||
|
||||
|
||||
class TestStatusLineSubgoalCount:
|
||||
def test_status_line_no_subgoals(self, hermes_home):
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sl-empty")
|
||||
mgr.set("ship it")
|
||||
line = mgr.status_line()
|
||||
assert "ship it" in line
|
||||
assert "subgoal" not in line.lower()
|
||||
|
||||
def test_status_line_with_subgoals(self, hermes_home):
|
||||
from hermes_cli.goals import GoalManager
|
||||
mgr = GoalManager(session_id="sl-with")
|
||||
mgr.set("ship it")
|
||||
mgr.add_subgoal("a")
|
||||
mgr.add_subgoal("b")
|
||||
line = mgr.status_line()
|
||||
assert "2 subgoals" in line
|
||||
|
||||
@@ -103,6 +103,33 @@ class TestPluginPickerInjection:
|
||||
visible = tools_config._visible_providers(browser, {})
|
||||
assert all(p.get("image_gen_plugin_name") is None for p in visible)
|
||||
|
||||
def test_post_setup_propagated_when_declared(self, monkeypatch):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
image_gen_registry.register_provider(_FakeProvider(
|
||||
"xai_img",
|
||||
schema={
|
||||
"name": "xAI Grok Imagine",
|
||||
"badge": "paid",
|
||||
"tag": "grok image",
|
||||
"env_vars": [],
|
||||
"post_setup": "xai_grok",
|
||||
},
|
||||
))
|
||||
|
||||
rows = tools_config._plugin_image_gen_providers()
|
||||
match = next(r for r in rows if r.get("image_gen_plugin_name") == "xai_img")
|
||||
assert match["post_setup"] == "xai_grok"
|
||||
|
||||
def test_post_setup_omitted_when_not_declared(self, monkeypatch):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
image_gen_registry.register_provider(_FakeProvider("plain_img"))
|
||||
|
||||
rows = tools_config._plugin_image_gen_providers()
|
||||
match = next(r for r in rows if r.get("image_gen_plugin_name") == "plain_img")
|
||||
assert "post_setup" not in match
|
||||
|
||||
|
||||
class TestPluginCatalog:
|
||||
def test_plugin_catalog_returns_models(self):
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
"""Behavior tests for hermes_cli.inventory.
|
||||
|
||||
Locks the invariants the three migrated consumers (web_server.py
|
||||
/api/model/options, tui_gateway model.options, tui_gateway model.save_key)
|
||||
depend on:
|
||||
|
||||
- load_picker_context() reproduces the inline 17-LOC config-slice exactly.
|
||||
- with_overrides() is truthy-only (empty agent attrs must not clobber).
|
||||
- build_models_payload() returns a stable {providers, model, provider}
|
||||
shape and delegates curation to list_authenticated_providers (does not
|
||||
call provider_model_ids per row).
|
||||
- canonical_order keys on slug membership, not is_user_defined — section
|
||||
3 of list_authenticated_providers sets is_user_defined=True for
|
||||
canonical slugs in the providers: dict, and that flag must NOT demote
|
||||
them to the tail.
|
||||
- picker_hints adds authenticated/auth_type/key_env/warning per row,
|
||||
matching the TUI ModelPickerDialog shape.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.inventory import (
|
||||
ConfigContext,
|
||||
build_models_payload,
|
||||
load_picker_context,
|
||||
)
|
||||
|
||||
|
||||
# ─── load_picker_context ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def _cfg(model=None, providers=None, custom_providers=None) -> dict:
|
||||
return {
|
||||
"model": model if model is not None else {},
|
||||
"providers": providers if providers is not None else {},
|
||||
"custom_providers": custom_providers if custom_providers is not None else [],
|
||||
}
|
||||
|
||||
|
||||
def test_load_picker_context_full_dict():
|
||||
cfg = _cfg(
|
||||
model={
|
||||
"default": "anthropic/claude-sonnet-4.6",
|
||||
"provider": "openrouter",
|
||||
"base_url": "https://openrouter.ai/api/v1",
|
||||
},
|
||||
providers={"openrouter": {}},
|
||||
custom_providers=[{"name": "Ollama", "base_url": "http://localhost:11434/v1"}],
|
||||
)
|
||||
with patch("hermes_cli.config.load_config", return_value=cfg):
|
||||
ctx = load_picker_context()
|
||||
assert ctx.current_model == "anthropic/claude-sonnet-4.6"
|
||||
assert ctx.current_provider == "openrouter"
|
||||
assert ctx.current_base_url == "https://openrouter.ai/api/v1"
|
||||
assert "openrouter" in ctx.user_providers
|
||||
# custom_providers comes from get_compatible_custom_providers, which
|
||||
# merges legacy list + v12+ keyed providers — both present here means
|
||||
# at least one row.
|
||||
assert isinstance(ctx.custom_providers, list)
|
||||
|
||||
|
||||
def test_load_picker_context_falls_back_to_name_when_default_missing():
|
||||
cfg = _cfg(model={"name": "gpt-5.4", "provider": "openai"})
|
||||
with patch("hermes_cli.config.load_config", return_value=cfg):
|
||||
ctx = load_picker_context()
|
||||
assert ctx.current_model == "gpt-5.4"
|
||||
assert ctx.current_provider == "openai"
|
||||
|
||||
|
||||
def test_load_picker_context_string_model_legacy_shape():
|
||||
"""config.model can be a bare string in older configs."""
|
||||
cfg = {"model": "some-model", "providers": {}, "custom_providers": []}
|
||||
with patch("hermes_cli.config.load_config", return_value=cfg):
|
||||
ctx = load_picker_context()
|
||||
assert ctx.current_model == "some-model"
|
||||
assert ctx.current_provider == ""
|
||||
assert ctx.current_base_url == ""
|
||||
|
||||
|
||||
def test_load_picker_context_empty_config():
|
||||
cfg = _cfg()
|
||||
with patch("hermes_cli.config.load_config", return_value=cfg):
|
||||
ctx = load_picker_context()
|
||||
assert ctx.current_provider == ""
|
||||
assert ctx.current_model == ""
|
||||
assert ctx.current_base_url == ""
|
||||
assert ctx.user_providers == {}
|
||||
assert ctx.custom_providers == []
|
||||
|
||||
|
||||
# ─── with_overrides ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _empty_ctx(provider="orig", model="orig-model", base_url="orig-url"):
|
||||
return ConfigContext(
|
||||
current_provider=provider,
|
||||
current_model=model,
|
||||
current_base_url=base_url,
|
||||
user_providers={},
|
||||
custom_providers=[],
|
||||
)
|
||||
|
||||
|
||||
def test_with_overrides_truthy_only_strings():
|
||||
"""Empty strings must NOT clobber disk config — TUI calls this with
|
||||
empty getattr(agent, 'provider', '') when no agent is spawned yet."""
|
||||
ctx = _empty_ctx()
|
||||
overlaid = ctx.with_overrides(
|
||||
current_provider="",
|
||||
current_model="",
|
||||
current_base_url="",
|
||||
)
|
||||
assert overlaid.current_provider == "orig"
|
||||
assert overlaid.current_model == "orig-model"
|
||||
assert overlaid.current_base_url == "orig-url"
|
||||
|
||||
|
||||
def test_with_overrides_truthy_value_replaces():
|
||||
ctx = _empty_ctx()
|
||||
overlaid = ctx.with_overrides(current_provider="anthropic")
|
||||
assert overlaid.current_provider == "anthropic"
|
||||
assert overlaid.current_model == "orig-model" # untouched
|
||||
|
||||
|
||||
def test_with_overrides_no_args_returns_self_or_equivalent():
|
||||
ctx = _empty_ctx()
|
||||
assert ctx.with_overrides() == ctx
|
||||
|
||||
|
||||
# ─── build_models_payload ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def _list_auth_returning(rows: list[dict]):
|
||||
"""Patch list_authenticated_providers to return a fixed row list."""
|
||||
return patch(
|
||||
"hermes_cli.model_switch.list_authenticated_providers",
|
||||
return_value=rows,
|
||||
)
|
||||
|
||||
|
||||
def test_build_models_payload_returns_expected_shape():
|
||||
rows = [
|
||||
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
|
||||
"total_models": 1, "is_current": True, "is_user_defined": False,
|
||||
"source": "built-in"},
|
||||
]
|
||||
ctx = _empty_ctx(provider="openrouter", model="m1", base_url="")
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(ctx)
|
||||
assert set(payload.keys()) == {"providers", "model", "provider"}
|
||||
assert payload["model"] == "m1"
|
||||
assert payload["provider"] == "openrouter"
|
||||
assert payload["providers"] == rows
|
||||
|
||||
|
||||
def test_build_models_payload_does_not_call_provider_model_ids():
|
||||
"""Curated lists must come from list_authenticated_providers, not
|
||||
provider_model_ids — that would pull TTS/embeddings/etc.
|
||||
"""
|
||||
rows = [{"slug": "nous", "name": "Nous", "models": ["hermes-4-405b"],
|
||||
"total_models": 1, "is_current": False, "is_user_defined": False,
|
||||
"source": "built-in"}]
|
||||
ctx = _empty_ctx()
|
||||
with _list_auth_returning(rows), \
|
||||
patch("hermes_cli.models.provider_model_ids") as mock_pm:
|
||||
build_models_payload(ctx)
|
||||
mock_pm.assert_not_called()
|
||||
|
||||
|
||||
def test_include_unconfigured_appends_canonical_skeletons():
|
||||
"""include_unconfigured=True adds CANONICAL_PROVIDERS rows that
|
||||
list_authenticated_providers didn't emit. Skeleton rows have empty
|
||||
models and source='canonical'."""
|
||||
rows = [
|
||||
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
|
||||
"total_models": 1, "is_current": True, "is_user_defined": False,
|
||||
"source": "built-in"},
|
||||
]
|
||||
ctx = _empty_ctx(provider="openrouter")
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(ctx, include_unconfigured=True)
|
||||
# All canonical providers other than openrouter should appear as
|
||||
# skeleton rows.
|
||||
from hermes_cli.models import CANONICAL_PROVIDERS
|
||||
|
||||
seen_slugs = {r["slug"] for r in payload["providers"]}
|
||||
for entry in CANONICAL_PROVIDERS:
|
||||
assert entry.slug in seen_slugs, f"missing {entry.slug}"
|
||||
# Skeletons have empty models and source='canonical'.
|
||||
skeletons = [r for r in payload["providers"]
|
||||
if r.get("source") == "canonical"]
|
||||
assert all(r["models"] == [] for r in skeletons)
|
||||
assert all(r["total_models"] == 0 for r in skeletons)
|
||||
|
||||
|
||||
def test_include_unconfigured_skips_already_present_slugs():
|
||||
"""If list_authenticated_providers already returned a row for a
|
||||
canonical slug, include_unconfigured must NOT duplicate it."""
|
||||
rows = [
|
||||
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
|
||||
"total_models": 1, "is_current": True, "is_user_defined": False,
|
||||
"source": "built-in"},
|
||||
]
|
||||
ctx = _empty_ctx()
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(ctx, include_unconfigured=True)
|
||||
or_rows = [r for r in payload["providers"] if r["slug"] == "openrouter"]
|
||||
assert len(or_rows) == 1
|
||||
assert or_rows[0]["models"] == ["m1"] # the authenticated row, not skeleton
|
||||
|
||||
|
||||
# ─── picker_hints ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_picker_hints_marks_authed_rows_authenticated():
|
||||
rows = [
|
||||
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
|
||||
"total_models": 1, "is_current": True, "is_user_defined": False,
|
||||
"source": "built-in"},
|
||||
]
|
||||
ctx = _empty_ctx()
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(ctx, picker_hints=True)
|
||||
assert payload["providers"][0]["authenticated"] is True
|
||||
|
||||
|
||||
def test_picker_hints_adds_warning_to_skeleton_rows():
|
||||
"""Skeleton rows (unconfigured canonical providers) must carry the
|
||||
setup hint the picker UI displays."""
|
||||
rows = []
|
||||
ctx = _empty_ctx()
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(
|
||||
ctx, include_unconfigured=True, picker_hints=True,
|
||||
)
|
||||
skeleton_rows = [r for r in payload["providers"]
|
||||
if r.get("source") == "canonical"]
|
||||
assert skeleton_rows, "test setup: expected at least one skeleton row"
|
||||
for row in skeleton_rows:
|
||||
assert row["authenticated"] is False
|
||||
assert "auth_type" in row
|
||||
assert "warning" in row
|
||||
# api_key providers get "paste X to activate" / others get the
|
||||
# hermes model fallback.
|
||||
assert (
|
||||
row["warning"].startswith("paste ")
|
||||
or row["warning"].startswith("run `hermes model`")
|
||||
)
|
||||
|
||||
|
||||
def test_picker_hints_api_key_warning_format():
|
||||
"""For api_key providers with a defined env var, the warning must
|
||||
point to that env var."""
|
||||
rows = []
|
||||
ctx = _empty_ctx()
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(
|
||||
ctx, include_unconfigured=True, picker_hints=True,
|
||||
)
|
||||
# anthropic uses api_key + ANTHROPIC_API_KEY.
|
||||
anthropic = next(
|
||||
r for r in payload["providers"] if r["slug"] == "anthropic"
|
||||
)
|
||||
assert "ANTHROPIC_API_KEY" in anthropic["warning"]
|
||||
assert anthropic["warning"].startswith("paste ")
|
||||
|
||||
|
||||
# ─── canonical_order ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_canonical_order_uses_slug_not_is_user_defined_flag():
|
||||
"""Section 3 of list_authenticated_providers sets is_user_defined=True
|
||||
for canonical slugs that appear in the providers: config dict.
|
||||
canonical_order MUST key on slug membership, not the flag — otherwise
|
||||
canonical providers configured via the keyed schema get demoted to
|
||||
the tail.
|
||||
"""
|
||||
from hermes_cli.models import CANONICAL_PROVIDERS
|
||||
|
||||
canonical_slug = CANONICAL_PROVIDERS[2].slug # any canonical
|
||||
rows = [
|
||||
# A truly-custom row (correct: is_user_defined=True)
|
||||
{"slug": "custom:Ollama", "name": "Ollama", "models": [],
|
||||
"total_models": 0, "is_current": False, "is_user_defined": True,
|
||||
"source": "user-config"},
|
||||
# A canonical row that the substrate flagged as user-defined
|
||||
# because the user configured it via providers: dict.
|
||||
{"slug": canonical_slug, "name": "x", "models": ["m1"],
|
||||
"total_models": 1, "is_current": False, "is_user_defined": True,
|
||||
"source": "built-in"},
|
||||
]
|
||||
ctx = _empty_ctx()
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(ctx, canonical_order=True)
|
||||
slugs = [r["slug"] for r in payload["providers"]]
|
||||
# Canonical-slug row must come BEFORE truly-custom rows, regardless
|
||||
# of is_user_defined.
|
||||
canonical_idx = slugs.index(canonical_slug)
|
||||
custom_idx = slugs.index("custom:Ollama")
|
||||
assert canonical_idx < custom_idx, (
|
||||
f"canonical {canonical_slug} demoted to tail "
|
||||
f"(canonical_idx={canonical_idx} > custom_idx={custom_idx})"
|
||||
)
|
||||
|
||||
|
||||
def test_canonical_order_with_unconfigured_preserves_full_universe():
|
||||
"""Combined picker call: include_unconfigured + picker_hints +
|
||||
canonical_order is the production TUI shape. Verify the result
|
||||
has CANONICAL_PROVIDERS in declaration order, hints applied,
|
||||
custom rows trailing.
|
||||
"""
|
||||
from hermes_cli.models import CANONICAL_PROVIDERS
|
||||
|
||||
rows = [
|
||||
{"slug": "custom:Ollama", "name": "Ollama", "models": [],
|
||||
"total_models": 0, "is_current": False, "is_user_defined": True,
|
||||
"source": "user-config"},
|
||||
]
|
||||
ctx = _empty_ctx()
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(
|
||||
ctx,
|
||||
include_unconfigured=True,
|
||||
picker_hints=True,
|
||||
canonical_order=True,
|
||||
)
|
||||
slugs = [r["slug"] for r in payload["providers"]]
|
||||
# First row: first canonical provider in declaration order.
|
||||
assert slugs[0] == CANONICAL_PROVIDERS[0].slug
|
||||
# Custom row trails canonical universe.
|
||||
assert slugs.index("custom:Ollama") >= len(CANONICAL_PROVIDERS)
|
||||
|
||||
|
||||
# ─── Integration: end-to-end through real load_picker_context ──────────
|
||||
|
||||
|
||||
def test_end_to_end_with_real_context_no_credentials_leak(monkeypatch):
|
||||
"""Full pipeline: real load_picker_context + real
|
||||
list_authenticated_providers. Verify no credential string ever
|
||||
appears in the returned payload, even with picker_hints=True."""
|
||||
canary = "sk-canary-XYZ-must-not-appear"
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", canary)
|
||||
monkeypatch.setenv("ANTHROPIC_API_KEY", canary)
|
||||
cfg = _cfg(model={"provider": "openrouter"})
|
||||
with patch("hermes_cli.config.load_config", return_value=cfg):
|
||||
ctx = load_picker_context()
|
||||
payload = build_models_payload(
|
||||
ctx, include_unconfigured=True, picker_hints=True,
|
||||
)
|
||||
import json as _json
|
||||
|
||||
assert canary not in _json.dumps(payload)
|
||||
|
||||
|
||||
def test_payload_shape_compatible_with_modelpickerdialog_frontend():
|
||||
"""Frontend (web/src/components/ModelPickerDialog.tsx) reads:
|
||||
name, slug, models, total_models, is_current, warning, authenticated.
|
||||
Verify every authenticated/skeleton row exposes those keys.
|
||||
"""
|
||||
rows = [
|
||||
{"slug": "openrouter", "name": "OpenRouter", "models": ["m1"],
|
||||
"total_models": 1, "is_current": True, "is_user_defined": False,
|
||||
"source": "built-in"},
|
||||
]
|
||||
ctx = _empty_ctx()
|
||||
with _list_auth_returning(rows):
|
||||
payload = build_models_payload(
|
||||
ctx, include_unconfigured=True, picker_hints=True,
|
||||
)
|
||||
required_keys = {"name", "slug", "models", "total_models", "is_current",
|
||||
"authenticated"}
|
||||
for row in payload["providers"]:
|
||||
missing = required_keys - row.keys()
|
||||
assert not missing, f"row {row['slug']} missing keys: {missing}"
|
||||
@@ -177,6 +177,40 @@ class TestProviderPersistsAfterModelSave:
|
||||
assert model.get("api_mode") == "codex_responses"
|
||||
assert config["agent"]["reasoning_effort"] == "high"
|
||||
|
||||
def test_named_custom_provider_preserves_explicit_api_mode(self, config_home):
|
||||
"""Named custom providers should re-activate with their saved api_mode."""
|
||||
import yaml
|
||||
|
||||
from hermes_cli.main import _model_flow_named_custom
|
||||
|
||||
provider_info = {
|
||||
"name": "Packy",
|
||||
"base_url": "https://packy.example.com/v1",
|
||||
"api_key": "sk-test",
|
||||
"model": "gpt-5.4",
|
||||
"api_mode": "codex_responses",
|
||||
}
|
||||
|
||||
# Patch fetch_api_models so the named custom flow returns one model;
|
||||
# patch simple_term_menu to force the input() fallback; patch input to
|
||||
# auto-select the first model from the fallback prompt.
|
||||
from unittest.mock import MagicMock
|
||||
fake_menu_module = MagicMock()
|
||||
fake_menu_module.TerminalMenu.side_effect = OSError("no tty in test")
|
||||
with patch("hermes_cli.auth._save_model_choice"), \
|
||||
patch("hermes_cli.auth.deactivate_provider"), \
|
||||
patch("hermes_cli.models.fetch_api_models", return_value=["gpt-5.4"]), \
|
||||
patch.dict("sys.modules", {"simple_term_menu": fake_menu_module}), \
|
||||
patch("builtins.input", return_value="1"):
|
||||
_model_flow_named_custom({}, provider_info)
|
||||
|
||||
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
||||
model = config.get("model")
|
||||
assert isinstance(model, dict)
|
||||
assert model.get("provider") == "custom"
|
||||
assert model.get("base_url") == "https://packy.example.com/v1"
|
||||
assert model.get("api_mode") == "codex_responses"
|
||||
|
||||
def test_copilot_acp_provider_saved_when_selected(self, config_home):
|
||||
"""_model_flow_copilot_acp should persist provider/base_url/model together."""
|
||||
from hermes_cli.main import _model_flow_copilot_acp
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Tests for the get_nous_auth_status() process-level cache.
|
||||
|
||||
The cache avoids re-validating Nous credentials on every menu paint —
|
||||
`hermes tools` → "All Platforms" used to fire ~31 OAuth refresh POSTs
|
||||
against portal.nousresearch.com during one render. The cache is keyed
|
||||
on auth.json mtime so login/logout flows invalidate naturally; tests
|
||||
and other writers can also call invalidate_nous_auth_status_cache().
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
def _seed_auth_file(tmp_path):
|
||||
"""Drop a placeholder auth.json into the test HERMES_HOME.
|
||||
|
||||
The exact content doesn't matter for cache-key purposes — only that
|
||||
the file exists and we can mutate it to bump mtime.
|
||||
"""
|
||||
auth = tmp_path / "auth.json"
|
||||
auth.write_text(json.dumps({"providers": {}}), encoding="utf-8")
|
||||
return auth
|
||||
|
||||
|
||||
def test_get_nous_auth_status_caches_consecutive_calls(tmp_path, monkeypatch):
|
||||
"""A second call within the TTL skips re-computing the snapshot."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_seed_auth_file(tmp_path)
|
||||
|
||||
from hermes_cli import auth as auth_mod
|
||||
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
|
||||
call_count = {"n": 0}
|
||||
|
||||
def fake_compute():
|
||||
call_count["n"] += 1
|
||||
return {"logged_in": False, "source": "auth_store", "call": call_count["n"]}
|
||||
|
||||
with patch.object(auth_mod, "_compute_nous_auth_status", side_effect=fake_compute):
|
||||
first = auth_mod.get_nous_auth_status()
|
||||
second = auth_mod.get_nous_auth_status()
|
||||
third = auth_mod.get_nous_auth_status()
|
||||
|
||||
assert call_count["n"] == 1, (
|
||||
f"_compute_nous_auth_status was called {call_count['n']}× — "
|
||||
"cache is not deduplicating within TTL."
|
||||
)
|
||||
# Each call returns a copy so callers can't mutate the cached dict.
|
||||
assert first == second == third
|
||||
first["mutated"] = True
|
||||
assert "mutated" not in auth_mod.get_nous_auth_status()
|
||||
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
|
||||
|
||||
def test_get_nous_auth_status_invalidates_on_auth_file_mtime(tmp_path, monkeypatch):
|
||||
"""Touching auth.json (login/logout) forces a re-compute."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
auth_path = _seed_auth_file(tmp_path)
|
||||
|
||||
from hermes_cli import auth as auth_mod
|
||||
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
|
||||
call_count = {"n": 0}
|
||||
|
||||
def fake_compute():
|
||||
call_count["n"] += 1
|
||||
return {"logged_in": False, "source": "auth_store", "call": call_count["n"]}
|
||||
|
||||
with patch.object(auth_mod, "_compute_nous_auth_status", side_effect=fake_compute):
|
||||
auth_mod.get_nous_auth_status()
|
||||
# Bump mtime forward so coarse-resolution filesystems still record
|
||||
# a change.
|
||||
future = auth_path.stat().st_mtime + 5.0
|
||||
os.utime(auth_path, (future, future))
|
||||
auth_mod.get_nous_auth_status()
|
||||
|
||||
assert call_count["n"] == 2, (
|
||||
"auth.json mtime change should invalidate the cache, but only "
|
||||
f"{call_count['n']} compute call(s) happened."
|
||||
)
|
||||
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
|
||||
|
||||
def test_invalidate_nous_auth_status_cache_forces_recompute(tmp_path, monkeypatch):
|
||||
"""Explicit invalidate forces the next call to re-compute."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_seed_auth_file(tmp_path)
|
||||
|
||||
from hermes_cli import auth as auth_mod
|
||||
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
|
||||
call_count = {"n": 0}
|
||||
|
||||
def fake_compute():
|
||||
call_count["n"] += 1
|
||||
return {"logged_in": False, "source": "auth_store"}
|
||||
|
||||
with patch.object(auth_mod, "_compute_nous_auth_status", side_effect=fake_compute):
|
||||
auth_mod.get_nous_auth_status()
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
auth_mod.get_nous_auth_status()
|
||||
|
||||
assert call_count["n"] == 2
|
||||
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
|
||||
|
||||
def test_get_nous_auth_status_caches_failure_path(tmp_path, monkeypatch):
|
||||
"""Logged-out snapshots are cached too — that's where the cost was.
|
||||
|
||||
Teknium's case: ~31 cache misses per `hermes tools` "All Platforms"
|
||||
menu paint, all returning logged_in=False after a failed refresh POST.
|
||||
The whole point of the cache is to memoise that failure path too.
|
||||
"""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_seed_auth_file(tmp_path)
|
||||
|
||||
from hermes_cli import auth as auth_mod
|
||||
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
|
||||
call_count = {"n": 0}
|
||||
|
||||
def fake_compute():
|
||||
call_count["n"] += 1
|
||||
return {"logged_in": False, "source": "auth_store", "error": "refresh failed"}
|
||||
|
||||
with patch.object(auth_mod, "_compute_nous_auth_status", side_effect=fake_compute):
|
||||
for _ in range(10):
|
||||
auth_mod.get_nous_auth_status()
|
||||
|
||||
assert call_count["n"] == 1, (
|
||||
f"Logged-out snapshots must cache; got {call_count['n']} computes for 10 calls."
|
||||
)
|
||||
|
||||
auth_mod.invalidate_nous_auth_status_cache()
|
||||
@@ -538,6 +538,95 @@ class TestPreToolCallBlocking:
|
||||
assert get_pre_tool_call_block_message("terminal", {}) == "first blocker"
|
||||
|
||||
|
||||
class TestThreadToolWhitelist:
|
||||
"""Tests for the thread-local tool whitelist used by background review forks."""
|
||||
|
||||
def test_allowed_tool_passes_through_to_hooks(self, monkeypatch):
|
||||
from hermes_cli.plugins import (
|
||||
set_thread_tool_whitelist,
|
||||
clear_thread_tool_whitelist,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [],
|
||||
)
|
||||
set_thread_tool_whitelist({"memory", "skill_manage"})
|
||||
try:
|
||||
assert get_pre_tool_call_block_message("memory", {}) is None
|
||||
finally:
|
||||
clear_thread_tool_whitelist()
|
||||
|
||||
def test_disallowed_tool_blocked_with_message(self, monkeypatch):
|
||||
from hermes_cli.plugins import (
|
||||
set_thread_tool_whitelist,
|
||||
clear_thread_tool_whitelist,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [],
|
||||
)
|
||||
set_thread_tool_whitelist(
|
||||
{"memory"}, deny_msg_fmt="denied: {tool_name}"
|
||||
)
|
||||
try:
|
||||
msg = get_pre_tool_call_block_message("terminal", {})
|
||||
assert msg == "denied: terminal"
|
||||
finally:
|
||||
clear_thread_tool_whitelist()
|
||||
|
||||
def test_clear_restores_unrestricted_behavior(self, monkeypatch):
|
||||
from hermes_cli.plugins import (
|
||||
set_thread_tool_whitelist,
|
||||
clear_thread_tool_whitelist,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [],
|
||||
)
|
||||
set_thread_tool_whitelist({"memory"})
|
||||
clear_thread_tool_whitelist()
|
||||
# After clearing, any tool should pass through to plugin hooks (which
|
||||
# return [] here, so result is None).
|
||||
assert get_pre_tool_call_block_message("terminal", {}) is None
|
||||
|
||||
def test_whitelist_is_thread_local(self, monkeypatch):
|
||||
"""Setting a whitelist in one thread must NOT leak into another."""
|
||||
import threading
|
||||
|
||||
from hermes_cli.plugins import (
|
||||
set_thread_tool_whitelist,
|
||||
clear_thread_tool_whitelist,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [],
|
||||
)
|
||||
|
||||
# Main thread: install a restrictive whitelist.
|
||||
set_thread_tool_whitelist({"memory"})
|
||||
try:
|
||||
assert get_pre_tool_call_block_message("terminal", {}) is not None
|
||||
|
||||
# Worker thread: should NOT inherit main thread's whitelist.
|
||||
result = {}
|
||||
|
||||
def worker():
|
||||
result["msg"] = get_pre_tool_call_block_message("terminal", {})
|
||||
|
||||
t = threading.Thread(target=worker)
|
||||
t.start()
|
||||
t.join()
|
||||
assert result["msg"] is None, (
|
||||
"thread-local whitelist leaked across threads"
|
||||
)
|
||||
finally:
|
||||
clear_thread_tool_whitelist()
|
||||
|
||||
|
||||
# ── TestPluginContext ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -29,8 +29,6 @@ from hermes_cli.profiles import (
|
||||
rename_profile,
|
||||
export_profile,
|
||||
import_profile,
|
||||
generate_bash_completion,
|
||||
generate_zsh_completion,
|
||||
_get_profiles_root,
|
||||
_get_default_hermes_home,
|
||||
seed_profile_skills,
|
||||
@@ -1013,32 +1011,6 @@ class TestProfileIsolation:
|
||||
assert (beta_dir / "skills").is_dir()
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# TestCompletion
|
||||
# ===================================================================
|
||||
|
||||
class TestCompletion:
|
||||
"""Tests for bash/zsh completion generators."""
|
||||
|
||||
def test_bash_completion_contains_complete(self):
|
||||
script = generate_bash_completion()
|
||||
assert len(script) > 0
|
||||
assert "complete" in script
|
||||
|
||||
def test_zsh_completion_contains_compdef(self):
|
||||
script = generate_zsh_completion()
|
||||
assert len(script) > 0
|
||||
assert "compdef" in script
|
||||
|
||||
def test_bash_completion_has_hermes_profiles_function(self):
|
||||
script = generate_bash_completion()
|
||||
assert "_hermes_profiles" in script
|
||||
|
||||
def test_zsh_completion_has_hermes_function(self):
|
||||
script = generate_zsh_completion()
|
||||
assert "_hermes" in script
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# TestGetProfilesRoot / TestGetDefaultHermesHome (internal helpers)
|
||||
# ===================================================================
|
||||
|
||||
@@ -0,0 +1,512 @@
|
||||
"""Tests for the `hermes proxy` subcommand and its upstream adapters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.proxy.adapters import ADAPTERS, get_adapter
|
||||
from hermes_cli.proxy.adapters.base import UpstreamAdapter, UpstreamCredential
|
||||
from hermes_cli.proxy.adapters.nous_portal import NousPortalAdapter
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Adapter registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_registry_lists_nous():
|
||||
assert "nous" in ADAPTERS
|
||||
|
||||
|
||||
def test_get_adapter_returns_instance():
|
||||
adapter = get_adapter("nous")
|
||||
assert isinstance(adapter, NousPortalAdapter)
|
||||
assert isinstance(adapter, UpstreamAdapter)
|
||||
|
||||
|
||||
def test_get_adapter_case_insensitive():
|
||||
assert isinstance(get_adapter("NOUS"), NousPortalAdapter)
|
||||
assert isinstance(get_adapter(" Nous "), NousPortalAdapter)
|
||||
|
||||
|
||||
def test_get_adapter_unknown_provider_raises():
|
||||
with pytest.raises(ValueError, match="anthropic"):
|
||||
get_adapter("anthropic") # not yet implemented
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NousPortalAdapter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_auth_store(hermes_home: Path, nous_state: Dict[str, Any]) -> Path:
|
||||
"""Write an auth.json with the given nous state into a hermetic HERMES_HOME."""
|
||||
auth_path = hermes_home / "auth.json"
|
||||
auth_path.write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {"nous": nous_state},
|
||||
}))
|
||||
return auth_path
|
||||
|
||||
|
||||
def test_nous_adapter_metadata():
|
||||
adapter = NousPortalAdapter()
|
||||
assert adapter.name == "nous"
|
||||
assert adapter.display_name == "Nous Portal"
|
||||
assert "/chat/completions" in adapter.allowed_paths
|
||||
assert "/embeddings" in adapter.allowed_paths
|
||||
assert "/completions" in adapter.allowed_paths
|
||||
assert "/models" in adapter.allowed_paths
|
||||
|
||||
|
||||
def test_nous_adapter_not_authenticated_when_no_auth_file(tmp_path, monkeypatch):
|
||||
# HERMES_HOME is already set by conftest, but make doubly sure
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
adapter = NousPortalAdapter()
|
||||
assert not adapter.is_authenticated()
|
||||
|
||||
|
||||
def test_nous_adapter_not_authenticated_when_provider_missing(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / "auth.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"providers": {},
|
||||
}))
|
||||
assert not NousPortalAdapter().is_authenticated()
|
||||
|
||||
|
||||
def test_nous_adapter_authenticated_with_agent_key(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_write_auth_store(tmp_path, {
|
||||
"agent_key": "ov-test-key",
|
||||
"agent_key_expires_at": "2099-01-01T00:00:00Z",
|
||||
"inference_base_url": "https://inference-api.nousresearch.com/v1",
|
||||
})
|
||||
assert NousPortalAdapter().is_authenticated()
|
||||
|
||||
|
||||
def test_nous_adapter_authenticated_with_refresh_token_only(tmp_path, monkeypatch):
|
||||
"""If access_token+refresh_token exist but no agent_key yet, we can still mint."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_write_auth_store(tmp_path, {
|
||||
"access_token": "access-tok",
|
||||
"refresh_token": "refresh-tok",
|
||||
})
|
||||
assert NousPortalAdapter().is_authenticated()
|
||||
|
||||
|
||||
def test_nous_adapter_get_credential_refreshes_and_persists(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_write_auth_store(tmp_path, {
|
||||
"access_token": "access-tok",
|
||||
"refresh_token": "refresh-tok",
|
||||
"client_id": "hermes-cli",
|
||||
"portal_base_url": "https://portal.nousresearch.com",
|
||||
"inference_base_url": "https://inference-api.nousresearch.com/v1",
|
||||
})
|
||||
|
||||
refreshed_state = {
|
||||
"access_token": "access-tok",
|
||||
"refresh_token": "refresh-tok",
|
||||
"client_id": "hermes-cli",
|
||||
"portal_base_url": "https://portal.nousresearch.com",
|
||||
"inference_base_url": "https://inference-api.nousresearch.com/v1",
|
||||
"agent_key": "minted-bearer",
|
||||
"agent_key_expires_at": "2099-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
with patch(
|
||||
"hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state",
|
||||
return_value=refreshed_state,
|
||||
) as mock_refresh:
|
||||
adapter = NousPortalAdapter()
|
||||
cred = adapter.get_credential()
|
||||
|
||||
mock_refresh.assert_called_once()
|
||||
assert cred.bearer == "minted-bearer"
|
||||
assert cred.base_url == "https://inference-api.nousresearch.com/v1"
|
||||
assert cred.expires_at == "2099-01-01T00:00:00Z"
|
||||
assert cred.token_type == "Bearer"
|
||||
|
||||
# Verify state was persisted back
|
||||
stored = json.loads((tmp_path / "auth.json").read_text())
|
||||
assert stored["providers"]["nous"]["agent_key"] == "minted-bearer"
|
||||
|
||||
|
||||
def test_nous_adapter_get_credential_raises_when_not_logged_in(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
adapter = NousPortalAdapter()
|
||||
with pytest.raises(RuntimeError, match="hermes login nous"):
|
||||
adapter.get_credential()
|
||||
|
||||
|
||||
def test_nous_adapter_get_credential_raises_on_refresh_failure(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_write_auth_store(tmp_path, {
|
||||
"access_token": "access-tok",
|
||||
"refresh_token": "refresh-tok",
|
||||
})
|
||||
|
||||
with patch(
|
||||
"hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state",
|
||||
side_effect=RuntimeError("Refresh session has been revoked"),
|
||||
):
|
||||
adapter = NousPortalAdapter()
|
||||
with pytest.raises(RuntimeError, match="Refresh session has been revoked"):
|
||||
adapter.get_credential()
|
||||
|
||||
|
||||
def test_nous_adapter_get_credential_raises_when_no_agent_key_returned(tmp_path, monkeypatch):
|
||||
"""If the refresh helper succeeds but produces no agent_key, we surface a clear error."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_write_auth_store(tmp_path, {
|
||||
"access_token": "access-tok",
|
||||
"refresh_token": "refresh-tok",
|
||||
})
|
||||
|
||||
with patch(
|
||||
"hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state",
|
||||
return_value={"access_token": "a", "refresh_token": "r"},
|
||||
):
|
||||
adapter = NousPortalAdapter()
|
||||
with pytest.raises(RuntimeError, match="did not return a usable agent_key"):
|
||||
adapter.get_credential()
|
||||
|
||||
|
||||
def test_nous_adapter_concurrent_refresh_serialized(tmp_path, monkeypatch):
|
||||
"""Two parallel get_credential() calls must serialize through the lock."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
_write_auth_store(tmp_path, {
|
||||
"access_token": "a", "refresh_token": "r",
|
||||
})
|
||||
|
||||
call_log: list = []
|
||||
in_flight = threading.Event()
|
||||
overlap_detected = threading.Event()
|
||||
counter = [0]
|
||||
counter_lock = threading.Lock()
|
||||
|
||||
def serializing_refresh(state, **kwargs):
|
||||
# If another thread is already inside refresh, the lock is broken.
|
||||
if in_flight.is_set():
|
||||
overlap_detected.set()
|
||||
in_flight.set()
|
||||
try:
|
||||
call_log.append(threading.current_thread().ident)
|
||||
# Simulate refresh latency so any race window is exposed.
|
||||
import time
|
||||
time.sleep(0.05)
|
||||
with counter_lock:
|
||||
counter[0] += 1
|
||||
idx = counter[0]
|
||||
return {
|
||||
**state,
|
||||
"agent_key": f"key-{idx}",
|
||||
"agent_key_expires_at": "2099-01-01T00:00:00Z",
|
||||
"inference_base_url": "https://inference-api.nousresearch.com/v1",
|
||||
}
|
||||
finally:
|
||||
in_flight.clear()
|
||||
|
||||
adapter = NousPortalAdapter()
|
||||
results: list = []
|
||||
errors: list = []
|
||||
|
||||
def worker():
|
||||
try:
|
||||
results.append(adapter.get_credential().bearer)
|
||||
except Exception as exc: # pragma: no cover - shouldn't happen
|
||||
errors.append(exc)
|
||||
|
||||
with patch(
|
||||
"hermes_cli.proxy.adapters.nous_portal.refresh_nous_oauth_from_state",
|
||||
side_effect=serializing_refresh,
|
||||
):
|
||||
threads = [threading.Thread(target=worker) for _ in range(3)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, f"workers errored: {errors}"
|
||||
assert len(results) == 3
|
||||
assert len(call_log) == 3
|
||||
assert not overlap_detected.is_set(), "refresh calls overlapped — lock is broken"
|
||||
assert all(r.startswith("key-") for r in results)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server: path filtering + forwarding
|
||||
#
|
||||
# We run the proxy AND a fake upstream as real aiohttp servers on ephemeral
|
||||
# ports. Avoids pytest-aiohttp's fixtures (extra dependency for one test file).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
aiohttp = pytest.importorskip("aiohttp")
|
||||
from aiohttp import web # noqa: E402
|
||||
|
||||
from hermes_cli.proxy.server import create_app # noqa: E402
|
||||
|
||||
|
||||
class FakeAdapter(UpstreamAdapter):
|
||||
"""A test adapter that returns a fixed credential without touching disk."""
|
||||
|
||||
def __init__(self, base_url: str, bearer: str = "test-bearer",
|
||||
allowed=None, raise_on_credential=False):
|
||||
self._base_url = base_url
|
||||
self._bearer = bearer
|
||||
self._allowed = frozenset(allowed or ["/chat/completions"])
|
||||
self._raise = raise_on_credential
|
||||
self.calls = 0
|
||||
|
||||
@property
|
||||
def name(self): return "fake"
|
||||
|
||||
@property
|
||||
def display_name(self): return "Fake Provider"
|
||||
|
||||
@property
|
||||
def allowed_paths(self): return self._allowed
|
||||
|
||||
def is_authenticated(self): return True
|
||||
|
||||
def get_credential(self):
|
||||
self.calls += 1
|
||||
if self._raise:
|
||||
raise RuntimeError("simulated auth failure")
|
||||
return UpstreamCredential(
|
||||
bearer=self._bearer, base_url=self._base_url,
|
||||
expires_at="2099-01-01T00:00:00Z",
|
||||
)
|
||||
|
||||
|
||||
async def _start_runner(app: "web.Application"):
|
||||
"""Spin up an aiohttp app on an ephemeral localhost port. Returns (runner, base_url)."""
|
||||
runner = web.AppRunner(app, access_log=None)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, host="127.0.0.1", port=0)
|
||||
await site.start()
|
||||
sockets = list(site._server.sockets) # type: ignore[union-attr]
|
||||
port = sockets[0].getsockname()[1]
|
||||
return runner, f"http://127.0.0.1:{port}"
|
||||
|
||||
|
||||
def _build_fake_upstream(captured: Dict[str, Any]) -> "web.Application":
|
||||
async def echo(request):
|
||||
body = await request.read()
|
||||
captured["requests"].append({
|
||||
"method": request.method,
|
||||
"path": request.path,
|
||||
"auth": request.headers.get("Authorization"),
|
||||
"body": body.decode("utf-8") if body else "",
|
||||
})
|
||||
return web.json_response({"echoed": True, "path": request.path})
|
||||
|
||||
async def sse(request):
|
||||
resp = web.StreamResponse(
|
||||
status=200, headers={"Content-Type": "text/event-stream"},
|
||||
)
|
||||
await resp.prepare(request)
|
||||
for chunk in [b"data: hello\n\n", b"data: world\n\n", b"data: [DONE]\n\n"]:
|
||||
await resp.write(chunk)
|
||||
await resp.write_eof()
|
||||
return resp
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_route("*", "/v1/chat/completions", echo)
|
||||
app.router.add_route("*", "/v1/embeddings", echo)
|
||||
app.router.add_route("*", "/v1/sse", sse)
|
||||
return app
|
||||
|
||||
|
||||
def test_server_forwards_chat_completions():
|
||||
async def run():
|
||||
captured: Dict[str, Any] = {"requests": []}
|
||||
upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured))
|
||||
adapter = FakeAdapter(f"{upstream_base}/v1", bearer="real-portal-key")
|
||||
proxy_runner, proxy_base = await _start_runner(create_app(adapter))
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
f"{proxy_base}/v1/chat/completions",
|
||||
json={"model": "Hermes-4-70B",
|
||||
"messages": [{"role": "user", "content": "hi"}]},
|
||||
headers={"Authorization": "Bearer client-dummy-key"},
|
||||
) as resp:
|
||||
assert resp.status == 200
|
||||
data = await resp.json()
|
||||
assert data["echoed"] is True
|
||||
|
||||
assert len(captured["requests"]) == 1
|
||||
req = captured["requests"][0]
|
||||
assert req["auth"] == "Bearer real-portal-key"
|
||||
assert "Hermes-4-70B" in req["body"]
|
||||
finally:
|
||||
await proxy_runner.cleanup()
|
||||
await upstream_runner.cleanup()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_server_rejects_disallowed_path():
|
||||
async def run():
|
||||
adapter = FakeAdapter("http://unused.example/v1", allowed=["/chat/completions"])
|
||||
runner, base = await _start_runner(create_app(adapter))
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(f"{base}/v1/random/endpoint") as resp:
|
||||
assert resp.status == 404
|
||||
body = await resp.json()
|
||||
assert body["error"]["type"] == "path_not_allowed"
|
||||
assert "/chat/completions" in body["error"]["message"]
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_server_returns_401_when_adapter_fails():
|
||||
async def run():
|
||||
adapter = FakeAdapter("http://unused.example/v1", raise_on_credential=True)
|
||||
runner, base = await _start_runner(create_app(adapter))
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(f"{base}/v1/chat/completions", json={}) as resp:
|
||||
assert resp.status == 401
|
||||
body = await resp.json()
|
||||
assert body["error"]["type"] == "upstream_auth_failed"
|
||||
assert "simulated auth failure" in body["error"]["message"]
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_server_health_endpoint():
|
||||
async def run():
|
||||
adapter = FakeAdapter("http://unused.example/v1")
|
||||
runner, base = await _start_runner(create_app(adapter))
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(f"{base}/health") as resp:
|
||||
assert resp.status == 200
|
||||
body = await resp.json()
|
||||
assert body["status"] == "ok"
|
||||
assert body["upstream"] == "Fake Provider"
|
||||
assert body["authenticated"] is True
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_server_streams_sse():
|
||||
async def run():
|
||||
captured: Dict[str, Any] = {"requests": []}
|
||||
upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured))
|
||||
adapter = FakeAdapter(f"{upstream_base}/v1", allowed=["/sse"])
|
||||
proxy_runner, proxy_base = await _start_runner(create_app(adapter))
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(f"{proxy_base}/v1/sse") as resp:
|
||||
assert resp.status == 200
|
||||
chunks = []
|
||||
async for chunk in resp.content.iter_any():
|
||||
chunks.append(chunk)
|
||||
full = b"".join(chunks)
|
||||
assert b"data: hello" in full
|
||||
assert b"data: [DONE]" in full
|
||||
finally:
|
||||
await proxy_runner.cleanup()
|
||||
await upstream_runner.cleanup()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_server_strips_client_auth_header():
|
||||
"""The client's Authorization header MUST NOT reach the upstream."""
|
||||
async def run():
|
||||
captured: Dict[str, Any] = {"requests": []}
|
||||
upstream_runner, upstream_base = await _start_runner(_build_fake_upstream(captured))
|
||||
adapter = FakeAdapter(f"{upstream_base}/v1", bearer="ours")
|
||||
proxy_runner, proxy_base = await _start_runner(create_app(adapter))
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
f"{proxy_base}/v1/chat/completions",
|
||||
json={},
|
||||
headers={"Authorization": "Bearer SHOULD_NOT_LEAK"},
|
||||
) as resp:
|
||||
await resp.read()
|
||||
assert captured["requests"][0]["auth"] == "Bearer ours"
|
||||
assert "SHOULD_NOT_LEAK" not in captured["requests"][0]["auth"]
|
||||
finally:
|
||||
await proxy_runner.cleanup()
|
||||
await upstream_runner.cleanup()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cmd_proxy_status_runs(capsys, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
from hermes_cli.proxy.cli import cmd_proxy_status
|
||||
|
||||
args = MagicMock()
|
||||
rc = cmd_proxy_status(args)
|
||||
assert rc == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "nous" in out
|
||||
assert "Nous Portal" in out
|
||||
assert "not logged in" in out
|
||||
|
||||
|
||||
def test_cmd_proxy_providers_runs(capsys):
|
||||
from hermes_cli.proxy.cli import cmd_proxy_list_providers
|
||||
|
||||
args = MagicMock()
|
||||
rc = cmd_proxy_list_providers(args)
|
||||
assert rc == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "nous" in out
|
||||
assert "Nous Portal" in out
|
||||
|
||||
|
||||
def test_cmd_proxy_start_refuses_unknown_provider(capsys):
|
||||
from hermes_cli.proxy.cli import cmd_proxy_start
|
||||
|
||||
args = MagicMock()
|
||||
args.provider = "no-such-provider"
|
||||
args.host = None
|
||||
args.port = None
|
||||
rc = cmd_proxy_start(args)
|
||||
assert rc == 2
|
||||
err = capsys.readouterr().err
|
||||
assert "no-such-provider" in err
|
||||
|
||||
|
||||
def test_cmd_proxy_start_refuses_when_unauthenticated(capsys, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
from hermes_cli.proxy.cli import cmd_proxy_start
|
||||
|
||||
args = MagicMock()
|
||||
args.provider = "nous"
|
||||
args.host = None
|
||||
args.port = None
|
||||
rc = cmd_proxy_start(args)
|
||||
assert rc == 2
|
||||
err = capsys.readouterr().err
|
||||
assert "hermes login nous" in err
|
||||
@@ -39,8 +39,6 @@ class TestExplicitAllowlist:
|
||||
"OPENROUTER_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"ANTHROPIC_API_KEY",
|
||||
"WANDB_API_KEY",
|
||||
"TINKER_API_KEY",
|
||||
"HONCHO_API_KEY",
|
||||
"FIRECRAWL_API_KEY",
|
||||
"BROWSERBASE_API_KEY",
|
||||
|
||||
@@ -573,48 +573,6 @@ def test_vercel_setup_prefills_project_and_team_from_link_file(tmp_path, monkeyp
|
||||
assert defaults[" Vercel team ID"] == "linked-team"
|
||||
|
||||
|
||||
def test_offer_launch_chat_relaunches_via_bin(monkeypatch):
|
||||
from hermes_cli import setup as setup_mod
|
||||
from hermes_cli import relaunch as relaunch_mod
|
||||
|
||||
monkeypatch.setattr(setup_mod, "prompt_yes_no", lambda *_args, **_kwargs: True)
|
||||
monkeypatch.setattr(relaunch_mod, "resolve_hermes_bin", lambda: "/usr/local/bin/hermes")
|
||||
|
||||
exec_calls = []
|
||||
|
||||
def fake_execvp(path, argv):
|
||||
exec_calls.append((path, argv))
|
||||
raise SystemExit(0)
|
||||
|
||||
monkeypatch.setattr(relaunch_mod.os, "execvp", fake_execvp)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
setup_mod._offer_launch_chat()
|
||||
|
||||
assert exec_calls == [("/usr/local/bin/hermes", ["/usr/local/bin/hermes", "chat"])]
|
||||
|
||||
|
||||
def test_offer_launch_chat_falls_back_to_module(monkeypatch):
|
||||
from hermes_cli import setup as setup_mod
|
||||
from hermes_cli import relaunch as relaunch_mod
|
||||
|
||||
monkeypatch.setattr(setup_mod, "prompt_yes_no", lambda *_args, **_kwargs: True)
|
||||
monkeypatch.setattr(relaunch_mod, "resolve_hermes_bin", lambda: None)
|
||||
|
||||
exec_calls = []
|
||||
|
||||
def fake_execvp(path, argv):
|
||||
exec_calls.append((path, argv))
|
||||
raise SystemExit(0)
|
||||
|
||||
monkeypatch.setattr(relaunch_mod.os, "execvp", fake_execvp)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
setup_mod._offer_launch_chat()
|
||||
|
||||
assert exec_calls == [(sys.executable, [sys.executable, "-m", "hermes_cli.main", "chat"])]
|
||||
|
||||
|
||||
def test_setup_slack_saves_home_channel(monkeypatch):
|
||||
"""_setup_slack() saves SLACK_HOME_CHANNEL when the user provides one."""
|
||||
saved = {}
|
||||
|
||||
@@ -18,4 +18,3 @@ def test_setup_hermes_script_has_termux_path():
|
||||
assert ".[termux]" in content
|
||||
assert "constraints-termux.txt" in content
|
||||
assert "$PREFIX/bin" in content
|
||||
assert "Skipping tinker-atropos on Termux" in content
|
||||
|
||||
@@ -262,7 +262,6 @@ class TestSetupWizardOpenclawIntegration:
|
||||
patch.object(setup_mod, "setup_tools"),
|
||||
patch.object(setup_mod, "save_config"),
|
||||
patch.object(setup_mod, "_print_setup_summary"),
|
||||
patch.object(setup_mod, "_offer_launch_chat"),
|
||||
):
|
||||
setup_mod.run_setup_wizard(args)
|
||||
|
||||
@@ -294,7 +293,6 @@ class TestSetupWizardOpenclawIntegration:
|
||||
patch.object(setup_mod, "setup_tools"),
|
||||
patch.object(setup_mod, "save_config"),
|
||||
patch.object(setup_mod, "_print_setup_summary"),
|
||||
patch.object(setup_mod, "_offer_launch_chat"),
|
||||
):
|
||||
setup_mod.run_setup_wizard(args)
|
||||
|
||||
@@ -327,7 +325,6 @@ class TestSetupWizardOpenclawIntegration:
|
||||
patch.object(setup_mod, "setup_tools"),
|
||||
patch.object(setup_mod, "save_config"),
|
||||
patch.object(setup_mod, "_print_setup_summary"),
|
||||
patch.object(setup_mod, "_offer_launch_chat"),
|
||||
):
|
||||
setup_mod.run_setup_wizard(args)
|
||||
|
||||
|
||||
@@ -63,7 +63,6 @@ def _enter_existing_install_patches(stack, **extra):
|
||||
("hermes_cli.setup.get_env_value", {"return_value": None}),
|
||||
("hermes_cli.auth.get_active_provider", {"return_value": "openrouter"}),
|
||||
("hermes_cli.setup._print_setup_summary", {}),
|
||||
("hermes_cli.setup._offer_launch_chat", {}),
|
||||
("hermes_cli.setup._offer_openclaw_migration", {"return_value": False}),
|
||||
]:
|
||||
stack.enter_context(patch(target, **kwargs))
|
||||
|
||||
@@ -199,6 +199,37 @@ class TestUserSkins:
|
||||
# Should inherit defaults for unspecified colors
|
||||
assert skin.get_color("banner_border") == "#CD7F32" # from default
|
||||
|
||||
def test_load_user_skin_invalid_section_types_fall_back_to_defaults(self, tmp_path, monkeypatch):
|
||||
from hermes_cli.skin_engine import load_skin
|
||||
|
||||
skins_dir = tmp_path / "skins"
|
||||
skins_dir.mkdir()
|
||||
import yaml
|
||||
|
||||
(skins_dir / "broken.yaml").write_text(
|
||||
yaml.dump(
|
||||
{
|
||||
"name": "broken",
|
||||
"colors": ["not", "a", "mapping"],
|
||||
"spinner": "invalid",
|
||||
"branding": ["also", "invalid"],
|
||||
"tool_emojis": ["invalid"],
|
||||
"tool_prefix": "!",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr("hermes_cli.skin_engine._skins_dir", lambda: skins_dir)
|
||||
|
||||
skin = load_skin("broken")
|
||||
|
||||
assert skin.name == "broken"
|
||||
assert skin.get_color("banner_title") == "#FFD700"
|
||||
assert skin.get_branding("agent_name") == "Hermes Agent"
|
||||
assert skin.spinner.get("waiting_faces", []) == []
|
||||
assert skin.tool_emojis == {}
|
||||
assert skin.tool_prefix == "!"
|
||||
|
||||
def test_list_skins_includes_user_skins(self, tmp_path, monkeypatch):
|
||||
from hermes_cli.skin_engine import list_skins
|
||||
skins_dir = tmp_path / "skins"
|
||||
|
||||
@@ -83,6 +83,12 @@ def test_get_platform_tools_default_telegram_includes_messaging():
|
||||
assert "messaging" in enabled
|
||||
|
||||
|
||||
def test_get_platform_tools_default_whatsapp_includes_web():
|
||||
enabled = _get_platform_tools({}, "whatsapp")
|
||||
|
||||
assert "web" in enabled
|
||||
|
||||
|
||||
def test_get_platform_tools_homeassistant_platform_keeps_homeassistant_toolset():
|
||||
enabled = _get_platform_tools({}, "homeassistant")
|
||||
|
||||
|
||||
@@ -305,6 +305,7 @@ def _setup_update_mocks(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(hermes_config, "get_missing_config_fields", lambda: [])
|
||||
monkeypatch.setattr(hermes_config, "check_config_version", lambda: (5, 5))
|
||||
monkeypatch.setattr(hermes_config, "migrate_config", lambda **kw: {"env_added": [], "config_added": []})
|
||||
monkeypatch.setattr(hermes_main, "_refresh_active_lazy_features", lambda: None)
|
||||
|
||||
|
||||
def test_cmd_update_retries_optional_extras_individually_when_all_fails(monkeypatch, tmp_path, capsys):
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Tests for plugin video_gen providers in the tools picker.
|
||||
|
||||
Covers the reconfigure path that previously failed to write
|
||||
``video_gen.provider`` when a user picked an xAI/etc. plugin backend
|
||||
through Reconfigure tool → Video Generation. The first-time configure
|
||||
path already handled it; the reconfigure path forgot to mirror it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
from agent.video_gen_provider import VideoGenProvider
|
||||
|
||||
|
||||
class _FakeVideoProvider(VideoGenProvider):
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
available: bool = True,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
models: Optional[List[Dict[str, Any]]] = None,
|
||||
):
|
||||
self._name = name
|
||||
self._available = available
|
||||
self._schema = schema or {
|
||||
"name": name.title(),
|
||||
"badge": "test",
|
||||
"tag": f"{name} test tag",
|
||||
"env_vars": [{"key": f"{name.upper()}_API_KEY", "prompt": f"{name} key"}],
|
||||
}
|
||||
self._models = models or [
|
||||
{
|
||||
"id": f"{name}-video-v1",
|
||||
"display": f"{name} v1",
|
||||
"speed": "~10s",
|
||||
"strengths": "test",
|
||||
"price": "$",
|
||||
},
|
||||
]
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
def list_models(self):
|
||||
return list(self._models)
|
||||
|
||||
def default_model(self):
|
||||
return self._models[0]["id"] if self._models else None
|
||||
|
||||
def get_setup_schema(self):
|
||||
return dict(self._schema)
|
||||
|
||||
def generate(self, prompt, **kw):
|
||||
return {"success": True, "video": f"{self._name}://{prompt}"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
class TestReconfigureWritesProvider:
|
||||
"""Regression tests for the video_gen reconfigure path.
|
||||
|
||||
Before the fix, _reconfigure_provider() handled image_gen_plugin_name
|
||||
in both the no-env-vars branch and the post-env-vars branch but
|
||||
missed video_gen_plugin_name in both. Picking xAI via Reconfigure
|
||||
tool → Video Generation silently no-op'd: the env var was already
|
||||
set, the env-var loop ran (Enter to keep), and the function fell
|
||||
through without ever writing config["video_gen"]["provider"].
|
||||
"""
|
||||
|
||||
def test_reconfigure_with_env_vars_already_set_writes_provider(
|
||||
self, monkeypatch, tmp_path
|
||||
):
|
||||
"""Env vars present and user accepts current value → still writes
|
||||
video_gen.provider via the post-env-vars branch."""
|
||||
from hermes_cli import tools_config
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
video_gen_registry.register_provider(_FakeVideoProvider("xai_fake"))
|
||||
|
||||
# Picker prompts replaced — no TTY in tests.
|
||||
monkeypatch.setattr(tools_config, "_prompt_choice", lambda *a, **kw: 0)
|
||||
# User presses Enter to keep the existing key.
|
||||
monkeypatch.setattr(tools_config, "_prompt", lambda *a, **kw: "")
|
||||
# Pretend the env var is already set so the reconfigure path
|
||||
# hits the "Kept current" branch.
|
||||
monkeypatch.setattr(
|
||||
tools_config,
|
||||
"get_env_value",
|
||||
lambda key: "sk-fake" if key == "XAI_FAKE_API_KEY" else "",
|
||||
)
|
||||
|
||||
config: dict = {}
|
||||
provider_row = {
|
||||
"name": "xAI",
|
||||
"env_vars": [{"key": "XAI_FAKE_API_KEY", "prompt": "xAI key"}],
|
||||
"video_gen_plugin_name": "xai_fake",
|
||||
}
|
||||
|
||||
tools_config._reconfigure_provider(provider_row, config)
|
||||
|
||||
assert config["video_gen"]["provider"] == "xai_fake"
|
||||
assert config["video_gen"]["model"] == "xai_fake-video-v1"
|
||||
assert config["video_gen"]["use_gateway"] is False
|
||||
|
||||
def test_reconfigure_with_no_env_vars_writes_provider(
|
||||
self, monkeypatch, tmp_path
|
||||
):
|
||||
"""No env vars at all (managed-style plugin) → writes
|
||||
video_gen.provider via the no-env-vars early-return branch."""
|
||||
from hermes_cli import tools_config
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
video_gen_registry.register_provider(_FakeVideoProvider(
|
||||
"noenv_video",
|
||||
schema={
|
||||
"name": "NoEnvVideo",
|
||||
"badge": "free",
|
||||
"tag": "",
|
||||
"env_vars": [],
|
||||
},
|
||||
))
|
||||
monkeypatch.setattr(tools_config, "_prompt_choice", lambda *a, **kw: 0)
|
||||
|
||||
config: dict = {}
|
||||
provider_row = {
|
||||
"name": "NoEnvVideo",
|
||||
"env_vars": [],
|
||||
"video_gen_plugin_name": "noenv_video",
|
||||
}
|
||||
|
||||
tools_config._reconfigure_provider(provider_row, config)
|
||||
|
||||
assert config["video_gen"]["provider"] == "noenv_video"
|
||||
assert config["video_gen"]["model"] == "noenv_video-video-v1"
|
||||
assert config["video_gen"]["use_gateway"] is False
|
||||
|
||||
|
||||
class TestPluginVideoProvidersRow:
|
||||
"""Tests for _plugin_video_gen_providers row contents."""
|
||||
|
||||
def test_post_setup_propagated_when_declared(self, monkeypatch):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
video_gen_registry.register_provider(_FakeVideoProvider(
|
||||
"xai_video",
|
||||
schema={
|
||||
"name": "xAI Grok Imagine",
|
||||
"badge": "paid",
|
||||
"tag": "grok video",
|
||||
"env_vars": [],
|
||||
"post_setup": "xai_grok",
|
||||
},
|
||||
))
|
||||
|
||||
rows = tools_config._plugin_video_gen_providers()
|
||||
match = next(r for r in rows if r.get("video_gen_plugin_name") == "xai_video")
|
||||
assert match["post_setup"] == "xai_grok"
|
||||
|
||||
def test_post_setup_omitted_when_not_declared(self, monkeypatch):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
video_gen_registry.register_provider(_FakeVideoProvider("plain_video"))
|
||||
|
||||
rows = tools_config._plugin_video_gen_providers()
|
||||
match = next(r for r in rows if r.get("video_gen_plugin_name") == "plain_video")
|
||||
assert "post_setup" not in match
|
||||
|
||||
|
||||
class TestVideoPluginProviderActive:
|
||||
"""Tests for _is_provider_active recognizing video_gen_plugin_name."""
|
||||
|
||||
def test_active_when_video_gen_provider_matches(self):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
config = {"video_gen": {"provider": "xai"}}
|
||||
row = {"name": "xAI Grok Imagine", "video_gen_plugin_name": "xai"}
|
||||
|
||||
assert tools_config._is_provider_active(row, config) is True
|
||||
|
||||
def test_inactive_when_video_gen_provider_differs(self):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
config = {"video_gen": {"provider": "fal"}}
|
||||
row = {"name": "xAI Grok Imagine", "video_gen_plugin_name": "xai"}
|
||||
|
||||
assert tools_config._is_provider_active(row, config) is False
|
||||
|
||||
def test_inactive_when_video_gen_section_missing(self):
|
||||
from hermes_cli import tools_config
|
||||
|
||||
row = {"name": "xAI Grok Imagine", "video_gen_plugin_name": "xai"}
|
||||
assert tools_config._is_provider_active(row, {}) is False
|
||||
|
||||
def test_detect_active_index_picks_video_plugin_match(self, monkeypatch):
|
||||
"""When xAI is the configured video_gen provider, the picker should
|
||||
default to the xAI row even if FAL_KEY happens to be set in env.
|
||||
|
||||
Regression: previously _detect_active_provider_index() saw
|
||||
_is_provider_active(xai) return False (no video_gen branch),
|
||||
skipped xAI (empty env_vars), and matched the FAL row via the
|
||||
env-var fallback — so the picker visually defaulted to FAL even
|
||||
though the user picked xAI. The xAI row uses empty env_vars
|
||||
because authentication is handled via xAI Grok OAuth (post_setup
|
||||
hook).
|
||||
"""
|
||||
from hermes_cli import tools_config
|
||||
|
||||
monkeypatch.setattr(
|
||||
tools_config,
|
||||
"get_env_value",
|
||||
lambda key: "fal-key" if key == "FAL_KEY" else "",
|
||||
)
|
||||
|
||||
config = {"video_gen": {"provider": "xai"}}
|
||||
providers = [
|
||||
{"name": "xAI Grok Imagine", "env_vars": [], "video_gen_plugin_name": "xai"},
|
||||
{
|
||||
"name": "FAL.ai",
|
||||
"env_vars": [{"key": "FAL_KEY", "prompt": "FAL"}],
|
||||
"video_gen_plugin_name": "fal",
|
||||
},
|
||||
]
|
||||
|
||||
assert tools_config._detect_active_provider_index(providers, config) == 0
|
||||
@@ -6,6 +6,8 @@ import os
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from hermes_cli.profiles import _get_default_hermes_home
|
||||
|
||||
import pytest
|
||||
|
||||
from plugins.memory.honcho.client import (
|
||||
@@ -349,18 +351,25 @@ class TestResolveConfigPath:
|
||||
result = resolve_config_path()
|
||||
assert result == local_cfg
|
||||
|
||||
def test_falls_back_to_global_when_no_local(self, tmp_path):
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir()
|
||||
# No honcho.json in HERMES_HOME — also isolate ~/.hermes so
|
||||
# the default-profile fallback doesn't hit the real filesystem.
|
||||
def test_falls_back_to_default_profile_when_no_local(self, tmp_path, monkeypatch):
|
||||
# Profile mode: HERMES_HOME points at ~/.hermes/profiles/<name>, so
|
||||
# _get_default_hermes_home() must resolve back to ~/.hermes — that's
|
||||
# the bug the HOME-anchored helper fixes (vs. blindly using Path.home()).
|
||||
fake_home = tmp_path / "fakehome"
|
||||
fake_home.mkdir()
|
||||
default_home = fake_home / ".hermes"
|
||||
profile_home = default_home / "profiles" / "work"
|
||||
profile_home.mkdir(parents=True)
|
||||
default_cfg = default_home / "honcho.json"
|
||||
default_cfg.write_text('{"apiKey": "default-key"}')
|
||||
|
||||
with patch.dict(os.environ, {"HERMES_HOME": str(hermes_home)}), \
|
||||
patch.object(Path, "home", return_value=fake_home):
|
||||
result = resolve_config_path()
|
||||
assert result == fake_home / ".honcho" / "config.json"
|
||||
monkeypatch.setattr(Path, "home", lambda: fake_home)
|
||||
monkeypatch.setenv("HERMES_HOME", str(profile_home))
|
||||
|
||||
result = resolve_config_path()
|
||||
|
||||
assert _get_default_hermes_home() == default_home
|
||||
assert result == default_cfg
|
||||
|
||||
def test_falls_back_to_global_without_hermes_home_env(self, tmp_path):
|
||||
fake_home = tmp_path / "fakehome"
|
||||
@@ -383,6 +392,28 @@ class TestResolveConfigPath:
|
||||
assert resolve_global_config_path() == fake_home / ".honcho" / "config.json"
|
||||
assert resolve_config_path() == fake_home / ".honcho" / "config.json"
|
||||
|
||||
def test_from_global_config_uses_default_profile_fallback(self, tmp_path, monkeypatch):
|
||||
# Profile mode: from_global_config() reads the default-profile honcho.json
|
||||
# via the HOME-anchored helper, not Path.home() / ".hermes".
|
||||
fake_home = tmp_path / "fakehome"
|
||||
fake_home.mkdir()
|
||||
default_home = fake_home / ".hermes"
|
||||
profile_home = default_home / "profiles" / "work"
|
||||
profile_home.mkdir(parents=True)
|
||||
default_cfg = default_home / "honcho.json"
|
||||
default_cfg.write_text(json.dumps({
|
||||
"apiKey": "default-key",
|
||||
"workspace": "default-ws",
|
||||
}))
|
||||
|
||||
monkeypatch.setattr(Path, "home", lambda: fake_home)
|
||||
monkeypatch.setenv("HERMES_HOME", str(profile_home))
|
||||
|
||||
config = HonchoClientConfig.from_global_config()
|
||||
|
||||
assert config.api_key == "default-key"
|
||||
assert config.workspace_id == "default-ws"
|
||||
|
||||
def test_from_global_config_uses_local_path(self, tmp_path):
|
||||
hermes_home = tmp_path / "hermes"
|
||||
hermes_home.mkdir()
|
||||
|
||||
@@ -72,10 +72,13 @@ class TestXAIImageGenProvider:
|
||||
|
||||
provider = XAIImageGenProvider()
|
||||
schema = provider.get_setup_schema()
|
||||
assert schema["name"] == "xAI (Grok)"
|
||||
assert schema["name"] == "xAI Grok Imagine (image)"
|
||||
assert schema["badge"] == "paid"
|
||||
assert len(schema["env_vars"]) == 1
|
||||
assert schema["env_vars"][0]["key"] == "XAI_API_KEY"
|
||||
# Auth resolution is delegated to the shared "xai_grok" post_setup
|
||||
# hook so the picker doesn't blindly prompt for XAI_API_KEY when the
|
||||
# user is already signed in via xAI Grok OAuth.
|
||||
assert schema["env_vars"] == []
|
||||
assert schema["post_setup"] == "xai_grok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import json
|
||||
import zipfile
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -156,6 +157,43 @@ def test_tool_add_resource_uploads_existing_local_directory_and_cleans_zip(tmp_p
|
||||
assert result["root_uri"] == "viking://resources/docs"
|
||||
|
||||
|
||||
def test_tool_add_resource_directory_zip_skips_symlink_escape(tmp_path):
|
||||
secret = tmp_path / "outside-secret.txt"
|
||||
secret.write_text("do not upload\n", encoding="utf-8")
|
||||
docs = tmp_path / "docs"
|
||||
docs.mkdir()
|
||||
(docs / "guide.md").write_text("# Guide\n", encoding="utf-8")
|
||||
link = docs / "leak.txt"
|
||||
try:
|
||||
link.symlink_to(secret)
|
||||
except OSError as exc:
|
||||
pytest.skip(f"symlinks unavailable in test environment: {exc}")
|
||||
|
||||
provider = OpenVikingMemoryProvider()
|
||||
provider._client = MagicMock()
|
||||
archive_entries = {}
|
||||
|
||||
def inspect_upload(path):
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
archive_entries["names"] = archive.namelist()
|
||||
archive_entries["payloads"] = {
|
||||
name: archive.read(name)
|
||||
for name in archive.namelist()
|
||||
}
|
||||
return "upload_docs.zip"
|
||||
|
||||
provider._client.upload_temp_file.side_effect = inspect_upload
|
||||
provider._client.post.return_value = {
|
||||
"status": "ok",
|
||||
"result": {"root_uri": "viking://resources/docs"},
|
||||
}
|
||||
|
||||
json.loads(provider._tool_add_resource({"url": str(docs)}))
|
||||
|
||||
assert archive_entries["names"] == ["guide.md"]
|
||||
assert b"do not upload" not in b"".join(archive_entries["payloads"].values())
|
||||
|
||||
|
||||
def test_tool_add_resource_cleans_local_directory_zip_when_add_fails(tmp_path):
|
||||
docs = tmp_path / "docs"
|
||||
docs.mkdir()
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -164,7 +165,542 @@ class TestHooksInert:
|
||||
|
||||
# Each hook should just return; no exceptions.
|
||||
mod.on_pre_llm_call(task_id="t", session_id="s", messages=[{"role": "user", "content": "hi"}])
|
||||
mod.on_pre_llm_request(task_id="t", session_id="s", api_call_count=1, messages=[])
|
||||
mod.on_pre_llm_request(task_id="t", session_id="s", api_call_count=1, request_messages=[])
|
||||
mod.on_post_llm_call(task_id="t", session_id="s", api_call_count=1)
|
||||
mod.on_pre_tool_call(tool_name="read_file", args={}, task_id="t", session_id="s")
|
||||
mod.on_post_tool_call(tool_name="read_file", args={}, result="ok", task_id="t", session_id="s")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Placeholder-credential guard (#23823).
|
||||
#
|
||||
# Regression coverage for the silent-failure bug: when an operator leaves
|
||||
# HERMES_LANGFUSE_PUBLIC_KEY / SECRET_KEY at a template value like
|
||||
# "placeholder", "test-key", or "your-langfuse-key", the SDK accepts the
|
||||
# credentials at construction time (it does no server-side validation
|
||||
# eagerly) but drops every trace at flush time, with no signal in the
|
||||
# Hermes logs. The fix in `_get_langfuse()` validates the documented
|
||||
# `pk-lf-` / `sk-lf-` prefix Langfuse always issues, surfaces a one-shot
|
||||
# warning naming the offending env var(s), and short-circuits via the
|
||||
# same `_INIT_FAILED` path used for missing credentials so subsequent
|
||||
# hook invocations don't re-log.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeLangfuse:
|
||||
"""Stand-in for the real :class:`langfuse.Langfuse` so tests don't
|
||||
need the optional ``langfuse`` SDK installed. The plugin's runtime
|
||||
gate refuses to proceed past ``if Langfuse is None`` when the SDK
|
||||
is missing, which would short-circuit before the placeholder check
|
||||
can fire. Patching ``plugin.Langfuse`` with this class lets the
|
||||
placeholder validator exercise its full code path."""
|
||||
|
||||
instances: list["_FakeLangfuse"] = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
_FakeLangfuse.instances.append(self)
|
||||
|
||||
|
||||
class TestPlaceholderKeyDetection:
|
||||
LOGGER_NAME = "plugins.observability.langfuse"
|
||||
|
||||
def _fresh_plugin(self, monkeypatch=None):
|
||||
mod_name = "plugins.observability.langfuse"
|
||||
sys.modules.pop(mod_name, None)
|
||||
mod = importlib.import_module(mod_name)
|
||||
if monkeypatch is not None:
|
||||
# Pretend the SDK is installed so `_get_langfuse()` actually
|
||||
# reaches the placeholder check. Real SDK calls are never
|
||||
# made because the placeholder/missing-credentials paths
|
||||
# return before constructing a client.
|
||||
_FakeLangfuse.instances.clear()
|
||||
monkeypatch.setattr(mod, "Langfuse", _FakeLangfuse, raising=False)
|
||||
return mod
|
||||
|
||||
@staticmethod
|
||||
def _clear_env(monkeypatch):
|
||||
for k in (
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "HERMES_LANGFUSE_SECRET_KEY",
|
||||
"LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
# -- helper unit tests (no SDK stub needed: these don't go through
|
||||
# _get_langfuse, they exercise the pure-Python helpers directly) ------
|
||||
|
||||
def test_redact_key_preview_empty(self, monkeypatch):
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
assert plugin._redact_key_preview("") == "<empty>"
|
||||
|
||||
def test_redact_key_preview_short_value_echoed(self, monkeypatch):
|
||||
"""Short placeholder strings are echoed in full so the operator
|
||||
can see exactly which template they forgot to replace."""
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
assert plugin._redact_key_preview("placeholder") == "'placeholder'"
|
||||
assert plugin._redact_key_preview("test-key") == "'test-key'"
|
||||
|
||||
def test_redact_key_preview_long_value_truncated(self, monkeypatch):
|
||||
"""If an operator pasted a real secret into the wrong env var the
|
||||
preview must NOT echo it in full — only the leading 6 chars."""
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
result = plugin._redact_key_preview("sk-lf-abcdefghijklmnop")
|
||||
assert "abcdefghij" not in result
|
||||
assert result.startswith("'sk-lf-")
|
||||
assert result.endswith("...'")
|
||||
|
||||
def test_validate_langfuse_key_accepts_documented_prefix(self, monkeypatch):
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
assert plugin._validate_langfuse_key(
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz"
|
||||
) is None
|
||||
assert plugin._validate_langfuse_key(
|
||||
"HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz"
|
||||
) is None
|
||||
|
||||
def test_validate_langfuse_key_rejects_wrong_prefix(self, monkeypatch):
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
msg = plugin._validate_langfuse_key(
|
||||
"HERMES_LANGFUSE_PUBLIC_KEY", "placeholder"
|
||||
)
|
||||
assert msg is not None
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in msg
|
||||
assert "pk-lf-" in msg
|
||||
|
||||
def test_validate_langfuse_key_unknown_name_passes(self, monkeypatch):
|
||||
"""Defensive: an env var with no registered prefix is trusted."""
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin()
|
||||
assert plugin._validate_langfuse_key("HERMES_LANGFUSE_BASE_URL", "anything") is None
|
||||
|
||||
# -- end-to-end _get_langfuse() behaviour --------------------------------
|
||||
# These tests pass `monkeypatch` to _fresh_plugin() so the helper can
|
||||
# stub out `Langfuse` (the optional SDK). Without that, every call
|
||||
# short-circuits at `if Langfuse is None` before reaching the
|
||||
# placeholder validator — masking the very behaviour we're testing.
|
||||
|
||||
def test_placeholder_public_key_warns_and_skips(self, monkeypatch, caplog):
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
text = caplog.text
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
|
||||
assert "'placeholder'" in text
|
||||
assert "pk-lf-" in text
|
||||
# The valid secret value must NOT appear (the var NAME does, in
|
||||
# the "or unset ..." hint, but the value preview shouldn't).
|
||||
assert "'sk-lf-" not in text
|
||||
# Never constructed the SDK client — short-circuited before that.
|
||||
assert _FakeLangfuse.instances == []
|
||||
|
||||
def test_placeholder_secret_key_warns_and_skips(self, monkeypatch, caplog):
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "test-key")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
text = caplog.text
|
||||
assert "HERMES_LANGFUSE_SECRET_KEY" in text
|
||||
assert "'test-key'" in text
|
||||
assert "sk-lf-" in text
|
||||
# The valid public value must NOT appear.
|
||||
assert "'pk-lf-" not in text
|
||||
assert _FakeLangfuse.instances == []
|
||||
|
||||
def test_both_placeholders_one_warning_with_both_keys(self, monkeypatch, caplog):
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
warnings = [r for r in caplog.records if r.levelname == "WARNING"
|
||||
and r.name == self.LOGGER_NAME]
|
||||
assert len(warnings) == 1, (
|
||||
f"Expected a single combined warning; got {len(warnings)}:\n"
|
||||
+ "\n".join(r.getMessage() for r in warnings)
|
||||
)
|
||||
text = warnings[0].getMessage()
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in text
|
||||
assert "HERMES_LANGFUSE_SECRET_KEY" in text
|
||||
|
||||
def test_repeated_calls_do_not_re_warn(self, monkeypatch, caplog):
|
||||
"""The cached ``_INIT_FAILED`` sentinel must short-circuit
|
||||
subsequent calls so each hook invocation isn't a fresh log
|
||||
line — otherwise a busy gateway will spam the operator's
|
||||
terminal."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
for _ in range(15):
|
||||
assert plugin._get_langfuse() is None
|
||||
warnings = [r for r in caplog.records if r.levelname == "WARNING"
|
||||
and r.name == self.LOGGER_NAME]
|
||||
assert len(warnings) == 1, (
|
||||
f"Warning fired {len(warnings)} times across 15 calls; "
|
||||
"expected 1 (cached via _INIT_FAILED)"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("placeholder", [
|
||||
"placeholder",
|
||||
"test-key",
|
||||
"your-langfuse-key",
|
||||
"change-me",
|
||||
"xxx",
|
||||
"dummy-key-here",
|
||||
"<your-key>",
|
||||
"REPLACE_ME",
|
||||
])
|
||||
def test_common_placeholders_detected(self, monkeypatch, caplog, placeholder):
|
||||
"""A grab-bag of values that real-world ``.env.example`` templates
|
||||
use as stand-ins. Any of them in either key must trip the guard."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", placeholder)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
|
||||
|
||||
def test_legacy_LANGFUSE_PUBLIC_KEY_also_validated(self, monkeypatch, caplog):
|
||||
"""The plugin reads both the canonical HERMES_-prefixed env var and
|
||||
the legacy bare ``LANGFUSE_PUBLIC_KEY``. The validator must run on
|
||||
whichever value ``_get_langfuse()`` actually consumed."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
# Warning names the canonical user-facing env var (the bare
|
||||
# LANGFUSE_PUBLIC_KEY is a backwards-compat alias for the
|
||||
# HERMES_-prefixed one — operators set the HERMES_-prefixed one).
|
||||
assert "HERMES_LANGFUSE_PUBLIC_KEY" in caplog.text
|
||||
assert "'placeholder'" in caplog.text
|
||||
|
||||
def test_missing_credentials_still_skip_silently(self, monkeypatch, caplog):
|
||||
"""Missing-creds is the documented opt-out path (operator hasn't
|
||||
configured the plugin yet) — it must remain SILENT. Regression
|
||||
guard against the placeholder validator accidentally running on
|
||||
empty values and re-introducing log noise for unconfigured
|
||||
installs."""
|
||||
self._clear_env(monkeypatch)
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
warnings = [r for r in caplog.records if r.levelname == "WARNING"
|
||||
and r.name == self.LOGGER_NAME]
|
||||
assert warnings == []
|
||||
|
||||
def test_sdk_not_installed_still_skips_silently(self, monkeypatch, caplog):
|
||||
"""If the langfuse SDK isn't installed at all, the placeholder
|
||||
check should never run — there's nothing the operator can do
|
||||
about a credential mismatch when the package is missing, and
|
||||
re-warning here would dilute the actually-actionable SDK-missing
|
||||
signal upstream. The ``Langfuse is None`` guard at the top of
|
||||
``_get_langfuse`` already handles this; this test pins that
|
||||
behaviour."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "placeholder")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "placeholder")
|
||||
# NO monkeypatch on Langfuse here — falls back to whatever the
|
||||
# plugin imported at module load (None if SDK absent).
|
||||
plugin = self._fresh_plugin()
|
||||
monkeypatch.setattr(plugin, "Langfuse", None, raising=False)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
assert plugin._get_langfuse() is None
|
||||
warnings = [r for r in caplog.records if r.levelname == "WARNING"
|
||||
and r.name == self.LOGGER_NAME]
|
||||
assert warnings == []
|
||||
|
||||
def test_valid_prefixes_do_not_trigger_placeholder_warning(self, monkeypatch, caplog):
|
||||
"""Real Langfuse keys (``pk-lf-…`` / ``sk-lf-…``) must pass the
|
||||
guard and proceed to SDK init. We stub the SDK constructor with
|
||||
a recording fake so the assertion can confirm BOTH that the
|
||||
placeholder warning didn't fire AND that the client was actually
|
||||
constructed — the latter is the success signal the bug report
|
||||
wanted."""
|
||||
self._clear_env(monkeypatch)
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_PUBLIC_KEY", "pk-lf-real-public-xyz")
|
||||
monkeypatch.setenv("HERMES_LANGFUSE_SECRET_KEY", "sk-lf-real-secret-xyz")
|
||||
plugin = self._fresh_plugin(monkeypatch)
|
||||
with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):
|
||||
client = plugin._get_langfuse()
|
||||
assert isinstance(client, _FakeLangfuse)
|
||||
assert client.kwargs["public_key"] == "pk-lf-real-public-xyz"
|
||||
assert client.kwargs["secret_key"] == "sk-lf-real-secret-xyz"
|
||||
assert "placeholders" not in caplog.text.lower(), (
|
||||
f"Valid Langfuse keys tripped the placeholder guard: {caplog.text!r}"
|
||||
)
|
||||
|
||||
|
||||
class TestRequestMessageCoercion:
|
||||
def test_prefers_request_messages_then_messages_then_history_then_user_message(self):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
assert mod._coerce_request_messages(
|
||||
request_messages=[{"role": "system", "content": "s"}],
|
||||
messages=[{"role": "user", "content": "m"}],
|
||||
conversation_history=[{"role": "user", "content": "h"}],
|
||||
user_message="u",
|
||||
) == [{"role": "system", "content": "s"}]
|
||||
assert mod._coerce_request_messages(
|
||||
messages=[{"role": "user", "content": "m"}],
|
||||
conversation_history=[{"role": "user", "content": "h"}],
|
||||
user_message="u",
|
||||
) == [{"role": "user", "content": "m"}]
|
||||
assert mod._coerce_request_messages(
|
||||
conversation_history=[{"role": "user", "content": "h"}],
|
||||
user_message="u",
|
||||
) == [{"role": "user", "content": "h"}]
|
||||
assert mod._coerce_request_messages(user_message="u") == [{"role": "user", "content": "u"}]
|
||||
|
||||
|
||||
class TestToolCallOutputBackfill:
|
||||
def test_post_tool_call_backfills_matching_turn_tool_call_output(self, monkeypatch):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
observation = object()
|
||||
state = mod.TraceState(trace_id="trace-1", root_ctx=None, root_span=None)
|
||||
state.tools["call-1"] = observation
|
||||
state.turn_tool_calls.append({
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"name": "web_extract",
|
||||
"arguments": '{"urls": ["https://example.com"]}',
|
||||
"function": {
|
||||
"name": "web_extract",
|
||||
"arguments": '{"urls": ["https://example.com"]}',
|
||||
},
|
||||
})
|
||||
|
||||
task_key = mod._trace_key("task-1", "session-1")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
ended = {}
|
||||
|
||||
def fake_end_observation(obs, *, output=None, metadata=None, usage_details=None, cost_details=None):
|
||||
ended["observation"] = obs
|
||||
ended["output"] = output
|
||||
ended["metadata"] = metadata
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end_observation)
|
||||
|
||||
mod.on_post_tool_call(
|
||||
tool_name="web_extract",
|
||||
args={"urls": ["https://example.com"]},
|
||||
result='{"results": [{"url": "https://example.com", "content": "Example Domain"}]}',
|
||||
task_id="task-1",
|
||||
session_id="session-1",
|
||||
tool_call_id="call-1",
|
||||
)
|
||||
|
||||
assert ended["observation"] is observation
|
||||
assert state.turn_tool_calls[0]["output"] == ended["output"]
|
||||
assert state.turn_tool_calls[0]["function"]["output"] == ended["output"]
|
||||
assert state.turn_tool_calls[0]["output"] == {
|
||||
"results": [{"url": "https://example.com", "content": "Example Domain"}]
|
||||
}
|
||||
|
||||
def test_serialize_messages_keeps_tool_name_and_call_id(self):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
messages = [{
|
||||
"role": "tool",
|
||||
"name": "web_extract",
|
||||
"tool_call_id": "call-1",
|
||||
"content": '{"ok": true}',
|
||||
}]
|
||||
|
||||
assert mod._serialize_messages(messages) == [{
|
||||
"role": "tool",
|
||||
"name": "web_extract",
|
||||
"tool_call_id": "call-1",
|
||||
"content": {"ok": True},
|
||||
}]
|
||||
|
||||
def test_serialize_tool_calls_emits_openai_style_function_shape(self):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
mod = importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
class _Fn:
|
||||
name = "web_extract"
|
||||
arguments = '{"urls": ["https://example.com"]}'
|
||||
|
||||
class _ToolCall:
|
||||
id = "call-1"
|
||||
type = "function"
|
||||
function = _Fn()
|
||||
|
||||
assert mod._serialize_tool_calls([_ToolCall()]) == [{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"name": "web_extract",
|
||||
"arguments": '{"urls": ["https://example.com"]}',
|
||||
"function": {
|
||||
"name": "web_extract",
|
||||
"arguments": '{"urls": ["https://example.com"]}',
|
||||
},
|
||||
}]
|
||||
|
||||
|
||||
class TestToolObservationKeying:
|
||||
"""Tests for pre/post tool_call observation matching when tool_call_id is absent."""
|
||||
|
||||
def _make_mod(self):
|
||||
sys.modules.pop("plugins.observability.langfuse", None)
|
||||
return importlib.import_module("plugins.observability.langfuse")
|
||||
|
||||
def test_empty_tool_call_id_single_tool_sets_output(self, monkeypatch):
|
||||
mod = self._make_mod()
|
||||
obs = object()
|
||||
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
|
||||
state.pending_tools_by_name.setdefault("my_tool", []).append(obs)
|
||||
|
||||
task_key = mod._trace_key("task-1", "sess-1")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
ended = {}
|
||||
|
||||
def fake_end(o, *, output=None, metadata=None, **kw):
|
||||
ended["obs"] = o
|
||||
ended["output"] = output
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end)
|
||||
|
||||
mod.on_post_tool_call(
|
||||
tool_name="my_tool",
|
||||
args={},
|
||||
result='{"ok": true}',
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
tool_call_id="",
|
||||
)
|
||||
|
||||
assert ended["obs"] is obs
|
||||
assert ended["output"] == {"ok": True}
|
||||
assert state.pending_tools_by_name.get("my_tool") is None
|
||||
|
||||
def test_empty_tool_call_id_observations_are_fifo_within_tool_name(self, monkeypatch):
|
||||
"""Two queued observations are consumed in FIFO order so the first
|
||||
post hook gets the first observation's output, not the second.
|
||||
|
||||
Sequential-on-one-thread coverage; the real concurrent case is
|
||||
guarded by ``_STATE_LOCK`` around every read-modify-write on
|
||||
``pending_tools_by_name`` and is exercised in
|
||||
``test_threaded_post_calls_preserve_fifo_under_lock`` below.
|
||||
"""
|
||||
mod = self._make_mod()
|
||||
obs_a, obs_b = object(), object()
|
||||
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
|
||||
state.pending_tools_by_name["web_extract"] = [obs_a, obs_b]
|
||||
|
||||
task_key = mod._trace_key("task-1", "sess-1")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_end(o, *, output=None, metadata=None, **kw):
|
||||
calls.append((o, output))
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end)
|
||||
|
||||
mod.on_post_tool_call(
|
||||
tool_name="web_extract", args={}, result='{"val": "a"}',
|
||||
task_id="task-1", session_id="sess-1", tool_call_id="",
|
||||
)
|
||||
mod.on_post_tool_call(
|
||||
tool_name="web_extract", args={}, result='{"val": "b"}',
|
||||
task_id="task-1", session_id="sess-1", tool_call_id="",
|
||||
)
|
||||
|
||||
assert calls[0] == (obs_a, {"val": "a"})
|
||||
assert calls[1] == (obs_b, {"val": "b"})
|
||||
assert state.pending_tools_by_name.get("web_extract") is None
|
||||
|
||||
def test_threaded_post_calls_preserve_fifo_under_lock(self, monkeypatch):
|
||||
"""The actual concurrency contract: when 8 threads race to drain
|
||||
the pending queue, no observation is consumed twice and none is
|
||||
lost. Validates ``_STATE_LOCK`` discipline, not Python list
|
||||
semantics."""
|
||||
import threading
|
||||
|
||||
mod = self._make_mod()
|
||||
n = 8
|
||||
observations = [object() for _ in range(n)]
|
||||
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
|
||||
state.pending_tools_by_name["web_extract"] = list(observations)
|
||||
|
||||
task_key = mod._trace_key("task-thr", "sess-thr")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
recorded: list = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def fake_end(o, *, output=None, metadata=None, **kw):
|
||||
with lock:
|
||||
recorded.append(o)
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end)
|
||||
|
||||
barrier = threading.Barrier(n)
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
mod.on_post_tool_call(
|
||||
tool_name="web_extract", args={}, result='{"ok": true}',
|
||||
task_id="task-thr", session_id="sess-thr", tool_call_id="",
|
||||
)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(n)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# Every observation was consumed exactly once; queue is empty.
|
||||
assert len(recorded) == n
|
||||
assert set(map(id, recorded)) == set(map(id, observations))
|
||||
assert state.pending_tools_by_name.get("web_extract") is None
|
||||
|
||||
def test_explicit_tool_call_id_uses_tools_dict(self, monkeypatch):
|
||||
"""When tool_call_id is present, pending_tools_by_name is not touched."""
|
||||
mod = self._make_mod()
|
||||
obs = object()
|
||||
state = mod.TraceState(trace_id="t", root_ctx=None, root_span=None)
|
||||
state.tools["call-99"] = obs
|
||||
|
||||
task_key = mod._trace_key("task-1", "sess-1")
|
||||
monkeypatch.setitem(mod._TRACE_STATE, task_key, state)
|
||||
|
||||
ended = {}
|
||||
|
||||
def fake_end(o, *, output=None, metadata=None, **kw):
|
||||
ended["obs"] = o
|
||||
ended["output"] = output
|
||||
|
||||
monkeypatch.setattr(mod, "_end_observation", fake_end)
|
||||
|
||||
mod.on_post_tool_call(
|
||||
tool_name="my_tool", args={}, result='{"status": "done"}',
|
||||
task_id="task-1", session_id="sess-1", tool_call_id="call-99",
|
||||
)
|
||||
|
||||
assert ended["obs"] is obs
|
||||
assert ended["output"] == {"status": "done"}
|
||||
assert not state.tools
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Make tests/plugins/video_gen a package."""
|
||||
@@ -0,0 +1,314 @@
|
||||
"""Tests for the FAL video gen plugin — family routing, payload shape."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
def test_fal_provider_registers():
|
||||
from plugins.video_gen.fal import FALVideoGenProvider, DEFAULT_MODEL
|
||||
|
||||
provider = FALVideoGenProvider()
|
||||
video_gen_registry.register_provider(provider)
|
||||
|
||||
assert video_gen_registry.get_provider("fal") is provider
|
||||
assert provider.display_name == "FAL"
|
||||
# DEFAULT_MODEL is the cheap-tier default
|
||||
assert provider.default_model() == DEFAULT_MODEL
|
||||
assert DEFAULT_MODEL in {"pixverse-v6", "ltx-2.3"}
|
||||
|
||||
|
||||
def test_fal_family_catalog():
|
||||
"""Each family declares both endpoints. The catalog covers the
|
||||
cheap + premium tiers Teknium listed."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES
|
||||
|
||||
expected = {
|
||||
# cheap
|
||||
"ltx-2.3", "pixverse-v6",
|
||||
# premium
|
||||
"veo3.1", "seedance-2.0", "kling-v3-4k", "happy-horse",
|
||||
}
|
||||
assert expected.issubset(set(FAL_FAMILIES.keys())), (
|
||||
f"missing families: {expected - set(FAL_FAMILIES.keys())}"
|
||||
)
|
||||
for fid, meta in FAL_FAMILIES.items():
|
||||
assert meta.get("text_endpoint"), f"{fid} missing text_endpoint"
|
||||
assert meta.get("image_endpoint"), f"{fid} missing image_endpoint"
|
||||
assert meta["text_endpoint"] != meta["image_endpoint"]
|
||||
assert meta.get("tier") in {"cheap", "premium"}, (
|
||||
f"{fid} has invalid tier"
|
||||
)
|
||||
|
||||
|
||||
def test_kling_4k_uses_start_image_url():
|
||||
"""Kling v3 4K's image-to-video endpoint expects start_image_url,
|
||||
not image_url. The family must declare image_param_key='start_image_url'."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["kling-v3-4k"]
|
||||
assert meta.get("image_param_key") == "start_image_url"
|
||||
payload = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url="https://example.com/i.png",
|
||||
duration=5,
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt=None,
|
||||
audio=None,
|
||||
seed=None,
|
||||
)
|
||||
assert payload.get("start_image_url") == "https://example.com/i.png"
|
||||
assert "image_url" not in payload
|
||||
|
||||
|
||||
def test_fal_list_models_advertises_both_modalities():
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
models = FALVideoGenProvider().list_models()
|
||||
for m in models:
|
||||
assert set(m["modalities"]) == {"text", "image"}, (
|
||||
f"{m['id']} doesn't advertise both modalities — every family "
|
||||
f"should have t2v + i2v"
|
||||
)
|
||||
|
||||
|
||||
def test_fal_unavailable_without_key(monkeypatch):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
monkeypatch.delenv("FAL_KEY", raising=False)
|
||||
assert FALVideoGenProvider().is_available() is False
|
||||
|
||||
|
||||
def test_fal_generate_requires_fal_key(monkeypatch):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
monkeypatch.delenv("FAL_KEY", raising=False)
|
||||
result = FALVideoGenProvider().generate("a happy dog")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "auth_required"
|
||||
|
||||
|
||||
class TestFamilyRouting:
|
||||
"""The headline behavior: image_url presence picks the endpoint."""
|
||||
|
||||
@pytest.fixture
|
||||
def with_fake_fal(self, monkeypatch):
|
||||
"""Stub fal_client.subscribe to capture which endpoint we hit."""
|
||||
import sys
|
||||
import types
|
||||
|
||||
captured = {"endpoint": None, "arguments": None}
|
||||
|
||||
fake = types.ModuleType("fal_client")
|
||||
def _subscribe(endpoint, arguments=None, with_logs=False):
|
||||
captured["endpoint"] = endpoint
|
||||
captured["arguments"] = arguments
|
||||
return {"video": {"url": "https://fake/out.mp4"}}
|
||||
fake.subscribe = _subscribe # type: ignore
|
||||
monkeypatch.setitem(sys.modules, "fal_client", fake)
|
||||
|
||||
# Reset the lazy global so it picks up our stub
|
||||
from plugins.video_gen import fal as fal_plugin
|
||||
fal_plugin._fal_client = None
|
||||
|
||||
monkeypatch.setenv("FAL_KEY", "test")
|
||||
return captured
|
||||
|
||||
def test_text_to_video_routes_to_text_endpoint(self, with_fake_fal):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"a dog running",
|
||||
model="pixverse-v6",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/text-to-video"
|
||||
assert result["modality"] == "text"
|
||||
assert with_fake_fal["arguments"]["prompt"] == "a dog running"
|
||||
assert "image_url" not in with_fake_fal["arguments"]
|
||||
|
||||
def test_image_to_video_routes_to_image_endpoint(self, with_fake_fal):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"animate this dog",
|
||||
model="pixverse-v6",
|
||||
image_url="https://example.com/dog.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert with_fake_fal["endpoint"] == "fal-ai/pixverse/v6/image-to-video"
|
||||
assert result["modality"] == "image"
|
||||
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
|
||||
|
||||
def test_default_family_text_routing(self, with_fake_fal):
|
||||
"""No model arg → DEFAULT_MODEL → text-to-video endpoint."""
|
||||
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
|
||||
|
||||
result = FALVideoGenProvider().generate("a dog")
|
||||
assert result["success"] is True
|
||||
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
|
||||
assert with_fake_fal["endpoint"] == expected_endpoint
|
||||
|
||||
def test_default_family_image_routing(self, with_fake_fal):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"animate this",
|
||||
image_url="https://example.com/i.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["image_endpoint"]
|
||||
assert with_fake_fal["endpoint"] == expected_endpoint
|
||||
|
||||
def test_unknown_family_falls_back_to_default(self, with_fake_fal):
|
||||
from plugins.video_gen.fal import FALVideoGenProvider, FAL_FAMILIES, DEFAULT_MODEL
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"x",
|
||||
model="not-a-real-family",
|
||||
)
|
||||
assert result["success"] is True
|
||||
expected_endpoint = FAL_FAMILIES[DEFAULT_MODEL]["text_endpoint"]
|
||||
assert with_fake_fal["endpoint"] == expected_endpoint
|
||||
|
||||
def test_premium_seedance_routing(self, with_fake_fal):
|
||||
"""Sanity check the premium-tier seedance routes correctly."""
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"a dog",
|
||||
model="seedance-2.0",
|
||||
image_url="https://example.com/dog.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert with_fake_fal["endpoint"] == "bytedance/seedance-2.0/image-to-video"
|
||||
# Seedance uses regular image_url (not start_image_url)
|
||||
assert with_fake_fal["arguments"]["image_url"] == "https://example.com/dog.png"
|
||||
|
||||
def test_kling_4k_remaps_image_param(self, with_fake_fal):
|
||||
"""Kling v3 4K image-to-video receives start_image_url, not image_url."""
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
result = FALVideoGenProvider().generate(
|
||||
"x",
|
||||
model="kling-v3-4k",
|
||||
image_url="https://example.com/frame.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert with_fake_fal["endpoint"] == "fal-ai/kling-video/v3/4k/image-to-video"
|
||||
assert with_fake_fal["arguments"].get("start_image_url") == "https://example.com/frame.png"
|
||||
assert "image_url" not in with_fake_fal["arguments"]
|
||||
|
||||
|
||||
class TestPayloadBuilder:
|
||||
def test_drops_unsupported_keys(self):
|
||||
"""Veo enum-clamps duration, supports aspect+resolution+audio+neg."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["veo3.1"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url=None,
|
||||
duration=12, # not in enum (4,6,8) — snap to 8
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt="ugly",
|
||||
audio=True,
|
||||
seed=42,
|
||||
)
|
||||
assert p["prompt"] == "x"
|
||||
assert p["duration"] == "8" # FAL queue API uses strings
|
||||
assert p["aspect_ratio"] == "16:9"
|
||||
assert p["resolution"] == "720p"
|
||||
assert p["generate_audio"] is True
|
||||
assert p["negative_prompt"] == "ugly"
|
||||
assert p["seed"] == 42
|
||||
|
||||
def test_pixverse_range_clamps_correctly(self):
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["pixverse-v6"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url="https://i.png",
|
||||
duration=99, # over max → 15
|
||||
aspect_ratio="16:9",
|
||||
resolution="540p",
|
||||
negative_prompt=None,
|
||||
audio=None,
|
||||
seed=None,
|
||||
)
|
||||
assert p["duration"] == "15"
|
||||
|
||||
def test_kling_4k_clamps_below_min(self):
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["kling-v3-4k"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url="https://i.png",
|
||||
duration=1, # below min (3) → 3
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt=None,
|
||||
audio=None,
|
||||
seed=None,
|
||||
)
|
||||
assert p["duration"] == "3"
|
||||
|
||||
def test_ltx_omits_duration_aspect_resolution(self):
|
||||
"""LTX 2.3 doesn't declare duration/aspect/resolution enums —
|
||||
the payload should NOT include those keys (let FAL default)."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["ltx-2.3"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="x",
|
||||
image_url=None,
|
||||
duration=8,
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt="ugly",
|
||||
audio=True,
|
||||
seed=None,
|
||||
)
|
||||
assert "duration" not in p
|
||||
assert "aspect_ratio" not in p
|
||||
assert "resolution" not in p
|
||||
# But audio + negative are advertised
|
||||
assert p["generate_audio"] is True
|
||||
assert p["negative_prompt"] == "ugly"
|
||||
|
||||
def test_happy_horse_minimal_payload(self):
|
||||
"""Happy Horse has sparse docs — payload should be minimal."""
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, _build_payload
|
||||
|
||||
meta = FAL_FAMILIES["happy-horse"]
|
||||
p = _build_payload(
|
||||
meta,
|
||||
prompt="a horse galloping",
|
||||
image_url=None,
|
||||
duration=8,
|
||||
aspect_ratio="16:9",
|
||||
resolution="720p",
|
||||
negative_prompt="watermark",
|
||||
audio=True,
|
||||
seed=None,
|
||||
)
|
||||
# Only prompt — no payload bloat for fields we can't verify
|
||||
assert p == {"prompt": "a horse galloping"}
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Smoke tests for the xAI video gen plugin — load & register surface."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
def test_xai_provider_registers():
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
provider = XAIVideoGenProvider()
|
||||
video_gen_registry.register_provider(provider)
|
||||
|
||||
assert video_gen_registry.get_provider("xai") is provider
|
||||
assert provider.display_name == "xAI"
|
||||
assert provider.default_model() == "grok-imagine-video"
|
||||
|
||||
|
||||
def test_xai_capabilities_text_and_image_only():
|
||||
"""xAI was previously advertised with edit/extend operations. The
|
||||
simplified surface only exposes text-to-video and image-to-video —
|
||||
confirm those are the only modalities advertised."""
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
caps = XAIVideoGenProvider().capabilities()
|
||||
assert caps["modalities"] == ["text", "image"]
|
||||
# No 'operations' key in the simplified surface
|
||||
assert "operations" not in caps
|
||||
assert caps["max_reference_images"] == 7
|
||||
|
||||
|
||||
def test_xai_unavailable_without_key(monkeypatch):
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
assert XAIVideoGenProvider().is_available() is False
|
||||
|
||||
|
||||
def test_xai_generate_requires_xai_key(monkeypatch):
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
result = XAIVideoGenProvider().generate("a happy dog")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "auth_required"
|
||||
|
||||
|
||||
def test_xai_available_with_oauth_only(monkeypatch):
|
||||
"""The plugin must honour xAI Grok OAuth credentials, not just
|
||||
XAI_API_KEY. Otherwise the agent's tool-availability check filters
|
||||
``video_generate`` out of the toolbelt and the agent silently falls
|
||||
back to whatever skill advertises video generation (e.g. comfyui).
|
||||
"""
|
||||
import plugins.video_gen.xai as xai_plugin
|
||||
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
monkeypatch.setattr(
|
||||
"tools.xai_http.resolve_xai_http_credentials",
|
||||
lambda: {
|
||||
"provider": "xai-oauth",
|
||||
"api_key": "oauth-bearer-token",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
},
|
||||
)
|
||||
|
||||
assert xai_plugin.XAIVideoGenProvider().is_available() is True
|
||||
|
||||
|
||||
def test_xai_resolved_credentials_threaded_through_request(monkeypatch):
|
||||
"""OAuth-resolved creds must reach the HTTP layer — bug class where
|
||||
``is_available()`` says yes but the request still hits with no key.
|
||||
"""
|
||||
import plugins.video_gen.xai as xai_plugin
|
||||
|
||||
monkeypatch.delenv("XAI_API_KEY", raising=False)
|
||||
monkeypatch.setattr(
|
||||
"tools.xai_http.resolve_xai_http_credentials",
|
||||
lambda: {
|
||||
"provider": "xai-oauth",
|
||||
"api_key": "oauth-bearer-token",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
},
|
||||
)
|
||||
|
||||
api_key, base_url = xai_plugin._resolve_xai_credentials()
|
||||
assert api_key == "oauth-bearer-token"
|
||||
assert base_url == "https://api.x.ai/v1"
|
||||
headers = xai_plugin._xai_headers(api_key)
|
||||
assert headers["Authorization"] == "Bearer oauth-bearer-token"
|
||||
|
||||
|
||||
def test_xai_no_operation_kwarg():
|
||||
"""The ABC's generate() signature no longer accepts 'operation'.
|
||||
Passing it through **kwargs should be ignored (forward-compat)."""
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
# We're not actually hitting the network — just verify the call
|
||||
# doesn't TypeError on the unexpected kwarg.
|
||||
# Will fail with auth_required (no XAI_API_KEY), but should NOT
|
||||
# fail with TypeError.
|
||||
result = XAIVideoGenProvider().generate("x", operation="generate")
|
||||
assert result["success"] is False
|
||||
# auth_required, NOT some signature error
|
||||
assert result["error_type"] in ("auth_required", "api_error")
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Integration tests for the xAI video gen plugin's simplified surface.
|
||||
|
||||
xAI exposes only text-to-video and image-to-video through the unified
|
||||
``video_generate`` tool. We assert the endpoint hit and the payload shape
|
||||
because routing is the part most likely to break silently.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, status: int = 200, payload: Optional[Dict[str, Any]] = None):
|
||||
self.status_code = status
|
||||
self._payload = payload or {}
|
||||
self.text = json.dumps(self._payload)
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
import httpx
|
||||
raise httpx.HTTPStatusError("err", request=None, response=self) # type: ignore
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
def __init__(self):
|
||||
self.posts: List[Dict[str, Any]] = []
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
return None
|
||||
|
||||
async def post(self, url, headers=None, json=None, timeout=None):
|
||||
self.posts.append({"url": url, "json": json})
|
||||
return _FakeResponse(200, {"request_id": "req-123"})
|
||||
|
||||
async def get(self, url, headers=None, timeout=None):
|
||||
return _FakeResponse(200, {
|
||||
"status": "done",
|
||||
"video": {"url": "https://xai-cdn/out.mp4", "duration": 8},
|
||||
"model": "grok-imagine-video",
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def xai_provider(monkeypatch):
|
||||
monkeypatch.setenv("XAI_API_KEY", "test-key")
|
||||
|
||||
import plugins.video_gen.xai as xai_plugin
|
||||
|
||||
captured: Dict[str, _FakeAsyncClient] = {}
|
||||
|
||||
def _client_factory():
|
||||
captured["client"] = _FakeAsyncClient()
|
||||
return captured["client"]
|
||||
|
||||
monkeypatch.setattr(xai_plugin.httpx, "AsyncClient", _client_factory)
|
||||
|
||||
async def _no_sleep(*a, **k):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
|
||||
|
||||
provider = xai_plugin.XAIVideoGenProvider()
|
||||
return provider, captured
|
||||
|
||||
|
||||
def _last_post(captured) -> Dict[str, Any]:
|
||||
return captured["client"].posts[-1]
|
||||
|
||||
|
||||
class TestXAIEndpoint:
|
||||
"""xAI uses one endpoint — ``/videos/generations`` — for both modes."""
|
||||
|
||||
def test_text_to_video_hits_generations(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate("a dog on a skateboard")
|
||||
assert result["success"] is True
|
||||
assert _last_post(captured)["url"].endswith("/videos/generations")
|
||||
assert result["modality"] == "text"
|
||||
|
||||
def test_image_to_video_hits_generations(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate(
|
||||
"animate this",
|
||||
image_url="https://example.com/cat.png",
|
||||
)
|
||||
assert result["success"] is True
|
||||
assert _last_post(captured)["url"].endswith("/videos/generations")
|
||||
assert result["modality"] == "image"
|
||||
|
||||
|
||||
class TestXAIPayload:
|
||||
def test_text_payload_has_no_image_field(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate("a dog at sunset")
|
||||
payload = _last_post(captured)["json"]
|
||||
assert payload["prompt"] == "a dog at sunset"
|
||||
assert "image" not in payload
|
||||
assert "reference_images" not in payload
|
||||
|
||||
def test_image_payload_has_image_field(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate("animate this", image_url="https://example.com/cat.png")
|
||||
payload = _last_post(captured)["json"]
|
||||
assert payload["image"] == {"url": "https://example.com/cat.png"}
|
||||
|
||||
def test_reference_images_payload(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate(
|
||||
"keep this character",
|
||||
reference_image_urls=[
|
||||
"https://example.com/a.png",
|
||||
"https://example.com/b.png",
|
||||
],
|
||||
)
|
||||
payload = _last_post(captured)["json"]
|
||||
assert payload["reference_images"] == [
|
||||
{"url": "https://example.com/a.png"},
|
||||
{"url": "https://example.com/b.png"},
|
||||
]
|
||||
|
||||
|
||||
class TestXAIValidation:
|
||||
def test_missing_prompt_rejects(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate("")
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "missing_prompt"
|
||||
# Never hit the network
|
||||
assert "client" not in captured or not captured["client"].posts
|
||||
|
||||
def test_image_plus_refs_rejects(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate(
|
||||
"x",
|
||||
image_url="https://example.com/i.png",
|
||||
reference_image_urls=["https://example.com/r.png"],
|
||||
)
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "conflicting_inputs"
|
||||
assert "client" not in captured or not captured["client"].posts
|
||||
|
||||
def test_too_many_references_rejects(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
result = provider.generate(
|
||||
"x",
|
||||
reference_image_urls=[f"https://example.com/r{i}.png" for i in range(8)],
|
||||
)
|
||||
assert result["success"] is False
|
||||
assert result["error_type"] == "too_many_references"
|
||||
|
||||
|
||||
class TestXAIClamping:
|
||||
def test_duration_clamped_to_15(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate("x", duration=30)
|
||||
assert _last_post(captured)["json"]["duration"] == 15
|
||||
|
||||
def test_duration_clamped_when_refs_present(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate(
|
||||
"x",
|
||||
duration=15,
|
||||
reference_image_urls=["https://example.com/r.png"],
|
||||
)
|
||||
# refs present caps to 10
|
||||
assert _last_post(captured)["json"]["duration"] == 10
|
||||
|
||||
def test_invalid_aspect_ratio_soft_clamps(self, xai_provider):
|
||||
provider, captured = xai_provider
|
||||
provider.generate("x", aspect_ratio="21:9")
|
||||
assert _last_post(captured)["json"]["aspect_ratio"] == "16:9"
|
||||
@@ -0,0 +1,475 @@
|
||||
"""Plugin-side tests for the web search provider migration (PR #25182).
|
||||
|
||||
Covers:
|
||||
|
||||
- All seven bundled plugins (brave-free, ddgs, searxng, exa, parallel,
|
||||
tavily, firecrawl) instantiate and self-report the expected
|
||||
capabilities + ABC-derived defaults.
|
||||
- Each plugin's ``is_available()`` correctly reflects env-var presence.
|
||||
- The web_search_registry resolves an active provider in the documented
|
||||
scenarios (explicit config wins ignoring availability, fallback walks
|
||||
legacy preference filtered by availability, unknown name falls back).
|
||||
- Plugin response shapes match the legacy bit-for-bit contract.
|
||||
|
||||
Per the dev skill: these tests use *real* imports from the plugin
|
||||
modules — no mocking of provider classes themselves — so the test
|
||||
catches drift in the ABC interface, the registry, and the plugin
|
||||
glue layer simultaneously.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Strip every web-provider env var so is_available() returns False."""
|
||||
for k in (
|
||||
"BRAVE_SEARCH_API_KEY",
|
||||
"SEARXNG_URL",
|
||||
"TAVILY_API_KEY",
|
||||
"TAVILY_BASE_URL",
|
||||
"EXA_API_KEY",
|
||||
"PARALLEL_API_KEY",
|
||||
"PARALLEL_SEARCH_MODE",
|
||||
"FIRECRAWL_API_KEY",
|
||||
"FIRECRAWL_API_URL",
|
||||
"FIRECRAWL_GATEWAY_URL",
|
||||
"TOOL_GATEWAY_DOMAIN",
|
||||
"TOOL_GATEWAY_USER_TOKEN",
|
||||
):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
|
||||
|
||||
def _ensure_plugins_loaded() -> None:
|
||||
"""Idempotently load plugins so the registry is populated."""
|
||||
from hermes_cli.plugins import _ensure_plugins_discovered
|
||||
|
||||
_ensure_plugins_discovered()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-plugin discovery + capability flags
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Each test starts with a clean web-provider env."""
|
||||
_clear_web_env(monkeypatch)
|
||||
|
||||
|
||||
class TestBundledPluginsRegister:
|
||||
"""All seven bundled web plugins discover and register correctly."""
|
||||
|
||||
def test_all_seven_plugins_present_in_registry(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import list_providers
|
||||
|
||||
names = sorted(p.name for p in list_providers())
|
||||
assert names == [
|
||||
"brave-free",
|
||||
"ddgs",
|
||||
"exa",
|
||||
"firecrawl",
|
||||
"parallel",
|
||||
"searxng",
|
||||
"tavily",
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name,expected_search,expected_extract,expected_crawl",
|
||||
[
|
||||
("brave-free", True, False, False),
|
||||
("ddgs", True, False, False),
|
||||
("searxng", True, False, False),
|
||||
("exa", True, True, False),
|
||||
("parallel", True, True, False),
|
||||
("tavily", True, True, True),
|
||||
# firecrawl: search + extract + crawl. Crawl was originally
|
||||
# disabled in the migration (fell through to a legacy inline
|
||||
# path); the follow-up commit enabled it natively.
|
||||
("firecrawl", True, True, True),
|
||||
],
|
||||
)
|
||||
def test_capability_flags_match_spec(
|
||||
self,
|
||||
plugin_name: str,
|
||||
expected_search: bool,
|
||||
expected_extract: bool,
|
||||
expected_crawl: bool,
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None, f"plugin {plugin_name!r} not registered"
|
||||
assert provider.supports_search() is expected_search
|
||||
assert provider.supports_extract() is expected_extract
|
||||
assert provider.supports_crawl() is expected_crawl
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl"],
|
||||
)
|
||||
def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None
|
||||
assert provider.name == plugin_name
|
||||
assert provider.display_name # any non-empty string
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name",
|
||||
["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl"],
|
||||
)
|
||||
def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None:
|
||||
"""``get_setup_schema()`` returns a dict the picker can consume."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
provider = get_provider(plugin_name)
|
||||
assert provider is not None
|
||||
schema = provider.get_setup_schema()
|
||||
assert isinstance(schema, dict)
|
||||
assert "name" in schema
|
||||
assert "env_vars" in schema
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# is_available() behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsAvailable:
|
||||
"""Each plugin's ``is_available()`` returns False without env config."""
|
||||
|
||||
def test_brave_free_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("brave-free")
|
||||
assert p is not None
|
||||
assert p.is_available() is False # no BRAVE_SEARCH_API_KEY
|
||||
monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_searxng_requires_url(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("searxng")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("tavily")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("TAVILY_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("exa")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("EXA_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_parallel_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("parallel")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
monkeypatch.setenv("PARALLEL_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_firecrawl_requires_either_key_or_url(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("firecrawl")
|
||||
assert p is not None
|
||||
assert p.is_available() is False
|
||||
|
||||
# Either FIRECRAWL_API_KEY or FIRECRAWL_API_URL lights it up.
|
||||
monkeypatch.setenv("FIRECRAWL_API_KEY", "real")
|
||||
assert p.is_available() is True
|
||||
monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
|
||||
monkeypatch.setenv("FIRECRAWL_API_URL", "http://localhost:3002")
|
||||
assert p.is_available() is True
|
||||
|
||||
def test_ddgs_always_available_when_package_importable(self) -> None:
|
||||
"""DDGS is the always-on fallback — no API key required.
|
||||
|
||||
It may report unavailable if the ``ddgs`` package itself isn't
|
||||
installed in the env (legitimate — the plugin's post_setup hook
|
||||
triggers pip install on first selection). We only assert that
|
||||
is_available() doesn't raise.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("ddgs")
|
||||
assert p is not None
|
||||
# Truthy or falsy, just must not raise.
|
||||
_ = bool(p.is_available())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry resolution semantics (Option B — conservative smart fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistryResolution:
|
||||
"""``_resolve()`` follows explicit-config + availability-filtered fallback."""
|
||||
|
||||
def test_explicit_configured_provider_returned_even_when_unavailable(
|
||||
self,
|
||||
) -> None:
|
||||
"""Explicit ``web.search_backend`` wins regardless of is_available().
|
||||
|
||||
Without availability filtering on the explicit path, the dispatcher
|
||||
would silently switch backends; with this check the dispatcher
|
||||
surfaces a precise "FOO_API_KEY is not set" error instead.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import _resolve, get_provider
|
||||
|
||||
# No BRAVE_SEARCH_API_KEY (fixture cleared it).
|
||||
result = _resolve("brave-free", capability="search")
|
||||
assert result is not None
|
||||
assert result.name == "brave-free"
|
||||
# Confirm it's the unavailable one — dispatcher will surface
|
||||
# a typed credential-missing error to the caller.
|
||||
assert result.is_available() is False
|
||||
|
||||
def test_unknown_configured_name_falls_back_to_available_provider(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Typo / uninstalled plugin → walk legacy preference, pick available."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import _resolve
|
||||
|
||||
monkeypatch.setenv("EXA_API_KEY", "real")
|
||||
result = _resolve("not-a-real-provider", capability="search")
|
||||
# Either ddgs (no-key fallback) or exa (the only available
|
||||
# premium provider) — both are valid. The point is the unknown
|
||||
# name shouldn't return None when SOMETHING is available.
|
||||
assert result is not None
|
||||
assert result.is_available() is True
|
||||
|
||||
def test_explicit_search_only_provider_for_extract_falls_back(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Asking for extract via a search-only backend → fall back.
|
||||
|
||||
``brave-free`` is search-only (``supports_extract() is False``).
|
||||
When the registry resolves it for an extract capability, the
|
||||
explicit-config branch rejects it as capability-incompatible
|
||||
and the fallback walk picks an extract-capable provider.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import _resolve
|
||||
|
||||
monkeypatch.setenv("EXA_API_KEY", "real")
|
||||
result = _resolve("brave-free", capability="extract")
|
||||
# Should land on exa (only extract-capable available provider).
|
||||
assert result is not None
|
||||
assert result.supports_extract() is True
|
||||
assert result.is_available() is True
|
||||
|
||||
def test_no_config_no_credentials_returns_none(
|
||||
self,
|
||||
) -> None:
|
||||
"""No backend configured AND no available providers → typically None.
|
||||
|
||||
``ddgs`` is the no-credential fallback; if its ``ddgs`` Python
|
||||
package is installed in the test env, ddgs will be picked.
|
||||
Otherwise the resolver returns None. Either outcome is correct.
|
||||
"""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import _resolve
|
||||
|
||||
result = _resolve(None, capability="search")
|
||||
if result is not None:
|
||||
# The only no-credential provider is ddgs; anything else
|
||||
# means an env var leaked in.
|
||||
assert result.is_available() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sync-vs-async extract detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAsyncExtractDispatch:
|
||||
"""The dispatcher detects async vs sync extract methods correctly."""
|
||||
|
||||
def test_parallel_extract_is_async(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("parallel")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.extract) is True
|
||||
|
||||
def test_firecrawl_extract_is_async(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("firecrawl")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.extract) is True
|
||||
|
||||
def test_exa_extract_is_sync(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("exa")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.extract) is False
|
||||
|
||||
def test_tavily_extract_is_sync(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("tavily")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.extract) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error response shape (preserved bit-for-bit from legacy)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestErrorResponseShapes:
|
||||
"""When credentials are missing, plugins return typed errors, not raises."""
|
||||
|
||||
def test_brave_free_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("brave-free")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
|
||||
def test_searxng_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("searxng")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
|
||||
def test_exa_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("exa")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
|
||||
def test_tavily_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("tavily")
|
||||
assert p is not None
|
||||
result = p.search("test", limit=5)
|
||||
assert isinstance(result, dict)
|
||||
assert result.get("success") is False
|
||||
assert "error" in result
|
||||
|
||||
def test_parallel_extract_returns_per_url_errors_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("parallel")
|
||||
assert p is not None
|
||||
result = asyncio.run(p.extract(["https://example.com"]))
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 1
|
||||
assert "error" in result[0]
|
||||
assert result[0]["url"] == "https://example.com"
|
||||
|
||||
def test_firecrawl_extract_returns_per_url_errors_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("firecrawl")
|
||||
assert p is not None
|
||||
# firecrawl extract returns [] when the website-policy gate rejects
|
||||
# the URL, or a per-URL error dict when the gate passes but the
|
||||
# firecrawl client fails. Use a URL the policy allows to make sure
|
||||
# we hit the credential-missing path.
|
||||
result = asyncio.run(p.extract(["https://example.com"]))
|
||||
assert isinstance(result, list)
|
||||
if result: # if anything came back, it should be an error entry
|
||||
assert "error" in result[0]
|
||||
|
||||
def test_tavily_crawl_returns_error_dict_when_unconfigured(self) -> None:
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("tavily")
|
||||
assert p is not None
|
||||
result = p.crawl("https://example.com")
|
||||
assert isinstance(result, dict)
|
||||
assert "results" in result
|
||||
assert isinstance(result["results"], list)
|
||||
if result["results"]:
|
||||
assert "error" in result["results"][0]
|
||||
|
||||
def test_firecrawl_crawl_returns_error_dict_when_unconfigured(self) -> None:
|
||||
"""firecrawl crawl is async (wraps SDK in to_thread); error must be
|
||||
surfaced via the per-page result shape, not raised."""
|
||||
_ensure_plugins_loaded()
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
p = get_provider("firecrawl")
|
||||
assert p is not None
|
||||
assert inspect.iscoroutinefunction(p.crawl)
|
||||
result = asyncio.run(p.crawl("https://example.com"))
|
||||
assert isinstance(result, dict)
|
||||
assert "results" in result
|
||||
assert isinstance(result["results"], list)
|
||||
# Without FIRECRAWL_API_KEY, the plugin's _get_firecrawl_client()
|
||||
# raises ValueError which is caught and returned as a per-page error.
|
||||
assert len(result["results"]) >= 1
|
||||
assert "error" in result["results"][0]
|
||||
assert result["results"][0]["url"] == "https://example.com"
|
||||
@@ -46,14 +46,14 @@ def test_bundled_plugins_discovered():
|
||||
assert (child / "plugin.yaml").exists(), f"{child.name} missing plugin.yaml"
|
||||
|
||||
|
||||
def test_all_33_profiles_register():
|
||||
"""After discovery, the registry must contain exactly 33 distinct profiles."""
|
||||
def test_all_34_profiles_register():
|
||||
"""After discovery, the registry must contain exactly 34 distinct profiles."""
|
||||
_clear_provider_caches()
|
||||
from providers import list_providers
|
||||
|
||||
profiles = list_providers()
|
||||
names = sorted(p.name for p in profiles)
|
||||
assert len(names) == 33, f"Expected 33 profiles, got {len(names)}: {names}"
|
||||
assert len(names) == 34, f"Expected 34 profiles, got {len(names)}: {names}"
|
||||
|
||||
# Spot-check representative providers from different categories
|
||||
for required in (
|
||||
|
||||
@@ -415,6 +415,32 @@ class TestHTTP413Compression:
|
||||
class TestPreflightCompression:
|
||||
"""Preflight compression should compress history before the first API call."""
|
||||
|
||||
def test_compress_context_emits_lifecycle_status_before_work(self, agent):
|
||||
"""Direct context compression should tell gateway users why the turn paused."""
|
||||
events = []
|
||||
agent.status_callback = lambda ev, msg: events.append((ev, msg))
|
||||
|
||||
def _fake_compress(messages, current_tokens=None, focus_topic=None):
|
||||
events.append(("compress", "started"))
|
||||
return [{"role": "user", "content": f"{SUMMARY_PREFIX}\nPrevious conversation"}]
|
||||
|
||||
with (
|
||||
patch.object(agent.context_compressor, "compress", side_effect=_fake_compress),
|
||||
patch.object(agent, "_build_system_prompt", return_value="new system prompt"),
|
||||
patch("run_agent.estimate_request_tokens_rough", return_value=42),
|
||||
):
|
||||
compressed, new_system_prompt = agent._compress_context(
|
||||
[{"role": "user", "content": "hello"}],
|
||||
"system prompt",
|
||||
approx_tokens=1234,
|
||||
)
|
||||
|
||||
assert compressed == [{"role": "user", "content": f"{SUMMARY_PREFIX}\nPrevious conversation"}]
|
||||
assert new_system_prompt == "new system prompt"
|
||||
assert events[0][0] == "lifecycle"
|
||||
assert "Compacting context" in events[0][1]
|
||||
assert events[1] == ("compress", "started")
|
||||
|
||||
def test_preflight_compresses_oversized_history(self, agent):
|
||||
"""When loaded history exceeds the model's context threshold, compress before API call."""
|
||||
agent.compression_enabled = True
|
||||
|
||||
@@ -1,505 +0,0 @@
|
||||
"""
|
||||
Tests for environments/agent_loop.py — HermesAgentLoop.
|
||||
|
||||
Tests the multi-turn agent engine using mocked servers, without needing
|
||||
real API keys or running servers.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure repo root is importable
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
|
||||
|
||||
try:
|
||||
from environments.agent_loop import (
|
||||
AgentResult,
|
||||
HermesAgentLoop,
|
||||
ToolError,
|
||||
_extract_reasoning_from_message,
|
||||
resize_tool_pool,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("atroposlib not installed", allow_module_level=True)
|
||||
|
||||
|
||||
# ─── Mock server infrastructure ─────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockFunction:
|
||||
name: str
|
||||
arguments: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockToolCall:
|
||||
id: str
|
||||
function: MockFunction
|
||||
type: str = "function"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockMessage:
|
||||
content: Optional[str]
|
||||
role: str = "assistant"
|
||||
tool_calls: Optional[List[MockToolCall]] = None
|
||||
reasoning_content: Optional[str] = None
|
||||
reasoning: Optional[str] = None
|
||||
reasoning_details: Optional[list] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockChoice:
|
||||
message: MockMessage
|
||||
finish_reason: str = "stop"
|
||||
index: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockChatCompletion:
|
||||
choices: List[MockChoice]
|
||||
id: str = "chatcmpl-mock"
|
||||
model: str = "mock-model"
|
||||
|
||||
|
||||
class MockServer:
|
||||
"""
|
||||
Mock server that returns pre-configured responses in sequence.
|
||||
Mimics the chat_completion() interface.
|
||||
"""
|
||||
|
||||
def __init__(self, responses: List[MockChatCompletion]):
|
||||
self.responses = responses
|
||||
self.call_count = 0
|
||||
self.call_history: List[Dict[str, Any]] = []
|
||||
|
||||
async def chat_completion(self, **kwargs) -> MockChatCompletion:
|
||||
self.call_history.append(kwargs)
|
||||
if self.call_count >= len(self.responses):
|
||||
# Return a simple text response if we run out
|
||||
return MockChatCompletion(
|
||||
choices=[MockChoice(message=MockMessage(content="Done."))]
|
||||
)
|
||||
resp = self.responses[self.call_count]
|
||||
self.call_count += 1
|
||||
return resp
|
||||
|
||||
|
||||
def make_text_response(content: str) -> MockChatCompletion:
|
||||
"""Create a simple text-only response (no tool calls)."""
|
||||
return MockChatCompletion(
|
||||
choices=[MockChoice(message=MockMessage(content=content))]
|
||||
)
|
||||
|
||||
|
||||
def make_tool_response(
|
||||
tool_name: str,
|
||||
arguments: dict,
|
||||
content: str = "",
|
||||
tool_call_id: str = "call_001",
|
||||
) -> MockChatCompletion:
|
||||
"""Create a response with a single tool call."""
|
||||
return MockChatCompletion(
|
||||
choices=[
|
||||
MockChoice(
|
||||
message=MockMessage(
|
||||
content=content,
|
||||
tool_calls=[
|
||||
MockToolCall(
|
||||
id=tool_call_id,
|
||||
function=MockFunction(
|
||||
name=tool_name,
|
||||
arguments=json.dumps(arguments),
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
finish_reason="tool_calls",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ─── Tests ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestAgentResult:
|
||||
def test_defaults(self):
|
||||
result = AgentResult(messages=[])
|
||||
assert result.messages == []
|
||||
assert result.managed_state is None
|
||||
assert result.turns_used == 0
|
||||
assert result.finished_naturally is False
|
||||
assert result.reasoning_per_turn == []
|
||||
assert result.tool_errors == []
|
||||
|
||||
|
||||
class TestExtractReasoning:
|
||||
def test_reasoning_content_field(self):
|
||||
msg = MockMessage(content="hello", reasoning_content="I think...")
|
||||
assert _extract_reasoning_from_message(msg) == "I think..."
|
||||
|
||||
def test_reasoning_field(self):
|
||||
msg = MockMessage(content="hello", reasoning="Let me consider...")
|
||||
assert _extract_reasoning_from_message(msg) == "Let me consider..."
|
||||
|
||||
def test_reasoning_details(self):
|
||||
detail = MagicMock()
|
||||
detail.text = "Detail reasoning"
|
||||
msg = MockMessage(content="hello", reasoning_details=[detail])
|
||||
assert _extract_reasoning_from_message(msg) == "Detail reasoning"
|
||||
|
||||
def test_reasoning_details_dict_format(self):
|
||||
msg = MockMessage(
|
||||
content="hello",
|
||||
reasoning_details=[{"text": "Dict reasoning"}],
|
||||
)
|
||||
assert _extract_reasoning_from_message(msg) == "Dict reasoning"
|
||||
|
||||
def test_no_reasoning(self):
|
||||
msg = MockMessage(content="hello")
|
||||
assert _extract_reasoning_from_message(msg) is None
|
||||
|
||||
def test_reasoning_content_takes_priority(self):
|
||||
msg = MockMessage(
|
||||
content="hello",
|
||||
reasoning_content="First",
|
||||
reasoning="Second",
|
||||
)
|
||||
assert _extract_reasoning_from_message(msg) == "First"
|
||||
|
||||
|
||||
class TestHermesAgentLoop:
|
||||
"""Test the agent loop with mock servers."""
|
||||
|
||||
@pytest.fixture
|
||||
def basic_tools(self):
|
||||
"""Minimal tool schema for testing."""
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "terminal",
|
||||
"description": "Run a command",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"command": {
|
||||
"type": "string",
|
||||
"description": "Command to run",
|
||||
}
|
||||
},
|
||||
"required": ["command"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "read_file",
|
||||
"description": "Read a file",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string"},
|
||||
},
|
||||
"required": ["path"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
@pytest.fixture
|
||||
def valid_names(self):
|
||||
return {"terminal", "read_file", "todo"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_simple_text_response(self, basic_tools, valid_names):
|
||||
"""Model responds with text only, no tool calls."""
|
||||
server = MockServer([make_text_response("Hello! How can I help?")])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.finished_naturally is True
|
||||
assert result.turns_used == 1
|
||||
assert len(result.messages) >= 2 # user + assistant
|
||||
assert result.messages[-1]["role"] == "assistant"
|
||||
assert result.messages[-1]["content"] == "Hello! How can I help?"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_call_then_text(self, basic_tools, valid_names):
|
||||
"""Model calls a tool, then responds with text."""
|
||||
server = MockServer([
|
||||
make_tool_response("todo", {"todos": [{"id": "1", "content": "test", "status": "pending"}]}),
|
||||
make_text_response("I created a todo for you."),
|
||||
])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Create a todo"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.finished_naturally is True
|
||||
assert result.turns_used == 2
|
||||
# Should have: user, assistant (tool_call), tool (result), assistant (text)
|
||||
roles = [m["role"] for m in result.messages]
|
||||
assert roles == ["user", "assistant", "tool", "assistant"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_turns_reached(self, basic_tools, valid_names):
|
||||
"""Model keeps calling tools until max_turns is hit."""
|
||||
# Create responses that always call a tool
|
||||
responses = [
|
||||
make_tool_response("todo", {"todos": [{"id": str(i), "content": f"task {i}", "status": "pending"}]}, tool_call_id=f"call_{i}")
|
||||
for i in range(10)
|
||||
]
|
||||
server = MockServer(responses)
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=3,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Keep going"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.finished_naturally is False
|
||||
assert result.turns_used == 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_tool_name(self, basic_tools, valid_names):
|
||||
"""Model calls a tool not in valid_tool_names."""
|
||||
server = MockServer([
|
||||
make_tool_response("nonexistent_tool", {"arg": "val"}),
|
||||
make_text_response("OK, that didn't work."),
|
||||
])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Call something weird"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Should record a tool error
|
||||
assert len(result.tool_errors) >= 1
|
||||
assert result.tool_errors[0].tool_name == "nonexistent_tool"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_response(self, basic_tools, valid_names):
|
||||
"""Server returns empty response."""
|
||||
server = MockServer([MockChatCompletion(choices=[])])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.finished_naturally is False
|
||||
assert result.turns_used == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_error_handling(self, basic_tools, valid_names):
|
||||
"""Server raises an exception."""
|
||||
|
||||
class FailingServer:
|
||||
async def chat_completion(self, **kwargs):
|
||||
raise ConnectionError("Server unreachable")
|
||||
|
||||
agent = HermesAgentLoop(
|
||||
server=FailingServer(),
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.finished_naturally is False
|
||||
assert result.turns_used == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tools_passed_to_server(self, basic_tools, valid_names):
|
||||
"""Verify tools are passed in the chat_completion kwargs."""
|
||||
server = MockServer([make_text_response("OK")])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
await agent.run(messages)
|
||||
|
||||
assert len(server.call_history) == 1
|
||||
assert "tools" in server.call_history[0]
|
||||
assert server.call_history[0]["tools"] == basic_tools
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_forwarded(self, basic_tools, valid_names):
|
||||
"""extra_body should be forwarded to server."""
|
||||
extra = {"provider": {"ignore": ["DeepInfra"]}}
|
||||
server = MockServer([make_text_response("OK")])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
extra_body=extra,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
await agent.run(messages)
|
||||
|
||||
assert server.call_history[0].get("extra_body") == extra
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_managed_state_returned(self, basic_tools, valid_names):
|
||||
"""If server has get_state(), result should include managed_state."""
|
||||
server = MockServer([make_text_response("OK")])
|
||||
server.get_state = lambda: {"nodes": [{"test": True}]}
|
||||
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.managed_state is not None
|
||||
assert "nodes" in result.managed_state
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_managed_state_without_get_state(self, basic_tools, valid_names):
|
||||
"""Regular server without get_state() should return None managed_state."""
|
||||
server = MockServer([make_text_response("OK")])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.managed_state is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_tool_blocked(self, basic_tools):
|
||||
"""Memory tool should return error in RL environments."""
|
||||
valid = {"terminal", "read_file", "todo", "memory"}
|
||||
server = MockServer([
|
||||
make_tool_response("memory", {"action": "add", "target": "user", "content": "test"}),
|
||||
make_text_response("Done"),
|
||||
])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Remember this"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Find the tool response
|
||||
tool_msgs = [m for m in result.messages if m["role"] == "tool"]
|
||||
assert len(tool_msgs) >= 1
|
||||
tool_result = json.loads(tool_msgs[0]["content"])
|
||||
assert "error" in tool_result
|
||||
assert "not available" in tool_result["error"].lower()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_search_blocked(self, basic_tools):
|
||||
"""session_search should return error in RL environments."""
|
||||
valid = {"terminal", "read_file", "todo", "session_search"}
|
||||
server = MockServer([
|
||||
make_tool_response("session_search", {"query": "test"}),
|
||||
make_text_response("Done"),
|
||||
])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "Search sessions"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
tool_msgs = [m for m in result.messages if m["role"] == "tool"]
|
||||
assert len(tool_msgs) >= 1
|
||||
tool_result = json.loads(tool_msgs[0]["content"])
|
||||
assert "error" in tool_result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_content_preserved(self, basic_tools, valid_names):
|
||||
"""Reasoning content should be extracted and preserved."""
|
||||
resp = MockChatCompletion(
|
||||
choices=[
|
||||
MockChoice(
|
||||
message=MockMessage(
|
||||
content="The answer is 42.",
|
||||
reasoning_content="Let me think about this step by step...",
|
||||
)
|
||||
)
|
||||
]
|
||||
)
|
||||
server = MockServer([resp])
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=basic_tools,
|
||||
valid_tool_names=valid_names,
|
||||
max_turns=10,
|
||||
)
|
||||
messages = [{"role": "user", "content": "What is the meaning of life?"}]
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert len(result.reasoning_per_turn) == 1
|
||||
assert result.reasoning_per_turn[0] == "Let me think about this step by step..."
|
||||
|
||||
|
||||
class TestResizeToolPool:
|
||||
def test_resize_works(self):
|
||||
"""resize_tool_pool should not raise."""
|
||||
resize_tool_pool(16) # Small pool for testing
|
||||
resize_tool_pool(128) # Restore default
|
||||
|
||||
def test_resize_shuts_down_previous_executor(self, monkeypatch):
|
||||
"""Replacing the global tool executor should shut down the old pool."""
|
||||
import environments.agent_loop as agent_loop_module
|
||||
|
||||
old_executor = MagicMock()
|
||||
new_executor = MagicMock()
|
||||
|
||||
monkeypatch.setattr(agent_loop_module, "_tool_executor", old_executor)
|
||||
monkeypatch.setattr(
|
||||
agent_loop_module.concurrent.futures,
|
||||
"ThreadPoolExecutor",
|
||||
MagicMock(return_value=new_executor),
|
||||
)
|
||||
|
||||
resize_tool_pool(16)
|
||||
|
||||
old_executor.shutdown.assert_called_once_with(wait=False)
|
||||
assert agent_loop_module._tool_executor is new_executor
|
||||
@@ -1,552 +0,0 @@
|
||||
"""Integration tests for HermesAgentLoop tool calling.
|
||||
|
||||
Tests the full agent loop with real LLM calls via OpenRouter.
|
||||
Uses stepfun/step-3.5-flash:free by default (zero cost), falls back
|
||||
to anthropic/claude-sonnet-4 if the free model is unavailable.
|
||||
|
||||
These tests verify:
|
||||
1. Single tool call: model calls a tool, gets result, responds
|
||||
2. Multi-tool call: model calls multiple tools in one turn
|
||||
3. Multi-turn: model calls tools across multiple turns
|
||||
4. Unknown tool rejection: model calling a non-existent tool gets an error
|
||||
5. Max turns: loop stops when max_turns is reached
|
||||
6. No tools: model responds without calling any tools
|
||||
7. Tool error handling: tool execution errors are captured
|
||||
|
||||
Run:
|
||||
pytest tests/test_agent_loop_tool_calling.py -v
|
||||
pytest tests/test_agent_loop_tool_calling.py -v -k "single" # run one test
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Set
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
# pytestmark removed — tests skip gracefully via OPENROUTER_API_KEY check on line 59
|
||||
|
||||
# Ensure repo root is importable
|
||||
_repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(_repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(_repo_root))
|
||||
|
||||
try:
|
||||
from environments.agent_loop import AgentResult, HermesAgentLoop
|
||||
from atroposlib.envs.server_handling.openai_server import OpenAIServer # noqa: F401
|
||||
except ImportError:
|
||||
pytest.skip("atroposlib not installed", allow_module_level=True)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Test infrastructure
|
||||
# =========================================================================
|
||||
|
||||
# Models to try, in order of preference (free first)
|
||||
_MODELS = [
|
||||
"stepfun/step-3.5-flash:free",
|
||||
"google/gemini-2.0-flash-001",
|
||||
"anthropic/claude-sonnet-4",
|
||||
]
|
||||
|
||||
def _get_api_key():
|
||||
key = os.getenv("OPENROUTER_API_KEY", "")
|
||||
if not key:
|
||||
pytest.skip("OPENROUTER_API_KEY not set")
|
||||
return key
|
||||
|
||||
|
||||
def _make_server(model: str = None):
|
||||
"""Create an OpenAI server for testing."""
|
||||
from atroposlib.envs.server_handling.openai_server import OpenAIServer
|
||||
from atroposlib.envs.server_handling.server_manager import APIServerConfig
|
||||
|
||||
config = APIServerConfig(
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
model_name=model or _MODELS[0],
|
||||
server_type="openai",
|
||||
api_key=_get_api_key(),
|
||||
health_check=False,
|
||||
)
|
||||
return OpenAIServer(config)
|
||||
|
||||
|
||||
async def _try_models(test_fn):
|
||||
"""Try running a test with each model until one works."""
|
||||
last_error = None
|
||||
for model in _MODELS:
|
||||
try:
|
||||
server = _make_server(model)
|
||||
return await test_fn(server, model)
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
if "rate" in str(e).lower() or "limit" in str(e).lower():
|
||||
continue # Rate limited, try next model
|
||||
raise # Real error
|
||||
pytest.skip(f"All models failed. Last error: {last_error}")
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Fake tools for testing
|
||||
# =========================================================================
|
||||
|
||||
# Simple calculator tool
|
||||
CALC_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calculate",
|
||||
"description": "Calculate a math expression. Returns the numeric result.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"expression": {
|
||||
"type": "string",
|
||||
"description": "Math expression to evaluate, e.g. '2 + 3'"
|
||||
}
|
||||
},
|
||||
"required": ["expression"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# Weather lookup tool
|
||||
WEATHER_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a city. Returns temperature and conditions.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": "City name, e.g. 'Tokyo'"
|
||||
}
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# Lookup tool (always succeeds)
|
||||
LOOKUP_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"description": "Look up a fact. Returns a short answer string.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "What to look up"
|
||||
}
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# Error tool (always fails)
|
||||
ERROR_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "failing_tool",
|
||||
"description": "A tool that always fails with an error.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"input": {"type": "string"}
|
||||
},
|
||||
"required": ["input"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _fake_tool_handler(tool_name: str, args: Dict[str, Any], **kwargs) -> str:
|
||||
"""Handle fake tool calls for testing."""
|
||||
if tool_name == "calculate":
|
||||
expr = args.get("expression", "0")
|
||||
try:
|
||||
# Safe eval for simple math
|
||||
result = eval(expr, {"__builtins__": {}}, {})
|
||||
return json.dumps({"result": result})
|
||||
except Exception as e:
|
||||
return json.dumps({"error": str(e)})
|
||||
|
||||
elif tool_name == "get_weather":
|
||||
city = args.get("city", "Unknown")
|
||||
# Return canned weather
|
||||
return json.dumps({
|
||||
"city": city,
|
||||
"temperature": 22,
|
||||
"conditions": "sunny",
|
||||
"humidity": 45,
|
||||
})
|
||||
|
||||
elif tool_name == "lookup":
|
||||
query = args.get("query", "")
|
||||
return json.dumps({"answer": f"The answer to '{query}' is 42."})
|
||||
|
||||
elif tool_name == "failing_tool":
|
||||
raise RuntimeError("This tool always fails!")
|
||||
|
||||
return json.dumps({"error": f"Unknown tool: {tool_name}"})
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Tests
|
||||
# =========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_tool_call():
|
||||
"""Model should call a single tool, get the result, and respond."""
|
||||
|
||||
async def _run(server, model):
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[WEATHER_TOOL],
|
||||
valid_tool_names={"get_weather"},
|
||||
max_turns=5,
|
||||
temperature=0.0,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What's the weather in Tokyo? Use the get_weather tool."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert isinstance(result, AgentResult)
|
||||
assert result.turns_used >= 2, f"Expected at least 2 turns (tool call + response), got {result.turns_used}"
|
||||
|
||||
# Verify a tool call happened
|
||||
tool_calls_found = False
|
||||
for msg in result.messages:
|
||||
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
for tc in msg["tool_calls"]:
|
||||
if tc["function"]["name"] == "get_weather":
|
||||
tool_calls_found = True
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert "city" in args
|
||||
assert tool_calls_found, "Model should have called get_weather"
|
||||
|
||||
# Verify tool result is in conversation
|
||||
tool_results = [m for m in result.messages if m.get("role") == "tool"]
|
||||
assert len(tool_results) >= 1, "Should have at least one tool result"
|
||||
|
||||
# Verify the final response references the weather
|
||||
final_msg = result.messages[-1]
|
||||
assert final_msg["role"] == "assistant"
|
||||
assert final_msg["content"], "Final response should have content"
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_tool_single_turn():
|
||||
"""Model should call multiple tools in a single turn."""
|
||||
|
||||
async def _run(server, model):
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[WEATHER_TOOL, CALC_TOOL],
|
||||
valid_tool_names={"get_weather", "calculate"},
|
||||
max_turns=5,
|
||||
temperature=0.0,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": (
|
||||
"I need two things at once: "
|
||||
"1) What's the weather in Paris? Use get_weather. "
|
||||
"2) What is 15 * 7? Use calculate. "
|
||||
"Call BOTH tools in a single response."
|
||||
)},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Count distinct tools called
|
||||
tools_called = set()
|
||||
for msg in result.messages:
|
||||
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
for tc in msg["tool_calls"]:
|
||||
tools_called.add(tc["function"]["name"])
|
||||
|
||||
# At minimum, both tools should have been called (maybe in different turns)
|
||||
assert "get_weather" in tools_called, f"get_weather not called. Called: {tools_called}"
|
||||
assert "calculate" in tools_called, f"calculate not called. Called: {tools_called}"
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_turn_conversation():
|
||||
"""Agent should handle multiple turns of tool calls."""
|
||||
|
||||
async def _run(server, model):
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[LOOKUP_TOOL, CALC_TOOL],
|
||||
valid_tool_names={"lookup", "calculate"},
|
||||
max_turns=10,
|
||||
temperature=0.0,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": (
|
||||
"First, use the lookup tool to look up 'meaning of life'. "
|
||||
"Then use calculate to compute 6 * 7. "
|
||||
"Do these in separate tool calls, one at a time."
|
||||
)},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Should have used both tools
|
||||
tools_called = set()
|
||||
for msg in result.messages:
|
||||
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
for tc in msg["tool_calls"]:
|
||||
tools_called.add(tc["function"]["name"])
|
||||
|
||||
assert "lookup" in tools_called, f"lookup not called. Called: {tools_called}"
|
||||
assert "calculate" in tools_called, f"calculate not called. Called: {tools_called}"
|
||||
|
||||
# Should finish naturally
|
||||
assert result.finished_naturally, "Should finish naturally after answering"
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_tool_rejected():
|
||||
"""If the model calls a tool not in valid_tool_names, it gets an error."""
|
||||
|
||||
async def _run(server, model):
|
||||
# Only allow "calculate" but give schema for both
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[CALC_TOOL, WEATHER_TOOL],
|
||||
valid_tool_names={"calculate"}, # weather NOT allowed
|
||||
max_turns=5,
|
||||
temperature=0.0,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What's the weather in London? Use get_weather."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Check if get_weather was called and rejected
|
||||
if result.tool_errors:
|
||||
weather_errors = [e for e in result.tool_errors if e.tool_name == "get_weather"]
|
||||
assert len(weather_errors) > 0, "get_weather should have been rejected"
|
||||
assert "Unknown tool" in weather_errors[0].error
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_turns_limit():
|
||||
"""Agent should stop after max_turns even if model keeps calling tools."""
|
||||
|
||||
async def _run(server, model):
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[LOOKUP_TOOL],
|
||||
valid_tool_names={"lookup"},
|
||||
max_turns=2, # Very low limit
|
||||
temperature=0.0,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": (
|
||||
"Keep looking up facts. Look up 'fact 1', then 'fact 2', "
|
||||
"then 'fact 3', then 'fact 4'. Do them one at a time."
|
||||
)},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.turns_used <= 2, f"Should stop at max_turns=2, used {result.turns_used}"
|
||||
assert not result.finished_naturally, "Should NOT finish naturally (hit max_turns)"
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_tools_direct_response():
|
||||
"""When no tools are useful, model should respond directly."""
|
||||
|
||||
async def _run(server, model):
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[WEATHER_TOOL],
|
||||
valid_tool_names={"get_weather"},
|
||||
max_turns=5,
|
||||
temperature=0.0,
|
||||
max_tokens=200,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 2 + 2? Just answer directly, no tools needed."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.finished_naturally, "Should finish naturally with a direct response"
|
||||
assert result.turns_used == 1, f"Should take exactly 1 turn for a direct answer, took {result.turns_used}"
|
||||
|
||||
final = result.messages[-1]
|
||||
assert final["role"] == "assistant"
|
||||
assert final["content"], "Should have text content"
|
||||
assert "4" in final["content"], "Should contain the answer '4'"
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_error_handling():
|
||||
"""Tool execution errors should be captured and reported to the model."""
|
||||
|
||||
async def _run(server, model):
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[ERROR_TOOL],
|
||||
valid_tool_names={"failing_tool"},
|
||||
max_turns=5,
|
||||
temperature=0.0,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Please call the failing_tool with input 'test'."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# The tool error should be recorded
|
||||
assert len(result.tool_errors) >= 1, "Should have at least one tool error"
|
||||
assert "RuntimeError" in result.tool_errors[0].error or "always fails" in result.tool_errors[0].error
|
||||
|
||||
# The error should be in the conversation as a tool result
|
||||
tool_results = [m for m in result.messages if m.get("role") == "tool"]
|
||||
assert len(tool_results) >= 1
|
||||
error_result = json.loads(tool_results[0]["content"])
|
||||
assert "error" in error_result
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_result_structure():
|
||||
"""Verify the AgentResult has all expected fields populated."""
|
||||
|
||||
async def _run(server, model):
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[CALC_TOOL],
|
||||
valid_tool_names={"calculate"},
|
||||
max_turns=5,
|
||||
temperature=0.0,
|
||||
max_tokens=300,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 3 + 4? Use the calculate tool."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Structural checks
|
||||
assert isinstance(result, AgentResult)
|
||||
assert isinstance(result.messages, list)
|
||||
assert len(result.messages) >= 3, "Should have user + assistant(tool) + tool_result + assistant(final)"
|
||||
assert isinstance(result.turns_used, int)
|
||||
assert result.turns_used > 0
|
||||
assert isinstance(result.finished_naturally, bool)
|
||||
assert isinstance(result.tool_errors, list)
|
||||
assert isinstance(result.reasoning_per_turn, list)
|
||||
|
||||
# Messages should follow OpenAI format
|
||||
for msg in result.messages:
|
||||
assert "role" in msg, f"Message missing 'role': {msg}"
|
||||
assert msg["role"] in ("system", "user", "assistant", "tool"), f"Invalid role: {msg['role']}"
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conversation_history_preserved():
|
||||
"""The full conversation history should be in result.messages."""
|
||||
|
||||
async def _run(server, model):
|
||||
agent = HermesAgentLoop(
|
||||
server=server,
|
||||
tool_schemas=[WEATHER_TOOL],
|
||||
valid_tool_names={"get_weather"},
|
||||
max_turns=5,
|
||||
temperature=0.0,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful weather assistant."},
|
||||
{"role": "user", "content": "What's the weather in Berlin? Use get_weather."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# System message should be preserved
|
||||
assert result.messages[0]["role"] == "system"
|
||||
assert "weather assistant" in result.messages[0]["content"]
|
||||
|
||||
# User message should be preserved
|
||||
assert result.messages[1]["role"] == "user"
|
||||
assert "Berlin" in result.messages[1]["content"]
|
||||
|
||||
# Should have assistant + tool + assistant sequence
|
||||
roles = [m["role"] for m in result.messages]
|
||||
assert "tool" in roles, "Should have tool results in conversation"
|
||||
|
||||
return result
|
||||
|
||||
await _try_models(_run)
|
||||
@@ -1,359 +0,0 @@
|
||||
"""Integration tests for HermesAgentLoop with a local vLLM server.
|
||||
|
||||
Tests the full Phase 2 flow: ManagedServer + tool calling with a real
|
||||
vLLM backend, producing actual token IDs and logprobs for RL training.
|
||||
|
||||
Requires a running vLLM server. Start one from the atropos directory:
|
||||
|
||||
python -m example_trainer.vllm_api_server \
|
||||
--model Qwen/Qwen3-4B-Thinking-2507 \
|
||||
--port 9001 \
|
||||
--gpu-memory-utilization 0.8 \
|
||||
--max-model-len=32000
|
||||
|
||||
Tests are automatically skipped if the server is not reachable.
|
||||
|
||||
Run:
|
||||
pytest tests/test_agent_loop_vllm.py -v
|
||||
pytest tests/test_agent_loop_vllm.py -v -k "single"
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
# Ensure repo root is importable
|
||||
_repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(_repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(_repo_root))
|
||||
|
||||
try:
|
||||
from environments.agent_loop import AgentResult, HermesAgentLoop
|
||||
except ImportError:
|
||||
pytest.skip("atroposlib not installed", allow_module_level=True)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Configuration
|
||||
# =========================================================================
|
||||
|
||||
VLLM_HOST = "localhost"
|
||||
VLLM_PORT = 9001
|
||||
VLLM_BASE_URL = f"http://{VLLM_HOST}:{VLLM_PORT}"
|
||||
VLLM_MODEL = "Qwen/Qwen3-4B-Thinking-2507"
|
||||
|
||||
|
||||
def _vllm_is_running() -> bool:
|
||||
"""Check if the vLLM server is reachable."""
|
||||
try:
|
||||
r = requests.get(f"{VLLM_BASE_URL}/health", timeout=3)
|
||||
return r.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# Skip all tests in this module if vLLM is not running
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _vllm_is_running(),
|
||||
reason=(
|
||||
f"vLLM server not reachable at {VLLM_BASE_URL}. "
|
||||
"Start it with: python -m example_trainer.vllm_api_server "
|
||||
f"--model {VLLM_MODEL} --port {VLLM_PORT} "
|
||||
"--gpu-memory-utilization 0.8 --max-model-len=32000"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Server setup
|
||||
# =========================================================================
|
||||
|
||||
def _make_server_manager():
|
||||
"""Create a ServerManager pointing to the local vLLM server."""
|
||||
from atroposlib.envs.server_handling.server_manager import (
|
||||
ServerManager,
|
||||
APIServerConfig,
|
||||
)
|
||||
|
||||
config = APIServerConfig(
|
||||
base_url=VLLM_BASE_URL,
|
||||
model_name=VLLM_MODEL,
|
||||
server_type="vllm",
|
||||
health_check=False,
|
||||
)
|
||||
sm = ServerManager([config], tool_parser="hermes")
|
||||
sm.servers[0].server_healthy = True
|
||||
return sm
|
||||
|
||||
|
||||
def _get_tokenizer():
|
||||
"""Load the tokenizer for the model."""
|
||||
from transformers import AutoTokenizer
|
||||
return AutoTokenizer.from_pretrained(VLLM_MODEL)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Fake tools
|
||||
# =========================================================================
|
||||
|
||||
WEATHER_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a city. Returns temperature and conditions.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": "City name, e.g. 'Tokyo'",
|
||||
}
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
CALC_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calculate",
|
||||
"description": "Calculate a math expression. Returns the numeric result.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"expression": {
|
||||
"type": "string",
|
||||
"description": "Math expression, e.g. '2 + 3'",
|
||||
}
|
||||
},
|
||||
"required": ["expression"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _fake_tool_handler(tool_name: str, args: Dict[str, Any], **kwargs) -> str:
|
||||
"""Handle fake tool calls for testing."""
|
||||
if tool_name == "get_weather":
|
||||
city = args.get("city", "Unknown")
|
||||
return json.dumps({
|
||||
"city": city,
|
||||
"temperature": 22,
|
||||
"conditions": "sunny",
|
||||
"humidity": 45,
|
||||
})
|
||||
elif tool_name == "calculate":
|
||||
expr = args.get("expression", "0")
|
||||
try:
|
||||
result = eval(expr, {"__builtins__": {}}, {})
|
||||
return json.dumps({"result": result})
|
||||
except Exception as e:
|
||||
return json.dumps({"error": str(e)})
|
||||
return json.dumps({"error": f"Unknown tool: {tool_name}"})
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Tests
|
||||
# =========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vllm_single_tool_call():
|
||||
"""vLLM model calls a tool, gets result, responds — full Phase 2 flow."""
|
||||
sm = _make_server_manager()
|
||||
tokenizer = _get_tokenizer()
|
||||
|
||||
async with sm.managed_server(tokenizer=tokenizer) as managed:
|
||||
agent = HermesAgentLoop(
|
||||
server=managed,
|
||||
tool_schemas=[WEATHER_TOOL],
|
||||
valid_tool_names={"get_weather"},
|
||||
max_turns=5,
|
||||
temperature=0.6,
|
||||
max_tokens=1000,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What's the weather in Tokyo? Use the get_weather tool."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert isinstance(result, AgentResult)
|
||||
assert result.turns_used >= 2, f"Expected at least 2 turns, got {result.turns_used}"
|
||||
|
||||
# Verify tool call happened
|
||||
tool_calls_found = False
|
||||
for msg in result.messages:
|
||||
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
for tc in msg["tool_calls"]:
|
||||
if tc["function"]["name"] == "get_weather":
|
||||
tool_calls_found = True
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert "city" in args
|
||||
assert tool_calls_found, "Model should have called get_weather"
|
||||
|
||||
# Verify tool results in conversation
|
||||
tool_results = [m for m in result.messages if m.get("role") == "tool"]
|
||||
assert len(tool_results) >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vllm_multi_tool_calls():
|
||||
"""vLLM model calls multiple tools across turns."""
|
||||
sm = _make_server_manager()
|
||||
tokenizer = _get_tokenizer()
|
||||
|
||||
async with sm.managed_server(tokenizer=tokenizer) as managed:
|
||||
agent = HermesAgentLoop(
|
||||
server=managed,
|
||||
tool_schemas=[WEATHER_TOOL, CALC_TOOL],
|
||||
valid_tool_names={"get_weather", "calculate"},
|
||||
max_turns=10,
|
||||
temperature=0.6,
|
||||
max_tokens=1000,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": (
|
||||
"I need two things: "
|
||||
"1) What's the weather in Paris? Use get_weather. "
|
||||
"2) What is 15 * 7? Use calculate."
|
||||
)},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Both tools should be called
|
||||
tools_called = set()
|
||||
for msg in result.messages:
|
||||
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
for tc in msg["tool_calls"]:
|
||||
tools_called.add(tc["function"]["name"])
|
||||
|
||||
assert "get_weather" in tools_called, f"get_weather not called. Called: {tools_called}"
|
||||
assert "calculate" in tools_called, f"calculate not called. Called: {tools_called}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vllm_managed_server_produces_nodes():
|
||||
"""ManagedServer should produce SequenceNodes with tokens and logprobs."""
|
||||
sm = _make_server_manager()
|
||||
tokenizer = _get_tokenizer()
|
||||
|
||||
async with sm.managed_server(tokenizer=tokenizer) as managed:
|
||||
agent = HermesAgentLoop(
|
||||
server=managed,
|
||||
tool_schemas=[WEATHER_TOOL],
|
||||
valid_tool_names={"get_weather"},
|
||||
max_turns=5,
|
||||
temperature=0.6,
|
||||
max_tokens=1000,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What's the weather in Berlin? Use get_weather."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Get the managed state — should have SequenceNodes
|
||||
state = managed.get_state()
|
||||
|
||||
assert state is not None, "ManagedServer should return state"
|
||||
nodes = state.get("nodes", [])
|
||||
assert len(nodes) >= 1, f"Should have at least 1 node, got {len(nodes)}"
|
||||
|
||||
node = nodes[0]
|
||||
assert hasattr(node, "tokens"), "Node should have tokens"
|
||||
assert hasattr(node, "logprobs"), "Node should have logprobs"
|
||||
assert len(node.tokens) > 0, "Tokens should not be empty"
|
||||
assert len(node.logprobs) > 0, "Logprobs should not be empty"
|
||||
assert len(node.tokens) == len(node.logprobs), (
|
||||
f"Tokens ({len(node.tokens)}) and logprobs ({len(node.logprobs)}) should have same length"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vllm_no_tools_direct_response():
|
||||
"""vLLM model should respond directly when no tools are needed."""
|
||||
sm = _make_server_manager()
|
||||
tokenizer = _get_tokenizer()
|
||||
|
||||
async with sm.managed_server(tokenizer=tokenizer) as managed:
|
||||
agent = HermesAgentLoop(
|
||||
server=managed,
|
||||
tool_schemas=[WEATHER_TOOL],
|
||||
valid_tool_names={"get_weather"},
|
||||
max_turns=5,
|
||||
temperature=0.6,
|
||||
max_tokens=500,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 2 + 2? Answer directly, no tools."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
assert result.finished_naturally, "Should finish naturally"
|
||||
assert result.turns_used == 1, f"Should take 1 turn, took {result.turns_used}"
|
||||
|
||||
final = result.messages[-1]
|
||||
assert final["role"] == "assistant"
|
||||
assert final["content"], "Should have content"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vllm_thinking_content_extracted():
|
||||
"""Qwen3-Thinking model should produce reasoning content."""
|
||||
sm = _make_server_manager()
|
||||
tokenizer = _get_tokenizer()
|
||||
|
||||
async with sm.managed_server(
|
||||
tokenizer=tokenizer,
|
||||
preserve_think_blocks=True,
|
||||
) as managed:
|
||||
agent = HermesAgentLoop(
|
||||
server=managed,
|
||||
tool_schemas=[CALC_TOOL],
|
||||
valid_tool_names={"calculate"},
|
||||
max_turns=5,
|
||||
temperature=0.6,
|
||||
max_tokens=1000,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What is 123 * 456? Use the calculate tool."},
|
||||
]
|
||||
|
||||
with patch("environments.agent_loop.handle_function_call", side_effect=_fake_tool_handler):
|
||||
result = await agent.run(messages)
|
||||
|
||||
# Qwen3-Thinking should generate <think> blocks
|
||||
# Check if any content contains thinking markers
|
||||
has_thinking = False
|
||||
for msg in result.messages:
|
||||
content = msg.get("content", "") or ""
|
||||
if "<think>" in content or "</think>" in content:
|
||||
has_thinking = True
|
||||
break
|
||||
|
||||
# Also check reasoning_per_turn
|
||||
has_reasoning = any(r for r in result.reasoning_per_turn if r)
|
||||
|
||||
# At least one of these should be true for a thinking model
|
||||
assert has_thinking or has_reasoning, (
|
||||
"Qwen3-Thinking should produce <think> blocks or reasoning content"
|
||||
)
|
||||
@@ -59,7 +59,7 @@ class TestTruncatedAnthropicResponseNormalization:
|
||||
nr = get_transport("anthropic_messages").normalize_response(response)
|
||||
|
||||
# The continuation block checks these two attributes:
|
||||
# assistant_message.content → appended to truncated_response_prefix
|
||||
# assistant_message.content → appended to truncated_response_parts
|
||||
# assistant_message.tool_calls → guards the text-retry branch
|
||||
assert nr.content is not None
|
||||
assert "partial response" in nr.content
|
||||
|
||||
@@ -20,6 +20,9 @@ def _bare_agent() -> AIAgent:
|
||||
agent._memory_store = object()
|
||||
agent._memory_enabled = True
|
||||
agent._user_profile_enabled = False
|
||||
agent._cached_system_prompt = "test-cached-system-prompt"
|
||||
import datetime as _dt
|
||||
agent.session_start = _dt.datetime(2026, 1, 1, 12, 0, 0)
|
||||
agent._MEMORY_REVIEW_PROMPT = "review memory"
|
||||
agent._SKILL_REVIEW_PROMPT = "review skills"
|
||||
agent._COMBINED_REVIEW_PROMPT = "review both"
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Tests that the background review fork inherits the parent's cached system prompt.
|
||||
|
||||
Regression coverage for issue #25322 (and PR #17276's first root cause): the
|
||||
background review's outbound HTTP request must carry the same system bytes as
|
||||
the parent's so Anthropic/OpenRouter's exact-prefix cache key matches.
|
||||
|
||||
Without this, every review rebuilds the system prompt from scratch — fresh
|
||||
``_hermes_now()`` timestamp, fresh ``session_id``, and a different skills
|
||||
prompt under the (former) narrow toolset — and the prefix-cache miss costs
|
||||
roughly the full uncached system-prompt cost per nudge (~26% end-to-end on
|
||||
Sonnet 4.5 per the contributor's measurement).
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
def _make_agent_stub(agent_cls):
|
||||
"""Create a minimal AIAgent-like object with just enough state for _spawn_background_review."""
|
||||
agent = object.__new__(agent_cls)
|
||||
agent.model = "test-model"
|
||||
agent.platform = "test"
|
||||
agent.provider = "openai"
|
||||
agent.session_id = "sess-123"
|
||||
agent.quiet_mode = True
|
||||
agent._memory_store = None
|
||||
agent._memory_enabled = True
|
||||
agent._user_profile_enabled = False
|
||||
agent._memory_nudge_interval = 5
|
||||
agent._skill_nudge_interval = 5
|
||||
agent.background_review_callback = None
|
||||
agent.status_callback = None
|
||||
agent._cached_system_prompt = (
|
||||
"PARENT-SYSTEM-PROMPT-BYTES — must be inherited verbatim "
|
||||
"for prefix-cache parity"
|
||||
)
|
||||
import datetime as _dt
|
||||
agent.session_start = _dt.datetime(2026, 1, 1, 12, 0, 0)
|
||||
agent._MEMORY_REVIEW_PROMPT = "review memory"
|
||||
agent._SKILL_REVIEW_PROMPT = "review skills"
|
||||
agent._COMBINED_REVIEW_PROMPT = "review both"
|
||||
return agent
|
||||
|
||||
|
||||
class _SyncThread:
|
||||
"""Drop-in replacement for threading.Thread that runs the target inline."""
|
||||
|
||||
def __init__(self, *, target=None, daemon=None, name=None):
|
||||
self._target = target
|
||||
|
||||
def start(self):
|
||||
if self._target:
|
||||
self._target()
|
||||
|
||||
|
||||
class _ReviewAgentRecorder:
|
||||
"""Stand-in for the review-fork AIAgent that records the prompt assignment."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._cached_system_prompt = None
|
||||
self._memory_write_origin = None
|
||||
self._memory_write_context = None
|
||||
self._memory_store = None
|
||||
self._memory_enabled = None
|
||||
self._user_profile_enabled = None
|
||||
self._memory_nudge_interval = None
|
||||
self._skill_nudge_interval = None
|
||||
self.suppress_status_output = None
|
||||
|
||||
def run_conversation(self, *args, **kwargs):
|
||||
raise RuntimeError("stop after recording state — don't actually call the API")
|
||||
|
||||
def shutdown_memory_provider(self):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_review_fork_inherits_parent_cached_system_prompt():
|
||||
"""The review fork's _cached_system_prompt must equal the parent's byte-for-byte.
|
||||
|
||||
Anthropic's prefix cache keys on exact bytes; any divergence (timestamp
|
||||
minute tick, fresh session_id, narrower skills_prompt) shifts the key
|
||||
and forces a full re-cache. Inheriting the parent's cached prompt is
|
||||
the cheap, mechanical fix.
|
||||
"""
|
||||
import run_agent
|
||||
|
||||
agent = _make_agent_stub(run_agent.AIAgent)
|
||||
|
||||
captured = {}
|
||||
parent_prompt = agent._cached_system_prompt
|
||||
|
||||
# Hook the assignment site: record what gets put on the review agent.
|
||||
real_recorder_init = _ReviewAgentRecorder.__init__
|
||||
|
||||
def _recorder_init(self, *args, **kwargs):
|
||||
real_recorder_init(self, *args, **kwargs)
|
||||
# The actual production code assigns _cached_system_prompt AFTER __init__,
|
||||
# so we need to capture it on attribute set. Use a property-style sentinel
|
||||
# via __setattr__ on this instance.
|
||||
|
||||
with patch.object(run_agent, "AIAgent", _ReviewAgentRecorder), \
|
||||
patch("threading.Thread", _SyncThread):
|
||||
# Wrap the recorder's __setattr__ so we can see the _cached_system_prompt
|
||||
# write that _spawn_background_review performs after construction.
|
||||
orig_setattr = _ReviewAgentRecorder.__setattr__
|
||||
|
||||
def _spy_setattr(self, name, value):
|
||||
if name == "_cached_system_prompt":
|
||||
captured["written_prompt"] = value
|
||||
orig_setattr(self, name, value)
|
||||
|
||||
with patch.object(_ReviewAgentRecorder, "__setattr__", _spy_setattr):
|
||||
agent._spawn_background_review(
|
||||
messages_snapshot=[],
|
||||
review_memory=True,
|
||||
review_skills=False,
|
||||
)
|
||||
|
||||
assert "written_prompt" in captured, (
|
||||
"_spawn_background_review never assigned _cached_system_prompt on the review agent"
|
||||
)
|
||||
assert captured["written_prompt"] == parent_prompt, (
|
||||
f"Review fork's _cached_system_prompt diverged from parent's. "
|
||||
f"Got {captured['written_prompt']!r}, expected {parent_prompt!r}. "
|
||||
"This breaks Anthropic/OpenRouter prefix-cache parity (#25322)."
|
||||
)
|
||||
|
||||
|
||||
def test_review_fork_pins_session_start_and_session_id():
|
||||
"""Defensive complement to cached-system-prompt inheritance.
|
||||
|
||||
Even though ``_cached_system_prompt`` inheritance short-circuits the
|
||||
normal rebuild path, pinning ``session_start`` and ``session_id`` to
|
||||
the parent's guarantees byte-identical output from any code path that
|
||||
re-renders parts of the system prompt (compression, plugin hooks).
|
||||
"""
|
||||
import run_agent
|
||||
|
||||
agent = _make_agent_stub(run_agent.AIAgent)
|
||||
|
||||
captured = {}
|
||||
|
||||
class _Recorder:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._cached_system_prompt = None
|
||||
self._memory_write_origin = None
|
||||
self._memory_write_context = None
|
||||
self._memory_store = None
|
||||
self._memory_enabled = None
|
||||
self._user_profile_enabled = None
|
||||
self._memory_nudge_interval = None
|
||||
self._skill_nudge_interval = None
|
||||
self.suppress_status_output = None
|
||||
self.session_start = None
|
||||
self.session_id = None
|
||||
|
||||
def run_conversation(self, *args, **kwargs):
|
||||
captured["session_start"] = self.session_start
|
||||
captured["session_id"] = self.session_id
|
||||
raise RuntimeError("stop after recording")
|
||||
|
||||
def shutdown_memory_provider(self):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
with patch.object(run_agent, "AIAgent", _Recorder), \
|
||||
patch("threading.Thread", _SyncThread):
|
||||
agent._spawn_background_review(
|
||||
messages_snapshot=[],
|
||||
review_memory=True,
|
||||
review_skills=False,
|
||||
)
|
||||
|
||||
assert captured.get("session_start") == agent.session_start, (
|
||||
"Review fork did not inherit parent's session_start — "
|
||||
"system-prompt rebuild paths would diverge."
|
||||
)
|
||||
assert captured.get("session_id") == agent.session_id, (
|
||||
"Review fork did not inherit parent's session_id — "
|
||||
"system-prompt rebuild paths would diverge."
|
||||
)
|
||||
@@ -1,8 +1,16 @@
|
||||
"""Tests that the background review agent is restricted to memory+skills toolsets.
|
||||
"""Tests that the background review agent restricts tools at runtime, not at schema time.
|
||||
|
||||
Regression coverage for issue #15204: the background skill-review agent
|
||||
inherited the full default toolset, allowing it to perform non-skill side
|
||||
effects (terminal, send_message, delegate_task, etc.).
|
||||
Regression coverage for issue #15204 (the background skill-review agent must
|
||||
not perform non-skill side effects like terminal, send_message, delegate_task)
|
||||
combined with issue #25322 / PR #17276 (the review fork must hit the parent's
|
||||
Anthropic/OpenRouter prefix cache).
|
||||
|
||||
Reconciling the two: the fork now inherits the parent's full ``tools`` schema
|
||||
so the cache-key matches, and enforces the memory+skills restriction at
|
||||
runtime via a thread-local whitelist on the existing
|
||||
``get_pre_tool_call_block_message`` gate. Safety is preserved mechanically
|
||||
(any non-whitelisted dispatch is blocked) without the schema-level narrowing
|
||||
that caused the prefix-cache miss.
|
||||
"""
|
||||
|
||||
import threading
|
||||
@@ -24,6 +32,9 @@ def _make_agent_stub(agent_cls):
|
||||
agent._skill_nudge_interval = 5
|
||||
agent.background_review_callback = None
|
||||
agent.status_callback = None
|
||||
agent._cached_system_prompt = None
|
||||
import datetime as _dt
|
||||
agent.session_start = _dt.datetime(2026, 1, 1, 12, 0, 0)
|
||||
agent._MEMORY_REVIEW_PROMPT = "review memory"
|
||||
agent._SKILL_REVIEW_PROMPT = "review skills"
|
||||
agent._COMBINED_REVIEW_PROMPT = "review both"
|
||||
@@ -41,15 +52,20 @@ class _SyncThread:
|
||||
self._target()
|
||||
|
||||
|
||||
def test_background_review_agent_uses_restricted_toolsets():
|
||||
"""The review agent must only have access to 'memory' and 'skills' toolsets."""
|
||||
def test_background_review_does_not_narrow_toolset_schema():
|
||||
"""The review fork must NOT pass enabled_toolsets to AIAgent.
|
||||
|
||||
Narrowing the schema diverges the ``tools`` cache key from the parent's,
|
||||
which sits above ``system`` in Anthropic's cache hierarchy and forces a
|
||||
full prefix-cache miss on every review (see #25322, PR #17276).
|
||||
"""
|
||||
import run_agent
|
||||
|
||||
agent = _make_agent_stub(run_agent.AIAgent)
|
||||
captured = {}
|
||||
|
||||
def _capture_init(self, *args, **kwargs):
|
||||
captured["enabled_toolsets"] = kwargs.get("enabled_toolsets")
|
||||
captured["enabled_toolsets"] = kwargs.get("enabled_toolsets", "UNSET")
|
||||
raise RuntimeError("stop after capturing init args")
|
||||
|
||||
with patch.object(run_agent.AIAgent, "__init__", _capture_init), \
|
||||
@@ -61,11 +77,71 @@ def test_background_review_agent_uses_restricted_toolsets():
|
||||
)
|
||||
|
||||
assert "enabled_toolsets" in captured, "AIAgent.__init__ was not called"
|
||||
assert sorted(captured["enabled_toolsets"]) == ["memory", "skills"]
|
||||
# The kwarg must be absent — letting AIAgent inherit the default full
|
||||
# toolset so the schema bytes match the parent's.
|
||||
assert captured["enabled_toolsets"] == "UNSET", (
|
||||
f"Review fork narrowed the toolset schema (got {captured['enabled_toolsets']!r}), "
|
||||
"which breaks prefix-cache parity with the parent."
|
||||
)
|
||||
|
||||
|
||||
def test_background_review_installs_thread_local_whitelist():
|
||||
"""The review fork must install a memory/skills-only thread-local whitelist.
|
||||
|
||||
The schema-level toolset narrowing was lifted (for prefix-cache parity),
|
||||
so #15204's safety contract now relies on the runtime whitelist gate to
|
||||
deny terminal/send_message/delegate_task at dispatch time. Verify the
|
||||
whitelist is set with exactly the memory+skills tool names.
|
||||
"""
|
||||
import run_agent
|
||||
from hermes_cli import plugins as _plugins
|
||||
|
||||
captured = {}
|
||||
|
||||
def _capture_whitelist(whitelist, deny_msg_fmt=None):
|
||||
captured["whitelist"] = set(whitelist)
|
||||
captured["deny_msg_fmt"] = deny_msg_fmt
|
||||
# Stop here — we just want to see what gets installed.
|
||||
raise RuntimeError("stop after capturing whitelist")
|
||||
|
||||
agent = _make_agent_stub(run_agent.AIAgent)
|
||||
|
||||
def _no_init(self, *args, **kwargs):
|
||||
# Don't crash AIAgent.__init__; let execution flow reach
|
||||
# set_thread_tool_whitelist.
|
||||
return None
|
||||
|
||||
with patch.object(run_agent.AIAgent, "__init__", _no_init), \
|
||||
patch.object(_plugins, "set_thread_tool_whitelist", _capture_whitelist), \
|
||||
patch("threading.Thread", _SyncThread):
|
||||
agent._spawn_background_review(
|
||||
messages_snapshot=[],
|
||||
review_memory=True,
|
||||
review_skills=False,
|
||||
)
|
||||
|
||||
assert "whitelist" in captured, "set_thread_tool_whitelist was not called"
|
||||
whitelist = captured["whitelist"]
|
||||
# memory + skills tools must be allowed
|
||||
assert "memory" in whitelist
|
||||
assert "skill_manage" in whitelist
|
||||
assert "skill_view" in whitelist
|
||||
assert "skills_list" in whitelist
|
||||
# dangerous tools must NOT be in the whitelist
|
||||
assert "terminal" not in whitelist
|
||||
assert "send_message" not in whitelist
|
||||
assert "delegate_task" not in whitelist
|
||||
assert "web_search" not in whitelist
|
||||
assert "execute_code" not in whitelist
|
||||
|
||||
|
||||
def test_background_review_agent_tools_are_limited():
|
||||
"""Verify the resolved memory+skills toolsets only contain memory and skill tools."""
|
||||
"""Verify the resolved memory+skills toolsets only contain memory and skill tools.
|
||||
|
||||
Sanity check on the source of truth for what the runtime whitelist is
|
||||
derived from — if a future PR adds e.g. `terminal` to the `memory`
|
||||
toolset, the review-fork safety contract silently breaks.
|
||||
"""
|
||||
from toolsets import resolve_multiple_toolsets
|
||||
|
||||
expected_tools = set(resolve_multiple_toolsets(["memory", "skills"]))
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
"""Integration test for the codex_app_server runtime path through AIAgent.
|
||||
|
||||
Verifies that:
|
||||
- api_mode='codex_app_server' is accepted on AIAgent construction
|
||||
- run_conversation() takes the early-return path and never enters the
|
||||
chat completions loop
|
||||
- Projected messages from a fake Codex session land in the messages list
|
||||
- tool_iterations from the codex session tick the skill nudge counter
|
||||
- Memory nudge counter ticks once per turn
|
||||
- The returned dict has the same shape as the chat_completions path
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import run_agent
|
||||
from agent.transports.codex_app_server_session import CodexAppServerSession, TurnResult
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_session(monkeypatch):
|
||||
"""Replace CodexAppServerSession with a stub that returns a fixed
|
||||
TurnResult, so we can drive AIAgent without spawning real codex."""
|
||||
|
||||
def fake_run_turn(self, user_input: str, **kwargs):
|
||||
return TurnResult(
|
||||
final_text=f"echo: {user_input}",
|
||||
projected_messages=[
|
||||
{"role": "assistant", "content": None,
|
||||
"tool_calls": [{"id": "exec_1", "type": "function",
|
||||
"function": {"name": "exec_command",
|
||||
"arguments": "{}"}}]},
|
||||
{"role": "tool", "tool_call_id": "exec_1", "content": "ok"},
|
||||
{"role": "assistant", "content": f"echo: {user_input}"},
|
||||
],
|
||||
tool_iterations=1,
|
||||
interrupted=False,
|
||||
error=None,
|
||||
turn_id="turn-stub-1",
|
||||
thread_id="thread-stub-1",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(CodexAppServerSession, "run_turn", fake_run_turn)
|
||||
monkeypatch.setattr(
|
||||
CodexAppServerSession, "ensure_started", lambda self: "thread-stub-1"
|
||||
)
|
||||
|
||||
|
||||
def _make_codex_agent():
|
||||
"""Construct an AIAgent in codex_app_server mode without contacting any
|
||||
real provider. We pass api_mode explicitly so the constructor takes the
|
||||
fast path for direct credentials."""
|
||||
return run_agent.AIAgent(
|
||||
api_key="stub",
|
||||
base_url="https://stub.invalid",
|
||||
provider="openai",
|
||||
api_mode="codex_app_server",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
|
||||
|
||||
class TestApiModeAccepted:
|
||||
def test_api_mode_is_codex_app_server(self):
|
||||
agent = _make_codex_agent()
|
||||
assert agent.api_mode == "codex_app_server"
|
||||
|
||||
|
||||
class TestRunConversationCodexPath:
|
||||
def test_run_conversation_returns_codex_shape(self, fake_session):
|
||||
agent = _make_codex_agent()
|
||||
# No background review fork during tests
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
result = agent.run_conversation("hello there")
|
||||
assert result["final_response"] == "echo: hello there"
|
||||
assert result["completed"] is True
|
||||
assert result["partial"] is False
|
||||
assert result["error"] is None
|
||||
assert result["api_calls"] == 1
|
||||
assert result["codex_thread_id"] == "thread-stub-1"
|
||||
assert result["codex_turn_id"] == "turn-stub-1"
|
||||
|
||||
def test_projected_messages_are_spliced(self, fake_session):
|
||||
agent = _make_codex_agent()
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
result = agent.run_conversation("hello")
|
||||
msgs = result["messages"]
|
||||
# User message + 3 projected (assistant tool_call + tool + assistant text)
|
||||
assert len(msgs) >= 4
|
||||
assert msgs[0]["role"] == "user"
|
||||
assert msgs[0]["content"] == "hello"
|
||||
# Last assistant message has the final text
|
||||
final = [m for m in msgs if m.get("role") == "assistant"
|
||||
and m.get("content") == "echo: hello"]
|
||||
assert final, f"expected final assistant message in {msgs}"
|
||||
|
||||
def test_nudge_counters_tick(self, fake_session):
|
||||
"""The skill nudge counter must accumulate tool_iterations across
|
||||
turns. The memory nudge counter is gated on memory being configured
|
||||
(which we skip via skip_memory=True), so we don't assert on it here —
|
||||
a separate test below covers that path explicitly."""
|
||||
agent = _make_codex_agent()
|
||||
agent._iters_since_skill = 0
|
||||
agent._user_turn_count = 0
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
agent.run_conversation("first")
|
||||
assert agent._iters_since_skill == 1 # one tool_iteration in fake turn
|
||||
# _user_turn_count is incremented by run_conversation pre-loop, not
|
||||
# by the codex helper — confirms we delegate that to the standard flow.
|
||||
assert agent._user_turn_count == 1
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
agent.run_conversation("second")
|
||||
assert agent._iters_since_skill == 2
|
||||
assert agent._user_turn_count == 2
|
||||
|
||||
def test_user_message_not_duplicated(self, fake_session):
|
||||
"""Regression guard: the user message must appear exactly once in
|
||||
the messages list. The standard run_conversation pre-loop appends
|
||||
it, and the codex helper must NOT append again."""
|
||||
agent = _make_codex_agent()
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
result = agent.run_conversation("ping unique 12345")
|
||||
user_count = sum(
|
||||
1 for m in result["messages"]
|
||||
if m.get("role") == "user" and m.get("content") == "ping unique 12345"
|
||||
)
|
||||
assert user_count == 1, f"user message appeared {user_count}× in {result['messages']}"
|
||||
|
||||
def test_background_review_NOT_invoked_below_threshold(self, fake_session):
|
||||
"""A single turn shouldn't trigger background review — counters
|
||||
haven't reached the nudge interval (default 10)."""
|
||||
agent = _make_codex_agent()
|
||||
agent._memory_nudge_interval = 10
|
||||
agent._skill_nudge_interval = 10
|
||||
agent._iters_since_skill = 0
|
||||
with patch.object(agent, "_spawn_background_review",
|
||||
return_value=None) as spawn:
|
||||
agent.run_conversation("ping")
|
||||
# Below threshold → review should NOT fire (was a real bug:
|
||||
# the helper was calling _spawn_background_review() with no
|
||||
# args after every turn, which would crash with TypeError).
|
||||
assert not spawn.called
|
||||
|
||||
def test_background_review_skill_trigger_fires_above_threshold(
|
||||
self, monkeypatch
|
||||
):
|
||||
"""When tool iterations cross the skill nudge interval, the
|
||||
background review fires with review_skills=True and the right
|
||||
messages_snapshot signature."""
|
||||
from agent.transports.codex_app_server_session import (
|
||||
CodexAppServerSession, TurnResult,
|
||||
)
|
||||
# Make the fake session report 10 tool iterations in one turn
|
||||
# (matching the default skill threshold).
|
||||
def fake_run_turn(self, user_input: str, **kwargs):
|
||||
return TurnResult(
|
||||
final_text=f"echo: {user_input}",
|
||||
projected_messages=[
|
||||
{"role": "assistant", "content": f"echo: {user_input}"},
|
||||
],
|
||||
tool_iterations=10,
|
||||
turn_id="t1", thread_id="th1",
|
||||
)
|
||||
monkeypatch.setattr(CodexAppServerSession, "run_turn", fake_run_turn)
|
||||
monkeypatch.setattr(
|
||||
CodexAppServerSession, "ensure_started", lambda self: "th1"
|
||||
)
|
||||
|
||||
agent = _make_codex_agent()
|
||||
agent._skill_nudge_interval = 10
|
||||
agent._iters_since_skill = 0
|
||||
# Make valid_tool_names include 'skill_manage' so the gate passes
|
||||
agent.valid_tool_names = set(getattr(agent, "valid_tool_names", set()))
|
||||
agent.valid_tool_names.add("skill_manage")
|
||||
|
||||
with patch.object(agent, "_spawn_background_review",
|
||||
return_value=None) as spawn:
|
||||
agent.run_conversation("do tool work")
|
||||
|
||||
assert spawn.called, "skill threshold tripped but review didn't fire"
|
||||
# Verify the call signature matches what _spawn_background_review
|
||||
# actually expects — this is the regression guard for the original
|
||||
# bug where the codex path called it with no args at all.
|
||||
call = spawn.call_args
|
||||
assert "messages_snapshot" in call.kwargs
|
||||
assert isinstance(call.kwargs["messages_snapshot"], list)
|
||||
assert call.kwargs["review_skills"] is True
|
||||
# Counter should be reset after the review fires
|
||||
assert agent._iters_since_skill == 0
|
||||
|
||||
def test_background_review_signature_never_breaks(self, fake_session):
|
||||
"""Even when no trigger fires, the helper must never call
|
||||
_spawn_background_review with the wrong signature. Run a turn,
|
||||
then run another turn after manually tripping the skill counter
|
||||
and confirm the call shape is the kwargs-only form the function
|
||||
actually accepts."""
|
||||
agent = _make_codex_agent()
|
||||
agent._skill_nudge_interval = 1 # very low so any iter trips it
|
||||
agent._iters_since_skill = 0
|
||||
agent.valid_tool_names = set(getattr(agent, "valid_tool_names", set()))
|
||||
agent.valid_tool_names.add("skill_manage")
|
||||
|
||||
with patch.object(agent, "_spawn_background_review",
|
||||
return_value=None) as spawn:
|
||||
agent.run_conversation("first")
|
||||
# The fake session reports tool_iterations=1, which trips
|
||||
# _skill_nudge_interval=1. So review should fire.
|
||||
assert spawn.called
|
||||
# Critical invariant: positional args must be empty, all real
|
||||
# args must be kwargs (matching _spawn_background_review's
|
||||
# actual signature).
|
||||
call = spawn.call_args
|
||||
assert call.args == (), (
|
||||
f"expected no positional args, got {call.args!r} — "
|
||||
"would crash _spawn_background_review at runtime"
|
||||
)
|
||||
assert "messages_snapshot" in call.kwargs
|
||||
|
||||
def test_chat_completions_loop_is_not_entered(self, fake_session):
|
||||
"""The early-return must bypass the regular API call loop entirely.
|
||||
We confirm by patching the SDK call and asserting it's never invoked."""
|
||||
agent = _make_codex_agent()
|
||||
# The chat_completions loop calls self.client.chat.completions.create(...)
|
||||
# If our early-return works, that path is dead.
|
||||
with patch.object(agent, "client") as client_mock, patch.object(
|
||||
agent, "_spawn_background_review", return_value=None
|
||||
):
|
||||
agent.run_conversation("hi")
|
||||
assert not client_mock.chat.completions.create.called
|
||||
|
||||
|
||||
class TestReviewForkApiModeDowngrade:
|
||||
"""When the parent agent runs on codex_app_server, the background
|
||||
review fork must downgrade to codex_responses — otherwise the fork
|
||||
can't dispatch agent-loop tools (memory, skill_manage) which is the
|
||||
whole point of the review."""
|
||||
|
||||
def test_codex_app_server_parent_downgrades_review_fork(self):
|
||||
"""Live test against the real _spawn_background_review code path:
|
||||
verify the review_agent gets api_mode=codex_responses when the
|
||||
parent is codex_app_server."""
|
||||
from unittest.mock import MagicMock, patch as _patch
|
||||
agent = _make_codex_agent()
|
||||
# Pretend memory + skills are configured so the review fork
|
||||
# reaches the AIAgent constructor.
|
||||
agent._memory_store = MagicMock()
|
||||
agent._memory_enabled = True
|
||||
agent._user_profile_enabled = True
|
||||
# Mock _current_main_runtime to return the parent's codex_app_server
|
||||
# state so we can confirm the helper detects + downgrades it.
|
||||
agent._current_main_runtime = lambda: {
|
||||
"api_mode": "codex_app_server",
|
||||
"base_url": "https://chatgpt.com/backend-api/codex",
|
||||
"api_key": "stub-token",
|
||||
}
|
||||
# Capture what AIAgent gets constructed with inside the helper.
|
||||
captured = {}
|
||||
|
||||
def _capture_init(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
# Set bare attributes the rest of the spawn function reads
|
||||
# so it can finish without exploding.
|
||||
self.api_mode = kwargs.get("api_mode")
|
||||
self.provider = kwargs.get("provider")
|
||||
self.model = kwargs.get("model")
|
||||
self._memory_write_origin = None
|
||||
self._memory_write_context = None
|
||||
self._memory_store = None
|
||||
self._memory_enabled = False
|
||||
self._user_profile_enabled = False
|
||||
self._memory_nudge_interval = 0
|
||||
self._skill_nudge_interval = 0
|
||||
self.suppress_status_output = False
|
||||
self._session_messages = []
|
||||
|
||||
def _no_op_run_conv(*a, **kw):
|
||||
return {"final_response": "", "messages": []}
|
||||
self.run_conversation = _no_op_run_conv
|
||||
|
||||
def _no_op_close(*a, **kw):
|
||||
return None
|
||||
self.close = _no_op_close
|
||||
|
||||
with _patch("run_agent.AIAgent.__init__", _capture_init):
|
||||
agent._spawn_background_review(
|
||||
messages_snapshot=[{"role": "user", "content": "x"}],
|
||||
review_memory=True,
|
||||
review_skills=False,
|
||||
)
|
||||
# Wait for the spawned thread to actually execute
|
||||
import time
|
||||
for _ in range(30):
|
||||
if "api_mode" in captured:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
|
||||
assert captured.get("api_mode") == "codex_responses", (
|
||||
f"review fork should be downgraded to codex_responses when "
|
||||
f"parent is codex_app_server; got {captured.get('api_mode')!r}"
|
||||
)
|
||||
|
||||
|
||||
class TestErrorHandling:
|
||||
def test_session_exception_returns_partial_with_error(self, monkeypatch):
|
||||
def boom_run_turn(self, user_input, **kwargs):
|
||||
raise RuntimeError("subprocess died")
|
||||
|
||||
monkeypatch.setattr(CodexAppServerSession, "ensure_started",
|
||||
lambda self: "t1")
|
||||
monkeypatch.setattr(CodexAppServerSession, "run_turn", boom_run_turn)
|
||||
|
||||
agent = _make_codex_agent()
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
result = agent.run_conversation("hi")
|
||||
assert result["completed"] is False
|
||||
assert result["partial"] is True
|
||||
assert "subprocess died" in result["error"]
|
||||
assert "codex-runtime auto" in result["final_response"]
|
||||
|
||||
def test_interrupted_turn_marked_partial(self, monkeypatch):
|
||||
def interrupted_turn(self, user_input, **kwargs):
|
||||
return TurnResult(
|
||||
final_text="",
|
||||
projected_messages=[],
|
||||
tool_iterations=0,
|
||||
interrupted=True,
|
||||
error="user interrupted",
|
||||
turn_id="t",
|
||||
thread_id="th",
|
||||
)
|
||||
monkeypatch.setattr(CodexAppServerSession, "ensure_started",
|
||||
lambda self: "th")
|
||||
monkeypatch.setattr(CodexAppServerSession, "run_turn", interrupted_turn)
|
||||
|
||||
agent = _make_codex_agent()
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
result = agent.run_conversation("hi")
|
||||
assert result["completed"] is False
|
||||
assert result["partial"] is True
|
||||
assert result["error"] == "user interrupted"
|
||||
|
||||
|
||||
class TestSessionRetirementOnRunAgent:
|
||||
"""run_agent.py side: when run_turn returns should_retire=True, the
|
||||
AIAgent must close + null _codex_session so the next turn respawns."""
|
||||
|
||||
def test_should_retire_drops_session(self, monkeypatch):
|
||||
closes = {"count": 0}
|
||||
|
||||
def fake_run_turn(self, user_input, **kwargs):
|
||||
return TurnResult(
|
||||
final_text="",
|
||||
projected_messages=[],
|
||||
tool_iterations=0,
|
||||
interrupted=True,
|
||||
error="turn timed out after 600.0s",
|
||||
turn_id="tu1",
|
||||
thread_id="th1",
|
||||
should_retire=True,
|
||||
)
|
||||
|
||||
def fake_close(self):
|
||||
closes["count"] += 1
|
||||
|
||||
monkeypatch.setattr(CodexAppServerSession, "ensure_started",
|
||||
lambda self: "th1")
|
||||
monkeypatch.setattr(CodexAppServerSession, "run_turn", fake_run_turn)
|
||||
monkeypatch.setattr(CodexAppServerSession, "close", fake_close)
|
||||
|
||||
agent = _make_codex_agent()
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
result = agent.run_conversation("hi")
|
||||
|
||||
# The session was closed and cleared
|
||||
assert closes["count"] == 1
|
||||
assert getattr(agent, "_codex_session", "MISSING") is None
|
||||
# Partial result was still returned (caller still sees the error)
|
||||
assert result["partial"] is True
|
||||
assert result["error"] == "turn timed out after 600.0s"
|
||||
|
||||
def test_normal_turn_keeps_session(self, fake_session):
|
||||
"""fake_session fixture returns should_retire=False (default).
|
||||
The session must stay attached for the next turn to reuse."""
|
||||
agent = _make_codex_agent()
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
agent.run_conversation("hi")
|
||||
# Session was lazily created and still attached.
|
||||
assert getattr(agent, "_codex_session", None) is not None
|
||||
|
||||
def test_exception_path_also_drops_session(self, monkeypatch):
|
||||
"""Even if run_turn raises (not just sets should_retire), we must
|
||||
drop the session — a thrown exception is the strongest possible
|
||||
signal the process is dead."""
|
||||
closes = {"count": 0}
|
||||
|
||||
def boom_run_turn(self, user_input, **kwargs):
|
||||
raise RuntimeError("codex segfaulted")
|
||||
|
||||
def fake_close(self):
|
||||
closes["count"] += 1
|
||||
|
||||
monkeypatch.setattr(CodexAppServerSession, "ensure_started",
|
||||
lambda self: "th1")
|
||||
monkeypatch.setattr(CodexAppServerSession, "run_turn", boom_run_turn)
|
||||
monkeypatch.setattr(CodexAppServerSession, "close", fake_close)
|
||||
|
||||
agent = _make_codex_agent()
|
||||
with patch.object(agent, "_spawn_background_review", return_value=None):
|
||||
result = agent.run_conversation("hi")
|
||||
|
||||
assert closes["count"] == 1
|
||||
assert agent._codex_session is None
|
||||
assert result["completed"] is False
|
||||
assert "codex segfaulted" in result["error"]
|
||||
@@ -16,6 +16,16 @@ from run_agent import AIAgent
|
||||
from agent.context_compressor import ContextCompressor
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _stable_aux_provider_config():
|
||||
"""Keep feasibility tests independent from the developer's config.yaml."""
|
||||
with patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", None, None, None, None),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def _make_agent(
|
||||
*,
|
||||
compression_enabled: bool = True,
|
||||
@@ -41,6 +51,7 @@ def _make_agent(
|
||||
agent.tool_progress_callback = None
|
||||
agent._compression_warning = None
|
||||
agent._aux_compression_context_length_config = None
|
||||
agent._custom_providers = []
|
||||
agent.tools = []
|
||||
|
||||
compressor = MagicMock(spec=ContextCompressor)
|
||||
@@ -182,6 +193,7 @@ def test_feasibility_check_passes_config_context_length(mock_get_client, mock_ct
|
||||
api_key="sk-custom",
|
||||
config_context_length=1_000_000,
|
||||
provider="openrouter",
|
||||
custom_providers=[],
|
||||
)
|
||||
|
||||
|
||||
@@ -205,6 +217,7 @@ def test_feasibility_check_ignores_invalid_context_length(mock_get_client, mock_
|
||||
api_key="sk-test",
|
||||
config_context_length=None,
|
||||
provider="openrouter",
|
||||
custom_providers=[],
|
||||
)
|
||||
|
||||
|
||||
@@ -258,6 +271,7 @@ def test_init_feasibility_check_uses_aux_context_override_from_config():
|
||||
api_key="sk-custom",
|
||||
config_context_length=1_000_000,
|
||||
provider="",
|
||||
custom_providers=[],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -166,6 +166,56 @@ class TestRecordFileMutationResult:
|
||||
)
|
||||
assert agent._turn_failed_file_mutations == {}
|
||||
|
||||
def test_write_file_with_lint_error_counts_as_landed(self):
|
||||
agent = _bare_agent()
|
||||
agent._record_file_mutation_result(
|
||||
"write_file",
|
||||
{"path": "/tmp/a.py", "content": "bad"},
|
||||
json.dumps({"error": "write failed"}),
|
||||
is_error=True,
|
||||
)
|
||||
assert "/tmp/a.py" in agent._turn_failed_file_mutations
|
||||
|
||||
result = json.dumps({
|
||||
"bytes_written": 24,
|
||||
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
|
||||
})
|
||||
|
||||
agent._record_file_mutation_result(
|
||||
"write_file",
|
||||
{"path": "/tmp/a.py", "content": "def nope(:\n"},
|
||||
result,
|
||||
is_error=True,
|
||||
)
|
||||
|
||||
assert agent._turn_failed_file_mutations == {}
|
||||
|
||||
def test_patch_with_lsp_diagnostics_counts_as_landed(self):
|
||||
agent = _bare_agent()
|
||||
agent._record_file_mutation_result(
|
||||
"patch",
|
||||
{"mode": "replace", "path": "/tmp/a.py", "old_string": "x", "new_string": "y"},
|
||||
json.dumps({"error": "Could not find old_string"}),
|
||||
is_error=True,
|
||||
)
|
||||
assert "/tmp/a.py" in agent._turn_failed_file_mutations
|
||||
|
||||
result = json.dumps({
|
||||
"success": True,
|
||||
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
|
||||
"files_modified": ["/tmp/a.py"],
|
||||
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
|
||||
})
|
||||
|
||||
agent._record_file_mutation_result(
|
||||
"patch",
|
||||
{"mode": "replace", "path": "/tmp/a.py", "old_string": "x", "new_string": "y"},
|
||||
result,
|
||||
is_error=True,
|
||||
)
|
||||
|
||||
assert agent._turn_failed_file_mutations == {}
|
||||
|
||||
def test_repeated_failure_keeps_first_error(self):
|
||||
agent = _bare_agent()
|
||||
agent._record_file_mutation_result(
|
||||
|
||||
@@ -2524,8 +2524,9 @@ class TestRunConversation:
|
||||
assert [call["api_call_count"] for call in pre_request_calls] == [1, 2]
|
||||
assert [call["api_call_count"] for call in post_request_calls] == [1, 2]
|
||||
assert all(call["session_id"] == agent.session_id for call in pre_request_calls)
|
||||
assert all("message_count" in c and "messages" not in c for c in pre_request_calls)
|
||||
assert all("usage" in c and "response" not in c for c in post_request_calls)
|
||||
assert all("message_count" in c and isinstance(c.get("request_messages"), list) for c in pre_request_calls)
|
||||
assert any(msg.get("role") == "user" and msg.get("content") == "search something" for msg in pre_request_calls[0]["request_messages"])
|
||||
assert all("usage" in c and "response" in c and "assistant_message" in c for c in post_request_calls)
|
||||
|
||||
def test_content_with_tool_calls_stays_silent_for_non_cli_quiet_mode(self, agent):
|
||||
self._setup_agent(agent)
|
||||
|
||||
@@ -578,6 +578,197 @@ def test_run_conversation_codex_refreshes_after_401_and_retries(monkeypatch):
|
||||
assert result["final_response"] == "Recovered after refresh"
|
||||
|
||||
|
||||
def _build_xai_oauth_agent(monkeypatch):
|
||||
_patch_agent_bootstrap(monkeypatch)
|
||||
agent = run_agent.AIAgent(
|
||||
model="grok-4.3",
|
||||
provider="xai-oauth",
|
||||
api_mode="codex_responses",
|
||||
base_url="https://api.x.ai/v1",
|
||||
api_key="xai-oauth-token",
|
||||
quiet_mode=True,
|
||||
max_iterations=4,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
agent._cleanup_task_resources = lambda task_id: None
|
||||
agent._persist_session = lambda messages, history=None: None
|
||||
agent._save_trajectory = lambda messages, user_message, completed: None
|
||||
agent._save_session_log = lambda messages: None
|
||||
return agent
|
||||
|
||||
|
||||
def test_build_api_kwargs_xai_oauth_sends_cache_key_via_extra_body(monkeypatch):
|
||||
"""xai-oauth + codex_responses must route prompt caching via the
|
||||
``prompt_cache_key`` body field on /v1/responses (xAI's documented
|
||||
Responses-API cache key — see docs.x.ai prompt-caching/maximizing-
|
||||
cache-hits).
|
||||
|
||||
We pass it through ``extra_body`` rather than as a top-level kwarg so
|
||||
the body field is serialized into JSON regardless of whether the
|
||||
installed openai SDK build still accepts ``prompt_cache_key`` on
|
||||
``Responses.stream()``. Older or trimmed SDK builds drop it from the
|
||||
signature and would otherwise raise ``TypeError`` before the request
|
||||
reaches api.x.ai. The ``x-grok-conv-id`` header is retained as a
|
||||
belt-and-braces fallback for clients/proxies that route on headers."""
|
||||
agent = _build_xai_oauth_agent(monkeypatch)
|
||||
kwargs = agent._build_api_kwargs(
|
||||
[
|
||||
{"role": "system", "content": "You are Hermes."},
|
||||
{"role": "user", "content": "Ping"},
|
||||
]
|
||||
)
|
||||
|
||||
assert kwargs.get("model") == "grok-4.3"
|
||||
# Top-level kwarg must NOT be set — that's the openai SDK
|
||||
# incompatibility this whole indirection exists to dodge.
|
||||
assert "prompt_cache_key" not in kwargs
|
||||
extra_body = kwargs.get("extra_body") or {}
|
||||
assert extra_body.get("prompt_cache_key"), (
|
||||
"xAI prompt-cache routing must travel via extra_body.prompt_cache_key "
|
||||
"for /v1/responses — body field is the documented surface."
|
||||
)
|
||||
headers = kwargs.get("extra_headers") or {}
|
||||
assert "x-grok-conv-id" in headers, (
|
||||
"x-grok-conv-id header kept as belt-and-braces fallback for clients "
|
||||
"that route on headers."
|
||||
)
|
||||
|
||||
|
||||
def test_run_conversation_xai_oauth_refreshes_after_401_and_retries(monkeypatch):
|
||||
"""xai-oauth speaks the Responses API just like codex. When the access
|
||||
token is rejected mid-call (401), the same proactive refresh-and-retry
|
||||
handler that fires for openai-codex must also fire for xai-oauth — the
|
||||
bug it caught: the gating condition checked only ``provider == "openai-codex"``,
|
||||
so xai-oauth 401s leaked straight to non-retryable abort path with no
|
||||
chance to swap in a freshly refreshed access token."""
|
||||
agent = _build_xai_oauth_agent(monkeypatch)
|
||||
calls = {"api": 0, "refresh": 0}
|
||||
|
||||
class _UnauthorizedError(RuntimeError):
|
||||
def __init__(self):
|
||||
super().__init__("Error code: 401 - unauthorized")
|
||||
self.status_code = 401
|
||||
|
||||
def _fake_api_call(api_kwargs):
|
||||
calls["api"] += 1
|
||||
if calls["api"] == 1:
|
||||
raise _UnauthorizedError()
|
||||
return _codex_message_response("Recovered after xAI refresh")
|
||||
|
||||
def _fake_refresh(*, force=True):
|
||||
calls["refresh"] += 1
|
||||
assert force is True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(agent, "_interruptible_api_call", _fake_api_call)
|
||||
monkeypatch.setattr(agent, "_try_refresh_codex_client_credentials", _fake_refresh)
|
||||
|
||||
result = agent.run_conversation("Say OK")
|
||||
|
||||
assert calls["api"] == 2
|
||||
assert calls["refresh"] == 1
|
||||
assert result["completed"] is True
|
||||
assert result["final_response"] == "Recovered after xAI refresh"
|
||||
|
||||
|
||||
def test_try_refresh_codex_client_credentials_handles_xai_oauth(monkeypatch):
|
||||
"""``_try_refresh_codex_client_credentials`` must rebuild the OpenAI
|
||||
client with freshly resolved xAI OAuth credentials when the active
|
||||
provider is xai-oauth. The function name is shared between codex and
|
||||
xai-oauth (both speak codex_responses) — covering both cases prevents
|
||||
silent regressions where the function gets gated to a single provider."""
|
||||
agent = _build_xai_oauth_agent(monkeypatch)
|
||||
closed = {"value": False}
|
||||
rebuilt = {"kwargs": None}
|
||||
|
||||
class _ExistingClient:
|
||||
def close(self):
|
||||
closed["value"] = True
|
||||
|
||||
class _RebuiltClient:
|
||||
pass
|
||||
|
||||
def _fake_openai(**kwargs):
|
||||
rebuilt["kwargs"] = kwargs
|
||||
return _RebuiltClient()
|
||||
|
||||
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
|
||||
# The pre-refresh guard reads the singleton with refresh_if_expiring=False
|
||||
# to verify that the agent's active key still matches; the actual
|
||||
# refresh later passes force_refresh=True. Both calls must succeed.
|
||||
return {
|
||||
"api_key": "fresh-xai-token" if force_refresh else agent.api_key,
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth.resolve_xai_oauth_runtime_credentials",
|
||||
_fake_resolve,
|
||||
)
|
||||
monkeypatch.setattr(run_agent, "OpenAI", _fake_openai)
|
||||
|
||||
agent.client = _ExistingClient()
|
||||
ok = agent._try_refresh_codex_client_credentials(force=True)
|
||||
|
||||
assert ok is True
|
||||
assert closed["value"] is True
|
||||
assert rebuilt["kwargs"]["api_key"] == "fresh-xai-token"
|
||||
assert rebuilt["kwargs"]["base_url"] == "https://api.x.ai/v1"
|
||||
assert isinstance(agent.client, _RebuiltClient)
|
||||
assert agent.api_key == "fresh-xai-token"
|
||||
|
||||
|
||||
def test_try_refresh_codex_client_credentials_skips_xai_oauth_when_singleton_differs(monkeypatch):
|
||||
"""An xai-oauth agent constructed with a non-singleton credential
|
||||
(e.g. a manual pool entry whose tokens belong to a different account
|
||||
than the loopback_pkce singleton, or an explicit ``api_key=`` arg)
|
||||
MUST NOT silently adopt the singleton's tokens on a 401 reactive
|
||||
refresh. Otherwise a 401 mid-conversation would re-route the rest
|
||||
of the conversation onto a different account, with no user feedback.
|
||||
|
||||
The credential pool's reactive recovery is the right channel for
|
||||
pool-managed credentials; this fallback path is for the singleton-
|
||||
only case and must short-circuit when the active key differs."""
|
||||
agent = _build_xai_oauth_agent(monkeypatch)
|
||||
# Agent is using "xai-oauth-token" (per the builder); singleton holds
|
||||
# a *different* account's token. No force_refresh should fire.
|
||||
refresh_calls = {"count": 0}
|
||||
|
||||
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
|
||||
if force_refresh:
|
||||
refresh_calls["count"] += 1
|
||||
return {
|
||||
"api_key": "singleton-account-token",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
}
|
||||
# The pre-refresh guard read — return the singleton's view of the
|
||||
# singleton's token, which is NOT what the agent is currently using.
|
||||
return {
|
||||
"api_key": "singleton-account-token",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth.resolve_xai_oauth_runtime_credentials",
|
||||
_fake_resolve,
|
||||
)
|
||||
|
||||
pre_refresh_key = agent.api_key
|
||||
ok = agent._try_refresh_codex_client_credentials(force=True)
|
||||
|
||||
assert ok is False, (
|
||||
"must not refresh when the active credential isn't the singleton; "
|
||||
"otherwise the conversation silently swaps accounts mid-flight."
|
||||
)
|
||||
assert refresh_calls["count"] == 0, (
|
||||
"force_refresh must not run — that would mutate the singleton's "
|
||||
"tokens on disk and consume its single-use refresh_token for an "
|
||||
"agent that wasn't even using the singleton."
|
||||
)
|
||||
assert agent.api_key == pre_refresh_key
|
||||
|
||||
|
||||
def test_run_conversation_copilot_refreshes_after_401_and_retries(monkeypatch):
|
||||
agent = _build_copilot_agent(monkeypatch)
|
||||
calls = {"api": 0, "refresh": 0}
|
||||
@@ -624,12 +815,18 @@ def test_try_refresh_codex_client_credentials_rebuilds_client(monkeypatch):
|
||||
rebuilt["kwargs"] = kwargs
|
||||
return _RebuiltClient()
|
||||
|
||||
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
|
||||
# Pre-refresh guard reads the singleton (refresh_if_expiring=False).
|
||||
# It must report the agent's current api_key so the equality check
|
||||
# passes; only then does the actual force_refresh run.
|
||||
return {
|
||||
"api_key": "new-codex-token" if force_refresh else agent.api_key,
|
||||
"base_url": "https://chatgpt.com/backend-api/codex",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth.resolve_codex_runtime_credentials",
|
||||
lambda force_refresh=True: {
|
||||
"api_key": "new-codex-token",
|
||||
"base_url": "https://chatgpt.com/backend-api/codex",
|
||||
},
|
||||
_fake_resolve,
|
||||
)
|
||||
monkeypatch.setattr(run_agent, "OpenAI", _fake_openai)
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ class TestStreamingAssemblyRepair:
|
||||
|
||||
These tests verify the REPAIR FUNCTION itself works correctly for the
|
||||
cases that arise during streaming assembly. Integration tests that
|
||||
exercise the full streaming path are in test_agent_loop_tool_calling.py.
|
||||
exercise the full streaming path are in run_agent.py's streaming tests.
|
||||
"""
|
||||
|
||||
# -- Truncation cases (most common streaming failure) --
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user