Merge main into bb/gui.
Resolve merge conflicts while preserving bb/gui dashboard paths and STT provider support.
This commit is contained in:
Brooklyn Nicholson
@@ -0,0 +1,262 @@
|
||||
"""Tests for cross-edit LSP delta filtering.
|
||||
|
||||
The delta-filter contract spans three pieces:
|
||||
|
||||
1. ``agent.lsp.manager._diag_key`` — strict equality key including
|
||||
the diagnostic's position range. Two diagnostics with the same
|
||||
content but different lines are NOT equal under this key (they
|
||||
are genuinely different diagnostics).
|
||||
2. ``agent.lsp.range_shift.build_line_shift`` — derives a function
|
||||
mapping pre-edit line numbers to post-edit line numbers from a
|
||||
pre/post text pair.
|
||||
3. ``agent.lsp.manager.LSPService.get_diagnostics_sync(line_shift=…)``
|
||||
— applies the shift to baseline diagnostics before computing the
|
||||
set-difference, so pre-existing errors at shifted lines hash
|
||||
equal to their post-edit counterparts and get filtered out.
|
||||
|
||||
These tests exercise the contract at the unit level; the E2E case
|
||||
(real LSP server, real shift) is covered in test_service.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from agent.lsp.client import _diagnostic_key
|
||||
from agent.lsp.manager import _diag_key
|
||||
from agent.lsp.range_shift import (
|
||||
build_line_shift,
|
||||
shift_baseline,
|
||||
shift_diagnostic_range,
|
||||
)
|
||||
|
||||
|
||||
def _diag(*, line: int, message: str = "Undefined variable",
|
||||
severity: int = 1, code: str = "reportUndefinedVariable",
|
||||
source: str = "Pyright", end_line: int | None = None) -> dict:
|
||||
if end_line is None:
|
||||
end_line = line
|
||||
return {
|
||||
"severity": severity,
|
||||
"code": code,
|
||||
"source": source,
|
||||
"message": message,
|
||||
"range": {
|
||||
"start": {"line": line, "character": 0},
|
||||
"end": {"line": end_line, "character": 10},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# _diag_key: strict equality (with range)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_diag_key_treats_shifted_diagnostics_as_distinct():
|
||||
"""Two diagnostics with the same message but at different lines hash
|
||||
differently — they are genuinely different diagnostics. The shift
|
||||
map is what makes them equal AFTER remapping; the key itself stays
|
||||
strict."""
|
||||
a = _diag(line=100)
|
||||
b = _diag(line=200)
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_matches_client_key_for_shifted_baseline():
|
||||
"""When a baseline diagnostic is remapped through a shift, its
|
||||
_diag_key must match the corresponding post-edit diagnostic's key
|
||||
at the same coordinates. This is the contract the delta filter
|
||||
relies on."""
|
||||
pre = _diag(line=200)
|
||||
# Edit deletes 14 lines above line 200, so the same error now
|
||||
# appears at line 186 post-edit.
|
||||
shift = lambda L: L - 14 if L >= 14 else L
|
||||
shifted = shift_diagnostic_range(pre, shift)
|
||||
assert shifted is not None
|
||||
post = _diag(line=186)
|
||||
assert _diag_key(shifted) == _diag_key(post)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_message():
|
||||
a = _diag(line=100, message="foo")
|
||||
b = _diag(line=100, message="bar")
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_severity():
|
||||
a = _diag(line=100, severity=1)
|
||||
b = _diag(line=100, severity=2)
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_distinguishes_source():
|
||||
a = _diag(line=100, source="Pyright")
|
||||
b = _diag(line=100, source="Ruff")
|
||||
assert _diag_key(a) != _diag_key(b)
|
||||
|
||||
|
||||
def test_diag_key_matches_client_key_byte_for_byte():
|
||||
"""The manager-side and client-side keys must agree on diagnostic
|
||||
identity — they're used by two layers that need to round-trip the
|
||||
same diagnostics through dedup and delta filtering."""
|
||||
d = _diag(line=42)
|
||||
assert _diag_key(d) == _diagnostic_key(d)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# build_line_shift
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_shift_identity_for_identical_content():
|
||||
shift = build_line_shift("a\nb\nc\n", "a\nb\nc\n")
|
||||
assert shift(0) == 0
|
||||
assert shift(1) == 1
|
||||
assert shift(2) == 2
|
||||
|
||||
|
||||
def test_shift_pure_deletion_above_line():
|
||||
"""Delete 2 lines at the top; everything below shifts up by 2."""
|
||||
pre = "line0\nline1\nline2\nline3\nline4\n"
|
||||
post = "line2\nline3\nline4\n" # deleted lines 0-1
|
||||
shift = build_line_shift(pre, post)
|
||||
# Pre lines 0,1 → deleted → None
|
||||
assert shift(0) is None
|
||||
assert shift(1) is None
|
||||
# Pre line 2 → post line 0
|
||||
assert shift(2) == 0
|
||||
# Pre line 4 → post line 2
|
||||
assert shift(4) == 2
|
||||
|
||||
|
||||
def test_shift_pure_insertion_above_line():
|
||||
"""Insert 3 lines at the top; everything below shifts down by 3."""
|
||||
pre = "line0\nline1\nline2\n"
|
||||
post = "new0\nnew1\nnew2\nline0\nline1\nline2\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
# Pre lines unchanged in identity, shifted by 3
|
||||
assert shift(0) == 3
|
||||
assert shift(1) == 4
|
||||
assert shift(2) == 5
|
||||
|
||||
|
||||
def test_shift_replacement_in_middle():
|
||||
"""Replace 2 lines in the middle with 1 line. Lines above
|
||||
unchanged; lines below shift up by 1."""
|
||||
pre = "a\nb\nc\nd\ne\n"
|
||||
post = "a\nb\nX\ne\n" # replaced lines 2,3 (c,d) with X
|
||||
shift = build_line_shift(pre, post)
|
||||
assert shift(0) == 0 # a → a
|
||||
assert shift(1) == 1 # b → b
|
||||
assert shift(2) is None # c → deleted
|
||||
assert shift(3) is None # d → deleted
|
||||
assert shift(4) == 3 # e → post line 3
|
||||
|
||||
|
||||
def test_shift_handles_empty_pre():
|
||||
"""First write of a file: pre is empty, post has content. Nothing
|
||||
to shift, so the function should be well-defined for empty pre."""
|
||||
shift = build_line_shift("", "hello\nworld\n")
|
||||
# Any pre line falls past the end of an empty pre — anchor at end of post
|
||||
assert shift(0) == 1
|
||||
|
||||
|
||||
def test_shift_handles_empty_post():
|
||||
"""File deleted to empty. Every pre line returns None."""
|
||||
shift = build_line_shift("line0\nline1\n", "")
|
||||
assert shift(0) is None
|
||||
assert shift(1) is None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# shift_diagnostic_range
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_shift_diag_remaps_start_and_end():
|
||||
pre = "a\nb\nc\nd\n"
|
||||
post = "X\na\nb\nc\nd\n" # one line inserted at top
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=2, end_line=2)
|
||||
remapped = shift_diagnostic_range(d, shift)
|
||||
assert remapped is not None
|
||||
assert remapped["range"]["start"]["line"] == 3
|
||||
assert remapped["range"]["end"]["line"] == 3
|
||||
|
||||
|
||||
def test_shift_diag_drops_diagnostic_in_deleted_region():
|
||||
pre = "a\nb\nc\nd\n"
|
||||
post = "a\nd\n" # deleted lines 1,2 (b,c)
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=1)
|
||||
assert shift_diagnostic_range(d, shift) is None
|
||||
|
||||
|
||||
def test_shift_diag_does_not_mutate_original():
|
||||
pre = "a\nb\n"
|
||||
post = "X\na\nb\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
d = _diag(line=0)
|
||||
original_line = d["range"]["start"]["line"]
|
||||
_ = shift_diagnostic_range(d, shift)
|
||||
assert d["range"]["start"]["line"] == original_line
|
||||
|
||||
|
||||
def test_shift_baseline_drops_deleted_and_remaps_rest():
|
||||
pre = "a\nb\nc\nd\ne\n"
|
||||
post = "a\ne\n" # deleted b,c,d
|
||||
shift = build_line_shift(pre, post)
|
||||
baseline = [
|
||||
_diag(line=0, message="err on a"),
|
||||
_diag(line=1, message="err on b"), # → deleted
|
||||
_diag(line=2, message="err on c"), # → deleted
|
||||
_diag(line=4, message="err on e"),
|
||||
]
|
||||
out = shift_baseline(baseline, shift)
|
||||
assert [d["message"] for d in out] == ["err on a", "err on e"]
|
||||
assert out[0]["range"]["start"]["line"] == 0
|
||||
assert out[1]["range"]["start"]["line"] == 1
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# End-to-end: simulate the delta-filter pipeline
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
def test_pipeline_filters_shifted_baseline_under_strict_key():
|
||||
"""The exact scenario the bug fix is for: an edit deletes lines,
|
||||
every diagnostic below shifts, and the delta filter (strict key
|
||||
+ shifted baseline) correctly identifies them as pre-existing."""
|
||||
pre = "line0\nline1\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\n"
|
||||
# Delete lines 2,3,4 — pre-existing errors at lines 7,8 should
|
||||
# appear at lines 4,5 post-edit and be filtered out.
|
||||
post = "line0\nline1\nline5\nline6\nline7\nline8\nline9\n"
|
||||
shift = build_line_shift(pre, post)
|
||||
|
||||
baseline = [_diag(line=7, message="X"), _diag(line=8, message="Y")]
|
||||
post_diags = [_diag(line=4, message="X"), _diag(line=5, message="Y")]
|
||||
|
||||
shifted_baseline = shift_baseline(baseline, shift)
|
||||
seen = {_diag_key(d) for d in shifted_baseline}
|
||||
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
|
||||
|
||||
# Both errors were pre-existing — filtered out.
|
||||
assert new_diags == []
|
||||
|
||||
|
||||
def test_pipeline_preserves_new_instance_at_different_line():
|
||||
"""The case content-only keys would miss: the model introduces a
|
||||
SECOND instance of the same error class at a new location. The
|
||||
new instance must surface."""
|
||||
pre = "good\ngood\ngood\n"
|
||||
post = "good\nbad\ngood\nbad\n" # added 2 new error lines
|
||||
shift = build_line_shift(pre, post)
|
||||
|
||||
baseline = [_diag(line=0, message="bad style")] # pre-existing
|
||||
post_diags = [
|
||||
_diag(line=0, message="bad style"), # pre-existing
|
||||
_diag(line=1, message="bad style"), # NEW — different line
|
||||
_diag(line=3, message="bad style"), # NEW — different line
|
||||
]
|
||||
|
||||
shifted_baseline = shift_baseline(baseline, shift)
|
||||
seen = {_diag_key(d) for d in shifted_baseline}
|
||||
new_diags = [d for d in post_diags if _diag_key(d) not in seen]
|
||||
|
||||
# Two genuinely new instances must be surfaced.
|
||||
assert len(new_diags) == 2
|
||||
assert {d["range"]["start"]["line"] for d in new_diags} == {1, 3}
|
||||
@@ -130,6 +130,35 @@ def test_service_e2e_delta_filter(mock_pyright):
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_service_e2e_delta_filter_with_line_shift(mock_pyright):
|
||||
"""End-to-end: an edit that shifts the diagnostic's line still
|
||||
filters correctly when ``line_shift`` is supplied.
|
||||
|
||||
The mock LSP server emits a fixed error at line 0; for this test
|
||||
we don't need to actually shift the server's output — we just
|
||||
need to prove that supplying a line_shift through the API works
|
||||
and doesn't break the existing delta path. The unit tests in
|
||||
test_delta_key.py cover the shift semantics in detail.
|
||||
"""
|
||||
repo = mock_pyright
|
||||
f = repo / "x.py"
|
||||
f.write_text("print('hi')\n")
|
||||
|
||||
svc = LSPService(
|
||||
enabled=True,
|
||||
wait_mode="document",
|
||||
wait_timeout=3.0,
|
||||
install_strategy="manual",
|
||||
)
|
||||
try:
|
||||
svc.snapshot_baseline(str(f))
|
||||
# Identity shift — should behave exactly like no shift.
|
||||
new_diags = svc.get_diagnostics_sync(str(f), line_shift=lambda L: L)
|
||||
assert new_diags == []
|
||||
finally:
|
||||
svc.shutdown()
|
||||
|
||||
|
||||
def test_service_status_includes_clients(mock_pyright):
|
||||
repo = mock_pyright
|
||||
f = repo / "x.py"
|
||||
|
||||
@@ -12,12 +12,24 @@ Covers:
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from contextlib import contextmanager
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch, PropertyMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _mock_botocore_session(*, return_value=None, side_effect=None):
|
||||
"""Patch botocore.session even when botocore is not installed."""
|
||||
botocore_mod = ModuleType("botocore")
|
||||
session_mod = ModuleType("botocore.session")
|
||||
session_mod.get_session = MagicMock(return_value=return_value, side_effect=side_effect)
|
||||
botocore_mod.session = session_mod
|
||||
with patch.dict("sys.modules", {"botocore": botocore_mod, "botocore.session": session_mod}):
|
||||
yield session_mod.get_session
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AWS credential detection
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -120,7 +132,7 @@ class TestResolveBedrocRegion:
|
||||
from unittest.mock import patch, MagicMock
|
||||
mock_session = MagicMock()
|
||||
mock_session.get_config_variable.return_value = None
|
||||
with patch("botocore.session.get_session", return_value=mock_session):
|
||||
with _mock_botocore_session(return_value=mock_session):
|
||||
assert resolve_bedrock_region({}) == "us-east-1"
|
||||
|
||||
def test_falls_back_to_botocore_profile_region(self):
|
||||
@@ -128,13 +140,13 @@ class TestResolveBedrocRegion:
|
||||
from unittest.mock import patch, MagicMock
|
||||
mock_session = MagicMock()
|
||||
mock_session.get_config_variable.return_value = "eu-central-1"
|
||||
with patch("botocore.session.get_session", return_value=mock_session):
|
||||
with _mock_botocore_session(return_value=mock_session):
|
||||
assert resolve_bedrock_region({}) == "eu-central-1"
|
||||
|
||||
def test_botocore_failure_falls_back_to_us_east_1(self):
|
||||
from agent.bedrock_adapter import resolve_bedrock_region
|
||||
from unittest.mock import patch
|
||||
with patch("botocore.session.get_session", side_effect=Exception("no botocore")):
|
||||
with _mock_botocore_session(side_effect=Exception("no botocore")):
|
||||
assert resolve_bedrock_region({}) == "us-east-1"
|
||||
|
||||
|
||||
|
||||
@@ -253,20 +253,24 @@ class TestErrorClassifierBedrock:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPackaging:
|
||||
"""Verify bedrock optional dependency is declared."""
|
||||
"""Verify Bedrock remains a declared lazy optional dependency."""
|
||||
|
||||
@staticmethod
|
||||
def _optional_dependencies():
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
content = (Path(__file__).parent.parent.parent / "pyproject.toml").read_text()
|
||||
return tomllib.loads(content)["project"]["optional-dependencies"]
|
||||
|
||||
def test_bedrock_extra_exists(self):
|
||||
import configparser
|
||||
from pathlib import Path
|
||||
# Read pyproject.toml to verify [bedrock] extra
|
||||
toml_path = Path(__file__).parent.parent.parent / "pyproject.toml"
|
||||
content = toml_path.read_text()
|
||||
assert 'bedrock = ["boto3' in content
|
||||
extras = self._optional_dependencies()
|
||||
assert "bedrock" in extras
|
||||
assert any(dep.startswith("boto3==") for dep in extras["bedrock"])
|
||||
|
||||
def test_bedrock_in_all_extra(self):
|
||||
from pathlib import Path
|
||||
content = (Path(__file__).parent.parent.parent / "pyproject.toml").read_text()
|
||||
assert '"hermes-agent[bedrock]"' in content
|
||||
def test_bedrock_is_not_eager_installed_by_all_extra(self):
|
||||
extras = self._optional_dependencies()
|
||||
assert "hermes-agent[bedrock]" not in extras["all"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -991,9 +991,12 @@ class TestCompressWithClient:
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=2)
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=2)
|
||||
|
||||
# Last head message (index 2) is "user" → summary should be "assistant"
|
||||
# NOTE: protect_first_n=2 preserves 2 non-system messages in addition to
|
||||
# the system prompt (always implicitly protected), yielding head [system,
|
||||
# user, user] with last head = user.
|
||||
msgs = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": "msg 1"},
|
||||
@@ -1059,11 +1062,13 @@ class TestCompressWithClient:
|
||||
mock_response.choices[0].message.content = "summary text"
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=3)
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=3)
|
||||
|
||||
# Head: [system, user, assistant] → last head = assistant
|
||||
# Tail: [user, assistant, user] → first tail = user
|
||||
# summary_role="user" collides with tail, "assistant" collides with head → merge
|
||||
# NOTE: protect_first_n=2 preserves 2 non-system messages in addition to
|
||||
# the system prompt (always implicitly protected).
|
||||
msgs = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": "msg 1"},
|
||||
@@ -1097,7 +1102,7 @@ class TestCompressWithClient:
|
||||
mock_response.choices[0].message.content = "summary text"
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=3, protect_last_n=3)
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=3)
|
||||
|
||||
msgs = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
@@ -1133,13 +1138,15 @@ class TestCompressWithClient:
|
||||
mock_response.choices[0].message.content = "summary text"
|
||||
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=2, protect_last_n=2)
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=1, protect_last_n=2)
|
||||
|
||||
# Head: [system, user] → last head = user
|
||||
# Tail: [assistant, user, assistant] → first tail = assistant
|
||||
# summary_role="assistant" collides with tail, "user" collides with head → merge
|
||||
# NOTE: protect_first_n=1 preserves 1 non-system message in addition to
|
||||
# the system prompt (always implicitly protected).
|
||||
# With min_tail=3, tail = last 3 messages (indices 5-7).
|
||||
# Need 8 messages: min_for_compress = 2+3+1 = 6, must have > 6.
|
||||
# Need 8 messages: _min_for_compress = head(2) + 3 + 1 = 6, must have > 6.
|
||||
msgs = [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": "msg 1"},
|
||||
@@ -1292,6 +1299,92 @@ class TestSummaryTargetRatio:
|
||||
c = ContextCompressor(model="test", quiet_mode=True)
|
||||
assert c.protect_last_n == 20
|
||||
|
||||
def test_default_protect_first_n_is_3(self):
|
||||
"""Default protect_first_n is 3 (system + 3 extra non-system messages =
|
||||
4 protected messages total when a system prompt is present). With the
|
||||
new semantics, the constructor default is 3 — the system prompt is
|
||||
always implicitly protected ON TOP OF protect_first_n non-system
|
||||
messages.
|
||||
"""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True)
|
||||
assert c.protect_first_n == 3
|
||||
|
||||
def test_protect_first_n_override(self):
|
||||
"""protect_first_n=0 should be honoured — for users who rely on rolling
|
||||
compaction and want NOTHING pinned at head except the system prompt
|
||||
(always implicitly protected)."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=0)
|
||||
assert c.protect_first_n == 0
|
||||
|
||||
def test_protect_first_n_0_preserves_only_system_prompt(self):
|
||||
"""End-to-end: when protect_first_n=0, compression should treat only
|
||||
the system prompt as head. All user/assistant messages between the
|
||||
system prompt and the protected tail become summarization candidates.
|
||||
|
||||
This is the cleanest configuration for long-running rolling-compaction
|
||||
sessions — no user/assistant turn gets pinned verbatim forever just
|
||||
because it happened to be early in the session."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(
|
||||
model="test",
|
||||
quiet_mode=True,
|
||||
protect_first_n=0,
|
||||
protect_last_n=2,
|
||||
)
|
||||
msgs = (
|
||||
[{"role": "system", "content": "System prompt"}]
|
||||
+ [{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg {i}"}
|
||||
for i in range(8)]
|
||||
)
|
||||
result = c.compress(msgs)
|
||||
# System prompt (msg[0]) survives as head
|
||||
assert result[0]["role"] == "system"
|
||||
assert result[0]["content"].startswith("System prompt")
|
||||
# The first user/assistant exchange (msg 0, msg 1) should NOT be pinned
|
||||
# as head verbatim — those would have been summarized or absorbed.
|
||||
# Under default protect_first_n=3, result[1..3] would be the literal
|
||||
# "msg 0" / "msg 1" / "msg 2"; with protect_first_n=0 they aren't.
|
||||
assert result[1].get("content") != "msg 0"
|
||||
# Last 2 messages are tail-protected under protect_last_n=2
|
||||
assert result[-1]["content"] == msgs[-1]["content"]
|
||||
|
||||
def test_protect_first_n_semantics_stable_without_system_prompt(self):
|
||||
"""Regression: gateway /compress handler strips the system prompt
|
||||
before calling compress(). protect_first_n must mean the same thing
|
||||
in both paths — "N non-system head messages" — so configuring
|
||||
protect_first_n=0 preserves NOTHING at the head regardless of whether
|
||||
the system prompt is in the messages list.
|
||||
|
||||
Bug this covers: under the old semantics, protect_first_n counted
|
||||
literally from messages[0]. In the gateway path (no system prompt)
|
||||
that meant protect_first_n=1 would pin the first user turn of the
|
||||
session forever — a user-reported complaint that a week-old
|
||||
resolved question kept getting reinserted into every compaction
|
||||
summary."""
|
||||
with patch("agent.context_compressor.get_model_context_length", return_value=100_000):
|
||||
c = ContextCompressor(
|
||||
model="test",
|
||||
quiet_mode=True,
|
||||
protect_first_n=0,
|
||||
protect_last_n=2,
|
||||
)
|
||||
# No system prompt — this is what the gateway passes to compress().
|
||||
msgs = [
|
||||
{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg {i}"}
|
||||
for i in range(10)
|
||||
]
|
||||
head_size = c._protect_head_size(msgs)
|
||||
# With no system prompt and protect_first_n=0 → head is empty.
|
||||
# The first user message is NOT pinned as head.
|
||||
assert head_size == 0
|
||||
|
||||
# And with protect_first_n=3 on the same no-system-prompt list →
|
||||
# head size is 3 (the three earliest non-system messages).
|
||||
c.protect_first_n = 3
|
||||
assert c._protect_head_size(msgs) == 3
|
||||
|
||||
|
||||
class TestTokenBudgetTailProtection:
|
||||
"""Tests for token-budget-based tail protection (PR #6240).
|
||||
|
||||
@@ -27,10 +27,12 @@ def _messages_with_handoff(summary_body: str):
|
||||
return [
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": f"{SUMMARY_PREFIX}\n{summary_body}"},
|
||||
{"role": "assistant", "content": "handoff acknowledged after resume"},
|
||||
{"role": "user", "content": "new user turn after resume"},
|
||||
{"role": "assistant", "content": "new assistant work after resume"},
|
||||
{"role": "user", "content": "more new work after resume"},
|
||||
{"role": "assistant", "content": "latest tail response"},
|
||||
{"role": "user", "content": "final active request stays in protected tail"},
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Tests for agent/display.py — build_tool_preview() and inline diff previews."""
|
||||
|
||||
import os
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -149,6 +150,27 @@ class TestCuteToolMessagePreviewLength:
|
||||
assert path in line
|
||||
assert "..." not in line
|
||||
|
||||
def test_write_file_lint_error_result_is_not_marked_failed(self):
|
||||
result = json.dumps({
|
||||
"bytes_written": 12,
|
||||
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
|
||||
})
|
||||
|
||||
line = get_cute_tool_message("write_file", {"path": "/tmp/a.py"}, 0.1, result=result)
|
||||
|
||||
assert "[error]" not in line
|
||||
|
||||
def test_patch_lsp_diagnostics_result_is_not_marked_failed(self):
|
||||
result = json.dumps({
|
||||
"success": True,
|
||||
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
|
||||
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
|
||||
})
|
||||
|
||||
line = get_cute_tool_message("patch", {"path": "/tmp/a.py"}, 0.1, result=result)
|
||||
|
||||
assert "[error]" not in line
|
||||
|
||||
|
||||
class TestEditDiffPreview:
|
||||
def test_extract_edit_diff_for_patch(self):
|
||||
|
||||
@@ -913,6 +913,35 @@ class TestTranslateStreamEvent:
|
||||
assert chunks[-1].choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
class TestMakeStreamChunk:
|
||||
def test_reasoning_only_chunk_has_content_none(self):
|
||||
from agent.gemini_cloudcode_adapter import _make_stream_chunk
|
||||
|
||||
chunk = _make_stream_chunk(model="m", reasoning="think")
|
||||
delta = chunk.choices[0].delta
|
||||
assert delta.content is None
|
||||
assert delta.reasoning == "think"
|
||||
|
||||
def test_content_only_chunk_has_reasoning_none(self):
|
||||
from agent.gemini_cloudcode_adapter import _make_stream_chunk
|
||||
|
||||
chunk = _make_stream_chunk(model="m", content="hello")
|
||||
delta = chunk.choices[0].delta
|
||||
assert delta.content == "hello"
|
||||
assert delta.reasoning is None
|
||||
assert delta.tool_calls is None
|
||||
|
||||
def test_finish_only_chunk_has_all_fields_none(self):
|
||||
from agent.gemini_cloudcode_adapter import _make_stream_chunk
|
||||
|
||||
chunk = _make_stream_chunk(model="m", finish_reason="stop")
|
||||
delta = chunk.choices[0].delta
|
||||
assert delta.content is None
|
||||
assert delta.reasoning is None
|
||||
assert delta.tool_calls is None
|
||||
assert chunk.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
class TestGeminiCloudCodeClient:
|
||||
def test_client_exposes_openai_interface(self):
|
||||
from agent.gemini_cloudcode_adapter import GeminiCloudCodeClient
|
||||
|
||||
@@ -7,6 +7,7 @@ from agent.tool_guardrails import (
|
||||
ToolCallGuardrailController,
|
||||
ToolCallSignature,
|
||||
canonical_tool_args,
|
||||
classify_tool_failure,
|
||||
)
|
||||
|
||||
|
||||
@@ -131,6 +132,21 @@ def test_success_resets_exact_signature_failure_streak():
|
||||
assert controller.before_call("web_search", args).action == "allow"
|
||||
|
||||
|
||||
def test_file_mutation_lint_error_result_is_not_a_tool_failure():
|
||||
write_result = json.dumps({
|
||||
"bytes_written": 12,
|
||||
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
|
||||
})
|
||||
patch_result = json.dumps({
|
||||
"success": True,
|
||||
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
|
||||
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
|
||||
})
|
||||
|
||||
assert classify_tool_failure("write_file", write_result) == (False, "")
|
||||
assert classify_tool_failure("patch", patch_result) == (False, "")
|
||||
|
||||
|
||||
def test_same_tool_varying_args_warns_by_default_without_halting():
|
||||
controller = ToolCallGuardrailController(
|
||||
ToolCallGuardrailConfig(same_tool_failure_warn_after=2, same_tool_failure_halt_after=3)
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Tests for shared tool result classification helpers."""
|
||||
|
||||
import json
|
||||
|
||||
from agent.tool_result_classification import file_mutation_result_landed
|
||||
|
||||
|
||||
def test_write_file_with_nested_lint_error_counts_as_landed():
|
||||
result = json.dumps({
|
||||
"bytes_written": 12,
|
||||
"lint": {"status": "error", "output": "SyntaxError: invalid syntax"},
|
||||
})
|
||||
|
||||
assert file_mutation_result_landed("write_file", result) is True
|
||||
|
||||
|
||||
def test_patch_with_nested_lsp_diagnostics_counts_as_landed():
|
||||
result = json.dumps({
|
||||
"success": True,
|
||||
"diff": "--- a/tmp.py\n+++ b/tmp.py\n",
|
||||
"lsp_diagnostics": "<diagnostics>ERROR [1:1] type mismatch</diagnostics>",
|
||||
})
|
||||
|
||||
assert file_mutation_result_landed("patch", result) is True
|
||||
|
||||
|
||||
def test_top_level_file_mutation_error_does_not_count_as_landed():
|
||||
result = json.dumps({"success": True, "error": "post-write verification failed"})
|
||||
|
||||
assert file_mutation_result_landed("patch", result) is False
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Tests for agent/video_gen_registry.py — provider registration & active lookup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import video_gen_registry
|
||||
from agent.video_gen_provider import VideoGenProvider
|
||||
|
||||
|
||||
class _FakeProvider(VideoGenProvider):
|
||||
def __init__(self, name: str, available: bool = True):
|
||||
self._name = name
|
||||
self._available = available
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
def generate(self, prompt, **kw):
|
||||
return {"success": True, "video": f"{self._name}://{prompt}"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry():
|
||||
video_gen_registry._reset_for_tests()
|
||||
yield
|
||||
video_gen_registry._reset_for_tests()
|
||||
|
||||
|
||||
class TestRegisterProvider:
|
||||
def test_register_and_lookup(self):
|
||||
provider = _FakeProvider("fake")
|
||||
video_gen_registry.register_provider(provider)
|
||||
assert video_gen_registry.get_provider("fake") is provider
|
||||
|
||||
def test_rejects_non_provider(self):
|
||||
with pytest.raises(TypeError):
|
||||
video_gen_registry.register_provider("not a provider") # type: ignore[arg-type]
|
||||
|
||||
def test_rejects_empty_name(self):
|
||||
class Empty(VideoGenProvider):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return ""
|
||||
|
||||
def generate(self, prompt, **kw):
|
||||
return {}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
video_gen_registry.register_provider(Empty())
|
||||
|
||||
def test_reregister_overwrites(self):
|
||||
a = _FakeProvider("same")
|
||||
b = _FakeProvider("same")
|
||||
video_gen_registry.register_provider(a)
|
||||
video_gen_registry.register_provider(b)
|
||||
assert video_gen_registry.get_provider("same") is b
|
||||
|
||||
def test_list_is_sorted(self):
|
||||
video_gen_registry.register_provider(_FakeProvider("zeta"))
|
||||
video_gen_registry.register_provider(_FakeProvider("alpha"))
|
||||
names = [p.name for p in video_gen_registry.list_providers()]
|
||||
assert names == ["alpha", "zeta"]
|
||||
|
||||
|
||||
class TestGetActiveProvider:
|
||||
def test_single_provider_autoresolves(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
video_gen_registry.register_provider(_FakeProvider("solo"))
|
||||
active = video_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "solo"
|
||||
|
||||
def test_no_provider_returns_none(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
assert video_gen_registry.get_active_provider() is None
|
||||
|
||||
def test_multi_without_config_returns_none(self, tmp_path, monkeypatch):
|
||||
"""Unlike image_gen (which falls back to 'fal'), video_gen has no
|
||||
legacy default — when there are multiple providers and no config,
|
||||
the registry returns None and the tool surfaces a helpful error.
|
||||
"""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
video_gen_registry.register_provider(_FakeProvider("xai"))
|
||||
video_gen_registry.register_provider(_FakeProvider("fal"))
|
||||
assert video_gen_registry.get_active_provider() is None
|
||||
|
||||
def test_config_selects_provider(self, tmp_path, monkeypatch):
|
||||
import yaml
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"video_gen": {"provider": "fal"}})
|
||||
)
|
||||
video_gen_registry.register_provider(_FakeProvider("xai"))
|
||||
video_gen_registry.register_provider(_FakeProvider("fal"))
|
||||
active = video_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "fal"
|
||||
|
||||
def test_unknown_config_falls_back(self, tmp_path, monkeypatch):
|
||||
"""If video_gen.provider names a provider that isn't registered,
|
||||
the single-provider fallback still applies."""
|
||||
import yaml
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"video_gen": {"provider": "ghost"}})
|
||||
)
|
||||
video_gen_registry.register_provider(_FakeProvider("only"))
|
||||
active = video_gen_registry.get_active_provider()
|
||||
assert active is not None and active.name == "only"
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Tests for the optional codex app-server runtime gate.
|
||||
|
||||
These are unit tests for the api_mode rewriter and the wire-level transport
|
||||
module. They do NOT require the `codex` CLI to be installed — that's
|
||||
covered by a separate live test gated on `codex --version`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.runtime_provider import (
|
||||
_VALID_API_MODES,
|
||||
_maybe_apply_codex_app_server_runtime,
|
||||
)
|
||||
|
||||
|
||||
class TestApiModeRegistration:
|
||||
"""The new api_mode must be registered or downstream parsing rejects it."""
|
||||
|
||||
def test_codex_app_server_is_a_valid_api_mode(self) -> None:
|
||||
assert "codex_app_server" in _VALID_API_MODES
|
||||
|
||||
def test_existing_api_modes_still_present(self) -> None:
|
||||
# Regression guard: don't accidentally delete other api_modes when
|
||||
# touching this set.
|
||||
for mode in (
|
||||
"chat_completions",
|
||||
"codex_responses",
|
||||
"anthropic_messages",
|
||||
"bedrock_converse",
|
||||
):
|
||||
assert mode in _VALID_API_MODES
|
||||
|
||||
|
||||
class TestMaybeApplyCodexAppServerRuntime:
|
||||
"""The opt-in helper that rewrites api_mode → codex_app_server."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_cfg",
|
||||
[
|
||||
None,
|
||||
{},
|
||||
{"openai_runtime": ""},
|
||||
{"openai_runtime": "auto"},
|
||||
{"openai_runtime": "AUTO"},
|
||||
{"other_key": "codex_app_server"}, # wrong key
|
||||
],
|
||||
)
|
||||
def test_default_off_for_openai(self, model_cfg) -> None:
|
||||
"""Default behavior is preserved when the flag is unset/auto."""
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai", api_mode="chat_completions", model_cfg=model_cfg
|
||||
)
|
||||
assert got == "chat_completions"
|
||||
|
||||
def test_opt_in_rewrites_openai(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai",
|
||||
api_mode="chat_completions",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
def test_opt_in_rewrites_openai_codex(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai-codex",
|
||||
api_mode="codex_responses",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
def test_case_insensitive(self) -> None:
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider="openai",
|
||||
api_mode="chat_completions",
|
||||
model_cfg={"openai_runtime": "Codex_App_Server"},
|
||||
)
|
||||
assert got == "codex_app_server"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider",
|
||||
[
|
||||
"anthropic",
|
||||
"openrouter",
|
||||
"xai",
|
||||
"qwen-oauth",
|
||||
"google-gemini-cli",
|
||||
"opencode-zen",
|
||||
"bedrock",
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_other_providers_never_rerouted(self, provider) -> None:
|
||||
"""Non-OpenAI providers MUST NOT be rerouted even with the flag set —
|
||||
codex's app-server can only run OpenAI/Codex auth flows."""
|
||||
got = _maybe_apply_codex_app_server_runtime(
|
||||
provider=provider,
|
||||
api_mode="anthropic_messages",
|
||||
model_cfg={"openai_runtime": "codex_app_server"},
|
||||
)
|
||||
assert got == "anthropic_messages", (
|
||||
f"provider={provider!r} should not be rerouted to codex_app_server"
|
||||
)
|
||||
|
||||
|
||||
class TestCodexAppServerModule:
|
||||
"""Module-surface tests for the JSON-RPC speaker. Don't require codex CLI."""
|
||||
|
||||
def test_module_imports(self) -> None:
|
||||
from agent.transports import codex_app_server
|
||||
|
||||
assert codex_app_server.MIN_CODEX_VERSION >= (0, 1, 0)
|
||||
assert callable(codex_app_server.parse_codex_version)
|
||||
assert callable(codex_app_server.check_codex_binary)
|
||||
|
||||
def test_parse_codex_version_valid(self) -> None:
|
||||
from agent.transports.codex_app_server import parse_codex_version
|
||||
|
||||
assert parse_codex_version("codex-cli 0.130.0") == (0, 130, 0)
|
||||
assert parse_codex_version("codex-cli 1.2.3 (extra metadata)") == (1, 2, 3)
|
||||
assert parse_codex_version("codex 99.0.1\n") == (99, 0, 1)
|
||||
|
||||
def test_parse_codex_version_invalid(self) -> None:
|
||||
from agent.transports.codex_app_server import parse_codex_version
|
||||
|
||||
assert parse_codex_version("nope") is None
|
||||
assert parse_codex_version("") is None
|
||||
assert parse_codex_version(None) is None # type: ignore[arg-type]
|
||||
|
||||
def test_check_binary_handles_missing_executable(self) -> None:
|
||||
from agent.transports.codex_app_server import check_codex_binary
|
||||
|
||||
ok, msg = check_codex_binary(codex_bin="/nonexistent/codex/binary/path")
|
||||
assert ok is False
|
||||
assert "not found" in msg.lower() or "no such" in msg.lower()
|
||||
|
||||
def test_codex_error_class_is_runtimeerror(self) -> None:
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
err = CodexAppServerError(code=-32600, message="boom")
|
||||
assert isinstance(err, RuntimeError)
|
||||
assert "boom" in str(err)
|
||||
assert "-32600" in str(err)
|
||||
|
||||
|
||||
class TestSpawnEnvIsolation:
|
||||
"""The codex spawn must NOT rewrite HOME — codex's shell tool spawns
|
||||
subprocesses (gh, git, npm, aws, gcloud, ...) that need to find their
|
||||
config in the real user $HOME. CODEX_HOME isolates codex's own state,
|
||||
HOME stays unchanged.
|
||||
|
||||
OpenClaw hit this footgun (openclaw/openclaw#81562) — they were
|
||||
rewriting HOME to a synthetic per-agent dir alongside CODEX_HOME,
|
||||
and then `gh auth status` / git config / etc. all broke inside codex
|
||||
shell calls. We avoid the same bug by only overlaying CODEX_HOME and
|
||||
RUST_LOG on top of os.environ.copy().
|
||||
"""
|
||||
|
||||
def test_spawn_env_preserves_HOME(self, monkeypatch):
|
||||
"""The spawn env must contain the parent process's HOME unchanged.
|
||||
Verifies via a subprocess-monkey-patch."""
|
||||
import subprocess
|
||||
from agent.transports import codex_app_server as cas
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakePopen:
|
||||
def __init__(self, cmd, *args, **kwargs):
|
||||
captured["env"] = kwargs.get("env", {}).copy()
|
||||
# Provide minimal Popen surface so __init__ doesn't crash
|
||||
# on attribute access during construction.
|
||||
self.stdin = None
|
||||
self.stdout = None
|
||||
self.stderr = None
|
||||
self.pid = 1
|
||||
self.returncode = None
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", FakePopen)
|
||||
monkeypatch.setenv("HOME", "/users/alice")
|
||||
|
||||
client = cas.CodexAppServerClient(codex_bin="codex")
|
||||
client._closed = True # so close() is a no-op
|
||||
|
||||
# The spawn env must have HOME=/users/alice unchanged
|
||||
assert captured["env"].get("HOME") == "/users/alice", (
|
||||
f"HOME got rewritten in codex spawn env: "
|
||||
f"{captured['env'].get('HOME')!r}. Codex's shell tool's "
|
||||
"subprocesses (gh, git, aws, npm) need the user's real HOME."
|
||||
)
|
||||
|
||||
def test_spawn_env_sets_CODEX_HOME_when_provided(self, monkeypatch):
|
||||
"""CODEX_HOME isolation must still work — that's the whole point
|
||||
of the codex_home arg."""
|
||||
import subprocess
|
||||
from agent.transports import codex_app_server as cas
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakePopen:
|
||||
def __init__(self, cmd, *args, **kwargs):
|
||||
captured["env"] = kwargs.get("env", {}).copy()
|
||||
self.stdin = None
|
||||
self.stdout = None
|
||||
self.stderr = None
|
||||
self.pid = 1
|
||||
self.returncode = None
|
||||
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", FakePopen)
|
||||
monkeypatch.setenv("HOME", "/users/alice")
|
||||
|
||||
client = cas.CodexAppServerClient(
|
||||
codex_bin="codex", codex_home="/tmp/profile/codex"
|
||||
)
|
||||
client._closed = True
|
||||
|
||||
assert captured["env"].get("CODEX_HOME") == "/tmp/profile/codex"
|
||||
# And HOME still passes through unchanged
|
||||
assert captured["env"].get("HOME") == "/users/alice"
|
||||
@@ -0,0 +1,976 @@
|
||||
"""Tests for CodexAppServerSession — drive turns through a mock client.
|
||||
|
||||
The session adapter has the most complex behavior of the three new modules:
|
||||
notification draining, server-request handling (approvals), interrupt,
|
||||
deadline timeouts. These tests pin all of that without spawning real codex.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.transports.codex_app_server_session import (
|
||||
CodexAppServerSession,
|
||||
TurnResult,
|
||||
_ServerRequestRouting,
|
||||
_approval_choice_to_codex_decision,
|
||||
)
|
||||
|
||||
|
||||
class FakeClient:
|
||||
"""Stand-in for CodexAppServerClient that records calls and lets the test
|
||||
drive the notification / server-request streams synchronously."""
|
||||
|
||||
def __init__(self, *, codex_bin: str = "codex", codex_home=None) -> None:
|
||||
self.codex_bin = codex_bin
|
||||
self.codex_home = codex_home
|
||||
self.requests: list[tuple[str, dict]] = []
|
||||
self.notifications_responses: list[dict] = []
|
||||
self.responses: list[tuple[Any, dict]] = []
|
||||
self.error_responses: list[tuple[Any, int, str]] = []
|
||||
self._initialized = False
|
||||
self._closed = False
|
||||
self._notifications: list[dict] = []
|
||||
self._server_requests: list[dict] = []
|
||||
self._request_handler = None # Optional[Callable[[str, dict], dict]]
|
||||
|
||||
# API matching CodexAppServerClient
|
||||
def initialize(self, **kwargs):
|
||||
self._initialized = True
|
||||
return {"userAgent": "fake/0.0.0", "codexHome": "/tmp",
|
||||
"platformOs": "linux", "platformFamily": "unix"}
|
||||
|
||||
def request(self, method: str, params: Optional[dict] = None, timeout: float = 30.0):
|
||||
self.requests.append((method, params or {}))
|
||||
if self._request_handler is not None:
|
||||
return self._request_handler(method, params or {})
|
||||
# Sensible defaults for protocol methods used by the session
|
||||
if method == "thread/start":
|
||||
return {"thread": {"id": "thread-fake-001"},
|
||||
"activePermissionProfile": {"id": "workspace-write"}}
|
||||
if method == "turn/start":
|
||||
return {"turn": {"id": "turn-fake-001"}}
|
||||
if method == "turn/interrupt":
|
||||
return {}
|
||||
return {}
|
||||
|
||||
def notify(self, method: str, params=None):
|
||||
pass
|
||||
|
||||
def respond(self, request_id, result):
|
||||
self.responses.append((request_id, result))
|
||||
|
||||
def respond_error(self, request_id, code, message, data=None):
|
||||
self.error_responses.append((request_id, code, message))
|
||||
|
||||
def take_notification(self, timeout: float = 0.0):
|
||||
if self._notifications:
|
||||
return self._notifications.pop(0)
|
||||
# Honor a tiny sleep so the loop doesn't hot-spin; the real client
|
||||
# blocks on a queue. For tests we want determinism.
|
||||
if timeout > 0:
|
||||
time.sleep(min(timeout, 0.001))
|
||||
return None
|
||||
|
||||
def take_server_request(self, timeout: float = 0.0):
|
||||
if self._server_requests:
|
||||
return self._server_requests.pop(0)
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
self._closed = True
|
||||
|
||||
def is_alive(self) -> bool:
|
||||
# Fake is "alive" until close() is called; tests that want a dead
|
||||
# subprocess can patch this attribute or call close() directly.
|
||||
return not self._closed
|
||||
|
||||
def stderr_tail(self, n: int = 20):
|
||||
return list(getattr(self, "_stderr_tail", []))[-n:]
|
||||
|
||||
# Test helpers
|
||||
def queue_notification(self, method: str, **params):
|
||||
self._notifications.append({"method": method, "params": params})
|
||||
|
||||
def queue_server_request(self, method: str, request_id: Any = "srv-1", **params):
|
||||
self._server_requests.append({"id": request_id, "method": method, "params": params})
|
||||
|
||||
def set_stderr_tail(self, lines):
|
||||
"""Test helper: seed stderr_tail() output for OAuth-refresh classifier tests."""
|
||||
self._stderr_tail = list(lines)
|
||||
|
||||
|
||||
def make_session(client: FakeClient, **kwargs) -> CodexAppServerSession:
|
||||
return CodexAppServerSession(
|
||||
cwd="/tmp",
|
||||
client_factory=lambda **kw: client,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ---- choice mapping ----
|
||||
|
||||
class TestApprovalChoiceMapping:
|
||||
@pytest.mark.parametrize("choice,expected", [
|
||||
("once", "accept"),
|
||||
("session", "acceptForSession"),
|
||||
("always", "acceptForSession"),
|
||||
("deny", "decline"),
|
||||
("anything-else", "decline"),
|
||||
])
|
||||
def test_mapping(self, choice, expected):
|
||||
assert _approval_choice_to_codex_decision(choice) == expected
|
||||
|
||||
|
||||
# ---- lifecycle ----
|
||||
|
||||
class TestLifecycle:
|
||||
def test_ensure_started_is_idempotent(self):
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
tid_a = s.ensure_started()
|
||||
tid_b = s.ensure_started()
|
||||
assert tid_a == tid_b == "thread-fake-001"
|
||||
# thread/start should be called exactly once
|
||||
method_calls = [m for (m, _) in client.requests if m == "thread/start"]
|
||||
assert len(method_calls) == 1
|
||||
|
||||
def test_thread_start_passes_cwd_only(self):
|
||||
"""thread/start carries cwd. We intentionally do NOT pass `permissions`
|
||||
on this codex version (experimentalApi-gated + requires matching
|
||||
config.toml [permissions] table). Letting codex use its default
|
||||
(read-only unless user configures otherwise) is the documented path."""
|
||||
client = FakeClient()
|
||||
s = make_session(client, permission_profile="workspace-write")
|
||||
s.ensure_started()
|
||||
method, params = next(r for r in client.requests if r[0] == "thread/start")
|
||||
assert params["cwd"] == "/tmp"
|
||||
assert "permissions" not in params # see session.ensure_started() comment
|
||||
|
||||
def test_close_idempotent(self):
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
s.ensure_started()
|
||||
s.close()
|
||||
s.close()
|
||||
assert client._closed is True
|
||||
|
||||
|
||||
# ---- turn loop ----
|
||||
|
||||
class TestRunTurn:
|
||||
def test_simple_text_turn_returns_final_message(self):
|
||||
client = FakeClient()
|
||||
client.queue_notification("turn/started", threadId="t", turn={"id": "tu1"})
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1", "text": "hello world"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed",
|
||||
threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.final_text == "hello world"
|
||||
assert r.interrupted is False
|
||||
assert r.error is None
|
||||
assert any(m["role"] == "assistant" and m.get("content") == "hello world"
|
||||
for m in r.projected_messages)
|
||||
# turn_id propagated for downstream session-DB linkage
|
||||
assert r.turn_id == "turn-fake-001"
|
||||
|
||||
def test_tool_iteration_counter_ticks(self):
|
||||
client = FakeClient()
|
||||
# Two completed exec items + one final agent message
|
||||
for i, item_id in enumerate(("ex1", "ex2"), start=1):
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={
|
||||
"type": "commandExecution", "id": item_id,
|
||||
"command": f"cmd{i}", "cwd": "/tmp",
|
||||
"status": "completed", "aggregatedOutput": "ok",
|
||||
"exitCode": 0, "commandActions": [],
|
||||
},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1", "text": "done"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("do stuff", turn_timeout=2.0)
|
||||
assert r.tool_iterations == 2
|
||||
# Each tool item produces (assistant, tool) — 2*2 + final assistant = 5 msgs
|
||||
assert len(r.projected_messages) == 5
|
||||
|
||||
def test_turn_start_failure_returns_error(self):
|
||||
client = FakeClient()
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
def boom(method, params):
|
||||
if method == "turn/start":
|
||||
raise CodexAppServerError(code=-32600, message="bad input")
|
||||
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.error is not None
|
||||
assert "bad input" in r.error
|
||||
assert r.final_text == ""
|
||||
|
||||
def test_turn_start_failure_attaches_redacted_stderr_tail(self):
|
||||
"""When codex stderr has content (non-OAuth), the tail gets attached
|
||||
to the user-facing error so config/provider problems are debuggable
|
||||
instead of just 'Internal error'. Secrets in stderr are redacted
|
||||
via agent.redact(force=True)."""
|
||||
client = FakeClient()
|
||||
client.set_stderr_tail([
|
||||
"ERROR: provider auth failed",
|
||||
"Authorization: Bearer sk-live-deadbeefdeadbeef",
|
||||
"url=https://api.example.com/v1?token=querysecret12345",
|
||||
])
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
def boom(method, params):
|
||||
if method == "turn/start":
|
||||
raise CodexAppServerError(code=-32603, message="Internal error")
|
||||
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.error is not None
|
||||
assert "turn/start failed" in r.error
|
||||
assert "Internal error" in r.error
|
||||
# Stderr tail attached
|
||||
assert "codex stderr" in r.error
|
||||
assert "provider auth failed" in r.error
|
||||
# Secrets redacted
|
||||
assert "sk-live-deadbeefdeadbeef" not in r.error
|
||||
assert "querysecret12345" not in r.error
|
||||
# Non-OAuth → should NOT retire (subprocess JSON-RPC is still healthy).
|
||||
assert r.should_retire is False
|
||||
|
||||
def test_turn_start_timeout_attaches_redacted_stderr_tail(self):
|
||||
"""A non-OAuth TimeoutError on turn/start surfaces with codex stderr
|
||||
context attached and marks the session for retirement."""
|
||||
client = FakeClient()
|
||||
client.set_stderr_tail([
|
||||
"WARN: provider request stalled",
|
||||
"Authorization: Bearer sk-stalled-secret-abc123",
|
||||
])
|
||||
|
||||
def stall(method, params):
|
||||
if method == "turn/start":
|
||||
raise TimeoutError("codex method 'turn/start' timed out after 10s")
|
||||
return {"thread": {"id": "t"}, "activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = stall
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.error is not None
|
||||
assert "turn/start timed out" in r.error
|
||||
assert "provider request stalled" in r.error
|
||||
assert "sk-stalled-secret-abc123" not in r.error
|
||||
assert r.should_retire is True
|
||||
|
||||
def test_startup_failure_returns_error_with_stderr(self):
|
||||
"""Codex thread/start failures during ensure_started() used to bubble
|
||||
up as uncaught exceptions. Now they return a TurnResult.error so
|
||||
AIAgent surfaces a clean diagnostic instead of crashing the turn."""
|
||||
client = FakeClient()
|
||||
client.set_stderr_tail([
|
||||
"FATAL: model_provider 'azure_foundry' not configured",
|
||||
])
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
def boom(method, params):
|
||||
if method == "thread/start":
|
||||
raise CodexAppServerError(code=-32603, message="Internal error")
|
||||
return {}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=2.0)
|
||||
assert r.error is not None
|
||||
assert "startup failed" in r.error
|
||||
assert "model_provider 'azure_foundry' not configured" in r.error
|
||||
assert r.should_retire is True
|
||||
assert r.final_text == ""
|
||||
|
||||
def test_interrupt_during_turn_issues_turn_interrupt(self):
|
||||
client = FakeClient()
|
||||
# Don't queue turn/completed — the loop has to interrupt out
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "commandExecution", "id": "x", "command": "sleep 60",
|
||||
"cwd": "/", "status": "inProgress",
|
||||
"aggregatedOutput": None, "exitCode": None,
|
||||
"commandActions": []},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
s = make_session(client)
|
||||
s.ensure_started()
|
||||
# Trip the interrupt before run_turn even consumes the notification.
|
||||
# The loop will see interrupt set on its first iteration and bail.
|
||||
s.request_interrupt()
|
||||
r = s.run_turn("loop forever", turn_timeout=2.0)
|
||||
assert r.interrupted is True
|
||||
# turn/interrupt was requested with the right turnId
|
||||
assert any(
|
||||
method == "turn/interrupt" and params.get("turnId") == "turn-fake-001"
|
||||
for (method, params) in client.requests
|
||||
)
|
||||
|
||||
def test_deadline_exceeded_records_error(self):
|
||||
client = FakeClient()
|
||||
# No notifications and no completion → must hit deadline
|
||||
s = make_session(client)
|
||||
r = s.run_turn("never finishes", turn_timeout=0.05,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.interrupted is True
|
||||
assert r.error and "timed out" in r.error
|
||||
|
||||
def test_failed_turn_records_error_from_turn_completed(self):
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "failed",
|
||||
"error": {"message": "model error"}},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("x", turn_timeout=1.0)
|
||||
assert r.error and "model error" in r.error
|
||||
|
||||
|
||||
# ---- approval bridge ----
|
||||
|
||||
class TestServerRequestRouting:
|
||||
def test_exec_approval_with_callback_approves_once(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"item/commandExecution/requestApproval", request_id="req-1",
|
||||
command="ls /tmp", cwd="/tmp",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
captured["command"] = command
|
||||
captured["description"] = description
|
||||
return "once"
|
||||
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert captured["command"] == "ls /tmp"
|
||||
# The session must have responded to the server request with "accept"
|
||||
assert ("req-1", {"decision": "accept"}) in client.responses
|
||||
|
||||
def test_exec_approval_no_callback_denies(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request("item/commandExecution/requestApproval", request_id="req-1",
|
||||
command="rm -rf /", cwd="/")
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client) # no approval_callback wired
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("req-1", {"decision": "decline"}) in client.responses
|
||||
|
||||
def test_apply_patch_approval_session_maps_to_session_decision(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"item/fileChange/requestApproval", request_id="req-2",
|
||||
itemId="fc-1",
|
||||
turnId="t1",
|
||||
threadId="th",
|
||||
startedAtMs=1234567890,
|
||||
reason="create new file with hello() function",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
return "session"
|
||||
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("req-2", {"decision": "acceptForSession"}) in client.responses
|
||||
|
||||
def test_unknown_server_request_replied_with_error(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request("totally/unknown", request_id="req-3")
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert any(
|
||||
rid == "req-3" and code == -32601
|
||||
for (rid, code, _msg) in client.error_responses
|
||||
)
|
||||
|
||||
def test_mcp_elicitation_for_hermes_tools_auto_accepts(self):
|
||||
"""When codex elicits on behalf of hermes-tools (our own callback),
|
||||
accept automatically — the user already opted in by enabling the
|
||||
runtime."""
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"mcpServer/elicitation/request", request_id="elic-1",
|
||||
threadId="t", turnId="tu1",
|
||||
serverName="hermes-tools",
|
||||
mode="form",
|
||||
message="confirm",
|
||||
requestedSchema={"type": "object", "properties": {}},
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("elic-1", {"action": "accept", "content": None, "_meta": None}) in client.responses
|
||||
|
||||
def test_mcp_elicitation_for_other_servers_declines(self):
|
||||
"""For third-party MCP servers we decline by default so users
|
||||
explicitly opt in through codex's own UI."""
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"mcpServer/elicitation/request", request_id="elic-2",
|
||||
threadId="t", turnId="tu1",
|
||||
serverName="some-third-party",
|
||||
mode="url",
|
||||
message="please log in",
|
||||
url="https://example.com/oauth",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("elic-2", {"action": "decline", "content": None, "_meta": None}) in client.responses
|
||||
|
||||
def test_routing_auto_approve_bypass(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request("item/commandExecution/requestApproval", request_id="r1",
|
||||
command="ls", cwd="/")
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
# No callback, but routing says auto-approve. Should approve.
|
||||
s = make_session(client, request_routing=_ServerRequestRouting(
|
||||
auto_approve_exec=True))
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
assert ("r1", {"decision": "accept"}) in client.responses
|
||||
|
||||
def test_callback_raises_falls_back_to_decline(self):
|
||||
client = FakeClient()
|
||||
client.queue_server_request("item/commandExecution/requestApproval", request_id="r1",
|
||||
command="ls", cwd="/")
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
|
||||
def boom(*a, **kw):
|
||||
raise RuntimeError("ui crashed")
|
||||
|
||||
s = make_session(client, approval_callback=boom)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
# Fail-closed: deny on callback exception
|
||||
assert ("r1", {"decision": "decline"}) in client.responses
|
||||
|
||||
|
||||
# ---- enriched approval prompts ----
|
||||
|
||||
class TestApprovalPromptEnrichment:
|
||||
"""Quirk #4: apply_patch prompt should show what's changing.
|
||||
Quirk #10: exec prompt should never show empty cwd."""
|
||||
|
||||
def test_exec_falls_back_to_session_cwd(self):
|
||||
"""When codex omits cwd from the approval params, the prompt shows
|
||||
the session cwd, not an empty string."""
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"item/commandExecution/requestApproval", request_id="r1",
|
||||
command="ls", # no cwd
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
captured = {}
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
captured["description"] = description
|
||||
return "once"
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
# Session cwd is /tmp by default in make_session()
|
||||
assert "/tmp" in captured["description"]
|
||||
assert "Codex requests exec in <unknown>" not in captured["description"]
|
||||
|
||||
def test_apply_patch_prompt_summarizes_pending_changes(self):
|
||||
"""When the projector has cached the fileChange item from item/started,
|
||||
the approval prompt surfaces the change summary."""
|
||||
client = FakeClient()
|
||||
# item/started fires first (carries the changes), then approval request
|
||||
client.queue_notification(
|
||||
"item/started",
|
||||
item={"type": "fileChange", "id": "fc-1",
|
||||
"changes": [
|
||||
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
|
||||
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
|
||||
]},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_server_request(
|
||||
"item/fileChange/requestApproval", request_id="req-2",
|
||||
itemId="fc-1", turnId="tu1", threadId="t",
|
||||
startedAtMs=1234567890,
|
||||
reason="add and update files",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
captured = {}
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
captured["command"] = command
|
||||
captured["description"] = description
|
||||
return "once"
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
# Both add and update kinds should be in the summary
|
||||
assert "1 add" in captured["command"] or "1 add" in captured["description"]
|
||||
assert "1 update" in captured["command"] or "1 update" in captured["description"]
|
||||
# And at least one of the paths
|
||||
joined = captured["command"] + " " + captured["description"]
|
||||
assert "/tmp/new.py" in joined or "/tmp/old.py" in joined
|
||||
|
||||
def test_apply_patch_prompt_works_without_cached_summary(self):
|
||||
"""When approval arrives before item/started (or without changes
|
||||
info), prompt falls back to whatever codex provided."""
|
||||
client = FakeClient()
|
||||
client.queue_server_request(
|
||||
"item/fileChange/requestApproval", request_id="req-2",
|
||||
itemId="fc-orphan", turnId="tu1", threadId="t",
|
||||
startedAtMs=1234567890,
|
||||
reason="apply some changes",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
captured = {}
|
||||
def cb(command, description, *, allow_permanent=True):
|
||||
captured["command"] = command
|
||||
return "once"
|
||||
s = make_session(client, approval_callback=cb)
|
||||
s.run_turn("hi", turn_timeout=1.0)
|
||||
# Falls back to the reason
|
||||
assert "apply some changes" in captured["command"]
|
||||
|
||||
|
||||
# ---- openclaw beta.8 parity: retire/wedge/oauth/abort marker ----
|
||||
|
||||
class TestSessionRetirement:
|
||||
"""Mirrors openclaw beta.8's resilience fixes:
|
||||
- retire timed-out app-server clients (should_retire on deadline)
|
||||
- post-tool completion watchdog (don't burn the full deadline after a
|
||||
tool result if codex goes silent)
|
||||
- <turn_aborted> raw marker as terminal (don't wait for turn/completed
|
||||
that never comes)
|
||||
- OAuth refresh failure classification (suggest `codex login` instead
|
||||
of raw RPC error strings)
|
||||
- dead subprocess detection between iterations
|
||||
"""
|
||||
|
||||
def test_deadline_marks_session_for_retirement(self):
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
r = s.run_turn(
|
||||
"never finishes",
|
||||
turn_timeout=0.05,
|
||||
notification_poll_timeout=0.01,
|
||||
)
|
||||
assert r.interrupted is True
|
||||
assert r.error and "timed out" in r.error
|
||||
assert r.should_retire is True, (
|
||||
"Deadline exhaustion must signal retirement so the next turn "
|
||||
"respawns codex instead of riding a wedged subprocess."
|
||||
)
|
||||
|
||||
def test_completed_turn_does_not_retire(self):
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1", "text": "hi"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=1.0)
|
||||
assert r.should_retire is False
|
||||
|
||||
def test_post_tool_quiet_watchdog_trips_and_retires(self):
|
||||
client = FakeClient()
|
||||
# One tool completion, then total silence — no further events,
|
||||
# no turn/completed. With a tiny post_tool_quiet_timeout the
|
||||
# watchdog must fire before the larger turn deadline.
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={
|
||||
"type": "commandExecution", "id": "ex1",
|
||||
"command": "echo hi", "cwd": "/tmp",
|
||||
"status": "completed", "aggregatedOutput": "hi",
|
||||
"exitCode": 0, "commandActions": [],
|
||||
},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn(
|
||||
"tool then silence",
|
||||
turn_timeout=5.0, # would be miserable to wait
|
||||
notification_poll_timeout=0.02,
|
||||
post_tool_quiet_timeout=0.15,
|
||||
)
|
||||
assert r.interrupted is True
|
||||
assert r.should_retire is True
|
||||
assert r.error and "silent" in r.error
|
||||
# Confirm we issued turn/interrupt to free codex compute
|
||||
assert any(method == "turn/interrupt" for (method, _) in client.requests)
|
||||
|
||||
def test_post_tool_watchdog_resets_on_further_activity(self):
|
||||
"""A tool completion followed by an agent message should NOT trip
|
||||
the watchdog — further activity = codex still alive."""
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={
|
||||
"type": "commandExecution", "id": "ex1",
|
||||
"command": "echo hi", "cwd": "/tmp",
|
||||
"status": "completed", "aggregatedOutput": "hi",
|
||||
"exitCode": 0, "commandActions": [],
|
||||
},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
# Non-tool activity immediately after — resets watchdog.
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1", "text": "tool finished"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={"id": "tu1", "status": "completed", "error": None},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn(
|
||||
"tool then talk", turn_timeout=2.0,
|
||||
notification_poll_timeout=0.01,
|
||||
post_tool_quiet_timeout=0.05,
|
||||
)
|
||||
# Tool ran, then text reset the watchdog, then turn/completed.
|
||||
# Should NOT be a retirement case.
|
||||
assert r.tool_iterations == 1
|
||||
assert r.final_text == "tool finished"
|
||||
assert r.should_retire is False
|
||||
assert r.interrupted is False
|
||||
|
||||
def test_turn_aborted_marker_in_text_is_terminal(self):
|
||||
"""If codex emits `<turn_aborted>` in agent text and never sends
|
||||
turn/completed, we still exit promptly instead of burning the
|
||||
deadline."""
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={
|
||||
"type": "agentMessage", "id": "m1",
|
||||
"text": "partial output... <turn_aborted>",
|
||||
},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
# Deliberately NO turn/completed notification queued.
|
||||
s = make_session(client)
|
||||
r = s.run_turn(
|
||||
"abort mid-turn", turn_timeout=2.0,
|
||||
notification_poll_timeout=0.01,
|
||||
)
|
||||
assert r.interrupted is True
|
||||
assert r.error and "turn_aborted" in r.error
|
||||
# Should have exited fast — not waited for the full 2s deadline.
|
||||
# (Can't measure wall clock reliably in CI; presence of the marker
|
||||
# error string instead of a "timed out" message is the proxy.)
|
||||
assert "timed out" not in r.error
|
||||
|
||||
def test_turn_aborted_self_closing_marker_also_terminal(self):
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"item/completed",
|
||||
item={"type": "agentMessage", "id": "m1",
|
||||
"text": "<turn_aborted/>"},
|
||||
threadId="t", turnId="tu1",
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("x", turn_timeout=2.0,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.interrupted is True
|
||||
assert r.error and "turn_aborted" in r.error
|
||||
|
||||
def test_oauth_refresh_failure_on_turn_start_suggests_login(self):
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
client = FakeClient()
|
||||
|
||||
def boom(method, params):
|
||||
if method == "turn/start":
|
||||
raise CodexAppServerError(
|
||||
code=-32603,
|
||||
message="auth refresh failed: invalid_grant",
|
||||
)
|
||||
return {"thread": {"id": "t"},
|
||||
"activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=1.0)
|
||||
assert r.error is not None
|
||||
assert "codex login" in r.error
|
||||
assert r.should_retire is True
|
||||
|
||||
def test_oauth_failure_from_stderr_on_turn_start_failure(self):
|
||||
"""If the RPC error itself is opaque but stderr shows an auth
|
||||
problem, we still classify it as a refresh failure."""
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
client = FakeClient()
|
||||
client.set_stderr_tail([
|
||||
"[2026-05-14T10:00:00Z WARN codex_core::auth] token refresh failed",
|
||||
"[2026-05-14T10:00:00Z ERROR codex_core] please log in again",
|
||||
])
|
||||
|
||||
def boom(method, params):
|
||||
if method == "turn/start":
|
||||
raise CodexAppServerError(code=-32603, message="rpc broke")
|
||||
return {"thread": {"id": "t"},
|
||||
"activePermissionProfile": {"id": "x"}}
|
||||
|
||||
client._request_handler = boom
|
||||
s = make_session(client)
|
||||
r = s.run_turn("hi", turn_timeout=1.0)
|
||||
assert r.error is not None
|
||||
assert "codex login" in r.error
|
||||
assert r.should_retire is True
|
||||
|
||||
def test_oauth_failure_in_turn_completed_error(self):
|
||||
"""A failed turn/completed whose error mentions auth/refresh
|
||||
triggers the re-auth hint + retirement."""
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={
|
||||
"id": "tu1", "status": "failed",
|
||||
"error": {"message": "401 Unauthorized: please reauthenticate"},
|
||||
},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("x", turn_timeout=1.0,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.error is not None
|
||||
assert "codex login" in r.error
|
||||
assert r.should_retire is True
|
||||
|
||||
def test_generic_turn_failure_does_not_trigger_oauth_hint(self):
|
||||
"""A boring model error must NOT rewrite the message into a fake
|
||||
re-auth hint. Conservative classifier."""
|
||||
client = FakeClient()
|
||||
client.queue_notification(
|
||||
"turn/completed", threadId="t",
|
||||
turn={
|
||||
"id": "tu1", "status": "failed",
|
||||
"error": {"message": "rate limit exceeded"},
|
||||
},
|
||||
)
|
||||
s = make_session(client)
|
||||
r = s.run_turn("x", turn_timeout=1.0,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.error is not None
|
||||
assert "codex login" not in r.error
|
||||
assert "rate limit exceeded" in r.error
|
||||
# Generic model failures don't retire — the session itself is fine
|
||||
assert r.should_retire is False
|
||||
|
||||
def test_dead_subprocess_detected_between_iterations(self):
|
||||
"""If codex dies (segfault, OOM, killed by its auth refresh
|
||||
thread), the inter-iteration is_alive check breaks the loop
|
||||
instead of waiting on a queue that will never fill."""
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
s.ensure_started()
|
||||
# Simulate subprocess death by setting _closed (FakeClient's
|
||||
# is_alive returns False when closed).
|
||||
client._closed = True
|
||||
client.set_stderr_tail([
|
||||
"thread 'tokio-runtime-worker' panicked at 'oauth: invalid_grant'",
|
||||
])
|
||||
r = s.run_turn("x", turn_timeout=2.0,
|
||||
notification_poll_timeout=0.01)
|
||||
assert r.should_retire is True
|
||||
# Stderr-derived auth hint takes precedence over generic message
|
||||
assert r.error and "codex login" in r.error
|
||||
|
||||
|
||||
# ---- thread/start cross-fill ----
|
||||
|
||||
class TestThreadStartCrossFill:
|
||||
"""Mirrors openclaw beta.8's tolerance for thread.id/sessionId aliasing."""
|
||||
|
||||
def test_thread_id_under_thread_key(self):
|
||||
client = FakeClient()
|
||||
s = make_session(client)
|
||||
tid = s.ensure_started()
|
||||
assert tid == "thread-fake-001"
|
||||
|
||||
def test_thread_session_id_alias_under_thread_key(self):
|
||||
client = FakeClient()
|
||||
client._request_handler = lambda method, params: (
|
||||
{"thread": {"sessionId": "alias-1"},
|
||||
"activePermissionProfile": {"id": "x"}}
|
||||
if method == "thread/start" else
|
||||
{"turn": {"id": "tu1"}} if method == "turn/start" else {}
|
||||
)
|
||||
s = make_session(client)
|
||||
tid = s.ensure_started()
|
||||
assert tid == "alias-1"
|
||||
|
||||
def test_top_level_session_id_fallback(self):
|
||||
client = FakeClient()
|
||||
client._request_handler = lambda method, params: (
|
||||
{"sessionId": "top-1"} if method == "thread/start" else
|
||||
{"turn": {"id": "tu1"}} if method == "turn/start" else {}
|
||||
)
|
||||
s = make_session(client)
|
||||
tid = s.ensure_started()
|
||||
assert tid == "top-1"
|
||||
|
||||
def test_missing_thread_id_raises(self):
|
||||
from agent.transports.codex_app_server import CodexAppServerError
|
||||
|
||||
client = FakeClient()
|
||||
client._request_handler = lambda method, params: (
|
||||
{"thread": {}, "activePermissionProfile": {"id": "x"}}
|
||||
if method == "thread/start" else
|
||||
{"turn": {"id": "tu1"}}
|
||||
)
|
||||
s = make_session(client)
|
||||
with pytest.raises(CodexAppServerError, match="no thread id"):
|
||||
s.ensure_started()
|
||||
|
||||
|
||||
class TestHasTurnAbortedMarker:
|
||||
"""Unit coverage for the marker matcher itself."""
|
||||
|
||||
def test_empty_string(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_has_turn_aborted_marker,
|
||||
)
|
||||
assert _has_turn_aborted_marker("") is False
|
||||
assert _has_turn_aborted_marker(None) is False # type: ignore[arg-type]
|
||||
|
||||
def test_plain_text_no_marker(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_has_turn_aborted_marker,
|
||||
)
|
||||
assert _has_turn_aborted_marker("normal response with no markers") is False
|
||||
|
||||
def test_open_marker(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_has_turn_aborted_marker,
|
||||
)
|
||||
assert _has_turn_aborted_marker("blah <turn_aborted> blah") is True
|
||||
|
||||
def test_self_closing_marker(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_has_turn_aborted_marker,
|
||||
)
|
||||
assert _has_turn_aborted_marker("<turn_aborted/>") is True
|
||||
|
||||
|
||||
class TestClassifyOAuthFailure:
|
||||
"""Unit coverage for the OAuth classifier; conservative on purpose."""
|
||||
|
||||
def test_invalid_grant_classified(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
hint = _classify_oauth_failure("error: invalid_grant returned by server")
|
||||
assert hint is not None
|
||||
assert "codex login" in hint
|
||||
|
||||
def test_token_refresh_classified(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
hint = _classify_oauth_failure("token refresh failed: network error")
|
||||
assert hint is not None
|
||||
assert "codex login" in hint
|
||||
|
||||
def test_401_classified(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
hint = _classify_oauth_failure("HTTP 401 Unauthorized")
|
||||
assert hint is not None
|
||||
|
||||
def test_generic_error_not_classified(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
assert _classify_oauth_failure("connection reset") is None
|
||||
assert _classify_oauth_failure("model returned bad json") is None
|
||||
assert _classify_oauth_failure("rate limit exceeded") is None
|
||||
|
||||
def test_empty_inputs(self):
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
assert _classify_oauth_failure() is None
|
||||
assert _classify_oauth_failure("") is None
|
||||
assert _classify_oauth_failure("", None) is None # type: ignore[arg-type]
|
||||
|
||||
def test_multi_string_search(self):
|
||||
"""Hint can come from any of the provided strings."""
|
||||
from agent.transports.codex_app_server_session import (
|
||||
_classify_oauth_failure,
|
||||
)
|
||||
hint = _classify_oauth_failure(
|
||||
"rpc returned -32603",
|
||||
"[stderr] token has expired, run codex login",
|
||||
)
|
||||
assert hint is not None
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Tests for CodexEventProjector — codex item/* events → Hermes messages list.
|
||||
|
||||
Drives projection against fixture notifications captured from codex 0.130.0
|
||||
plus synthetic ones for item types we couldn't auth-test live."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.transports.codex_event_projector import (
|
||||
CodexEventProjector,
|
||||
ProjectionResult,
|
||||
_deterministic_call_id,
|
||||
_format_tool_args,
|
||||
)
|
||||
|
||||
|
||||
# --- Fixture: real `commandExecution` notification captured from codex 0.130.0
|
||||
COMMAND_EXEC_COMPLETED = {
|
||||
"method": "item/completed",
|
||||
"params": {
|
||||
"item": {
|
||||
"type": "commandExecution",
|
||||
"id": "f8a75c66-a89e-4fd7-8bcf-2d58e664fa9e",
|
||||
"command": "/bin/bash -lc 'echo hello && ls /tmp | head -3'",
|
||||
"cwd": "/tmp",
|
||||
"processId": None,
|
||||
"source": "userShell",
|
||||
"status": "completed",
|
||||
"commandActions": [
|
||||
{"type": "listFiles", "command": "ls /tmp", "path": "tmp"}
|
||||
],
|
||||
"aggregatedOutput": "hello\naa_lang.json\n",
|
||||
"exitCode": 0,
|
||||
"durationMs": 10,
|
||||
},
|
||||
"threadId": "019e1a94-352b-71e1-b214-e5c67c9ec190",
|
||||
"turnId": "019e1a94-3553-7940-8af3-4ca57142deb7",
|
||||
"completedAtMs": 1778562381151,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestProjectionInvariants:
|
||||
"""Universal invariants that must hold across all projection paths."""
|
||||
|
||||
def test_streaming_deltas_dont_materialize(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
for delta_method in (
|
||||
"item/commandExecution/outputDelta",
|
||||
"item/agentMessage/delta",
|
||||
"item/reasoning/delta",
|
||||
):
|
||||
r = p.project({"method": delta_method, "params": {"delta": "x"}})
|
||||
assert r.messages == [], (
|
||||
f"{delta_method} should NOT produce messages — only "
|
||||
f"item/completed materializes"
|
||||
)
|
||||
assert r.is_tool_iteration is False
|
||||
assert r.final_text is None
|
||||
|
||||
def test_turn_started_and_completed_are_silent(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
for method in ("turn/started", "turn/completed", "thread/started"):
|
||||
r = p.project({"method": method, "params": {}})
|
||||
assert r.messages == []
|
||||
|
||||
def test_unknown_method_silent(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project({"method": "totally/unknown", "params": {}})
|
||||
assert r.messages == []
|
||||
|
||||
|
||||
class TestCommandExecutionProjection:
|
||||
"""Real captured notification → assistant tool_call + tool result."""
|
||||
|
||||
def test_command_completed_produces_two_messages(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project(COMMAND_EXEC_COMPLETED)
|
||||
assert len(r.messages) == 2
|
||||
assert r.is_tool_iteration is True
|
||||
|
||||
def test_first_message_is_assistant_tool_call(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assistant = msgs[0]
|
||||
assert assistant["role"] == "assistant"
|
||||
assert assistant["content"] is None
|
||||
assert len(assistant["tool_calls"]) == 1
|
||||
tc = assistant["tool_calls"][0]
|
||||
assert tc["type"] == "function"
|
||||
assert tc["function"]["name"] == "exec_command"
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert "echo hello" in args["command"]
|
||||
assert args["cwd"] == "/tmp"
|
||||
|
||||
def test_second_message_is_tool_result_correlating_by_id(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assistant, tool = msgs
|
||||
assert tool["role"] == "tool"
|
||||
assert tool["tool_call_id"] == assistant["tool_calls"][0]["id"]
|
||||
assert "hello" in tool["content"]
|
||||
|
||||
def test_nonzero_exit_code_annotated_in_tool_result(self) -> None:
|
||||
item = {**COMMAND_EXEC_COMPLETED["params"]["item"], "exitCode": 2,
|
||||
"aggregatedOutput": "boom"}
|
||||
notif = {
|
||||
"method": "item/completed",
|
||||
"params": {**COMMAND_EXEC_COMPLETED["params"], "item": item},
|
||||
}
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project(notif).messages
|
||||
assert "[exit 2]" in msgs[1]["content"]
|
||||
assert "boom" in msgs[1]["content"]
|
||||
|
||||
def test_deterministic_call_id_across_replay(self) -> None:
|
||||
# Same item id → same call_id (prefix cache must stay valid).
|
||||
p1 = CodexEventProjector()
|
||||
p2 = CodexEventProjector()
|
||||
a = p1.project(COMMAND_EXEC_COMPLETED).messages
|
||||
b = p2.project(COMMAND_EXEC_COMPLETED).messages
|
||||
assert a[0]["tool_calls"][0]["id"] == b[0]["tool_calls"][0]["id"]
|
||||
|
||||
|
||||
class TestAgentMessageProjection:
|
||||
"""assistant text → final_text + assistant message."""
|
||||
|
||||
def test_agent_message_projects_to_assistant(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
r = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "agentMessage", "id": "x",
|
||||
"text": "hi there"}},
|
||||
})
|
||||
assert r.final_text == "hi there"
|
||||
assert r.messages == [{"role": "assistant", "content": "hi there"}]
|
||||
assert r.is_tool_iteration is False
|
||||
|
||||
def test_pending_reasoning_attaches_to_next_assistant_message(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
# First a reasoning item lands
|
||||
r1 = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "reasoning", "id": "r1",
|
||||
"summary": ["thinking..."],
|
||||
"content": ["step 1", "step 2"]}},
|
||||
})
|
||||
assert r1.messages == [] # reasoning alone produces no message
|
||||
# Then the assistant message
|
||||
r2 = p.project({
|
||||
"method": "item/completed",
|
||||
"params": {"item": {"type": "agentMessage", "id": "a1",
|
||||
"text": "ok"}},
|
||||
})
|
||||
assistant = r2.messages[0]
|
||||
assert "reasoning" in assistant
|
||||
assert "thinking" in assistant["reasoning"]
|
||||
assert "step 1" in assistant["reasoning"]
|
||||
|
||||
def test_reasoning_consumed_after_attaching(self) -> None:
|
||||
p = CodexEventProjector()
|
||||
p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "reasoning", "id": "r1", "summary": ["once"], "content": []}}})
|
||||
first = p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "agentMessage", "id": "a", "text": "first"}}}).messages[0]
|
||||
second = p.project({"method": "item/completed", "params": {"item": {
|
||||
"type": "agentMessage", "id": "b", "text": "second"}}}).messages[0]
|
||||
assert "reasoning" in first
|
||||
assert "reasoning" not in second
|
||||
|
||||
|
||||
class TestFileChangeProjection:
|
||||
def test_file_change_summary_no_inlined_content(self) -> None:
|
||||
item = {
|
||||
"type": "fileChange",
|
||||
"id": "fc1",
|
||||
"status": "applied",
|
||||
"changes": [
|
||||
{"kind": {"type": "add"}, "path": "/tmp/new.py"},
|
||||
{"kind": {"type": "update"}, "path": "/tmp/old.py"},
|
||||
],
|
||||
}
|
||||
p = CodexEventProjector()
|
||||
msgs = p.project({"method": "item/completed",
|
||||
"params": {"item": item}}).messages
|
||||
assert len(msgs) == 2
|
||||
tc = msgs[0]["tool_calls"][0]
|
||||
assert tc["function"]["name"] == "apply_patch"
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert len(args["changes"]) == 2
|
||||
assert all("kind" in c and "path" in c for c in args["changes"])
|
||||
assert "applied" in msgs[1]["content"]
|
||||
|
||||
|
||||
class TestMcpToolCallProjection:
|
||||
def test_mcp_tool_call_namespaced(self) -> None:
|
||||
item = {
|
||||
"type": "mcpToolCall",
|
||||
"id": "m1",
|
||||
"server": "obsidian",
|
||||
"tool": "search_notes",
|
||||
"status": "completed",
|
||||
"arguments": {"query": "hermes"},
|
||||
"result": {"content": [{"text": "found"}]},
|
||||
"error": None,
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert msgs[0]["tool_calls"][0]["function"]["name"] == "mcp.obsidian.search_notes"
|
||||
assert "found" in msgs[1]["content"]
|
||||
|
||||
def test_mcp_error_surfaced(self) -> None:
|
||||
item = {
|
||||
"type": "mcpToolCall", "id": "m2",
|
||||
"server": "x", "tool": "y", "status": "failed",
|
||||
"arguments": {}, "result": None,
|
||||
"error": {"code": -1, "message": "no"},
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert "error" in msgs[1]["content"]
|
||||
|
||||
|
||||
class TestUserAndOpaqueProjection:
|
||||
def test_user_message_text_fragments_only(self) -> None:
|
||||
item = {
|
||||
"type": "userMessage", "id": "u1",
|
||||
"content": [
|
||||
{"type": "text", "text": "hello"},
|
||||
{"type": "image", "url": "http://x/y"},
|
||||
{"type": "text", "text": "world"},
|
||||
],
|
||||
}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert msgs[0]["role"] == "user"
|
||||
assert "hello" in msgs[0]["content"]
|
||||
assert "world" in msgs[0]["content"]
|
||||
|
||||
def test_opaque_item_recorded_without_fabricated_tool_calls(self) -> None:
|
||||
item = {"type": "plan", "id": "p1", "text": "do the thing"}
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0]["role"] == "assistant"
|
||||
assert "plan" in msgs[0]["content"].lower()
|
||||
assert "tool_calls" not in msgs[0]
|
||||
|
||||
|
||||
class TestHelpers:
|
||||
def test_deterministic_call_id_stable(self) -> None:
|
||||
assert _deterministic_call_id("exec", "abc") == _deterministic_call_id("exec", "abc")
|
||||
assert _deterministic_call_id("exec", "abc") != _deterministic_call_id("exec", "xyz")
|
||||
|
||||
def test_deterministic_call_id_handles_missing_id(self) -> None:
|
||||
# Should not raise, should be stable for same item type
|
||||
a = _deterministic_call_id("exec", "")
|
||||
b = _deterministic_call_id("exec", "")
|
||||
assert a == b
|
||||
assert "exec" in a
|
||||
|
||||
def test_format_tool_args_sorted_keys(self) -> None:
|
||||
# Sorted keys = deterministic across replays = prefix cache stays valid
|
||||
a = _format_tool_args({"b": 1, "a": 2})
|
||||
b = _format_tool_args({"a": 2, "b": 1})
|
||||
assert a == b
|
||||
|
||||
|
||||
class TestRoleAlternationInvariant:
|
||||
"""The project must never emit two assistant messages back-to-back from
|
||||
one item — that breaks Hermes' message alternation invariant."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"item",
|
||||
[
|
||||
{"type": "commandExecution", "id": "c1", "command": "x",
|
||||
"cwd": "/", "status": "completed", "aggregatedOutput": "",
|
||||
"exitCode": 0, "commandActions": []},
|
||||
{"type": "fileChange", "id": "f1", "status": "applied",
|
||||
"changes": []},
|
||||
{"type": "mcpToolCall", "id": "m1", "server": "s", "tool": "t",
|
||||
"status": "completed", "arguments": {}, "result": None,
|
||||
"error": None},
|
||||
{"type": "dynamicToolCall", "id": "d1", "tool": "x",
|
||||
"arguments": {}, "status": "completed",
|
||||
"contentItems": [], "success": True},
|
||||
],
|
||||
)
|
||||
def test_tool_items_emit_assistant_then_tool(self, item) -> None:
|
||||
msgs = CodexEventProjector().project(
|
||||
{"method": "item/completed", "params": {"item": item}}
|
||||
).messages
|
||||
assert len(msgs) == 2
|
||||
assert msgs[0]["role"] == "assistant"
|
||||
assert msgs[1]["role"] == "tool"
|
||||
assert msgs[1]["tool_call_id"] == msgs[0]["tool_calls"][0]["id"]
|
||||
@@ -100,6 +100,44 @@ class TestCodexBuildKwargs:
|
||||
)
|
||||
assert "prompt_cache_key" not in kw
|
||||
|
||||
def test_xai_responses_sends_cache_key_via_extra_body(self, transport):
|
||||
"""xAI's Responses API documents ``prompt_cache_key`` as the
|
||||
body-level cache-routing key (the ``x-grok-conv-id`` header is
|
||||
Chat-Completions-only). Passing it via ``extra_body`` is robust
|
||||
against openai SDK builds whose ``Responses.stream()`` kwarg
|
||||
signature ever drops the field — the body field still serializes
|
||||
and reaches xAI either way. The ``x-grok-conv-id`` header is kept
|
||||
as a belt-and-braces fallback so cache routing survives even
|
||||
when the body field would be stripped by an intermediate proxy.
|
||||
Ref: https://docs.x.ai/developers/advanced-api-usage/prompt-caching/maximizing-cache-hits
|
||||
"""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
session_id="conv-xai-1",
|
||||
is_xai_responses=True,
|
||||
)
|
||||
assert "prompt_cache_key" not in kw
|
||||
assert kw.get("extra_body", {}).get("prompt_cache_key") == "conv-xai-1"
|
||||
assert kw.get("extra_headers", {}).get("x-grok-conv-id") == "conv-xai-1"
|
||||
|
||||
def test_xai_responses_extra_body_preserves_caller_fields(self, transport):
|
||||
"""When the caller already supplies ``extra_body`` (e.g. via
|
||||
request_overrides), the xAI cache-key injection must merge into
|
||||
the existing dict instead of overwriting it. Caller-supplied
|
||||
``prompt_cache_key`` wins (setdefault semantics) so user overrides
|
||||
aren't silently clobbered by the transport."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.3", messages=messages, tools=[],
|
||||
session_id="conv-xai-1",
|
||||
is_xai_responses=True,
|
||||
request_overrides={"extra_body": {"prompt_cache_key": "caller-override", "other_field": 42}},
|
||||
)
|
||||
eb = kw.get("extra_body", {})
|
||||
assert eb.get("prompt_cache_key") == "caller-override"
|
||||
assert eb.get("other_field") == 42
|
||||
|
||||
def test_max_tokens(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Tests for the hermes-tools-as-MCP server module surface.
|
||||
|
||||
We don't run a live MCP session in unit tests — that requires the codex
|
||||
subprocess + client + an event loop. These tests pin the static
|
||||
contract: the module imports, the EXPOSED_TOOLS list is sane, and the
|
||||
build helper assembles a server when the SDK is present.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestModuleSurface:
|
||||
def test_module_imports_clean(self):
|
||||
from agent.transports import hermes_tools_mcp_server as m
|
||||
assert callable(m.main)
|
||||
assert callable(m._build_server)
|
||||
assert isinstance(m.EXPOSED_TOOLS, tuple)
|
||||
assert len(m.EXPOSED_TOOLS) > 0
|
||||
|
||||
def test_exposed_tools_are_safe_subset(self):
|
||||
"""We MUST NOT expose tools codex already has, because codex'
|
||||
own builtins are better-integrated with its sandbox + approvals.
|
||||
Specifically: no terminal/shell, no read_file/write_file, no
|
||||
patch — those are codex's built-in tools."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
forbidden = {
|
||||
"terminal", "shell", "read_file", "write_file", "patch",
|
||||
"search_files", "process",
|
||||
}
|
||||
leaked = forbidden & set(EXPOSED_TOOLS)
|
||||
assert not leaked, (
|
||||
f"these tools must NOT be exposed via the codex callback "
|
||||
f"because codex has built-in equivalents: {leaked}"
|
||||
)
|
||||
|
||||
def test_expected_hermes_specific_tools_listed(self):
|
||||
"""The Hermes-specific tools should be present so users on the
|
||||
codex runtime keep access to them."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for required in (
|
||||
"web_search",
|
||||
"web_extract",
|
||||
"browser_navigate",
|
||||
"vision_analyze",
|
||||
"image_generate",
|
||||
"skill_view",
|
||||
):
|
||||
assert required in EXPOSED_TOOLS, f"missing {required!r}"
|
||||
|
||||
def test_agent_loop_tools_not_exposed(self):
|
||||
"""delegate_task / memory / session_search / todo require the
|
||||
running AIAgent context to dispatch, so a stateless MCP callback
|
||||
can't drive them. They must NOT be in EXPOSED_TOOLS."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for agent_loop_tool in ("delegate_task", "memory", "session_search", "todo"):
|
||||
assert agent_loop_tool not in EXPOSED_TOOLS, (
|
||||
f"{agent_loop_tool!r} requires the agent loop context "
|
||||
"and can't be reached through a stateless MCP callback"
|
||||
)
|
||||
|
||||
def test_kanban_worker_tools_exposed(self):
|
||||
"""Kanban workers run as `hermes chat -q` subprocesses; if they
|
||||
come up on the codex_app_server runtime, the worker can do the
|
||||
actual work via codex's shell but needs the kanban tools through
|
||||
the MCP callback to report back to the kernel. Without these
|
||||
tools available, the worker would hang at completion time."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
# Worker handoff tools — every dispatched worker uses at least
|
||||
# one of {complete, block, comment} to close out its task.
|
||||
for worker_tool in (
|
||||
"kanban_complete",
|
||||
"kanban_block",
|
||||
"kanban_comment",
|
||||
"kanban_heartbeat",
|
||||
):
|
||||
assert worker_tool in EXPOSED_TOOLS, (
|
||||
f"{worker_tool!r} missing from codex callback — kanban "
|
||||
"workers on codex_app_server runtime would hang"
|
||||
)
|
||||
|
||||
def test_kanban_orchestrator_tools_exposed(self):
|
||||
"""Orchestrator agents need to dispatch new tasks, query the
|
||||
board, and unblock/link tasks. Exposed so an orchestrator on
|
||||
codex_app_server can do its job."""
|
||||
from agent.transports.hermes_tools_mcp_server import EXPOSED_TOOLS
|
||||
for orch_tool in (
|
||||
"kanban_create",
|
||||
"kanban_show",
|
||||
"kanban_list",
|
||||
"kanban_unblock",
|
||||
"kanban_link",
|
||||
):
|
||||
assert orch_tool in EXPOSED_TOOLS, (
|
||||
f"{orch_tool!r} missing from codex callback"
|
||||
)
|
||||
|
||||
|
||||
class TestMain:
|
||||
def test_main_returns_2_when_mcp_unavailable(self, monkeypatch):
|
||||
"""When the mcp package isn't installed, main() should exit
|
||||
cleanly with code 2 and an install hint, not crash."""
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
def boom_build(*a, **kw):
|
||||
raise ImportError("mcp not installed")
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", boom_build)
|
||||
rc = m.main(["--verbose"])
|
||||
assert rc == 2
|
||||
|
||||
def test_main_handles_keyboard_interrupt(self, monkeypatch):
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
class FakeServer:
|
||||
def run(self):
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", lambda: FakeServer())
|
||||
rc = m.main([])
|
||||
assert rc == 0
|
||||
|
||||
def test_main_returns_1_on_runtime_error(self, monkeypatch):
|
||||
import agent.transports.hermes_tools_mcp_server as m
|
||||
|
||||
class CrashingServer:
|
||||
def run(self):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(m, "_build_server", lambda: CrashingServer())
|
||||
rc = m.main([])
|
||||
assert rc == 1
|
||||
Reference in New Issue
Block a user