Merge remote-tracking branch 'origin/main' into bb/gui
# Conflicts: # apps/dashboard/src/i18n/af.ts # apps/dashboard/src/i18n/de.ts # apps/dashboard/src/i18n/es.ts # apps/dashboard/src/i18n/fr.ts # apps/dashboard/src/i18n/ga.ts # apps/dashboard/src/i18n/hu.ts # apps/dashboard/src/i18n/it.ts # apps/dashboard/src/i18n/ja.ts # apps/dashboard/src/i18n/ko.ts # apps/dashboard/src/i18n/pt.ts # apps/dashboard/src/i18n/ru.ts # apps/dashboard/src/i18n/tr.ts # apps/dashboard/src/i18n/uk.ts # apps/dashboard/src/i18n/zh-hant.ts # gateway/config.py # hermes_cli/main.py # plugins/strike-freedom-cockpit/README.md # tui_gateway/server.py
This commit is contained in:
Brooklyn Nicholson
@@ -2123,6 +2123,227 @@ class TestCodexAuxiliaryAdapterTimeout:
|
||||
assert time.monotonic() - started < 0.14
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Issue #23432 — auxiliary timeout poisons cached client; later aux calls fail
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestAuxiliaryClientPoisonedCacheEviction:
|
||||
"""Connection/timeout errors must evict the cached aux client.
|
||||
|
||||
Otherwise the next auxiliary call (compression retry, memory flush,
|
||||
background review) reuses the closed httpx transport and fails with
|
||||
``Connection error`` even though the main provider route is healthy.
|
||||
See https://github.com/NousResearch/hermes-agent/issues/23432.
|
||||
"""
|
||||
|
||||
def test_evict_cached_client_instance_drops_direct_match(self):
|
||||
from agent.auxiliary_client import (
|
||||
_client_cache, _client_cache_lock, _evict_cached_client_instance,
|
||||
)
|
||||
|
||||
target = MagicMock(name="target_client")
|
||||
other = MagicMock(name="other_client")
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
_client_cache[("openrouter", False, None, None, None)] = (target, "x", None)
|
||||
_client_cache[("anthropic", False, None, None, None)] = (other, "y", None)
|
||||
try:
|
||||
assert _evict_cached_client_instance(target) is True
|
||||
assert ("openrouter", False, None, None, None) not in _client_cache
|
||||
assert ("anthropic", False, None, None, None) in _client_cache
|
||||
finally:
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
|
||||
def test_evict_cached_client_instance_walks_codex_wrapper(self):
|
||||
"""Closing the underlying OpenAI client must evict the Codex shim."""
|
||||
from agent.auxiliary_client import (
|
||||
_client_cache, _client_cache_lock, _evict_cached_client_instance,
|
||||
CodexAuxiliaryClient,
|
||||
)
|
||||
|
||||
real = SimpleNamespace(api_key="k", base_url="https://chatgpt.com/backend-api/codex",
|
||||
responses=SimpleNamespace(stream=lambda **k: None),
|
||||
close=lambda: None)
|
||||
wrapper = CodexAuxiliaryClient(real, "gpt-5.5")
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
_client_cache[("openai-codex", False, None, None, None)] = (wrapper, "gpt-5.5", None)
|
||||
try:
|
||||
# Eviction by the inner OpenAI client must remove the wrapper entry.
|
||||
assert _evict_cached_client_instance(real) is True
|
||||
assert ("openai-codex", False, None, None, None) not in _client_cache
|
||||
finally:
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
|
||||
def test_evict_cached_client_instance_handles_none_and_misses(self):
|
||||
from agent.auxiliary_client import _evict_cached_client_instance
|
||||
|
||||
assert _evict_cached_client_instance(None) is False
|
||||
assert _evict_cached_client_instance(MagicMock()) is False
|
||||
|
||||
def test_evict_cached_client_instance_walks_async_wrapper(self):
|
||||
"""async_mode is part of the cache key so sync and async share the same
|
||||
underlying OpenAI client across two distinct cache entries. A single
|
||||
timeout that closes the leaf must evict BOTH — otherwise the async
|
||||
entry survives, keeps reusing the dead transport, and every async
|
||||
aux call (compression, vision, session_search) fails fast with
|
||||
'Connection error' until gateway restart even while the sync route
|
||||
recovers.
|
||||
|
||||
Regression for the async-side gap left by #23482, which fixed the
|
||||
sync wrapper's _real_client walk but missed the async wrappers.
|
||||
"""
|
||||
from agent.auxiliary_client import (
|
||||
_client_cache, _client_cache_lock, _evict_cached_client_instance,
|
||||
CodexAuxiliaryClient, AsyncCodexAuxiliaryClient,
|
||||
)
|
||||
|
||||
real = SimpleNamespace(api_key="k", base_url="https://chatgpt.com/backend-api/codex",
|
||||
responses=SimpleNamespace(stream=lambda **k: None),
|
||||
close=lambda: None)
|
||||
sync_wrapper = CodexAuxiliaryClient(real, "gpt-5.5")
|
||||
async_wrapper = AsyncCodexAuxiliaryClient(sync_wrapper)
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
_client_cache[("openai-codex", False, None, None, None)] = (sync_wrapper, "gpt-5.5", None)
|
||||
_client_cache[("openai-codex", True, None, None, None)] = (async_wrapper, "gpt-5.5", None)
|
||||
try:
|
||||
assert _evict_cached_client_instance(real) is True
|
||||
assert ("openai-codex", False, None, None, None) not in _client_cache
|
||||
assert ("openai-codex", True, None, None, None) not in _client_cache, (
|
||||
"async cache entry survived eviction — wrapper is missing _real_client"
|
||||
)
|
||||
finally:
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
|
||||
def test_codex_timeout_evicts_cached_wrapper(self):
|
||||
"""The timeout closer evicts the cache entry that wraps the closed client."""
|
||||
from agent.auxiliary_client import (
|
||||
_client_cache, _client_cache_lock,
|
||||
_CodexCompletionsAdapter, CodexAuxiliaryClient,
|
||||
)
|
||||
|
||||
class SlowAliveStream:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def __iter__(self):
|
||||
for _ in range(20):
|
||||
time.sleep(0.01)
|
||||
yield SimpleNamespace(type="response.in_progress")
|
||||
|
||||
def get_final_response(self): # pragma: no cover — timeout fires first
|
||||
return SimpleNamespace(output=[], usage=None)
|
||||
|
||||
closed = {"flag": False}
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self):
|
||||
self.responses = SimpleNamespace(stream=lambda **k: SlowAliveStream())
|
||||
self.api_key = "k"
|
||||
self.base_url = "https://chatgpt.com/backend-api/codex"
|
||||
|
||||
def close(self):
|
||||
closed["flag"] = True
|
||||
|
||||
fake_real = FakeClient()
|
||||
wrapper = CodexAuxiliaryClient(fake_real, "gpt-5.5")
|
||||
cache_key = ("openai-codex", False, None, None, None)
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
_client_cache[cache_key] = (wrapper, "gpt-5.5", None)
|
||||
try:
|
||||
adapter = _CodexCompletionsAdapter(fake_real, "gpt-5.5")
|
||||
with pytest.raises(TimeoutError):
|
||||
adapter.create(
|
||||
messages=[{"role": "user", "content": "x"}],
|
||||
timeout=0.05,
|
||||
)
|
||||
assert closed["flag"] is True, "timeout closer must close inner client"
|
||||
assert cache_key not in _client_cache, (
|
||||
"timeout closer must evict cache entry that wraps the closed client"
|
||||
)
|
||||
finally:
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
|
||||
def test_call_llm_evicts_on_connection_error_with_explicit_provider(self):
|
||||
"""Connection error on an explicit provider must drop the cached client.
|
||||
|
||||
This is the exact reporter scenario: ``auxiliary.compression.provider:
|
||||
main`` (resolves to ``openai-codex``) → no fallback chain runs (not
|
||||
auto), but the cached client was poisoned by a prior timeout and must
|
||||
be evicted so the next call rebuilds.
|
||||
"""
|
||||
from agent.auxiliary_client import _client_cache, _client_cache_lock
|
||||
|
||||
poisoned = MagicMock(name="poisoned_client")
|
||||
poisoned.base_url = "https://chatgpt.com/backend-api/codex"
|
||||
poisoned.chat.completions.create.side_effect = ConnectionError("transport closed")
|
||||
|
||||
cache_key = ("openai-codex", False, None, None, None)
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
_client_cache[cache_key] = (poisoned, "gpt-5.5", None)
|
||||
|
||||
try:
|
||||
with patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("openai-codex", "gpt-5.5", None, None, None),
|
||||
), patch(
|
||||
"agent.auxiliary_client._get_cached_client",
|
||||
return_value=(poisoned, "gpt-5.5"),
|
||||
):
|
||||
with pytest.raises(ConnectionError):
|
||||
call_llm(
|
||||
task="compression",
|
||||
messages=[{"role": "user", "content": "x"}],
|
||||
)
|
||||
assert cache_key not in _client_cache, (
|
||||
"connection error must evict cached client so the next call rebuilds"
|
||||
)
|
||||
finally:
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_call_llm_evicts_on_connection_error_with_explicit_provider(self):
|
||||
from agent.auxiliary_client import _client_cache, _client_cache_lock
|
||||
|
||||
poisoned = MagicMock(name="poisoned_async_client")
|
||||
poisoned.base_url = "https://chatgpt.com/backend-api/codex"
|
||||
poisoned.chat.completions.create = AsyncMock(side_effect=ConnectionError("transport closed"))
|
||||
|
||||
cache_key = ("openai-codex", True, None, None, None)
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
_client_cache[cache_key] = (poisoned, "gpt-5.5", None)
|
||||
|
||||
try:
|
||||
with patch(
|
||||
"agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("openai-codex", "gpt-5.5", None, None, None),
|
||||
), patch(
|
||||
"agent.auxiliary_client._get_cached_client",
|
||||
return_value=(poisoned, "gpt-5.5"),
|
||||
):
|
||||
with pytest.raises(ConnectionError):
|
||||
await async_call_llm(
|
||||
task="compression",
|
||||
messages=[{"role": "user", "content": "x"}],
|
||||
)
|
||||
assert cache_key not in _client_cache
|
||||
finally:
|
||||
with _client_cache_lock:
|
||||
_client_cache.clear()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_call_kwargs — tool dedup at API boundary
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -2311,3 +2532,165 @@ class TestAnthropicExplicitApiKey:
|
||||
assert mock_build.call_args.args[0] == "explicit-fallback-key", (
|
||||
"resolve_provider_client must forward explicit_api_key to _try_anthropic()"
|
||||
)
|
||||
|
||||
|
||||
# ── Auxiliary unhealthy-provider TTL cache (issue #23570) ────────────────
|
||||
|
||||
|
||||
class TestAuxUnhealthyCache:
|
||||
"""Recently-402'd providers are skipped on subsequent aux calls.
|
||||
|
||||
Without this, every compression / title-gen / session-search call on a
|
||||
long session retries a depleted OpenRouter (~1 RTT to 402) before
|
||||
falling back to the next provider. The TTL cache hides the unhealthy
|
||||
provider for ``_AUX_UNHEALTHY_TTL_SECONDS`` so the chain skips it.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
from agent.auxiliary_client import _reset_aux_unhealthy_cache
|
||||
_reset_aux_unhealthy_cache()
|
||||
|
||||
def teardown_method(self):
|
||||
from agent.auxiliary_client import _reset_aux_unhealthy_cache
|
||||
_reset_aux_unhealthy_cache()
|
||||
|
||||
def test_mark_then_skip(self):
|
||||
from agent.auxiliary_client import (
|
||||
_mark_provider_unhealthy,
|
||||
_is_provider_unhealthy,
|
||||
)
|
||||
assert _is_provider_unhealthy("openrouter") is False
|
||||
_mark_provider_unhealthy("openrouter")
|
||||
assert _is_provider_unhealthy("openrouter") is True
|
||||
|
||||
def test_ttl_expiry_evicts(self):
|
||||
from agent.auxiliary_client import (
|
||||
_mark_provider_unhealthy,
|
||||
_is_provider_unhealthy,
|
||||
_aux_unhealthy_until,
|
||||
)
|
||||
_mark_provider_unhealthy("openrouter", ttl=0.01)
|
||||
assert _is_provider_unhealthy("openrouter") is True
|
||||
import time
|
||||
time.sleep(0.02)
|
||||
# Lazy eviction: first lookup after expiry returns False AND removes the entry.
|
||||
assert _is_provider_unhealthy("openrouter") is False
|
||||
assert "openrouter" not in _aux_unhealthy_until
|
||||
|
||||
def test_alias_normalization(self):
|
||||
"""'codex' should normalize to 'openai-codex' so the cache lookup
|
||||
matches the chain label."""
|
||||
from agent.auxiliary_client import (
|
||||
_mark_provider_unhealthy,
|
||||
_is_provider_unhealthy,
|
||||
)
|
||||
_mark_provider_unhealthy("codex")
|
||||
assert _is_provider_unhealthy("openai-codex") is True
|
||||
|
||||
def test_resolve_auto_skips_unhealthy_step2(self):
|
||||
"""_resolve_auto Step-2 chain skips unhealthy providers."""
|
||||
from agent.auxiliary_client import (
|
||||
_resolve_auto,
|
||||
_mark_provider_unhealthy,
|
||||
)
|
||||
nous_client = MagicMock()
|
||||
# Mark OpenRouter unhealthy → chain should skip it and pick nous.
|
||||
_mark_provider_unhealthy("openrouter")
|
||||
with patch("agent.auxiliary_client._read_main_provider", return_value=""), \
|
||||
patch("agent.auxiliary_client._read_main_model", return_value=""), \
|
||||
patch("agent.auxiliary_client._try_openrouter") as or_try, \
|
||||
patch("agent.auxiliary_client._try_nous", return_value=(nous_client, "nous-model")), \
|
||||
patch("agent.auxiliary_client._try_custom_endpoint", return_value=(None, None)), \
|
||||
patch("agent.auxiliary_client._resolve_api_key_provider", return_value=(None, None)):
|
||||
client, model = _resolve_auto()
|
||||
assert client is nous_client
|
||||
assert model == "nous-model"
|
||||
# The skipped provider's _try_* should NOT have been called at all.
|
||||
or_try.assert_not_called()
|
||||
|
||||
def test_resolve_auto_skips_unhealthy_main_in_step1(self):
|
||||
"""Step-1 also consults the unhealthy cache so a depleted main
|
||||
provider doesn't burn a 402 RTT every aux call. Falls through to
|
||||
Step-2 chain (which also respects the cache)."""
|
||||
from agent.auxiliary_client import (
|
||||
_resolve_auto,
|
||||
_mark_provider_unhealthy,
|
||||
)
|
||||
nous_client = MagicMock()
|
||||
_mark_provider_unhealthy("openrouter")
|
||||
with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \
|
||||
patch("agent.auxiliary_client._read_main_model", return_value="anthropic/claude-sonnet-4.6"), \
|
||||
patch("agent.auxiliary_client.resolve_provider_client") as step1, \
|
||||
patch("agent.auxiliary_client._try_openrouter") as or_try, \
|
||||
patch("agent.auxiliary_client._try_nous", return_value=(nous_client, "n-model")), \
|
||||
patch("agent.auxiliary_client._try_custom_endpoint", return_value=(None, None)), \
|
||||
patch("agent.auxiliary_client._resolve_api_key_provider", return_value=(None, None)):
|
||||
client, model = _resolve_auto()
|
||||
# Step-1 was bypassed — resolve_provider_client never invoked
|
||||
step1.assert_not_called()
|
||||
# Step-2 also skipped openrouter and landed on nous
|
||||
or_try.assert_not_called()
|
||||
assert client is nous_client
|
||||
|
||||
def test_payment_fallback_skips_unhealthy(self):
|
||||
"""_try_payment_fallback also consults the unhealthy cache so a 402
|
||||
on OpenRouter doesn't cause a second OR call within the same chain
|
||||
iteration if it gets re-entered."""
|
||||
from agent.auxiliary_client import (
|
||||
_try_payment_fallback,
|
||||
_mark_provider_unhealthy,
|
||||
)
|
||||
nous_client = MagicMock()
|
||||
# Mark BOTH the failed provider (openrouter) and a sibling (custom)
|
||||
# unhealthy. The chain should still find nous.
|
||||
_mark_provider_unhealthy("local/custom")
|
||||
with patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"), \
|
||||
patch("agent.auxiliary_client._try_openrouter") as or_try, \
|
||||
patch("agent.auxiliary_client._try_nous", return_value=(nous_client, "n-model")), \
|
||||
patch("agent.auxiliary_client._try_custom_endpoint") as custom_try, \
|
||||
patch("agent.auxiliary_client._resolve_api_key_provider", return_value=(None, None)):
|
||||
client, model, label = _try_payment_fallback("openrouter", task="compression")
|
||||
assert client is nous_client
|
||||
assert label == "nous"
|
||||
# OR is skipped via skip_chain_labels (failed provider), custom via unhealthy cache.
|
||||
or_try.assert_not_called()
|
||||
custom_try.assert_not_called()
|
||||
|
||||
def test_call_llm_marks_provider_unhealthy_on_402(self, monkeypatch):
|
||||
"""A 402 from call_llm causes the provider to be marked unhealthy
|
||||
so the next call skips it instead of re-trying the same depleted
|
||||
endpoint."""
|
||||
from agent.auxiliary_client import (
|
||||
call_llm,
|
||||
_is_provider_unhealthy,
|
||||
)
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
||||
|
||||
primary_client = MagicMock()
|
||||
# base_url tells _recoverable_pool_provider() that this is OpenRouter
|
||||
# (resolved_provider="auto" doesn't carry that information by itself).
|
||||
primary_client.base_url = "https://openrouter.ai/api/v1/"
|
||||
err = Exception("Payment Required: insufficient credits")
|
||||
err.status_code = 402
|
||||
primary_client.chat.completions.create.side_effect = err
|
||||
|
||||
nous_client = MagicMock()
|
||||
nous_resp = MagicMock()
|
||||
nous_resp.choices = [MagicMock(message=MagicMock(content="ok"))]
|
||||
nous_client.chat.completions.create.return_value = nous_resp
|
||||
|
||||
with patch("agent.auxiliary_client._get_cached_client",
|
||||
return_value=(primary_client, "google/gemini-3-flash-preview")), \
|
||||
patch("agent.auxiliary_client._resolve_task_provider_model",
|
||||
return_value=("auto", "google/gemini-3-flash-preview", None, None, None)), \
|
||||
patch("agent.auxiliary_client._try_payment_fallback",
|
||||
return_value=(nous_client, "n-model", "nous")), \
|
||||
patch("agent.auxiliary_client._build_call_kwargs",
|
||||
return_value={"model": "n-model", "messages": [{"role": "user", "content": "hi"}]}):
|
||||
assert _is_provider_unhealthy("openrouter") is False
|
||||
call_llm(
|
||||
task="compression",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
# After the 402, OpenRouter is in the unhealthy cache.
|
||||
assert _is_provider_unhealthy("openrouter") is True
|
||||
|
||||
@@ -1020,3 +1020,106 @@ def test_rename_summary_mixed_consolidation_and_pruning(curator_env):
|
||||
assert merge_idx < drop_idx, "consolidated should render before pruned"
|
||||
assert "merge-me → umbrella" in lines[merge_idx]
|
||||
assert "drop-me — pruned (stale)" in lines[drop_idx]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pin hint — surfaces `hermes curator pin <umbrella>` in the rename block so
|
||||
# users learn the command exists at the moment they care (a consolidation
|
||||
# just landed against their library). The hint is gated on having at least
|
||||
# one umbrella destination — pruned-only runs skip it.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_rename_summary_pin_hint_appears_when_consolidation_produced_umbrella(curator_env):
|
||||
"""When at least one skill was absorbed into an umbrella, hint at pinning it."""
|
||||
result = curator_env._build_rename_summary(
|
||||
before_names={"pdf-extraction", "docx-extraction", "document-tools"},
|
||||
after_report=[{"name": "document-tools", "state": "active"}],
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "skill_manage",
|
||||
"arguments": json.dumps({
|
||||
"action": "delete",
|
||||
"name": "pdf-extraction",
|
||||
"absorbed_into": "document-tools",
|
||||
}),
|
||||
},
|
||||
{
|
||||
"name": "skill_manage",
|
||||
"arguments": json.dumps({
|
||||
"action": "delete",
|
||||
"name": "docx-extraction",
|
||||
"absorbed_into": "document-tools",
|
||||
}),
|
||||
},
|
||||
],
|
||||
model_final="",
|
||||
)
|
||||
assert "hermes curator pin document-tools" in result
|
||||
assert "keep an umbrella stable" in result
|
||||
|
||||
|
||||
def test_rename_summary_pin_hint_skipped_for_pruned_only_runs(curator_env):
|
||||
"""Pruned-only runs have nothing surviving to pin — hint should not appear."""
|
||||
result = curator_env._build_rename_summary(
|
||||
before_names={"old-flaky-thing", "another-stale", "keeper"},
|
||||
after_report=[{"name": "keeper", "state": "active"}],
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "skill_manage",
|
||||
"arguments": json.dumps({
|
||||
"action": "delete",
|
||||
"name": "old-flaky-thing",
|
||||
"absorbed_into": "",
|
||||
}),
|
||||
},
|
||||
{
|
||||
"name": "skill_manage",
|
||||
"arguments": json.dumps({
|
||||
"action": "delete",
|
||||
"name": "another-stale",
|
||||
"absorbed_into": "",
|
||||
}),
|
||||
},
|
||||
],
|
||||
model_final="",
|
||||
)
|
||||
# Block still renders (skills were archived) but no pin hint.
|
||||
assert "archived 2 skill(s):" in result
|
||||
assert "hermes curator pin" not in result
|
||||
assert "keep an umbrella stable" not in result
|
||||
|
||||
|
||||
def test_rename_summary_pin_hint_picks_one_umbrella_when_multiple_absorbed(curator_env):
|
||||
"""Multiple umbrellas → hint shows one example (alphabetically first), not a list."""
|
||||
result = curator_env._build_rename_summary(
|
||||
before_names={"a-skill", "b-skill", "umbrella-zeta", "umbrella-alpha"},
|
||||
after_report=[
|
||||
{"name": "umbrella-zeta", "state": "active"},
|
||||
{"name": "umbrella-alpha", "state": "active"},
|
||||
],
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "skill_manage",
|
||||
"arguments": json.dumps({
|
||||
"action": "delete",
|
||||
"name": "a-skill",
|
||||
"absorbed_into": "umbrella-zeta",
|
||||
}),
|
||||
},
|
||||
{
|
||||
"name": "skill_manage",
|
||||
"arguments": json.dumps({
|
||||
"action": "delete",
|
||||
"name": "b-skill",
|
||||
"absorbed_into": "umbrella-alpha",
|
||||
}),
|
||||
},
|
||||
],
|
||||
model_final="",
|
||||
)
|
||||
# Sorted picks alphabetically first.
|
||||
assert "hermes curator pin umbrella-alpha" in result
|
||||
# Exactly one hint line, not one per umbrella.
|
||||
pin_lines = [ln for ln in result.splitlines() if "hermes curator pin" in ln]
|
||||
assert len(pin_lines) == 1
|
||||
|
||||
@@ -156,7 +156,12 @@ def test_t_missing_key_in_non_english_falls_back_to_english(tmp_path, monkeypatc
|
||||
(fake_locales / "zh.yaml").write_text("# intentionally empty\n", encoding="utf-8")
|
||||
monkeypatch.setattr(i18n, "_locales_dir", lambda: fake_locales)
|
||||
i18n.reset_language_cache()
|
||||
assert i18n.t("foo", lang="zh") == "English Foo"
|
||||
try:
|
||||
assert i18n.t("foo", lang="zh") == "English Foo"
|
||||
finally:
|
||||
# Clear the cache on teardown so subsequent tests don't see the
|
||||
# fake "foo: English Foo" catalog instead of the real locales/*.yaml.
|
||||
i18n.reset_language_cache()
|
||||
|
||||
|
||||
def test_t_unknown_language_uses_english():
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Tests for `agent.markdown_tables.realign_markdown_tables`.
|
||||
|
||||
These cover the alignment guarantee on CJK / wide-character tables and
|
||||
the conservative no-op behaviour on non-table input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from textwrap import dedent
|
||||
|
||||
from wcwidth import wcswidth
|
||||
|
||||
from agent.markdown_tables import (
|
||||
is_table_divider,
|
||||
looks_like_table_row,
|
||||
realign_markdown_tables,
|
||||
split_table_row,
|
||||
)
|
||||
|
||||
|
||||
def _column_offsets(line: str) -> list[int]:
|
||||
"""Return the display-cell index of every ``|`` in ``line``."""
|
||||
|
||||
cells: list[int] = []
|
||||
width = 0
|
||||
for ch in line:
|
||||
if ch == "|":
|
||||
cells.append(width)
|
||||
# wcswidth on a single char; clamp negatives.
|
||||
w = wcswidth(ch)
|
||||
width += w if w > 0 else 1
|
||||
return cells
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# split_table_row / is_table_divider / looks_like_table_row
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_split_strips_outer_pipes_and_trims():
|
||||
assert split_table_row("| a | b | c |") == ["a", "b", "c"]
|
||||
assert split_table_row("|配置|状态|") == ["配置", "状态"]
|
||||
assert split_table_row("a | b | c") == ["a", "b", "c"]
|
||||
|
||||
|
||||
def test_is_table_divider_handles_alignment_colons():
|
||||
assert is_table_divider("|---|---|")
|
||||
assert is_table_divider("| :--- | ---: | :---: |")
|
||||
assert not is_table_divider("| - | - |") # 1 dash is not a divider
|
||||
assert not is_table_divider("| a | b |")
|
||||
assert not is_table_divider("---") # single column, no pipes
|
||||
|
||||
|
||||
def test_looks_like_table_row():
|
||||
assert looks_like_table_row("| a | b |")
|
||||
assert looks_like_table_row("a | b | c") # no leading pipe, ≥2 pipes
|
||||
assert not looks_like_table_row("not a table")
|
||||
assert not looks_like_table_row("a | b") # one pipe, no leading pipe
|
||||
assert not looks_like_table_row("")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# realign_markdown_tables
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_no_op_on_text_without_tables():
|
||||
text = "Hello world\nThis has no | pipes table.\n"
|
||||
assert realign_markdown_tables(text) == text
|
||||
|
||||
|
||||
def test_no_op_when_pipes_but_no_divider():
|
||||
text = "echo a | grep b\necho c | wc -l\n"
|
||||
assert realign_markdown_tables(text) == text
|
||||
|
||||
|
||||
def test_cjk_table_pipes_align_across_rows():
|
||||
# Model-emitted (under-padded for CJK) input.
|
||||
src = dedent(
|
||||
"""\
|
||||
| 配置 | Config | 论文 (%) | 复现 (%) | 差值 | 状态 |
|
||||
|------|--------|---------|---------|------|------|
|
||||
| Vicuna (report) | dense | 79.30 | 未完成 | - | × |
|
||||
| ChatGLM | chat | 37.60 | 37.82 | +0.22 | ✓ |
|
||||
| 通义千问 | qwen | (无) | 报错 | - | × |
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src).rstrip("\n").split("\n")
|
||||
|
||||
# All rows in the rebuilt block must have pipes at identical display
|
||||
# columns — that's the alignment guarantee.
|
||||
offsets = [_column_offsets(row) for row in out]
|
||||
assert all(o == offsets[0] for o in offsets), (
|
||||
"rebuilt table rows do not share pipe column offsets:\n"
|
||||
+ "\n".join(out)
|
||||
)
|
||||
# And we expect 7 pipes per row (6 columns + outer borders).
|
||||
assert len(offsets[0]) == 7
|
||||
|
||||
|
||||
def test_emoji_with_cjk_table_aligns():
|
||||
src = dedent(
|
||||
"""\
|
||||
| 模型 | 状态 | 备注 |
|
||||
|------|------|------|
|
||||
| 千问 | ✅ | 通过 |
|
||||
| Claude | ✅ | 推理强 |
|
||||
| 文心一言 | ❌ | 报错 |
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src).rstrip("\n").split("\n")
|
||||
offsets = [_column_offsets(row) for row in out]
|
||||
# The emoji-with-variation-selector case (⚠️) intentionally tolerates
|
||||
# 1-cell drift; bare emoji like ✅ / ❌ have stable wcwidth and must
|
||||
# align. Use bare emoji here so the assertion is hard.
|
||||
assert all(o == offsets[0] for o in offsets), (
|
||||
"emoji+CJK rows do not share pipe column offsets:\n" + "\n".join(out)
|
||||
)
|
||||
|
||||
|
||||
def test_already_aligned_ascii_table_remains_aligned():
|
||||
src = dedent(
|
||||
"""\
|
||||
| a | b |
|
||||
|-----|-----|
|
||||
| 1 | 2 |
|
||||
| foo | bar |
|
||||
"""
|
||||
)
|
||||
out = realign_markdown_tables(src).rstrip("\n").split("\n")
|
||||
offsets = [_column_offsets(row) for row in out]
|
||||
assert all(o == offsets[0] for o in offsets)
|
||||
|
||||
|
||||
def test_passes_non_table_lines_through_around_a_table():
|
||||
src = dedent(
|
||||
"""\
|
||||
Here is a comparison:
|
||||
|
||||
| 模型 | 状态 |
|
||||
|------|------|
|
||||
| 千问 | 通过 |
|
||||
|
||||
And some prose after.
|
||||
"""
|
||||
)
|
||||
|
||||
out = realign_markdown_tables(src)
|
||||
assert out.startswith("Here is a comparison:\n")
|
||||
assert out.endswith("And some prose after.\n")
|
||||
# And the table lines are aligned.
|
||||
block = [ln for ln in out.split("\n") if "|" in ln]
|
||||
offsets = [_column_offsets(row) for row in block]
|
||||
assert all(o == offsets[0] for o in offsets)
|
||||
|
||||
|
||||
def test_handles_ragged_rows_by_padding_short_rows():
|
||||
src = dedent(
|
||||
"""\
|
||||
| a | b | c |
|
||||
|---|---|---|
|
||||
| 1 | 2 |
|
||||
| x | y | z |
|
||||
"""
|
||||
)
|
||||
out = realign_markdown_tables(src).rstrip("\n").split("\n")
|
||||
offsets = [_column_offsets(row) for row in out]
|
||||
# Short rows must be padded out so they have the same pipe count
|
||||
# and column positions as the header.
|
||||
assert all(len(o) == len(offsets[0]) for o in offsets)
|
||||
assert all(o == offsets[0] for o in offsets)
|
||||
|
||||
|
||||
def test_multiple_tables_in_one_text():
|
||||
src = dedent(
|
||||
"""\
|
||||
First:
|
||||
|
||||
| 配置 | 值 |
|
||||
|------|----|
|
||||
| 通义 | 1 |
|
||||
|
||||
Second:
|
||||
|
||||
| model | n |
|
||||
|-------|---|
|
||||
| gpt | 2 |
|
||||
"""
|
||||
)
|
||||
out = realign_markdown_tables(src)
|
||||
# Each table block individually aligns.
|
||||
blocks: list[list[str]] = []
|
||||
current: list[str] = []
|
||||
for line in out.split("\n"):
|
||||
if "|" in line:
|
||||
current.append(line)
|
||||
elif current:
|
||||
blocks.append(current)
|
||||
current = []
|
||||
if current:
|
||||
blocks.append(current)
|
||||
|
||||
assert len(blocks) == 2
|
||||
for block in blocks:
|
||||
offsets = [_column_offsets(row) for row in block]
|
||||
assert all(o == offsets[0] for o in offsets), (
|
||||
f"block did not align:\n" + "\n".join(block)
|
||||
)
|
||||
@@ -0,0 +1,991 @@
|
||||
"""Unit tests for the plugin LLM facade (``agent.plugin_llm``).
|
||||
|
||||
These tests exercise the trust gate, JSON parsing, schema validation,
|
||||
image input encoding, and the auxiliary-client invocation contract.
|
||||
The auxiliary client itself is stubbed via ``make_plugin_llm_for_test``
|
||||
so we don't hit real providers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.plugin_llm import (
|
||||
PluginLlm,
|
||||
PluginLlmCompleteResult,
|
||||
PluginLlmImageInput,
|
||||
PluginLlmStructuredResult,
|
||||
PluginLlmTextInput,
|
||||
PluginLlmTrustError,
|
||||
_build_structured_messages,
|
||||
_check_overrides,
|
||||
_coerce_allowlist,
|
||||
_parse_structured_text,
|
||||
_strip_code_fences,
|
||||
_TrustPolicy,
|
||||
make_plugin_llm_for_test,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fake_response(text: str, *, prompt: int = 4, completion: int = 6) -> SimpleNamespace:
|
||||
"""Build an OpenAI-shaped response with the given text + token usage."""
|
||||
return SimpleNamespace(
|
||||
choices=[
|
||||
SimpleNamespace(
|
||||
message=SimpleNamespace(content=text, role="assistant"),
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
usage=SimpleNamespace(
|
||||
prompt_tokens=prompt,
|
||||
completion_tokens=completion,
|
||||
total_tokens=prompt + completion,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _trusted_policy(plugin_id: str = "trusted-plugin", **overrides: Any) -> _TrustPolicy:
|
||||
defaults = dict(
|
||||
allow_provider_override=True,
|
||||
allowed_providers=None,
|
||||
allow_any_provider=True,
|
||||
allow_model_override=True,
|
||||
allowed_models=None,
|
||||
allow_any_model=True,
|
||||
allow_agent_id_override=True,
|
||||
allow_profile_override=True,
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return _TrustPolicy(plugin_id=plugin_id, **defaults)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Trust gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTrustGate:
|
||||
def test_default_policy_blocks_provider_override(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the provider"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider="anthropic",
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_default_policy_blocks_model_override(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the model"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model="claude-3-5-sonnet",
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_default_policy_blocks_agent_override(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
with pytest.raises(PluginLlmTrustError, match="non-default agent id"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model=None,
|
||||
requested_agent_id="ada",
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_default_policy_blocks_profile_override(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the auth profile"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile="work",
|
||||
)
|
||||
|
||||
def test_overrides_independent(self):
|
||||
"""Each override is gated independently — turning on
|
||||
``allow_model_override`` does NOT also grant provider override."""
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="model-only",
|
||||
allow_model_override=True,
|
||||
allow_any_model=True,
|
||||
)
|
||||
# model alone passes
|
||||
_, m, _, _ = _check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model="gpt-4o",
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
assert m == "gpt-4o"
|
||||
# provider alone is still denied
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the provider"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider="anthropic",
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_provider_allowlist_rejects_non_listed(self):
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="restricted",
|
||||
allow_provider_override=True,
|
||||
allowed_providers=frozenset({"openrouter", "anthropic"}),
|
||||
allow_any_provider=False,
|
||||
)
|
||||
with pytest.raises(PluginLlmTrustError, match="not in plugins.entries"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider="openai",
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_provider_allowlist_accepts_listed_case_insensitively(self):
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="restricted",
|
||||
allow_provider_override=True,
|
||||
allowed_providers=frozenset({"openrouter"}),
|
||||
allow_any_provider=False,
|
||||
)
|
||||
p, _, _, _ = _check_overrides(
|
||||
policy,
|
||||
requested_provider="OpenRouter",
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
assert p == "OpenRouter"
|
||||
|
||||
def test_model_allowlist_rejects_non_listed(self):
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="restricted",
|
||||
allow_model_override=True,
|
||||
allowed_models=frozenset({"openai/gpt-4o-mini"}),
|
||||
allow_any_model=False,
|
||||
)
|
||||
with pytest.raises(PluginLlmTrustError, match="not in plugins.entries"):
|
||||
_check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model="anthropic/claude-3-opus",
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
|
||||
def test_model_allowlist_accepts_listed_case_insensitively(self):
|
||||
policy = _TrustPolicy(
|
||||
plugin_id="restricted",
|
||||
allow_model_override=True,
|
||||
allowed_models=frozenset({"openai/gpt-4o-mini"}),
|
||||
allow_any_model=False,
|
||||
)
|
||||
_, m, _, _ = _check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model="OpenAI/GPT-4o-mini",
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
assert m == "OpenAI/GPT-4o-mini"
|
||||
|
||||
def test_no_overrides_passes_through(self):
|
||||
policy = _TrustPolicy(plugin_id="locked")
|
||||
result = _check_overrides(
|
||||
policy,
|
||||
requested_provider=None,
|
||||
requested_model=None,
|
||||
requested_agent_id=None,
|
||||
requested_profile=None,
|
||||
)
|
||||
assert result == (None, None, None, None)
|
||||
|
||||
def test_all_overrides_when_fully_trusted(self):
|
||||
policy = _trusted_policy()
|
||||
result = _check_overrides(
|
||||
policy,
|
||||
requested_provider="openrouter",
|
||||
requested_model="anthropic/claude-3-5-sonnet",
|
||||
requested_agent_id="ada",
|
||||
requested_profile="work",
|
||||
)
|
||||
assert result == ("openrouter", "anthropic/claude-3-5-sonnet", "ada", "work")
|
||||
|
||||
|
||||
class TestAllowlistCoercion:
|
||||
def test_missing_yields_none(self):
|
||||
ranges, allow_any = _coerce_allowlist(None)
|
||||
assert ranges is None
|
||||
assert allow_any is False
|
||||
|
||||
def test_list_of_strings(self):
|
||||
ranges, allow_any = _coerce_allowlist(["A", "B"])
|
||||
assert ranges == frozenset({"a", "b"})
|
||||
assert allow_any is False
|
||||
|
||||
def test_star_alone_means_any(self):
|
||||
ranges, allow_any = _coerce_allowlist(["*"])
|
||||
assert ranges == frozenset()
|
||||
assert allow_any is True
|
||||
|
||||
def test_star_plus_specific_keeps_specifics(self):
|
||||
ranges, allow_any = _coerce_allowlist(["*", "openrouter"])
|
||||
assert ranges == frozenset({"openrouter"})
|
||||
assert allow_any is True
|
||||
|
||||
def test_non_list_yields_none(self):
|
||||
ranges, allow_any = _coerce_allowlist("openrouter")
|
||||
assert ranges is None
|
||||
assert allow_any is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structured message building
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStructuredMessageBuilding:
|
||||
def test_text_only_input(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Extract the action items",
|
||||
inputs=[PluginLlmTextInput(text="meeting notes go here")],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
assert len(messages) == 1
|
||||
assert messages[0]["role"] == "user"
|
||||
parts = messages[0]["content"]
|
||||
assert parts[0]["type"] == "text"
|
||||
assert "Extract the action items" in parts[0]["text"]
|
||||
assert parts[1] == {"type": "text", "text": "meeting notes go here"}
|
||||
|
||||
def test_json_mode_adds_system_directive(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Summarise",
|
||||
inputs=[PluginLlmTextInput(text="content")],
|
||||
json_mode=True,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
assert messages[0]["role"] == "system"
|
||||
assert "JSON object" in messages[0]["content"]
|
||||
|
||||
def test_schema_name_appended_to_header(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Extract fields",
|
||||
inputs=[PluginLlmTextInput(text="data")],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name="action.items",
|
||||
system_prompt=None,
|
||||
)
|
||||
header = messages[0]["content"][0]["text"]
|
||||
assert "Schema name: action.items" in header
|
||||
|
||||
def test_image_bytes_encoded_as_data_url(self):
|
||||
png_bytes = b"\x89PNG\r\n\x1a\nfake"
|
||||
messages = _build_structured_messages(
|
||||
instructions="Read the image",
|
||||
inputs=[
|
||||
PluginLlmImageInput(data=png_bytes, mime_type="image/png"),
|
||||
PluginLlmTextInput(text="prefer printed text"),
|
||||
],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
parts = messages[0]["content"]
|
||||
assert parts[1]["type"] == "image_url"
|
||||
url = parts[1]["image_url"]["url"]
|
||||
assert url.startswith("data:image/png;base64,")
|
||||
decoded = base64.b64decode(url.split(",", 1)[1])
|
||||
assert decoded == png_bytes
|
||||
assert parts[2] == {"type": "text", "text": "prefer printed text"}
|
||||
|
||||
def test_image_url_passed_through(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Caption this",
|
||||
inputs=[PluginLlmImageInput(url="https://example.com/cat.jpg")],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
img_part = messages[0]["content"][1]
|
||||
assert img_part["type"] == "image_url"
|
||||
assert img_part["image_url"]["url"] == "https://example.com/cat.jpg"
|
||||
|
||||
def test_dict_inputs_normalized(self):
|
||||
messages = _build_structured_messages(
|
||||
instructions="Test",
|
||||
inputs=[
|
||||
{"type": "text", "text": "hello"},
|
||||
{"type": "image", "url": "https://x.example/y.png"},
|
||||
],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
parts = messages[0]["content"]
|
||||
assert parts[1]["text"] == "hello"
|
||||
assert parts[2]["image_url"]["url"] == "https://x.example/y.png"
|
||||
|
||||
def test_invalid_input_block_rejected(self):
|
||||
with pytest.raises(ValueError, match="Unknown input block"):
|
||||
_build_structured_messages(
|
||||
instructions="Test",
|
||||
inputs=[{"type": "audio", "data": b""}],
|
||||
json_mode=False,
|
||||
json_schema=None,
|
||||
schema_name=None,
|
||||
system_prompt=None,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestJsonParsing:
|
||||
def test_strip_code_fences_with_json_label(self):
|
||||
assert _strip_code_fences('```json\n{"a":1}\n```') == '{"a":1}'
|
||||
|
||||
def test_strip_code_fences_without_label(self):
|
||||
assert _strip_code_fences("```\nfoo\n```") == "foo"
|
||||
|
||||
def test_strip_code_fences_no_fence(self):
|
||||
assert _strip_code_fences('{"a":1}') == '{"a":1}'
|
||||
|
||||
def test_parse_returns_text_when_not_json_mode(self):
|
||||
parsed, ct = _parse_structured_text(
|
||||
text='{"a": 1}', json_mode=False, json_schema=None
|
||||
)
|
||||
assert parsed is None
|
||||
assert ct == "text"
|
||||
|
||||
def test_parse_valid_json_with_json_mode(self):
|
||||
parsed, ct = _parse_structured_text(
|
||||
text='{"language": "French", "is_question": true}',
|
||||
json_mode=True,
|
||||
json_schema=None,
|
||||
)
|
||||
assert parsed == {"language": "French", "is_question": True}
|
||||
assert ct == "json"
|
||||
|
||||
def test_parse_strips_code_fences_before_loading(self):
|
||||
parsed, ct = _parse_structured_text(
|
||||
text='Here you go:\n```json\n{"ok": true}\n```',
|
||||
json_mode=True,
|
||||
json_schema=None,
|
||||
)
|
||||
assert parsed == {"ok": True}
|
||||
assert ct == "json"
|
||||
|
||||
def test_parse_returns_text_on_invalid_json(self):
|
||||
parsed, ct = _parse_structured_text(
|
||||
text="not even close to json",
|
||||
json_mode=True,
|
||||
json_schema=None,
|
||||
)
|
||||
assert parsed is None
|
||||
assert ct == "text"
|
||||
|
||||
def test_schema_validation_rejects_mismatch(self):
|
||||
pytest.importorskip("jsonschema")
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"language": {"type": "string"}},
|
||||
"required": ["language"],
|
||||
}
|
||||
with pytest.raises(ValueError, match="did not match schema"):
|
||||
_parse_structured_text(
|
||||
text='{"is_question": true}',
|
||||
json_mode=False,
|
||||
json_schema=schema,
|
||||
)
|
||||
|
||||
def test_schema_validation_accepts_match(self):
|
||||
pytest.importorskip("jsonschema")
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"language": {"type": "string"}},
|
||||
"required": ["language"],
|
||||
}
|
||||
parsed, ct = _parse_structured_text(
|
||||
text='{"language": "French"}',
|
||||
json_mode=False,
|
||||
json_schema=schema,
|
||||
)
|
||||
assert parsed == {"language": "French"}
|
||||
assert ct == "json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end facade
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPluginLlmFacade:
|
||||
def test_complete_uses_active_model_by_default(self):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return "auto", "default", _fake_response("Hello world.")
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
result = llm.complete([{"role": "user", "content": "hi"}])
|
||||
assert isinstance(result, PluginLlmCompleteResult)
|
||||
assert result.text == "Hello world."
|
||||
assert captured["provider_override"] is None
|
||||
assert captured["model_override"] is None
|
||||
assert captured["profile_override"] is None
|
||||
assert result.usage.input_tokens == 4
|
||||
assert result.usage.total_tokens == 10
|
||||
|
||||
def test_complete_rejects_provider_override_without_trust(self):
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=lambda **_: ("x", "y", _fake_response("")),
|
||||
)
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the provider"):
|
||||
llm.complete(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
provider="openrouter",
|
||||
)
|
||||
|
||||
def test_complete_rejects_model_override_without_trust(self):
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=lambda **_: ("x", "y", _fake_response("")),
|
||||
)
|
||||
with pytest.raises(PluginLlmTrustError, match="cannot override the model"):
|
||||
llm.complete(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
model="anthropic/claude-3-opus",
|
||||
)
|
||||
|
||||
def test_complete_passes_through_trusted_overrides(self):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return "anthropic", "claude-3-opus", _fake_response("ok")
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_trusted_policy("my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
result = llm.complete(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
provider="anthropic",
|
||||
model="claude-3-opus",
|
||||
profile="work",
|
||||
agent_id="ada",
|
||||
temperature=0.0,
|
||||
max_tokens=128,
|
||||
timeout=10.0,
|
||||
purpose="extract",
|
||||
)
|
||||
# The recorded provider/model in the result come from the override,
|
||||
# since the stub caller echoed those values.
|
||||
assert result.provider == "anthropic"
|
||||
assert result.model == "claude-3-opus"
|
||||
assert captured["provider_override"] == "anthropic"
|
||||
assert captured["model_override"] == "claude-3-opus"
|
||||
assert captured["profile_override"] == "work"
|
||||
assert captured["temperature"] == 0.0
|
||||
assert captured["max_tokens"] == 128
|
||||
assert captured["timeout"] == 10.0
|
||||
|
||||
def test_complete_structured_returns_parsed_json(self):
|
||||
def fake_caller(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response(
|
||||
'{"language": "French", "is_question": true, "confidence": 0.99}'
|
||||
)
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
result = llm.complete_structured(
|
||||
instructions="Detect language",
|
||||
input=[PluginLlmTextInput(text="Comment ça va?")],
|
||||
json_mode=True,
|
||||
)
|
||||
assert isinstance(result, PluginLlmStructuredResult)
|
||||
assert result.parsed == {
|
||||
"language": "French",
|
||||
"is_question": True,
|
||||
"confidence": 0.99,
|
||||
}
|
||||
assert result.content_type == "json"
|
||||
|
||||
def test_complete_structured_returns_text_on_unparseable_response(self):
|
||||
def fake_caller(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response("Sorry, I can't help with that.")
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
result = llm.complete_structured(
|
||||
instructions="Detect language",
|
||||
input=[PluginLlmTextInput(text="x")],
|
||||
json_mode=True,
|
||||
)
|
||||
assert result.parsed is None
|
||||
assert result.content_type == "text"
|
||||
assert result.text.startswith("Sorry")
|
||||
|
||||
def test_complete_structured_validates_against_schema(self):
|
||||
pytest.importorskip("jsonschema")
|
||||
|
||||
def fake_caller(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response('{"unrelated": "field"}')
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"language": {"type": "string"}},
|
||||
"required": ["language"],
|
||||
}
|
||||
with pytest.raises(ValueError, match="did not match schema"):
|
||||
llm.complete_structured(
|
||||
instructions="Detect language",
|
||||
input=[PluginLlmTextInput(text="x")],
|
||||
json_schema=schema,
|
||||
)
|
||||
|
||||
def test_complete_structured_requires_instructions(self):
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=MagicMock(),
|
||||
)
|
||||
with pytest.raises(ValueError, match="non-empty instructions"):
|
||||
llm.complete_structured(
|
||||
instructions=" ",
|
||||
input=[PluginLlmTextInput(text="x")],
|
||||
)
|
||||
|
||||
def test_complete_structured_requires_at_least_one_input(self):
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=MagicMock(),
|
||||
)
|
||||
with pytest.raises(ValueError, match="at least one input"):
|
||||
llm.complete_structured(
|
||||
instructions="Extract",
|
||||
input=[],
|
||||
)
|
||||
|
||||
def test_complete_structured_emits_response_format_extra_body(self):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return "openai", "gpt-4o", _fake_response('{"a": 1}')
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
schema = {"type": "object"}
|
||||
llm.complete_structured(
|
||||
instructions="Test",
|
||||
input=[PluginLlmTextInput(text="x")],
|
||||
json_schema=schema,
|
||||
)
|
||||
rf = captured["extra_body"]["response_format"]
|
||||
assert rf["type"] == "json_schema"
|
||||
assert rf["json_schema"]["schema"] == schema
|
||||
|
||||
def test_complete_structured_with_image_passes_image_url_part(self):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return "openai", "gpt-4o", _fake_response('{"caption": "ok"}')
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
png = b"fake-bytes"
|
||||
llm.complete_structured(
|
||||
instructions="Caption this",
|
||||
input=[PluginLlmImageInput(data=png, mime_type="image/png")],
|
||||
json_mode=True,
|
||||
)
|
||||
msgs = captured["messages"]
|
||||
user_msg = next(m for m in msgs if m["role"] == "user")
|
||||
image_parts = [p for p in user_msg["content"] if p.get("type") == "image_url"]
|
||||
assert len(image_parts) == 1
|
||||
assert image_parts[0]["image_url"]["url"].startswith("data:image/png;base64,")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async surface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAsyncSurface:
|
||||
def test_acomplete_uses_async_caller(self):
|
||||
async def fake_async(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response("async hello")
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
async_caller=fake_async,
|
||||
)
|
||||
|
||||
async def _run() -> PluginLlmCompleteResult:
|
||||
return await llm.acomplete([{"role": "user", "content": "hi"}])
|
||||
|
||||
result = asyncio.run(_run())
|
||||
assert result.text == "async hello"
|
||||
assert result.provider == "openai"
|
||||
|
||||
def test_acomplete_structured_parses_json(self):
|
||||
async def fake_async(**_kwargs):
|
||||
return "openai", "gpt-4o", _fake_response('{"x": 42}')
|
||||
|
||||
llm = make_plugin_llm_for_test(
|
||||
plugin_id="my-plugin",
|
||||
policy=_TrustPolicy(plugin_id="my-plugin"),
|
||||
async_caller=fake_async,
|
||||
)
|
||||
|
||||
async def _run() -> PluginLlmStructuredResult:
|
||||
return await llm.acomplete_structured(
|
||||
instructions="Extract x",
|
||||
input=[PluginLlmTextInput(text="data")],
|
||||
json_mode=True,
|
||||
)
|
||||
|
||||
result = asyncio.run(_run())
|
||||
assert result.parsed == {"x": 42}
|
||||
assert result.content_type == "json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config-driven trust gate (round-trip via plugins.entries.<id>.llm)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConfigDrivenPolicy:
|
||||
def test_policy_loaded_from_yaml(self, tmp_path, monkeypatch):
|
||||
from agent.plugin_llm import _resolve_trust_policy
|
||||
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
"""
|
||||
plugins:
|
||||
entries:
|
||||
my-plugin:
|
||||
llm:
|
||||
allow_provider_override: true
|
||||
allowed_providers: [openrouter, anthropic]
|
||||
allow_model_override: true
|
||||
allowed_models:
|
||||
- openai/gpt-4o-mini
|
||||
- anthropic/claude-3-5-haiku
|
||||
allow_profile_override: false
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
from hermes_cli import config as _config_mod
|
||||
_config_mod._config_cache = None # type: ignore[attr-defined]
|
||||
|
||||
policy = _resolve_trust_policy("my-plugin")
|
||||
assert policy.allow_provider_override is True
|
||||
assert policy.allow_model_override is True
|
||||
assert policy.allow_profile_override is False
|
||||
assert policy.allowed_providers == frozenset({"openrouter", "anthropic"})
|
||||
assert policy.allowed_models == frozenset({
|
||||
"openai/gpt-4o-mini", "anthropic/claude-3-5-haiku",
|
||||
})
|
||||
|
||||
def test_missing_plugin_entry_yields_default_deny(self, tmp_path, monkeypatch):
|
||||
from agent.plugin_llm import _resolve_trust_policy
|
||||
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text("plugins: {}\n", encoding="utf-8")
|
||||
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
||||
from hermes_cli import config as _config_mod
|
||||
_config_mod._config_cache = None # type: ignore[attr-defined]
|
||||
|
||||
policy = _resolve_trust_policy("never-configured")
|
||||
assert policy.allow_provider_override is False
|
||||
assert policy.allow_model_override is False
|
||||
assert policy.allow_profile_override is False
|
||||
assert policy.allow_agent_id_override is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin context wiring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPluginContextIntegration:
|
||||
def test_ctx_llm_is_lazy_singleton(self):
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(name="test-plugin", source="test", key="test-plugin")
|
||||
manager = PluginManager()
|
||||
ctx = PluginContext(manifest, manager)
|
||||
first = ctx.llm
|
||||
second = ctx.llm
|
||||
assert first is second
|
||||
assert isinstance(first, PluginLlm)
|
||||
assert first._plugin_id == "test-plugin" # type: ignore[attr-defined]
|
||||
|
||||
def test_ctx_llm_uses_manifest_key_for_policy(self):
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(
|
||||
name="bare-name", source="test", key="image_gen/openai"
|
||||
)
|
||||
manager = PluginManager()
|
||||
ctx = PluginContext(manifest, manager)
|
||||
assert ctx.llm._plugin_id == "image_gen/openai" # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Attribution (result.provider / result.model / audit log)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAttribution:
|
||||
"""Verifies that the result object and the audit log carry the real
|
||||
provider/model that ``call_llm`` ended up using, NOT the placeholder
|
||||
fallbacks ('auto', 'default') from earlier drafts."""
|
||||
|
||||
def test_explicit_overrides_recorded_when_no_response_model(self):
|
||||
from agent.plugin_llm import _resolve_attribution
|
||||
|
||||
# Response with no .model attribute — overrides win.
|
||||
response = SimpleNamespace(choices=[], usage=None)
|
||||
provider, model = _resolve_attribution(
|
||||
provider_override="openrouter",
|
||||
model_override="anthropic/claude-3-5-sonnet",
|
||||
response=response,
|
||||
)
|
||||
assert provider == "openrouter"
|
||||
assert model == "anthropic/claude-3-5-sonnet"
|
||||
|
||||
def test_response_model_wins_over_model_override(self):
|
||||
"""Providers often canonicalise the model name (e.g. ``gpt-4o``
|
||||
→ ``gpt-4o-2024-08-06``). Whatever they actually returned wins
|
||||
for the recorded model so the audit log reflects reality."""
|
||||
from agent.plugin_llm import _resolve_attribution
|
||||
|
||||
response = SimpleNamespace(model="gpt-4o-2024-08-06", choices=[])
|
||||
provider, model = _resolve_attribution(
|
||||
provider_override="openrouter",
|
||||
model_override="openai/gpt-4o",
|
||||
response=response,
|
||||
)
|
||||
assert model == "gpt-4o-2024-08-06"
|
||||
# Provider override is unaffected by response.model.
|
||||
assert provider == "openrouter"
|
||||
|
||||
def test_falls_back_to_main_provider_and_model_when_no_overrides(self, monkeypatch):
|
||||
"""When the plugin doesn't override anything, attribution
|
||||
reflects the user's active main provider/model rather than
|
||||
misleading placeholders."""
|
||||
from agent import plugin_llm
|
||||
import agent.auxiliary_client as ac
|
||||
|
||||
monkeypatch.setattr(ac, "_read_main_provider", lambda: "openrouter")
|
||||
monkeypatch.setattr(ac, "_read_main_model", lambda: "anthropic/claude-3-5-sonnet")
|
||||
|
||||
response = SimpleNamespace(choices=[]) # no .model attribute
|
||||
provider, model = plugin_llm._resolve_attribution(
|
||||
provider_override=None,
|
||||
model_override=None,
|
||||
response=response,
|
||||
)
|
||||
assert provider == "openrouter"
|
||||
assert model == "anthropic/claude-3-5-sonnet"
|
||||
|
||||
def test_response_model_used_even_when_no_overrides(self, monkeypatch):
|
||||
"""The provider's canonical model name should still flow through
|
||||
when no overrides are set."""
|
||||
from agent import plugin_llm
|
||||
import agent.auxiliary_client as ac
|
||||
|
||||
monkeypatch.setattr(ac, "_read_main_provider", lambda: "openrouter")
|
||||
monkeypatch.setattr(ac, "_read_main_model", lambda: "openai/gpt-4o")
|
||||
|
||||
response = SimpleNamespace(model="openai/gpt-4o-2024-08-06", choices=[])
|
||||
provider, model = plugin_llm._resolve_attribution(
|
||||
provider_override=None,
|
||||
model_override=None,
|
||||
response=response,
|
||||
)
|
||||
assert provider == "openrouter"
|
||||
assert model == "openai/gpt-4o-2024-08-06"
|
||||
|
||||
def test_placeholder_fallback_only_when_everything_is_empty(self, monkeypatch):
|
||||
"""If main_provider/main_model are unset AND there's no override
|
||||
AND the response has no .model, fall through to the safety
|
||||
placeholders so the result object never has empty strings."""
|
||||
from agent import plugin_llm
|
||||
import agent.auxiliary_client as ac
|
||||
|
||||
monkeypatch.setattr(ac, "_read_main_provider", lambda: "")
|
||||
monkeypatch.setattr(ac, "_read_main_model", lambda: "")
|
||||
|
||||
response = SimpleNamespace(choices=[])
|
||||
provider, model = plugin_llm._resolve_attribution(
|
||||
provider_override=None,
|
||||
model_override=None,
|
||||
response=response,
|
||||
)
|
||||
assert provider == "auto"
|
||||
assert model == "default"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hook-mode integration (ctx.llm called from a post_tool_call callback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHookMode:
|
||||
"""The docs page promises ``ctx.llm`` works from inside lifecycle
|
||||
hooks. This exercises that path: register a ``post_tool_call``
|
||||
callback that calls ``ctx.llm.complete``, fire the hook through
|
||||
the real ``invoke_hook`` machinery, and check the call landed."""
|
||||
|
||||
def test_complete_works_from_post_tool_call_hook(self):
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(name="hook-plugin", source="test", key="hook-plugin")
|
||||
manager = PluginManager()
|
||||
ctx = PluginContext(manifest, manager)
|
||||
|
||||
# Replace ctx.llm with a stub that records what the hook called.
|
||||
captured: list = []
|
||||
|
||||
def fake_caller(**kwargs):
|
||||
captured.append(kwargs)
|
||||
return "openrouter", "openai/gpt-4o", _fake_response("rewrote it")
|
||||
|
||||
ctx._llm = make_plugin_llm_for_test( # type: ignore[attr-defined]
|
||||
plugin_id="hook-plugin",
|
||||
policy=_TrustPolicy(plugin_id="hook-plugin"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
|
||||
# Plugin registers a hook that runs ctx.llm.complete on every tool call.
|
||||
def rewrite_error_hook(*, tool_name, args, result, **_):
|
||||
if "Traceback" in (result or ""):
|
||||
rewritten = ctx.llm.complete(
|
||||
messages=[
|
||||
{"role": "system", "content": "Rewrite errors plainly."},
|
||||
{"role": "user", "content": result},
|
||||
],
|
||||
max_tokens=64,
|
||||
purpose="hook-plugin.rewrite",
|
||||
)
|
||||
# Real hook would return the rewritten text via
|
||||
# transform_tool_result; here we just capture for the assert.
|
||||
captured.append({"hook_returned": rewritten.text})
|
||||
|
||||
ctx.register_hook("post_tool_call", rewrite_error_hook)
|
||||
|
||||
# Fire the hook the same way the agent core does it.
|
||||
manager.invoke_hook(
|
||||
"post_tool_call",
|
||||
tool_name="terminal",
|
||||
args={"command": "boom"},
|
||||
result="Traceback (most recent call last):\n RuntimeError",
|
||||
)
|
||||
|
||||
# Verify ctx.llm.complete fired through the hook.
|
||||
assert len(captured) == 2 # one llm call + one hook return record
|
||||
llm_call = captured[0]
|
||||
assert "messages" in llm_call
|
||||
assert any("rewrite" in m.get("content", "").lower()
|
||||
for m in llm_call["messages"] if isinstance(m, dict))
|
||||
hook_record = captured[1]
|
||||
assert hook_record["hook_returned"] == "rewrote it"
|
||||
|
||||
def test_complete_works_from_post_tool_call_hook_when_async_caller_set(self):
|
||||
"""Hooks fired synchronously should still work with sync
|
||||
ctx.llm.complete even if other callsites use async."""
|
||||
from hermes_cli.plugins import PluginContext, PluginManifest, PluginManager
|
||||
|
||||
manifest = PluginManifest(name="hook-async", source="test", key="hook-async")
|
||||
manager = PluginManager()
|
||||
ctx = PluginContext(manifest, manager)
|
||||
|
||||
def fake_caller(**_):
|
||||
return "openrouter", "model-x", _fake_response("ok")
|
||||
|
||||
ctx._llm = make_plugin_llm_for_test( # type: ignore[attr-defined]
|
||||
plugin_id="hook-async",
|
||||
policy=_TrustPolicy(plugin_id="hook-async"),
|
||||
sync_caller=fake_caller,
|
||||
)
|
||||
|
||||
called: list = []
|
||||
|
||||
def hook(**kwargs):
|
||||
r = ctx.llm.complete(messages=[{"role": "user", "content": "x"}])
|
||||
called.append(r.text)
|
||||
|
||||
ctx.register_hook("post_tool_call", hook)
|
||||
manager.invoke_hook("post_tool_call", tool_name="x", args={}, result="y")
|
||||
assert called == ["ok"]
|
||||
@@ -6,6 +6,8 @@ import pytest
|
||||
from agent.prompt_caching import (
|
||||
_apply_cache_marker,
|
||||
apply_anthropic_cache_control,
|
||||
apply_anthropic_cache_control_long_lived,
|
||||
mark_tools_for_long_lived_cache,
|
||||
)
|
||||
|
||||
|
||||
@@ -141,3 +143,132 @@ class TestApplyAnthropicCacheControl:
|
||||
elif "cache_control" in msg:
|
||||
count += 1
|
||||
assert count <= 4
|
||||
|
||||
|
||||
class TestMarkToolsForLongLivedCache:
|
||||
def test_returns_unchanged_for_empty_tools(self):
|
||||
assert mark_tools_for_long_lived_cache(None) is None
|
||||
assert mark_tools_for_long_lived_cache([]) == []
|
||||
|
||||
def test_marks_only_last_tool(self):
|
||||
tools = [
|
||||
{"type": "function", "function": {"name": "a"}},
|
||||
{"type": "function", "function": {"name": "b"}},
|
||||
{"type": "function", "function": {"name": "c"}},
|
||||
]
|
||||
out = mark_tools_for_long_lived_cache(tools)
|
||||
assert "cache_control" not in out[0]
|
||||
assert "cache_control" not in out[1]
|
||||
assert out[2]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
|
||||
def test_does_not_mutate_input(self):
|
||||
tools = [{"type": "function", "function": {"name": "a"}}]
|
||||
mark_tools_for_long_lived_cache(tools)
|
||||
assert "cache_control" not in tools[0]
|
||||
|
||||
def test_5m_ttl_drops_ttl_field(self):
|
||||
tools = [{"type": "function", "function": {"name": "a"}}]
|
||||
out = mark_tools_for_long_lived_cache(tools, long_lived_ttl="5m")
|
||||
assert out[0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
class TestApplyAnthropicCacheControlLongLived:
|
||||
def test_empty_messages(self):
|
||||
assert apply_anthropic_cache_control_long_lived([]) == []
|
||||
|
||||
def test_marks_first_block_of_split_system(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [
|
||||
{"type": "text", "text": "STABLE"},
|
||||
{"type": "text", "text": "CONTEXT"},
|
||||
{"type": "text", "text": "VOLATILE"},
|
||||
]},
|
||||
{"role": "user", "content": "msg1"},
|
||||
{"role": "assistant", "content": "msg2"},
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(msgs)
|
||||
sys_blocks = out[0]["content"]
|
||||
assert sys_blocks[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
assert "cache_control" not in sys_blocks[1]
|
||||
assert "cache_control" not in sys_blocks[2]
|
||||
|
||||
def test_rolling_marker_on_last_2_messages(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "S"}]},
|
||||
{"role": "user", "content": "u1"},
|
||||
{"role": "assistant", "content": "a1"},
|
||||
{"role": "user", "content": "u2"},
|
||||
{"role": "assistant", "content": "a2"},
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(msgs)
|
||||
|
||||
def has_marker(m):
|
||||
c = m.get("content")
|
||||
if isinstance(c, list) and c and isinstance(c[-1], dict):
|
||||
return "cache_control" in c[-1]
|
||||
return "cache_control" in m
|
||||
|
||||
# u1 and a1 (older messages) should NOT be marked
|
||||
assert not has_marker(out[1])
|
||||
assert not has_marker(out[2])
|
||||
# u2 and a2 (last 2) SHOULD be marked
|
||||
assert has_marker(out[3])
|
||||
assert has_marker(out[4])
|
||||
|
||||
def test_rolling_marker_uses_5m_ttl(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "S"}]},
|
||||
{"role": "user", "content": "u1"},
|
||||
{"role": "assistant", "content": "a1"},
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(
|
||||
msgs, long_lived_ttl="1h", rolling_ttl="5m",
|
||||
)
|
||||
# Last user message: cache_control on the wrapped text part should be 5m
|
||||
last = out[-1]
|
||||
c = last["content"]
|
||||
assert isinstance(c, list)
|
||||
assert c[-1]["cache_control"] == {"type": "ephemeral"} # 5m has no ttl key
|
||||
|
||||
def test_string_system_falls_back_to_envelope_marker(self):
|
||||
"""When the caller didn't split the system message, we still place a marker."""
|
||||
msgs = [
|
||||
{"role": "system", "content": "Single string system"},
|
||||
{"role": "user", "content": "u1"},
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(msgs)
|
||||
sys_content = out[0]["content"]
|
||||
# Wrapped into a list and the (now sole) block gets the 1h marker
|
||||
assert isinstance(sys_content, list)
|
||||
assert sys_content[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
|
||||
def test_does_not_mutate_input(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "S"}]},
|
||||
{"role": "user", "content": "u1"},
|
||||
]
|
||||
before = copy.deepcopy(msgs)
|
||||
apply_anthropic_cache_control_long_lived(msgs)
|
||||
assert msgs == before
|
||||
|
||||
def test_max_4_breakpoints_with_split_system(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "S"}, {"type": "text", "text": "V"}]},
|
||||
] + [
|
||||
{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg{i}"}
|
||||
for i in range(10)
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(msgs)
|
||||
count = 0
|
||||
for m in out:
|
||||
c = m.get("content")
|
||||
if isinstance(c, list):
|
||||
for item in c:
|
||||
if isinstance(item, dict) and "cache_control" in item:
|
||||
count += 1
|
||||
elif "cache_control" in m:
|
||||
count += 1
|
||||
# 1 system block + last 2 messages = 3 breakpoints from this function.
|
||||
# tools[-1] is marked separately (not via this function), so a 4th
|
||||
# breakpoint can be added at API-call time.
|
||||
assert count == 3
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Live E2E: long-lived prefix caching on Claude via OpenRouter.
|
||||
|
||||
Run only when LIVE_OR_KEY env var is set. Skipped under the normal hermetic
|
||||
test suite (which unsets credentials).
|
||||
"""
|
||||
import os, sys, tempfile, time, shutil, pytest
|
||||
|
||||
|
||||
# Probe for the key BEFORE conftest unsets it
|
||||
_LIVE_KEY = os.environ.get("OPENROUTER_API_KEY") or os.environ.get("LIVE_OR_KEY")
|
||||
if not _LIVE_KEY:
|
||||
# Try to read directly from .env
|
||||
env_path = os.path.expanduser("~/.hermes/.env")
|
||||
if os.path.exists(env_path):
|
||||
with open(env_path) as f:
|
||||
for line in f:
|
||||
if line.startswith("OPENROUTER_API_KEY="):
|
||||
_LIVE_KEY = line.strip().split("=", 1)[1].strip().strip('"').strip("'")
|
||||
break
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _LIVE_KEY,
|
||||
reason="set OPENROUTER_API_KEY (or LIVE_OR_KEY) to run live cache test",
|
||||
)
|
||||
|
||||
|
||||
def test_long_lived_prefix_cache_e2e_openrouter(tmp_path, monkeypatch):
|
||||
"""Two AIAgent runs in fresh sessions: call 1 writes cache, call 2 reads it."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
# The hermetic conftest unsets OPENROUTER_API_KEY — restore for this test
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", _LIVE_KEY)
|
||||
|
||||
# Minimal config — but with enough toolset/guidance to exceed Anthropic's
|
||||
# ~1024-token minimum-cacheable-prefix threshold. Anthropic silently
|
||||
# ignores cache_control markers on small blocks.
|
||||
import yaml
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({
|
||||
"model": {"provider": "openrouter", "default": "anthropic/claude-haiku-4.5"},
|
||||
"prompt_caching": {"long_lived_prefix": True, "long_lived_ttl": "1h", "cache_ttl": "5m"},
|
||||
"agent": {"tool_use_enforcement": True}, # adds substantial guidance text
|
||||
"memory": {"provider": ""},
|
||||
"compression": {"enabled": False},
|
||||
}))
|
||||
|
||||
from run_agent import AIAgent
|
||||
|
||||
def make_agent():
|
||||
return AIAgent(
|
||||
api_key=_LIVE_KEY,
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
provider="openrouter",
|
||||
model="anthropic/claude-haiku-4.5",
|
||||
api_mode="chat_completions",
|
||||
# Use the default toolset roster — the tools array (~13k tokens
|
||||
# for ~35 tools) is what carries the bulk of the cross-session
|
||||
# cache value. With a tiny toolset the cached prefix can fall
|
||||
# below Anthropic Haiku's 2048-token minimum cacheable size and
|
||||
# the marker is silently ignored.
|
||||
enabled_toolsets=None,
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
save_trajectories=False,
|
||||
)
|
||||
|
||||
a1 = make_agent()
|
||||
assert a1._use_prompt_caching is True, "policy should enable caching for Claude on OR"
|
||||
assert a1._use_long_lived_prefix_cache is True, "long-lived path should activate"
|
||||
parts = a1._build_system_prompt_parts()
|
||||
print(f"\nstable={len(parts['stable']):,} ctx={len(parts['context']):,} volatile={len(parts['volatile']):,} chars")
|
||||
print(f"tool count: {len(a1.tools or [])}")
|
||||
|
||||
# Use distinct user messages each call so OpenRouter's response cache
|
||||
# doesn't short-circuit the upstream Anthropic call (we need real
|
||||
# Anthropic billing visibility to verify cache_creation/cache_read).
|
||||
USER_1 = "Reply with the single word ALPHA."
|
||||
USER_2 = "Reply with the single word BRAVO."
|
||||
|
||||
print("\n--- Call 1 (cold) ---")
|
||||
r1 = a1.run_conversation(USER_1, conversation_history=[])
|
||||
print(f"final_response[:80]: {(r1.get('final_response') or '')[:80]!r}")
|
||||
cr1 = a1.session_cache_read_tokens
|
||||
cw1 = a1.session_cache_write_tokens
|
||||
print(f"call1: cache_read={cr1} cache_write={cw1}")
|
||||
|
||||
# Wait so cache settles, then fresh agent (NEW SESSION) for cross-session read
|
||||
time.sleep(2)
|
||||
a2 = make_agent()
|
||||
assert a2.session_id != a1.session_id, "second agent must have a new session"
|
||||
|
||||
print("\n--- Call 2 (warm, NEW session, different user msg) ---")
|
||||
r2 = a2.run_conversation(USER_2, conversation_history=[])
|
||||
print(f"final_response[:80]: {(r2.get('final_response') or '')[:80]!r}")
|
||||
cr2 = a2.session_cache_read_tokens
|
||||
cw2 = a2.session_cache_write_tokens
|
||||
print(f"call2: cache_read={cr2} cache_write={cw2}")
|
||||
|
||||
print(f"\n=== VERDICT ===")
|
||||
print(f" call1 wrote {cw1:,} cache tokens, read {cr1:,}")
|
||||
print(f" call2 wrote {cw2:,} cache tokens, read {cr2:,}")
|
||||
if cw1:
|
||||
print(f" cross-session read fraction: cr2/cw1 = {cr2/cw1:.2%}")
|
||||
|
||||
# Assertions
|
||||
assert cw1 > 0, f"call 1 must write cache (got {cw1}); long-lived layout not reaching wire"
|
||||
assert cr2 > 0, (
|
||||
f"call 2 must read cache cross-session (got {cr2}); "
|
||||
f"stable prefix is not byte-stable across sessions"
|
||||
)
|
||||
assert cr2 >= 1000, f"cache_read on call 2 ({cr2}) too small to indicate real reuse"
|
||||
@@ -180,6 +180,119 @@ class TestCodexBuildKwargs:
|
||||
# "minimal" should be clamped to "low" for xAI as well
|
||||
assert kw.get("reasoning", {}).get("effort") == "low"
|
||||
|
||||
# --- Grok reasoning-effort capability allowlist ---
|
||||
# api.x.ai 400s with "Model X does not support parameter reasoningEffort"
|
||||
# on grok-4 / grok-4-fast / grok-3 / grok-code-fast / grok-4.20-0309-*.
|
||||
# Those models reason natively but don't expose the dial. The transport
|
||||
# must omit the `reasoning` key for them while keeping the encrypted
|
||||
# reasoning content include so we can capture native reasoning tokens.
|
||||
|
||||
def test_xai_grok_4_omits_reasoning_effort(self, transport):
|
||||
"""grok-4 / grok-4-0709 reject reasoning.effort with HTTP 400."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
for model in ("grok-4", "grok-4-0709"):
|
||||
kw = transport.build_kwargs(
|
||||
model=model, messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw, (
|
||||
f"{model} must not receive a reasoning key (xAI rejects it)"
|
||||
)
|
||||
# Still capture native reasoning tokens
|
||||
assert "reasoning.encrypted_content" in kw.get("include", [])
|
||||
|
||||
def test_xai_grok_4_fast_omits_reasoning_effort(self, transport):
|
||||
"""grok-4-fast and grok-4-1-fast variants reject reasoning.effort."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
for model in (
|
||||
"grok-4-fast-reasoning",
|
||||
"grok-4-fast-non-reasoning",
|
||||
"grok-4-1-fast-reasoning",
|
||||
"grok-4-1-fast-non-reasoning",
|
||||
):
|
||||
kw = transport.build_kwargs(
|
||||
model=model, messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "low"},
|
||||
)
|
||||
assert "reasoning" not in kw, (
|
||||
f"{model} must not receive a reasoning key (xAI rejects it)"
|
||||
)
|
||||
|
||||
def test_xai_grok_3_non_mini_omits_reasoning_effort(self, transport):
|
||||
"""Plain grok-3 rejects reasoning.effort — only grok-3-mini accepts it."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-3", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "medium"},
|
||||
)
|
||||
assert "reasoning" not in kw
|
||||
|
||||
def test_xai_grok_3_mini_keeps_reasoning_effort(self, transport):
|
||||
"""grok-3-mini and -fast variants do accept the effort dial."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
for model in ("grok-3-mini", "grok-3-mini-fast"):
|
||||
kw = transport.build_kwargs(
|
||||
model=model, messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert kw.get("reasoning") == {"effort": "high"}
|
||||
|
||||
def test_xai_grok_4_20_0309_variants_omit_reasoning_effort(self, transport):
|
||||
"""grok-4.20-0309-(non-)reasoning reject the effort dial.
|
||||
|
||||
Counterintuitively, only grok-4.20-multi-agent-0309 accepts it.
|
||||
"""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
for model in ("grok-4.20-0309-reasoning", "grok-4.20-0309-non-reasoning"):
|
||||
kw = transport.build_kwargs(
|
||||
model=model, messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw, f"{model} must not receive reasoning"
|
||||
|
||||
def test_xai_grok_4_20_multi_agent_keeps_reasoning_effort(self, transport):
|
||||
"""grok-4.20-multi-agent-0309 is the one grok-4.20 variant that accepts effort."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-4.20-multi-agent-0309", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "low"},
|
||||
)
|
||||
assert kw.get("reasoning") == {"effort": "low"}
|
||||
|
||||
def test_xai_grok_code_fast_omits_reasoning_effort(self, transport):
|
||||
"""grok-code-fast-1 rejects reasoning.effort."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="grok-code-fast-1", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw
|
||||
|
||||
def test_xai_aggregator_prefix_stripped(self, transport):
|
||||
"""`x-ai/grok-3-mini` (OpenRouter-style slug) still resolves correctly."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
# Effort-capable
|
||||
kw = transport.build_kwargs(
|
||||
model="x-ai/grok-3-mini", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert kw.get("reasoning") == {"effort": "high"}
|
||||
# Effort-incapable
|
||||
kw = transport.build_kwargs(
|
||||
model="x-ai/grok-4-0709", messages=messages, tools=[],
|
||||
is_xai_responses=True,
|
||||
reasoning_config={"effort": "high"},
|
||||
)
|
||||
assert "reasoning" not in kw
|
||||
|
||||
|
||||
class TestCodexValidateResponse:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user