Merge remote-tracking branch 'origin/main' into bb/gui
# Conflicts: # tui_gateway/server.py
This commit is contained in:
Brooklyn Nicholson
@@ -0,0 +1,61 @@
|
||||
"""Tests for agent.portal_tags — Nous Portal request tag contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def test_hermes_client_tag_includes_current_version():
|
||||
"""The client tag must reflect hermes_cli.__version__ verbatim."""
|
||||
from hermes_cli import __version__
|
||||
from agent.portal_tags import hermes_client_tag
|
||||
|
||||
assert hermes_client_tag() == f"client=hermes-client-v{__version__}"
|
||||
|
||||
|
||||
def test_hermes_client_tag_format():
|
||||
"""The client tag has the exact shape Nous Portal expects."""
|
||||
from agent.portal_tags import hermes_client_tag
|
||||
|
||||
tag = hermes_client_tag()
|
||||
assert tag.startswith("client=hermes-client-v")
|
||||
# No spaces, no commas — single tag value
|
||||
assert " " not in tag
|
||||
assert "," not in tag
|
||||
|
||||
|
||||
def test_nous_portal_tags_contains_product_and_client():
|
||||
"""Every Nous Portal request gets BOTH the product tag and the version tag."""
|
||||
from agent.portal_tags import hermes_client_tag, nous_portal_tags
|
||||
|
||||
tags = nous_portal_tags()
|
||||
assert "product=hermes-agent" in tags
|
||||
assert hermes_client_tag() in tags
|
||||
assert len(tags) == 2
|
||||
|
||||
|
||||
def test_nous_portal_tags_returns_fresh_list():
|
||||
"""Callers mutate the returned list; we must not share state across calls."""
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
|
||||
a = nous_portal_tags()
|
||||
a.append("client=test-mutation")
|
||||
b = nous_portal_tags()
|
||||
assert "client=test-mutation" not in b
|
||||
|
||||
|
||||
def test_auxiliary_client_nous_extra_body_uses_helper():
|
||||
"""auxiliary_client.NOUS_EXTRA_BODY must match the canonical helper output."""
|
||||
from agent.auxiliary_client import NOUS_EXTRA_BODY
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
|
||||
assert NOUS_EXTRA_BODY == {"tags": nous_portal_tags()}
|
||||
|
||||
|
||||
def test_nous_provider_profile_uses_helper():
|
||||
"""The Nous provider profile (main agent loop) must use the canonical tags."""
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
from providers import get_provider_profile
|
||||
|
||||
profile = get_provider_profile("nous")
|
||||
assert profile is not None
|
||||
body = profile.build_extra_body()
|
||||
assert body["tags"] == nous_portal_tags()
|
||||
@@ -6,8 +6,6 @@ import pytest
|
||||
from agent.prompt_caching import (
|
||||
_apply_cache_marker,
|
||||
apply_anthropic_cache_control,
|
||||
apply_anthropic_cache_control_long_lived,
|
||||
mark_tools_for_long_lived_cache,
|
||||
)
|
||||
|
||||
|
||||
@@ -143,132 +141,3 @@ class TestApplyAnthropicCacheControl:
|
||||
elif "cache_control" in msg:
|
||||
count += 1
|
||||
assert count <= 4
|
||||
|
||||
|
||||
class TestMarkToolsForLongLivedCache:
|
||||
def test_returns_unchanged_for_empty_tools(self):
|
||||
assert mark_tools_for_long_lived_cache(None) is None
|
||||
assert mark_tools_for_long_lived_cache([]) == []
|
||||
|
||||
def test_marks_only_last_tool(self):
|
||||
tools = [
|
||||
{"type": "function", "function": {"name": "a"}},
|
||||
{"type": "function", "function": {"name": "b"}},
|
||||
{"type": "function", "function": {"name": "c"}},
|
||||
]
|
||||
out = mark_tools_for_long_lived_cache(tools)
|
||||
assert "cache_control" not in out[0]
|
||||
assert "cache_control" not in out[1]
|
||||
assert out[2]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
|
||||
def test_does_not_mutate_input(self):
|
||||
tools = [{"type": "function", "function": {"name": "a"}}]
|
||||
mark_tools_for_long_lived_cache(tools)
|
||||
assert "cache_control" not in tools[0]
|
||||
|
||||
def test_5m_ttl_drops_ttl_field(self):
|
||||
tools = [{"type": "function", "function": {"name": "a"}}]
|
||||
out = mark_tools_for_long_lived_cache(tools, long_lived_ttl="5m")
|
||||
assert out[0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
|
||||
class TestApplyAnthropicCacheControlLongLived:
|
||||
def test_empty_messages(self):
|
||||
assert apply_anthropic_cache_control_long_lived([]) == []
|
||||
|
||||
def test_marks_first_block_of_split_system(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [
|
||||
{"type": "text", "text": "STABLE"},
|
||||
{"type": "text", "text": "CONTEXT"},
|
||||
{"type": "text", "text": "VOLATILE"},
|
||||
]},
|
||||
{"role": "user", "content": "msg1"},
|
||||
{"role": "assistant", "content": "msg2"},
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(msgs)
|
||||
sys_blocks = out[0]["content"]
|
||||
assert sys_blocks[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
assert "cache_control" not in sys_blocks[1]
|
||||
assert "cache_control" not in sys_blocks[2]
|
||||
|
||||
def test_rolling_marker_on_last_2_messages(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "S"}]},
|
||||
{"role": "user", "content": "u1"},
|
||||
{"role": "assistant", "content": "a1"},
|
||||
{"role": "user", "content": "u2"},
|
||||
{"role": "assistant", "content": "a2"},
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(msgs)
|
||||
|
||||
def has_marker(m):
|
||||
c = m.get("content")
|
||||
if isinstance(c, list) and c and isinstance(c[-1], dict):
|
||||
return "cache_control" in c[-1]
|
||||
return "cache_control" in m
|
||||
|
||||
# u1 and a1 (older messages) should NOT be marked
|
||||
assert not has_marker(out[1])
|
||||
assert not has_marker(out[2])
|
||||
# u2 and a2 (last 2) SHOULD be marked
|
||||
assert has_marker(out[3])
|
||||
assert has_marker(out[4])
|
||||
|
||||
def test_rolling_marker_uses_5m_ttl(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "S"}]},
|
||||
{"role": "user", "content": "u1"},
|
||||
{"role": "assistant", "content": "a1"},
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(
|
||||
msgs, long_lived_ttl="1h", rolling_ttl="5m",
|
||||
)
|
||||
# Last user message: cache_control on the wrapped text part should be 5m
|
||||
last = out[-1]
|
||||
c = last["content"]
|
||||
assert isinstance(c, list)
|
||||
assert c[-1]["cache_control"] == {"type": "ephemeral"} # 5m has no ttl key
|
||||
|
||||
def test_string_system_falls_back_to_envelope_marker(self):
|
||||
"""When the caller didn't split the system message, we still place a marker."""
|
||||
msgs = [
|
||||
{"role": "system", "content": "Single string system"},
|
||||
{"role": "user", "content": "u1"},
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(msgs)
|
||||
sys_content = out[0]["content"]
|
||||
# Wrapped into a list and the (now sole) block gets the 1h marker
|
||||
assert isinstance(sys_content, list)
|
||||
assert sys_content[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
|
||||
|
||||
def test_does_not_mutate_input(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "S"}]},
|
||||
{"role": "user", "content": "u1"},
|
||||
]
|
||||
before = copy.deepcopy(msgs)
|
||||
apply_anthropic_cache_control_long_lived(msgs)
|
||||
assert msgs == before
|
||||
|
||||
def test_max_4_breakpoints_with_split_system(self):
|
||||
msgs = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "S"}, {"type": "text", "text": "V"}]},
|
||||
] + [
|
||||
{"role": "user" if i % 2 == 0 else "assistant", "content": f"msg{i}"}
|
||||
for i in range(10)
|
||||
]
|
||||
out = apply_anthropic_cache_control_long_lived(msgs)
|
||||
count = 0
|
||||
for m in out:
|
||||
c = m.get("content")
|
||||
if isinstance(c, list):
|
||||
for item in c:
|
||||
if isinstance(item, dict) and "cache_control" in item:
|
||||
count += 1
|
||||
elif "cache_control" in m:
|
||||
count += 1
|
||||
# 1 system block + last 2 messages = 3 breakpoints from this function.
|
||||
# tools[-1] is marked separately (not via this function), so a 4th
|
||||
# breakpoint can be added at API-call time.
|
||||
assert count == 3
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
"""Live E2E: long-lived prefix caching on Claude via OpenRouter.
|
||||
|
||||
Run only when LIVE_OR_KEY env var is set. Skipped under the normal hermetic
|
||||
test suite (which unsets credentials).
|
||||
"""
|
||||
import os, sys, tempfile, time, shutil, pytest
|
||||
|
||||
|
||||
# Probe for the key BEFORE conftest unsets it
|
||||
_LIVE_KEY = os.environ.get("OPENROUTER_API_KEY") or os.environ.get("LIVE_OR_KEY")
|
||||
if not _LIVE_KEY:
|
||||
# Try to read directly from .env
|
||||
env_path = os.path.expanduser("~/.hermes/.env")
|
||||
if os.path.exists(env_path):
|
||||
with open(env_path) as f:
|
||||
for line in f:
|
||||
if line.startswith("OPENROUTER_API_KEY="):
|
||||
_LIVE_KEY = line.strip().split("=", 1)[1].strip().strip('"').strip("'")
|
||||
break
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _LIVE_KEY,
|
||||
reason="set OPENROUTER_API_KEY (or LIVE_OR_KEY) to run live cache test",
|
||||
)
|
||||
|
||||
|
||||
def test_long_lived_prefix_cache_e2e_openrouter(tmp_path, monkeypatch):
|
||||
"""Two AIAgent runs in fresh sessions: call 1 writes cache, call 2 reads it."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
# The hermetic conftest unsets OPENROUTER_API_KEY — restore for this test
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", _LIVE_KEY)
|
||||
|
||||
# Minimal config — but with enough toolset/guidance to exceed Anthropic's
|
||||
# ~1024-token minimum-cacheable-prefix threshold. Anthropic silently
|
||||
# ignores cache_control markers on small blocks.
|
||||
import yaml
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({
|
||||
"model": {"provider": "openrouter", "default": "anthropic/claude-haiku-4.5"},
|
||||
"prompt_caching": {"long_lived_prefix": True, "long_lived_ttl": "1h", "cache_ttl": "5m"},
|
||||
"agent": {"tool_use_enforcement": True}, # adds substantial guidance text
|
||||
"memory": {"provider": ""},
|
||||
"compression": {"enabled": False},
|
||||
}))
|
||||
|
||||
from run_agent import AIAgent
|
||||
|
||||
def make_agent():
|
||||
return AIAgent(
|
||||
api_key=_LIVE_KEY,
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
provider="openrouter",
|
||||
model="anthropic/claude-haiku-4.5",
|
||||
api_mode="chat_completions",
|
||||
# Use the default toolset roster — the tools array (~13k tokens
|
||||
# for ~35 tools) is what carries the bulk of the cross-session
|
||||
# cache value. With a tiny toolset the cached prefix can fall
|
||||
# below Anthropic Haiku's 2048-token minimum cacheable size and
|
||||
# the marker is silently ignored.
|
||||
enabled_toolsets=None,
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
save_trajectories=False,
|
||||
)
|
||||
|
||||
a1 = make_agent()
|
||||
assert a1._use_prompt_caching is True, "policy should enable caching for Claude on OR"
|
||||
assert a1._use_long_lived_prefix_cache is True, "long-lived path should activate"
|
||||
parts = a1._build_system_prompt_parts()
|
||||
print(f"\nstable={len(parts['stable']):,} ctx={len(parts['context']):,} volatile={len(parts['volatile']):,} chars")
|
||||
print(f"tool count: {len(a1.tools or [])}")
|
||||
|
||||
# Use distinct user messages each call so OpenRouter's response cache
|
||||
# doesn't short-circuit the upstream Anthropic call (we need real
|
||||
# Anthropic billing visibility to verify cache_creation/cache_read).
|
||||
USER_1 = "Reply with the single word ALPHA."
|
||||
USER_2 = "Reply with the single word BRAVO."
|
||||
|
||||
print("\n--- Call 1 (cold) ---")
|
||||
r1 = a1.run_conversation(USER_1, conversation_history=[])
|
||||
print(f"final_response[:80]: {(r1.get('final_response') or '')[:80]!r}")
|
||||
cr1 = a1.session_cache_read_tokens
|
||||
cw1 = a1.session_cache_write_tokens
|
||||
print(f"call1: cache_read={cr1} cache_write={cw1}")
|
||||
|
||||
# Wait so cache settles, then fresh agent (NEW SESSION) for cross-session read
|
||||
time.sleep(2)
|
||||
a2 = make_agent()
|
||||
assert a2.session_id != a1.session_id, "second agent must have a new session"
|
||||
|
||||
print("\n--- Call 2 (warm, NEW session, different user msg) ---")
|
||||
r2 = a2.run_conversation(USER_2, conversation_history=[])
|
||||
print(f"final_response[:80]: {(r2.get('final_response') or '')[:80]!r}")
|
||||
cr2 = a2.session_cache_read_tokens
|
||||
cw2 = a2.session_cache_write_tokens
|
||||
print(f"call2: cache_read={cr2} cache_write={cw2}")
|
||||
|
||||
print(f"\n=== VERDICT ===")
|
||||
print(f" call1 wrote {cw1:,} cache tokens, read {cr1:,}")
|
||||
print(f" call2 wrote {cw2:,} cache tokens, read {cr2:,}")
|
||||
if cw1:
|
||||
print(f" cross-session read fraction: cr2/cw1 = {cr2/cw1:.2%}")
|
||||
|
||||
# Assertions
|
||||
assert cw1 > 0, f"call 1 must write cache (got {cw1}); long-lived layout not reaching wire"
|
||||
assert cr2 > 0, (
|
||||
f"call 2 must read cache cross-session (got {cr2}); "
|
||||
f"stable prefix is not byte-stable across sessions"
|
||||
)
|
||||
assert cr2 >= 1000, f"cache_read on call 2 ({cr2}) too small to indicate real reuse"
|
||||
@@ -147,11 +147,12 @@ class TestChatCompletionsBuildKwargs:
|
||||
]
|
||||
|
||||
def test_nous_tags(self, transport):
|
||||
from agent.portal_tags import nous_portal_tags
|
||||
from providers import get_provider_profile
|
||||
profile = get_provider_profile("nous")
|
||||
msgs = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(model="gpt-4o", messages=msgs, provider_profile=profile)
|
||||
assert kw["extra_body"]["tags"] == ["product=hermes-agent"]
|
||||
assert kw["extra_body"]["tags"] == nous_portal_tags()
|
||||
|
||||
def test_reasoning_default(self, transport):
|
||||
msgs = [{"role": "user", "content": "Hi"}]
|
||||
|
||||
Reference in New Issue
Block a user