fix(background_review): propagate parent toolset config to keep tools[] cache-stable
## Summary The background skill/memory-review fork constructed a child `AIAgent` without propagating `enabled_toolsets` / `disabled_toolsets` from the parent. When the parent narrowed its toolset (via `hermes tools disable` or `config.yaml`), the fork's default `enabled_toolsets=None` expanded to "all registered tools" — and the fork's outbound request body sent a wider `tools[]` array than the parent's main-turn request. Anthropic's prompt-cache key includes the `tools[]` array byte-for-byte, so this divergence forked the cache lineage on every nudge and forced a full prefix rewrite. On a captured ~4 hour Claude-via-Hermes session this cost roughly 4.3 M cache-write tokens — about half of those attributable to the per-nudge alternation between the main turn's narrowed `tools[]` and the review fork's wider `tools[]`. ## Goal Extend the byte-stability invariant established by PR #17276 (which fixed `system`) to the `tools[]` slot of the request body, so the review fork's outbound request hits the parent's warmed Anthropic prefix cache regardless of how the parent's toolset is configured. ## Implementation Two-line change in `agent/background_review.py`: pass `enabled_toolsets=getattr(agent, "enabled_toolsets", None)` and the matching `disabled_toolsets` kwarg into the `AIAgent(...)` call inside `_spawn_background_review`. Adds an explanatory block comment that calls out the cache-key dependency and the relationship to PR #17276. The post-construction runtime whitelist (`set_thread_tool_whitelist({memory, skills})`) is untouched — it still gates which tools the model is allowed to *dispatch*. This change aligns only what the request body *transmits*, not what the review is allowed to do, so the safety contract from issue #15204 remains intact. ## Testing - `tests/run_agent/test_background_review_cache_parity.py`: new `test_review_fork_inherits_parent_toolset_config` asserts the parent's `enabled_toolsets` and `disabled_toolsets` reach the review-fork constructor as kwargs. - `tests/run_agent/test_background_review_toolset_restriction.py`: the existing `test_background_review_does_not_narrow_toolset_schema` was inverted (its old "must NOT pass enabled_toolsets" rule was built on the assumption that the parent always ran with the registry default — wrong in practice when the parent is narrowed). Renamed to `test_background_review_matches_parent_toolset_config` and updated to assert the parent's value propagates verbatim. - Verified the new positive test fails without the fix and passes with it. - Full suite for `test_background_review*`: ``` $ python -m pytest tests/run_agent/test_background_review.py \ tests/run_agent/test_background_review_summary.py \ tests/run_agent/test_background_review_toolset_restriction.py \ tests/run_agent/test_background_review_cache_parity.py -q 18 passed in 1.85s ``` ## Scope - `agent/background_review.py`: 2 added kwargs + explanatory comment. - Two test files: one new positive test, one inverted existing test. - No production code paths outside the review fork; no schema changes; no public-API changes. Refs: ziliangpeng/hermes-agent#1 (root-cause analysis with wire-level cache-write measurements). Extends PR #17276's `system`-bytes invariant to the `tools[]` slot.
This commit is contained in:
parent
6c26727bb3
commit
c3a09f7835
@ -390,6 +390,24 @@ def _run_review_in_thread(
|
|||||||
# parent below so memory(action="add") writes from
|
# parent below so memory(action="add") writes from
|
||||||
# the review still land on disk; the review just
|
# the review still land on disk; the review just
|
||||||
# has zero side effects on external providers.
|
# has zero side effects on external providers.
|
||||||
|
# Inherit the parent's toolset configuration so the review
|
||||||
|
# fork's outbound request body has byte-identical ``tools[]``
|
||||||
|
# with the parent's last main-turn request. Without this,
|
||||||
|
# ``enabled_toolsets=None`` defaults to "all registered tools"
|
||||||
|
# and the fork transmits every tool descriptor (including any
|
||||||
|
# the user has disabled via ``hermes tools disable``), while
|
||||||
|
# the parent transmits only its narrower configured set —
|
||||||
|
# making the two requests diverge in ``tools[]`` even though
|
||||||
|
# they share ``messages[0..N]`` and ``system`` byte-for-byte.
|
||||||
|
# Anthropic's prompt-cache key includes ``tools[]``, so any
|
||||||
|
# divergence forks the cache lineage and forces a full
|
||||||
|
# prefix rewrite (~100-200K tokens per turn for long convs).
|
||||||
|
# The post-construction whitelist (``set_thread_tool_whitelist``
|
||||||
|
# below) still restricts which tools the model is allowed
|
||||||
|
# to dispatch — this change only aligns what the request
|
||||||
|
# body transmits, not what the review is allowed to do.
|
||||||
|
# This extends the byte-stability invariant established by
|
||||||
|
# PR #17276 (which fixed ``system``) to ``tools[]``.
|
||||||
review_agent = AIAgent(
|
review_agent = AIAgent(
|
||||||
model=agent.model,
|
model=agent.model,
|
||||||
max_iterations=16,
|
max_iterations=16,
|
||||||
@ -401,6 +419,8 @@ def _run_review_in_thread(
|
|||||||
api_key=_parent_runtime.get("api_key") or None,
|
api_key=_parent_runtime.get("api_key") or None,
|
||||||
credential_pool=getattr(agent, "_credential_pool", None),
|
credential_pool=getattr(agent, "_credential_pool", None),
|
||||||
parent_session_id=agent.session_id,
|
parent_session_id=agent.session_id,
|
||||||
|
enabled_toolsets=getattr(agent, "enabled_toolsets", None),
|
||||||
|
disabled_toolsets=getattr(agent, "disabled_toolsets", None),
|
||||||
skip_memory=True,
|
skip_memory=True,
|
||||||
)
|
)
|
||||||
review_agent._memory_write_origin = "background_review"
|
review_agent._memory_write_origin = "background_review"
|
||||||
|
|||||||
@ -38,6 +38,13 @@ def _make_agent_stub(agent_cls):
|
|||||||
agent._MEMORY_REVIEW_PROMPT = "review memory"
|
agent._MEMORY_REVIEW_PROMPT = "review memory"
|
||||||
agent._SKILL_REVIEW_PROMPT = "review skills"
|
agent._SKILL_REVIEW_PROMPT = "review skills"
|
||||||
agent._COMBINED_REVIEW_PROMPT = "review both"
|
agent._COMBINED_REVIEW_PROMPT = "review both"
|
||||||
|
# Parent's toolset configuration — must be propagated to the review
|
||||||
|
# fork so ``tools[]`` matches byte-for-byte. Without these set on the
|
||||||
|
# stub, ``getattr(agent, ..., None)`` would return None on both sides
|
||||||
|
# and the test wouldn't catch a regression where the fork is built
|
||||||
|
# without the kwargs at all.
|
||||||
|
agent.enabled_toolsets = ["memory", "skills", "terminal"]
|
||||||
|
agent.disabled_toolsets = ["spotify", "feishu_doc"]
|
||||||
return agent
|
return agent
|
||||||
|
|
||||||
|
|
||||||
@ -183,3 +190,73 @@ def test_review_fork_pins_session_start_and_session_id():
|
|||||||
"Review fork did not inherit parent's session_id — "
|
"Review fork did not inherit parent's session_id — "
|
||||||
"system-prompt rebuild paths would diverge."
|
"system-prompt rebuild paths would diverge."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_review_fork_inherits_parent_toolset_config():
|
||||||
|
"""The review fork must receive ``enabled_toolsets`` / ``disabled_toolsets``
|
||||||
|
from the parent so the outbound request body's ``tools[]`` field matches
|
||||||
|
byte-for-byte.
|
||||||
|
|
||||||
|
Without this, ``enabled_toolsets=None`` defaults to "all registered tools"
|
||||||
|
and the fork sends every tool descriptor (e.g. Spotify, Feishu, video)
|
||||||
|
even when the parent disabled them via ``hermes tools disable``. Anthropic's
|
||||||
|
prompt cache keys on the byte-exact ``tools[]`` array, so divergence here
|
||||||
|
forks the cache lineage and forces a full prefix rewrite per nudge
|
||||||
|
(~100-200 K cache-write tokens for long conversations).
|
||||||
|
|
||||||
|
This is the same byte-stability invariant as
|
||||||
|
``test_review_fork_inherits_parent_cached_system_prompt`` but for the
|
||||||
|
``tools[]`` slot of the request body, not the ``system`` slot.
|
||||||
|
"""
|
||||||
|
import run_agent
|
||||||
|
|
||||||
|
agent = _make_agent_stub(run_agent.AIAgent)
|
||||||
|
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
class _Recorder:
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
captured["enabled_toolsets"] = kwargs.get("enabled_toolsets")
|
||||||
|
captured["disabled_toolsets"] = kwargs.get("disabled_toolsets")
|
||||||
|
# Minimal post-init attrs the surrounding code touches.
|
||||||
|
self._cached_system_prompt = None
|
||||||
|
self._memory_write_origin = None
|
||||||
|
self._memory_write_context = None
|
||||||
|
self._memory_store = None
|
||||||
|
self._memory_enabled = None
|
||||||
|
self._user_profile_enabled = None
|
||||||
|
self._memory_nudge_interval = None
|
||||||
|
self._skill_nudge_interval = None
|
||||||
|
self.suppress_status_output = None
|
||||||
|
self.session_start = None
|
||||||
|
self.session_id = None
|
||||||
|
|
||||||
|
def run_conversation(self, *args, **kwargs):
|
||||||
|
raise RuntimeError("stop after recording — don't actually call the API")
|
||||||
|
|
||||||
|
def shutdown_memory_provider(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
with patch.object(run_agent, "AIAgent", _Recorder), \
|
||||||
|
patch("threading.Thread", _SyncThread):
|
||||||
|
agent._spawn_background_review(
|
||||||
|
messages_snapshot=[],
|
||||||
|
review_memory=True,
|
||||||
|
review_skills=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert captured.get("enabled_toolsets") == agent.enabled_toolsets, (
|
||||||
|
f"Review fork did not receive parent's enabled_toolsets. "
|
||||||
|
f"Got {captured.get('enabled_toolsets')!r}, expected {agent.enabled_toolsets!r}. "
|
||||||
|
"This causes ``tools[]`` to diverge between main turns and review nudges, "
|
||||||
|
"breaking Anthropic prompt-cache parity."
|
||||||
|
)
|
||||||
|
assert captured.get("disabled_toolsets") == agent.disabled_toolsets, (
|
||||||
|
f"Review fork did not receive parent's disabled_toolsets. "
|
||||||
|
f"Got {captured.get('disabled_toolsets')!r}, expected {agent.disabled_toolsets!r}. "
|
||||||
|
"This causes ``tools[]`` to diverge between main turns and review nudges, "
|
||||||
|
"breaking Anthropic prompt-cache parity."
|
||||||
|
)
|
||||||
|
|||||||
@ -38,6 +38,13 @@ def _make_agent_stub(agent_cls):
|
|||||||
agent._MEMORY_REVIEW_PROMPT = "review memory"
|
agent._MEMORY_REVIEW_PROMPT = "review memory"
|
||||||
agent._SKILL_REVIEW_PROMPT = "review skills"
|
agent._SKILL_REVIEW_PROMPT = "review skills"
|
||||||
agent._COMBINED_REVIEW_PROMPT = "review both"
|
agent._COMBINED_REVIEW_PROMPT = "review both"
|
||||||
|
# Parent's toolset configuration must propagate to the review fork
|
||||||
|
# so the request body's ``tools[]`` array is byte-identical. Without
|
||||||
|
# propagation, ``enabled_toolsets=None`` expands to "all registered
|
||||||
|
# tools" — which diverges from a parent that narrowed its set via
|
||||||
|
# ``hermes tools disable`` or config.
|
||||||
|
agent.enabled_toolsets = ["memory", "skills", "terminal"]
|
||||||
|
agent.disabled_toolsets = ["spotify", "feishu_doc"]
|
||||||
return agent
|
return agent
|
||||||
|
|
||||||
|
|
||||||
@ -52,12 +59,25 @@ class _SyncThread:
|
|||||||
self._target()
|
self._target()
|
||||||
|
|
||||||
|
|
||||||
def test_background_review_does_not_narrow_toolset_schema():
|
def test_background_review_matches_parent_toolset_config():
|
||||||
"""The review fork must NOT pass enabled_toolsets to AIAgent.
|
"""The review fork must propagate the parent's ``enabled_toolsets`` to AIAgent.
|
||||||
|
|
||||||
Narrowing the schema diverges the ``tools`` cache key from the parent's,
|
Earlier guidance (the old "do NOT pass enabled_toolsets" rule) assumed the
|
||||||
which sits above ``system`` in Anthropic's cache hierarchy and forces a
|
parent always ran with the registry default. In practice the parent is
|
||||||
full prefix-cache miss on every review (see #25322, PR #17276).
|
often narrowed via ``hermes tools disable`` or ``config.yaml``, and
|
||||||
|
leaving ``enabled_toolsets=None`` on the fork makes it expand to ALL
|
||||||
|
registered tools — which is what *diverges* from the parent and breaks
|
||||||
|
Anthropic's prefix cache key on ``tools[]``.
|
||||||
|
|
||||||
|
The correct rule is symmetric: whatever the parent has, the fork
|
||||||
|
must have the same. When the parent's value is ``None``, the fork's
|
||||||
|
must also be ``None`` (and they'll both expand identically). When
|
||||||
|
the parent narrows, the fork inherits the narrowed set verbatim.
|
||||||
|
|
||||||
|
(Schema-level alignment with the parent + post-construction runtime
|
||||||
|
whitelist remain the safety contract for #15204 — the whitelist
|
||||||
|
blocks dispatch of non-memory/skill tools regardless of what schemas
|
||||||
|
are sent over the wire.)
|
||||||
"""
|
"""
|
||||||
import run_agent
|
import run_agent
|
||||||
|
|
||||||
@ -66,6 +86,7 @@ def test_background_review_does_not_narrow_toolset_schema():
|
|||||||
|
|
||||||
def _capture_init(self, *args, **kwargs):
|
def _capture_init(self, *args, **kwargs):
|
||||||
captured["enabled_toolsets"] = kwargs.get("enabled_toolsets", "UNSET")
|
captured["enabled_toolsets"] = kwargs.get("enabled_toolsets", "UNSET")
|
||||||
|
captured["disabled_toolsets"] = kwargs.get("disabled_toolsets", "UNSET")
|
||||||
raise RuntimeError("stop after capturing init args")
|
raise RuntimeError("stop after capturing init args")
|
||||||
|
|
||||||
with patch.object(run_agent.AIAgent, "__init__", _capture_init), \
|
with patch.object(run_agent.AIAgent, "__init__", _capture_init), \
|
||||||
@ -77,11 +98,18 @@ def test_background_review_does_not_narrow_toolset_schema():
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert "enabled_toolsets" in captured, "AIAgent.__init__ was not called"
|
assert "enabled_toolsets" in captured, "AIAgent.__init__ was not called"
|
||||||
# The kwarg must be absent — letting AIAgent inherit the default full
|
# The kwargs must equal the parent's so the ``tools[]`` request-body
|
||||||
# toolset so the schema bytes match the parent's.
|
# bytes match the parent's last main-turn request.
|
||||||
assert captured["enabled_toolsets"] == "UNSET", (
|
assert captured["enabled_toolsets"] == agent.enabled_toolsets, (
|
||||||
f"Review fork narrowed the toolset schema (got {captured['enabled_toolsets']!r}), "
|
f"Review fork did not propagate parent's enabled_toolsets "
|
||||||
"which breaks prefix-cache parity with the parent."
|
f"(got {captured['enabled_toolsets']!r}, expected {agent.enabled_toolsets!r}). "
|
||||||
|
"This causes ``tools[]`` to diverge from the parent — Anthropic's "
|
||||||
|
"prompt-cache key includes ``tools[]``, so divergence forks the "
|
||||||
|
"cache lineage and forces a full prefix rewrite per nudge."
|
||||||
|
)
|
||||||
|
assert captured["disabled_toolsets"] == agent.disabled_toolsets, (
|
||||||
|
f"Review fork did not propagate parent's disabled_toolsets "
|
||||||
|
f"(got {captured['disabled_toolsets']!r}, expected {agent.disabled_toolsets!r})."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user