fix(agent): demote non-coding skill categories to names-only — never hide skills (#44342)
Real-world failure with the original index pruning: under the default auto
posture, an agent-created ops skill in a demoted category vanished from the
prompt's skill index mid-project, and the agent silently fell back to a
stale sibling skill instead. The "discovery-only" premise didn't hold —
models do not reach for skills_list to rediscover what the index stops
showing them, and agent-created skills are the model's accumulated project
memory (runbooks, pitfalls, operating rules).
Gating pruning behind the opt-in focus mode was the wrong fix too: users
opening a worktree don't know the config exists, so the index-noise win
would effectively never ship.
Instead, the coding posture now DEMOTES non-coding categories rather than
hiding them: each demoted category renders as a single names-only line
("gaming [names only]: allthemons10-ops, mc-backup") with a footer note
explaining the omitted descriptions. Every skill name stays in the prompt,
so memory-anchored recall ("load <name>") keeps working in every mode,
while the description noise is still cut. Applies in auto/on/focus alike;
the general posture demotes nothing. Deny-list semantics unchanged —
unknown/custom categories and coding-adjacent ones keep full entries.
API renamed to match the honest semantics: hidden_skill_categories →
compact_skill_categories, build_skills_system_prompt(hidden_categories=) →
compact_categories=.
This commit is contained in:
@@ -368,20 +368,24 @@ class TestProfiles:
|
||||
assert cc.GENERAL_PROFILE.toolset is None
|
||||
assert cc.GENERAL_PROFILE.guidance == ""
|
||||
|
||||
def test_skill_pruning_scoped_to_coding_posture(self, tmp_path):
|
||||
# Coding posture hides clearly-non-coding categories; coding-adjacent
|
||||
# ones stay visible (deny-list semantics).
|
||||
def test_skill_demotion_scoped_to_coding_posture(self, tmp_path):
|
||||
# Coding posture demotes clearly-non-coding categories to names-only
|
||||
# in the index (never hides them — agent-created skills are the
|
||||
# model's project memory and must stay recallable by name).
|
||||
# Coding-adjacent categories keep full entries (deny-list semantics).
|
||||
_git_init(tmp_path)
|
||||
coding = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={})
|
||||
hidden = coding.hidden_skill_categories()
|
||||
assert "social-media" in hidden and "smart-home" in hidden
|
||||
for kept in ("github", "devops", "software-development", "data-science"):
|
||||
assert kept not in hidden
|
||||
# General posture hides nothing.
|
||||
general = cc.resolve_runtime_mode(
|
||||
platform="telegram", cwd=tmp_path, config={}
|
||||
)
|
||||
assert general.hidden_skill_categories() == frozenset()
|
||||
for raw in ("auto", "on", "focus"):
|
||||
mode = cc.resolve_runtime_mode(
|
||||
platform="cli", cwd=tmp_path, config={"agent": {"coding_context": raw}}
|
||||
)
|
||||
assert mode.is_coding is True
|
||||
compact = mode.compact_skill_categories()
|
||||
assert "social-media" in compact and "smart-home" in compact
|
||||
for kept in ("github", "devops", "software-development", "data-science"):
|
||||
assert kept not in compact
|
||||
# General posture demotes nothing.
|
||||
general = cc.resolve_runtime_mode(platform="telegram", cwd=tmp_path, config={})
|
||||
assert general.compact_skill_categories() == frozenset()
|
||||
|
||||
|
||||
# ── detection signals ───────────────────────────────────────────────────────
|
||||
|
||||
@@ -276,8 +276,14 @@ class TestBuildSkillsSystemPrompt:
|
||||
# "search" should appear only once per category
|
||||
assert result.count("- search") == 1
|
||||
|
||||
def test_hidden_categories_pruned_with_note(self, monkeypatch, tmp_path):
|
||||
"""Posture-driven pruning drops whole categories and discloses it."""
|
||||
def test_compact_categories_demoted_to_names_only(self, monkeypatch, tmp_path):
|
||||
"""Posture-driven demotion keeps every skill NAME visible.
|
||||
|
||||
Demoted categories lose their descriptions, never their entries —
|
||||
full pruning caused silent capability loss in a real workflow
|
||||
(agent-created skills are the model's project memory, and models
|
||||
don't rediscover them via skills_list once the index goes quiet).
|
||||
"""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
for cat, name in (("social-media", "tweet-stuff"), ("github", "pr-review")):
|
||||
d = tmp_path / "skills" / cat / name
|
||||
@@ -287,14 +293,18 @@ class TestBuildSkillsSystemPrompt:
|
||||
)
|
||||
|
||||
result = build_skills_system_prompt(
|
||||
hidden_categories=frozenset({"social-media"})
|
||||
compact_categories=frozenset({"social-media"})
|
||||
)
|
||||
assert "pr-review" in result
|
||||
assert "tweet-stuff" not in result
|
||||
# Disclosure note so the model knows the full catalog exists.
|
||||
assert "skills_list" in result
|
||||
# Coding-adjacent category keeps its full entry.
|
||||
assert "pr-review" in result and "Does pr-review things" in result
|
||||
# Demoted category: name stays visible, description is dropped.
|
||||
assert "tweet-stuff" in result
|
||||
assert "Does tweet-stuff things" not in result
|
||||
assert "social-media [names only]" in result
|
||||
# Disclosure note explains the demotion and how to load.
|
||||
assert "skill_view" in result
|
||||
|
||||
def test_hidden_categories_prune_nested_and_miss_cache_separately(
|
||||
def test_compact_categories_demote_nested_and_miss_cache_separately(
|
||||
self, monkeypatch, tmp_path
|
||||
):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
@@ -303,14 +313,16 @@ class TestBuildSkillsSystemPrompt:
|
||||
(d / "SKILL.md").write_text(
|
||||
"---\nname: thread-writer\ndescription: Write threads\n---\n"
|
||||
)
|
||||
# Nested category ("social-media/twitter") pruned via its parent.
|
||||
pruned = build_skills_system_prompt(
|
||||
hidden_categories=frozenset({"social-media"})
|
||||
# Nested category ("social-media/twitter") demoted via its parent:
|
||||
# name visible, description gone.
|
||||
compact = build_skills_system_prompt(
|
||||
compact_categories=frozenset({"social-media"})
|
||||
)
|
||||
assert "thread-writer" not in pruned
|
||||
# Unfiltered call must not be served from the filtered cache entry.
|
||||
assert "thread-writer" in compact
|
||||
assert "Write threads" not in compact
|
||||
# Unfiltered call must not be served from the compacted cache entry.
|
||||
full = build_skills_system_prompt()
|
||||
assert "thread-writer" in full
|
||||
assert "Write threads" in full
|
||||
|
||||
def test_excludes_incompatible_platform_skills(self, monkeypatch, tmp_path):
|
||||
"""Skills with platforms: [macos] should not appear on Linux."""
|
||||
|
||||
Reference in New Issue
Block a user