feat(curator): prune built-in skills after inactivity + track usage for all skills (#36701)

Two related changes to the skill curator:

1. Built-in pruning. New curator.prune_builtins config (default on) lets the
   curator archive bundled built-in skills after the inactivity period, not
   just agent-created ones. A .curator_suppressed list tells the update-time
   re-seeder (tools/skills_sync) to leave pruned built-ins archived, so the
   prune is durable across `hermes update`. Built-ins are seeded with a
   baseline record on first sight, so the inactivity clock starts at upgrade
   time -- no mass-prune on the first run. Hub-installed skills are never
   pruned regardless of the flag. Restoring a built-in clears its suppression.

2. Usage tracking for all skills. Telemetry (view/use/patch) was wrongly gated
   behind curation-eligibility, so built-ins were tracked only when prunable
   and hub skills never. Telemetry is observability and is now decoupled from
   curation: every skill accrues usage counts regardless of provenance, while
   lifecycle mutators (set_state/set_pinned/mark_agent_created) stay
   curation-gated. New usage_report() + provenance() expose all skills with an
   agent/bundled/hub tag.
This commit is contained in:
Teknium
2026-06-01 02:07:32 -07:00
committed by GitHub
parent 0622a70eb4
commit 70e1571d89
8 changed files with 622 additions and 73 deletions
+91 -25
View File
@@ -18,7 +18,13 @@ def _bump_view_many(hermes_home: str, skill_name: str, iterations: int) -> None:
@pytest.fixture
def skills_home(tmp_path, monkeypatch):
"""Isolated HERMES_HOME with a clean skills/ dir for each test."""
"""Isolated HERMES_HOME with a clean skills/ dir for each test.
Pins ``curator.prune_builtins`` OFF so the bundled/hub-protection tests in
this module exercise the off-path semantics regardless of the shipped
default. Tests that want built-ins to be curation-eligible flip it back on
explicitly via ``monkeypatch.setattr(mod, "_prune_builtins_enabled", ...)``.
"""
home = tmp_path / ".hermes"
home.mkdir()
(home / "skills").mkdir()
@@ -28,6 +34,7 @@ def skills_home(tmp_path, monkeypatch):
import importlib
import tools.skill_usage as mod
importlib.reload(mod)
monkeypatch.setattr(mod, "_prune_builtins_enabled", lambda: False)
return home
@@ -520,27 +527,35 @@ def test_agent_created_report_derives_activity_from_view_and_patch(skills_home,
# ---------------------------------------------------------------------------
# Provenance guard — telemetry must not leak records for bundled/hub skills
# Telemetry vs curation — usage is tracked for ALL skills; curation is not
# ---------------------------------------------------------------------------
def test_bump_view_no_op_for_bundled_skill(skills_home):
"""Telemetry bumps on bundled skills are dropped — the sidecar must stay
focused on agent-created skills only."""
from tools.skill_usage import bump_view, load_usage
def test_bump_view_tracks_bundled_skill(skills_home):
"""Telemetry IS recorded for bundled skills (observability), but the record
must NOT make the skill a curation candidate by itself."""
from tools.skill_usage import (
bump_view, load_usage, list_agent_created_skill_names,
)
skills_dir = skills_home / "skills"
_write_skill(skills_dir, "ship-bundled")
(skills_dir / ".bundled_manifest").write_text(
"ship-bundled:abc\n", encoding="utf-8",
)
bump_view("ship-bundled")
assert "ship-bundled" not in load_usage(), (
"bundled skill leaked into .usage.json"
rec = load_usage().get("ship-bundled")
assert isinstance(rec, dict), "bundled skill telemetry should be recorded"
assert rec["view_count"] == 1
# Pruning is off by default in this fixture → not a curation candidate.
assert "ship-bundled" not in list_agent_created_skill_names()
def test_bump_patch_tracks_hub_skill(skills_home):
from tools.skill_usage import (
bump_patch, load_usage, list_agent_created_skill_names,
)
def test_bump_patch_no_op_for_hub_skill(skills_home):
from tools.skill_usage import bump_patch, load_usage
skills_dir = skills_home / "skills"
_write_skill(skills_dir, "from-hub")
hub = skills_dir / ".hub"
hub.mkdir()
(hub / "lock.json").write_text(
@@ -548,12 +563,17 @@ def test_bump_patch_no_op_for_hub_skill(skills_home):
)
bump_patch("from-hub")
assert "from-hub" not in load_usage()
rec = load_usage().get("from-hub")
assert isinstance(rec, dict), "hub skill telemetry should be recorded"
assert rec["patch_count"] == 1
# Hub skills are NEVER curation candidates regardless of any flag.
assert "from-hub" not in list_agent_created_skill_names()
def test_bump_use_no_op_for_hub_skill(skills_home):
def test_bump_use_tracks_hub_skill(skills_home):
from tools.skill_usage import bump_use, load_usage
skills_dir = skills_home / "skills"
_write_skill(skills_dir, "from-hub")
hub = skills_dir / ".hub"
hub.mkdir()
(hub / "lock.json").write_text(
@@ -561,7 +581,9 @@ def test_bump_use_no_op_for_hub_skill(skills_home):
)
bump_use("from-hub")
assert "from-hub" not in load_usage()
rec = load_usage().get("from-hub")
assert isinstance(rec, dict)
assert rec["use_count"] == 1
def test_set_state_no_op_for_bundled_skill(skills_home):
@@ -593,12 +615,17 @@ def test_restore_refuses_to_shadow_bundled_skill(skills_home):
assert "bundled" in msg.lower() or "shadow" in msg.lower()
def test_end_to_end_no_code_path_mutates_bundled_skill(skills_home):
"""The combined guarantee: no curator code path can archive, mark stale,
set-state, or persist telemetry for a bundled or hub-installed skill."""
def test_end_to_end_telemetry_tracked_but_lifecycle_refused(skills_home):
"""The combined guarantee under decoupled telemetry/curation:
- Usage telemetry (view/use/patch) IS recorded for bundled & hub skills.
- Lifecycle mutations (set_state, set_pinned, archive) are REFUSED for them
(with pruning off, the fixture default), so no state/pinned/archived flag
lands and the directories stay on disk.
"""
from tools.skill_usage import (
bump_view, bump_use, bump_patch, set_state, set_pinned,
archive_skill, load_usage, STATE_STALE, STATE_ARCHIVED,
archive_skill, load_usage, STATE_ACTIVE, STATE_STALE, STATE_ARCHIVED,
)
skills_dir = skills_home / "skills"
_write_skill(skills_dir, "bundled-one")
@@ -614,7 +641,6 @@ def test_end_to_end_no_code_path_mutates_bundled_skill(skills_home):
json.dumps({"installed": {"hub-one": {}}}), encoding="utf-8",
)
# Hammer every mutator at the bundled/hub names
for name in ("bundled-one", "hub-one"):
bump_view(name)
bump_use(name)
@@ -625,15 +651,55 @@ def test_end_to_end_no_code_path_mutates_bundled_skill(skills_home):
ok, _msg = archive_skill(name)
assert not ok, f"archive_skill(\"{name}\") should refuse"
# Sidecar must be clean of all three
data = load_usage()
assert "bundled-one" not in data
assert "hub-one" not in data
# Telemetry landed for both.
for name in ("bundled-one", "hub-one"):
assert name in data, f"{name} telemetry should be recorded"
assert data[name]["view_count"] == 1
assert data[name]["use_count"] == 1
assert data[name]["patch_count"] == 1
# But lifecycle mutators were refused — state stays the default, never
# archived/stale/pinned, and created_by is never agent.
assert data[name]["state"] == STATE_ACTIVE
assert data[name]["archived_at"] is None
assert data[name]["pinned"] is False
assert data[name].get("created_by") != "agent"
# Directories must still be in place on disk
# Directories must still be in place on disk.
assert (skills_dir / "bundled-one" / "SKILL.md").exists()
assert (skills_dir / "hub-one" / "SKILL.md").exists()
# The agent-created skill can still be mutated normally
# The agent-created skill can still be mutated normally.
bump_view("mine")
assert load_usage()["mine"]["view_count"] == 1
def test_usage_report_covers_all_provenance(skills_home):
"""usage_report() surfaces every skill with provenance, unlike the
curator-scoped agent_created_report()."""
from tools.skill_usage import (
bump_use, usage_report, mark_agent_created,
)
skills_dir = skills_home / "skills"
_write_skill(skills_dir, "bundled-one")
_write_skill(skills_dir, "hub-one")
_write_skill(skills_dir, "mine")
(skills_dir / ".bundled_manifest").write_text("bundled-one:abc\n", encoding="utf-8")
hub = skills_dir / ".hub"
hub.mkdir()
(hub / "lock.json").write_text(
json.dumps({"installed": {"hub-one": {}}}), encoding="utf-8",
)
mark_agent_created("mine")
for n in ("bundled-one", "hub-one", "mine"):
bump_use(n)
rows = {r["name"]: r for r in usage_report()}
assert set(rows) == {"bundled-one", "hub-one", "mine"}
assert rows["bundled-one"]["provenance"] == "bundled"
assert rows["hub-one"]["provenance"] == "hub"
assert rows["mine"]["provenance"] == "agent"
# All carry real usage now.
for n in rows:
assert rows[n]["use_count"] == 1
assert rows[n]["_persisted"] is True