fix(skills): bound ClawHub catalog walk to requested page on cold start (#43395)

Browse renders one page but the cold-cache fallback walked the entire
50k+ ClawHub catalog, then sliced off the first N — pure waste behind the
12s budget band-aid. _load_catalog_index now takes max_items: browse's
empty-query path bounds the walk to its limit and stops early; the offline
index builder still passes limit=0 (unbounded) and walks to exhaustion.
A bounded walk is partial, so it is not written to the shared full-catalog
cache (same poison-guard as the budget-truncated case).
This commit is contained in:
Teknium
2026-06-10 01:01:53 -07:00
committed by GitHub
parent 6a30cfca82
commit eee1da45f0
2 changed files with 140 additions and 10 deletions
+34 -10
View File
@@ -2119,12 +2119,13 @@ class ClawHubSource(SkillSource):
if results:
return results
else:
# Empty query: route through the paginating catalog walker so the
# full ClawHub catalog (20k+ skills) lands in the index. The
# single-request listing path below caps at one page (200 items)
# regardless of `limit`, which silently truncates the public
# skills index. The catalog walker follows `nextCursor`.
catalog = self._load_catalog_index()
# Empty query: route through the paginating catalog walker. When
# the full catalog is already disk-cached this returns it whole and
# the caller paginates client-side. On a cold cache, bound the walk
# to `limit` so a browse command renders its first page without
# walking the entire 50k+ catalog (max_items=0 → unbounded, used
# only by the offline index builder via search("", limit=0)).
catalog = self._load_catalog_index(max_items=limit if limit > 0 else 0)
if catalog:
return self._dedupe_results(catalog)[:limit] if limit > 0 else self._dedupe_results(catalog)
@@ -2249,7 +2250,21 @@ class ClawHubSource(SkillSource):
_write_index_cache(cache_key, [_skill_meta_to_dict(s) for s in results])
return results
def _load_catalog_index(self) -> List[SkillMeta]:
def _load_catalog_index(self, max_items: int = 0) -> List[SkillMeta]:
"""Walk the ClawHub catalog via cursor pagination.
``max_items`` bounds the walk: once at least that many distinct skills
have been gathered the walk stops early. This is what browse's
cold-start fallback wants — it only renders one page, so walking the
entire 50k+ catalog just to slice off the first N is pure waste.
``max_items=0`` (the default, used by the offline index builder) means
walk to exhaustion.
Caching: only a *complete* catalog (cursor exhausted or page cap) is
written to the shared ``clawhub_catalog_v1`` cache. A walk truncated by
``max_items`` OR the wall-clock budget is partial, so caching it would
poison the full-catalog cache with an incomplete slice.
"""
cache_key = "clawhub_catalog_v1"
cached = _read_index_cache(cache_key)
if cached is not None:
@@ -2266,6 +2281,7 @@ class ClawHubSource(SkillSource):
max_pages = 750
deadline = time.monotonic() + self.CATALOG_WALK_BUDGET_SECONDS
hit_deadline = False
hit_max_items = False
for _ in range(max_pages):
if time.monotonic() > deadline:
@@ -2308,10 +2324,18 @@ class ClawHubSource(SkillSource):
if not isinstance(cursor, str) or not cursor:
break
# Browse's cold-start fallback only renders one page, so stop as
# soon as we have enough to satisfy the caller's bound. The index
# builder passes max_items=0 (unbounded) and walks to exhaustion.
if max_items > 0 and len(results) >= max_items:
hit_max_items = True
break
# Only cache a walk that reached a natural stop (cursor exhausted or
# page cap). A walk truncated by the wall-clock budget is partial, so
# writing it would poison the cache with incomplete catalog data.
if not hit_deadline:
# page cap). A walk truncated by the wall-clock budget OR by max_items
# is partial, so writing it would poison the shared full-catalog cache
# with incomplete data.
if not hit_deadline and not hit_max_items:
_write_index_cache(cache_key, [_skill_meta_to_dict(s) for s in results])
return results