fix(skills-hub): stop shipping a degenerate index when GitHub taps collapse (#42347)
The Skills Hub lost every api.github.com-backed source — the OpenAI, Anthropic, HuggingFace, NVIDIA, gstack, Claude Marketplace and Well-Known tabs all vanished — while ClawHub/skills.sh/LobeHub/browse.sh survived. A GitHub API rate limit during the docs-deploy crawl zeroed all three api.github.com sources (github / claude-marketplace / well-known) at once. Two compounding bugs let the broken index reach the live site: 1. build_skills_index.py wrote the output file BEFORE the health check, so even when the github floor (30) tripped and the script exited 2, the degenerate file was already on disk. deploy-site.yml then swallowed the exit code with `|| echo non-fatal` and extract-skills.py read the partial index. Fix: run the health check first, write the file only when healthy, exit without writing on failure. Removed the non-fatal swallow in deploy-site.yml so a collapse fails the deploy and the last good site stays live (Pages serves the previous build). 2. The build-time GitHub listing path returned [] on a 403 rate-limit without retrying or flagging it, so a rate-limited crawl looked identical to an empty source. Fix: a shared _github_get() helper on GitHubSource with retry/backoff (honors Retry-After / X-RateLimit-Reset on 403/429, backs off on 5xx + transport errors) and flags is_rate_limited. Routed _list_skills_in_repo and _fetch_file_content through it; gave ClaudeMarketplaceSource a persistent GitHubSource + is_rate_limited so the builder can name the rate limit as the cause instead of '0 results'. Added tests/scripts/test_build_skills_index_health.py pinning both contracts: a degenerate crawl exits non-zero and writes no file; a healthy crawl writes the index with github/claude-marketplace/well-known all present.
This commit is contained in:
Teknium
+111
-30
@@ -550,11 +550,8 @@ class GitHubSource(SkillSource):
|
||||
return [SkillMeta(**s) for s in cached]
|
||||
|
||||
url = f"https://api.github.com/repos/{repo}/contents/{path.rstrip('/')}"
|
||||
try:
|
||||
resp = httpx.get(url, headers=self.auth.get_headers(), timeout=15, follow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
return []
|
||||
except httpx.HTTPError:
|
||||
resp = self._github_get(url)
|
||||
if resp is None or resp.status_code != 200:
|
||||
return []
|
||||
|
||||
entries = resp.json()
|
||||
@@ -639,15 +636,98 @@ class GitHubSource(SkillSource):
|
||||
|
||||
def _check_rate_limit_response(self, resp: "httpx.Response") -> None:
|
||||
"""Flag the instance as rate-limited when GitHub returns 403 + exhausted quota."""
|
||||
if resp.status_code == 403:
|
||||
if resp.status_code in (403, 429):
|
||||
remaining = resp.headers.get("X-RateLimit-Remaining", "")
|
||||
if remaining == "0":
|
||||
if remaining == "0" or resp.status_code == 429:
|
||||
self._rate_limited = True
|
||||
logger.warning(
|
||||
"GitHub API rate limit exhausted (unauthenticated: 60 req/hr). "
|
||||
"Set GITHUB_TOKEN or install the gh CLI to raise the limit to 5,000/hr."
|
||||
)
|
||||
|
||||
def _github_get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
params: Optional[Dict] = None,
|
||||
headers: Optional[Dict] = None,
|
||||
timeout: float = 15.0,
|
||||
max_retries: int = 3,
|
||||
) -> Optional["httpx.Response"]:
|
||||
"""GET against the GitHub API with retry/backoff on transient failures.
|
||||
|
||||
Returns the final ``httpx.Response`` (caller inspects status) or
|
||||
``None`` when every attempt raised a transport error.
|
||||
|
||||
Retries on:
|
||||
- 403/429 with ``X-RateLimit-Remaining: 0`` — waits until the
|
||||
reset time (capped) when the header is present, else exponential
|
||||
backoff. This is the all-GitHub-tap-collapse case: a single
|
||||
shared rate limit zeroes github + claude-marketplace + well-known
|
||||
at once during the index build.
|
||||
- 5xx and connection/timeout errors — exponential backoff.
|
||||
|
||||
On terminal rate-limit exhaustion the instance is flagged via
|
||||
``_check_rate_limit_response`` so the build can fail loud instead of
|
||||
silently shipping an index with the GitHub sources dropped to zero.
|
||||
"""
|
||||
hdrs = headers if headers is not None else self.auth.get_headers()
|
||||
backoff = 1.0
|
||||
last_resp: Optional["httpx.Response"] = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
resp = httpx.get(
|
||||
url, params=params, headers=hdrs,
|
||||
timeout=timeout, follow_redirects=True,
|
||||
)
|
||||
except httpx.HTTPError as e:
|
||||
logger.debug("GitHub GET %s failed (attempt %d/%d): %s",
|
||||
url, attempt + 1, max_retries, e)
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, 30.0)
|
||||
continue
|
||||
return None
|
||||
|
||||
last_resp = resp
|
||||
if resp.status_code == 200:
|
||||
return resp
|
||||
|
||||
# Rate-limited: honor the reset header when present, else back off.
|
||||
if resp.status_code in (403, 429):
|
||||
remaining = resp.headers.get("X-RateLimit-Remaining", "")
|
||||
is_rl = remaining == "0" or resp.status_code == 429
|
||||
if is_rl and attempt < max_retries - 1:
|
||||
wait = backoff
|
||||
reset = resp.headers.get("X-RateLimit-Reset", "")
|
||||
retry_after = resp.headers.get("Retry-After", "")
|
||||
if retry_after.isdigit():
|
||||
wait = min(float(retry_after), 60.0)
|
||||
elif reset.isdigit():
|
||||
delta = float(reset) - time.time()
|
||||
if 0 < delta <= 60.0:
|
||||
wait = delta
|
||||
logger.debug(
|
||||
"GitHub rate limited on %s, waiting %.1fs (attempt %d/%d)",
|
||||
url, wait, attempt + 1, max_retries,
|
||||
)
|
||||
time.sleep(wait)
|
||||
backoff = min(backoff * 2, 30.0)
|
||||
continue
|
||||
# Out of retries (or not a rate-limit 403) — flag and return.
|
||||
self._check_rate_limit_response(resp)
|
||||
return resp
|
||||
|
||||
# 5xx — retry; 4xx (other than rate limit) — return immediately.
|
||||
if 500 <= resp.status_code < 600 and attempt < max_retries - 1:
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, 30.0)
|
||||
continue
|
||||
return resp
|
||||
|
||||
return last_resp
|
||||
|
||||
|
||||
def _download_directory(self, repo: str, path: str) -> Dict[str, str]:
|
||||
"""Recursively download all text files from a GitHub directory.
|
||||
|
||||
@@ -768,17 +848,12 @@ class GitHubSource(SkillSource):
|
||||
def _fetch_file_content(self, repo: str, path: str) -> Optional[str]:
|
||||
"""Fetch a single file's content from GitHub."""
|
||||
url = f"https://api.github.com/repos/{repo}/contents/{path}"
|
||||
try:
|
||||
resp = httpx.get(
|
||||
url,
|
||||
headers={**self.auth.get_headers(), "Accept": "application/vnd.github.v3.raw"},
|
||||
timeout=15, follow_redirects=True,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
return resp.text
|
||||
self._check_rate_limit_response(resp)
|
||||
except httpx.HTTPError as e:
|
||||
logger.debug("GitHub contents API fetch failed: %s", e)
|
||||
resp = self._github_get(
|
||||
url,
|
||||
headers={**self.auth.get_headers(), "Accept": "application/vnd.github.v3.raw"},
|
||||
)
|
||||
if resp is not None and resp.status_code == 200:
|
||||
return resp.text
|
||||
return None
|
||||
|
||||
def _get_skillsh_groupings(self, repo: str) -> Optional[Dict[str, str]]:
|
||||
@@ -2373,10 +2448,19 @@ class ClaudeMarketplaceSource(SkillSource):
|
||||
|
||||
def __init__(self, auth: GitHubAuth):
|
||||
self.auth = auth
|
||||
# Persistent GitHubSource so rate-limit state survives across the
|
||||
# marketplace-index fetch + per-skill inspect calls and can be
|
||||
# surfaced to the index builder (see is_rate_limited).
|
||||
self.github = GitHubSource(auth=auth)
|
||||
|
||||
def source_id(self) -> str:
|
||||
return "claude-marketplace"
|
||||
|
||||
@property
|
||||
def is_rate_limited(self) -> bool:
|
||||
"""Whether the underlying GitHub API hit a rate limit during the crawl."""
|
||||
return self.github.is_rate_limited
|
||||
|
||||
def trust_level_for(self, identifier: str) -> str:
|
||||
parts = identifier.split("/", 2)
|
||||
if len(parts) >= 2:
|
||||
@@ -2415,15 +2499,13 @@ class ClaudeMarketplaceSource(SkillSource):
|
||||
|
||||
def fetch(self, identifier: str) -> Optional[SkillBundle]:
|
||||
# Delegate to GitHub Contents API since marketplace skills live in GitHub repos
|
||||
gh = GitHubSource(auth=self.auth)
|
||||
bundle = gh.fetch(identifier)
|
||||
bundle = self.github.fetch(identifier)
|
||||
if bundle:
|
||||
bundle.source = "claude-marketplace"
|
||||
return bundle
|
||||
|
||||
def inspect(self, identifier: str) -> Optional[SkillMeta]:
|
||||
gh = GitHubSource(auth=self.auth)
|
||||
meta = gh.inspect(identifier)
|
||||
meta = self.github.inspect(identifier)
|
||||
if meta:
|
||||
meta.source = "claude-marketplace"
|
||||
meta.trust_level = self.trust_level_for(identifier)
|
||||
@@ -2437,16 +2519,15 @@ class ClaudeMarketplaceSource(SkillSource):
|
||||
return cached
|
||||
|
||||
url = f"https://api.github.com/repos/{repo}/contents/.claude-plugin/marketplace.json"
|
||||
resp = self.github._github_get(
|
||||
url,
|
||||
headers={**self.auth.get_headers(), "Accept": "application/vnd.github.v3.raw"},
|
||||
)
|
||||
if resp is None or resp.status_code != 200:
|
||||
return []
|
||||
try:
|
||||
resp = httpx.get(
|
||||
url,
|
||||
headers={**self.auth.get_headers(), "Accept": "application/vnd.github.v3.raw"},
|
||||
timeout=15,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return []
|
||||
data = json.loads(resp.text)
|
||||
except (httpx.HTTPError, json.JSONDecodeError):
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
|
||||
plugins = data.get("plugins", [])
|
||||
|
||||
Reference in New Issue
Block a user