fix(model_metadata): consult DEFAULT_CONTEXT_LENGTHS before 256K fallback on custom endpoints
Problem: get_model_context_length() had an early return at the end of the custom-endpoint probe branch (step 3) that returned DEFAULT_FALLBACK_CONTEXT (256K) without ever consulting the hardcoded DEFAULT_CONTEXT_LENGTHS catalog (step 8). Models served through a custom/proxied gateway (e.g. corporate Anthropic proxy) that didn't expose Ollama or local-server endpoints would hit this path and get capped at 256K, even when the model name clearly matched a known entry in the catalog (e.g. claude-opus-4-8 → 1M). Changes: - agent/model_metadata.py: Before returning DEFAULT_FALLBACK_CONTEXT at the end of the custom-endpoint branch, consult DEFAULT_CONTEXT_LENGTHS using the same longest-key-first fuzzy matching as step 8. Only fall through to 256K if no catalog entry matches. - tests/agent/test_model_metadata.py: Updated existing test and added new test covering the custom-endpoint → catalog fallback behavior. Fixes #38865
This commit is contained in:
@@ -18,6 +18,7 @@ from unittest.mock import patch, MagicMock
|
||||
from agent.model_metadata import (
|
||||
CONTEXT_PROBE_TIERS,
|
||||
DEFAULT_CONTEXT_LENGTHS,
|
||||
DEFAULT_FALLBACK_CONTEXT,
|
||||
_strip_provider_prefix,
|
||||
estimate_tokens_rough,
|
||||
estimate_messages_tokens_rough,
|
||||
@@ -773,17 +774,24 @@ class TestGetModelContextLength:
|
||||
|
||||
@patch("agent.model_metadata.fetch_model_metadata")
|
||||
@patch("agent.model_metadata.fetch_endpoint_model_metadata")
|
||||
def test_custom_endpoint_without_metadata_skips_name_based_default(self, mock_endpoint_fetch, mock_fetch):
|
||||
def test_custom_endpoint_without_metadata_falls_back_to_catalog(self, mock_endpoint_fetch, mock_fetch):
|
||||
"""Custom endpoint with no metadata should fall back to the hardcoded
|
||||
catalog (not 256K) when the model name matches a known entry.
|
||||
|
||||
Previously this returned CONTEXT_PROBE_TIERS[0] (256K) because the
|
||||
custom-endpoint branch short-circuited before the catalog lookup.
|
||||
See #38865.
|
||||
"""
|
||||
mock_fetch.return_value = {}
|
||||
mock_endpoint_fetch.return_value = {}
|
||||
|
||||
# GLM-5-TEE matches the "glm" entry in DEFAULT_CONTEXT_LENGTHS
|
||||
result = get_model_context_length(
|
||||
"zai-org/GLM-5-TEE",
|
||||
base_url="https://llm.chutes.ai/v1",
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
assert result == CONTEXT_PROBE_TIERS[0]
|
||||
assert result == 202752 # "glm" entry in DEFAULT_CONTEXT_LENGTHS
|
||||
|
||||
@patch("agent.model_metadata.fetch_model_metadata")
|
||||
@patch("agent.model_metadata.fetch_endpoint_model_metadata")
|
||||
@@ -858,6 +866,64 @@ class TestGetModelContextLength:
|
||||
|
||||
assert result == 200000
|
||||
|
||||
@patch("agent.model_metadata.fetch_model_metadata")
|
||||
def test_custom_endpoint_falls_back_to_hardcoded_catalog(self, mock_fetch):
|
||||
"""Custom/proxied endpoint that fails all probes should still resolve
|
||||
via DEFAULT_CONTEXT_LENGTHS instead of returning 256K.
|
||||
|
||||
Regression test for #38865: a corporate Anthropic proxy (custom
|
||||
base_url) caused the custom-endpoint branch to short-circuit before
|
||||
the catalog lookup, capping context at 256K even for models like
|
||||
claude-opus-4-8 that are in the hardcoded catalog with 1M.
|
||||
"""
|
||||
mock_fetch.return_value = {}
|
||||
|
||||
# Patch all the probe functions that the custom-endpoint branch calls
|
||||
# so they all fail (return None/empty), simulating a proxy that
|
||||
# doesn't expose Ollama or local-server endpoints.
|
||||
with (
|
||||
patch(
|
||||
"agent.model_metadata._resolve_endpoint_context_length",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"agent.model_metadata._query_ollama_api_show",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"agent.model_metadata._query_local_context_length",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"agent.model_metadata.is_local_endpoint",
|
||||
return_value=False,
|
||||
),
|
||||
):
|
||||
# A known model behind a custom proxy should resolve to its
|
||||
# catalog value (1M), NOT the 256K fallback.
|
||||
ctx = get_model_context_length(
|
||||
"claude-opus-4-8",
|
||||
base_url="https://my-gateway.example.com/v1/claude",
|
||||
)
|
||||
assert ctx == 1000000, f"Expected 1000000, got {ctx}"
|
||||
|
||||
# Another known model
|
||||
ctx2 = get_model_context_length(
|
||||
"claude-sonnet-4-6",
|
||||
base_url="https://my-gateway.example.com/v1/claude",
|
||||
)
|
||||
assert ctx2 == 1000000, f"Expected 1000000, got {ctx2}"
|
||||
|
||||
# An unknown model on a custom endpoint should still fall back
|
||||
# to 256K (no catalog match).
|
||||
ctx3 = get_model_context_length(
|
||||
"totally-unknown-model",
|
||||
base_url="https://my-gateway.example.com/v1/claude",
|
||||
)
|
||||
assert ctx3 == DEFAULT_FALLBACK_CONTEXT, (
|
||||
f"Expected {DEFAULT_FALLBACK_CONTEXT}, got {ctx3}"
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Bedrock context resolution — must run BEFORE custom-endpoint probe
|
||||
|
||||
Reference in New Issue
Block a user