fix(auxiliary): retry transient transport error once before fallback (#16587)

A one-off transient transport failure (streaming-close / incomplete
chunked read / 5xx / 408) on an auxiliary LLM call escalated straight to
provider/model fallback (or, for context compression, dropped the summary
and entered cooldown), even when an immediate retry on the same provider
would have succeeded.

Add a single same-target retry at the top of call_llm() and
async_call_llm() — before the existing except-chain — gated on a new
_is_transient_transport_error() that reuses the canonical
_is_connection_error() detector plus a 5xx/408 status check. A second
failure (or any non-transient error: auth, other 4xx, malformed payload)
falls through to first_err and the existing fallback handling unchanged.

This lives in call_llm so every auxiliary task (compression, memory flush,
title generation, session search, vision) shares one transient-retry
surface, rather than each caller re-implementing it. The context
compressor needs no change — it calls call_llm and inherits the retry; its
existing fallback-to-main path (#18458) now composes naturally (retry the
aux model once, then fall back to main only if the retry also fails).

Co-authored-by: ARegalado1 <alberto.regalado@ymail.com>
This commit is contained in:
teknium1
2026-06-08 01:05:45 -07:00
committed by Teknium
co-authored by ARegalado1
parent 4107076128
commit 02a4d66951
3 changed files with 160 additions and 4 deletions
+57 -4
View File
@@ -2476,6 +2476,25 @@ def _is_connection_error(exc: Exception) -> bool:
return False
def _is_transient_transport_error(exc: Exception) -> bool:
"""Return True for a one-off transport blip worth retrying ONCE on the
same provider before any provider/model fallback.
Covers connection/streaming-close errors (via the canonical
``_is_connection_error`` detector, shared so the two cannot drift) plus a
pure 5xx/408 HTTP status. Deliberately narrow: this is the "retry the
same target once" gate, distinct from ``_is_payment_error`` /
``_is_auth_error`` / ``_is_rate_limit_error`` which the except-chain
handles by switching provider, refreshing creds, or rotating the pool.
"""
if _is_connection_error(exc):
return True
status = getattr(exc, "status_code", None) or getattr(
getattr(exc, "response", None), "status_code", None
)
return isinstance(status, int) and (status == 408 or 500 <= status < 600)
def _is_auth_error(exc: Exception) -> bool:
"""Detect auth failures that should trigger provider-specific refresh."""
status = getattr(exc, "status_code", None)
@@ -5147,8 +5166,28 @@ def call_llm(
# Handle unsupported temperature, max_tokens vs max_completion_tokens retry,
# then payment fallback.
try:
return _validate_llm_response(
client.chat.completions.create(**kwargs), task)
# Retry ONCE on the same provider for a one-off transient transport
# blip (streaming-close / incomplete chunked read / 5xx / 408) before
# the except-chain below escalates to provider/model fallback. A
# single dropped connection shouldn't abandon an otherwise-healthy
# provider. A second failure (or any non-transient error) falls
# through to ``first_err`` and the existing fallback handling
# unchanged. This is the unified home for the transient retry that
# every auxiliary task (compression, memory flush, title-gen,
# session-search, vision) shares. (PR #16587)
try:
return _validate_llm_response(
client.chat.completions.create(**kwargs), task)
except Exception as transient_err:
if not _is_transient_transport_error(transient_err):
raise
logger.info(
"Auxiliary %s: transient transport error; retrying once on "
"the same provider before fallback: %s",
task or "call", transient_err,
)
return _validate_llm_response(
client.chat.completions.create(**kwargs), task)
except Exception as first_err:
if "temperature" in kwargs and _is_unsupported_temperature_error(first_err):
retry_kwargs = dict(kwargs)
@@ -5614,8 +5653,22 @@ async def async_call_llm(
kwargs["messages"] = _convert_openai_images_to_anthropic(kwargs["messages"])
try:
return _validate_llm_response(
await client.chat.completions.create(**kwargs), task)
# Retry ONCE on the same provider for a transient transport blip
# before the except-chain escalates to fallback — see call_llm()
# for the rationale. (PR #16587)
try:
return _validate_llm_response(
await client.chat.completions.create(**kwargs), task)
except Exception as transient_err:
if not _is_transient_transport_error(transient_err):
raise
logger.info(
"Auxiliary %s (async): transient transport error; retrying "
"once on the same provider before fallback: %s",
task or "call", transient_err,
)
return _validate_llm_response(
await client.chat.completions.create(**kwargs), task)
except Exception as first_err:
if "temperature" in kwargs and _is_unsupported_temperature_error(first_err):
retry_kwargs = dict(kwargs)