Teknium
d848ea7109
When a gateway session hits a non-retryable error (e.g. invalid model ID → HTTP 400), the agent fails and returns. But if the session keeps receiving messages (or something periodically recreates agents), each attempt spawns a new AIAgent — reinitializing MCP server connections, burning CPU — only to hit the same 400 error again. On a 4-core server, this pegs an entire core per stuck session and accumulates 300+ minutes of CPU time over hours. Fix: add a per-session consecutive failure counter in the gateway runner. - Track consecutive non-retryable failures per session key - After 3 consecutive failures (_MAX_CONSECUTIVE_FAILURES), block further agent creation for that session and notify the user: '⚠️ This session has failed N times in a row with a non-retryable error. Use /reset to start a new session.' - Evict the cached agent when the circuit breaker engages to prevent stale state from accumulating - Reset the counter on successful agent runs - Clear the counter on /reset and /new so users can recover - Uses getattr() pattern so bare GatewayRunner instances (common in tests using object.__new__) don't crash Tests: - 8 new tests in test_circuit_breaker.py covering counter behavior, threshold, reset, session isolation, and bare-runner safety Addresses #7130.
98 lines
4.0 KiB
Python
98 lines
4.0 KiB
Python
"""Tests for the gateway consecutive-failure circuit breaker (#7130).
|
|
|
|
When a session hits N consecutive non-retryable failures (e.g. invalid
|
|
model ID → 400), the gateway stops recreating agents and tells the user
|
|
to fix their config. /reset clears the breaker.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
|
|
|
|
from gateway.run import GatewayRunner, _MAX_CONSECUTIVE_FAILURES
|
|
|
|
|
|
class TestCircuitBreaker:
|
|
"""Circuit breaker prevents CPU-burning restart loops on persistent errors."""
|
|
|
|
def _make_runner(self):
|
|
"""Create a minimal GatewayRunner without full __init__."""
|
|
runner = object.__new__(GatewayRunner)
|
|
runner._session_consecutive_failures = {}
|
|
runner._agent_cache = {}
|
|
runner._agent_cache_lock = MagicMock()
|
|
return runner
|
|
|
|
def test_failure_counter_increments(self):
|
|
runner = self._make_runner()
|
|
key = "test:session:1"
|
|
runner._session_consecutive_failures[key] = 0
|
|
runner._session_consecutive_failures[key] += 1
|
|
assert runner._session_consecutive_failures[key] == 1
|
|
|
|
def test_success_resets_counter(self):
|
|
runner = self._make_runner()
|
|
key = "test:session:1"
|
|
runner._session_consecutive_failures[key] = 2
|
|
# Simulate success: pop the key
|
|
runner._session_consecutive_failures.pop(key, None)
|
|
assert key not in runner._session_consecutive_failures
|
|
|
|
def test_max_consecutive_failures_is_reasonable(self):
|
|
"""The threshold should be low enough to stop loops quickly."""
|
|
assert 2 <= _MAX_CONSECUTIVE_FAILURES <= 10
|
|
|
|
def test_circuit_breaker_blocks_after_threshold(self):
|
|
"""After N failures, the circuit breaker should be tripped."""
|
|
runner = self._make_runner()
|
|
key = "test:session:1"
|
|
runner._session_consecutive_failures[key] = _MAX_CONSECUTIVE_FAILURES
|
|
count = runner._session_consecutive_failures.get(key, 0)
|
|
assert count >= _MAX_CONSECUTIVE_FAILURES
|
|
|
|
def test_reset_clears_circuit_breaker(self):
|
|
"""The /reset path clears the failure counter."""
|
|
runner = self._make_runner()
|
|
key = "test:session:1"
|
|
runner._session_consecutive_failures[key] = _MAX_CONSECUTIVE_FAILURES
|
|
|
|
# Simulate what the reset handler does
|
|
runner._session_consecutive_failures.pop(key, None)
|
|
assert key not in runner._session_consecutive_failures
|
|
|
|
def test_evict_cached_agent_on_circuit_break(self):
|
|
"""When circuit breaker engages, the cached agent should be evicted."""
|
|
runner = self._make_runner()
|
|
key = "test:session:1"
|
|
runner._agent_cache[key] = (MagicMock(), "sig")
|
|
runner._session_consecutive_failures[key] = _MAX_CONSECUTIVE_FAILURES
|
|
|
|
# Simulate eviction
|
|
runner._evict_cached_agent(key)
|
|
assert key not in runner._agent_cache
|
|
|
|
def test_different_sessions_track_independently(self):
|
|
"""Failures in session A should not affect session B."""
|
|
runner = self._make_runner()
|
|
runner._session_consecutive_failures["session:a"] = _MAX_CONSECUTIVE_FAILURES
|
|
runner._session_consecutive_failures["session:b"] = 1
|
|
|
|
assert runner._session_consecutive_failures["session:a"] >= _MAX_CONSECUTIVE_FAILURES
|
|
assert runner._session_consecutive_failures["session:b"] < _MAX_CONSECUTIVE_FAILURES
|
|
|
|
def test_getattr_pattern_safe_for_bare_runner(self):
|
|
"""The getattr pattern should not crash on bare runners without __init__."""
|
|
runner = object.__new__(GatewayRunner)
|
|
# No _session_consecutive_failures attribute set
|
|
failures = getattr(runner, "_session_consecutive_failures", None)
|
|
assert failures is None
|
|
# The circuit breaker check uses getattr().get() which would fail
|
|
# on None, but the code uses getattr(self, ..., {}).get() pattern
|
|
count = getattr(runner, "_session_consecutive_failures", {}).get("any_key", 0)
|
|
assert count == 0
|