fix(model): require confirmation for expensive model selections
Rebased onto current main and re-ported across the restructured surfaces: model flows now thread confirm_provider/base_url/api_key through hermes_cli/model_setup_flows.py, the Discord picker lives in plugins/platforms/discord/adapter.py, and the web dashboard picker applies chat-mode switches via config.set so the expensive-model confirmation can ride the response. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Robin Fernandes
committed by
Teknium
Teknium
co-authored by
Claude Fable 5
Claude Fable 5
parent
4eadef18a9
commit
af978ecb17
@@ -192,6 +192,32 @@ def test_custom_endpoint_models_api_pricing_is_supported(monkeypatch):
|
||||
assert float(entry.output_cost_per_million) == 2.0
|
||||
|
||||
|
||||
def test_nous_portal_pricing_preserves_vendor_prefixed_model_ids(monkeypatch):
|
||||
seen = {}
|
||||
|
||||
def _fake_fetch_endpoint_model_metadata(base_url, api_key=None):
|
||||
seen["base_url"] = base_url
|
||||
return {
|
||||
"openai/gpt-5.5-pro": {
|
||||
"pricing": {
|
||||
"prompt": "0.000025",
|
||||
"completion": "0.000125",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"agent.usage_pricing.fetch_endpoint_model_metadata",
|
||||
_fake_fetch_endpoint_model_metadata,
|
||||
)
|
||||
|
||||
entry = get_pricing_entry("openai/gpt-5.5-pro", provider="nous")
|
||||
|
||||
assert seen["base_url"] == "https://inference-api.nousresearch.com/v1"
|
||||
assert float(entry.input_cost_per_million) == 25.0
|
||||
assert float(entry.output_cost_per_million) == 125.0
|
||||
|
||||
|
||||
def test_deepseek_v4_pro_pricing_entry_exists():
|
||||
"""Regression test: deepseek-v4-pro must have a pricing entry.
|
||||
|
||||
|
||||
@@ -80,3 +80,91 @@ async def test_model_picker_clears_controls_before_running_switch_callback():
|
||||
interaction.response.edit_message.assert_awaited_once()
|
||||
interaction.response.defer.assert_not_called()
|
||||
interaction.edit_original_response.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expensive_model_requires_confirmation(monkeypatch):
|
||||
events: list[object] = []
|
||||
|
||||
async def on_model_selected(chat_id: str, model_id: str, provider_slug: str) -> str:
|
||||
events.append(("switch", chat_id, model_id, provider_slug))
|
||||
return "Model switched"
|
||||
|
||||
async def edit_message(**kwargs):
|
||||
events.append(
|
||||
(
|
||||
"edit",
|
||||
kwargs["embed"].title,
|
||||
kwargs["embed"].description,
|
||||
kwargs["view"],
|
||||
)
|
||||
)
|
||||
|
||||
async def edit_original_response(**kwargs):
|
||||
events.append((
|
||||
"final-edit",
|
||||
kwargs["embed"].title,
|
||||
kwargs["embed"].description,
|
||||
kwargs["view"],
|
||||
))
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.model_cost_guard.expensive_model_warning",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
message="!!! EXPENSIVE MODEL WARNING !!!\ndid you mean to select openai/gpt-5.5?"
|
||||
),
|
||||
)
|
||||
|
||||
view = ModelPickerView(
|
||||
providers=[
|
||||
{
|
||||
"slug": "openrouter",
|
||||
"name": "OpenRouter",
|
||||
"models": ["openai/gpt-5.5-pro"],
|
||||
"total_models": 1,
|
||||
"is_current": True,
|
||||
}
|
||||
],
|
||||
current_model="openai/gpt-5.5",
|
||||
current_provider="openrouter",
|
||||
session_key="session-1",
|
||||
on_model_selected=on_model_selected,
|
||||
allowed_user_ids={"123"}, # matches the interaction user; empty = fail-closed
|
||||
)
|
||||
view._selected_provider = "openrouter"
|
||||
|
||||
interaction = SimpleNamespace(
|
||||
user=SimpleNamespace(id=123),
|
||||
channel_id=456,
|
||||
data={"values": ["openai/gpt-5.5-pro"]},
|
||||
response=SimpleNamespace(
|
||||
send_message=AsyncMock(),
|
||||
edit_message=AsyncMock(side_effect=edit_message),
|
||||
),
|
||||
edit_original_response=AsyncMock(side_effect=edit_original_response),
|
||||
)
|
||||
|
||||
await view._on_model_selected(interaction)
|
||||
|
||||
assert events == [
|
||||
(
|
||||
"edit",
|
||||
"⚠ Expensive Model Warning",
|
||||
"!!! EXPENSIVE MODEL WARNING !!!\ndid you mean to select openai/gpt-5.5?",
|
||||
view,
|
||||
),
|
||||
]
|
||||
assert view.resolved is False
|
||||
|
||||
await view._on_expensive_confirm(interaction)
|
||||
|
||||
assert events[1:] == [
|
||||
(
|
||||
"edit",
|
||||
"⚙ Switching Model",
|
||||
"Switching to `openai/gpt-5.5-pro`...",
|
||||
None,
|
||||
),
|
||||
("switch", "456", "openai/gpt-5.5-pro", "openrouter"),
|
||||
("final-edit", "⚙ Model Switched", "Model switched", None),
|
||||
]
|
||||
|
||||
@@ -91,10 +91,6 @@ class TestTelegramModelPicker:
|
||||
query.answer = AsyncMock()
|
||||
query.edit_message_text = AsyncMock()
|
||||
|
||||
update = MagicMock()
|
||||
update.callback_query = query
|
||||
context = MagicMock()
|
||||
|
||||
await adapter._handle_model_picker_callback(query, "mb", "12345")
|
||||
|
||||
edit_kwargs = query.edit_message_text.call_args[1]
|
||||
@@ -133,17 +129,11 @@ class TestTelegramModelPicker:
|
||||
|
||||
await adapter._handle_model_picker_callback(query, "mm:0", "12345")
|
||||
|
||||
# The callback was invoked with the selected model
|
||||
callback.assert_awaited_once()
|
||||
# edit_message_text MUST be called on the success path (this is the
|
||||
# regression we're guarding).
|
||||
query.edit_message_text.assert_awaited()
|
||||
edit_kwargs = query.edit_message_text.call_args[1]
|
||||
assert "MARKDOWN_V2" in repr(edit_kwargs["parse_mode"])
|
||||
# The dynamic result text was routed through format_message
|
||||
# (backtick code blocks survive escaping).
|
||||
assert "`gpt-5`" in edit_kwargs["text"]
|
||||
# State is cleaned up after a successful switch.
|
||||
assert "12345" not in adapter._model_picker_state
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -184,7 +174,7 @@ class TestTelegramModelPicker:
|
||||
providers = [
|
||||
{"slug": "minimax", "name": "MiniMax", "total_models": 2},
|
||||
{"slug": "minimax-cn", "name": "MiniMax (China)", "total_models": 3},
|
||||
{"slug": "xai", "name": "xAI", "total_models": 1}, # lone group member
|
||||
{"slug": "xai", "name": "xAI", "total_models": 1},
|
||||
]
|
||||
|
||||
await adapter.send_model_picker(
|
||||
@@ -197,14 +187,11 @@ class TestTelegramModelPicker:
|
||||
metadata=None,
|
||||
)
|
||||
|
||||
# Top-level keyboard: MiniMax family folded into one group button;
|
||||
# xai (lone member) degraded to a direct provider button.
|
||||
assert "mpg:minimax" in built
|
||||
assert "mp:xai" in built
|
||||
assert "mp:minimax" not in built
|
||||
assert "mp:minimax-cn" not in built
|
||||
|
||||
# Drill into the MiniMax group → members appear as mp: buttons + back.
|
||||
built.clear()
|
||||
query = AsyncMock()
|
||||
query.message = MagicMock()
|
||||
@@ -216,7 +203,49 @@ class TestTelegramModelPicker:
|
||||
|
||||
assert "mp:minimax" in built
|
||||
assert "mp:minimax-cn" in built
|
||||
assert "mb" in built # back-to-providers button present
|
||||
assert "mb" in built
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expensive_model_requires_confirmation(self, monkeypatch):
|
||||
adapter = _make_adapter()
|
||||
callback = AsyncMock(return_value="Switched to `openai/gpt-5.5-pro`")
|
||||
adapter._model_picker_state["12345"] = {
|
||||
"providers": [
|
||||
{"slug": "openrouter", "name": "OpenRouter", "total_models": 1, "is_current": True}
|
||||
],
|
||||
"current_model": "model_1",
|
||||
"current_provider": "openrouter",
|
||||
"session_key": "s",
|
||||
"on_model_selected": callback,
|
||||
"selected_provider": "openrouter",
|
||||
"model_list": ["openai/gpt-5.5-pro"],
|
||||
"msg_id": 42,
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.model_cost_guard.expensive_model_warning",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
message="!!! EXPENSIVE MODEL WARNING !!!\ndid you mean to select openai/gpt-5.5?"
|
||||
),
|
||||
)
|
||||
|
||||
query = AsyncMock()
|
||||
query.message = MagicMock()
|
||||
query.message.chat_id = 12345
|
||||
query.answer = AsyncMock()
|
||||
query.edit_message_text = AsyncMock()
|
||||
|
||||
await adapter._handle_model_picker_callback(query, "mm:0", "12345")
|
||||
|
||||
callback.assert_not_awaited()
|
||||
assert "12345" in adapter._model_picker_state
|
||||
first_edit = query.edit_message_text.call_args[1]
|
||||
assert "EXPENSIVE MODEL WARNING" in first_edit["text"]
|
||||
assert first_edit["reply_markup"] is not None
|
||||
|
||||
await adapter._handle_model_picker_callback(query, "mc:0", "12345")
|
||||
|
||||
callback.assert_awaited_once_with("12345", "openai/gpt-5.5-pro", "openrouter")
|
||||
assert "12345" not in adapter._model_picker_state
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retries_without_thread_when_thread_not_found(self):
|
||||
|
||||
@@ -133,7 +133,7 @@ def test_model_command_uses_runtime_access_token_for_codex_list(monkeypatch):
|
||||
captured["access_token"] = access_token
|
||||
return ["gpt-5.2-codex", "gpt-5.2"]
|
||||
|
||||
def _fake_prompt_model_selection(model_ids, current_model=""):
|
||||
def _fake_prompt_model_selection(model_ids, current_model="", **_kwargs):
|
||||
captured["model_ids"] = list(model_ids)
|
||||
captured["current_model"] = current_model
|
||||
return None
|
||||
@@ -181,7 +181,7 @@ def test_model_command_prompts_to_reuse_or_reauthenticate_codex_session(monkeypa
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth._prompt_model_selection",
|
||||
lambda model_ids, current_model="": None,
|
||||
lambda model_ids, current_model="", **_kwargs: None,
|
||||
)
|
||||
|
||||
_model_flow_openai_codex({}, current_model="gpt-5.4")
|
||||
@@ -219,7 +219,7 @@ def test_model_command_uses_existing_codex_session_without_relogin(monkeypatch):
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth._prompt_model_selection",
|
||||
lambda model_ids, current_model="": None,
|
||||
lambda model_ids, current_model="", **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth._login_openai_codex",
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
from decimal import Decimal
|
||||
|
||||
from agent.models_dev import ModelInfo
|
||||
from agent.usage_pricing import PricingEntry
|
||||
from hermes_cli.model_cost_guard import expensive_model_warning
|
||||
|
||||
|
||||
def test_no_warning_when_known_prices_are_at_threshold():
|
||||
info = ModelInfo(
|
||||
id="edge/model",
|
||||
name="edge/model",
|
||||
family="",
|
||||
provider_id="test",
|
||||
cost_input=20.0,
|
||||
cost_output=100.0,
|
||||
)
|
||||
|
||||
assert expensive_model_warning("edge/model", provider="test", model_info=info) is None
|
||||
|
||||
|
||||
def test_warns_when_models_dev_input_price_exceeds_threshold():
|
||||
info = ModelInfo(
|
||||
id="expensive/input",
|
||||
name="expensive/input",
|
||||
family="",
|
||||
provider_id="test",
|
||||
cost_input=20.01,
|
||||
cost_output=1.0,
|
||||
)
|
||||
|
||||
warning = expensive_model_warning(
|
||||
"expensive/input",
|
||||
provider="test",
|
||||
model_info=info,
|
||||
)
|
||||
|
||||
assert warning is not None
|
||||
assert warning.input_cost_per_million == Decimal("20.01")
|
||||
assert "EXPENSIVE MODEL WARNING" in warning.message
|
||||
assert "$20/M input" in warning.message
|
||||
|
||||
|
||||
def test_warns_when_pricing_entry_output_price_exceeds_threshold(monkeypatch):
|
||||
monkeypatch.setattr("agent.models_dev.get_model_info", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(
|
||||
"agent.usage_pricing.get_pricing_entry",
|
||||
lambda *_args, **_kwargs: PricingEntry(
|
||||
input_cost_per_million=Decimal("1.00"),
|
||||
output_cost_per_million=Decimal("100.01"),
|
||||
source="provider_models_api",
|
||||
),
|
||||
)
|
||||
|
||||
warning = expensive_model_warning("provider/expensive-output", provider="openrouter")
|
||||
|
||||
assert warning is not None
|
||||
assert warning.output_cost_per_million == Decimal("100.01")
|
||||
assert "$100.01/M" in warning.message
|
||||
|
||||
|
||||
def test_openai_gpt55_pro_adds_suggestion(monkeypatch):
|
||||
monkeypatch.setattr("agent.models_dev.get_model_info", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(
|
||||
"agent.usage_pricing.get_pricing_entry",
|
||||
lambda *_args, **_kwargs: PricingEntry(
|
||||
input_cost_per_million=Decimal("25"),
|
||||
output_cost_per_million=Decimal("125"),
|
||||
source="provider_models_api",
|
||||
),
|
||||
)
|
||||
|
||||
warning = expensive_model_warning("openai/gpt-5.5-pro", provider="openrouter")
|
||||
|
||||
assert warning is not None
|
||||
assert "did you mean to select openai/gpt-5.5?" in warning.message
|
||||
|
||||
|
||||
def test_openai_gpt55_pro_warns_for_nous_portal_pricing(monkeypatch):
|
||||
monkeypatch.setattr("agent.models_dev.get_model_info", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(
|
||||
"agent.usage_pricing.fetch_endpoint_model_metadata",
|
||||
lambda base_url, api_key="": {
|
||||
"openai/gpt-5.5-pro": {
|
||||
"pricing": {
|
||||
"prompt": "0.000025",
|
||||
"completion": "0.000125",
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
warning = expensive_model_warning("openai/gpt-5.5-pro", provider="nous")
|
||||
|
||||
assert warning is not None
|
||||
assert warning.input_cost_per_million == Decimal("25.000000")
|
||||
assert warning.output_cost_per_million == Decimal("125.000000")
|
||||
assert "did you mean to select openai/gpt-5.5?" in warning.message
|
||||
@@ -0,0 +1,64 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from hermes_cli.model_switch import ModelSwitchResult
|
||||
|
||||
|
||||
def _bound(fn, instance):
|
||||
return fn.__get__(instance, type(instance))
|
||||
|
||||
|
||||
def test_prompt_toolkit_model_picker_defers_confirmation_off_key_handler(monkeypatch):
|
||||
import cli as cli_mod
|
||||
|
||||
result = ModelSwitchResult(
|
||||
success=True,
|
||||
new_model="openai/gpt-5.5-pro",
|
||||
target_provider="nous",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.model_switch.switch_model",
|
||||
lambda **_kwargs: result,
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
class _Thread:
|
||||
def __init__(self, *, target, args, daemon):
|
||||
captured["target"] = target
|
||||
captured["args"] = args
|
||||
captured["daemon"] = daemon
|
||||
|
||||
def start(self):
|
||||
captured["started"] = True
|
||||
|
||||
monkeypatch.setattr(cli_mod.threading, "Thread", _Thread)
|
||||
|
||||
self_ = SimpleNamespace(
|
||||
_app=object(),
|
||||
_model_picker_state={
|
||||
"stage": "model",
|
||||
"provider_data": {"slug": "nous"},
|
||||
"model_list": ["openai/gpt-5.5-pro"],
|
||||
"selected": 0,
|
||||
"user_provs": None,
|
||||
"custom_provs": None,
|
||||
},
|
||||
provider="nous",
|
||||
model="openai/gpt-5.5",
|
||||
base_url="",
|
||||
api_key="",
|
||||
_restore_modal_input_snapshot=lambda: None,
|
||||
_invalidate=lambda **_kwargs: None,
|
||||
)
|
||||
self_._close_model_picker = _bound(cli_mod.HermesCLI._close_model_picker, self_)
|
||||
self_._confirm_and_apply_model_switch_result = (
|
||||
lambda *_args: captured.setdefault("ran_inline", True)
|
||||
)
|
||||
|
||||
_bound(cli_mod.HermesCLI._handle_model_picker_selection, self_)()
|
||||
|
||||
assert self_._model_picker_state is None
|
||||
assert captured["started"] is True
|
||||
assert captured["daemon"] is True
|
||||
assert captured["args"] == (result, False)
|
||||
assert "ran_inline" not in captured
|
||||
@@ -2,6 +2,7 @@
|
||||
cannot initialize (e.g. non-TTY, curses unavailable, terminal error)."""
|
||||
|
||||
import subprocess
|
||||
from types import SimpleNamespace
|
||||
|
||||
from hermes_cli.config import load_config, save_config
|
||||
|
||||
@@ -24,6 +25,46 @@ def test_prompt_model_selection_falls_back_on_menu_runtime_error(monkeypatch):
|
||||
assert selected == "model-b"
|
||||
|
||||
|
||||
def test_prompt_model_selection_requires_expensive_confirmation(monkeypatch, capsys):
|
||||
from hermes_cli.auth import _prompt_model_selection
|
||||
|
||||
monkeypatch.setattr("hermes_cli.curses_ui.curses_radiolist", _raise_menu)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.model_cost_guard.expensive_model_warning",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(message="EXPENSIVE MODEL WARNING"),
|
||||
)
|
||||
responses = iter(["1", "n"])
|
||||
monkeypatch.setattr("builtins.input", lambda _prompt="": next(responses))
|
||||
|
||||
selected = _prompt_model_selection(
|
||||
["openai/gpt-5.5-pro"],
|
||||
confirm_provider="nous",
|
||||
)
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert selected is None
|
||||
assert "EXPENSIVE MODEL WARNING" in out
|
||||
|
||||
|
||||
def test_prompt_model_selection_allows_confirmed_expensive_model(monkeypatch):
|
||||
from hermes_cli.auth import _prompt_model_selection
|
||||
|
||||
monkeypatch.setattr("hermes_cli.curses_ui.curses_radiolist", _raise_menu)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.model_cost_guard.expensive_model_warning",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(message="EXPENSIVE MODEL WARNING"),
|
||||
)
|
||||
responses = iter(["1", "y"])
|
||||
monkeypatch.setattr("builtins.input", lambda _prompt="": next(responses))
|
||||
|
||||
selected = _prompt_model_selection(
|
||||
["openai/gpt-5.5-pro"],
|
||||
confirm_provider="nous",
|
||||
)
|
||||
|
||||
assert selected == "openai/gpt-5.5-pro"
|
||||
|
||||
|
||||
def test_prompt_reasoning_effort_falls_back_on_menu_runtime_error(monkeypatch):
|
||||
from hermes_cli.main import _prompt_reasoning_effort_selection
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import os
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -1069,6 +1070,41 @@ class TestWebServerEndpoints:
|
||||
assert "GATEWAY_PROXY_URL" not in managed
|
||||
assert "GATEWAY_PROXY_URL" in _MESSAGING_KEYS_PAGE_KEYS
|
||||
|
||||
def test_model_set_requires_confirmation_for_expensive_model(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.model_cost_guard.expensive_model_warning",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(message="EXPENSIVE MODEL WARNING"),
|
||||
)
|
||||
|
||||
resp = self.client.post(
|
||||
"/api/model/set",
|
||||
json={
|
||||
"scope": "main",
|
||||
"provider": "nous",
|
||||
"model": "openai/gpt-5.5-pro",
|
||||
},
|
||||
)
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["ok"] is False
|
||||
assert data["confirm_required"] is True
|
||||
assert data["confirm_message"] == "EXPENSIVE MODEL WARNING"
|
||||
|
||||
confirmed = self.client.post(
|
||||
"/api/model/set",
|
||||
json={
|
||||
"scope": "main",
|
||||
"provider": "nous",
|
||||
"model": "openai/gpt-5.5-pro",
|
||||
"confirm_expensive_model": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert confirmed.status_code == 200
|
||||
assert confirmed.json()["ok"] is True
|
||||
|
||||
|
||||
def test_reveal_env_var(self, tmp_path):
|
||||
"""POST /api/env/reveal should return the real unredacted value."""
|
||||
from hermes_cli.config import save_env_value
|
||||
|
||||
@@ -2397,7 +2397,7 @@ def test_config_set_model_waits_for_lazy_agent_before_switch(monkeypatch):
|
||||
target["agent"] = agent
|
||||
agent_ready.set()
|
||||
|
||||
def fake_apply(sid, target, raw):
|
||||
def fake_apply(sid, target, raw, **kwargs):
|
||||
calls.append(("apply", sid, target.get("agent"), raw))
|
||||
if target.get("agent") is not agent:
|
||||
raise AssertionError("model switch ran before lazy agent was ready")
|
||||
@@ -2424,7 +2424,7 @@ def test_config_set_model_uses_live_switch_path(monkeypatch):
|
||||
server._sessions["sid"] = _session()
|
||||
seen = {}
|
||||
|
||||
def _fake_apply(sid, session, raw):
|
||||
def _fake_apply(sid, session, raw, **_kwargs):
|
||||
seen["args"] = (sid, session["session_key"], raw)
|
||||
return {"value": "new/model", "warning": "catalog unreachable"}
|
||||
|
||||
@@ -2442,6 +2442,74 @@ def test_config_set_model_uses_live_switch_path(monkeypatch):
|
||||
assert seen["args"] == ("sid", "session-key", "new/model")
|
||||
|
||||
|
||||
def test_config_set_model_requires_confirmation_for_expensive_model(monkeypatch):
|
||||
class _Agent:
|
||||
provider = "openrouter"
|
||||
model = "old/model"
|
||||
base_url = ""
|
||||
api_key = "sk-or"
|
||||
switched = False
|
||||
|
||||
def switch_model(self, **_kwargs):
|
||||
self.switched = True
|
||||
|
||||
result = types.SimpleNamespace(
|
||||
success=True,
|
||||
new_model="openai/gpt-5.5-pro",
|
||||
target_provider="openrouter",
|
||||
api_key="sk-or",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
api_mode="chat_completions",
|
||||
warning_message="",
|
||||
model_info=types.SimpleNamespace(
|
||||
has_cost_data=lambda: True,
|
||||
cost_input=25.0,
|
||||
cost_output=125.0,
|
||||
),
|
||||
)
|
||||
|
||||
agent = _Agent()
|
||||
server._sessions["sid"] = _session(agent=agent)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.model_switch.switch_model", lambda **_kwargs: result
|
||||
)
|
||||
monkeypatch.setattr(server, "_restart_slash_worker", lambda sid, session: None)
|
||||
monkeypatch.setattr(server, "_emit", lambda *args, **kwargs: None)
|
||||
|
||||
resp = server.handle_request(
|
||||
{
|
||||
"id": "1",
|
||||
"method": "config.set",
|
||||
"params": {
|
||||
"session_id": "sid",
|
||||
"key": "model",
|
||||
"value": "openai/gpt-5.5-pro --provider openrouter",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert resp["result"]["confirm_required"] is True
|
||||
assert "did you mean to select openai/gpt-5.5?" in resp["result"]["confirm_message"]
|
||||
assert agent.switched is False
|
||||
|
||||
confirmed = server.handle_request(
|
||||
{
|
||||
"id": "2",
|
||||
"method": "config.set",
|
||||
"params": {
|
||||
"session_id": "sid",
|
||||
"key": "model",
|
||||
"value": "openai/gpt-5.5-pro --provider openrouter",
|
||||
"confirm_expensive_model": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert confirmed["result"]["confirm_required"] is False
|
||||
assert confirmed["result"]["value"] == "openai/gpt-5.5-pro"
|
||||
assert agent.switched is True
|
||||
|
||||
|
||||
def test_config_set_model_global_persists(monkeypatch):
|
||||
class _Agent:
|
||||
provider = "openrouter"
|
||||
@@ -3944,7 +4012,7 @@ def test_config_set_model_rejects_while_running(monkeypatch):
|
||||
"""/model via config.set must reject during an in-flight turn."""
|
||||
seen = {"called": False}
|
||||
|
||||
def _fake_apply(sid, session, raw):
|
||||
def _fake_apply(sid, session, raw, **_kwargs):
|
||||
seen["called"] = True
|
||||
return {"value": raw, "warning": ""}
|
||||
|
||||
@@ -3978,7 +4046,7 @@ def test_config_set_model_allowed_when_idle(monkeypatch):
|
||||
"""Regression guard: idle sessions can still switch models."""
|
||||
seen = {"called": False}
|
||||
|
||||
def _fake_apply(sid, session, raw):
|
||||
def _fake_apply(sid, session, raw, **_kwargs):
|
||||
seen["called"] = True
|
||||
return {"value": "newmodel", "warning": ""}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user