feat(model): honor discover_models in terminal hermes model named-custom flow
The terminal `hermes model` wizard (_model_flow_named_custom) always live-probed a custom provider's /models endpoint, ignoring the configured `models:` list. For plans whose endpoint exposes a large catalog (e.g. Baidu Qianfan Coding Plan returns 100+ models for a 2-3 model plan) the picker flooded with models the user can't use. This wires `discover_models` (and the `models:` list) through _named_custom_provider_map into the flow and honors `discover_models: false` the same way the slash-command picker (model_switch.py sections 3 & 4) does: - Default stays True — live probe, no behaviour change. - discover_models: false → use the configured `models:` list verbatim, skip the probe (string 'false'/'no'/'0' normalised to False). - If the probe is on but returns empty, fall back to the configured list instead of forcing manual entry. Closes #18726
This commit is contained in:
@@ -563,3 +563,133 @@ class TestCustomProviderModelSwitch:
|
||||
# clobber it via _preserve_env_ref_templates).
|
||||
assert entry["api_key"] == "${HERMES_CRS_HENKEE_KEY}"
|
||||
assert "cr_live_secret_xyz" not in saved_text
|
||||
|
||||
|
||||
class TestCustomProviderDiscoverModels:
|
||||
"""#18726: honor ``discover_models: false`` in the terminal ``hermes model``
|
||||
named-custom flow so the picker shows the configured ``models:`` subset
|
||||
instead of the endpoint's full live catalog."""
|
||||
|
||||
def test_discover_false_uses_configured_list_and_skips_probe(self, config_home):
|
||||
"""discover_models: false + configured models → no live probe, the
|
||||
configured list is used verbatim."""
|
||||
from hermes_cli.main import _model_flow_named_custom
|
||||
|
||||
provider_info = {
|
||||
"name": "Baidu Coding",
|
||||
"base_url": "https://qianfan.baidubce.com/v2/coding",
|
||||
"api_key": "sk-test",
|
||||
"discover_models": False,
|
||||
"models": {"kimi-k2.5": {}, "glm-5": {}},
|
||||
"model": "kimi-k2.5",
|
||||
}
|
||||
|
||||
with patch("hermes_cli.models.fetch_api_models") as mock_fetch, \
|
||||
patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
|
||||
patch("builtins.input", return_value="2"), \
|
||||
patch("builtins.print"):
|
||||
_model_flow_named_custom({}, provider_info)
|
||||
|
||||
# The live /models endpoint must NOT be probed when discovery is off.
|
||||
mock_fetch.assert_not_called()
|
||||
|
||||
def test_discover_false_saves_choice_from_configured_list(self, config_home):
|
||||
"""User picks the 2nd configured model; it persists, list-driven."""
|
||||
import yaml
|
||||
from hermes_cli.main import _model_flow_named_custom
|
||||
|
||||
provider_info = {
|
||||
"name": "Baidu Coding",
|
||||
"base_url": "https://qianfan.baidubce.com/v2/coding",
|
||||
"api_key": "sk-test",
|
||||
"discover_models": False,
|
||||
"models": {"kimi-k2.5": {}, "glm-5": {}},
|
||||
"model": "kimi-k2.5",
|
||||
}
|
||||
|
||||
with patch("hermes_cli.models.fetch_api_models") as mock_fetch, \
|
||||
patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
|
||||
patch("builtins.input", return_value="2"), \
|
||||
patch("builtins.print"):
|
||||
_model_flow_named_custom({}, provider_info)
|
||||
|
||||
mock_fetch.assert_not_called()
|
||||
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
||||
model = config.get("model")
|
||||
assert isinstance(model, dict)
|
||||
assert model["default"] == "glm-5"
|
||||
|
||||
def test_default_still_probes_when_discover_unset(self, config_home):
|
||||
"""Default (discover_models unset → True) keeps live-probe behaviour
|
||||
even when a models: list is configured — Option B opt-out semantics."""
|
||||
from hermes_cli.main import _model_flow_named_custom
|
||||
|
||||
provider_info = {
|
||||
"name": "My Gateway",
|
||||
"base_url": "https://gw.example.com/v1",
|
||||
"api_key": "sk-test",
|
||||
"models": {"subset-a": {}}, # configured, but discovery NOT disabled
|
||||
"model": "subset-a",
|
||||
}
|
||||
|
||||
with patch(
|
||||
"hermes_cli.models.fetch_api_models",
|
||||
return_value=["live-a", "live-b", "live-c"],
|
||||
) as mock_fetch, \
|
||||
patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
|
||||
patch("builtins.input", return_value="1"), \
|
||||
patch("builtins.print"):
|
||||
_model_flow_named_custom({}, provider_info)
|
||||
|
||||
# Probe MUST still run — configured models: alone does not whitelist.
|
||||
mock_fetch.assert_called_once_with(
|
||||
"sk-test",
|
||||
"https://gw.example.com/v1",
|
||||
timeout=8.0,
|
||||
)
|
||||
|
||||
def test_probe_empty_falls_back_to_configured_list(self, config_home):
|
||||
"""When discovery is on but the probe returns nothing, fall back to the
|
||||
configured models: list instead of forcing manual entry."""
|
||||
import yaml
|
||||
from hermes_cli.main import _model_flow_named_custom
|
||||
|
||||
provider_info = {
|
||||
"name": "My Gateway",
|
||||
"base_url": "https://gw.example.com/v1",
|
||||
"api_key": "sk-test",
|
||||
"models": {"fallback-a": {}, "fallback-b": {}},
|
||||
"model": "fallback-a",
|
||||
}
|
||||
|
||||
with patch("hermes_cli.models.fetch_api_models", return_value=[]), \
|
||||
patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
|
||||
patch("builtins.input", return_value="2"), \
|
||||
patch("builtins.print"):
|
||||
_model_flow_named_custom({}, provider_info)
|
||||
|
||||
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
||||
model = config.get("model")
|
||||
assert isinstance(model, dict)
|
||||
assert model["default"] == "fallback-b"
|
||||
|
||||
def test_discover_false_string_is_normalised(self, config_home):
|
||||
"""String 'false' (hand-edited configs) disables discovery too."""
|
||||
from hermes_cli.main import _model_flow_named_custom
|
||||
|
||||
provider_info = {
|
||||
"name": "Baidu Coding",
|
||||
"base_url": "https://qianfan.baidubce.com/v2/coding",
|
||||
"api_key": "sk-test",
|
||||
"discover_models": "false",
|
||||
"models": {"kimi-k2.5": {}, "glm-5": {}},
|
||||
"model": "kimi-k2.5",
|
||||
}
|
||||
|
||||
with patch("hermes_cli.models.fetch_api_models") as mock_fetch, \
|
||||
patch("hermes_cli.curses_ui.curses_radiolist", side_effect=ImportError), \
|
||||
patch("builtins.input", return_value="1"), \
|
||||
patch("builtins.print"):
|
||||
_model_flow_named_custom({}, provider_info)
|
||||
|
||||
mock_fetch.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user