refactor(vision): consolidate native fast-path gate into one shared helper

The fast-path decision (native routing + provider allowlist OR
supports_vision override) lived inline in vision_analyze and was copied
into browser_vision. Extract it to _should_use_native_vision_fast_path()
so both tools share one source of truth.

- vision_tools: gate logic now one helper; vision_analyze calls it in 3 lines
- browser_tool: thin envelope decoration over the shared helper, not a copy
- browser_vision typed Union[str, Dict] to match its real return shape
- tests slimmed to target the override path + text-mode-wins invariant
This commit is contained in:
teknium1
2026-05-29 03:58:56 -07:00
committed by Teknium
parent c3f28c651d
commit c77a697fa4
4 changed files with 140 additions and 193 deletions
+36 -58
View File
@@ -62,7 +62,7 @@ import tempfile
import threading
import time
import requests
from typing import Dict, Any, Optional, List, Tuple
from typing import Dict, Any, Optional, List, Tuple, Union
from pathlib import Path
from agent.auxiliary_client import call_llm
from hermes_constants import get_hermes_home
@@ -3044,16 +3044,16 @@ def browser_get_images(task_id: Optional[str] = None) -> str:
return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False)
def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> str:
def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> Union[str, Dict[str, Any]]:
"""
Take a screenshot of the current page for visual inspection.
This tool captures what's visually displayed in the browser. When the
active model supports native vision, the screenshot is attached directly
to the conversation so the model can inspect it on the next turn.
Otherwise Hermes falls back to the auxiliary vision model. Useful for
understanding visual content that the text-based snapshot may not capture
(CAPTCHAs, verification challenges, images, complex layouts, etc.).
Captures what's visually displayed in the browser. When the active model
supports native vision, the screenshot is attached directly to the
conversation so the model can inspect it on the next turn; otherwise Hermes
falls back to the auxiliary vision model and returns a text analysis. Useful
for visual content the text-based snapshot may not capture (CAPTCHAs,
verification challenges, images, complex layouts, etc.).
The screenshot is saved persistently and its file path is returned so it
can be shared with users via MEDIA:<path> in the response.
@@ -3064,8 +3064,8 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
task_id: Task identifier for session isolation
Returns:
Either a JSON string with vision analysis results and screenshot_path,
or a multimodal tool-result envelope with the screenshot and metadata.
A JSON string with vision analysis results and screenshot_path, or a
multimodal tool-result envelope carrying the screenshot and metadata.
"""
if _is_camofox_mode():
from tools.browser_camofox import camofox_vision
@@ -3190,55 +3190,33 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
_screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii")
data_url = f"data:image/png;base64,{_screenshot_b64}"
# Fast path: when the active main model supports native vision AND the
# provider supports image content inside tool results, short-circuit
# the auxiliary LLM and return the image bytes as a multimodal
# tool-result envelope. The user can force native vision with the
# supports_vision override. The main model sees the pixels directly on its
# next turn — no aux call, no information loss, no extra latency.
try:
from agent.auxiliary_client import _read_main_model, _read_main_provider
from agent.image_routing import decide_image_input_mode, _lookup_supports_vision
from hermes_cli.config import load_config
from tools.vision_tools import (
_build_native_vision_tool_result,
_supports_media_in_tool_results,
)
# Fast path: when native image routing is in effect for the active main
# model, attach the screenshot directly instead of describing it through
# an auxiliary vision LLM. The model inspects the pixels on its next
# turn — no aux call, no information loss. Consistent with vision_analyze.
from tools.vision_tools import (
_build_native_vision_tool_result,
_should_use_native_vision_fast_path,
)
_provider = _read_main_provider()
_model = _read_main_model()
_cfg = load_config()
_mode = decide_image_input_mode(_provider, _model, _cfg)
_supports_vision = _lookup_supports_vision(_provider, _model, _cfg) is True
if _mode == "native" and (
_supports_media_in_tool_results(_provider, _model)
or _supports_vision
):
native_result = _build_native_vision_tool_result(
image_url=str(screenshot_path),
question=question,
image_data_url=data_url,
image_size_bytes=len(_screenshot_bytes),
)
native_result.setdefault("meta", {})
native_result["meta"]["screenshot_path"] = str(screenshot_path)
if _lp_fallback_warning:
native_result["meta"]["fallback_warning"] = _lp_fallback_warning
if annotate and result.get("data", {}).get("annotations"):
native_result["meta"]["annotations"] = result["data"]["annotations"]
text_parts = native_result.get("content") or []
if text_parts and isinstance(text_parts[0], dict) and text_parts[0].get("type") == "text":
text_parts[0]["text"] = (
str(text_parts[0].get("text", ""))
+ f"\n\nScreenshot path: {screenshot_path}"
)
native_result["text_summary"] = (
str(native_result.get("text_summary") or "")
+ f" Screenshot path: {screenshot_path}"
).strip()
return native_result
except Exception:
pass
if _should_use_native_vision_fast_path():
native_result = _build_native_vision_tool_result(
image_url=str(screenshot_path),
question=question,
image_data_url=data_url,
image_size_bytes=len(_screenshot_bytes),
)
meta = native_result.setdefault("meta", {})
meta["screenshot_path"] = str(screenshot_path)
if _lp_fallback_warning:
meta["fallback_warning"] = _lp_fallback_warning
if annotate and result.get("data", {}).get("annotations"):
meta["annotations"] = result["data"]["annotations"]
native_result["text_summary"] = (
f"{native_result.get('text_summary', '')} "
f"Screenshot path: {screenshot_path}"
).strip()
return native_result
vision_prompt = (
f"You are analyzing a screenshot of a web browser.\n\n"