refactor(vision): consolidate native fast-path gate into one shared helper
The fast-path decision (native routing + provider allowlist OR supports_vision override) lived inline in vision_analyze and was copied into browser_vision. Extract it to _should_use_native_vision_fast_path() so both tools share one source of truth. - vision_tools: gate logic now one helper; vision_analyze calls it in 3 lines - browser_tool: thin envelope decoration over the shared helper, not a copy - browser_vision typed Union[str, Dict] to match its real return shape - tests slimmed to target the override path + text-mode-wins invariant
This commit is contained in:
+36
-58
@@ -62,7 +62,7 @@ import tempfile
|
||||
import threading
|
||||
import time
|
||||
import requests
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
from typing import Dict, Any, Optional, List, Tuple, Union
|
||||
from pathlib import Path
|
||||
from agent.auxiliary_client import call_llm
|
||||
from hermes_constants import get_hermes_home
|
||||
@@ -3044,16 +3044,16 @@ def browser_get_images(task_id: Optional[str] = None) -> str:
|
||||
return json.dumps(_copy_fallback_warning(response, result), ensure_ascii=False)
|
||||
|
||||
|
||||
def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> str:
|
||||
def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> Union[str, Dict[str, Any]]:
|
||||
"""
|
||||
Take a screenshot of the current page for visual inspection.
|
||||
|
||||
This tool captures what's visually displayed in the browser. When the
|
||||
active model supports native vision, the screenshot is attached directly
|
||||
to the conversation so the model can inspect it on the next turn.
|
||||
Otherwise Hermes falls back to the auxiliary vision model. Useful for
|
||||
understanding visual content that the text-based snapshot may not capture
|
||||
(CAPTCHAs, verification challenges, images, complex layouts, etc.).
|
||||
Captures what's visually displayed in the browser. When the active model
|
||||
supports native vision, the screenshot is attached directly to the
|
||||
conversation so the model can inspect it on the next turn; otherwise Hermes
|
||||
falls back to the auxiliary vision model and returns a text analysis. Useful
|
||||
for visual content the text-based snapshot may not capture (CAPTCHAs,
|
||||
verification challenges, images, complex layouts, etc.).
|
||||
|
||||
The screenshot is saved persistently and its file path is returned so it
|
||||
can be shared with users via MEDIA:<path> in the response.
|
||||
@@ -3064,8 +3064,8 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
|
||||
task_id: Task identifier for session isolation
|
||||
|
||||
Returns:
|
||||
Either a JSON string with vision analysis results and screenshot_path,
|
||||
or a multimodal tool-result envelope with the screenshot and metadata.
|
||||
A JSON string with vision analysis results and screenshot_path, or a
|
||||
multimodal tool-result envelope carrying the screenshot and metadata.
|
||||
"""
|
||||
if _is_camofox_mode():
|
||||
from tools.browser_camofox import camofox_vision
|
||||
@@ -3190,55 +3190,33 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
|
||||
_screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii")
|
||||
data_url = f"data:image/png;base64,{_screenshot_b64}"
|
||||
|
||||
# Fast path: when the active main model supports native vision AND the
|
||||
# provider supports image content inside tool results, short-circuit
|
||||
# the auxiliary LLM and return the image bytes as a multimodal
|
||||
# tool-result envelope. The user can force native vision with the
|
||||
# supports_vision override. The main model sees the pixels directly on its
|
||||
# next turn — no aux call, no information loss, no extra latency.
|
||||
try:
|
||||
from agent.auxiliary_client import _read_main_model, _read_main_provider
|
||||
from agent.image_routing import decide_image_input_mode, _lookup_supports_vision
|
||||
from hermes_cli.config import load_config
|
||||
from tools.vision_tools import (
|
||||
_build_native_vision_tool_result,
|
||||
_supports_media_in_tool_results,
|
||||
)
|
||||
# Fast path: when native image routing is in effect for the active main
|
||||
# model, attach the screenshot directly instead of describing it through
|
||||
# an auxiliary vision LLM. The model inspects the pixels on its next
|
||||
# turn — no aux call, no information loss. Consistent with vision_analyze.
|
||||
from tools.vision_tools import (
|
||||
_build_native_vision_tool_result,
|
||||
_should_use_native_vision_fast_path,
|
||||
)
|
||||
|
||||
_provider = _read_main_provider()
|
||||
_model = _read_main_model()
|
||||
_cfg = load_config()
|
||||
_mode = decide_image_input_mode(_provider, _model, _cfg)
|
||||
_supports_vision = _lookup_supports_vision(_provider, _model, _cfg) is True
|
||||
if _mode == "native" and (
|
||||
_supports_media_in_tool_results(_provider, _model)
|
||||
or _supports_vision
|
||||
):
|
||||
native_result = _build_native_vision_tool_result(
|
||||
image_url=str(screenshot_path),
|
||||
question=question,
|
||||
image_data_url=data_url,
|
||||
image_size_bytes=len(_screenshot_bytes),
|
||||
)
|
||||
native_result.setdefault("meta", {})
|
||||
native_result["meta"]["screenshot_path"] = str(screenshot_path)
|
||||
if _lp_fallback_warning:
|
||||
native_result["meta"]["fallback_warning"] = _lp_fallback_warning
|
||||
if annotate and result.get("data", {}).get("annotations"):
|
||||
native_result["meta"]["annotations"] = result["data"]["annotations"]
|
||||
text_parts = native_result.get("content") or []
|
||||
if text_parts and isinstance(text_parts[0], dict) and text_parts[0].get("type") == "text":
|
||||
text_parts[0]["text"] = (
|
||||
str(text_parts[0].get("text", ""))
|
||||
+ f"\n\nScreenshot path: {screenshot_path}"
|
||||
)
|
||||
native_result["text_summary"] = (
|
||||
str(native_result.get("text_summary") or "")
|
||||
+ f" Screenshot path: {screenshot_path}"
|
||||
).strip()
|
||||
return native_result
|
||||
except Exception:
|
||||
pass
|
||||
if _should_use_native_vision_fast_path():
|
||||
native_result = _build_native_vision_tool_result(
|
||||
image_url=str(screenshot_path),
|
||||
question=question,
|
||||
image_data_url=data_url,
|
||||
image_size_bytes=len(_screenshot_bytes),
|
||||
)
|
||||
meta = native_result.setdefault("meta", {})
|
||||
meta["screenshot_path"] = str(screenshot_path)
|
||||
if _lp_fallback_warning:
|
||||
meta["fallback_warning"] = _lp_fallback_warning
|
||||
if annotate and result.get("data", {}).get("annotations"):
|
||||
meta["annotations"] = result["data"]["annotations"]
|
||||
native_result["text_summary"] = (
|
||||
f"{native_result.get('text_summary', '')} "
|
||||
f"Screenshot path: {screenshot_path}"
|
||||
).strip()
|
||||
return native_result
|
||||
|
||||
vision_prompt = (
|
||||
f"You are analyzing a screenshot of a web browser.\n\n"
|
||||
|
||||
Reference in New Issue
Block a user