test: extend test coverage to native image routing

2026-05-29 03:58:56 -07:00
parent f8b8dffccf
commit 2402ec5e7b
2 changed files with 201 additions and 27 deletions
@@ -250,6 +250,131 @@ class TestBrowserVisionConfig:
        assert mock_llm.call_args.kwargs["temperature"] == 0.1
        assert mock_llm.call_args.kwargs["timeout"] == 120.0

+    def test_browser_vision_native_fast_path_returns_multimodal(self, tmp_path):
+        from agent.auxiliary_client import clear_runtime_main, set_runtime_main
+        from tools.browser_tool import browser_vision
+
+        shots_dir, screenshot = self._setup_screenshot(tmp_path)
+        annotations = [{"id": 1, "label": "Search box"}]
+        set_runtime_main("brand-new-provider", "llava-v1.6")
+        try:
+            with (
+                patch("hermes_constants.get_hermes_dir", return_value=shots_dir),
+                patch("tools.browser_tool._cleanup_old_screenshots"),
+                patch(
+                    "tools.browser_tool._run_browser_command",
+                    return_value={
+                        "success": True,
+                        "data": {
+                            "path": str(screenshot),
+                            "annotations": annotations,
+                        },
+                    },
+                ),
+                patch(
+                    "hermes_cli.config.load_config",
+                    return_value={"model": {"supports_vision": True}},
+                ),
+                patch("tools.browser_tool._get_vision_model") as mock_get_vision_model,
+                patch("tools.browser_tool.call_llm") as mock_llm,
+            ):
+                result = browser_vision(
+                    "what is on the page?", annotate=True, task_id="test"
+                )
+        finally:
+            clear_runtime_main()
+
+        assert isinstance(result, dict)
+        assert result["_multimodal"] is True
+        assert result["meta"]["screenshot_path"] == str(screenshot)
+        assert result["meta"]["annotations"] == annotations
+        assert any(p.get("type") == "image_url" for p in result["content"])
+        assert "what is on the page?" in result["content"][0]["text"]
+        assert str(screenshot) in result["content"][0]["text"]
+        assert "Screenshot path:" in result["text_summary"]
+        mock_get_vision_model.assert_not_called()
+        mock_llm.assert_not_called()
+
+    def test_browser_vision_native_mode_without_supports_vision_uses_aux_llm(self, tmp_path):
+        from agent.auxiliary_client import clear_runtime_main, set_runtime_main
+        from tools.browser_tool import browser_vision
+
+        shots_dir, screenshot = self._setup_screenshot(tmp_path)
+        mock_response = MagicMock()
+        mock_choice = MagicMock()
+        mock_choice.message.content = "Fallback screenshot analysis"
+        mock_response.choices = [mock_choice]
+
+        set_runtime_main("brand-new-provider", "opaque-model")
+        try:
+            with (
+                patch("hermes_constants.get_hermes_dir", return_value=shots_dir),
+                patch("tools.browser_tool._cleanup_old_screenshots"),
+                patch(
+                    "tools.browser_tool._run_browser_command",
+                    return_value={"success": True, "data": {"path": str(screenshot)}},
+                ),
+                patch(
+                    "hermes_cli.config.load_config",
+                    return_value={"agent": {"image_input_mode": "native"}},
+                ),
+                patch("tools.browser_tool._get_vision_model", return_value="test-model"),
+                patch("tools.browser_tool.call_llm", return_value=mock_response) as mock_llm,
+            ):
+                result = json.loads(browser_vision("what is on the page?", task_id="test"))
+        finally:
+            clear_runtime_main()
+
+        assert result["success"] is True
+        assert result["analysis"] == "Fallback screenshot analysis"
+        assert result["screenshot_path"] == str(screenshot)
+        mock_llm.assert_called_once()
+        kwargs = mock_llm.call_args.kwargs
+        assert kwargs["task"] == "vision"
+        assert kwargs["model"] == "test-model"
+        assert kwargs["messages"][0]["content"][1]["type"] == "image_url"
+        assert kwargs["messages"][0]["content"][1]["image_url"]["url"].startswith(
+            "data:image/png;base64,"
+        )
+
+    def test_browser_vision_text_mode_blocks_native_fast_path(self, tmp_path):
+        from agent.auxiliary_client import clear_runtime_main, set_runtime_main
+        from tools.browser_tool import browser_vision
+
+        shots_dir, screenshot = self._setup_screenshot(tmp_path)
+        mock_response = MagicMock()
+        mock_choice = MagicMock()
+        mock_choice.message.content = "Text-mode screenshot analysis"
+        mock_response.choices = [mock_choice]
+
+        set_runtime_main("brand-new-provider", "llava-v1.6")
+        try:
+            with (
+                patch("hermes_constants.get_hermes_dir", return_value=shots_dir),
+                patch("tools.browser_tool._cleanup_old_screenshots"),
+                patch(
+                    "tools.browser_tool._run_browser_command",
+                    return_value={"success": True, "data": {"path": str(screenshot)}},
+                ),
+                patch(
+                    "hermes_cli.config.load_config",
+                    return_value={
+                        "agent": {"image_input_mode": "text"},
+                        "model": {"supports_vision": True},
+                    },
+                ),
+                patch("tools.browser_tool._get_vision_model", return_value="test-model"),
+                patch("tools.browser_tool.call_llm", return_value=mock_response) as mock_llm,
+            ):
+                result = json.loads(browser_vision("what is on the page?", task_id="test"))
+        finally:
+            clear_runtime_main()
+
+        assert result["success"] is True
+        assert result["analysis"] == "Text-mode screenshot analysis"
+        assert result["screenshot_path"] == str(screenshot)
+        mock_llm.assert_called_once()
+

 # ── auto-recording config ────────────────────────────────────────────