Merge pull request #41430 from helix4u/fix-url-tools-unicode-normalization

fix(tools): percent-encode non-ascii URL components

tests/tools/test_browser_secret_exfil.py

@@ -42,6 +42,25 @@ class TestBrowserSecretExfil:
         # Should NOT be blocked by secret detection
         assert "API key or token" not in parsed.get("error", "")
 
+    def test_normalizes_non_ascii_url_before_navigation(self):
+        from tools.browser_tool import browser_navigate
+
+        captured = {}
+
+        def mock_run(_session_key, command, args, **_kwargs):
+            if command == "open":
+                captured["url"] = args[0]
+            return {"success": True, "data": {"title": "ok", "url": args[0]}}
+
+        with patch("tools.browser_tool._run_browser_command", side_effect=mock_run), \
+             patch("tools.browser_tool._get_session_info", return_value={"_first_nav": False}), \
+             patch("tools.browser_tool._is_local_backend", return_value=True):
+            result = browser_navigate("https://wttr.in/Köln")
+
+        parsed = json.loads(result)
+        assert parsed["success"] is True
+        assert captured["url"] == "https://wttr.in/K%C3%B6ln"
+
 
 class TestWebExtractSecretExfil:
     """Verify web_extract_tool blocks URLs containing secrets."""
@@ -65,6 +84,56 @@ class TestWebExtractSecretExfil:
         # Should fail for API/config reason, not secret blocking
         assert "API key" not in parsed.get("error", "") or "Blocked" not in parsed.get("error", "")
 
+    @pytest.mark.asyncio
+    async def test_normalizes_non_ascii_url_before_extract_provider(self, monkeypatch):
+        from agent.web_search_provider import WebSearchProvider
+        from agent import web_search_registry
+        from tools import web_tools
+
+        class FakeExtractProvider(WebSearchProvider):
+            @property
+            def name(self) -> str:
+                return "fake-extract"
+
+            def is_available(self) -> bool:
+                return True
+
+            def supports_search(self) -> bool:
+                return False
+
+            def supports_extract(self) -> bool:
+                return True
+
+            def extract(self, urls, **_kwargs):
+                return [
+                    {
+                        "url": urls[0],
+                        "title": "ok",
+                        "content": "ok",
+                        "raw_content": "ok",
+                    }
+                ]
+
+        async def allow_url(_url: str) -> bool:
+            return True
+
+        web_search_registry._reset_for_tests()
+        web_search_registry.register_provider(FakeExtractProvider())
+        monkeypatch.setattr(web_tools, "_ensure_web_plugins_loaded", lambda: None)
+        monkeypatch.setattr(web_tools, "_get_extract_backend", lambda: "fake-extract")
+        monkeypatch.setattr(web_tools, "async_is_safe_url", allow_url)
+
+        try:
+            result = await web_tools.web_extract_tool(
+                urls=["https://wttr.in/Köln"],
+                use_llm_processing=False,
+            )
+        finally:
+            web_search_registry._reset_for_tests()
+
+        parsed = json.loads(result)
+        assert parsed["results"][0]["url"] == "https://wttr.in/K%C3%B6ln"
+
 
 class TestBrowserSnapshotRedaction:
     """Verify secrets in page snapshots are redacted before auxiliary LLM calls."""

tests/tools/test_url_safety.py

@@ -7,6 +7,7 @@ from tools.url_safety import (
     is_safe_url,
     async_is_safe_url,
     is_always_blocked_url,
+    normalize_url_for_request,
     _is_blocked_ip,
     _global_allow_private_urls,
     _reset_allow_private_cache,
@@ -16,6 +17,32 @@ import ipaddress
 import pytest
 
 
+class TestNormalizeUrlForRequest:
+    def test_percent_encodes_non_ascii_path(self):
+        assert (
+            normalize_url_for_request("https://wttr.in/Köln")
+            == "https://wttr.in/K%C3%B6ln"
+        )
+
+    def test_preserves_existing_percent_escapes(self):
+        assert (
+            normalize_url_for_request("https://wttr.in/K%C3%B6ln")
+            == "https://wttr.in/K%C3%B6ln"
+        )
+
+    def test_preserves_reserved_query_syntax(self):
+        assert (
+            normalize_url_for_request("https://example.com/search?q=Köln&lang=de")
+            == "https://example.com/search?q=K%C3%B6ln&lang=de"
+        )
+
+    def test_idna_encodes_hostname(self):
+        assert (
+            normalize_url_for_request("https://münich.example/Köln")
+            == "https://xn--mnich-kva.example/K%C3%B6ln"
+        )
+
+
 class TestIsSafeUrl:
     def test_public_url_allowed(self):
         with patch("socket.getaddrinfo", return_value=[