fix(tools): percent-encode non-ascii URL components

This commit is contained in:
helix4u
2026-06-07 11:42:26 -06:00
parent cb3e41e2fd
commit 333f01bc7f
5 changed files with 161 additions and 6 deletions
+13 -5
View File
@@ -102,7 +102,7 @@ from tools.tool_backend_helpers import ( # noqa: F401
nous_tool_gateway_unavailable_message,
prefers_gateway,
)
from tools.url_safety import async_is_safe_url
from tools.url_safety import async_is_safe_url, normalize_url_for_request
import sys
logger = logging.getLogger(__name__)
@@ -902,17 +902,25 @@ async def web_extract_tool(
# URL-decode first so percent-encoded secrets (%73k- = sk-) are caught.
from agent.redact import _PREFIX_RE
from urllib.parse import unquote
normalized_urls: List[str] = []
for _url in urls:
if _PREFIX_RE.search(_url) or _PREFIX_RE.search(unquote(_url)):
normalized_url = normalize_url_for_request(_url)
if (
_PREFIX_RE.search(_url)
or _PREFIX_RE.search(unquote(_url))
or _PREFIX_RE.search(normalized_url)
or _PREFIX_RE.search(unquote(normalized_url))
):
return json.dumps({
"success": False,
"error": "Blocked: URL contains what appears to be an API key or token. "
"Secrets must not be sent in URLs.",
})
normalized_urls.append(normalized_url)
debug_call_data = {
"parameters": {
"urls": urls,
"urls": normalized_urls,
"format": format,
"use_llm_processing": use_llm_processing,
"model": model,
@@ -928,12 +936,12 @@ async def web_extract_tool(
}
try:
logger.info("Extracting content from %d URL(s)", len(urls))
logger.info("Extracting content from %d URL(s)", len(normalized_urls))
# ── SSRF protection — filter out private/internal URLs before any backend ──
safe_urls = []
ssrf_blocked: List[Dict[str, Any]] = []
for url in urls:
for url in normalized_urls:
if not await async_is_safe_url(url):
ssrf_blocked.append({
"url": url, "title": "", "content": "",