Merge branch 'main' of github.com:NousResearch/hermes-agent into bb/gui
This commit is contained in:
+62
-6
@@ -1482,6 +1482,34 @@ def _run_browser_command(
|
||||
if "AGENT_BROWSER_IDLE_TIMEOUT_MS" not in browser_env:
|
||||
idle_ms = str(BROWSER_SESSION_INACTIVITY_TIMEOUT * 1000)
|
||||
browser_env["AGENT_BROWSER_IDLE_TIMEOUT_MS"] = idle_ms
|
||||
|
||||
# Inject --no-sandbox when needed (issue #15765):
|
||||
# - Running as root: Chromium always refuses to start without it
|
||||
# - Ubuntu 23.10+ / AppArmor systems: unprivileged user namespaces
|
||||
# are restricted, causing Chromium to exit with "No usable sandbox"
|
||||
# even for non-root users running under systemd or containers.
|
||||
if "AGENT_BROWSER_CHROME_FLAGS" not in browser_env:
|
||||
_needs_sandbox_bypass = False
|
||||
if hasattr(os, "geteuid") and os.geteuid() == 0:
|
||||
_needs_sandbox_bypass = True
|
||||
logger.debug("browser: running as root — injecting --no-sandbox")
|
||||
else:
|
||||
# Detect AppArmor user namespace restrictions (Ubuntu 23.10+)
|
||||
_userns_restrict = "/proc/sys/kernel/apparmor_restrict_unprivileged_userns"
|
||||
try:
|
||||
with open(_userns_restrict) as _f:
|
||||
if _f.read().strip() == "1":
|
||||
_needs_sandbox_bypass = True
|
||||
logger.debug(
|
||||
"browser: AppArmor userns restrictions detected — "
|
||||
"injecting --no-sandbox"
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
if _needs_sandbox_bypass:
|
||||
browser_env["AGENT_BROWSER_CHROME_FLAGS"] = (
|
||||
"--no-sandbox --disable-dev-shm-usage"
|
||||
)
|
||||
|
||||
# Use temp files for stdout/stderr instead of pipes.
|
||||
# agent-browser starts a background daemon that inherits file
|
||||
@@ -2757,17 +2785,40 @@ def _chromium_search_roots() -> List[str]:
|
||||
def _chromium_installed() -> bool:
|
||||
"""Return True when a usable Chromium (or headless-shell) build is on disk.
|
||||
|
||||
Checks, in order:
|
||||
|
||||
1. ``AGENT_BROWSER_EXECUTABLE_PATH`` env var — the official way to point
|
||||
agent-browser at a pre-installed Chrome/Chromium.
|
||||
2. System Chrome/Chromium in PATH (``google-chrome``, ``chromium-browser``,
|
||||
``chrome``).
|
||||
3. Playwright's browser cache (current logic) — directories containing
|
||||
``chromium-*`` or ``chromium_headless_shell-*``.
|
||||
|
||||
agent-browser (0.26+) downloads Playwright's chromium / headless-shell
|
||||
builds into ``PLAYWRIGHT_BROWSERS_PATH`` and won't start without them.
|
||||
When the CLI is present but no browser build is, the first browser tool
|
||||
call hangs for the full command timeout (often ~30s each) before
|
||||
surfacing a useless error. Guarding the tool behind this check prevents
|
||||
advertising a capability that will fail at runtime.
|
||||
builds into ``PLAYWRIGHT_BROWSERS_PATH`` and won't start without at least
|
||||
one of the three above being present. Without a browser binary the CLI
|
||||
hangs on first use until the command timeout fires (often ~30s). Guarding
|
||||
the tool behind this check prevents advertising a capability that will
|
||||
fail at runtime.
|
||||
"""
|
||||
global _cached_chromium_installed
|
||||
if _cached_chromium_installed is not None:
|
||||
return _cached_chromium_installed
|
||||
|
||||
# 1. AGENT_BROWSER_EXECUTABLE_PATH — explicit user-configured browser
|
||||
ab_path = os.environ.get("AGENT_BROWSER_EXECUTABLE_PATH", "").strip()
|
||||
if ab_path:
|
||||
if os.path.isfile(ab_path) or shutil.which(ab_path):
|
||||
_cached_chromium_installed = True
|
||||
return True
|
||||
|
||||
# 2. System Chrome/Chromium in PATH (common names)
|
||||
system_chrome = shutil.which("google-chrome") or shutil.which("chromium-browser") or shutil.which("chrome")
|
||||
if system_chrome:
|
||||
_cached_chromium_installed = True
|
||||
return True
|
||||
|
||||
# 3. Playwright browser cache (legacy — chromium-* / chromium_headless_shell-* dirs)
|
||||
for root in _chromium_search_roots():
|
||||
if not root or not os.path.isdir(root):
|
||||
continue
|
||||
@@ -2817,7 +2868,12 @@ def check_browser_requirements() -> bool:
|
||||
if _is_camofox_mode():
|
||||
return True
|
||||
|
||||
# The agent-browser CLI is always required
|
||||
# CDP override mode can connect to an existing remote/local browser endpoint
|
||||
# without requiring the local agent-browser binary on PATH.
|
||||
if _get_cdp_override():
|
||||
return True
|
||||
|
||||
# The agent-browser CLI is required for local launch and cloud-provider flows.
|
||||
try:
|
||||
browser_cmd = _find_agent_browser()
|
||||
except FileNotFoundError:
|
||||
|
||||
+10
-1
@@ -128,6 +128,15 @@ def _resolve_model_override(model_obj: Optional[Dict[str, Any]]) -> tuple:
|
||||
return (None, None)
|
||||
model_name = (model_obj.get("model") or "").strip() or None
|
||||
provider_name = (model_obj.get("provider") or "").strip() or None
|
||||
# Bare "custom" is an incomplete spec — the canonical form is
|
||||
# "custom:<name>" matching a custom_providers entry. LLMs frequently
|
||||
# supply the bare type because the schema does not advertise the
|
||||
# ":<name>" suffix, which used to bypass the pinning path below and
|
||||
# leave the job stored with an unresolvable "custom" provider. Treat
|
||||
# the bare value as "no provider supplied" so the current main
|
||||
# provider gets pinned instead.
|
||||
if provider_name == "custom":
|
||||
provider_name = None
|
||||
if model_name and not provider_name:
|
||||
# Pin to the current main provider so the job is stable
|
||||
try:
|
||||
@@ -513,7 +522,7 @@ Important safety rule: cron-run sessions should not recursively schedule more cr
|
||||
"properties": {
|
||||
"provider": {
|
||||
"type": "string",
|
||||
"description": "Provider name (e.g. 'openrouter', 'anthropic'). Omit to use and pin the current provider."
|
||||
"description": "Provider name (e.g. 'openrouter', 'anthropic', or 'custom:<name>' for a provider defined in custom_providers config — always include the ':<name>' suffix, never pass the bare 'custom'). Omit to use and pin the current provider."
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
|
||||
+51
-20
@@ -483,8 +483,8 @@ _HEARTBEAT_INTERVAL = 30 # seconds between parent activity heartbeats during de
|
||||
# The idle ceiling stays tight so genuinely stuck children don't mask the gateway
|
||||
# timeout. The in-tool ceiling is much higher so legit long-running tools get
|
||||
# time to finish; child_timeout_seconds (default 600s) is still the hard cap.
|
||||
_HEARTBEAT_STALE_CYCLES_IDLE = 5 # 5 * 30s = 150s idle between turns → stale
|
||||
_HEARTBEAT_STALE_CYCLES_IN_TOOL = 20 # 20 * 30s = 600s stuck on same tool → stale
|
||||
_HEARTBEAT_STALE_CYCLES_IDLE = 15 # 15 * 30s = 450s idle between turns → stale
|
||||
_HEARTBEAT_STALE_CYCLES_IN_TOOL = 40 # 40 * 30s = 1200s stuck on same tool → stale
|
||||
DEFAULT_TOOLSETS = ["terminal", "file", "web"]
|
||||
|
||||
|
||||
@@ -1026,6 +1026,29 @@ def _build_child_agent(
|
||||
except Exception as exc:
|
||||
logger.debug("Could not load delegation reasoning_effort: %s", exc)
|
||||
|
||||
# Inherit the parent's fallback provider chain so subagents can recover
|
||||
# from rate-limits and credential exhaustion exactly like the top-level
|
||||
# agent does. _fallback_chain is a list accepted by AIAgent's
|
||||
# fallback_model parameter (which handles both list and dict forms).
|
||||
parent_fallback = getattr(parent_agent, "_fallback_chain", None) or None
|
||||
|
||||
# Inherit the parent's OpenRouter provider-preference filters by default
|
||||
# (so subagents routed to the same provider honour the same routing
|
||||
# constraints). BUT: when `delegation.provider` is set the user is
|
||||
# explicitly asking the child to run on a different provider, and
|
||||
# parent-level OpenRouter filters (e.g. `only=["Anthropic"]`) would
|
||||
# silently force the child back onto the parent's provider. Clear the
|
||||
# filters in that case so the delegated provider is honoured.
|
||||
child_providers_allowed = getattr(parent_agent, "providers_allowed", None)
|
||||
child_providers_ignored = getattr(parent_agent, "providers_ignored", None)
|
||||
child_providers_order = getattr(parent_agent, "providers_order", None)
|
||||
child_provider_sort = getattr(parent_agent, "provider_sort", None)
|
||||
if override_provider:
|
||||
child_providers_allowed = None
|
||||
child_providers_ignored = None
|
||||
child_providers_order = None
|
||||
child_provider_sort = None
|
||||
|
||||
child = AIAgent(
|
||||
base_url=effective_base_url,
|
||||
api_key=effective_api_key,
|
||||
@@ -1038,6 +1061,7 @@ def _build_child_agent(
|
||||
max_tokens=getattr(parent_agent, "max_tokens", None),
|
||||
reasoning_config=child_reasoning,
|
||||
prefill_messages=getattr(parent_agent, "prefill_messages", None),
|
||||
fallback_model=parent_fallback,
|
||||
enabled_toolsets=child_toolsets,
|
||||
quiet_mode=True,
|
||||
ephemeral_system_prompt=child_prompt,
|
||||
@@ -1049,10 +1073,10 @@ def _build_child_agent(
|
||||
thinking_callback=child_thinking_cb,
|
||||
session_db=getattr(parent_agent, "_session_db", None),
|
||||
parent_session_id=getattr(parent_agent, "session_id", None),
|
||||
providers_allowed=parent_agent.providers_allowed,
|
||||
providers_ignored=parent_agent.providers_ignored,
|
||||
providers_order=parent_agent.providers_order,
|
||||
provider_sort=parent_agent.provider_sort,
|
||||
providers_allowed=child_providers_allowed,
|
||||
providers_ignored=child_providers_ignored,
|
||||
providers_order=child_providers_order,
|
||||
provider_sort=child_provider_sort,
|
||||
tool_progress_callback=child_progress_cb,
|
||||
iteration_budget=None, # fresh budget per subagent
|
||||
)
|
||||
@@ -2230,11 +2254,17 @@ def _resolve_delegation_credentials(cfg: dict, parent_agent) -> dict:
|
||||
"""Resolve credentials for subagent delegation.
|
||||
|
||||
If ``delegation.base_url`` is configured, subagents use that direct
|
||||
OpenAI-compatible endpoint. Otherwise, if ``delegation.provider`` is
|
||||
configured, the full credential bundle (base_url, api_key, api_mode,
|
||||
provider) is resolved via the runtime provider system — the same path used
|
||||
by CLI/gateway startup. This lets subagents run on a completely different
|
||||
provider:model pair.
|
||||
OpenAI-compatible endpoint. ``delegation.api_key`` overrides the key; when
|
||||
omitted, ``api_key`` is returned as ``None`` so ``_build_child_agent``
|
||||
inherits the parent agent's key (``effective_api_key = override_api_key or
|
||||
parent_api_key``). This lets providers that store their key outside
|
||||
``OPENAI_API_KEY`` (e.g. ``MINIMAX_API_KEY``, ``DASHSCOPE_API_KEY``) work
|
||||
without a duplicate config entry.
|
||||
|
||||
Otherwise, if ``delegation.provider`` is configured, the full credential
|
||||
bundle (base_url, api_key, api_mode, provider) is resolved via the runtime
|
||||
provider system — the same path used by CLI/gateway startup. This lets
|
||||
subagents run on a completely different provider:model pair.
|
||||
|
||||
If neither base_url nor provider is configured, returns None values so the
|
||||
child inherits everything from the parent agent.
|
||||
@@ -2247,12 +2277,13 @@ def _resolve_delegation_credentials(cfg: dict, parent_agent) -> dict:
|
||||
configured_api_key = str(cfg.get("api_key") or "").strip() or None
|
||||
|
||||
if configured_base_url:
|
||||
api_key = configured_api_key or os.getenv("OPENAI_API_KEY", "").strip()
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
"Delegation base_url is configured but no API key was found. "
|
||||
"Set delegation.api_key or OPENAI_API_KEY."
|
||||
)
|
||||
# When delegation.api_key is not set, return None so _build_child_agent
|
||||
# falls back to the parent agent's API key via the credential inheritance
|
||||
# path (effective_api_key = override_api_key or parent_api_key). This
|
||||
# lets providers that store their key in a non-OPENAI_API_KEY env var
|
||||
# (e.g. MINIMAX_API_KEY, DASHSCOPE_API_KEY) work without requiring
|
||||
# callers to duplicate the key under delegation.api_key.
|
||||
api_key = configured_api_key # None → inherited from parent in _build_child_agent
|
||||
|
||||
base_lower = configured_base_url.lower()
|
||||
provider = "custom"
|
||||
@@ -2292,7 +2323,7 @@ def _resolve_delegation_credentials(cfg: dict, parent_agent) -> dict:
|
||||
try:
|
||||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||||
|
||||
runtime = resolve_runtime_provider(requested=configured_provider)
|
||||
runtime = resolve_runtime_provider(requested=configured_provider, target_model=configured_model)
|
||||
except Exception as exc:
|
||||
raise ValueError(
|
||||
f"Cannot resolve delegation provider '{configured_provider}': {exc}. "
|
||||
@@ -2330,7 +2361,7 @@ def _load_config() -> dict:
|
||||
try:
|
||||
from cli import CLI_CONFIG
|
||||
|
||||
cfg = CLI_CONFIG.get("delegation", {})
|
||||
cfg = CLI_CONFIG.get("delegation") or {}
|
||||
if cfg:
|
||||
return cfg
|
||||
except Exception:
|
||||
@@ -2339,7 +2370,7 @@ def _load_config() -> dict:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
full = load_config()
|
||||
return full.get("delegation", {})
|
||||
return full.get("delegation") or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
@@ -405,7 +405,8 @@ class BaseEnvironment(ABC):
|
||||
# Preserve bare ``~`` expansion, but rewrite ``~/...`` through
|
||||
# ``$HOME`` so suffixes with spaces remain a single shell word.
|
||||
quoted_cwd = self._quote_cwd_for_cd(cwd)
|
||||
parts.append(f"builtin cd {quoted_cwd} || exit 126")
|
||||
# ``--`` keeps hyphen-prefixed directory names from being parsed as options.
|
||||
parts.append(f"builtin cd -- {quoted_cwd} || exit 126")
|
||||
|
||||
# Run the actual command
|
||||
parts.append(f"eval '{escaped}'")
|
||||
|
||||
@@ -53,6 +53,27 @@ WRITE_DENIED_PATHS = build_write_denied_paths(_HOME)
|
||||
WRITE_DENIED_PREFIXES = build_write_denied_prefixes(_HOME)
|
||||
|
||||
|
||||
_OSC_SEQUENCE_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")
|
||||
_FENCE_MARKER_RE = re.compile(r"'?\x07?__HERMES_FENCE_[A-Za-z0-9]+__\x07?'?")
|
||||
|
||||
|
||||
def _strip_terminal_fence_leaks(text: str) -> str:
|
||||
"""Strip leaked terminal fence wrappers from file read output."""
|
||||
if not text:
|
||||
return text
|
||||
|
||||
cleaned_lines: List[str] = []
|
||||
for line in text.splitlines(keepends=True):
|
||||
had_terminal_wrapper = "__HERMES_FENCE_" in line or "\x1b]" in line
|
||||
cleaned = _OSC_SEQUENCE_RE.sub("", line)
|
||||
cleaned = _FENCE_MARKER_RE.sub("", cleaned)
|
||||
cleaned = cleaned.replace("\x07", "")
|
||||
if had_terminal_wrapper and cleaned.strip("'\r\n\t ") == "":
|
||||
continue
|
||||
cleaned_lines.append(cleaned)
|
||||
return "".join(cleaned_lines)
|
||||
|
||||
|
||||
def _get_safe_write_root() -> Optional[str]:
|
||||
"""Return the resolved HERMES_WRITE_SAFE_ROOT path, or None if unset.
|
||||
|
||||
@@ -511,8 +532,9 @@ class ShellFileOperations(FileOperations):
|
||||
# File not found - try to suggest similar files
|
||||
return self._suggest_similar_files(path)
|
||||
|
||||
stat_output = _strip_terminal_fence_leaks(stat_result.stdout)
|
||||
try:
|
||||
file_size = int(stat_result.stdout.strip())
|
||||
file_size = int(stat_output.strip())
|
||||
except ValueError:
|
||||
file_size = 0
|
||||
|
||||
@@ -536,8 +558,9 @@ class ShellFileOperations(FileOperations):
|
||||
# Read a sample to check for binary content
|
||||
sample_cmd = f"head -c 1000 {self._escape_shell_arg(path)} 2>/dev/null"
|
||||
sample_result = self._exec(sample_cmd)
|
||||
sample_output = _strip_terminal_fence_leaks(sample_result.stdout)
|
||||
|
||||
if self._is_likely_binary(path, sample_result.stdout):
|
||||
if self._is_likely_binary(path, sample_output):
|
||||
return ReadResult(
|
||||
is_binary=True,
|
||||
file_size=file_size,
|
||||
@@ -551,12 +574,14 @@ class ShellFileOperations(FileOperations):
|
||||
|
||||
if read_result.exit_code != 0:
|
||||
return ReadResult(error=f"Failed to read file: {read_result.stdout}")
|
||||
read_output = _strip_terminal_fence_leaks(read_result.stdout)
|
||||
|
||||
# Get total line count
|
||||
wc_cmd = f"wc -l < {self._escape_shell_arg(path)}"
|
||||
wc_result = self._exec(wc_cmd)
|
||||
wc_output = _strip_terminal_fence_leaks(wc_result.stdout)
|
||||
try:
|
||||
total_lines = int(wc_result.stdout.strip())
|
||||
total_lines = int(wc_output.strip())
|
||||
except ValueError:
|
||||
total_lines = 0
|
||||
|
||||
@@ -567,7 +592,7 @@ class ShellFileOperations(FileOperations):
|
||||
hint = f"Use offset={end_line + 1} to continue reading (showing {offset}-{end_line} of {total_lines} lines)"
|
||||
|
||||
return ReadResult(
|
||||
content=self._add_line_numbers(read_result.stdout, offset),
|
||||
content=self._add_line_numbers(read_output, offset),
|
||||
total_lines=total_lines,
|
||||
file_size=file_size,
|
||||
truncated=truncated,
|
||||
@@ -637,14 +662,16 @@ class ShellFileOperations(FileOperations):
|
||||
stat_result = self._exec(stat_cmd)
|
||||
if stat_result.exit_code != 0:
|
||||
return self._suggest_similar_files(path)
|
||||
stat_output = _strip_terminal_fence_leaks(stat_result.stdout)
|
||||
try:
|
||||
file_size = int(stat_result.stdout.strip())
|
||||
file_size = int(stat_output.strip())
|
||||
except ValueError:
|
||||
file_size = 0
|
||||
if self._is_image(path):
|
||||
return ReadResult(is_image=True, is_binary=True, file_size=file_size)
|
||||
sample_result = self._exec(f"head -c 1000 {self._escape_shell_arg(path)} 2>/dev/null")
|
||||
if self._is_likely_binary(path, sample_result.stdout):
|
||||
sample_output = _strip_terminal_fence_leaks(sample_result.stdout)
|
||||
if self._is_likely_binary(path, sample_output):
|
||||
return ReadResult(
|
||||
is_binary=True, file_size=file_size,
|
||||
error="Binary file — cannot display as text."
|
||||
@@ -652,7 +679,10 @@ class ShellFileOperations(FileOperations):
|
||||
cat_result = self._exec(f"cat {self._escape_shell_arg(path)}")
|
||||
if cat_result.exit_code != 0:
|
||||
return ReadResult(error=f"Failed to read file: {cat_result.stdout}")
|
||||
return ReadResult(content=cat_result.stdout, file_size=file_size)
|
||||
return ReadResult(
|
||||
content=_strip_terminal_fence_leaks(cat_result.stdout),
|
||||
file_size=file_size,
|
||||
)
|
||||
|
||||
def delete_file(self, path: str) -> WriteResult:
|
||||
"""Delete a file via rm."""
|
||||
|
||||
+3
-3
@@ -570,7 +570,7 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str =
|
||||
|
||||
# ── Redact secrets (after guard check to skip oversized content) ──
|
||||
if result.content:
|
||||
result.content = redact_sensitive_text(result.content)
|
||||
result.content = redact_sensitive_text(result.content, code_file=True)
|
||||
result_dict["content"] = result.content
|
||||
|
||||
# Large-file hint: if the file is big and the caller didn't ask
|
||||
@@ -993,7 +993,7 @@ def search_tool(pattern: str, target: str = "content", path: str = ".",
|
||||
if hasattr(result, 'matches'):
|
||||
for m in result.matches:
|
||||
if hasattr(m, 'content') and m.content:
|
||||
m.content = redact_sensitive_text(m.content)
|
||||
m.content = redact_sensitive_text(m.content, code_file=True)
|
||||
result_dict = result.to_dict()
|
||||
|
||||
if count >= 3:
|
||||
@@ -1137,7 +1137,7 @@ def _handle_search_files(args, **kw):
|
||||
output_mode=args.get("output_mode", "content"), context=args.get("context", 0), task_id=tid)
|
||||
|
||||
|
||||
registry.register(name="read_file", toolset="file", schema=READ_FILE_SCHEMA, handler=_handle_read_file, check_fn=_check_file_reqs, emoji="📖", max_result_size_chars=float('inf'))
|
||||
registry.register(name="read_file", toolset="file", schema=READ_FILE_SCHEMA, handler=_handle_read_file, check_fn=_check_file_reqs, emoji="📖", max_result_size_chars=100_000)
|
||||
registry.register(name="write_file", toolset="file", schema=WRITE_FILE_SCHEMA, handler=_handle_write_file, check_fn=_check_file_reqs, emoji="✍️", max_result_size_chars=100_000)
|
||||
registry.register(name="patch", toolset="file", schema=PATCH_SCHEMA, handler=_handle_patch, check_fn=_check_file_reqs, emoji="🔧", max_result_size_chars=100_000)
|
||||
registry.register(name="search_files", toolset="file", schema=SEARCH_FILES_SCHEMA, handler=_handle_search_files, check_fn=_check_file_reqs, emoji="🔎", max_result_size_chars=100_000)
|
||||
|
||||
+64
-5
@@ -40,13 +40,31 @@ logger = logging.getLogger(__name__)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _check_kanban_mode() -> bool:
|
||||
"""Tools are available iff the current process has ``HERMES_KANBAN_TASK``
|
||||
set in its env, which the dispatcher sets when spawning a worker.
|
||||
"""Tools are available when:
|
||||
|
||||
Humans running ``hermes chat`` see zero kanban tools. Workers spawned
|
||||
by the kanban dispatcher (gateway-embedded by default) see all seven.
|
||||
1. ``HERMES_KANBAN_TASK`` is set (dispatcher-spawned worker), OR
|
||||
2. The current profile has ``kanban`` in its toolsets config
|
||||
(orchestrator profiles like techlead that route work via Kanban).
|
||||
|
||||
Humans running ``hermes chat`` without the kanban toolset see zero
|
||||
kanban tools. Workers spawned by the kanban dispatcher (gateway-
|
||||
embedded by default) and orchestrator profiles with the kanban
|
||||
toolset enabled see all seven.
|
||||
"""
|
||||
return bool(os.environ.get("HERMES_KANBAN_TASK"))
|
||||
if os.environ.get("HERMES_KANBAN_TASK"):
|
||||
return True
|
||||
|
||||
# Check if the current profile has the kanban toolset enabled.
|
||||
# Uses load_config() which has mtime-based caching, so this adds
|
||||
# negligible overhead. The check_fn results are further TTL-cached
|
||||
# (~30s) by the tool registry.
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
cfg = load_config()
|
||||
toolsets = cfg.get("toolsets", [])
|
||||
return "kanban" in toolsets
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -61,6 +79,38 @@ def _default_task_id(arg: Optional[str]) -> Optional[str]:
|
||||
return env_tid or None
|
||||
|
||||
|
||||
def _enforce_worker_task_ownership(tid: str) -> Optional[str]:
|
||||
"""Reject worker-driven destructive calls on foreign task IDs.
|
||||
|
||||
A process spawned by the dispatcher has ``HERMES_KANBAN_TASK`` set
|
||||
to its own task id. Tools like ``kanban_complete`` / ``kanban_block``
|
||||
/ ``kanban_heartbeat`` mutate run-lifecycle state, so a buggy or
|
||||
prompt-injected worker that passed an explicit ``task_id`` for some
|
||||
other task could corrupt sibling or cross-tenant runs (see #19534).
|
||||
|
||||
Orchestrator profiles (kanban toolset enabled but **no**
|
||||
``HERMES_KANBAN_TASK`` in env) aren't subject to this check — their
|
||||
job is routing, and they sometimes legitimately close out child
|
||||
tasks or reopen blocked ones. Workers are narrowly scoped to their
|
||||
one task.
|
||||
|
||||
Returns ``None`` when the call is allowed, or a tool-error string
|
||||
when it must be rejected. Callers should ``return`` the error
|
||||
verbatim.
|
||||
"""
|
||||
env_tid = os.environ.get("HERMES_KANBAN_TASK")
|
||||
if not env_tid:
|
||||
# Orchestrator or CLI context — no task-scope restriction.
|
||||
return None
|
||||
if tid != env_tid:
|
||||
return tool_error(
|
||||
f"worker is scoped to task {env_tid}; refusing to mutate "
|
||||
f"{tid}. Use kanban_comment to hand off information to other "
|
||||
f"tasks, or kanban_create to spawn follow-up work."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _connect():
|
||||
"""Import + connect lazily so the module imports cleanly in non-kanban
|
||||
contexts (e.g. test rigs that import every tool module)."""
|
||||
@@ -154,6 +204,9 @@ def _handle_complete(args: dict, **kw) -> str:
|
||||
return tool_error(
|
||||
"task_id is required (or set HERMES_KANBAN_TASK in the env)"
|
||||
)
|
||||
ownership_err = _enforce_worker_task_ownership(tid)
|
||||
if ownership_err:
|
||||
return ownership_err
|
||||
summary = args.get("summary")
|
||||
metadata = args.get("metadata")
|
||||
result = args.get("result")
|
||||
@@ -192,6 +245,9 @@ def _handle_block(args: dict, **kw) -> str:
|
||||
return tool_error(
|
||||
"task_id is required (or set HERMES_KANBAN_TASK in the env)"
|
||||
)
|
||||
ownership_err = _enforce_worker_task_ownership(tid)
|
||||
if ownership_err:
|
||||
return ownership_err
|
||||
reason = args.get("reason")
|
||||
if not reason or not str(reason).strip():
|
||||
return tool_error("reason is required — explain what input you need")
|
||||
@@ -220,6 +276,9 @@ def _handle_heartbeat(args: dict, **kw) -> str:
|
||||
return tool_error(
|
||||
"task_id is required (or set HERMES_KANBAN_TASK in the env)"
|
||||
)
|
||||
ownership_err = _enforce_worker_task_ownership(tid)
|
||||
if ownership_err:
|
||||
return ownership_err
|
||||
note = args.get("note")
|
||||
try:
|
||||
kb, conn = _connect()
|
||||
|
||||
+7
-3
@@ -53,7 +53,7 @@ logger = logging.getLogger(__name__)
|
||||
# Lazy imports -- MCP SDK with OAuth support is optional
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_OAUTH_AVAILABLE = False
|
||||
_OAUTH_AVAILABLE=False
|
||||
try:
|
||||
from mcp.client.auth import OAuthClientProvider
|
||||
from mcp.shared.auth import (
|
||||
@@ -61,12 +61,16 @@ try:
|
||||
OAuthClientMetadata,
|
||||
OAuthToken,
|
||||
)
|
||||
from pydantic import AnyUrl
|
||||
|
||||
_OAUTH_AVAILABLE = True
|
||||
_OAUTH_AVAILABLE=True
|
||||
except ImportError:
|
||||
logger.debug("MCP OAuth types not available -- OAuth MCP auth disabled")
|
||||
|
||||
try:
|
||||
from pydantic import AnyUrl
|
||||
except ImportError:
|
||||
AnyUrl = None # type: ignore[assignment, misc]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
|
||||
@@ -1667,6 +1667,7 @@ _SESSION_EXPIRED_MARKERS: tuple = (
|
||||
"session expired",
|
||||
"session not found",
|
||||
"unknown session",
|
||||
"session terminated",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -10,9 +10,10 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, Optional
|
||||
import ssl
|
||||
import time
|
||||
from email.utils import formatdate
|
||||
from typing import Dict, Optional
|
||||
|
||||
from agent.redact import redact_sensitive_text
|
||||
|
||||
@@ -588,11 +589,28 @@ async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None,
|
||||
last_result = result
|
||||
return last_result
|
||||
|
||||
# --- Feishu: native media attachment support via adapter ---
|
||||
if platform == Platform.FEISHU and media_files:
|
||||
last_result = None
|
||||
for i, chunk in enumerate(chunks):
|
||||
is_last = (i == len(chunks) - 1)
|
||||
result = await _send_feishu(
|
||||
pconfig,
|
||||
chat_id,
|
||||
chunk,
|
||||
media_files=media_files if is_last else None,
|
||||
thread_id=thread_id,
|
||||
)
|
||||
if isinstance(result, dict) and result.get("error"):
|
||||
return result
|
||||
last_result = result
|
||||
return last_result
|
||||
|
||||
# --- Non-media platforms ---
|
||||
if media_files and not message.strip():
|
||||
return {
|
||||
"error": (
|
||||
f"send_message MEDIA delivery is currently only supported for telegram, discord, matrix, weixin, signal and yuanbao; "
|
||||
f"send_message MEDIA delivery is currently only supported for telegram, discord, matrix, weixin, signal, yuanbao and feishu; "
|
||||
f"target {platform.value} had only media attachments"
|
||||
)
|
||||
}
|
||||
@@ -600,7 +618,7 @@ async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None,
|
||||
if media_files:
|
||||
warning = (
|
||||
f"MEDIA attachments were omitted for {platform.value}; "
|
||||
"native send_message media delivery is currently only supported for telegram, discord, matrix, weixin, signal and yuanbao"
|
||||
"native send_message media delivery is currently only supported for telegram, discord, matrix, weixin, signal, yuanbao and feishu"
|
||||
)
|
||||
|
||||
last_result = None
|
||||
@@ -1652,8 +1670,8 @@ async def _send_qqbot(pconfig, chat_id, message):
|
||||
"""Send via QQBot using the REST API directly (no WebSocket needed).
|
||||
|
||||
Uses the QQ Bot Open Platform REST endpoints to get an access token
|
||||
and post a message. Works for guild channels without requiring
|
||||
a running gateway adapter.
|
||||
and post a message. Supports guild channels, C2C (private) chats,
|
||||
and group chats by trying the appropriate endpoints.
|
||||
"""
|
||||
try:
|
||||
import httpx
|
||||
@@ -1682,20 +1700,40 @@ async def _send_qqbot(pconfig, chat_id, message):
|
||||
return _error(f"QQBot: no access_token in response")
|
||||
|
||||
# Step 2: Send message via REST
|
||||
# QQ Bot API has separate endpoints for channels, C2C, and groups.
|
||||
# We try them in order: channel first, then fallback to C2C.
|
||||
headers = {
|
||||
"Authorization": f"QQBot {access_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
url = f"https://api.sgroup.qq.com/channels/{chat_id}/messages"
|
||||
payload = {"content": message[:4000], "msg_type": 0}
|
||||
|
||||
# Try channel endpoint first (works for guild channels)
|
||||
url = f"https://api.sgroup.qq.com/channels/{chat_id}/messages"
|
||||
resp = await client.post(url, json=payload, headers=headers)
|
||||
if resp.status_code in (200, 201):
|
||||
data = resp.json()
|
||||
return {"success": True, "platform": "qqbot", "chat_id": chat_id,
|
||||
"message_id": data.get("id")}
|
||||
else:
|
||||
return _error(f"QQBot send failed: {resp.status_code} {resp.text}")
|
||||
|
||||
# If channel endpoint failed (likely "频道不存在"), try C2C endpoint
|
||||
url_c2c = f"https://api.sgroup.qq.com/v2/users/{chat_id}/messages"
|
||||
resp_c2c = await client.post(url_c2c, json=payload, headers=headers)
|
||||
if resp_c2c.status_code in (200, 201):
|
||||
data = resp_c2c.json()
|
||||
return {"success": True, "platform": "qqbot", "chat_id": chat_id,
|
||||
"message_id": data.get("id")}
|
||||
|
||||
# If C2C also failed, try group endpoint
|
||||
url_group = f"https://api.sgroup.qq.com/v2/groups/{chat_id}/messages"
|
||||
resp_group = await client.post(url_group, json=payload, headers=headers)
|
||||
if resp_group.status_code in (200, 201):
|
||||
data = resp_group.json()
|
||||
return {"success": True, "platform": "qqbot", "chat_id": chat_id,
|
||||
"message_id": data.get("id")}
|
||||
|
||||
# All endpoints failed — return the most informative error
|
||||
return _error(f"QQBot send failed: channel={resp.status_code} c2c={resp_c2c.status_code} group={resp_group.status_code}")
|
||||
except Exception as e:
|
||||
return _error(f"QQBot send failed: {e}")
|
||||
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
Session Search Tool - Long-Term Conversation Recall
|
||||
|
||||
Searches past session transcripts in SQLite via FTS5, then summarizes the top
|
||||
matching sessions using a cheap/fast model (same pattern as web_extract).
|
||||
matching sessions using the configured auxiliary session_search model (same
|
||||
pattern as web_extract). By default, auxiliary "auto" routing uses the main
|
||||
chat provider/model unless the user overrides auxiliary.session_search.
|
||||
Returns focused summaries of past conversations rather than raw transcripts,
|
||||
keeping the main model's context window clean.
|
||||
|
||||
@@ -11,7 +13,7 @@ Flow:
|
||||
1. FTS5 search finds matching messages ranked by relevance
|
||||
2. Groups by session, takes the top N unique sessions (default 3)
|
||||
3. Loads each session's conversation, truncates to ~100k chars centered on matches
|
||||
4. Sends to Gemini Flash with a focused summarization prompt
|
||||
4. Sends to the configured auxiliary model with a focused summarization prompt
|
||||
5. Returns per-session summaries with metadata
|
||||
"""
|
||||
|
||||
@@ -330,7 +332,8 @@ def session_search(
|
||||
"""
|
||||
Search past sessions and return focused summaries of matching conversations.
|
||||
|
||||
Uses FTS5 to find matches, then summarizes the top sessions with Gemini Flash.
|
||||
Uses FTS5 to find matches, then summarizes the top sessions with the
|
||||
configured auxiliary session_search model.
|
||||
The current session is excluded from results since the agent already has that context.
|
||||
"""
|
||||
if db is None:
|
||||
@@ -483,7 +486,7 @@ def session_search(
|
||||
}, ensure_ascii=False)
|
||||
|
||||
summaries = []
|
||||
for (session_id, match_info, conversation_text, _), result in zip(tasks, results):
|
||||
for (session_id, match_info, conversation_text, session_meta), result in zip(tasks, results):
|
||||
if isinstance(result, Exception):
|
||||
logging.warning(
|
||||
"Failed to summarize session %s: %s",
|
||||
@@ -491,11 +494,18 @@ def session_search(
|
||||
)
|
||||
result = None
|
||||
|
||||
# Prefer resolved parent session metadata over FTS5 match metadata.
|
||||
# match_info carries source/model from the *child* session that contained
|
||||
# the FTS5 hit; after _resolve_to_parent() the session_id points to the
|
||||
# root, so session_meta has the authoritative platform/source for the
|
||||
# session the user actually cares about (#15909).
|
||||
entry = {
|
||||
"session_id": session_id,
|
||||
"when": _format_timestamp(match_info.get("session_started")),
|
||||
"source": match_info.get("source", "unknown"),
|
||||
"model": match_info.get("model"),
|
||||
"when": _format_timestamp(
|
||||
session_meta.get("started_at") or match_info.get("session_started")
|
||||
),
|
||||
"source": session_meta.get("source") or match_info.get("source", "unknown"),
|
||||
"model": session_meta.get("model") or match_info.get("model"),
|
||||
}
|
||||
|
||||
if result:
|
||||
|
||||
@@ -784,10 +784,17 @@ def skill_manage(
|
||||
pass
|
||||
# Curator telemetry: bump patch_count on edit/patch/write_file (the actions
|
||||
# that mutate an existing skill's guidance), drop the record on delete.
|
||||
# Best-effort; telemetry failures never break the tool.
|
||||
# Only mark a skill as agent-created when the background self-improvement
|
||||
# review fork creates it — foreground `skill_manage(create)` calls are
|
||||
# user-directed, and those skills belong to the user (the curator must
|
||||
# not touch them). Best-effort; telemetry failures never break the tool.
|
||||
try:
|
||||
from tools.skill_usage import bump_patch, forget
|
||||
if action in ("patch", "edit", "write_file", "remove_file"):
|
||||
from tools.skill_usage import bump_patch, forget, mark_agent_created
|
||||
from tools.skill_provenance import is_background_review
|
||||
if action == "create":
|
||||
if is_background_review():
|
||||
mark_agent_created(name)
|
||||
elif action in ("patch", "edit", "write_file", "remove_file"):
|
||||
bump_patch(name)
|
||||
elif action == "delete":
|
||||
forget(name)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Skill write-origin provenance — ContextVar for distinguishing agent-sediment skill writes from foreground user-directed writes.
|
||||
|
||||
The curator only consolidates/prunes skills it autonomously created via the
|
||||
background self-improvement review fork. Skills a user asks a foreground
|
||||
agent to write belong to the user and must never be auto-curated.
|
||||
|
||||
This module exposes a ContextVar that run_agent.py sets before each tool
|
||||
loop so tool handlers (e.g. skill_manage create) can check whether they
|
||||
are executing inside the background-review fork.
|
||||
|
||||
The signal piggybacks on AIAgent._memory_write_origin, which is already
|
||||
set to "background_review" for review-fork instances (see
|
||||
_spawn_background_review in run_agent.py) and defaults to "assistant_tool"
|
||||
for normal (foreground) agents.
|
||||
|
||||
Usage:
|
||||
from tools.skill_provenance import (
|
||||
set_current_write_origin,
|
||||
reset_current_write_origin,
|
||||
get_current_write_origin,
|
||||
)
|
||||
|
||||
token = set_current_write_origin("background_review")
|
||||
try:
|
||||
... # tool runs here
|
||||
finally:
|
||||
reset_current_write_origin(token)
|
||||
|
||||
# inside a tool:
|
||||
if get_current_write_origin() == "background_review":
|
||||
mark_agent_created(skill_name)
|
||||
"""
|
||||
|
||||
import contextvars
|
||||
|
||||
|
||||
_write_origin: contextvars.ContextVar[str] = contextvars.ContextVar(
|
||||
"skill_write_origin",
|
||||
default="foreground",
|
||||
)
|
||||
|
||||
# The sentinel value the background review fork uses; mirrors
|
||||
# run_agent.py's AIAgent._memory_write_origin override in
|
||||
# _spawn_background_review().
|
||||
BACKGROUND_REVIEW = "background_review"
|
||||
|
||||
|
||||
def set_current_write_origin(origin: str) -> contextvars.Token[str]:
|
||||
"""Bind the active write origin to the current context.
|
||||
|
||||
Returns a Token the caller must pass to reset_current_write_origin
|
||||
in a finally block.
|
||||
"""
|
||||
return _write_origin.set(origin or "foreground")
|
||||
|
||||
|
||||
def reset_current_write_origin(token: contextvars.Token[str]) -> None:
|
||||
"""Restore the prior write origin context."""
|
||||
_write_origin.reset(token)
|
||||
|
||||
|
||||
def get_current_write_origin() -> str:
|
||||
"""Return the active write origin.
|
||||
|
||||
Default: "foreground" — any tool call made by a regular (non-review)
|
||||
agent, from the CLI, the gateway, cron, or a subagent.
|
||||
|
||||
"background_review" — the self-improvement review fork; only skills
|
||||
created under this origin should be marked agent-created for curator
|
||||
management.
|
||||
"""
|
||||
return _write_origin.get()
|
||||
|
||||
|
||||
def is_background_review() -> bool:
|
||||
"""Convenience: True iff the current write origin is the background
|
||||
review fork."""
|
||||
return get_current_write_origin() == BACKGROUND_REVIEW
|
||||
+33
-9
@@ -11,8 +11,9 @@ Design notes:
|
||||
- Atomic writes via tempfile + os.replace (same pattern as .bundled_manifest).
|
||||
- All counter bumps are best-effort: failures log at DEBUG and return silently.
|
||||
A broken sidecar never breaks the underlying tool call.
|
||||
- Provenance filter: "agent-created" == not in .bundled_manifest AND not in
|
||||
.hub/lock.json. The curator only ever mutates agent-created skills.
|
||||
- Provenance filter: curator-managed skills are explicitly marked when
|
||||
created through skill_manage. Bundled / hub-installed skills stay
|
||||
off-limits, and manually authored skills are not inferred from location.
|
||||
|
||||
Lifecycle states:
|
||||
active -> default
|
||||
@@ -149,11 +150,13 @@ def _read_hub_installed_names() -> Set[str]:
|
||||
|
||||
|
||||
def list_agent_created_skill_names() -> List[str]:
|
||||
"""Enumerate skills that were authored by the agent (or user), NOT by a
|
||||
bundled or hub-installed source.
|
||||
"""Enumerate skills explicitly authored by the agent.
|
||||
|
||||
The curator operates exclusively on this set. Bundled / hub skills are
|
||||
maintained by their upstream sources and must never be pruned here.
|
||||
The curator operates exclusively on this set. Skills are only eligible
|
||||
after ``skill_manage(action="create")`` marks them in ``.usage.json``;
|
||||
manually authored skills must not be inferred from filesystem location.
|
||||
Bundled / hub skills are maintained by their upstream sources and must
|
||||
never be pruned here.
|
||||
"""
|
||||
base = _skills_dir()
|
||||
if not base.exists():
|
||||
@@ -161,6 +164,7 @@ def list_agent_created_skill_names() -> List[str]:
|
||||
bundled = _read_bundled_manifest_names()
|
||||
hub = _read_hub_installed_names()
|
||||
off_limits = bundled | hub
|
||||
usage = load_usage()
|
||||
|
||||
names: List[str] = []
|
||||
# Top-level SKILL.md files (flat layout) AND nested category/skill/SKILL.md
|
||||
@@ -176,6 +180,8 @@ def list_agent_created_skill_names() -> List[str]:
|
||||
name = _read_skill_name(skill_md, fallback=skill_md.parent.name)
|
||||
if name in off_limits:
|
||||
continue
|
||||
if not _is_curator_managed_record(usage.get(name)):
|
||||
continue
|
||||
names.append(name)
|
||||
return sorted(set(names))
|
||||
|
||||
@@ -207,12 +213,20 @@ def is_agent_created(skill_name: str) -> bool:
|
||||
return skill_name not in off_limits
|
||||
|
||||
|
||||
def _is_curator_managed_record(record: Any) -> bool:
|
||||
"""Return True when a usage record opts a skill into curator management."""
|
||||
if not isinstance(record, dict):
|
||||
return False
|
||||
return record.get("created_by") == "agent" or record.get("agent_created") is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sidecar I/O
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _empty_record() -> Dict[str, Any]:
|
||||
return {
|
||||
"created_by": None,
|
||||
"use_count": 0,
|
||||
"view_count": 0,
|
||||
"last_used_at": None,
|
||||
@@ -287,9 +301,8 @@ def _mutate(skill_name: str, mutator) -> None:
|
||||
"""Load, apply *mutator(record)* in place, save. Best-effort.
|
||||
|
||||
Bundled and hub-installed skills are NEVER recorded in the sidecar.
|
||||
This keeps .usage.json focused on agent-created skills (the only ones
|
||||
the curator considers) and prevents stale counters from hanging around
|
||||
for upstream-managed skills.
|
||||
Local manual skills may still accrue usage telemetry, but they only
|
||||
become curator-managed when ``created_by`` is explicitly marked.
|
||||
"""
|
||||
if not skill_name:
|
||||
return
|
||||
@@ -336,6 +349,17 @@ def bump_patch(skill_name: str) -> None:
|
||||
_mutate(skill_name, _apply)
|
||||
|
||||
|
||||
def mark_agent_created(skill_name: str) -> None:
|
||||
"""Opt a skill created by skill_manage into curator management.
|
||||
|
||||
Viewing or invoking a manually authored skill may still create telemetry,
|
||||
but only this explicit marker makes it eligible for automatic curation.
|
||||
"""
|
||||
def _apply(rec: Dict[str, Any]) -> None:
|
||||
rec["created_by"] = "agent"
|
||||
_mutate(skill_name, _apply)
|
||||
|
||||
|
||||
def set_state(skill_name: str, state: str) -> None:
|
||||
"""Set lifecycle state. No-op if *state* is invalid."""
|
||||
if state not in _VALID_STATES:
|
||||
|
||||
+5
-1
@@ -2801,7 +2801,11 @@ def bundle_content_hash(bundle: SkillBundle) -> str:
|
||||
"""Compute a deterministic hash for an in-memory skill bundle."""
|
||||
h = hashlib.sha256()
|
||||
for rel_path in sorted(bundle.files):
|
||||
h.update(bundle.files[rel_path].encode("utf-8"))
|
||||
content = bundle.files[rel_path]
|
||||
if isinstance(content, bytes):
|
||||
h.update(content)
|
||||
else:
|
||||
h.update(content.encode("utf-8"))
|
||||
return f"sha256:{h.hexdigest()[:16]}"
|
||||
|
||||
|
||||
|
||||
@@ -440,6 +440,8 @@ async def vision_analyze_tool(
|
||||
- For local file paths, the file is used directly and NOT deleted
|
||||
- Supports common image formats (JPEG, PNG, GIF, WebP, etc.)
|
||||
"""
|
||||
if not isinstance(user_prompt, str):
|
||||
user_prompt = str(user_prompt) if user_prompt is not None else ""
|
||||
debug_call_data = {
|
||||
"parameters": {
|
||||
"image_url": image_url,
|
||||
@@ -801,3 +803,366 @@ registry.register(
|
||||
is_async=True,
|
||||
emoji="👁️",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Video Analysis Tool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Extension → MIME. avi/mkv fall back to mp4.
|
||||
_VIDEO_MIME_TYPES = {
|
||||
".mp4": "video/mp4",
|
||||
".webm": "video/webm",
|
||||
".mov": "video/mov",
|
||||
".avi": "video/mp4",
|
||||
".mkv": "video/mp4",
|
||||
".mpeg": "video/mpeg",
|
||||
".mpg": "video/mpeg",
|
||||
}
|
||||
|
||||
_MAX_VIDEO_BASE64_BYTES = 50 * 1024 * 1024 # 50 MB hard cap
|
||||
_VIDEO_SIZE_WARN_BYTES = 20 * 1024 * 1024
|
||||
|
||||
|
||||
def _detect_video_mime_type(video_path: Path) -> Optional[str]:
|
||||
"""Return a video MIME type based on file extension, or None if unsupported."""
|
||||
ext = video_path.suffix.lower()
|
||||
return _VIDEO_MIME_TYPES.get(ext)
|
||||
|
||||
|
||||
def _video_to_base64_data_url(video_path: Path, mime_type: Optional[str] = None) -> str:
|
||||
"""Convert a video file to a base64-encoded data URL."""
|
||||
data = video_path.read_bytes()
|
||||
encoded = base64.b64encode(data).decode("ascii")
|
||||
mime = mime_type or _VIDEO_MIME_TYPES.get(video_path.suffix.lower(), "video/mp4")
|
||||
return f"data:{mime};base64,{encoded}"
|
||||
|
||||
|
||||
async def _download_video(video_url: str, destination: Path, max_retries: int = 3) -> Path:
|
||||
"""Download video from URL with SSRF protection and retry."""
|
||||
import asyncio
|
||||
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def _ssrf_redirect_guard(response):
|
||||
if response.is_redirect and response.next_request:
|
||||
redirect_url = str(response.next_request.url)
|
||||
from tools.url_safety import is_safe_url
|
||||
if not is_safe_url(redirect_url):
|
||||
raise ValueError(
|
||||
f"Blocked redirect to private/internal address: {redirect_url}"
|
||||
)
|
||||
|
||||
last_error = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
blocked = check_website_access(video_url)
|
||||
if blocked:
|
||||
raise PermissionError(blocked["message"])
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
timeout=60.0,
|
||||
follow_redirects=True,
|
||||
event_hooks={"response": [_ssrf_redirect_guard]},
|
||||
) as client:
|
||||
response = await client.get(
|
||||
video_url,
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "video/*,*/*;q=0.8",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
cl = response.headers.get("content-length")
|
||||
if cl and int(cl) > _MAX_VIDEO_BASE64_BYTES:
|
||||
raise ValueError(
|
||||
f"Video too large ({int(cl)} bytes, max {_MAX_VIDEO_BASE64_BYTES})"
|
||||
)
|
||||
|
||||
final_url = str(response.url)
|
||||
blocked = check_website_access(final_url)
|
||||
if blocked:
|
||||
raise PermissionError(blocked["message"])
|
||||
|
||||
body = response.content
|
||||
if len(body) > _MAX_VIDEO_BASE64_BYTES:
|
||||
raise ValueError(
|
||||
f"Video too large ({len(body)} bytes, max {_MAX_VIDEO_BASE64_BYTES})"
|
||||
)
|
||||
destination.write_bytes(body)
|
||||
|
||||
return destination
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2 ** (attempt + 1)
|
||||
logger.warning("Video download failed (attempt %s/%s): %s", attempt + 1, max_retries, str(e)[:50])
|
||||
await asyncio.sleep(wait_time)
|
||||
else:
|
||||
logger.error(
|
||||
"Video download failed after %s attempts: %s",
|
||||
max_retries, str(e)[:100], exc_info=True,
|
||||
)
|
||||
|
||||
if last_error is None:
|
||||
raise RuntimeError(
|
||||
f"_download_video exited retry loop without attempting (max_retries={max_retries})"
|
||||
)
|
||||
raise last_error
|
||||
|
||||
|
||||
async def video_analyze_tool(
|
||||
video_url: str,
|
||||
user_prompt: str,
|
||||
model: str = None,
|
||||
) -> str:
|
||||
"""Analyze a video via multimodal LLM. Returns JSON {success, analysis}."""
|
||||
if not isinstance(user_prompt, str):
|
||||
user_prompt = str(user_prompt) if user_prompt is not None else ""
|
||||
debug_call_data = {
|
||||
"parameters": {
|
||||
"video_url": video_url,
|
||||
"user_prompt": user_prompt[:200] + "..." if len(user_prompt) > 200 else user_prompt,
|
||||
"model": model,
|
||||
},
|
||||
"error": None,
|
||||
"success": False,
|
||||
"analysis_length": 0,
|
||||
"model_used": model,
|
||||
"video_size_bytes": 0,
|
||||
}
|
||||
|
||||
temp_video_path = None
|
||||
should_cleanup = True
|
||||
|
||||
try:
|
||||
from tools.interrupt import is_interrupted
|
||||
if is_interrupted():
|
||||
return tool_error("Interrupted", success=False)
|
||||
|
||||
logger.info("Analyzing video: %s", video_url[:60])
|
||||
logger.info("User prompt: %s", user_prompt[:100])
|
||||
|
||||
# Resolve local path vs remote URL
|
||||
resolved_url = video_url
|
||||
if resolved_url.startswith("file://"):
|
||||
resolved_url = resolved_url[len("file://"):]
|
||||
local_path = Path(os.path.expanduser(resolved_url))
|
||||
|
||||
if local_path.is_file():
|
||||
logger.info("Using local video file: %s", video_url)
|
||||
temp_video_path = local_path
|
||||
should_cleanup = False
|
||||
elif _validate_image_url(video_url):
|
||||
blocked = check_website_access(video_url)
|
||||
if blocked:
|
||||
raise PermissionError(blocked["message"])
|
||||
temp_dir = get_hermes_dir("cache/video", "temp_video_files")
|
||||
temp_video_path = temp_dir / f"temp_video_{uuid.uuid4()}.mp4"
|
||||
await _download_video(video_url, temp_video_path)
|
||||
should_cleanup = True
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid video source. Provide an HTTP/HTTPS URL or a valid local file path."
|
||||
)
|
||||
|
||||
video_size_bytes = temp_video_path.stat().st_size
|
||||
video_size_mb = video_size_bytes / (1024 * 1024)
|
||||
logger.info("Video ready (%.1f MB)", video_size_mb)
|
||||
|
||||
detected_mime = _detect_video_mime_type(temp_video_path)
|
||||
if not detected_mime:
|
||||
raise ValueError(
|
||||
f"Unsupported video format: '{temp_video_path.suffix}'. "
|
||||
f"Supported: {', '.join(sorted(_VIDEO_MIME_TYPES.keys()))}"
|
||||
)
|
||||
|
||||
if video_size_bytes > _VIDEO_SIZE_WARN_BYTES:
|
||||
logger.warning("Video is %.1f MB — may be slow or rejected", video_size_mb)
|
||||
|
||||
video_data_url = _video_to_base64_data_url(temp_video_path, mime_type=detected_mime)
|
||||
data_size_mb = len(video_data_url) / (1024 * 1024)
|
||||
|
||||
if len(video_data_url) > _MAX_VIDEO_BASE64_BYTES:
|
||||
raise ValueError(
|
||||
f"Video too large for API: base64 payload is {data_size_mb:.1f} MB "
|
||||
f"(limit {_MAX_VIDEO_BASE64_BYTES / (1024 * 1024):.0f} MB). "
|
||||
f"Compress or trim the video and retry."
|
||||
)
|
||||
|
||||
debug_call_data["video_size_bytes"] = video_size_bytes
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": user_prompt,
|
||||
},
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": video_data_url,
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
vision_timeout = 180.0
|
||||
vision_temperature = 0.1
|
||||
try:
|
||||
from hermes_cli.config import cfg_get, load_config
|
||||
_cfg = load_config()
|
||||
_vision_cfg = cfg_get(_cfg, "auxiliary", "vision", default={})
|
||||
_vt = _vision_cfg.get("timeout")
|
||||
if _vt is not None:
|
||||
vision_timeout = max(float(_vt), 180.0)
|
||||
_vtemp = _vision_cfg.get("temperature")
|
||||
if _vtemp is not None:
|
||||
vision_temperature = float(_vtemp)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
call_kwargs = {
|
||||
"task": "vision",
|
||||
"messages": messages,
|
||||
"temperature": vision_temperature,
|
||||
"max_tokens": 4000,
|
||||
"timeout": vision_timeout,
|
||||
}
|
||||
if model:
|
||||
call_kwargs["model"] = model
|
||||
|
||||
response = await async_call_llm(**call_kwargs)
|
||||
analysis = extract_content_or_reasoning(response)
|
||||
|
||||
if not analysis:
|
||||
logger.warning("Empty video response, retrying once")
|
||||
response = await async_call_llm(**call_kwargs)
|
||||
analysis = extract_content_or_reasoning(response)
|
||||
|
||||
analysis_length = len(analysis) if analysis else 0
|
||||
logger.info("Video analysis completed (%s characters)", analysis_length)
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"analysis": analysis or "There was a problem with the request and the video could not be analyzed.",
|
||||
}
|
||||
|
||||
debug_call_data["success"] = True
|
||||
debug_call_data["analysis_length"] = analysis_length
|
||||
_debug.log_call("video_analyze_tool", debug_call_data)
|
||||
_debug.save()
|
||||
|
||||
return json.dumps(result, indent=2, ensure_ascii=False)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error analyzing video: {str(e)}"
|
||||
logger.error("%s", error_msg, exc_info=True)
|
||||
|
||||
err_str = str(e).lower()
|
||||
if any(hint in err_str for hint in (
|
||||
"402", "insufficient", "payment required", "credits", "billing",
|
||||
)):
|
||||
analysis = (
|
||||
"Insufficient credits or payment required. Please top up your "
|
||||
f"API provider account and try again. Error: {e}"
|
||||
)
|
||||
elif any(hint in err_str for hint in (
|
||||
"does not support", "not support video",
|
||||
"content_policy", "multimodal",
|
||||
"unrecognized request argument", "video input",
|
||||
"video_url",
|
||||
)):
|
||||
analysis = (
|
||||
f"The model does not support video analysis or the request was "
|
||||
f"rejected. Ensure you're using a video-capable model "
|
||||
f"(e.g. google/gemini-2.5-flash). Error: {e}"
|
||||
)
|
||||
elif any(hint in err_str for hint in (
|
||||
"too large", "payload", "413", "content_too_large",
|
||||
"request_too_large", "exceeds", "size limit",
|
||||
)):
|
||||
analysis = (
|
||||
"The video is too large for the API. Try compressing or trimming "
|
||||
f"the video (max ~50 MB). Error: {e}"
|
||||
)
|
||||
else:
|
||||
analysis = (
|
||||
"There was a problem with the request and the video could not "
|
||||
f"be analyzed. Error: {e}"
|
||||
)
|
||||
|
||||
result = {
|
||||
"success": False,
|
||||
"error": error_msg,
|
||||
"analysis": analysis,
|
||||
}
|
||||
|
||||
debug_call_data["error"] = error_msg
|
||||
_debug.log_call("video_analyze_tool", debug_call_data)
|
||||
_debug.save()
|
||||
|
||||
return json.dumps(result, indent=2, ensure_ascii=False)
|
||||
|
||||
finally:
|
||||
if should_cleanup and temp_video_path and temp_video_path.exists():
|
||||
try:
|
||||
temp_video_path.unlink()
|
||||
logger.debug("Cleaned up temporary video file")
|
||||
except Exception as cleanup_error:
|
||||
logger.warning(
|
||||
"Could not delete temporary file: %s", cleanup_error, exc_info=True
|
||||
)
|
||||
|
||||
|
||||
VIDEO_ANALYZE_SCHEMA = {
|
||||
"name": "video_analyze",
|
||||
"description": (
|
||||
"Analyze a video from a URL or local file path using a multimodal AI model. "
|
||||
"Sends the video to a video-capable model (e.g. Gemini) for understanding. "
|
||||
"Use this for video files — for images, use vision_analyze instead. "
|
||||
"Supports mp4, webm, mov, avi, mkv, mpeg formats. "
|
||||
"Note: large videos (>20 MB) may be slow; max ~50 MB."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"video_url": {
|
||||
"type": "string",
|
||||
"description": "Video URL (http/https) or local file path to analyze.",
|
||||
},
|
||||
"question": {
|
||||
"type": "string",
|
||||
"description": "Your specific question about the video. The AI will describe what happens in the video and answer your question.",
|
||||
},
|
||||
},
|
||||
"required": ["video_url", "question"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _handle_video_analyze(args: Dict[str, Any], **kw: Any) -> Awaitable[str]:
|
||||
video_url = args.get("video_url", "")
|
||||
question = args.get("question", "")
|
||||
full_prompt = (
|
||||
"Fully describe and explain everything happening in this video, "
|
||||
"including visual content, motion, audio cues, text overlays, and scene "
|
||||
f"transitions. Then answer the following question:\n\n{question}"
|
||||
)
|
||||
model = os.getenv("AUXILIARY_VIDEO_MODEL", "").strip() or os.getenv("AUXILIARY_VISION_MODEL", "").strip() or None
|
||||
return video_analyze_tool(video_url, full_prompt, model)
|
||||
|
||||
|
||||
registry.register(
|
||||
name="video_analyze",
|
||||
toolset="video",
|
||||
schema=VIDEO_ANALYZE_SCHEMA,
|
||||
handler=_handle_video_analyze,
|
||||
check_fn=check_vision_requirements,
|
||||
is_async=True,
|
||||
emoji="🎬",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user