chore: uptick
This commit is contained in:
+318
-136
@@ -162,6 +162,13 @@ from agent.display import (
|
||||
_detect_tool_failure,
|
||||
get_tool_emoji as _get_tool_emoji,
|
||||
)
|
||||
from agent.tool_guardrails import (
|
||||
ToolCallGuardrailConfig,
|
||||
ToolCallGuardrailController,
|
||||
ToolGuardrailDecision,
|
||||
append_toolguard_guidance,
|
||||
toolguard_synthetic_result,
|
||||
)
|
||||
from agent.trajectory import (
|
||||
convert_scratchpad_to_think, has_incomplete_scratchpad,
|
||||
save_trajectory as _save_trajectory_to_file,
|
||||
@@ -1150,6 +1157,8 @@ class AIAgent:
|
||||
# Tool execution state — allows _vprint during tool execution
|
||||
# even when stream consumers are registered (no tokens streaming then)
|
||||
self._executing_tools = False
|
||||
self._tool_guardrails = ToolCallGuardrailController()
|
||||
self._tool_guardrail_halt_decision: ToolGuardrailDecision | None = None
|
||||
|
||||
# Interrupt mechanism for breaking out of tool loops
|
||||
self._interrupt_requested = False
|
||||
@@ -1623,30 +1632,12 @@ class AIAgent:
|
||||
self._session_db = session_db
|
||||
self._parent_session_id = parent_session_id
|
||||
self._last_flushed_db_idx = 0 # tracks DB-write cursor to prevent duplicate writes
|
||||
if self._session_db:
|
||||
try:
|
||||
self._session_db.create_session(
|
||||
session_id=self.session_id,
|
||||
source=self.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
|
||||
model=self.model,
|
||||
model_config={
|
||||
"max_iterations": self.max_iterations,
|
||||
"reasoning_config": reasoning_config,
|
||||
"max_tokens": max_tokens,
|
||||
},
|
||||
user_id=None,
|
||||
parent_session_id=self._parent_session_id,
|
||||
)
|
||||
except Exception as e:
|
||||
# Transient SQLite lock contention (e.g. CLI and gateway writing
|
||||
# concurrently) must NOT permanently disable session_search for
|
||||
# this agent. Keep _session_db alive — subsequent message
|
||||
# flushes and session_search calls will still work once the
|
||||
# lock clears. The session row may be missing from the index
|
||||
# for this run, but that is recoverable (flushes upsert rows).
|
||||
logger.warning(
|
||||
"Session DB create_session failed (session_search still available): %s", e
|
||||
)
|
||||
self._session_db_created = False # DB row deferred to run_conversation()
|
||||
self._session_init_model_config = {
|
||||
"max_iterations": self.max_iterations,
|
||||
"reasoning_config": reasoning_config,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
|
||||
# In-memory todo list for task planning (one per agent/session)
|
||||
from tools.todo_tool import TodoStore
|
||||
@@ -1658,6 +1649,14 @@ class AIAgent:
|
||||
_agent_cfg = _load_agent_config()
|
||||
except Exception:
|
||||
_agent_cfg = {}
|
||||
try:
|
||||
self._tool_guardrails = ToolCallGuardrailController(
|
||||
ToolCallGuardrailConfig.from_mapping(
|
||||
_agent_cfg.get("tool_loop_guardrails", {})
|
||||
)
|
||||
)
|
||||
except Exception as _tlg_err:
|
||||
logger.warning("Tool loop guardrail config ignored: %s", _tlg_err)
|
||||
# Cache only the derived auxiliary compression context override that is
|
||||
# needed later by the startup feasibility check. Avoid exposing a
|
||||
# broad pseudo-public config object on the agent instance.
|
||||
@@ -2153,6 +2152,28 @@ class AIAgent:
|
||||
"is_anthropic_oauth": self._is_anthropic_oauth,
|
||||
})
|
||||
|
||||
def _ensure_db_session(self) -> None:
|
||||
"""Create session DB row on first use. Disables _session_db on failure."""
|
||||
if self._session_db_created or not self._session_db:
|
||||
return
|
||||
try:
|
||||
self._session_db.create_session(
|
||||
session_id=self.session_id,
|
||||
source=self.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
|
||||
model=self.model,
|
||||
model_config=self._session_init_model_config,
|
||||
system_prompt=self._cached_system_prompt,
|
||||
user_id=None,
|
||||
parent_session_id=self._parent_session_id,
|
||||
)
|
||||
self._session_db_created = True
|
||||
except Exception as e:
|
||||
# Transient failure (e.g. SQLite lock). Keep _session_db alive —
|
||||
# _session_db_created stays False so next run_conversation() retries.
|
||||
logger.warning(
|
||||
"Session DB creation failed (will retry next turn): %s", e
|
||||
)
|
||||
|
||||
def reset_session_state(self):
|
||||
"""Reset all session-scoped token counters to 0 for a fresh session.
|
||||
|
||||
@@ -3702,14 +3723,9 @@ class AIAgent:
|
||||
return
|
||||
self._apply_persist_user_message_override(messages)
|
||||
try:
|
||||
# If create_session() failed at startup (e.g. transient lock), the
|
||||
# session row may not exist yet. ensure_session() uses INSERT OR
|
||||
# IGNORE so it is a no-op when the row is already there.
|
||||
self._session_db.ensure_session(
|
||||
self.session_id,
|
||||
source=self.platform or "cli",
|
||||
model=self.model,
|
||||
)
|
||||
# Retry row creation if the earlier attempt failed transiently.
|
||||
if not self._session_db_created:
|
||||
self._ensure_db_session()
|
||||
start_idx = len(conversation_history) if conversation_history else 0
|
||||
flush_from = max(start_idx, self._last_flushed_db_idx)
|
||||
for msg in messages[flush_from:]:
|
||||
@@ -4982,8 +4998,8 @@ class AIAgent:
|
||||
def _get_tool_call_id_static(tc) -> str:
|
||||
"""Extract call ID from a tool_call entry (dict or object)."""
|
||||
if isinstance(tc, dict):
|
||||
return tc.get("id", "") or ""
|
||||
return getattr(tc, "id", "") or ""
|
||||
return tc.get("call_id", "") or tc.get("id", "") or ""
|
||||
return getattr(tc, "call_id", "") or getattr(tc, "id", "") or ""
|
||||
|
||||
_VALID_API_ROLES = frozenset({"system", "user", "assistant", "tool", "function", "developer"})
|
||||
|
||||
@@ -8586,9 +8602,13 @@ class AIAgent:
|
||||
# message. Without it, replaying the persisted message causes
|
||||
# HTTP 400 ("The reasoning_content in the thinking mode must
|
||||
# be passed back to the API"). Include streamed reasoning
|
||||
# text when captured; otherwise pad with empty string.
|
||||
# Refs #15250, #17400.
|
||||
msg["reasoning_content"] = reasoning_text or ""
|
||||
# text when captured; otherwise pad with a single space —
|
||||
# DeepSeek V4 Pro tightened validation and rejects empty
|
||||
# string ("The reasoning content in the thinking mode must
|
||||
# be passed back to the API"). A space satisfies non-empty
|
||||
# checks everywhere without leaking fabricated reasoning.
|
||||
# Refs #15250, #17400, #17341.
|
||||
msg["reasoning_content"] = reasoning_text or " "
|
||||
|
||||
# Additive fallback (refs #16844, #16884). Streaming-only providers
|
||||
# (glm, MiniMax, gpt-5.x via aigw, Anthropic via openai-compat shims)
|
||||
@@ -8743,11 +8763,20 @@ class AIAgent:
|
||||
return
|
||||
|
||||
# 1. Explicit reasoning_content already set — preserve it verbatim
|
||||
# (includes DeepSeek/Kimi's own empty-string placeholder written at
|
||||
# creation time, and any valid reasoning content from the same provider).
|
||||
# (includes DeepSeek/Kimi's own space-placeholder written at creation
|
||||
# time, and any valid reasoning content from the same provider).
|
||||
#
|
||||
# Exception: sessions persisted BEFORE #17341 have empty-string
|
||||
# placeholders pinned at creation time. DeepSeek V4 Pro rejects
|
||||
# those with HTTP 400. When the active provider enforces the
|
||||
# thinking-mode echo, upgrade "" → " " on replay so stale history
|
||||
# doesn't 400 the user on the next turn.
|
||||
existing = source_msg.get("reasoning_content")
|
||||
if isinstance(existing, str):
|
||||
api_msg["reasoning_content"] = existing
|
||||
if existing == "" and self._needs_thinking_reasoning_pad():
|
||||
api_msg["reasoning_content"] = " "
|
||||
else:
|
||||
api_msg["reasoning_content"] = existing
|
||||
return
|
||||
|
||||
needs_thinking_pad = self._needs_thinking_reasoning_pad()
|
||||
@@ -8759,8 +8788,10 @@ class AIAgent:
|
||||
# pins reasoning_content at creation time for tool-call turns, so the
|
||||
# shape (reasoning set, reasoning_content absent, tool_calls present)
|
||||
# is unreachable from same-provider DeepSeek history after this fix.
|
||||
# Inject "" to satisfy the API without leaking another provider's
|
||||
# chain of thought to DeepSeek/Kimi.
|
||||
# Inject a single space to satisfy the API without leaking another
|
||||
# provider's chain of thought to DeepSeek/Kimi. Space (not "")
|
||||
# because DeepSeek V4 Pro rejects empty-string reasoning_content
|
||||
# in thinking mode (refs #17341).
|
||||
normalized_reasoning = source_msg.get("reasoning")
|
||||
if (
|
||||
needs_thinking_pad
|
||||
@@ -8768,7 +8799,7 @@ class AIAgent:
|
||||
and isinstance(normalized_reasoning, str)
|
||||
and normalized_reasoning
|
||||
):
|
||||
api_msg["reasoning_content"] = ""
|
||||
api_msg["reasoning_content"] = " "
|
||||
return
|
||||
|
||||
# 3. Healthy session: promote 'reasoning' field to 'reasoning_content'
|
||||
@@ -8781,12 +8812,15 @@ class AIAgent:
|
||||
return
|
||||
|
||||
# 4. DeepSeek / Kimi thinking mode: all assistant messages need
|
||||
# reasoning_content. Inject "" to satisfy the provider's requirement
|
||||
# when no explicit reasoning content is present. Covers both
|
||||
# tool-call turns (already-poisoned history with no reasoning at all)
|
||||
# and plain text turns.
|
||||
# reasoning_content. Inject a single space to satisfy the provider's
|
||||
# requirement when no explicit reasoning content is present. Covers
|
||||
# both tool-call turns (already-poisoned history with no reasoning
|
||||
# at all) and plain text turns. Space (not "") because DeepSeek V4
|
||||
# Pro tightened validation and rejects empty string with HTTP 400
|
||||
# ("The reasoning content in the thinking mode must be passed back
|
||||
# to the API"). Refs #17341.
|
||||
if needs_thinking_pad:
|
||||
api_msg["reasoning_content"] = ""
|
||||
api_msg["reasoning_content"] = " "
|
||||
return
|
||||
|
||||
# 5. reasoning_content was present but not a string (e.g. None after
|
||||
@@ -9021,12 +9055,15 @@ class AIAgent:
|
||||
self.session_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}"
|
||||
# Update session_log_file to point to the new session's JSON file
|
||||
self.session_log_file = self.logs_dir / f"session_{self.session_id}.json"
|
||||
self._session_db_created = False
|
||||
self._session_db.create_session(
|
||||
session_id=self.session_id,
|
||||
source=self.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
|
||||
model=self.model,
|
||||
model_config=self._session_init_model_config,
|
||||
parent_session_id=old_session_id,
|
||||
)
|
||||
self._session_db_created = True
|
||||
# Auto-number the title for the continuation session
|
||||
if old_title:
|
||||
try:
|
||||
@@ -9084,9 +9121,14 @@ class AIAgent:
|
||||
|
||||
# Update token estimate after compaction so pressure calculations
|
||||
# use the post-compression count, not the stale pre-compression one.
|
||||
_compressed_est = (
|
||||
estimate_tokens_rough(new_system_prompt)
|
||||
+ estimate_messages_tokens_rough(compressed)
|
||||
# Use estimate_request_tokens_rough() so tool schemas are included —
|
||||
# with 50+ tools enabled, schemas alone can add 20-30K tokens, and
|
||||
# omitting them delays the next compression cycle far past the
|
||||
# configured threshold (issue #14695).
|
||||
_compressed_est = estimate_request_tokens_rough(
|
||||
compressed,
|
||||
system_prompt=new_system_prompt or "",
|
||||
tools=self.tools or None,
|
||||
)
|
||||
self.context_compressor.last_prompt_tokens = _compressed_est
|
||||
self.context_compressor.last_completion_tokens = 0
|
||||
@@ -9107,6 +9149,44 @@ class AIAgent:
|
||||
)
|
||||
return compressed, new_system_prompt
|
||||
|
||||
def _set_tool_guardrail_halt(self, decision: ToolGuardrailDecision) -> None:
|
||||
"""Record the first guardrail decision that should stop this turn."""
|
||||
if decision.should_halt and self._tool_guardrail_halt_decision is None:
|
||||
self._tool_guardrail_halt_decision = decision
|
||||
|
||||
def _toolguard_controlled_halt_response(self, decision: ToolGuardrailDecision) -> str:
|
||||
tool = decision.tool_name or "a tool"
|
||||
return (
|
||||
f"I stopped retrying {tool} because it hit the tool-call guardrail "
|
||||
f"({decision.code}) after {decision.count} repeated non-progressing "
|
||||
"attempts. The last tool result explains the blocker; the next step is "
|
||||
"to change strategy instead of repeating the same call."
|
||||
)
|
||||
|
||||
def _append_guardrail_observation(
|
||||
self,
|
||||
tool_name: str,
|
||||
function_args: dict,
|
||||
function_result: str,
|
||||
*,
|
||||
failed: bool,
|
||||
) -> str:
|
||||
decision = self._tool_guardrails.after_call(
|
||||
tool_name,
|
||||
function_args,
|
||||
function_result,
|
||||
failed=failed,
|
||||
)
|
||||
if decision.action in {"warn", "halt"}:
|
||||
function_result = append_toolguard_guidance(function_result, decision)
|
||||
if decision.should_halt:
|
||||
self._set_tool_guardrail_halt(decision)
|
||||
return function_result
|
||||
|
||||
def _guardrail_block_result(self, decision: ToolGuardrailDecision) -> str:
|
||||
self._set_tool_guardrail_halt(decision)
|
||||
return toolguard_synthetic_result(decision)
|
||||
|
||||
def _execute_tool_calls(self, assistant_message, messages: list, effective_task_id: str, api_call_count: int = 0) -> None:
|
||||
"""Execute tool calls from the assistant message and append results to messages.
|
||||
|
||||
@@ -9150,7 +9230,8 @@ class AIAgent:
|
||||
)
|
||||
|
||||
def _invoke_tool(self, function_name: str, function_args: dict, effective_task_id: str,
|
||||
tool_call_id: Optional[str] = None, messages: list = None) -> str:
|
||||
tool_call_id: Optional[str] = None, messages: list = None,
|
||||
pre_tool_block_checked: bool = False) -> str:
|
||||
"""Invoke a single tool and return the result string. No display logic.
|
||||
|
||||
Handles both agent-level tools (todo, memory, etc.) and registry-dispatched
|
||||
@@ -9159,13 +9240,14 @@ class AIAgent:
|
||||
"""
|
||||
# Check plugin hooks for a block directive before executing anything.
|
||||
block_message: Optional[str] = None
|
||||
try:
|
||||
from hermes_cli.plugins import get_pre_tool_call_block_message
|
||||
block_message = get_pre_tool_call_block_message(
|
||||
function_name, function_args, task_id=effective_task_id or "",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if not pre_tool_block_checked:
|
||||
try:
|
||||
from hermes_cli.plugins import get_pre_tool_call_block_message
|
||||
block_message = get_pre_tool_call_block_message(
|
||||
function_name, function_args, task_id=effective_task_id or "",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if block_message is not None:
|
||||
return json.dumps({"error": block_message}, ensure_ascii=False)
|
||||
|
||||
@@ -9317,13 +9399,31 @@ class AIAgent:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
parsed_calls.append((tool_call, function_name, function_args))
|
||||
block_result = None
|
||||
blocked_by_guardrail = False
|
||||
try:
|
||||
from hermes_cli.plugins import get_pre_tool_call_block_message
|
||||
block_message = get_pre_tool_call_block_message(
|
||||
function_name, function_args, task_id=effective_task_id or "",
|
||||
)
|
||||
except Exception:
|
||||
block_message = None
|
||||
|
||||
if block_message is not None:
|
||||
block_result = json.dumps({"error": block_message}, ensure_ascii=False)
|
||||
else:
|
||||
guardrail_decision = self._tool_guardrails.before_call(function_name, function_args)
|
||||
if not guardrail_decision.allows_execution:
|
||||
block_result = self._guardrail_block_result(guardrail_decision)
|
||||
blocked_by_guardrail = True
|
||||
|
||||
parsed_calls.append((tool_call, function_name, function_args, block_result, blocked_by_guardrail))
|
||||
|
||||
# ── Logging / callbacks ──────────────────────────────────────────
|
||||
tool_names_str = ", ".join(name for _, name, _ in parsed_calls)
|
||||
tool_names_str = ", ".join(name for _, name, _, _, _ in parsed_calls)
|
||||
if not self.quiet_mode:
|
||||
print(f" ⚡ Concurrent: {num_tools} tool calls — {tool_names_str}")
|
||||
for i, (tc, name, args) in enumerate(parsed_calls, 1):
|
||||
for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls, 1):
|
||||
args_str = json.dumps(args, ensure_ascii=False)
|
||||
if self.verbose_logging:
|
||||
print(f" 📞 Tool {i}: {name}({list(args.keys())})")
|
||||
@@ -9332,7 +9432,9 @@ class AIAgent:
|
||||
args_preview = args_str[:self.log_prefix_chars] + "..." if len(args_str) > self.log_prefix_chars else args_str
|
||||
print(f" 📞 Tool {i}: {name}({list(args.keys())}) - {args_preview}")
|
||||
|
||||
for tc, name, args in parsed_calls:
|
||||
for tc, name, args, block_result, blocked_by_guardrail in parsed_calls:
|
||||
if block_result is not None:
|
||||
continue
|
||||
if self.tool_progress_callback:
|
||||
try:
|
||||
preview = _build_tool_preview(name, args)
|
||||
@@ -9340,7 +9442,9 @@ class AIAgent:
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool progress callback error: {cb_err}")
|
||||
|
||||
for tc, name, args in parsed_calls:
|
||||
for tc, name, args, block_result, blocked_by_guardrail in parsed_calls:
|
||||
if block_result is not None:
|
||||
continue
|
||||
if self.tool_start_callback:
|
||||
try:
|
||||
self.tool_start_callback(tc.id, name, args)
|
||||
@@ -9348,8 +9452,11 @@ class AIAgent:
|
||||
logging.debug(f"Tool start callback error: {cb_err}")
|
||||
|
||||
# ── Concurrent execution ─────────────────────────────────────────
|
||||
# Each slot holds (function_name, function_args, function_result, duration, error_flag)
|
||||
# Each slot holds (function_name, function_args, function_result, duration, error_flag, blocked_flag)
|
||||
results = [None] * num_tools
|
||||
for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls):
|
||||
if block_result is not None:
|
||||
results[i] = (name, args, block_result, 0.0, True, True)
|
||||
|
||||
# Touch activity before launching workers so the gateway knows
|
||||
# we're executing tools (not stuck).
|
||||
@@ -9404,7 +9511,14 @@ class AIAgent:
|
||||
pass
|
||||
start = time.time()
|
||||
try:
|
||||
result = self._invoke_tool(function_name, function_args, effective_task_id, tool_call.id, messages=messages)
|
||||
result = self._invoke_tool(
|
||||
function_name,
|
||||
function_args,
|
||||
effective_task_id,
|
||||
tool_call.id,
|
||||
messages=messages,
|
||||
pre_tool_block_checked=True,
|
||||
)
|
||||
except Exception as tool_error:
|
||||
result = f"Error executing tool '{function_name}': {tool_error}"
|
||||
logger.error("_invoke_tool raised for %s: %s", function_name, tool_error, exc_info=True)
|
||||
@@ -9414,7 +9528,7 @@ class AIAgent:
|
||||
logger.info("tool %s failed (%.2fs): %s", function_name, duration, result[:200])
|
||||
else:
|
||||
logger.info("tool %s completed (%.2fs, %d chars)", function_name, duration, len(result))
|
||||
results[index] = (function_name, function_args, result, duration, is_error)
|
||||
results[index] = (function_name, function_args, result, duration, is_error, False)
|
||||
# Tear down worker-tid tracking. Clear any interrupt bit we may
|
||||
# have set so the next task scheduled onto this recycled tid
|
||||
# starts with a clean slate.
|
||||
@@ -9440,61 +9554,67 @@ class AIAgent:
|
||||
spinner.start()
|
||||
|
||||
try:
|
||||
max_workers = min(num_tools, _MAX_TOOL_WORKERS)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = []
|
||||
for i, (tc, name, args) in enumerate(parsed_calls):
|
||||
# Propagate ContextVars (e.g. _approval_session_key); mirrors asyncio.to_thread.
|
||||
ctx = contextvars.copy_context()
|
||||
f = executor.submit(ctx.run, _run_tool, i, tc, name, args)
|
||||
futures.append(f)
|
||||
runnable_calls = [
|
||||
(i, tc, name, args)
|
||||
for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls)
|
||||
if block_result is None
|
||||
]
|
||||
futures = []
|
||||
if runnable_calls:
|
||||
max_workers = min(len(runnable_calls), _MAX_TOOL_WORKERS)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
for i, tc, name, args in runnable_calls:
|
||||
# Propagate ContextVars (e.g. _approval_session_key); mirrors asyncio.to_thread.
|
||||
ctx = contextvars.copy_context()
|
||||
f = executor.submit(ctx.run, _run_tool, i, tc, name, args)
|
||||
futures.append(f)
|
||||
|
||||
# Wait for all to complete with periodic heartbeats so the
|
||||
# gateway's inactivity monitor doesn't kill us during long
|
||||
# concurrent tool batches. Also check for user interrupts
|
||||
# so we don't block indefinitely when the user sends /stop
|
||||
# or a new message during concurrent tool execution.
|
||||
_conc_start = time.time()
|
||||
_interrupt_logged = False
|
||||
while True:
|
||||
done, not_done = concurrent.futures.wait(
|
||||
futures, timeout=5.0,
|
||||
)
|
||||
if not not_done:
|
||||
break
|
||||
|
||||
# Check for interrupt — the per-thread interrupt signal
|
||||
# already causes individual tools (terminal, execute_code)
|
||||
# to abort, but tools without interrupt checks (web_search,
|
||||
# read_file) will run to completion. Cancel any futures
|
||||
# that haven't started yet so we don't block on them.
|
||||
if self._interrupt_requested:
|
||||
if not _interrupt_logged:
|
||||
_interrupt_logged = True
|
||||
self._vprint(
|
||||
f"{self.log_prefix}⚡ Interrupt: cancelling "
|
||||
f"{len(not_done)} pending concurrent tool(s)",
|
||||
force=True,
|
||||
)
|
||||
for f in not_done:
|
||||
f.cancel()
|
||||
# Give already-running tools a moment to notice the
|
||||
# per-thread interrupt signal and exit gracefully.
|
||||
concurrent.futures.wait(not_done, timeout=3.0)
|
||||
break
|
||||
|
||||
_conc_elapsed = int(time.time() - _conc_start)
|
||||
# Heartbeat every ~30s (6 × 5s poll intervals)
|
||||
if _conc_elapsed > 0 and _conc_elapsed % 30 < 6:
|
||||
_still_running = [
|
||||
parsed_calls[futures.index(f)][1]
|
||||
for f in not_done
|
||||
if f in futures
|
||||
]
|
||||
self._touch_activity(
|
||||
f"concurrent tools running ({_conc_elapsed}s, "
|
||||
f"{len(not_done)} remaining: {', '.join(_still_running[:3])})"
|
||||
# Wait for all to complete with periodic heartbeats so the
|
||||
# gateway's inactivity monitor doesn't kill us during long
|
||||
# concurrent tool batches. Also check for user interrupts
|
||||
# so we don't block indefinitely when the user sends /stop
|
||||
# or a new message during concurrent tool execution.
|
||||
_conc_start = time.time()
|
||||
_interrupt_logged = False
|
||||
while True:
|
||||
done, not_done = concurrent.futures.wait(
|
||||
futures, timeout=5.0,
|
||||
)
|
||||
if not not_done:
|
||||
break
|
||||
|
||||
# Check for interrupt — the per-thread interrupt signal
|
||||
# already causes individual tools (terminal, execute_code)
|
||||
# to abort, but tools without interrupt checks (web_search,
|
||||
# read_file) will run to completion. Cancel any futures
|
||||
# that haven't started yet so we don't block on them.
|
||||
if self._interrupt_requested:
|
||||
if not _interrupt_logged:
|
||||
_interrupt_logged = True
|
||||
self._vprint(
|
||||
f"{self.log_prefix}⚡ Interrupt: cancelling "
|
||||
f"{len(not_done)} pending concurrent tool(s)",
|
||||
force=True,
|
||||
)
|
||||
for f in not_done:
|
||||
f.cancel()
|
||||
# Give already-running tools a moment to notice the
|
||||
# per-thread interrupt signal and exit gracefully.
|
||||
concurrent.futures.wait(not_done, timeout=3.0)
|
||||
break
|
||||
|
||||
_conc_elapsed = int(time.time() - _conc_start)
|
||||
# Heartbeat every ~30s (6 × 5s poll intervals)
|
||||
if _conc_elapsed > 0 and _conc_elapsed % 30 < 6:
|
||||
_still_running = [
|
||||
parsed_calls[futures.index(f)][1]
|
||||
for f in not_done
|
||||
if f in futures
|
||||
]
|
||||
self._touch_activity(
|
||||
f"concurrent tools running ({_conc_elapsed}s, "
|
||||
f"{len(not_done)} remaining: {', '.join(_still_running[:3])})"
|
||||
)
|
||||
finally:
|
||||
if spinner:
|
||||
# Build a summary message for the spinner stop
|
||||
@@ -9503,8 +9623,9 @@ class AIAgent:
|
||||
spinner.stop(f"⚡ {completed}/{num_tools} tools completed in {total_dur:.1f}s total")
|
||||
|
||||
# ── Post-execution: display per-tool results ─────────────────────
|
||||
for i, (tc, name, args) in enumerate(parsed_calls):
|
||||
for i, (tc, name, args, block_result, blocked_by_guardrail) in enumerate(parsed_calls):
|
||||
r = results[i]
|
||||
blocked = False
|
||||
if r is None:
|
||||
# Tool was cancelled (interrupt) or thread didn't return
|
||||
if self._interrupt_requested:
|
||||
@@ -9513,13 +9634,21 @@ class AIAgent:
|
||||
function_result = f"Error executing tool '{name}': thread did not return a result"
|
||||
tool_duration = 0.0
|
||||
else:
|
||||
function_name, function_args, function_result, tool_duration, is_error = r
|
||||
function_name, function_args, function_result, tool_duration, is_error, blocked = r
|
||||
|
||||
if not blocked:
|
||||
function_result = self._append_guardrail_observation(
|
||||
function_name,
|
||||
function_args,
|
||||
function_result,
|
||||
failed=is_error,
|
||||
)
|
||||
|
||||
if is_error:
|
||||
result_preview = function_result[:200] if len(function_result) > 200 else function_result
|
||||
logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview)
|
||||
|
||||
if self.tool_progress_callback:
|
||||
if not blocked and self.tool_progress_callback:
|
||||
try:
|
||||
self.tool_progress_callback(
|
||||
"tool.completed", function_name, None, None,
|
||||
@@ -9547,7 +9676,7 @@ class AIAgent:
|
||||
self._current_tool = None
|
||||
self._touch_activity(f"tool completed: {name} ({tool_duration:.1f}s)")
|
||||
|
||||
if self.tool_complete_callback:
|
||||
if not blocked and self.tool_complete_callback:
|
||||
try:
|
||||
self.tool_complete_callback(tc.id, name, args, function_result)
|
||||
except Exception as cb_err:
|
||||
@@ -9629,9 +9758,17 @@ class AIAgent:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if _block_msg is not None:
|
||||
# Tool blocked by plugin policy — skip counter resets.
|
||||
# Execution is handled below in the tool dispatch chain.
|
||||
_guardrail_block_decision: ToolGuardrailDecision | None = None
|
||||
if _block_msg is None:
|
||||
guardrail_decision = self._tool_guardrails.before_call(function_name, function_args)
|
||||
if not guardrail_decision.allows_execution:
|
||||
_guardrail_block_decision = guardrail_decision
|
||||
|
||||
_execution_blocked = _block_msg is not None or _guardrail_block_decision is not None
|
||||
|
||||
if _execution_blocked:
|
||||
# Tool blocked by plugin or guardrail policy — skip counters,
|
||||
# callbacks, checkpointing, activity mutation, and real execution.
|
||||
pass
|
||||
else:
|
||||
# Reset nudge counters when the relevant tool is actually used
|
||||
@@ -9649,35 +9786,35 @@ class AIAgent:
|
||||
args_preview = args_str[:self.log_prefix_chars] + "..." if len(args_str) > self.log_prefix_chars else args_str
|
||||
print(f" 📞 Tool {i}: {function_name}({list(function_args.keys())}) - {args_preview}")
|
||||
|
||||
if _block_msg is None:
|
||||
if not _execution_blocked:
|
||||
self._current_tool = function_name
|
||||
self._touch_activity(f"executing tool: {function_name}")
|
||||
|
||||
# Set activity callback for long-running tool execution (terminal
|
||||
# commands, etc.) so the gateway's inactivity monitor doesn't kill
|
||||
# the agent while a command is running.
|
||||
if _block_msg is None:
|
||||
if not _execution_blocked:
|
||||
try:
|
||||
from tools.environments.base import set_activity_callback
|
||||
set_activity_callback(self._touch_activity)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if _block_msg is None and self.tool_progress_callback:
|
||||
if not _execution_blocked and self.tool_progress_callback:
|
||||
try:
|
||||
preview = _build_tool_preview(function_name, function_args)
|
||||
self.tool_progress_callback("tool.started", function_name, preview, function_args)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool progress callback error: {cb_err}")
|
||||
|
||||
if _block_msg is None and self.tool_start_callback:
|
||||
if not _execution_blocked and self.tool_start_callback:
|
||||
try:
|
||||
self.tool_start_callback(tool_call.id, function_name, function_args)
|
||||
except Exception as cb_err:
|
||||
logging.debug(f"Tool start callback error: {cb_err}")
|
||||
|
||||
# Checkpoint: snapshot working dir before file-mutating tools
|
||||
if _block_msg is None and function_name in ("write_file", "patch") and self._checkpoint_mgr.enabled:
|
||||
if not _execution_blocked and function_name in ("write_file", "patch") and self._checkpoint_mgr.enabled:
|
||||
try:
|
||||
file_path = function_args.get("path", "")
|
||||
if file_path:
|
||||
@@ -9689,7 +9826,7 @@ class AIAgent:
|
||||
pass # never block tool execution
|
||||
|
||||
# Checkpoint before destructive terminal commands
|
||||
if _block_msg is None and function_name == "terminal" and self._checkpoint_mgr.enabled:
|
||||
if not _execution_blocked and function_name == "terminal" and self._checkpoint_mgr.enabled:
|
||||
try:
|
||||
cmd = function_args.get("command", "")
|
||||
if _is_destructive_command(cmd):
|
||||
@@ -9706,6 +9843,11 @@ class AIAgent:
|
||||
# Tool blocked by plugin policy — return error without executing.
|
||||
function_result = json.dumps({"error": _block_msg}, ensure_ascii=False)
|
||||
tool_duration = 0.0
|
||||
elif _guardrail_block_decision is not None:
|
||||
# Tool blocked by tool-loop guardrail — synthesize exactly one
|
||||
# tool result for the original tool_call_id without executing.
|
||||
function_result = self._guardrail_block_result(_guardrail_block_decision)
|
||||
tool_duration = 0.0
|
||||
elif function_name == "todo":
|
||||
from tools.todo_tool import todo_tool as _todo_tool
|
||||
function_result = _todo_tool(
|
||||
@@ -9889,12 +10031,22 @@ class AIAgent:
|
||||
# Log tool errors to the persistent error log so [error] tags
|
||||
# in the UI always have a corresponding detailed entry on disk.
|
||||
_is_error_result, _ = _detect_tool_failure(function_name, function_result)
|
||||
if not _execution_blocked:
|
||||
function_result = self._append_guardrail_observation(
|
||||
function_name,
|
||||
function_args,
|
||||
function_result,
|
||||
failed=_is_error_result,
|
||||
)
|
||||
result_preview = function_result if self.verbose_logging else (
|
||||
function_result[:200] if len(function_result) > 200 else function_result
|
||||
)
|
||||
if _is_error_result:
|
||||
logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview)
|
||||
else:
|
||||
logger.info("tool %s completed (%.2fs, %d chars)", function_name, tool_duration, len(function_result))
|
||||
|
||||
if self.tool_progress_callback:
|
||||
if not _execution_blocked and self.tool_progress_callback:
|
||||
try:
|
||||
self.tool_progress_callback(
|
||||
"tool.completed", function_name, None, None,
|
||||
@@ -9910,7 +10062,7 @@ class AIAgent:
|
||||
logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s")
|
||||
logging.debug(f"Tool result ({len(function_result)} chars): {function_result}")
|
||||
|
||||
if self.tool_complete_callback:
|
||||
if not _execution_blocked and self.tool_complete_callback:
|
||||
try:
|
||||
self.tool_complete_callback(tool_call.id, function_name, function_args, function_result)
|
||||
except Exception as cb_err:
|
||||
@@ -10013,6 +10165,13 @@ class AIAgent:
|
||||
for idx, pfm in enumerate(self.prefill_messages):
|
||||
api_messages.insert(sys_offset + idx, pfm.copy())
|
||||
|
||||
# Same safety net as the main loop: repair tool-call/result
|
||||
# pairing before asking for a final summary. Compression and
|
||||
# session resume can leave a tool result whose parent assistant
|
||||
# tool_call was summarized away; Responses API rejects that as
|
||||
# "No tool call found for function call output".
|
||||
api_messages = self._sanitize_api_messages(api_messages)
|
||||
|
||||
# Same safety net as the main loop: drop thinking-only assistant
|
||||
# turns so Anthropic-family providers don't 400 the summary call.
|
||||
api_messages = self._drop_thinking_only_and_merge_users(api_messages)
|
||||
@@ -10194,6 +10353,8 @@ class AIAgent:
|
||||
# Installed once, transparent when streams are healthy, prevents crash on write.
|
||||
_install_safe_stdio()
|
||||
|
||||
self._ensure_db_session()
|
||||
|
||||
# Tag all log records on this thread with the session ID so
|
||||
# ``hermes logs --session <id>`` can filter a single conversation.
|
||||
from hermes_logging import set_session_context
|
||||
@@ -10237,6 +10398,8 @@ class AIAgent:
|
||||
self._last_content_tools_all_housekeeping = False
|
||||
self._mute_post_response = False
|
||||
self._unicode_sanitization_passes = 0
|
||||
self._tool_guardrails.reset_for_turn()
|
||||
self._tool_guardrail_halt_decision = None
|
||||
|
||||
# Pre-turn connection health check: detect and clean up dead TCP
|
||||
# connections left over from provider outages or dropped streams.
|
||||
@@ -13034,6 +13197,16 @@ class AIAgent:
|
||||
|
||||
self._execute_tool_calls(assistant_message, messages, effective_task_id, api_call_count)
|
||||
|
||||
if self._tool_guardrail_halt_decision is not None:
|
||||
decision = self._tool_guardrail_halt_decision
|
||||
_turn_exit_reason = "guardrail_halt"
|
||||
final_response = self._toolguard_controlled_halt_response(decision)
|
||||
self._emit_status(
|
||||
f"⚠️ Tool guardrail halted {decision.tool_name}: {decision.code}"
|
||||
)
|
||||
messages.append({"role": "assistant", "content": final_response})
|
||||
break
|
||||
|
||||
# Reset per-turn retry counters after successful tool
|
||||
# execution so a single truncation doesn't poison the
|
||||
# entire conversation.
|
||||
@@ -13077,7 +13250,13 @@ class AIAgent:
|
||||
# causing premature compression. (#12026)
|
||||
_real_tokens = _compressor.last_prompt_tokens
|
||||
else:
|
||||
_real_tokens = estimate_messages_tokens_rough(messages)
|
||||
# Include tool schemas — with 50+ tools enabled
|
||||
# these add 20-30K tokens the messages-only
|
||||
# estimate misses, which can skip compression
|
||||
# past the configured threshold (#14695).
|
||||
_real_tokens = estimate_request_tokens_rough(
|
||||
messages, tools=self.tools or None
|
||||
)
|
||||
|
||||
if self.compression_enabled and _compressor.should_compress(_real_tokens):
|
||||
self._safe_print(" ⟳ compacting context…")
|
||||
@@ -13560,6 +13739,7 @@ class AIAgent:
|
||||
"messages": messages,
|
||||
"api_calls": api_call_count,
|
||||
"completed": completed,
|
||||
"turn_exit_reason": _turn_exit_reason,
|
||||
"partial": False, # True only when stopped due to invalid tool calls
|
||||
"interrupted": interrupted,
|
||||
"response_previewed": getattr(self, "_response_was_previewed", False),
|
||||
@@ -13579,6 +13759,8 @@ class AIAgent:
|
||||
"cost_status": self.session_cost_status,
|
||||
"cost_source": self.session_cost_source,
|
||||
}
|
||||
if self._tool_guardrail_halt_decision is not None:
|
||||
result["guardrail"] = self._tool_guardrail_halt_decision.to_metadata()
|
||||
# If a /steer landed after the final assistant turn (no more tool
|
||||
# batches to drain into), hand it back to the caller so it can be
|
||||
# delivered as the next user turn instead of being silently lost.
|
||||
|
||||
Reference in New Issue
Block a user