feat(observability): observer-grade telemetry hooks + NeMo-Relay plugin

Adds backend-neutral observer hooks for plugins: session, turn, API
request, tool, approval, and subagent lifecycle events with stable
correlation IDs (session_id, task_id, turn_id, api_request_id,
tool_call_id, parent/child subagent ids). Extends VALID_HOOKS with
api_request_error and subagent_start.

Hot path is zero-cost when no plugin subscribes: has_hook()/presence
checks gate all payload construction, request payloads are returned
by reference when no middleware rewrites, and the sanitized response
payload no longer embeds raw response objects.

Bundles the optional NeMo-Relay observability plugin
(plugins/observability/nemo_relay) as an in-repo consumer of the new
hooks, peer to the existing langfuse plugin. Fails open when the
optional nemo-relay package is not installed.

Authored-by: Bryan Bednarski <bbednarski@nvidia.com>
Salvaged from #29722 onto current main.
This commit is contained in:
Bryan Bednarski
2026-06-03 06:36:46 -07:00
committed by Teknium
parent a78c73f3aa
commit 0d9b7132ff
26 changed files with 3188 additions and 140 deletions
+47 -6
View File
@@ -1,4 +1,5 @@
from unittest.mock import MagicMock, patch
from types import SimpleNamespace
from hermes_cli.plugins import VALID_HOOKS, PluginManager
from cli import HermesCLI
@@ -20,12 +21,18 @@ def test_session_finalize_on_reset(mock_invoke_hook):
cli.new_session(silent=True)
# Check if on_session_finalize was called for the old session
mock_invoke_hook.assert_any_call(
"on_session_finalize", session_id="test-session-id", platform="cli"
assert any(
c.args == ("on_session_finalize",)
and c.kwargs["session_id"] == "test-session-id"
and c.kwargs["platform"] == "cli"
for c in mock_invoke_hook.call_args_list
)
# Check if on_session_reset was called for the new session
mock_invoke_hook.assert_any_call(
"on_session_reset", session_id=cli.session_id, platform="cli"
assert any(
c.args == ("on_session_reset",)
and c.kwargs["session_id"] == cli.session_id
and c.kwargs["platform"] == "cli"
for c in mock_invoke_hook.call_args_list
)
@@ -41,11 +48,45 @@ def test_session_finalize_on_cleanup(mock_invoke_hook):
cli_mod._run_cleanup()
mock_invoke_hook.assert_any_call(
"on_session_finalize", session_id="cleanup-session-id", platform="cli"
assert any(
c.args == ("on_session_finalize",)
and c.kwargs["session_id"] == "cleanup-session-id"
and c.kwargs["platform"] == "cli"
and c.kwargs["reason"] == "shutdown"
for c in mock_invoke_hook.call_args_list
)
@patch("hermes_cli.plugins.invoke_hook")
def test_interrupted_session_end_helper_emits_observer_shape(mock_invoke_hook):
"""Verify quiet single-query interruption emits a correlated session end."""
import cli as cli_mod
mock_agent = MagicMock()
mock_agent.session_id = "agent-session-id"
mock_agent.model = "test-model"
mock_agent.platform = "cli"
mock_agent._current_task_id = "task-1"
mock_agent._current_turn_id = "turn-1"
mock_agent._current_api_request_id = "api-1"
cli = SimpleNamespace(agent=mock_agent, session_id="cli-session-id")
cli_mod._emit_interrupted_session_end(cli, reason="keyboard_interrupt")
mock_agent.interrupt.assert_called_once_with("keyboard interrupt")
assert cli.session_id == "agent-session-id"
mock_invoke_hook.assert_called_once()
call = mock_invoke_hook.call_args
assert call.args == ("on_session_end",)
assert call.kwargs["session_id"] == "agent-session-id"
assert call.kwargs["task_id"] == "task-1"
assert call.kwargs["turn_id"] == "turn-1"
assert call.kwargs["api_request_id"] == "api-1"
assert call.kwargs["completed"] is False
assert call.kwargs["interrupted"] is True
assert call.kwargs["reason"] == "keyboard_interrupt"
@patch("hermes_cli.plugins.invoke_hook")
def test_hook_errors_are_caught(mock_invoke_hook):
"""Verify hook exceptions are caught and don't crash the agent."""