feat(observability): observer-grade telemetry hooks + NeMo-Relay plugin

Adds backend-neutral observer hooks for plugins: session, turn, API
request, tool, approval, and subagent lifecycle events with stable
correlation IDs (session_id, task_id, turn_id, api_request_id,
tool_call_id, parent/child subagent ids). Extends VALID_HOOKS with
api_request_error and subagent_start.

Hot path is zero-cost when no plugin subscribes: has_hook()/presence
checks gate all payload construction, request payloads are returned
by reference when no middleware rewrites, and the sanitized response
payload no longer embeds raw response objects.

Bundles the optional NeMo-Relay observability plugin
(plugins/observability/nemo_relay) as an in-repo consumer of the new
hooks, peer to the existing langfuse plugin. Fails open when the
optional nemo-relay package is not installed.

Authored-by: Bryan Bednarski <bbednarski@nvidia.com>
Salvaged from #29722 onto current main.
This commit is contained in:
Bryan Bednarski
2026-06-03 06:36:46 -07:00
committed by Teknium
parent a78c73f3aa
commit 0d9b7132ff
26 changed files with 3188 additions and 140 deletions
+24 -1
View File
@@ -323,6 +323,8 @@ class TestPluginHooks:
def test_valid_hooks_include_request_scoped_api_hooks(self):
assert "pre_api_request" in VALID_HOOKS
assert "post_api_request" in VALID_HOOKS
assert "api_request_error" in VALID_HOOKS
assert "subagent_start" in VALID_HOOKS
assert "transform_terminal_output" in VALID_HOOKS
assert "transform_tool_result" in VALID_HOOKS
assert "transform_llm_output" in VALID_HOOKS
@@ -369,6 +371,26 @@ class TestPluginHooks:
# Should not raise
mgr.invoke_hook("pre_tool_call", tool_name="test", args={}, task_id="t1")
def test_invoke_hook_adds_observer_schema_version(self, tmp_path, monkeypatch):
"""invoke_hook() supplies the observer schema version for all hooks."""
plugins_dir = tmp_path / "hermes_test" / "plugins"
_make_plugin_dir(
plugins_dir,
"schema_plugin",
register_body=(
'ctx.register_hook("pre_tool_call", '
'lambda **kw: kw.get("telemetry_schema_version"))'
),
)
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes_test"))
mgr = PluginManager()
mgr.discover_and_load()
assert mgr.invoke_hook("pre_tool_call", tool_name="test", args={}) == [
"hermes.observer.v1"
]
def test_hook_exception_does_not_propagate(self, tmp_path, monkeypatch):
"""A hook callback that raises does NOT crash the caller."""
plugins_dir = tmp_path / "hermes_test" / "plugins"
@@ -435,6 +457,8 @@ class TestPluginHooks:
mgr = PluginManager()
mgr.discover_and_load()
assert mgr.has_hook("pre_api_request") is True
assert mgr.has_hook("post_api_request") is False
results = mgr.invoke_hook(
"pre_api_request",
session_id="s1",
@@ -488,7 +512,6 @@ class TestPluginHooks:
assert any("on_banana" in record.message for record in caplog.records)
class TestPreToolCallBlocking:
"""Tests for the pre_tool_call block directive helper."""