feat(observability): observer-grade telemetry hooks + NeMo-Relay plugin
Adds backend-neutral observer hooks for plugins: session, turn, API request, tool, approval, and subagent lifecycle events with stable correlation IDs (session_id, task_id, turn_id, api_request_id, tool_call_id, parent/child subagent ids). Extends VALID_HOOKS with api_request_error and subagent_start. Hot path is zero-cost when no plugin subscribes: has_hook()/presence checks gate all payload construction, request payloads are returned by reference when no middleware rewrites, and the sanitized response payload no longer embeds raw response objects. Bundles the optional NeMo-Relay observability plugin (plugins/observability/nemo_relay) as an in-repo consumer of the new hooks, peer to the existing langfuse plugin. Fails open when the optional nemo-relay package is not installed. Authored-by: Bryan Bednarski <bbednarski@nvidia.com> Salvaged from #29722 onto current main.
This commit is contained in:
@@ -323,6 +323,8 @@ class TestPluginHooks:
|
||||
def test_valid_hooks_include_request_scoped_api_hooks(self):
|
||||
assert "pre_api_request" in VALID_HOOKS
|
||||
assert "post_api_request" in VALID_HOOKS
|
||||
assert "api_request_error" in VALID_HOOKS
|
||||
assert "subagent_start" in VALID_HOOKS
|
||||
assert "transform_terminal_output" in VALID_HOOKS
|
||||
assert "transform_tool_result" in VALID_HOOKS
|
||||
assert "transform_llm_output" in VALID_HOOKS
|
||||
@@ -369,6 +371,26 @@ class TestPluginHooks:
|
||||
# Should not raise
|
||||
mgr.invoke_hook("pre_tool_call", tool_name="test", args={}, task_id="t1")
|
||||
|
||||
def test_invoke_hook_adds_observer_schema_version(self, tmp_path, monkeypatch):
|
||||
"""invoke_hook() supplies the observer schema version for all hooks."""
|
||||
plugins_dir = tmp_path / "hermes_test" / "plugins"
|
||||
_make_plugin_dir(
|
||||
plugins_dir,
|
||||
"schema_plugin",
|
||||
register_body=(
|
||||
'ctx.register_hook("pre_tool_call", '
|
||||
'lambda **kw: kw.get("telemetry_schema_version"))'
|
||||
),
|
||||
)
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes_test"))
|
||||
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert mgr.invoke_hook("pre_tool_call", tool_name="test", args={}) == [
|
||||
"hermes.observer.v1"
|
||||
]
|
||||
|
||||
def test_hook_exception_does_not_propagate(self, tmp_path, monkeypatch):
|
||||
"""A hook callback that raises does NOT crash the caller."""
|
||||
plugins_dir = tmp_path / "hermes_test" / "plugins"
|
||||
@@ -435,6 +457,8 @@ class TestPluginHooks:
|
||||
mgr = PluginManager()
|
||||
mgr.discover_and_load()
|
||||
|
||||
assert mgr.has_hook("pre_api_request") is True
|
||||
assert mgr.has_hook("post_api_request") is False
|
||||
results = mgr.invoke_hook(
|
||||
"pre_api_request",
|
||||
session_id="s1",
|
||||
@@ -488,7 +512,6 @@ class TestPluginHooks:
|
||||
|
||||
assert any("on_banana" in record.message for record in caplog.records)
|
||||
|
||||
|
||||
class TestPreToolCallBlocking:
|
||||
"""Tests for the pre_tool_call block directive helper."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user