gateway: capture real provider-reported cost (openrouter usage accounting)

Cost displays were estimates from a pricing table; on OpenRouter the
status bar never reflected what was actually charged. Now cost is
provider-REPORTED only, end to end:

- OpenRouter requests carry usage:{include:true} (profile + legacy
  transport paths); the response usage.cost field (credits, 1:1 USD)
  is captured per call into agent.session_actual_cost_usd and
  persisted to the sessions DB actual_cost_usd column (NULL-safe:
  unreported calls never touch the stored value).
- Nous keeps its x-nous-credits-* header capture; the header delta
  now surfaces as the session's real cost via real_session_cost_usd.
- Providers that report nothing accumulate NOTHING: cost fields stay
  absent/None (the TUI hides its cost segment), never a fabricated
  $0.00 and never an estimate. _get_usage, gateway /usage and the
  CLI usage page all switched off estimate_usage_cost for display.
- Per-model session accumulator (session_model_usage) records real
  per-call counts and provider-reported cost per model.
This commit is contained in:
alt-glitch
2026-06-11 00:14:21 +05:30
parent ba3fe7027c
commit 85546bb9e2
13 changed files with 427 additions and 86 deletions
+186
View File
@@ -0,0 +1,186 @@
"""Real provider-reported cost capture — never estimated, absent ≠ zero.
Covers the three fixture shapes from the cost-tracking fix:
- OpenRouter usage accounting: response ``usage.cost`` present → accumulates.
- Nous: ``x-nous-credits-*`` headers present → header delta accumulates.
- Provider reports nothing → cost stays None/absent (NOT zero-as-real).
"""
from types import SimpleNamespace
import pytest
from agent.usage_pricing import extract_provider_cost_usd, real_session_cost_usd
# ── extract_provider_cost_usd — the per-response REAL cost reader ────────────
class TestExtractProviderCost:
def test_openrouter_usage_cost_attr(self):
usage = SimpleNamespace(prompt_tokens=10, completion_tokens=5, cost=0.001234)
assert extract_provider_cost_usd(usage) == pytest.approx(0.001234)
def test_dict_shaped_usage(self):
assert extract_provider_cost_usd({"cost": 0.5}) == pytest.approx(0.5)
def test_reported_zero_is_real_zero(self):
# Free-tier models really cost $0 — distinct from "not reported".
usage = SimpleNamespace(cost=0)
assert extract_provider_cost_usd(usage) == 0.0
def test_absent_cost_is_none_not_zero(self):
usage = SimpleNamespace(prompt_tokens=10, completion_tokens=5)
assert extract_provider_cost_usd(usage) is None
assert extract_provider_cost_usd({"prompt_tokens": 10}) is None
def test_none_usage_is_none(self):
assert extract_provider_cost_usd(None) is None
def test_garbage_cost_values_are_none(self):
for bad in ("0.01", True, float("nan"), float("inf"), -0.5, [], {}):
assert extract_provider_cost_usd(SimpleNamespace(cost=bad)) is None, bad
# ── real_session_cost_usd — the session accumulator surface ─────────────────
class _FakeAgent:
def __init__(self, actual=None, credits_micros=None):
self.session_actual_cost_usd = actual
self._credits_micros = credits_micros
def get_credits_spent_micros(self):
return self._credits_micros
class TestRealSessionCost:
def test_nothing_reported_is_none(self):
assert real_session_cost_usd(_FakeAgent()) is None
def test_openrouter_accumulator_only(self):
assert real_session_cost_usd(_FakeAgent(actual=0.42)) == pytest.approx(0.42)
def test_nous_credits_delta_only(self):
# 123_400 micros = $0.1234
assert real_session_cost_usd(
_FakeAgent(credits_micros=123_400)
) == pytest.approx(0.1234)
def test_both_sources_sum(self):
assert real_session_cost_usd(
_FakeAgent(actual=0.10, credits_micros=200_000)
) == pytest.approx(0.30)
def test_negative_credits_delta_clamped(self):
# A mid-session top-up makes the delta negative — never show negative cost.
assert real_session_cost_usd(_FakeAgent(credits_micros=-50_000)) == 0.0
def test_agent_without_credits_method(self):
agent = SimpleNamespace(session_actual_cost_usd=None)
assert real_session_cost_usd(agent) is None
def test_non_numeric_actual_ignored(self):
agent = _FakeAgent()
agent.session_actual_cost_usd = "0.42" # corrupted attr → ignore
assert real_session_cost_usd(agent) is None
# ── Nous header fixture → real accumulator (full _capture_credits path) ─────
def _nous_headers(remaining_micros: int) -> dict:
return {
"x-nous-credits-version": "1",
"x-nous-credits-remaining-micros": str(remaining_micros),
"x-nous-credits-remaining-usd": f"{remaining_micros / 1_000_000:.2f}",
"x-nous-credits-subscription-micros": str(remaining_micros),
"x-nous-credits-subscription-usd": f"{remaining_micros / 1_000_000:.2f}",
"x-nous-credits-rollover-micros": "0",
"x-nous-credits-purchased-micros": "0",
"x-nous-credits-purchased-usd": "0.00",
"x-nous-credits-denominator-kind": "none",
"x-nous-credits-paid-access": "true",
"x-nous-credits-as-of-ms": "1717000000000",
}
def _bare_nous_agent():
"""Minimal AIAgent shell exercising the real _capture_credits path."""
from run_agent import AIAgent
agent = object.__new__(AIAgent)
agent.provider = "nous"
agent._credits_state = None
agent._credits_session_start_micros = None
agent.notice_callback = None
agent.notice_clear_callback = None
agent.session_actual_cost_usd = None
return agent
class TestNousHeaderAccumulation:
def test_headers_accumulate_into_real_session_cost(self, monkeypatch):
monkeypatch.delenv("HERMES_DEV_CREDITS_FIXTURE", raising=False)
agent = _bare_nous_agent()
# First response latches the session-start balance ($10.00).
agent._capture_credits(SimpleNamespace(headers=_nous_headers(10_000_000)))
assert real_session_cost_usd(agent) == 0.0 # real zero: headers seen, $0 spent
# Second response: balance dropped by $0.25 → real reported spend.
agent._capture_credits(SimpleNamespace(headers=_nous_headers(9_750_000)))
assert real_session_cost_usd(agent) == pytest.approx(0.25)
def test_no_headers_means_no_cost(self, monkeypatch):
monkeypatch.delenv("HERMES_DEV_CREDITS_FIXTURE", raising=False)
agent = _bare_nous_agent()
agent._capture_credits(SimpleNamespace(headers={"content-type": "application/json"}))
assert real_session_cost_usd(agent) is None
# ── OpenRouter request param — usage accounting must be requested ────────────
class TestOpenRouterUsageParam:
def test_profile_extra_body_requests_usage_accounting(self):
import importlib.util
from pathlib import Path
from providers import get_provider_profile
profile = get_provider_profile("openrouter")
if profile is None:
# Force plugin discovery in minimal test envs.
plugin = Path(__file__).resolve().parents[2] / "plugins" / "model-providers" / "openrouter" / "__init__.py"
spec = importlib.util.spec_from_file_location("_or_plugin", plugin)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
profile = mod.openrouter
body = profile.build_extra_body(session_id="s-1")
assert body["usage"] == {"include": True}
def test_legacy_transport_path_requests_usage_accounting(self):
from agent.transports.chat_completions import ChatCompletionsTransport
transport = ChatCompletionsTransport()
kwargs = transport.build_kwargs(
model="anthropic/claude-sonnet-4.6",
messages=[{"role": "user", "content": "hi"}],
tools=None,
is_openrouter=True,
)
assert kwargs["extra_body"]["usage"] == {"include": True}
def test_non_openrouter_does_not_send_usage_param(self):
from agent.transports.chat_completions import ChatCompletionsTransport
transport = ChatCompletionsTransport()
kwargs = transport.build_kwargs(
model="deepseek-chat",
messages=[{"role": "user", "content": "hi"}],
tools=None,
is_openrouter=False,
)
assert "usage" not in (kwargs.get("extra_body") or {})
+16 -13
View File
@@ -523,7 +523,7 @@ class TestCLIStatusBar:
class TestCLIUsageReport:
def test_show_usage_includes_estimated_cost(self, capsys):
def test_show_usage_reports_real_provider_cost(self, capsys):
cli_obj = _attach_agent(
_make_cli(),
prompt_tokens=10_230,
@@ -535,20 +535,22 @@ class TestCLIUsageReport:
compressions=1,
)
cli_obj.verbose = False
# Provider-reported cost (e.g. OpenRouter usage accounting accumulator).
cli_obj.agent.session_actual_cost_usd = 0.0640
cli_obj._show_usage()
output = capsys.readouterr().out
assert "Model:" in output
assert "Cost status:" in output
assert "Cost source:" in output
assert "Total cost:" in output
assert "Cost (provider-reported):" in output
assert "$" in output
assert "0.064" in output
assert "Session duration:" in output
assert "Compressions:" in output
def test_show_usage_marks_unknown_pricing(self, capsys):
def test_show_usage_unreported_cost_is_not_a_dollar_figure(self, capsys):
"""No estimation: when the provider reports nothing, /usage must NOT
fabricate a dollar amount — not even $0.00."""
cli_obj = _attach_agent(
_make_cli(model="local/my-custom-model"),
prompt_tokens=1_000,
@@ -563,13 +565,15 @@ class TestCLIUsageReport:
cli_obj._show_usage()
output = capsys.readouterr().out
assert "Total cost:" in output
assert "n/a" in output
assert "Pricing unknown for local/my-custom-model" in output
assert "not reported by provider" in output
assert "Cost (provider-reported):" not in output
assert "$0.00" not in output
def test_zero_priced_provider_models_stay_unknown(self, capsys):
def test_show_usage_never_estimates_even_with_known_pricing(self, capsys):
"""A model with a pricing-table entry must still show NO cost when the
provider reported nothing (hard requirement: real cost only)."""
cli_obj = _attach_agent(
_make_cli(model="glm-5"),
_make_cli(model="anthropic/claude-sonnet-4-6"),
prompt_tokens=1_000,
completion_tokens=500,
total_tokens=1_500,
@@ -582,9 +586,8 @@ class TestCLIUsageReport:
cli_obj._show_usage()
output = capsys.readouterr().out
assert "Total cost:" in output
assert "n/a" in output
assert "Pricing unknown for glm-5" in output
assert "not reported by provider" in output
assert "Cost (provider-reported):" not in output
class TestStatusBarWidthSource:
+51 -19
View File
@@ -21,11 +21,16 @@ def _make_mock_agent(**overrides):
"session_output_tokens": 10_000,
"session_cache_read_tokens": 5_000,
"session_cache_write_tokens": 2_000,
# Real provider-reported cost: None = nothing reported (the default).
"session_actual_cost_usd": None,
}
defaults.update(overrides)
for k, v in defaults.items():
setattr(agent, k, v)
# No Nous credits headers seen unless a test overrides this.
agent.get_credits_spent_micros = MagicMock(return_value=None)
# Rate limit state
rl = MagicMock()
rl.has_data = True
@@ -72,13 +77,11 @@ class TestUsageCachedAgent:
@pytest.mark.asyncio
async def test_cached_agent_shows_detailed_usage(self):
agent = _make_mock_agent()
agent = _make_mock_agent(session_actual_cost_usd=0.1234)
runner = _make_runner(SK, cached_agent=agent)
event = MagicMock()
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
mock_cost.return_value = MagicMock(amount_usd=0.1234, status="estimated")
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "claude-sonnet-4.6" in result
@@ -99,9 +102,7 @@ class TestUsageCachedAgent:
runner = _make_runner(SK, agent=running, cached_agent=cached)
event = MagicMock()
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
mock_cost.return_value = MagicMock(amount_usd=None, status="unknown")
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "80,000" in result # running agent's total
@@ -117,9 +118,7 @@ class TestUsageCachedAgent:
runner._running_agents[SK] = _AGENT_PENDING_SENTINEL
event = MagicMock()
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
mock_cost.return_value = MagicMock(amount_usd=None, status="unknown")
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "claude-sonnet-4.6" in result
@@ -153,9 +152,7 @@ class TestUsageCachedAgent:
runner = _make_runner(SK, cached_agent=agent)
event = MagicMock()
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
mock_cost.return_value = MagicMock(amount_usd=None, status="unknown")
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "Cache read" not in result
@@ -168,9 +165,7 @@ class TestUsageCachedAgent:
runner = _make_runner(SK, cached_agent=agent)
event = MagicMock()
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
mock_cost.return_value = MagicMock(amount_usd=None, status="included")
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "Cost: included" in result
@@ -199,9 +194,7 @@ class TestUsageAccountSection:
"Session: 85% remaining (15% used)",
],
)
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
mock_cost.return_value = MagicMock(amount_usd=None, status="included")
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "📊 **Session Token Usage**" in result
@@ -256,3 +249,42 @@ class TestUsageAccountSection:
assert account_call["kwargs"]["base_url"] == "https://chatgpt.com/backend-api/codex"
assert "📊 **Session Info**" in result
assert "📈 **Account limits**" in result
class TestUsageRealCostOnly:
"""Cost lines are provider-REPORTED only — never estimated, never $0.00."""
@pytest.mark.asyncio
async def test_unreported_cost_renders_no_cost_line(self):
agent = _make_mock_agent() # openrouter, nothing reported
runner = _make_runner(SK, cached_agent=agent)
event = MagicMock()
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "Cost:" not in result
assert "$0.00" not in result
@pytest.mark.asyncio
async def test_nous_credits_delta_renders_as_cost(self):
agent = _make_mock_agent(provider="nous", model="Hermes-4.1-405B")
agent.get_credits_spent_micros = MagicMock(return_value=123_400)
runner = _make_runner(SK, cached_agent=agent)
event = MagicMock()
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "$0.1234" in result
@pytest.mark.asyncio
async def test_openrouter_reported_cost_renders(self):
agent = _make_mock_agent(session_actual_cost_usd=0.9876)
runner = _make_runner(SK, cached_agent=agent)
event = MagicMock()
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
result = await runner._handle_usage_command(event)
assert "$0.9876" in result
+3 -1
View File
@@ -110,7 +110,9 @@ class TestOpenRouterProfile:
def test_extra_body_no_prefs(self):
p = get_provider_profile("openrouter")
body = p.build_extra_body()
assert body == {}
# Usage accounting is always requested (real provider-reported cost);
# nothing else should appear without prefs/session.
assert body == {"usage": {"include": True}}
def test_pareto_min_coding_score_emitted_for_pareto_model(self):
"""min_coding_score → plugins block when model is openrouter/pareto-code."""