gateway: capture real provider-reported cost (openrouter usage accounting)
Cost displays were estimates from a pricing table; on OpenRouter the
status bar never reflected what was actually charged. Now cost is
provider-REPORTED only, end to end:
- OpenRouter requests carry usage:{include:true} (profile + legacy
transport paths); the response usage.cost field (credits, 1:1 USD)
is captured per call into agent.session_actual_cost_usd and
persisted to the sessions DB actual_cost_usd column (NULL-safe:
unreported calls never touch the stored value).
- Nous keeps its x-nous-credits-* header capture; the header delta
now surfaces as the session's real cost via real_session_cost_usd.
- Providers that report nothing accumulate NOTHING: cost fields stay
absent/None (the TUI hides its cost segment), never a fabricated
$0.00 and never an estimate. _get_usage, gateway /usage and the
CLI usage page all switched off estimate_usage_cost for display.
- Per-model session accumulator (session_model_usage) records real
per-call counts and provider-reported cost per model.
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
"""Real provider-reported cost capture — never estimated, absent ≠ zero.
|
||||
|
||||
Covers the three fixture shapes from the cost-tracking fix:
|
||||
- OpenRouter usage accounting: response ``usage.cost`` present → accumulates.
|
||||
- Nous: ``x-nous-credits-*`` headers present → header delta accumulates.
|
||||
- Provider reports nothing → cost stays None/absent (NOT zero-as-real).
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.usage_pricing import extract_provider_cost_usd, real_session_cost_usd
|
||||
|
||||
|
||||
# ── extract_provider_cost_usd — the per-response REAL cost reader ────────────
|
||||
|
||||
|
||||
class TestExtractProviderCost:
|
||||
def test_openrouter_usage_cost_attr(self):
|
||||
usage = SimpleNamespace(prompt_tokens=10, completion_tokens=5, cost=0.001234)
|
||||
assert extract_provider_cost_usd(usage) == pytest.approx(0.001234)
|
||||
|
||||
def test_dict_shaped_usage(self):
|
||||
assert extract_provider_cost_usd({"cost": 0.5}) == pytest.approx(0.5)
|
||||
|
||||
def test_reported_zero_is_real_zero(self):
|
||||
# Free-tier models really cost $0 — distinct from "not reported".
|
||||
usage = SimpleNamespace(cost=0)
|
||||
assert extract_provider_cost_usd(usage) == 0.0
|
||||
|
||||
def test_absent_cost_is_none_not_zero(self):
|
||||
usage = SimpleNamespace(prompt_tokens=10, completion_tokens=5)
|
||||
assert extract_provider_cost_usd(usage) is None
|
||||
assert extract_provider_cost_usd({"prompt_tokens": 10}) is None
|
||||
|
||||
def test_none_usage_is_none(self):
|
||||
assert extract_provider_cost_usd(None) is None
|
||||
|
||||
def test_garbage_cost_values_are_none(self):
|
||||
for bad in ("0.01", True, float("nan"), float("inf"), -0.5, [], {}):
|
||||
assert extract_provider_cost_usd(SimpleNamespace(cost=bad)) is None, bad
|
||||
|
||||
|
||||
# ── real_session_cost_usd — the session accumulator surface ─────────────────
|
||||
|
||||
|
||||
class _FakeAgent:
|
||||
def __init__(self, actual=None, credits_micros=None):
|
||||
self.session_actual_cost_usd = actual
|
||||
self._credits_micros = credits_micros
|
||||
|
||||
def get_credits_spent_micros(self):
|
||||
return self._credits_micros
|
||||
|
||||
|
||||
class TestRealSessionCost:
|
||||
def test_nothing_reported_is_none(self):
|
||||
assert real_session_cost_usd(_FakeAgent()) is None
|
||||
|
||||
def test_openrouter_accumulator_only(self):
|
||||
assert real_session_cost_usd(_FakeAgent(actual=0.42)) == pytest.approx(0.42)
|
||||
|
||||
def test_nous_credits_delta_only(self):
|
||||
# 123_400 micros = $0.1234
|
||||
assert real_session_cost_usd(
|
||||
_FakeAgent(credits_micros=123_400)
|
||||
) == pytest.approx(0.1234)
|
||||
|
||||
def test_both_sources_sum(self):
|
||||
assert real_session_cost_usd(
|
||||
_FakeAgent(actual=0.10, credits_micros=200_000)
|
||||
) == pytest.approx(0.30)
|
||||
|
||||
def test_negative_credits_delta_clamped(self):
|
||||
# A mid-session top-up makes the delta negative — never show negative cost.
|
||||
assert real_session_cost_usd(_FakeAgent(credits_micros=-50_000)) == 0.0
|
||||
|
||||
def test_agent_without_credits_method(self):
|
||||
agent = SimpleNamespace(session_actual_cost_usd=None)
|
||||
assert real_session_cost_usd(agent) is None
|
||||
|
||||
def test_non_numeric_actual_ignored(self):
|
||||
agent = _FakeAgent()
|
||||
agent.session_actual_cost_usd = "0.42" # corrupted attr → ignore
|
||||
assert real_session_cost_usd(agent) is None
|
||||
|
||||
|
||||
# ── Nous header fixture → real accumulator (full _capture_credits path) ─────
|
||||
|
||||
|
||||
def _nous_headers(remaining_micros: int) -> dict:
|
||||
return {
|
||||
"x-nous-credits-version": "1",
|
||||
"x-nous-credits-remaining-micros": str(remaining_micros),
|
||||
"x-nous-credits-remaining-usd": f"{remaining_micros / 1_000_000:.2f}",
|
||||
"x-nous-credits-subscription-micros": str(remaining_micros),
|
||||
"x-nous-credits-subscription-usd": f"{remaining_micros / 1_000_000:.2f}",
|
||||
"x-nous-credits-rollover-micros": "0",
|
||||
"x-nous-credits-purchased-micros": "0",
|
||||
"x-nous-credits-purchased-usd": "0.00",
|
||||
"x-nous-credits-denominator-kind": "none",
|
||||
"x-nous-credits-paid-access": "true",
|
||||
"x-nous-credits-as-of-ms": "1717000000000",
|
||||
}
|
||||
|
||||
|
||||
def _bare_nous_agent():
|
||||
"""Minimal AIAgent shell exercising the real _capture_credits path."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
agent = object.__new__(AIAgent)
|
||||
agent.provider = "nous"
|
||||
agent._credits_state = None
|
||||
agent._credits_session_start_micros = None
|
||||
agent.notice_callback = None
|
||||
agent.notice_clear_callback = None
|
||||
agent.session_actual_cost_usd = None
|
||||
return agent
|
||||
|
||||
|
||||
class TestNousHeaderAccumulation:
|
||||
def test_headers_accumulate_into_real_session_cost(self, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_DEV_CREDITS_FIXTURE", raising=False)
|
||||
agent = _bare_nous_agent()
|
||||
|
||||
# First response latches the session-start balance ($10.00).
|
||||
agent._capture_credits(SimpleNamespace(headers=_nous_headers(10_000_000)))
|
||||
assert real_session_cost_usd(agent) == 0.0 # real zero: headers seen, $0 spent
|
||||
|
||||
# Second response: balance dropped by $0.25 → real reported spend.
|
||||
agent._capture_credits(SimpleNamespace(headers=_nous_headers(9_750_000)))
|
||||
assert real_session_cost_usd(agent) == pytest.approx(0.25)
|
||||
|
||||
def test_no_headers_means_no_cost(self, monkeypatch):
|
||||
monkeypatch.delenv("HERMES_DEV_CREDITS_FIXTURE", raising=False)
|
||||
agent = _bare_nous_agent()
|
||||
agent._capture_credits(SimpleNamespace(headers={"content-type": "application/json"}))
|
||||
assert real_session_cost_usd(agent) is None
|
||||
|
||||
|
||||
# ── OpenRouter request param — usage accounting must be requested ────────────
|
||||
|
||||
|
||||
class TestOpenRouterUsageParam:
|
||||
def test_profile_extra_body_requests_usage_accounting(self):
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
from providers import get_provider_profile
|
||||
|
||||
profile = get_provider_profile("openrouter")
|
||||
if profile is None:
|
||||
# Force plugin discovery in minimal test envs.
|
||||
plugin = Path(__file__).resolve().parents[2] / "plugins" / "model-providers" / "openrouter" / "__init__.py"
|
||||
spec = importlib.util.spec_from_file_location("_or_plugin", plugin)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
profile = mod.openrouter
|
||||
|
||||
body = profile.build_extra_body(session_id="s-1")
|
||||
assert body["usage"] == {"include": True}
|
||||
|
||||
def test_legacy_transport_path_requests_usage_accounting(self):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
transport = ChatCompletionsTransport()
|
||||
kwargs = transport.build_kwargs(
|
||||
model="anthropic/claude-sonnet-4.6",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=None,
|
||||
is_openrouter=True,
|
||||
)
|
||||
assert kwargs["extra_body"]["usage"] == {"include": True}
|
||||
|
||||
def test_non_openrouter_does_not_send_usage_param(self):
|
||||
from agent.transports.chat_completions import ChatCompletionsTransport
|
||||
|
||||
transport = ChatCompletionsTransport()
|
||||
kwargs = transport.build_kwargs(
|
||||
model="deepseek-chat",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=None,
|
||||
is_openrouter=False,
|
||||
)
|
||||
assert "usage" not in (kwargs.get("extra_body") or {})
|
||||
@@ -523,7 +523,7 @@ class TestCLIStatusBar:
|
||||
|
||||
|
||||
class TestCLIUsageReport:
|
||||
def test_show_usage_includes_estimated_cost(self, capsys):
|
||||
def test_show_usage_reports_real_provider_cost(self, capsys):
|
||||
cli_obj = _attach_agent(
|
||||
_make_cli(),
|
||||
prompt_tokens=10_230,
|
||||
@@ -535,20 +535,22 @@ class TestCLIUsageReport:
|
||||
compressions=1,
|
||||
)
|
||||
cli_obj.verbose = False
|
||||
# Provider-reported cost (e.g. OpenRouter usage accounting accumulator).
|
||||
cli_obj.agent.session_actual_cost_usd = 0.0640
|
||||
|
||||
cli_obj._show_usage()
|
||||
output = capsys.readouterr().out
|
||||
|
||||
assert "Model:" in output
|
||||
assert "Cost status:" in output
|
||||
assert "Cost source:" in output
|
||||
assert "Total cost:" in output
|
||||
assert "Cost (provider-reported):" in output
|
||||
assert "$" in output
|
||||
assert "0.064" in output
|
||||
assert "Session duration:" in output
|
||||
assert "Compressions:" in output
|
||||
|
||||
def test_show_usage_marks_unknown_pricing(self, capsys):
|
||||
def test_show_usage_unreported_cost_is_not_a_dollar_figure(self, capsys):
|
||||
"""No estimation: when the provider reports nothing, /usage must NOT
|
||||
fabricate a dollar amount — not even $0.00."""
|
||||
cli_obj = _attach_agent(
|
||||
_make_cli(model="local/my-custom-model"),
|
||||
prompt_tokens=1_000,
|
||||
@@ -563,13 +565,15 @@ class TestCLIUsageReport:
|
||||
cli_obj._show_usage()
|
||||
output = capsys.readouterr().out
|
||||
|
||||
assert "Total cost:" in output
|
||||
assert "n/a" in output
|
||||
assert "Pricing unknown for local/my-custom-model" in output
|
||||
assert "not reported by provider" in output
|
||||
assert "Cost (provider-reported):" not in output
|
||||
assert "$0.00" not in output
|
||||
|
||||
def test_zero_priced_provider_models_stay_unknown(self, capsys):
|
||||
def test_show_usage_never_estimates_even_with_known_pricing(self, capsys):
|
||||
"""A model with a pricing-table entry must still show NO cost when the
|
||||
provider reported nothing (hard requirement: real cost only)."""
|
||||
cli_obj = _attach_agent(
|
||||
_make_cli(model="glm-5"),
|
||||
_make_cli(model="anthropic/claude-sonnet-4-6"),
|
||||
prompt_tokens=1_000,
|
||||
completion_tokens=500,
|
||||
total_tokens=1_500,
|
||||
@@ -582,9 +586,8 @@ class TestCLIUsageReport:
|
||||
cli_obj._show_usage()
|
||||
output = capsys.readouterr().out
|
||||
|
||||
assert "Total cost:" in output
|
||||
assert "n/a" in output
|
||||
assert "Pricing unknown for glm-5" in output
|
||||
assert "not reported by provider" in output
|
||||
assert "Cost (provider-reported):" not in output
|
||||
|
||||
|
||||
class TestStatusBarWidthSource:
|
||||
|
||||
@@ -21,11 +21,16 @@ def _make_mock_agent(**overrides):
|
||||
"session_output_tokens": 10_000,
|
||||
"session_cache_read_tokens": 5_000,
|
||||
"session_cache_write_tokens": 2_000,
|
||||
# Real provider-reported cost: None = nothing reported (the default).
|
||||
"session_actual_cost_usd": None,
|
||||
}
|
||||
defaults.update(overrides)
|
||||
for k, v in defaults.items():
|
||||
setattr(agent, k, v)
|
||||
|
||||
# No Nous credits headers seen unless a test overrides this.
|
||||
agent.get_credits_spent_micros = MagicMock(return_value=None)
|
||||
|
||||
# Rate limit state
|
||||
rl = MagicMock()
|
||||
rl.has_data = True
|
||||
@@ -72,13 +77,11 @@ class TestUsageCachedAgent:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cached_agent_shows_detailed_usage(self):
|
||||
agent = _make_mock_agent()
|
||||
agent = _make_mock_agent(session_actual_cost_usd=0.1234)
|
||||
runner = _make_runner(SK, cached_agent=agent)
|
||||
event = MagicMock()
|
||||
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
|
||||
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
|
||||
mock_cost.return_value = MagicMock(amount_usd=0.1234, status="estimated")
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "claude-sonnet-4.6" in result
|
||||
@@ -99,9 +102,7 @@ class TestUsageCachedAgent:
|
||||
runner = _make_runner(SK, agent=running, cached_agent=cached)
|
||||
event = MagicMock()
|
||||
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
|
||||
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
|
||||
mock_cost.return_value = MagicMock(amount_usd=None, status="unknown")
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "80,000" in result # running agent's total
|
||||
@@ -117,9 +118,7 @@ class TestUsageCachedAgent:
|
||||
runner._running_agents[SK] = _AGENT_PENDING_SENTINEL
|
||||
event = MagicMock()
|
||||
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
|
||||
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
|
||||
mock_cost.return_value = MagicMock(amount_usd=None, status="unknown")
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "claude-sonnet-4.6" in result
|
||||
@@ -153,9 +152,7 @@ class TestUsageCachedAgent:
|
||||
runner = _make_runner(SK, cached_agent=agent)
|
||||
event = MagicMock()
|
||||
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
|
||||
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
|
||||
mock_cost.return_value = MagicMock(amount_usd=None, status="unknown")
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "Cache read" not in result
|
||||
@@ -168,9 +165,7 @@ class TestUsageCachedAgent:
|
||||
runner = _make_runner(SK, cached_agent=agent)
|
||||
event = MagicMock()
|
||||
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
|
||||
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
|
||||
mock_cost.return_value = MagicMock(amount_usd=None, status="included")
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "Cost: included" in result
|
||||
@@ -199,9 +194,7 @@ class TestUsageAccountSection:
|
||||
"Session: 85% remaining (15% used)",
|
||||
],
|
||||
)
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"), \
|
||||
patch("agent.usage_pricing.estimate_usage_cost") as mock_cost:
|
||||
mock_cost.return_value = MagicMock(amount_usd=None, status="included")
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "📊 **Session Token Usage**" in result
|
||||
@@ -256,3 +249,42 @@ class TestUsageAccountSection:
|
||||
assert account_call["kwargs"]["base_url"] == "https://chatgpt.com/backend-api/codex"
|
||||
assert "📊 **Session Info**" in result
|
||||
assert "📈 **Account limits**" in result
|
||||
|
||||
|
||||
class TestUsageRealCostOnly:
|
||||
"""Cost lines are provider-REPORTED only — never estimated, never $0.00."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unreported_cost_renders_no_cost_line(self):
|
||||
agent = _make_mock_agent() # openrouter, nothing reported
|
||||
runner = _make_runner(SK, cached_agent=agent)
|
||||
event = MagicMock()
|
||||
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "Cost:" not in result
|
||||
assert "$0.00" not in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nous_credits_delta_renders_as_cost(self):
|
||||
agent = _make_mock_agent(provider="nous", model="Hermes-4.1-405B")
|
||||
agent.get_credits_spent_micros = MagicMock(return_value=123_400)
|
||||
runner = _make_runner(SK, cached_agent=agent)
|
||||
event = MagicMock()
|
||||
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "$0.1234" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openrouter_reported_cost_renders(self):
|
||||
agent = _make_mock_agent(session_actual_cost_usd=0.9876)
|
||||
runner = _make_runner(SK, cached_agent=agent)
|
||||
event = MagicMock()
|
||||
|
||||
with patch("agent.rate_limit_tracker.format_rate_limit_compact", return_value="RPM: 50/60"):
|
||||
result = await runner._handle_usage_command(event)
|
||||
|
||||
assert "$0.9876" in result
|
||||
|
||||
@@ -110,7 +110,9 @@ class TestOpenRouterProfile:
|
||||
def test_extra_body_no_prefs(self):
|
||||
p = get_provider_profile("openrouter")
|
||||
body = p.build_extra_body()
|
||||
assert body == {}
|
||||
# Usage accounting is always requested (real provider-reported cost);
|
||||
# nothing else should appear without prefs/session.
|
||||
assert body == {"usage": {"include": True}}
|
||||
|
||||
def test_pareto_min_coding_score_emitted_for_pareto_model(self):
|
||||
"""min_coding_score → plugins block when model is openrouter/pareto-code."""
|
||||
|
||||
Reference in New Issue
Block a user