agentloop-profiler 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentloop/__init__.py +136 -0
- agentloop/ablation_protocol.py +229 -0
- agentloop/ablations.py +564 -0
- agentloop/aggregate_cli.py +88 -0
- agentloop/aggregate_stats.py +175 -0
- agentloop/aggregates.py +532 -0
- agentloop/audit.py +39 -0
- agentloop/autoinstrument.py +145 -0
- agentloop/batch_experiments.py +295 -0
- agentloop/batching.py +95 -0
- agentloop/budget_types.py +335 -0
- agentloop/budgets.py +539 -0
- agentloop/calibration.py +872 -0
- agentloop/checkpoint_state.py +306 -0
- agentloop/checkpoint_store.py +86 -0
- agentloop/ci.py +202 -0
- agentloop/cli.py +1314 -0
- agentloop/client.py +228 -0
- agentloop/completion.py +345 -0
- agentloop/completion_types.py +199 -0
- agentloop/config.py +50 -0
- agentloop/context_controls.py +443 -0
- agentloop/context_experiments.py +337 -0
- agentloop/context_types.py +216 -0
- agentloop/costs.py +667 -0
- agentloop/cursor_validation.py +48 -0
- agentloop/decorators.py +445 -0
- agentloop/demo.py +300 -0
- agentloop/doctor.py +341 -0
- agentloop/drift.py +361 -0
- agentloop/drift_cli.py +147 -0
- agentloop/drift_types.py +110 -0
- agentloop/drift_windows.py +340 -0
- agentloop/entrypoint.py +201 -0
- agentloop/estimates.py +133 -0
- agentloop/events.py +75 -0
- agentloop/experiment_artifacts.py +80 -0
- agentloop/experiment_observations.py +169 -0
- agentloop/experiment_reports.py +545 -0
- agentloop/experiment_types.py +355 -0
- agentloop/experiments.py +392 -0
- agentloop/exporters.py +117 -0
- agentloop/finding_benchmark_types.py +131 -0
- agentloop/finding_benchmarks.py +424 -0
- agentloop/findings.py +419 -0
- agentloop/graph.py +464 -0
- agentloop/harness.py +825 -0
- agentloop/harness_bench/__init__.py +1 -0
- agentloop/harness_bench/cli.py +77 -0
- agentloop/harness_bench/drivers.py +96 -0
- agentloop/harness_bench/offline.py +507 -0
- agentloop/harness_evidence.py +585 -0
- agentloop/harness_usage.py +91 -0
- agentloop/html_report.py +860 -0
- agentloop/integrations/__init__.py +1 -0
- agentloop/integrations/crewai.py +144 -0
- agentloop/integrations/harbor/__init__.py +1 -0
- agentloop/integrations/harbor/atif.py +982 -0
- agentloop/integrations/harbor/atif_validation.py +329 -0
- agentloop/integrations/harbor/cli.py +207 -0
- agentloop/integrations/harbor/manifest.py +86 -0
- agentloop/integrations/harbor/trial_evidence.py +196 -0
- agentloop/integrations/harbor/trials.py +1109 -0
- agentloop/integrations/harbor/verifier.py +118 -0
- agentloop/integrations/langgraph.py +150 -0
- agentloop/integrations/langgraph_harness.py +466 -0
- agentloop/integrations/langgraph_recovery.py +366 -0
- agentloop/integrations/omnigent/__init__.py +1 -0
- agentloop/integrations/omnigent/cli.py +78 -0
- agentloop/integrations/omnigent/evidence.py +93 -0
- agentloop/integrations/omnigent/telemetry.py +422 -0
- agentloop/integrations/openai.py +415 -0
- agentloop/integrations/openai_agents.py +291 -0
- agentloop/integrations/sdk.py +82 -0
- agentloop/integrations/vercel_ai.py +83 -0
- agentloop/interoperability/__init__.py +5 -0
- agentloop/interoperability/artifacts.py +112 -0
- agentloop/interoperability/cli.py +59 -0
- agentloop/interoperability/cohort_evidence.py +175 -0
- agentloop/interoperability/cohorts.py +681 -0
- agentloop/interoperability/contracts.py +427 -0
- agentloop/interoperability/coordination.py +554 -0
- agentloop/interoperability/evidence.py +215 -0
- agentloop/interoperability/otlp_jsonl.py +775 -0
- agentloop/interoperability/otlp_validation.py +496 -0
- agentloop/interoperability/registry.py +561 -0
- agentloop/interoperability/validation.py +333 -0
- agentloop/intervention_service.py +110 -0
- agentloop/intervention_store.py +118 -0
- agentloop/interventions.py +190 -0
- agentloop/issues.py +90 -0
- agentloop/judgment_adapters.py +125 -0
- agentloop/judgment_benchmark_types.py +98 -0
- agentloop/judgment_benchmarks.py +606 -0
- agentloop/judgment_types.py +241 -0
- agentloop/judgment_views.py +38 -0
- agentloop/judgments.py +342 -0
- agentloop/loop_guards.py +274 -0
- agentloop/loop_types.py +66 -0
- agentloop/markdown.py +102 -0
- agentloop/metrics.py +342 -0
- agentloop/migrations.py +303 -0
- agentloop/model_routing.py +470 -0
- agentloop/onboarding.py +315 -0
- agentloop/operations.py +44 -0
- agentloop/optimizer.py +199 -0
- agentloop/otel.py +715 -0
- agentloop/otel_ids.py +81 -0
- agentloop/otel_semantics.py +134 -0
- agentloop/parallelism.py +141 -0
- agentloop/patches.py +603 -0
- agentloop/plan_export.py +126 -0
- agentloop/policy_promotion.py +325 -0
- agentloop/promotion_evidence.py +304 -0
- agentloop/promotion_types.py +209 -0
- agentloop/quality.py +446 -0
- agentloop/ranking.py +529 -0
- agentloop/recovery_evidence.py +96 -0
- agentloop/replay.py +753 -0
- agentloop/retention.py +375 -0
- agentloop/retention_types.py +113 -0
- agentloop/reuse_experiments.py +247 -0
- agentloop/routing_checks.py +116 -0
- agentloop/routing_types.py +241 -0
- agentloop/rules.py +396 -0
- agentloop/runtime.py +286 -0
- agentloop/savings.py +158 -0
- agentloop/scheduling_plan.py +99 -0
- agentloop/scheduling_types.py +169 -0
- agentloop/schema.py +257 -0
- agentloop/semantic_rules.py +200 -0
- agentloop/semantic_waste.py +345 -0
- agentloop/semantic_waste_types.py +88 -0
- agentloop/server.py +494 -0
- agentloop/store.py +1429 -0
- agentloop/structural_experiments.py +431 -0
- agentloop/structural_types.py +113 -0
- agentloop/structured_quality.py +1020 -0
- agentloop/studies.py +629 -0
- agentloop/study_statistics.py +58 -0
- agentloop/substitution_evidence.py +199 -0
- agentloop/substitution_export.py +190 -0
- agentloop/substitution_reports.py +407 -0
- agentloop/substitution_types.py +284 -0
- agentloop/substitutions.py +267 -0
- agentloop/timing.py +142 -0
- agentloop/tokens.py +188 -0
- agentloop/tool_scheduling.py +410 -0
- agentloop/trace_sources.py +31 -0
- agentloop/tracer.py +494 -0
- agentloop/value.py +306 -0
- agentloop/version.py +3 -0
- agentloop/workflow_types.py +264 -0
- agentloop/workflows.py +284 -0
- agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
- agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
- agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
- agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
- agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
- agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
- agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
- dashboard/__init__.py +5 -0
- dashboard/app.py +922 -0
- dashboard/streamlit_compat.py +63 -0
- dashboard/trace_upload.py +57 -0
- dashboard/value_view.py +105 -0
agentloop/costs.py
ADDED
|
@@ -0,0 +1,667 @@
|
|
|
1
|
+
"""Provider-aware model cost estimation with an explicit unknown state.
|
|
2
|
+
|
|
3
|
+
The core rule of this module: **a cost is only ever reported when it can be
|
|
4
|
+
justified.** A model whose pricing is not known does not silently borrow a
|
|
5
|
+
generic rate — it returns an ``unknown`` estimate with ``amount_usd = None``, so
|
|
6
|
+
downstream reports can distinguish a measured or calculated cost from a missing
|
|
7
|
+
one instead of presenting a fabricated number as observed.
|
|
8
|
+
|
|
9
|
+
Three kinds of cost are represented (`CostState`):
|
|
10
|
+
|
|
11
|
+
- ``provider_reported`` — the caller supplied the actual billed cost (e.g. from a
|
|
12
|
+
provider usage field), which is used verbatim.
|
|
13
|
+
- ``calculated`` — computed from a known per-model rate in the pricing table.
|
|
14
|
+
- ``unknown`` — no rate is known and no reported cost was supplied; ``amount_usd``
|
|
15
|
+
is ``None``.
|
|
16
|
+
|
|
17
|
+
Pricing is offline and configurable. The built-in table carries provenance (a
|
|
18
|
+
``source`` and an ``as_of`` date) and is a point-in-time snapshot — provider
|
|
19
|
+
prices change, so treat the built-ins as a convenience, not an authority, and
|
|
20
|
+
override them for anything that matters. Users supply their own per-model rates
|
|
21
|
+
without editing this package by pointing ``AGENTLOOP_PRICING_FILE`` at a JSON
|
|
22
|
+
file (see ``load_pricing_table``), or by passing a ``PricingTable`` explicitly.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
import math
|
|
29
|
+
import os
|
|
30
|
+
from dataclasses import dataclass, field
|
|
31
|
+
from functools import lru_cache
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any, Literal
|
|
34
|
+
|
|
35
|
+
CostState = Literal["provider_reported", "calculated", "unknown"]
|
|
36
|
+
CostStatus = Literal["complete", "partial", "unknown", "empty"]
|
|
37
|
+
|
|
38
|
+
# Environment variable pointing at a JSON pricing-override file. Kept here (not in
|
|
39
|
+
# config.py) so the cost module stays self-contained and importable without the
|
|
40
|
+
# rest of the package's configuration surface.
|
|
41
|
+
PRICING_FILE_ENV = "AGENTLOOP_PRICING_FILE"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def is_cost_evaluable(status: str | None) -> bool:
|
|
45
|
+
"""Return whether a cost total is complete enough for comparisons."""
|
|
46
|
+
|
|
47
|
+
return status in {"complete", "empty"}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def format_cost_usd(
|
|
51
|
+
amount_usd: float | None,
|
|
52
|
+
status: str | None = "complete",
|
|
53
|
+
*,
|
|
54
|
+
decimals: int = 4,
|
|
55
|
+
token_status: str | None = None,
|
|
56
|
+
) -> str:
|
|
57
|
+
"""Format a cost without presenting an inexact total as an exact amount.
|
|
58
|
+
|
|
59
|
+
Two things can make a cost inexact, and both are qualified in the rendered
|
|
60
|
+
string. ``status`` covers pricing completeness: a ``partial`` total is a known
|
|
61
|
+
lower bound. ``token_status`` covers the basis underneath — a calculated cost
|
|
62
|
+
is a rate multiplied by a token count, so a count that was approximated from
|
|
63
|
+
word counts yields an approximate cost, however precise the arithmetic looks.
|
|
64
|
+
Pass the report's ``token_status`` to have that qualified too; omitting it
|
|
65
|
+
keeps the pricing-only behavior callers had before.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
status = "complete" if status is None else status
|
|
69
|
+
if (
|
|
70
|
+
amount_usd is None
|
|
71
|
+
or not _is_finite_number(amount_usd)
|
|
72
|
+
or amount_usd < 0
|
|
73
|
+
or status not in {"complete", "empty", "partial"}
|
|
74
|
+
):
|
|
75
|
+
return "unavailable"
|
|
76
|
+
rendered = f"${float(amount_usd):,.{decimals}f}"
|
|
77
|
+
if status == "partial":
|
|
78
|
+
rendered = f"{rendered} (known lower bound)"
|
|
79
|
+
if token_status is not None and token_status not in {"exact", "empty"}:
|
|
80
|
+
rendered = f"{rendered} (from {token_status} tokens)"
|
|
81
|
+
return rendered
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _is_finite_number(value: Any) -> bool:
|
|
85
|
+
"""Return whether value is a supported finite real number.
|
|
86
|
+
|
|
87
|
+
Values must also fit Python's floating-point calculations; extremely large
|
|
88
|
+
integers are rejected instead of surfacing ``OverflowError`` later.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
92
|
+
return False
|
|
93
|
+
try:
|
|
94
|
+
return math.isfinite(float(value))
|
|
95
|
+
except OverflowError:
|
|
96
|
+
return False
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _is_integral_number(value: Any) -> bool:
|
|
100
|
+
return _is_finite_number(value) and (
|
|
101
|
+
isinstance(value, int) or (isinstance(value, float) and value.is_integer())
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _validated_token_counts(
|
|
106
|
+
input_tokens: Any, output_tokens: Any, cached_input_tokens: Any = 0
|
|
107
|
+
) -> tuple[int, int, int]:
|
|
108
|
+
if not _is_integral_number(input_tokens) or not _is_integral_number(output_tokens):
|
|
109
|
+
raise ValueError("input_tokens and output_tokens must be finite integers")
|
|
110
|
+
resolved_input = int(input_tokens)
|
|
111
|
+
resolved_output = int(output_tokens)
|
|
112
|
+
if resolved_input < 0 or resolved_output < 0:
|
|
113
|
+
raise ValueError("input_tokens and output_tokens must be non-negative")
|
|
114
|
+
if not _is_integral_number(cached_input_tokens):
|
|
115
|
+
raise ValueError("cached_input_tokens must be a finite integer")
|
|
116
|
+
resolved_cached = int(cached_input_tokens)
|
|
117
|
+
if not 0 <= resolved_cached <= resolved_input:
|
|
118
|
+
raise ValueError(
|
|
119
|
+
"cached_input_tokens must satisfy 0 <= cached_input_tokens <= input_tokens"
|
|
120
|
+
)
|
|
121
|
+
return resolved_input, resolved_output, resolved_cached
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass(frozen=True)
|
|
125
|
+
class ModelPricing:
|
|
126
|
+
"""Per-model pricing in USD per million tokens, with provenance.
|
|
127
|
+
|
|
128
|
+
``cached_input_usd_per_mtok`` is the (usually discounted) rate for cached
|
|
129
|
+
input tokens; when ``None`` the plain input rate is used for any cached
|
|
130
|
+
tokens, so a caller can always pass ``cached_input_tokens`` without needing
|
|
131
|
+
every model to declare a cache rate.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
input_usd_per_mtok: float
|
|
135
|
+
output_usd_per_mtok: float
|
|
136
|
+
provider: str
|
|
137
|
+
source: str
|
|
138
|
+
as_of: str
|
|
139
|
+
cached_input_usd_per_mtok: float | None = None
|
|
140
|
+
# This rate is only valid up to this many input tokens; above it the entry
|
|
141
|
+
# does not apply (many providers charge a higher tier for long context, e.g.
|
|
142
|
+
# Gemini above 200K). ``None`` means the rate has no declared upper bound.
|
|
143
|
+
max_input_tokens: int | None = None
|
|
144
|
+
|
|
145
|
+
def __post_init__(self) -> None:
|
|
146
|
+
for name, value in (
|
|
147
|
+
("provider", self.provider),
|
|
148
|
+
("source", self.source),
|
|
149
|
+
("as_of", self.as_of),
|
|
150
|
+
):
|
|
151
|
+
if not isinstance(value, str) or not value.strip():
|
|
152
|
+
raise ValueError(f"{name} must be a non-empty string")
|
|
153
|
+
object.__setattr__(self, name, value.strip())
|
|
154
|
+
for name, rate in (
|
|
155
|
+
("input_usd_per_mtok", self.input_usd_per_mtok),
|
|
156
|
+
("output_usd_per_mtok", self.output_usd_per_mtok),
|
|
157
|
+
):
|
|
158
|
+
if not _is_finite_number(rate) or rate < 0:
|
|
159
|
+
raise ValueError(f"{name} must be a finite, non-negative number")
|
|
160
|
+
cached_rate = self.cached_input_usd_per_mtok
|
|
161
|
+
if cached_rate is not None and (not _is_finite_number(cached_rate) or cached_rate < 0):
|
|
162
|
+
raise ValueError("cached_input_usd_per_mtok must be a finite, non-negative number")
|
|
163
|
+
if self.max_input_tokens is not None and (
|
|
164
|
+
not _is_integral_number(self.max_input_tokens) or self.max_input_tokens <= 0
|
|
165
|
+
):
|
|
166
|
+
raise ValueError("max_input_tokens must be a positive integer")
|
|
167
|
+
|
|
168
|
+
def applies_to(self, input_tokens: int) -> bool:
|
|
169
|
+
resolved_input, _, _ = _validated_token_counts(input_tokens, 0)
|
|
170
|
+
return self.max_input_tokens is None or resolved_input <= self.max_input_tokens
|
|
171
|
+
|
|
172
|
+
def cost_usd(
|
|
173
|
+
self, input_tokens: int, output_tokens: int, cached_input_tokens: int = 0
|
|
174
|
+
) -> float:
|
|
175
|
+
input_tokens, output_tokens, cached_input_tokens = _validated_token_counts(
|
|
176
|
+
input_tokens, output_tokens, cached_input_tokens
|
|
177
|
+
)
|
|
178
|
+
billable_input = input_tokens - cached_input_tokens
|
|
179
|
+
cached_rate = (
|
|
180
|
+
self.cached_input_usd_per_mtok
|
|
181
|
+
if self.cached_input_usd_per_mtok is not None
|
|
182
|
+
else self.input_usd_per_mtok
|
|
183
|
+
)
|
|
184
|
+
cost = (
|
|
185
|
+
(billable_input / 1_000_000) * self.input_usd_per_mtok
|
|
186
|
+
+ (cached_input_tokens / 1_000_000) * cached_rate
|
|
187
|
+
+ (output_tokens / 1_000_000) * self.output_usd_per_mtok
|
|
188
|
+
)
|
|
189
|
+
return round(cost, 6)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@dataclass(frozen=True)
|
|
193
|
+
class CostEstimate:
|
|
194
|
+
"""The cost of a single model call, with enough context to audit it.
|
|
195
|
+
|
|
196
|
+
``amount_usd`` is ``None`` exactly when ``state == "unknown"``.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
state: CostState
|
|
200
|
+
amount_usd: float | None
|
|
201
|
+
model: str | None
|
|
202
|
+
provider: str | None
|
|
203
|
+
pricing_source: str | None
|
|
204
|
+
pricing_as_of: str | None
|
|
205
|
+
input_tokens: int
|
|
206
|
+
output_tokens: int
|
|
207
|
+
cached_input_tokens: int = 0
|
|
208
|
+
# Populated only when ``state == "unknown"``: a short machine-usable reason
|
|
209
|
+
# (e.g. ``unpriced_model``, ``context_over_threshold``,
|
|
210
|
+
# ``unsupported_billing_mode``, ``invalid_metadata``) so consumers can tell
|
|
211
|
+
# "no rate for this model" apart from "we refused to price this call".
|
|
212
|
+
unknown_reason: str | None = None
|
|
213
|
+
|
|
214
|
+
def __post_init__(self) -> None:
|
|
215
|
+
if (self.state == "unknown") != (self.amount_usd is None):
|
|
216
|
+
raise ValueError("amount_usd must be None exactly when state is unknown")
|
|
217
|
+
if self.amount_usd is not None and (
|
|
218
|
+
not _is_finite_number(self.amount_usd) or self.amount_usd < 0
|
|
219
|
+
):
|
|
220
|
+
raise ValueError("amount_usd must be a finite, non-negative number")
|
|
221
|
+
|
|
222
|
+
@property
|
|
223
|
+
def is_known(self) -> bool:
|
|
224
|
+
return self.state != "unknown"
|
|
225
|
+
|
|
226
|
+
def to_dict(self) -> dict[str, Any]:
|
|
227
|
+
return {
|
|
228
|
+
"state": self.state,
|
|
229
|
+
"amount_usd": self.amount_usd,
|
|
230
|
+
"model": self.model,
|
|
231
|
+
"provider": self.provider,
|
|
232
|
+
"pricing_source": self.pricing_source,
|
|
233
|
+
"pricing_as_of": self.pricing_as_of,
|
|
234
|
+
"input_tokens": self.input_tokens,
|
|
235
|
+
"output_tokens": self.output_tokens,
|
|
236
|
+
"cached_input_tokens": self.cached_input_tokens,
|
|
237
|
+
"unknown_reason": self.unknown_reason,
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
# Built-in pricing snapshot. Provenance is attached to every entry so a report
|
|
242
|
+
# can state where a rate came from and how current it is. These are a convenience
|
|
243
|
+
# for common models only — they are NOT kept up to date automatically and must be
|
|
244
|
+
# overridden for anything cost-sensitive (see docs/PRICING.md).
|
|
245
|
+
_OPENAI_SOURCE = "https://openai.com/api/pricing/"
|
|
246
|
+
_ANTHROPIC_SOURCE = "https://www.anthropic.com/pricing"
|
|
247
|
+
_GOOGLE_SOURCE = "https://ai.google.dev/gemini-api/docs/pricing"
|
|
248
|
+
_OPENAI_AS_OF = "2025-06-01"
|
|
249
|
+
_ANTHROPIC_AS_OF = "2025-06-01"
|
|
250
|
+
_GOOGLE_AS_OF = "2025-06-17"
|
|
251
|
+
|
|
252
|
+
# Gemini 2.5 charges a higher tier above 200K input tokens; the built-in flat
|
|
253
|
+
# rate is only correct at or below that, so above it we return unknown rather
|
|
254
|
+
# than under-report. A deployment that needs the long-context tier priced adds
|
|
255
|
+
# its own override (see docs/PRICING.md).
|
|
256
|
+
_GEMINI_STANDARD_CONTEXT = 200_000
|
|
257
|
+
|
|
258
|
+
_BUILTIN_PRICING: dict[str, ModelPricing] = {
|
|
259
|
+
# OpenAI. The original three rates this module shipped with are preserved so
|
|
260
|
+
# previously-priced traces keep the same numbers.
|
|
261
|
+
"gpt-4.1": ModelPricing(2.00, 8.00, "openai", _OPENAI_SOURCE, _OPENAI_AS_OF, 0.50),
|
|
262
|
+
"gpt-4.1-mini": ModelPricing(0.40, 1.60, "openai", _OPENAI_SOURCE, _OPENAI_AS_OF, 0.10),
|
|
263
|
+
"gpt-4o-mini": ModelPricing(0.15, 0.60, "openai", _OPENAI_SOURCE, _OPENAI_AS_OF, 0.075),
|
|
264
|
+
"gpt-4o": ModelPricing(2.50, 10.00, "openai", _OPENAI_SOURCE, _OPENAI_AS_OF, 1.25),
|
|
265
|
+
# Anthropic.
|
|
266
|
+
"claude-sonnet-4": ModelPricing(
|
|
267
|
+
3.00, 15.00, "anthropic", _ANTHROPIC_SOURCE, _ANTHROPIC_AS_OF, 0.30
|
|
268
|
+
),
|
|
269
|
+
"claude-opus-4": ModelPricing(
|
|
270
|
+
15.00, 75.00, "anthropic", _ANTHROPIC_SOURCE, _ANTHROPIC_AS_OF, 1.50
|
|
271
|
+
),
|
|
272
|
+
"claude-3-5-haiku": ModelPricing(
|
|
273
|
+
0.80, 4.00, "anthropic", _ANTHROPIC_SOURCE, _ANTHROPIC_AS_OF, 0.08
|
|
274
|
+
),
|
|
275
|
+
# Google. Pro has a higher price tier above the standard-context threshold;
|
|
276
|
+
# Flash pricing is flat across its supported context window.
|
|
277
|
+
"gemini-2.5-pro": ModelPricing(
|
|
278
|
+
1.25,
|
|
279
|
+
10.00,
|
|
280
|
+
"google",
|
|
281
|
+
_GOOGLE_SOURCE,
|
|
282
|
+
_GOOGLE_AS_OF,
|
|
283
|
+
max_input_tokens=_GEMINI_STANDARD_CONTEXT,
|
|
284
|
+
),
|
|
285
|
+
"gemini-2.5-flash": ModelPricing(
|
|
286
|
+
0.30,
|
|
287
|
+
2.50,
|
|
288
|
+
"google",
|
|
289
|
+
_GOOGLE_SOURCE,
|
|
290
|
+
_GOOGLE_AS_OF,
|
|
291
|
+
),
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
@dataclass
|
|
296
|
+
class PricingTable:
|
|
297
|
+
"""A resolvable set of model rates: user overrides layered over built-ins.
|
|
298
|
+
|
|
299
|
+
Resolution normalizes a model identifier so dated snapshots and common
|
|
300
|
+
prefixes map onto a base rate (e.g. ``gpt-4.1-2025-04-14`` and
|
|
301
|
+
``openai/gpt-4.1`` both resolve to ``gpt-4.1``). An override always wins over
|
|
302
|
+
a built-in with the same key.
|
|
303
|
+
"""
|
|
304
|
+
|
|
305
|
+
rates: dict[str, ModelPricing] = field(default_factory=dict)
|
|
306
|
+
|
|
307
|
+
def resolve(
|
|
308
|
+
self,
|
|
309
|
+
model: str | None,
|
|
310
|
+
provider: str | None = None,
|
|
311
|
+
billing_mode: str | None = None,
|
|
312
|
+
) -> ModelPricing | None:
|
|
313
|
+
"""Resolve a rate, treating an explicit provider as a hard constraint.
|
|
314
|
+
|
|
315
|
+
When a provider is specified (as the ``provider`` argument or a
|
|
316
|
+
``provider/model`` prefix), a bare-model entry only matches if its own
|
|
317
|
+
``provider`` equals it — so ``azure/gpt-4o`` or ``provider="bedrock"``
|
|
318
|
+
will not silently borrow the OpenAI or Anthropic rate for a same-named
|
|
319
|
+
model. A key that was itself provider-qualified (e.g. a user override
|
|
320
|
+
keyed ``"azure/gpt-4o"``) is trusted as an explicit intent.
|
|
321
|
+
|
|
322
|
+
A non-standard ``billing_mode`` (e.g. ``batch``) only matches a
|
|
323
|
+
mode-qualified key; it never falls back to the standard rate.
|
|
324
|
+
"""
|
|
325
|
+
if not model:
|
|
326
|
+
return None
|
|
327
|
+
effective_provider = provider or _provider_prefix(model)
|
|
328
|
+
mode = _normalize_billing_mode(billing_mode)
|
|
329
|
+
for candidate, provider_qualified in _resolution_candidates(model, provider, mode):
|
|
330
|
+
pricing = self.rates.get(candidate)
|
|
331
|
+
if pricing is None:
|
|
332
|
+
continue
|
|
333
|
+
if provider_qualified:
|
|
334
|
+
return pricing
|
|
335
|
+
if effective_provider and pricing.provider.lower() != effective_provider.lower():
|
|
336
|
+
continue
|
|
337
|
+
return pricing
|
|
338
|
+
return None
|
|
339
|
+
|
|
340
|
+
def with_overrides(self, overrides: dict[str, ModelPricing]) -> PricingTable:
|
|
341
|
+
merged = dict(self.rates)
|
|
342
|
+
merged.update(overrides)
|
|
343
|
+
return PricingTable(rates=merged)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _normalize_key(model: str) -> str:
|
|
347
|
+
return model.strip().lower()
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
_STANDARD_BILLING_MODES = {"", "standard", "default", "sync", "on-demand"}
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _normalize_billing_mode(billing_mode: str | None) -> str | None:
|
|
354
|
+
if billing_mode is None:
|
|
355
|
+
return None
|
|
356
|
+
mode = billing_mode.strip().lower()
|
|
357
|
+
return None if mode in _STANDARD_BILLING_MODES else mode
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _provider_prefix(model: str) -> str | None:
|
|
361
|
+
normalized = _normalize_key(model)
|
|
362
|
+
for separator in ("/", ":"):
|
|
363
|
+
if separator in normalized:
|
|
364
|
+
return normalized.split(separator, 1)[0]
|
|
365
|
+
return None
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _resolution_candidates(
|
|
369
|
+
model: str, provider: str | None, billing_mode: str | None
|
|
370
|
+
) -> list[tuple[str, bool]]:
|
|
371
|
+
"""Ordered ``(key, provider_qualified)`` pairs for a model, most specific first.
|
|
372
|
+
|
|
373
|
+
Handles ``provider/model`` prefixes, ``provider:model`` prefixes, and dated
|
|
374
|
+
snapshot suffixes (a trailing ``-YYYY-MM-DD`` or ``-YYYYMMDD``) so a specific
|
|
375
|
+
snapshot falls back to its base model rate when only the base is known. When
|
|
376
|
+
``billing_mode`` is non-standard, every key is mode-suffixed (``model#mode``)
|
|
377
|
+
and the plain keys are omitted, so a non-standard mode never resolves to a
|
|
378
|
+
standard rate.
|
|
379
|
+
"""
|
|
380
|
+
normalized = _normalize_key(model)
|
|
381
|
+
base: list[tuple[str, bool]] = []
|
|
382
|
+
seen: set[str] = set()
|
|
383
|
+
|
|
384
|
+
def _add(value: str, provider_qualified: bool) -> None:
|
|
385
|
+
if value and value not in seen:
|
|
386
|
+
seen.add(value)
|
|
387
|
+
base.append((value, provider_qualified))
|
|
388
|
+
|
|
389
|
+
# A provider-qualified key is the most specific: try "provider/model" first.
|
|
390
|
+
if provider:
|
|
391
|
+
_add(f"{_normalize_key(provider)}/{normalized}", True)
|
|
392
|
+
|
|
393
|
+
# Strip a leading "provider/" or "provider:" that came in the model string.
|
|
394
|
+
bare = normalized
|
|
395
|
+
for separator in ("/", ":"):
|
|
396
|
+
if separator in bare:
|
|
397
|
+
bare = bare.split(separator, 1)[1]
|
|
398
|
+
_add(normalized, False)
|
|
399
|
+
_add(bare, False)
|
|
400
|
+
|
|
401
|
+
# Strip a trailing dated snapshot suffix so gpt-4.1-2025-04-14 -> gpt-4.1.
|
|
402
|
+
for snapshot_base in (_strip_snapshot(normalized), _strip_snapshot(bare)):
|
|
403
|
+
_add(snapshot_base, False)
|
|
404
|
+
|
|
405
|
+
if billing_mode is None:
|
|
406
|
+
return base
|
|
407
|
+
return [(f"{key}#{billing_mode}", qualified) for key, qualified in base]
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _strip_snapshot(model: str) -> str:
|
|
411
|
+
parts = model.split("-")
|
|
412
|
+
# Drop a trailing YYYY-MM-DD (three parts) or YYYYMMDD (one part) date.
|
|
413
|
+
if len(parts) >= 4 and _looks_like_date_parts(parts[-3:]):
|
|
414
|
+
return "-".join(parts[:-3])
|
|
415
|
+
if len(parts) >= 2 and parts[-1].isdigit() and len(parts[-1]) == 8:
|
|
416
|
+
return "-".join(parts[:-1])
|
|
417
|
+
return model
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _looks_like_date_parts(parts: list[str]) -> bool:
|
|
421
|
+
if len(parts) != 3:
|
|
422
|
+
return False
|
|
423
|
+
year, month, day = parts
|
|
424
|
+
return (
|
|
425
|
+
year.isdigit()
|
|
426
|
+
and len(year) == 4
|
|
427
|
+
and month.isdigit()
|
|
428
|
+
and len(month) == 2
|
|
429
|
+
and day.isdigit()
|
|
430
|
+
and len(day) == 2
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _pricing_from_mapping(key: str, raw: dict[str, Any]) -> ModelPricing:
|
|
435
|
+
"""Build a ModelPricing from a JSON/dict override entry, with clear errors."""
|
|
436
|
+
try:
|
|
437
|
+
input_raw = raw["input_usd_per_mtok"]
|
|
438
|
+
output_raw = raw["output_usd_per_mtok"]
|
|
439
|
+
if isinstance(input_raw, bool) or isinstance(output_raw, bool):
|
|
440
|
+
raise TypeError("rates must be numeric, not boolean")
|
|
441
|
+
input_rate = float(input_raw)
|
|
442
|
+
output_rate = float(output_raw)
|
|
443
|
+
except (KeyError, OverflowError, TypeError, ValueError) as exc:
|
|
444
|
+
raise ValueError(
|
|
445
|
+
f"pricing override for {key!r} must set numeric 'input_usd_per_mtok' "
|
|
446
|
+
f"and 'output_usd_per_mtok' ({exc})"
|
|
447
|
+
) from exc
|
|
448
|
+
cached_raw = raw.get("cached_input_usd_per_mtok")
|
|
449
|
+
if isinstance(cached_raw, bool):
|
|
450
|
+
raise ValueError(
|
|
451
|
+
f"pricing override for {key!r} 'cached_input_usd_per_mtok' must be numeric, not boolean"
|
|
452
|
+
)
|
|
453
|
+
try:
|
|
454
|
+
cached_rate = None if cached_raw is None else float(cached_raw)
|
|
455
|
+
except (OverflowError, TypeError, ValueError) as exc:
|
|
456
|
+
raise ValueError(
|
|
457
|
+
f"pricing override for {key!r} must set a numeric 'cached_input_usd_per_mtok' ({exc})"
|
|
458
|
+
) from exc
|
|
459
|
+
max_input_raw = raw.get("max_input_tokens")
|
|
460
|
+
if max_input_raw is not None and not _is_integral_number(max_input_raw):
|
|
461
|
+
raise ValueError(f"pricing override for {key!r} must set an integer 'max_input_tokens'")
|
|
462
|
+
try:
|
|
463
|
+
return ModelPricing(
|
|
464
|
+
input_usd_per_mtok=input_rate,
|
|
465
|
+
output_usd_per_mtok=output_rate,
|
|
466
|
+
provider=raw.get("provider", "custom"),
|
|
467
|
+
source=raw.get("source", "user override"),
|
|
468
|
+
as_of=raw.get("as_of", "user-provided"),
|
|
469
|
+
cached_input_usd_per_mtok=cached_rate,
|
|
470
|
+
max_input_tokens=None if max_input_raw is None else int(max_input_raw),
|
|
471
|
+
)
|
|
472
|
+
except ValueError as exc:
|
|
473
|
+
raise ValueError(f"pricing override for {key!r} is invalid: {exc}") from exc
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def load_overrides_from_file(path: str | os.PathLike[str]) -> dict[str, ModelPricing]:
|
|
477
|
+
"""Load per-model pricing overrides from a JSON file.
|
|
478
|
+
|
|
479
|
+
The file maps a model key (matched after normalization — a bare model name,
|
|
480
|
+
``provider/model``, or a dated snapshot are all valid keys) to an object with
|
|
481
|
+
``input_usd_per_mtok``/``output_usd_per_mtok`` and optional ``provider``,
|
|
482
|
+
``source``, ``as_of``, and ``cached_input_usd_per_mtok``. A top-level
|
|
483
|
+
``models`` wrapper object is accepted for readability.
|
|
484
|
+
"""
|
|
485
|
+
text = Path(path).read_text(encoding="utf-8")
|
|
486
|
+
data = json.loads(text)
|
|
487
|
+
if not isinstance(data, dict):
|
|
488
|
+
raise ValueError(f"pricing file {os.fspath(path)} must contain a JSON object")
|
|
489
|
+
entries = data.get("models") if isinstance(data.get("models"), dict) else data
|
|
490
|
+
overrides: dict[str, ModelPricing] = {}
|
|
491
|
+
for key, raw in entries.items():
|
|
492
|
+
if not isinstance(raw, dict):
|
|
493
|
+
raise ValueError(f"pricing entry for {key!r} must be an object")
|
|
494
|
+
overrides[_normalize_key(key)] = _pricing_from_mapping(key, raw)
|
|
495
|
+
return overrides
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
@lru_cache(maxsize=8)
|
|
499
|
+
def _cached_overrides_from_file(
|
|
500
|
+
path: str, mtime_ns: int, ctime_ns: int, size: int
|
|
501
|
+
) -> dict[str, ModelPricing]:
|
|
502
|
+
"""Reuse parsed overrides until the file's observable identity changes."""
|
|
503
|
+
_ = (mtime_ns, ctime_ns, size)
|
|
504
|
+
return load_overrides_from_file(path)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def load_pricing_table(
|
|
508
|
+
*,
|
|
509
|
+
overrides: dict[str, ModelPricing] | None = None,
|
|
510
|
+
include_builtins: bool = True,
|
|
511
|
+
env: dict[str, str] | None = None,
|
|
512
|
+
) -> PricingTable:
|
|
513
|
+
"""Build the effective pricing table: built-ins, then file, then explicit overrides.
|
|
514
|
+
|
|
515
|
+
Precedence (later wins): built-in snapshot < ``AGENTLOOP_PRICING_FILE`` <
|
|
516
|
+
``overrides`` passed here. Set ``include_builtins=False`` to price *only*
|
|
517
|
+
against user-provided rates (so every unlisted model is explicitly unknown).
|
|
518
|
+
"""
|
|
519
|
+
environ = os.environ if env is None else env
|
|
520
|
+
table = PricingTable(rates=dict(_BUILTIN_PRICING) if include_builtins else {})
|
|
521
|
+
|
|
522
|
+
file_path = environ.get(PRICING_FILE_ENV)
|
|
523
|
+
if file_path:
|
|
524
|
+
normalized_path = os.path.abspath(os.fspath(file_path))
|
|
525
|
+
file_stat = os.stat(normalized_path)
|
|
526
|
+
file_overrides = _cached_overrides_from_file(
|
|
527
|
+
normalized_path,
|
|
528
|
+
file_stat.st_mtime_ns,
|
|
529
|
+
file_stat.st_ctime_ns,
|
|
530
|
+
file_stat.st_size,
|
|
531
|
+
)
|
|
532
|
+
table = table.with_overrides(file_overrides)
|
|
533
|
+
|
|
534
|
+
if overrides:
|
|
535
|
+
table = table.with_overrides({_normalize_key(k): v for k, v in overrides.items()})
|
|
536
|
+
|
|
537
|
+
return table
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _unknown(
|
|
541
|
+
model: str | None,
|
|
542
|
+
provider: str | None,
|
|
543
|
+
input_tokens: int,
|
|
544
|
+
output_tokens: int,
|
|
545
|
+
cached_input_tokens: int,
|
|
546
|
+
reason: str,
|
|
547
|
+
) -> CostEstimate:
|
|
548
|
+
return CostEstimate(
|
|
549
|
+
state="unknown",
|
|
550
|
+
amount_usd=None,
|
|
551
|
+
model=model,
|
|
552
|
+
provider=provider,
|
|
553
|
+
pricing_source=None,
|
|
554
|
+
pricing_as_of=None,
|
|
555
|
+
input_tokens=input_tokens,
|
|
556
|
+
output_tokens=output_tokens,
|
|
557
|
+
cached_input_tokens=cached_input_tokens,
|
|
558
|
+
unknown_reason=reason,
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def estimate_cost(
|
|
563
|
+
model: str | None,
|
|
564
|
+
input_tokens: int,
|
|
565
|
+
output_tokens: int,
|
|
566
|
+
*,
|
|
567
|
+
provider: str | None = None,
|
|
568
|
+
cached_input_tokens: int = 0,
|
|
569
|
+
billing_mode: str | None = None,
|
|
570
|
+
provider_reported_cost_usd: float | None = None,
|
|
571
|
+
pricing: PricingTable | None = None,
|
|
572
|
+
) -> CostEstimate:
|
|
573
|
+
"""Estimate the cost of one model call, preferring reported > calculated > unknown.
|
|
574
|
+
|
|
575
|
+
- If ``provider_reported_cost_usd`` is given, it's used verbatim
|
|
576
|
+
(``provider_reported``) — the provider's own number always beats a
|
|
577
|
+
calculation. A non-finite or negative reported cost is rejected with a
|
|
578
|
+
``ValueError`` (report construction catches this and yields ``unknown``).
|
|
579
|
+
- Otherwise, if the model resolves to a known rate — under the effective
|
|
580
|
+
provider, billing mode, and context threshold — the cost is
|
|
581
|
+
``calculated``.
|
|
582
|
+
- Otherwise the estimate is ``unknown`` with ``amount_usd = None`` and an
|
|
583
|
+
``unknown_reason``; no generic rate is substituted.
|
|
584
|
+
|
|
585
|
+
``cached_input_tokens`` must satisfy ``0 <= cached <= input_tokens``.
|
|
586
|
+
"""
|
|
587
|
+
table = pricing if pricing is not None else load_pricing_table()
|
|
588
|
+
|
|
589
|
+
input_tokens, output_tokens, cached_input_tokens = _validated_token_counts(
|
|
590
|
+
input_tokens, output_tokens, cached_input_tokens
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
if model is not None and not isinstance(model, str):
|
|
594
|
+
raise ValueError("model must be a string or None")
|
|
595
|
+
if provider is not None and (not isinstance(provider, str) or not provider.strip()):
|
|
596
|
+
raise ValueError("provider must be a non-empty string or None")
|
|
597
|
+
provider = provider.strip() if provider is not None else None
|
|
598
|
+
if billing_mode is not None and not isinstance(billing_mode, str):
|
|
599
|
+
raise ValueError("billing_mode must be a string or None")
|
|
600
|
+
|
|
601
|
+
if provider_reported_cost_usd is not None:
|
|
602
|
+
if not _is_finite_number(provider_reported_cost_usd) or provider_reported_cost_usd < 0:
|
|
603
|
+
raise ValueError("provider_reported_cost_usd must be a finite, non-negative number")
|
|
604
|
+
resolved = table.resolve(model, provider, billing_mode)
|
|
605
|
+
return CostEstimate(
|
|
606
|
+
state="provider_reported",
|
|
607
|
+
amount_usd=float(provider_reported_cost_usd),
|
|
608
|
+
model=model,
|
|
609
|
+
provider=provider or (resolved.provider if resolved else None),
|
|
610
|
+
pricing_source="provider-reported",
|
|
611
|
+
pricing_as_of=None,
|
|
612
|
+
input_tokens=input_tokens,
|
|
613
|
+
output_tokens=output_tokens,
|
|
614
|
+
cached_input_tokens=cached_input_tokens,
|
|
615
|
+
)
|
|
616
|
+
|
|
617
|
+
resolved = table.resolve(model, provider, billing_mode)
|
|
618
|
+
if resolved is None:
|
|
619
|
+
mode = _normalize_billing_mode(billing_mode)
|
|
620
|
+
reason = (
|
|
621
|
+
"unsupported_billing_mode"
|
|
622
|
+
if mode is not None and table.resolve(model, provider) is not None
|
|
623
|
+
else "unpriced_model"
|
|
624
|
+
)
|
|
625
|
+
return _unknown(model, provider, input_tokens, output_tokens, cached_input_tokens, reason)
|
|
626
|
+
|
|
627
|
+
if not resolved.applies_to(input_tokens):
|
|
628
|
+
# A rate exists but only for a smaller context; refuse to under-report.
|
|
629
|
+
return _unknown(
|
|
630
|
+
model,
|
|
631
|
+
resolved.provider,
|
|
632
|
+
input_tokens,
|
|
633
|
+
output_tokens,
|
|
634
|
+
cached_input_tokens,
|
|
635
|
+
"context_over_threshold",
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
return CostEstimate(
|
|
639
|
+
state="calculated",
|
|
640
|
+
amount_usd=resolved.cost_usd(input_tokens, output_tokens, cached_input_tokens),
|
|
641
|
+
model=model,
|
|
642
|
+
provider=resolved.provider,
|
|
643
|
+
pricing_source=resolved.source,
|
|
644
|
+
pricing_as_of=resolved.as_of,
|
|
645
|
+
input_tokens=input_tokens,
|
|
646
|
+
output_tokens=output_tokens,
|
|
647
|
+
cached_input_tokens=cached_input_tokens,
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def estimate_cost_usd(model: str | None, input_tokens: int, output_tokens: int) -> float | None:
|
|
652
|
+
"""Backward-compatible thin wrapper returning just the amount.
|
|
653
|
+
|
|
654
|
+
**Behavior change:** this now returns ``None`` for a model with no known
|
|
655
|
+
rate, instead of the old fabricated ``$1/M`` in / ``$3/M`` out default. Use
|
|
656
|
+
:func:`estimate_cost` for the full provider/provenance/state context.
|
|
657
|
+
"""
|
|
658
|
+
return estimate_cost(model, input_tokens, output_tokens).amount_usd
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
# Legacy export: a plain {model: {input, output}} view of the built-in rates for
|
|
662
|
+
# any external reader that imported MODEL_PRICES. It intentionally no longer has a
|
|
663
|
+
# "default" entry — the absence of a default is the whole point of this module.
|
|
664
|
+
MODEL_PRICES: dict[str, dict[str, float]] = {
|
|
665
|
+
name: {"input": pricing.input_usd_per_mtok, "output": pricing.output_usd_per_mtok}
|
|
666
|
+
for name, pricing in _BUILTIN_PRICING.items()
|
|
667
|
+
}
|