agentloop-profiler 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. agentloop/__init__.py +136 -0
  2. agentloop/ablation_protocol.py +229 -0
  3. agentloop/ablations.py +564 -0
  4. agentloop/aggregate_cli.py +88 -0
  5. agentloop/aggregate_stats.py +175 -0
  6. agentloop/aggregates.py +532 -0
  7. agentloop/audit.py +39 -0
  8. agentloop/autoinstrument.py +145 -0
  9. agentloop/batch_experiments.py +295 -0
  10. agentloop/batching.py +95 -0
  11. agentloop/budget_types.py +335 -0
  12. agentloop/budgets.py +539 -0
  13. agentloop/calibration.py +872 -0
  14. agentloop/checkpoint_state.py +306 -0
  15. agentloop/checkpoint_store.py +86 -0
  16. agentloop/ci.py +202 -0
  17. agentloop/cli.py +1314 -0
  18. agentloop/client.py +228 -0
  19. agentloop/completion.py +345 -0
  20. agentloop/completion_types.py +199 -0
  21. agentloop/config.py +50 -0
  22. agentloop/context_controls.py +443 -0
  23. agentloop/context_experiments.py +337 -0
  24. agentloop/context_types.py +216 -0
  25. agentloop/costs.py +667 -0
  26. agentloop/cursor_validation.py +48 -0
  27. agentloop/decorators.py +445 -0
  28. agentloop/demo.py +300 -0
  29. agentloop/doctor.py +341 -0
  30. agentloop/drift.py +361 -0
  31. agentloop/drift_cli.py +147 -0
  32. agentloop/drift_types.py +110 -0
  33. agentloop/drift_windows.py +340 -0
  34. agentloop/entrypoint.py +201 -0
  35. agentloop/estimates.py +133 -0
  36. agentloop/events.py +75 -0
  37. agentloop/experiment_artifacts.py +80 -0
  38. agentloop/experiment_observations.py +169 -0
  39. agentloop/experiment_reports.py +545 -0
  40. agentloop/experiment_types.py +355 -0
  41. agentloop/experiments.py +392 -0
  42. agentloop/exporters.py +117 -0
  43. agentloop/finding_benchmark_types.py +131 -0
  44. agentloop/finding_benchmarks.py +424 -0
  45. agentloop/findings.py +419 -0
  46. agentloop/graph.py +464 -0
  47. agentloop/harness.py +825 -0
  48. agentloop/harness_bench/__init__.py +1 -0
  49. agentloop/harness_bench/cli.py +77 -0
  50. agentloop/harness_bench/drivers.py +96 -0
  51. agentloop/harness_bench/offline.py +507 -0
  52. agentloop/harness_evidence.py +585 -0
  53. agentloop/harness_usage.py +91 -0
  54. agentloop/html_report.py +860 -0
  55. agentloop/integrations/__init__.py +1 -0
  56. agentloop/integrations/crewai.py +144 -0
  57. agentloop/integrations/harbor/__init__.py +1 -0
  58. agentloop/integrations/harbor/atif.py +982 -0
  59. agentloop/integrations/harbor/atif_validation.py +329 -0
  60. agentloop/integrations/harbor/cli.py +207 -0
  61. agentloop/integrations/harbor/manifest.py +86 -0
  62. agentloop/integrations/harbor/trial_evidence.py +196 -0
  63. agentloop/integrations/harbor/trials.py +1109 -0
  64. agentloop/integrations/harbor/verifier.py +118 -0
  65. agentloop/integrations/langgraph.py +150 -0
  66. agentloop/integrations/langgraph_harness.py +466 -0
  67. agentloop/integrations/langgraph_recovery.py +366 -0
  68. agentloop/integrations/omnigent/__init__.py +1 -0
  69. agentloop/integrations/omnigent/cli.py +78 -0
  70. agentloop/integrations/omnigent/evidence.py +93 -0
  71. agentloop/integrations/omnigent/telemetry.py +422 -0
  72. agentloop/integrations/openai.py +415 -0
  73. agentloop/integrations/openai_agents.py +291 -0
  74. agentloop/integrations/sdk.py +82 -0
  75. agentloop/integrations/vercel_ai.py +83 -0
  76. agentloop/interoperability/__init__.py +5 -0
  77. agentloop/interoperability/artifacts.py +112 -0
  78. agentloop/interoperability/cli.py +59 -0
  79. agentloop/interoperability/cohort_evidence.py +175 -0
  80. agentloop/interoperability/cohorts.py +681 -0
  81. agentloop/interoperability/contracts.py +427 -0
  82. agentloop/interoperability/coordination.py +554 -0
  83. agentloop/interoperability/evidence.py +215 -0
  84. agentloop/interoperability/otlp_jsonl.py +775 -0
  85. agentloop/interoperability/otlp_validation.py +496 -0
  86. agentloop/interoperability/registry.py +561 -0
  87. agentloop/interoperability/validation.py +333 -0
  88. agentloop/intervention_service.py +110 -0
  89. agentloop/intervention_store.py +118 -0
  90. agentloop/interventions.py +190 -0
  91. agentloop/issues.py +90 -0
  92. agentloop/judgment_adapters.py +125 -0
  93. agentloop/judgment_benchmark_types.py +98 -0
  94. agentloop/judgment_benchmarks.py +606 -0
  95. agentloop/judgment_types.py +241 -0
  96. agentloop/judgment_views.py +38 -0
  97. agentloop/judgments.py +342 -0
  98. agentloop/loop_guards.py +274 -0
  99. agentloop/loop_types.py +66 -0
  100. agentloop/markdown.py +102 -0
  101. agentloop/metrics.py +342 -0
  102. agentloop/migrations.py +303 -0
  103. agentloop/model_routing.py +470 -0
  104. agentloop/onboarding.py +315 -0
  105. agentloop/operations.py +44 -0
  106. agentloop/optimizer.py +199 -0
  107. agentloop/otel.py +715 -0
  108. agentloop/otel_ids.py +81 -0
  109. agentloop/otel_semantics.py +134 -0
  110. agentloop/parallelism.py +141 -0
  111. agentloop/patches.py +603 -0
  112. agentloop/plan_export.py +126 -0
  113. agentloop/policy_promotion.py +325 -0
  114. agentloop/promotion_evidence.py +304 -0
  115. agentloop/promotion_types.py +209 -0
  116. agentloop/quality.py +446 -0
  117. agentloop/ranking.py +529 -0
  118. agentloop/recovery_evidence.py +96 -0
  119. agentloop/replay.py +753 -0
  120. agentloop/retention.py +375 -0
  121. agentloop/retention_types.py +113 -0
  122. agentloop/reuse_experiments.py +247 -0
  123. agentloop/routing_checks.py +116 -0
  124. agentloop/routing_types.py +241 -0
  125. agentloop/rules.py +396 -0
  126. agentloop/runtime.py +286 -0
  127. agentloop/savings.py +158 -0
  128. agentloop/scheduling_plan.py +99 -0
  129. agentloop/scheduling_types.py +169 -0
  130. agentloop/schema.py +257 -0
  131. agentloop/semantic_rules.py +200 -0
  132. agentloop/semantic_waste.py +345 -0
  133. agentloop/semantic_waste_types.py +88 -0
  134. agentloop/server.py +494 -0
  135. agentloop/store.py +1429 -0
  136. agentloop/structural_experiments.py +431 -0
  137. agentloop/structural_types.py +113 -0
  138. agentloop/structured_quality.py +1020 -0
  139. agentloop/studies.py +629 -0
  140. agentloop/study_statistics.py +58 -0
  141. agentloop/substitution_evidence.py +199 -0
  142. agentloop/substitution_export.py +190 -0
  143. agentloop/substitution_reports.py +407 -0
  144. agentloop/substitution_types.py +284 -0
  145. agentloop/substitutions.py +267 -0
  146. agentloop/timing.py +142 -0
  147. agentloop/tokens.py +188 -0
  148. agentloop/tool_scheduling.py +410 -0
  149. agentloop/trace_sources.py +31 -0
  150. agentloop/tracer.py +494 -0
  151. agentloop/value.py +306 -0
  152. agentloop/version.py +3 -0
  153. agentloop/workflow_types.py +264 -0
  154. agentloop/workflows.py +284 -0
  155. agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
  156. agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
  157. agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
  158. agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
  159. agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
  160. agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
  161. agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
  162. dashboard/__init__.py +5 -0
  163. dashboard/app.py +922 -0
  164. dashboard/streamlit_compat.py +63 -0
  165. dashboard/trace_upload.py +57 -0
  166. dashboard/value_view.py +105 -0
agentloop/costs.py ADDED
@@ -0,0 +1,667 @@
1
+ """Provider-aware model cost estimation with an explicit unknown state.
2
+
3
+ The core rule of this module: **a cost is only ever reported when it can be
4
+ justified.** A model whose pricing is not known does not silently borrow a
5
+ generic rate — it returns an ``unknown`` estimate with ``amount_usd = None``, so
6
+ downstream reports can distinguish a measured or calculated cost from a missing
7
+ one instead of presenting a fabricated number as observed.
8
+
9
+ Three kinds of cost are represented (`CostState`):
10
+
11
+ - ``provider_reported`` — the caller supplied the actual billed cost (e.g. from a
12
+ provider usage field), which is used verbatim.
13
+ - ``calculated`` — computed from a known per-model rate in the pricing table.
14
+ - ``unknown`` — no rate is known and no reported cost was supplied; ``amount_usd``
15
+ is ``None``.
16
+
17
+ Pricing is offline and configurable. The built-in table carries provenance (a
18
+ ``source`` and an ``as_of`` date) and is a point-in-time snapshot — provider
19
+ prices change, so treat the built-ins as a convenience, not an authority, and
20
+ override them for anything that matters. Users supply their own per-model rates
21
+ without editing this package by pointing ``AGENTLOOP_PRICING_FILE`` at a JSON
22
+ file (see ``load_pricing_table``), or by passing a ``PricingTable`` explicitly.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import json
28
+ import math
29
+ import os
30
+ from dataclasses import dataclass, field
31
+ from functools import lru_cache
32
+ from pathlib import Path
33
+ from typing import Any, Literal
34
+
35
+ CostState = Literal["provider_reported", "calculated", "unknown"]
36
+ CostStatus = Literal["complete", "partial", "unknown", "empty"]
37
+
38
+ # Environment variable pointing at a JSON pricing-override file. Kept here (not in
39
+ # config.py) so the cost module stays self-contained and importable without the
40
+ # rest of the package's configuration surface.
41
+ PRICING_FILE_ENV = "AGENTLOOP_PRICING_FILE"
42
+
43
+
44
+ def is_cost_evaluable(status: str | None) -> bool:
45
+ """Return whether a cost total is complete enough for comparisons."""
46
+
47
+ return status in {"complete", "empty"}
48
+
49
+
50
+ def format_cost_usd(
51
+ amount_usd: float | None,
52
+ status: str | None = "complete",
53
+ *,
54
+ decimals: int = 4,
55
+ token_status: str | None = None,
56
+ ) -> str:
57
+ """Format a cost without presenting an inexact total as an exact amount.
58
+
59
+ Two things can make a cost inexact, and both are qualified in the rendered
60
+ string. ``status`` covers pricing completeness: a ``partial`` total is a known
61
+ lower bound. ``token_status`` covers the basis underneath — a calculated cost
62
+ is a rate multiplied by a token count, so a count that was approximated from
63
+ word counts yields an approximate cost, however precise the arithmetic looks.
64
+ Pass the report's ``token_status`` to have that qualified too; omitting it
65
+ keeps the pricing-only behavior callers had before.
66
+ """
67
+
68
+ status = "complete" if status is None else status
69
+ if (
70
+ amount_usd is None
71
+ or not _is_finite_number(amount_usd)
72
+ or amount_usd < 0
73
+ or status not in {"complete", "empty", "partial"}
74
+ ):
75
+ return "unavailable"
76
+ rendered = f"${float(amount_usd):,.{decimals}f}"
77
+ if status == "partial":
78
+ rendered = f"{rendered} (known lower bound)"
79
+ if token_status is not None and token_status not in {"exact", "empty"}:
80
+ rendered = f"{rendered} (from {token_status} tokens)"
81
+ return rendered
82
+
83
+
84
+ def _is_finite_number(value: Any) -> bool:
85
+ """Return whether value is a supported finite real number.
86
+
87
+ Values must also fit Python's floating-point calculations; extremely large
88
+ integers are rejected instead of surfacing ``OverflowError`` later.
89
+ """
90
+
91
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
92
+ return False
93
+ try:
94
+ return math.isfinite(float(value))
95
+ except OverflowError:
96
+ return False
97
+
98
+
99
+ def _is_integral_number(value: Any) -> bool:
100
+ return _is_finite_number(value) and (
101
+ isinstance(value, int) or (isinstance(value, float) and value.is_integer())
102
+ )
103
+
104
+
105
+ def _validated_token_counts(
106
+ input_tokens: Any, output_tokens: Any, cached_input_tokens: Any = 0
107
+ ) -> tuple[int, int, int]:
108
+ if not _is_integral_number(input_tokens) or not _is_integral_number(output_tokens):
109
+ raise ValueError("input_tokens and output_tokens must be finite integers")
110
+ resolved_input = int(input_tokens)
111
+ resolved_output = int(output_tokens)
112
+ if resolved_input < 0 or resolved_output < 0:
113
+ raise ValueError("input_tokens and output_tokens must be non-negative")
114
+ if not _is_integral_number(cached_input_tokens):
115
+ raise ValueError("cached_input_tokens must be a finite integer")
116
+ resolved_cached = int(cached_input_tokens)
117
+ if not 0 <= resolved_cached <= resolved_input:
118
+ raise ValueError(
119
+ "cached_input_tokens must satisfy 0 <= cached_input_tokens <= input_tokens"
120
+ )
121
+ return resolved_input, resolved_output, resolved_cached
122
+
123
+
124
+ @dataclass(frozen=True)
125
+ class ModelPricing:
126
+ """Per-model pricing in USD per million tokens, with provenance.
127
+
128
+ ``cached_input_usd_per_mtok`` is the (usually discounted) rate for cached
129
+ input tokens; when ``None`` the plain input rate is used for any cached
130
+ tokens, so a caller can always pass ``cached_input_tokens`` without needing
131
+ every model to declare a cache rate.
132
+ """
133
+
134
+ input_usd_per_mtok: float
135
+ output_usd_per_mtok: float
136
+ provider: str
137
+ source: str
138
+ as_of: str
139
+ cached_input_usd_per_mtok: float | None = None
140
+ # This rate is only valid up to this many input tokens; above it the entry
141
+ # does not apply (many providers charge a higher tier for long context, e.g.
142
+ # Gemini above 200K). ``None`` means the rate has no declared upper bound.
143
+ max_input_tokens: int | None = None
144
+
145
+ def __post_init__(self) -> None:
146
+ for name, value in (
147
+ ("provider", self.provider),
148
+ ("source", self.source),
149
+ ("as_of", self.as_of),
150
+ ):
151
+ if not isinstance(value, str) or not value.strip():
152
+ raise ValueError(f"{name} must be a non-empty string")
153
+ object.__setattr__(self, name, value.strip())
154
+ for name, rate in (
155
+ ("input_usd_per_mtok", self.input_usd_per_mtok),
156
+ ("output_usd_per_mtok", self.output_usd_per_mtok),
157
+ ):
158
+ if not _is_finite_number(rate) or rate < 0:
159
+ raise ValueError(f"{name} must be a finite, non-negative number")
160
+ cached_rate = self.cached_input_usd_per_mtok
161
+ if cached_rate is not None and (not _is_finite_number(cached_rate) or cached_rate < 0):
162
+ raise ValueError("cached_input_usd_per_mtok must be a finite, non-negative number")
163
+ if self.max_input_tokens is not None and (
164
+ not _is_integral_number(self.max_input_tokens) or self.max_input_tokens <= 0
165
+ ):
166
+ raise ValueError("max_input_tokens must be a positive integer")
167
+
168
+ def applies_to(self, input_tokens: int) -> bool:
169
+ resolved_input, _, _ = _validated_token_counts(input_tokens, 0)
170
+ return self.max_input_tokens is None or resolved_input <= self.max_input_tokens
171
+
172
+ def cost_usd(
173
+ self, input_tokens: int, output_tokens: int, cached_input_tokens: int = 0
174
+ ) -> float:
175
+ input_tokens, output_tokens, cached_input_tokens = _validated_token_counts(
176
+ input_tokens, output_tokens, cached_input_tokens
177
+ )
178
+ billable_input = input_tokens - cached_input_tokens
179
+ cached_rate = (
180
+ self.cached_input_usd_per_mtok
181
+ if self.cached_input_usd_per_mtok is not None
182
+ else self.input_usd_per_mtok
183
+ )
184
+ cost = (
185
+ (billable_input / 1_000_000) * self.input_usd_per_mtok
186
+ + (cached_input_tokens / 1_000_000) * cached_rate
187
+ + (output_tokens / 1_000_000) * self.output_usd_per_mtok
188
+ )
189
+ return round(cost, 6)
190
+
191
+
192
+ @dataclass(frozen=True)
193
+ class CostEstimate:
194
+ """The cost of a single model call, with enough context to audit it.
195
+
196
+ ``amount_usd`` is ``None`` exactly when ``state == "unknown"``.
197
+ """
198
+
199
+ state: CostState
200
+ amount_usd: float | None
201
+ model: str | None
202
+ provider: str | None
203
+ pricing_source: str | None
204
+ pricing_as_of: str | None
205
+ input_tokens: int
206
+ output_tokens: int
207
+ cached_input_tokens: int = 0
208
+ # Populated only when ``state == "unknown"``: a short machine-usable reason
209
+ # (e.g. ``unpriced_model``, ``context_over_threshold``,
210
+ # ``unsupported_billing_mode``, ``invalid_metadata``) so consumers can tell
211
+ # "no rate for this model" apart from "we refused to price this call".
212
+ unknown_reason: str | None = None
213
+
214
+ def __post_init__(self) -> None:
215
+ if (self.state == "unknown") != (self.amount_usd is None):
216
+ raise ValueError("amount_usd must be None exactly when state is unknown")
217
+ if self.amount_usd is not None and (
218
+ not _is_finite_number(self.amount_usd) or self.amount_usd < 0
219
+ ):
220
+ raise ValueError("amount_usd must be a finite, non-negative number")
221
+
222
+ @property
223
+ def is_known(self) -> bool:
224
+ return self.state != "unknown"
225
+
226
+ def to_dict(self) -> dict[str, Any]:
227
+ return {
228
+ "state": self.state,
229
+ "amount_usd": self.amount_usd,
230
+ "model": self.model,
231
+ "provider": self.provider,
232
+ "pricing_source": self.pricing_source,
233
+ "pricing_as_of": self.pricing_as_of,
234
+ "input_tokens": self.input_tokens,
235
+ "output_tokens": self.output_tokens,
236
+ "cached_input_tokens": self.cached_input_tokens,
237
+ "unknown_reason": self.unknown_reason,
238
+ }
239
+
240
+
241
+ # Built-in pricing snapshot. Provenance is attached to every entry so a report
242
+ # can state where a rate came from and how current it is. These are a convenience
243
+ # for common models only — they are NOT kept up to date automatically and must be
244
+ # overridden for anything cost-sensitive (see docs/PRICING.md).
245
+ _OPENAI_SOURCE = "https://openai.com/api/pricing/"
246
+ _ANTHROPIC_SOURCE = "https://www.anthropic.com/pricing"
247
+ _GOOGLE_SOURCE = "https://ai.google.dev/gemini-api/docs/pricing"
248
+ _OPENAI_AS_OF = "2025-06-01"
249
+ _ANTHROPIC_AS_OF = "2025-06-01"
250
+ _GOOGLE_AS_OF = "2025-06-17"
251
+
252
+ # Gemini 2.5 charges a higher tier above 200K input tokens; the built-in flat
253
+ # rate is only correct at or below that, so above it we return unknown rather
254
+ # than under-report. A deployment that needs the long-context tier priced adds
255
+ # its own override (see docs/PRICING.md).
256
+ _GEMINI_STANDARD_CONTEXT = 200_000
257
+
258
+ _BUILTIN_PRICING: dict[str, ModelPricing] = {
259
+ # OpenAI. The original three rates this module shipped with are preserved so
260
+ # previously-priced traces keep the same numbers.
261
+ "gpt-4.1": ModelPricing(2.00, 8.00, "openai", _OPENAI_SOURCE, _OPENAI_AS_OF, 0.50),
262
+ "gpt-4.1-mini": ModelPricing(0.40, 1.60, "openai", _OPENAI_SOURCE, _OPENAI_AS_OF, 0.10),
263
+ "gpt-4o-mini": ModelPricing(0.15, 0.60, "openai", _OPENAI_SOURCE, _OPENAI_AS_OF, 0.075),
264
+ "gpt-4o": ModelPricing(2.50, 10.00, "openai", _OPENAI_SOURCE, _OPENAI_AS_OF, 1.25),
265
+ # Anthropic.
266
+ "claude-sonnet-4": ModelPricing(
267
+ 3.00, 15.00, "anthropic", _ANTHROPIC_SOURCE, _ANTHROPIC_AS_OF, 0.30
268
+ ),
269
+ "claude-opus-4": ModelPricing(
270
+ 15.00, 75.00, "anthropic", _ANTHROPIC_SOURCE, _ANTHROPIC_AS_OF, 1.50
271
+ ),
272
+ "claude-3-5-haiku": ModelPricing(
273
+ 0.80, 4.00, "anthropic", _ANTHROPIC_SOURCE, _ANTHROPIC_AS_OF, 0.08
274
+ ),
275
+ # Google. Pro has a higher price tier above the standard-context threshold;
276
+ # Flash pricing is flat across its supported context window.
277
+ "gemini-2.5-pro": ModelPricing(
278
+ 1.25,
279
+ 10.00,
280
+ "google",
281
+ _GOOGLE_SOURCE,
282
+ _GOOGLE_AS_OF,
283
+ max_input_tokens=_GEMINI_STANDARD_CONTEXT,
284
+ ),
285
+ "gemini-2.5-flash": ModelPricing(
286
+ 0.30,
287
+ 2.50,
288
+ "google",
289
+ _GOOGLE_SOURCE,
290
+ _GOOGLE_AS_OF,
291
+ ),
292
+ }
293
+
294
+
295
+ @dataclass
296
+ class PricingTable:
297
+ """A resolvable set of model rates: user overrides layered over built-ins.
298
+
299
+ Resolution normalizes a model identifier so dated snapshots and common
300
+ prefixes map onto a base rate (e.g. ``gpt-4.1-2025-04-14`` and
301
+ ``openai/gpt-4.1`` both resolve to ``gpt-4.1``). An override always wins over
302
+ a built-in with the same key.
303
+ """
304
+
305
+ rates: dict[str, ModelPricing] = field(default_factory=dict)
306
+
307
+ def resolve(
308
+ self,
309
+ model: str | None,
310
+ provider: str | None = None,
311
+ billing_mode: str | None = None,
312
+ ) -> ModelPricing | None:
313
+ """Resolve a rate, treating an explicit provider as a hard constraint.
314
+
315
+ When a provider is specified (as the ``provider`` argument or a
316
+ ``provider/model`` prefix), a bare-model entry only matches if its own
317
+ ``provider`` equals it — so ``azure/gpt-4o`` or ``provider="bedrock"``
318
+ will not silently borrow the OpenAI or Anthropic rate for a same-named
319
+ model. A key that was itself provider-qualified (e.g. a user override
320
+ keyed ``"azure/gpt-4o"``) is trusted as an explicit intent.
321
+
322
+ A non-standard ``billing_mode`` (e.g. ``batch``) only matches a
323
+ mode-qualified key; it never falls back to the standard rate.
324
+ """
325
+ if not model:
326
+ return None
327
+ effective_provider = provider or _provider_prefix(model)
328
+ mode = _normalize_billing_mode(billing_mode)
329
+ for candidate, provider_qualified in _resolution_candidates(model, provider, mode):
330
+ pricing = self.rates.get(candidate)
331
+ if pricing is None:
332
+ continue
333
+ if provider_qualified:
334
+ return pricing
335
+ if effective_provider and pricing.provider.lower() != effective_provider.lower():
336
+ continue
337
+ return pricing
338
+ return None
339
+
340
+ def with_overrides(self, overrides: dict[str, ModelPricing]) -> PricingTable:
341
+ merged = dict(self.rates)
342
+ merged.update(overrides)
343
+ return PricingTable(rates=merged)
344
+
345
+
346
+ def _normalize_key(model: str) -> str:
347
+ return model.strip().lower()
348
+
349
+
350
+ _STANDARD_BILLING_MODES = {"", "standard", "default", "sync", "on-demand"}
351
+
352
+
353
+ def _normalize_billing_mode(billing_mode: str | None) -> str | None:
354
+ if billing_mode is None:
355
+ return None
356
+ mode = billing_mode.strip().lower()
357
+ return None if mode in _STANDARD_BILLING_MODES else mode
358
+
359
+
360
+ def _provider_prefix(model: str) -> str | None:
361
+ normalized = _normalize_key(model)
362
+ for separator in ("/", ":"):
363
+ if separator in normalized:
364
+ return normalized.split(separator, 1)[0]
365
+ return None
366
+
367
+
368
+ def _resolution_candidates(
369
+ model: str, provider: str | None, billing_mode: str | None
370
+ ) -> list[tuple[str, bool]]:
371
+ """Ordered ``(key, provider_qualified)`` pairs for a model, most specific first.
372
+
373
+ Handles ``provider/model`` prefixes, ``provider:model`` prefixes, and dated
374
+ snapshot suffixes (a trailing ``-YYYY-MM-DD`` or ``-YYYYMMDD``) so a specific
375
+ snapshot falls back to its base model rate when only the base is known. When
376
+ ``billing_mode`` is non-standard, every key is mode-suffixed (``model#mode``)
377
+ and the plain keys are omitted, so a non-standard mode never resolves to a
378
+ standard rate.
379
+ """
380
+ normalized = _normalize_key(model)
381
+ base: list[tuple[str, bool]] = []
382
+ seen: set[str] = set()
383
+
384
+ def _add(value: str, provider_qualified: bool) -> None:
385
+ if value and value not in seen:
386
+ seen.add(value)
387
+ base.append((value, provider_qualified))
388
+
389
+ # A provider-qualified key is the most specific: try "provider/model" first.
390
+ if provider:
391
+ _add(f"{_normalize_key(provider)}/{normalized}", True)
392
+
393
+ # Strip a leading "provider/" or "provider:" that came in the model string.
394
+ bare = normalized
395
+ for separator in ("/", ":"):
396
+ if separator in bare:
397
+ bare = bare.split(separator, 1)[1]
398
+ _add(normalized, False)
399
+ _add(bare, False)
400
+
401
+ # Strip a trailing dated snapshot suffix so gpt-4.1-2025-04-14 -> gpt-4.1.
402
+ for snapshot_base in (_strip_snapshot(normalized), _strip_snapshot(bare)):
403
+ _add(snapshot_base, False)
404
+
405
+ if billing_mode is None:
406
+ return base
407
+ return [(f"{key}#{billing_mode}", qualified) for key, qualified in base]
408
+
409
+
410
+ def _strip_snapshot(model: str) -> str:
411
+ parts = model.split("-")
412
+ # Drop a trailing YYYY-MM-DD (three parts) or YYYYMMDD (one part) date.
413
+ if len(parts) >= 4 and _looks_like_date_parts(parts[-3:]):
414
+ return "-".join(parts[:-3])
415
+ if len(parts) >= 2 and parts[-1].isdigit() and len(parts[-1]) == 8:
416
+ return "-".join(parts[:-1])
417
+ return model
418
+
419
+
420
+ def _looks_like_date_parts(parts: list[str]) -> bool:
421
+ if len(parts) != 3:
422
+ return False
423
+ year, month, day = parts
424
+ return (
425
+ year.isdigit()
426
+ and len(year) == 4
427
+ and month.isdigit()
428
+ and len(month) == 2
429
+ and day.isdigit()
430
+ and len(day) == 2
431
+ )
432
+
433
+
434
+ def _pricing_from_mapping(key: str, raw: dict[str, Any]) -> ModelPricing:
435
+ """Build a ModelPricing from a JSON/dict override entry, with clear errors."""
436
+ try:
437
+ input_raw = raw["input_usd_per_mtok"]
438
+ output_raw = raw["output_usd_per_mtok"]
439
+ if isinstance(input_raw, bool) or isinstance(output_raw, bool):
440
+ raise TypeError("rates must be numeric, not boolean")
441
+ input_rate = float(input_raw)
442
+ output_rate = float(output_raw)
443
+ except (KeyError, OverflowError, TypeError, ValueError) as exc:
444
+ raise ValueError(
445
+ f"pricing override for {key!r} must set numeric 'input_usd_per_mtok' "
446
+ f"and 'output_usd_per_mtok' ({exc})"
447
+ ) from exc
448
+ cached_raw = raw.get("cached_input_usd_per_mtok")
449
+ if isinstance(cached_raw, bool):
450
+ raise ValueError(
451
+ f"pricing override for {key!r} 'cached_input_usd_per_mtok' must be numeric, not boolean"
452
+ )
453
+ try:
454
+ cached_rate = None if cached_raw is None else float(cached_raw)
455
+ except (OverflowError, TypeError, ValueError) as exc:
456
+ raise ValueError(
457
+ f"pricing override for {key!r} must set a numeric 'cached_input_usd_per_mtok' ({exc})"
458
+ ) from exc
459
+ max_input_raw = raw.get("max_input_tokens")
460
+ if max_input_raw is not None and not _is_integral_number(max_input_raw):
461
+ raise ValueError(f"pricing override for {key!r} must set an integer 'max_input_tokens'")
462
+ try:
463
+ return ModelPricing(
464
+ input_usd_per_mtok=input_rate,
465
+ output_usd_per_mtok=output_rate,
466
+ provider=raw.get("provider", "custom"),
467
+ source=raw.get("source", "user override"),
468
+ as_of=raw.get("as_of", "user-provided"),
469
+ cached_input_usd_per_mtok=cached_rate,
470
+ max_input_tokens=None if max_input_raw is None else int(max_input_raw),
471
+ )
472
+ except ValueError as exc:
473
+ raise ValueError(f"pricing override for {key!r} is invalid: {exc}") from exc
474
+
475
+
476
+ def load_overrides_from_file(path: str | os.PathLike[str]) -> dict[str, ModelPricing]:
477
+ """Load per-model pricing overrides from a JSON file.
478
+
479
+ The file maps a model key (matched after normalization — a bare model name,
480
+ ``provider/model``, or a dated snapshot are all valid keys) to an object with
481
+ ``input_usd_per_mtok``/``output_usd_per_mtok`` and optional ``provider``,
482
+ ``source``, ``as_of``, and ``cached_input_usd_per_mtok``. A top-level
483
+ ``models`` wrapper object is accepted for readability.
484
+ """
485
+ text = Path(path).read_text(encoding="utf-8")
486
+ data = json.loads(text)
487
+ if not isinstance(data, dict):
488
+ raise ValueError(f"pricing file {os.fspath(path)} must contain a JSON object")
489
+ entries = data.get("models") if isinstance(data.get("models"), dict) else data
490
+ overrides: dict[str, ModelPricing] = {}
491
+ for key, raw in entries.items():
492
+ if not isinstance(raw, dict):
493
+ raise ValueError(f"pricing entry for {key!r} must be an object")
494
+ overrides[_normalize_key(key)] = _pricing_from_mapping(key, raw)
495
+ return overrides
496
+
497
+
498
+ @lru_cache(maxsize=8)
499
+ def _cached_overrides_from_file(
500
+ path: str, mtime_ns: int, ctime_ns: int, size: int
501
+ ) -> dict[str, ModelPricing]:
502
+ """Reuse parsed overrides until the file's observable identity changes."""
503
+ _ = (mtime_ns, ctime_ns, size)
504
+ return load_overrides_from_file(path)
505
+
506
+
507
+ def load_pricing_table(
508
+ *,
509
+ overrides: dict[str, ModelPricing] | None = None,
510
+ include_builtins: bool = True,
511
+ env: dict[str, str] | None = None,
512
+ ) -> PricingTable:
513
+ """Build the effective pricing table: built-ins, then file, then explicit overrides.
514
+
515
+ Precedence (later wins): built-in snapshot < ``AGENTLOOP_PRICING_FILE`` <
516
+ ``overrides`` passed here. Set ``include_builtins=False`` to price *only*
517
+ against user-provided rates (so every unlisted model is explicitly unknown).
518
+ """
519
+ environ = os.environ if env is None else env
520
+ table = PricingTable(rates=dict(_BUILTIN_PRICING) if include_builtins else {})
521
+
522
+ file_path = environ.get(PRICING_FILE_ENV)
523
+ if file_path:
524
+ normalized_path = os.path.abspath(os.fspath(file_path))
525
+ file_stat = os.stat(normalized_path)
526
+ file_overrides = _cached_overrides_from_file(
527
+ normalized_path,
528
+ file_stat.st_mtime_ns,
529
+ file_stat.st_ctime_ns,
530
+ file_stat.st_size,
531
+ )
532
+ table = table.with_overrides(file_overrides)
533
+
534
+ if overrides:
535
+ table = table.with_overrides({_normalize_key(k): v for k, v in overrides.items()})
536
+
537
+ return table
538
+
539
+
540
+ def _unknown(
541
+ model: str | None,
542
+ provider: str | None,
543
+ input_tokens: int,
544
+ output_tokens: int,
545
+ cached_input_tokens: int,
546
+ reason: str,
547
+ ) -> CostEstimate:
548
+ return CostEstimate(
549
+ state="unknown",
550
+ amount_usd=None,
551
+ model=model,
552
+ provider=provider,
553
+ pricing_source=None,
554
+ pricing_as_of=None,
555
+ input_tokens=input_tokens,
556
+ output_tokens=output_tokens,
557
+ cached_input_tokens=cached_input_tokens,
558
+ unknown_reason=reason,
559
+ )
560
+
561
+
562
+ def estimate_cost(
563
+ model: str | None,
564
+ input_tokens: int,
565
+ output_tokens: int,
566
+ *,
567
+ provider: str | None = None,
568
+ cached_input_tokens: int = 0,
569
+ billing_mode: str | None = None,
570
+ provider_reported_cost_usd: float | None = None,
571
+ pricing: PricingTable | None = None,
572
+ ) -> CostEstimate:
573
+ """Estimate the cost of one model call, preferring reported > calculated > unknown.
574
+
575
+ - If ``provider_reported_cost_usd`` is given, it's used verbatim
576
+ (``provider_reported``) — the provider's own number always beats a
577
+ calculation. A non-finite or negative reported cost is rejected with a
578
+ ``ValueError`` (report construction catches this and yields ``unknown``).
579
+ - Otherwise, if the model resolves to a known rate — under the effective
580
+ provider, billing mode, and context threshold — the cost is
581
+ ``calculated``.
582
+ - Otherwise the estimate is ``unknown`` with ``amount_usd = None`` and an
583
+ ``unknown_reason``; no generic rate is substituted.
584
+
585
+ ``cached_input_tokens`` must satisfy ``0 <= cached <= input_tokens``.
586
+ """
587
+ table = pricing if pricing is not None else load_pricing_table()
588
+
589
+ input_tokens, output_tokens, cached_input_tokens = _validated_token_counts(
590
+ input_tokens, output_tokens, cached_input_tokens
591
+ )
592
+
593
+ if model is not None and not isinstance(model, str):
594
+ raise ValueError("model must be a string or None")
595
+ if provider is not None and (not isinstance(provider, str) or not provider.strip()):
596
+ raise ValueError("provider must be a non-empty string or None")
597
+ provider = provider.strip() if provider is not None else None
598
+ if billing_mode is not None and not isinstance(billing_mode, str):
599
+ raise ValueError("billing_mode must be a string or None")
600
+
601
+ if provider_reported_cost_usd is not None:
602
+ if not _is_finite_number(provider_reported_cost_usd) or provider_reported_cost_usd < 0:
603
+ raise ValueError("provider_reported_cost_usd must be a finite, non-negative number")
604
+ resolved = table.resolve(model, provider, billing_mode)
605
+ return CostEstimate(
606
+ state="provider_reported",
607
+ amount_usd=float(provider_reported_cost_usd),
608
+ model=model,
609
+ provider=provider or (resolved.provider if resolved else None),
610
+ pricing_source="provider-reported",
611
+ pricing_as_of=None,
612
+ input_tokens=input_tokens,
613
+ output_tokens=output_tokens,
614
+ cached_input_tokens=cached_input_tokens,
615
+ )
616
+
617
+ resolved = table.resolve(model, provider, billing_mode)
618
+ if resolved is None:
619
+ mode = _normalize_billing_mode(billing_mode)
620
+ reason = (
621
+ "unsupported_billing_mode"
622
+ if mode is not None and table.resolve(model, provider) is not None
623
+ else "unpriced_model"
624
+ )
625
+ return _unknown(model, provider, input_tokens, output_tokens, cached_input_tokens, reason)
626
+
627
+ if not resolved.applies_to(input_tokens):
628
+ # A rate exists but only for a smaller context; refuse to under-report.
629
+ return _unknown(
630
+ model,
631
+ resolved.provider,
632
+ input_tokens,
633
+ output_tokens,
634
+ cached_input_tokens,
635
+ "context_over_threshold",
636
+ )
637
+
638
+ return CostEstimate(
639
+ state="calculated",
640
+ amount_usd=resolved.cost_usd(input_tokens, output_tokens, cached_input_tokens),
641
+ model=model,
642
+ provider=resolved.provider,
643
+ pricing_source=resolved.source,
644
+ pricing_as_of=resolved.as_of,
645
+ input_tokens=input_tokens,
646
+ output_tokens=output_tokens,
647
+ cached_input_tokens=cached_input_tokens,
648
+ )
649
+
650
+
651
+ def estimate_cost_usd(model: str | None, input_tokens: int, output_tokens: int) -> float | None:
652
+ """Backward-compatible thin wrapper returning just the amount.
653
+
654
+ **Behavior change:** this now returns ``None`` for a model with no known
655
+ rate, instead of the old fabricated ``$1/M`` in / ``$3/M`` out default. Use
656
+ :func:`estimate_cost` for the full provider/provenance/state context.
657
+ """
658
+ return estimate_cost(model, input_tokens, output_tokens).amount_usd
659
+
660
+
661
+ # Legacy export: a plain {model: {input, output}} view of the built-in rates for
662
+ # any external reader that imported MODEL_PRICES. It intentionally no longer has a
663
+ # "default" entry — the absence of a default is the whole point of this module.
664
+ MODEL_PRICES: dict[str, dict[str, float]] = {
665
+ name: {"input": pricing.input_usd_per_mtok, "output": pricing.output_usd_per_mtok}
666
+ for name, pricing in _BUILTIN_PRICING.items()
667
+ }