tokencat 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencat/__init__.py +5 -0
- tokencat/__main__.py +5 -0
- tokencat/cli.py +451 -0
- tokencat/core/__init__.py +1 -0
- tokencat/core/aggregate.py +209 -0
- tokencat/core/filters.py +21 -0
- tokencat/core/models.py +286 -0
- tokencat/core/pricing.py +302 -0
- tokencat/core/privacy.py +10 -0
- tokencat/core/render.py +251 -0
- tokencat/core/serialize.py +101 -0
- tokencat/core/time.py +76 -0
- tokencat/pricing/__init__.py +1 -0
- tokencat/pricing/catalog.json +107 -0
- tokencat/providers/__init__.py +1 -0
- tokencat/providers/base.py +15 -0
- tokencat/providers/codex.py +467 -0
- tokencat/providers/copilot.py +73 -0
- tokencat/providers/gemini.py +110 -0
- tokencat/providers/registry.py +34 -0
- tokencat-0.1.0.dist-info/METADATA +229 -0
- tokencat-0.1.0.dist-info/RECORD +26 -0
- tokencat-0.1.0.dist-info/WHEEL +5 -0
- tokencat-0.1.0.dist-info/entry_points.txt +2 -0
- tokencat-0.1.0.dist-info/licenses/LICENSE +674 -0
- tokencat-0.1.0.dist-info/top_level.txt +1 -0
tokencat/__init__.py
ADDED
tokencat/__main__.py
ADDED
tokencat/cli.py
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
from tokencat.core.aggregate import aggregate_daily, aggregate_models, aggregate_summary, build_dashboard_overview
|
|
12
|
+
from tokencat.core.models import PricingCatalog, PricingCoverage, ProviderName, ScanFilters
|
|
13
|
+
from tokencat.core.pricing import apply_pricing, load_pricing_catalog, refresh_builtin_pricing
|
|
14
|
+
from tokencat.core.render import render_dashboard, render_pricing_summary
|
|
15
|
+
from tokencat.core.serialize import (
|
|
16
|
+
serialize_daily_records,
|
|
17
|
+
serialize_filters,
|
|
18
|
+
serialize_pricing_catalog,
|
|
19
|
+
serialize_pricing_coverage,
|
|
20
|
+
serialize_session,
|
|
21
|
+
serialize_status,
|
|
22
|
+
)
|
|
23
|
+
from tokencat.core.time import local_now, parse_datetime_value
|
|
24
|
+
from tokencat.providers.registry import scan_providers
|
|
25
|
+
|
|
26
|
+
app = typer.Typer(help="TokenCat: local-first, read-only token and usage inspector for AI coding agents.", invoke_without_command=True)
|
|
27
|
+
pricing_app = typer.Typer(help="Inspect and refresh the local pricing catalog.")
|
|
28
|
+
app.add_typer(pricing_app, name="pricing")
|
|
29
|
+
console = Console(highlight=False)
|
|
30
|
+
|
|
31
|
+
ProviderOption = Annotated[
|
|
32
|
+
list[ProviderName] | None,
|
|
33
|
+
typer.Option(
|
|
34
|
+
"--provider",
|
|
35
|
+
help="Filter to one or more providers.",
|
|
36
|
+
case_sensitive=False,
|
|
37
|
+
),
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def build_filters(
|
|
42
|
+
providers: list[ProviderName] | None,
|
|
43
|
+
since: str | None,
|
|
44
|
+
until: str | None,
|
|
45
|
+
limit: int | None,
|
|
46
|
+
model: str | None,
|
|
47
|
+
show_title: bool,
|
|
48
|
+
show_path: bool,
|
|
49
|
+
) -> ScanFilters:
|
|
50
|
+
provider_set = set(providers) if providers else None
|
|
51
|
+
try:
|
|
52
|
+
since_value = parse_datetime_value(since, bound="since")
|
|
53
|
+
until_value = parse_datetime_value(until, bound="until")
|
|
54
|
+
except ValueError as exc:
|
|
55
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
56
|
+
|
|
57
|
+
return ScanFilters(
|
|
58
|
+
providers=provider_set,
|
|
59
|
+
since=since_value,
|
|
60
|
+
until=until_value,
|
|
61
|
+
limit=limit,
|
|
62
|
+
model=model,
|
|
63
|
+
show_title=show_title,
|
|
64
|
+
show_path=show_path,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@app.callback()
|
|
69
|
+
def main(
|
|
70
|
+
ctx: typer.Context,
|
|
71
|
+
providers: ProviderOption = None,
|
|
72
|
+
since: Annotated[str | None, typer.Option("--since", help="Relative like 7d/24h or ISO date/datetime.")] = "7d",
|
|
73
|
+
until: Annotated[str | None, typer.Option("--until", help="Relative like 7d/24h or ISO date/datetime.")] = None,
|
|
74
|
+
no_price: Annotated[bool, typer.Option("--no-price", help="Disable pricing and cost estimation.")] = False,
|
|
75
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit structured JSON instead of styled dashboard output.")] = False,
|
|
76
|
+
) -> None:
|
|
77
|
+
if ctx.invoked_subcommand is None:
|
|
78
|
+
dashboard(providers=providers, since=since, until=until, no_price=no_price, json_output=json_output)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@app.command()
|
|
82
|
+
def dashboard(
|
|
83
|
+
providers: ProviderOption = None,
|
|
84
|
+
since: Annotated[str | None, typer.Option("--since", help="Relative like 7d/24h or ISO date/datetime.")] = "7d",
|
|
85
|
+
until: Annotated[str | None, typer.Option("--until", help="Relative like 7d/24h or ISO date/datetime.")] = None,
|
|
86
|
+
no_price: Annotated[bool, typer.Option("--no-price", help="Disable pricing and cost estimation.")] = False,
|
|
87
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit structured JSON instead of the dashboard.")] = False,
|
|
88
|
+
) -> None:
|
|
89
|
+
filters = build_filters(providers, since, until, limit=None, model=None, show_title=False, show_path=False)
|
|
90
|
+
result, catalog, coverage = _scan_with_pricing(filters, pricing_enabled=not no_price)
|
|
91
|
+
summary_data = aggregate_summary(result.sessions, pricing_coverage=coverage)
|
|
92
|
+
daily = aggregate_daily(result.sessions)
|
|
93
|
+
top_models = aggregate_models(result.sessions)
|
|
94
|
+
overview = build_dashboard_overview(summary_data, top_models, result.statuses)
|
|
95
|
+
recent_sessions = result.sessions[:6]
|
|
96
|
+
time_label = _format_window_label(filters)
|
|
97
|
+
|
|
98
|
+
payload = {
|
|
99
|
+
"generated_at": local_now().isoformat(),
|
|
100
|
+
"filters": serialize_filters(filters),
|
|
101
|
+
"providers": [serialize_status(status) for status in result.statuses],
|
|
102
|
+
"summary": {
|
|
103
|
+
"overview": overview,
|
|
104
|
+
"daily": serialize_daily_records(daily),
|
|
105
|
+
"top_models": top_models[:8],
|
|
106
|
+
"recent_sessions": [serialize_session(record, show_title=False, show_path=False) for record in recent_sessions],
|
|
107
|
+
"pricing": {
|
|
108
|
+
"catalog": serialize_pricing_catalog(catalog),
|
|
109
|
+
"coverage": serialize_pricing_coverage(coverage),
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
"warnings": result.warnings,
|
|
113
|
+
}
|
|
114
|
+
if json_output:
|
|
115
|
+
console.print_json(json.dumps(payload, ensure_ascii=False))
|
|
116
|
+
return
|
|
117
|
+
|
|
118
|
+
render_dashboard(
|
|
119
|
+
console,
|
|
120
|
+
time_label=time_label,
|
|
121
|
+
statuses=result.statuses,
|
|
122
|
+
overview=overview,
|
|
123
|
+
daily=daily[-7:],
|
|
124
|
+
sessions=recent_sessions,
|
|
125
|
+
pricing_catalog=catalog,
|
|
126
|
+
pricing_coverage=coverage,
|
|
127
|
+
warnings=result.warnings,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@app.command()
|
|
132
|
+
def doctor(
|
|
133
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit structured JSON instead of tables.")] = False,
|
|
134
|
+
) -> None:
|
|
135
|
+
filters = ScanFilters()
|
|
136
|
+
result = scan_providers(filters)
|
|
137
|
+
catalog = load_pricing_catalog()
|
|
138
|
+
pricing_summary = {
|
|
139
|
+
"catalog": serialize_pricing_catalog(catalog),
|
|
140
|
+
}
|
|
141
|
+
payload = {
|
|
142
|
+
"generated_at": datetime.now().astimezone().isoformat(),
|
|
143
|
+
"filters": serialize_filters(filters),
|
|
144
|
+
"providers": [serialize_status(status) for status in result.statuses],
|
|
145
|
+
"summary": pricing_summary,
|
|
146
|
+
"warnings": result.warnings,
|
|
147
|
+
}
|
|
148
|
+
if json_output:
|
|
149
|
+
console.print_json(json.dumps(payload, ensure_ascii=False))
|
|
150
|
+
return
|
|
151
|
+
|
|
152
|
+
table = Table(title="TokenCat Doctor")
|
|
153
|
+
table.add_column("Provider")
|
|
154
|
+
table.add_column("Status")
|
|
155
|
+
table.add_column("Found Paths")
|
|
156
|
+
table.add_column("Ignored Paths")
|
|
157
|
+
table.add_column("Reasons")
|
|
158
|
+
for status in result.statuses:
|
|
159
|
+
table.add_row(
|
|
160
|
+
status.provider.value,
|
|
161
|
+
status.status.value,
|
|
162
|
+
"\n".join(str(path) for path in status.found_paths) or "-",
|
|
163
|
+
"\n".join(str(path) for path in status.ignored_paths) or "-",
|
|
164
|
+
"\n".join(status.reasons + status.warnings) or "-",
|
|
165
|
+
)
|
|
166
|
+
console.print(table)
|
|
167
|
+
render_pricing_summary(console, catalog=catalog, coverage=None, unknown_models=[])
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@app.command()
|
|
171
|
+
def summary(
|
|
172
|
+
providers: ProviderOption = None,
|
|
173
|
+
since: Annotated[str | None, typer.Option("--since", help="Relative like 7d/24h or ISO date/datetime.")] = None,
|
|
174
|
+
until: Annotated[str | None, typer.Option("--until", help="Relative like 7d/24h or ISO date/datetime.")] = None,
|
|
175
|
+
limit: Annotated[int | None, typer.Option("--limit", min=1, help="Cap matching sessions before aggregation.")] = None,
|
|
176
|
+
no_price: Annotated[bool, typer.Option("--no-price", help="Disable pricing and cost estimation.")] = False,
|
|
177
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit structured JSON instead of tables.")] = False,
|
|
178
|
+
) -> None:
|
|
179
|
+
filters = build_filters(providers, since, until, limit, model=None, show_title=False, show_path=False)
|
|
180
|
+
result, _, coverage = _scan_with_pricing(filters, pricing_enabled=not no_price)
|
|
181
|
+
summary_data = aggregate_summary(result.sessions, pricing_coverage=coverage)
|
|
182
|
+
payload = {
|
|
183
|
+
"generated_at": local_now().isoformat(),
|
|
184
|
+
"filters": serialize_filters(filters),
|
|
185
|
+
"providers": [serialize_status(status) for status in result.statuses],
|
|
186
|
+
"summary": summary_data,
|
|
187
|
+
"warnings": result.warnings,
|
|
188
|
+
}
|
|
189
|
+
if json_output:
|
|
190
|
+
console.print_json(json.dumps(payload, ensure_ascii=False))
|
|
191
|
+
return
|
|
192
|
+
|
|
193
|
+
overall = Table(title="TokenCat Summary")
|
|
194
|
+
overall.add_column("Metric")
|
|
195
|
+
overall.add_column("Value")
|
|
196
|
+
overall.add_row("Sessions", str(summary_data["session_count"]))
|
|
197
|
+
overall.add_row("Models", str(summary_data["model_count"]))
|
|
198
|
+
overall.add_row("Estimated API Cost", _format_cost(summary_data["estimated_cost"]["total_cost"]))
|
|
199
|
+
if summary_data.get("pricing_coverage"):
|
|
200
|
+
overall.add_row("Priced Coverage", _format_ratio(summary_data["pricing_coverage"]["priced_ratio"]))
|
|
201
|
+
overall.add_row("Unknown Models", ", ".join(summary_data["pricing_coverage"]["unknown_models"]) or "-")
|
|
202
|
+
for name, value in _token_rows(summary_data["token_totals"]).items():
|
|
203
|
+
overall.add_row(name, value)
|
|
204
|
+
console.print(overall)
|
|
205
|
+
|
|
206
|
+
providers_table = Table(title="By Provider")
|
|
207
|
+
providers_table.add_column("Provider")
|
|
208
|
+
providers_table.add_column("Sessions")
|
|
209
|
+
providers_table.add_column("Models")
|
|
210
|
+
providers_table.add_column("Total Tokens")
|
|
211
|
+
providers_table.add_column("Est Cost")
|
|
212
|
+
for provider_name, provider_summary in summary_data["providers"].items():
|
|
213
|
+
providers_table.add_row(
|
|
214
|
+
provider_name,
|
|
215
|
+
str(provider_summary["session_count"]),
|
|
216
|
+
str(provider_summary["model_count"]),
|
|
217
|
+
_format_tokens(provider_summary["token_totals"]["total"]),
|
|
218
|
+
_format_cost(provider_summary["estimated_cost"]["total_cost"]),
|
|
219
|
+
)
|
|
220
|
+
console.print(providers_table)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@app.command()
|
|
224
|
+
def sessions(
|
|
225
|
+
providers: ProviderOption = None,
|
|
226
|
+
since: Annotated[str | None, typer.Option("--since", help="Relative like 7d/24h or ISO date/datetime.")] = "7d",
|
|
227
|
+
until: Annotated[str | None, typer.Option("--until", help="Relative like 7d/24h or ISO date/datetime.")] = None,
|
|
228
|
+
limit: Annotated[int | None, typer.Option("--limit", min=1, help="Maximum number of sessions to show.")] = 50,
|
|
229
|
+
model: Annotated[str | None, typer.Option("--model", help="Only include sessions that used this model.")] = None,
|
|
230
|
+
show_title: Annotated[bool, typer.Option("--show-title", help="Show local session titles when available.")] = False,
|
|
231
|
+
show_path: Annotated[bool, typer.Option("--show-path", help="Show local paths/source refs when available.")] = False,
|
|
232
|
+
no_price: Annotated[bool, typer.Option("--no-price", help="Disable pricing and cost estimation.")] = False,
|
|
233
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit structured JSON instead of tables.")] = False,
|
|
234
|
+
) -> None:
|
|
235
|
+
filters = build_filters(providers, since, until, limit, model, show_title, show_path)
|
|
236
|
+
result, _, _ = _scan_with_pricing(filters, pricing_enabled=not no_price)
|
|
237
|
+
payload = {
|
|
238
|
+
"generated_at": local_now().isoformat(),
|
|
239
|
+
"filters": serialize_filters(filters),
|
|
240
|
+
"providers": [serialize_status(status) for status in result.statuses],
|
|
241
|
+
"items": [serialize_session(record, show_title=show_title, show_path=show_path) for record in result.sessions],
|
|
242
|
+
"warnings": result.warnings,
|
|
243
|
+
}
|
|
244
|
+
if json_output:
|
|
245
|
+
console.print_json(json.dumps(payload, ensure_ascii=False))
|
|
246
|
+
return
|
|
247
|
+
|
|
248
|
+
table = Table(title="TokenCat Sessions")
|
|
249
|
+
table.add_column("Anon ID")
|
|
250
|
+
table.add_column("Provider")
|
|
251
|
+
table.add_column("Updated")
|
|
252
|
+
table.add_column("Primary Model")
|
|
253
|
+
table.add_column("Attr")
|
|
254
|
+
table.add_column("Total Tokens")
|
|
255
|
+
if not no_price:
|
|
256
|
+
table.add_column("Est Cost", justify="right")
|
|
257
|
+
table.add_column("Pricing")
|
|
258
|
+
if show_title:
|
|
259
|
+
table.add_column("Title")
|
|
260
|
+
if show_path:
|
|
261
|
+
table.add_column("Path")
|
|
262
|
+
|
|
263
|
+
for record in result.sessions:
|
|
264
|
+
row = [
|
|
265
|
+
record.anon_session_id,
|
|
266
|
+
record.provider.value,
|
|
267
|
+
_format_datetime(record.updated_at or record.started_at),
|
|
268
|
+
record.primary_model or "-",
|
|
269
|
+
record.attribution_status or "-",
|
|
270
|
+
_format_tokens(record.token_totals.total),
|
|
271
|
+
]
|
|
272
|
+
if not no_price:
|
|
273
|
+
row.append(_format_cost(record.estimated_cost.total_cost if record.estimated_cost is not None else 0.0))
|
|
274
|
+
row.append(record.pricing_status or "-")
|
|
275
|
+
if show_title:
|
|
276
|
+
row.append(record.title or "-")
|
|
277
|
+
if show_path:
|
|
278
|
+
path_value = record.cwd or (str(record.source_refs[0]) if record.source_refs else "-")
|
|
279
|
+
row.append(path_value)
|
|
280
|
+
table.add_row(*row)
|
|
281
|
+
console.print(table)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
@app.command()
|
|
285
|
+
def models(
|
|
286
|
+
providers: ProviderOption = None,
|
|
287
|
+
since: Annotated[str | None, typer.Option("--since", help="Relative like 7d/24h or ISO date/datetime.")] = "7d",
|
|
288
|
+
until: Annotated[str | None, typer.Option("--until", help="Relative like 7d/24h or ISO date/datetime.")] = None,
|
|
289
|
+
limit: Annotated[int | None, typer.Option("--limit", min=1, help="Maximum number of rows to show.")] = None,
|
|
290
|
+
no_price: Annotated[bool, typer.Option("--no-price", help="Disable pricing and cost estimation.")] = False,
|
|
291
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit structured JSON instead of tables.")] = False,
|
|
292
|
+
) -> None:
|
|
293
|
+
filters = build_filters(providers, since, until, limit=None, model=None, show_title=False, show_path=False)
|
|
294
|
+
result, _, _ = _scan_with_pricing(filters, pricing_enabled=not no_price)
|
|
295
|
+
items = aggregate_models(result.sessions)
|
|
296
|
+
if limit is not None:
|
|
297
|
+
items = items[:limit]
|
|
298
|
+
payload = {
|
|
299
|
+
"generated_at": local_now().isoformat(),
|
|
300
|
+
"filters": serialize_filters(filters),
|
|
301
|
+
"providers": [serialize_status(status) for status in result.statuses],
|
|
302
|
+
"items": items,
|
|
303
|
+
"warnings": result.warnings,
|
|
304
|
+
}
|
|
305
|
+
if json_output:
|
|
306
|
+
console.print_json(json.dumps(payload, ensure_ascii=False))
|
|
307
|
+
return
|
|
308
|
+
|
|
309
|
+
table = Table(title="TokenCat Models")
|
|
310
|
+
table.add_column("Provider")
|
|
311
|
+
table.add_column("Model")
|
|
312
|
+
table.add_column("Attr")
|
|
313
|
+
table.add_column("Sessions")
|
|
314
|
+
table.add_column("Messages")
|
|
315
|
+
table.add_column("Total Tokens")
|
|
316
|
+
table.add_column("Input")
|
|
317
|
+
table.add_column("Output")
|
|
318
|
+
table.add_column("Cached")
|
|
319
|
+
if not no_price:
|
|
320
|
+
table.add_column("Est Cost", justify="right")
|
|
321
|
+
table.add_column("Coverage", justify="right")
|
|
322
|
+
|
|
323
|
+
for item in items:
|
|
324
|
+
tokens = item["token_totals"]
|
|
325
|
+
row = [
|
|
326
|
+
item["provider"],
|
|
327
|
+
item["model"],
|
|
328
|
+
item.get("attribution_status") or "-",
|
|
329
|
+
str(item["session_count"]),
|
|
330
|
+
str(item["message_count"]),
|
|
331
|
+
_format_tokens(tokens["total"]),
|
|
332
|
+
_format_tokens(tokens["input"]),
|
|
333
|
+
_format_tokens((tokens["output"] or 0) + (tokens["reasoning"] or 0)),
|
|
334
|
+
_format_tokens(tokens["cached"]),
|
|
335
|
+
]
|
|
336
|
+
if not no_price:
|
|
337
|
+
estimated = item.get("estimated_cost") or {}
|
|
338
|
+
row.append(_format_cost(estimated.get("total_cost", 0.0)))
|
|
339
|
+
row.append(_format_ratio(item.get("priced_token_coverage", 0.0)))
|
|
340
|
+
table.add_row(*row)
|
|
341
|
+
console.print(table)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
@pricing_app.command("show")
|
|
345
|
+
def pricing_show(
|
|
346
|
+
providers: ProviderOption = None,
|
|
347
|
+
since: Annotated[str | None, typer.Option("--since", help="Relative like 7d/24h or ISO date/datetime.")] = None,
|
|
348
|
+
until: Annotated[str | None, typer.Option("--until", help="Relative like 7d/24h or ISO date/datetime.")] = None,
|
|
349
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit structured JSON instead of tables.")] = False,
|
|
350
|
+
) -> None:
|
|
351
|
+
filters = build_filters(providers, since, until, limit=None, model=None, show_title=False, show_path=False)
|
|
352
|
+
result, catalog, coverage = _scan_with_pricing(filters, pricing_enabled=True)
|
|
353
|
+
unknown = coverage.unknown_models if coverage is not None else []
|
|
354
|
+
payload = {
|
|
355
|
+
"generated_at": local_now().isoformat(),
|
|
356
|
+
"filters": serialize_filters(filters),
|
|
357
|
+
"providers": [serialize_status(status) for status in result.statuses],
|
|
358
|
+
"summary": {
|
|
359
|
+
"pricing": {
|
|
360
|
+
"catalog": serialize_pricing_catalog(catalog),
|
|
361
|
+
"coverage": serialize_pricing_coverage(coverage),
|
|
362
|
+
"unknown_models": unknown,
|
|
363
|
+
}
|
|
364
|
+
},
|
|
365
|
+
"warnings": result.warnings,
|
|
366
|
+
}
|
|
367
|
+
if json_output:
|
|
368
|
+
console.print_json(json.dumps(payload, ensure_ascii=False))
|
|
369
|
+
return
|
|
370
|
+
render_pricing_summary(console, catalog=catalog, coverage=coverage, unknown_models=unknown)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
@pricing_app.command("refresh")
|
|
374
|
+
def pricing_refresh(
|
|
375
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit structured JSON instead of tables.")] = False,
|
|
376
|
+
) -> None:
|
|
377
|
+
warnings: list[str] = []
|
|
378
|
+
try:
|
|
379
|
+
catalog = refresh_builtin_pricing()
|
|
380
|
+
except Exception as exc: # pragma: no cover - exercised in tests by function patching
|
|
381
|
+
catalog = load_pricing_catalog()
|
|
382
|
+
warnings.append(str(exc))
|
|
383
|
+
|
|
384
|
+
payload = {
|
|
385
|
+
"generated_at": local_now().isoformat(),
|
|
386
|
+
"filters": serialize_filters(ScanFilters()),
|
|
387
|
+
"providers": [],
|
|
388
|
+
"summary": {
|
|
389
|
+
"pricing": {
|
|
390
|
+
"catalog": serialize_pricing_catalog(catalog),
|
|
391
|
+
"coverage": None,
|
|
392
|
+
"unknown_models": [],
|
|
393
|
+
}
|
|
394
|
+
},
|
|
395
|
+
"warnings": warnings,
|
|
396
|
+
}
|
|
397
|
+
if json_output:
|
|
398
|
+
console.print_json(json.dumps(payload, ensure_ascii=False))
|
|
399
|
+
return
|
|
400
|
+
render_pricing_summary(console, catalog=catalog, coverage=None, unknown_models=[])
|
|
401
|
+
if warnings:
|
|
402
|
+
console.print("\n".join(warnings))
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _scan_with_pricing(filters: ScanFilters, *, pricing_enabled: bool) -> tuple[object, PricingCatalog | None, PricingCoverage | None]:
|
|
406
|
+
result = scan_providers(filters)
|
|
407
|
+
if not pricing_enabled:
|
|
408
|
+
return result, None, None
|
|
409
|
+
catalog = load_pricing_catalog()
|
|
410
|
+
coverage = apply_pricing(result.sessions, catalog)
|
|
411
|
+
return result, catalog, coverage
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _token_rows(tokens: dict[str, int | None]) -> dict[str, str]:
|
|
415
|
+
return {
|
|
416
|
+
"Input Tokens": _format_tokens(tokens["input"]),
|
|
417
|
+
"Output Tokens": _format_tokens((tokens["output"] or 0) + (tokens["reasoning"] or 0)),
|
|
418
|
+
"Cached Tokens": _format_tokens(tokens["cached"]),
|
|
419
|
+
"Tool Tokens": _format_tokens(tokens["tool"]),
|
|
420
|
+
"Total Tokens": _format_tokens(tokens["total"]),
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _format_datetime(value: datetime | None) -> str:
|
|
425
|
+
return value.isoformat(timespec="seconds") if value is not None else "-"
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _format_cost(value: float | None) -> str:
|
|
429
|
+
return f"${(value or 0.0):,.2f}"
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _format_ratio(value: float) -> str:
|
|
433
|
+
return f"{value * 100:.1f}%"
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _format_window_label(filters: ScanFilters) -> str:
|
|
437
|
+
start = filters.since.astimezone().date().isoformat() if filters.since is not None else "start"
|
|
438
|
+
end = filters.until.astimezone().date().isoformat() if filters.until is not None else local_now().date().isoformat()
|
|
439
|
+
return f"{start} -> {end}"
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _format_tokens(value: int | None) -> str:
|
|
443
|
+
number = float(value or 0)
|
|
444
|
+
abs_number = abs(number)
|
|
445
|
+
if abs_number >= 1_000_000_000:
|
|
446
|
+
return f"{int(number):,} ({number / 1_000_000_000:.1f}B)"
|
|
447
|
+
if abs_number >= 1_000_000:
|
|
448
|
+
return f"{int(number):,} ({number / 1_000_000:.1f}M)"
|
|
449
|
+
if abs_number >= 1_000:
|
|
450
|
+
return f"{int(number):,} ({number / 1_000:.1f}K)"
|
|
451
|
+
return f"{int(number):,}"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Core TokenCat types and helpers."""
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from datetime import date
|
|
5
|
+
|
|
6
|
+
from tokencat.core.models import (
|
|
7
|
+
CostEstimate,
|
|
8
|
+
DailyModelUsageRecord,
|
|
9
|
+
DailyUsageRecord,
|
|
10
|
+
ModelUsage,
|
|
11
|
+
PricingCoverage,
|
|
12
|
+
ProviderName,
|
|
13
|
+
SessionRecord,
|
|
14
|
+
TokenTotals,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def aggregate_summary(records: list[SessionRecord], *, pricing_coverage: PricingCoverage | None = None) -> dict[str, object]:
|
|
19
|
+
totals = TokenTotals.zero()
|
|
20
|
+
provider_totals: dict[str, dict[str, object]] = {}
|
|
21
|
+
overall_models: set[str] = set()
|
|
22
|
+
total_cost = CostEstimate()
|
|
23
|
+
|
|
24
|
+
by_provider: dict[ProviderName, list[SessionRecord]] = defaultdict(list)
|
|
25
|
+
for record in records:
|
|
26
|
+
by_provider[record.provider].append(record)
|
|
27
|
+
totals.add(record.token_totals)
|
|
28
|
+
overall_models.update(record.models)
|
|
29
|
+
if record.estimated_cost is not None:
|
|
30
|
+
total_cost.add(record.estimated_cost)
|
|
31
|
+
|
|
32
|
+
for provider, provider_records in sorted(by_provider.items(), key=lambda item: item[0].value):
|
|
33
|
+
provider_tokens = TokenTotals.zero()
|
|
34
|
+
provider_models: set[str] = set()
|
|
35
|
+
provider_cost = CostEstimate()
|
|
36
|
+
for record in provider_records:
|
|
37
|
+
provider_tokens.add(record.token_totals)
|
|
38
|
+
provider_models.update(record.models)
|
|
39
|
+
if record.estimated_cost is not None:
|
|
40
|
+
provider_cost.add(record.estimated_cost)
|
|
41
|
+
provider_totals[provider.value] = {
|
|
42
|
+
"session_count": len(provider_records),
|
|
43
|
+
"model_count": len(provider_models),
|
|
44
|
+
"token_totals": provider_tokens.to_dict(),
|
|
45
|
+
"estimated_cost": provider_cost.to_dict(),
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
summary = {
|
|
49
|
+
"session_count": len(records),
|
|
50
|
+
"model_count": len(overall_models),
|
|
51
|
+
"token_totals": totals.to_dict(),
|
|
52
|
+
"estimated_cost": total_cost.to_dict(),
|
|
53
|
+
"providers": provider_totals,
|
|
54
|
+
}
|
|
55
|
+
if pricing_coverage is not None:
|
|
56
|
+
summary["pricing_coverage"] = pricing_coverage.to_dict()
|
|
57
|
+
return summary
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def aggregate_models(records: list[SessionRecord]) -> list[dict[str, object]]:
|
|
61
|
+
buckets: dict[tuple[str, str], ModelUsage] = {}
|
|
62
|
+
sessions_per_model: dict[tuple[str, str], set[str]] = defaultdict(set)
|
|
63
|
+
attribution_statuses: dict[tuple[str, str], set[str]] = defaultdict(set)
|
|
64
|
+
pricing_models: dict[tuple[str, str], set[str]] = defaultdict(set)
|
|
65
|
+
fallback_flags: dict[tuple[str, str], bool] = defaultdict(bool)
|
|
66
|
+
|
|
67
|
+
for record in records:
|
|
68
|
+
for model_name, usage in record.model_usage.items():
|
|
69
|
+
key = (record.provider.value, model_name)
|
|
70
|
+
bucket = buckets.setdefault(key, ModelUsage(model=model_name, tokens=TokenTotals.zero(), estimated_cost=CostEstimate()))
|
|
71
|
+
bucket.add(usage.tokens, message_count=usage.message_count)
|
|
72
|
+
if usage.estimated_cost is not None:
|
|
73
|
+
bucket.estimated_cost.add(usage.estimated_cost)
|
|
74
|
+
if usage.attribution_status is not None:
|
|
75
|
+
attribution_statuses[key].add(usage.attribution_status)
|
|
76
|
+
if usage.pricing_model is not None:
|
|
77
|
+
pricing_models[key].add(usage.pricing_model)
|
|
78
|
+
fallback_flags[key] = fallback_flags[key] or usage.is_fallback_model
|
|
79
|
+
sessions_per_model[key].add(record.anon_session_id)
|
|
80
|
+
|
|
81
|
+
items: list[dict[str, object]] = []
|
|
82
|
+
for (provider, model), usage in buckets.items():
|
|
83
|
+
total_tokens = usage.tokens.total or 0
|
|
84
|
+
priced_tokens = usage.tokens.total if usage.estimated_cost and usage.estimated_cost.total_cost > 0 else 0
|
|
85
|
+
statuses = attribution_statuses[(provider, model)]
|
|
86
|
+
attribution_status = "fallback" if "fallback" in statuses or fallback_flags[(provider, model)] else "exact" if statuses else None
|
|
87
|
+
resolved_pricing_models = sorted(pricing_models[(provider, model)])
|
|
88
|
+
items.append(
|
|
89
|
+
{
|
|
90
|
+
"provider": provider,
|
|
91
|
+
"model": model,
|
|
92
|
+
"session_count": len(sessions_per_model[(provider, model)]),
|
|
93
|
+
"message_count": usage.message_count,
|
|
94
|
+
"token_totals": usage.tokens.to_dict(),
|
|
95
|
+
"estimated_cost": usage.estimated_cost.to_dict() if usage.estimated_cost is not None else None,
|
|
96
|
+
"priced_token_coverage": round((priced_tokens or 0) / total_tokens, 4) if total_tokens else 0.0,
|
|
97
|
+
"attribution_status": attribution_status,
|
|
98
|
+
"pricing_model": resolved_pricing_models[0] if len(resolved_pricing_models) == 1 else None,
|
|
99
|
+
"is_fallback_model": fallback_flags[(provider, model)],
|
|
100
|
+
}
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
items.sort(key=lambda item: (-((item["estimated_cost"] or {}).get("total_cost", 0) if item.get("estimated_cost") else 0), -(item["token_totals"]["total"] or 0), item["provider"], item["model"]))
|
|
104
|
+
return items
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def aggregate_daily(records: list[SessionRecord]) -> list[DailyUsageRecord]:
|
|
108
|
+
buckets: dict[date, DailyUsageRecord] = {}
|
|
109
|
+
model_buckets: dict[date, dict[tuple[ProviderName, str], DailyModelUsageRecord]] = defaultdict(dict)
|
|
110
|
+
for record in records:
|
|
111
|
+
timestamp = record.updated_at or record.started_at
|
|
112
|
+
if timestamp is None:
|
|
113
|
+
continue
|
|
114
|
+
day = timestamp.date()
|
|
115
|
+
bucket = buckets.setdefault(day, DailyUsageRecord(date=day))
|
|
116
|
+
bucket.providers.add(record.provider)
|
|
117
|
+
bucket.token_totals.add(record.token_totals)
|
|
118
|
+
bucket.session_count += 1
|
|
119
|
+
bucket.total_tokens += record.token_totals.total or 0
|
|
120
|
+
if record.estimated_cost is not None:
|
|
121
|
+
bucket.estimated_cost.add(record.estimated_cost)
|
|
122
|
+
priced_tokens = 0
|
|
123
|
+
seen_models_for_session: set[tuple[ProviderName, str]] = set()
|
|
124
|
+
for model_name, usage in record.model_usage.items():
|
|
125
|
+
key = (record.provider, model_name)
|
|
126
|
+
model_bucket = model_buckets[day].setdefault(
|
|
127
|
+
key,
|
|
128
|
+
DailyModelUsageRecord(
|
|
129
|
+
provider=record.provider,
|
|
130
|
+
model=model_name,
|
|
131
|
+
token_totals=TokenTotals.zero(),
|
|
132
|
+
estimated_cost=CostEstimate(),
|
|
133
|
+
),
|
|
134
|
+
)
|
|
135
|
+
model_bucket.token_totals.add(usage.tokens)
|
|
136
|
+
if usage.estimated_cost is not None:
|
|
137
|
+
model_bucket.estimated_cost.add(usage.estimated_cost)
|
|
138
|
+
if key not in seen_models_for_session:
|
|
139
|
+
model_bucket.session_count += 1
|
|
140
|
+
seen_models_for_session.add(key)
|
|
141
|
+
model_bucket.attribution_status = _pick_attribution_status(model_bucket.attribution_status, usage.attribution_status)
|
|
142
|
+
model_bucket.pricing_status = _pick_pricing_status(model_bucket.pricing_status, usage.pricing_status)
|
|
143
|
+
if usage.pricing_status in {"priced", "fallback_priced"}:
|
|
144
|
+
model_bucket.priced_tokens += usage.tokens.total or 0
|
|
145
|
+
priced_tokens += usage.tokens.total or 0
|
|
146
|
+
bucket.priced_tokens += priced_tokens
|
|
147
|
+
|
|
148
|
+
items: list[DailyUsageRecord] = []
|
|
149
|
+
for day in sorted(buckets):
|
|
150
|
+
bucket = buckets[day]
|
|
151
|
+
day_models = sorted(
|
|
152
|
+
model_buckets[day].values(),
|
|
153
|
+
key=lambda item: (
|
|
154
|
+
_daily_model_sort_rank(item.pricing_status),
|
|
155
|
+
-(item.token_totals.total or 0),
|
|
156
|
+
item.provider.value,
|
|
157
|
+
item.model,
|
|
158
|
+
),
|
|
159
|
+
)
|
|
160
|
+
bucket.models = day_models
|
|
161
|
+
items.append(bucket)
|
|
162
|
+
return items
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def build_dashboard_overview(summary: dict[str, object], top_models: list[dict[str, object]], statuses: list[object]) -> dict[str, object]:
|
|
166
|
+
pricing = summary.get("pricing_coverage") or {}
|
|
167
|
+
return {
|
|
168
|
+
**summary,
|
|
169
|
+
"top_models": top_models[:5],
|
|
170
|
+
"secondary_metrics": {
|
|
171
|
+
"priced_coverage": pricing.get("priced_ratio", 0.0),
|
|
172
|
+
"unknown_model_tokens": pricing.get("unknown_model_tokens", 0),
|
|
173
|
+
"unattributed_token_count": pricing.get("unattributed_token_count", 0),
|
|
174
|
+
"provider_count": len([status for status in statuses if getattr(getattr(status, "status", None), "value", None) == "supported"]),
|
|
175
|
+
},
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _daily_model_sort_rank(pricing_status: str | None) -> int:
|
|
180
|
+
if pricing_status in {"unknown_model", "unattributed"}:
|
|
181
|
+
return 1
|
|
182
|
+
return 0
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _pick_attribution_status(current: str | None, incoming: str | None) -> str | None:
|
|
186
|
+
if current == "exact" or incoming is None:
|
|
187
|
+
return current
|
|
188
|
+
if incoming == "exact" or current is None:
|
|
189
|
+
return incoming
|
|
190
|
+
if incoming == "fallback":
|
|
191
|
+
return "fallback"
|
|
192
|
+
return current or incoming
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _pick_pricing_status(current: str | None, incoming: str | None) -> str | None:
|
|
196
|
+
order = {
|
|
197
|
+
None: 0,
|
|
198
|
+
"priced": 1,
|
|
199
|
+
"fallback_priced": 2,
|
|
200
|
+
"partial": 3,
|
|
201
|
+
"unknown_model": 4,
|
|
202
|
+
"unattributed": 5,
|
|
203
|
+
"unpriced": 6,
|
|
204
|
+
}
|
|
205
|
+
if current is None:
|
|
206
|
+
return incoming
|
|
207
|
+
if incoming is None:
|
|
208
|
+
return current
|
|
209
|
+
return incoming if order[incoming] > order[current] else current
|