opencode-models 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
opencode_models/cli.py ADDED
@@ -0,0 +1,1172 @@
1
+ """Command-line interface.
2
+
3
+ Usage::
4
+
5
+ python3 -m opencode_models fetch # refresh data/ from opencode.ai
6
+
7
+ Exit codes: 0 success, 1 usage or data error (message on stdout).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import argparse
13
+ import os
14
+ import sys
15
+ import urllib.request
16
+ from dataclasses import replace
17
+ from pathlib import Path
18
+
19
+ from . import __version__, sources
20
+ from . import arena as arena_mod
21
+ from .arena import ArenaCache, merge_from_file
22
+ from .auth import load_key, save_key
23
+ from .availability import available_ids, load_available, offered
24
+ from .benchmarks import available
25
+ from .boards import fetch_boards, load_boards, merge_boards, save_boards
26
+ from .data import load_models
27
+ from .fetch import http_get, refresh
28
+ from .live import LIVE_CONFIG_URL, fetch_live, save_live
29
+ from .overrides import load_curated, off_band_models, source_tier
30
+ from .pricing import (
31
+ FREE_TIER_NOMINAL_RATES,
32
+ FREE_TIER_SNAPSHOT_NAME,
33
+ load_free_nominals,
34
+ load_prices,
35
+ refresh_free_nominals,
36
+ refresh_prices,
37
+ )
38
+ from .ranking import (
39
+ BAND_FREE,
40
+ BAND_INCLUDED,
41
+ BAND_PAYG,
42
+ RECOMMEND_FLOOR,
43
+ build_rows,
44
+ dedupe_providers,
45
+ grade_for,
46
+ grade_rows,
47
+ merge_rows,
48
+ next_up,
49
+ order_rows,
50
+ partition_rows,
51
+ plans_of,
52
+ ratio,
53
+ )
54
+ from .scoring import (
55
+ _TRACKED,
56
+ MIN_AXES,
57
+ _folded,
58
+ canonical,
59
+ combined,
60
+ derived_scores,
61
+ scales,
62
+ )
63
+ from .tables import column_widths, format_row, render
64
+
65
+ #: Where the data lives for an installed tool: $OPENCODE_MODELS_DATA, else the
66
+ #: XDG user-data dir (~/.local/share/opencode-models). `fetch` populates it and
67
+ #: every other command reads it; a repo checkout passes --data ./data.
68
+ DATA_ENV = "OPENCODE_MODELS_DATA"
69
+
70
+
71
+ def default_data_dir() -> Path:
72
+ """The user data directory: $OPENCODE_MODELS_DATA, else XDG."""
73
+ env = os.environ.get(DATA_ENV)
74
+ if env:
75
+ return Path(env)
76
+ xdg = os.environ.get("XDG_DATA_HOME")
77
+ base = Path(xdg) if xdg else Path.home() / ".local" / "share"
78
+ return base / "opencode-models"
79
+
80
+
81
+ DEFAULT_DATA = default_data_dir()
82
+
83
+ #: Representative cost: 20% input, 60% cached read, 20% output.
84
+ COST_NOTE = "cost 20% in / 60% cached / 20% out"
85
+ BAND_LABELS = (
86
+ (BAND_FREE, "free — use first"),
87
+ (BAND_INCLUDED, "included usage"),
88
+ (BAND_PAYG, "pay-as-you-go"),
89
+ )
90
+ #: Fixed columns, right-aligned; the model name takes the rest of the line.
91
+ #: Costs count cents ($/100) so a cheap line is a readable number.
92
+ #:
93
+ #: Two layouts, both built around the one score: the default shows the
94
+ #: model's score beside its cost and plan, and ``--benchmark X`` ranks on
95
+ #: one benchmark and adds that benchmark's tag. The per-axis raw numbers
96
+ #: live in `scores`, not here.
97
+ AXIS_HEADERS = ("SCORE", "¢/1M", "SCORE/$", "PROV")
98
+ SINGLE_HEADERS = ("SCORE", "BENCH", "¢/1M", "SCORE/$", "PROV")
99
+
100
+ #: Every row is prefixed by its rank marker, outside the column grid and
101
+ #: without a header of its own: ++ at 90+, + at 75+, o at 60+, - at 50+,
102
+ #: -- below that, · unranked (no score at all). The header line is indented
103
+ #: by the same width so the columns still line up.
104
+ GRADE_WIDTH = 5
105
+ RULE_WIDTH = 60
106
+
107
+
108
+ def _style(text: str) -> str:
109
+ """Bold when talking to a terminal, plain when piped or captured."""
110
+ return f"\033[1m{text}\033[0m" if sys.stdout.isatty() else text
111
+
112
+
113
+ def _row_line(cells: tuple[str, ...], widths: tuple[int, ...]) -> str:
114
+ return format_row(cells, widths)
115
+
116
+
117
+ def _header(widths: tuple[int, ...], axis: bool = True) -> str:
118
+ headings = AXIS_HEADERS if axis else SINGLE_HEADERS
119
+ return _style(format_row(headings, widths) + " MODEL")
120
+
121
+
122
+ def _section(title: str, widths: tuple[int, ...], axis: bool = True) -> None:
123
+ """Open a table: blank line, rule-wrapped title, highlighted header."""
124
+ print()
125
+ print(f"── {title} " + "─" * max(4, RULE_WIDTH - len(title)))
126
+ print(" " * GRADE_WIDTH + _header(widths, axis))
127
+
128
+
129
+ def _fail(msg: str) -> int:
130
+ print(msg)
131
+ return 1
132
+
133
+
134
+ def _load(data_dir: Path):
135
+ """Upstream records, arena scores, curated overlays and official boards.
136
+
137
+ Returns the models, the curated layer (which models carry overrides), and
138
+ the board-vs-registry disagreements — so the console can mark curated rows
139
+ and `overrides` can report the boards' numbers replacing self-reports.
140
+ """
141
+ curated = load_curated(data_dir / "opencode", data_dir / "overrides")
142
+ models = merge_from_file(curated.models, data_dir / "aiarena")
143
+ models, board_conflicts = merge_boards(models, load_boards(data_dir / "boards"))
144
+ return models, curated, board_conflicts
145
+
146
+
147
+ #: How much of a board row's provenance the NOTE column carries.
148
+ _NOTE_WIDTH = 52
149
+
150
+ #: Source tiers, short enough to sit beside a number without dominating the
151
+ #: column. The long names are in the footer rather than in every row.
152
+ _TIER_SHORT = {1: "indep", 2: "board", 3: "self"}
153
+
154
+
155
+ def _tier_word(source) -> str:
156
+ return _TIER_SHORT[source_tier(source)]
157
+
158
+
159
+ def _note_text(c) -> str:
160
+ """A conflict's provenance, trimmed to one line.
161
+
162
+ The board rows carry the run that produced the number - agent, effort,
163
+ rank, CI and trial count - and that is the part worth reading, so it
164
+ goes in the table rather than on a second line under it. A longer note
165
+ is cut: the full record is in ``data/boards/``.
166
+ """
167
+ text = " ".join((c.reason or "").split())
168
+ return text if len(text) <= _NOTE_WIDTH else text[: _NOTE_WIDTH - 1] + "…"
169
+
170
+
171
+ def _print_conflicts(conflicts, orphans=()) -> int:
172
+ """Disagreements between two sources, as a table with the value used.
173
+
174
+ Three numbers per row would be one too many, so the two reported values
175
+ carry their source tier and USED says which of them is in the view.
176
+ The arrow used to run upstream → override, which read as if the console
177
+ had taken the second number even where it kept the first.
178
+ """
179
+ rows = []
180
+ for c in conflicts:
181
+ rows.append(
182
+ (
183
+ c.model_id,
184
+ c.benchmark,
185
+ f"{c.upstream:g} {_tier_word(c.upstream_source)}",
186
+ f"{c.override:g} {_tier_word(c.override_source)}",
187
+ f"{c.used:g}",
188
+ _note_text(c),
189
+ )
190
+ )
191
+ orphans = [
192
+ (model_id, "", "", "", "", "override file has no upstream record")
193
+ for model_id in sorted(orphans)
194
+ ]
195
+ rows += orphans
196
+ if not rows:
197
+ return 0
198
+ headings = ("MODEL", "BENCHMARK", "CATALOG", "REPORTED", "USED", "NOTE")
199
+ print(render(headings, rows, aligns="<", indent=" "))
200
+ return len(conflicts) + len(orphans)
201
+
202
+
203
+ def _cmd_conflicts(args: argparse.Namespace) -> int:
204
+ """Curated overrides that disagree with upstream, ranked by provenance.
205
+
206
+ Each disagreement shows both values with the tier of their source, and
207
+ whether the override applies (its source is at least as good as
208
+ upstream's) or is merely recorded. To accept upstream, delete the
209
+ entry in data/overrides/.
210
+ """
211
+ data_dir = Path(args.data)
212
+ if not data_dir.is_dir():
213
+ return _fail(f"no data directory at {data_dir} — run `fetch` first")
214
+ models, curated, board_conflicts = _load(data_dir)
215
+ n = _print_conflicts(curated.conflicts, curated.orphans)
216
+ if board_conflicts:
217
+ print("— official boards —")
218
+ _print_conflicts(board_conflicts)
219
+ for mid, s21, s40, resid in off_band_models(curated.models):
220
+ print(
221
+ f"{mid} Terminal-Bench 2.1 {s21} vs 4.0 {s40} — "
222
+ f"{resid:+.0f} off the 2.1→4.0 band, check the run"
223
+ )
224
+ print(
225
+ f"{len(curated.curated)} curated models · {n} curated "
226
+ f"disagreements · {len(board_conflicts)} from official boards"
227
+ )
228
+ print(
229
+ "tiers: self = vendor self-report · board = official board · "
230
+ "indep = independent evaluator"
231
+ )
232
+ return 0
233
+
234
+
235
+ #: How often the long model-file walk reports: one line per file would bury
236
+ #: the per-source announcements, so every fiftieth file plus the last one.
237
+ _FETCH_TICK = 50
238
+
239
+
240
+ def _model_tick(done: int, total: int, name: str) -> None:
241
+ if done % _FETCH_TICK == 0 or done == total:
242
+ print(f" opencode.ai/data: {done}/{total} model files ...")
243
+
244
+
245
+ def _api_get(key: str):
246
+ """Authed transport for the key-gated Console API (30s per request)."""
247
+
248
+ def get(url: str) -> str:
249
+ req = urllib.request.Request(
250
+ url,
251
+ headers={
252
+ "Authorization": f"Bearer {key}",
253
+ "User-Agent": f"opencode-models/{__version__}",
254
+ },
255
+ )
256
+ try:
257
+ with urllib.request.urlopen(req, timeout=30) as resp:
258
+ return resp.read().decode("utf-8", errors="replace")
259
+ except urllib.error.HTTPError as exc:
260
+ exc.close() # see http_get: callers never read error bodies
261
+ raise
262
+
263
+ return get
264
+
265
+
266
+ def _cmd_auth(args: argparse.Namespace) -> int:
267
+ """Ask for the Console service-account key and save it owner-only.
268
+
269
+ The prompt never echoes, the key is never printed back, and the file
270
+ is created (or tightened) to owner-only access. The saved key is then
271
+ verified against the live config API — a rejection fails the command
272
+ but keeps the file, so an offline first run is not punished.
273
+ """
274
+ import getpass
275
+
276
+ if not sys.stdin.isatty():
277
+ return _fail(
278
+ "auth needs an interactive terminal — pipe a key via "
279
+ "OPENCODE_API_KEY instead"
280
+ )
281
+ key = getpass.getpass("OPENCODE_API_KEY: ").strip()
282
+ if not key:
283
+ return _fail("no key entered — nothing saved")
284
+ path = save_key(key)
285
+ print(f"saved to {path} (owner-only)")
286
+ try:
287
+ live = fetch_live(_api_get(key))
288
+ except (OSError, ValueError, KeyError) as exc:
289
+ print(f"warning: key saved but not verified ({exc})")
290
+ return 0
291
+ count = len(set().union(*live.values()))
292
+ print(
293
+ f"verified live: {count} models across "
294
+ f"{len([t for t in live.values() if t])} tiers"
295
+ )
296
+ return 0
297
+
298
+
299
+ def _display_names(data_dir: Path) -> dict[str, str]:
300
+ """Display names by slug or id, from the catalog records on disk.
301
+
302
+ Absent catalog, absent names: callers fall back to the slug itself.
303
+ """
304
+ names = {}
305
+ for model in load_models(Path(data_dir) / "opencode"):
306
+ names[model.model_id] = model.name
307
+ names[model.model_id.split("/")[-1]] = model.name
308
+ return names
309
+
310
+
311
+ def _model_name(names: dict[str, str], slug: str) -> str:
312
+ """The catalog's display name for a tier slug, or the slug itself.
313
+
314
+ Tier pages list free variants under their own slug
315
+ (``step-5-preview-free``) while the catalog carries the base model,
316
+ so the ``-free`` suffix reaches back to it.
317
+ """
318
+ if slug in names:
319
+ return names[slug]
320
+ if slug.endswith("-free") and slug[: -len("-free")] in names:
321
+ return names[slug[: -len("-free")]]
322
+ return slug
323
+
324
+
325
+ def _cmd_models(args: argparse.Namespace) -> int:
326
+ """The models the plans offer right now, per tier.
327
+
328
+ Live from the Console config API when ``OPENCODE_API_KEY`` is set —
329
+ recorded to ``available/`` on the way, so `list` and `next` unlist
330
+ the dropped models immediately. Else the snapshots the last ``fetch``
331
+ left behind, read-only.
332
+ """
333
+ data_dir = Path(args.data)
334
+ key = load_key()
335
+ tiers: dict[str, set[str]] | None = None
336
+ if key:
337
+ try:
338
+ tiers = fetch_live(_api_get(key))
339
+ except (OSError, ValueError, KeyError) as exc:
340
+ print(
341
+ f"warning: live config unreachable ({exc}) — showing the last snapshots"
342
+ )
343
+ tiers = None
344
+ else:
345
+ save_live(data_dir, tiers)
346
+ origin = (
347
+ f"live from {LIVE_CONFIG_URL} (recorded to {data_dir / 'available'})"
348
+ )
349
+ if tiers is None:
350
+ known = load_available(data_dir)
351
+ if all(ids is None for ids in known.values()):
352
+ return _fail(
353
+ f"no availability known in {data_dir} — run `fetch` "
354
+ f"first (with OPENCODE_API_KEY for the live set)"
355
+ )
356
+ tiers = {t: ids for t, ids in known.items() if ids is not None}
357
+ origin = f"snapshots in {data_dir} (run `fetch` to refresh)"
358
+ unknown = sorted(t for t, ids in known.items() if ids is None)
359
+ else:
360
+ unknown = []
361
+ names = _display_names(data_dir)
362
+ by_slug: dict[str, str] = {}
363
+ for tier in sorted(tiers):
364
+ for slug in sorted(tiers[tier]):
365
+ if slug not in by_slug or tier == "go":
366
+ by_slug[slug] = tier
367
+ rows = []
368
+ for slug, tier in by_slug.items():
369
+ endpoint = f"opencode-go/{slug}" if tier == "go" else f"opencode/{slug}"
370
+ name = _model_name(names, slug)
371
+ # the endpoint already carries the slug: a missing display name
372
+ # reads "—" (as a missing score does), not the slug twice
373
+ rows.append((endpoint, name if name != slug else "—"))
374
+ rows.sort(key=lambda row: (row[1].lower(), row[0]))
375
+ counts = " + ".join(f"{len(tiers[t])} {t}" for t in sorted(tiers))
376
+ print(origin)
377
+ print(
378
+ f"{len(by_slug)} models · {counts}"
379
+ + (f" · {', '.join(unknown)} unknown" if unknown else "")
380
+ )
381
+ print(render(["ENDPOINT", "MODEL"], rows, aligns=("<", "<"), indent=" "))
382
+ return 0
383
+
384
+
385
+ def _cmd_fetch(args: argparse.Namespace) -> int:
386
+ data_dir = Path(args.data)
387
+ key = load_key()
388
+ only: set[str] | None = None
389
+ if args.all_models:
390
+ print("fetching the full catalog (--all)")
391
+ elif key:
392
+ print(f"fetch live model set from {LIVE_CONFIG_URL} ...")
393
+ try:
394
+ live = fetch_live(_api_get(key))
395
+ except (OSError, ValueError, KeyError) as exc:
396
+ print(
397
+ f"warning: live config unreachable ({exc}) — "
398
+ f"downloading the full catalog"
399
+ )
400
+ else:
401
+ saved_live = save_live(data_dir, live)
402
+ print(f"saved {', '.join(saved_live)} to {data_dir / 'available'}")
403
+ only = set().union(*live.values()) or set()
404
+ if only is None and not args.all_models:
405
+ only = available_ids(data_dir)
406
+ if only is not None:
407
+ print(
408
+ f"{len(only)} models offered — downloading those "
409
+ f"(--all for the full catalog)"
410
+ )
411
+ print("fetch model records from opencode.ai/data ...")
412
+ saved = refresh(data_dir, get=http_get, progress=_model_tick, only=only)
413
+ print(f"saved {len(saved)} model files to {data_dir / 'opencode'}")
414
+ print("fetch tier pricing from opencode.ai/docs ...")
415
+ try:
416
+ prices = refresh_prices(data_dir / "tier_prices", get=http_get)
417
+ except (OSError, ValueError) as exc: # docs down must not fail the fetch
418
+ print(f"warning: tier pricing docs not refreshed ({exc})")
419
+ else:
420
+ if prices:
421
+ print(f"saved {', '.join(prices)} to {data_dir / 'tier_prices'}")
422
+ else:
423
+ print(
424
+ "warning: tier pricing docs parsed to nothing, "
425
+ "keeping existing snapshots"
426
+ )
427
+ print("fetch arena Elo from datasets-server.huggingface.co ...")
428
+ try:
429
+ elo = arena_mod.lookup_arena_elo(get=http_get)
430
+ except (OSError, ValueError) as exc: # one dead source must not stop the rest
431
+ print(f"warning: arena Elo not refreshed ({exc})")
432
+ else:
433
+ cache = ArenaCache(data_dir / "aiarena")
434
+ cache.save(elo)
435
+ print(
436
+ f"saved {len(elo)} arena Elo scores to "
437
+ f"{data_dir / 'aiarena' / arena_mod.CACHE_NAME}"
438
+ )
439
+ if not key:
440
+ print(
441
+ "free-tier nominals not refreshed (no OPENCODE_API_KEY) — "
442
+ "keeping the existing snapshot"
443
+ )
444
+ else:
445
+ try:
446
+ free_slugs = {
447
+ e.model_id
448
+ for prices in load_prices(data_dir / "tier_prices").values()
449
+ for e in prices
450
+ if e.free and e.model_id
451
+ }
452
+ print("fetch usage export from opencode.ai/console ...")
453
+ nominals = refresh_free_nominals(
454
+ data_dir / "usage", data_dir / "opencode", _api_get(key), free_slugs
455
+ )
456
+ except (OSError, ValueError) as exc:
457
+ print(f"warning: free-tier nominals not refreshed ({exc})")
458
+ else:
459
+ print(
460
+ f"saved {len(nominals)} free-tier nominals to "
461
+ f"{data_dir / 'usage' / FREE_TIER_SNAPSHOT_NAME}"
462
+ if nominals is not None
463
+ else "warning: free-tier nominals not refreshed "
464
+ "(anchor model absent from the export)"
465
+ )
466
+ curated = load_curated(data_dir / "opencode", data_dir / "overrides")
467
+ n = _print_conflicts(curated.conflicts, curated.orphans)
468
+ if n:
469
+ print(
470
+ f"{n} curated override(s) disagree with upstream — "
471
+ f"run `overrides` to review"
472
+ )
473
+ boards = fetch_boards(
474
+ get=http_get, progress=lambda done, total, desc: print(f"fetch {desc} ...")
475
+ )
476
+ saved = save_boards(data_dir / "boards", boards)
477
+ rows = sum(len(v) for v in boards.values())
478
+ if saved:
479
+ print(
480
+ f"saved {rows} board rows from {', '.join(saved)} to {data_dir / 'boards'}"
481
+ )
482
+ return 0
483
+
484
+
485
+ def _column_label(name: str) -> str:
486
+ """The short name a benchmark column carries: "Terminal Bench 2.1" → "TB 2.1".
487
+
488
+ Column headers have to fit beside each other, and there are enough
489
+ benchmarks to score on that the full names run past any terminal width.
490
+ """
491
+ if name == "LMArena Elo":
492
+ return "ARENA"
493
+ if name == "DeepSWE 1.1":
494
+ # the version lives in the legend ("DeepSWE = DeepSWE 1.1"):
495
+ # exact match, so a future revision keeps its number until renamed
496
+ return "DeepSWE"
497
+ for family, short in (("Terminal Bench", "TB"), ("SWE Bench", "SWE")):
498
+ if name.startswith(family):
499
+ rest = name[len(family) :]
500
+ if short == "SWE":
501
+ # "SWE Verified" is the widest header in the scores table;
502
+ # "Vrfd" keeps the column parseable at a glance
503
+ rest = rest.replace("Verified", "Vrfd")
504
+ return short + rest
505
+ return name
506
+
507
+
508
+ def _score_columns(models, table: dict) -> list[tuple[str, str, str]]:
509
+ """The benchmarks worth a column, as ``(key, header, full name)``.
510
+
511
+ One column per benchmark we score on, in the order ``scoring._TRACKED``
512
+ lists them — the axis the console ranks on first — and only where the
513
+ catalog actually has numbers. Two registries spelling one benchmark
514
+ differently share a column.
515
+ """
516
+ spellings: dict[str, set[str]] = {}
517
+ for model in models:
518
+ for name in derived_scores(model, table):
519
+ spellings.setdefault(canonical(name), set()).add(name)
520
+ columns = []
521
+ for key in _TRACKED:
522
+ folded = _folded(key)
523
+ if folded not in spellings:
524
+ continue
525
+ # prefer the hyphenated spelling for the full name: it is how the
526
+ # benchmark is written everywhere else, including in the README
527
+ name = sorted(spellings[folded], key=lambda n: (n.count("-") == 0, n))[0]
528
+ columns.append((folded, _column_label(folded), name))
529
+ return columns
530
+
531
+
532
+ def _score_cells(model, table: dict, columns: list[tuple[str, str, str]]) -> list[str]:
533
+ """One cell per column: each raw number with where it lands, best first.
534
+
535
+ The raw number is what the evaluator published; the derived number is
536
+ where that sits on the benchmark's 0-100 scale across this catalog.
537
+ Both are shown, because a score without its position in the field says
538
+ nothing on its own. A column the model was not measured on is blank
539
+ rather than zero — zero would read as "worst in the catalog".
540
+ """
541
+ derived = derived_scores(model, table)
542
+ cells = []
543
+ for key, _label, _name in columns:
544
+ parts = sorted(
545
+ ((name, derived[name]) for name in derived if canonical(name) == key),
546
+ key=lambda part: (-part[1], part[0]),
547
+ )
548
+ cells.append(" ".join(f"{model.scores[n]:g}→{d:.0f}" for n, d in parts))
549
+ return cells
550
+
551
+
552
+ def _cmd_scores(args: argparse.Namespace) -> int:
553
+ """Every model's score, one column per benchmark we score on.
554
+
555
+ Each cell holds the raw number and where it lands on that benchmark's
556
+ 0-100 scale, because the raw numbers do not compare across benchmarks.
557
+ How a model's score is derived from those cells is documented rather
558
+ than printed: see the README's "How a model's score is computed" and
559
+ ``scoring.py``. This command only reads: `fetch` owns every network call,
560
+ including the arena Elo cache.
561
+ """
562
+ data_dir = Path(args.data)
563
+ models, _curated, _boards = _load(data_dir)
564
+ table = scales(models)
565
+ on_offer = available_ids(data_dir)
566
+ shown = (
567
+ models
568
+ if getattr(args, "all_models", False) or not on_offer
569
+ else [m for m in models if offered(m.model_id, on_offer)]
570
+ )
571
+ ranked = sorted(
572
+ (m for m in shown if m.scores),
573
+ key=lambda m: -(combined(derived_scores(m, table)) or 0),
574
+ )
575
+ unscored = [m for m in shown if not m.scores]
576
+ columns = _score_columns(models, table)
577
+
578
+ # four counts, because "models with scores" used to mean "models with
579
+ # any benchmark" - which overstated the scored field by a third - and
580
+ # "on one benchmark" used to fold in models with no scored benchmark
581
+ # at all
582
+ counts = {}
583
+ for m in ranked:
584
+ counts.setdefault(len(derived_scores(m, table)), []).append(m)
585
+ scored = [m for m in ranked if combined(derived_scores(m, table)) is not None]
586
+ one = counts.get(1, [])
587
+ zerod = [m for m in ranked if not derived_scores(m, table)]
588
+ print(
589
+ f"{len(models)} models · {len(columns)} benchmarks scored on · "
590
+ f"{len(scored)} with scores · {len(one)} on one benchmark · "
591
+ f"{len(zerod)} with none scored · {len(unscored)} without any"
592
+ )
593
+ print()
594
+ cells = [_score_cells(m, table, columns) for m in ranked]
595
+ # the same markers as `list`, over the models shown here: rank position
596
+ # in this ordering, · where there is no score to rank
597
+ marks = {id(m): grade_for(combined(derived_scores(m, table))) for m in ranked}
598
+ print("GRADE: ++ ≥90 · + ≥75 · o 60–75 · - 50–60 · -- <50 · · unranked")
599
+ heads = ["MODEL", "GRADE", "SCORE"] + [label for _k, label, _n in columns]
600
+ grid = []
601
+ for model, row in zip(ranked, cells, strict=True):
602
+ derived = derived_scores(model, table)
603
+ score = combined(derived)
604
+ grid.append(
605
+ [
606
+ model.name,
607
+ marks.get(id(model), "·"),
608
+ f"{score:.0f}" if score is not None else "—",
609
+ ]
610
+ + row
611
+ )
612
+ # measured, not fixed: MODEL fits the shown rows (not the whole
613
+ # catalog) and the fixed columns fit their headers. The benchmark
614
+ # columns keep double gaps — their labels hold single spaces
615
+ # ("SWE Vrfd"), so the gap is what delimits them.
616
+ fixed_aligns = ["<", ">", ">"]
617
+ widths = column_widths(heads, grid)
618
+ lines = []
619
+ for cells in [heads] + grid:
620
+ left = format_row(cells[:3], widths[:3], fixed_aligns, gap=" ")
621
+ right = format_row(cells[3:], widths[3:], "<", gap=" ")
622
+ lines.append((" " + left + (" " + right if right else "")).rstrip())
623
+ print("\n".join(lines))
624
+ legend = [f"{label} = {name}" for _key, label, name in columns if label != name]
625
+ if legend:
626
+ print()
627
+ print(" " + " · ".join(legend))
628
+ if unscored:
629
+ print(
630
+ f" {len(unscored)} with no benchmarks: "
631
+ + ", ".join(sorted(m.name for m in unscored)[:10])
632
+ )
633
+ print(' how a score is derived: README, "How a model\'s score is computed"')
634
+ return 0
635
+
636
+
637
+ def _entries_for(source: str, data_dir: Path):
638
+ """Documented pricing lines for a tier (Go first: interchangeable, and
639
+ its rates are the ones with included usage attached)."""
640
+ prices = load_prices(data_dir / "tier_prices")
641
+ if source in prices:
642
+ return prices[source]
643
+ if source == "all":
644
+ return prices.get("go", []) + prices.get("zen", [])
645
+ return []
646
+
647
+
648
+ def _bench_tag(benchmark: str | None) -> str:
649
+ if not benchmark:
650
+ return "-"
651
+ folded = benchmark.lower().replace(" ", "-")
652
+ if folded.startswith("terminal-bench-4"):
653
+ return "TB4" # harder scale, ranked on its own
654
+ if folded.startswith("terminal-bench"):
655
+ return "TB"
656
+ if folded.startswith("swe-bench"):
657
+ return "SWE"
658
+ return benchmark[:5]
659
+
660
+
661
+ def _cells(row, axis: bool = True) -> tuple[str, ...]:
662
+ """The fixed columns of one row, as text: score, cost, value, plan.
663
+
664
+ The score is the model's combined 0-100 number, rounded for the column;
665
+ the value is score per dollar — free lines divide by their nominal
666
+ rate, so the cheapest free models read highest ("free" only where the
667
+ price is exactly zero, "-" with no score). The best-value table orders
668
+ on it, so the column that explains the order stays printed. *axis* is
669
+ kept for the single-benchmark layout, which adds that benchmark's tag.
670
+ Prices are cents, as elsewhere.
671
+ """
672
+ no_cost = row.free or row.price == 0
673
+ price = "0" if no_cost else f"{round(row.price * 100)}"
674
+ score = "-" if row.score is None else f"{round(row.score)}"
675
+ if axis:
676
+ return (score, price, _pick_value(row), row.provider)
677
+ return (score, _bench_tag(row.benchmark), price, _pick_value(row), row.provider)
678
+
679
+
680
+ def _widths(rows, axis: bool = True) -> tuple[int, ...]:
681
+ """Widest cell per column across the whole view, header included."""
682
+ headings = AXIS_HEADERS if axis else SINGLE_HEADERS
683
+ return tuple(column_widths(headings, [_cells(r, axis) for r in rows]))
684
+
685
+
686
+ def _print_row(
687
+ row, widths: tuple[int, ...], curated: set[str] = frozenset(), axis: bool = True
688
+ ) -> None:
689
+ name = f"{row.name} *" if row.model.model_id in curated else row.name
690
+ print(
691
+ f"{row.grade:>{GRADE_WIDTH - 1}} "
692
+ + _row_line(_cells(row, axis), widths)
693
+ + f" {name}"
694
+ )
695
+
696
+
697
+ def _select(data_dir: Path, source: str | None, price: str = "weighted"):
698
+ """Which providers a view covers, or None if the name is not one.
699
+
700
+ Providers are zen and go, plus ``-`` for catalog models in neither tier.
701
+ """
702
+ tiers = sources.available_sources()
703
+ if source is None:
704
+ return list(tiers), "tiers" # both, merged
705
+ if source == "all":
706
+ return list(tiers) + ["-"], "all"
707
+ if source in tiers:
708
+ return [source], "tier"
709
+ return None
710
+
711
+
712
+ def _current_offered(
713
+ data_dir: Path,
714
+ bypass: bool = False,
715
+ ) -> set[str] | None:
716
+ """Model ids offered right now, or None for no restriction.
717
+
718
+ With ``OPENCODE_API_KEY`` the live config API is the record — it is
719
+ also snapshotted to ``available/`` on the way, so offline runs inherit
720
+ the liveness. Without a key (or when the API is unreachable) the last
721
+ snapshots decide. *bypass* (`--all`) skips every filter.
722
+ """
723
+ if bypass:
724
+ return None
725
+ key = load_key()
726
+ if key:
727
+ try:
728
+ live = fetch_live(_api_get(key))
729
+ except (OSError, ValueError, KeyError) as exc:
730
+ print(
731
+ f"warning: live config unreachable ({exc}) — using the last snapshots"
732
+ )
733
+ else:
734
+ ids = set().union(*live.values())
735
+ if not ids:
736
+ print("warning: live config lists no models — using the last snapshots")
737
+ else:
738
+ save_live(data_dir, live)
739
+ return ids
740
+ else:
741
+ print(
742
+ "no API token found — availability may be stale, some models "
743
+ "may be unavailable (`auth` for live data, `--all` for the "
744
+ "full catalog)"
745
+ )
746
+ return available_ids(data_dir)
747
+
748
+
749
+ def _in_tier_slug_list(model_id: str, slugs: set[str]) -> bool:
750
+ """Whether *model_id* is claimed by a tier's shipped slug list.
751
+
752
+ Mirrors :func:`sources.members` matching: slug, full id, or the
753
+ ``-free`` variant of the slug. Used to keep tier-listed models out of
754
+ the catalog-only pool even when a tier dropped them from its offer.
755
+ """
756
+ slug = model_id.split("/")[-1]
757
+ return slug in slugs or model_id in slugs or f"{slug}-free" in slugs
758
+
759
+
760
+ def _rows_for(
761
+ data_dir: Path, selected: list[str], price: str, all_models: bool = False
762
+ ):
763
+ """Build, merge and dedupe the rows for *selected* providers.
764
+
765
+ Shared by `list` and `next` so a recommendation can never disagree with
766
+ the ranking it was drawn from.
767
+
768
+ Unless *all_models*, a row is a model the plans actually offer: the
769
+ live config API is the record when keyed, else the snapshots. A model
770
+ the plans dropped is neither listed nor scored. The scales are still
771
+ built over the whole catalog, so a score does not move when a plan
772
+ changes what it sells.
773
+ """
774
+ models, curated, _ = _load(data_dir)
775
+ on_offer = _current_offered(data_dir, bypass=all_models)
776
+ # the fetched free-tier nominals win per model; the committed table is
777
+ # only the offline fallback for whatever the snapshot does not cover
778
+ snapshot = load_free_nominals(data_dir)
779
+ free_rates = {**FREE_TIER_NOMINAL_RATES, **snapshot} if snapshot else None
780
+ # one scale for the whole catalog: per-tier scales would make the same
781
+ # score mean two different things in the merged view
782
+ table = scales(models)
783
+ groups, listed, counts, tier_slugs = [], [], {}, {}
784
+ covered: set[str] = set()
785
+ # slugs shipped for every selected tier: a model listed by any of them
786
+ # belongs to that tier (or is dropped by availability) and must not be
787
+ # routed to the catalog-only "-" pool, even when the tier dropped it
788
+ selected_tier_slugs: set[str] = set()
789
+ for provider in selected:
790
+ if provider != "-":
791
+ selected_tier_slugs |= sources.load_source(provider)
792
+ for provider in selected:
793
+ if provider == "-":
794
+ pool = [
795
+ m
796
+ for m in models
797
+ if m.model_id not in covered
798
+ and not _in_tier_slug_list(m.model_id, selected_tier_slugs)
799
+ ]
800
+ free_ids = sources.members(models, None)[1]
801
+ free_ids &= {m.model_id for m in pool}
802
+ else:
803
+ tier_slugs[provider] = sources.load_source(provider)
804
+ pool, free_ids = sources.members(models, tier_slugs[provider])
805
+ if on_offer is not None:
806
+ pool = [m for m in pool if offered(m.model_id, on_offer)]
807
+ free_ids &= {m.model_id for m in pool}
808
+ counts[provider] = len(pool)
809
+ covered |= {m.model_id for m in pool}
810
+ listed += pool
811
+ groups.append(
812
+ build_rows(
813
+ pool,
814
+ _entries_for(provider, data_dir),
815
+ free_ids,
816
+ axis=price,
817
+ provider=provider,
818
+ table=table,
819
+ free_rates=free_rates,
820
+ )
821
+ )
822
+ rows = dedupe_providers(merge_rows(*groups))
823
+ return rows, listed, curated, counts, tier_slugs, models
824
+
825
+
826
+ def _cmd_list(args: argparse.Namespace) -> int:
827
+ data_dir = Path(args.data)
828
+ if not data_dir.is_dir():
829
+ return _fail(f"no data directory at {data_dir} — run `fetch` first")
830
+
831
+ source = args.source
832
+ chosen = _select(data_dir, source)
833
+ if chosen is None:
834
+ return _fail(
835
+ f"unknown source {source!r} — available: "
836
+ f"{', '.join(sources.available_sources())}, all"
837
+ )
838
+ selected, scope = chosen
839
+
840
+ rows, listed, curated, counts, tier_slugs, models = _rows_for(
841
+ data_dir, selected, args.price, getattr(args, "all_models", False)
842
+ )
843
+ if not listed:
844
+ return _fail(f"no model of {source or 'zen+go'} is in {data_dir} — run `fetch`")
845
+ # one line per (model, variant, score, cost) before the culls see them
846
+ if args.benchmark:
847
+ wanted = args.benchmark
848
+ if wanted not in dict(available(listed)):
849
+ listing = ", ".join(f"{n} ({c})" for n, c in list(available(listed))[:10])
850
+ return _fail(f"unknown benchmark {wanted!r} — available: {listing}")
851
+ rows = [
852
+ replace(r, score=r.model.scores[wanted], benchmark=wanted)
853
+ for r in rows
854
+ if wanted in r.model.scores
855
+ ]
856
+ rows.sort(key=lambda r: (-ratio(r.score, r.price), -r.score))
857
+ order_note = f"benchmark {wanted} · sorted by score per $"
858
+ else:
859
+ rows = order_rows(rows)
860
+ order_note = "ranked by score per $, best value first"
861
+ # the marker reads the score itself, so a row in the cost or
862
+ # worst-score table shows the same grade it earned in best value
863
+ rows = grade_rows(rows)
864
+ # the layout: every axis side by side, or the single ranked benchmark
865
+ axis = not args.benchmark
866
+ if not rows:
867
+ return _fail(
868
+ f"no {source or 'zen+go'} model in {data_dir} has benchmark scores"
869
+ )
870
+
871
+ if scope == "tiers":
872
+ head = f"zen+go: {counts.get('zen', 0)} zen + {counts.get('go', 0)} go models"
873
+ elif scope == "tier":
874
+ head = f"{source}: {counts[source]}/{len(tier_slugs[source])} models in catalog"
875
+ else:
876
+ head = f"all: {len(models)} models in catalog"
877
+ print(f"{head} · {len(rows)} lines · {order_note} · {COST_NOTE}")
878
+ if not getattr(args, "all_models", False):
879
+ plans = "Go/Zen plans" if source in (None, "all") else f"{source} plan"
880
+ print(
881
+ f"Only models available right now on the {plans} — "
882
+ f"full list with --all-models."
883
+ )
884
+ print("GRADE: ++ ≥90 · + ≥75 · o 60–75 · - 50–60 · -- <50 · · unranked")
885
+
886
+ # the split always happens — the cull is what keeps the worst-scoring
887
+ # and priciest models out of the view — only the reporting is optional
888
+ main, worst, pricey = partition_rows(rows)
889
+ # one set of column widths for the whole view, so the three tables line
890
+ # up with each other too
891
+ widths = _widths(rows, axis)
892
+
893
+ _section(f"best value · {len(main)} lines", widths, axis)
894
+ for band, label in BAND_LABELS:
895
+ group = [r for r in main if r.band == band]
896
+ if group:
897
+ print(f" {label} · {len(group)}")
898
+ for row in group:
899
+ _print_row(row, widths, curated.curated, axis)
900
+
901
+ if args.tables == "all":
902
+ _section(f"worst score (bottom 30%) · {len(worst)} lines", widths, axis)
903
+ for row in worst:
904
+ _print_row(row, widths, curated.curated, axis)
905
+
906
+ _section(f"highest cost (top 30%) · {len(pricey)} lines", widths, axis)
907
+ for row in pricey:
908
+ _print_row(row, widths, curated.curated, axis)
909
+ return 0
910
+
911
+
912
+ #: How each band is named in the sequence. The rules behind these words —
913
+ #: what an allowance costs, when a balance is drawn down, what "free" does
914
+ #: not mean — are written down once in LIMITS.md rather than restated on
915
+ #: every run.
916
+ NEXT_BANDS = {
917
+ "free": "free",
918
+ "included": "included",
919
+ "payg": "payg",
920
+ "catalog": "catalog",
921
+ }
922
+
923
+
924
+ def _pick_value(row) -> str:
925
+ """A pick's score per dollar, free lines included.
926
+
927
+ Free lines carry their nominal rate, so they divide like any other
928
+ cost — the cheapest free models read highest. "free" is only the
929
+ degenerate zero-price fallback; "-" means there is no score to value.
930
+ """
931
+ if row.score is None:
932
+ return "-"
933
+ if row.price == 0:
934
+ return "free"
935
+ return f"{round(ratio(row.score, row.price))}"
936
+
937
+
938
+ def _endpoint(row) -> str:
939
+ """A pick as an API identifier: provider prefix plus the plan-local id.
940
+
941
+ Display names are not callable, so the view shows what goes in the
942
+ request instead: ``opencode-go/step-5-preview-free``. The prefix follows
943
+ the plans on the line, preferring Go where both list it; the id is the
944
+ tier's own entry where documented, else the catalog slug. Variants and
945
+ free tags are display concerns and stay out of it.
946
+ """
947
+ plans = plans_of(row)
948
+ prefix = "opencode-go" if "go" in plans else "opencode"
949
+ if row.entry is not None and row.entry.model_id:
950
+ return f"{prefix}/{row.entry.model_id}"
951
+ return f"{prefix}/{row.model.model_id.split('/')[-1]}"
952
+
953
+
954
+ def _next_band(row) -> str:
955
+ # a row sold by both tiers is labelled "go,zen", so membership decides,
956
+ # not equality with a single plan name
957
+ if not {"zen", "go"} & plans_of(row):
958
+ return NEXT_BANDS["catalog"]
959
+ return NEXT_BANDS[
960
+ "included" if row.band == "included" else ("free" if row.free else "payg")
961
+ ]
962
+
963
+
964
+ #: The `next` grid, declared: one text column per field, measured at
965
+ #: print time so a wide provider (`go,zen`) or a long score widens its
966
+ #: column instead of shifting the rest of the row.
967
+ NEXT_HEADERS = ("#", "PLAN", "SCORE", "¢/1M", "SCORE/$", "ENDPOINT", "BAND")
968
+ NEXT_ALIGNS = ("<", "<", ">", ">", ">", "<", "<")
969
+ NEXT_MIN_WIDTHS = (2, 5, 6, 5, 7, 0, 0)
970
+
971
+
972
+ def _next_table(picks: list) -> str:
973
+ """The picks as an aligned grid: position, plan, score, cost, value,
974
+ endpoint, band."""
975
+ rows = []
976
+ for i, row in enumerate(picks, 1):
977
+ _score, price, value, _prov = _cells(row)
978
+ rows.append(
979
+ [
980
+ str(i),
981
+ row.provider,
982
+ f"{round(row.score)}",
983
+ price,
984
+ value,
985
+ _endpoint(row),
986
+ _next_band(row),
987
+ ]
988
+ )
989
+ return render(
990
+ NEXT_HEADERS,
991
+ rows,
992
+ aligns=NEXT_ALIGNS,
993
+ min_widths=NEXT_MIN_WIDTHS,
994
+ indent=" ",
995
+ gap=" ",
996
+ )
997
+
998
+
999
+ def _cmd_next(args: argparse.Namespace) -> int:
1000
+ """A short ordered sequence: free and paid picks interleaved band by band."""
1001
+ data_dir = Path(args.data)
1002
+ if not data_dir.is_dir():
1003
+ return _fail(f"no data directory at {data_dir} — run `fetch` first")
1004
+ chosen = _select(data_dir, args.source)
1005
+ if chosen is None:
1006
+ return _fail(
1007
+ f"unknown source {args.source!r} — available: "
1008
+ f"{', '.join(sources.available_sources())}, all"
1009
+ )
1010
+ rows, listed, _curated, _counts, _slugs, _models = _rows_for(
1011
+ data_dir, chosen[0], args.price, getattr(args, "all_models", False)
1012
+ )
1013
+ if not listed:
1014
+ return _fail(
1015
+ f"no model of {args.source or 'zen+go'} is in {data_dir} "
1016
+ f"— run `fetch` first"
1017
+ )
1018
+ picks = next_up(grade_rows(order_rows(rows)), count=args.count)
1019
+ if not picks:
1020
+ return _fail(
1021
+ f"no model of {args.source or 'zen+go'} scores on "
1022
+ f"{MIN_AXES}+ benchmarks in {data_dir} — nothing to "
1023
+ f"recommend yet"
1024
+ )
1025
+
1026
+ print(
1027
+ f"Next {len(picks)} recommended models, "
1028
+ f">{RECOMMEND_FLOOR:.0%} overall score, free and paid picks "
1029
+ f"interleaved, alternating Zen/Go plans."
1030
+ )
1031
+ print()
1032
+ print(_next_table(picks))
1033
+ return 0
1034
+
1035
+
1036
+ def main(argv: list[str] | None = None) -> int:
1037
+ parser = argparse.ArgumentParser(
1038
+ prog="opencode-models",
1039
+ description="Price/score Pareto frontier of models from opencode.ai/data",
1040
+ )
1041
+ parser.add_argument(
1042
+ "--data",
1043
+ default=str(DEFAULT_DATA),
1044
+ help="data directory (default: %(default)s)",
1045
+ )
1046
+ parser.add_argument(
1047
+ "--version", action="version", version=f"%(prog)s {__version__}"
1048
+ )
1049
+ sub = parser.add_subparsers(dest="command", required=True)
1050
+
1051
+ sub.add_parser("fetch", help="refresh data from opencode.ai").add_argument(
1052
+ "--all",
1053
+ dest="all_models",
1054
+ action="store_true",
1055
+ help="download the full model catalog, not just the models the "
1056
+ "plans offer (kept records are never deleted either way)",
1057
+ )
1058
+ sub.add_parser(
1059
+ "models",
1060
+ help="the models the plans offer right now, "
1061
+ "as endpoints beside display names: live "
1062
+ "from the Console config API with "
1063
+ "OPENCODE_API_KEY, else the last fetch "
1064
+ "snapshots",
1065
+ )
1066
+ sub.add_parser(
1067
+ "auth",
1068
+ help="save the Console service-account key "
1069
+ "to an owner-only file (prompted, never "
1070
+ "echoed); verifies it live when it can",
1071
+ )
1072
+ sub.add_parser(
1073
+ "overrides",
1074
+ help="every benchmark number more than one source "
1075
+ "reports, and which value the console keeps",
1076
+ )
1077
+
1078
+ p_list = sub.add_parser(
1079
+ "list",
1080
+ help="three tables of Console models across providers "
1081
+ "(zen/go): best value (free/included/pay-as-you-go), "
1082
+ "worst score, highest cost",
1083
+ )
1084
+ p_list.add_argument(
1085
+ "--source",
1086
+ help="provider to list: zen or go (default: both, "
1087
+ "merged) — 'all' adds the catalog models that "
1088
+ "are in neither tier",
1089
+ )
1090
+ p_list.add_argument(
1091
+ "--price",
1092
+ choices=("weighted", "input", "output"),
1093
+ default="weighted",
1094
+ help="cost of a line (default: weighted 20%% in, 60%% cached, 20%% out)",
1095
+ )
1096
+ p_list.add_argument(
1097
+ "--tables",
1098
+ choices=("best", "all"),
1099
+ default="best",
1100
+ help="best: print only the best-value table. all: "
1101
+ "also print the worst-score and highest-cost "
1102
+ "tables. Either way the cull happens — those "
1103
+ "rows are out of best value (default: %(default)s)",
1104
+ )
1105
+ p_list.add_argument(
1106
+ "--all-models",
1107
+ "--all",
1108
+ dest="all_models",
1109
+ action="store_true",
1110
+ help="include models the plans no longer offer; the "
1111
+ "default shows only what a plan currently lists",
1112
+ )
1113
+ p_list.add_argument(
1114
+ "--benchmark",
1115
+ help="rank on one benchmark instead of combining "
1116
+ "Terminal-Bench 2.x (first) with SWE-Bench and "
1117
+ "Terminal-Bench 4.0",
1118
+ )
1119
+
1120
+ p_next = sub.add_parser(
1121
+ "next",
1122
+ help="the next few models to use, in order: free and paid "
1123
+ "picks interleaved, best score then best score per dollar",
1124
+ )
1125
+ p_next.add_argument(
1126
+ "--count",
1127
+ type=int,
1128
+ default=6,
1129
+ help="how many to recommend (default: %(default)s)",
1130
+ )
1131
+ p_next.add_argument(
1132
+ "--source", help="provider to draw from: zen or go (default: both, merged)"
1133
+ )
1134
+ p_next.add_argument(
1135
+ "--price",
1136
+ choices=("weighted", "input", "output"),
1137
+ default="weighted",
1138
+ help="cost of a line (default: weighted 20%% in, 60%% cached, 20%% out)",
1139
+ )
1140
+ p_next.add_argument(
1141
+ "--all-models",
1142
+ "--all",
1143
+ dest="all_models",
1144
+ action="store_true",
1145
+ help="include models the plans no longer offer; the "
1146
+ "default shows only what a plan currently lists",
1147
+ )
1148
+
1149
+ p_scores = sub.add_parser("scores", help="manage cached arena.ai Elo scores")
1150
+ p_scores.add_argument(
1151
+ "--all-models",
1152
+ "--all",
1153
+ dest="all_models",
1154
+ action="store_true",
1155
+ help="include models the plans no longer offer; the "
1156
+ "default shows only what a plan currently lists",
1157
+ )
1158
+
1159
+ args = parser.parse_args(argv)
1160
+ return {
1161
+ "fetch": _cmd_fetch,
1162
+ "models": _cmd_models,
1163
+ "auth": _cmd_auth,
1164
+ "overrides": _cmd_conflicts,
1165
+ "list": _cmd_list,
1166
+ "next": _cmd_next,
1167
+ "scores": _cmd_scores,
1168
+ }[args.command](args)
1169
+
1170
+
1171
+ if __name__ == "__main__":
1172
+ sys.exit(main())