opencode-models 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencode_models/__init__.py +8 -0
- opencode_models/__main__.py +5 -0
- opencode_models/arena.py +108 -0
- opencode_models/auth.py +48 -0
- opencode_models/availability.py +102 -0
- opencode_models/benchmarks.py +19 -0
- opencode_models/boards.py +518 -0
- opencode_models/cli.py +1172 -0
- opencode_models/data.py +82 -0
- opencode_models/fetch.py +101 -0
- opencode_models/identity.py +100 -0
- opencode_models/labelmap.py +84 -0
- opencode_models/live.py +88 -0
- opencode_models/overrides.py +252 -0
- opencode_models/pareto.py +34 -0
- opencode_models/pricing.py +415 -0
- opencode_models/ranking.py +591 -0
- opencode_models/scoring.py +230 -0
- opencode_models/seed.py +184 -0
- opencode_models/sources/go.txt +45 -0
- opencode_models/sources/zen.txt +87 -0
- opencode_models/sources.py +55 -0
- opencode_models/tables.py +94 -0
- opencode_models-0.1.0.dist-info/METADATA +544 -0
- opencode_models-0.1.0.dist-info/RECORD +28 -0
- opencode_models-0.1.0.dist-info/WHEEL +4 -0
- opencode_models-0.1.0.dist-info/entry_points.txt +2 -0
- opencode_models-0.1.0.dist-info/licenses/LICENSE +21 -0
opencode_models/cli.py
ADDED
|
@@ -0,0 +1,1172 @@
|
|
|
1
|
+
"""Command-line interface.
|
|
2
|
+
|
|
3
|
+
Usage::
|
|
4
|
+
|
|
5
|
+
python3 -m opencode_models fetch # refresh data/ from opencode.ai
|
|
6
|
+
|
|
7
|
+
Exit codes: 0 success, 1 usage or data error (message on stdout).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
import urllib.request
|
|
16
|
+
from dataclasses import replace
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from . import __version__, sources
|
|
20
|
+
from . import arena as arena_mod
|
|
21
|
+
from .arena import ArenaCache, merge_from_file
|
|
22
|
+
from .auth import load_key, save_key
|
|
23
|
+
from .availability import available_ids, load_available, offered
|
|
24
|
+
from .benchmarks import available
|
|
25
|
+
from .boards import fetch_boards, load_boards, merge_boards, save_boards
|
|
26
|
+
from .data import load_models
|
|
27
|
+
from .fetch import http_get, refresh
|
|
28
|
+
from .live import LIVE_CONFIG_URL, fetch_live, save_live
|
|
29
|
+
from .overrides import load_curated, off_band_models, source_tier
|
|
30
|
+
from .pricing import (
|
|
31
|
+
FREE_TIER_NOMINAL_RATES,
|
|
32
|
+
FREE_TIER_SNAPSHOT_NAME,
|
|
33
|
+
load_free_nominals,
|
|
34
|
+
load_prices,
|
|
35
|
+
refresh_free_nominals,
|
|
36
|
+
refresh_prices,
|
|
37
|
+
)
|
|
38
|
+
from .ranking import (
|
|
39
|
+
BAND_FREE,
|
|
40
|
+
BAND_INCLUDED,
|
|
41
|
+
BAND_PAYG,
|
|
42
|
+
RECOMMEND_FLOOR,
|
|
43
|
+
build_rows,
|
|
44
|
+
dedupe_providers,
|
|
45
|
+
grade_for,
|
|
46
|
+
grade_rows,
|
|
47
|
+
merge_rows,
|
|
48
|
+
next_up,
|
|
49
|
+
order_rows,
|
|
50
|
+
partition_rows,
|
|
51
|
+
plans_of,
|
|
52
|
+
ratio,
|
|
53
|
+
)
|
|
54
|
+
from .scoring import (
|
|
55
|
+
_TRACKED,
|
|
56
|
+
MIN_AXES,
|
|
57
|
+
_folded,
|
|
58
|
+
canonical,
|
|
59
|
+
combined,
|
|
60
|
+
derived_scores,
|
|
61
|
+
scales,
|
|
62
|
+
)
|
|
63
|
+
from .tables import column_widths, format_row, render
|
|
64
|
+
|
|
65
|
+
#: Where the data lives for an installed tool: $OPENCODE_MODELS_DATA, else the
|
|
66
|
+
#: XDG user-data dir (~/.local/share/opencode-models). `fetch` populates it and
|
|
67
|
+
#: every other command reads it; a repo checkout passes --data ./data.
|
|
68
|
+
DATA_ENV = "OPENCODE_MODELS_DATA"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def default_data_dir() -> Path:
|
|
72
|
+
"""The user data directory: $OPENCODE_MODELS_DATA, else XDG."""
|
|
73
|
+
env = os.environ.get(DATA_ENV)
|
|
74
|
+
if env:
|
|
75
|
+
return Path(env)
|
|
76
|
+
xdg = os.environ.get("XDG_DATA_HOME")
|
|
77
|
+
base = Path(xdg) if xdg else Path.home() / ".local" / "share"
|
|
78
|
+
return base / "opencode-models"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
DEFAULT_DATA = default_data_dir()
|
|
82
|
+
|
|
83
|
+
#: Representative cost: 20% input, 60% cached read, 20% output.
|
|
84
|
+
COST_NOTE = "cost 20% in / 60% cached / 20% out"
|
|
85
|
+
BAND_LABELS = (
|
|
86
|
+
(BAND_FREE, "free — use first"),
|
|
87
|
+
(BAND_INCLUDED, "included usage"),
|
|
88
|
+
(BAND_PAYG, "pay-as-you-go"),
|
|
89
|
+
)
|
|
90
|
+
#: Fixed columns, right-aligned; the model name takes the rest of the line.
|
|
91
|
+
#: Costs count cents ($/100) so a cheap line is a readable number.
|
|
92
|
+
#:
|
|
93
|
+
#: Two layouts, both built around the one score: the default shows the
|
|
94
|
+
#: model's score beside its cost and plan, and ``--benchmark X`` ranks on
|
|
95
|
+
#: one benchmark and adds that benchmark's tag. The per-axis raw numbers
|
|
96
|
+
#: live in `scores`, not here.
|
|
97
|
+
AXIS_HEADERS = ("SCORE", "¢/1M", "SCORE/$", "PROV")
|
|
98
|
+
SINGLE_HEADERS = ("SCORE", "BENCH", "¢/1M", "SCORE/$", "PROV")
|
|
99
|
+
|
|
100
|
+
#: Every row is prefixed by its rank marker, outside the column grid and
|
|
101
|
+
#: without a header of its own: ++ at 90+, + at 75+, o at 60+, - at 50+,
|
|
102
|
+
#: -- below that, · unranked (no score at all). The header line is indented
|
|
103
|
+
#: by the same width so the columns still line up.
|
|
104
|
+
GRADE_WIDTH = 5
|
|
105
|
+
RULE_WIDTH = 60
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _style(text: str) -> str:
|
|
109
|
+
"""Bold when talking to a terminal, plain when piped or captured."""
|
|
110
|
+
return f"\033[1m{text}\033[0m" if sys.stdout.isatty() else text
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _row_line(cells: tuple[str, ...], widths: tuple[int, ...]) -> str:
|
|
114
|
+
return format_row(cells, widths)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _header(widths: tuple[int, ...], axis: bool = True) -> str:
|
|
118
|
+
headings = AXIS_HEADERS if axis else SINGLE_HEADERS
|
|
119
|
+
return _style(format_row(headings, widths) + " MODEL")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _section(title: str, widths: tuple[int, ...], axis: bool = True) -> None:
|
|
123
|
+
"""Open a table: blank line, rule-wrapped title, highlighted header."""
|
|
124
|
+
print()
|
|
125
|
+
print(f"── {title} " + "─" * max(4, RULE_WIDTH - len(title)))
|
|
126
|
+
print(" " * GRADE_WIDTH + _header(widths, axis))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _fail(msg: str) -> int:
|
|
130
|
+
print(msg)
|
|
131
|
+
return 1
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _load(data_dir: Path):
|
|
135
|
+
"""Upstream records, arena scores, curated overlays and official boards.
|
|
136
|
+
|
|
137
|
+
Returns the models, the curated layer (which models carry overrides), and
|
|
138
|
+
the board-vs-registry disagreements — so the console can mark curated rows
|
|
139
|
+
and `overrides` can report the boards' numbers replacing self-reports.
|
|
140
|
+
"""
|
|
141
|
+
curated = load_curated(data_dir / "opencode", data_dir / "overrides")
|
|
142
|
+
models = merge_from_file(curated.models, data_dir / "aiarena")
|
|
143
|
+
models, board_conflicts = merge_boards(models, load_boards(data_dir / "boards"))
|
|
144
|
+
return models, curated, board_conflicts
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
#: How much of a board row's provenance the NOTE column carries.
|
|
148
|
+
_NOTE_WIDTH = 52
|
|
149
|
+
|
|
150
|
+
#: Source tiers, short enough to sit beside a number without dominating the
|
|
151
|
+
#: column. The long names are in the footer rather than in every row.
|
|
152
|
+
_TIER_SHORT = {1: "indep", 2: "board", 3: "self"}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _tier_word(source) -> str:
|
|
156
|
+
return _TIER_SHORT[source_tier(source)]
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _note_text(c) -> str:
|
|
160
|
+
"""A conflict's provenance, trimmed to one line.
|
|
161
|
+
|
|
162
|
+
The board rows carry the run that produced the number - agent, effort,
|
|
163
|
+
rank, CI and trial count - and that is the part worth reading, so it
|
|
164
|
+
goes in the table rather than on a second line under it. A longer note
|
|
165
|
+
is cut: the full record is in ``data/boards/``.
|
|
166
|
+
"""
|
|
167
|
+
text = " ".join((c.reason or "").split())
|
|
168
|
+
return text if len(text) <= _NOTE_WIDTH else text[: _NOTE_WIDTH - 1] + "…"
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _print_conflicts(conflicts, orphans=()) -> int:
|
|
172
|
+
"""Disagreements between two sources, as a table with the value used.
|
|
173
|
+
|
|
174
|
+
Three numbers per row would be one too many, so the two reported values
|
|
175
|
+
carry their source tier and USED says which of them is in the view.
|
|
176
|
+
The arrow used to run upstream → override, which read as if the console
|
|
177
|
+
had taken the second number even where it kept the first.
|
|
178
|
+
"""
|
|
179
|
+
rows = []
|
|
180
|
+
for c in conflicts:
|
|
181
|
+
rows.append(
|
|
182
|
+
(
|
|
183
|
+
c.model_id,
|
|
184
|
+
c.benchmark,
|
|
185
|
+
f"{c.upstream:g} {_tier_word(c.upstream_source)}",
|
|
186
|
+
f"{c.override:g} {_tier_word(c.override_source)}",
|
|
187
|
+
f"{c.used:g}",
|
|
188
|
+
_note_text(c),
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
orphans = [
|
|
192
|
+
(model_id, "", "", "", "", "override file has no upstream record")
|
|
193
|
+
for model_id in sorted(orphans)
|
|
194
|
+
]
|
|
195
|
+
rows += orphans
|
|
196
|
+
if not rows:
|
|
197
|
+
return 0
|
|
198
|
+
headings = ("MODEL", "BENCHMARK", "CATALOG", "REPORTED", "USED", "NOTE")
|
|
199
|
+
print(render(headings, rows, aligns="<", indent=" "))
|
|
200
|
+
return len(conflicts) + len(orphans)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _cmd_conflicts(args: argparse.Namespace) -> int:
|
|
204
|
+
"""Curated overrides that disagree with upstream, ranked by provenance.
|
|
205
|
+
|
|
206
|
+
Each disagreement shows both values with the tier of their source, and
|
|
207
|
+
whether the override applies (its source is at least as good as
|
|
208
|
+
upstream's) or is merely recorded. To accept upstream, delete the
|
|
209
|
+
entry in data/overrides/.
|
|
210
|
+
"""
|
|
211
|
+
data_dir = Path(args.data)
|
|
212
|
+
if not data_dir.is_dir():
|
|
213
|
+
return _fail(f"no data directory at {data_dir} — run `fetch` first")
|
|
214
|
+
models, curated, board_conflicts = _load(data_dir)
|
|
215
|
+
n = _print_conflicts(curated.conflicts, curated.orphans)
|
|
216
|
+
if board_conflicts:
|
|
217
|
+
print("— official boards —")
|
|
218
|
+
_print_conflicts(board_conflicts)
|
|
219
|
+
for mid, s21, s40, resid in off_band_models(curated.models):
|
|
220
|
+
print(
|
|
221
|
+
f"{mid} Terminal-Bench 2.1 {s21} vs 4.0 {s40} — "
|
|
222
|
+
f"{resid:+.0f} off the 2.1→4.0 band, check the run"
|
|
223
|
+
)
|
|
224
|
+
print(
|
|
225
|
+
f"{len(curated.curated)} curated models · {n} curated "
|
|
226
|
+
f"disagreements · {len(board_conflicts)} from official boards"
|
|
227
|
+
)
|
|
228
|
+
print(
|
|
229
|
+
"tiers: self = vendor self-report · board = official board · "
|
|
230
|
+
"indep = independent evaluator"
|
|
231
|
+
)
|
|
232
|
+
return 0
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
#: How often the long model-file walk reports: one line per file would bury
|
|
236
|
+
#: the per-source announcements, so every fiftieth file plus the last one.
|
|
237
|
+
_FETCH_TICK = 50
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _model_tick(done: int, total: int, name: str) -> None:
|
|
241
|
+
if done % _FETCH_TICK == 0 or done == total:
|
|
242
|
+
print(f" opencode.ai/data: {done}/{total} model files ...")
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _api_get(key: str):
|
|
246
|
+
"""Authed transport for the key-gated Console API (30s per request)."""
|
|
247
|
+
|
|
248
|
+
def get(url: str) -> str:
|
|
249
|
+
req = urllib.request.Request(
|
|
250
|
+
url,
|
|
251
|
+
headers={
|
|
252
|
+
"Authorization": f"Bearer {key}",
|
|
253
|
+
"User-Agent": f"opencode-models/{__version__}",
|
|
254
|
+
},
|
|
255
|
+
)
|
|
256
|
+
try:
|
|
257
|
+
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
258
|
+
return resp.read().decode("utf-8", errors="replace")
|
|
259
|
+
except urllib.error.HTTPError as exc:
|
|
260
|
+
exc.close() # see http_get: callers never read error bodies
|
|
261
|
+
raise
|
|
262
|
+
|
|
263
|
+
return get
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _cmd_auth(args: argparse.Namespace) -> int:
|
|
267
|
+
"""Ask for the Console service-account key and save it owner-only.
|
|
268
|
+
|
|
269
|
+
The prompt never echoes, the key is never printed back, and the file
|
|
270
|
+
is created (or tightened) to owner-only access. The saved key is then
|
|
271
|
+
verified against the live config API — a rejection fails the command
|
|
272
|
+
but keeps the file, so an offline first run is not punished.
|
|
273
|
+
"""
|
|
274
|
+
import getpass
|
|
275
|
+
|
|
276
|
+
if not sys.stdin.isatty():
|
|
277
|
+
return _fail(
|
|
278
|
+
"auth needs an interactive terminal — pipe a key via "
|
|
279
|
+
"OPENCODE_API_KEY instead"
|
|
280
|
+
)
|
|
281
|
+
key = getpass.getpass("OPENCODE_API_KEY: ").strip()
|
|
282
|
+
if not key:
|
|
283
|
+
return _fail("no key entered — nothing saved")
|
|
284
|
+
path = save_key(key)
|
|
285
|
+
print(f"saved to {path} (owner-only)")
|
|
286
|
+
try:
|
|
287
|
+
live = fetch_live(_api_get(key))
|
|
288
|
+
except (OSError, ValueError, KeyError) as exc:
|
|
289
|
+
print(f"warning: key saved but not verified ({exc})")
|
|
290
|
+
return 0
|
|
291
|
+
count = len(set().union(*live.values()))
|
|
292
|
+
print(
|
|
293
|
+
f"verified live: {count} models across "
|
|
294
|
+
f"{len([t for t in live.values() if t])} tiers"
|
|
295
|
+
)
|
|
296
|
+
return 0
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _display_names(data_dir: Path) -> dict[str, str]:
|
|
300
|
+
"""Display names by slug or id, from the catalog records on disk.
|
|
301
|
+
|
|
302
|
+
Absent catalog, absent names: callers fall back to the slug itself.
|
|
303
|
+
"""
|
|
304
|
+
names = {}
|
|
305
|
+
for model in load_models(Path(data_dir) / "opencode"):
|
|
306
|
+
names[model.model_id] = model.name
|
|
307
|
+
names[model.model_id.split("/")[-1]] = model.name
|
|
308
|
+
return names
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _model_name(names: dict[str, str], slug: str) -> str:
|
|
312
|
+
"""The catalog's display name for a tier slug, or the slug itself.
|
|
313
|
+
|
|
314
|
+
Tier pages list free variants under their own slug
|
|
315
|
+
(``step-5-preview-free``) while the catalog carries the base model,
|
|
316
|
+
so the ``-free`` suffix reaches back to it.
|
|
317
|
+
"""
|
|
318
|
+
if slug in names:
|
|
319
|
+
return names[slug]
|
|
320
|
+
if slug.endswith("-free") and slug[: -len("-free")] in names:
|
|
321
|
+
return names[slug[: -len("-free")]]
|
|
322
|
+
return slug
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _cmd_models(args: argparse.Namespace) -> int:
|
|
326
|
+
"""The models the plans offer right now, per tier.
|
|
327
|
+
|
|
328
|
+
Live from the Console config API when ``OPENCODE_API_KEY`` is set —
|
|
329
|
+
recorded to ``available/`` on the way, so `list` and `next` unlist
|
|
330
|
+
the dropped models immediately. Else the snapshots the last ``fetch``
|
|
331
|
+
left behind, read-only.
|
|
332
|
+
"""
|
|
333
|
+
data_dir = Path(args.data)
|
|
334
|
+
key = load_key()
|
|
335
|
+
tiers: dict[str, set[str]] | None = None
|
|
336
|
+
if key:
|
|
337
|
+
try:
|
|
338
|
+
tiers = fetch_live(_api_get(key))
|
|
339
|
+
except (OSError, ValueError, KeyError) as exc:
|
|
340
|
+
print(
|
|
341
|
+
f"warning: live config unreachable ({exc}) — showing the last snapshots"
|
|
342
|
+
)
|
|
343
|
+
tiers = None
|
|
344
|
+
else:
|
|
345
|
+
save_live(data_dir, tiers)
|
|
346
|
+
origin = (
|
|
347
|
+
f"live from {LIVE_CONFIG_URL} (recorded to {data_dir / 'available'})"
|
|
348
|
+
)
|
|
349
|
+
if tiers is None:
|
|
350
|
+
known = load_available(data_dir)
|
|
351
|
+
if all(ids is None for ids in known.values()):
|
|
352
|
+
return _fail(
|
|
353
|
+
f"no availability known in {data_dir} — run `fetch` "
|
|
354
|
+
f"first (with OPENCODE_API_KEY for the live set)"
|
|
355
|
+
)
|
|
356
|
+
tiers = {t: ids for t, ids in known.items() if ids is not None}
|
|
357
|
+
origin = f"snapshots in {data_dir} (run `fetch` to refresh)"
|
|
358
|
+
unknown = sorted(t for t, ids in known.items() if ids is None)
|
|
359
|
+
else:
|
|
360
|
+
unknown = []
|
|
361
|
+
names = _display_names(data_dir)
|
|
362
|
+
by_slug: dict[str, str] = {}
|
|
363
|
+
for tier in sorted(tiers):
|
|
364
|
+
for slug in sorted(tiers[tier]):
|
|
365
|
+
if slug not in by_slug or tier == "go":
|
|
366
|
+
by_slug[slug] = tier
|
|
367
|
+
rows = []
|
|
368
|
+
for slug, tier in by_slug.items():
|
|
369
|
+
endpoint = f"opencode-go/{slug}" if tier == "go" else f"opencode/{slug}"
|
|
370
|
+
name = _model_name(names, slug)
|
|
371
|
+
# the endpoint already carries the slug: a missing display name
|
|
372
|
+
# reads "—" (as a missing score does), not the slug twice
|
|
373
|
+
rows.append((endpoint, name if name != slug else "—"))
|
|
374
|
+
rows.sort(key=lambda row: (row[1].lower(), row[0]))
|
|
375
|
+
counts = " + ".join(f"{len(tiers[t])} {t}" for t in sorted(tiers))
|
|
376
|
+
print(origin)
|
|
377
|
+
print(
|
|
378
|
+
f"{len(by_slug)} models · {counts}"
|
|
379
|
+
+ (f" · {', '.join(unknown)} unknown" if unknown else "")
|
|
380
|
+
)
|
|
381
|
+
print(render(["ENDPOINT", "MODEL"], rows, aligns=("<", "<"), indent=" "))
|
|
382
|
+
return 0
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _cmd_fetch(args: argparse.Namespace) -> int:
|
|
386
|
+
data_dir = Path(args.data)
|
|
387
|
+
key = load_key()
|
|
388
|
+
only: set[str] | None = None
|
|
389
|
+
if args.all_models:
|
|
390
|
+
print("fetching the full catalog (--all)")
|
|
391
|
+
elif key:
|
|
392
|
+
print(f"fetch live model set from {LIVE_CONFIG_URL} ...")
|
|
393
|
+
try:
|
|
394
|
+
live = fetch_live(_api_get(key))
|
|
395
|
+
except (OSError, ValueError, KeyError) as exc:
|
|
396
|
+
print(
|
|
397
|
+
f"warning: live config unreachable ({exc}) — "
|
|
398
|
+
f"downloading the full catalog"
|
|
399
|
+
)
|
|
400
|
+
else:
|
|
401
|
+
saved_live = save_live(data_dir, live)
|
|
402
|
+
print(f"saved {', '.join(saved_live)} to {data_dir / 'available'}")
|
|
403
|
+
only = set().union(*live.values()) or set()
|
|
404
|
+
if only is None and not args.all_models:
|
|
405
|
+
only = available_ids(data_dir)
|
|
406
|
+
if only is not None:
|
|
407
|
+
print(
|
|
408
|
+
f"{len(only)} models offered — downloading those "
|
|
409
|
+
f"(--all for the full catalog)"
|
|
410
|
+
)
|
|
411
|
+
print("fetch model records from opencode.ai/data ...")
|
|
412
|
+
saved = refresh(data_dir, get=http_get, progress=_model_tick, only=only)
|
|
413
|
+
print(f"saved {len(saved)} model files to {data_dir / 'opencode'}")
|
|
414
|
+
print("fetch tier pricing from opencode.ai/docs ...")
|
|
415
|
+
try:
|
|
416
|
+
prices = refresh_prices(data_dir / "tier_prices", get=http_get)
|
|
417
|
+
except (OSError, ValueError) as exc: # docs down must not fail the fetch
|
|
418
|
+
print(f"warning: tier pricing docs not refreshed ({exc})")
|
|
419
|
+
else:
|
|
420
|
+
if prices:
|
|
421
|
+
print(f"saved {', '.join(prices)} to {data_dir / 'tier_prices'}")
|
|
422
|
+
else:
|
|
423
|
+
print(
|
|
424
|
+
"warning: tier pricing docs parsed to nothing, "
|
|
425
|
+
"keeping existing snapshots"
|
|
426
|
+
)
|
|
427
|
+
print("fetch arena Elo from datasets-server.huggingface.co ...")
|
|
428
|
+
try:
|
|
429
|
+
elo = arena_mod.lookup_arena_elo(get=http_get)
|
|
430
|
+
except (OSError, ValueError) as exc: # one dead source must not stop the rest
|
|
431
|
+
print(f"warning: arena Elo not refreshed ({exc})")
|
|
432
|
+
else:
|
|
433
|
+
cache = ArenaCache(data_dir / "aiarena")
|
|
434
|
+
cache.save(elo)
|
|
435
|
+
print(
|
|
436
|
+
f"saved {len(elo)} arena Elo scores to "
|
|
437
|
+
f"{data_dir / 'aiarena' / arena_mod.CACHE_NAME}"
|
|
438
|
+
)
|
|
439
|
+
if not key:
|
|
440
|
+
print(
|
|
441
|
+
"free-tier nominals not refreshed (no OPENCODE_API_KEY) — "
|
|
442
|
+
"keeping the existing snapshot"
|
|
443
|
+
)
|
|
444
|
+
else:
|
|
445
|
+
try:
|
|
446
|
+
free_slugs = {
|
|
447
|
+
e.model_id
|
|
448
|
+
for prices in load_prices(data_dir / "tier_prices").values()
|
|
449
|
+
for e in prices
|
|
450
|
+
if e.free and e.model_id
|
|
451
|
+
}
|
|
452
|
+
print("fetch usage export from opencode.ai/console ...")
|
|
453
|
+
nominals = refresh_free_nominals(
|
|
454
|
+
data_dir / "usage", data_dir / "opencode", _api_get(key), free_slugs
|
|
455
|
+
)
|
|
456
|
+
except (OSError, ValueError) as exc:
|
|
457
|
+
print(f"warning: free-tier nominals not refreshed ({exc})")
|
|
458
|
+
else:
|
|
459
|
+
print(
|
|
460
|
+
f"saved {len(nominals)} free-tier nominals to "
|
|
461
|
+
f"{data_dir / 'usage' / FREE_TIER_SNAPSHOT_NAME}"
|
|
462
|
+
if nominals is not None
|
|
463
|
+
else "warning: free-tier nominals not refreshed "
|
|
464
|
+
"(anchor model absent from the export)"
|
|
465
|
+
)
|
|
466
|
+
curated = load_curated(data_dir / "opencode", data_dir / "overrides")
|
|
467
|
+
n = _print_conflicts(curated.conflicts, curated.orphans)
|
|
468
|
+
if n:
|
|
469
|
+
print(
|
|
470
|
+
f"{n} curated override(s) disagree with upstream — "
|
|
471
|
+
f"run `overrides` to review"
|
|
472
|
+
)
|
|
473
|
+
boards = fetch_boards(
|
|
474
|
+
get=http_get, progress=lambda done, total, desc: print(f"fetch {desc} ...")
|
|
475
|
+
)
|
|
476
|
+
saved = save_boards(data_dir / "boards", boards)
|
|
477
|
+
rows = sum(len(v) for v in boards.values())
|
|
478
|
+
if saved:
|
|
479
|
+
print(
|
|
480
|
+
f"saved {rows} board rows from {', '.join(saved)} to {data_dir / 'boards'}"
|
|
481
|
+
)
|
|
482
|
+
return 0
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _column_label(name: str) -> str:
|
|
486
|
+
"""The short name a benchmark column carries: "Terminal Bench 2.1" → "TB 2.1".
|
|
487
|
+
|
|
488
|
+
Column headers have to fit beside each other, and there are enough
|
|
489
|
+
benchmarks to score on that the full names run past any terminal width.
|
|
490
|
+
"""
|
|
491
|
+
if name == "LMArena Elo":
|
|
492
|
+
return "ARENA"
|
|
493
|
+
if name == "DeepSWE 1.1":
|
|
494
|
+
# the version lives in the legend ("DeepSWE = DeepSWE 1.1"):
|
|
495
|
+
# exact match, so a future revision keeps its number until renamed
|
|
496
|
+
return "DeepSWE"
|
|
497
|
+
for family, short in (("Terminal Bench", "TB"), ("SWE Bench", "SWE")):
|
|
498
|
+
if name.startswith(family):
|
|
499
|
+
rest = name[len(family) :]
|
|
500
|
+
if short == "SWE":
|
|
501
|
+
# "SWE Verified" is the widest header in the scores table;
|
|
502
|
+
# "Vrfd" keeps the column parseable at a glance
|
|
503
|
+
rest = rest.replace("Verified", "Vrfd")
|
|
504
|
+
return short + rest
|
|
505
|
+
return name
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _score_columns(models, table: dict) -> list[tuple[str, str, str]]:
|
|
509
|
+
"""The benchmarks worth a column, as ``(key, header, full name)``.
|
|
510
|
+
|
|
511
|
+
One column per benchmark we score on, in the order ``scoring._TRACKED``
|
|
512
|
+
lists them — the axis the console ranks on first — and only where the
|
|
513
|
+
catalog actually has numbers. Two registries spelling one benchmark
|
|
514
|
+
differently share a column.
|
|
515
|
+
"""
|
|
516
|
+
spellings: dict[str, set[str]] = {}
|
|
517
|
+
for model in models:
|
|
518
|
+
for name in derived_scores(model, table):
|
|
519
|
+
spellings.setdefault(canonical(name), set()).add(name)
|
|
520
|
+
columns = []
|
|
521
|
+
for key in _TRACKED:
|
|
522
|
+
folded = _folded(key)
|
|
523
|
+
if folded not in spellings:
|
|
524
|
+
continue
|
|
525
|
+
# prefer the hyphenated spelling for the full name: it is how the
|
|
526
|
+
# benchmark is written everywhere else, including in the README
|
|
527
|
+
name = sorted(spellings[folded], key=lambda n: (n.count("-") == 0, n))[0]
|
|
528
|
+
columns.append((folded, _column_label(folded), name))
|
|
529
|
+
return columns
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _score_cells(model, table: dict, columns: list[tuple[str, str, str]]) -> list[str]:
|
|
533
|
+
"""One cell per column: each raw number with where it lands, best first.
|
|
534
|
+
|
|
535
|
+
The raw number is what the evaluator published; the derived number is
|
|
536
|
+
where that sits on the benchmark's 0-100 scale across this catalog.
|
|
537
|
+
Both are shown, because a score without its position in the field says
|
|
538
|
+
nothing on its own. A column the model was not measured on is blank
|
|
539
|
+
rather than zero — zero would read as "worst in the catalog".
|
|
540
|
+
"""
|
|
541
|
+
derived = derived_scores(model, table)
|
|
542
|
+
cells = []
|
|
543
|
+
for key, _label, _name in columns:
|
|
544
|
+
parts = sorted(
|
|
545
|
+
((name, derived[name]) for name in derived if canonical(name) == key),
|
|
546
|
+
key=lambda part: (-part[1], part[0]),
|
|
547
|
+
)
|
|
548
|
+
cells.append(" ".join(f"{model.scores[n]:g}→{d:.0f}" for n, d in parts))
|
|
549
|
+
return cells
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _cmd_scores(args: argparse.Namespace) -> int:
|
|
553
|
+
"""Every model's score, one column per benchmark we score on.
|
|
554
|
+
|
|
555
|
+
Each cell holds the raw number and where it lands on that benchmark's
|
|
556
|
+
0-100 scale, because the raw numbers do not compare across benchmarks.
|
|
557
|
+
How a model's score is derived from those cells is documented rather
|
|
558
|
+
than printed: see the README's "How a model's score is computed" and
|
|
559
|
+
``scoring.py``. This command only reads: `fetch` owns every network call,
|
|
560
|
+
including the arena Elo cache.
|
|
561
|
+
"""
|
|
562
|
+
data_dir = Path(args.data)
|
|
563
|
+
models, _curated, _boards = _load(data_dir)
|
|
564
|
+
table = scales(models)
|
|
565
|
+
on_offer = available_ids(data_dir)
|
|
566
|
+
shown = (
|
|
567
|
+
models
|
|
568
|
+
if getattr(args, "all_models", False) or not on_offer
|
|
569
|
+
else [m for m in models if offered(m.model_id, on_offer)]
|
|
570
|
+
)
|
|
571
|
+
ranked = sorted(
|
|
572
|
+
(m for m in shown if m.scores),
|
|
573
|
+
key=lambda m: -(combined(derived_scores(m, table)) or 0),
|
|
574
|
+
)
|
|
575
|
+
unscored = [m for m in shown if not m.scores]
|
|
576
|
+
columns = _score_columns(models, table)
|
|
577
|
+
|
|
578
|
+
# four counts, because "models with scores" used to mean "models with
|
|
579
|
+
# any benchmark" - which overstated the scored field by a third - and
|
|
580
|
+
# "on one benchmark" used to fold in models with no scored benchmark
|
|
581
|
+
# at all
|
|
582
|
+
counts = {}
|
|
583
|
+
for m in ranked:
|
|
584
|
+
counts.setdefault(len(derived_scores(m, table)), []).append(m)
|
|
585
|
+
scored = [m for m in ranked if combined(derived_scores(m, table)) is not None]
|
|
586
|
+
one = counts.get(1, [])
|
|
587
|
+
zerod = [m for m in ranked if not derived_scores(m, table)]
|
|
588
|
+
print(
|
|
589
|
+
f"{len(models)} models · {len(columns)} benchmarks scored on · "
|
|
590
|
+
f"{len(scored)} with scores · {len(one)} on one benchmark · "
|
|
591
|
+
f"{len(zerod)} with none scored · {len(unscored)} without any"
|
|
592
|
+
)
|
|
593
|
+
print()
|
|
594
|
+
cells = [_score_cells(m, table, columns) for m in ranked]
|
|
595
|
+
# the same markers as `list`, over the models shown here: rank position
|
|
596
|
+
# in this ordering, · where there is no score to rank
|
|
597
|
+
marks = {id(m): grade_for(combined(derived_scores(m, table))) for m in ranked}
|
|
598
|
+
print("GRADE: ++ ≥90 · + ≥75 · o 60–75 · - 50–60 · -- <50 · · unranked")
|
|
599
|
+
heads = ["MODEL", "GRADE", "SCORE"] + [label for _k, label, _n in columns]
|
|
600
|
+
grid = []
|
|
601
|
+
for model, row in zip(ranked, cells, strict=True):
|
|
602
|
+
derived = derived_scores(model, table)
|
|
603
|
+
score = combined(derived)
|
|
604
|
+
grid.append(
|
|
605
|
+
[
|
|
606
|
+
model.name,
|
|
607
|
+
marks.get(id(model), "·"),
|
|
608
|
+
f"{score:.0f}" if score is not None else "—",
|
|
609
|
+
]
|
|
610
|
+
+ row
|
|
611
|
+
)
|
|
612
|
+
# measured, not fixed: MODEL fits the shown rows (not the whole
|
|
613
|
+
# catalog) and the fixed columns fit their headers. The benchmark
|
|
614
|
+
# columns keep double gaps — their labels hold single spaces
|
|
615
|
+
# ("SWE Vrfd"), so the gap is what delimits them.
|
|
616
|
+
fixed_aligns = ["<", ">", ">"]
|
|
617
|
+
widths = column_widths(heads, grid)
|
|
618
|
+
lines = []
|
|
619
|
+
for cells in [heads] + grid:
|
|
620
|
+
left = format_row(cells[:3], widths[:3], fixed_aligns, gap=" ")
|
|
621
|
+
right = format_row(cells[3:], widths[3:], "<", gap=" ")
|
|
622
|
+
lines.append((" " + left + (" " + right if right else "")).rstrip())
|
|
623
|
+
print("\n".join(lines))
|
|
624
|
+
legend = [f"{label} = {name}" for _key, label, name in columns if label != name]
|
|
625
|
+
if legend:
|
|
626
|
+
print()
|
|
627
|
+
print(" " + " · ".join(legend))
|
|
628
|
+
if unscored:
|
|
629
|
+
print(
|
|
630
|
+
f" {len(unscored)} with no benchmarks: "
|
|
631
|
+
+ ", ".join(sorted(m.name for m in unscored)[:10])
|
|
632
|
+
)
|
|
633
|
+
print(' how a score is derived: README, "How a model\'s score is computed"')
|
|
634
|
+
return 0
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def _entries_for(source: str, data_dir: Path):
|
|
638
|
+
"""Documented pricing lines for a tier (Go first: interchangeable, and
|
|
639
|
+
its rates are the ones with included usage attached)."""
|
|
640
|
+
prices = load_prices(data_dir / "tier_prices")
|
|
641
|
+
if source in prices:
|
|
642
|
+
return prices[source]
|
|
643
|
+
if source == "all":
|
|
644
|
+
return prices.get("go", []) + prices.get("zen", [])
|
|
645
|
+
return []
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def _bench_tag(benchmark: str | None) -> str:
|
|
649
|
+
if not benchmark:
|
|
650
|
+
return "-"
|
|
651
|
+
folded = benchmark.lower().replace(" ", "-")
|
|
652
|
+
if folded.startswith("terminal-bench-4"):
|
|
653
|
+
return "TB4" # harder scale, ranked on its own
|
|
654
|
+
if folded.startswith("terminal-bench"):
|
|
655
|
+
return "TB"
|
|
656
|
+
if folded.startswith("swe-bench"):
|
|
657
|
+
return "SWE"
|
|
658
|
+
return benchmark[:5]
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
def _cells(row, axis: bool = True) -> tuple[str, ...]:
|
|
662
|
+
"""The fixed columns of one row, as text: score, cost, value, plan.
|
|
663
|
+
|
|
664
|
+
The score is the model's combined 0-100 number, rounded for the column;
|
|
665
|
+
the value is score per dollar — free lines divide by their nominal
|
|
666
|
+
rate, so the cheapest free models read highest ("free" only where the
|
|
667
|
+
price is exactly zero, "-" with no score). The best-value table orders
|
|
668
|
+
on it, so the column that explains the order stays printed. *axis* is
|
|
669
|
+
kept for the single-benchmark layout, which adds that benchmark's tag.
|
|
670
|
+
Prices are cents, as elsewhere.
|
|
671
|
+
"""
|
|
672
|
+
no_cost = row.free or row.price == 0
|
|
673
|
+
price = "0" if no_cost else f"{round(row.price * 100)}"
|
|
674
|
+
score = "-" if row.score is None else f"{round(row.score)}"
|
|
675
|
+
if axis:
|
|
676
|
+
return (score, price, _pick_value(row), row.provider)
|
|
677
|
+
return (score, _bench_tag(row.benchmark), price, _pick_value(row), row.provider)
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def _widths(rows, axis: bool = True) -> tuple[int, ...]:
|
|
681
|
+
"""Widest cell per column across the whole view, header included."""
|
|
682
|
+
headings = AXIS_HEADERS if axis else SINGLE_HEADERS
|
|
683
|
+
return tuple(column_widths(headings, [_cells(r, axis) for r in rows]))
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _print_row(
|
|
687
|
+
row, widths: tuple[int, ...], curated: set[str] = frozenset(), axis: bool = True
|
|
688
|
+
) -> None:
|
|
689
|
+
name = f"{row.name} *" if row.model.model_id in curated else row.name
|
|
690
|
+
print(
|
|
691
|
+
f"{row.grade:>{GRADE_WIDTH - 1}} "
|
|
692
|
+
+ _row_line(_cells(row, axis), widths)
|
|
693
|
+
+ f" {name}"
|
|
694
|
+
)
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def _select(data_dir: Path, source: str | None, price: str = "weighted"):
|
|
698
|
+
"""Which providers a view covers, or None if the name is not one.
|
|
699
|
+
|
|
700
|
+
Providers are zen and go, plus ``-`` for catalog models in neither tier.
|
|
701
|
+
"""
|
|
702
|
+
tiers = sources.available_sources()
|
|
703
|
+
if source is None:
|
|
704
|
+
return list(tiers), "tiers" # both, merged
|
|
705
|
+
if source == "all":
|
|
706
|
+
return list(tiers) + ["-"], "all"
|
|
707
|
+
if source in tiers:
|
|
708
|
+
return [source], "tier"
|
|
709
|
+
return None
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def _current_offered(
|
|
713
|
+
data_dir: Path,
|
|
714
|
+
bypass: bool = False,
|
|
715
|
+
) -> set[str] | None:
|
|
716
|
+
"""Model ids offered right now, or None for no restriction.
|
|
717
|
+
|
|
718
|
+
With ``OPENCODE_API_KEY`` the live config API is the record — it is
|
|
719
|
+
also snapshotted to ``available/`` on the way, so offline runs inherit
|
|
720
|
+
the liveness. Without a key (or when the API is unreachable) the last
|
|
721
|
+
snapshots decide. *bypass* (`--all`) skips every filter.
|
|
722
|
+
"""
|
|
723
|
+
if bypass:
|
|
724
|
+
return None
|
|
725
|
+
key = load_key()
|
|
726
|
+
if key:
|
|
727
|
+
try:
|
|
728
|
+
live = fetch_live(_api_get(key))
|
|
729
|
+
except (OSError, ValueError, KeyError) as exc:
|
|
730
|
+
print(
|
|
731
|
+
f"warning: live config unreachable ({exc}) — using the last snapshots"
|
|
732
|
+
)
|
|
733
|
+
else:
|
|
734
|
+
ids = set().union(*live.values())
|
|
735
|
+
if not ids:
|
|
736
|
+
print("warning: live config lists no models — using the last snapshots")
|
|
737
|
+
else:
|
|
738
|
+
save_live(data_dir, live)
|
|
739
|
+
return ids
|
|
740
|
+
else:
|
|
741
|
+
print(
|
|
742
|
+
"no API token found — availability may be stale, some models "
|
|
743
|
+
"may be unavailable (`auth` for live data, `--all` for the "
|
|
744
|
+
"full catalog)"
|
|
745
|
+
)
|
|
746
|
+
return available_ids(data_dir)
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def _in_tier_slug_list(model_id: str, slugs: set[str]) -> bool:
|
|
750
|
+
"""Whether *model_id* is claimed by a tier's shipped slug list.
|
|
751
|
+
|
|
752
|
+
Mirrors :func:`sources.members` matching: slug, full id, or the
|
|
753
|
+
``-free`` variant of the slug. Used to keep tier-listed models out of
|
|
754
|
+
the catalog-only pool even when a tier dropped them from its offer.
|
|
755
|
+
"""
|
|
756
|
+
slug = model_id.split("/")[-1]
|
|
757
|
+
return slug in slugs or model_id in slugs or f"{slug}-free" in slugs
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def _rows_for(
|
|
761
|
+
data_dir: Path, selected: list[str], price: str, all_models: bool = False
|
|
762
|
+
):
|
|
763
|
+
"""Build, merge and dedupe the rows for *selected* providers.
|
|
764
|
+
|
|
765
|
+
Shared by `list` and `next` so a recommendation can never disagree with
|
|
766
|
+
the ranking it was drawn from.
|
|
767
|
+
|
|
768
|
+
Unless *all_models*, a row is a model the plans actually offer: the
|
|
769
|
+
live config API is the record when keyed, else the snapshots. A model
|
|
770
|
+
the plans dropped is neither listed nor scored. The scales are still
|
|
771
|
+
built over the whole catalog, so a score does not move when a plan
|
|
772
|
+
changes what it sells.
|
|
773
|
+
"""
|
|
774
|
+
models, curated, _ = _load(data_dir)
|
|
775
|
+
on_offer = _current_offered(data_dir, bypass=all_models)
|
|
776
|
+
# the fetched free-tier nominals win per model; the committed table is
|
|
777
|
+
# only the offline fallback for whatever the snapshot does not cover
|
|
778
|
+
snapshot = load_free_nominals(data_dir)
|
|
779
|
+
free_rates = {**FREE_TIER_NOMINAL_RATES, **snapshot} if snapshot else None
|
|
780
|
+
# one scale for the whole catalog: per-tier scales would make the same
|
|
781
|
+
# score mean two different things in the merged view
|
|
782
|
+
table = scales(models)
|
|
783
|
+
groups, listed, counts, tier_slugs = [], [], {}, {}
|
|
784
|
+
covered: set[str] = set()
|
|
785
|
+
# slugs shipped for every selected tier: a model listed by any of them
|
|
786
|
+
# belongs to that tier (or is dropped by availability) and must not be
|
|
787
|
+
# routed to the catalog-only "-" pool, even when the tier dropped it
|
|
788
|
+
selected_tier_slugs: set[str] = set()
|
|
789
|
+
for provider in selected:
|
|
790
|
+
if provider != "-":
|
|
791
|
+
selected_tier_slugs |= sources.load_source(provider)
|
|
792
|
+
for provider in selected:
|
|
793
|
+
if provider == "-":
|
|
794
|
+
pool = [
|
|
795
|
+
m
|
|
796
|
+
for m in models
|
|
797
|
+
if m.model_id not in covered
|
|
798
|
+
and not _in_tier_slug_list(m.model_id, selected_tier_slugs)
|
|
799
|
+
]
|
|
800
|
+
free_ids = sources.members(models, None)[1]
|
|
801
|
+
free_ids &= {m.model_id for m in pool}
|
|
802
|
+
else:
|
|
803
|
+
tier_slugs[provider] = sources.load_source(provider)
|
|
804
|
+
pool, free_ids = sources.members(models, tier_slugs[provider])
|
|
805
|
+
if on_offer is not None:
|
|
806
|
+
pool = [m for m in pool if offered(m.model_id, on_offer)]
|
|
807
|
+
free_ids &= {m.model_id for m in pool}
|
|
808
|
+
counts[provider] = len(pool)
|
|
809
|
+
covered |= {m.model_id for m in pool}
|
|
810
|
+
listed += pool
|
|
811
|
+
groups.append(
|
|
812
|
+
build_rows(
|
|
813
|
+
pool,
|
|
814
|
+
_entries_for(provider, data_dir),
|
|
815
|
+
free_ids,
|
|
816
|
+
axis=price,
|
|
817
|
+
provider=provider,
|
|
818
|
+
table=table,
|
|
819
|
+
free_rates=free_rates,
|
|
820
|
+
)
|
|
821
|
+
)
|
|
822
|
+
rows = dedupe_providers(merge_rows(*groups))
|
|
823
|
+
return rows, listed, curated, counts, tier_slugs, models
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def _cmd_list(args: argparse.Namespace) -> int:
|
|
827
|
+
data_dir = Path(args.data)
|
|
828
|
+
if not data_dir.is_dir():
|
|
829
|
+
return _fail(f"no data directory at {data_dir} — run `fetch` first")
|
|
830
|
+
|
|
831
|
+
source = args.source
|
|
832
|
+
chosen = _select(data_dir, source)
|
|
833
|
+
if chosen is None:
|
|
834
|
+
return _fail(
|
|
835
|
+
f"unknown source {source!r} — available: "
|
|
836
|
+
f"{', '.join(sources.available_sources())}, all"
|
|
837
|
+
)
|
|
838
|
+
selected, scope = chosen
|
|
839
|
+
|
|
840
|
+
rows, listed, curated, counts, tier_slugs, models = _rows_for(
|
|
841
|
+
data_dir, selected, args.price, getattr(args, "all_models", False)
|
|
842
|
+
)
|
|
843
|
+
if not listed:
|
|
844
|
+
return _fail(f"no model of {source or 'zen+go'} is in {data_dir} — run `fetch`")
|
|
845
|
+
# one line per (model, variant, score, cost) before the culls see them
|
|
846
|
+
if args.benchmark:
|
|
847
|
+
wanted = args.benchmark
|
|
848
|
+
if wanted not in dict(available(listed)):
|
|
849
|
+
listing = ", ".join(f"{n} ({c})" for n, c in list(available(listed))[:10])
|
|
850
|
+
return _fail(f"unknown benchmark {wanted!r} — available: {listing}")
|
|
851
|
+
rows = [
|
|
852
|
+
replace(r, score=r.model.scores[wanted], benchmark=wanted)
|
|
853
|
+
for r in rows
|
|
854
|
+
if wanted in r.model.scores
|
|
855
|
+
]
|
|
856
|
+
rows.sort(key=lambda r: (-ratio(r.score, r.price), -r.score))
|
|
857
|
+
order_note = f"benchmark {wanted} · sorted by score per $"
|
|
858
|
+
else:
|
|
859
|
+
rows = order_rows(rows)
|
|
860
|
+
order_note = "ranked by score per $, best value first"
|
|
861
|
+
# the marker reads the score itself, so a row in the cost or
|
|
862
|
+
# worst-score table shows the same grade it earned in best value
|
|
863
|
+
rows = grade_rows(rows)
|
|
864
|
+
# the layout: every axis side by side, or the single ranked benchmark
|
|
865
|
+
axis = not args.benchmark
|
|
866
|
+
if not rows:
|
|
867
|
+
return _fail(
|
|
868
|
+
f"no {source or 'zen+go'} model in {data_dir} has benchmark scores"
|
|
869
|
+
)
|
|
870
|
+
|
|
871
|
+
if scope == "tiers":
|
|
872
|
+
head = f"zen+go: {counts.get('zen', 0)} zen + {counts.get('go', 0)} go models"
|
|
873
|
+
elif scope == "tier":
|
|
874
|
+
head = f"{source}: {counts[source]}/{len(tier_slugs[source])} models in catalog"
|
|
875
|
+
else:
|
|
876
|
+
head = f"all: {len(models)} models in catalog"
|
|
877
|
+
print(f"{head} · {len(rows)} lines · {order_note} · {COST_NOTE}")
|
|
878
|
+
if not getattr(args, "all_models", False):
|
|
879
|
+
plans = "Go/Zen plans" if source in (None, "all") else f"{source} plan"
|
|
880
|
+
print(
|
|
881
|
+
f"Only models available right now on the {plans} — "
|
|
882
|
+
f"full list with --all-models."
|
|
883
|
+
)
|
|
884
|
+
print("GRADE: ++ ≥90 · + ≥75 · o 60–75 · - 50–60 · -- <50 · · unranked")
|
|
885
|
+
|
|
886
|
+
# the split always happens — the cull is what keeps the worst-scoring
|
|
887
|
+
# and priciest models out of the view — only the reporting is optional
|
|
888
|
+
main, worst, pricey = partition_rows(rows)
|
|
889
|
+
# one set of column widths for the whole view, so the three tables line
|
|
890
|
+
# up with each other too
|
|
891
|
+
widths = _widths(rows, axis)
|
|
892
|
+
|
|
893
|
+
_section(f"best value · {len(main)} lines", widths, axis)
|
|
894
|
+
for band, label in BAND_LABELS:
|
|
895
|
+
group = [r for r in main if r.band == band]
|
|
896
|
+
if group:
|
|
897
|
+
print(f" {label} · {len(group)}")
|
|
898
|
+
for row in group:
|
|
899
|
+
_print_row(row, widths, curated.curated, axis)
|
|
900
|
+
|
|
901
|
+
if args.tables == "all":
|
|
902
|
+
_section(f"worst score (bottom 30%) · {len(worst)} lines", widths, axis)
|
|
903
|
+
for row in worst:
|
|
904
|
+
_print_row(row, widths, curated.curated, axis)
|
|
905
|
+
|
|
906
|
+
_section(f"highest cost (top 30%) · {len(pricey)} lines", widths, axis)
|
|
907
|
+
for row in pricey:
|
|
908
|
+
_print_row(row, widths, curated.curated, axis)
|
|
909
|
+
return 0
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
#: How each band is named in the sequence. The rules behind these words —
|
|
913
|
+
#: what an allowance costs, when a balance is drawn down, what "free" does
|
|
914
|
+
#: not mean — are written down once in LIMITS.md rather than restated on
|
|
915
|
+
#: every run.
|
|
916
|
+
NEXT_BANDS = {
|
|
917
|
+
"free": "free",
|
|
918
|
+
"included": "included",
|
|
919
|
+
"payg": "payg",
|
|
920
|
+
"catalog": "catalog",
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def _pick_value(row) -> str:
|
|
925
|
+
"""A pick's score per dollar, free lines included.
|
|
926
|
+
|
|
927
|
+
Free lines carry their nominal rate, so they divide like any other
|
|
928
|
+
cost — the cheapest free models read highest. "free" is only the
|
|
929
|
+
degenerate zero-price fallback; "-" means there is no score to value.
|
|
930
|
+
"""
|
|
931
|
+
if row.score is None:
|
|
932
|
+
return "-"
|
|
933
|
+
if row.price == 0:
|
|
934
|
+
return "free"
|
|
935
|
+
return f"{round(ratio(row.score, row.price))}"
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
def _endpoint(row) -> str:
|
|
939
|
+
"""A pick as an API identifier: provider prefix plus the plan-local id.
|
|
940
|
+
|
|
941
|
+
Display names are not callable, so the view shows what goes in the
|
|
942
|
+
request instead: ``opencode-go/step-5-preview-free``. The prefix follows
|
|
943
|
+
the plans on the line, preferring Go where both list it; the id is the
|
|
944
|
+
tier's own entry where documented, else the catalog slug. Variants and
|
|
945
|
+
free tags are display concerns and stay out of it.
|
|
946
|
+
"""
|
|
947
|
+
plans = plans_of(row)
|
|
948
|
+
prefix = "opencode-go" if "go" in plans else "opencode"
|
|
949
|
+
if row.entry is not None and row.entry.model_id:
|
|
950
|
+
return f"{prefix}/{row.entry.model_id}"
|
|
951
|
+
return f"{prefix}/{row.model.model_id.split('/')[-1]}"
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
def _next_band(row) -> str:
|
|
955
|
+
# a row sold by both tiers is labelled "go,zen", so membership decides,
|
|
956
|
+
# not equality with a single plan name
|
|
957
|
+
if not {"zen", "go"} & plans_of(row):
|
|
958
|
+
return NEXT_BANDS["catalog"]
|
|
959
|
+
return NEXT_BANDS[
|
|
960
|
+
"included" if row.band == "included" else ("free" if row.free else "payg")
|
|
961
|
+
]
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
#: The `next` grid, declared: one text column per field, measured at
|
|
965
|
+
#: print time so a wide provider (`go,zen`) or a long score widens its
|
|
966
|
+
#: column instead of shifting the rest of the row.
|
|
967
|
+
NEXT_HEADERS = ("#", "PLAN", "SCORE", "¢/1M", "SCORE/$", "ENDPOINT", "BAND")
|
|
968
|
+
NEXT_ALIGNS = ("<", "<", ">", ">", ">", "<", "<")
|
|
969
|
+
NEXT_MIN_WIDTHS = (2, 5, 6, 5, 7, 0, 0)
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def _next_table(picks: list) -> str:
|
|
973
|
+
"""The picks as an aligned grid: position, plan, score, cost, value,
|
|
974
|
+
endpoint, band."""
|
|
975
|
+
rows = []
|
|
976
|
+
for i, row in enumerate(picks, 1):
|
|
977
|
+
_score, price, value, _prov = _cells(row)
|
|
978
|
+
rows.append(
|
|
979
|
+
[
|
|
980
|
+
str(i),
|
|
981
|
+
row.provider,
|
|
982
|
+
f"{round(row.score)}",
|
|
983
|
+
price,
|
|
984
|
+
value,
|
|
985
|
+
_endpoint(row),
|
|
986
|
+
_next_band(row),
|
|
987
|
+
]
|
|
988
|
+
)
|
|
989
|
+
return render(
|
|
990
|
+
NEXT_HEADERS,
|
|
991
|
+
rows,
|
|
992
|
+
aligns=NEXT_ALIGNS,
|
|
993
|
+
min_widths=NEXT_MIN_WIDTHS,
|
|
994
|
+
indent=" ",
|
|
995
|
+
gap=" ",
|
|
996
|
+
)
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _cmd_next(args: argparse.Namespace) -> int:
|
|
1000
|
+
"""A short ordered sequence: free and paid picks interleaved band by band."""
|
|
1001
|
+
data_dir = Path(args.data)
|
|
1002
|
+
if not data_dir.is_dir():
|
|
1003
|
+
return _fail(f"no data directory at {data_dir} — run `fetch` first")
|
|
1004
|
+
chosen = _select(data_dir, args.source)
|
|
1005
|
+
if chosen is None:
|
|
1006
|
+
return _fail(
|
|
1007
|
+
f"unknown source {args.source!r} — available: "
|
|
1008
|
+
f"{', '.join(sources.available_sources())}, all"
|
|
1009
|
+
)
|
|
1010
|
+
rows, listed, _curated, _counts, _slugs, _models = _rows_for(
|
|
1011
|
+
data_dir, chosen[0], args.price, getattr(args, "all_models", False)
|
|
1012
|
+
)
|
|
1013
|
+
if not listed:
|
|
1014
|
+
return _fail(
|
|
1015
|
+
f"no model of {args.source or 'zen+go'} is in {data_dir} "
|
|
1016
|
+
f"— run `fetch` first"
|
|
1017
|
+
)
|
|
1018
|
+
picks = next_up(grade_rows(order_rows(rows)), count=args.count)
|
|
1019
|
+
if not picks:
|
|
1020
|
+
return _fail(
|
|
1021
|
+
f"no model of {args.source or 'zen+go'} scores on "
|
|
1022
|
+
f"{MIN_AXES}+ benchmarks in {data_dir} — nothing to "
|
|
1023
|
+
f"recommend yet"
|
|
1024
|
+
)
|
|
1025
|
+
|
|
1026
|
+
print(
|
|
1027
|
+
f"Next {len(picks)} recommended models, "
|
|
1028
|
+
f">{RECOMMEND_FLOOR:.0%} overall score, free and paid picks "
|
|
1029
|
+
f"interleaved, alternating Zen/Go plans."
|
|
1030
|
+
)
|
|
1031
|
+
print()
|
|
1032
|
+
print(_next_table(picks))
|
|
1033
|
+
return 0
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1037
|
+
parser = argparse.ArgumentParser(
|
|
1038
|
+
prog="opencode-models",
|
|
1039
|
+
description="Price/score Pareto frontier of models from opencode.ai/data",
|
|
1040
|
+
)
|
|
1041
|
+
parser.add_argument(
|
|
1042
|
+
"--data",
|
|
1043
|
+
default=str(DEFAULT_DATA),
|
|
1044
|
+
help="data directory (default: %(default)s)",
|
|
1045
|
+
)
|
|
1046
|
+
parser.add_argument(
|
|
1047
|
+
"--version", action="version", version=f"%(prog)s {__version__}"
|
|
1048
|
+
)
|
|
1049
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
1050
|
+
|
|
1051
|
+
sub.add_parser("fetch", help="refresh data from opencode.ai").add_argument(
|
|
1052
|
+
"--all",
|
|
1053
|
+
dest="all_models",
|
|
1054
|
+
action="store_true",
|
|
1055
|
+
help="download the full model catalog, not just the models the "
|
|
1056
|
+
"plans offer (kept records are never deleted either way)",
|
|
1057
|
+
)
|
|
1058
|
+
sub.add_parser(
|
|
1059
|
+
"models",
|
|
1060
|
+
help="the models the plans offer right now, "
|
|
1061
|
+
"as endpoints beside display names: live "
|
|
1062
|
+
"from the Console config API with "
|
|
1063
|
+
"OPENCODE_API_KEY, else the last fetch "
|
|
1064
|
+
"snapshots",
|
|
1065
|
+
)
|
|
1066
|
+
sub.add_parser(
|
|
1067
|
+
"auth",
|
|
1068
|
+
help="save the Console service-account key "
|
|
1069
|
+
"to an owner-only file (prompted, never "
|
|
1070
|
+
"echoed); verifies it live when it can",
|
|
1071
|
+
)
|
|
1072
|
+
sub.add_parser(
|
|
1073
|
+
"overrides",
|
|
1074
|
+
help="every benchmark number more than one source "
|
|
1075
|
+
"reports, and which value the console keeps",
|
|
1076
|
+
)
|
|
1077
|
+
|
|
1078
|
+
p_list = sub.add_parser(
|
|
1079
|
+
"list",
|
|
1080
|
+
help="three tables of Console models across providers "
|
|
1081
|
+
"(zen/go): best value (free/included/pay-as-you-go), "
|
|
1082
|
+
"worst score, highest cost",
|
|
1083
|
+
)
|
|
1084
|
+
p_list.add_argument(
|
|
1085
|
+
"--source",
|
|
1086
|
+
help="provider to list: zen or go (default: both, "
|
|
1087
|
+
"merged) — 'all' adds the catalog models that "
|
|
1088
|
+
"are in neither tier",
|
|
1089
|
+
)
|
|
1090
|
+
p_list.add_argument(
|
|
1091
|
+
"--price",
|
|
1092
|
+
choices=("weighted", "input", "output"),
|
|
1093
|
+
default="weighted",
|
|
1094
|
+
help="cost of a line (default: weighted 20%% in, 60%% cached, 20%% out)",
|
|
1095
|
+
)
|
|
1096
|
+
p_list.add_argument(
|
|
1097
|
+
"--tables",
|
|
1098
|
+
choices=("best", "all"),
|
|
1099
|
+
default="best",
|
|
1100
|
+
help="best: print only the best-value table. all: "
|
|
1101
|
+
"also print the worst-score and highest-cost "
|
|
1102
|
+
"tables. Either way the cull happens — those "
|
|
1103
|
+
"rows are out of best value (default: %(default)s)",
|
|
1104
|
+
)
|
|
1105
|
+
p_list.add_argument(
|
|
1106
|
+
"--all-models",
|
|
1107
|
+
"--all",
|
|
1108
|
+
dest="all_models",
|
|
1109
|
+
action="store_true",
|
|
1110
|
+
help="include models the plans no longer offer; the "
|
|
1111
|
+
"default shows only what a plan currently lists",
|
|
1112
|
+
)
|
|
1113
|
+
p_list.add_argument(
|
|
1114
|
+
"--benchmark",
|
|
1115
|
+
help="rank on one benchmark instead of combining "
|
|
1116
|
+
"Terminal-Bench 2.x (first) with SWE-Bench and "
|
|
1117
|
+
"Terminal-Bench 4.0",
|
|
1118
|
+
)
|
|
1119
|
+
|
|
1120
|
+
p_next = sub.add_parser(
|
|
1121
|
+
"next",
|
|
1122
|
+
help="the next few models to use, in order: free and paid "
|
|
1123
|
+
"picks interleaved, best score then best score per dollar",
|
|
1124
|
+
)
|
|
1125
|
+
p_next.add_argument(
|
|
1126
|
+
"--count",
|
|
1127
|
+
type=int,
|
|
1128
|
+
default=6,
|
|
1129
|
+
help="how many to recommend (default: %(default)s)",
|
|
1130
|
+
)
|
|
1131
|
+
p_next.add_argument(
|
|
1132
|
+
"--source", help="provider to draw from: zen or go (default: both, merged)"
|
|
1133
|
+
)
|
|
1134
|
+
p_next.add_argument(
|
|
1135
|
+
"--price",
|
|
1136
|
+
choices=("weighted", "input", "output"),
|
|
1137
|
+
default="weighted",
|
|
1138
|
+
help="cost of a line (default: weighted 20%% in, 60%% cached, 20%% out)",
|
|
1139
|
+
)
|
|
1140
|
+
p_next.add_argument(
|
|
1141
|
+
"--all-models",
|
|
1142
|
+
"--all",
|
|
1143
|
+
dest="all_models",
|
|
1144
|
+
action="store_true",
|
|
1145
|
+
help="include models the plans no longer offer; the "
|
|
1146
|
+
"default shows only what a plan currently lists",
|
|
1147
|
+
)
|
|
1148
|
+
|
|
1149
|
+
p_scores = sub.add_parser("scores", help="manage cached arena.ai Elo scores")
|
|
1150
|
+
p_scores.add_argument(
|
|
1151
|
+
"--all-models",
|
|
1152
|
+
"--all",
|
|
1153
|
+
dest="all_models",
|
|
1154
|
+
action="store_true",
|
|
1155
|
+
help="include models the plans no longer offer; the "
|
|
1156
|
+
"default shows only what a plan currently lists",
|
|
1157
|
+
)
|
|
1158
|
+
|
|
1159
|
+
args = parser.parse_args(argv)
|
|
1160
|
+
return {
|
|
1161
|
+
"fetch": _cmd_fetch,
|
|
1162
|
+
"models": _cmd_models,
|
|
1163
|
+
"auth": _cmd_auth,
|
|
1164
|
+
"overrides": _cmd_conflicts,
|
|
1165
|
+
"list": _cmd_list,
|
|
1166
|
+
"next": _cmd_next,
|
|
1167
|
+
"scores": _cmd_scores,
|
|
1168
|
+
}[args.command](args)
|
|
1169
|
+
|
|
1170
|
+
|
|
1171
|
+
if __name__ == "__main__":
|
|
1172
|
+
sys.exit(main())
|