opencode-models 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. opencode_models-0.1.0/.githooks/pre-commit +24 -0
  2. opencode_models-0.1.0/.githooks/pre-push +17 -0
  3. opencode_models-0.1.0/.github/dependabot.yml +24 -0
  4. opencode_models-0.1.0/.github/workflows/ci.yml +88 -0
  5. opencode_models-0.1.0/.github/workflows/release.yml +34 -0
  6. opencode_models-0.1.0/.github/workflows/secret-scan.yml +36 -0
  7. opencode_models-0.1.0/.gitignore +21 -0
  8. opencode_models-0.1.0/.opencode/skills/backfill-scores/SKILL.md +284 -0
  9. opencode_models-0.1.0/.plan/.TEMPLATE.md +22 -0
  10. opencode_models-0.1.0/.plan/.schema.json +22 -0
  11. opencode_models-0.1.0/.plan/public-release.md +67 -0
  12. opencode_models-0.1.0/.plan/robustness.md +22 -0
  13. opencode_models-0.1.0/.plan/v01-review.md +84 -0
  14. opencode_models-0.1.0/AGENTS.md +107 -0
  15. opencode_models-0.1.0/CHANGELOG.md +13 -0
  16. opencode_models-0.1.0/CONTRIBUTING.md +65 -0
  17. opencode_models-0.1.0/LICENSE +21 -0
  18. opencode_models-0.1.0/LIMITS.md +288 -0
  19. opencode_models-0.1.0/PKG-INFO +544 -0
  20. opencode_models-0.1.0/README.md +504 -0
  21. opencode_models-0.1.0/SECURITY.md +49 -0
  22. opencode_models-0.1.0/data/overrides/README.md +81 -0
  23. opencode_models-0.1.0/data/overrides/alibaba__qwen3-8-flash-next.json +1 -0
  24. opencode_models-0.1.0/data/overrides/alibaba__qwen3-8-max-0902.json +1 -0
  25. opencode_models-0.1.0/data/overrides/anthropic__claude-fable-5-1.json +1 -0
  26. opencode_models-0.1.0/data/overrides/anthropic__claude-opus-5.json +1 -0
  27. opencode_models-0.1.0/data/overrides/deepseek__deepseek-v4-flash.json +1 -0
  28. opencode_models-0.1.0/data/overrides/inclusionai__ling-3-0-flash-fin.json +1 -0
  29. opencode_models-0.1.0/data/overrides/moonshotai__kimi-k3.json +1 -0
  30. opencode_models-0.1.0/data/overrides/openai__gpt-6-astra.json +1 -0
  31. opencode_models-0.1.0/data/overrides/stepfun__step-5-preview.json +1 -0
  32. opencode_models-0.1.0/justfile +37 -0
  33. opencode_models-0.1.0/opencode_models/__init__.py +8 -0
  34. opencode_models-0.1.0/opencode_models/__main__.py +5 -0
  35. opencode_models-0.1.0/opencode_models/arena.py +108 -0
  36. opencode_models-0.1.0/opencode_models/auth.py +48 -0
  37. opencode_models-0.1.0/opencode_models/availability.py +102 -0
  38. opencode_models-0.1.0/opencode_models/benchmarks.py +19 -0
  39. opencode_models-0.1.0/opencode_models/boards.py +518 -0
  40. opencode_models-0.1.0/opencode_models/cli.py +1172 -0
  41. opencode_models-0.1.0/opencode_models/data.py +82 -0
  42. opencode_models-0.1.0/opencode_models/fetch.py +101 -0
  43. opencode_models-0.1.0/opencode_models/identity.py +100 -0
  44. opencode_models-0.1.0/opencode_models/labelmap.py +84 -0
  45. opencode_models-0.1.0/opencode_models/live.py +88 -0
  46. opencode_models-0.1.0/opencode_models/overrides.py +252 -0
  47. opencode_models-0.1.0/opencode_models/pareto.py +34 -0
  48. opencode_models-0.1.0/opencode_models/pricing.py +415 -0
  49. opencode_models-0.1.0/opencode_models/ranking.py +591 -0
  50. opencode_models-0.1.0/opencode_models/scoring.py +230 -0
  51. opencode_models-0.1.0/opencode_models/seed.py +184 -0
  52. opencode_models-0.1.0/opencode_models/sources/go.txt +45 -0
  53. opencode_models-0.1.0/opencode_models/sources/zen.txt +87 -0
  54. opencode_models-0.1.0/opencode_models/sources.py +55 -0
  55. opencode_models-0.1.0/opencode_models/tables.py +94 -0
  56. opencode_models-0.1.0/pyproject.toml +64 -0
  57. opencode_models-0.1.0/tests/__init__.py +0 -0
  58. opencode_models-0.1.0/tests/test_arena.py +131 -0
  59. opencode_models-0.1.0/tests/test_auth.py +68 -0
  60. opencode_models-0.1.0/tests/test_availability.py +167 -0
  61. opencode_models-0.1.0/tests/test_benchmarks.py +55 -0
  62. opencode_models-0.1.0/tests/test_boards.py +731 -0
  63. opencode_models-0.1.0/tests/test_cli.py +3628 -0
  64. opencode_models-0.1.0/tests/test_data.py +131 -0
  65. opencode_models-0.1.0/tests/test_fetch.py +245 -0
  66. opencode_models-0.1.0/tests/test_identity.py +123 -0
  67. opencode_models-0.1.0/tests/test_live.py +49 -0
  68. opencode_models-0.1.0/tests/test_next.py +366 -0
  69. opencode_models-0.1.0/tests/test_overrides.py +493 -0
  70. opencode_models-0.1.0/tests/test_pareto.py +73 -0
  71. opencode_models-0.1.0/tests/test_pricing.py +362 -0
  72. opencode_models-0.1.0/tests/test_ranking.py +770 -0
  73. opencode_models-0.1.0/tests/test_scoring.py +467 -0
  74. opencode_models-0.1.0/tests/test_seed.py +201 -0
  75. opencode_models-0.1.0/tests/test_sources.py +103 -0
  76. opencode_models-0.1.0/tests/test_tables.py +92 -0
  77. opencode_models-0.1.0/tests/test_views.py +495 -0
  78. opencode_models-0.1.0/uv.lock +43 -0
@@ -0,0 +1,24 @@
1
+ #!/bin/sh
2
+ # Fast local gates, no new installs: a secret-pattern sweep over the tree
3
+ # plus ruff lint and format check from the project's own venv.
4
+ # Fails the commit on findings. (CI runs the real gitleaks scanner on
5
+ # every push and PR; this hook is the cheap local tripwire.)
6
+
7
+ # 1. Tracked secret patterns (same sweep as `just audit`, but failing).
8
+ # The pattern is split with '' so this file never matches itself.
9
+ if grep -rniE 'BEGIN (RSA |OPENSSH |EC )?PRIVATE ''KEY|aws_secret_''access_key|ghp_''[A-Za-z0-9]{36}|github_pat_''[A-Za-z0-9]{22}' --exclude-dir=.venv --exclude-dir=.git --exclude-dir=__pycache__ . ; then
10
+ echo "pre-commit: secret-looking content above — remove it before committing" >&2
11
+ exit 1
12
+ fi
13
+
14
+ # 2. Lint + format via the project's own venv (installed by `just bootstrap`).
15
+ # Missing venv is not a failure: warn and let CI be the backstop.
16
+ if command -v uv >/dev/null 2>&1 && uv run --no-sync ruff --version >/dev/null 2>&1; then
17
+ uv run --no-sync ruff check opencode_models tests || exit 1
18
+ uv run --no-sync ruff format --check opencode_models tests || {
19
+ echo "pre-commit: run 'just lint' (or 'uv run ruff format opencode_models tests') and re-stage" >&2
20
+ exit 1
21
+ }
22
+ else
23
+ echo "pre-commit: ruff not available (run 'just bootstrap' once) — lint skipped, CI still checks" >&2
24
+ fi
@@ -0,0 +1,17 @@
1
+ #!/bin/sh
2
+ # Block pushing the local-data branch: it holds the full fetched dataset
3
+ # (hundreds of third-party records plus workspace-derived snapshots) and
4
+ # must never leave this machine. The branch is intentionally never pushed;
5
+ # this hook makes that rule mechanical instead of remembered.
6
+ #
7
+ # Invoked by git with lines "<local ref> <local sha> <remote ref>
8
+ # <remote sha>" on stdin; exit non-zero rejects the push.
9
+
10
+ while read local_ref _local_sha _remote_ref _remote_sha; do
11
+ case "$local_ref" in
12
+ refs/heads/local-data)
13
+ echo "refusing to push $local_ref: the fetched dataset must stay local (see README, Data)" >&2
14
+ exit 1
15
+ ;;
16
+ esac
17
+ done
@@ -0,0 +1,24 @@
1
+ version: 2
2
+
3
+ updates:
4
+ # The package itself is stdlib-only; the Actions pinned in
5
+ # .github/workflows are what drifts. All minor/patch action updates
6
+ # land in one grouped weekly PR instead of one PR each; majors stay
7
+ # individual so a breaking action change gets its own review.
8
+ - package-ecosystem: github-actions
9
+ directory: /
10
+ schedule:
11
+ interval: weekly
12
+ # delay version updates N days after release: avoids yanked-release
13
+ # haste, and satisfies the zizmor dependabot-cooldown audit
14
+ cooldown:
15
+ default-days: 7
16
+ open-pull-requests-limit: 3
17
+ groups:
18
+ github-actions:
19
+ applies-to: version-updates
20
+ patterns:
21
+ - "*"
22
+ update-types:
23
+ - minor
24
+ - patch
@@ -0,0 +1,88 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ # Least privilege: jobs get read-only repo access by default.
9
+ permissions:
10
+ contents: read
11
+
12
+ # One run per ref: a new push cancels the older in-progress run.
13
+ concurrency:
14
+ group: ${{ github.workflow }}-${{ github.ref }}
15
+ cancel-in-progress: true
16
+
17
+ jobs:
18
+ test:
19
+ runs-on: ubuntu-latest
20
+ timeout-minutes: 10
21
+ strategy:
22
+ fail-fast: false
23
+ matrix:
24
+ python-version: ["3.10", "3.12", "3.14"]
25
+ steps:
26
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
27
+ with:
28
+ persist-credentials: false
29
+ - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
30
+ with:
31
+ enable-cache: true
32
+ - name: sync (with the pinned lockfile)
33
+ run: uv sync --python ${{ matrix.python-version }}
34
+ - name: unit tests
35
+ run: uv run --no-sync python -m unittest discover -s tests -v
36
+ - name: the console resolves and runs from the entry point
37
+ # no scraped dataset in CI: it is fetched on demand (see README)
38
+ run: uv run --no-sync opencode-models --help
39
+
40
+ build:
41
+ runs-on: ubuntu-latest
42
+ timeout-minutes: 10
43
+ steps:
44
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
45
+ with:
46
+ persist-credentials: false
47
+ - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
48
+ - name: wheel + sdist build
49
+ run: uv build
50
+ - name: install the wheel into a throwaway tool env
51
+ run: |
52
+ uv tool install --force dist/*.whl
53
+ opencode-models --help >/dev/null
54
+
55
+ lint:
56
+ runs-on: ubuntu-latest
57
+ timeout-minutes: 10
58
+ steps:
59
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
60
+ with:
61
+ persist-credentials: false
62
+ - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
63
+ - name: sync (with the pinned lockfile, dev tools included)
64
+ run: uv sync --python "3.12"
65
+ # same commands as `just lint`; ruff itself comes from the locked
66
+ # dev group, never from the network at CI time beyond the lockfile
67
+ - name: lint, format check and complexity gate
68
+ run: |
69
+ uv run --no-sync ruff check opencode_models tests
70
+ uv run --no-sync ruff format --check opencode_models tests
71
+
72
+ zizmor:
73
+ runs-on: ubuntu-latest
74
+ timeout-minutes: 10
75
+ steps:
76
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
77
+ with:
78
+ persist-credentials: false
79
+ # No SARIF upload: code scanning is not enabled for this repo, and
80
+ # the upload is what kept failing — not the audit. The audit still
81
+ # runs and fails this step on findings. If code scanning gets
82
+ # enabled, drop this flag and add a job-level `permissions` block
83
+ # with `security-events: write` (upload) plus `actions: read`
84
+ # (run association).
85
+ - name: audit workflows with zizmor
86
+ uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1
87
+ with:
88
+ advanced-security: false
@@ -0,0 +1,34 @@
1
+ name: release
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*"
7
+
8
+ # Least privilege: checkout needs repo read access; publishing uses
9
+ # a short-lived OIDC token instead of a stored PyPI secret.
10
+ permissions:
11
+ contents: read
12
+
13
+ jobs:
14
+ publish:
15
+ runs-on: ubuntu-latest
16
+ timeout-minutes: 10
17
+ permissions:
18
+ contents: read
19
+ id-token: write # trusted publishing (PyPI must list this repo as a trusted publisher first)
20
+ steps:
21
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
22
+ with:
23
+ persist-credentials: false
24
+ - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
25
+ with:
26
+ # no cache: releases are rare enough that a cold sync is cheap,
27
+ # and it removes the cache-poisoning surface entirely
28
+ enable-cache: false
29
+ - name: check the tag matches the package version
30
+ run: test "${GITHUB_REF#refs/tags/v}" = "$(grep -m1 '^version *=' pyproject.toml | cut -d'"' -f2)"
31
+ - name: wheel + sdist build
32
+ run: uv build
33
+ - name: publish to PyPI
34
+ uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
@@ -0,0 +1,36 @@
1
+ name: secret-scan
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ schedule:
8
+ - cron: "0 6 * * 1"
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ concurrency:
14
+ group: ${{ github.workflow }}-${{ github.ref }}
15
+ cancel-in-progress: true
16
+
17
+ jobs:
18
+ gitleaks:
19
+ runs-on: ubuntu-latest
20
+ timeout-minutes: 10
21
+ # contents: read comes from the top level; pull-requests: read lets
22
+ # the action resolve the PR range it scans on pull_request events.
23
+ permissions:
24
+ contents: read
25
+ pull-requests: read
26
+ steps:
27
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
28
+ with:
29
+ fetch-depth: 0
30
+ persist-credentials: false
31
+ - name: scan history for leaked secrets
32
+ uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2
33
+ # v2 requires the token explicitly on pull_request events, where it
34
+ # needs the API to determine the scan range
35
+ env:
36
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -0,0 +1,21 @@
1
+ __pycache__/
2
+ *.pyc
3
+ *.egg-info/
4
+ .venv/
5
+ dist/
6
+ # No fetched page or payload is committed. This `*.html` rule also caught the
7
+ # test fixtures once, silently: nothing under tests/ is HTML any more (the
8
+ # board shapes are written out in tests/test_boards.py), and a `git add -A`
9
+ # that skips a file is easy to miss until CI fails on a clean checkout.
10
+ *.html
11
+ !README.md
12
+
13
+ # The scraped/fetched dataset is deliberately not in this branch's history:
14
+ # `opencode-models fetch` populates it, and the `local-data` branch keeps a
15
+ # committed copy. The curated overlays in data/overrides/ ARE tracked.
16
+ data/opencode/
17
+ data/tier_prices/
18
+ data/aiarena/
19
+ data/boards/
20
+ data/usage/
21
+ data/available/
@@ -0,0 +1,284 @@
1
+ ---
2
+ name: backfill-scores
3
+ description: Use when an opencode-models model has only one scored benchmark and is therefore unranked, or when asked to find, research or add benchmark numbers — covers where to look, which sources are actually scrapable, how to write a data/overrides JSON file, and how to verify it landed.
4
+ ---
5
+
6
+ # Backfilling missing benchmark scores
7
+
8
+ A model needs two of the four scored benchmarks to carry a score at all
9
+ (`scoring.MIN_AXES`), so the models worth chasing are the ones sitting on
10
+ one. This is how to find them, source a number, write it down, and prove it
11
+ landed.
12
+
13
+ The rule everything else serves: **a number without a source is not data.**
14
+ The loader drops any override entry that has no `source`, so a file without
15
+ one is inert — it will not appear, fail, or warn.
16
+
17
+ ## 1. Find what is missing
18
+
19
+ The target list is models that are **listed in a Zen or Go plan** and carry
20
+ **one** scored benchmark. Both filters matter:
21
+
22
+ - A model no plan lists is a model about to be unlisted. A number for it
23
+ changes nothing, because nothing recommends it.
24
+ - A model with no scored benchmark at all is a different problem (nothing to
25
+ pair with, and usually nothing published). A model with *one* is a missing
26
+ pair away from being ranked.
27
+
28
+ ```sh
29
+ PYTHONPATH=. uv run python -c "
30
+ from pathlib import Path
31
+ from opencode_models.cli import _load
32
+ from opencode_models.scoring import scales, derived_scores, canonical, _TRACKED
33
+ from opencode_models.availability import load_available, offered
34
+ from opencode_models import sources
35
+
36
+ models, *_ = _load(Path('data'))
37
+ table = scales(models)
38
+ # availability is the provider's own snapshot, not the committed slug lists:
39
+ # a model a plan has dropped is not worth a number
40
+ offered = {m.model_id for tier, ids in load_available(Path('data')).items()
41
+ if ids for m in sources.members(models, ids)[0]}
42
+ for m in models:
43
+ derived = derived_scores(m, table)
44
+ plan = next((t for t, ids in load_available(Path('data')).items()
45
+ if ids and offered(m.model_id, ids)), None)
46
+ if not plan or len(derived) != 1:
47
+ continue
48
+ has = canonical(next(iter(derived)))
49
+ missing = [b.replace('Terminal-Bench ', 'TB ').replace('SWE-Bench ', 'SWE ')
50
+ for b in _TRACKED if canonical(b) != has]
51
+ print(f'{plan:4} {m.model_id:34} has {has:20} missing {\", \".join(missing)}')
52
+ "
53
+ ```
54
+
55
+ **Skip old and superseded models.** A nano, a mini, a two-generations-back
56
+ release, and anything the provider is about to retire will not change a
57
+ recommendation. Chase the new frontier and the opencode-exclusive ones first;
58
+ those are where a second axis moves a row.
59
+
60
+ Coverage per benchmark, for a wider view — the `scores` command prints one
61
+ column per benchmark with model counts:
62
+
63
+ ```sh
64
+ uv run opencode-models --data data scores
65
+ ```
66
+
67
+ Filter that list down before spending time on it: an unreleased model or a
68
+ fine-tune nobody benchmarks will never turn up a number, and a 7B model is
69
+ not on Terminal-Bench at all. Prefer models that are **new, strong, and
70
+ opencode-exclusive** — those are the ones where a missing score actually
71
+ changes the ranking.
72
+
73
+ ## 2. Check what we already fetch first
74
+
75
+ `data/boards/` holds the official boards, already parsed. If the model is on
76
+ one of them, the score is already there and nothing needs writing:
77
+
78
+ ```sh
79
+ python3 -c "
80
+ import json
81
+ for name in ('tbench', 'swebench'):
82
+ for r in json.load(open(f'data/boards/{name}.json')):
83
+ if 'grok' in r['model_name'].lower(): print(r)
84
+ "
85
+ ```
86
+
87
+ Those two boards are fetched by `opencode-models fetch` and merged by tier. Do
88
+ not hand-copy a number that is already in there.
89
+
90
+ **Checked and empty for the thin models (2026-10).** Of the 26 Zen/Go models
91
+ sitting on exactly one axis, neither board holds a second scored axis for any
92
+ of them: `tbench.json` carries only Terminal-Bench 4.0 rows we already have,
93
+ and the one SWE-bench row that matches (Kimi K2.5) is Multilingual, which is
94
+ not scored on. Do not re-run this check hoping for a different answer — it
95
+ only changes when the boards themselves gain rows, and `fetch` reports that.
96
+
97
+ ## 3. Where to look, in order
98
+
99
+ ### Vendor pages, and what they actually publish
100
+
101
+ Vendor launch pages **are** scrapable — `curl -sL` with a real User-Agent
102
+ returns 200–300 KB of HTML — but for new frontier models they publish
103
+ Terminal-Bench 4.0 and little else. Checked on Anthropic's Haiku 5.5 and
104
+ Sonnet 5.5 pages and xAI's Grok 4.7 post: all three carry Terminal-Bench 4.0,
105
+ none carries SWE-bench or Terminal-Bench 2.1.
106
+
107
+ Normalise before concluding a page lacks a benchmark: vendor HTML writes
108
+ "SWE‑bench" with U+2011, "SWE-bench™" with a trademark, and non-breaking
109
+ spaces. A plain-space search misses all three and reports a false negative.
110
+
111
+ ```sh
112
+ python3 - <<'EOF'
113
+ import re, subprocess, unicodedata
114
+ html = subprocess.run(["curl", "-sL", "--max-time", "30",
115
+ "-H", "User-Agent: opencode-models/0.1", URL],
116
+ capture_output=True, text=True).stdout
117
+ for a, b in (("\u2011", "-"), ("\u00a0", " "), ("\u2122", ""), ("\u00ae", "")):
118
+ html = html.replace(a, b)
119
+ html = unicodedata.normalize("NFKC", html)
120
+ print(sorted({m.group(0) for m in
121
+ re.finditer(r"(terminal[\s-]*bench|swe[\s-]*bench)[^<>\n]{0,30}",
122
+ html, re.I)}))
123
+ EOF
124
+ ```
125
+
126
+ This is the structural finding behind the thin-model list: the axis vendors
127
+ publish is Terminal-Bench 4.0, and it is the one axis a model can be alone
128
+ on. A model announced today will usually have exactly one scored axis.
129
+
130
+ ### Hugging Face model cards
131
+
132
+ Public and reliable to read, and they carry the metrics a lab chose to
133
+ publish. Kimi K3's card has Terminal-Bench 2.1 (88.3) in an HTML results
134
+ table — the benchmark name is in one `<td>`, the numbers in the next ones, so
135
+ parse the table rather than searching for a "name: value" pattern. MiMo-V2.5's
136
+ card carries no scored axis at all. A 29-byte card means the repo name is
137
+ wrong (`Entry not found`), not that the model is unbenchmarked — search
138
+ `https://huggingface.co/api/models?search=<name>` for the real id.
139
+
140
+ ### Primary for opencode-exclusive and stealth models
141
+
142
+ **OpenCode's own Reddit and Twitter** — `r/opencode` and the OpenCode
143
+ accounts on X. These are where a model appears before it is in any
144
+ aggregator: a stealth launch, a "we just shipped X" post, a screenshot of a
145
+ leaderboard. They are also the weakest possible source: they are the
146
+ vendor's own claim. Cite them as tier 3 unless the post links an
147
+ independent run, and say so in the note.
148
+
149
+ When a model is announced with no evaluation at all, that is a finding, not
150
+ a blocker — write a file with no benchmarks? No. Leave it out and say the
151
+ model is unbenchmarked, exactly as LongCat-2.5-Preview was.
152
+
153
+ ### Scrapable, verified
154
+
155
+ | source | how | notes |
156
+ |---|---|---|
157
+ | **Hugging Face datasets-server** | `https://datasets-server.huggingface.co/rows?dataset=<org>/<name>&config=…&split=…&offset=0&length=100` | public JSON, no auth. Already used: `opencode-models fetch` reads arena Elo from `openaccess-ai-collective/chatbot-arena-elo-scores` (fields `elo_score`, `chatbot_name`) |
158
+ | **Terminal-Bench 4.0 board** | `https://www.tbench.ai/leaderboard/terminal-bench/4.0` | parsed by `fetch`; per row: accuracy ± CI95, agent, reasoning effort, date, rank |
159
+ | **SWE-bench** | `https://www.swebench.com/` | parsed by `fetch`; subsets Verified, Multilingual, Multimodal, Lite; `checked` entries only |
160
+ | **aider polyglot** | `https://aider.chat/docs/leaderboards/` | a plain `<tr>` table — but its newest entry is GPT-5-era (Aug 2025), so only use it for older models, never for ranking current ones |
161
+
162
+ ### Verified NOT scrapable — do not waste a pass
163
+
164
+ StepFun's model pages (JS shell, ~2KB served; numbers render in a browser),
165
+ Artificial Analysis eval pages (e.g. `/evaluations/terminalbench-4-0`):
166
+ machine-readable model registry with exact run slugs (which settles
167
+ "Opus 5 vs 5.5" questions), but chart scores are client-rendered - only
168
+ the top three are stated in the page prose. A screenshot transcription is
169
+ not a source: one transcribed bar collided with the prose, so nine adopted
170
+ cells were reverted. Per-model pages are 4MB app shells, no data API. Vals
171
+ AI, Scale's SWE-Atlas, DeepSWE, LiveBench, the Epoch AI benchmarking
172
+ dashboard, and the Open LLM Leaderboard HF space: all render their numbers in
173
+ a browser. **Check whether the data is embedded before writing a source off**
174
+ — tbench.ai looks exactly like one of these and is not, because a Next.js
175
+ page streams the data it renders from. Look for `self.__next_f`,
176
+ `<script type="application/json">`, or a `_next/data` route before giving up.
177
+
178
+ ## 4. When a source is protected: ask, precisely
179
+
180
+ When the only remaining source is one a scraper cannot reach, hand the user a
181
+ request they can answer in one sitting. Vague asks ("can you find benchmarks
182
+ for X?") come back with nothing. Give them, per model, **only the axes it
183
+ lacks** — asking for a number the catalog already holds wastes their time and
184
+ yours:
185
+
186
+ ```
187
+ Model: moonshotai/kimi-k3 (Kimi K3) — Zen
188
+ Has: Terminal-Bench 2.1 88.3
189
+ Looking for (one of these is enough to make it rankable):
190
+ - SWE-Bench Verified (% resolved), or SWE-Bench Pro
191
+ - Terminal-Bench 4.0 (pass@1)
192
+ What makes a number usable:
193
+ - which harness/agent ran it (Claude Code, Codex, mini-swe-agent, …)
194
+ - reasoning effort, if the board distinguishes it
195
+ - the date, and the CI or trial count if published
196
+ - the exact URL to the run, not a screenshot
197
+ Why it matters: one axis is not a score (scoring.MIN_AXES), so this model is
198
+ unranked and never recommended, however strong that one number is.
199
+ ```
200
+
201
+ A bare score with no harness is worth far less than the ranking suggests —
202
+ Terminal-Bench 2.x and 4.0 are different scales, and a run on a bespoke
203
+ scaffold is not comparable with a standardized board.
204
+
205
+ Batch the ask: one message listing the models, each with the single axis it
206
+ most needs, beats 26 separate ones. Prioritise by how strong the model
207
+ already looks on its one axis — a model at 88 on Terminal-Bench 2.1 that
208
+ cannot be ranked is the most expensive gap in the console.
209
+
210
+ ## 5. Write the file
211
+
212
+ `data/overrides/<lab>__<model>.json`, matching the upstream record filename.
213
+ Format and full rules are in `data/overrides/README.md`; the short version:
214
+
215
+ ```json
216
+ {
217
+ "model": {
218
+ "id": "stepfun/step-5-preview",
219
+ "note": "why this model needs an overlay, in one or two sentences",
220
+ "benchmarks": [
221
+ {
222
+ "name": "Terminal-Bench",
223
+ "version": "2.1",
224
+ "score": 85.0,
225
+ "metric": "pass@1",
226
+ "source": "https://…",
227
+ "note": "who ran it, when, on what scaffold, and any caveat"
228
+ }
229
+ ]
230
+ }
231
+ }
232
+ ```
233
+
234
+ Rules that are enforced, not advisory:
235
+
236
+ - **`source` is required.** The loader drops an entry without one.
237
+ - **`id` must name its lab** (`lab/model`), matching the catalog.
238
+ - **The key is (`name`, `version`)**, the same key the catalog uses — a
239
+ benchmark with no version lands on its own axis.
240
+ - **Do not add a model file for a model that has no upstream record.** The
241
+ entry would sit there matching nothing.
242
+ - Nothing ever writes here. `fetch` refuses to touch `data/overrides/`.
243
+
244
+ Source tiers, which decide whether your value applies:
245
+
246
+ | tier | kind | example |
247
+ |---|---|---|
248
+ | 1 | independent evaluator, one harness | artificialanalysis.ai |
249
+ | 2 | official board, scaffold recorded per row | tbench.ai, swebench.com |
250
+ | 3 | vendor post, card, aggregator | x.ai, opencode posts, benchmarklist.com |
251
+
252
+ A tier-1 override replaces a tier-3 upstream number. A tier-3 claim against
253
+ an independent upstream is **recorded but not applied** — it shows up in
254
+ `conflicts` and the upstream value stays. Choose the source honestly; the
255
+ tier decides the outcome anyway.
256
+
257
+ ## 6. Verify
258
+
259
+ ```sh
260
+ just test # guard tests read the real files
261
+ uv run opencode-models --data data conflicts # did it apply, or just get noted?
262
+ uv run opencode-models --data data list --benchmark "Terminal-Bench 2.1"
263
+ ```
264
+
265
+ What to look for:
266
+
267
+ 1. `just test` — the committed overrides are checked for a `source` on every
268
+ entry and a lab-qualified id. A file that fails here is not loadable.
269
+ 2. `conflicts` — `override wins` means your value is in the view;
270
+ `override noted, upstream kept` means your source was tiered *below* the
271
+ one already there, so nothing changed. That is the tier policy working,
272
+ not a bug.
273
+ 3. `list --benchmark "Terminal-Bench 2.1"` — the model should appear, with
274
+ `*` beside its name marking it curated.
275
+
276
+ If a model you expected to be picked does not appear, the most likely cause
277
+ is the tier: a vendor number cannot displace an independent one.
278
+
279
+ ## Etiquette
280
+
281
+ One sequential pass per source, real `User-Agent: opencode-models/0.1`, 30s
282
+ timeouts. No credentials, no telemetry, no scraping a source that asks not
283
+ to be scraped. A board that publishes a JSON API is meant to be read; a
284
+ social feed is meant to be read by a person.
@@ -0,0 +1,22 @@
1
+ ---
2
+ status: open
3
+ type: feature
4
+ priority: medium
5
+ area: backend
6
+ ---
7
+
8
+ ## Notes
9
+
10
+ Short description and context. Agents append progress notes below, newest first. Do not edit existing lines — append
11
+ only.
12
+
13
+ ## Status values
14
+
15
+ | status | meaning |
16
+ | ------------- | ------------------------------------------------------------- |
17
+ | `idea` | aspirational — not fully settled, not ready to start |
18
+ | `backlog` | accepted but deferred — consider when stepping back to plan |
19
+ | `open` | ready to be picked up |
20
+ | `in-progress` | actively being worked on |
21
+ | `blocked` | cannot proceed; note the blocker in the body |
22
+ | `done` | completed |
@@ -0,0 +1,22 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "description": "Schema for .plan/*.md YAML frontmatter. Filename (without .md) is the unique task key.",
4
+ "type": "object",
5
+ "required": ["status", "type", "priority"],
6
+ "properties": {
7
+ "status": {
8
+ "enum": ["idea", "backlog", "open", "in-progress", "blocked", "done"]
9
+ },
10
+ "type": {
11
+ "enum": ["bug", "feature", "chore", "decision", "perf"]
12
+ },
13
+ "priority": {
14
+ "enum": ["high", "medium", "low"]
15
+ },
16
+ "area": {
17
+ "description": "Optional grouping label for filtering (e.g. backend, frontend, devops, testing)",
18
+ "type": "string"
19
+ }
20
+ },
21
+ "additionalProperties": false
22
+ }
@@ -0,0 +1,67 @@
1
+ ---
2
+ status: in-progress
3
+ type: chore
4
+ priority: high
5
+ area: devops
6
+ ---
7
+
8
+ ## Notes
9
+
10
+ Public-release hardening, decided 2026-10-20. Decisions (same day, with
11
+ maintainer — they supersede the plan below where they differ): the
12
+ email stays everywhere (`codesam` name is fine too) — LICENSE,
13
+ commit identities, pyproject contact unchanged, no history rewrite.
14
+ README contact paths are GitHub issues + private security advisories
15
+ only. The gitleaks pre-commit hook needs its own toolchain
16
+ (golang/docker image) or a system binary — fails the local-and-cheap
17
+ test, so the local audit-grep hook stands in and CI keeps running real
18
+ gitleaks.
19
+
20
+ ## Steps
21
+
22
+ - [x] 1. ~~LICENSE identity + history rewrite.~~ DROPPED by decision:
23
+ email and `codesam` name stay as-is everywhere; no rewrite, no
24
+ force-push.
25
+ - [x] 2. README due-diligence paragraph on benchmark data (GitHub-only
26
+ contact paths). Added as "### Data due diligence" under ## Data.
27
+ - [x] 3. Client-side git hooks, no new installs (`.githooks/`, wired via
28
+ `just bootstrap` without clobbering an existing `core.hooksPath`):
29
+ `pre-push` refuses `refs/heads/local-data`; `pre-commit` runs the
30
+ secret-pattern sweep and `ruff check` + `ruff format --check` via the
31
+ project's own venv, with a graceful skip when ruff is unavailable.
32
+ Verified: pre-commit passes on a clean tree, fails on planted token
33
+ fixtures (a fake GitHub PAT and a fake AWS key); pre-push refuses a
34
+ `local-data` push and allows normal branches (scratch-repo end-to-end
35
+ run).
36
+
37
+ ## Open questions — all settled 2026-10-20 (see Notes above for answers)
38
+
39
+ - Q1: commit identities — settled: leave unchanged.
40
+ - Q2: `pyproject.toml` author contact — settled: leave unchanged.
41
+ - Q3: due-diligence paragraph contact path — settled: GitHub issues +
42
+ private security advisories.
43
+
44
+ ## Verification (revised — no rewrite, no force-push anywhere)
45
+
46
+ - `just lint`, `just test` green on a fresh checkout after `just bootstrap`.
47
+ - Hooks fire on trial runs: `pre-commit` blocks a planted secret-pattern
48
+ file and fails a planted lint violation; `pre-push` refuses a
49
+ `local-data` push and allows a normal branch push.
50
+ - `git config core.hooksPath` untouched when already set to something
51
+ else (bootstrap must not clobber an existing hooks setup).
52
+
53
+ ## Remaining scope (reopened 2026-10-20 — not started, needs direction)
54
+
55
+ - [ ] Release process: reserve the PyPI name with a first publish;
56
+ single-source the version (currently hardcoded in `pyproject.toml`
57
+ and as the `opencode-rank/0.1` User-Agent); add CHANGELOG.md;
58
+ tag-triggered publish workflow.
59
+ - [ ] Repo hygiene: CONTRIBUTING.md only — decided against issue/PR
60
+ templates and against a CODE_OF_CONDUCT (small single-maintainer tool;
61
+ GitHub issues + private security advisories already cover reporting).
62
+ - [ ] GitHub push protection + secret scanning at public-launch time
63
+ (free for public repos; the local grep and CI gitleaks stay as the
64
+ pre-launch layers).
65
+ - [ ] Source-site Terms of Service review for republished benchmark
66
+ numbers — deferred counsel work; the README due-diligence note is the
67
+ interim public stance.
@@ -0,0 +1,22 @@
1
+ ---
2
+ status: in-progress
3
+ type: feature
4
+ priority: high
5
+ area: cli
6
+ ---
7
+
8
+ ## Notes
9
+
10
+ `fetch` dies with a traceback when the catalog sitemap is unreachable, and
11
+ `list` / `next` / `models` silently use arbitrarily old snapshots. Two
12
+ fixes, both in console output behavior, no ranking changes:
13
+
14
+ 1. Catalog fetch failure degrades to a warning (`keeping existing records`),
15
+ same shape as the existing per-source warnings. Exit code stays 0.
16
+ 2. `list`, `next`, and the `models` snapshot path warn per tier when a
17
+ `tier_prices/<tier>.json` snapshot is older than 7 days
18
+ (`STALE_AFTER_DAYS`), by file mtime. Missing files stay silent (unknown
19
+ is already reported as unknown).
20
+
21
+ Out of scope: exit-code changes, auto-refresh, freshness of `data/available`
22
+ (keyed runs re-fetch it live), docs beyond one README line.