opencode-models 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencode_models-0.1.0/.githooks/pre-commit +24 -0
- opencode_models-0.1.0/.githooks/pre-push +17 -0
- opencode_models-0.1.0/.github/dependabot.yml +24 -0
- opencode_models-0.1.0/.github/workflows/ci.yml +88 -0
- opencode_models-0.1.0/.github/workflows/release.yml +34 -0
- opencode_models-0.1.0/.github/workflows/secret-scan.yml +36 -0
- opencode_models-0.1.0/.gitignore +21 -0
- opencode_models-0.1.0/.opencode/skills/backfill-scores/SKILL.md +284 -0
- opencode_models-0.1.0/.plan/.TEMPLATE.md +22 -0
- opencode_models-0.1.0/.plan/.schema.json +22 -0
- opencode_models-0.1.0/.plan/public-release.md +67 -0
- opencode_models-0.1.0/.plan/robustness.md +22 -0
- opencode_models-0.1.0/.plan/v01-review.md +84 -0
- opencode_models-0.1.0/AGENTS.md +107 -0
- opencode_models-0.1.0/CHANGELOG.md +13 -0
- opencode_models-0.1.0/CONTRIBUTING.md +65 -0
- opencode_models-0.1.0/LICENSE +21 -0
- opencode_models-0.1.0/LIMITS.md +288 -0
- opencode_models-0.1.0/PKG-INFO +544 -0
- opencode_models-0.1.0/README.md +504 -0
- opencode_models-0.1.0/SECURITY.md +49 -0
- opencode_models-0.1.0/data/overrides/README.md +81 -0
- opencode_models-0.1.0/data/overrides/alibaba__qwen3-8-flash-next.json +1 -0
- opencode_models-0.1.0/data/overrides/alibaba__qwen3-8-max-0902.json +1 -0
- opencode_models-0.1.0/data/overrides/anthropic__claude-fable-5-1.json +1 -0
- opencode_models-0.1.0/data/overrides/anthropic__claude-opus-5.json +1 -0
- opencode_models-0.1.0/data/overrides/deepseek__deepseek-v4-flash.json +1 -0
- opencode_models-0.1.0/data/overrides/inclusionai__ling-3-0-flash-fin.json +1 -0
- opencode_models-0.1.0/data/overrides/moonshotai__kimi-k3.json +1 -0
- opencode_models-0.1.0/data/overrides/openai__gpt-6-astra.json +1 -0
- opencode_models-0.1.0/data/overrides/stepfun__step-5-preview.json +1 -0
- opencode_models-0.1.0/justfile +37 -0
- opencode_models-0.1.0/opencode_models/__init__.py +8 -0
- opencode_models-0.1.0/opencode_models/__main__.py +5 -0
- opencode_models-0.1.0/opencode_models/arena.py +108 -0
- opencode_models-0.1.0/opencode_models/auth.py +48 -0
- opencode_models-0.1.0/opencode_models/availability.py +102 -0
- opencode_models-0.1.0/opencode_models/benchmarks.py +19 -0
- opencode_models-0.1.0/opencode_models/boards.py +518 -0
- opencode_models-0.1.0/opencode_models/cli.py +1172 -0
- opencode_models-0.1.0/opencode_models/data.py +82 -0
- opencode_models-0.1.0/opencode_models/fetch.py +101 -0
- opencode_models-0.1.0/opencode_models/identity.py +100 -0
- opencode_models-0.1.0/opencode_models/labelmap.py +84 -0
- opencode_models-0.1.0/opencode_models/live.py +88 -0
- opencode_models-0.1.0/opencode_models/overrides.py +252 -0
- opencode_models-0.1.0/opencode_models/pareto.py +34 -0
- opencode_models-0.1.0/opencode_models/pricing.py +415 -0
- opencode_models-0.1.0/opencode_models/ranking.py +591 -0
- opencode_models-0.1.0/opencode_models/scoring.py +230 -0
- opencode_models-0.1.0/opencode_models/seed.py +184 -0
- opencode_models-0.1.0/opencode_models/sources/go.txt +45 -0
- opencode_models-0.1.0/opencode_models/sources/zen.txt +87 -0
- opencode_models-0.1.0/opencode_models/sources.py +55 -0
- opencode_models-0.1.0/opencode_models/tables.py +94 -0
- opencode_models-0.1.0/pyproject.toml +64 -0
- opencode_models-0.1.0/tests/__init__.py +0 -0
- opencode_models-0.1.0/tests/test_arena.py +131 -0
- opencode_models-0.1.0/tests/test_auth.py +68 -0
- opencode_models-0.1.0/tests/test_availability.py +167 -0
- opencode_models-0.1.0/tests/test_benchmarks.py +55 -0
- opencode_models-0.1.0/tests/test_boards.py +731 -0
- opencode_models-0.1.0/tests/test_cli.py +3628 -0
- opencode_models-0.1.0/tests/test_data.py +131 -0
- opencode_models-0.1.0/tests/test_fetch.py +245 -0
- opencode_models-0.1.0/tests/test_identity.py +123 -0
- opencode_models-0.1.0/tests/test_live.py +49 -0
- opencode_models-0.1.0/tests/test_next.py +366 -0
- opencode_models-0.1.0/tests/test_overrides.py +493 -0
- opencode_models-0.1.0/tests/test_pareto.py +73 -0
- opencode_models-0.1.0/tests/test_pricing.py +362 -0
- opencode_models-0.1.0/tests/test_ranking.py +770 -0
- opencode_models-0.1.0/tests/test_scoring.py +467 -0
- opencode_models-0.1.0/tests/test_seed.py +201 -0
- opencode_models-0.1.0/tests/test_sources.py +103 -0
- opencode_models-0.1.0/tests/test_tables.py +92 -0
- opencode_models-0.1.0/tests/test_views.py +495 -0
- opencode_models-0.1.0/uv.lock +43 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Fast local gates, no new installs: a secret-pattern sweep over the tree
|
|
3
|
+
# plus ruff lint and format check from the project's own venv.
|
|
4
|
+
# Fails the commit on findings. (CI runs the real gitleaks scanner on
|
|
5
|
+
# every push and PR; this hook is the cheap local tripwire.)
|
|
6
|
+
|
|
7
|
+
# 1. Tracked secret patterns (same sweep as `just audit`, but failing).
|
|
8
|
+
# The pattern is split with '' so this file never matches itself.
|
|
9
|
+
if grep -rniE 'BEGIN (RSA |OPENSSH |EC )?PRIVATE ''KEY|aws_secret_''access_key|ghp_''[A-Za-z0-9]{36}|github_pat_''[A-Za-z0-9]{22}' --exclude-dir=.venv --exclude-dir=.git --exclude-dir=__pycache__ . ; then
|
|
10
|
+
echo "pre-commit: secret-looking content above — remove it before committing" >&2
|
|
11
|
+
exit 1
|
|
12
|
+
fi
|
|
13
|
+
|
|
14
|
+
# 2. Lint + format via the project's own venv (installed by `just bootstrap`).
|
|
15
|
+
# Missing venv is not a failure: warn and let CI be the backstop.
|
|
16
|
+
if command -v uv >/dev/null 2>&1 && uv run --no-sync ruff --version >/dev/null 2>&1; then
|
|
17
|
+
uv run --no-sync ruff check opencode_models tests || exit 1
|
|
18
|
+
uv run --no-sync ruff format --check opencode_models tests || {
|
|
19
|
+
echo "pre-commit: run 'just lint' (or 'uv run ruff format opencode_models tests') and re-stage" >&2
|
|
20
|
+
exit 1
|
|
21
|
+
}
|
|
22
|
+
else
|
|
23
|
+
echo "pre-commit: ruff not available (run 'just bootstrap' once) — lint skipped, CI still checks" >&2
|
|
24
|
+
fi
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Block pushing the local-data branch: it holds the full fetched dataset
|
|
3
|
+
# (hundreds of third-party records plus workspace-derived snapshots) and
|
|
4
|
+
# must never leave this machine. The branch is intentionally never pushed;
|
|
5
|
+
# this hook makes that rule mechanical instead of remembered.
|
|
6
|
+
#
|
|
7
|
+
# Invoked by git with lines "<local ref> <local sha> <remote ref>
|
|
8
|
+
# <remote sha>" on stdin; exit non-zero rejects the push.
|
|
9
|
+
|
|
10
|
+
while read local_ref _local_sha _remote_ref _remote_sha; do
|
|
11
|
+
case "$local_ref" in
|
|
12
|
+
refs/heads/local-data)
|
|
13
|
+
echo "refusing to push $local_ref: the fetched dataset must stay local (see README, Data)" >&2
|
|
14
|
+
exit 1
|
|
15
|
+
;;
|
|
16
|
+
esac
|
|
17
|
+
done
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
|
|
3
|
+
updates:
|
|
4
|
+
# The package itself is stdlib-only; the Actions pinned in
|
|
5
|
+
# .github/workflows are what drifts. All minor/patch action updates
|
|
6
|
+
# land in one grouped weekly PR instead of one PR each; majors stay
|
|
7
|
+
# individual so a breaking action change gets its own review.
|
|
8
|
+
- package-ecosystem: github-actions
|
|
9
|
+
directory: /
|
|
10
|
+
schedule:
|
|
11
|
+
interval: weekly
|
|
12
|
+
# delay version updates N days after release: avoids yanked-release
|
|
13
|
+
# haste, and satisfies the zizmor dependabot-cooldown audit
|
|
14
|
+
cooldown:
|
|
15
|
+
default-days: 7
|
|
16
|
+
open-pull-requests-limit: 3
|
|
17
|
+
groups:
|
|
18
|
+
github-actions:
|
|
19
|
+
applies-to: version-updates
|
|
20
|
+
patterns:
|
|
21
|
+
- "*"
|
|
22
|
+
update-types:
|
|
23
|
+
- minor
|
|
24
|
+
- patch
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
# Least privilege: jobs get read-only repo access by default.
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
# One run per ref: a new push cancels the older in-progress run.
|
|
13
|
+
concurrency:
|
|
14
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
15
|
+
cancel-in-progress: true
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
test:
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
timeout-minutes: 10
|
|
21
|
+
strategy:
|
|
22
|
+
fail-fast: false
|
|
23
|
+
matrix:
|
|
24
|
+
python-version: ["3.10", "3.12", "3.14"]
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
27
|
+
with:
|
|
28
|
+
persist-credentials: false
|
|
29
|
+
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
|
|
30
|
+
with:
|
|
31
|
+
enable-cache: true
|
|
32
|
+
- name: sync (with the pinned lockfile)
|
|
33
|
+
run: uv sync --python ${{ matrix.python-version }}
|
|
34
|
+
- name: unit tests
|
|
35
|
+
run: uv run --no-sync python -m unittest discover -s tests -v
|
|
36
|
+
- name: the console resolves and runs from the entry point
|
|
37
|
+
# no scraped dataset in CI: it is fetched on demand (see README)
|
|
38
|
+
run: uv run --no-sync opencode-models --help
|
|
39
|
+
|
|
40
|
+
build:
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
timeout-minutes: 10
|
|
43
|
+
steps:
|
|
44
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
45
|
+
with:
|
|
46
|
+
persist-credentials: false
|
|
47
|
+
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
|
|
48
|
+
- name: wheel + sdist build
|
|
49
|
+
run: uv build
|
|
50
|
+
- name: install the wheel into a throwaway tool env
|
|
51
|
+
run: |
|
|
52
|
+
uv tool install --force dist/*.whl
|
|
53
|
+
opencode-models --help >/dev/null
|
|
54
|
+
|
|
55
|
+
lint:
|
|
56
|
+
runs-on: ubuntu-latest
|
|
57
|
+
timeout-minutes: 10
|
|
58
|
+
steps:
|
|
59
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
60
|
+
with:
|
|
61
|
+
persist-credentials: false
|
|
62
|
+
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
|
|
63
|
+
- name: sync (with the pinned lockfile, dev tools included)
|
|
64
|
+
run: uv sync --python "3.12"
|
|
65
|
+
# same commands as `just lint`; ruff itself comes from the locked
|
|
66
|
+
# dev group, never from the network at CI time beyond the lockfile
|
|
67
|
+
- name: lint, format check and complexity gate
|
|
68
|
+
run: |
|
|
69
|
+
uv run --no-sync ruff check opencode_models tests
|
|
70
|
+
uv run --no-sync ruff format --check opencode_models tests
|
|
71
|
+
|
|
72
|
+
zizmor:
|
|
73
|
+
runs-on: ubuntu-latest
|
|
74
|
+
timeout-minutes: 10
|
|
75
|
+
steps:
|
|
76
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
77
|
+
with:
|
|
78
|
+
persist-credentials: false
|
|
79
|
+
# No SARIF upload: code scanning is not enabled for this repo, and
|
|
80
|
+
# the upload is what kept failing — not the audit. The audit still
|
|
81
|
+
# runs and fails this step on findings. If code scanning gets
|
|
82
|
+
# enabled, drop this flag and add a job-level `permissions` block
|
|
83
|
+
# with `security-events: write` (upload) plus `actions: read`
|
|
84
|
+
# (run association).
|
|
85
|
+
- name: audit workflows with zizmor
|
|
86
|
+
uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1
|
|
87
|
+
with:
|
|
88
|
+
advanced-security: false
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
# Least privilege: checkout needs repo read access; publishing uses
|
|
9
|
+
# a short-lived OIDC token instead of a stored PyPI secret.
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
publish:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
timeout-minutes: 10
|
|
17
|
+
permissions:
|
|
18
|
+
contents: read
|
|
19
|
+
id-token: write # trusted publishing (PyPI must list this repo as a trusted publisher first)
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
22
|
+
with:
|
|
23
|
+
persist-credentials: false
|
|
24
|
+
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
|
|
25
|
+
with:
|
|
26
|
+
# no cache: releases are rare enough that a cold sync is cheap,
|
|
27
|
+
# and it removes the cache-poisoning surface entirely
|
|
28
|
+
enable-cache: false
|
|
29
|
+
- name: check the tag matches the package version
|
|
30
|
+
run: test "${GITHUB_REF#refs/tags/v}" = "$(grep -m1 '^version *=' pyproject.toml | cut -d'"' -f2)"
|
|
31
|
+
- name: wheel + sdist build
|
|
32
|
+
run: uv build
|
|
33
|
+
- name: publish to PyPI
|
|
34
|
+
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: secret-scan
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
schedule:
|
|
8
|
+
- cron: "0 6 * * 1"
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
concurrency:
|
|
14
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
15
|
+
cancel-in-progress: true
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
gitleaks:
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
timeout-minutes: 10
|
|
21
|
+
# contents: read comes from the top level; pull-requests: read lets
|
|
22
|
+
# the action resolve the PR range it scans on pull_request events.
|
|
23
|
+
permissions:
|
|
24
|
+
contents: read
|
|
25
|
+
pull-requests: read
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
28
|
+
with:
|
|
29
|
+
fetch-depth: 0
|
|
30
|
+
persist-credentials: false
|
|
31
|
+
- name: scan history for leaked secrets
|
|
32
|
+
uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2
|
|
33
|
+
# v2 requires the token explicitly on pull_request events, where it
|
|
34
|
+
# needs the API to determine the scan range
|
|
35
|
+
env:
|
|
36
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.pyc
|
|
3
|
+
*.egg-info/
|
|
4
|
+
.venv/
|
|
5
|
+
dist/
|
|
6
|
+
# No fetched page or payload is committed. This `*.html` rule also caught the
|
|
7
|
+
# test fixtures once, silently: nothing under tests/ is HTML any more (the
|
|
8
|
+
# board shapes are written out in tests/test_boards.py), and a `git add -A`
|
|
9
|
+
# that skips a file is easy to miss until CI fails on a clean checkout.
|
|
10
|
+
*.html
|
|
11
|
+
!README.md
|
|
12
|
+
|
|
13
|
+
# The scraped/fetched dataset is deliberately not in this branch's history:
|
|
14
|
+
# `opencode-models fetch` populates it, and the `local-data` branch keeps a
|
|
15
|
+
# committed copy. The curated overlays in data/overrides/ ARE tracked.
|
|
16
|
+
data/opencode/
|
|
17
|
+
data/tier_prices/
|
|
18
|
+
data/aiarena/
|
|
19
|
+
data/boards/
|
|
20
|
+
data/usage/
|
|
21
|
+
data/available/
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: backfill-scores
|
|
3
|
+
description: Use when an opencode-models model has only one scored benchmark and is therefore unranked, or when asked to find, research or add benchmark numbers — covers where to look, which sources are actually scrapable, how to write a data/overrides JSON file, and how to verify it landed.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Backfilling missing benchmark scores
|
|
7
|
+
|
|
8
|
+
A model needs two of the four scored benchmarks to carry a score at all
|
|
9
|
+
(`scoring.MIN_AXES`), so the models worth chasing are the ones sitting on
|
|
10
|
+
one. This is how to find them, source a number, write it down, and prove it
|
|
11
|
+
landed.
|
|
12
|
+
|
|
13
|
+
The rule everything else serves: **a number without a source is not data.**
|
|
14
|
+
The loader drops any override entry that has no `source`, so a file without
|
|
15
|
+
one is inert — it will not appear, fail, or warn.
|
|
16
|
+
|
|
17
|
+
## 1. Find what is missing
|
|
18
|
+
|
|
19
|
+
The target list is models that are **listed in a Zen or Go plan** and carry
|
|
20
|
+
**one** scored benchmark. Both filters matter:
|
|
21
|
+
|
|
22
|
+
- A model no plan lists is a model about to be unlisted. A number for it
|
|
23
|
+
changes nothing, because nothing recommends it.
|
|
24
|
+
- A model with no scored benchmark at all is a different problem (nothing to
|
|
25
|
+
pair with, and usually nothing published). A model with *one* is a missing
|
|
26
|
+
pair away from being ranked.
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
PYTHONPATH=. uv run python -c "
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from opencode_models.cli import _load
|
|
32
|
+
from opencode_models.scoring import scales, derived_scores, canonical, _TRACKED
|
|
33
|
+
from opencode_models.availability import load_available, offered
|
|
34
|
+
from opencode_models import sources
|
|
35
|
+
|
|
36
|
+
models, *_ = _load(Path('data'))
|
|
37
|
+
table = scales(models)
|
|
38
|
+
# availability is the provider's own snapshot, not the committed slug lists:
|
|
39
|
+
# a model a plan has dropped is not worth a number
|
|
40
|
+
offered = {m.model_id for tier, ids in load_available(Path('data')).items()
|
|
41
|
+
if ids for m in sources.members(models, ids)[0]}
|
|
42
|
+
for m in models:
|
|
43
|
+
derived = derived_scores(m, table)
|
|
44
|
+
plan = next((t for t, ids in load_available(Path('data')).items()
|
|
45
|
+
if ids and offered(m.model_id, ids)), None)
|
|
46
|
+
if not plan or len(derived) != 1:
|
|
47
|
+
continue
|
|
48
|
+
has = canonical(next(iter(derived)))
|
|
49
|
+
missing = [b.replace('Terminal-Bench ', 'TB ').replace('SWE-Bench ', 'SWE ')
|
|
50
|
+
for b in _TRACKED if canonical(b) != has]
|
|
51
|
+
print(f'{plan:4} {m.model_id:34} has {has:20} missing {\", \".join(missing)}')
|
|
52
|
+
"
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
**Skip old and superseded models.** A nano, a mini, a two-generations-back
|
|
56
|
+
release, and anything the provider is about to retire will not change a
|
|
57
|
+
recommendation. Chase the new frontier and the opencode-exclusive ones first;
|
|
58
|
+
those are where a second axis moves a row.
|
|
59
|
+
|
|
60
|
+
Coverage per benchmark, for a wider view — the `scores` command prints one
|
|
61
|
+
column per benchmark with model counts:
|
|
62
|
+
|
|
63
|
+
```sh
|
|
64
|
+
uv run opencode-models --data data scores
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Filter that list down before spending time on it: an unreleased model or a
|
|
68
|
+
fine-tune nobody benchmarks will never turn up a number, and a 7B model is
|
|
69
|
+
not on Terminal-Bench at all. Prefer models that are **new, strong, and
|
|
70
|
+
opencode-exclusive** — those are the ones where a missing score actually
|
|
71
|
+
changes the ranking.
|
|
72
|
+
|
|
73
|
+
## 2. Check what we already fetch first
|
|
74
|
+
|
|
75
|
+
`data/boards/` holds the official boards, already parsed. If the model is on
|
|
76
|
+
one of them, the score is already there and nothing needs writing:
|
|
77
|
+
|
|
78
|
+
```sh
|
|
79
|
+
python3 -c "
|
|
80
|
+
import json
|
|
81
|
+
for name in ('tbench', 'swebench'):
|
|
82
|
+
for r in json.load(open(f'data/boards/{name}.json')):
|
|
83
|
+
if 'grok' in r['model_name'].lower(): print(r)
|
|
84
|
+
"
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Those two boards are fetched by `opencode-models fetch` and merged by tier. Do
|
|
88
|
+
not hand-copy a number that is already in there.
|
|
89
|
+
|
|
90
|
+
**Checked and empty for the thin models (2026-10).** Of the 26 Zen/Go models
|
|
91
|
+
sitting on exactly one axis, neither board holds a second scored axis for any
|
|
92
|
+
of them: `tbench.json` carries only Terminal-Bench 4.0 rows we already have,
|
|
93
|
+
and the one SWE-bench row that matches (Kimi K2.5) is Multilingual, which is
|
|
94
|
+
not scored on. Do not re-run this check hoping for a different answer — it
|
|
95
|
+
only changes when the boards themselves gain rows, and `fetch` reports that.
|
|
96
|
+
|
|
97
|
+
## 3. Where to look, in order
|
|
98
|
+
|
|
99
|
+
### Vendor pages, and what they actually publish
|
|
100
|
+
|
|
101
|
+
Vendor launch pages **are** scrapable — `curl -sL` with a real User-Agent
|
|
102
|
+
returns 200–300 KB of HTML — but for new frontier models they publish
|
|
103
|
+
Terminal-Bench 4.0 and little else. Checked on Anthropic's Haiku 5.5 and
|
|
104
|
+
Sonnet 5.5 pages and xAI's Grok 4.7 post: all three carry Terminal-Bench 4.0,
|
|
105
|
+
none carries SWE-bench or Terminal-Bench 2.1.
|
|
106
|
+
|
|
107
|
+
Normalise before concluding a page lacks a benchmark: vendor HTML writes
|
|
108
|
+
"SWE‑bench" with U+2011, "SWE-bench™" with a trademark, and non-breaking
|
|
109
|
+
spaces. A plain-space search misses all three and reports a false negative.
|
|
110
|
+
|
|
111
|
+
```sh
|
|
112
|
+
python3 - <<'EOF'
|
|
113
|
+
import re, subprocess, unicodedata
|
|
114
|
+
html = subprocess.run(["curl", "-sL", "--max-time", "30",
|
|
115
|
+
"-H", "User-Agent: opencode-models/0.1", URL],
|
|
116
|
+
capture_output=True, text=True).stdout
|
|
117
|
+
for a, b in (("\u2011", "-"), ("\u00a0", " "), ("\u2122", ""), ("\u00ae", "")):
|
|
118
|
+
html = html.replace(a, b)
|
|
119
|
+
html = unicodedata.normalize("NFKC", html)
|
|
120
|
+
print(sorted({m.group(0) for m in
|
|
121
|
+
re.finditer(r"(terminal[\s-]*bench|swe[\s-]*bench)[^<>\n]{0,30}",
|
|
122
|
+
html, re.I)}))
|
|
123
|
+
EOF
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
This is the structural finding behind the thin-model list: the axis vendors
|
|
127
|
+
publish is Terminal-Bench 4.0, and it is the one axis a model can be alone
|
|
128
|
+
on. A model announced today will usually have exactly one scored axis.
|
|
129
|
+
|
|
130
|
+
### Hugging Face model cards
|
|
131
|
+
|
|
132
|
+
Public and reliable to read, and they carry the metrics a lab chose to
|
|
133
|
+
publish. Kimi K3's card has Terminal-Bench 2.1 (88.3) in an HTML results
|
|
134
|
+
table — the benchmark name is in one `<td>`, the numbers in the next ones, so
|
|
135
|
+
parse the table rather than searching for a "name: value" pattern. MiMo-V2.5's
|
|
136
|
+
card carries no scored axis at all. A 29-byte card means the repo name is
|
|
137
|
+
wrong (`Entry not found`), not that the model is unbenchmarked — search
|
|
138
|
+
`https://huggingface.co/api/models?search=<name>` for the real id.
|
|
139
|
+
|
|
140
|
+
### Primary for opencode-exclusive and stealth models
|
|
141
|
+
|
|
142
|
+
**OpenCode's own Reddit and Twitter** — `r/opencode` and the OpenCode
|
|
143
|
+
accounts on X. These are where a model appears before it is in any
|
|
144
|
+
aggregator: a stealth launch, a "we just shipped X" post, a screenshot of a
|
|
145
|
+
leaderboard. They are also the weakest possible source: they are the
|
|
146
|
+
vendor's own claim. Cite them as tier 3 unless the post links an
|
|
147
|
+
independent run, and say so in the note.
|
|
148
|
+
|
|
149
|
+
When a model is announced with no evaluation at all, that is a finding, not
|
|
150
|
+
a blocker — write a file with no benchmarks? No. Leave it out and say the
|
|
151
|
+
model is unbenchmarked, exactly as LongCat-2.5-Preview was.
|
|
152
|
+
|
|
153
|
+
### Scrapable, verified
|
|
154
|
+
|
|
155
|
+
| source | how | notes |
|
|
156
|
+
|---|---|---|
|
|
157
|
+
| **Hugging Face datasets-server** | `https://datasets-server.huggingface.co/rows?dataset=<org>/<name>&config=…&split=…&offset=0&length=100` | public JSON, no auth. Already used: `opencode-models fetch` reads arena Elo from `openaccess-ai-collective/chatbot-arena-elo-scores` (fields `elo_score`, `chatbot_name`) |
|
|
158
|
+
| **Terminal-Bench 4.0 board** | `https://www.tbench.ai/leaderboard/terminal-bench/4.0` | parsed by `fetch`; per row: accuracy ± CI95, agent, reasoning effort, date, rank |
|
|
159
|
+
| **SWE-bench** | `https://www.swebench.com/` | parsed by `fetch`; subsets Verified, Multilingual, Multimodal, Lite; `checked` entries only |
|
|
160
|
+
| **aider polyglot** | `https://aider.chat/docs/leaderboards/` | a plain `<tr>` table — but its newest entry is GPT-5-era (Aug 2025), so only use it for older models, never for ranking current ones |
|
|
161
|
+
|
|
162
|
+
### Verified NOT scrapable — do not waste a pass
|
|
163
|
+
|
|
164
|
+
StepFun's model pages (JS shell, ~2KB served; numbers render in a browser),
|
|
165
|
+
Artificial Analysis eval pages (e.g. `/evaluations/terminalbench-4-0`):
|
|
166
|
+
machine-readable model registry with exact run slugs (which settles
|
|
167
|
+
"Opus 5 vs 5.5" questions), but chart scores are client-rendered - only
|
|
168
|
+
the top three are stated in the page prose. A screenshot transcription is
|
|
169
|
+
not a source: one transcribed bar collided with the prose, so nine adopted
|
|
170
|
+
cells were reverted. Per-model pages are 4MB app shells, no data API. Vals
|
|
171
|
+
AI, Scale's SWE-Atlas, DeepSWE, LiveBench, the Epoch AI benchmarking
|
|
172
|
+
dashboard, and the Open LLM Leaderboard HF space: all render their numbers in
|
|
173
|
+
a browser. **Check whether the data is embedded before writing a source off**
|
|
174
|
+
— tbench.ai looks exactly like one of these and is not, because a Next.js
|
|
175
|
+
page streams the data it renders from. Look for `self.__next_f`,
|
|
176
|
+
`<script type="application/json">`, or a `_next/data` route before giving up.
|
|
177
|
+
|
|
178
|
+
## 4. When a source is protected: ask, precisely
|
|
179
|
+
|
|
180
|
+
When the only remaining source is one a scraper cannot reach, hand the user a
|
|
181
|
+
request they can answer in one sitting. Vague asks ("can you find benchmarks
|
|
182
|
+
for X?") come back with nothing. Give them, per model, **only the axes it
|
|
183
|
+
lacks** — asking for a number the catalog already holds wastes their time and
|
|
184
|
+
yours:
|
|
185
|
+
|
|
186
|
+
```
|
|
187
|
+
Model: moonshotai/kimi-k3 (Kimi K3) — Zen
|
|
188
|
+
Has: Terminal-Bench 2.1 88.3
|
|
189
|
+
Looking for (one of these is enough to make it rankable):
|
|
190
|
+
- SWE-Bench Verified (% resolved), or SWE-Bench Pro
|
|
191
|
+
- Terminal-Bench 4.0 (pass@1)
|
|
192
|
+
What makes a number usable:
|
|
193
|
+
- which harness/agent ran it (Claude Code, Codex, mini-swe-agent, …)
|
|
194
|
+
- reasoning effort, if the board distinguishes it
|
|
195
|
+
- the date, and the CI or trial count if published
|
|
196
|
+
- the exact URL to the run, not a screenshot
|
|
197
|
+
Why it matters: one axis is not a score (scoring.MIN_AXES), so this model is
|
|
198
|
+
unranked and never recommended, however strong that one number is.
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
A bare score with no harness is worth far less than the ranking suggests —
|
|
202
|
+
Terminal-Bench 2.x and 4.0 are different scales, and a run on a bespoke
|
|
203
|
+
scaffold is not comparable with a standardized board.
|
|
204
|
+
|
|
205
|
+
Batch the ask: one message listing the models, each with the single axis it
|
|
206
|
+
most needs, beats 26 separate ones. Prioritise by how strong the model
|
|
207
|
+
already looks on its one axis — a model at 88 on Terminal-Bench 2.1 that
|
|
208
|
+
cannot be ranked is the most expensive gap in the console.
|
|
209
|
+
|
|
210
|
+
## 5. Write the file
|
|
211
|
+
|
|
212
|
+
`data/overrides/<lab>__<model>.json`, matching the upstream record filename.
|
|
213
|
+
Format and full rules are in `data/overrides/README.md`; the short version:
|
|
214
|
+
|
|
215
|
+
```json
|
|
216
|
+
{
|
|
217
|
+
"model": {
|
|
218
|
+
"id": "stepfun/step-5-preview",
|
|
219
|
+
"note": "why this model needs an overlay, in one or two sentences",
|
|
220
|
+
"benchmarks": [
|
|
221
|
+
{
|
|
222
|
+
"name": "Terminal-Bench",
|
|
223
|
+
"version": "2.1",
|
|
224
|
+
"score": 85.0,
|
|
225
|
+
"metric": "pass@1",
|
|
226
|
+
"source": "https://…",
|
|
227
|
+
"note": "who ran it, when, on what scaffold, and any caveat"
|
|
228
|
+
}
|
|
229
|
+
]
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
Rules that are enforced, not advisory:
|
|
235
|
+
|
|
236
|
+
- **`source` is required.** The loader drops an entry without one.
|
|
237
|
+
- **`id` must name its lab** (`lab/model`), matching the catalog.
|
|
238
|
+
- **The key is (`name`, `version`)**, the same key the catalog uses — a
|
|
239
|
+
benchmark with no version lands on its own axis.
|
|
240
|
+
- **Do not add a model file for a model that has no upstream record.** The
|
|
241
|
+
entry would sit there matching nothing.
|
|
242
|
+
- Nothing ever writes here. `fetch` refuses to touch `data/overrides/`.
|
|
243
|
+
|
|
244
|
+
Source tiers, which decide whether your value applies:
|
|
245
|
+
|
|
246
|
+
| tier | kind | example |
|
|
247
|
+
|---|---|---|
|
|
248
|
+
| 1 | independent evaluator, one harness | artificialanalysis.ai |
|
|
249
|
+
| 2 | official board, scaffold recorded per row | tbench.ai, swebench.com |
|
|
250
|
+
| 3 | vendor post, card, aggregator | x.ai, opencode posts, benchmarklist.com |
|
|
251
|
+
|
|
252
|
+
A tier-1 override replaces a tier-3 upstream number. A tier-3 claim against
|
|
253
|
+
an independent upstream is **recorded but not applied** — it shows up in
|
|
254
|
+
`conflicts` and the upstream value stays. Choose the source honestly; the
|
|
255
|
+
tier decides the outcome anyway.
|
|
256
|
+
|
|
257
|
+
## 6. Verify
|
|
258
|
+
|
|
259
|
+
```sh
|
|
260
|
+
just test # guard tests read the real files
|
|
261
|
+
uv run opencode-models --data data conflicts # did it apply, or just get noted?
|
|
262
|
+
uv run opencode-models --data data list --benchmark "Terminal-Bench 2.1"
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
What to look for:
|
|
266
|
+
|
|
267
|
+
1. `just test` — the committed overrides are checked for a `source` on every
|
|
268
|
+
entry and a lab-qualified id. A file that fails here is not loadable.
|
|
269
|
+
2. `conflicts` — `override wins` means your value is in the view;
|
|
270
|
+
`override noted, upstream kept` means your source was tiered *below* the
|
|
271
|
+
one already there, so nothing changed. That is the tier policy working,
|
|
272
|
+
not a bug.
|
|
273
|
+
3. `list --benchmark "Terminal-Bench 2.1"` — the model should appear, with
|
|
274
|
+
`*` beside its name marking it curated.
|
|
275
|
+
|
|
276
|
+
If a model you expected to be picked does not appear, the most likely cause
|
|
277
|
+
is the tier: a vendor number cannot displace an independent one.
|
|
278
|
+
|
|
279
|
+
## Etiquette
|
|
280
|
+
|
|
281
|
+
One sequential pass per source, real `User-Agent: opencode-models/0.1`, 30s
|
|
282
|
+
timeouts. No credentials, no telemetry, no scraping a source that asks not
|
|
283
|
+
to be scraped. A board that publishes a JSON API is meant to be read; a
|
|
284
|
+
social feed is meant to be read by a person.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
---
|
|
2
|
+
status: open
|
|
3
|
+
type: feature
|
|
4
|
+
priority: medium
|
|
5
|
+
area: backend
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Notes
|
|
9
|
+
|
|
10
|
+
Short description and context. Agents append progress notes below, newest first. Do not edit existing lines — append
|
|
11
|
+
only.
|
|
12
|
+
|
|
13
|
+
## Status values
|
|
14
|
+
|
|
15
|
+
| status | meaning |
|
|
16
|
+
| ------------- | ------------------------------------------------------------- |
|
|
17
|
+
| `idea` | aspirational — not fully settled, not ready to start |
|
|
18
|
+
| `backlog` | accepted but deferred — consider when stepping back to plan |
|
|
19
|
+
| `open` | ready to be picked up |
|
|
20
|
+
| `in-progress` | actively being worked on |
|
|
21
|
+
| `blocked` | cannot proceed; note the blocker in the body |
|
|
22
|
+
| `done` | completed |
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"description": "Schema for .plan/*.md YAML frontmatter. Filename (without .md) is the unique task key.",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"required": ["status", "type", "priority"],
|
|
6
|
+
"properties": {
|
|
7
|
+
"status": {
|
|
8
|
+
"enum": ["idea", "backlog", "open", "in-progress", "blocked", "done"]
|
|
9
|
+
},
|
|
10
|
+
"type": {
|
|
11
|
+
"enum": ["bug", "feature", "chore", "decision", "perf"]
|
|
12
|
+
},
|
|
13
|
+
"priority": {
|
|
14
|
+
"enum": ["high", "medium", "low"]
|
|
15
|
+
},
|
|
16
|
+
"area": {
|
|
17
|
+
"description": "Optional grouping label for filtering (e.g. backend, frontend, devops, testing)",
|
|
18
|
+
"type": "string"
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"additionalProperties": false
|
|
22
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
---
|
|
2
|
+
status: in-progress
|
|
3
|
+
type: chore
|
|
4
|
+
priority: high
|
|
5
|
+
area: devops
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Notes
|
|
9
|
+
|
|
10
|
+
Public-release hardening, decided 2026-10-20. Decisions (same day, with
|
|
11
|
+
maintainer — they supersede the plan below where they differ): the
|
|
12
|
+
email stays everywhere (`codesam` name is fine too) — LICENSE,
|
|
13
|
+
commit identities, pyproject contact unchanged, no history rewrite.
|
|
14
|
+
README contact paths are GitHub issues + private security advisories
|
|
15
|
+
only. The gitleaks pre-commit hook needs its own toolchain
|
|
16
|
+
(golang/docker image) or a system binary — fails the local-and-cheap
|
|
17
|
+
test, so the local audit-grep hook stands in and CI keeps running real
|
|
18
|
+
gitleaks.
|
|
19
|
+
|
|
20
|
+
## Steps
|
|
21
|
+
|
|
22
|
+
- [x] 1. ~~LICENSE identity + history rewrite.~~ DROPPED by decision:
|
|
23
|
+
email and `codesam` name stay as-is everywhere; no rewrite, no
|
|
24
|
+
force-push.
|
|
25
|
+
- [x] 2. README due-diligence paragraph on benchmark data (GitHub-only
|
|
26
|
+
contact paths). Added as "### Data due diligence" under ## Data.
|
|
27
|
+
- [x] 3. Client-side git hooks, no new installs (`.githooks/`, wired via
|
|
28
|
+
`just bootstrap` without clobbering an existing `core.hooksPath`):
|
|
29
|
+
`pre-push` refuses `refs/heads/local-data`; `pre-commit` runs the
|
|
30
|
+
secret-pattern sweep and `ruff check` + `ruff format --check` via the
|
|
31
|
+
project's own venv, with a graceful skip when ruff is unavailable.
|
|
32
|
+
Verified: pre-commit passes on a clean tree, fails on planted token
|
|
33
|
+
fixtures (a fake GitHub PAT and a fake AWS key); pre-push refuses a
|
|
34
|
+
`local-data` push and allows normal branches (scratch-repo end-to-end
|
|
35
|
+
run).
|
|
36
|
+
|
|
37
|
+
## Open questions — all settled 2026-10-20 (see Notes above for answers)
|
|
38
|
+
|
|
39
|
+
- Q1: commit identities — settled: leave unchanged.
|
|
40
|
+
- Q2: `pyproject.toml` author contact — settled: leave unchanged.
|
|
41
|
+
- Q3: due-diligence paragraph contact path — settled: GitHub issues +
|
|
42
|
+
private security advisories.
|
|
43
|
+
|
|
44
|
+
## Verification (revised — no rewrite, no force-push anywhere)
|
|
45
|
+
|
|
46
|
+
- `just lint`, `just test` green on a fresh checkout after `just bootstrap`.
|
|
47
|
+
- Hooks fire on trial runs: `pre-commit` blocks a planted secret-pattern
|
|
48
|
+
file and fails a planted lint violation; `pre-push` refuses a
|
|
49
|
+
`local-data` push and allows a normal branch push.
|
|
50
|
+
- `git config core.hooksPath` untouched when already set to something
|
|
51
|
+
else (bootstrap must not clobber an existing hooks setup).
|
|
52
|
+
|
|
53
|
+
## Remaining scope (reopened 2026-10-20 — not started, needs direction)
|
|
54
|
+
|
|
55
|
+
- [ ] Release process: reserve the PyPI name with a first publish;
|
|
56
|
+
single-source the version (currently hardcoded in `pyproject.toml`
|
|
57
|
+
and as the `opencode-rank/0.1` User-Agent); add CHANGELOG.md;
|
|
58
|
+
tag-triggered publish workflow.
|
|
59
|
+
- [ ] Repo hygiene: CONTRIBUTING.md only — decided against issue/PR
|
|
60
|
+
templates and against a CODE_OF_CONDUCT (small single-maintainer tool;
|
|
61
|
+
GitHub issues + private security advisories already cover reporting).
|
|
62
|
+
- [ ] GitHub push protection + secret scanning at public-launch time
|
|
63
|
+
(free for public repos; the local grep and CI gitleaks stay as the
|
|
64
|
+
pre-launch layers).
|
|
65
|
+
- [ ] Source-site Terms of Service review for republished benchmark
|
|
66
|
+
numbers — deferred counsel work; the README due-diligence note is the
|
|
67
|
+
interim public stance.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
---
|
|
2
|
+
status: in-progress
|
|
3
|
+
type: feature
|
|
4
|
+
priority: high
|
|
5
|
+
area: cli
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Notes
|
|
9
|
+
|
|
10
|
+
`fetch` dies with a traceback when the catalog sitemap is unreachable, and
|
|
11
|
+
`list` / `next` / `models` silently use arbitrarily old snapshots. Two
|
|
12
|
+
fixes, both in console output behavior, no ranking changes:
|
|
13
|
+
|
|
14
|
+
1. Catalog fetch failure degrades to a warning (`keeping existing records`),
|
|
15
|
+
same shape as the existing per-source warnings. Exit code stays 0.
|
|
16
|
+
2. `list`, `next`, and the `models` snapshot path warn per tier when a
|
|
17
|
+
`tier_prices/<tier>.json` snapshot is older than 7 days
|
|
18
|
+
(`STALE_AFTER_DAYS`), by file mtime. Missing files stay silent (unknown
|
|
19
|
+
is already reported as unknown).
|
|
20
|
+
|
|
21
|
+
Out of scope: exit-code changes, auto-refresh, freshness of `data/available`
|
|
22
|
+
(keyed runs re-fetch it live), docs beyond one README line.
|