roadmodel 0.2.83__tar.gz → 0.2.85__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {roadmodel-0.2.83 → roadmodel-0.2.85}/PKG-INFO +2 -1
- {roadmodel-0.2.83 → roadmodel-0.2.85}/README.md +1 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmap-refresh.md +15 -2
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/prompt-phase-roadmap.md +14 -1
- {roadmodel-0.2.83 → roadmodel-0.2.85}/pyproject.toml +1 -1
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/prompt-phase-roadmap.md +14 -1
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmap-refresh.md +15 -2
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/__init__.py +1 -1
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/cli.py +124 -11
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/scoring.py +330 -14
- {roadmodel-0.2.83 → roadmodel-0.2.85}/.github/README.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/.gitignore +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/LICENSE +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/NOTICE +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/benchmarks.json +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/catalog.json +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmap-phase.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmap-project.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmap-step.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmodel-upgrade.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/context-windows.json +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/effort-tokens.json +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/model-selector.txt +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/model-tier-cost-scale.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/settings-display.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/phase-roadmap-template.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/planning-kit-how-to-use.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/project-roadmap-template.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/prompt-project-roadmap.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/user-context.example.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/hatch_build.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/infra/README.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/infra/gcp-killswitch/README.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/infra/supabase/README.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/benchmarks.json +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/catalog.json +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/context-windows.json +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/effort-tokens.json +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/model-selector.txt +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/model-tier-cost-scale.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/phase-roadmap-template.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/planning-kit-how-to-use.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/project-roadmap-template.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/prompt-project-roadmap.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmap-phase.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmap-project.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmap-step.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmodel-upgrade.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/settings-display.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/user-context.example.md +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/agent_context.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/agent_rules.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/update_projects.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/updater_lock.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/upgrade_fleet.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/worktree_memory.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/__main__.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/config.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/cost.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/errors.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/mcp_server.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/private_files.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/__init__.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/_safety.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/anthropic.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/google.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/openai.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/openai_compatible.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/registry.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/recommend.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/usage.py +0 -0
- {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/user_context.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: roadmodel
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.85
|
|
4
4
|
Summary: BYO-key CLI that recommends the right AI model, platform, and settings for a prompt.
|
|
5
5
|
Project-URL: Homepage, https://roadmodel.ai
|
|
6
6
|
Project-URL: Repository, https://github.com/nathanramoscfa/roadmodel
|
|
@@ -607,6 +607,7 @@ server and the reasoning-effort default in parity automatically.
|
|
|
607
607
|
| `roadmodel recommend --model` | Override the specific model ID on the chosen provider. |
|
|
608
608
|
| `roadmodel recommend --user-context PATH` | Override the user-context.md location for this invocation. |
|
|
609
609
|
| `roadmodel score --category C --complexity X` | Rank (model, platform, effort) candidates in code for a classified task — no engine call. See [docs/scoring-model.md](docs/scoring-model.md). |
|
|
610
|
+
| `roadmodel backup --category C --complexity X --model M --effort E` | Name the cross-maker backup for a pick, by the rule /recommend uses — no engine call. Prints the roadmap Settings Backup cell, and first a `MODEL` cell when a newer version of the pick's model (same platform, no dearer) scores level or better at its effort. |
|
|
610
611
|
| `roadmodel catalog show` | Print the bundled `model-selector.txt` (use `--doc tier-cost-scale` for the price doc). |
|
|
611
612
|
| `roadmodel catalog path` | Print the on-disk path of the bundled catalog document (same `--doc` flag). |
|
|
612
613
|
| `roadmodel context show` | Print the resolved `user-context.md` file. |
|
|
@@ -356,6 +356,7 @@ server and the reasoning-effort default in parity automatically.
|
|
|
356
356
|
| `roadmodel recommend --model` | Override the specific model ID on the chosen provider. |
|
|
357
357
|
| `roadmodel recommend --user-context PATH` | Override the user-context.md location for this invocation. |
|
|
358
358
|
| `roadmodel score --category C --complexity X` | Rank (model, platform, effort) candidates in code for a classified task — no engine call. See [docs/scoring-model.md](docs/scoring-model.md). |
|
|
359
|
+
| `roadmodel backup --category C --complexity X --model M --effort E` | Name the cross-maker backup for a pick, by the rule /recommend uses — no engine call. Prints the roadmap Settings Backup cell, and first a `MODEL` cell when a newer version of the pick's model (same platform, no dearer) scores level or better at its effort. |
|
|
359
360
|
| `roadmodel catalog show` | Print the bundled `model-selector.txt` (use `--doc tier-cost-scale` for the price doc). |
|
|
360
361
|
| `roadmodel catalog path` | Print the on-disk path of the bundled catalog document (same `--doc` flag). |
|
|
361
362
|
| `roadmodel context show` | Print the resolved `user-context.md` file. |
|
|
@@ -180,8 +180,21 @@ exclusion in the selector, and include a backup model.
|
|
|
180
180
|
A changed backup alone counts as a difference; name it in that line.
|
|
181
181
|
- Every table carries a `Backup` row directly under `Model`: the
|
|
182
182
|
selector's BACKUP with its own platform and dial in one cell
|
|
183
|
-
(`GPT-5.6 Terra — Codex · Intelligence Medium`), or `None`.
|
|
184
|
-
|
|
183
|
+
(`GPT-5.6 Terra — Codex · Intelligence Medium`), or `None`. Where
|
|
184
|
+
`roadmodel` is installed, the cell is the one `roadmodel backup
|
|
185
|
+
--category <c> --complexity <x> [--novel] --model <the step's model>
|
|
186
|
+
--effort <the step's effort>` prints for the step's classification —
|
|
187
|
+
the rule /recommend backs a pick up by, which takes the cheaper of two
|
|
188
|
+
substitutes the benchmarks cannot tell apart — and it outranks the
|
|
189
|
+
selector's BACKUP. When the command also prints a `MODEL` line, that
|
|
190
|
+
cell replaces the step's model — a newer version of the same model
|
|
191
|
+
(GPT-6 Sol → GPT-6.1 Sol), on the same platform, no dearer, that
|
|
192
|
+
scores level or better at the step's effort; the selector's letters
|
|
193
|
+
are measured at each model's top effort, and a step that runs lower
|
|
194
|
+
can reverse them. That is a difference: rewrite the table, its Model
|
|
195
|
+
rationale and its selection block with the `MODEL` cell, keep the
|
|
196
|
+
Backup cell printed with it, and add the `Settings updated` line.
|
|
197
|
+
A table written before roadmodel 0.2.41 has no such row: add it from the
|
|
185
198
|
backup the step already names (its Model rationale's closing
|
|
186
199
|
sentence, or its entry in the Model selection blocks) — a layout
|
|
187
200
|
fix, not a new pick, so it takes no `Settings updated` line.
|
|
@@ -103,7 +103,20 @@ selector in `@planning/model-selector.txt` (prices from
|
|
|
103
103
|
You are the engine — do not call any external API. Honor every
|
|
104
104
|
availability exclusion in the selector, and write each step's backup
|
|
105
105
|
model into its Settings table's Backup row, with the backup's own
|
|
106
|
-
platform and dial.
|
|
106
|
+
platform and dial. Where `roadmodel` is installed, the Backup row is
|
|
107
|
+
the cell `roadmodel backup --category <c> --complexity <x> [--novel]
|
|
108
|
+
--model <the step's model> --effort <the step's effort>` prints for the
|
|
109
|
+
step's own classification: the rule /recommend backs a pick up by, in
|
|
110
|
+
code, which takes the cheaper of two substitutes the benchmarks cannot
|
|
111
|
+
tell apart. The selector's BACKUP stands only where it is not
|
|
112
|
+
installed. When the command also prints a `MODEL` line, that cell
|
|
113
|
+
replaces the step's model: a newer version of the same model (GPT-6 Sol
|
|
114
|
+
→ GPT-6.1 Sol), on the same platform, no dearer, that scores level or
|
|
115
|
+
better at the step's effort. The selector's letters are measured at each
|
|
116
|
+
model's top effort, and a step that runs lower can reverse them. Write
|
|
117
|
+
the `MODEL` cell into the Model row, the Model rationale and the step's
|
|
118
|
+
selection block, citing the newer model's figures at the step's effort,
|
|
119
|
+
and keep the Backup cell printed with it.
|
|
107
120
|
|
|
108
121
|
Honor the template's style rules: 80-column prose, zero `{{...}}`
|
|
109
122
|
tokens left, `**Status:** Not started` directly under the title and
|
|
@@ -103,7 +103,20 @@ selector in `@planning/model-selector.txt` (prices from
|
|
|
103
103
|
You are the engine — do not call any external API. Honor every
|
|
104
104
|
availability exclusion in the selector, and write each step's backup
|
|
105
105
|
model into its Settings table's Backup row, with the backup's own
|
|
106
|
-
platform and dial.
|
|
106
|
+
platform and dial. Where `roadmodel` is installed, the Backup row is
|
|
107
|
+
the cell `roadmodel backup --category <c> --complexity <x> [--novel]
|
|
108
|
+
--model <the step's model> --effort <the step's effort>` prints for the
|
|
109
|
+
step's own classification: the rule /recommend backs a pick up by, in
|
|
110
|
+
code, which takes the cheaper of two substitutes the benchmarks cannot
|
|
111
|
+
tell apart. The selector's BACKUP stands only where it is not
|
|
112
|
+
installed. When the command also prints a `MODEL` line, that cell
|
|
113
|
+
replaces the step's model: a newer version of the same model (GPT-6 Sol
|
|
114
|
+
→ GPT-6.1 Sol), on the same platform, no dearer, that scores level or
|
|
115
|
+
better at the step's effort. The selector's letters are measured at each
|
|
116
|
+
model's top effort, and a step that runs lower can reverse them. Write
|
|
117
|
+
the `MODEL` cell into the Model row, the Model rationale and the step's
|
|
118
|
+
selection block, citing the newer model's figures at the step's effort,
|
|
119
|
+
and keep the Backup cell printed with it.
|
|
107
120
|
|
|
108
121
|
Honor the template's style rules: 80-column prose, zero `{{...}}`
|
|
109
122
|
tokens left, `**Status:** Not started` directly under the title and
|
|
@@ -180,8 +180,21 @@ exclusion in the selector, and include a backup model.
|
|
|
180
180
|
A changed backup alone counts as a difference; name it in that line.
|
|
181
181
|
- Every table carries a `Backup` row directly under `Model`: the
|
|
182
182
|
selector's BACKUP with its own platform and dial in one cell
|
|
183
|
-
(`GPT-5.6 Terra — Codex · Intelligence Medium`), or `None`.
|
|
184
|
-
|
|
183
|
+
(`GPT-5.6 Terra — Codex · Intelligence Medium`), or `None`. Where
|
|
184
|
+
`roadmodel` is installed, the cell is the one `roadmodel backup
|
|
185
|
+
--category <c> --complexity <x> [--novel] --model <the step's model>
|
|
186
|
+
--effort <the step's effort>` prints for the step's classification —
|
|
187
|
+
the rule /recommend backs a pick up by, which takes the cheaper of two
|
|
188
|
+
substitutes the benchmarks cannot tell apart — and it outranks the
|
|
189
|
+
selector's BACKUP. When the command also prints a `MODEL` line, that
|
|
190
|
+
cell replaces the step's model — a newer version of the same model
|
|
191
|
+
(GPT-6 Sol → GPT-6.1 Sol), on the same platform, no dearer, that
|
|
192
|
+
scores level or better at the step's effort; the selector's letters
|
|
193
|
+
are measured at each model's top effort, and a step that runs lower
|
|
194
|
+
can reverse them. That is a difference: rewrite the table, its Model
|
|
195
|
+
rationale and its selection block with the `MODEL` cell, keep the
|
|
196
|
+
Backup cell printed with it, and add the `Settings updated` line.
|
|
197
|
+
A table written before roadmodel 0.2.41 has no such row: add it from the
|
|
185
198
|
backup the step already names (its Model rationale's closing
|
|
186
199
|
sentence, or its entry in the Model selection blocks) — a layout
|
|
187
200
|
fix, not a new pick, so it takes no `Settings updated` line.
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# src/roadmodel/__init__.py
|
|
2
|
-
__version__ = "0.2.
|
|
2
|
+
__version__ = "0.2.85"
|
|
@@ -418,6 +418,23 @@ def recommend(
|
|
|
418
418
|
click.echo(_format_structured_text(structured), nl=False)
|
|
419
419
|
|
|
420
420
|
|
|
421
|
+
def _scoring_user_context(user_context_path: Path | None) -> str:
|
|
422
|
+
"""The user-context text the scorer reads: ``--user-context``, else
|
|
423
|
+
ROADMODEL_USER_CONTEXT, else the default path; empty when none exists. A
|
|
424
|
+
path named explicitly that does not exist is an error."""
|
|
425
|
+
if user_context_path is not None and not user_context_path.expanduser().exists():
|
|
426
|
+
raise UserContextNotFoundError(user_context_path.expanduser())
|
|
427
|
+
env_user_context = os.environ.get("ROADMODEL_USER_CONTEXT")
|
|
428
|
+
if (
|
|
429
|
+
user_context_path is None
|
|
430
|
+
and env_user_context
|
|
431
|
+
and not Path(env_user_context).expanduser().exists()
|
|
432
|
+
):
|
|
433
|
+
raise UserContextNotFoundError(Path(env_user_context).expanduser())
|
|
434
|
+
resolved = user_context.resolve(cli_path=user_context_path)
|
|
435
|
+
return user_context.read(resolved) if resolved.exists() else ""
|
|
436
|
+
|
|
437
|
+
|
|
421
438
|
@cli.command("score")
|
|
422
439
|
@click.option(
|
|
423
440
|
"--category",
|
|
@@ -474,17 +491,7 @@ def score_command(
|
|
|
474
491
|
funding, consumption headroom and Usage-pool status from user-context.md.
|
|
475
492
|
Every term is printed so a pick can be audited line by line.
|
|
476
493
|
"""
|
|
477
|
-
|
|
478
|
-
raise UserContextNotFoundError(user_context_path.expanduser())
|
|
479
|
-
env_user_context = os.environ.get("ROADMODEL_USER_CONTEXT")
|
|
480
|
-
if (
|
|
481
|
-
user_context_path is None
|
|
482
|
-
and env_user_context
|
|
483
|
-
and not Path(env_user_context).expanduser().exists()
|
|
484
|
-
):
|
|
485
|
-
raise UserContextNotFoundError(Path(env_user_context).expanduser())
|
|
486
|
-
resolved = user_context.resolve(cli_path=user_context_path)
|
|
487
|
-
text = user_context.read(resolved) if resolved.exists() else ""
|
|
494
|
+
text = _scoring_user_context(user_context_path)
|
|
488
495
|
posture = (budget or scoring.declared_budget(text)).lower()
|
|
489
496
|
try:
|
|
490
497
|
task = scoring.Task(
|
|
@@ -504,6 +511,112 @@ def score_command(
|
|
|
504
511
|
click.echo(scoring.render_text(ranking, limit=top))
|
|
505
512
|
|
|
506
513
|
|
|
514
|
+
@cli.command("backup")
|
|
515
|
+
@click.option(
|
|
516
|
+
"--category",
|
|
517
|
+
required=True,
|
|
518
|
+
type=click.Choice(list(scoring.CATEGORIES), case_sensitive=False),
|
|
519
|
+
help="PRIMARY task category of the pick.",
|
|
520
|
+
)
|
|
521
|
+
@click.option(
|
|
522
|
+
"--complexity",
|
|
523
|
+
required=True,
|
|
524
|
+
type=click.Choice(list(scoring.COMPLEXITIES), case_sensitive=False),
|
|
525
|
+
help="Overall complexity of the pick's task.",
|
|
526
|
+
)
|
|
527
|
+
@click.option("--novel", is_flag=True, default=False, help="High complexity AND novel.")
|
|
528
|
+
@click.option("--model", required=True, help="The pick's model: catalog id or name.")
|
|
529
|
+
@click.option("--effort", required=True, help="The pick's effort, as its dial names it.")
|
|
530
|
+
@click.option(
|
|
531
|
+
"--budget",
|
|
532
|
+
type=click.Choice(list(scoring.BUDGETS), case_sensitive=False),
|
|
533
|
+
default=None,
|
|
534
|
+
help="Budget posture; defaults to the user-context's declared Budget priority.",
|
|
535
|
+
)
|
|
536
|
+
@click.option(
|
|
537
|
+
"--output",
|
|
538
|
+
type=click.Choice(["text", "json"], case_sensitive=False),
|
|
539
|
+
default="text",
|
|
540
|
+
show_default=True,
|
|
541
|
+
)
|
|
542
|
+
@click.option(
|
|
543
|
+
"--user-context",
|
|
544
|
+
"user_context_path",
|
|
545
|
+
type=click.Path(path_type=Path),
|
|
546
|
+
default=None,
|
|
547
|
+
help="Path to user-context.md override.",
|
|
548
|
+
)
|
|
549
|
+
def backup_command(
|
|
550
|
+
category: str,
|
|
551
|
+
complexity: str,
|
|
552
|
+
novel: bool,
|
|
553
|
+
model: str,
|
|
554
|
+
effort: str,
|
|
555
|
+
budget: str | None,
|
|
556
|
+
output: str,
|
|
557
|
+
user_context_path: Path | None,
|
|
558
|
+
) -> None:
|
|
559
|
+
"""Name the cross-maker backup for a pick, in code — no engine call.
|
|
560
|
+
|
|
561
|
+
The rule /recommend backs each pick up by: another maker's model that does
|
|
562
|
+
the job as well, for the least; where the benchmarks cannot tell two
|
|
563
|
+
substitutes apart, the cheaper. Prints the roadmap Settings Backup cell.
|
|
564
|
+
|
|
565
|
+
When a newer version of the pick's model (same maker and series, on the
|
|
566
|
+
same platform, no dearer) scores level or better at the pick's effort, it
|
|
567
|
+
first prints that model on a MODEL line: the step runs it instead, and the
|
|
568
|
+
backup printed is its backup.
|
|
569
|
+
"""
|
|
570
|
+
text = _scoring_user_context(user_context_path)
|
|
571
|
+
posture = (budget or scoring.declared_budget(text)).lower()
|
|
572
|
+
try:
|
|
573
|
+
task = scoring.Task(category.lower(), complexity.lower(), novel, posture)
|
|
574
|
+
successor = scoring.successor_for_pick(task, text, model, effort)
|
|
575
|
+
runs = successor.candidate.model_id if successor is not None else model
|
|
576
|
+
backup = scoring.backup_for_pick(task, text, runs, effort)
|
|
577
|
+
except ValueError as exc:
|
|
578
|
+
raise click.ClickException(str(exc)) from exc
|
|
579
|
+
cell = scoring.backup_cell(backup) if backup is not None else "None"
|
|
580
|
+
model_cell = scoring.backup_cell(successor) if successor is not None else None
|
|
581
|
+
if output.lower() == "json":
|
|
582
|
+
payload: dict[str, Any] = {
|
|
583
|
+
"model": None,
|
|
584
|
+
"model_cell": model_cell,
|
|
585
|
+
"backup": None,
|
|
586
|
+
"cell": cell,
|
|
587
|
+
}
|
|
588
|
+
if successor is not None:
|
|
589
|
+
s = successor.candidate
|
|
590
|
+
payload["model"] = {
|
|
591
|
+
"model_id": s.model_id,
|
|
592
|
+
"model": s.model_name,
|
|
593
|
+
"platform_id": s.platform_id,
|
|
594
|
+
"platform": s.platform_name,
|
|
595
|
+
"effort": successor.effort,
|
|
596
|
+
"native_effort": successor.native,
|
|
597
|
+
"replaces": successor.replaces,
|
|
598
|
+
}
|
|
599
|
+
if backup is not None:
|
|
600
|
+
c = backup.candidate
|
|
601
|
+
payload["backup"] = {
|
|
602
|
+
"model_id": c.model_id,
|
|
603
|
+
"model": c.model_name,
|
|
604
|
+
"platform_id": c.platform_id,
|
|
605
|
+
"platform": c.platform_name,
|
|
606
|
+
"effort": backup.effort,
|
|
607
|
+
"native_effort": backup.native,
|
|
608
|
+
}
|
|
609
|
+
click.echo(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
610
|
+
return
|
|
611
|
+
if successor is not None:
|
|
612
|
+
click.echo(f"MODEL {model_cell}")
|
|
613
|
+
click.echo(
|
|
614
|
+
f" replaces {successor.replaces}: a newer version, no dearer, "
|
|
615
|
+
"level or better at this effort"
|
|
616
|
+
)
|
|
617
|
+
click.echo(f"BACKUP {cell}")
|
|
618
|
+
|
|
619
|
+
|
|
507
620
|
@cli.command("cost")
|
|
508
621
|
@click.option("--model", required=True, help="Catalog model id or display name.")
|
|
509
622
|
@click.option("--platform", required=True, help="Access method id or display name.")
|
|
@@ -137,8 +137,19 @@ _RANK_SCALED_KEYS: Final[frozenset[str]] = frozenset(
|
|
|
137
137
|
|
|
138
138
|
# Editorial letter → quality points. Midpoints of five equal bands.
|
|
139
139
|
LETTER_QUALITY: Final[dict[str, float]] = {"S": 90.0, "A": 70.0, "B": 50.0, "C": 30.0, "D": 10.0}
|
|
140
|
-
# When AA has measured the model: weight on the scaled evidence vs
|
|
140
|
+
# When AA has measured the model in speed: weight on the scaled evidence vs
|
|
141
|
+
# the letter (speed's letter is an estimate, not derived from tokens/s).
|
|
141
142
|
EVIDENCE_WEIGHT: Final[float] = 0.7
|
|
143
|
+
# The five derived categories: update/derive_ratings.py letters them from the
|
|
144
|
+
# very benchmark CATEGORY_EVIDENCE names, so a measured model's letter there
|
|
145
|
+
# is a staircase of evidence already counted. Blending it in adds no
|
|
146
|
+
# information, only a cliff at each band edge: GPT-6 Astra's 52.7 on the AA
|
|
147
|
+
# Index reads S and GPT-6.1 Sol's 51.8 reads A, six quality points for 0.9
|
|
148
|
+
# of index. A model measured there is scored on its evidence alone; its
|
|
149
|
+
# letter stands in only where AA has not measured it.
|
|
150
|
+
DERIVED_LETTER_CATEGORIES: Final[frozenset[str]] = frozenset(
|
|
151
|
+
{"coding", "planning", "agentic", "long-context", "knowledge"}
|
|
152
|
+
)
|
|
142
153
|
# A letter with no AA measurement behind it is an editorial prior, not
|
|
143
154
|
# evidence: discount it so a measured model wins an otherwise-tied letter.
|
|
144
155
|
UNMEASURED_DISCOUNT: Final[float] = 5.0
|
|
@@ -886,7 +897,8 @@ def _quality(
|
|
|
886
897
|
level: str | None = None,
|
|
887
898
|
) -> tuple[float, str, str]:
|
|
888
899
|
"""The model's quality for the task's category: its headline evidence, or
|
|
889
|
-
with ``level`` its evidence at that effort
|
|
900
|
+
with ``level`` its evidence at that effort; in speed blended with its
|
|
901
|
+
letter, in a derived category (:data:`DERIVED_LETTER_CATEGORIES`) alone."""
|
|
890
902
|
raw_tiers = model.get("tiers")
|
|
891
903
|
tiers: dict[str, Any] = raw_tiers if isinstance(raw_tiers, dict) else {}
|
|
892
904
|
letter = str(tiers.get(task.category, "C")).upper()
|
|
@@ -898,6 +910,8 @@ def _quality(
|
|
|
898
910
|
points = _evidence_points(str(model.get("id", "")), task, bench, scale, level)
|
|
899
911
|
if points is None:
|
|
900
912
|
return max(0.0, letter_q - UNMEASURED_DISCOUNT), "letter", letter
|
|
913
|
+
if task.category in DERIVED_LETTER_CATEGORIES:
|
|
914
|
+
return points, f"aa:{CATEGORY_EVIDENCE[task.category]}", letter
|
|
901
915
|
q = EVIDENCE_WEIGHT * points + (1.0 - EVIDENCE_WEIGHT) * letter_q
|
|
902
916
|
return q, f"aa:{CATEGORY_EVIDENCE[task.category]}+letter", letter
|
|
903
917
|
|
|
@@ -1244,10 +1258,24 @@ def rank(
|
|
|
1244
1258
|
backup: Candidate | None = None
|
|
1245
1259
|
warning: str | None = None
|
|
1246
1260
|
if primary is not None:
|
|
1247
|
-
|
|
1248
|
-
if c.provider != primary.provider and c.funding != "unfunded"
|
|
1249
|
-
|
|
1250
|
-
|
|
1261
|
+
others = [
|
|
1262
|
+
c for c in candidates[1:] if c.provider != primary.provider and c.funding != "unfunded"
|
|
1263
|
+
]
|
|
1264
|
+
if others:
|
|
1265
|
+
# The best-scoring other maker's model, else the cheapest one the
|
|
1266
|
+
# evidence cannot tell from it (:func:`_near_tied`), whatever the
|
|
1267
|
+
# posture: at budget best the score barely prices a model, and a
|
|
1268
|
+
# fraction of an AA point within noise would buy a dearer backup.
|
|
1269
|
+
ref = others[0]
|
|
1270
|
+
band = _aa_band(bench, cat)
|
|
1271
|
+
|
|
1272
|
+
def aa(c: Candidate) -> float | None:
|
|
1273
|
+
return _evidence(bench, c.model_id, FRONTIER_INDEX, c.evidence_level)
|
|
1274
|
+
|
|
1275
|
+
backup = min(
|
|
1276
|
+
(c for c in others if _near_tied(c, aa(c), ref, aa(ref), task.category, band)),
|
|
1277
|
+
key=lambda c: (c.effective_cost_usd, -c.score, c.model_id),
|
|
1278
|
+
)
|
|
1251
1279
|
if backup is None:
|
|
1252
1280
|
funded_providers = sorted({c.provider for c in candidates if c.funding != "unfunded"})
|
|
1253
1281
|
warning = (
|
|
@@ -1452,6 +1480,33 @@ def _level_with(a: Candidate, b: Candidate, category: str) -> bool:
|
|
|
1452
1480
|
return a.evidence_points >= b.evidence_points - TIE_BAND.get(category, DEFAULT_TIE_BAND)
|
|
1453
1481
|
|
|
1454
1482
|
|
|
1483
|
+
def _near_tied(
|
|
1484
|
+
a: Candidate,
|
|
1485
|
+
a_aa: float | None,
|
|
1486
|
+
b: Candidate,
|
|
1487
|
+
b_aa: float | None,
|
|
1488
|
+
category: str,
|
|
1489
|
+
aa_band: float,
|
|
1490
|
+
) -> bool:
|
|
1491
|
+
"""Whether the evidence cannot tell ``a`` from ``b``: each level with the
|
|
1492
|
+
other in the category (:func:`_level_with`, both ways), and their AA
|
|
1493
|
+
Indexes (``a_aa``, ``b_aa``) no more than ``aa_band`` apart where both are
|
|
1494
|
+
measured. One-way levelness is not enough: every stronger model is level
|
|
1495
|
+
with a weaker one."""
|
|
1496
|
+
return (
|
|
1497
|
+
_level_with(a, b, category)
|
|
1498
|
+
and _level_with(b, a, category)
|
|
1499
|
+
and (a_aa is None or b_aa is None or abs(a_aa - b_aa) <= aa_band)
|
|
1500
|
+
)
|
|
1501
|
+
|
|
1502
|
+
|
|
1503
|
+
def _aa_band(bench: dict[str, Any], catalog: dict[str, Any]) -> float:
|
|
1504
|
+
"""The AA Index's noise band in its own points: DEFAULT_TIE_BAND on the
|
|
1505
|
+
0–100 scale it is read on, mapped back onto the index's measured range."""
|
|
1506
|
+
gscale = _evidence_scale(catalog, bench, FRONTIER_INDEX)
|
|
1507
|
+
return 0.0 if gscale is None else DEFAULT_TIE_BAND * (gscale.hi - gscale.lo) / 100.0
|
|
1508
|
+
|
|
1509
|
+
|
|
1455
1510
|
def _tie_break(p: FrontierPoint, category: str) -> tuple[float, float]:
|
|
1456
1511
|
"""Between points level in the category: the higher AA Index, then the
|
|
1457
1512
|
cheaper. Speed work too: QUALITY is the ladder's top, and COST and
|
|
@@ -1680,8 +1735,9 @@ def ladder(
|
|
|
1680
1735
|
adequate point above COST.
|
|
1681
1736
|
- Each rung carries a BACKUP from another maker (:func:`_backup_for`):
|
|
1682
1737
|
the cheapest point on the other makers' frontier that does the job as
|
|
1683
|
-
well as the rung, else the
|
|
1684
|
-
|
|
1738
|
+
well as the rung, else the strongest substitute, giving way to a
|
|
1739
|
+
cheaper one the evidence cannot tell from it. ``backup_warning`` says
|
|
1740
|
+
why a rung has none.
|
|
1685
1741
|
|
|
1686
1742
|
None when no pool model carries an AA Index (no frontier to read)."""
|
|
1687
1743
|
cat = catalog if catalog is not None else _cost._load_catalog()
|
|
@@ -1826,8 +1882,7 @@ def ladder(
|
|
|
1826
1882
|
cost_rung = settle(_below(cost_rung, balanced_rung, cat))
|
|
1827
1883
|
# General capability is level within the same noise band, read on the
|
|
1828
1884
|
# AA Index's own 0–100 scale (the planning evidence).
|
|
1829
|
-
|
|
1830
|
-
aa_band = 0.0 if gscale is None else DEFAULT_TIE_BAND * (gscale.hi - gscale.lo) / 100.0
|
|
1885
|
+
aa_band = _aa_band(bench, cat)
|
|
1831
1886
|
rungs = {
|
|
1832
1887
|
r.tier: replace(r, backup=_backup_for(r, settings, task, headroom, bench, cat, aa_band))
|
|
1833
1888
|
for r in (cost_rung, balanced_rung, quality_rung)
|
|
@@ -1894,10 +1949,13 @@ def _backup_for(
|
|
|
1894
1949
|
those already paid for ahead of per-token ones. A point matches the rung
|
|
1895
1950
|
when it stands level with it in the task's category
|
|
1896
1951
|
(:func:`_level_with`) and in general capability (an AA Index no more than
|
|
1897
|
-
``aa_band`` below). The
|
|
1952
|
+
``aa_band`` below). The reference is the cheapest match; with none, the
|
|
1898
1953
|
substitute the QUALITY rule would pick (:func:`_top`: strongest in the
|
|
1899
1954
|
category, a near tie going to the higher AA Index); with no adequate
|
|
1900
|
-
point, the one that falls shortest of the bar.
|
|
1955
|
+
point, the one that falls shortest of the bar. The backup is the cheapest
|
|
1956
|
+
of the points within the noise bands of the reference both ways
|
|
1957
|
+
(:func:`_near_tied`): the evidence cannot tell them apart, so paying more
|
|
1958
|
+
buys nothing it can show. It runs at its
|
|
1901
1959
|
point's measured effort, else at the rung's posture's effort, on its own
|
|
1902
1960
|
platform. None when the pool holds no other
|
|
1903
1961
|
maker's model."""
|
|
@@ -1924,13 +1982,26 @@ def _backup_for(
|
|
|
1924
1982
|
if _level_with(p.candidate, r.candidate, task.category)
|
|
1925
1983
|
and (r.aa_index is None or aa(p) >= r.aa_index - aa_band)
|
|
1926
1984
|
]
|
|
1927
|
-
|
|
1985
|
+
ref = (
|
|
1928
1986
|
min(matches, key=lambda p: (p.price_usd, -aa(p)))
|
|
1929
1987
|
if matches
|
|
1930
1988
|
else _top(near, task.category)
|
|
1931
1989
|
)
|
|
1932
1990
|
else:
|
|
1933
|
-
|
|
1991
|
+
ref = max(front, key=_nearest)
|
|
1992
|
+
# Of the points the evidence cannot tell from the reference, the cheapest:
|
|
1993
|
+
# a near-identical model at a quarter of the price is not passed over for
|
|
1994
|
+
# a fraction of an AA point. A peer may fall short of the bar where the
|
|
1995
|
+
# reference clears it, by less than the noise: a shortfall the evidence
|
|
1996
|
+
# cannot show. A prepaid reference keeps a prepaid backup.
|
|
1997
|
+
prepaid = ref.candidate.scarcity < 1.0
|
|
1998
|
+
peers = [
|
|
1999
|
+
p
|
|
2000
|
+
for p in front
|
|
2001
|
+
if (p.candidate.scarcity < 1.0 or not prepaid)
|
|
2002
|
+
and _near_tied(p.candidate, p.aa_index, ref.candidate, ref.aa_index, task.category, aa_band)
|
|
2003
|
+
]
|
|
2004
|
+
chosen = min(peers, key=lambda p: (p.price_usd, -p.candidate.score))
|
|
1934
2005
|
c = chosen.candidate
|
|
1935
2006
|
if c.evidence_level is not None:
|
|
1936
2007
|
effort = c.evidence_level
|
|
@@ -1940,6 +2011,251 @@ def _backup_for(
|
|
|
1940
2011
|
return Backup(chosen, effort, native_level(c.platform_id, c.model_id, effort, catalog=catalog))
|
|
1941
2012
|
|
|
1942
2013
|
|
|
2014
|
+
def _model_ref_id(ref: str, catalog: dict[str, Any]) -> str:
|
|
2015
|
+
"""The catalog id ``ref`` names, matched spelling-free as
|
|
2016
|
+
recommend._model_key does ("Claude Opus 5.5", "Opus 5.5" and
|
|
2017
|
+
"claude-opus-5-5" are one model); ``ref`` itself on a miss."""
|
|
2018
|
+
|
|
2019
|
+
def key(text: str) -> str:
|
|
2020
|
+
words = re.split(r"[\s_.-]+", text.lower())
|
|
2021
|
+
return " ".join(sorted(w for w in words if w and w != "claude"))
|
|
2022
|
+
|
|
2023
|
+
want = key(ref)
|
|
2024
|
+
for m in catalog.get("models", []):
|
|
2025
|
+
if isinstance(m, dict) and want in {key(str(m.get("id", ""))), key(str(m.get("name", "")))}:
|
|
2026
|
+
return str(m.get("id", ""))
|
|
2027
|
+
return ref
|
|
2028
|
+
|
|
2029
|
+
|
|
2030
|
+
@dataclass(frozen=True)
|
|
2031
|
+
class _Pick:
|
|
2032
|
+
"""A pick made outside the ladder (a roadmap step's Settings), with the
|
|
2033
|
+
pool and scales it is read against (:func:`_pick`)."""
|
|
2034
|
+
|
|
2035
|
+
candidate: Candidate # the model, on the platform the pool reaches it best
|
|
2036
|
+
level: str # the scorer's level it runs at
|
|
2037
|
+
ranking: Ranking
|
|
2038
|
+
base: Task
|
|
2039
|
+
models: dict[str, dict[str, Any]]
|
|
2040
|
+
bench: dict[str, Any]
|
|
2041
|
+
composite: dict[str, Any]
|
|
2042
|
+
scales: tuple[EvidenceScale | None, EvidenceScale | None]
|
|
2043
|
+
catalog: dict[str, Any]
|
|
2044
|
+
|
|
2045
|
+
def at(self, c: Candidate, level: str | None) -> Candidate:
|
|
2046
|
+
"""``c`` read at ``level`` (None: its headline row; see :func:`_at_level`)."""
|
|
2047
|
+
return _at_level(
|
|
2048
|
+
c,
|
|
2049
|
+
level,
|
|
2050
|
+
self.models[c.model_id],
|
|
2051
|
+
self.base,
|
|
2052
|
+
self.composite,
|
|
2053
|
+
self.scales,
|
|
2054
|
+
self.ranking.lam,
|
|
2055
|
+
self.ranking.k_points_per_decade,
|
|
2056
|
+
)
|
|
2057
|
+
|
|
2058
|
+
def measured(self, model_id: str) -> bool:
|
|
2059
|
+
return self.level in measured_levels(self.bench, model_id)
|
|
2060
|
+
|
|
2061
|
+
|
|
2062
|
+
def _pick(
|
|
2063
|
+
task: Task,
|
|
2064
|
+
user_context_text: str,
|
|
2065
|
+
model: str,
|
|
2066
|
+
effort: str,
|
|
2067
|
+
unavailable_models: list[str] | None,
|
|
2068
|
+
catalog: dict[str, Any] | None,
|
|
2069
|
+
benchmarks: dict[str, Any] | None,
|
|
2070
|
+
) -> _Pick:
|
|
2071
|
+
"""``model`` (catalog id or name) run at ``effort`` (a scorer or native
|
|
2072
|
+
level) for ``task``, over the pool /recommend ranks. ValueError when the
|
|
2073
|
+
model is not one this user-context can reach."""
|
|
2074
|
+
cat = catalog if catalog is not None else _cost._load_catalog()
|
|
2075
|
+
bench = benchmarks if benchmarks is not None else _load_benchmarks()
|
|
2076
|
+
model_id = _model_ref_id(model, cat)
|
|
2077
|
+
base = Task(task.category, task.complexity, task.novel, "balanced")
|
|
2078
|
+
ranking = rank(
|
|
2079
|
+
base,
|
|
2080
|
+
user_context_text,
|
|
2081
|
+
unavailable_models=unavailable_models,
|
|
2082
|
+
catalog=cat,
|
|
2083
|
+
benchmarks=bench,
|
|
2084
|
+
effort_aware=False,
|
|
2085
|
+
)
|
|
2086
|
+
picked = next((c for c in ranking.candidates if c.model_id == model_id), None)
|
|
2087
|
+
models = {str(m.get("id", "")): m for m in cat.get("models", []) if isinstance(m, dict)}
|
|
2088
|
+
if picked is None or model_id not in models:
|
|
2089
|
+
raise ValueError(f"{model!r} is not a model this user-context can reach")
|
|
2090
|
+
composite = with_composites(bench)
|
|
2091
|
+
scales = (
|
|
2092
|
+
_evidence_scale(cat, composite, CATEGORY_EVIDENCE[base.category]),
|
|
2093
|
+
_evidence_scale(cat, composite, CATEGORY_EVIDENCE["planning"]),
|
|
2094
|
+
)
|
|
2095
|
+
return _Pick(
|
|
2096
|
+
picked, _scorer_level(effort), ranking, base, models, bench, composite, scales, cat
|
|
2097
|
+
)
|
|
2098
|
+
|
|
2099
|
+
|
|
2100
|
+
def backup_for_pick(
|
|
2101
|
+
task: Task,
|
|
2102
|
+
user_context_text: str,
|
|
2103
|
+
model: str,
|
|
2104
|
+
effort: str,
|
|
2105
|
+
*,
|
|
2106
|
+
unavailable_models: list[str] | None = None,
|
|
2107
|
+
catalog: dict[str, Any] | None = None,
|
|
2108
|
+
benchmarks: dict[str, Any] | None = None,
|
|
2109
|
+
) -> Backup | None:
|
|
2110
|
+
"""The backup for a pick made outside the ladder (a roadmap step's
|
|
2111
|
+
Settings): ``model`` (catalog id or name) run at ``effort`` (a scorer or
|
|
2112
|
+
native level), backed up by the rule a /recommend rung follows
|
|
2113
|
+
(:func:`_backup_for`), over the same pool, so the two never disagree.
|
|
2114
|
+
``task.budget`` sets the posture an unmeasured backup's effort follows.
|
|
2115
|
+
None when the pool holds no other maker's model. ValueError when the
|
|
2116
|
+
model is not one this user-context can reach."""
|
|
2117
|
+
p = _pick(task, user_context_text, model, effort, unavailable_models, catalog, benchmarks)
|
|
2118
|
+
funded = [c for c in p.ranking.candidates if c.funding != "unfunded"]
|
|
2119
|
+
pool = funded or p.ranking.candidates
|
|
2120
|
+
headroom = consumption_headroom(user_context_text or "")
|
|
2121
|
+
settings = effort_settings(
|
|
2122
|
+
pool,
|
|
2123
|
+
p.base,
|
|
2124
|
+
headroom,
|
|
2125
|
+
lam=p.ranking.lam,
|
|
2126
|
+
k=p.ranking.k_points_per_decade,
|
|
2127
|
+
catalog=p.catalog,
|
|
2128
|
+
benchmarks=p.bench,
|
|
2129
|
+
)
|
|
2130
|
+
model_id = p.candidate.model_id
|
|
2131
|
+
at = p.at(p.candidate, p.level if p.measured(model_id) else None)
|
|
2132
|
+
point = FrontierPoint(
|
|
2133
|
+
at, _evidence(p.bench, model_id, FRONTIER_INDEX, at.evidence_level), frontier_price(at)
|
|
2134
|
+
)
|
|
2135
|
+
tier = next(t for t, b in TIER_BUDGET.items() if b == task.budget)
|
|
2136
|
+
return _backup_for(
|
|
2137
|
+
Rung(tier, point, p.level),
|
|
2138
|
+
settings,
|
|
2139
|
+
task,
|
|
2140
|
+
headroom,
|
|
2141
|
+
p.bench,
|
|
2142
|
+
p.catalog,
|
|
2143
|
+
_aa_band(p.bench, p.catalog),
|
|
2144
|
+
)
|
|
2145
|
+
|
|
2146
|
+
|
|
2147
|
+
def _series(model_id: str) -> frozenset[str]:
|
|
2148
|
+
"""A model id's name words, its version numbers and the vendor prefix
|
|
2149
|
+
aside: `gpt-6-sol` and `gpt-6.1-sol` are one series, `gpt-6-sol` and
|
|
2150
|
+
`gpt-6-luna` two. Mirrors update/rating_guard.py same_series."""
|
|
2151
|
+
return frozenset(re.findall(r"[a-z]+", model_id.lower())) - {"claude"}
|
|
2152
|
+
|
|
2153
|
+
|
|
2154
|
+
def _version(model_id: str) -> tuple[int, ...]:
|
|
2155
|
+
return tuple(int(n) for n in re.findall(r"\d+", model_id))
|
|
2156
|
+
|
|
2157
|
+
|
|
2158
|
+
@dataclass(frozen=True)
|
|
2159
|
+
class Successor:
|
|
2160
|
+
"""A newer version of a pick's model that runs in its place
|
|
2161
|
+
(:func:`successor_for_pick`)."""
|
|
2162
|
+
|
|
2163
|
+
candidate: Candidate # the newer model, read at the pick's effort
|
|
2164
|
+
effort: str # the scorer's level
|
|
2165
|
+
native: str | None
|
|
2166
|
+
replaces: str # the pick's model name
|
|
2167
|
+
|
|
2168
|
+
@property
|
|
2169
|
+
def level(self) -> str:
|
|
2170
|
+
return self.native or self.effort
|
|
2171
|
+
|
|
2172
|
+
|
|
2173
|
+
def successor_for_pick(
|
|
2174
|
+
task: Task,
|
|
2175
|
+
user_context_text: str,
|
|
2176
|
+
model: str,
|
|
2177
|
+
effort: str,
|
|
2178
|
+
*,
|
|
2179
|
+
unavailable_models: list[str] | None = None,
|
|
2180
|
+
catalog: dict[str, Any] | None = None,
|
|
2181
|
+
benchmarks: dict[str, Any] | None = None,
|
|
2182
|
+
) -> Successor | None:
|
|
2183
|
+
"""The newer version of ``model`` a pick made outside the ladder should run
|
|
2184
|
+
instead: the same maker's next release in the same series (GPT-6 Sol →
|
|
2185
|
+
GPT-6.1 Sol), funded on the pick's platform, no dearer per token, and, run
|
|
2186
|
+
at ``effort``, level with it in the task's category (:func:`_level_with`)
|
|
2187
|
+
and at least as high on the AA Intelligence Index. The newest such version;
|
|
2188
|
+
None when there is none. ValueError when ``model`` is not one this
|
|
2189
|
+
user-context can reach.
|
|
2190
|
+
|
|
2191
|
+
The selector reads each model's letters at its top effort, and a step that
|
|
2192
|
+
runs lower can reverse them: GPT-6 Sol letters S in coding and GPT-6.1 Sol
|
|
2193
|
+
A on a 3-point SciCode gap at max, yet at medium 6.1 Sol leads it on
|
|
2194
|
+
Terminal-Bench 4.0 by 29 points at the same price. The two are compared
|
|
2195
|
+
at ``effort`` where AA has measured both there, else on their headline
|
|
2196
|
+
rows, so neither is credited with a run the other is not."""
|
|
2197
|
+
p = _pick(task, user_context_text, model, effort, unavailable_models, catalog, benchmarks)
|
|
2198
|
+
old = p.candidate
|
|
2199
|
+
series, version = _series(old.model_id), _version(old.model_id)
|
|
2200
|
+
best: Successor | None = None
|
|
2201
|
+
for c in p.ranking.candidates:
|
|
2202
|
+
if (
|
|
2203
|
+
c.platform_id != old.platform_id
|
|
2204
|
+
or c.provider != old.provider
|
|
2205
|
+
or _series(c.model_id) != series
|
|
2206
|
+
or _version(c.model_id) <= version
|
|
2207
|
+
or c.blended_price_usd > old.blended_price_usd
|
|
2208
|
+
):
|
|
2209
|
+
continue
|
|
2210
|
+
both = p.measured(old.model_id) and p.measured(c.model_id)
|
|
2211
|
+
level = p.level if both else None
|
|
2212
|
+
was, now = p.at(old, level), p.at(c, level)
|
|
2213
|
+
aa_was = _evidence(p.bench, old.model_id, FRONTIER_INDEX, level)
|
|
2214
|
+
aa_now = _evidence(p.bench, c.model_id, FRONTIER_INDEX, level)
|
|
2215
|
+
if not _level_with(now, was, task.category):
|
|
2216
|
+
continue
|
|
2217
|
+
if aa_now is None or (aa_was is not None and aa_now < aa_was):
|
|
2218
|
+
continue
|
|
2219
|
+
if best is not None and (
|
|
2220
|
+
_version(c.model_id),
|
|
2221
|
+
-c.blended_price_usd,
|
|
2222
|
+
) <= (_version(best.candidate.model_id), -best.candidate.blended_price_usd):
|
|
2223
|
+
continue
|
|
2224
|
+
native = native_level(c.platform_id, c.model_id, p.level, catalog=p.catalog)
|
|
2225
|
+
best = Successor(now, p.level, native, old.model_name)
|
|
2226
|
+
return best
|
|
2227
|
+
|
|
2228
|
+
|
|
2229
|
+
# A platform's effort dial as a Settings table names it (docs/settings-display.md):
|
|
2230
|
+
# Intelligence on OpenAI's reasoning surfaces, Effort elsewhere; Cursor has none.
|
|
2231
|
+
_DIAL_NAMES: Final[dict[str, str | None]] = {
|
|
2232
|
+
"codex-cli": "Intelligence",
|
|
2233
|
+
"codex-api": "Intelligence",
|
|
2234
|
+
"openai-api": "Intelligence",
|
|
2235
|
+
"cursor": None,
|
|
2236
|
+
}
|
|
2237
|
+
_LEVEL_WORDS: Final[dict[str, str]] = {
|
|
2238
|
+
"minimal": "Minimal",
|
|
2239
|
+
"low": "Low",
|
|
2240
|
+
"medium": "Medium",
|
|
2241
|
+
"high": "High",
|
|
2242
|
+
"xhigh": "XHigh",
|
|
2243
|
+
"max": "Max",
|
|
2244
|
+
"ultra": "Ultra",
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
|
|
2248
|
+
def backup_cell(backup: Backup | Successor) -> str:
|
|
2249
|
+
"""``backup`` as a roadmap Settings table's Backup cell:
|
|
2250
|
+
``GPT-6.1 Sol — Codex · Intelligence XHigh``. A :class:`Successor` reads
|
|
2251
|
+
the same way, as the model the step runs."""
|
|
2252
|
+
c = backup.candidate
|
|
2253
|
+
dial = _DIAL_NAMES.get(c.platform_id, "Effort")
|
|
2254
|
+
level = _LEVEL_WORDS.get(normalize_level(backup.level), backup.level)
|
|
2255
|
+
head = f"{c.model_name} — {c.platform_name}"
|
|
2256
|
+
return head if dial is None else f"{head} · {dial} {level}"
|
|
2257
|
+
|
|
2258
|
+
|
|
1943
2259
|
def _below(lower: Rung, upper: Rung, catalog: dict[str, Any]) -> Rung:
|
|
1944
2260
|
"""``lower`` run at least one effort level below ``upper`` when both name
|
|
1945
2261
|
the same model on the same platform and that path costs something
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|