roadmodel 0.2.83__tar.gz → 0.2.85__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {roadmodel-0.2.83 → roadmodel-0.2.85}/PKG-INFO +2 -1
  2. {roadmodel-0.2.83 → roadmodel-0.2.85}/README.md +1 -0
  3. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmap-refresh.md +15 -2
  4. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/prompt-phase-roadmap.md +14 -1
  5. {roadmodel-0.2.83 → roadmodel-0.2.85}/pyproject.toml +1 -1
  6. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/prompt-phase-roadmap.md +14 -1
  7. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmap-refresh.md +15 -2
  8. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/__init__.py +1 -1
  9. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/cli.py +124 -11
  10. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/scoring.py +330 -14
  11. {roadmodel-0.2.83 → roadmodel-0.2.85}/.github/README.md +0 -0
  12. {roadmodel-0.2.83 → roadmodel-0.2.85}/.gitignore +0 -0
  13. {roadmodel-0.2.83 → roadmodel-0.2.85}/LICENSE +0 -0
  14. {roadmodel-0.2.83 → roadmodel-0.2.85}/NOTICE +0 -0
  15. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/benchmarks.json +0 -0
  16. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/catalog.json +0 -0
  17. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmap-phase.md +0 -0
  18. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmap-project.md +0 -0
  19. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmap-step.md +0 -0
  20. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/claude-commands/roadmodel-upgrade.md +0 -0
  21. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/context-windows.json +0 -0
  22. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/effort-tokens.json +0 -0
  23. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/model-selector.txt +0 -0
  24. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/model-tier-cost-scale.md +0 -0
  25. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/settings-display.md +0 -0
  26. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/phase-roadmap-template.md +0 -0
  27. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/planning-kit-how-to-use.md +0 -0
  28. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/project-roadmap-template.md +0 -0
  29. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/templates/prompt-project-roadmap.md +0 -0
  30. {roadmodel-0.2.83 → roadmodel-0.2.85}/docs/user-context.example.md +0 -0
  31. {roadmodel-0.2.83 → roadmodel-0.2.85}/hatch_build.py +0 -0
  32. {roadmodel-0.2.83 → roadmodel-0.2.85}/infra/README.md +0 -0
  33. {roadmodel-0.2.83 → roadmodel-0.2.85}/infra/gcp-killswitch/README.md +0 -0
  34. {roadmodel-0.2.83 → roadmodel-0.2.85}/infra/supabase/README.md +0 -0
  35. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/benchmarks.json +0 -0
  36. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/catalog.json +0 -0
  37. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/context-windows.json +0 -0
  38. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/effort-tokens.json +0 -0
  39. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/model-selector.txt +0 -0
  40. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/model-tier-cost-scale.md +0 -0
  41. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/phase-roadmap-template.md +0 -0
  42. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/planning-kit-how-to-use.md +0 -0
  43. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/project-roadmap-template.md +0 -0
  44. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/prompt-project-roadmap.md +0 -0
  45. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmap-phase.md +0 -0
  46. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmap-project.md +0 -0
  47. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmap-step.md +0 -0
  48. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/roadmodel-upgrade.md +0 -0
  49. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/settings-display.md +0 -0
  50. {roadmodel-0.2.83 → roadmodel-0.2.85}/roadmodel/data/user-context.example.md +0 -0
  51. {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/agent_context.py +0 -0
  52. {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/agent_rules.py +0 -0
  53. {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/update_projects.py +0 -0
  54. {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/updater_lock.py +0 -0
  55. {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/upgrade_fleet.py +0 -0
  56. {roadmodel-0.2.83 → roadmodel-0.2.85}/scripts/worktree_memory.py +0 -0
  57. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/__main__.py +0 -0
  58. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/config.py +0 -0
  59. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/cost.py +0 -0
  60. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/errors.py +0 -0
  61. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/mcp_server.py +0 -0
  62. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/private_files.py +0 -0
  63. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/__init__.py +0 -0
  64. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/_safety.py +0 -0
  65. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/anthropic.py +0 -0
  66. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/google.py +0 -0
  67. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/openai.py +0 -0
  68. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/openai_compatible.py +0 -0
  69. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/providers/registry.py +0 -0
  70. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/recommend.py +0 -0
  71. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/usage.py +0 -0
  72. {roadmodel-0.2.83 → roadmodel-0.2.85}/src/roadmodel/user_context.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: roadmodel
3
- Version: 0.2.83
3
+ Version: 0.2.85
4
4
  Summary: BYO-key CLI that recommends the right AI model, platform, and settings for a prompt.
5
5
  Project-URL: Homepage, https://roadmodel.ai
6
6
  Project-URL: Repository, https://github.com/nathanramoscfa/roadmodel
@@ -607,6 +607,7 @@ server and the reasoning-effort default in parity automatically.
607
607
  | `roadmodel recommend --model` | Override the specific model ID on the chosen provider. |
608
608
  | `roadmodel recommend --user-context PATH` | Override the user-context.md location for this invocation. |
609
609
  | `roadmodel score --category C --complexity X` | Rank (model, platform, effort) candidates in code for a classified task — no engine call. See [docs/scoring-model.md](docs/scoring-model.md). |
610
+ | `roadmodel backup --category C --complexity X --model M --effort E` | Name the cross-maker backup for a pick, by the rule /recommend uses — no engine call. Prints the roadmap Settings Backup cell, and first a `MODEL` cell when a newer version of the pick's model (same platform, no dearer) scores level or better at its effort. |
610
611
  | `roadmodel catalog show` | Print the bundled `model-selector.txt` (use `--doc tier-cost-scale` for the price doc). |
611
612
  | `roadmodel catalog path` | Print the on-disk path of the bundled catalog document (same `--doc` flag). |
612
613
  | `roadmodel context show` | Print the resolved `user-context.md` file. |
@@ -356,6 +356,7 @@ server and the reasoning-effort default in parity automatically.
356
356
  | `roadmodel recommend --model` | Override the specific model ID on the chosen provider. |
357
357
  | `roadmodel recommend --user-context PATH` | Override the user-context.md location for this invocation. |
358
358
  | `roadmodel score --category C --complexity X` | Rank (model, platform, effort) candidates in code for a classified task — no engine call. See [docs/scoring-model.md](docs/scoring-model.md). |
359
+ | `roadmodel backup --category C --complexity X --model M --effort E` | Name the cross-maker backup for a pick, by the rule /recommend uses — no engine call. Prints the roadmap Settings Backup cell, and first a `MODEL` cell when a newer version of the pick's model (same platform, no dearer) scores level or better at its effort. |
359
360
  | `roadmodel catalog show` | Print the bundled `model-selector.txt` (use `--doc tier-cost-scale` for the price doc). |
360
361
  | `roadmodel catalog path` | Print the on-disk path of the bundled catalog document (same `--doc` flag). |
361
362
  | `roadmodel context show` | Print the resolved `user-context.md` file. |
@@ -180,8 +180,21 @@ exclusion in the selector, and include a backup model.
180
180
  A changed backup alone counts as a difference; name it in that line.
181
181
  - Every table carries a `Backup` row directly under `Model`: the
182
182
  selector's BACKUP with its own platform and dial in one cell
183
- (`GPT-5.6 Terra — Codex · Intelligence Medium`), or `None`. A table
184
- written before roadmodel 0.2.41 has no such row: add it from the
183
+ (`GPT-5.6 Terra — Codex · Intelligence Medium`), or `None`. Where
184
+ `roadmodel` is installed, the cell is the one `roadmodel backup
185
+ --category <c> --complexity <x> [--novel] --model <the step's model>
186
+ --effort <the step's effort>` prints for the step's classification —
187
+ the rule /recommend backs a pick up by, which takes the cheaper of two
188
+ substitutes the benchmarks cannot tell apart — and it outranks the
189
+ selector's BACKUP. When the command also prints a `MODEL` line, that
190
+ cell replaces the step's model — a newer version of the same model
191
+ (GPT-6 Sol → GPT-6.1 Sol), on the same platform, no dearer, that
192
+ scores level or better at the step's effort; the selector's letters
193
+ are measured at each model's top effort, and a step that runs lower
194
+ can reverse them. That is a difference: rewrite the table, its Model
195
+ rationale and its selection block with the `MODEL` cell, keep the
196
+ Backup cell printed with it, and add the `Settings updated` line.
197
+ A table written before roadmodel 0.2.41 has no such row: add it from the
185
198
  backup the step already names (its Model rationale's closing
186
199
  sentence, or its entry in the Model selection blocks) — a layout
187
200
  fix, not a new pick, so it takes no `Settings updated` line.
@@ -103,7 +103,20 @@ selector in `@planning/model-selector.txt` (prices from
103
103
  You are the engine — do not call any external API. Honor every
104
104
  availability exclusion in the selector, and write each step's backup
105
105
  model into its Settings table's Backup row, with the backup's own
106
- platform and dial.
106
+ platform and dial. Where `roadmodel` is installed, the Backup row is
107
+ the cell `roadmodel backup --category <c> --complexity <x> [--novel]
108
+ --model <the step's model> --effort <the step's effort>` prints for the
109
+ step's own classification: the rule /recommend backs a pick up by, in
110
+ code, which takes the cheaper of two substitutes the benchmarks cannot
111
+ tell apart. The selector's BACKUP stands only where it is not
112
+ installed. When the command also prints a `MODEL` line, that cell
113
+ replaces the step's model: a newer version of the same model (GPT-6 Sol
114
+ → GPT-6.1 Sol), on the same platform, no dearer, that scores level or
115
+ better at the step's effort. The selector's letters are measured at each
116
+ model's top effort, and a step that runs lower can reverse them. Write
117
+ the `MODEL` cell into the Model row, the Model rationale and the step's
118
+ selection block, citing the newer model's figures at the step's effort,
119
+ and keep the Backup cell printed with it.
107
120
 
108
121
  Honor the template's style rules: 80-column prose, zero `{{...}}`
109
122
  tokens left, `**Status:** Not started` directly under the title and
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "roadmodel"
7
- version = "0.2.83"
7
+ version = "0.2.85"
8
8
  description = "BYO-key CLI that recommends the right AI model, platform, and settings for a prompt."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -103,7 +103,20 @@ selector in `@planning/model-selector.txt` (prices from
103
103
  You are the engine — do not call any external API. Honor every
104
104
  availability exclusion in the selector, and write each step's backup
105
105
  model into its Settings table's Backup row, with the backup's own
106
- platform and dial.
106
+ platform and dial. Where `roadmodel` is installed, the Backup row is
107
+ the cell `roadmodel backup --category <c> --complexity <x> [--novel]
108
+ --model <the step's model> --effort <the step's effort>` prints for the
109
+ step's own classification: the rule /recommend backs a pick up by, in
110
+ code, which takes the cheaper of two substitutes the benchmarks cannot
111
+ tell apart. The selector's BACKUP stands only where it is not
112
+ installed. When the command also prints a `MODEL` line, that cell
113
+ replaces the step's model: a newer version of the same model (GPT-6 Sol
114
+ → GPT-6.1 Sol), on the same platform, no dearer, that scores level or
115
+ better at the step's effort. The selector's letters are measured at each
116
+ model's top effort, and a step that runs lower can reverse them. Write
117
+ the `MODEL` cell into the Model row, the Model rationale and the step's
118
+ selection block, citing the newer model's figures at the step's effort,
119
+ and keep the Backup cell printed with it.
107
120
 
108
121
  Honor the template's style rules: 80-column prose, zero `{{...}}`
109
122
  tokens left, `**Status:** Not started` directly under the title and
@@ -180,8 +180,21 @@ exclusion in the selector, and include a backup model.
180
180
  A changed backup alone counts as a difference; name it in that line.
181
181
  - Every table carries a `Backup` row directly under `Model`: the
182
182
  selector's BACKUP with its own platform and dial in one cell
183
- (`GPT-5.6 Terra — Codex · Intelligence Medium`), or `None`. A table
184
- written before roadmodel 0.2.41 has no such row: add it from the
183
+ (`GPT-5.6 Terra — Codex · Intelligence Medium`), or `None`. Where
184
+ `roadmodel` is installed, the cell is the one `roadmodel backup
185
+ --category <c> --complexity <x> [--novel] --model <the step's model>
186
+ --effort <the step's effort>` prints for the step's classification —
187
+ the rule /recommend backs a pick up by, which takes the cheaper of two
188
+ substitutes the benchmarks cannot tell apart — and it outranks the
189
+ selector's BACKUP. When the command also prints a `MODEL` line, that
190
+ cell replaces the step's model — a newer version of the same model
191
+ (GPT-6 Sol → GPT-6.1 Sol), on the same platform, no dearer, that
192
+ scores level or better at the step's effort; the selector's letters
193
+ are measured at each model's top effort, and a step that runs lower
194
+ can reverse them. That is a difference: rewrite the table, its Model
195
+ rationale and its selection block with the `MODEL` cell, keep the
196
+ Backup cell printed with it, and add the `Settings updated` line.
197
+ A table written before roadmodel 0.2.41 has no such row: add it from the
185
198
  backup the step already names (its Model rationale's closing
186
199
  sentence, or its entry in the Model selection blocks) — a layout
187
200
  fix, not a new pick, so it takes no `Settings updated` line.
@@ -1,2 +1,2 @@
1
1
  # src/roadmodel/__init__.py
2
- __version__ = "0.2.83"
2
+ __version__ = "0.2.85"
@@ -418,6 +418,23 @@ def recommend(
418
418
  click.echo(_format_structured_text(structured), nl=False)
419
419
 
420
420
 
421
+ def _scoring_user_context(user_context_path: Path | None) -> str:
422
+ """The user-context text the scorer reads: ``--user-context``, else
423
+ ROADMODEL_USER_CONTEXT, else the default path; empty when none exists. A
424
+ path named explicitly that does not exist is an error."""
425
+ if user_context_path is not None and not user_context_path.expanduser().exists():
426
+ raise UserContextNotFoundError(user_context_path.expanduser())
427
+ env_user_context = os.environ.get("ROADMODEL_USER_CONTEXT")
428
+ if (
429
+ user_context_path is None
430
+ and env_user_context
431
+ and not Path(env_user_context).expanduser().exists()
432
+ ):
433
+ raise UserContextNotFoundError(Path(env_user_context).expanduser())
434
+ resolved = user_context.resolve(cli_path=user_context_path)
435
+ return user_context.read(resolved) if resolved.exists() else ""
436
+
437
+
421
438
  @cli.command("score")
422
439
  @click.option(
423
440
  "--category",
@@ -474,17 +491,7 @@ def score_command(
474
491
  funding, consumption headroom and Usage-pool status from user-context.md.
475
492
  Every term is printed so a pick can be audited line by line.
476
493
  """
477
- if user_context_path is not None and not user_context_path.expanduser().exists():
478
- raise UserContextNotFoundError(user_context_path.expanduser())
479
- env_user_context = os.environ.get("ROADMODEL_USER_CONTEXT")
480
- if (
481
- user_context_path is None
482
- and env_user_context
483
- and not Path(env_user_context).expanduser().exists()
484
- ):
485
- raise UserContextNotFoundError(Path(env_user_context).expanduser())
486
- resolved = user_context.resolve(cli_path=user_context_path)
487
- text = user_context.read(resolved) if resolved.exists() else ""
494
+ text = _scoring_user_context(user_context_path)
488
495
  posture = (budget or scoring.declared_budget(text)).lower()
489
496
  try:
490
497
  task = scoring.Task(
@@ -504,6 +511,112 @@ def score_command(
504
511
  click.echo(scoring.render_text(ranking, limit=top))
505
512
 
506
513
 
514
+ @cli.command("backup")
515
+ @click.option(
516
+ "--category",
517
+ required=True,
518
+ type=click.Choice(list(scoring.CATEGORIES), case_sensitive=False),
519
+ help="PRIMARY task category of the pick.",
520
+ )
521
+ @click.option(
522
+ "--complexity",
523
+ required=True,
524
+ type=click.Choice(list(scoring.COMPLEXITIES), case_sensitive=False),
525
+ help="Overall complexity of the pick's task.",
526
+ )
527
+ @click.option("--novel", is_flag=True, default=False, help="High complexity AND novel.")
528
+ @click.option("--model", required=True, help="The pick's model: catalog id or name.")
529
+ @click.option("--effort", required=True, help="The pick's effort, as its dial names it.")
530
+ @click.option(
531
+ "--budget",
532
+ type=click.Choice(list(scoring.BUDGETS), case_sensitive=False),
533
+ default=None,
534
+ help="Budget posture; defaults to the user-context's declared Budget priority.",
535
+ )
536
+ @click.option(
537
+ "--output",
538
+ type=click.Choice(["text", "json"], case_sensitive=False),
539
+ default="text",
540
+ show_default=True,
541
+ )
542
+ @click.option(
543
+ "--user-context",
544
+ "user_context_path",
545
+ type=click.Path(path_type=Path),
546
+ default=None,
547
+ help="Path to user-context.md override.",
548
+ )
549
+ def backup_command(
550
+ category: str,
551
+ complexity: str,
552
+ novel: bool,
553
+ model: str,
554
+ effort: str,
555
+ budget: str | None,
556
+ output: str,
557
+ user_context_path: Path | None,
558
+ ) -> None:
559
+ """Name the cross-maker backup for a pick, in code — no engine call.
560
+
561
+ The rule /recommend backs each pick up by: another maker's model that does
562
+ the job as well, for the least; where the benchmarks cannot tell two
563
+ substitutes apart, the cheaper. Prints the roadmap Settings Backup cell.
564
+
565
+ When a newer version of the pick's model (same maker and series, on the
566
+ same platform, no dearer) scores level or better at the pick's effort, it
567
+ first prints that model on a MODEL line: the step runs it instead, and the
568
+ backup printed is its backup.
569
+ """
570
+ text = _scoring_user_context(user_context_path)
571
+ posture = (budget or scoring.declared_budget(text)).lower()
572
+ try:
573
+ task = scoring.Task(category.lower(), complexity.lower(), novel, posture)
574
+ successor = scoring.successor_for_pick(task, text, model, effort)
575
+ runs = successor.candidate.model_id if successor is not None else model
576
+ backup = scoring.backup_for_pick(task, text, runs, effort)
577
+ except ValueError as exc:
578
+ raise click.ClickException(str(exc)) from exc
579
+ cell = scoring.backup_cell(backup) if backup is not None else "None"
580
+ model_cell = scoring.backup_cell(successor) if successor is not None else None
581
+ if output.lower() == "json":
582
+ payload: dict[str, Any] = {
583
+ "model": None,
584
+ "model_cell": model_cell,
585
+ "backup": None,
586
+ "cell": cell,
587
+ }
588
+ if successor is not None:
589
+ s = successor.candidate
590
+ payload["model"] = {
591
+ "model_id": s.model_id,
592
+ "model": s.model_name,
593
+ "platform_id": s.platform_id,
594
+ "platform": s.platform_name,
595
+ "effort": successor.effort,
596
+ "native_effort": successor.native,
597
+ "replaces": successor.replaces,
598
+ }
599
+ if backup is not None:
600
+ c = backup.candidate
601
+ payload["backup"] = {
602
+ "model_id": c.model_id,
603
+ "model": c.model_name,
604
+ "platform_id": c.platform_id,
605
+ "platform": c.platform_name,
606
+ "effort": backup.effort,
607
+ "native_effort": backup.native,
608
+ }
609
+ click.echo(json.dumps(payload, indent=2, ensure_ascii=False))
610
+ return
611
+ if successor is not None:
612
+ click.echo(f"MODEL {model_cell}")
613
+ click.echo(
614
+ f" replaces {successor.replaces}: a newer version, no dearer, "
615
+ "level or better at this effort"
616
+ )
617
+ click.echo(f"BACKUP {cell}")
618
+
619
+
507
620
  @cli.command("cost")
508
621
  @click.option("--model", required=True, help="Catalog model id or display name.")
509
622
  @click.option("--platform", required=True, help="Access method id or display name.")
@@ -137,8 +137,19 @@ _RANK_SCALED_KEYS: Final[frozenset[str]] = frozenset(
137
137
 
138
138
  # Editorial letter → quality points. Midpoints of five equal bands.
139
139
  LETTER_QUALITY: Final[dict[str, float]] = {"S": 90.0, "A": 70.0, "B": 50.0, "C": 30.0, "D": 10.0}
140
- # When AA has measured the model: weight on the scaled evidence vs the letter.
140
+ # When AA has measured the model in speed: weight on the scaled evidence vs
141
+ # the letter (speed's letter is an estimate, not derived from tokens/s).
141
142
  EVIDENCE_WEIGHT: Final[float] = 0.7
143
+ # The five derived categories: update/derive_ratings.py letters them from the
144
+ # very benchmark CATEGORY_EVIDENCE names, so a measured model's letter there
145
+ # is a staircase of evidence already counted. Blending it in adds no
146
+ # information, only a cliff at each band edge: GPT-6 Astra's 52.7 on the AA
147
+ # Index reads S and GPT-6.1 Sol's 51.8 reads A, six quality points for 0.9
148
+ # of index. A model measured there is scored on its evidence alone; its
149
+ # letter stands in only where AA has not measured it.
150
+ DERIVED_LETTER_CATEGORIES: Final[frozenset[str]] = frozenset(
151
+ {"coding", "planning", "agentic", "long-context", "knowledge"}
152
+ )
142
153
  # A letter with no AA measurement behind it is an editorial prior, not
143
154
  # evidence: discount it so a measured model wins an otherwise-tied letter.
144
155
  UNMEASURED_DISCOUNT: Final[float] = 5.0
@@ -886,7 +897,8 @@ def _quality(
886
897
  level: str | None = None,
887
898
  ) -> tuple[float, str, str]:
888
899
  """The model's quality for the task's category: its headline evidence, or
889
- with ``level`` its evidence at that effort, blended with its letter."""
900
+ with ``level`` its evidence at that effort; in speed blended with its
901
+ letter, in a derived category (:data:`DERIVED_LETTER_CATEGORIES`) alone."""
890
902
  raw_tiers = model.get("tiers")
891
903
  tiers: dict[str, Any] = raw_tiers if isinstance(raw_tiers, dict) else {}
892
904
  letter = str(tiers.get(task.category, "C")).upper()
@@ -898,6 +910,8 @@ def _quality(
898
910
  points = _evidence_points(str(model.get("id", "")), task, bench, scale, level)
899
911
  if points is None:
900
912
  return max(0.0, letter_q - UNMEASURED_DISCOUNT), "letter", letter
913
+ if task.category in DERIVED_LETTER_CATEGORIES:
914
+ return points, f"aa:{CATEGORY_EVIDENCE[task.category]}", letter
901
915
  q = EVIDENCE_WEIGHT * points + (1.0 - EVIDENCE_WEIGHT) * letter_q
902
916
  return q, f"aa:{CATEGORY_EVIDENCE[task.category]}+letter", letter
903
917
 
@@ -1244,10 +1258,24 @@ def rank(
1244
1258
  backup: Candidate | None = None
1245
1259
  warning: str | None = None
1246
1260
  if primary is not None:
1247
- for c in candidates[1:]:
1248
- if c.provider != primary.provider and c.funding != "unfunded":
1249
- backup = c
1250
- break
1261
+ others = [
1262
+ c for c in candidates[1:] if c.provider != primary.provider and c.funding != "unfunded"
1263
+ ]
1264
+ if others:
1265
+ # The best-scoring other maker's model, else the cheapest one the
1266
+ # evidence cannot tell from it (:func:`_near_tied`), whatever the
1267
+ # posture: at budget best the score barely prices a model, and a
1268
+ # fraction of an AA point within noise would buy a dearer backup.
1269
+ ref = others[0]
1270
+ band = _aa_band(bench, cat)
1271
+
1272
+ def aa(c: Candidate) -> float | None:
1273
+ return _evidence(bench, c.model_id, FRONTIER_INDEX, c.evidence_level)
1274
+
1275
+ backup = min(
1276
+ (c for c in others if _near_tied(c, aa(c), ref, aa(ref), task.category, band)),
1277
+ key=lambda c: (c.effective_cost_usd, -c.score, c.model_id),
1278
+ )
1251
1279
  if backup is None:
1252
1280
  funded_providers = sorted({c.provider for c in candidates if c.funding != "unfunded"})
1253
1281
  warning = (
@@ -1452,6 +1480,33 @@ def _level_with(a: Candidate, b: Candidate, category: str) -> bool:
1452
1480
  return a.evidence_points >= b.evidence_points - TIE_BAND.get(category, DEFAULT_TIE_BAND)
1453
1481
 
1454
1482
 
1483
+ def _near_tied(
1484
+ a: Candidate,
1485
+ a_aa: float | None,
1486
+ b: Candidate,
1487
+ b_aa: float | None,
1488
+ category: str,
1489
+ aa_band: float,
1490
+ ) -> bool:
1491
+ """Whether the evidence cannot tell ``a`` from ``b``: each level with the
1492
+ other in the category (:func:`_level_with`, both ways), and their AA
1493
+ Indexes (``a_aa``, ``b_aa``) no more than ``aa_band`` apart where both are
1494
+ measured. One-way levelness is not enough: every stronger model is level
1495
+ with a weaker one."""
1496
+ return (
1497
+ _level_with(a, b, category)
1498
+ and _level_with(b, a, category)
1499
+ and (a_aa is None or b_aa is None or abs(a_aa - b_aa) <= aa_band)
1500
+ )
1501
+
1502
+
1503
+ def _aa_band(bench: dict[str, Any], catalog: dict[str, Any]) -> float:
1504
+ """The AA Index's noise band in its own points: DEFAULT_TIE_BAND on the
1505
+ 0–100 scale it is read on, mapped back onto the index's measured range."""
1506
+ gscale = _evidence_scale(catalog, bench, FRONTIER_INDEX)
1507
+ return 0.0 if gscale is None else DEFAULT_TIE_BAND * (gscale.hi - gscale.lo) / 100.0
1508
+
1509
+
1455
1510
  def _tie_break(p: FrontierPoint, category: str) -> tuple[float, float]:
1456
1511
  """Between points level in the category: the higher AA Index, then the
1457
1512
  cheaper. Speed work too: QUALITY is the ladder's top, and COST and
@@ -1680,8 +1735,9 @@ def ladder(
1680
1735
  adequate point above COST.
1681
1736
  - Each rung carries a BACKUP from another maker (:func:`_backup_for`):
1682
1737
  the cheapest point on the other makers' frontier that does the job as
1683
- well as the rung, else the nearest below it. ``backup_warning`` says why
1684
- a rung has none.
1738
+ well as the rung, else the strongest substitute, giving way to a
1739
+ cheaper one the evidence cannot tell from it. ``backup_warning`` says
1740
+ why a rung has none.
1685
1741
 
1686
1742
  None when no pool model carries an AA Index (no frontier to read)."""
1687
1743
  cat = catalog if catalog is not None else _cost._load_catalog()
@@ -1826,8 +1882,7 @@ def ladder(
1826
1882
  cost_rung = settle(_below(cost_rung, balanced_rung, cat))
1827
1883
  # General capability is level within the same noise band, read on the
1828
1884
  # AA Index's own 0–100 scale (the planning evidence).
1829
- gscale = _evidence_scale(cat, bench, FRONTIER_INDEX)
1830
- aa_band = 0.0 if gscale is None else DEFAULT_TIE_BAND * (gscale.hi - gscale.lo) / 100.0
1885
+ aa_band = _aa_band(bench, cat)
1831
1886
  rungs = {
1832
1887
  r.tier: replace(r, backup=_backup_for(r, settings, task, headroom, bench, cat, aa_band))
1833
1888
  for r in (cost_rung, balanced_rung, quality_rung)
@@ -1894,10 +1949,13 @@ def _backup_for(
1894
1949
  those already paid for ahead of per-token ones. A point matches the rung
1895
1950
  when it stands level with it in the task's category
1896
1951
  (:func:`_level_with`) and in general capability (an AA Index no more than
1897
- ``aa_band`` below). The backup is the cheapest match; with none, the
1952
+ ``aa_band`` below). The reference is the cheapest match; with none, the
1898
1953
  substitute the QUALITY rule would pick (:func:`_top`: strongest in the
1899
1954
  category, a near tie going to the higher AA Index); with no adequate
1900
- point, the one that falls shortest of the bar. It runs at its
1955
+ point, the one that falls shortest of the bar. The backup is the cheapest
1956
+ of the points within the noise bands of the reference both ways
1957
+ (:func:`_near_tied`): the evidence cannot tell them apart, so paying more
1958
+ buys nothing it can show. It runs at its
1901
1959
  point's measured effort, else at the rung's posture's effort, on its own
1902
1960
  platform. None when the pool holds no other
1903
1961
  maker's model."""
@@ -1924,13 +1982,26 @@ def _backup_for(
1924
1982
  if _level_with(p.candidate, r.candidate, task.category)
1925
1983
  and (r.aa_index is None or aa(p) >= r.aa_index - aa_band)
1926
1984
  ]
1927
- chosen = (
1985
+ ref = (
1928
1986
  min(matches, key=lambda p: (p.price_usd, -aa(p)))
1929
1987
  if matches
1930
1988
  else _top(near, task.category)
1931
1989
  )
1932
1990
  else:
1933
- chosen = max(front, key=_nearest)
1991
+ ref = max(front, key=_nearest)
1992
+ # Of the points the evidence cannot tell from the reference, the cheapest:
1993
+ # a near-identical model at a quarter of the price is not passed over for
1994
+ # a fraction of an AA point. A peer may fall short of the bar where the
1995
+ # reference clears it, by less than the noise: a shortfall the evidence
1996
+ # cannot show. A prepaid reference keeps a prepaid backup.
1997
+ prepaid = ref.candidate.scarcity < 1.0
1998
+ peers = [
1999
+ p
2000
+ for p in front
2001
+ if (p.candidate.scarcity < 1.0 or not prepaid)
2002
+ and _near_tied(p.candidate, p.aa_index, ref.candidate, ref.aa_index, task.category, aa_band)
2003
+ ]
2004
+ chosen = min(peers, key=lambda p: (p.price_usd, -p.candidate.score))
1934
2005
  c = chosen.candidate
1935
2006
  if c.evidence_level is not None:
1936
2007
  effort = c.evidence_level
@@ -1940,6 +2011,251 @@ def _backup_for(
1940
2011
  return Backup(chosen, effort, native_level(c.platform_id, c.model_id, effort, catalog=catalog))
1941
2012
 
1942
2013
 
2014
+ def _model_ref_id(ref: str, catalog: dict[str, Any]) -> str:
2015
+ """The catalog id ``ref`` names, matched spelling-free as
2016
+ recommend._model_key does ("Claude Opus 5.5", "Opus 5.5" and
2017
+ "claude-opus-5-5" are one model); ``ref`` itself on a miss."""
2018
+
2019
+ def key(text: str) -> str:
2020
+ words = re.split(r"[\s_.-]+", text.lower())
2021
+ return " ".join(sorted(w for w in words if w and w != "claude"))
2022
+
2023
+ want = key(ref)
2024
+ for m in catalog.get("models", []):
2025
+ if isinstance(m, dict) and want in {key(str(m.get("id", ""))), key(str(m.get("name", "")))}:
2026
+ return str(m.get("id", ""))
2027
+ return ref
2028
+
2029
+
2030
+ @dataclass(frozen=True)
2031
+ class _Pick:
2032
+ """A pick made outside the ladder (a roadmap step's Settings), with the
2033
+ pool and scales it is read against (:func:`_pick`)."""
2034
+
2035
+ candidate: Candidate # the model, on the platform the pool reaches it best
2036
+ level: str # the scorer's level it runs at
2037
+ ranking: Ranking
2038
+ base: Task
2039
+ models: dict[str, dict[str, Any]]
2040
+ bench: dict[str, Any]
2041
+ composite: dict[str, Any]
2042
+ scales: tuple[EvidenceScale | None, EvidenceScale | None]
2043
+ catalog: dict[str, Any]
2044
+
2045
+ def at(self, c: Candidate, level: str | None) -> Candidate:
2046
+ """``c`` read at ``level`` (None: its headline row; see :func:`_at_level`)."""
2047
+ return _at_level(
2048
+ c,
2049
+ level,
2050
+ self.models[c.model_id],
2051
+ self.base,
2052
+ self.composite,
2053
+ self.scales,
2054
+ self.ranking.lam,
2055
+ self.ranking.k_points_per_decade,
2056
+ )
2057
+
2058
+ def measured(self, model_id: str) -> bool:
2059
+ return self.level in measured_levels(self.bench, model_id)
2060
+
2061
+
2062
+ def _pick(
2063
+ task: Task,
2064
+ user_context_text: str,
2065
+ model: str,
2066
+ effort: str,
2067
+ unavailable_models: list[str] | None,
2068
+ catalog: dict[str, Any] | None,
2069
+ benchmarks: dict[str, Any] | None,
2070
+ ) -> _Pick:
2071
+ """``model`` (catalog id or name) run at ``effort`` (a scorer or native
2072
+ level) for ``task``, over the pool /recommend ranks. ValueError when the
2073
+ model is not one this user-context can reach."""
2074
+ cat = catalog if catalog is not None else _cost._load_catalog()
2075
+ bench = benchmarks if benchmarks is not None else _load_benchmarks()
2076
+ model_id = _model_ref_id(model, cat)
2077
+ base = Task(task.category, task.complexity, task.novel, "balanced")
2078
+ ranking = rank(
2079
+ base,
2080
+ user_context_text,
2081
+ unavailable_models=unavailable_models,
2082
+ catalog=cat,
2083
+ benchmarks=bench,
2084
+ effort_aware=False,
2085
+ )
2086
+ picked = next((c for c in ranking.candidates if c.model_id == model_id), None)
2087
+ models = {str(m.get("id", "")): m for m in cat.get("models", []) if isinstance(m, dict)}
2088
+ if picked is None or model_id not in models:
2089
+ raise ValueError(f"{model!r} is not a model this user-context can reach")
2090
+ composite = with_composites(bench)
2091
+ scales = (
2092
+ _evidence_scale(cat, composite, CATEGORY_EVIDENCE[base.category]),
2093
+ _evidence_scale(cat, composite, CATEGORY_EVIDENCE["planning"]),
2094
+ )
2095
+ return _Pick(
2096
+ picked, _scorer_level(effort), ranking, base, models, bench, composite, scales, cat
2097
+ )
2098
+
2099
+
2100
+ def backup_for_pick(
2101
+ task: Task,
2102
+ user_context_text: str,
2103
+ model: str,
2104
+ effort: str,
2105
+ *,
2106
+ unavailable_models: list[str] | None = None,
2107
+ catalog: dict[str, Any] | None = None,
2108
+ benchmarks: dict[str, Any] | None = None,
2109
+ ) -> Backup | None:
2110
+ """The backup for a pick made outside the ladder (a roadmap step's
2111
+ Settings): ``model`` (catalog id or name) run at ``effort`` (a scorer or
2112
+ native level), backed up by the rule a /recommend rung follows
2113
+ (:func:`_backup_for`), over the same pool, so the two never disagree.
2114
+ ``task.budget`` sets the posture an unmeasured backup's effort follows.
2115
+ None when the pool holds no other maker's model. ValueError when the
2116
+ model is not one this user-context can reach."""
2117
+ p = _pick(task, user_context_text, model, effort, unavailable_models, catalog, benchmarks)
2118
+ funded = [c for c in p.ranking.candidates if c.funding != "unfunded"]
2119
+ pool = funded or p.ranking.candidates
2120
+ headroom = consumption_headroom(user_context_text or "")
2121
+ settings = effort_settings(
2122
+ pool,
2123
+ p.base,
2124
+ headroom,
2125
+ lam=p.ranking.lam,
2126
+ k=p.ranking.k_points_per_decade,
2127
+ catalog=p.catalog,
2128
+ benchmarks=p.bench,
2129
+ )
2130
+ model_id = p.candidate.model_id
2131
+ at = p.at(p.candidate, p.level if p.measured(model_id) else None)
2132
+ point = FrontierPoint(
2133
+ at, _evidence(p.bench, model_id, FRONTIER_INDEX, at.evidence_level), frontier_price(at)
2134
+ )
2135
+ tier = next(t for t, b in TIER_BUDGET.items() if b == task.budget)
2136
+ return _backup_for(
2137
+ Rung(tier, point, p.level),
2138
+ settings,
2139
+ task,
2140
+ headroom,
2141
+ p.bench,
2142
+ p.catalog,
2143
+ _aa_band(p.bench, p.catalog),
2144
+ )
2145
+
2146
+
2147
+ def _series(model_id: str) -> frozenset[str]:
2148
+ """A model id's name words, its version numbers and the vendor prefix
2149
+ aside: `gpt-6-sol` and `gpt-6.1-sol` are one series, `gpt-6-sol` and
2150
+ `gpt-6-luna` two. Mirrors update/rating_guard.py same_series."""
2151
+ return frozenset(re.findall(r"[a-z]+", model_id.lower())) - {"claude"}
2152
+
2153
+
2154
+ def _version(model_id: str) -> tuple[int, ...]:
2155
+ return tuple(int(n) for n in re.findall(r"\d+", model_id))
2156
+
2157
+
2158
+ @dataclass(frozen=True)
2159
+ class Successor:
2160
+ """A newer version of a pick's model that runs in its place
2161
+ (:func:`successor_for_pick`)."""
2162
+
2163
+ candidate: Candidate # the newer model, read at the pick's effort
2164
+ effort: str # the scorer's level
2165
+ native: str | None
2166
+ replaces: str # the pick's model name
2167
+
2168
+ @property
2169
+ def level(self) -> str:
2170
+ return self.native or self.effort
2171
+
2172
+
2173
+ def successor_for_pick(
2174
+ task: Task,
2175
+ user_context_text: str,
2176
+ model: str,
2177
+ effort: str,
2178
+ *,
2179
+ unavailable_models: list[str] | None = None,
2180
+ catalog: dict[str, Any] | None = None,
2181
+ benchmarks: dict[str, Any] | None = None,
2182
+ ) -> Successor | None:
2183
+ """The newer version of ``model`` a pick made outside the ladder should run
2184
+ instead: the same maker's next release in the same series (GPT-6 Sol →
2185
+ GPT-6.1 Sol), funded on the pick's platform, no dearer per token, and, run
2186
+ at ``effort``, level with it in the task's category (:func:`_level_with`)
2187
+ and at least as high on the AA Intelligence Index. The newest such version;
2188
+ None when there is none. ValueError when ``model`` is not one this
2189
+ user-context can reach.
2190
+
2191
+ The selector reads each model's letters at its top effort, and a step that
2192
+ runs lower can reverse them: GPT-6 Sol letters S in coding and GPT-6.1 Sol
2193
+ A on a 3-point SciCode gap at max, yet at medium 6.1 Sol leads it on
2194
+ Terminal-Bench 4.0 by 29 points at the same price. The two are compared
2195
+ at ``effort`` where AA has measured both there, else on their headline
2196
+ rows, so neither is credited with a run the other is not."""
2197
+ p = _pick(task, user_context_text, model, effort, unavailable_models, catalog, benchmarks)
2198
+ old = p.candidate
2199
+ series, version = _series(old.model_id), _version(old.model_id)
2200
+ best: Successor | None = None
2201
+ for c in p.ranking.candidates:
2202
+ if (
2203
+ c.platform_id != old.platform_id
2204
+ or c.provider != old.provider
2205
+ or _series(c.model_id) != series
2206
+ or _version(c.model_id) <= version
2207
+ or c.blended_price_usd > old.blended_price_usd
2208
+ ):
2209
+ continue
2210
+ both = p.measured(old.model_id) and p.measured(c.model_id)
2211
+ level = p.level if both else None
2212
+ was, now = p.at(old, level), p.at(c, level)
2213
+ aa_was = _evidence(p.bench, old.model_id, FRONTIER_INDEX, level)
2214
+ aa_now = _evidence(p.bench, c.model_id, FRONTIER_INDEX, level)
2215
+ if not _level_with(now, was, task.category):
2216
+ continue
2217
+ if aa_now is None or (aa_was is not None and aa_now < aa_was):
2218
+ continue
2219
+ if best is not None and (
2220
+ _version(c.model_id),
2221
+ -c.blended_price_usd,
2222
+ ) <= (_version(best.candidate.model_id), -best.candidate.blended_price_usd):
2223
+ continue
2224
+ native = native_level(c.platform_id, c.model_id, p.level, catalog=p.catalog)
2225
+ best = Successor(now, p.level, native, old.model_name)
2226
+ return best
2227
+
2228
+
2229
+ # A platform's effort dial as a Settings table names it (docs/settings-display.md):
2230
+ # Intelligence on OpenAI's reasoning surfaces, Effort elsewhere; Cursor has none.
2231
+ _DIAL_NAMES: Final[dict[str, str | None]] = {
2232
+ "codex-cli": "Intelligence",
2233
+ "codex-api": "Intelligence",
2234
+ "openai-api": "Intelligence",
2235
+ "cursor": None,
2236
+ }
2237
+ _LEVEL_WORDS: Final[dict[str, str]] = {
2238
+ "minimal": "Minimal",
2239
+ "low": "Low",
2240
+ "medium": "Medium",
2241
+ "high": "High",
2242
+ "xhigh": "XHigh",
2243
+ "max": "Max",
2244
+ "ultra": "Ultra",
2245
+ }
2246
+
2247
+
2248
+ def backup_cell(backup: Backup | Successor) -> str:
2249
+ """``backup`` as a roadmap Settings table's Backup cell:
2250
+ ``GPT-6.1 Sol — Codex · Intelligence XHigh``. A :class:`Successor` reads
2251
+ the same way, as the model the step runs."""
2252
+ c = backup.candidate
2253
+ dial = _DIAL_NAMES.get(c.platform_id, "Effort")
2254
+ level = _LEVEL_WORDS.get(normalize_level(backup.level), backup.level)
2255
+ head = f"{c.model_name} — {c.platform_name}"
2256
+ return head if dial is None else f"{head} · {dial} {level}"
2257
+
2258
+
1943
2259
  def _below(lower: Rung, upper: Rung, catalog: dict[str, Any]) -> Rung:
1944
2260
  """``lower`` run at least one effort level below ``upper`` when both name
1945
2261
  the same model on the same platform and that path costs something
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes