roadmodel 0.2.62__tar.gz → 0.2.64__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {roadmodel-0.2.62 → roadmodel-0.2.64}/PKG-INFO +1 -1
- {roadmodel-0.2.62 → roadmodel-0.2.64}/pyproject.toml +1 -1
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/__init__.py +1 -1
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/recommend.py +84 -7
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/scoring.py +191 -27
- {roadmodel-0.2.62 → roadmodel-0.2.64}/.gitignore +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/LICENSE +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/NOTICE +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/README.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/benchmarks.json +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/catalog.json +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/model-selector.txt +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/model-tier-cost-scale.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/settings-display.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/templates/phase-roadmap-template.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/templates/planning-kit-how-to-use.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/templates/project-roadmap-template.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/templates/prompt-phase-roadmap.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/templates/prompt-project-roadmap.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/docs/user-context.example.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/hatch_build.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/infra/README.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/infra/gcp-killswitch/README.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/infra/supabase/README.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/benchmarks.json +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/catalog.json +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/model-selector.txt +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/model-tier-cost-scale.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/phase-roadmap-template.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/planning-kit-how-to-use.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/project-roadmap-template.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/prompt-phase-roadmap.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/prompt-project-roadmap.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/settings-display.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/roadmodel/data/user-context.example.md +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/scripts/update_projects.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/__main__.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/cli.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/config.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/cost.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/errors.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/mcp_server.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/providers/__init__.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/providers/anthropic.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/providers/google.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/providers/openai.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/providers/openai_compatible.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/providers/registry.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/usage.py +0 -0
- {roadmodel-0.2.62 → roadmodel-0.2.64}/src/roadmodel/user_context.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: roadmodel
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.64
|
|
4
4
|
Summary: BYO-key CLI that recommends the right AI model, platform, and settings for a prompt.
|
|
5
5
|
Project-URL: Homepage, https://roadmodel.ai
|
|
6
6
|
Project-URL: Repository, https://github.com/nathanramoscfa/roadmodel
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
# src/roadmodel/__init__.py
|
|
2
|
-
__version__ = "0.2.
|
|
2
|
+
__version__ = "0.2.64"
|
|
@@ -501,8 +501,10 @@ _SAAS_LADDER_TABLE_HEADER: Final = (
|
|
|
501
501
|
"gives both the same settings because a lower effort saves this user nothing or "
|
|
502
502
|
"the dial has no lower level; say which in that block's RATIONALE, and never "
|
|
503
503
|
"invent another difference.\n"
|
|
504
|
-
"- BACKUP
|
|
505
|
-
"
|
|
504
|
+
"- Each block's BACKUP is the model the row names after `backup` in that tier's "
|
|
505
|
+
"cell, copied verbatim; code checks it against the row and replaces one that "
|
|
506
|
+
"differs. Where a cell names no backup, BACKUP obeys the Step 7 HARD "
|
|
507
|
+
"cross-provider rule and names a model this user can reach.\n"
|
|
506
508
|
"- Each RATIONALE is three labelled segments (TASK: / PICK: / EFFORT:), one crisp "
|
|
507
509
|
"sentence each. PICK says why the row's model fits its tier in the table's terms "
|
|
508
510
|
"(where it sits on the user's frontier, its AA Index); EFFORT says why the setting "
|
|
@@ -1293,7 +1295,14 @@ def _table_row_key(
|
|
|
1293
1295
|
def _rung_pick_sentence(row: scoring.Ladder, tier: str) -> str:
|
|
1294
1296
|
r = row.rungs[tier]
|
|
1295
1297
|
c = r.candidate
|
|
1296
|
-
|
|
1298
|
+
aa = "not measured" if r.point.aa_index is None else f"{r.point.aa_index:g}"
|
|
1299
|
+
facts = f"AA Index {aa}, ${c.blended_price_usd:.2f} per 1M tokens at list price"
|
|
1300
|
+
if r.specialist:
|
|
1301
|
+
return (
|
|
1302
|
+
f"{c.model_name} is the strongest model you can run for {row.task.category} work: "
|
|
1303
|
+
f"its {row.task.category} rating ({c.letter}) is above every model on your "
|
|
1304
|
+
f"cost/quality frontier ({facts})."
|
|
1305
|
+
)
|
|
1297
1306
|
if tier == "quality":
|
|
1298
1307
|
return f"{c.model_name} is the strongest model on your cost/quality frontier for {row.task.category} work ({facts})."
|
|
1299
1308
|
if tier == "cost":
|
|
@@ -1331,6 +1340,60 @@ def _enforce_table_row(ladder: dict[str, dict[str, str]], row: scoring.Ladder) -
|
|
|
1331
1340
|
return rewritten
|
|
1332
1341
|
|
|
1333
1342
|
|
|
1343
|
+
def _enforce_table_backup(ladder: dict[str, dict[str, str]], row: scoring.Ladder) -> list[str]:
|
|
1344
|
+
"""Make every rung's BACKUP the row's: the best model from another maker
|
|
1345
|
+
the user can run at no higher list price (scoring._backup_for). A rung
|
|
1346
|
+
whose row names no backup (no other maker funded) keeps the engine's,
|
|
1347
|
+
under the Step 7 guard. Returns the tiers whose backup it set."""
|
|
1348
|
+
adjusted: list[str] = []
|
|
1349
|
+
for tier in _LADDER_TIERS:
|
|
1350
|
+
backup = row.rungs[tier].backup
|
|
1351
|
+
if backup is None:
|
|
1352
|
+
continue
|
|
1353
|
+
b = backup.candidate
|
|
1354
|
+
written = ladder[tier].get("backup", "")
|
|
1355
|
+
if written and (
|
|
1356
|
+
cost.model_id_of(written) == b.model_id
|
|
1357
|
+
or _model_key(written) in {_model_key(b.model_id), _model_key(b.model_name)}
|
|
1358
|
+
):
|
|
1359
|
+
continue
|
|
1360
|
+
ladder[tier]["backup"] = b.model_name
|
|
1361
|
+
adjusted.append(tier)
|
|
1362
|
+
return adjusted
|
|
1363
|
+
|
|
1364
|
+
|
|
1365
|
+
def _attach_backup_plans(picks: dict[str, dict[str, Any]], row: scoring.Ladder) -> None:
|
|
1366
|
+
"""Give each pick whose BACKUP is the row's a ``backup_plan``: the
|
|
1367
|
+
platform and effort the row runs that backup at, so a caller can show the
|
|
1368
|
+
backup on the surface the scorer chose (the user's subscription surface
|
|
1369
|
+
where one reaches it) instead of guessing one. A backup the guard replaced
|
|
1370
|
+
gets none."""
|
|
1371
|
+
for tier in _LADDER_TIERS:
|
|
1372
|
+
backup = row.rungs[tier].backup
|
|
1373
|
+
if backup is None:
|
|
1374
|
+
continue
|
|
1375
|
+
b = backup.candidate
|
|
1376
|
+
if picks[tier].get("backup") != cost.canonical_model_name(b.model_name):
|
|
1377
|
+
continue
|
|
1378
|
+
picks[tier]["backup_plan"] = {
|
|
1379
|
+
"model": picks[tier]["backup"],
|
|
1380
|
+
"platform": b.platform_name,
|
|
1381
|
+
"effort": _effort_word(backup.level),
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
|
|
1385
|
+
def _mark_specialist(picks: dict[str, dict[str, Any]], row: scoring.Ladder) -> None:
|
|
1386
|
+
"""Flag the QUALITY pick ``specialist`` with its category when the row
|
|
1387
|
+
puts a category specialist there and the pick still names it."""
|
|
1388
|
+
rung = row.rungs["quality"]
|
|
1389
|
+
if not rung.specialist:
|
|
1390
|
+
return
|
|
1391
|
+
if picks["quality"].get("model") != cost.canonical_model_name(rung.candidate.model_name):
|
|
1392
|
+
return
|
|
1393
|
+
picks["quality"]["specialist"] = True
|
|
1394
|
+
picks["quality"]["specialist_category"] = row.task.category
|
|
1395
|
+
|
|
1396
|
+
|
|
1334
1397
|
# Claude Code's Ultracode is a session setting above its Max level: it stands
|
|
1335
1398
|
# where the row says max.
|
|
1336
1399
|
_ULTRACODE_PLATFORM: Final = "claude-code"
|
|
@@ -1410,10 +1473,15 @@ def recommend_structured_ladder(
|
|
|
1410
1473
|
ladder table (scoring.ladder_table) goes into the prompt, the engine
|
|
1411
1474
|
classifies the task and copies its row, and every rung is then made the
|
|
1412
1475
|
row's (:func:`_enforce_table_row`), effort included on every surface
|
|
1413
|
-
whose dial the catalog documents (:func:`_enforce_table_effort`)
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1476
|
+
whose dial the catalog documents (:func:`_enforce_table_effort`), and
|
|
1477
|
+
BACKUP included where the row names one (:func:`_enforce_table_backup`;
|
|
1478
|
+
such a pick also carries a ``backup_plan`` with the backup's platform and
|
|
1479
|
+
effort). A QUALITY pick the row gives a category specialist carries
|
|
1480
|
+
``specialist: True`` and ``specialist_category``. ``guard`` then reports
|
|
1481
|
+
``mode: "frontier"``, the row, how it was found, the tiers rewritten, the
|
|
1482
|
+
tiers whose effort was set, the tiers whose backup was set and the
|
|
1483
|
+
specialist (or None), and is healthy: two rungs on one model are the
|
|
1484
|
+
table's answer, not a collapse.
|
|
1417
1485
|
Without a table, or a response the table cannot place, the engine's own
|
|
1418
1486
|
picks stand under the tier-distinctness guard as before.
|
|
1419
1487
|
|
|
@@ -1460,7 +1528,13 @@ def recommend_structured_ladder(
|
|
|
1460
1528
|
"found": found,
|
|
1461
1529
|
"rewritten": _enforce_table_row(ladder, row),
|
|
1462
1530
|
"effort_set": _enforce_table_effort(ladder, row),
|
|
1531
|
+
"backup_set": _enforce_table_backup(ladder, row),
|
|
1463
1532
|
"frontier": [p.candidate.model_name for p in row.frontier],
|
|
1533
|
+
"specialist": (
|
|
1534
|
+
{"tier": "quality", "category": row.task.category}
|
|
1535
|
+
if row.rungs["quality"].specialist
|
|
1536
|
+
else None
|
|
1537
|
+
),
|
|
1464
1538
|
}
|
|
1465
1539
|
else:
|
|
1466
1540
|
frontier_guard = {"mode": "frontier", "classification": None, "found": found}
|
|
@@ -1474,6 +1548,9 @@ def recommend_structured_ladder(
|
|
|
1474
1548
|
)
|
|
1475
1549
|
for tier in _LADDER_TIERS
|
|
1476
1550
|
}
|
|
1551
|
+
if frontier_guard is not None and frontier_guard["classification"] is not None:
|
|
1552
|
+
_attach_backup_plans(picks, table[frontier_guard["classification"]])
|
|
1553
|
+
_mark_specialist(picks, table[frontier_guard["classification"]])
|
|
1477
1554
|
guard = _ladder_tier_guard(picks)
|
|
1478
1555
|
if frontier_guard is not None:
|
|
1479
1556
|
guard.update(frontier_guard)
|
|
@@ -56,7 +56,7 @@ from __future__ import annotations
|
|
|
56
56
|
import json
|
|
57
57
|
import math
|
|
58
58
|
import re
|
|
59
|
-
from dataclasses import asdict, dataclass, field
|
|
59
|
+
from dataclasses import asdict, dataclass, field, replace
|
|
60
60
|
from datetime import datetime, timedelta, timezone
|
|
61
61
|
from importlib import resources
|
|
62
62
|
from typing import Any, Final
|
|
@@ -1016,7 +1016,9 @@ FRONTIER_INDEX: Final[str] = "artificial_analysis_intelligence_index"
|
|
|
1016
1016
|
@dataclass(frozen=True)
|
|
1017
1017
|
class FrontierPoint:
|
|
1018
1018
|
candidate: Candidate
|
|
1019
|
-
|
|
1019
|
+
# The AA Intelligence Index; every frontier point has one, and only a
|
|
1020
|
+
# category specialist (off the frontier) may be unmeasured.
|
|
1021
|
+
aa_index: float | None
|
|
1020
1022
|
# Blended list $/1M, the price /models plots. List price, not what the
|
|
1021
1023
|
# operator pays: on a subscription every covered model costs them $0, which
|
|
1022
1024
|
# would collapse the frontier to its top model; list price is what a pick
|
|
@@ -1024,6 +1026,24 @@ class FrontierPoint:
|
|
|
1024
1026
|
price_usd: float
|
|
1025
1027
|
|
|
1026
1028
|
|
|
1029
|
+
@dataclass(frozen=True)
|
|
1030
|
+
class Backup:
|
|
1031
|
+
"""A rung's cross-maker substitute, read off the frontier of the other
|
|
1032
|
+
makers' models (see :func:`ladder`)."""
|
|
1033
|
+
|
|
1034
|
+
point: FrontierPoint
|
|
1035
|
+
effort: str
|
|
1036
|
+
native: str | None = None
|
|
1037
|
+
|
|
1038
|
+
@property
|
|
1039
|
+
def candidate(self) -> Candidate:
|
|
1040
|
+
return self.point.candidate
|
|
1041
|
+
|
|
1042
|
+
@property
|
|
1043
|
+
def level(self) -> str:
|
|
1044
|
+
return self.native or self.effort
|
|
1045
|
+
|
|
1046
|
+
|
|
1027
1047
|
@dataclass(frozen=True)
|
|
1028
1048
|
class Rung:
|
|
1029
1049
|
tier: str
|
|
@@ -1033,6 +1053,11 @@ class Rung:
|
|
|
1033
1053
|
# The same level as the platform's own dial names it (native_level), or
|
|
1034
1054
|
# None when the catalog documents no dial for the platform and model.
|
|
1035
1055
|
native: str | None = None
|
|
1056
|
+
# The best cross-maker substitute at no higher list price; None when the
|
|
1057
|
+
# pool holds no other maker's model.
|
|
1058
|
+
backup: Backup | None = None
|
|
1059
|
+
# True for a QUALITY rung held by a category specialist off the frontier.
|
|
1060
|
+
specialist: bool = False
|
|
1036
1061
|
|
|
1037
1062
|
@property
|
|
1038
1063
|
def candidate(self) -> Candidate:
|
|
@@ -1052,6 +1077,8 @@ class Ladder:
|
|
|
1052
1077
|
# complexity's requirement, cheapest first.
|
|
1053
1078
|
adequate: list[FrontierPoint]
|
|
1054
1079
|
rungs: dict[str, Rung]
|
|
1080
|
+
# Why a rung has no backup: the pool holds no other maker's model.
|
|
1081
|
+
backup_warning: str | None = None
|
|
1055
1082
|
|
|
1056
1083
|
def to_dict(self) -> dict[str, Any]:
|
|
1057
1084
|
def point(p: FrontierPoint) -> dict[str, Any]:
|
|
@@ -1072,9 +1099,24 @@ class Ladder:
|
|
|
1072
1099
|
"frontier": [point(p) for p in self.frontier],
|
|
1073
1100
|
"adequate": [p.candidate.model_id for p in self.adequate],
|
|
1074
1101
|
"rungs": {
|
|
1075
|
-
t: {
|
|
1102
|
+
t: {
|
|
1103
|
+
**point(r.point),
|
|
1104
|
+
"effort": r.effort,
|
|
1105
|
+
"native_effort": r.native,
|
|
1106
|
+
"specialist": r.specialist,
|
|
1107
|
+
"backup": (
|
|
1108
|
+
{
|
|
1109
|
+
**point(r.backup.point),
|
|
1110
|
+
"effort": r.backup.effort,
|
|
1111
|
+
"native_effort": r.backup.native,
|
|
1112
|
+
}
|
|
1113
|
+
if r.backup is not None
|
|
1114
|
+
else None
|
|
1115
|
+
),
|
|
1116
|
+
}
|
|
1076
1117
|
for t, r in self.rungs.items()
|
|
1077
1118
|
},
|
|
1119
|
+
"backup_warning": self.backup_warning,
|
|
1078
1120
|
}
|
|
1079
1121
|
|
|
1080
1122
|
|
|
@@ -1084,20 +1126,70 @@ def frontier(candidates: list[Candidate], bench: dict[str, Any]) -> list[Frontie
|
|
|
1084
1126
|
at its price or less. The rule web/lib/benchmark-grid.ts paretoFrontier
|
|
1085
1127
|
draws on /models: a tie on price goes to the higher index, a tie on index to
|
|
1086
1128
|
the cheaper price. Unmeasured models have no place on it."""
|
|
1087
|
-
|
|
1088
|
-
|
|
1129
|
+
measured = [
|
|
1130
|
+
(c, aa)
|
|
1089
1131
|
for c in candidates
|
|
1090
1132
|
for aa in [_evidence(bench, c.model_id, FRONTIER_INDEX)]
|
|
1091
1133
|
if aa is not None
|
|
1092
1134
|
]
|
|
1093
|
-
|
|
1135
|
+
measured.sort(key=lambda m: (m[0].blended_price_usd, -m[1], m[0].model_id))
|
|
1094
1136
|
out: list[FrontierPoint] = []
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1137
|
+
best = -math.inf
|
|
1138
|
+
for c, aa in measured:
|
|
1139
|
+
if aa > best:
|
|
1140
|
+
out.append(FrontierPoint(c, aa, c.blended_price_usd))
|
|
1141
|
+
best = aa
|
|
1098
1142
|
return out
|
|
1099
1143
|
|
|
1100
1144
|
|
|
1145
|
+
def _strength(p: FrontierPoint) -> tuple[float, float, float]:
|
|
1146
|
+
"""A point's standing for the task's category: its category quality, then
|
|
1147
|
+
the AA Index, then the cheaper price."""
|
|
1148
|
+
return (p.candidate.quality, -1.0 if p.aa_index is None else p.aa_index, -p.price_usd)
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
# Letters from weakest to strongest.
|
|
1152
|
+
LETTER_ORDER: Final[str] = "DCBAS"
|
|
1153
|
+
# Categories whose evidence is something other than the AA Intelligence Index,
|
|
1154
|
+
# the frontier's own axis: there a model off the frontier can be clearly
|
|
1155
|
+
# stronger for the category (every category but planning).
|
|
1156
|
+
SPECIALIST_CATEGORIES: Final[frozenset[str]] = frozenset(
|
|
1157
|
+
cat for cat, key in CATEGORY_EVIDENCE.items() if key != FRONTIER_INDEX
|
|
1158
|
+
)
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
def _specialist(
|
|
1162
|
+
task: Task,
|
|
1163
|
+
pool: list[Candidate],
|
|
1164
|
+
front: list[FrontierPoint],
|
|
1165
|
+
top: FrontierPoint,
|
|
1166
|
+
bench: dict[str, Any],
|
|
1167
|
+
) -> FrontierPoint | None:
|
|
1168
|
+
"""A category specialist for QUALITY: the pool model off the frontier
|
|
1169
|
+
whose letter in the task's category is strictly above that of ``top``
|
|
1170
|
+
(the strongest adequate frontier point) and whose category quality is
|
|
1171
|
+
strictly above it too; the strongest such model, else None. Only for
|
|
1172
|
+
SPECIALIST_CATEGORIES."""
|
|
1173
|
+
if task.category not in SPECIALIST_CATEGORIES:
|
|
1174
|
+
return None
|
|
1175
|
+
on_front = {p.candidate.model_id for p in front}
|
|
1176
|
+
rank = LETTER_ORDER.find
|
|
1177
|
+
t = top.candidate
|
|
1178
|
+
stronger = [
|
|
1179
|
+
c
|
|
1180
|
+
for c in pool
|
|
1181
|
+
if c.model_id not in on_front and rank(c.letter) > rank(t.letter) and c.quality > t.quality
|
|
1182
|
+
]
|
|
1183
|
+
if not stronger:
|
|
1184
|
+
return None
|
|
1185
|
+
best = max(
|
|
1186
|
+
stronger, key=lambda c: (c.quality, rank(c.letter), -c.blended_price_usd, c.model_id)
|
|
1187
|
+
)
|
|
1188
|
+
return FrontierPoint(
|
|
1189
|
+
best, _evidence(bench, best.model_id, FRONTIER_INDEX), best.blended_price_usd
|
|
1190
|
+
)
|
|
1191
|
+
|
|
1192
|
+
|
|
1101
1193
|
def ladder(
|
|
1102
1194
|
task: Task,
|
|
1103
1195
|
user_context_text: str,
|
|
@@ -1129,6 +1221,15 @@ def ladder(
|
|
|
1129
1221
|
level they may converge. On an `uncapped` pool every free rung runs at
|
|
1130
1222
|
the top effort and two rungs on one model converge: a lower effort saves
|
|
1131
1223
|
that operator nothing.
|
|
1224
|
+
- A category specialist (:func:`_specialist`) may take QUALITY in a
|
|
1225
|
+
category whose evidence is not the AA Index: a pool model off the
|
|
1226
|
+
frontier whose letter and category quality are both strictly above
|
|
1227
|
+
the strongest adequate frontier point's. It runs at the best posture's
|
|
1228
|
+
effort; COST and BALANCED stay on the frontier, BALANCED then spanning
|
|
1229
|
+
every adequate point above COST.
|
|
1230
|
+
- Each rung carries a BACKUP from another maker (:func:`_backup_for`):
|
|
1231
|
+
the best adequate point on the other makers' frontier at no higher list
|
|
1232
|
+
price. ``backup_warning`` says why a rung has none.
|
|
1132
1233
|
|
|
1133
1234
|
None when no pool model carries an AA Index (no frontier to read)."""
|
|
1134
1235
|
cat = catalog if catalog is not None else _cost._load_catalog()
|
|
@@ -1146,15 +1247,14 @@ def ladder(
|
|
|
1146
1247
|
if not front:
|
|
1147
1248
|
return None
|
|
1148
1249
|
|
|
1149
|
-
def strength(p: FrontierPoint) -> tuple[float, float, float]:
|
|
1150
|
-
return (p.candidate.quality, p.aa_index, -p.price_usd)
|
|
1151
|
-
|
|
1152
1250
|
adequate = [p for p in front if p.candidate.requirement_penalty == 0]
|
|
1153
1251
|
if not adequate:
|
|
1154
|
-
adequate = [max(front, key=
|
|
1155
|
-
top = max(adequate, key=
|
|
1252
|
+
adequate = [max(front, key=_strength)]
|
|
1253
|
+
top = max(adequate, key=_strength)
|
|
1156
1254
|
span = adequate[: adequate.index(top) + 1]
|
|
1157
1255
|
headroom = consumption_headroom(user_context_text or "")
|
|
1256
|
+
pool = funded or ranking.candidates
|
|
1257
|
+
specialist = _specialist(task, pool, front, top, bench)
|
|
1158
1258
|
|
|
1159
1259
|
def rung(tier: str, p: FrontierPoint) -> Rung:
|
|
1160
1260
|
t = Task(task.category, task.complexity, task.novel, TIER_BUDGET[tier])
|
|
@@ -1166,8 +1266,14 @@ def ladder(
|
|
|
1166
1266
|
return a.candidate.model_id == b.candidate.model_id and a.level == b.level
|
|
1167
1267
|
|
|
1168
1268
|
cost_rung = rung("cost", span[0])
|
|
1169
|
-
quality_rung =
|
|
1170
|
-
|
|
1269
|
+
quality_rung = (
|
|
1270
|
+
replace(rung("quality", specialist), specialist=True)
|
|
1271
|
+
if specialist is not None
|
|
1272
|
+
else rung("quality", top)
|
|
1273
|
+
)
|
|
1274
|
+
# With a specialist on QUALITY, BALANCED spans every adequate frontier point
|
|
1275
|
+
# above COST, the frontier's strongest included.
|
|
1276
|
+
between = span[1:] if specialist is not None else span[1:-1]
|
|
1171
1277
|
if between:
|
|
1172
1278
|
balanced_rung = rung(
|
|
1173
1279
|
"balanced", max(between, key=lambda p: (p.candidate.score, -p.price_usd))
|
|
@@ -1175,16 +1281,61 @@ def ladder(
|
|
|
1175
1281
|
else:
|
|
1176
1282
|
# Two adequate points or one: BALANCED runs one of them at the balanced
|
|
1177
1283
|
# posture, whichever differs from both other rungs; it converges only
|
|
1178
|
-
# when neither does.
|
|
1179
|
-
options = [rung("balanced", span[0])
|
|
1284
|
+
# when neither does. Beside a specialist it stays on the frontier.
|
|
1285
|
+
options = [rung("balanced", span[0])]
|
|
1286
|
+
if specialist is None:
|
|
1287
|
+
options.append(rung("balanced", top))
|
|
1180
1288
|
balanced_rung = next(
|
|
1181
1289
|
(r for r in options if not same(r, cost_rung) and not same(r, quality_rung)),
|
|
1182
1290
|
options[0],
|
|
1183
1291
|
)
|
|
1184
1292
|
balanced_rung = _below(balanced_rung, quality_rung, cat)
|
|
1185
1293
|
cost_rung = _below(cost_rung, balanced_rung, cat)
|
|
1186
|
-
rungs = {
|
|
1187
|
-
|
|
1294
|
+
rungs = {
|
|
1295
|
+
r.tier: replace(r, backup=_backup_for(r, pool, task, headroom, bench, cat))
|
|
1296
|
+
for r in (cost_rung, balanced_rung, quality_rung)
|
|
1297
|
+
}
|
|
1298
|
+
warning: str | None = None
|
|
1299
|
+
if any(r.backup is None for r in rungs.values()):
|
|
1300
|
+
makers = sorted({c.provider for c in pool})
|
|
1301
|
+
warning = (
|
|
1302
|
+
f"No cross-provider backup: the models this user can run come from "
|
|
1303
|
+
f"{', '.join(makers) or 'no maker'} only. Add a second maker's subscription or "
|
|
1304
|
+
"API key so an outage or an exhausted pool has somewhere to go."
|
|
1305
|
+
)
|
|
1306
|
+
return Ladder(task=base, frontier=front, adequate=adequate, rungs=rungs, backup_warning=warning)
|
|
1307
|
+
|
|
1308
|
+
|
|
1309
|
+
def _backup_for(
|
|
1310
|
+
rung: Rung,
|
|
1311
|
+
pool: list[Candidate],
|
|
1312
|
+
task: Task,
|
|
1313
|
+
headroom: str,
|
|
1314
|
+
bench: dict[str, Any],
|
|
1315
|
+
catalog: dict[str, Any],
|
|
1316
|
+
) -> Backup | None:
|
|
1317
|
+
"""The rung's backup: the best substitute from another maker the operator
|
|
1318
|
+
can run, at no higher list price.
|
|
1319
|
+
|
|
1320
|
+
Over the pool's models from a maker other than the rung model's, draw the
|
|
1321
|
+
frontier and keep its adequate points. The backup is the adequate point
|
|
1322
|
+
with the highest list price at or below the rung's; else the cheapest
|
|
1323
|
+
adequate point above it; else, with no adequate point, the frontier's
|
|
1324
|
+
strongest for the category. It runs at the rung's posture's effort on its
|
|
1325
|
+
own platform. None when the pool holds no other maker's model."""
|
|
1326
|
+
front = frontier([c for c in pool if c.provider != rung.candidate.provider], bench)
|
|
1327
|
+
if not front:
|
|
1328
|
+
return None
|
|
1329
|
+
adequate = [p for p in front if p.candidate.requirement_penalty == 0]
|
|
1330
|
+
if adequate:
|
|
1331
|
+
within = [p for p in adequate if p.price_usd <= rung.point.price_usd]
|
|
1332
|
+
chosen = within[-1] if within else adequate[0]
|
|
1333
|
+
else:
|
|
1334
|
+
chosen = max(front, key=_strength)
|
|
1335
|
+
t = Task(task.category, task.complexity, task.novel, TIER_BUDGET[rung.tier])
|
|
1336
|
+
c = chosen.candidate
|
|
1337
|
+
effort = effort_for(t, headroom, c.scarcity)
|
|
1338
|
+
return Backup(chosen, effort, native_level(c.platform_id, c.model_id, effort, catalog=catalog))
|
|
1188
1339
|
|
|
1189
1340
|
|
|
1190
1341
|
def _below(lower: Rung, upper: Rung, catalog: dict[str, Any]) -> Rung:
|
|
@@ -1255,6 +1406,10 @@ def ladder_table(
|
|
|
1255
1406
|
return out
|
|
1256
1407
|
|
|
1257
1408
|
|
|
1409
|
+
def _aa_text(p: FrontierPoint) -> str:
|
|
1410
|
+
return "not measured" if p.aa_index is None else f"{p.aa_index:g}"
|
|
1411
|
+
|
|
1412
|
+
|
|
1258
1413
|
def render_ladder_table(table: dict[str, Ladder]) -> str:
|
|
1259
1414
|
"""The table as the engine reads it: one row per classification, each rung
|
|
1260
1415
|
as model, platform and effort, under the frontier it was read from."""
|
|
@@ -1267,18 +1422,27 @@ def render_ladder_table(table: dict[str, Ladder]) -> str:
|
|
|
1267
1422
|
"the AA Intelligence Index than any they can run for less), cheapest first:",
|
|
1268
1423
|
" "
|
|
1269
1424
|
+ " < ".join(
|
|
1270
|
-
f"{p.candidate.model_name} (AA {p
|
|
1425
|
+
f"{p.candidate.model_name} (AA {_aa_text(p)}, ${p.candidate.blended_price_usd:.2f}/1M list)"
|
|
1271
1426
|
for p in any_ladder.frontier
|
|
1272
1427
|
),
|
|
1273
1428
|
"Rows: <category>/<complexity>[/novel]: COST | BALANCED | QUALITY, each "
|
|
1274
|
-
"<model> @ <platform> · <effort>, the effort in the platform's own level words
|
|
1429
|
+
"<model> @ <platform> · <effort>, the effort in the platform's own level words, "
|
|
1430
|
+
"then `backup <model> @ <platform>`: that pick's BACKUP. A QUALITY cell marked "
|
|
1431
|
+
"`(top for <category> work)` names a model off the frontier whose rating for that "
|
|
1432
|
+
"category is above every frontier model's.",
|
|
1275
1433
|
]
|
|
1434
|
+
|
|
1435
|
+
def cell(t: str, r: Rung, category: str) -> str:
|
|
1436
|
+
text = f"{t.upper()} = {r.candidate.model_name} @ {r.candidate.platform_name} · {r.level}"
|
|
1437
|
+
if r.specialist:
|
|
1438
|
+
text += f" (top for {category} work)"
|
|
1439
|
+
if r.backup is not None:
|
|
1440
|
+
b = r.backup.candidate
|
|
1441
|
+
text += f", backup {b.model_name} @ {b.platform_name}"
|
|
1442
|
+
return text
|
|
1443
|
+
|
|
1276
1444
|
for key, lad in table.items():
|
|
1277
|
-
cells = " | ".join(
|
|
1278
|
-
f"{t.upper()} = {r.candidate.model_name} @ {r.candidate.platform_name} · {r.level}"
|
|
1279
|
-
for t in LADDER_TIERS
|
|
1280
|
-
for r in [lad.rungs[t]]
|
|
1281
|
-
)
|
|
1445
|
+
cells = " | ".join(cell(t, lad.rungs[t], lad.task.category) for t in LADDER_TIERS)
|
|
1282
1446
|
lines.append(f"{key}: {cells}")
|
|
1283
1447
|
lines.append("</ladder-table>")
|
|
1284
1448
|
return "\n".join(lines)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|