pi-smart-router 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/delegation-runtime.ts +12 -2
- package/README.md +57 -10
- package/config/benchmark-profiles.json +145 -0
- package/config/models.yaml.example +5 -0
- package/config/routing-calibration.json.example +14 -2
- package/dist/api/explain/router-explain.d.ts +4 -1
- package/dist/api/explain/router-explain.d.ts.map +1 -1
- package/dist/api/explain/router-explain.js +9 -9
- package/dist/api/explain/router-explain.js.map +1 -1
- package/dist/config/defaults.d.ts +4 -1
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +10 -0
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/pi-model-mapper.d.ts +12 -2
- package/dist/config/pi-model-mapper.d.ts.map +1 -1
- package/dist/config/pi-model-mapper.js +91 -6
- package/dist/config/pi-model-mapper.js.map +1 -1
- package/dist/domain/delegation/delegation-context.d.ts +23 -1
- package/dist/domain/delegation/delegation-context.d.ts.map +1 -1
- package/dist/domain/delegation/delegation-context.js +93 -0
- package/dist/domain/delegation/delegation-context.js.map +1 -1
- package/dist/domain/matching/hydra-input.d.ts +6 -5
- package/dist/domain/matching/hydra-input.d.ts.map +1 -1
- package/dist/domain/matching/hydra-input.js +73 -6
- package/dist/domain/matching/hydra-input.js.map +1 -1
- package/dist/domain/pinning/cache-breakeven.d.ts +51 -0
- package/dist/domain/pinning/cache-breakeven.d.ts.map +1 -0
- package/dist/domain/pinning/cache-breakeven.js +124 -0
- package/dist/domain/pinning/cache-breakeven.js.map +1 -0
- package/dist/domain/pinning/saar-session-state.d.ts +36 -0
- package/dist/domain/pinning/saar-session-state.d.ts.map +1 -0
- package/dist/domain/pinning/saar-session-state.js +74 -0
- package/dist/domain/pinning/saar-session-state.js.map +1 -0
- package/dist/domain/pinning/session-pinner.d.ts +35 -3
- package/dist/domain/pinning/session-pinner.d.ts.map +1 -1
- package/dist/domain/pinning/session-pinner.js +149 -1
- package/dist/domain/pinning/session-pinner.js.map +1 -1
- package/dist/domain/pipeline/router-pipeline.d.ts +24 -1
- package/dist/domain/pipeline/router-pipeline.d.ts.map +1 -1
- package/dist/domain/pipeline/router-pipeline.js +157 -10
- package/dist/domain/pipeline/router-pipeline.js.map +1 -1
- package/dist/domain/routing/isotonic-calibrator.d.ts +56 -0
- package/dist/domain/routing/isotonic-calibrator.d.ts.map +1 -0
- package/dist/domain/routing/isotonic-calibrator.js +187 -0
- package/dist/domain/routing/isotonic-calibrator.js.map +1 -0
- package/dist/domain/routing/p-success-classifier.d.ts +53 -7
- package/dist/domain/routing/p-success-classifier.d.ts.map +1 -1
- package/dist/domain/routing/p-success-classifier.js +205 -21
- package/dist/domain/routing/p-success-classifier.js.map +1 -1
- package/dist/domain/routing/tool-history-guard.d.ts +21 -5
- package/dist/domain/routing/tool-history-guard.d.ts.map +1 -1
- package/dist/domain/routing/tool-history-guard.js +77 -10
- package/dist/domain/routing/tool-history-guard.js.map +1 -1
- package/dist/domain/types/entities.d.ts +54 -0
- package/dist/domain/types/entities.d.ts.map +1 -1
- package/dist/domain/types/index.d.ts +1 -1
- package/dist/domain/types/index.d.ts.map +1 -1
- package/dist/domain/types/schemas.d.ts +24 -0
- package/dist/domain/types/schemas.d.ts.map +1 -1
- package/dist/domain/types/schemas.js +54 -0
- package/dist/domain/types/schemas.js.map +1 -1
- package/dist/infra/gemini-provider.d.ts.map +1 -1
- package/dist/infra/gemini-provider.js +2 -1
- package/dist/infra/gemini-provider.js.map +1 -1
- package/dist/infrastructure/persistence/sqlite-store.d.ts.map +1 -1
- package/dist/infrastructure/persistence/sqlite-store.js +3 -0
- package/dist/infrastructure/persistence/sqlite-store.js.map +1 -1
- package/dist/infrastructure/telemetry/routing-telemetry.d.ts +30 -2
- package/dist/infrastructure/telemetry/routing-telemetry.d.ts.map +1 -1
- package/dist/infrastructure/telemetry/routing-telemetry.js +225 -3
- package/dist/infrastructure/telemetry/routing-telemetry.js.map +1 -1
- package/package.json +6 -3
- package/specs/001-build-smart-router/contracts/telemetry-contrib.schema.json +29 -1
- package/src/api/explain/router-explain.ts +14 -10
- package/src/config/defaults.ts +18 -1
- package/src/config/pi-model-mapper.ts +110 -6
- package/src/domain/delegation/delegation-context.ts +124 -0
- package/src/domain/matching/hydra-input.ts +86 -7
- package/src/domain/pinning/cache-breakeven.ts +219 -0
- package/src/domain/pinning/saar-session-state.ts +93 -0
- package/src/domain/pinning/session-pinner.ts +244 -3
- package/src/domain/pipeline/router-pipeline.ts +213 -10
- package/src/domain/routing/isotonic-calibrator.ts +255 -0
- package/src/domain/routing/p-success-classifier.ts +299 -26
- package/src/domain/routing/tool-history-guard.ts +108 -10
- package/src/domain/types/entities.ts +60 -0
- package/src/domain/types/index.ts +4 -0
- package/src/domain/types/schemas.ts +69 -0
- package/src/infra/gemini-provider.ts +2 -1
- package/src/infrastructure/persistence/sqlite-store.ts +6 -0
- package/src/infrastructure/telemetry/routing-telemetry.ts +331 -7
|
@@ -13,7 +13,11 @@ import {
|
|
|
13
13
|
} from '@earendil-works/pi-ai/compat';
|
|
14
14
|
import type { ModelRegistry } from '@earendil-works/pi-coding-agent';
|
|
15
15
|
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
isGoogleDelegationTarget,
|
|
18
|
+
normalizeDelegationContext,
|
|
19
|
+
repairGeminiReplayContext,
|
|
20
|
+
} from '../../../src/domain/delegation/delegation-context.js';
|
|
17
21
|
import {
|
|
18
22
|
computeOutputHeadroom,
|
|
19
23
|
type OutputHeadroomConfig,
|
|
@@ -147,9 +151,15 @@ export function buildDelegationContext(
|
|
|
147
151
|
? deps.executionLedger.getLastExecution(sessionId)
|
|
148
152
|
: null;
|
|
149
153
|
|
|
150
|
-
|
|
154
|
+
const normalized = normalizeDelegationContext(context, targetModel, {
|
|
151
155
|
sessionExecution,
|
|
152
156
|
});
|
|
157
|
+
|
|
158
|
+
if (isGoogleDelegationTarget(targetModel)) {
|
|
159
|
+
return repairGeminiReplayContext(normalized, targetModel, sessionExecution);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return normalized;
|
|
153
163
|
}
|
|
154
164
|
|
|
155
165
|
export function createErrorMessage(
|
package/README.md
CHANGED
|
@@ -45,7 +45,7 @@ pi-smart-router builds on ideas from several production and research routing sys
|
|
|
45
45
|
- **Adopted:** [GitHub Copilot HyDRA](https://arxiv.org/abs/2409.08379) (shortfall matching decoupled from model identities), Zero-Tier local edge-cache pattern, [Weave Router](https://github.com/workweave/router) session pinning and multi-objective selection
|
|
46
46
|
- **Rejected:** FrugalGPT sequential cascading (tail latency), RouteLLM matrix factorization (confounder vulnerability), turn-by-turn dynamic routing (cache destruction)
|
|
47
47
|
|
|
48
|
-
See [docs/PRD.md](docs/PRD.md) for full architectural justification.
|
|
48
|
+
See [docs/PRD.md](docs/PRD.md) for full architectural justification, [docs/deep-research.md](docs/deep-research.md) for the research survey, [docs/routing-roadmap.md](docs/routing-roadmap.md) for the prioritized quality backlog, [docs/gemini-research.md](docs/gemini-research.md) for the second-source agent-router report, and [docs/research/README.md](docs/research/README.md) for research provenance.
|
|
49
49
|
|
|
50
50
|
## Prerequisites
|
|
51
51
|
|
|
@@ -169,7 +169,7 @@ pi exposes two different **auto** models. They are easy to confuse but play diff
|
|
|
169
169
|
| `smart-router/auto` | `smart-router` (this extension) | Runs the routing pipeline on every turn and **delegates** to whichever underlying model HyDRA selects |
|
|
170
170
|
| `cursor/auto` | `cursor` (pi registry) | Cursor's opaque auto model — **direct** inference target when selected; Cursor picks the backend model |
|
|
171
171
|
|
|
172
|
-
**Recommended dogfood setup:** use `/model smart-router/auto` so routing, pinning, and telemetry stay active. Enable `cursor/auto` (and other Cursor models such as `composer-latest`) in your scoped fleet so the router can select them when appropriate — for example on planning turns or when the [Gemini tool-history guard](#gemini-thought_signature-400-errors) excludes Google
|
|
172
|
+
**Recommended dogfood setup:** use `/model smart-router/auto` so routing, pinning, and telemetry stay active. Enable `cursor/auto` (and other Cursor models such as `composer-latest`) in your scoped fleet so the router can select them when appropriate — for example on planning turns or when the [Gemini tool-history guard](#gemini-thought_signature-400-errors) excludes unrepairable Google replay state.
|
|
173
173
|
|
|
174
174
|
**When to pin `/model cursor/auto` directly (bypass the router):**
|
|
175
175
|
|
|
@@ -181,7 +181,7 @@ pi exposes two different **auto** models. They are easy to confuse but play diff
|
|
|
181
181
|
|
|
182
182
|
- You want cost/capability-aware model selection across your full authenticated fleet
|
|
183
183
|
- You rely on session pinning, failover, or `/smart-router status` / `history` telemetry
|
|
184
|
-
- Tool-heavy sessions
|
|
184
|
+
- Tool-heavy sessions with Gemini economical models work via in-repo replay repair; add `cursor/auto` for unrepairable Google replay edge cases (see [pi-smart-router#85](https://github.com/beettlle/pi-smart-router/issues/85))
|
|
185
185
|
|
|
186
186
|
Cursor models (`cursor/*`, `composer-*`, and the opaque fleet id `default`) map to **frontier-cloud** tier in `pi-model-mapper.ts` so HyDRA can score them against Gemini and Claude instead of treating them as unknown economical models ([pi-smart-router#40](https://github.com/beettlle/pi-smart-router/issues/40), [pi-smart-router#70](https://github.com/beettlle/pi-smart-router/issues/70)). Related: [pi-smart-router#23](https://github.com/beettlle/pi-smart-router/issues/23) (turn envelope / pin order), [pi-smart-router#37](https://github.com/beettlle/pi-smart-router/issues/37) (Gemini `thought_signature` errors).
|
|
187
187
|
|
|
@@ -343,9 +343,34 @@ Cluster IDs are stable reason-code prefixes (`cluster_low_stakes_general`, `clus
|
|
|
343
343
|
| `SMART_ROUTER_DATASET` | (unset) | Set to `1` to opt in to privacy-safe routing dataset capture (metadata and feature fields only; 30-day / 10k-row retention). Prompt text, messages, and tool arguments are never stored. Required for outcome labels and P(success) training export. See [#8](https://github.com/beettlle/pi-smart-router/issues/8). |
|
|
344
344
|
| `SMART_ROUTER_DATASET_FINGERPRINT` | (unset) | Set to `1` (requires `SMART_ROUTER_DATASET=1`) to store an install-local HMAC-SHA256 fingerprint of each normalized prompt for duplicate detection within this install. The install pepper lives in `.pi-smart-router/.dataset-key` (gitignored) and is never exported. **Warning:** short or common prompts are vulnerable to offline rainbow-table guessing; use only when you accept that tradeoff. See [#10](https://github.com/beettlle/pi-smart-router/issues/10). |
|
|
345
345
|
| `MODELS_YAML_PATH` | `./config/models.yaml` | Fleet catalog path (library API only) |
|
|
346
|
+
| `SMART_ROUTER_PLANNING_TURN_BUFFER` | `2` | SAAR planning buffer: frontier planning turns allowed before hard-lock ([v0.2.0 Continuity](https://github.com/beettlle/pi-smart-router/issues/72)) |
|
|
347
|
+
| `SMART_ROUTER_PREFIX_CACHE_WEIGHT` | `0.20` | SAAR weight on warm prefix value in cache breakeven math (0–1; [#73](https://github.com/beettlle/pi-smart-router/issues/73)) |
|
|
348
|
+
| `SMART_ROUTER_IDLE_TIMEOUT_SECONDS` | `300` | SAAR idle seconds before pin reopens for full re-route |
|
|
349
|
+
| `SMART_ROUTER_SWITCH_THRESHOLD` | `0.5` | SAAR switch score gate (0–1) for tier upgrades during hard-lock |
|
|
346
350
|
| `ROUTER_SAFE_DEFAULT_TIER` | `economical-cloud` | Fallback tier on any routing failure |
|
|
347
351
|
| `LITELLM_PRICING_URL` | — | LiteLLM pricing JSON source |
|
|
348
352
|
|
|
353
|
+
### SAAR session pin and cache breakeven (v0.2.0 Continuity)
|
|
354
|
+
|
|
355
|
+
v0.2.0 adds **Session-Aware Agentic Routing (SAAR)** pin knobs ([#72](https://github.com/beettlle/pi-smart-router/issues/72)) and a **cache breakeven gate** ([#73](https://github.com/beettlle/pi-smart-router/issues/73)) that blocks tier switches when `marginal_savings + future_cache_value <= cache_reprime_cost` — preventing cheap-turn savings from invalidating a warm prefix cache.
|
|
356
|
+
|
|
357
|
+
| Knob | Env var | Default | Effect |
|
|
358
|
+
|------|---------|---------|--------|
|
|
359
|
+
| Planning buffer | `SMART_ROUTER_PLANNING_TURN_BUFFER` | `2` | First N turns may route planning to frontier while pin metadata stays economical |
|
|
360
|
+
| Prefix cache weight | `SMART_ROUTER_PREFIX_CACHE_WEIGHT` | `0.20` | Discounted future cache credit in breakeven |
|
|
361
|
+
| Idle reopen | `SMART_ROUTER_IDLE_TIMEOUT_SECONDS` | `300` | Seconds of inactivity before SAAR resets and pin reopens |
|
|
362
|
+
| Hard-lock upgrade gate | `SMART_ROUTER_SWITCH_THRESHOLD` | `0.5` | Score threshold for tier upgrades after buffer exhaust |
|
|
363
|
+
|
|
364
|
+
**Dogfood verification (multi-turn planning session)**
|
|
365
|
+
|
|
366
|
+
1. Start pi with routing logs: `SMART_ROUTER_LOG_ROUTING=1 pi` (optional: tune SAAR env vars above).
|
|
367
|
+
2. Run `/model smart-router/auto` and begin a multi-turn planning session (planning turns mixed with tool results).
|
|
368
|
+
3. Inspect stderr JSON lines — confirm `saar_summary.buffer_active` / `saar_reason_code: saar_buffer_active` on early planning turns, then `hard_lock: true` / `saar_hard_lock` after the buffer exhausts.
|
|
369
|
+
4. On a warm pinned session, trigger a `tool_result` sub-route — when breakeven fails, expect `breakeven_summary.decision: "blocked"` and `breakeven_reason_code: breakeven_blocked` while the pin holds.
|
|
370
|
+
5. Use `pi router explain` (or `POST /v1/route/explain`) on the same session — `features.breakeven` and `features.saar` mirror telemetry fields for operator audit.
|
|
371
|
+
|
|
372
|
+
See [routing-roadmap.md](docs/routing-roadmap.md) §2 P0 for design context.
|
|
373
|
+
|
|
349
374
|
### P(success) training export (baseline classifier)
|
|
350
375
|
|
|
351
376
|
When `SMART_ROUTER_DATASET=1`, the router records privacy-safe dataset rows and behavioral outcome labels (model override, compaction pin break, `/smart-router feedback good|bad`). Export labeled training data from pi:
|
|
@@ -458,18 +483,21 @@ The `GatewayDispatch` layer wraps the pipeline with:
|
|
|
458
483
|
|
|
459
484
|
If Gemini returns **400 INVALID_ARGUMENT** mentioning `thought_signature`, the router treats this as a **protocol validation error** (incomplete tool-call replay), not provider unavailability — it will **not** failover to another model.
|
|
460
485
|
|
|
461
|
-
|
|
486
|
+
See [Google's thought signatures documentation](https://ai.google.dev/gemini-api/docs/generate-content/thought-signatures).
|
|
462
487
|
|
|
463
|
-
**
|
|
488
|
+
**Primary fix — replay repair (SP-127/128):** before every Google-target delegation, smart-router repairs tool-call replay state: prior turns keep captured `thoughtSignature` values; tool calls missing a signature receive the Google-accepted skip sentinel so pi-ai can replay without a 400. Typical Gemini-first tool loops on `/model smart-router/auto` no longer require `/new` or switching away from Google models.
|
|
464
489
|
|
|
465
|
-
**
|
|
490
|
+
**Narrowed guard fail-safe (SP-129):** sessions with **unrepairable** Google-origin replay state (e.g. redacted thinking blocks paired with tool calls) exclude Gemini from routing (`reason_code: gemini_tool_history_excluded`) unless the operator sets `force_model_id` via `/model`. Repairable Google tool history is delegated normally.
|
|
466
491
|
|
|
467
|
-
**
|
|
492
|
+
**Empty fleet fail-safe (SP-084):** when the guard filters every model in the scoped fleet (e.g. Google/Gemini-only dogfood configs with unrepairable replay risk), the router throws an actionable error instead of delegating with `selected_model_id: unknown`. Add a non-Google model such as `openai/gpt-4o-mini` or `cursor/auto` to the fleet, start `/new`, or pin `/model` to force a specific model.
|
|
468
493
|
|
|
469
|
-
|
|
470
|
-
2. Switch to a non-Google model (e.g. `/model openai/gpt-4o-mini`) for tool-heavy sessions until the upstream fix lands.
|
|
494
|
+
**If you still see a `thought_signature` error:**
|
|
471
495
|
|
|
472
|
-
|
|
496
|
+
1. Start a fresh session with `/new` in pi (clears unrepairable history).
|
|
497
|
+
2. Switch to a non-Google model (e.g. `/model openai/gpt-4o-mini`) for that session.
|
|
498
|
+
3. Upstream: [pi#6342](https://github.com/earendil-works/pi/issues/6342) tracks pi preserving thought signatures in session replay; smart-router repair covers the common cross-model routing case without waiting on that fix.
|
|
499
|
+
|
|
500
|
+
Related: [pi-smart-router#37](https://github.com/beettlle/pi-smart-router/issues/37), [pi-smart-router#38](https://github.com/beettlle/pi-smart-router/issues/38), [pi-smart-router#40](https://github.com/beettlle/pi-smart-router/issues/40), [pi-smart-router#41](https://github.com/beettlle/pi-smart-router/issues/41), [pi-smart-router#85](https://github.com/beettlle/pi-smart-router/issues/85).
|
|
473
501
|
|
|
474
502
|
### Explain endpoint (library API)
|
|
475
503
|
|
|
@@ -537,6 +565,25 @@ Contributors must run `npm run build` before publishing or consuming the library
|
|
|
537
565
|
| `npm run routing:calibration-aggregate` | Aggregate community telemetry for calibration |
|
|
538
566
|
| `npm run routing:train-calibration` | Train routing calibration artifact bundle |
|
|
539
567
|
| `npm run routing:verify-calibration` | Verify calibration bundle against benchmark prompts |
|
|
568
|
+
| `npm run routing:ingest-benchmarks` | Regenerate `config/benchmark-profiles.json` from leaderboard fixtures |
|
|
569
|
+
| `npm run routing:verify-benchmark-profiles` | CI smoke: assert checked-in profiles match fixture ingest |
|
|
570
|
+
|
|
571
|
+
### Benchmark profile refresh
|
|
572
|
+
|
|
573
|
+
Capability scores in `config/benchmark-profiles.json` are grounded from public leaderboard snapshots under `tests/fixtures/benchmark-leaderboards/`. Each artifact records provenance (`source_urls`, `scrape_date`, `catalog_freeze_date`) in its header.
|
|
574
|
+
|
|
575
|
+
**Operator policy:**
|
|
576
|
+
|
|
577
|
+
1. **PR smoke** — `.github/workflows/benchmark-profile-refresh.yml` runs on PRs that touch fixtures, ingest, or the checked-in artifact. It executes `npm run routing:verify-benchmark-profiles` so fixture edits cannot drift from `config/benchmark-profiles.json`.
|
|
578
|
+
2. **Monthly refresh** — the same workflow runs on the 1st of each month (06:00 UTC) and via `workflow_dispatch`. It re-ingests fixtures, updates `catalog_freeze_date` to the run date, and opens a PR when model scores change.
|
|
579
|
+
3. **Manual updates** — after editing fixture snapshots, run `npm run routing:ingest-benchmarks` (optionally `--catalog-freeze-date YYYY-MM-DD`) and commit the regenerated `config/benchmark-profiles.json` with the PR.
|
|
580
|
+
|
|
581
|
+
Regenerate locally:
|
|
582
|
+
|
|
583
|
+
```bash
|
|
584
|
+
npm run routing:ingest-benchmarks
|
|
585
|
+
npm run routing:verify-benchmark-profiles
|
|
586
|
+
```
|
|
540
587
|
|
|
541
588
|
### Releasing
|
|
542
589
|
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"provenance": {
|
|
4
|
+
"source_urls": {
|
|
5
|
+
"swebench_verified": "https://www.swebench.com/",
|
|
6
|
+
"terminal_bench": "https://www.tbench.ai/leaderboard",
|
|
7
|
+
"livecodebench": "https://livecodebench.github.io/leaderboard.html",
|
|
8
|
+
"bfcl": "https://gorilla.cs.berkeley.edu/leaderboard.html"
|
|
9
|
+
},
|
|
10
|
+
"scrape_date": "2026-07-09",
|
|
11
|
+
"catalog_freeze_date": "2026-07-09"
|
|
12
|
+
},
|
|
13
|
+
"models": [
|
|
14
|
+
{
|
|
15
|
+
"model_id": "claude-3.5-haiku",
|
|
16
|
+
"capabilities": {
|
|
17
|
+
"reasoning": 0.5685,
|
|
18
|
+
"code_gen": 0.587,
|
|
19
|
+
"tool_use": 0.6095
|
|
20
|
+
},
|
|
21
|
+
"benchmark_sources": {
|
|
22
|
+
"bfcl": {
|
|
23
|
+
"raw_score": 70.6,
|
|
24
|
+
"normalized": 0.706
|
|
25
|
+
},
|
|
26
|
+
"livecodebench": {
|
|
27
|
+
"raw_score": 55,
|
|
28
|
+
"normalized": 0.55
|
|
29
|
+
},
|
|
30
|
+
"swebench_verified": {
|
|
31
|
+
"raw_score": 62.4,
|
|
32
|
+
"normalized": 0.624
|
|
33
|
+
},
|
|
34
|
+
"terminal_bench": {
|
|
35
|
+
"raw_score": 51.3,
|
|
36
|
+
"normalized": 0.513
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"model_id": "claude-opus-4-5",
|
|
42
|
+
"capabilities": {
|
|
43
|
+
"reasoning": 0.767,
|
|
44
|
+
"code_gen": 0.7915,
|
|
45
|
+
"tool_use": 0.8035
|
|
46
|
+
},
|
|
47
|
+
"benchmark_sources": {
|
|
48
|
+
"bfcl": {
|
|
49
|
+
"raw_score": 88.2,
|
|
50
|
+
"normalized": 0.882
|
|
51
|
+
},
|
|
52
|
+
"livecodebench": {
|
|
53
|
+
"raw_score": 77.4,
|
|
54
|
+
"normalized": 0.774
|
|
55
|
+
},
|
|
56
|
+
"swebench_verified": {
|
|
57
|
+
"raw_score": 80.9,
|
|
58
|
+
"normalized": 0.809
|
|
59
|
+
},
|
|
60
|
+
"terminal_bench": {
|
|
61
|
+
"raw_score": 72.5,
|
|
62
|
+
"normalized": 0.725
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"model_id": "claude-sonnet-4-6",
|
|
68
|
+
"capabilities": {
|
|
69
|
+
"reasoning": 0.738,
|
|
70
|
+
"code_gen": 0.7635,
|
|
71
|
+
"tool_use": 0.7625
|
|
72
|
+
},
|
|
73
|
+
"benchmark_sources": {
|
|
74
|
+
"bfcl": {
|
|
75
|
+
"raw_score": 84.5,
|
|
76
|
+
"normalized": 0.845
|
|
77
|
+
},
|
|
78
|
+
"livecodebench": {
|
|
79
|
+
"raw_score": 73.1,
|
|
80
|
+
"normalized": 0.731
|
|
81
|
+
},
|
|
82
|
+
"swebench_verified": {
|
|
83
|
+
"raw_score": 79.6,
|
|
84
|
+
"normalized": 0.7959999999999999
|
|
85
|
+
},
|
|
86
|
+
"terminal_bench": {
|
|
87
|
+
"raw_score": 68,
|
|
88
|
+
"normalized": 0.68
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"model_id": "gemini-2.5-flash",
|
|
94
|
+
"capabilities": {
|
|
95
|
+
"reasoning": 0.5295,
|
|
96
|
+
"code_gen": 0.5525,
|
|
97
|
+
"tool_use": 0.5735
|
|
98
|
+
},
|
|
99
|
+
"benchmark_sources": {
|
|
100
|
+
"bfcl": {
|
|
101
|
+
"raw_score": 66.9,
|
|
102
|
+
"normalized": 0.669
|
|
103
|
+
},
|
|
104
|
+
"livecodebench": {
|
|
105
|
+
"raw_score": 52.4,
|
|
106
|
+
"normalized": 0.524
|
|
107
|
+
},
|
|
108
|
+
"swebench_verified": {
|
|
109
|
+
"raw_score": 58.1,
|
|
110
|
+
"normalized": 0.581
|
|
111
|
+
},
|
|
112
|
+
"terminal_bench": {
|
|
113
|
+
"raw_score": 47.8,
|
|
114
|
+
"normalized": 0.478
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"model_id": "gpt-5.3-codex",
|
|
120
|
+
"capabilities": {
|
|
121
|
+
"reasoning": 0.796,
|
|
122
|
+
"code_gen": 0.833,
|
|
123
|
+
"tool_use": 0.8215
|
|
124
|
+
},
|
|
125
|
+
"benchmark_sources": {
|
|
126
|
+
"bfcl": {
|
|
127
|
+
"raw_score": 90.1,
|
|
128
|
+
"normalized": 0.9009999999999999
|
|
129
|
+
},
|
|
130
|
+
"livecodebench": {
|
|
131
|
+
"raw_score": 81.6,
|
|
132
|
+
"normalized": 0.816
|
|
133
|
+
},
|
|
134
|
+
"swebench_verified": {
|
|
135
|
+
"raw_score": 85,
|
|
136
|
+
"normalized": 0.85
|
|
137
|
+
},
|
|
138
|
+
"terminal_bench": {
|
|
139
|
+
"raw_score": 74.2,
|
|
140
|
+
"normalized": 0.742
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
]
|
|
145
|
+
}
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# Example fleet catalog — copy to config/models.yaml and customize.
|
|
2
2
|
# See specs/001-build-smart-router/data-model.md (ModelProfile) and docs/PRD.md §5.
|
|
3
|
+
#
|
|
4
|
+
# Capability scores in this file are static examples. At runtime, mapPiModelToProfile
|
|
5
|
+
# prefers benchmark-grounded profiles from config/benchmark-profiles.json (SP-136)
|
|
6
|
+
# when a model id has an ingest row; regex/pattern defaults apply otherwise.
|
|
7
|
+
# Regenerate the artifact with: npm run routing:ingest-benchmarks
|
|
3
8
|
|
|
4
9
|
models:
|
|
5
10
|
- id: local-gemma-4-7b
|
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version":
|
|
2
|
+
"version": 2,
|
|
3
3
|
"minimum_training_samples": {
|
|
4
4
|
"hydra_projection": 100,
|
|
5
5
|
"triage_thresholds": 50,
|
|
6
6
|
"p_success_weights": 30,
|
|
7
|
+
"isotonic_calibrator": 30,
|
|
7
8
|
"routing_centroids": 10
|
|
8
9
|
},
|
|
9
10
|
"hydra_projection": {
|
|
10
|
-
"version":
|
|
11
|
+
"version": 2,
|
|
11
12
|
"embedding_dim": 384,
|
|
13
|
+
"prefix_schema_version": 2,
|
|
14
|
+
"prefix_flag_count": 7,
|
|
12
15
|
"weights": [
|
|
13
16
|
0,
|
|
14
17
|
0,
|
|
@@ -1205,6 +1208,15 @@
|
|
|
1205
1208
|
],
|
|
1206
1209
|
"trained_sample_count": 0
|
|
1207
1210
|
},
|
|
1211
|
+
"isotonic_calibrator": {
|
|
1212
|
+
"version": 1,
|
|
1213
|
+
"min_training_samples": 30,
|
|
1214
|
+
"x_knots": [0, 1],
|
|
1215
|
+
"y_knots": [0, 1],
|
|
1216
|
+
"trained_sample_count": 0,
|
|
1217
|
+
"holdout_ece_raw": null,
|
|
1218
|
+
"holdout_ece_calibrated": null
|
|
1219
|
+
},
|
|
1208
1220
|
"routing_centroids": {
|
|
1209
1221
|
"version": 1,
|
|
1210
1222
|
"embedding_dim": 384,
|
|
@@ -12,9 +12,10 @@
|
|
|
12
12
|
* - MUST produce a decision identical to the live pipeline for the same input
|
|
13
13
|
* - MUST emit no RoutingTelemetry with upstream cost (routing_latency_ms only)
|
|
14
14
|
*/
|
|
15
|
-
import type { ModelProfile, RoutingDecision } from '../../domain/types/index.js';
|
|
15
|
+
import type { ModelProfile, RoutingDecision, SaarConfig } from '../../domain/types/index.js';
|
|
16
16
|
import { RouterPipeline } from '../../domain/pipeline/router-pipeline.js';
|
|
17
17
|
import type { ClusterMatcher } from '../../domain/matching/cluster-matcher.js';
|
|
18
|
+
import type { SessionPinner } from '../../domain/pinning/session-pinner.js';
|
|
18
19
|
export interface ExplainValidationError {
|
|
19
20
|
readonly error: 'validation_failed';
|
|
20
21
|
readonly details: readonly string[];
|
|
@@ -36,6 +37,8 @@ export interface ExplainHandlerDeps {
|
|
|
36
37
|
readonly fleet: readonly ModelProfile[];
|
|
37
38
|
readonly pipeline: RouterPipeline;
|
|
38
39
|
readonly clusterMatcher?: ClusterMatcher;
|
|
40
|
+
readonly sessionPinner?: SessionPinner;
|
|
41
|
+
readonly saarConfig?: SaarConfig;
|
|
39
42
|
}
|
|
40
43
|
/**
|
|
41
44
|
* Create an explain handler bound to a shared pipeline instance.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router-explain.d.ts","sourceRoot":"","sources":["../../../src/api/explain/router-explain.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAA2B,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"router-explain.d.ts","sourceRoot":"","sources":["../../../src/api/explain/router-explain.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAA2B,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAEtH,OAAO,EAAE,cAAc,EAAE,MAAM,0CAA0C,CAAC;AAE1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0CAA0C,CAAC;AAC/E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wCAAwC,CAAC;AAK5E,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,KAAK,EAAE,mBAAmB,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;CAChC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;CACvC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;CAChC;AAED,MAAM,MAAM,aAAa,GAAG,cAAc,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;AAIpF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,SAAS,YAAY,EAAE,CAAC;IACxC,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,QAAQ,CAAC,cAAc,CAAC,EAAE,cAAc,CAAC;IACzC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;CAClC;AAID;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,kBAAkB,IAU7B,SAAS,OAAO,KAAG,OAAO,CAAC,aAAa,CAAC,CAyCxE"}
|
|
@@ -23,7 +23,13 @@ import { enrichRoutingDecisionForExplain } from '../../infrastructure/telemetry/
|
|
|
23
23
|
* session pin state and stage ordering are identical.
|
|
24
24
|
*/
|
|
25
25
|
export function createExplainHandler(deps) {
|
|
26
|
-
const { fleet, pipeline, clusterMatcher } = deps;
|
|
26
|
+
const { fleet, pipeline, clusterMatcher, sessionPinner, saarConfig } = deps;
|
|
27
|
+
const explainEnrichmentOptions = {
|
|
28
|
+
fleet,
|
|
29
|
+
...(clusterMatcher !== undefined ? { clusterMatcher } : {}),
|
|
30
|
+
...(sessionPinner !== undefined ? { sessionPinner } : {}),
|
|
31
|
+
...(saarConfig !== undefined ? { saarConfig } : {}),
|
|
32
|
+
};
|
|
27
33
|
return async function explain(rawBody) {
|
|
28
34
|
const parsed = RoutingRequestSchema.safeParse(rawBody);
|
|
29
35
|
if (!parsed.success) {
|
|
@@ -41,10 +47,7 @@ export function createExplainHandler(deps) {
|
|
|
41
47
|
const decision = await pipeline.route(request);
|
|
42
48
|
return {
|
|
43
49
|
status: 200,
|
|
44
|
-
body: await enrichRoutingDecisionForExplain(request, decision,
|
|
45
|
-
fleet,
|
|
46
|
-
...(clusterMatcher !== undefined ? { clusterMatcher } : {}),
|
|
47
|
-
}),
|
|
50
|
+
body: await enrichRoutingDecisionForExplain(request, decision, explainEnrichmentOptions),
|
|
48
51
|
};
|
|
49
52
|
}
|
|
50
53
|
catch {
|
|
@@ -60,10 +63,7 @@ export function createExplainHandler(deps) {
|
|
|
60
63
|
};
|
|
61
64
|
return {
|
|
62
65
|
status: 503,
|
|
63
|
-
body: await enrichRoutingDecisionForExplain(request, decision,
|
|
64
|
-
fleet,
|
|
65
|
-
...(clusterMatcher !== undefined ? { clusterMatcher } : {}),
|
|
66
|
-
}),
|
|
66
|
+
body: await enrichRoutingDecisionForExplain(request, decision, explainEnrichmentOptions),
|
|
67
67
|
};
|
|
68
68
|
}
|
|
69
69
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router-explain.js","sourceRoot":"","sources":["../../../src/api/explain/router-explain.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAErE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uCAAuC,CAAC;
|
|
1
|
+
{"version":3,"file":"router-explain.js","sourceRoot":"","sources":["../../../src/api/explain/router-explain.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAErE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uCAAuC,CAAC;AAGzE,OAAO,EAAE,+BAA+B,EAAE,MAAM,qDAAqD,CAAC;AAoCtG,iFAAiF;AAEjF;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAwB;IAC3D,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAE5E,MAAM,wBAAwB,GAAG;QAC/B,KAAK;QACL,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,GAAG,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpD,CAAC;IAEF,OAAO,KAAK,UAAU,OAAO,CAAC,OAAgB;QAC5C,MAAM,MAAM,GAAG,oBAAoB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAEvD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE;oBACJ,KAAK,EAAE,mBAAmB;oBAC1B,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CACvD;iBACF;aACF,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,MAAM,+BAA+B,CAAC,OAAO,EAAE,QAAQ,EAAE,wBAAwB,CAAC;aACzF,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,aAAa,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAoB;gBAChC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,iBAAiB,EAAE,aAAa,EAAE,EAAE,IAAI,SAAS;gBACjD,IAAI,EAAE,aAAa,EAAE,IAAI,IAAI,kBAAkB;gBAC/C,KAAK,EAAE,UAAU;gBACjB,WAAW,EAAE,cAAc;gBAC3B,kBAAkB,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC7C,UAAU,EAAE,IAAI;aACjB,CAAC;YACF,OAAO;gBACL,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,MAAM,+BAA+B,CAAC,OAAO,EAAE,QAAQ,EAAE,wBAAwB,CAAC;aACzF,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAMD;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,IAAmB;IAC3C,+EAA+E;IAC/E,wEAAwE;IACxE,MAAM,MAAM,GAA0F;QACpG,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC;IAEF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAW,EAAE,CAAC,CAAC;YACtD,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC,CAAC;IACN,CAAC;IACD,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;QAAE,MAAM,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;IACvE,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS;QAAE,MAAM,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC;IACzF,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS;QAAE,MAAM,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC;IACtF,IAAI,IAAI,CAAC,sBAAsB,KAAK,SAAS;QAAE,MAAM,CAAC,wBAAwB,CAAC,GAAG,IAAI,CAAC,sBAAsB,CAAC;IAE9G,OAAO,MAAwB,CAAC;AAClC,CAAC"}
|
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
* Operator configuration defaults (FR-021).
|
|
3
3
|
* Values sourced from specs/001-build-smart-router/data-model.md § Configuration (Operator).
|
|
4
4
|
*/
|
|
5
|
-
import type
|
|
5
|
+
import { type OperatorConfig } from '../domain/types/schemas.js';
|
|
6
|
+
export { DEFAULT_SAAR_CONFIG, resolveSaarConfigFromEnv } from '../domain/types/schemas.js';
|
|
7
|
+
/** Merge operator env overrides onto defaults (SAAR section only today). */
|
|
8
|
+
export declare function resolveOperatorConfigFromEnv(base?: OperatorConfig): OperatorConfig;
|
|
6
9
|
export declare const DEFAULT_OPERATOR_CONFIG: Readonly<OperatorConfig>;
|
|
7
10
|
//# sourceMappingURL=defaults.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/config/defaults.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/config/defaults.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAGL,KAAK,cAAc,EACpB,MAAM,4BAA4B,CAAC;AAGpC,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AAE3F,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAC1C,IAAI,GAAE,cAAwC,GAC7C,cAAc,CAKhB;AAED,eAAO,MAAM,uBAAuB,EAAE,QAAQ,CAAC,cAAc,CA2BnD,CAAC"}
|
package/dist/config/defaults.js
CHANGED
|
@@ -2,7 +2,16 @@
|
|
|
2
2
|
* Operator configuration defaults (FR-021).
|
|
3
3
|
* Values sourced from specs/001-build-smart-router/data-model.md § Configuration (Operator).
|
|
4
4
|
*/
|
|
5
|
+
import { DEFAULT_SAAR_CONFIG, resolveSaarConfigFromEnv, } from '../domain/types/schemas.js';
|
|
5
6
|
import { DEFAULT_LOW_INTENSITY_WEIGHTS } from '../domain/routing/tier-features.js';
|
|
7
|
+
export { DEFAULT_SAAR_CONFIG, resolveSaarConfigFromEnv } from '../domain/types/schemas.js';
|
|
8
|
+
/** Merge operator env overrides onto defaults (SAAR section only today). */
|
|
9
|
+
export function resolveOperatorConfigFromEnv(base = DEFAULT_OPERATOR_CONFIG) {
|
|
10
|
+
return {
|
|
11
|
+
...base,
|
|
12
|
+
saar: resolveSaarConfigFromEnv(base.saar),
|
|
13
|
+
};
|
|
14
|
+
}
|
|
6
15
|
export const DEFAULT_OPERATOR_CONFIG = {
|
|
7
16
|
frugality: {
|
|
8
17
|
lambda_cost: 0.5,
|
|
@@ -29,5 +38,6 @@ export const DEFAULT_OPERATOR_CONFIG = {
|
|
|
29
38
|
low_threshold: 0.35,
|
|
30
39
|
p_success_alpha: 0.5,
|
|
31
40
|
},
|
|
41
|
+
saar: DEFAULT_SAAR_CONFIG,
|
|
32
42
|
};
|
|
33
43
|
//# sourceMappingURL=defaults.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"defaults.js","sourceRoot":"","sources":["../../src/config/defaults.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"defaults.js","sourceRoot":"","sources":["../../src/config/defaults.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,mBAAmB,EACnB,wBAAwB,GAEzB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,6BAA6B,EAAE,MAAM,oCAAoC,CAAC;AAEnF,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AAE3F,4EAA4E;AAC5E,MAAM,UAAU,4BAA4B,CAC1C,OAAuB,uBAAuB;IAE9C,OAAO;QACL,GAAG,IAAI;QACP,IAAI,EAAE,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,uBAAuB,GAA6B;IAC/D,SAAS,EAAE;QACT,WAAW,EAAE,GAAG;QAChB,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,IAAI;KACvB;IACD,eAAe,EAAE;QACf,SAAS,EAAE,CAAC;KACb;IACD,OAAO,EAAE;QACP,cAAc,EAAE,EAAE;KACnB;IACD,KAAK,EAAE;QACL,kBAAkB,EAAE,EAAE;QACtB,4BAA4B,EAAE,CAAC;QAC/B,qBAAqB,EAAE,EAAE;KAC1B;IACD,KAAK,EAAE;QACL,mBAAmB,EAAE,0BAA0B;KAChD;IACD,aAAa,EAAE;QACb,OAAO,EAAE,6BAA6B;QACtC,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,GAAG;KACrB;IACD,IAAI,EAAE,mBAAmB;CACjB,CAAC"}
|
|
@@ -2,10 +2,20 @@
|
|
|
2
2
|
* Pi model registry → ModelProfile mapper.
|
|
3
3
|
*
|
|
4
4
|
* Maps pi `Model` objects (provider + id) to router fleet entries using
|
|
5
|
-
* pattern-based lookup for known families.
|
|
6
|
-
*
|
|
5
|
+
* pattern-based lookup for known families. When `config/benchmark-profiles.json`
|
|
6
|
+
* contains a row for the model id (SP-134/136 ingest output), capability
|
|
7
|
+
* vectors are grounded in benchmark scores instead of regex defaults.
|
|
8
|
+
* Unknown models or missing benchmark rows receive conservative pattern defaults.
|
|
7
9
|
*/
|
|
8
10
|
import type { ModelLimits, ModelProfile, Tier } from '../domain/types/entities.js';
|
|
11
|
+
/** Checked-in ingest artifact from `npm run routing:ingest-benchmarks` (SP-134). */
|
|
12
|
+
export declare const DEFAULT_BENCHMARK_PROFILES_PATH: string;
|
|
13
|
+
/**
|
|
14
|
+
* Test hook — override benchmark artifact path (null disables benchmark grounding).
|
|
15
|
+
*/
|
|
16
|
+
export declare function setBenchmarkProfilesPathForTests(filePath: string | null): void;
|
|
17
|
+
/** Test hook — clear cached benchmark artifact between cases. */
|
|
18
|
+
export declare function resetBenchmarkProfilesCacheForTests(): void;
|
|
9
19
|
/** Pi registry `Model.cost` shape — per-token USD rates. */
|
|
10
20
|
export interface PiRegistryCost {
|
|
11
21
|
readonly input: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pi-model-mapper.d.ts","sourceRoot":"","sources":["../../src/config/pi-model-mapper.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"pi-model-mapper.d.ts","sourceRoot":"","sources":["../../src/config/pi-model-mapper.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAOH,OAAO,KAAK,EAEV,WAAW,EAGX,YAAY,EACZ,IAAI,EACL,MAAM,6BAA6B,CAAC;AAErC,oFAAoF;AACpF,eAAO,MAAM,+BAA+B,QAA+C,CAAC;AAqB5F;;GAEG;AACH,wBAAgB,gCAAgC,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAG9E;AAED,iEAAiE;AACjE,wBAAgB,mCAAmC,IAAI,IAAI,CAG1D;AA+DD,4DAA4D;AAC5D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC;CAChC;AAiBD,yFAAyF;AACzF,eAAO,MAAM,oBAAoB,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAIpE,CAAC;AAEF,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,IAAI,GAAG,WAAW,CAE/D;AAwDD;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,IAAM,CAAC;AAqIpD;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,YAAY,CAuBrE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,GAAG,YAAY,EAAE,CAEpF"}
|
|
@@ -2,9 +2,94 @@
|
|
|
2
2
|
* Pi model registry → ModelProfile mapper.
|
|
3
3
|
*
|
|
4
4
|
* Maps pi `Model` objects (provider + id) to router fleet entries using
|
|
5
|
-
* pattern-based lookup for known families.
|
|
6
|
-
*
|
|
5
|
+
* pattern-based lookup for known families. When `config/benchmark-profiles.json`
|
|
6
|
+
* contains a row for the model id (SP-134/136 ingest output), capability
|
|
7
|
+
* vectors are grounded in benchmark scores instead of regex defaults.
|
|
8
|
+
* Unknown models or missing benchmark rows receive conservative pattern defaults.
|
|
7
9
|
*/
|
|
10
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
11
|
+
import { resolve } from 'node:path';
|
|
12
|
+
import { z } from 'zod';
|
|
13
|
+
/** Checked-in ingest artifact from `npm run routing:ingest-benchmarks` (SP-134). */
|
|
14
|
+
export const DEFAULT_BENCHMARK_PROFILES_PATH = resolve('config', 'benchmark-profiles.json');
|
|
15
|
+
const benchmarkCapabilitiesSchema = z.object({
|
|
16
|
+
reasoning: z.number().min(0).max(1),
|
|
17
|
+
code_gen: z.number().min(0).max(1),
|
|
18
|
+
tool_use: z.number().min(0).max(1),
|
|
19
|
+
});
|
|
20
|
+
const benchmarkModelRowSchema = z.object({
|
|
21
|
+
model_id: z.string().min(1),
|
|
22
|
+
capabilities: benchmarkCapabilitiesSchema,
|
|
23
|
+
});
|
|
24
|
+
const benchmarkProfilesArtifactSchema = z.object({
|
|
25
|
+
version: z.literal(1),
|
|
26
|
+
models: z.array(benchmarkModelRowSchema).min(1),
|
|
27
|
+
});
|
|
28
|
+
let benchmarkProfilesPathOverride;
|
|
29
|
+
let benchmarkCapabilitiesByModelId;
|
|
30
|
+
/**
|
|
31
|
+
* Test hook — override benchmark artifact path (null disables benchmark grounding).
|
|
32
|
+
*/
|
|
33
|
+
export function setBenchmarkProfilesPathForTests(filePath) {
|
|
34
|
+
benchmarkProfilesPathOverride = filePath;
|
|
35
|
+
benchmarkCapabilitiesByModelId = undefined;
|
|
36
|
+
}
|
|
37
|
+
/** Test hook — clear cached benchmark artifact between cases. */
|
|
38
|
+
export function resetBenchmarkProfilesCacheForTests() {
|
|
39
|
+
benchmarkProfilesPathOverride = undefined;
|
|
40
|
+
benchmarkCapabilitiesByModelId = undefined;
|
|
41
|
+
}
|
|
42
|
+
function resolveBenchmarkProfilesPath() {
|
|
43
|
+
if (benchmarkProfilesPathOverride === null) {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
return benchmarkProfilesPathOverride ?? DEFAULT_BENCHMARK_PROFILES_PATH;
|
|
47
|
+
}
|
|
48
|
+
function loadBenchmarkCapabilitiesMap() {
|
|
49
|
+
if (benchmarkCapabilitiesByModelId !== undefined) {
|
|
50
|
+
return benchmarkCapabilitiesByModelId;
|
|
51
|
+
}
|
|
52
|
+
const filePath = resolveBenchmarkProfilesPath();
|
|
53
|
+
if (filePath === null || !existsSync(filePath)) {
|
|
54
|
+
benchmarkCapabilitiesByModelId = null;
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
const raw = readFileSync(filePath, 'utf8');
|
|
59
|
+
const parsed = JSON.parse(raw);
|
|
60
|
+
const result = benchmarkProfilesArtifactSchema.safeParse(parsed);
|
|
61
|
+
if (!result.success) {
|
|
62
|
+
throw new Error(result.error.message);
|
|
63
|
+
}
|
|
64
|
+
const map = new Map();
|
|
65
|
+
for (const row of result.data.models) {
|
|
66
|
+
map.set(row.model_id, { ...row.capabilities });
|
|
67
|
+
}
|
|
68
|
+
benchmarkCapabilitiesByModelId = map;
|
|
69
|
+
return map;
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
console.warn('benchmark profiles artifact invalid; using regex capability defaults', {
|
|
73
|
+
path: filePath,
|
|
74
|
+
error: err instanceof Error ? err.message : String(err),
|
|
75
|
+
});
|
|
76
|
+
benchmarkCapabilitiesByModelId = null;
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
function lookupBenchmarkCapabilities(modelId) {
|
|
81
|
+
return loadBenchmarkCapabilitiesMap()?.get(modelId);
|
|
82
|
+
}
|
|
83
|
+
function withBenchmarkCapabilities(defaults, modelId) {
|
|
84
|
+
const grounded = lookupBenchmarkCapabilities(modelId);
|
|
85
|
+
if (grounded === undefined) {
|
|
86
|
+
return defaults;
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
...defaults,
|
|
90
|
+
capabilities: { ...grounded },
|
|
91
|
+
};
|
|
92
|
+
}
|
|
8
93
|
const LOCAL_PROVIDERS = new Set(['lmstudio', 'ollama']);
|
|
9
94
|
/** Conservative context limits when YAML and LiteLLM registry have no entry (SP-092). */
|
|
10
95
|
export const DEFAULT_MODEL_LIMITS = {
|
|
@@ -196,16 +281,16 @@ export function mapPiModelToProfile(input) {
|
|
|
196
281
|
id: input.id,
|
|
197
282
|
...(input.name !== undefined ? { name: input.name } : {}),
|
|
198
283
|
};
|
|
199
|
-
return buildProfile(localInput, LOCAL_DEFAULTS);
|
|
284
|
+
return buildProfile(localInput, withBenchmarkCapabilities(LOCAL_DEFAULTS, input.id));
|
|
200
285
|
}
|
|
201
286
|
if (input.id === OPAQUE_FLEET_DEFAULT_ID) {
|
|
202
|
-
return buildProfile(input, CURSOR_AUTO_DEFAULTS);
|
|
287
|
+
return buildProfile(input, withBenchmarkCapabilities(CURSOR_AUTO_DEFAULTS, input.id));
|
|
203
288
|
}
|
|
204
289
|
const matched = matchPatternRules(input.id);
|
|
205
290
|
if (matched) {
|
|
206
|
-
return buildProfile(input, matched);
|
|
291
|
+
return buildProfile(input, withBenchmarkCapabilities(matched, input.id));
|
|
207
292
|
}
|
|
208
|
-
return buildProfile(input, UNKNOWN_DEFAULTS);
|
|
293
|
+
return buildProfile(input, withBenchmarkCapabilities(UNKNOWN_DEFAULTS, input.id));
|
|
209
294
|
}
|
|
210
295
|
/**
|
|
211
296
|
* Map an array of pi registry models to a router fleet catalog.
|