pi-smart-router 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/extension-setup.ts +11 -4
- package/.pi/extensions/smart-router/fleet-bootstrap.ts +53 -4
- package/.pi/extensions/smart-router/index.ts +2 -0
- package/README.md +41 -2
- package/config/benchmark-profiles.json +22 -0
- package/config/p-success-weights.json +36 -0
- package/config/p-success-weights.json.example +4 -1
- package/dist/config/pi-model-mapper.d.ts +24 -5
- package/dist/config/pi-model-mapper.d.ts.map +1 -1
- package/dist/config/pi-model-mapper.js +67 -24
- package/dist/config/pi-model-mapper.js.map +1 -1
- package/package.json +2 -1
- package/src/config/pi-model-mapper.ts +100 -36
|
@@ -4,8 +4,8 @@ import {
|
|
|
4
4
|
type ExtensionAPI,
|
|
5
5
|
} from '@earendil-works/pi-coding-agent';
|
|
6
6
|
|
|
7
|
+
import { resolveOperatorConfigFromEnv } from '../../../src/config/defaults.js';
|
|
7
8
|
import { ExecutionLedger } from '../../../src/domain/delegation/execution-ledger.js';
|
|
8
|
-
import { SessionPinner } from '../../../src/domain/pinning/session-pinner.js';
|
|
9
9
|
import type { SessionRoutingSnapshot } from '../../../src/infrastructure/telemetry/outcome-recorder.js';
|
|
10
10
|
import { createRouterFromFleet, LifecycleHookState } from '../../../src/index.js';
|
|
11
11
|
|
|
@@ -14,7 +14,11 @@ import {
|
|
|
14
14
|
createExtensionDatasetRecorder,
|
|
15
15
|
createExtensionOutcomeRecorder,
|
|
16
16
|
} from './dataset-export.js';
|
|
17
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
createDispatchOptions,
|
|
19
|
+
createOperatorAwareSessionPinner,
|
|
20
|
+
initHydraMatcher,
|
|
21
|
+
} from './fleet-bootstrap.js';
|
|
18
22
|
import { setupSessionHooks } from './session-lifecycle.js';
|
|
19
23
|
import { createStreamSimple } from './stream-delegation.js';
|
|
20
24
|
import type { SmartRouterRuntime } from './types.js';
|
|
@@ -35,7 +39,8 @@ export async function createSmartRouterRuntime(cwd: string): Promise<{
|
|
|
35
39
|
const modelRegistry = ModelRegistry.inMemory(authStorage);
|
|
36
40
|
const hydraMatcher = await initHydraMatcher();
|
|
37
41
|
const store = createExtensionStore(cwd);
|
|
38
|
-
const
|
|
42
|
+
const operatorConfig = resolveOperatorConfigFromEnv();
|
|
43
|
+
const sessionPinner = createOperatorAwareSessionPinner(store, operatorConfig);
|
|
39
44
|
const executionLedger = new ExecutionLedger();
|
|
40
45
|
const lifecycleHookState = new LifecycleHookState();
|
|
41
46
|
const datasetNotify: DatasetNotify = {
|
|
@@ -62,7 +67,9 @@ export async function createSmartRouterRuntime(cwd: string): Promise<{
|
|
|
62
67
|
sessionRouting,
|
|
63
68
|
streamDeps: {
|
|
64
69
|
router: createRouterFromFleet([], {
|
|
65
|
-
...createDispatchOptions(store, sessionPinner, hydraMatcher
|
|
70
|
+
...createDispatchOptions(store, sessionPinner, hydraMatcher, {
|
|
71
|
+
operatorConfig,
|
|
72
|
+
}),
|
|
66
73
|
lifecycleHookState,
|
|
67
74
|
}),
|
|
68
75
|
modelRegistry,
|
|
@@ -6,13 +6,18 @@ import {
|
|
|
6
6
|
} from '@earendil-works/pi-coding-agent';
|
|
7
7
|
|
|
8
8
|
import { mapFleetFromRegistry } from '../../../src/config/pi-model-mapper.js';
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
DEFAULT_OPERATOR_CONFIG,
|
|
11
|
+
resolveOperatorConfigFromEnv,
|
|
12
|
+
} from '../../../src/config/defaults.js';
|
|
10
13
|
import {
|
|
11
14
|
HydraMatcher,
|
|
12
15
|
createOnnxEmbeddingProvider,
|
|
13
16
|
} from '../../../src/domain/matching/hydra-matcher.js';
|
|
14
17
|
import { SessionPinner } from '../../../src/domain/pinning/session-pinner.js';
|
|
15
18
|
import type { ModelProfile, PriceCatalog } from '../../../src/domain/types/index.js';
|
|
19
|
+
import type { QuotaWindowPosition } from '../../../src/domain/types/entities.js';
|
|
20
|
+
import type { OperatorConfig } from '../../../src/domain/types/schemas.js';
|
|
16
21
|
import type { StorePort } from '../../../src/domain/types/store-port.js';
|
|
17
22
|
import { getDefaultSystemInfo } from '../../../src/infrastructure/hardware/hardware-probe.js';
|
|
18
23
|
import { DEFAULT_LOCAL_CONFIG } from '../../../src/infrastructure/local/local-zero-tier.js';
|
|
@@ -27,6 +32,16 @@ import { resolveModelScope } from './pi-model-scope.js';
|
|
|
27
32
|
import type { FleetMode, SmartRouterRuntime } from './types.js';
|
|
28
33
|
import { resolveRateLimiter } from './utils.js';
|
|
29
34
|
|
|
35
|
+
/** Optional overrides for extension dispatch wiring (SP-173). */
|
|
36
|
+
export interface CreateDispatchOptionsExtras {
|
|
37
|
+
/** Base operator config before env merge; defaults to DEFAULT_OPERATOR_CONFIG. */
|
|
38
|
+
readonly operatorConfig?: OperatorConfig;
|
|
39
|
+
/** Live price catalog when fleet discovery has loaded one. */
|
|
40
|
+
readonly priceCatalog?: PriceCatalog | null;
|
|
41
|
+
/** Rolling subscription quota position when available. */
|
|
42
|
+
readonly quotaWindowPosition?: QuotaWindowPosition;
|
|
43
|
+
}
|
|
44
|
+
|
|
30
45
|
/** Minimal settings surface used for scoped fleet discovery. */
|
|
31
46
|
export interface ScopedSettingsReader {
|
|
32
47
|
getEnabledModels(): string[] | null | undefined;
|
|
@@ -152,26 +167,58 @@ export function createDispatchOptions(
|
|
|
152
167
|
store: StorePort,
|
|
153
168
|
sessionPinner: SessionPinner,
|
|
154
169
|
hydraMatcher?: HydraMatcher,
|
|
170
|
+
extras?: CreateDispatchOptionsExtras,
|
|
155
171
|
): GatewayDispatchOptions {
|
|
172
|
+
const operatorConfig = resolveOperatorConfigFromEnv(
|
|
173
|
+
extras?.operatorConfig ?? DEFAULT_OPERATOR_CONFIG,
|
|
174
|
+
);
|
|
156
175
|
const telemetryEmitter = new RoutingTelemetryEmitter({
|
|
157
176
|
onRecord: (record) => {
|
|
158
177
|
store.appendTelemetry(record);
|
|
159
178
|
},
|
|
179
|
+
sessionPinner,
|
|
180
|
+
saarConfig: operatorConfig.saar,
|
|
181
|
+
...(extras?.priceCatalog !== undefined ? { priceCatalog: extras.priceCatalog } : {}),
|
|
182
|
+
...(extras?.quotaWindowPosition !== undefined
|
|
183
|
+
? { quotaWindowPosition: extras.quotaWindowPosition }
|
|
184
|
+
: {}),
|
|
160
185
|
});
|
|
161
186
|
const rateLimiter = resolveRateLimiter(store);
|
|
162
187
|
|
|
163
188
|
return {
|
|
164
189
|
sessionPinner,
|
|
165
|
-
hardwareConfig:
|
|
190
|
+
hardwareConfig: operatorConfig.local,
|
|
166
191
|
systemInfoProvider: getDefaultSystemInfo,
|
|
167
192
|
localConfig: DEFAULT_LOCAL_CONFIG,
|
|
168
|
-
loopEscalationConfig:
|
|
193
|
+
loopEscalationConfig: operatorConfig.loop_escalation,
|
|
194
|
+
saarConfig: operatorConfig.saar,
|
|
195
|
+
planningDelegateConfig: operatorConfig.planning_delegate,
|
|
196
|
+
pinOnlyFallback: operatorConfig.pin_only_fallback,
|
|
197
|
+
...(extras?.priceCatalog !== undefined ? { priceCatalog: extras.priceCatalog } : {}),
|
|
198
|
+
...(extras?.quotaWindowPosition !== undefined
|
|
199
|
+
? { quotaWindowPosition: extras.quotaWindowPosition }
|
|
200
|
+
: {}),
|
|
169
201
|
...(hydraMatcher ? { hydraMatcher } : {}),
|
|
170
202
|
...(rateLimiter ? { rateLimiter } : {}),
|
|
171
203
|
telemetryEmitter,
|
|
172
204
|
};
|
|
173
205
|
}
|
|
174
206
|
|
|
207
|
+
/**
|
|
208
|
+
* Build a SessionPinner wired with operator SAAR / pin-only settings (SP-173).
|
|
209
|
+
* No operator-config.json loader exists yet — env + optional base config only.
|
|
210
|
+
*/
|
|
211
|
+
export function createOperatorAwareSessionPinner(
|
|
212
|
+
store: StorePort,
|
|
213
|
+
operatorConfig: OperatorConfig = resolveOperatorConfigFromEnv(),
|
|
214
|
+
): SessionPinner {
|
|
215
|
+
return new SessionPinner({
|
|
216
|
+
store,
|
|
217
|
+
saarConfig: operatorConfig.saar,
|
|
218
|
+
pinOnlyFallback: operatorConfig.pin_only_fallback,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
|
|
175
222
|
export async function rebuildFleet(
|
|
176
223
|
runtime: SmartRouterRuntime,
|
|
177
224
|
pi: ExtensionAPI,
|
|
@@ -189,7 +236,9 @@ export async function rebuildFleet(
|
|
|
189
236
|
runtime.priceCatalog = catalog;
|
|
190
237
|
runtime.fleetScopeFingerprint = fingerprint;
|
|
191
238
|
const router = createRouterFromFleet(fleet, {
|
|
192
|
-
...createDispatchOptions(runtime.store, runtime.sessionPinner, runtime.hydraMatcher
|
|
239
|
+
...createDispatchOptions(runtime.store, runtime.sessionPinner, runtime.hydraMatcher, {
|
|
240
|
+
priceCatalog: catalog,
|
|
241
|
+
}),
|
|
193
242
|
lifecycleHookState: runtime.lifecycleHookState,
|
|
194
243
|
});
|
|
195
244
|
router.register(createHooksAdapter(pi));
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
} from './dataset-export.js';
|
|
25
25
|
import {
|
|
26
26
|
createDispatchOptions,
|
|
27
|
+
createOperatorAwareSessionPinner,
|
|
27
28
|
discoverFleet,
|
|
28
29
|
formatLmuStatus,
|
|
29
30
|
initHydraMatcher,
|
|
@@ -62,6 +63,7 @@ export {
|
|
|
62
63
|
createDispatchOptions,
|
|
63
64
|
createExtensionDatasetRecorder,
|
|
64
65
|
createExtensionOutcomeRecorder,
|
|
66
|
+
createOperatorAwareSessionPinner,
|
|
65
67
|
createSmartRouterRuntime,
|
|
66
68
|
createStreamSimple,
|
|
67
69
|
deriveTurnType,
|
package/README.md
CHANGED
|
@@ -464,12 +464,40 @@ When `SMART_ROUTER_DATASET=1`, the router records privacy-safe dataset rows and
|
|
|
464
464
|
|
|
465
465
|
Each JSONL row joins dataset features with `success_label` and `outcome_signals`. Success means no negative outcome signals were recorded for that `request_id` (for example `model_override` or `feedback_bad` mark failure). Prompt plaintext is never included.
|
|
466
466
|
|
|
467
|
-
|
|
467
|
+
**Dogfood artifact (SP-175):** the repo ships a non-example `config/p-success-weights.json` trained on the synthetic fixture at `scripts/fixtures/p-success-synthetic-train.jsonl` (**provenance: synthetic/fixture**, not community contrib — 40 labeled feature-vector rows, no prompt text). With `trained_sample_count ≥ 30`, the low-intensity gate uses trained logistic scores instead of neutral `0.5`. Missing or invalid artifacts still fall back safely to neutral defaults.
|
|
468
|
+
|
|
469
|
+
**Operator train / reload (no prompt text):**
|
|
470
|
+
|
|
471
|
+
```bash
|
|
472
|
+
# 1) Opt in + dogfood, then export privacy-safe labeled JSONL (features + labels only)
|
|
473
|
+
SMART_ROUTER_DATASET=1
|
|
474
|
+
# …run sessions with /model smart-router/auto and /smart-router feedback…
|
|
475
|
+
/smart-router export dataset --limit 200
|
|
476
|
+
|
|
477
|
+
# 2) Train standalone weights (≥30 labeled rows required)
|
|
478
|
+
npm run routing:train-p-success -- --input path/to/export.jsonl --output config/p-success-weights.json
|
|
479
|
+
|
|
480
|
+
# Or regenerate the checked-in dogfood weights from the synthetic fixture:
|
|
481
|
+
npm run routing:train-p-success
|
|
482
|
+
|
|
483
|
+
# 3) Optional: merge isotonic into an existing calibration bundle (does not rewrite hydra/centroids)
|
|
484
|
+
npm run routing:train-p-success -- --input path/to/export.jsonl \
|
|
485
|
+
--calibration-output config/routing-calibration.json
|
|
486
|
+
|
|
487
|
+
# Full Phase-3 bundle (also refreshes standalone p-success-weights.json when the gate is met):
|
|
488
|
+
npm run routing:train-calibration -- --input path/to/aggregated.jsonl
|
|
489
|
+
```
|
|
490
|
+
|
|
491
|
+
Reload is file-based: replace `config/p-success-weights.json` (and optionally `config/routing-calibration.json` for isotonic) and restart the host agent — no prompt text is ever written into training artifacts.
|
|
492
|
+
|
|
493
|
+
**Isotonic gap:** serve-time isotonic calibration loads from `config/routing-calibration.json` (`isotonic_calibrator`). The checked-in dogfood path ships trained **logistic** weights only; isotonic is produced when you pass `--calibration-output` or run `routing:train-calibration` with ≥30 labeled samples. Until that bundle exists, the pipeline uses raw logistic `P(success)` (identity / no-op calibrator) and still exposes `p_success_raw` vs `p_success_calibrated` / `p_success_cheap` on explain and telemetry.
|
|
494
|
+
|
|
495
|
+
Library helpers (see `src/domain/routing/p-success-classifier.ts`):
|
|
468
496
|
|
|
469
497
|
- `trainFromExportJsonl(exportContent)` — fit coefficients from labeled JSONL
|
|
470
498
|
- `predictPSuccessCheap(features, weights)` — returns `P_success_cheap` in `[0, 1]`
|
|
471
499
|
|
|
472
|
-
|
|
500
|
+
**Minimum sample guidance:** collect at least **30** labeled economical-tier rows before relying on non-neutral predictions; below that threshold the classifier returns neutral `P_success_cheap = 0.5`. **Online inference** is active in the low-intensity gate; without trained weights the router uses neutral defaults until you add `config/p-success-weights.json`.
|
|
473
501
|
|
|
474
502
|
### Community telemetry contribution (calibration)
|
|
475
503
|
|
|
@@ -744,6 +772,7 @@ Contributors must run `npm run build` before publishing or consuming the library
|
|
|
744
772
|
| `npm run routing:bootstrap-centroids` | Regenerate `config/routing-centroids.json` from cluster catalog |
|
|
745
773
|
| `npm run routing:calibration-aggregate` | Aggregate community telemetry for calibration |
|
|
746
774
|
| `npm run routing:train-calibration` | Train routing calibration artifact bundle |
|
|
775
|
+
| `npm run routing:train-p-success` | Train standalone `config/p-success-weights.json` (synthetic fixture by default) |
|
|
747
776
|
| `npm run routing:verify-calibration` | Verify calibration bundle against benchmark prompts |
|
|
748
777
|
| `npm run routing:ingest-benchmarks` | Regenerate `config/benchmark-profiles.json` from leaderboard fixtures |
|
|
749
778
|
| `npm run routing:verify-benchmark-profiles` | CI smoke: assert checked-in profiles match fixture ingest |
|
|
@@ -782,6 +811,16 @@ npm run routing:eval-replay
|
|
|
782
811
|
|
|
783
812
|
Capability scores in `config/benchmark-profiles.json` are grounded from public leaderboard snapshots under `tests/fixtures/benchmark-leaderboards/`. Each artifact records provenance (`source_urls`, `scrape_date`, `catalog_freeze_date`) in its header.
|
|
784
813
|
|
|
814
|
+
**Fleet ID aliases (SP-174):** live pi/Cursor scoped-fleet model IDs often differ from leaderboard `model_id` strings. The artifact’s optional `aliases` map sends those fleet IDs to an existing grounded row (never invents scores). `mapPiModelToProfile` sets `capability_source` to `benchmark` when a direct row or alias hits, otherwise `pattern_default`. Operators can also call `getCapabilitySource(modelId)` / `resolveBenchmarkModelId(modelId)`.
|
|
815
|
+
|
|
816
|
+
**Add a new fleet ID after ingest:**
|
|
817
|
+
|
|
818
|
+
1. Ensure the canonical model has fixture scores (edit `tests/fixtures/benchmark-leaderboards/*.json` if needed).
|
|
819
|
+
2. Run `npm run routing:ingest-benchmarks` (and commit the regenerated `config/benchmark-profiles.json`).
|
|
820
|
+
3. Add `"your-fleet-id": "canonical-model_id"` under `aliases` in `config/benchmark-profiles.json` (target must already appear in `models[].model_id`).
|
|
821
|
+
4. Re-run ingest anytime — the CLI **preserves** existing `aliases` from the output file. Seed defaults live in `DEFAULT_FLEET_BENCHMARK_ALIASES` when no prior artifact exists.
|
|
822
|
+
5. Confirm with `npm run routing:verify-benchmark-profiles` and a mapper unit test that `capability_source === 'benchmark'` for the fleet id.
|
|
823
|
+
|
|
785
824
|
**Operator policy:**
|
|
786
825
|
|
|
787
826
|
1. **PR smoke** — `.github/workflows/benchmark-profile-refresh.yml` runs on PRs that touch fixtures, ingest, or the checked-in artifact. It executes `npm run routing:verify-benchmark-profiles` so fixture edits cannot drift from `config/benchmark-profiles.json`.
|
|
@@ -10,6 +10,28 @@
|
|
|
10
10
|
"scrape_date": "2026-07-09",
|
|
11
11
|
"catalog_freeze_date": "2026-07-09"
|
|
12
12
|
},
|
|
13
|
+
"aliases": {
|
|
14
|
+
"claude-3-5-sonnet": "claude-sonnet-4-6",
|
|
15
|
+
"claude-3.5-sonnet": "claude-sonnet-4-6",
|
|
16
|
+
"claude-3.5-sonnet-latest": "claude-sonnet-4-6",
|
|
17
|
+
"claude-opus-4": "claude-opus-4-5",
|
|
18
|
+
"claude-opus-4-20250514": "claude-opus-4-5",
|
|
19
|
+
"claude-sonnet-4": "claude-sonnet-4-6",
|
|
20
|
+
"claude-sonnet-4-20250514": "claude-sonnet-4-6",
|
|
21
|
+
"composer-1": "gpt-5.3-codex",
|
|
22
|
+
"composer-latest": "gpt-5.3-codex",
|
|
23
|
+
"cursor/auto": "gpt-5.3-codex",
|
|
24
|
+
"cursor/composer-latest": "gpt-5.3-codex",
|
|
25
|
+
"default": "gpt-5.3-codex",
|
|
26
|
+
"gemini-2.0-flash": "gemini-2.5-flash",
|
|
27
|
+
"gemini-2.5-flash-lite": "gemini-2.5-flash",
|
|
28
|
+
"gemini-2.5-flash-preview": "gemini-2.5-flash",
|
|
29
|
+
"gemini-flash-latest": "gemini-2.5-flash",
|
|
30
|
+
"gpt-5": "gpt-5.3-codex",
|
|
31
|
+
"gpt-5-codex": "gpt-5.3-codex",
|
|
32
|
+
"gpt-5.3": "gpt-5.3-codex",
|
|
33
|
+
"gpt-5.5": "gpt-5.3-codex"
|
|
34
|
+
},
|
|
13
35
|
"models": [
|
|
14
36
|
{
|
|
15
37
|
"model_id": "claude-3.5-haiku",
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"min_training_samples": 30,
|
|
4
|
+
"feature_names": [
|
|
5
|
+
"prompt_length_norm",
|
|
6
|
+
"estimated_input_tokens_norm",
|
|
7
|
+
"triage_cyclomatic_score",
|
|
8
|
+
"requirement_reasoning",
|
|
9
|
+
"requirement_code_gen",
|
|
10
|
+
"requirement_tool_use",
|
|
11
|
+
"has_tool_context",
|
|
12
|
+
"compaction_flag",
|
|
13
|
+
"routing_latency_norm",
|
|
14
|
+
"economical_tier"
|
|
15
|
+
],
|
|
16
|
+
"intercept": 4.101841550841884,
|
|
17
|
+
"coefficients": [
|
|
18
|
+
-3.9458158506645473,
|
|
19
|
+
-3.9458158506645473,
|
|
20
|
+
-4.6221357618419106,
|
|
21
|
+
-4.702014247565688,
|
|
22
|
+
-1.9442887494123497,
|
|
23
|
+
-2.945430358303044,
|
|
24
|
+
-2.8373347243581435,
|
|
25
|
+
0,
|
|
26
|
+
0.09441030879566878,
|
|
27
|
+
4.101841550841884
|
|
28
|
+
],
|
|
29
|
+
"trained_sample_count": 40,
|
|
30
|
+
"provenance": {
|
|
31
|
+
"source": "synthetic_fixture",
|
|
32
|
+
"task": "SP-175",
|
|
33
|
+
"note": "Privacy-safe feature vectors + labels only; no prompt text.",
|
|
34
|
+
"trained_at": "2026-07-10"
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -15,5 +15,8 @@
|
|
|
15
15
|
],
|
|
16
16
|
"intercept": 0,
|
|
17
17
|
"coefficients": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
|
|
18
|
-
"trained_sample_count": 0
|
|
18
|
+
"trained_sample_count": 0,
|
|
19
|
+
"provenance": {
|
|
20
|
+
"note": "Untrained template. Prefer npm run routing:train-p-success (or copy from config/p-success-weights.json after training). Never commit prompt text."
|
|
21
|
+
}
|
|
19
22
|
}
|
|
@@ -3,19 +3,37 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Maps pi `Model` objects (provider + id) to router fleet entries using
|
|
5
5
|
* pattern-based lookup for known families. When `config/benchmark-profiles.json`
|
|
6
|
-
* contains a row for the model id (SP-134/136 ingest output),
|
|
7
|
-
* vectors are grounded in benchmark
|
|
8
|
-
* Unknown models or missing benchmark rows
|
|
6
|
+
* contains a row for the model id (SP-134/136 ingest output), or a fleet alias
|
|
7
|
+
* pointing at such a row (SP-174), capability vectors are grounded in benchmark
|
|
8
|
+
* scores instead of regex defaults. Unknown models or missing benchmark rows
|
|
9
|
+
* receive conservative pattern defaults. Mapped profiles include
|
|
10
|
+
* `capability_source` (`benchmark` | `pattern_default`) for explain/telemetry.
|
|
9
11
|
*/
|
|
10
12
|
import type { ModelLimits, ModelProfile, Tier } from '../domain/types/entities.js';
|
|
11
13
|
/** Checked-in ingest artifact from `npm run routing:ingest-benchmarks` (SP-134). */
|
|
12
14
|
export declare const DEFAULT_BENCHMARK_PROFILES_PATH: string;
|
|
15
|
+
/** Whether HyDRA capabilities came from the ingest artifact (or alias) vs regex defaults. */
|
|
16
|
+
export type CapabilitySource = 'benchmark' | 'pattern_default';
|
|
17
|
+
/** ModelProfile plus operator-visible capability provenance (SP-174). */
|
|
18
|
+
export interface MappedModelProfile extends ModelProfile {
|
|
19
|
+
readonly capability_source: CapabilitySource;
|
|
20
|
+
}
|
|
13
21
|
/**
|
|
14
22
|
* Test hook — override benchmark artifact path (null disables benchmark grounding).
|
|
15
23
|
*/
|
|
16
24
|
export declare function setBenchmarkProfilesPathForTests(filePath: string | null): void;
|
|
17
25
|
/** Test hook — clear cached benchmark artifact between cases. */
|
|
18
26
|
export declare function resetBenchmarkProfilesCacheForTests(): void;
|
|
27
|
+
/**
|
|
28
|
+
* Resolve a scoped-fleet model id to the canonical ingest `model_id` via aliases.
|
|
29
|
+
* Returns the input id when no alias exists.
|
|
30
|
+
*/
|
|
31
|
+
export declare function resolveBenchmarkModelId(modelId: string): string;
|
|
32
|
+
/**
|
|
33
|
+
* Whether capabilities for this model id resolve from the benchmark artifact
|
|
34
|
+
* (direct row or fleet alias) vs pattern/family defaults.
|
|
35
|
+
*/
|
|
36
|
+
export declare function getCapabilitySource(modelId: string): CapabilitySource;
|
|
19
37
|
/** Pi registry `Model.cost` shape — per-token USD rates. */
|
|
20
38
|
export interface PiRegistryCost {
|
|
21
39
|
readonly input: number;
|
|
@@ -41,10 +59,11 @@ export declare function getDefaultLimitsForTier(tier: Tier): ModelLimits;
|
|
|
41
59
|
export declare const DEFAULT_CURSOR_QUOTA_COST_PER_1M = 3;
|
|
42
60
|
/**
|
|
43
61
|
* Map a pi model registry entry to a router ModelProfile.
|
|
62
|
+
* Includes `capability_source` for operators (benchmark vs pattern_default).
|
|
44
63
|
*/
|
|
45
|
-
export declare function mapPiModelToProfile(input: PiModelInput):
|
|
64
|
+
export declare function mapPiModelToProfile(input: PiModelInput): MappedModelProfile;
|
|
46
65
|
/**
|
|
47
66
|
* Map an array of pi registry models to a router fleet catalog.
|
|
48
67
|
*/
|
|
49
|
-
export declare function mapFleetFromRegistry(models: readonly PiModelInput[]):
|
|
68
|
+
export declare function mapFleetFromRegistry(models: readonly PiModelInput[]): MappedModelProfile[];
|
|
50
69
|
//# sourceMappingURL=pi-model-mapper.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pi-model-mapper.d.ts","sourceRoot":"","sources":["../../src/config/pi-model-mapper.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"pi-model-mapper.d.ts","sourceRoot":"","sources":["../../src/config/pi-model-mapper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAOH,OAAO,KAAK,EAEV,WAAW,EAGX,YAAY,EACZ,IAAI,EACL,MAAM,6BAA6B,CAAC;AAErC,oFAAoF;AACpF,eAAO,MAAM,+BAA+B,QAA+C,CAAC;AAE5F,6FAA6F;AAC7F,MAAM,MAAM,gBAAgB,GAAG,WAAW,GAAG,iBAAiB,CAAC;AAE/D,yEAAyE;AACzE,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,QAAQ,CAAC,iBAAiB,EAAE,gBAAgB,CAAC;CAC9C;AA2BD;;GAEG;AACH,wBAAgB,gCAAgC,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAG9E;AAED,iEAAiE;AACjE,wBAAgB,mCAAmC,IAAI,IAAI,CAG1D;AA+CD;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAM/D;AAkBD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,gBAAgB,CAErE;AA4BD,4DAA4D;AAC5D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC;CAChC;AASD,yFAAyF;AACzF,eAAO,MAAM,oBAAoB,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAIpE,CAAC;AAEF,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,IAAI,GAAG,WAAW,CAE/D;AAwDD;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,IAAM,CAAC;AA0IpD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,kBAAkB,CA2B3E;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,GAAG,kBAAkB,EAAE,CAE1F"}
|
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Maps pi `Model` objects (provider + id) to router fleet entries using
|
|
5
5
|
* pattern-based lookup for known families. When `config/benchmark-profiles.json`
|
|
6
|
-
* contains a row for the model id (SP-134/136 ingest output),
|
|
7
|
-
* vectors are grounded in benchmark
|
|
8
|
-
* Unknown models or missing benchmark rows
|
|
6
|
+
* contains a row for the model id (SP-134/136 ingest output), or a fleet alias
|
|
7
|
+
* pointing at such a row (SP-174), capability vectors are grounded in benchmark
|
|
8
|
+
* scores instead of regex defaults. Unknown models or missing benchmark rows
|
|
9
|
+
* receive conservative pattern defaults. Mapped profiles include
|
|
10
|
+
* `capability_source` (`benchmark` | `pattern_default`) for explain/telemetry.
|
|
9
11
|
*/
|
|
10
12
|
import { existsSync, readFileSync } from 'node:fs';
|
|
11
13
|
import { resolve } from 'node:path';
|
|
@@ -23,21 +25,22 @@ const benchmarkModelRowSchema = z.object({
|
|
|
23
25
|
});
|
|
24
26
|
const benchmarkProfilesArtifactSchema = z.object({
|
|
25
27
|
version: z.literal(1),
|
|
28
|
+
aliases: z.record(z.string().min(1), z.string().min(1)).optional(),
|
|
26
29
|
models: z.array(benchmarkModelRowSchema).min(1),
|
|
27
30
|
});
|
|
28
31
|
let benchmarkProfilesPathOverride;
|
|
29
|
-
let
|
|
32
|
+
let benchmarkProfilesCache;
|
|
30
33
|
/**
|
|
31
34
|
* Test hook — override benchmark artifact path (null disables benchmark grounding).
|
|
32
35
|
*/
|
|
33
36
|
export function setBenchmarkProfilesPathForTests(filePath) {
|
|
34
37
|
benchmarkProfilesPathOverride = filePath;
|
|
35
|
-
|
|
38
|
+
benchmarkProfilesCache = undefined;
|
|
36
39
|
}
|
|
37
40
|
/** Test hook — clear cached benchmark artifact between cases. */
|
|
38
41
|
export function resetBenchmarkProfilesCacheForTests() {
|
|
39
42
|
benchmarkProfilesPathOverride = undefined;
|
|
40
|
-
|
|
43
|
+
benchmarkProfilesCache = undefined;
|
|
41
44
|
}
|
|
42
45
|
function resolveBenchmarkProfilesPath() {
|
|
43
46
|
if (benchmarkProfilesPathOverride === null) {
|
|
@@ -45,13 +48,13 @@ function resolveBenchmarkProfilesPath() {
|
|
|
45
48
|
}
|
|
46
49
|
return benchmarkProfilesPathOverride ?? DEFAULT_BENCHMARK_PROFILES_PATH;
|
|
47
50
|
}
|
|
48
|
-
function
|
|
49
|
-
if (
|
|
50
|
-
return
|
|
51
|
+
function loadBenchmarkProfilesCache() {
|
|
52
|
+
if (benchmarkProfilesCache !== undefined) {
|
|
53
|
+
return benchmarkProfilesCache;
|
|
51
54
|
}
|
|
52
55
|
const filePath = resolveBenchmarkProfilesPath();
|
|
53
56
|
if (filePath === null || !existsSync(filePath)) {
|
|
54
|
-
|
|
57
|
+
benchmarkProfilesCache = null;
|
|
55
58
|
return null;
|
|
56
59
|
}
|
|
57
60
|
try {
|
|
@@ -61,33 +64,67 @@ function loadBenchmarkCapabilitiesMap() {
|
|
|
61
64
|
if (!result.success) {
|
|
62
65
|
throw new Error(result.error.message);
|
|
63
66
|
}
|
|
64
|
-
const
|
|
67
|
+
const capabilitiesByModelId = new Map();
|
|
65
68
|
for (const row of result.data.models) {
|
|
66
|
-
|
|
69
|
+
capabilitiesByModelId.set(row.model_id, { ...row.capabilities });
|
|
67
70
|
}
|
|
68
|
-
|
|
69
|
-
|
|
71
|
+
const aliases = new Map(Object.entries(result.data.aliases ?? {}));
|
|
72
|
+
benchmarkProfilesCache = { capabilitiesByModelId, aliases };
|
|
73
|
+
return benchmarkProfilesCache;
|
|
70
74
|
}
|
|
71
75
|
catch (err) {
|
|
72
76
|
console.warn('benchmark profiles artifact invalid; using regex capability defaults', {
|
|
73
77
|
path: filePath,
|
|
74
78
|
error: err instanceof Error ? err.message : String(err),
|
|
75
79
|
});
|
|
76
|
-
|
|
80
|
+
benchmarkProfilesCache = null;
|
|
77
81
|
return null;
|
|
78
82
|
}
|
|
79
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Resolve a scoped-fleet model id to the canonical ingest `model_id` via aliases.
|
|
86
|
+
* Returns the input id when no alias exists.
|
|
87
|
+
*/
|
|
88
|
+
export function resolveBenchmarkModelId(modelId) {
|
|
89
|
+
const cache = loadBenchmarkProfilesCache();
|
|
90
|
+
if (cache === null) {
|
|
91
|
+
return modelId;
|
|
92
|
+
}
|
|
93
|
+
return cache.aliases.get(modelId) ?? modelId;
|
|
94
|
+
}
|
|
80
95
|
function lookupBenchmarkCapabilities(modelId) {
|
|
81
|
-
|
|
96
|
+
const cache = loadBenchmarkProfilesCache();
|
|
97
|
+
if (cache === null) {
|
|
98
|
+
return undefined;
|
|
99
|
+
}
|
|
100
|
+
const direct = cache.capabilitiesByModelId.get(modelId);
|
|
101
|
+
if (direct !== undefined) {
|
|
102
|
+
return direct;
|
|
103
|
+
}
|
|
104
|
+
const canonical = cache.aliases.get(modelId);
|
|
105
|
+
if (canonical === undefined) {
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
return cache.capabilitiesByModelId.get(canonical);
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Whether capabilities for this model id resolve from the benchmark artifact
|
|
112
|
+
* (direct row or fleet alias) vs pattern/family defaults.
|
|
113
|
+
*/
|
|
114
|
+
export function getCapabilitySource(modelId) {
|
|
115
|
+
return lookupBenchmarkCapabilities(modelId) !== undefined ? 'benchmark' : 'pattern_default';
|
|
82
116
|
}
|
|
83
117
|
function withBenchmarkCapabilities(defaults, modelId) {
|
|
84
118
|
const grounded = lookupBenchmarkCapabilities(modelId);
|
|
85
119
|
if (grounded === undefined) {
|
|
86
|
-
return defaults;
|
|
120
|
+
return { defaults, capability_source: 'pattern_default' };
|
|
87
121
|
}
|
|
88
122
|
return {
|
|
89
|
-
|
|
90
|
-
|
|
123
|
+
defaults: {
|
|
124
|
+
...defaults,
|
|
125
|
+
capabilities: { ...grounded },
|
|
126
|
+
},
|
|
127
|
+
capability_source: 'benchmark',
|
|
91
128
|
};
|
|
92
129
|
}
|
|
93
130
|
const LOCAL_PROVIDERS = new Set(['lmstudio', 'ollama']);
|
|
@@ -245,7 +282,7 @@ function resolveFallbackCost(input, defaults) {
|
|
|
245
282
|
const fromRegistry = deriveFallbackCostPer1M(input.cost);
|
|
246
283
|
return fromRegistry ?? defaults.pricing.fallback_cost_per_1m;
|
|
247
284
|
}
|
|
248
|
-
function buildProfile(input, defaults) {
|
|
285
|
+
function buildProfile(input, defaults, capability_source) {
|
|
249
286
|
const provider = input.provider.trim();
|
|
250
287
|
const registryKey = `${normalizeProvider(provider)}/${input.id}`;
|
|
251
288
|
const profile = {
|
|
@@ -260,6 +297,7 @@ function buildProfile(input, defaults) {
|
|
|
260
297
|
? { quota_cost_per_1m: defaults.pricing.quota_cost_per_1m }
|
|
261
298
|
: {}),
|
|
262
299
|
},
|
|
300
|
+
capability_source,
|
|
263
301
|
};
|
|
264
302
|
const withPerformance = defaults.performance !== undefined
|
|
265
303
|
? { ...profile, performance: { ...defaults.performance } }
|
|
@@ -271,6 +309,7 @@ function buildProfile(input, defaults) {
|
|
|
271
309
|
}
|
|
272
310
|
/**
|
|
273
311
|
* Map a pi model registry entry to a router ModelProfile.
|
|
312
|
+
* Includes `capability_source` for operators (benchmark vs pattern_default).
|
|
274
313
|
*/
|
|
275
314
|
export function mapPiModelToProfile(input) {
|
|
276
315
|
const provider = normalizeProvider(input.provider);
|
|
@@ -281,16 +320,20 @@ export function mapPiModelToProfile(input) {
|
|
|
281
320
|
id: input.id,
|
|
282
321
|
...(input.name !== undefined ? { name: input.name } : {}),
|
|
283
322
|
};
|
|
284
|
-
|
|
323
|
+
const grounded = withBenchmarkCapabilities(LOCAL_DEFAULTS, input.id);
|
|
324
|
+
return buildProfile(localInput, grounded.defaults, grounded.capability_source);
|
|
285
325
|
}
|
|
286
326
|
if (input.id === OPAQUE_FLEET_DEFAULT_ID) {
|
|
287
|
-
|
|
327
|
+
const grounded = withBenchmarkCapabilities(CURSOR_AUTO_DEFAULTS, input.id);
|
|
328
|
+
return buildProfile(input, grounded.defaults, grounded.capability_source);
|
|
288
329
|
}
|
|
289
330
|
const matched = matchPatternRules(input.id);
|
|
290
331
|
if (matched) {
|
|
291
|
-
|
|
332
|
+
const grounded = withBenchmarkCapabilities(matched, input.id);
|
|
333
|
+
return buildProfile(input, grounded.defaults, grounded.capability_source);
|
|
292
334
|
}
|
|
293
|
-
|
|
335
|
+
const grounded = withBenchmarkCapabilities(UNKNOWN_DEFAULTS, input.id);
|
|
336
|
+
return buildProfile(input, grounded.defaults, grounded.capability_source);
|
|
294
337
|
}
|
|
295
338
|
/**
|
|
296
339
|
* Map an array of pi registry models to a router fleet catalog.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pi-model-mapper.js","sourceRoot":"","sources":["../../src/config/pi-model-mapper.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"pi-model-mapper.js","sourceRoot":"","sources":["../../src/config/pi-model-mapper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,oFAAoF;AACpF,MAAM,CAAC,MAAM,+BAA+B,GAAG,OAAO,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAC;AAU5F,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACnC,CAAC,CAAC;AAEH,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,YAAY,EAAE,2BAA2B;CAC1C,CAAC,CAAC;AAEH,MAAM,+BAA+B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrB,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAClE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAChD,CAAC,CAAC;AAOH,IAAI,6BAAwD,CAAC;AAC7D,IAAI,sBAAiE,CAAC;AAEtE;;GAEG;AACH,MAAM,UAAU,gCAAgC,CAAC,QAAuB;IACtE,6BAA6B,GAAG,QAAQ,CAAC;IACzC,sBAAsB,GAAG,SAAS,CAAC;AACrC,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,mCAAmC;IACjD,6BAA6B,GAAG,SAAS,CAAC;IAC1C,sBAAsB,GAAG,SAAS,CAAC;AACrC,CAAC;AAED,SAAS,4BAA4B;IACnC,IAAI,6BAA6B,KAAK,IAAI,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,6BAA6B,IAAI,+BAA+B,CAAC;AAC1E,CAAC;AAED,SAAS,0BAA0B;IACjC,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;QACzC,OAAO,sBAAsB,CAAC;IAChC,CAAC;IAED,MAAM,QAAQ,GAAG,4BAA4B,EAAE,CAAC;IAChD,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,sBAAsB,GAAG,IAAI,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,+BAA+B,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACjE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAA6B,CAAC;QACnE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACrC,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,GAAG,CACrB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAC1C,CAAC;QACF,sBAAsB,GAAG,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC;QAC5D,OAAO,sBAAsB,CAAC;IAChC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,sEAAsE,EAAE;YACnF,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,sBAAsB,GAAG,IAAI,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAe;IACrD,MAAM,KAAK,GAAG,0BAA0B,EAAE,CAAC;IAC3C,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC;AAC/C,CAAC;AAED,SAAS,2BAA2B,CAAC,OAAe;IAClD,MAAM,KAAK,GAAG,0BAA0B,EAAE,CAAC;IAC3C,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACxD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,CAAC,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AACpD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAe;IACjD,OAAO,2BAA2B,CAAC,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC;AAC9F,CAAC;AAUD,SAAS,yBAAyB,CAChC,QAA6B,EAC7B,OAAe;IAEf,MAAM,QAAQ,GAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;IACtD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,CAAC;IAC5D,CAAC;IAED,OAAO;QACL,QAAQ,EAAE;YACR,GAAG,QAAQ;YACX,YAAY,EAAE,EAAE,GAAG,QAAQ,EAAE;SAC9B;QACD,iBAAiB,EAAE,WAAW;KAC/B,CAAC;AACJ,CAAC;AAsBD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;AAExD,yFAAyF;AACzF,MAAM,CAAC,MAAM,oBAAoB,GAAwC;IACvE,WAAW,EAAE,EAAE,gBAAgB,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE;IACnE,kBAAkB,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,iBAAiB,EAAE,KAAK,EAAE;IAC3E,gBAAgB,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE;CAC3E,CAAC;AAEF,MAAM,UAAU,uBAAuB,CAAC,IAAU;IAChD,OAAO,EAAE,GAAG,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,MAAM,cAAc,GAAwB;IAC1C,IAAI,EAAE,WAAW;IACjB,YAAY,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE;IAC9D,WAAW,EAAE;QACX,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,GAAG;QACrB,cAAc,EAAE,IAAI;KACrB;IACD,OAAO,EAAE;QACP,YAAY,EAAE,YAAY;QAC1B,oBAAoB,EAAE,GAAG;KAC1B;IACD,QAAQ,EAAE,0BAA0B;CACrC,CAAC;AAEF,MAAM,iBAAiB,GAAwB;IAC7C,IAAI,EAAE,gBAAgB;IACtB,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE;IACjE,WAAW,EAAE;QACX,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,GAAG;QACrB,cAAc,EAAE,IAAI;KACrB;IACD,OAAO,EAAE;QACP,oBAAoB,EAAE,GAAG;KAC1B;CACF,CAAC;AAEF,MAAM,mBAAmB,GAAwB;IAC/C,IAAI,EAAE,kBAAkB;IACxB,YAAY,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE;IAC/D,WAAW,EAAE;QACX,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,IAAI;QACtB,cAAc,EAAE,IAAI;KACrB;IACD,OAAO,EAAE;QACP,oBAAoB,EAAE,GAAG;KAC1B;CACF,CAAC;AAEF,MAAM,gBAAgB,GAAwB;IAC5C,IAAI,EAAE,kBAAkB;IACxB,YAAY,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE;IAC/D,WAAW,EAAE;QACX,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,GAAG;QACrB,cAAc,EAAE,IAAI;KACrB;IACD,OAAO,EAAE;QACP,oBAAoB,EAAE,GAAG;KAC1B;CACF,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAEpD;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,SAAS,CAAC;AAE1C;;;;;;GAMG;AACH,MAAM,oBAAoB,GAAwB;IAChD,IAAI,EAAE,gBAAgB;IACtB,YAAY,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC/D,WAAW,EAAE;QACX,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,GAAG;QACrB,cAAc,EAAE,KAAK;KACtB;IACD,OAAO,EAAE;QACP,oBAAoB,EAAE,GAAG;QACzB,iBAAiB,EAAE,gCAAgC;KACpD;CACF,CAAC;AAEF;;;GAGG;AACH,MAAM,iBAAiB,GAAwB;IAC7C,IAAI,EAAE,gBAAgB;IACtB,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE;IAChE,WAAW,EAAE;QACX,cAAc,EAAE,GAAG;QACnB,gBAAgB,EAAE,IAAI;QACtB,cAAc,EAAE,KAAK;KACtB;IACD,OAAO,EAAE;QACP,oBAAoB,EAAE,GAAG;QACzB,iBAAiB,EAAE,gCAAgC;KACpD;CACF,CAAC;AAEF,wCAAwC;AACxC,MAAM,mBAAmB,GAAgC;IACvD,EAAE,OAAO,EAAE,kDAAkD,EAAE,QAAQ,EAAE,iBAAiB,EAAE;IAC5F,EAAE,OAAO,EAAE,0BAA0B,EAAE,QAAQ,EAAE,mBAAmB,EAAE;IACtE,EAAE,OAAO,EAAE,wBAAwB,EAAE,QAAQ,EAAE,iBAAiB,EAAE;IAClE,EAAE,OAAO,EAAE,mDAAmD,EAAE,QAAQ,EAAE,mBAAmB,EAAE;IAC/F,EAAE,OAAO,EAAE,2CAA2C,EAAE,QAAQ,EAAE,iBAAiB,EAAE;IACrF,EAAE,OAAO,EAAE,qBAAqB,EAAE,QAAQ,EAAE,iBAAiB,EAAE;IAC/D,EAAE,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,iBAAiB,EAAE;IACxD,EAAE,OAAO,EAAE,6BAA6B,EAAE,QAAQ,EAAE,mBAAmB,EAAE;IACzE,EAAE,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,iBAAiB,EAAE;IAC1D,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,oBAAoB,EAAE;CAC1D,CAAC;AAEF,SAAS,iBAAiB,CAAC,QAAgB;IACzC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,iBAAiB,CAAC,EAAU;IACnC,KAAK,MAAM,IAAI,IAAI,mBAAmB,EAAE,CAAC;QACvC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,SAAS,uBAAuB,CAAC,IAAoB;IACnD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC/B,IAAI,KAAK,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,GAAG,SAAS,CAAC;IACrC,MAAM,WAAW,GAAG,MAAM,GAAG,SAAS,CAAC;IACvC,OAAO,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAmB,EACnB,QAA6B;IAE7B,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,QAAQ,CAAC,OAAO,CAAC,oBAAoB,CAAC;IAC/C,CAAC;IAED,MAAM,YAAY,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzD,OAAO,YAAY,IAAI,QAAQ,CAAC,OAAO,CAAC,oBAAoB,CAAC;AAC/D,CAAC;AAED,SAAS,YAAY,CACnB,KAAmB,EACnB,QAA6B,EAC7B,iBAAmC;IAEnC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,WAAW,GAAG,GAAG,iBAAiB,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,EAAE,CAAC;IAEjE,MAAM,OAAO,GAAuB;QAClC,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ;QACR,YAAY,EAAE,EAAE,GAAG,QAAQ,CAAC,YAAY,EAAE;QAC1C,OAAO,EAAE;YACP,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,YAAY,IAAI,WAAW;YAC1D,oBAAoB,EAAE,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC;YAC1D,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,iBAAiB,KAAK,SAAS;gBAClD,CAAC,CAAC,EAAE,iBAAiB,EAAE,QAAQ,CAAC,OAAO,CAAC,iBAAiB,EAAE;gBAC3D,CAAC,CAAC,EAAE,CAAC;SACR;QACD,iBAAiB;KAClB,CAAC;IAEF,MAAM,eAAe,GACnB,QAAQ,CAAC,WAAW,KAAK,SAAS;QAChC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,WAAW,EAAE,EAAE,GAAG,QAAQ,CAAC,WAAW,EAAE,EAAE;QAC1D,CAAC,CAAC,OAAO,CAAC;IAEd,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACpC,OAAO,EAAE,GAAG,eAAe,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAC7D,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAmB;IACrD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAEnD,IAAI,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,8EAA8E;QAC9E,MAAM,UAAU,GAAiB;YAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1D,CAAC;QACF,MAAM,QAAQ,GAAG,yBAAyB,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;QACrE,OAAO,YAAY,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IACjF,CAAC;IAED,IAAI,KAAK,CAAC,EAAE,KAAK,uBAAuB,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,yBAAyB,CAAC,oBAAoB,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3E,OAAO,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5C,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,yBAAyB,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9D,OAAO,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,QAAQ,GAAG,yBAAyB,CAAC,gBAAgB,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,iBAAiB,CAAC,CAAC;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAA+B;IAClE,OAAO,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;AACzC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-smart-router",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Auto-model router middleware for the pi.dev coding agent",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -62,6 +62,7 @@
|
|
|
62
62
|
"routing:bootstrap-centroids": "node --experimental-strip-types scripts/bootstrap-routing-centroids.ts",
|
|
63
63
|
"routing:calibration-aggregate": "node --experimental-strip-types scripts/calibration-aggregate.ts",
|
|
64
64
|
"routing:train-calibration": "node --experimental-strip-types scripts/train-routing-calibration.ts",
|
|
65
|
+
"routing:train-p-success": "tsx scripts/train-p-success-weights.ts",
|
|
65
66
|
"routing:ingest-benchmarks": "tsx scripts/ingest-benchmark-profiles.ts",
|
|
66
67
|
"routing:verify-benchmark-profiles": "vitest run tests/unit/ingest-benchmark-profiles.test.ts -t \"checked-in artifact matches fixture ingest\"",
|
|
67
68
|
"routing:verify-calibration": "bash -c 'if [[ \" $* \" == *\" --skip-embed \"* ]]; then vitest run tests/unit/train-routing-calibration.test.ts -t \"verifyRoutingCalibration passes|evaluates benchmark triage gates\"; else npm run build && tsx scripts/verify-routing-calibration.ts \"$@\"; fi' --",
|
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Maps pi `Model` objects (provider + id) to router fleet entries using
|
|
5
5
|
* pattern-based lookup for known families. When `config/benchmark-profiles.json`
|
|
6
|
-
* contains a row for the model id (SP-134/136 ingest output),
|
|
7
|
-
* vectors are grounded in benchmark
|
|
8
|
-
* Unknown models or missing benchmark rows
|
|
6
|
+
* contains a row for the model id (SP-134/136 ingest output), or a fleet alias
|
|
7
|
+
* pointing at such a row (SP-174), capability vectors are grounded in benchmark
|
|
8
|
+
* scores instead of regex defaults. Unknown models or missing benchmark rows
|
|
9
|
+
* receive conservative pattern defaults. Mapped profiles include
|
|
10
|
+
* `capability_source` (`benchmark` | `pattern_default`) for explain/telemetry.
|
|
9
11
|
*/
|
|
10
12
|
|
|
11
13
|
import { existsSync, readFileSync } from 'node:fs';
|
|
@@ -25,6 +27,14 @@ import type {
|
|
|
25
27
|
/** Checked-in ingest artifact from `npm run routing:ingest-benchmarks` (SP-134). */
|
|
26
28
|
export const DEFAULT_BENCHMARK_PROFILES_PATH = resolve('config', 'benchmark-profiles.json');
|
|
27
29
|
|
|
30
|
+
/** Whether HyDRA capabilities came from the ingest artifact (or alias) vs regex defaults. */
|
|
31
|
+
export type CapabilitySource = 'benchmark' | 'pattern_default';
|
|
32
|
+
|
|
33
|
+
/** ModelProfile plus operator-visible capability provenance (SP-174). */
|
|
34
|
+
export interface MappedModelProfile extends ModelProfile {
|
|
35
|
+
readonly capability_source: CapabilitySource;
|
|
36
|
+
}
|
|
37
|
+
|
|
28
38
|
const benchmarkCapabilitiesSchema = z.object({
|
|
29
39
|
reasoning: z.number().min(0).max(1),
|
|
30
40
|
code_gen: z.number().min(0).max(1),
|
|
@@ -38,24 +48,30 @@ const benchmarkModelRowSchema = z.object({
|
|
|
38
48
|
|
|
39
49
|
const benchmarkProfilesArtifactSchema = z.object({
|
|
40
50
|
version: z.literal(1),
|
|
51
|
+
aliases: z.record(z.string().min(1), z.string().min(1)).optional(),
|
|
41
52
|
models: z.array(benchmarkModelRowSchema).min(1),
|
|
42
53
|
});
|
|
43
54
|
|
|
55
|
+
interface BenchmarkProfilesCache {
|
|
56
|
+
readonly capabilitiesByModelId: ReadonlyMap<string, ModelCapabilities>;
|
|
57
|
+
readonly aliases: ReadonlyMap<string, string>;
|
|
58
|
+
}
|
|
59
|
+
|
|
44
60
|
let benchmarkProfilesPathOverride: string | null | undefined;
|
|
45
|
-
let
|
|
61
|
+
let benchmarkProfilesCache: BenchmarkProfilesCache | null | undefined;
|
|
46
62
|
|
|
47
63
|
/**
|
|
48
64
|
* Test hook — override benchmark artifact path (null disables benchmark grounding).
|
|
49
65
|
*/
|
|
50
66
|
export function setBenchmarkProfilesPathForTests(filePath: string | null): void {
|
|
51
67
|
benchmarkProfilesPathOverride = filePath;
|
|
52
|
-
|
|
68
|
+
benchmarkProfilesCache = undefined;
|
|
53
69
|
}
|
|
54
70
|
|
|
55
71
|
/** Test hook — clear cached benchmark artifact between cases. */
|
|
56
72
|
export function resetBenchmarkProfilesCacheForTests(): void {
|
|
57
73
|
benchmarkProfilesPathOverride = undefined;
|
|
58
|
-
|
|
74
|
+
benchmarkProfilesCache = undefined;
|
|
59
75
|
}
|
|
60
76
|
|
|
61
77
|
function resolveBenchmarkProfilesPath(): string | null {
|
|
@@ -65,14 +81,14 @@ function resolveBenchmarkProfilesPath(): string | null {
|
|
|
65
81
|
return benchmarkProfilesPathOverride ?? DEFAULT_BENCHMARK_PROFILES_PATH;
|
|
66
82
|
}
|
|
67
83
|
|
|
68
|
-
function
|
|
69
|
-
if (
|
|
70
|
-
return
|
|
84
|
+
function loadBenchmarkProfilesCache(): BenchmarkProfilesCache | null {
|
|
85
|
+
if (benchmarkProfilesCache !== undefined) {
|
|
86
|
+
return benchmarkProfilesCache;
|
|
71
87
|
}
|
|
72
88
|
|
|
73
89
|
const filePath = resolveBenchmarkProfilesPath();
|
|
74
90
|
if (filePath === null || !existsSync(filePath)) {
|
|
75
|
-
|
|
91
|
+
benchmarkProfilesCache = null;
|
|
76
92
|
return null;
|
|
77
93
|
}
|
|
78
94
|
|
|
@@ -84,38 +100,84 @@ function loadBenchmarkCapabilitiesMap(): ReadonlyMap<string, ModelCapabilities>
|
|
|
84
100
|
throw new Error(result.error.message);
|
|
85
101
|
}
|
|
86
102
|
|
|
87
|
-
const
|
|
103
|
+
const capabilitiesByModelId = new Map<string, ModelCapabilities>();
|
|
88
104
|
for (const row of result.data.models) {
|
|
89
|
-
|
|
105
|
+
capabilitiesByModelId.set(row.model_id, { ...row.capabilities });
|
|
90
106
|
}
|
|
91
|
-
|
|
92
|
-
|
|
107
|
+
const aliases = new Map<string, string>(
|
|
108
|
+
Object.entries(result.data.aliases ?? {}),
|
|
109
|
+
);
|
|
110
|
+
benchmarkProfilesCache = { capabilitiesByModelId, aliases };
|
|
111
|
+
return benchmarkProfilesCache;
|
|
93
112
|
} catch (err: unknown) {
|
|
94
113
|
console.warn('benchmark profiles artifact invalid; using regex capability defaults', {
|
|
95
114
|
path: filePath,
|
|
96
115
|
error: err instanceof Error ? err.message : String(err),
|
|
97
116
|
});
|
|
98
|
-
|
|
117
|
+
benchmarkProfilesCache = null;
|
|
99
118
|
return null;
|
|
100
119
|
}
|
|
101
120
|
}
|
|
102
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Resolve a scoped-fleet model id to the canonical ingest `model_id` via aliases.
|
|
124
|
+
* Returns the input id when no alias exists.
|
|
125
|
+
*/
|
|
126
|
+
export function resolveBenchmarkModelId(modelId: string): string {
|
|
127
|
+
const cache = loadBenchmarkProfilesCache();
|
|
128
|
+
if (cache === null) {
|
|
129
|
+
return modelId;
|
|
130
|
+
}
|
|
131
|
+
return cache.aliases.get(modelId) ?? modelId;
|
|
132
|
+
}
|
|
133
|
+
|
|
103
134
|
function lookupBenchmarkCapabilities(modelId: string): ModelCapabilities | undefined {
|
|
104
|
-
|
|
135
|
+
const cache = loadBenchmarkProfilesCache();
|
|
136
|
+
if (cache === null) {
|
|
137
|
+
return undefined;
|
|
138
|
+
}
|
|
139
|
+
const direct = cache.capabilitiesByModelId.get(modelId);
|
|
140
|
+
if (direct !== undefined) {
|
|
141
|
+
return direct;
|
|
142
|
+
}
|
|
143
|
+
const canonical = cache.aliases.get(modelId);
|
|
144
|
+
if (canonical === undefined) {
|
|
145
|
+
return undefined;
|
|
146
|
+
}
|
|
147
|
+
return cache.capabilitiesByModelId.get(canonical);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Whether capabilities for this model id resolve from the benchmark artifact
|
|
152
|
+
* (direct row or fleet alias) vs pattern/family defaults.
|
|
153
|
+
*/
|
|
154
|
+
export function getCapabilitySource(modelId: string): CapabilitySource {
|
|
155
|
+
return lookupBenchmarkCapabilities(modelId) !== undefined ? 'benchmark' : 'pattern_default';
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
interface ModelFamilyDefaults {
|
|
159
|
+
readonly tier: Tier;
|
|
160
|
+
readonly capabilities: ModelCapabilities;
|
|
161
|
+
readonly performance?: ModelPerformance;
|
|
162
|
+
readonly pricing: ModelPricing;
|
|
163
|
+
readonly endpoint?: string;
|
|
105
164
|
}
|
|
106
165
|
|
|
107
166
|
function withBenchmarkCapabilities(
|
|
108
167
|
defaults: ModelFamilyDefaults,
|
|
109
168
|
modelId: string,
|
|
110
|
-
): ModelFamilyDefaults {
|
|
169
|
+
): { readonly defaults: ModelFamilyDefaults; readonly capability_source: CapabilitySource } {
|
|
111
170
|
const grounded = lookupBenchmarkCapabilities(modelId);
|
|
112
171
|
if (grounded === undefined) {
|
|
113
|
-
return defaults;
|
|
172
|
+
return { defaults, capability_source: 'pattern_default' };
|
|
114
173
|
}
|
|
115
174
|
|
|
116
175
|
return {
|
|
117
|
-
|
|
118
|
-
|
|
176
|
+
defaults: {
|
|
177
|
+
...defaults,
|
|
178
|
+
capabilities: { ...grounded },
|
|
179
|
+
},
|
|
180
|
+
capability_source: 'benchmark',
|
|
119
181
|
};
|
|
120
182
|
}
|
|
121
183
|
|
|
@@ -134,14 +196,6 @@ export interface PiModelInput {
|
|
|
134
196
|
readonly cost?: PiRegistryCost;
|
|
135
197
|
}
|
|
136
198
|
|
|
137
|
-
interface ModelFamilyDefaults {
|
|
138
|
-
readonly tier: Tier;
|
|
139
|
-
readonly capabilities: ModelCapabilities;
|
|
140
|
-
readonly performance?: ModelPerformance;
|
|
141
|
-
readonly pricing: ModelPricing;
|
|
142
|
-
readonly endpoint?: string;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
199
|
interface ModelPatternRule {
|
|
146
200
|
readonly pattern: RegExp;
|
|
147
201
|
readonly defaults: ModelFamilyDefaults;
|
|
@@ -323,11 +377,15 @@ function resolveFallbackCost(
|
|
|
323
377
|
return fromRegistry ?? defaults.pricing.fallback_cost_per_1m;
|
|
324
378
|
}
|
|
325
379
|
|
|
326
|
-
function buildProfile(
|
|
380
|
+
function buildProfile(
|
|
381
|
+
input: PiModelInput,
|
|
382
|
+
defaults: ModelFamilyDefaults,
|
|
383
|
+
capability_source: CapabilitySource,
|
|
384
|
+
): MappedModelProfile {
|
|
327
385
|
const provider = input.provider.trim();
|
|
328
386
|
const registryKey = `${normalizeProvider(provider)}/${input.id}`;
|
|
329
387
|
|
|
330
|
-
const profile:
|
|
388
|
+
const profile: MappedModelProfile = {
|
|
331
389
|
id: input.id,
|
|
332
390
|
tier: defaults.tier,
|
|
333
391
|
provider,
|
|
@@ -339,6 +397,7 @@ function buildProfile(input: PiModelInput, defaults: ModelFamilyDefaults): Model
|
|
|
339
397
|
? { quota_cost_per_1m: defaults.pricing.quota_cost_per_1m }
|
|
340
398
|
: {}),
|
|
341
399
|
},
|
|
400
|
+
capability_source,
|
|
342
401
|
};
|
|
343
402
|
|
|
344
403
|
const withPerformance =
|
|
@@ -355,8 +414,9 @@ function buildProfile(input: PiModelInput, defaults: ModelFamilyDefaults): Model
|
|
|
355
414
|
|
|
356
415
|
/**
|
|
357
416
|
* Map a pi model registry entry to a router ModelProfile.
|
|
417
|
+
* Includes `capability_source` for operators (benchmark vs pattern_default).
|
|
358
418
|
*/
|
|
359
|
-
export function mapPiModelToProfile(input: PiModelInput):
|
|
419
|
+
export function mapPiModelToProfile(input: PiModelInput): MappedModelProfile {
|
|
360
420
|
const provider = normalizeProvider(input.provider);
|
|
361
421
|
|
|
362
422
|
if (LOCAL_PROVIDERS.has(provider)) {
|
|
@@ -366,24 +426,28 @@ export function mapPiModelToProfile(input: PiModelInput): ModelProfile {
|
|
|
366
426
|
id: input.id,
|
|
367
427
|
...(input.name !== undefined ? { name: input.name } : {}),
|
|
368
428
|
};
|
|
369
|
-
|
|
429
|
+
const grounded = withBenchmarkCapabilities(LOCAL_DEFAULTS, input.id);
|
|
430
|
+
return buildProfile(localInput, grounded.defaults, grounded.capability_source);
|
|
370
431
|
}
|
|
371
432
|
|
|
372
433
|
if (input.id === OPAQUE_FLEET_DEFAULT_ID) {
|
|
373
|
-
|
|
434
|
+
const grounded = withBenchmarkCapabilities(CURSOR_AUTO_DEFAULTS, input.id);
|
|
435
|
+
return buildProfile(input, grounded.defaults, grounded.capability_source);
|
|
374
436
|
}
|
|
375
437
|
|
|
376
438
|
const matched = matchPatternRules(input.id);
|
|
377
439
|
if (matched) {
|
|
378
|
-
|
|
440
|
+
const grounded = withBenchmarkCapabilities(matched, input.id);
|
|
441
|
+
return buildProfile(input, grounded.defaults, grounded.capability_source);
|
|
379
442
|
}
|
|
380
443
|
|
|
381
|
-
|
|
444
|
+
const grounded = withBenchmarkCapabilities(UNKNOWN_DEFAULTS, input.id);
|
|
445
|
+
return buildProfile(input, grounded.defaults, grounded.capability_source);
|
|
382
446
|
}
|
|
383
447
|
|
|
384
448
|
/**
|
|
385
449
|
* Map an array of pi registry models to a router fleet catalog.
|
|
386
450
|
*/
|
|
387
|
-
export function mapFleetFromRegistry(models: readonly PiModelInput[]):
|
|
451
|
+
export function mapFleetFromRegistry(models: readonly PiModelInput[]): MappedModelProfile[] {
|
|
388
452
|
return models.map(mapPiModelToProfile);
|
|
389
453
|
}
|