pi-smart-router 1.2.2 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/fleet-bootstrap.ts +22 -0
- package/README.md +51 -5
- package/config/benchmark-profiles.json +126 -3
- package/dist/config/defaults.d.ts +2 -2
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +6 -3
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/pi-model-mapper.d.ts +37 -1
- package/dist/config/pi-model-mapper.d.ts.map +1 -1
- package/dist/config/pi-model-mapper.js +123 -7
- package/dist/config/pi-model-mapper.js.map +1 -1
- package/dist/domain/judgment/answer-mapper.d.ts +49 -0
- package/dist/domain/judgment/answer-mapper.d.ts.map +1 -0
- package/dist/domain/judgment/answer-mapper.js +124 -0
- package/dist/domain/judgment/answer-mapper.js.map +1 -0
- package/dist/domain/judgment/index.d.ts +8 -0
- package/dist/domain/judgment/index.d.ts.map +1 -0
- package/dist/domain/judgment/index.js +8 -0
- package/dist/domain/judgment/index.js.map +1 -0
- package/dist/domain/judgment/judge-state.d.ts +70 -0
- package/dist/domain/judgment/judge-state.d.ts.map +1 -0
- package/dist/domain/judgment/judge-state.js +227 -0
- package/dist/domain/judgment/judge-state.js.map +1 -0
- package/dist/domain/judgment/questions.d.ts +58 -0
- package/dist/domain/judgment/questions.d.ts.map +1 -0
- package/dist/domain/judgment/questions.js +157 -0
- package/dist/domain/judgment/questions.js.map +1 -0
- package/dist/domain/pipeline/pipeline-stage.d.ts +11 -0
- package/dist/domain/pipeline/pipeline-stage.d.ts.map +1 -1
- package/dist/domain/pipeline/pipeline-stage.js +3 -0
- package/dist/domain/pipeline/pipeline-stage.js.map +1 -1
- package/dist/domain/pipeline/router-pipeline.d.ts +32 -6
- package/dist/domain/pipeline/router-pipeline.d.ts.map +1 -1
- package/dist/domain/pipeline/router-pipeline.js +49 -4
- package/dist/domain/pipeline/router-pipeline.js.map +1 -1
- package/dist/domain/pipeline/typed-judgment-stage.d.ts +44 -0
- package/dist/domain/pipeline/typed-judgment-stage.d.ts.map +1 -0
- package/dist/domain/pipeline/typed-judgment-stage.js +144 -0
- package/dist/domain/pipeline/typed-judgment-stage.js.map +1 -0
- package/dist/domain/ports/typed-judgment-port.d.ts +138 -0
- package/dist/domain/ports/typed-judgment-port.d.ts.map +1 -0
- package/dist/domain/ports/typed-judgment-port.js +28 -0
- package/dist/domain/ports/typed-judgment-port.js.map +1 -0
- package/dist/domain/types/entities.d.ts +42 -1
- package/dist/domain/types/entities.d.ts.map +1 -1
- package/dist/domain/types/index.d.ts +1 -1
- package/dist/domain/types/index.d.ts.map +1 -1
- package/dist/domain/types/schemas.d.ts +140 -0
- package/dist/domain/types/schemas.d.ts.map +1 -1
- package/dist/domain/types/schemas.js +191 -0
- package/dist/domain/types/schemas.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.d.ts +67 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.d.ts.map +1 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.js +411 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.js.map +1 -0
- package/package.json +3 -3
- package/src/config/defaults.ts +8 -1
- package/src/config/pi-model-mapper.ts +149 -7
- package/src/domain/judgment/answer-mapper.ts +210 -0
- package/src/domain/judgment/index.ts +32 -0
- package/src/domain/judgment/judge-state.ts +294 -0
- package/src/domain/judgment/questions.ts +198 -0
- package/src/domain/pipeline/pipeline-stage.ts +17 -0
- package/src/domain/pipeline/router-pipeline.ts +83 -6
- package/src/domain/pipeline/typed-judgment-stage.ts +165 -0
- package/src/domain/ports/typed-judgment-port.ts +186 -0
- package/src/domain/types/entities.ts +54 -0
- package/src/domain/types/index.ts +1 -0
- package/src/domain/types/schemas.ts +206 -0
- package/src/index.ts +13 -2
- package/src/infrastructure/judgment/jev-typed-judgment-adapter.ts +564 -0
|
@@ -8,6 +8,21 @@
|
|
|
8
8
|
* scores instead of regex defaults. Unknown models or missing benchmark rows
|
|
9
9
|
* receive conservative pattern defaults. Mapped profiles include
|
|
10
10
|
* `capability_source` (`benchmark` | `pattern_default`) for explain/telemetry.
|
|
11
|
+
*
|
|
12
|
+
* Mixed capability-source fairness (SP-312 / #193): raw ingest scores live on a
|
|
13
|
+
* compressed scale (~0.5–0.85) while pattern defaults sit at 0.7–0.95, so a
|
|
14
|
+
* benchmark-grounded model ranked below every ungrounded same-tier model purely
|
|
15
|
+
* because of its capability source. Grounded vectors are therefore rescaled per
|
|
16
|
+
* dimension against the max of their own pattern-family cohort, anchored at the
|
|
17
|
+
* matched family default: the best-measured grounded model in a family ties the
|
|
18
|
+
* ungrounded family default, and every other grounded model ranks below it in
|
|
19
|
+
* proportion to its measured distance from the best — remaining differences are
|
|
20
|
+
* explained by benchmark standing, not capability source. The transform is
|
|
21
|
+
* strictly monotonic per dimension, so grounded relative order is preserved.
|
|
22
|
+
* The rescale is lift-only: a grounded model is never pushed below its measured
|
|
23
|
+
* scores, so families whose pattern default sits below the measured cohort
|
|
24
|
+
* scale (e.g. UNKNOWN_DEFAULTS rows like gpt-5.3-codex) keep their raw scores
|
|
25
|
+
* instead of regressing. See `withBenchmarkCapabilities`.
|
|
11
26
|
*/
|
|
12
27
|
|
|
13
28
|
import { existsSync, readFileSync } from 'node:fs';
|
|
@@ -62,6 +77,13 @@ const benchmarkProfilesArtifactSchema = z.object({
|
|
|
62
77
|
interface BenchmarkProfilesCache {
|
|
63
78
|
readonly capabilitiesByModelId: ReadonlyMap<string, ModelCapabilities>;
|
|
64
79
|
readonly aliases: ReadonlyMap<string, string>;
|
|
80
|
+
/**
|
|
81
|
+
* SP-312: canonical model_id → per-dimension max across the pattern-family
|
|
82
|
+
* cohort its benchmark row belongs to (family = first matching
|
|
83
|
+
* `MODEL_PATTERN_RULES` entry, else UNKNOWN_DEFAULTS). Anchors the fairness
|
|
84
|
+
* rescale in `withBenchmarkCapabilities`.
|
|
85
|
+
*/
|
|
86
|
+
readonly cohortMaxByModelId: ReadonlyMap<string, ModelCapabilities>;
|
|
65
87
|
}
|
|
66
88
|
|
|
67
89
|
let benchmarkProfilesPathOverride: string | null | undefined;
|
|
@@ -114,7 +136,11 @@ function loadBenchmarkProfilesCache(): BenchmarkProfilesCache | null {
|
|
|
114
136
|
const aliases = new Map<string, string>(
|
|
115
137
|
Object.entries(result.data.aliases ?? {}),
|
|
116
138
|
);
|
|
117
|
-
benchmarkProfilesCache = {
|
|
139
|
+
benchmarkProfilesCache = {
|
|
140
|
+
capabilitiesByModelId,
|
|
141
|
+
aliases,
|
|
142
|
+
cohortMaxByModelId: computeFamilyCohortMax(result.data.models),
|
|
143
|
+
};
|
|
118
144
|
return benchmarkProfilesCache;
|
|
119
145
|
} catch (err: unknown) {
|
|
120
146
|
console.warn('benchmark profiles artifact invalid; using regex capability defaults', {
|
|
@@ -138,20 +164,77 @@ export function resolveBenchmarkModelId(modelId: string): string {
|
|
|
138
164
|
return cache.aliases.get(modelId) ?? modelId;
|
|
139
165
|
}
|
|
140
166
|
|
|
141
|
-
|
|
167
|
+
/** A resolved benchmark row: capabilities plus the canonical row id (SP-312). */
|
|
168
|
+
interface GroundedBenchmarkRow {
|
|
169
|
+
readonly capabilities: ModelCapabilities;
|
|
170
|
+
readonly rowId: string;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function lookupBenchmarkRow(modelId: string): GroundedBenchmarkRow | undefined {
|
|
142
174
|
const cache = loadBenchmarkProfilesCache();
|
|
143
175
|
if (cache === null) {
|
|
144
176
|
return undefined;
|
|
145
177
|
}
|
|
146
178
|
const direct = cache.capabilitiesByModelId.get(modelId);
|
|
147
179
|
if (direct !== undefined) {
|
|
148
|
-
return direct;
|
|
180
|
+
return { capabilities: direct, rowId: modelId };
|
|
149
181
|
}
|
|
150
182
|
const canonical = cache.aliases.get(modelId);
|
|
151
183
|
if (canonical === undefined) {
|
|
152
184
|
return undefined;
|
|
153
185
|
}
|
|
154
|
-
|
|
186
|
+
const capabilities = cache.capabilitiesByModelId.get(canonical);
|
|
187
|
+
return capabilities === undefined ? undefined : { capabilities, rowId: canonical };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function lookupBenchmarkCapabilities(modelId: string): ModelCapabilities | undefined {
|
|
191
|
+
return lookupBenchmarkRow(modelId)?.capabilities;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Per-dimension max of the pattern-family cohort the model's benchmark row
|
|
196
|
+
* belongs to (SP-312). Exported for fairness audits and tests.
|
|
197
|
+
*/
|
|
198
|
+
export function getBenchmarkFamilyCohortMax(modelId: string): ModelCapabilities | undefined {
|
|
199
|
+
const cache = loadBenchmarkProfilesCache();
|
|
200
|
+
if (cache === null) {
|
|
201
|
+
return undefined;
|
|
202
|
+
}
|
|
203
|
+
const row = lookupBenchmarkRow(modelId);
|
|
204
|
+
if (row === undefined) {
|
|
205
|
+
return undefined;
|
|
206
|
+
}
|
|
207
|
+
const max = cache.cohortMaxByModelId.get(row.rowId);
|
|
208
|
+
return max === undefined ? undefined : { ...max };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* SP-312 / #193 — rescale a grounded capability vector onto the pattern-default
|
|
213
|
+
* scale of its model family. Per dimension:
|
|
214
|
+
* `score × max(1, familyDefault / cohortMax)`, where `cohortMax` is the max
|
|
215
|
+
* across the pattern-family cohort the benchmark row belongs to. Properties:
|
|
216
|
+
* - Strictly monotonic in the raw score → grounded relative order preserved.
|
|
217
|
+
* - The cohort-best grounded model reaches the family default exactly (ties
|
|
218
|
+
* ungrounded same-tier models) whenever the family default sits above the
|
|
219
|
+
* measured cohort scale.
|
|
220
|
+
* - Lift-only: never pushes a grounded model below its measured scores, so
|
|
221
|
+
* rows whose family default is below the cohort scale (e.g. UNKNOWN_DEFAULTS
|
|
222
|
+
* rows) are not penalized for being benchmarked.
|
|
223
|
+
* A degenerate cohort (max ≤ 0) cannot differentiate — keep the raw score
|
|
224
|
+
* rather than divide by zero.
|
|
225
|
+
*/
|
|
226
|
+
export function scaleBenchmarkCapabilitiesToFamily(
|
|
227
|
+
grounded: ModelCapabilities,
|
|
228
|
+
familyDefault: ModelCapabilities,
|
|
229
|
+
familyCohortMax: ModelCapabilities,
|
|
230
|
+
): ModelCapabilities {
|
|
231
|
+
const scale = (score: number, defaultScore: number, cohortMaxScore: number): number =>
|
|
232
|
+
cohortMaxScore > 0 ? score * Math.max(1, defaultScore / cohortMaxScore) : score;
|
|
233
|
+
return {
|
|
234
|
+
reasoning: scale(grounded.reasoning, familyDefault.reasoning, familyCohortMax.reasoning),
|
|
235
|
+
code_gen: scale(grounded.code_gen, familyDefault.code_gen, familyCohortMax.code_gen),
|
|
236
|
+
tool_use: scale(grounded.tool_use, familyDefault.tool_use, familyCohortMax.tool_use),
|
|
237
|
+
};
|
|
155
238
|
}
|
|
156
239
|
|
|
157
240
|
/**
|
|
@@ -174,15 +257,25 @@ function withBenchmarkCapabilities(
|
|
|
174
257
|
defaults: ModelFamilyDefaults,
|
|
175
258
|
modelId: string,
|
|
176
259
|
): { readonly defaults: ModelFamilyDefaults; readonly capability_source: CapabilitySource } {
|
|
177
|
-
const
|
|
178
|
-
if (
|
|
260
|
+
const row = lookupBenchmarkRow(modelId);
|
|
261
|
+
if (row === undefined) {
|
|
179
262
|
return { defaults, capability_source: 'pattern_default' };
|
|
180
263
|
}
|
|
181
264
|
|
|
265
|
+
// SP-312 / #193: rescale grounded scores onto the family-default scale so a
|
|
266
|
+
// benchmark-grounded model no longer loses to an ungrounded same-tier model
|
|
267
|
+
// purely because of capability source (see header comment).
|
|
268
|
+
const cache = loadBenchmarkProfilesCache();
|
|
269
|
+
const cohortMax = cache?.cohortMaxByModelId.get(row.rowId);
|
|
270
|
+
const capabilities =
|
|
271
|
+
cohortMax === undefined
|
|
272
|
+
? { ...row.capabilities }
|
|
273
|
+
: scaleBenchmarkCapabilitiesToFamily(row.capabilities, defaults.capabilities, cohortMax);
|
|
274
|
+
|
|
182
275
|
return {
|
|
183
276
|
defaults: {
|
|
184
277
|
...defaults,
|
|
185
|
-
capabilities
|
|
278
|
+
capabilities,
|
|
186
279
|
},
|
|
187
280
|
capability_source: 'benchmark',
|
|
188
281
|
};
|
|
@@ -336,18 +429,36 @@ const COMPOSER_DEFAULTS: ModelFamilyDefaults = {
|
|
|
336
429
|
/** Ordered rules — first match wins. */
|
|
337
430
|
const MODEL_PATTERN_RULES: readonly ModelPatternRule[] = [
|
|
338
431
|
{ pattern: /claude[-_.]?opus|claude[-_.]?sonnet|opus|sonnet/i, defaults: FRONTIER_DEFAULTS },
|
|
432
|
+
// SP-310 / #193: Fable is a frontier Claude family (claude-fable-5-1).
|
|
433
|
+
{ pattern: /claude[-_.]?fable/i, defaults: FRONTIER_DEFAULTS },
|
|
339
434
|
{ pattern: /claude[-_.]?haiku|haiku/i, defaults: ECONOMICAL_DEFAULTS },
|
|
435
|
+
// SP-310 / #193: any other Claude id is a current/future flagship family —
|
|
436
|
+
// frontier, not UNKNOWN_DEFAULTS. Must stay after the Haiku rule so
|
|
437
|
+
// claude-haiku-* keeps economical defaults. Anchored to the start of the id
|
|
438
|
+
// (or a provider/ prefix) so unrelated ids merely containing 'claude'
|
|
439
|
+
// (e.g. openai-claude-bridge) do not inherit frontier defaults.
|
|
440
|
+
{ pattern: /(?:^|\/)claude/i, defaults: FRONTIER_DEFAULTS },
|
|
340
441
|
{ pattern: /gpt[-_.]?5\.(?:[5-9]|\d{2,})|gpt-5-[5-9]/i, defaults: FRONTIER_DEFAULTS },
|
|
341
442
|
{ pattern: /gpt[-_.]?5\.1|gpt[-_.]?5[-_.]?mini|gpt[-_.]?mini/i, defaults: ECONOMICAL_DEFAULTS },
|
|
342
443
|
{ pattern: /gemini[-_.]?2\.5[-_.]?pro|gemini-2-5-pro/i, defaults: FRONTIER_DEFAULTS },
|
|
343
444
|
{ pattern: /gemini[-_.]?3.*pro/i, defaults: FRONTIER_DEFAULTS },
|
|
344
445
|
{ pattern: /gemini.*pro/i, defaults: FRONTIER_DEFAULTS },
|
|
345
446
|
{ pattern: /gemini.*flash|gemini-flash/i, defaults: ECONOMICAL_DEFAULTS },
|
|
447
|
+
// SP-310 / #193: lite variants stay economical even at major >= 4 — keep
|
|
448
|
+
// ahead of the generic Gemini 4+ frontier rule.
|
|
449
|
+
{ pattern: /gemini.*lite/i, defaults: ECONOMICAL_DEFAULTS },
|
|
450
|
+
// SP-310 / #193: Gemini major version >= 4 (gemini-4-argon, gemini-4-pro)
|
|
451
|
+
// is frontier. Placed after the flash/lite rules so gemini-4-flash and
|
|
452
|
+
// gemini-4-lite stay economical.
|
|
453
|
+
{ pattern: /gemini[-_.]?(?:[4-9]|\d{2,})/i, defaults: FRONTIER_DEFAULTS },
|
|
346
454
|
// Peak-pricing vendor families (SP-244 / #165) — recognize Z.ai GLM and
|
|
347
455
|
// DeepSeek ids explicitly instead of UNKNOWN_DEFAULTS so the peak/off-peak
|
|
348
456
|
// adapters (SP-243) pair with sensible economical-tier capability defaults.
|
|
349
457
|
{ pattern: /^glm[-_.]?\d/i, defaults: ECONOMICAL_DEFAULTS },
|
|
350
458
|
{ pattern: /deepseek/i, defaults: ECONOMICAL_DEFAULTS },
|
|
459
|
+
// SP-314 / #195: Kimi K3 / K3.1 are frontier models; require a version or
|
|
460
|
+
// suffix boundary so substring-only ids such as foo-k30 stay unknown.
|
|
461
|
+
{ pattern: /^(kimi[-_.]?)?k3(\.\d+)?(-|$)/i, defaults: FRONTIER_DEFAULTS },
|
|
351
462
|
{ pattern: /^composer[-_]/i, defaults: COMPOSER_DEFAULTS },
|
|
352
463
|
{ pattern: /^cursor\//i, defaults: CURSOR_AUTO_DEFAULTS },
|
|
353
464
|
];
|
|
@@ -365,6 +476,37 @@ function matchPatternRules(id: string): ModelFamilyDefaults | undefined {
|
|
|
365
476
|
return undefined;
|
|
366
477
|
}
|
|
367
478
|
|
|
479
|
+
/**
|
|
480
|
+
* SP-312: group benchmark rows by their matched pattern family (UNKNOWN_DEFAULTS
|
|
481
|
+
* when no rule matches) and return, per canonical row id, the per-dimension max
|
|
482
|
+
* across that family cohort. Called lazily from `loadBenchmarkProfilesCache`,
|
|
483
|
+
* so the pattern-rule consts are already initialized at call time.
|
|
484
|
+
*/
|
|
485
|
+
function computeFamilyCohortMax(
|
|
486
|
+
models: readonly { readonly model_id: string; readonly capabilities: ModelCapabilities }[],
|
|
487
|
+
): ReadonlyMap<string, ModelCapabilities> {
|
|
488
|
+
const cohortMaxByFamily = new Map<ModelFamilyDefaults, ModelCapabilities>();
|
|
489
|
+
const familyByModelId = new Map<string, ModelFamilyDefaults>();
|
|
490
|
+
for (const row of models) {
|
|
491
|
+
const family = matchPatternRules(row.model_id) ?? UNKNOWN_DEFAULTS;
|
|
492
|
+
familyByModelId.set(row.model_id, family);
|
|
493
|
+
const existing = cohortMaxByFamily.get(family);
|
|
494
|
+
cohortMaxByFamily.set(family, {
|
|
495
|
+
reasoning: Math.max(existing?.reasoning ?? 0, row.capabilities.reasoning),
|
|
496
|
+
code_gen: Math.max(existing?.code_gen ?? 0, row.capabilities.code_gen),
|
|
497
|
+
tool_use: Math.max(existing?.tool_use ?? 0, row.capabilities.tool_use),
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
const cohortMaxByModelId = new Map<string, ModelCapabilities>();
|
|
501
|
+
for (const [modelId, family] of familyByModelId) {
|
|
502
|
+
const max = cohortMaxByFamily.get(family);
|
|
503
|
+
if (max !== undefined) {
|
|
504
|
+
cohortMaxByModelId.set(modelId, max);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
return cohortMaxByModelId;
|
|
508
|
+
}
|
|
509
|
+
|
|
368
510
|
/**
|
|
369
511
|
* Registry cost → USD per 1M tokens: average of input and output rates.
|
|
370
512
|
* Pi registry `Model.cost` rates are already denominated per 1M tokens
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed-judgment answer mapper (SP-306, GitHub #185) — pure domain, no I/O.
|
|
3
|
+
*
|
|
4
|
+
* Maps raw {@link TypedJudgmentAnswers} (keyed by question id, SP-304 wire
|
|
5
|
+
* types) onto the normalized {@link TypedJudgment} signal consumed by the
|
|
6
|
+
* routing pipeline:
|
|
7
|
+
* - score answers are normalized by `(levels - 1)` onto 0–1;
|
|
8
|
+
* - noul answers pass through as the calibrated P(yes) from the wire;
|
|
9
|
+
* - a missing answer maps its fields to `null` (no signal);
|
|
10
|
+
* - malformed answers are **rejected** with `TypedJudgmentError`
|
|
11
|
+
* (`kind: 'invalid_response'`): unknown choice, answer/question kind
|
|
12
|
+
* mismatch, out-of-range or non-finite scores, out-of-range confidence,
|
|
13
|
+
* out-of-range or non-finite noul probability.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import {
|
|
17
|
+
TypedJudgmentError,
|
|
18
|
+
type TypedJudgmentAnswer,
|
|
19
|
+
type TypedJudgmentAnswers,
|
|
20
|
+
type TypedJudgmentChoiceAnswer,
|
|
21
|
+
type TypedJudgmentChoiceQuestion,
|
|
22
|
+
type TypedJudgmentNoulAnswer,
|
|
23
|
+
type TypedJudgmentScoreAnswer,
|
|
24
|
+
type TypedJudgmentScoreQuestion,
|
|
25
|
+
} from '../ports/typed-judgment-port.js';
|
|
26
|
+
import {
|
|
27
|
+
CAPABILITY_DESERVED_QUESTION,
|
|
28
|
+
CODE_GEN_QUESTION,
|
|
29
|
+
TASK_KIND_QUESTION,
|
|
30
|
+
TOOL_USE_QUESTION,
|
|
31
|
+
type TaskKind,
|
|
32
|
+
} from './questions.js';
|
|
33
|
+
|
|
34
|
+
/** Normalized, pipeline-ready typed judgment. Fields are `null` when the
|
|
35
|
+
* corresponding question was not answered (partial results allowed). */
|
|
36
|
+
export interface TypedJudgment {
|
|
37
|
+
/** Selected task kind, or `null` when `task_kind` was not answered. */
|
|
38
|
+
readonly taskKind: TaskKind | null;
|
|
39
|
+
/** Judge confidence in `taskKind` (0–1), or `null`. */
|
|
40
|
+
readonly taskKindConfidence: number | null;
|
|
41
|
+
/** `capability_deserved` normalized to 0–1 (`score / (levels - 1)`), or `null`. */
|
|
42
|
+
readonly capabilityDeserved: number | null;
|
|
43
|
+
/** Probability the task needs deep reasoning (noul P(yes)), or `null`. */
|
|
44
|
+
readonly deepReasoningP: number | null;
|
|
45
|
+
/** `code_gen` normalized to 0–1, or `null`. */
|
|
46
|
+
readonly codeGen: number | null;
|
|
47
|
+
/** `tool_use` normalized to 0–1, or `null`. */
|
|
48
|
+
readonly toolUse: number | null;
|
|
49
|
+
/** Probability the request is destructive (noul P(yes)), or `null`. */
|
|
50
|
+
readonly destructiveP: number | null;
|
|
51
|
+
/**
|
|
52
|
+
* Judge confidence (0–1) per answered question that carries one (choice
|
|
53
|
+
* and score answers). Noul answers carry no confidence on the wire.
|
|
54
|
+
*/
|
|
55
|
+
readonly confidences: Readonly<Record<string, number>>;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function invalid(message: string): never {
|
|
59
|
+
throw new TypedJudgmentError('invalid_response', message);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function requireConfidence(value: number, questionId: string): number {
|
|
63
|
+
if (!Number.isFinite(value) || value < 0 || value > 1) {
|
|
64
|
+
invalid(`question '${questionId}': confidence out of range 0–1`);
|
|
65
|
+
}
|
|
66
|
+
return value;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function mapChoiceAnswer(
|
|
70
|
+
question: TypedJudgmentChoiceQuestion,
|
|
71
|
+
answer: TypedJudgmentAnswer,
|
|
72
|
+
): TypedJudgmentChoiceAnswer {
|
|
73
|
+
if (answer.kind !== 'choice') {
|
|
74
|
+
invalid(
|
|
75
|
+
`question '${question.id}': expected a choice answer, got '${answer.kind}'`,
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
if (!question.options.includes(answer.choice)) {
|
|
79
|
+
invalid(
|
|
80
|
+
`question '${question.id}': unknown choice '${answer.choice}'`,
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
requireConfidence(answer.confidence, question.id);
|
|
84
|
+
return answer;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function mapScoreAnswer(
|
|
88
|
+
question: TypedJudgmentScoreQuestion,
|
|
89
|
+
answer: TypedJudgmentAnswer,
|
|
90
|
+
): number {
|
|
91
|
+
if (answer.kind !== 'score') {
|
|
92
|
+
invalid(
|
|
93
|
+
`question '${question.id}': expected a score answer, got '${answer.kind}'`,
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
const scoreAnswer = answer as TypedJudgmentScoreAnswer;
|
|
97
|
+
const levels = question.anchors.length;
|
|
98
|
+
const max = levels - 1;
|
|
99
|
+
if (max < 1) {
|
|
100
|
+
invalid(
|
|
101
|
+
`question '${question.id}': score scale needs at least two anchors`,
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
if (
|
|
105
|
+
!Number.isFinite(scoreAnswer.score) ||
|
|
106
|
+
!Number.isInteger(scoreAnswer.score) ||
|
|
107
|
+
scoreAnswer.score < 0 ||
|
|
108
|
+
scoreAnswer.score > max
|
|
109
|
+
) {
|
|
110
|
+
invalid(
|
|
111
|
+
`question '${question.id}': score ${String(scoreAnswer.score)} out of range 0–${max}`,
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
requireConfidence(scoreAnswer.confidence, question.id);
|
|
115
|
+
// Normalized by (levels - 1) onto 0–1 (#185).
|
|
116
|
+
return scoreAnswer.score / max;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function mapNoulAnswer(
|
|
120
|
+
questionId: string,
|
|
121
|
+
answer: TypedJudgmentAnswer,
|
|
122
|
+
): number {
|
|
123
|
+
if (answer.kind !== 'noul') {
|
|
124
|
+
invalid(
|
|
125
|
+
`question '${questionId}': expected a noul answer, got '${answer.kind}'`,
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
const probability = (answer as TypedJudgmentNoulAnswer).noul;
|
|
129
|
+
if (!Number.isFinite(probability) || probability < 0 || probability > 1) {
|
|
130
|
+
invalid(`question '${questionId}': noul probability out of range 0–1`);
|
|
131
|
+
}
|
|
132
|
+
return probability;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Map raw judge answers to the normalized {@link TypedJudgment} signal.
|
|
137
|
+
*
|
|
138
|
+
* @throws {TypedJudgmentError} with `kind: 'invalid_response'` when an
|
|
139
|
+
* answer is malformed (unknown choice, kind mismatch, out-of-range
|
|
140
|
+
* score/confidence). Callers follow the port's fail-open contract: any
|
|
141
|
+
* rejection means "no judgment available".
|
|
142
|
+
*/
|
|
143
|
+
export function mapTypedJudgment(
|
|
144
|
+
answers: TypedJudgmentAnswers,
|
|
145
|
+
): TypedJudgment {
|
|
146
|
+
const confidences: Record<string, number> = {};
|
|
147
|
+
|
|
148
|
+
let taskKind: TaskKind | null = null;
|
|
149
|
+
let taskKindConfidence: number | null = null;
|
|
150
|
+
const taskKindAnswer = answers[TASK_KIND_QUESTION.id];
|
|
151
|
+
if (taskKindAnswer !== undefined) {
|
|
152
|
+
const mapped = mapChoiceAnswer(TASK_KIND_QUESTION, taskKindAnswer);
|
|
153
|
+
taskKind = mapped.choice as TaskKind;
|
|
154
|
+
taskKindConfidence = mapped.confidence;
|
|
155
|
+
confidences[TASK_KIND_QUESTION.id] = mapped.confidence;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
let capabilityDeserved: number | null = null;
|
|
159
|
+
const capabilityAnswer = answers[CAPABILITY_DESERVED_QUESTION.id];
|
|
160
|
+
if (capabilityAnswer !== undefined) {
|
|
161
|
+
capabilityDeserved = mapScoreAnswer(
|
|
162
|
+
CAPABILITY_DESERVED_QUESTION,
|
|
163
|
+
capabilityAnswer,
|
|
164
|
+
);
|
|
165
|
+
confidences[CAPABILITY_DESERVED_QUESTION.id] = (
|
|
166
|
+
capabilityAnswer as TypedJudgmentScoreAnswer
|
|
167
|
+
).confidence;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
let codeGen: number | null = null;
|
|
171
|
+
const codeGenAnswer = answers[CODE_GEN_QUESTION.id];
|
|
172
|
+
if (codeGenAnswer !== undefined) {
|
|
173
|
+
codeGen = mapScoreAnswer(CODE_GEN_QUESTION, codeGenAnswer);
|
|
174
|
+
confidences[CODE_GEN_QUESTION.id] = (
|
|
175
|
+
codeGenAnswer as TypedJudgmentScoreAnswer
|
|
176
|
+
).confidence;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
let toolUse: number | null = null;
|
|
180
|
+
const toolUseAnswer = answers[TOOL_USE_QUESTION.id];
|
|
181
|
+
if (toolUseAnswer !== undefined) {
|
|
182
|
+
toolUse = mapScoreAnswer(TOOL_USE_QUESTION, toolUseAnswer);
|
|
183
|
+
confidences[TOOL_USE_QUESTION.id] = (
|
|
184
|
+
toolUseAnswer as TypedJudgmentScoreAnswer
|
|
185
|
+
).confidence;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
let deepReasoningP: number | null = null;
|
|
189
|
+
const deepReasoningAnswer = answers['needs_deep_reasoning'];
|
|
190
|
+
if (deepReasoningAnswer !== undefined) {
|
|
191
|
+
deepReasoningP = mapNoulAnswer('needs_deep_reasoning', deepReasoningAnswer);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
let destructiveP: number | null = null;
|
|
195
|
+
const destructiveAnswer = answers['destructive'];
|
|
196
|
+
if (destructiveAnswer !== undefined) {
|
|
197
|
+
destructiveP = mapNoulAnswer('destructive', destructiveAnswer);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
return {
|
|
201
|
+
taskKind,
|
|
202
|
+
taskKindConfidence,
|
|
203
|
+
capabilityDeserved,
|
|
204
|
+
deepReasoningP,
|
|
205
|
+
codeGen,
|
|
206
|
+
toolUse,
|
|
207
|
+
destructiveP,
|
|
208
|
+
confidences,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed-judgment domain module (SP-306, #185) — barrel export.
|
|
3
|
+
* Pure domain: no I/O, no infrastructure dependencies.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export {
|
|
7
|
+
ACTIVITY_LINE_MAX_CHARS,
|
|
8
|
+
ACK_DEEP_SESSION_MESSAGE_THRESHOLD,
|
|
9
|
+
JUDGE_STATE_TOKEN_CAP,
|
|
10
|
+
WORKSPACE_MAX_CHARS,
|
|
11
|
+
buildJudgeState,
|
|
12
|
+
isAcknowledgement,
|
|
13
|
+
type IsAcknowledgementOptions,
|
|
14
|
+
type JudgeStateOptions,
|
|
15
|
+
} from './judge-state.js';
|
|
16
|
+
|
|
17
|
+
export {
|
|
18
|
+
CAPABILITY_DESERVED_QUESTION,
|
|
19
|
+
CODE_GEN_QUESTION,
|
|
20
|
+
DESTRUCTIVE_QUESTION,
|
|
21
|
+
NEEDS_DEEP_REASONING_QUESTION,
|
|
22
|
+
TASK_KIND_OPTIONS,
|
|
23
|
+
TASK_KIND_QUESTION,
|
|
24
|
+
TOOL_USE_QUESTION,
|
|
25
|
+
TYPED_JUDGMENT_QUESTIONS,
|
|
26
|
+
TYPED_JUDGMENT_QUESTION_IDS,
|
|
27
|
+
TYPED_JUDGMENT_QUESTION_SET_VERSION,
|
|
28
|
+
type TaskKind,
|
|
29
|
+
type TypedJudgmentSixQuestionId,
|
|
30
|
+
} from './questions.js';
|
|
31
|
+
|
|
32
|
+
export { mapTypedJudgment, type TypedJudgment } from './answer-mapper.js';
|