pi-smart-router 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/index.ts +9 -0
- package/.pi/extensions/smart-router/pi-model-scope.ts +127 -20
- package/.pi/extensions/smart-router/planning-delegate.ts +318 -0
- package/.pi/extensions/smart-router/route-and-delegate.ts +21 -1
- package/.pi/extensions/smart-router/types.ts +3 -0
- package/README.md +51 -0
- package/config/benchmark-profiles.json +145 -0
- package/config/models.yaml.example +5 -0
- package/config/routing-calibration.json.example +14 -2
- package/dist/config/defaults.d.ts +2 -2
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +5 -3
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/pi-model-mapper.d.ts +12 -2
- package/dist/config/pi-model-mapper.d.ts.map +1 -1
- package/dist/config/pi-model-mapper.js +91 -6
- package/dist/config/pi-model-mapper.js.map +1 -1
- package/dist/domain/matching/hydra-input.d.ts +6 -5
- package/dist/domain/matching/hydra-input.d.ts.map +1 -1
- package/dist/domain/matching/hydra-input.js +73 -6
- package/dist/domain/matching/hydra-input.js.map +1 -1
- package/dist/domain/pipeline/router-pipeline.d.ts +22 -1
- package/dist/domain/pipeline/router-pipeline.d.ts.map +1 -1
- package/dist/domain/pipeline/router-pipeline.js +135 -22
- package/dist/domain/pipeline/router-pipeline.js.map +1 -1
- package/dist/domain/routing/isotonic-calibrator.d.ts +56 -0
- package/dist/domain/routing/isotonic-calibrator.d.ts.map +1 -0
- package/dist/domain/routing/isotonic-calibrator.js +187 -0
- package/dist/domain/routing/isotonic-calibrator.js.map +1 -0
- package/dist/domain/routing/p-success-classifier.d.ts +53 -7
- package/dist/domain/routing/p-success-classifier.d.ts.map +1 -1
- package/dist/domain/routing/p-success-classifier.js +205 -21
- package/dist/domain/routing/p-success-classifier.js.map +1 -1
- package/dist/domain/types/entities.d.ts +54 -0
- package/dist/domain/types/entities.d.ts.map +1 -1
- package/dist/domain/types/index.d.ts +1 -1
- package/dist/domain/types/index.d.ts.map +1 -1
- package/dist/domain/types/schemas.d.ts +29 -0
- package/dist/domain/types/schemas.d.ts.map +1 -1
- package/dist/domain/types/schemas.js +57 -0
- package/dist/domain/types/schemas.js.map +1 -1
- package/dist/infrastructure/persistence/sqlite-store.d.ts.map +1 -1
- package/dist/infrastructure/persistence/sqlite-store.js +2 -1
- package/dist/infrastructure/persistence/sqlite-store.js.map +1 -1
- package/dist/infrastructure/telemetry/routing-telemetry.d.ts +20 -1
- package/dist/infrastructure/telemetry/routing-telemetry.d.ts.map +1 -1
- package/dist/infrastructure/telemetry/routing-telemetry.js +82 -2
- package/dist/infrastructure/telemetry/routing-telemetry.js.map +1 -1
- package/package.json +6 -3
- package/specs/001-build-smart-router/contracts/telemetry-contrib.schema.json +29 -1
- package/src/config/defaults.ts +6 -2
- package/src/config/pi-model-mapper.ts +110 -6
- package/src/domain/matching/hydra-input.ts +86 -7
- package/src/domain/pipeline/router-pipeline.ts +195 -29
- package/src/domain/routing/isotonic-calibrator.ts +255 -0
- package/src/domain/routing/p-success-classifier.ts +299 -26
- package/src/domain/types/entities.ts +58 -0
- package/src/domain/types/index.ts +4 -0
- package/src/domain/types/schemas.ts +73 -0
- package/src/infrastructure/persistence/sqlite-store.ts +2 -0
- package/src/infrastructure/telemetry/routing-telemetry.ts +127 -7
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Online isotonic P(success) calibrator for low_intensity gate (SP-133).
|
|
3
|
+
*
|
|
4
|
+
* Loads monotonic knot tables from routing-calibration bundle at serve time.
|
|
5
|
+
* O(log n) binary search lookup with <5ms budget; falls back to raw logistic
|
|
6
|
+
* when the artifact is missing or under-trained.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
9
|
+
import { resolve } from 'node:path';
|
|
10
|
+
import { z } from 'zod';
|
|
11
|
+
import { MIN_TRAINING_SAMPLES } from './p-success-classifier.js';
|
|
12
|
+
export const ISOTONIC_CALIBRATOR_ARTIFACT_VERSION = 1;
|
|
13
|
+
export const ISOTONIC_LOOKUP_BUDGET_MS = 5;
|
|
14
|
+
export const DEFAULT_ROUTING_CALIBRATION_PATH = resolve('config', 'routing-calibration.json');
|
|
15
|
+
export class IsotonicCalibratorLoaderError extends Error {
|
|
16
|
+
name = 'IsotonicCalibratorLoaderError';
|
|
17
|
+
constructor(message, options) {
|
|
18
|
+
super(message, options);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const IsotonicCalibratorArtifactSchema = z.object({
|
|
22
|
+
version: z.literal(ISOTONIC_CALIBRATOR_ARTIFACT_VERSION),
|
|
23
|
+
min_training_samples: z.number().int().min(0),
|
|
24
|
+
x_knots: z.array(z.number().finite().min(0).max(1)).min(2),
|
|
25
|
+
y_knots: z.array(z.number().finite().min(0).max(1)).min(2),
|
|
26
|
+
trained_sample_count: z.number().int().min(0),
|
|
27
|
+
holdout_ece_raw: z.number().finite().min(0).max(1).nullable(),
|
|
28
|
+
holdout_ece_calibrated: z.number().finite().min(0).max(1).nullable(),
|
|
29
|
+
});
|
|
30
|
+
function clamp01(value) {
|
|
31
|
+
if (value <= 0)
|
|
32
|
+
return 0;
|
|
33
|
+
if (value >= 1)
|
|
34
|
+
return 1;
|
|
35
|
+
return value;
|
|
36
|
+
}
|
|
37
|
+
function formatZodIssues(error) {
|
|
38
|
+
return error.issues
|
|
39
|
+
.map((issue) => ` - ${issue.path.join('.')}: ${issue.message}`)
|
|
40
|
+
.join('\n');
|
|
41
|
+
}
|
|
42
|
+
/** True when artifact has enough labeled samples for serve-time calibration. */
|
|
43
|
+
export function isIsotonicCalibratorTrained(artifact) {
|
|
44
|
+
return artifact.trained_sample_count >= artifact.min_training_samples;
|
|
45
|
+
}
|
|
46
|
+
export function createDefaultIsotonicCalibratorArtifact() {
|
|
47
|
+
return {
|
|
48
|
+
version: ISOTONIC_CALIBRATOR_ARTIFACT_VERSION,
|
|
49
|
+
min_training_samples: MIN_TRAINING_SAMPLES,
|
|
50
|
+
x_knots: [0, 1],
|
|
51
|
+
y_knots: [0, 1],
|
|
52
|
+
trained_sample_count: 0,
|
|
53
|
+
holdout_ece_raw: null,
|
|
54
|
+
holdout_ece_calibrated: null,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/** Piecewise-constant isotonic lookup with O(log n) binary search on knots. */
|
|
58
|
+
export function applyIsotonicLookup(rawScore, xKnots, yKnots) {
|
|
59
|
+
if (xKnots.length === 0 || yKnots.length === 0 || xKnots.length !== yKnots.length) {
|
|
60
|
+
return clamp01(rawScore);
|
|
61
|
+
}
|
|
62
|
+
const score = clamp01(rawScore);
|
|
63
|
+
if (score <= xKnots[0]) {
|
|
64
|
+
return clamp01(yKnots[0]);
|
|
65
|
+
}
|
|
66
|
+
const lastIndex = xKnots.length - 1;
|
|
67
|
+
if (score >= xKnots[lastIndex]) {
|
|
68
|
+
return clamp01(yKnots[lastIndex]);
|
|
69
|
+
}
|
|
70
|
+
let low = 0;
|
|
71
|
+
let high = lastIndex;
|
|
72
|
+
while (low < high) {
|
|
73
|
+
const mid = Math.floor((low + high + 1) / 2);
|
|
74
|
+
if (xKnots[mid] <= score) {
|
|
75
|
+
low = mid;
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
high = mid - 1;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return clamp01(yKnots[low]);
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Apply isotonic calibration to a raw logistic P(success) score.
|
|
85
|
+
* Returns the raw score when the artifact is missing or under-trained.
|
|
86
|
+
*/
|
|
87
|
+
export function applyIsotonicCalibrator(rawScore, artifact) {
|
|
88
|
+
if (artifact === null || !isIsotonicCalibratorTrained(artifact)) {
|
|
89
|
+
return clamp01(rawScore);
|
|
90
|
+
}
|
|
91
|
+
return applyIsotonicLookup(rawScore, artifact.x_knots, artifact.y_knots);
|
|
92
|
+
}
|
|
93
|
+
/** Apply isotonic calibration with elapsed timing guard for the online routing budget. */
|
|
94
|
+
export function applyIsotonicCalibratorTimed(rawScore, artifact, budgetMs = ISOTONIC_LOOKUP_BUDGET_MS) {
|
|
95
|
+
const start = performance.now();
|
|
96
|
+
const calibration_applied = artifact !== null && isIsotonicCalibratorTrained(artifact);
|
|
97
|
+
const calibrated = applyIsotonicCalibrator(rawScore, artifact);
|
|
98
|
+
const elapsed_ms = performance.now() - start;
|
|
99
|
+
if (elapsed_ms > budgetMs) {
|
|
100
|
+
console.warn('Isotonic P(success) lookup exceeded latency budget', {
|
|
101
|
+
elapsed_ms,
|
|
102
|
+
budget_ms: budgetMs,
|
|
103
|
+
knot_count: artifact?.x_knots.length ?? 0,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
return {
|
|
107
|
+
calibrated,
|
|
108
|
+
elapsed_ms,
|
|
109
|
+
within_budget: elapsed_ms <= budgetMs,
|
|
110
|
+
calibration_applied,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/** Parse and validate an isotonic calibrator artifact from JSON text. */
|
|
114
|
+
export function parseIsotonicCalibratorJson(raw) {
|
|
115
|
+
let parsed;
|
|
116
|
+
try {
|
|
117
|
+
parsed = JSON.parse(raw);
|
|
118
|
+
}
|
|
119
|
+
catch (err) {
|
|
120
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
121
|
+
throw new IsotonicCalibratorLoaderError(`Failed to parse JSON: ${message}`, { cause: err });
|
|
122
|
+
}
|
|
123
|
+
const result = IsotonicCalibratorArtifactSchema.safeParse(parsed);
|
|
124
|
+
if (!result.success) {
|
|
125
|
+
throw new IsotonicCalibratorLoaderError(`Invalid isotonic calibrator artifact:\n${formatZodIssues(result.error)}`, { cause: result.error });
|
|
126
|
+
}
|
|
127
|
+
return result.data;
|
|
128
|
+
}
|
|
129
|
+
/** Extract isotonic_calibrator from a routing-calibration bundle object. */
|
|
130
|
+
export function parseIsotonicCalibratorFromBundle(parsed) {
|
|
131
|
+
if (typeof parsed !== 'object' || parsed === null) {
|
|
132
|
+
throw new IsotonicCalibratorLoaderError('Routing calibration bundle must be a JSON object');
|
|
133
|
+
}
|
|
134
|
+
const isotonic = parsed.isotonic_calibrator;
|
|
135
|
+
if (isotonic === undefined) {
|
|
136
|
+
throw new IsotonicCalibratorLoaderError('Routing calibration bundle missing isotonic_calibrator');
|
|
137
|
+
}
|
|
138
|
+
const result = IsotonicCalibratorArtifactSchema.safeParse(isotonic);
|
|
139
|
+
if (!result.success) {
|
|
140
|
+
throw new IsotonicCalibratorLoaderError(`Invalid isotonic_calibrator in routing calibration bundle:\n${formatZodIssues(result.error)}`, { cause: result.error });
|
|
141
|
+
}
|
|
142
|
+
return result.data;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Load isotonic calibrator from routing-calibration bundle on disk.
|
|
146
|
+
* Returns null when the bundle file is missing.
|
|
147
|
+
*/
|
|
148
|
+
export function loadIsotonicCalibrator(options) {
|
|
149
|
+
const filePath = options?.filePath ?? DEFAULT_ROUTING_CALIBRATION_PATH;
|
|
150
|
+
if (!existsSync(filePath)) {
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
let raw;
|
|
154
|
+
try {
|
|
155
|
+
raw = readFileSync(filePath, 'utf8');
|
|
156
|
+
}
|
|
157
|
+
catch (err) {
|
|
158
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
159
|
+
throw new IsotonicCalibratorLoaderError(`Failed to read routing calibration file: ${message}`, {
|
|
160
|
+
cause: err,
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
let parsed;
|
|
164
|
+
try {
|
|
165
|
+
parsed = JSON.parse(raw);
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
169
|
+
throw new IsotonicCalibratorLoaderError(`Failed to parse routing calibration JSON: ${message}`, {
|
|
170
|
+
cause: err,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
return parseIsotonicCalibratorFromBundle(parsed);
|
|
174
|
+
}
|
|
175
|
+
/** Resolve calibrator for online inference — missing or invalid artifacts fall back safely. */
|
|
176
|
+
export function resolveIsotonicCalibrator(options) {
|
|
177
|
+
try {
|
|
178
|
+
return loadIsotonicCalibrator(options);
|
|
179
|
+
}
|
|
180
|
+
catch (err) {
|
|
181
|
+
console.warn('Isotonic calibrator artifact invalid; using raw logistic fallback', {
|
|
182
|
+
error: err instanceof Error ? err.message : String(err),
|
|
183
|
+
});
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
//# sourceMappingURL=isotonic-calibrator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"isotonic-calibrator.js","sourceRoot":"","sources":["../../../src/domain/routing/isotonic-calibrator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAU,CAAC;AAC/D,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAC3C,MAAM,CAAC,MAAM,gCAAgC,GAAG,OAAO,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAC;AAuB9F,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IACpC,IAAI,GAAG,+BAA+B,CAAC;IAEzD,YAAY,OAAe,EAAE,OAAsB;QACjD,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,gCAAgC,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,oCAAoC,CAAC;IACxD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7D,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACrE,CAAC,CAAC;AAEH,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,KAA+E;IACtG,OAAO,KAAK,CAAC,MAAM;SAChB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;SAC/D,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,2BAA2B,CAAC,QAAoC;IAC9E,OAAO,QAAQ,CAAC,oBAAoB,IAAI,QAAQ,CAAC,oBAAoB,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,uCAAuC;IACrD,OAAO;QACL,OAAO,EAAE,oCAAoC;QAC7C,oBAAoB,EAAE,oBAAoB;QAC1C,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QACf,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QACf,oBAAoB,EAAE,CAAC;QACvB,eAAe,EAAE,IAAI;QACrB,sBAAsB,EAAE,IAAI;KAC7B,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,mBAAmB,CACjC,QAAgB,EAChB,MAAyB,EACzB,MAAyB;IAEzB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QAClF,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChC,IAAI,KAAK,IAAI,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IACpC,IAAI,KAAK,IAAI,MAAM,CAAC,SAAS,CAAE,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,MAAM,CAAC,SAAS,CAAE,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,SAAS,CAAC;IACrB,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,MAAM,CAAC,GAAG,CAAE,IAAI,KAAK,EAAE,CAAC;YAC1B,GAAG,GAAG,GAAG,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACrC,QAAgB,EAChB,QAA2C;IAE3C,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,2BAA2B,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChE,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;AAC3E,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,4BAA4B,CAC1C,QAAgB,EAChB,QAA2C,EAC3C,WAAmB,yBAAyB;IAE5C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,mBAAmB,GAAG,QAAQ,KAAK,IAAI,IAAI,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IACvF,MAAM,UAAU,GAAG,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/D,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IAE7C,IAAI,UAAU,GAAG,QAAQ,EAAE,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,oDAAoD,EAAE;YACjE,UAAU;YACV,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC;SAC1C,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,UAAU;QACV,UAAU;QACV,aAAa,EAAE,UAAU,IAAI,QAAQ;QACrC,mBAAmB;KACpB,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,2BAA2B,CAAC,GAAW;IACrD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,6BAA6B,CAAC,yBAAyB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,6BAA6B,CACrC,0CAA0C,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACzE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CACxB,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,iCAAiC,CAAC,MAAe;IAC/D,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,IAAI,6BAA6B,CAAC,kDAAkD,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,QAAQ,GAAI,MAAkC,CAAC,mBAAmB,CAAC;IACzE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,6BAA6B,CAAC,wDAAwD,CAAC,CAAC;IACpG,CAAC;IAED,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACpE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,6BAA6B,CACrC,+DAA+D,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC9F,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CACxB,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,OAAuC;IAEvC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,gCAAgC,CAAC;IAEvE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,6BAA6B,CAAC,4CAA4C,OAAO,EAAE,EAAE;YAC7F,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,6BAA6B,CAAC,6CAA6C,OAAO,EAAE,EAAE;YAC9F,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC;IAED,OAAO,iCAAiC,CAAC,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,yBAAyB,CACvC,OAAuC;IAEvC,IAAI,CAAC;QACH,OAAO,sBAAsB,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,mEAAmE,EAAE;YAChF,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -11,10 +11,37 @@ import type { OutcomeSignalType, RoutingDatasetRecord, RoutingOutcomeRecord } fr
|
|
|
11
11
|
export declare const MIN_TRAINING_SAMPLES = 30;
|
|
12
12
|
/** Neutral probability when training data is insufficient or artifact missing. */
|
|
13
13
|
export declare const NEUTRAL_P_SUCCESS = 0.5;
|
|
14
|
+
/** Behavioral outcome signals that mark cheap-tier training failure (SP-062). */
|
|
15
|
+
export declare const BEHAVIORAL_FAILURE_OUTCOME_SIGNALS: readonly OutcomeSignalType[];
|
|
16
|
+
/** Verifier-grade failure proxies derived from privacy-safe telemetry scalars (SP-131). */
|
|
17
|
+
export declare const VERIFIER_FAILURE_OUTCOME_SIGNALS: readonly ["tool_failure_chain", "stop_reason_invalid", "reprompt_detected", "high_edit_distance"];
|
|
18
|
+
export type VerifierFailureOutcomeSignal = (typeof VERIFIER_FAILURE_OUTCOME_SIGNALS)[number];
|
|
19
|
+
/** Execution telemetry failure signals (SP-104 review follow-up, SP-131). */
|
|
20
|
+
export declare const EXECUTION_FAILURE_OUTCOME_SIGNALS: readonly ["provider_failover", "stop_reason_length", "infra_error"];
|
|
21
|
+
export type ExecutionFailureOutcomeSignal = (typeof EXECUTION_FAILURE_OUTCOME_SIGNALS)[number];
|
|
22
|
+
/** Union of all training label signals consumed by P(success) export and aggregate paths. */
|
|
23
|
+
export type TrainingOutcomeSignal = OutcomeSignalType | VerifierFailureOutcomeSignal | ExecutionFailureOutcomeSignal;
|
|
14
24
|
/** Outcome signals that mark a routing attempt as unsuccessful for cheap-tier training. */
|
|
15
|
-
export declare const FAILURE_OUTCOME_SIGNALS: readonly
|
|
16
|
-
/**
|
|
25
|
+
export declare const FAILURE_OUTCOME_SIGNALS: readonly TrainingOutcomeSignal[];
|
|
26
|
+
/** @deprecated Use EXECUTION_FAILURE_OUTCOME_SIGNALS */
|
|
17
27
|
export declare const FUTURE_FAILURE_SIGNALS: readonly ["provider_failover", "stop_reason_length", "infra_error"];
|
|
28
|
+
/** Privacy-safe scalar failure proxies exported in calibration contrib rows (SP-131). */
|
|
29
|
+
export declare const P_SUCCESS_FAILURE_PROXY_FIELDS: readonly ["tool_failure_chain_count", "stop_reason_invalid", "reprompt_rate", "edit_distance_proxy"];
|
|
30
|
+
export type PSuccessFailureProxyField = (typeof P_SUCCESS_FAILURE_PROXY_FIELDS)[number];
|
|
31
|
+
export interface PSuccessFailureProxies {
|
|
32
|
+
readonly tool_failure_chain_count: number | null;
|
|
33
|
+
readonly stop_reason_invalid: boolean | null;
|
|
34
|
+
readonly reprompt_rate: number | null;
|
|
35
|
+
readonly edit_distance_proxy: number | null;
|
|
36
|
+
}
|
|
37
|
+
/** Minimum identical tool failures before labeling a tool-failure chain (loop escalation uses 3). */
|
|
38
|
+
export declare const TOOL_FAILURE_CHAIN_LABEL_THRESHOLD = 2;
|
|
39
|
+
/** Normalized re-prompt rate at or above this marks failure. */
|
|
40
|
+
export declare const REPROMPT_RATE_FAILURE_THRESHOLD = 0.5;
|
|
41
|
+
/** Normalized edit-distance proxy at or above this marks likely re-prompt / correction. */
|
|
42
|
+
export declare const EDIT_DISTANCE_PROXY_FAILURE_THRESHOLD = 0.6;
|
|
43
|
+
/** Provider stop reasons treated as invalid task completion for training labels. */
|
|
44
|
+
export declare const INVALID_STOP_REASONS: Set<string>;
|
|
18
45
|
export declare const P_SUCCESS_FEATURE_NAMES: readonly ["prompt_length_norm", "estimated_input_tokens_norm", "triage_cyclomatic_score", "requirement_reasoning", "requirement_code_gen", "requirement_tool_use", "has_tool_context", "compaction_flag", "routing_latency_norm", "economical_tier"];
|
|
19
46
|
export type PSuccessFeatureName = (typeof P_SUCCESS_FEATURE_NAMES)[number];
|
|
20
47
|
export interface PSuccessFeatures {
|
|
@@ -75,21 +102,40 @@ export interface LabeledTrainingSample {
|
|
|
75
102
|
readonly request_id: string;
|
|
76
103
|
readonly features: PSuccessFeatures;
|
|
77
104
|
readonly success: boolean;
|
|
78
|
-
readonly outcome_signals: readonly
|
|
105
|
+
readonly outcome_signals: readonly TrainingOutcomeSignal[];
|
|
106
|
+
readonly failure_proxies: PSuccessFailureProxies;
|
|
79
107
|
}
|
|
80
108
|
export interface DatasetExportJoinRow extends Record<string, unknown> {
|
|
81
109
|
readonly request_id: string;
|
|
82
110
|
readonly success_label: boolean | null;
|
|
83
|
-
readonly outcome_signals: readonly
|
|
111
|
+
readonly outcome_signals: readonly TrainingOutcomeSignal[];
|
|
112
|
+
readonly tool_failure_chain_count: number | null;
|
|
113
|
+
readonly stop_reason_invalid: boolean | null;
|
|
114
|
+
readonly reprompt_rate: number | null;
|
|
115
|
+
readonly edit_distance_proxy: number | null;
|
|
84
116
|
}
|
|
85
117
|
/** Map a dataset row to a normalized feature vector for training or inference. */
|
|
86
118
|
export declare function extractPSuccessFeatures(record: Pick<RoutingDatasetRecord, 'prompt_length_chars' | 'estimated_input_tokens' | 'triage_cyclomatic_score' | 'requirement_reasoning' | 'requirement_code_gen' | 'requirement_tool_use' | 'has_tool_context' | 'compaction_flag' | 'routing_latency_ms' | 'tier'>): PSuccessFeatures;
|
|
87
119
|
export declare function featuresToVector(features: PSuccessFeatures): number[];
|
|
88
|
-
/**
|
|
89
|
-
export declare function
|
|
120
|
+
/** Extract privacy-safe failure proxy scalars from a contrib or export row. */
|
|
121
|
+
export declare function extractFailureProxies(record: Record<string, unknown>): PSuccessFailureProxies;
|
|
122
|
+
/** Map failure proxy scalars to verifier-grade training outcome signals. */
|
|
123
|
+
export declare function deriveVerifierFailureSignals(proxies: PSuccessFailureProxies): readonly VerifierFailureOutcomeSignal[];
|
|
124
|
+
/** Derive success label from behavioral outcomes plus optional failure proxies. */
|
|
125
|
+
export declare function deriveSuccessLabel(outcomes: readonly RoutingOutcomeRecord[], options?: {
|
|
126
|
+
readonly failureProxies?: PSuccessFailureProxies;
|
|
127
|
+
}): {
|
|
128
|
+
success: boolean | null;
|
|
129
|
+
outcome_signals: readonly TrainingOutcomeSignal[];
|
|
130
|
+
};
|
|
131
|
+
/** Derive training labels from a privacy-safe export/contrib row (no prompt text). */
|
|
132
|
+
export declare function deriveSuccessLabelFromExportRow(record: Record<string, unknown>): {
|
|
90
133
|
success: boolean | null;
|
|
91
|
-
outcome_signals: readonly
|
|
134
|
+
outcome_signals: readonly TrainingOutcomeSignal[];
|
|
135
|
+
failure_proxies: PSuccessFailureProxies;
|
|
92
136
|
};
|
|
137
|
+
/** Attach normalized failure proxy fields for calibration export rows. */
|
|
138
|
+
export declare function attachFailureProxiesToExport(exportRecord: Record<string, unknown>): Record<string, unknown>;
|
|
93
139
|
/** Group outcomes by request_id for export joins. */
|
|
94
140
|
export declare function indexOutcomesByRequestId(outcomes: readonly RoutingOutcomeRecord[]): Map<string, RoutingOutcomeRecord[]>;
|
|
95
141
|
/** Join dataset rows with outcome labels for training export. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"p-success-classifier.d.ts","sourceRoot":"","sources":["../../../src/domain/routing/p-success-classifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,gFAAgF;AAChF,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,kFAAkF;AAClF,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC,2FAA2F;AAC3F,eAAO,MAAM,uBAAuB,EAAE,SAAS,
|
|
1
|
+
{"version":3,"file":"p-success-classifier.d.ts","sourceRoot":"","sources":["../../../src/domain/routing/p-success-classifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,gFAAgF;AAChF,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,kFAAkF;AAClF,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC,iFAAiF;AACjF,eAAO,MAAM,kCAAkC,EAAE,SAAS,iBAAiB,EAGjE,CAAC;AAEX,2FAA2F;AAC3F,eAAO,MAAM,gCAAgC,mGAKnC,CAAC;AAEX,MAAM,MAAM,4BAA4B,GAAG,CAAC,OAAO,gCAAgC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7F,6EAA6E;AAC7E,eAAO,MAAM,iCAAiC,qEAIpC,CAAC;AAEX,MAAM,MAAM,6BAA6B,GAAG,CAAC,OAAO,iCAAiC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE/F,6FAA6F;AAC7F,MAAM,MAAM,qBAAqB,GAC7B,iBAAiB,GACjB,4BAA4B,GAC5B,6BAA6B,CAAC;AAElC,2FAA2F;AAC3F,eAAO,MAAM,uBAAuB,EAAE,SAAS,qBAAqB,EAI1D,CAAC;AAEX,wDAAwD;AACxD,eAAO,MAAM,sBAAsB,qEAAoC,CAAC;AAExE,yFAAyF;AACzF,eAAO,MAAM,8BAA8B,sGAKjC,CAAC;AAEX,MAAM,MAAM,yBAAyB,GAAG,CAAC,OAAO,8BAA8B,CAAC,CAAC,MAAM,CAAC,CAAC;AAExF,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7C;AAED,qGAAqG;AACrG,eAAO,MAAM,kCAAkC,IAAI,CAAC;AAEpD,gEAAgE;AAChE,eAAO,MAAM,+BAA+B,MAAM,CAAC;AAEnD,2FAA2F;AAC3F,eAAO,MAAM,qCAAqC,MAAM,CAAC;AAIzD,oFAAoF;AACpF,eAAO,MAAM,oBAAoB,aAQ/B,CAAC;AAEH,eAAO,MAAM,uBAAuB,sPAW1B,CAAC;AAEX,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3E,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,gDAAgD;AAChD,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAE/C,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;iBAOhC,CAAC;AAEH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;IACpB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,CAAC;CAC7E;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,qBAAa,0BAA2B,SAAQ,KAAK;IACnD,SAAkB,IAAI,gCAAgC;gBAE1C,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY;CAGpD;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAC3D,QAAQ,CAAC,eAAe,EAAE,sBAAsB,CAAC;CAClD;AAED,MAAM,WAAW,oBAAqB,SAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACnE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,eAAe,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAC3D,QAAQ,CAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7C;AA6BD,kFAAkF;AAClF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,IAAI,CACV,oBAAoB,EAClB,qBAAqB,GACrB,wBAAwB,GACxB,yBAAyB,GACzB,uBAAuB,GACvB,sBAAsB,GACtB,sBAAsB,GACtB,kBAAkB,GAClB,iBAAiB,GACjB,oBAAoB,GACpB,MAAM,CACT,GACA,gBAAgB,CAiBlB;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,MAAM,EAAE,CAErE;AAuFD,+EAA+E;AAC/E,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,CAO7F;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,sBAAsB,GAC9B,SAAS,4BAA4B,EAAE,CA6BzC;AAmBD,mFAAmF;AACnF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,EACzC,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,cAAc,CAAC,EAAE,sBAAsB,CAAA;CAAE,GAC7D;IAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAAC,eAAe,EAAE,SAAS,qBAAqB,EAAE,CAAA;CAAE,CAqBhF;AAED,sFAAsF;AACtF,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B;IACD,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,eAAe,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAClD,eAAe,EAAE,sBAAsB,CAAC;CACzC,CA2BA;AAED,0EAA0E;AAC1E,wBAAgB,4BAA4B,CAC1C,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACpC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAUzB;AAED,qDAAqD;AACrD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,GACxC,GAAG,CAAC,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAUrC;AAED,iEAAiE;AACjE,wBAAgB,uBAAuB,CACrC,cAAc,EAAE,SAAS,oBAAoB,EAAE,EAC/C,cAAc,EAAE,SAAS,oBAAoB,EAAE,GAC9C,qBAAqB,EAAE,CAgBzB;AAED,0DAA0D;AAC1D,wBAAgB,2BAA2B,CACzC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,GACxC,oBAAoB,CAetB;AAUD,8EAA8E;AAC9E,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,SAAS,qBAAqB,EAAE,EACzC,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACrE,eAAe,CA4CjB;AAED,kEAAkE;AAClE,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,IAAI,CAsBlF;AAED,oEAAoE;AACpE,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,eAAe,CAOnE;AAED,6EAA6E;AAC7E,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,GACvB,MAAM,CAYR;AAED,yFAAyF;AACzF,wBAAgB,4BAA4B,IAAI,eAAe,CAS9D;AAQD,uEAAuE;AACvE,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAkBrE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,eAAe,GAAG,IAAI,CAgBhG;AAED,4FAA4F;AAC5F,wBAAgB,sBAAsB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,eAAe,CAS5F;AAED,gFAAgF;AAChF,wBAAgB,8BAA8B,CAC5C,YAAY,EAAE,iBAAiB,EAC/B,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,GAClF,gBAAgB,CAgBlB;AAED,8EAA8E;AAC9E,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,GACvB,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,CAS/C;AAED,uFAAuF;AACvF,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,EACxB,QAAQ,GAAE,MAAsC,GAC/C,wBAAwB,CAmB1B"}
|
|
@@ -11,17 +11,56 @@ import { z } from 'zod';
|
|
|
11
11
|
export const MIN_TRAINING_SAMPLES = 30;
|
|
12
12
|
/** Neutral probability when training data is insufficient or artifact missing. */
|
|
13
13
|
export const NEUTRAL_P_SUCCESS = 0.5;
|
|
14
|
-
/**
|
|
15
|
-
export const
|
|
14
|
+
/** Behavioral outcome signals that mark cheap-tier training failure (SP-062). */
|
|
15
|
+
export const BEHAVIORAL_FAILURE_OUTCOME_SIGNALS = [
|
|
16
16
|
'model_override',
|
|
17
17
|
'feedback_bad',
|
|
18
18
|
];
|
|
19
|
-
/**
|
|
20
|
-
export const
|
|
19
|
+
/** Verifier-grade failure proxies derived from privacy-safe telemetry scalars (SP-131). */
|
|
20
|
+
export const VERIFIER_FAILURE_OUTCOME_SIGNALS = [
|
|
21
|
+
'tool_failure_chain',
|
|
22
|
+
'stop_reason_invalid',
|
|
23
|
+
'reprompt_detected',
|
|
24
|
+
'high_edit_distance',
|
|
25
|
+
];
|
|
26
|
+
/** Execution telemetry failure signals (SP-104 review follow-up, SP-131). */
|
|
27
|
+
export const EXECUTION_FAILURE_OUTCOME_SIGNALS = [
|
|
21
28
|
'provider_failover',
|
|
22
29
|
'stop_reason_length',
|
|
23
30
|
'infra_error',
|
|
24
31
|
];
|
|
32
|
+
/** Outcome signals that mark a routing attempt as unsuccessful for cheap-tier training. */
|
|
33
|
+
export const FAILURE_OUTCOME_SIGNALS = [
|
|
34
|
+
...BEHAVIORAL_FAILURE_OUTCOME_SIGNALS,
|
|
35
|
+
...VERIFIER_FAILURE_OUTCOME_SIGNALS,
|
|
36
|
+
...EXECUTION_FAILURE_OUTCOME_SIGNALS,
|
|
37
|
+
];
|
|
38
|
+
/** @deprecated Use EXECUTION_FAILURE_OUTCOME_SIGNALS */
|
|
39
|
+
export const FUTURE_FAILURE_SIGNALS = EXECUTION_FAILURE_OUTCOME_SIGNALS;
|
|
40
|
+
/** Privacy-safe scalar failure proxies exported in calibration contrib rows (SP-131). */
|
|
41
|
+
export const P_SUCCESS_FAILURE_PROXY_FIELDS = [
|
|
42
|
+
'tool_failure_chain_count',
|
|
43
|
+
'stop_reason_invalid',
|
|
44
|
+
'reprompt_rate',
|
|
45
|
+
'edit_distance_proxy',
|
|
46
|
+
];
|
|
47
|
+
/** Minimum identical tool failures before labeling a tool-failure chain (loop escalation uses 3). */
|
|
48
|
+
export const TOOL_FAILURE_CHAIN_LABEL_THRESHOLD = 2;
|
|
49
|
+
/** Normalized re-prompt rate at or above this marks failure. */
|
|
50
|
+
export const REPROMPT_RATE_FAILURE_THRESHOLD = 0.5;
|
|
51
|
+
/** Normalized edit-distance proxy at or above this marks likely re-prompt / correction. */
|
|
52
|
+
export const EDIT_DISTANCE_PROXY_FAILURE_THRESHOLD = 0.6;
|
|
53
|
+
const PROMPT_LENGTH_NORM_FOR_EDIT_PROXY = 8_000;
|
|
54
|
+
/** Provider stop reasons treated as invalid task completion for training labels. */
|
|
55
|
+
export const INVALID_STOP_REASONS = new Set([
|
|
56
|
+
'length',
|
|
57
|
+
'max_tokens',
|
|
58
|
+
'content_filter',
|
|
59
|
+
'tool_calls',
|
|
60
|
+
'function_call',
|
|
61
|
+
'unknown',
|
|
62
|
+
'error',
|
|
63
|
+
]);
|
|
25
64
|
export const P_SUCCESS_FEATURE_NAMES = [
|
|
26
65
|
'prompt_length_norm',
|
|
27
66
|
'estimated_input_tokens_norm',
|
|
@@ -94,9 +133,118 @@ export function extractPSuccessFeatures(record) {
|
|
|
94
133
|
export function featuresToVector(features) {
|
|
95
134
|
return P_SUCCESS_FEATURE_NAMES.map((name) => features[name]);
|
|
96
135
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
136
|
+
function isTrainingOutcomeSignal(value) {
|
|
137
|
+
return (BEHAVIORAL_FAILURE_OUTCOME_SIGNALS.includes(value) ||
|
|
138
|
+
VERIFIER_FAILURE_OUTCOME_SIGNALS.includes(value) ||
|
|
139
|
+
EXECUTION_FAILURE_OUTCOME_SIGNALS.includes(value) ||
|
|
140
|
+
value === 'compaction_pin_break' ||
|
|
141
|
+
value === 'feedback_good');
|
|
142
|
+
}
|
|
143
|
+
function intOrNull(value) {
|
|
144
|
+
return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : null;
|
|
145
|
+
}
|
|
146
|
+
function boolOrNull(value) {
|
|
147
|
+
return typeof value === 'boolean' ? value : null;
|
|
148
|
+
}
|
|
149
|
+
function rateOrNull(value) {
|
|
150
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
return clamp01(value);
|
|
154
|
+
}
|
|
155
|
+
function normalizeStopReasonInvalid(record) {
|
|
156
|
+
const explicit = boolOrNull(record.stop_reason_invalid);
|
|
157
|
+
if (explicit !== null) {
|
|
158
|
+
return explicit;
|
|
159
|
+
}
|
|
160
|
+
const stopReason = record.stop_reason;
|
|
161
|
+
if (typeof stopReason !== 'string' || stopReason.length === 0) {
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
return INVALID_STOP_REASONS.has(stopReason.toLowerCase());
|
|
165
|
+
}
|
|
166
|
+
function normalizeRepromptRate(record) {
|
|
167
|
+
const explicit = rateOrNull(record.reprompt_rate);
|
|
168
|
+
if (explicit !== null) {
|
|
169
|
+
return explicit;
|
|
170
|
+
}
|
|
171
|
+
const repromptCount = intOrNull(record.reprompt_count);
|
|
172
|
+
if (repromptCount === null) {
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
const turnIndex = intOrNull(record.turn_index_in_session);
|
|
176
|
+
const denominator = turnIndex !== null && turnIndex > 0 ? turnIndex : repromptCount + 1;
|
|
177
|
+
return clamp01(repromptCount / denominator);
|
|
178
|
+
}
|
|
179
|
+
function normalizeEditDistanceProxy(record) {
|
|
180
|
+
const explicit = rateOrNull(record.edit_distance_proxy);
|
|
181
|
+
if (explicit !== null) {
|
|
182
|
+
return explicit;
|
|
183
|
+
}
|
|
184
|
+
const delta = intOrNull(record.prompt_length_delta);
|
|
185
|
+
if (delta !== null) {
|
|
186
|
+
return clamp01(Math.abs(delta) / PROMPT_LENGTH_NORM_FOR_EDIT_PROXY);
|
|
187
|
+
}
|
|
188
|
+
const current = intOrNull(record.prompt_length_chars);
|
|
189
|
+
const prior = intOrNull(record.prior_prompt_length_chars);
|
|
190
|
+
if (current === null || prior === null) {
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
const maxLength = Math.max(current, prior, 1);
|
|
194
|
+
return clamp01(Math.abs(current - prior) / maxLength);
|
|
195
|
+
}
|
|
196
|
+
function normalizeToolFailureChainCount(record) {
|
|
197
|
+
const explicit = intOrNull(record.tool_failure_chain_count);
|
|
198
|
+
if (explicit !== null) {
|
|
199
|
+
return explicit;
|
|
200
|
+
}
|
|
201
|
+
return intOrNull(record.consecutive_tool_failures);
|
|
202
|
+
}
|
|
203
|
+
/** Extract privacy-safe failure proxy scalars from a contrib or export row. */
|
|
204
|
+
export function extractFailureProxies(record) {
|
|
205
|
+
return {
|
|
206
|
+
tool_failure_chain_count: normalizeToolFailureChainCount(record),
|
|
207
|
+
stop_reason_invalid: normalizeStopReasonInvalid(record),
|
|
208
|
+
reprompt_rate: normalizeRepromptRate(record),
|
|
209
|
+
edit_distance_proxy: normalizeEditDistanceProxy(record),
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
/** Map failure proxy scalars to verifier-grade training outcome signals. */
|
|
213
|
+
export function deriveVerifierFailureSignals(proxies) {
|
|
214
|
+
const signals = [];
|
|
215
|
+
if (proxies.tool_failure_chain_count !== null &&
|
|
216
|
+
proxies.tool_failure_chain_count >= TOOL_FAILURE_CHAIN_LABEL_THRESHOLD) {
|
|
217
|
+
signals.push('tool_failure_chain');
|
|
218
|
+
}
|
|
219
|
+
if (proxies.stop_reason_invalid === true) {
|
|
220
|
+
signals.push('stop_reason_invalid');
|
|
221
|
+
}
|
|
222
|
+
if (proxies.reprompt_rate !== null &&
|
|
223
|
+
proxies.reprompt_rate >= REPROMPT_RATE_FAILURE_THRESHOLD) {
|
|
224
|
+
signals.push('reprompt_detected');
|
|
225
|
+
}
|
|
226
|
+
if (proxies.edit_distance_proxy !== null &&
|
|
227
|
+
proxies.edit_distance_proxy >= EDIT_DISTANCE_PROXY_FAILURE_THRESHOLD) {
|
|
228
|
+
signals.push('high_edit_distance');
|
|
229
|
+
}
|
|
230
|
+
return signals;
|
|
231
|
+
}
|
|
232
|
+
function mergeTrainingOutcomeSignals(...groups) {
|
|
233
|
+
return [...new Set(groups.flat())];
|
|
234
|
+
}
|
|
235
|
+
function parseTrainingOutcomeSignals(raw) {
|
|
236
|
+
if (!Array.isArray(raw)) {
|
|
237
|
+
return [];
|
|
238
|
+
}
|
|
239
|
+
return raw.filter((value) => typeof value === 'string' && isTrainingOutcomeSignal(value));
|
|
240
|
+
}
|
|
241
|
+
/** Derive success label from behavioral outcomes plus optional failure proxies. */
|
|
242
|
+
export function deriveSuccessLabel(outcomes, options) {
|
|
243
|
+
const behavioralSignals = [...new Set(outcomes.map((entry) => entry.signal_type))];
|
|
244
|
+
const verifierSignals = options?.failureProxies
|
|
245
|
+
? deriveVerifierFailureSignals(options.failureProxies)
|
|
246
|
+
: [];
|
|
247
|
+
const signals = mergeTrainingOutcomeSignals(behavioralSignals, verifierSignals);
|
|
100
248
|
if (signals.some((signal) => FAILURE_OUTCOME_SIGNALS.includes(signal))) {
|
|
101
249
|
return { success: false, outcome_signals: signals };
|
|
102
250
|
}
|
|
@@ -109,6 +257,41 @@ export function deriveSuccessLabel(outcomes) {
|
|
|
109
257
|
// compaction_pin_break and other neutral signals — treat as unlabeled success.
|
|
110
258
|
return { success: true, outcome_signals: signals };
|
|
111
259
|
}
|
|
260
|
+
/** Derive training labels from a privacy-safe export/contrib row (no prompt text). */
|
|
261
|
+
export function deriveSuccessLabelFromExportRow(record) {
|
|
262
|
+
const failure_proxies = extractFailureProxies(record);
|
|
263
|
+
const existingSignals = parseTrainingOutcomeSignals(record.outcome_signals);
|
|
264
|
+
const verifierSignals = deriveVerifierFailureSignals(failure_proxies);
|
|
265
|
+
const mergedSignals = mergeTrainingOutcomeSignals(existingSignals, verifierSignals);
|
|
266
|
+
if (mergedSignals.some((signal) => FAILURE_OUTCOME_SIGNALS.includes(signal))) {
|
|
267
|
+
return { success: false, outcome_signals: mergedSignals, failure_proxies };
|
|
268
|
+
}
|
|
269
|
+
if (mergedSignals.includes('feedback_good')) {
|
|
270
|
+
return { success: true, outcome_signals: mergedSignals, failure_proxies };
|
|
271
|
+
}
|
|
272
|
+
if (typeof record.success_label === 'boolean') {
|
|
273
|
+
return {
|
|
274
|
+
success: record.success_label,
|
|
275
|
+
outcome_signals: mergedSignals,
|
|
276
|
+
failure_proxies,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
if (mergedSignals.length === 0) {
|
|
280
|
+
return { success: null, outcome_signals: mergedSignals, failure_proxies };
|
|
281
|
+
}
|
|
282
|
+
return { success: true, outcome_signals: mergedSignals, failure_proxies };
|
|
283
|
+
}
|
|
284
|
+
/** Attach normalized failure proxy fields for calibration export rows. */
|
|
285
|
+
export function attachFailureProxiesToExport(exportRecord) {
|
|
286
|
+
const proxies = extractFailureProxies(exportRecord);
|
|
287
|
+
return {
|
|
288
|
+
...exportRecord,
|
|
289
|
+
tool_failure_chain_count: proxies.tool_failure_chain_count,
|
|
290
|
+
stop_reason_invalid: proxies.stop_reason_invalid,
|
|
291
|
+
reprompt_rate: proxies.reprompt_rate,
|
|
292
|
+
edit_distance_proxy: proxies.edit_distance_proxy,
|
|
293
|
+
};
|
|
294
|
+
}
|
|
112
295
|
/** Group outcomes by request_id for export joins. */
|
|
113
296
|
export function indexOutcomesByRequestId(outcomes) {
|
|
114
297
|
const byRequest = new Map();
|
|
@@ -124,23 +307,31 @@ export function joinDatasetWithOutcomes(datasetRecords, outcomeRecords) {
|
|
|
124
307
|
const outcomesByRequest = indexOutcomesByRequestId(outcomeRecords);
|
|
125
308
|
return datasetRecords.map((record) => {
|
|
126
309
|
const linked = outcomesByRequest.get(record.request_id) ?? [];
|
|
127
|
-
const
|
|
310
|
+
const failure_proxies = extractFailureProxies(record);
|
|
311
|
+
const { success, outcome_signals } = deriveSuccessLabel(linked, { failureProxies: failure_proxies });
|
|
128
312
|
return {
|
|
129
313
|
request_id: record.request_id,
|
|
130
314
|
features: extractPSuccessFeatures(record),
|
|
131
315
|
success: success ?? true,
|
|
132
316
|
outcome_signals,
|
|
317
|
+
failure_proxies,
|
|
133
318
|
};
|
|
134
319
|
});
|
|
135
320
|
}
|
|
136
321
|
/** Attach join labels to a privacy-safe export object. */
|
|
137
322
|
export function attachOutcomeLabelsToExport(exportRecord, outcomes) {
|
|
138
|
-
const
|
|
323
|
+
const withProxies = attachFailureProxiesToExport(exportRecord);
|
|
324
|
+
const failure_proxies = extractFailureProxies(withProxies);
|
|
325
|
+
const { success, outcome_signals } = deriveSuccessLabel(outcomes, { failureProxies: failure_proxies });
|
|
139
326
|
return {
|
|
140
|
-
...
|
|
327
|
+
...withProxies,
|
|
141
328
|
request_id: String(exportRecord.request_id ?? ''),
|
|
142
329
|
success_label: success,
|
|
143
330
|
outcome_signals,
|
|
331
|
+
tool_failure_chain_count: failure_proxies.tool_failure_chain_count,
|
|
332
|
+
stop_reason_invalid: failure_proxies.stop_reason_invalid,
|
|
333
|
+
reprompt_rate: failure_proxies.reprompt_rate,
|
|
334
|
+
edit_distance_proxy: failure_proxies.edit_distance_proxy,
|
|
144
335
|
};
|
|
145
336
|
}
|
|
146
337
|
function dot(a, b) {
|
|
@@ -202,21 +393,14 @@ export function parseTrainingExportLine(line) {
|
|
|
202
393
|
if (typeof requestId !== 'string' || requestId.length === 0) {
|
|
203
394
|
return null;
|
|
204
395
|
}
|
|
205
|
-
const
|
|
206
|
-
let success = true;
|
|
207
|
-
if (typeof successLabel === 'boolean') {
|
|
208
|
-
success = successLabel;
|
|
209
|
-
}
|
|
210
|
-
const rawSignals = parsed.outcome_signals;
|
|
211
|
-
const outcome_signals = Array.isArray(rawSignals)
|
|
212
|
-
? rawSignals.filter((value) => typeof value === 'string')
|
|
213
|
-
: [];
|
|
396
|
+
const labeled = deriveSuccessLabelFromExportRow(parsed);
|
|
214
397
|
const record = parsed;
|
|
215
398
|
return {
|
|
216
399
|
request_id: requestId,
|
|
217
400
|
features: extractPSuccessFeatures(record),
|
|
218
|
-
success: success === false ? false : true,
|
|
219
|
-
outcome_signals,
|
|
401
|
+
success: labeled.success === false ? false : true,
|
|
402
|
+
outcome_signals: labeled.outcome_signals,
|
|
403
|
+
failure_proxies: labeled.failure_proxies,
|
|
220
404
|
};
|
|
221
405
|
}
|
|
222
406
|
/** Train weights from privacy-safe labeled JSONL export content. */
|