pi-smart-router 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/.pi/extensions/smart-router/index.ts +9 -0
  2. package/.pi/extensions/smart-router/pi-model-scope.ts +127 -20
  3. package/.pi/extensions/smart-router/planning-delegate.ts +318 -0
  4. package/.pi/extensions/smart-router/route-and-delegate.ts +21 -1
  5. package/.pi/extensions/smart-router/types.ts +3 -0
  6. package/README.md +51 -0
  7. package/config/benchmark-profiles.json +145 -0
  8. package/config/models.yaml.example +5 -0
  9. package/config/routing-calibration.json.example +14 -2
  10. package/dist/config/defaults.d.ts +2 -2
  11. package/dist/config/defaults.d.ts.map +1 -1
  12. package/dist/config/defaults.js +5 -3
  13. package/dist/config/defaults.js.map +1 -1
  14. package/dist/config/pi-model-mapper.d.ts +12 -2
  15. package/dist/config/pi-model-mapper.d.ts.map +1 -1
  16. package/dist/config/pi-model-mapper.js +91 -6
  17. package/dist/config/pi-model-mapper.js.map +1 -1
  18. package/dist/domain/matching/hydra-input.d.ts +6 -5
  19. package/dist/domain/matching/hydra-input.d.ts.map +1 -1
  20. package/dist/domain/matching/hydra-input.js +73 -6
  21. package/dist/domain/matching/hydra-input.js.map +1 -1
  22. package/dist/domain/pipeline/router-pipeline.d.ts +22 -1
  23. package/dist/domain/pipeline/router-pipeline.d.ts.map +1 -1
  24. package/dist/domain/pipeline/router-pipeline.js +135 -22
  25. package/dist/domain/pipeline/router-pipeline.js.map +1 -1
  26. package/dist/domain/routing/isotonic-calibrator.d.ts +56 -0
  27. package/dist/domain/routing/isotonic-calibrator.d.ts.map +1 -0
  28. package/dist/domain/routing/isotonic-calibrator.js +187 -0
  29. package/dist/domain/routing/isotonic-calibrator.js.map +1 -0
  30. package/dist/domain/routing/p-success-classifier.d.ts +53 -7
  31. package/dist/domain/routing/p-success-classifier.d.ts.map +1 -1
  32. package/dist/domain/routing/p-success-classifier.js +205 -21
  33. package/dist/domain/routing/p-success-classifier.js.map +1 -1
  34. package/dist/domain/types/entities.d.ts +54 -0
  35. package/dist/domain/types/entities.d.ts.map +1 -1
  36. package/dist/domain/types/index.d.ts +1 -1
  37. package/dist/domain/types/index.d.ts.map +1 -1
  38. package/dist/domain/types/schemas.d.ts +29 -0
  39. package/dist/domain/types/schemas.d.ts.map +1 -1
  40. package/dist/domain/types/schemas.js +57 -0
  41. package/dist/domain/types/schemas.js.map +1 -1
  42. package/dist/infrastructure/persistence/sqlite-store.d.ts.map +1 -1
  43. package/dist/infrastructure/persistence/sqlite-store.js +2 -1
  44. package/dist/infrastructure/persistence/sqlite-store.js.map +1 -1
  45. package/dist/infrastructure/telemetry/routing-telemetry.d.ts +20 -1
  46. package/dist/infrastructure/telemetry/routing-telemetry.d.ts.map +1 -1
  47. package/dist/infrastructure/telemetry/routing-telemetry.js +82 -2
  48. package/dist/infrastructure/telemetry/routing-telemetry.js.map +1 -1
  49. package/package.json +6 -3
  50. package/specs/001-build-smart-router/contracts/telemetry-contrib.schema.json +29 -1
  51. package/src/config/defaults.ts +6 -2
  52. package/src/config/pi-model-mapper.ts +110 -6
  53. package/src/domain/matching/hydra-input.ts +86 -7
  54. package/src/domain/pipeline/router-pipeline.ts +195 -29
  55. package/src/domain/routing/isotonic-calibrator.ts +255 -0
  56. package/src/domain/routing/p-success-classifier.ts +299 -26
  57. package/src/domain/types/entities.ts +58 -0
  58. package/src/domain/types/index.ts +4 -0
  59. package/src/domain/types/schemas.ts +73 -0
  60. package/src/infrastructure/persistence/sqlite-store.ts +2 -0
  61. package/src/infrastructure/telemetry/routing-telemetry.ts +127 -7
@@ -0,0 +1,187 @@
1
+ /**
2
+ * Online isotonic P(success) calibrator for low_intensity gate (SP-133).
3
+ *
4
+ * Loads monotonic knot tables from routing-calibration bundle at serve time.
5
+ * O(log n) binary search lookup with <5ms budget; falls back to raw logistic
6
+ * when the artifact is missing or under-trained.
7
+ */
8
+ import { existsSync, readFileSync } from 'node:fs';
9
+ import { resolve } from 'node:path';
10
+ import { z } from 'zod';
11
+ import { MIN_TRAINING_SAMPLES } from './p-success-classifier.js';
12
+ export const ISOTONIC_CALIBRATOR_ARTIFACT_VERSION = 1;
13
+ export const ISOTONIC_LOOKUP_BUDGET_MS = 5;
14
+ export const DEFAULT_ROUTING_CALIBRATION_PATH = resolve('config', 'routing-calibration.json');
15
+ export class IsotonicCalibratorLoaderError extends Error {
16
+ name = 'IsotonicCalibratorLoaderError';
17
+ constructor(message, options) {
18
+ super(message, options);
19
+ }
20
+ }
21
+ const IsotonicCalibratorArtifactSchema = z.object({
22
+ version: z.literal(ISOTONIC_CALIBRATOR_ARTIFACT_VERSION),
23
+ min_training_samples: z.number().int().min(0),
24
+ x_knots: z.array(z.number().finite().min(0).max(1)).min(2),
25
+ y_knots: z.array(z.number().finite().min(0).max(1)).min(2),
26
+ trained_sample_count: z.number().int().min(0),
27
+ holdout_ece_raw: z.number().finite().min(0).max(1).nullable(),
28
+ holdout_ece_calibrated: z.number().finite().min(0).max(1).nullable(),
29
+ });
30
+ function clamp01(value) {
31
+ if (value <= 0)
32
+ return 0;
33
+ if (value >= 1)
34
+ return 1;
35
+ return value;
36
+ }
37
+ function formatZodIssues(error) {
38
+ return error.issues
39
+ .map((issue) => ` - ${issue.path.join('.')}: ${issue.message}`)
40
+ .join('\n');
41
+ }
42
+ /** True when artifact has enough labeled samples for serve-time calibration. */
43
+ export function isIsotonicCalibratorTrained(artifact) {
44
+ return artifact.trained_sample_count >= artifact.min_training_samples;
45
+ }
46
+ export function createDefaultIsotonicCalibratorArtifact() {
47
+ return {
48
+ version: ISOTONIC_CALIBRATOR_ARTIFACT_VERSION,
49
+ min_training_samples: MIN_TRAINING_SAMPLES,
50
+ x_knots: [0, 1],
51
+ y_knots: [0, 1],
52
+ trained_sample_count: 0,
53
+ holdout_ece_raw: null,
54
+ holdout_ece_calibrated: null,
55
+ };
56
+ }
57
+ /** Piecewise-constant isotonic lookup with O(log n) binary search on knots. */
58
+ export function applyIsotonicLookup(rawScore, xKnots, yKnots) {
59
+ if (xKnots.length === 0 || yKnots.length === 0 || xKnots.length !== yKnots.length) {
60
+ return clamp01(rawScore);
61
+ }
62
+ const score = clamp01(rawScore);
63
+ if (score <= xKnots[0]) {
64
+ return clamp01(yKnots[0]);
65
+ }
66
+ const lastIndex = xKnots.length - 1;
67
+ if (score >= xKnots[lastIndex]) {
68
+ return clamp01(yKnots[lastIndex]);
69
+ }
70
+ let low = 0;
71
+ let high = lastIndex;
72
+ while (low < high) {
73
+ const mid = Math.floor((low + high + 1) / 2);
74
+ if (xKnots[mid] <= score) {
75
+ low = mid;
76
+ }
77
+ else {
78
+ high = mid - 1;
79
+ }
80
+ }
81
+ return clamp01(yKnots[low]);
82
+ }
83
+ /**
84
+ * Apply isotonic calibration to a raw logistic P(success) score.
85
+ * Returns the raw score when the artifact is missing or under-trained.
86
+ */
87
+ export function applyIsotonicCalibrator(rawScore, artifact) {
88
+ if (artifact === null || !isIsotonicCalibratorTrained(artifact)) {
89
+ return clamp01(rawScore);
90
+ }
91
+ return applyIsotonicLookup(rawScore, artifact.x_knots, artifact.y_knots);
92
+ }
93
+ /** Apply isotonic calibration with elapsed timing guard for the online routing budget. */
94
+ export function applyIsotonicCalibratorTimed(rawScore, artifact, budgetMs = ISOTONIC_LOOKUP_BUDGET_MS) {
95
+ const start = performance.now();
96
+ const calibration_applied = artifact !== null && isIsotonicCalibratorTrained(artifact);
97
+ const calibrated = applyIsotonicCalibrator(rawScore, artifact);
98
+ const elapsed_ms = performance.now() - start;
99
+ if (elapsed_ms > budgetMs) {
100
+ console.warn('Isotonic P(success) lookup exceeded latency budget', {
101
+ elapsed_ms,
102
+ budget_ms: budgetMs,
103
+ knot_count: artifact?.x_knots.length ?? 0,
104
+ });
105
+ }
106
+ return {
107
+ calibrated,
108
+ elapsed_ms,
109
+ within_budget: elapsed_ms <= budgetMs,
110
+ calibration_applied,
111
+ };
112
+ }
113
+ /** Parse and validate an isotonic calibrator artifact from JSON text. */
114
+ export function parseIsotonicCalibratorJson(raw) {
115
+ let parsed;
116
+ try {
117
+ parsed = JSON.parse(raw);
118
+ }
119
+ catch (err) {
120
+ const message = err instanceof Error ? err.message : String(err);
121
+ throw new IsotonicCalibratorLoaderError(`Failed to parse JSON: ${message}`, { cause: err });
122
+ }
123
+ const result = IsotonicCalibratorArtifactSchema.safeParse(parsed);
124
+ if (!result.success) {
125
+ throw new IsotonicCalibratorLoaderError(`Invalid isotonic calibrator artifact:\n${formatZodIssues(result.error)}`, { cause: result.error });
126
+ }
127
+ return result.data;
128
+ }
129
+ /** Extract isotonic_calibrator from a routing-calibration bundle object. */
130
+ export function parseIsotonicCalibratorFromBundle(parsed) {
131
+ if (typeof parsed !== 'object' || parsed === null) {
132
+ throw new IsotonicCalibratorLoaderError('Routing calibration bundle must be a JSON object');
133
+ }
134
+ const isotonic = parsed.isotonic_calibrator;
135
+ if (isotonic === undefined) {
136
+ throw new IsotonicCalibratorLoaderError('Routing calibration bundle missing isotonic_calibrator');
137
+ }
138
+ const result = IsotonicCalibratorArtifactSchema.safeParse(isotonic);
139
+ if (!result.success) {
140
+ throw new IsotonicCalibratorLoaderError(`Invalid isotonic_calibrator in routing calibration bundle:\n${formatZodIssues(result.error)}`, { cause: result.error });
141
+ }
142
+ return result.data;
143
+ }
144
+ /**
145
+ * Load isotonic calibrator from routing-calibration bundle on disk.
146
+ * Returns null when the bundle file is missing.
147
+ */
148
+ export function loadIsotonicCalibrator(options) {
149
+ const filePath = options?.filePath ?? DEFAULT_ROUTING_CALIBRATION_PATH;
150
+ if (!existsSync(filePath)) {
151
+ return null;
152
+ }
153
+ let raw;
154
+ try {
155
+ raw = readFileSync(filePath, 'utf8');
156
+ }
157
+ catch (err) {
158
+ const message = err instanceof Error ? err.message : String(err);
159
+ throw new IsotonicCalibratorLoaderError(`Failed to read routing calibration file: ${message}`, {
160
+ cause: err,
161
+ });
162
+ }
163
+ let parsed;
164
+ try {
165
+ parsed = JSON.parse(raw);
166
+ }
167
+ catch (err) {
168
+ const message = err instanceof Error ? err.message : String(err);
169
+ throw new IsotonicCalibratorLoaderError(`Failed to parse routing calibration JSON: ${message}`, {
170
+ cause: err,
171
+ });
172
+ }
173
+ return parseIsotonicCalibratorFromBundle(parsed);
174
+ }
175
+ /** Resolve calibrator for online inference — missing or invalid artifacts fall back safely. */
176
+ export function resolveIsotonicCalibrator(options) {
177
+ try {
178
+ return loadIsotonicCalibrator(options);
179
+ }
180
+ catch (err) {
181
+ console.warn('Isotonic calibrator artifact invalid; using raw logistic fallback', {
182
+ error: err instanceof Error ? err.message : String(err),
183
+ });
184
+ return null;
185
+ }
186
+ }
187
+ //# sourceMappingURL=isotonic-calibrator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"isotonic-calibrator.js","sourceRoot":"","sources":["../../../src/domain/routing/isotonic-calibrator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEjE,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAU,CAAC;AAC/D,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAC3C,MAAM,CAAC,MAAM,gCAAgC,GAAG,OAAO,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAC;AAuB9F,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IACpC,IAAI,GAAG,+BAA+B,CAAC;IAEzD,YAAY,OAAe,EAAE,OAAsB;QACjD,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,gCAAgC,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,oCAAoC,CAAC;IACxD,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7D,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACrE,CAAC,CAAC;AAEH,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,KAA+E;IACtG,OAAO,KAAK,CAAC,MAAM;SAChB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;SAC/D,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,2BAA2B,CAAC,QAAoC;IAC9E,OAAO,QAAQ,CAAC,oBAAoB,IAAI,QAAQ,CAAC,oBAAoB,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,uCAAuC;IACrD,OAAO;QACL,OAAO,EAAE,oCAAoC;QAC7C,oBAAoB,EAAE,oBAAoB;QAC1C,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QACf,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QACf,oBAAoB,EAAE,CAAC;QACvB,eAAe,EAAE,IAAI;QACrB,sBAAsB,EAAE,IAAI;KAC7B,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,mBAAmB,CACjC,QAAgB,EAChB,MAAyB,EACzB,MAAyB;IAEzB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QAClF,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChC,IAAI,KAAK,IAAI,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IACpC,IAAI,KAAK,IAAI,MAAM,CAAC,SAAS,CAAE,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,MAAM,CAAC,SAAS,CAAE,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,SAAS,CAAC;IACrB,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,MAAM,CAAC,GAAG,CAAE,IAAI,KAAK,EAAE,CAAC;YAC1B,GAAG,GAAG,GAAG,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACrC,QAAgB,EAChB,QAA2C;IAE3C,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,2BAA2B,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChE,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;AAC3E,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,4BAA4B,CAC1C,QAAgB,EAChB,QAA2C,EAC3C,WAAmB,yBAAyB;IAE5C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,mBAAmB,GAAG,QAAQ,KAAK,IAAI,IAAI,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IACvF,MAAM,UAAU,GAAG,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/D,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IAE7C,IAAI,UAAU,GAAG,QAAQ,EAAE,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,oDAAoD,EAAE;YACjE,UAAU;YACV,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC;SAC1C,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,UAAU;QACV,UAAU;QACV,aAAa,EAAE,UAAU,IAAI,QAAQ;QACrC,mBAAmB;KACpB,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,2BAA2B,CAAC,GAAW;IACrD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,6BAA6B,CAAC,yBAAyB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,6BAA6B,CACrC,0CAA0C,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACzE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CACxB,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,iCAAiC,CAAC,MAAe;IAC/D,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,IAAI,6BAA6B,CAAC,kDAAkD,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,QAAQ,GAAI,MAAkC,CAAC,mBAAmB,CAAC;IACzE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,6BAA6B,CAAC,wDAAwD,CAAC,CAAC;IACpG,CAAC;IAED,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACpE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,6BAA6B,CACrC,+DAA+D,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC9F,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CACxB,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,OAAuC;IAEvC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,gCAAgC,CAAC;IAEvE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,6BAA6B,CAAC,4CAA4C,OAAO,EAAE,EAAE;YAC7F,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,6BAA6B,CAAC,6CAA6C,OAAO,EAAE,EAAE;YAC9F,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC;IAED,OAAO,iCAAiC,CAAC,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,yBAAyB,CACvC,OAAuC;IAEvC,IAAI,CAAC;QACH,OAAO,sBAAsB,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,mEAAmE,EAAE;YAChF,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
@@ -11,10 +11,37 @@ import type { OutcomeSignalType, RoutingDatasetRecord, RoutingOutcomeRecord } fr
11
11
  export declare const MIN_TRAINING_SAMPLES = 30;
12
12
  /** Neutral probability when training data is insufficient or artifact missing. */
13
13
  export declare const NEUTRAL_P_SUCCESS = 0.5;
14
+ /** Behavioral outcome signals that mark cheap-tier training failure (SP-062). */
15
+ export declare const BEHAVIORAL_FAILURE_OUTCOME_SIGNALS: readonly OutcomeSignalType[];
16
+ /** Verifier-grade failure proxies derived from privacy-safe telemetry scalars (SP-131). */
17
+ export declare const VERIFIER_FAILURE_OUTCOME_SIGNALS: readonly ["tool_failure_chain", "stop_reason_invalid", "reprompt_detected", "high_edit_distance"];
18
+ export type VerifierFailureOutcomeSignal = (typeof VERIFIER_FAILURE_OUTCOME_SIGNALS)[number];
19
+ /** Execution telemetry failure signals (SP-104 review follow-up, SP-131). */
20
+ export declare const EXECUTION_FAILURE_OUTCOME_SIGNALS: readonly ["provider_failover", "stop_reason_length", "infra_error"];
21
+ export type ExecutionFailureOutcomeSignal = (typeof EXECUTION_FAILURE_OUTCOME_SIGNALS)[number];
22
+ /** Union of all training label signals consumed by P(success) export and aggregate paths. */
23
+ export type TrainingOutcomeSignal = OutcomeSignalType | VerifierFailureOutcomeSignal | ExecutionFailureOutcomeSignal;
14
24
  /** Outcome signals that mark a routing attempt as unsuccessful for cheap-tier training. */
15
- export declare const FAILURE_OUTCOME_SIGNALS: readonly OutcomeSignalType[];
16
- /** Outcome signals reserved for future execution telemetry (failover, length, infra). */
25
+ export declare const FAILURE_OUTCOME_SIGNALS: readonly TrainingOutcomeSignal[];
26
+ /** @deprecated Use EXECUTION_FAILURE_OUTCOME_SIGNALS */
17
27
  export declare const FUTURE_FAILURE_SIGNALS: readonly ["provider_failover", "stop_reason_length", "infra_error"];
28
+ /** Privacy-safe scalar failure proxies exported in calibration contrib rows (SP-131). */
29
+ export declare const P_SUCCESS_FAILURE_PROXY_FIELDS: readonly ["tool_failure_chain_count", "stop_reason_invalid", "reprompt_rate", "edit_distance_proxy"];
30
+ export type PSuccessFailureProxyField = (typeof P_SUCCESS_FAILURE_PROXY_FIELDS)[number];
31
+ export interface PSuccessFailureProxies {
32
+ readonly tool_failure_chain_count: number | null;
33
+ readonly stop_reason_invalid: boolean | null;
34
+ readonly reprompt_rate: number | null;
35
+ readonly edit_distance_proxy: number | null;
36
+ }
37
+ /** Minimum identical tool failures before labeling a tool-failure chain (loop escalation uses 3). */
38
+ export declare const TOOL_FAILURE_CHAIN_LABEL_THRESHOLD = 2;
39
+ /** Normalized re-prompt rate at or above this marks failure. */
40
+ export declare const REPROMPT_RATE_FAILURE_THRESHOLD = 0.5;
41
+ /** Normalized edit-distance proxy at or above this marks likely re-prompt / correction. */
42
+ export declare const EDIT_DISTANCE_PROXY_FAILURE_THRESHOLD = 0.6;
43
+ /** Provider stop reasons treated as invalid task completion for training labels. */
44
+ export declare const INVALID_STOP_REASONS: Set<string>;
18
45
  export declare const P_SUCCESS_FEATURE_NAMES: readonly ["prompt_length_norm", "estimated_input_tokens_norm", "triage_cyclomatic_score", "requirement_reasoning", "requirement_code_gen", "requirement_tool_use", "has_tool_context", "compaction_flag", "routing_latency_norm", "economical_tier"];
19
46
  export type PSuccessFeatureName = (typeof P_SUCCESS_FEATURE_NAMES)[number];
20
47
  export interface PSuccessFeatures {
@@ -75,21 +102,40 @@ export interface LabeledTrainingSample {
75
102
  readonly request_id: string;
76
103
  readonly features: PSuccessFeatures;
77
104
  readonly success: boolean;
78
- readonly outcome_signals: readonly OutcomeSignalType[];
105
+ readonly outcome_signals: readonly TrainingOutcomeSignal[];
106
+ readonly failure_proxies: PSuccessFailureProxies;
79
107
  }
80
108
  export interface DatasetExportJoinRow extends Record<string, unknown> {
81
109
  readonly request_id: string;
82
110
  readonly success_label: boolean | null;
83
- readonly outcome_signals: readonly OutcomeSignalType[];
111
+ readonly outcome_signals: readonly TrainingOutcomeSignal[];
112
+ readonly tool_failure_chain_count: number | null;
113
+ readonly stop_reason_invalid: boolean | null;
114
+ readonly reprompt_rate: number | null;
115
+ readonly edit_distance_proxy: number | null;
84
116
  }
85
117
  /** Map a dataset row to a normalized feature vector for training or inference. */
86
118
  export declare function extractPSuccessFeatures(record: Pick<RoutingDatasetRecord, 'prompt_length_chars' | 'estimated_input_tokens' | 'triage_cyclomatic_score' | 'requirement_reasoning' | 'requirement_code_gen' | 'requirement_tool_use' | 'has_tool_context' | 'compaction_flag' | 'routing_latency_ms' | 'tier'>): PSuccessFeatures;
87
119
  export declare function featuresToVector(features: PSuccessFeatures): number[];
88
- /** Derive success label from outcome signals keyed to a request. */
89
- export declare function deriveSuccessLabel(outcomes: readonly RoutingOutcomeRecord[]): {
120
+ /** Extract privacy-safe failure proxy scalars from a contrib or export row. */
121
+ export declare function extractFailureProxies(record: Record<string, unknown>): PSuccessFailureProxies;
122
+ /** Map failure proxy scalars to verifier-grade training outcome signals. */
123
+ export declare function deriveVerifierFailureSignals(proxies: PSuccessFailureProxies): readonly VerifierFailureOutcomeSignal[];
124
+ /** Derive success label from behavioral outcomes plus optional failure proxies. */
125
+ export declare function deriveSuccessLabel(outcomes: readonly RoutingOutcomeRecord[], options?: {
126
+ readonly failureProxies?: PSuccessFailureProxies;
127
+ }): {
128
+ success: boolean | null;
129
+ outcome_signals: readonly TrainingOutcomeSignal[];
130
+ };
131
+ /** Derive training labels from a privacy-safe export/contrib row (no prompt text). */
132
+ export declare function deriveSuccessLabelFromExportRow(record: Record<string, unknown>): {
90
133
  success: boolean | null;
91
- outcome_signals: readonly OutcomeSignalType[];
134
+ outcome_signals: readonly TrainingOutcomeSignal[];
135
+ failure_proxies: PSuccessFailureProxies;
92
136
  };
137
+ /** Attach normalized failure proxy fields for calibration export rows. */
138
+ export declare function attachFailureProxiesToExport(exportRecord: Record<string, unknown>): Record<string, unknown>;
93
139
  /** Group outcomes by request_id for export joins. */
94
140
  export declare function indexOutcomesByRequestId(outcomes: readonly RoutingOutcomeRecord[]): Map<string, RoutingOutcomeRecord[]>;
95
141
  /** Join dataset rows with outcome labels for training export. */
@@ -1 +1 @@
1
- {"version":3,"file":"p-success-classifier.d.ts","sourceRoot":"","sources":["../../../src/domain/routing/p-success-classifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,gFAAgF;AAChF,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,kFAAkF;AAClF,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC,2FAA2F;AAC3F,eAAO,MAAM,uBAAuB,EAAE,SAAS,iBAAiB,EAGtD,CAAC;AAEX,yFAAyF;AACzF,eAAO,MAAM,sBAAsB,qEAIzB,CAAC;AAEX,eAAO,MAAM,uBAAuB,sPAW1B,CAAC;AAEX,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3E,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,gDAAgD;AAChD,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAE/C,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;iBAOhC,CAAC;AAEH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;IACpB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,CAAC;CAC7E;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,qBAAa,0BAA2B,SAAQ,KAAK;IACnD,SAAkB,IAAI,gCAAgC;gBAE1C,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY;CAGpD;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,SAAS,iBAAiB,EAAE,CAAC;CACxD;AAED,MAAM,WAAW,oBAAqB,SAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACnE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,eAAe,EAAE,SAAS,iBAAiB,EAAE,CAAC;CACxD;AA6BD,kFAAkF;AAClF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,IAAI,CACV,oBAAoB,EAClB,qBAAqB,GACrB,wBAAwB,GACxB,yBAAyB,GACzB,uBAAuB,GACvB,sBAAsB,GACtB,sBAAsB,GACtB,kBAAkB,GAClB,iBAAiB,GACjB,oBAAoB,GACpB,MAAM,CACT,GACA,gBAAgB,CAiBlB;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,MAAM,EAAE,CAErE;AAED,oEAAoE;AACpE,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,GACxC;IAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAAC,eAAe,EAAE,SAAS,iBAAiB,EAAE,CAAA;CAAE,CAiB5E;AAED,qDAAqD;AACrD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,GACxC,GAAG,CAAC,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAUrC;AAED,iEAAiE;AACjE,wBAAgB,uBAAuB,CACrC,cAAc,EAAE,SAAS,oBAAoB,EAAE,EAC/C,cAAc,EAAE,SAAS,oBAAoB,EAAE,GAC9C,qBAAqB,EAAE,CAczB;AAED,0DAA0D;AAC1D,wBAAgB,2BAA2B,CACzC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,GACxC,oBAAoB,CAStB;AAUD,8EAA8E;AAC9E,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,SAAS,qBAAqB,EAAE,EACzC,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACrE,eAAe,CA4CjB;AAED,kEAAkE;AAClE,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,IAAI,CA+BlF;AAED,oEAAoE;AACpE,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,eAAe,CAOnE;AAED,6EAA6E;AAC7E,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,GACvB,MAAM,CAYR;AAED,yFAAyF;AACzF,wBAAgB,4BAA4B,IAAI,eAAe,CAS9D;AAQD,uEAAuE;AACvE,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAkBrE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,eAAe,GAAG,IAAI,CAgBhG;AAED,4FAA4F;AAC5F,wBAAgB,sBAAsB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,eAAe,CAS5F;AAED,gFAAgF;AAChF,wBAAgB,8BAA8B,CAC5C,YAAY,EAAE,iBAAiB,EAC/B,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,GAClF,gBAAgB,CAgBlB;AAED,8EAA8E;AAC9E,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,GACvB,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,CAS/C;AAED,uFAAuF;AACvF,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,EACxB,QAAQ,GAAE,MAAsC,GAC/C,wBAAwB,CAmB1B"}
1
+ {"version":3,"file":"p-success-classifier.d.ts","sourceRoot":"","sources":["../../../src/domain/routing/p-success-classifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,KAAK,EACV,iBAAiB,EACjB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAE3B,gFAAgF;AAChF,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,kFAAkF;AAClF,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC,iFAAiF;AACjF,eAAO,MAAM,kCAAkC,EAAE,SAAS,iBAAiB,EAGjE,CAAC;AAEX,2FAA2F;AAC3F,eAAO,MAAM,gCAAgC,mGAKnC,CAAC;AAEX,MAAM,MAAM,4BAA4B,GAAG,CAAC,OAAO,gCAAgC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7F,6EAA6E;AAC7E,eAAO,MAAM,iCAAiC,qEAIpC,CAAC;AAEX,MAAM,MAAM,6BAA6B,GAAG,CAAC,OAAO,iCAAiC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE/F,6FAA6F;AAC7F,MAAM,MAAM,qBAAqB,GAC7B,iBAAiB,GACjB,4BAA4B,GAC5B,6BAA6B,CAAC;AAElC,2FAA2F;AAC3F,eAAO,MAAM,uBAAuB,EAAE,SAAS,qBAAqB,EAI1D,CAAC;AAEX,wDAAwD;AACxD,eAAO,MAAM,sBAAsB,qEAAoC,CAAC;AAExE,yFAAyF;AACzF,eAAO,MAAM,8BAA8B,sGAKjC,CAAC;AAEX,MAAM,MAAM,yBAAyB,GAAG,CAAC,OAAO,8BAA8B,CAAC,CAAC,MAAM,CAAC,CAAC;AAExF,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7C;AAED,qGAAqG;AACrG,eAAO,MAAM,kCAAkC,IAAI,CAAC;AAEpD,gEAAgE;AAChE,eAAO,MAAM,+BAA+B,MAAM,CAAC;AAEnD,2FAA2F;AAC3F,eAAO,MAAM,qCAAqC,MAAM,CAAC;AAIzD,oFAAoF;AACpF,eAAO,MAAM,oBAAoB,aAQ/B,CAAC;AAEH,eAAO,MAAM,uBAAuB,sPAW1B,CAAC;AAEX,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3E,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;IAC7C,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,gDAAgD;AAChD,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAE/C,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;iBAOhC,CAAC;AAEH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;IACpB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,CAAC;CAC7E;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,qBAAa,0BAA2B,SAAQ,KAAK;IACnD,SAAkB,IAAI,gCAAgC;gBAE1C,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY;CAGpD;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAC3D,QAAQ,CAAC,eAAe,EAAE,sBAAsB,CAAC;CAClD;AAED,MAAM,WAAW,oBAAqB,SAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACnE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,OAAO,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,eAAe,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAC3D,QAAQ,CAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7C;AA6BD,kFAAkF;AAClF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,IAAI,CACV,oBAAoB,EAClB,qBAAqB,GACrB,wBAAwB,GACxB,yBAAyB,GACzB,uBAAuB,GACvB,sBAAsB,GACtB,sBAAsB,GACtB,kBAAkB,GAClB,iBAAiB,GACjB,oBAAoB,GACpB,MAAM,CACT,GACA,gBAAgB,CAiBlB;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,MAAM,EAAE,CAErE;AAuFD,+EAA+E;AAC/E,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,sBAAsB,CAO7F;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,sBAAsB,GAC9B,SAAS,4BAA4B,EAAE,CA6BzC;AAmBD,mFAAmF;AACnF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,EACzC,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,cAAc,CAAC,EAAE,sBAAsB,CAAA;CAAE,GAC7D;IAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAAC,eAAe,EAAE,SAAS,qBAAqB,EAAE,CAAA;CAAE,CAqBhF;AAED,sFAAsF;AACtF,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B;IACD,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,eAAe,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAClD,eAAe,EAAE,sBAAsB,CAAC;CACzC,CA2BA;AAED,0EAA0E;AAC1E,wBAAgB,4BAA4B,CAC1C,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACpC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAUzB;AAED,qDAAqD;AACrD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,GACxC,GAAG,CAAC,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAUrC;AAED,iEAAiE;AACjE,wBAAgB,uBAAuB,CACrC,cAAc,EAAE,SAAS,oBAAoB,EAAE,EAC/C,cAAc,EAAE,SAAS,oBAAoB,EAAE,GAC9C,qBAAqB,EAAE,CAgBzB;AAED,0DAA0D;AAC1D,wBAAgB,2BAA2B,CACzC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,QAAQ,EAAE,SAAS,oBAAoB,EAAE,GACxC,oBAAoB,CAetB;AAUD,8EAA8E;AAC9E,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,SAAS,qBAAqB,EAAE,EACzC,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACrE,eAAe,CA4CjB;AAED,kEAAkE;AAClE,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,IAAI,CAsBlF;AAED,oEAAoE;AACpE,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,eAAe,CAOnE;AAED,6EAA6E;AAC7E,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,GACvB,MAAM,CAYR;AAED,yFAAyF;AACzF,wBAAgB,4BAA4B,IAAI,eAAe,CAS9D;AAQD,uEAAuE;AACvE,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAkBrE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,eAAe,GAAG,IAAI,CAgBhG;AAED,4FAA4F;AAC5F,wBAAgB,sBAAsB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,eAAe,CAS5F;AAED,gFAAgF;AAChF,wBAAgB,8BAA8B,CAC5C,YAAY,EAAE,iBAAiB,EAC/B,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,GAClF,gBAAgB,CAgBlB;AAED,8EAA8E;AAC9E,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,GACvB,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,CAS/C;AAED,uFAAuF;AACvF,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,eAAe,EACxB,QAAQ,GAAE,MAAsC,GAC/C,wBAAwB,CAmB1B"}
@@ -11,17 +11,56 @@ import { z } from 'zod';
11
11
  export const MIN_TRAINING_SAMPLES = 30;
12
12
  /** Neutral probability when training data is insufficient or artifact missing. */
13
13
  export const NEUTRAL_P_SUCCESS = 0.5;
14
- /** Outcome signals that mark a routing attempt as unsuccessful for cheap-tier training. */
15
- export const FAILURE_OUTCOME_SIGNALS = [
14
+ /** Behavioral outcome signals that mark cheap-tier training failure (SP-062). */
15
+ export const BEHAVIORAL_FAILURE_OUTCOME_SIGNALS = [
16
16
  'model_override',
17
17
  'feedback_bad',
18
18
  ];
19
- /** Outcome signals reserved for future execution telemetry (failover, length, infra). */
20
- export const FUTURE_FAILURE_SIGNALS = [
19
+ /** Verifier-grade failure proxies derived from privacy-safe telemetry scalars (SP-131). */
20
+ export const VERIFIER_FAILURE_OUTCOME_SIGNALS = [
21
+ 'tool_failure_chain',
22
+ 'stop_reason_invalid',
23
+ 'reprompt_detected',
24
+ 'high_edit_distance',
25
+ ];
26
+ /** Execution telemetry failure signals (SP-104 review follow-up, SP-131). */
27
+ export const EXECUTION_FAILURE_OUTCOME_SIGNALS = [
21
28
  'provider_failover',
22
29
  'stop_reason_length',
23
30
  'infra_error',
24
31
  ];
32
+ /** Outcome signals that mark a routing attempt as unsuccessful for cheap-tier training. */
33
+ export const FAILURE_OUTCOME_SIGNALS = [
34
+ ...BEHAVIORAL_FAILURE_OUTCOME_SIGNALS,
35
+ ...VERIFIER_FAILURE_OUTCOME_SIGNALS,
36
+ ...EXECUTION_FAILURE_OUTCOME_SIGNALS,
37
+ ];
38
+ /** @deprecated Use EXECUTION_FAILURE_OUTCOME_SIGNALS */
39
+ export const FUTURE_FAILURE_SIGNALS = EXECUTION_FAILURE_OUTCOME_SIGNALS;
40
+ /** Privacy-safe scalar failure proxies exported in calibration contrib rows (SP-131). */
41
+ export const P_SUCCESS_FAILURE_PROXY_FIELDS = [
42
+ 'tool_failure_chain_count',
43
+ 'stop_reason_invalid',
44
+ 'reprompt_rate',
45
+ 'edit_distance_proxy',
46
+ ];
47
+ /** Minimum identical tool failures before labeling a tool-failure chain (loop escalation uses 3). */
48
+ export const TOOL_FAILURE_CHAIN_LABEL_THRESHOLD = 2;
49
+ /** Normalized re-prompt rate at or above this marks failure. */
50
+ export const REPROMPT_RATE_FAILURE_THRESHOLD = 0.5;
51
+ /** Normalized edit-distance proxy at or above this marks likely re-prompt / correction. */
52
+ export const EDIT_DISTANCE_PROXY_FAILURE_THRESHOLD = 0.6;
53
+ const PROMPT_LENGTH_NORM_FOR_EDIT_PROXY = 8_000;
54
+ /** Provider stop reasons treated as invalid task completion for training labels. */
55
+ export const INVALID_STOP_REASONS = new Set([
56
+ 'length',
57
+ 'max_tokens',
58
+ 'content_filter',
59
+ 'tool_calls',
60
+ 'function_call',
61
+ 'unknown',
62
+ 'error',
63
+ ]);
25
64
  export const P_SUCCESS_FEATURE_NAMES = [
26
65
  'prompt_length_norm',
27
66
  'estimated_input_tokens_norm',
@@ -94,9 +133,118 @@ export function extractPSuccessFeatures(record) {
94
133
  export function featuresToVector(features) {
95
134
  return P_SUCCESS_FEATURE_NAMES.map((name) => features[name]);
96
135
  }
97
- /** Derive success label from outcome signals keyed to a request. */
98
- export function deriveSuccessLabel(outcomes) {
99
- const signals = [...new Set(outcomes.map((entry) => entry.signal_type))];
136
+ function isTrainingOutcomeSignal(value) {
137
+ return (BEHAVIORAL_FAILURE_OUTCOME_SIGNALS.includes(value) ||
138
+ VERIFIER_FAILURE_OUTCOME_SIGNALS.includes(value) ||
139
+ EXECUTION_FAILURE_OUTCOME_SIGNALS.includes(value) ||
140
+ value === 'compaction_pin_break' ||
141
+ value === 'feedback_good');
142
+ }
143
+ function intOrNull(value) {
144
+ return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : null;
145
+ }
146
+ function boolOrNull(value) {
147
+ return typeof value === 'boolean' ? value : null;
148
+ }
149
+ function rateOrNull(value) {
150
+ if (typeof value !== 'number' || !Number.isFinite(value)) {
151
+ return null;
152
+ }
153
+ return clamp01(value);
154
+ }
155
+ function normalizeStopReasonInvalid(record) {
156
+ const explicit = boolOrNull(record.stop_reason_invalid);
157
+ if (explicit !== null) {
158
+ return explicit;
159
+ }
160
+ const stopReason = record.stop_reason;
161
+ if (typeof stopReason !== 'string' || stopReason.length === 0) {
162
+ return null;
163
+ }
164
+ return INVALID_STOP_REASONS.has(stopReason.toLowerCase());
165
+ }
166
+ function normalizeRepromptRate(record) {
167
+ const explicit = rateOrNull(record.reprompt_rate);
168
+ if (explicit !== null) {
169
+ return explicit;
170
+ }
171
+ const repromptCount = intOrNull(record.reprompt_count);
172
+ if (repromptCount === null) {
173
+ return null;
174
+ }
175
+ const turnIndex = intOrNull(record.turn_index_in_session);
176
+ const denominator = turnIndex !== null && turnIndex > 0 ? turnIndex : repromptCount + 1;
177
+ return clamp01(repromptCount / denominator);
178
+ }
179
+ function normalizeEditDistanceProxy(record) {
180
+ const explicit = rateOrNull(record.edit_distance_proxy);
181
+ if (explicit !== null) {
182
+ return explicit;
183
+ }
184
+ const delta = intOrNull(record.prompt_length_delta);
185
+ if (delta !== null) {
186
+ return clamp01(Math.abs(delta) / PROMPT_LENGTH_NORM_FOR_EDIT_PROXY);
187
+ }
188
+ const current = intOrNull(record.prompt_length_chars);
189
+ const prior = intOrNull(record.prior_prompt_length_chars);
190
+ if (current === null || prior === null) {
191
+ return null;
192
+ }
193
+ const maxLength = Math.max(current, prior, 1);
194
+ return clamp01(Math.abs(current - prior) / maxLength);
195
+ }
196
+ function normalizeToolFailureChainCount(record) {
197
+ const explicit = intOrNull(record.tool_failure_chain_count);
198
+ if (explicit !== null) {
199
+ return explicit;
200
+ }
201
+ return intOrNull(record.consecutive_tool_failures);
202
+ }
203
+ /** Extract privacy-safe failure proxy scalars from a contrib or export row. */
204
+ export function extractFailureProxies(record) {
205
+ return {
206
+ tool_failure_chain_count: normalizeToolFailureChainCount(record),
207
+ stop_reason_invalid: normalizeStopReasonInvalid(record),
208
+ reprompt_rate: normalizeRepromptRate(record),
209
+ edit_distance_proxy: normalizeEditDistanceProxy(record),
210
+ };
211
+ }
212
+ /** Map failure proxy scalars to verifier-grade training outcome signals. */
213
+ export function deriveVerifierFailureSignals(proxies) {
214
+ const signals = [];
215
+ if (proxies.tool_failure_chain_count !== null &&
216
+ proxies.tool_failure_chain_count >= TOOL_FAILURE_CHAIN_LABEL_THRESHOLD) {
217
+ signals.push('tool_failure_chain');
218
+ }
219
+ if (proxies.stop_reason_invalid === true) {
220
+ signals.push('stop_reason_invalid');
221
+ }
222
+ if (proxies.reprompt_rate !== null &&
223
+ proxies.reprompt_rate >= REPROMPT_RATE_FAILURE_THRESHOLD) {
224
+ signals.push('reprompt_detected');
225
+ }
226
+ if (proxies.edit_distance_proxy !== null &&
227
+ proxies.edit_distance_proxy >= EDIT_DISTANCE_PROXY_FAILURE_THRESHOLD) {
228
+ signals.push('high_edit_distance');
229
+ }
230
+ return signals;
231
+ }
232
+ function mergeTrainingOutcomeSignals(...groups) {
233
+ return [...new Set(groups.flat())];
234
+ }
235
+ function parseTrainingOutcomeSignals(raw) {
236
+ if (!Array.isArray(raw)) {
237
+ return [];
238
+ }
239
+ return raw.filter((value) => typeof value === 'string' && isTrainingOutcomeSignal(value));
240
+ }
241
+ /** Derive success label from behavioral outcomes plus optional failure proxies. */
242
+ export function deriveSuccessLabel(outcomes, options) {
243
+ const behavioralSignals = [...new Set(outcomes.map((entry) => entry.signal_type))];
244
+ const verifierSignals = options?.failureProxies
245
+ ? deriveVerifierFailureSignals(options.failureProxies)
246
+ : [];
247
+ const signals = mergeTrainingOutcomeSignals(behavioralSignals, verifierSignals);
100
248
  if (signals.some((signal) => FAILURE_OUTCOME_SIGNALS.includes(signal))) {
101
249
  return { success: false, outcome_signals: signals };
102
250
  }
@@ -109,6 +257,41 @@ export function deriveSuccessLabel(outcomes) {
109
257
  // compaction_pin_break and other neutral signals — treat as unlabeled success.
110
258
  return { success: true, outcome_signals: signals };
111
259
  }
260
+ /** Derive training labels from a privacy-safe export/contrib row (no prompt text). */
261
+ export function deriveSuccessLabelFromExportRow(record) {
262
+ const failure_proxies = extractFailureProxies(record);
263
+ const existingSignals = parseTrainingOutcomeSignals(record.outcome_signals);
264
+ const verifierSignals = deriveVerifierFailureSignals(failure_proxies);
265
+ const mergedSignals = mergeTrainingOutcomeSignals(existingSignals, verifierSignals);
266
+ if (mergedSignals.some((signal) => FAILURE_OUTCOME_SIGNALS.includes(signal))) {
267
+ return { success: false, outcome_signals: mergedSignals, failure_proxies };
268
+ }
269
+ if (mergedSignals.includes('feedback_good')) {
270
+ return { success: true, outcome_signals: mergedSignals, failure_proxies };
271
+ }
272
+ if (typeof record.success_label === 'boolean') {
273
+ return {
274
+ success: record.success_label,
275
+ outcome_signals: mergedSignals,
276
+ failure_proxies,
277
+ };
278
+ }
279
+ if (mergedSignals.length === 0) {
280
+ return { success: null, outcome_signals: mergedSignals, failure_proxies };
281
+ }
282
+ return { success: true, outcome_signals: mergedSignals, failure_proxies };
283
+ }
284
+ /** Attach normalized failure proxy fields for calibration export rows. */
285
+ export function attachFailureProxiesToExport(exportRecord) {
286
+ const proxies = extractFailureProxies(exportRecord);
287
+ return {
288
+ ...exportRecord,
289
+ tool_failure_chain_count: proxies.tool_failure_chain_count,
290
+ stop_reason_invalid: proxies.stop_reason_invalid,
291
+ reprompt_rate: proxies.reprompt_rate,
292
+ edit_distance_proxy: proxies.edit_distance_proxy,
293
+ };
294
+ }
112
295
  /** Group outcomes by request_id for export joins. */
113
296
  export function indexOutcomesByRequestId(outcomes) {
114
297
  const byRequest = new Map();
@@ -124,23 +307,31 @@ export function joinDatasetWithOutcomes(datasetRecords, outcomeRecords) {
124
307
  const outcomesByRequest = indexOutcomesByRequestId(outcomeRecords);
125
308
  return datasetRecords.map((record) => {
126
309
  const linked = outcomesByRequest.get(record.request_id) ?? [];
127
- const { success, outcome_signals } = deriveSuccessLabel(linked);
310
+ const failure_proxies = extractFailureProxies(record);
311
+ const { success, outcome_signals } = deriveSuccessLabel(linked, { failureProxies: failure_proxies });
128
312
  return {
129
313
  request_id: record.request_id,
130
314
  features: extractPSuccessFeatures(record),
131
315
  success: success ?? true,
132
316
  outcome_signals,
317
+ failure_proxies,
133
318
  };
134
319
  });
135
320
  }
136
321
  /** Attach join labels to a privacy-safe export object. */
137
322
  export function attachOutcomeLabelsToExport(exportRecord, outcomes) {
138
- const { success, outcome_signals } = deriveSuccessLabel(outcomes);
323
+ const withProxies = attachFailureProxiesToExport(exportRecord);
324
+ const failure_proxies = extractFailureProxies(withProxies);
325
+ const { success, outcome_signals } = deriveSuccessLabel(outcomes, { failureProxies: failure_proxies });
139
326
  return {
140
- ...exportRecord,
327
+ ...withProxies,
141
328
  request_id: String(exportRecord.request_id ?? ''),
142
329
  success_label: success,
143
330
  outcome_signals,
331
+ tool_failure_chain_count: failure_proxies.tool_failure_chain_count,
332
+ stop_reason_invalid: failure_proxies.stop_reason_invalid,
333
+ reprompt_rate: failure_proxies.reprompt_rate,
334
+ edit_distance_proxy: failure_proxies.edit_distance_proxy,
144
335
  };
145
336
  }
146
337
  function dot(a, b) {
@@ -202,21 +393,14 @@ export function parseTrainingExportLine(line) {
202
393
  if (typeof requestId !== 'string' || requestId.length === 0) {
203
394
  return null;
204
395
  }
205
- const successLabel = parsed.success_label;
206
- let success = true;
207
- if (typeof successLabel === 'boolean') {
208
- success = successLabel;
209
- }
210
- const rawSignals = parsed.outcome_signals;
211
- const outcome_signals = Array.isArray(rawSignals)
212
- ? rawSignals.filter((value) => typeof value === 'string')
213
- : [];
396
+ const labeled = deriveSuccessLabelFromExportRow(parsed);
214
397
  const record = parsed;
215
398
  return {
216
399
  request_id: requestId,
217
400
  features: extractPSuccessFeatures(record),
218
- success: success === false ? false : true,
219
- outcome_signals,
401
+ success: labeled.success === false ? false : true,
402
+ outcome_signals: labeled.outcome_signals,
403
+ failure_proxies: labeled.failure_proxies,
220
404
  };
221
405
  }
222
406
  /** Train weights from privacy-safe labeled JSONL export content. */