pi-smart-router 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/delegation-runtime.ts +12 -2
- package/.pi/extensions/smart-router/fleet-bootstrap.ts +22 -0
- package/.pi/extensions/smart-router/routing-context.ts +85 -11
- package/README.md +47 -1
- package/config/benchmark-profiles.json +124 -33
- package/dist/config/defaults.d.ts +2 -2
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +6 -3
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/pi-model-mapper.d.ts +37 -1
- package/dist/config/pi-model-mapper.d.ts.map +1 -1
- package/dist/config/pi-model-mapper.js +120 -7
- package/dist/config/pi-model-mapper.js.map +1 -1
- package/dist/domain/delegation/delegation-context.d.ts +49 -0
- package/dist/domain/delegation/delegation-context.d.ts.map +1 -1
- package/dist/domain/delegation/delegation-context.js +92 -0
- package/dist/domain/delegation/delegation-context.js.map +1 -1
- package/dist/domain/judgment/answer-mapper.d.ts +49 -0
- package/dist/domain/judgment/answer-mapper.d.ts.map +1 -0
- package/dist/domain/judgment/answer-mapper.js +124 -0
- package/dist/domain/judgment/answer-mapper.js.map +1 -0
- package/dist/domain/judgment/index.d.ts +8 -0
- package/dist/domain/judgment/index.d.ts.map +1 -0
- package/dist/domain/judgment/index.js +8 -0
- package/dist/domain/judgment/index.js.map +1 -0
- package/dist/domain/judgment/judge-state.d.ts +70 -0
- package/dist/domain/judgment/judge-state.d.ts.map +1 -0
- package/dist/domain/judgment/judge-state.js +227 -0
- package/dist/domain/judgment/judge-state.js.map +1 -0
- package/dist/domain/judgment/questions.d.ts +58 -0
- package/dist/domain/judgment/questions.d.ts.map +1 -0
- package/dist/domain/judgment/questions.js +157 -0
- package/dist/domain/judgment/questions.js.map +1 -0
- package/dist/domain/matching/hydra-input.d.ts.map +1 -1
- package/dist/domain/matching/hydra-input.js +4 -17
- package/dist/domain/matching/hydra-input.js.map +1 -1
- package/dist/domain/pinning/loop-escalation.d.ts.map +1 -1
- package/dist/domain/pinning/loop-escalation.js +1 -49
- package/dist/domain/pinning/loop-escalation.js.map +1 -1
- package/dist/domain/pinning/tool-failure.d.ts +20 -0
- package/dist/domain/pinning/tool-failure.d.ts.map +1 -0
- package/dist/domain/pinning/tool-failure.js +62 -0
- package/dist/domain/pinning/tool-failure.js.map +1 -0
- package/dist/domain/pipeline/pipeline-stage.d.ts +11 -0
- package/dist/domain/pipeline/pipeline-stage.d.ts.map +1 -1
- package/dist/domain/pipeline/pipeline-stage.js +3 -0
- package/dist/domain/pipeline/pipeline-stage.js.map +1 -1
- package/dist/domain/pipeline/router-pipeline.d.ts +32 -6
- package/dist/domain/pipeline/router-pipeline.d.ts.map +1 -1
- package/dist/domain/pipeline/router-pipeline.js +49 -4
- package/dist/domain/pipeline/router-pipeline.js.map +1 -1
- package/dist/domain/pipeline/typed-judgment-stage.d.ts +44 -0
- package/dist/domain/pipeline/typed-judgment-stage.d.ts.map +1 -0
- package/dist/domain/pipeline/typed-judgment-stage.js +144 -0
- package/dist/domain/pipeline/typed-judgment-stage.js.map +1 -0
- package/dist/domain/ports/typed-judgment-port.d.ts +138 -0
- package/dist/domain/ports/typed-judgment-port.d.ts.map +1 -0
- package/dist/domain/ports/typed-judgment-port.js +28 -0
- package/dist/domain/ports/typed-judgment-port.js.map +1 -0
- package/dist/domain/types/entities.d.ts +42 -1
- package/dist/domain/types/entities.d.ts.map +1 -1
- package/dist/domain/types/index.d.ts +1 -1
- package/dist/domain/types/index.d.ts.map +1 -1
- package/dist/domain/types/schemas.d.ts +140 -0
- package/dist/domain/types/schemas.d.ts.map +1 -1
- package/dist/domain/types/schemas.js +191 -0
- package/dist/domain/types/schemas.js.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.d.ts +67 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.d.ts.map +1 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.js +411 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.js.map +1 -0
- package/dist/infrastructure/persistence/sqlite-store.d.ts.map +1 -1
- package/dist/infrastructure/persistence/sqlite-store.js +37 -9
- package/dist/infrastructure/persistence/sqlite-store.js.map +1 -1
- package/package.json +1 -1
- package/src/config/defaults.ts +8 -1
- package/src/config/pi-model-mapper.ts +146 -7
- package/src/domain/delegation/delegation-context.ts +123 -0
- package/src/domain/judgment/answer-mapper.ts +210 -0
- package/src/domain/judgment/index.ts +32 -0
- package/src/domain/judgment/judge-state.ts +294 -0
- package/src/domain/judgment/questions.ts +198 -0
- package/src/domain/matching/hydra-input.ts +4 -19
- package/src/domain/pinning/loop-escalation.ts +1 -52
- package/src/domain/pinning/tool-failure.ts +66 -0
- package/src/domain/pipeline/pipeline-stage.ts +17 -0
- package/src/domain/pipeline/router-pipeline.ts +83 -6
- package/src/domain/pipeline/typed-judgment-stage.ts +165 -0
- package/src/domain/ports/typed-judgment-port.ts +186 -0
- package/src/domain/types/entities.ts +54 -0
- package/src/domain/types/index.ts +1 -0
- package/src/domain/types/schemas.ts +206 -0
- package/src/index.ts +19 -2
- package/src/infrastructure/judgment/jev-typed-judgment-adapter.ts +564 -0
- package/src/infrastructure/persistence/sqlite-store.ts +45 -9
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed-judgment question set (SP-306, GitHub #185 / epic #183) — constants,
|
|
3
|
+
* one module, no I/O. Typed with the SP-304 question wire types
|
|
4
|
+
* (`src/domain/ports/typed-judgment-port.ts`).
|
|
5
|
+
*
|
|
6
|
+
* Measured on local OpenJev (epic #183): a single isolated yes/no question
|
|
7
|
+
* was unstable, while anchored level descriptions were stable — so every
|
|
8
|
+
* ordinal question here carries anchored levels and every noul carries
|
|
9
|
+
* explicit true/false criteria.
|
|
10
|
+
*
|
|
11
|
+
* Attribution: the level descriptions and criteria for `task_kind`,
|
|
12
|
+
* `capability_deserved`, and `needs_deep_reasoning` are adapted from
|
|
13
|
+
* pi-jev-model-router (`extensions/pi-jev-model-router/jev.ts` and
|
|
14
|
+
* `config.ts`), MIT License, © 2026 Shah Rukh.
|
|
15
|
+
* https://github.com/da-vinci-noob/pi-jev-model-router
|
|
16
|
+
*
|
|
17
|
+
* The `complexity` question from the reference router is deliberately
|
|
18
|
+
* omitted — dropped pending the #191 eval ablation.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type {
|
|
22
|
+
TypedJudgmentChoiceQuestion,
|
|
23
|
+
TypedJudgmentNoulQuestion,
|
|
24
|
+
TypedJudgmentQuestion,
|
|
25
|
+
TypedJudgmentScoreQuestion,
|
|
26
|
+
} from '../ports/typed-judgment-port.js';
|
|
27
|
+
|
|
28
|
+
/** Stable ids of the six typed-judgment questions. */
|
|
29
|
+
export const TYPED_JUDGMENT_QUESTION_IDS = [
|
|
30
|
+
'task_kind',
|
|
31
|
+
'capability_deserved',
|
|
32
|
+
'needs_deep_reasoning',
|
|
33
|
+
'code_gen',
|
|
34
|
+
'tool_use',
|
|
35
|
+
'destructive',
|
|
36
|
+
] as const;
|
|
37
|
+
|
|
38
|
+
export type TypedJudgmentSixQuestionId =
|
|
39
|
+
(typeof TYPED_JUDGMENT_QUESTION_IDS)[number];
|
|
40
|
+
|
|
41
|
+
/** The ten task-kind options, in canonical order. */
|
|
42
|
+
export const TASK_KIND_OPTIONS = [
|
|
43
|
+
'plan',
|
|
44
|
+
'implement',
|
|
45
|
+
'debug',
|
|
46
|
+
'refactor',
|
|
47
|
+
'review',
|
|
48
|
+
'research',
|
|
49
|
+
'explain',
|
|
50
|
+
'operate',
|
|
51
|
+
'write',
|
|
52
|
+
'chat',
|
|
53
|
+
] as const;
|
|
54
|
+
|
|
55
|
+
export type TaskKind = (typeof TASK_KIND_OPTIONS)[number];
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Per-kind descriptions embedded in the `task_kind` prompt so the judge sees
|
|
59
|
+
* anchored definitions, not bare labels. Adapted from pi-jev-model-router
|
|
60
|
+
* `config.ts` `TASK_KINDS` (MIT, © 2026 Shah Rukh).
|
|
61
|
+
*/
|
|
62
|
+
const TASK_KIND_DESCRIPTIONS: Readonly<Record<TaskKind, string>> = {
|
|
63
|
+
plan: 'Deciding what to build, sequencing work, or designing an approach before editing',
|
|
64
|
+
implement:
|
|
65
|
+
'Writing or changing code, scripts, or configuration to produce a concrete result',
|
|
66
|
+
write:
|
|
67
|
+
'Producing prose, documentation, comments, or other non-code content from scratch',
|
|
68
|
+
debug:
|
|
69
|
+
'Diagnosing a failure, error, or unexpected behavior and finding its root cause',
|
|
70
|
+
refactor: 'Restructuring existing code without changing intended behavior',
|
|
71
|
+
review: 'Auditing code, a diff, a document, or a plan for problems and risks',
|
|
72
|
+
research:
|
|
73
|
+
'Searching, reading, and synthesizing external information or unfamiliar APIs',
|
|
74
|
+
explain: 'Answering a question or explaining how something works',
|
|
75
|
+
operate: 'Running commands, tooling, git, deploys, or environment setup',
|
|
76
|
+
chat: 'Small talk, acknowledgements, or a request with no real work attached',
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const TASK_KIND_PROMPT = [
|
|
80
|
+
'Which single kind of work does `latest_user_message` ask for? Judge the work the user wants done, not the topic they mention. Read `recent_activity` when the request is a short follow-up that only makes sense in context. Pick the closest kind even when the request is ambiguous.',
|
|
81
|
+
...TASK_KIND_OPTIONS.map(
|
|
82
|
+
(kind) => `- ${kind}: ${TASK_KIND_DESCRIPTIONS[kind]}`,
|
|
83
|
+
),
|
|
84
|
+
].join('\n');
|
|
85
|
+
|
|
86
|
+
/** `task_kind` — categorical, ten anchored options. */
|
|
87
|
+
export const TASK_KIND_QUESTION: TypedJudgmentChoiceQuestion = {
|
|
88
|
+
id: 'task_kind',
|
|
89
|
+
kind: 'choice',
|
|
90
|
+
prompt: TASK_KIND_PROMPT,
|
|
91
|
+
options: TASK_KIND_OPTIONS,
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* `capability_deserved` — 4 anchored levels. Setting price aside: how much
|
|
96
|
+
* model capability does this request deserve? Anchors adapted from
|
|
97
|
+
* pi-jev-model-router (MIT, © 2026 Shah Rukh).
|
|
98
|
+
*/
|
|
99
|
+
export const CAPABILITY_DESERVED_QUESTION: TypedJudgmentScoreQuestion = {
|
|
100
|
+
id: 'capability_deserved',
|
|
101
|
+
kind: 'score',
|
|
102
|
+
prompt:
|
|
103
|
+
'Setting price aside entirely, how much model capability does this request deserve to get a good outcome? Judge by stakes, difficulty, and how much a stronger model would measurably improve the result.',
|
|
104
|
+
anchors: [
|
|
105
|
+
{ score: 0, description: 'Minimal: any fast small model answers this just as well' },
|
|
106
|
+
{ score: 1, description: 'Standard: a competent mid-tier model is enough' },
|
|
107
|
+
{ score: 2, description: 'High: a strong frontier model materially improves the outcome' },
|
|
108
|
+
{ score: 3, description: 'Maximum: correctness matters more than cost; use the best available' },
|
|
109
|
+
],
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* `needs_deep_reasoning` — noul with explicit true/false criteria. Criteria
|
|
114
|
+
* adapted from pi-jev-model-router (MIT, © 2026 Shah Rukh).
|
|
115
|
+
*/
|
|
116
|
+
export const NEEDS_DEEP_REASONING_QUESTION: TypedJudgmentNoulQuestion = {
|
|
117
|
+
id: 'needs_deep_reasoning',
|
|
118
|
+
kind: 'noul',
|
|
119
|
+
prompt:
|
|
120
|
+
'Does answering `latest_user_message` well require extended multi-step reasoning (algorithm design, subtle debugging, proof, careful long-horizon planning) rather than recall, lookup, or a short direct edit?',
|
|
121
|
+
criteria_true:
|
|
122
|
+
'The work hinges on reasoning through non-obvious steps or edge cases',
|
|
123
|
+
criteria_false:
|
|
124
|
+
'The work is recall, lookup, formatting, or a short direct change',
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
/** `code_gen` — 4 anchored levels of code-generation demand. */
|
|
128
|
+
export const CODE_GEN_QUESTION: TypedJudgmentScoreQuestion = {
|
|
129
|
+
id: 'code_gen',
|
|
130
|
+
kind: 'score',
|
|
131
|
+
prompt:
|
|
132
|
+
'How much code or configuration must be written or modified to satisfy `latest_user_message` well?',
|
|
133
|
+
anchors: [
|
|
134
|
+
{ score: 0, description: 'None: no code or configuration is written or modified' },
|
|
135
|
+
{ score: 1, description: 'Small: a short snippet, single-line change, or simple config value' },
|
|
136
|
+
{ score: 2, description: 'Moderate: a function, test, or file-scale change using familiar patterns' },
|
|
137
|
+
{ score: 3, description: 'Substantial: multi-file or architecturally significant code generation' },
|
|
138
|
+
],
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
/** `tool_use` — 4 anchored levels of tool-execution demand. */
|
|
142
|
+
export const TOOL_USE_QUESTION: TypedJudgmentScoreQuestion = {
|
|
143
|
+
id: 'tool_use',
|
|
144
|
+
kind: 'score',
|
|
145
|
+
prompt:
|
|
146
|
+
'How much tool execution (shell, file, search, or agentic tool calls) does satisfying `latest_user_message` require? Use `session` and `recent_activity` to judge whether a tool loop is already underway.',
|
|
147
|
+
anchors: [
|
|
148
|
+
{ score: 0, description: 'None: the answer needs no tool execution' },
|
|
149
|
+
{ score: 1, description: 'Light: at most a couple of simple read-only tool calls' },
|
|
150
|
+
{ score: 2, description: 'Moderate: several tool calls or state-changing commands in a known sequence' },
|
|
151
|
+
{ score: 3, description: 'Heavy: long agentic tool loops, orchestration across many calls, or recovery from tool failures' },
|
|
152
|
+
],
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
/** `destructive` — noul guarding irreversible actions. */
|
|
156
|
+
export const DESTRUCTIVE_QUESTION: TypedJudgmentNoulQuestion = {
|
|
157
|
+
id: 'destructive',
|
|
158
|
+
kind: 'noul',
|
|
159
|
+
prompt:
|
|
160
|
+
'Could satisfying `latest_user_message` destroy, overwrite, or irreversibly modify data, state, or history?',
|
|
161
|
+
criteria_true:
|
|
162
|
+
'The request could delete, overwrite, or irreversibly modify data, state, or history (e.g. recursive deletes, force pushes, dropped tables, branch resets, mass edits without a checkpoint)',
|
|
163
|
+
criteria_false:
|
|
164
|
+
'The request is read-only, additive, or easily reversible',
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* The six typed-judgment questions, in canonical order. This is the only
|
|
169
|
+
* module that defines them — consumers import, never redefine.
|
|
170
|
+
*/
|
|
171
|
+
export const TYPED_JUDGMENT_QUESTIONS: readonly TypedJudgmentQuestion[] = [
|
|
172
|
+
TASK_KIND_QUESTION,
|
|
173
|
+
CAPABILITY_DESERVED_QUESTION,
|
|
174
|
+
NEEDS_DEEP_REASONING_QUESTION,
|
|
175
|
+
CODE_GEN_QUESTION,
|
|
176
|
+
TOOL_USE_QUESTION,
|
|
177
|
+
DESTRUCTIVE_QUESTION,
|
|
178
|
+
];
|
|
179
|
+
|
|
180
|
+
/** FNV-1a 32-bit hex — deterministic across runs, no crypto dependency. */
|
|
181
|
+
function fnv1aHex(input: string): string {
|
|
182
|
+
let hash = 0x811c9dc5;
|
|
183
|
+
for (let i = 0; i < input.length; i++) {
|
|
184
|
+
hash ^= input.charCodeAt(i);
|
|
185
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
186
|
+
}
|
|
187
|
+
return hash.toString(16).padStart(8, '0');
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Stable question-set version hash. Derived deterministically from the
|
|
192
|
+
* canonical serialization of {@link TYPED_JUDGMENT_QUESTIONS}; any wording,
|
|
193
|
+
* option, or anchor change produces a new hash. Consumed by the #191 answer
|
|
194
|
+
* cache to key cached judgments to the exact question set that produced them.
|
|
195
|
+
*/
|
|
196
|
+
export const TYPED_JUDGMENT_QUESTION_SET_VERSION = `tjq-v1-${fnv1aHex(
|
|
197
|
+
JSON.stringify(TYPED_JUDGMENT_QUESTIONS),
|
|
198
|
+
)}`;
|
|
@@ -8,18 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import type { Message, RoutingRequest } from '../types/index.js';
|
|
10
10
|
import type { TriageResult } from '../triage/triage-engine.js';
|
|
11
|
-
|
|
12
|
-
const FAILURE_PATTERNS = [
|
|
13
|
-
'error',
|
|
14
|
-
'fail',
|
|
15
|
-
'exception',
|
|
16
|
-
'timed out',
|
|
17
|
-
'timeout',
|
|
18
|
-
'econnrefused',
|
|
19
|
-
'enotfound',
|
|
20
|
-
'econnreset',
|
|
21
|
-
'epipe',
|
|
22
|
-
] as const;
|
|
11
|
+
import { isToolFailure } from '../pinning/tool-failure.js';
|
|
23
12
|
|
|
24
13
|
function resolveMessageCount(request: RoutingRequest): number {
|
|
25
14
|
return request.messages?.length ?? 0;
|
|
@@ -45,14 +34,10 @@ function resolveCompactionFlag(request: RoutingRequest): 0 | 1 {
|
|
|
45
34
|
return request.compaction_flag ? 1 : 0;
|
|
46
35
|
}
|
|
47
36
|
|
|
48
|
-
function looksLikeToolFailure(content: string): boolean {
|
|
49
|
-
const lower = content.toLowerCase();
|
|
50
|
-
return FAILURE_PATTERNS.some((pattern) => lower.includes(pattern));
|
|
51
|
-
}
|
|
52
|
-
|
|
53
37
|
/**
|
|
54
38
|
* Observational loop-pressure flag: 1 when the latest tool message looks
|
|
55
|
-
* like a failure.
|
|
39
|
+
* like a failure. Uses the shared loop-escalation failure detection
|
|
40
|
+
* (`is_error` → `status` → tightened body heuristic; SP-303 / #186).
|
|
56
41
|
*/
|
|
57
42
|
function resolveLoopPressure(request: RoutingRequest): 0 | 1 {
|
|
58
43
|
const messages = request.messages;
|
|
@@ -63,7 +48,7 @@ function resolveLoopPressure(request: RoutingRequest): 0 | 1 {
|
|
|
63
48
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
64
49
|
const message = messages[i]!;
|
|
65
50
|
if (message.role === 'tool') {
|
|
66
|
-
return
|
|
51
|
+
return isToolFailure(message) ? 1 : 0;
|
|
67
52
|
}
|
|
68
53
|
}
|
|
69
54
|
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
24
|
import type { Message, ModelProfile, RoutingRequest, SessionPin } from '../types/index.js';
|
|
25
|
+
import { isToolFailure } from './tool-failure.js';
|
|
25
26
|
|
|
26
27
|
// ─── Types ────────────────────────────────────────────────────────────────────
|
|
27
28
|
|
|
@@ -46,32 +47,6 @@ export const ZERO_TIER_TOOL_CHURN_SIGNATURE = 'zt:tool_churn' as const;
|
|
|
46
47
|
|
|
47
48
|
// ─── Failure signature extraction ─────────────────────────────────────────────
|
|
48
49
|
|
|
49
|
-
const FAILURE_PATTERNS = [
|
|
50
|
-
'error',
|
|
51
|
-
'fail',
|
|
52
|
-
'exception',
|
|
53
|
-
'timed out',
|
|
54
|
-
'timeout',
|
|
55
|
-
'econnrefused',
|
|
56
|
-
'enotfound',
|
|
57
|
-
'econnreset',
|
|
58
|
-
'epipe',
|
|
59
|
-
] as const;
|
|
60
|
-
|
|
61
|
-
/** Rate-limit and quota failures whose body text may omit generic error keywords. */
|
|
62
|
-
const RATE_LIMIT_PATTERNS = [
|
|
63
|
-
'rate limit',
|
|
64
|
-
'429',
|
|
65
|
-
'too many requests',
|
|
66
|
-
'quota exceeded',
|
|
67
|
-
] as const;
|
|
68
|
-
|
|
69
|
-
/** Auth/access failures that are not rate limits but still indicate tool failure. */
|
|
70
|
-
const AUTH_DENIED_PATTERNS = [
|
|
71
|
-
'permission denied',
|
|
72
|
-
'access denied',
|
|
73
|
-
] as const;
|
|
74
|
-
|
|
75
50
|
/** Host/agent signals that the model invoked a tool the runtime does not expose. */
|
|
76
51
|
const UNSUPPORTED_TOOL_PATTERNS = [
|
|
77
52
|
'unknown tool',
|
|
@@ -125,32 +100,6 @@ function lastToolMessage(request: RoutingRequest): Message | null {
|
|
|
125
100
|
return null;
|
|
126
101
|
}
|
|
127
102
|
|
|
128
|
-
/**
|
|
129
|
-
* Decide whether a tool-result message represents a failure.
|
|
130
|
-
* Prefers structured host signals (`is_error` / `status`); falls back to a
|
|
131
|
-
* tightened body heuristic only when no structured signal is present.
|
|
132
|
-
*/
|
|
133
|
-
function isToolFailure(msg: Message): boolean {
|
|
134
|
-
if (msg.is_error === true) return true;
|
|
135
|
-
if (msg.is_error === false) return false;
|
|
136
|
-
if (msg.status !== undefined) {
|
|
137
|
-
return msg.status >= 400;
|
|
138
|
-
}
|
|
139
|
-
return looksLikeFailure(msg.content ?? '');
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
function looksLikeFailure(content: string): boolean {
|
|
143
|
-
const lower = content.toLowerCase();
|
|
144
|
-
// Benign negations must not be counted as failures.
|
|
145
|
-
if (/\bno (error|errors|failure|failures)\b/.test(lower)) return false;
|
|
146
|
-
if (/\bwithout (an? )?(error|failure)\b/.test(lower)) return false;
|
|
147
|
-
return (
|
|
148
|
-
FAILURE_PATTERNS.some((p) => lower.includes(p)) ||
|
|
149
|
-
RATE_LIMIT_PATTERNS.some((p) => lower.includes(p)) ||
|
|
150
|
-
AUTH_DENIED_PATTERNS.some((p) => lower.includes(p))
|
|
151
|
-
);
|
|
152
|
-
}
|
|
153
|
-
|
|
154
103
|
/** Deterministic djb2 hash of normalised error content. */
|
|
155
104
|
function computeSignature(content: string): string {
|
|
156
105
|
const normalized = content.trim().slice(0, 256).toLowerCase();
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared tool-failure detection — SP-303, GitHub #186.
|
|
3
|
+
*
|
|
4
|
+
* Single source of truth for deciding whether a tool-result message
|
|
5
|
+
* represents a failure. Used by loop escalation (pin break) and the
|
|
6
|
+
* HyDRA `loop:` metadata flag.
|
|
7
|
+
*
|
|
8
|
+
* Precedence: structured host signals (`is_error`, then `status`) win;
|
|
9
|
+
* a tightened body heuristic applies only when no structured signal
|
|
10
|
+
* is present. Benign negations ("no errors", "without failure") never
|
|
11
|
+
* count as failures.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { Message } from '../types/index.js';
|
|
15
|
+
|
|
16
|
+
const FAILURE_PATTERNS = [
|
|
17
|
+
'error',
|
|
18
|
+
'fail',
|
|
19
|
+
'exception',
|
|
20
|
+
'timed out',
|
|
21
|
+
'timeout',
|
|
22
|
+
'econnrefused',
|
|
23
|
+
'enotfound',
|
|
24
|
+
'econnreset',
|
|
25
|
+
'epipe',
|
|
26
|
+
] as const;
|
|
27
|
+
|
|
28
|
+
/** Rate-limit and quota failures whose body text may omit generic error keywords. */
|
|
29
|
+
const RATE_LIMIT_PATTERNS = [
|
|
30
|
+
'rate limit',
|
|
31
|
+
'429',
|
|
32
|
+
'too many requests',
|
|
33
|
+
'quota exceeded',
|
|
34
|
+
] as const;
|
|
35
|
+
|
|
36
|
+
/** Auth/access failures that are not rate limits but still indicate tool failure. */
|
|
37
|
+
const AUTH_DENIED_PATTERNS = [
|
|
38
|
+
'permission denied',
|
|
39
|
+
'access denied',
|
|
40
|
+
] as const;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Decide whether a tool-result message represents a failure.
|
|
44
|
+
* Prefers structured host signals (`is_error` / `status`); falls back to a
|
|
45
|
+
* tightened body heuristic only when no structured signal is present.
|
|
46
|
+
*/
|
|
47
|
+
export function isToolFailure(msg: Message): boolean {
|
|
48
|
+
if (msg.is_error === true) return true;
|
|
49
|
+
if (msg.is_error === false) return false;
|
|
50
|
+
if (msg.status !== undefined) {
|
|
51
|
+
return msg.status >= 400;
|
|
52
|
+
}
|
|
53
|
+
return looksLikeFailure(msg.content ?? '');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function looksLikeFailure(content: string): boolean {
|
|
57
|
+
const lower = content.toLowerCase();
|
|
58
|
+
// Benign negations must not be counted as failures.
|
|
59
|
+
if (/\bno (error|errors|failure|failures)\b/.test(lower)) return false;
|
|
60
|
+
if (/\bwithout (an? )?(error|failure)\b/.test(lower)) return false;
|
|
61
|
+
return (
|
|
62
|
+
FAILURE_PATTERNS.some((p) => lower.includes(p)) ||
|
|
63
|
+
RATE_LIMIT_PATTERNS.some((p) => lower.includes(p)) ||
|
|
64
|
+
AUTH_DENIED_PATTERNS.some((p) => lower.includes(p))
|
|
65
|
+
);
|
|
66
|
+
}
|
|
@@ -25,6 +25,7 @@ import type { MatchResult } from '../matching/hydra-matcher.js';
|
|
|
25
25
|
import type { ClusterMatchResult } from '../matching/cluster-matcher.js';
|
|
26
26
|
import type { ExpectedCostBreakdown } from '../routing/expected-cost.js';
|
|
27
27
|
import type { PrewarmOutcome } from '../routing/speculative-prewarm.js';
|
|
28
|
+
import type { TypedJudgment } from '../judgment/answer-mapper.js';
|
|
28
29
|
import type {
|
|
29
30
|
PipelineOptions,
|
|
30
31
|
PipelineStageName,
|
|
@@ -119,6 +120,19 @@ export interface RoutingContext {
|
|
|
119
120
|
|
|
120
121
|
/** Speculative prewarm outcome for explain/telemetry (SP-217, #117). */
|
|
121
122
|
prewarmOutcome: PrewarmOutcome | null;
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* typed_judgment output for the current user-prompt turn (SP-307, #187).
|
|
126
|
+
* `null` when the stage skipped or the judge call failed (fail open) —
|
|
127
|
+
* later stages must behave exactly as if the field were absent.
|
|
128
|
+
*/
|
|
129
|
+
typedJudgment: TypedJudgment | null;
|
|
130
|
+
|
|
131
|
+
/** Why typed_judgment skipped/failed (`typed_judgment_*` codes), or null on success. */
|
|
132
|
+
typedJudgmentSkipReason: string | null;
|
|
133
|
+
|
|
134
|
+
/** Wall-clock judge-call latency in ms; null when the port was not called. */
|
|
135
|
+
typedJudgmentLatencyMs: number | null;
|
|
122
136
|
}
|
|
123
137
|
|
|
124
138
|
/**
|
|
@@ -159,6 +173,9 @@ export function createRoutingContext(args: {
|
|
|
159
173
|
routePath: null,
|
|
160
174
|
routePathConfidence: null,
|
|
161
175
|
prewarmOutcome: null,
|
|
176
|
+
typedJudgment: null,
|
|
177
|
+
typedJudgmentSkipReason: null,
|
|
178
|
+
typedJudgmentLatencyMs: null,
|
|
162
179
|
};
|
|
163
180
|
}
|
|
164
181
|
|
|
@@ -2,10 +2,13 @@
|
|
|
2
2
|
* Pipeline stage orchestrator — FR-001, FR-006, FR-022.
|
|
3
3
|
*
|
|
4
4
|
* Runs stages sequentially with early-exit on decision.
|
|
5
|
-
* Documented order (SP-119, #69):
|
|
6
|
-
* hardware_probe → loop_escalation → turn_envelope →
|
|
7
|
-
* → session_pin → triage → local_zero → triage_cloud_fallback
|
|
8
|
-
* → safe_default → context_overflow_fallback
|
|
5
|
+
* Documented order (SP-119, #69; typed_judgment inserted by SP-307, #187):
|
|
6
|
+
* hardware_probe → loop_escalation → turn_envelope → typed_judgment → context_fit
|
|
7
|
+
* → low_intensity → session_pin → triage → local_zero → triage_cloud_fallback
|
|
8
|
+
* → hydra_match → safe_default → context_overflow_fallback
|
|
9
|
+
* typed_judgment never decides (fail open, SP-307); route() is serialized per
|
|
10
|
+
* pipeline (SP-230), so judge latency adds to user-turn routing latency only —
|
|
11
|
+
* tool_result turns pay nothing.
|
|
9
12
|
* Any stage failure falls back to safeCloudDefault(); never throws to host.
|
|
10
13
|
*
|
|
11
14
|
* SP-273 + SP-274 / #143: every stage now lives in a focused module behind
|
|
@@ -28,8 +31,17 @@ import type {
|
|
|
28
31
|
RoutingRequest,
|
|
29
32
|
SaarConfig,
|
|
30
33
|
} from '../types/index.js';
|
|
31
|
-
import type {
|
|
32
|
-
|
|
34
|
+
import type {
|
|
35
|
+
QuotaWindowPosition,
|
|
36
|
+
TypedJudgmentSidecar,
|
|
37
|
+
} from '../types/entities.js';
|
|
38
|
+
import type {
|
|
39
|
+
LowIntensityConfig,
|
|
40
|
+
LocalZeroConfig,
|
|
41
|
+
TypedJudgmentConfig,
|
|
42
|
+
VirtualCostV2Config,
|
|
43
|
+
} from '../types/schemas.js';
|
|
44
|
+
import type { TypedJudgmentPort } from '../ports/typed-judgment-port.js';
|
|
33
45
|
import type {
|
|
34
46
|
HardwareProbeConfig,
|
|
35
47
|
HardwareProbePort,
|
|
@@ -81,6 +93,10 @@ import {
|
|
|
81
93
|
import { createHydraMatchStage } from './hydra-match-stage.js';
|
|
82
94
|
import { createHardwareProbeStage } from './hardware-probe-stage.js';
|
|
83
95
|
import { createTurnEnvelopeStage } from './turn-envelope-stage.js';
|
|
96
|
+
import {
|
|
97
|
+
createTypedJudgmentStage,
|
|
98
|
+
TYPED_JUDGMENT_DISABLED,
|
|
99
|
+
} from './typed-judgment-stage.js';
|
|
84
100
|
import { createContextFitStage } from './context-fit-stage.js';
|
|
85
101
|
import { createLowIntensityStage } from './low-intensity-stage.js';
|
|
86
102
|
import { createLocalZeroStage } from './local-zero-stage.js';
|
|
@@ -116,6 +132,7 @@ export const PIPELINE_STAGE_ORDER = [
|
|
|
116
132
|
'hardware_probe',
|
|
117
133
|
'loop_escalation',
|
|
118
134
|
'turn_envelope',
|
|
135
|
+
'typed_judgment',
|
|
119
136
|
'context_fit',
|
|
120
137
|
'low_intensity',
|
|
121
138
|
'session_pin',
|
|
@@ -210,6 +227,20 @@ export interface PipelineOptions {
|
|
|
210
227
|
readonly learnedRouteStore?: LearnedRouteStore;
|
|
211
228
|
/** Compiled operator pattern pack overlay for the degraded sandwich (SP-212, #119). */
|
|
212
229
|
readonly patternPack?: CompiledPatternPack;
|
|
230
|
+
/**
|
|
231
|
+
* Typed judgment port (SP-304, #184; consumed by SP-307, #187). Injected by
|
|
232
|
+
* the composition root in SP-308; missing port OR missing/disabled
|
|
233
|
+
* `typedJudgmentConfig` → the typed_judgment stage skips with
|
|
234
|
+
* `typed_judgment_disabled` (fail open).
|
|
235
|
+
*/
|
|
236
|
+
readonly typedJudgmentPort?: TypedJudgmentPort;
|
|
237
|
+
/**
|
|
238
|
+
* Typed judgment operator knobs (SP-304). Missing config, missing
|
|
239
|
+
* `typedJudgmentPort`, or `enabled: false` → the typed_judgment stage
|
|
240
|
+
* skips with `typed_judgment_disabled` and routing is byte-identical
|
|
241
|
+
* to today.
|
|
242
|
+
*/
|
|
243
|
+
readonly typedJudgmentConfig?: TypedJudgmentConfig;
|
|
213
244
|
}
|
|
214
245
|
|
|
215
246
|
// ─── Orchestrator ────────────────────────────────────────────────────────────
|
|
@@ -229,6 +260,7 @@ export class RouterPipeline {
|
|
|
229
260
|
private readonly hardwareProbePipelineStage: PipelineStage = createHardwareProbeStage();
|
|
230
261
|
private readonly loopEscalationPipelineStage: PipelineStage = createLoopEscalationStage();
|
|
231
262
|
private readonly turnEnvelopePipelineStage: PipelineStage = createTurnEnvelopeStage();
|
|
263
|
+
private readonly typedJudgmentPipelineStage: PipelineStage = createTypedJudgmentStage();
|
|
232
264
|
private readonly contextFitPipelineStage: PipelineStage = createContextFitStage();
|
|
233
265
|
private readonly lowIntensityPipelineStage: PipelineStage = createLowIntensityStage();
|
|
234
266
|
private readonly sessionPinPipelineStage: PipelineStage = createSessionPinStage();
|
|
@@ -266,6 +298,7 @@ export class RouterPipeline {
|
|
|
266
298
|
{ name: 'hardware_probe', run: this.hardwareProbeStage.bind(this) },
|
|
267
299
|
{ name: 'loop_escalation', run: this.loopEscalation.bind(this) },
|
|
268
300
|
{ name: 'turn_envelope', run: this.turnEnvelope.bind(this) },
|
|
301
|
+
{ name: 'typed_judgment', run: this.typedJudgment.bind(this) },
|
|
269
302
|
{ name: 'context_fit', run: this.contextFitStage.bind(this) },
|
|
270
303
|
{ name: 'low_intensity', run: this.lowIntensityGate.bind(this) },
|
|
271
304
|
{ name: 'session_pin', run: this.sessionPin.bind(this) },
|
|
@@ -394,6 +427,7 @@ export class RouterPipeline {
|
|
|
394
427
|
decision: RoutingDecision,
|
|
395
428
|
): RoutingDecision {
|
|
396
429
|
const ctx = this.requireRouteContext();
|
|
430
|
+
const typedJudgmentSidecar = this.buildTypedJudgmentSidecar(ctx);
|
|
397
431
|
const features: RoutingFeatureSidecar = {
|
|
398
432
|
triage: ctx.triageResult
|
|
399
433
|
? {
|
|
@@ -443,6 +477,14 @@ export class RouterPipeline {
|
|
|
443
477
|
: ctx.prewarmOutcome.reason,
|
|
444
478
|
}
|
|
445
479
|
: {}),
|
|
480
|
+
// SP-308 / #187: typed-judgment signal sidecar. Present only when the
|
|
481
|
+
// stage was wired and enabled (judged turns carry the normalized
|
|
482
|
+
// signal; skipped turns carry the skip reason). Absent when unwired or
|
|
483
|
+
// disabled so the wire decision stays byte-identical (mirrors the
|
|
484
|
+
// embedding/cascade opt-in pattern).
|
|
485
|
+
...(typedJudgmentSidecar !== undefined
|
|
486
|
+
? { typed_judgment: typedJudgmentSidecar }
|
|
487
|
+
: {}),
|
|
446
488
|
};
|
|
447
489
|
|
|
448
490
|
const withBaseFeatures = { ...decision, features };
|
|
@@ -458,6 +500,36 @@ export class RouterPipeline {
|
|
|
458
500
|
);
|
|
459
501
|
}
|
|
460
502
|
|
|
503
|
+
/**
|
|
504
|
+
* Map the SP-307 typed_judgment stage context onto the SP-308 sidecar, or
|
|
505
|
+
* return undefined when the stage was unwired/disabled (skip reason
|
|
506
|
+
* `typed_judgment_disabled`) so the wire decision stays byte-identical.
|
|
507
|
+
*/
|
|
508
|
+
private buildTypedJudgmentSidecar(
|
|
509
|
+
ctx: RoutingContext,
|
|
510
|
+
): TypedJudgmentSidecar | undefined {
|
|
511
|
+
if (
|
|
512
|
+
ctx.typedJudgment === null &&
|
|
513
|
+
(ctx.typedJudgmentSkipReason === null ||
|
|
514
|
+
ctx.typedJudgmentSkipReason === TYPED_JUDGMENT_DISABLED)
|
|
515
|
+
) {
|
|
516
|
+
return undefined;
|
|
517
|
+
}
|
|
518
|
+
const judgment = ctx.typedJudgment;
|
|
519
|
+
return {
|
|
520
|
+
task_kind: judgment?.taskKind ?? null,
|
|
521
|
+
task_kind_confidence: judgment?.taskKindConfidence ?? null,
|
|
522
|
+
capability_deserved: judgment?.capabilityDeserved ?? null,
|
|
523
|
+
deep_reasoning_p: judgment?.deepReasoningP ?? null,
|
|
524
|
+
code_gen: judgment?.codeGen ?? null,
|
|
525
|
+
tool_use: judgment?.toolUse ?? null,
|
|
526
|
+
destructive_p: judgment?.destructiveP ?? null,
|
|
527
|
+
confidences: judgment?.confidences ?? {},
|
|
528
|
+
skip_reason: ctx.typedJudgmentSkipReason,
|
|
529
|
+
latency_ms: ctx.typedJudgmentLatencyMs,
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
|
|
461
533
|
/** Merge pipeline-recorded local_zero gate skip reasons into tier_selection (SP-164). */
|
|
462
534
|
private attachLocalZeroGateSkipReasons(
|
|
463
535
|
decision: RoutingDecision,
|
|
@@ -653,6 +725,11 @@ export class RouterPipeline {
|
|
|
653
725
|
return this.runStageWithContext(this.turnEnvelopePipelineStage);
|
|
654
726
|
}
|
|
655
727
|
|
|
728
|
+
private async typedJudgment(request: RoutingRequest): Promise<StageResult> {
|
|
729
|
+
void request;
|
|
730
|
+
return this.runStageWithContext(this.typedJudgmentPipelineStage);
|
|
731
|
+
}
|
|
732
|
+
|
|
656
733
|
private async contextFitStage(request: RoutingRequest): Promise<StageResult> {
|
|
657
734
|
void request;
|
|
658
735
|
return this.runStageWithContext(this.contextFitPipelineStage);
|