pi-smart-router 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/fleet-bootstrap.ts +22 -0
- package/README.md +47 -1
- package/config/benchmark-profiles.json +124 -33
- package/dist/config/defaults.d.ts +2 -2
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +6 -3
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/pi-model-mapper.d.ts +37 -1
- package/dist/config/pi-model-mapper.d.ts.map +1 -1
- package/dist/config/pi-model-mapper.js +120 -7
- package/dist/config/pi-model-mapper.js.map +1 -1
- package/dist/domain/judgment/answer-mapper.d.ts +49 -0
- package/dist/domain/judgment/answer-mapper.d.ts.map +1 -0
- package/dist/domain/judgment/answer-mapper.js +124 -0
- package/dist/domain/judgment/answer-mapper.js.map +1 -0
- package/dist/domain/judgment/index.d.ts +8 -0
- package/dist/domain/judgment/index.d.ts.map +1 -0
- package/dist/domain/judgment/index.js +8 -0
- package/dist/domain/judgment/index.js.map +1 -0
- package/dist/domain/judgment/judge-state.d.ts +70 -0
- package/dist/domain/judgment/judge-state.d.ts.map +1 -0
- package/dist/domain/judgment/judge-state.js +227 -0
- package/dist/domain/judgment/judge-state.js.map +1 -0
- package/dist/domain/judgment/questions.d.ts +58 -0
- package/dist/domain/judgment/questions.d.ts.map +1 -0
- package/dist/domain/judgment/questions.js +157 -0
- package/dist/domain/judgment/questions.js.map +1 -0
- package/dist/domain/pipeline/pipeline-stage.d.ts +11 -0
- package/dist/domain/pipeline/pipeline-stage.d.ts.map +1 -1
- package/dist/domain/pipeline/pipeline-stage.js +3 -0
- package/dist/domain/pipeline/pipeline-stage.js.map +1 -1
- package/dist/domain/pipeline/router-pipeline.d.ts +32 -6
- package/dist/domain/pipeline/router-pipeline.d.ts.map +1 -1
- package/dist/domain/pipeline/router-pipeline.js +49 -4
- package/dist/domain/pipeline/router-pipeline.js.map +1 -1
- package/dist/domain/pipeline/typed-judgment-stage.d.ts +44 -0
- package/dist/domain/pipeline/typed-judgment-stage.d.ts.map +1 -0
- package/dist/domain/pipeline/typed-judgment-stage.js +144 -0
- package/dist/domain/pipeline/typed-judgment-stage.js.map +1 -0
- package/dist/domain/ports/typed-judgment-port.d.ts +138 -0
- package/dist/domain/ports/typed-judgment-port.d.ts.map +1 -0
- package/dist/domain/ports/typed-judgment-port.js +28 -0
- package/dist/domain/ports/typed-judgment-port.js.map +1 -0
- package/dist/domain/types/entities.d.ts +42 -1
- package/dist/domain/types/entities.d.ts.map +1 -1
- package/dist/domain/types/index.d.ts +1 -1
- package/dist/domain/types/index.d.ts.map +1 -1
- package/dist/domain/types/schemas.d.ts +140 -0
- package/dist/domain/types/schemas.d.ts.map +1 -1
- package/dist/domain/types/schemas.js +191 -0
- package/dist/domain/types/schemas.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.d.ts +67 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.d.ts.map +1 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.js +411 -0
- package/dist/infrastructure/judgment/jev-typed-judgment-adapter.js.map +1 -0
- package/package.json +1 -1
- package/src/config/defaults.ts +8 -1
- package/src/config/pi-model-mapper.ts +146 -7
- package/src/domain/judgment/answer-mapper.ts +210 -0
- package/src/domain/judgment/index.ts +32 -0
- package/src/domain/judgment/judge-state.ts +294 -0
- package/src/domain/judgment/questions.ts +198 -0
- package/src/domain/pipeline/pipeline-stage.ts +17 -0
- package/src/domain/pipeline/router-pipeline.ts +83 -6
- package/src/domain/pipeline/typed-judgment-stage.ts +165 -0
- package/src/domain/ports/typed-judgment-port.ts +186 -0
- package/src/domain/types/entities.ts +54 -0
- package/src/domain/types/index.ts +1 -0
- package/src/domain/types/schemas.ts +206 -0
- package/src/index.ts +13 -2
- package/src/infrastructure/judgment/jev-typed-judgment-adapter.ts +564 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* typed_judgment stage (SP-307, GitHub #187 / epic #183) — non-deciding, fail-open.
|
|
3
|
+
*
|
|
4
|
+
* Computes one typed judgment per user-prompt turn via the SP-304
|
|
5
|
+
* {@link TypedJudgmentPort} and stores the normalized signal (SP-306
|
|
6
|
+
* `mapTypedJudgment`) on {@link RoutingContext} for later consumers
|
|
7
|
+
* (low_intensity / local_zero / hydra_match / adaptive reasoning — later
|
|
8
|
+
* trains #188–#190). This stage **never decides**: it always returns
|
|
9
|
+
* `decided: false`, and any timeout, error, or abort leaves
|
|
10
|
+
* `context.typedJudgment = null` with a `typed_judgment_*` skip reason so
|
|
11
|
+
* every later stage behaves exactly as when the stage is unwired.
|
|
12
|
+
*
|
|
13
|
+
* Stage position: directly after `turn_envelope` and before `context_fit`
|
|
14
|
+
* (the first stage that can early-exit on a cheap tier), so the judgment is
|
|
15
|
+
* available to every downstream consumer.
|
|
16
|
+
*
|
|
17
|
+
* Latency note: `route()` is serialized per pipeline (SP-230), so judge
|
|
18
|
+
* latency adds to user-turn routing latency only; `tool_result` turns (and
|
|
19
|
+
* disabled configs) pay nothing — the port is never invoked.
|
|
20
|
+
*
|
|
21
|
+
* Privacy: the judge state is built by SP-306 `buildJudgeState` (token-capped,
|
|
22
|
+
* no spend/budget keys). Errors are logged redacted via
|
|
23
|
+
* `redactPromptFromError`; prompt or judge-state text is never logged.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import {
|
|
27
|
+
buildJudgeState,
|
|
28
|
+
isAcknowledgement,
|
|
29
|
+
mapTypedJudgment,
|
|
30
|
+
TYPED_JUDGMENT_QUESTIONS,
|
|
31
|
+
} from '../judgment/index.js';
|
|
32
|
+
import { TypedJudgmentError } from '../ports/typed-judgment-port.js';
|
|
33
|
+
import { classifyTurnEnvelope } from '../triage/turn-envelope.js';
|
|
34
|
+
import type { Message, RoutingRequest } from '../types/index.js';
|
|
35
|
+
import type { PipelineStage, RoutingContext } from './pipeline-stage.js';
|
|
36
|
+
import type { StageResult } from './router-pipeline.js';
|
|
37
|
+
import { redactPromptFromError } from './stage-helpers.js';
|
|
38
|
+
|
|
39
|
+
// ─── Skip reason codes ───────────────────────────────────────────────────────
|
|
40
|
+
|
|
41
|
+
/** Stage unwired (no port / no config) or `typed_judgment.enabled` is false. */
|
|
42
|
+
export const TYPED_JUDGMENT_DISABLED = 'typed_judgment_disabled';
|
|
43
|
+
/** Tool-result turns never invoke the judge (they pay zero judge latency). */
|
|
44
|
+
export const TYPED_JUDGMENT_TOOL_RESULT = 'typed_judgment_tool_result';
|
|
45
|
+
/** Acknowledgement-only turn (SP-306 `isAcknowledgement`) — no judgment needed. */
|
|
46
|
+
export const TYPED_JUDGMENT_ACKNOWLEDGEMENT = 'typed_judgment_acknowledgement';
|
|
47
|
+
/** Explicit force_model_id override — judgment would not influence routing. */
|
|
48
|
+
export const TYPED_JUDGMENT_FORCE_MODEL = 'typed_judgment_force_model';
|
|
49
|
+
/** Judge call hit the `timeout_ms` budget or was aborted. */
|
|
50
|
+
export const TYPED_JUDGMENT_TIMEOUT = 'typed_judgment_timeout';
|
|
51
|
+
/** Judge call or answer mapping failed (fail open). */
|
|
52
|
+
export const TYPED_JUDGMENT_ERROR = 'typed_judgment_error';
|
|
53
|
+
|
|
54
|
+
/** Skip result helper — records the reason and never decides. */
|
|
55
|
+
function skip(context: RoutingContext, reason: string): StageResult {
|
|
56
|
+
context.typedJudgment = null;
|
|
57
|
+
context.typedJudgmentSkipReason = reason;
|
|
58
|
+
context.typedJudgmentLatencyMs = null;
|
|
59
|
+
return { decided: false, stage: 'typed_judgment' };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Latest user message text (mirrors SP-306 `buildJudgeState` precedence):
|
|
64
|
+
* the last non-empty `user` message, else `prompt_text`. `prompt_text` is a
|
|
65
|
+
* required wire field, so every non-tool_result turn carries a user message.
|
|
66
|
+
*/
|
|
67
|
+
function resolveLatestUserMessageText(request: RoutingRequest): string {
|
|
68
|
+
const messages = request.messages;
|
|
69
|
+
if (messages) {
|
|
70
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
71
|
+
const message: Message = messages[i]!;
|
|
72
|
+
if (message.role === 'user' && message.content.trim()) {
|
|
73
|
+
return message.content;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return request.prompt_text;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Judge-call timeout classification: explicit abort/timeout kinds or a fired signal. */
|
|
81
|
+
function isTimeoutLike(error: unknown, signal: AbortSignal): boolean {
|
|
82
|
+
if (signal.aborted) return true;
|
|
83
|
+
return (
|
|
84
|
+
error instanceof TypedJudgmentError &&
|
|
85
|
+
(error.kind === 'timeout' || error.kind === 'aborted')
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Step 2a+: compute the typed judgment for user-prompt turns.
|
|
91
|
+
* Skip reasons: disabled / tool_result / acknowledgement / force_model;
|
|
92
|
+
* failure reasons: timeout / error. Always `decided: false` (fail open).
|
|
93
|
+
*/
|
|
94
|
+
export function createTypedJudgmentStage(): PipelineStage {
|
|
95
|
+
return {
|
|
96
|
+
name: 'typed_judgment',
|
|
97
|
+
async run(context: RoutingContext): Promise<StageResult> {
|
|
98
|
+
const { request } = context;
|
|
99
|
+
const port = context.options.typedJudgmentPort;
|
|
100
|
+
const config = context.options.typedJudgmentConfig;
|
|
101
|
+
|
|
102
|
+
if (!port || !config || !config.enabled) {
|
|
103
|
+
return skip(context, TYPED_JUDGMENT_DISABLED);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// SP-209 parity with turn_envelope: an explicit force is resolved by
|
|
107
|
+
// session_pin; a judgment cannot influence that path.
|
|
108
|
+
if (request.force_model_id) {
|
|
109
|
+
return skip(context, TYPED_JUDGMENT_FORCE_MODEL);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const turnType = request.turn_type ?? classifyTurnEnvelope(request.messages);
|
|
113
|
+
if (turnType === 'tool_result') {
|
|
114
|
+
return skip(context, TYPED_JUDGMENT_TOOL_RESULT);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const userText = resolveLatestUserMessageText(request);
|
|
118
|
+
// An empty user message (degenerate blank prompt) carries no judgeable
|
|
119
|
+
// instruction — treat it like an acknowledgement-class no-op turn.
|
|
120
|
+
if (
|
|
121
|
+
!userText.trim() ||
|
|
122
|
+
isAcknowledgement(userText, {
|
|
123
|
+
sessionDepth: request.messages?.length ?? 0,
|
|
124
|
+
})
|
|
125
|
+
) {
|
|
126
|
+
return skip(context, TYPED_JUDGMENT_ACKNOWLEDGEMENT);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const controller = new AbortController();
|
|
130
|
+
const timer = setTimeout(() => {
|
|
131
|
+
controller.abort();
|
|
132
|
+
}, config.timeout_ms);
|
|
133
|
+
|
|
134
|
+
const startedAt = Date.now();
|
|
135
|
+
try {
|
|
136
|
+
const state = buildJudgeState(request);
|
|
137
|
+
const answers = await port.judge(state, TYPED_JUDGMENT_QUESTIONS, {
|
|
138
|
+
signal: controller.signal,
|
|
139
|
+
});
|
|
140
|
+
context.typedJudgment = mapTypedJudgment(answers);
|
|
141
|
+
context.typedJudgmentSkipReason = null;
|
|
142
|
+
context.typedJudgmentLatencyMs = Date.now() - startedAt;
|
|
143
|
+
return { decided: false, stage: 'typed_judgment' };
|
|
144
|
+
} catch (error: unknown) {
|
|
145
|
+
const reason = isTimeoutLike(error, controller.signal)
|
|
146
|
+
? TYPED_JUDGMENT_TIMEOUT
|
|
147
|
+
: TYPED_JUDGMENT_ERROR;
|
|
148
|
+
const latencyMs = Date.now() - startedAt;
|
|
149
|
+
console.warn('typed_judgment stage failed; continuing without judgment', {
|
|
150
|
+
stage: 'typed_judgment',
|
|
151
|
+
reason,
|
|
152
|
+
request_id: request.request_id,
|
|
153
|
+
session_id: request.session_id,
|
|
154
|
+
error: redactPromptFromError(error, request.prompt_text),
|
|
155
|
+
});
|
|
156
|
+
const result = skip(context, reason);
|
|
157
|
+
// Judge was invoked — keep latency even though the outcome is a skip.
|
|
158
|
+
context.typedJudgmentLatencyMs = latencyMs;
|
|
159
|
+
return result;
|
|
160
|
+
} finally {
|
|
161
|
+
clearTimeout(timer);
|
|
162
|
+
}
|
|
163
|
+
},
|
|
164
|
+
};
|
|
165
|
+
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed judgment port (SP-304, #184 partial — domain side).
|
|
3
|
+
*
|
|
4
|
+
* Domain-owned contract for asking a calibrated, schema-safe semantic
|
|
5
|
+
* question about a turn (Jev-compatible "systemone" typed judgments). Unlike
|
|
6
|
+
* {@link HttpFetchPort} in `local-runtime-port.ts` (GET-style pings), this
|
|
7
|
+
* port carries structured questions and returns typed answers keyed by
|
|
8
|
+
* question id.
|
|
9
|
+
*
|
|
10
|
+
* This module is **types only** — it performs no I/O. The HTTP adapter lands
|
|
11
|
+
* in SP-305 (`src/infrastructure/`), and pipeline wiring lands in
|
|
12
|
+
* SP-307/SP-308. Dependency direction: domain owns the port; infrastructure
|
|
13
|
+
* implements.
|
|
14
|
+
*
|
|
15
|
+
* Fail-open contract: `judge` may reject (always with a
|
|
16
|
+
* {@link TypedJudgmentError}). Callers treat any rejection as "no judgment
|
|
17
|
+
* available" and continue routing without it — a typed-judgment outage must
|
|
18
|
+
* never break routing (constitution §11: fail secure by default).
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
// ─── Question wire types ─────────────────────────────────────────────────────
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Stable question identifier. Answers in {@link TypedJudgmentAnswers} are
|
|
25
|
+
* keyed by this id so callers correlate questions and answers without
|
|
26
|
+
* relying on array order.
|
|
27
|
+
*/
|
|
28
|
+
export type TypedJudgmentQuestionId = string;
|
|
29
|
+
|
|
30
|
+
/** Shared fields for every Jev question kind. */
|
|
31
|
+
interface TypedJudgmentQuestionBase {
|
|
32
|
+
/** Stable caller-assigned id; echoed back as the answer map key. */
|
|
33
|
+
readonly id: TypedJudgmentQuestionId;
|
|
34
|
+
/** Natural-language question text presented to the judge. */
|
|
35
|
+
readonly prompt: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Categorical question: pick exactly one of `options`. */
|
|
39
|
+
export interface TypedJudgmentChoiceQuestion extends TypedJudgmentQuestionBase {
|
|
40
|
+
readonly kind: 'choice';
|
|
41
|
+
/** Candidate labels; the answer's `choice` is one of these strings. */
|
|
42
|
+
readonly options: readonly string[];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Anchor describing one level of an ordinal score scale. */
|
|
46
|
+
export interface TypedJudgmentScoreAnchor {
|
|
47
|
+
/** Ordinal score value. */
|
|
48
|
+
readonly score: number;
|
|
49
|
+
/** Human-readable anchor description for this level (e.g. "fully stated"). */
|
|
50
|
+
readonly description: string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Ordinal question: assign a score on an anchored scale. */
|
|
54
|
+
export interface TypedJudgmentScoreQuestion extends TypedJudgmentQuestionBase {
|
|
55
|
+
readonly kind: 'score';
|
|
56
|
+
/** Anchored levels defining the scale; at least two, ascending. */
|
|
57
|
+
readonly anchors: readonly TypedJudgmentScoreAnchor[];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Binary ("noul") question: yes/no judged against explicit true/false
|
|
62
|
+
* criteria so the judge cannot free-style the boundary. The answer is the
|
|
63
|
+
* calibrated probability that the true-criterion holds (P(yes) ∈ [0,1]),
|
|
64
|
+
* not a thresholded boolean — see {@link TypedJudgmentNoulAnswer}.
|
|
65
|
+
*/
|
|
66
|
+
export interface TypedJudgmentNoulQuestion extends TypedJudgmentQuestionBase {
|
|
67
|
+
readonly kind: 'noul';
|
|
68
|
+
/** Criterion describing the event whose probability (P(yes)) is estimated. */
|
|
69
|
+
readonly criteria_true: string;
|
|
70
|
+
/** Criterion describing the complement event (probability 1 − P(yes)). */
|
|
71
|
+
readonly criteria_false: string;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The three Jev answer kinds. */
|
|
75
|
+
export type TypedJudgmentQuestion =
|
|
76
|
+
| TypedJudgmentChoiceQuestion
|
|
77
|
+
| TypedJudgmentScoreQuestion
|
|
78
|
+
| TypedJudgmentNoulQuestion;
|
|
79
|
+
|
|
80
|
+
// ─── Answer types ────────────────────────────────────────────────────────────
|
|
81
|
+
|
|
82
|
+
/** Answer to a {@link TypedJudgmentChoiceQuestion}. */
|
|
83
|
+
export interface TypedJudgmentChoiceAnswer {
|
|
84
|
+
readonly kind: 'choice';
|
|
85
|
+
/** The selected option (one of the question's `options`). */
|
|
86
|
+
readonly choice: string;
|
|
87
|
+
/** Calibrated per-option probabilities (keys = question options). */
|
|
88
|
+
readonly probabilities: Readonly<Record<string, number>>;
|
|
89
|
+
/** Judge confidence in the selection (0–1). */
|
|
90
|
+
readonly confidence: number;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Answer to a {@link TypedJudgmentScoreQuestion}. */
|
|
94
|
+
export interface TypedJudgmentScoreAnswer {
|
|
95
|
+
readonly kind: 'score';
|
|
96
|
+
/** The assigned ordinal score. */
|
|
97
|
+
readonly score: number;
|
|
98
|
+
/** Score → anchor description legend echoed by the judge. */
|
|
99
|
+
readonly legend: Readonly<Record<string, string>>;
|
|
100
|
+
/** Calibrated per-score probabilities (keys = stringified scores). */
|
|
101
|
+
readonly probabilities: Readonly<Record<string, number>>;
|
|
102
|
+
/** Judge confidence in the score (0–1). */
|
|
103
|
+
readonly confidence: number;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Answer to a {@link TypedJudgmentNoulQuestion}. */
|
|
107
|
+
export interface TypedJudgmentNoulAnswer {
|
|
108
|
+
readonly kind: 'noul';
|
|
109
|
+
/**
|
|
110
|
+
* Calibrated probability that the true-criterion holds — P(yes) ∈ [0,1],
|
|
111
|
+
* as returned on the Jev wire (`{noul: P(yes)}`). Kept as a probability
|
|
112
|
+
* (not a thresholded boolean) so callers can apply their own confidence
|
|
113
|
+
* gates; SP-305 corrected this from `boolean` to match the wire contract.
|
|
114
|
+
*/
|
|
115
|
+
readonly noul: number;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export type TypedJudgmentAnswer =
|
|
119
|
+
| TypedJudgmentChoiceAnswer
|
|
120
|
+
| TypedJudgmentScoreAnswer
|
|
121
|
+
| TypedJudgmentNoulAnswer;
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Answers returned by {@link TypedJudgmentPort.judge}, keyed by the question
|
|
125
|
+
* `id` supplied by the caller. A question with no entry was not answered
|
|
126
|
+
* (partial results are allowed; callers treat missing ids as no-signal).
|
|
127
|
+
*/
|
|
128
|
+
export type TypedJudgmentAnswers = Readonly<
|
|
129
|
+
Record<TypedJudgmentQuestionId, TypedJudgmentAnswer>
|
|
130
|
+
>;
|
|
131
|
+
|
|
132
|
+
// ─── Error shape ─────────────────────────────────────────────────────────────
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Machine-readable failure classification for typed-judgment calls. The
|
|
136
|
+
* adapter (SP-305) maps HTTP status codes and transport failures onto these
|
|
137
|
+
* kinds so callers branch on `kind` without parsing provider error text
|
|
138
|
+
* (which may contain request/response content and must not leak).
|
|
139
|
+
*/
|
|
140
|
+
export type TypedJudgmentErrorKind =
|
|
141
|
+
| 'bad_request'
|
|
142
|
+
| 'validation'
|
|
143
|
+
| 'auth'
|
|
144
|
+
| 'rate_limited'
|
|
145
|
+
| 'overloaded'
|
|
146
|
+
| 'timeout'
|
|
147
|
+
| 'aborted'
|
|
148
|
+
| 'invalid_response'
|
|
149
|
+
| 'transport';
|
|
150
|
+
|
|
151
|
+
export class TypedJudgmentError extends Error {
|
|
152
|
+
readonly kind: TypedJudgmentErrorKind;
|
|
153
|
+
|
|
154
|
+
constructor(kind: TypedJudgmentErrorKind, message: string, options?: ErrorOptions) {
|
|
155
|
+
super(message, options);
|
|
156
|
+
this.name = 'TypedJudgmentError';
|
|
157
|
+
this.kind = kind;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// ─── Port ────────────────────────────────────────────────────────────────────
|
|
162
|
+
|
|
163
|
+
export interface TypedJudgmentJudgeOptions {
|
|
164
|
+
/** Caller-owned abort signal (turn deadline / cancellation). */
|
|
165
|
+
readonly signal: AbortSignal;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Port for typed (schema-safe) semantic judgments about a turn.
|
|
170
|
+
*
|
|
171
|
+
* The adapter is injected by the composition root; the pipeline consumes the
|
|
172
|
+
* port via options (SP-307). Implementations must honor `options.signal` and
|
|
173
|
+
* reject with {@link TypedJudgmentError} (`kind: 'aborted'`) on abort.
|
|
174
|
+
*
|
|
175
|
+
* Fail-open contract: any rejection means "no judgment available" — callers
|
|
176
|
+
* continue routing without typed-judgment signal. Implementations must never
|
|
177
|
+
* resolve with malformed answers; schema failures reject with
|
|
178
|
+
* `kind: 'invalid_response'`.
|
|
179
|
+
*/
|
|
180
|
+
export interface TypedJudgmentPort {
|
|
181
|
+
judge(
|
|
182
|
+
state: string,
|
|
183
|
+
questions: readonly TypedJudgmentQuestion[],
|
|
184
|
+
options: TypedJudgmentJudgeOptions,
|
|
185
|
+
): Promise<TypedJudgmentAnswers>;
|
|
186
|
+
}
|
|
@@ -37,6 +37,7 @@ export type RoutingStage =
|
|
|
37
37
|
| 'hardware_probe'
|
|
38
38
|
| 'loop_escalation'
|
|
39
39
|
| 'turn_envelope'
|
|
40
|
+
| 'typed_judgment'
|
|
40
41
|
| 'context_fit'
|
|
41
42
|
| 'low_intensity'
|
|
42
43
|
| 'session_pin'
|
|
@@ -354,6 +355,51 @@ export interface RequirementVector {
|
|
|
354
355
|
readonly tool_use: number;
|
|
355
356
|
}
|
|
356
357
|
|
|
358
|
+
/**
|
|
359
|
+
* Normalized typed-judgment signal sidecar (SP-308, #187).
|
|
360
|
+
* Mirrors the SP-306 `TypedJudgment` signal plus stage observability
|
|
361
|
+
* (skip reason, judge latency). Privacy-safe: categorical labels, calibrated
|
|
362
|
+
* 0–1 scores, and machine reason codes only — no prompt text, judge-state
|
|
363
|
+
* text, or question prompts. The `task_kind` union mirrors `TaskKind` in
|
|
364
|
+
* src/domain/judgment/questions.ts (entities.ts is import-free).
|
|
365
|
+
*/
|
|
366
|
+
export interface TypedJudgmentSidecar {
|
|
367
|
+
/** Selected task kind, or null when unanswered or the stage skipped. */
|
|
368
|
+
readonly task_kind:
|
|
369
|
+
| 'plan'
|
|
370
|
+
| 'implement'
|
|
371
|
+
| 'debug'
|
|
372
|
+
| 'refactor'
|
|
373
|
+
| 'review'
|
|
374
|
+
| 'research'
|
|
375
|
+
| 'explain'
|
|
376
|
+
| 'operate'
|
|
377
|
+
| 'write'
|
|
378
|
+
| 'chat'
|
|
379
|
+
| null;
|
|
380
|
+
/** Judge confidence in `task_kind` (0–1), or null. */
|
|
381
|
+
readonly task_kind_confidence: number | null;
|
|
382
|
+
/** `capability_deserved` normalized to 0–1, or null. */
|
|
383
|
+
readonly capability_deserved: number | null;
|
|
384
|
+
/** P(needs deep reasoning) — noul calibrated probability, or null. */
|
|
385
|
+
readonly deep_reasoning_p: number | null;
|
|
386
|
+
/** `code_gen` normalized to 0–1, or null. */
|
|
387
|
+
readonly code_gen: number | null;
|
|
388
|
+
/** `tool_use` normalized to 0–1, or null. */
|
|
389
|
+
readonly tool_use: number | null;
|
|
390
|
+
/** P(destructive) — noul calibrated probability, or null. */
|
|
391
|
+
readonly destructive_p: number | null;
|
|
392
|
+
/**
|
|
393
|
+
* Judge confidence (0–1) per answered question id (choice/score answers
|
|
394
|
+
* only). Empty record when the stage skipped or no answer carried one.
|
|
395
|
+
*/
|
|
396
|
+
readonly confidences: Readonly<Record<string, number>>;
|
|
397
|
+
/** Why the stage skipped/failed (`typed_judgment_*` code), or null on success. */
|
|
398
|
+
readonly skip_reason: string | null;
|
|
399
|
+
/** Judge call latency in ms; null when the port was never invoked. */
|
|
400
|
+
readonly latency_ms: number | null;
|
|
401
|
+
}
|
|
402
|
+
|
|
357
403
|
/**
|
|
358
404
|
* Privacy-safe routing feature sidecar for dataset capture (SP-057).
|
|
359
405
|
* Metadata and routing signals only — no prompt text, messages, or tool arguments.
|
|
@@ -419,6 +465,14 @@ export interface RoutingFeatureSidecar {
|
|
|
419
465
|
* encoder (degrade, never mix; SP-292, #173). Null when no fallback fired.
|
|
420
466
|
*/
|
|
421
467
|
readonly cascade_fallback_reason?: string | null;
|
|
468
|
+
/**
|
|
469
|
+
* Typed-judgment signal sidecar (SP-308, #187). Present only when the
|
|
470
|
+
* typed_judgment stage was wired and enabled (judged turns carry the
|
|
471
|
+
* normalized signal; skipped turns carry the skip reason). Absent (not
|
|
472
|
+
* null) when the stage is unwired or disabled so the wire decision stays
|
|
473
|
+
* byte-identical (mirrors the embedding/cascade opt-in pattern).
|
|
474
|
+
*/
|
|
475
|
+
readonly typed_judgment?: TypedJudgmentSidecar;
|
|
422
476
|
}
|
|
423
477
|
|
|
424
478
|
export interface RoutingDecision {
|
|
@@ -45,6 +45,7 @@ export const RoutingStageSchema = z.enum([
|
|
|
45
45
|
'hardware_probe',
|
|
46
46
|
'loop_escalation',
|
|
47
47
|
'turn_envelope',
|
|
48
|
+
'typed_judgment',
|
|
48
49
|
'context_fit',
|
|
49
50
|
'low_intensity',
|
|
50
51
|
'session_pin',
|
|
@@ -311,6 +312,45 @@ export type Encoder = z.infer<typeof EncoderSchema>;
|
|
|
311
312
|
|
|
312
313
|
export const DEFAULT_ENCODER: Encoder = 'minilm';
|
|
313
314
|
|
|
315
|
+
/**
|
|
316
|
+
* Task-kind labels for the typed-judgment sidecar (SP-308, #187). Inlined to
|
|
317
|
+
* mirror `TASK_KIND_OPTIONS` in src/domain/judgment/questions.ts without an
|
|
318
|
+
* import cycle (schemas.ts stays domain-judgment-free).
|
|
319
|
+
*/
|
|
320
|
+
export const TypedJudgmentTaskKindSchema = z.enum([
|
|
321
|
+
'plan',
|
|
322
|
+
'implement',
|
|
323
|
+
'debug',
|
|
324
|
+
'refactor',
|
|
325
|
+
'review',
|
|
326
|
+
'research',
|
|
327
|
+
'explain',
|
|
328
|
+
'operate',
|
|
329
|
+
'write',
|
|
330
|
+
'chat',
|
|
331
|
+
]);
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Normalized typed-judgment signal sidecar (SP-308, #187). Privacy-safe:
|
|
335
|
+
* categorical labels, calibrated 0–1 scores, machine reason codes only — no
|
|
336
|
+
* prompt text or judge-state text. Mirrors {@link TypedJudgmentSidecar} in
|
|
337
|
+
* entities.ts.
|
|
338
|
+
*/
|
|
339
|
+
export const TypedJudgmentSidecarSchema = z
|
|
340
|
+
.object({
|
|
341
|
+
task_kind: TypedJudgmentTaskKindSchema.nullable(),
|
|
342
|
+
task_kind_confidence: z.number().min(0).max(1).nullable(),
|
|
343
|
+
capability_deserved: z.number().min(0).max(1).nullable(),
|
|
344
|
+
deep_reasoning_p: z.number().min(0).max(1).nullable(),
|
|
345
|
+
code_gen: z.number().min(0).max(1).nullable(),
|
|
346
|
+
tool_use: z.number().min(0).max(1).nullable(),
|
|
347
|
+
destructive_p: z.number().min(0).max(1).nullable(),
|
|
348
|
+
confidences: z.record(z.string(), z.number().min(0).max(1)),
|
|
349
|
+
skip_reason: z.string().nullable(),
|
|
350
|
+
latency_ms: z.number().nonnegative().nullable(),
|
|
351
|
+
})
|
|
352
|
+
.strict();
|
|
353
|
+
|
|
314
354
|
/**
|
|
315
355
|
* Privacy-safe routing feature sidecar attached to live decisions (SP-057).
|
|
316
356
|
* Mirrors {@link RoutingFeatureSidecar} in entities.ts.
|
|
@@ -345,6 +385,9 @@ export const RoutingFeatureSidecarSchema = z
|
|
|
345
385
|
token_estimate: z.number().nullable().optional(),
|
|
346
386
|
cascade_threshold: z.number().nullable().optional(),
|
|
347
387
|
cascade_fallback_reason: z.string().nullable().optional(),
|
|
388
|
+
// Typed-judgment signal sidecar (SP-308, #187); present only when the
|
|
389
|
+
// typed_judgment stage is wired and enabled, absent when disabled.
|
|
390
|
+
typed_judgment: TypedJudgmentSidecarSchema.optional(),
|
|
348
391
|
})
|
|
349
392
|
.strict();
|
|
350
393
|
|
|
@@ -998,6 +1041,167 @@ export const RoutingClustersFileSchema = z
|
|
|
998
1041
|
}
|
|
999
1042
|
});
|
|
1000
1043
|
|
|
1044
|
+
// ─── Typed judgment (SP-304, #184) ──────────────────────────────────────────
|
|
1045
|
+
|
|
1046
|
+
/** Loopback hosts accepted for `typed_judgment.endpoint` without `allow_remote`. */
|
|
1047
|
+
const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '::1']);
|
|
1048
|
+
|
|
1049
|
+
/** Parse an endpoint URL and return its normalized hostname (brackets stripped). */
|
|
1050
|
+
function parseEndpointHostname(endpoint: string): string | undefined {
|
|
1051
|
+
let url: URL;
|
|
1052
|
+
try {
|
|
1053
|
+
url = new URL(endpoint);
|
|
1054
|
+
} catch {
|
|
1055
|
+
return undefined;
|
|
1056
|
+
}
|
|
1057
|
+
const hostname = url.hostname;
|
|
1058
|
+
return hostname.startsWith('[') && hostname.endsWith(']')
|
|
1059
|
+
? hostname.slice(1, -1)
|
|
1060
|
+
: hostname;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
/**
|
|
1064
|
+
* Typed judgment (Jev-compatible "systemone") operator knobs (SP-304, #184).
|
|
1065
|
+
*
|
|
1066
|
+
* The router can ask a calibrated, schema-safe semantic judge about a turn
|
|
1067
|
+
* via `TypedJudgmentPort` (src/domain/ports/typed-judgment-port.ts). Default
|
|
1068
|
+
* OFF: existing installs pay zero added cost and no network call is made.
|
|
1069
|
+
*
|
|
1070
|
+
* Fail closed on remote endpoints: `superRefine` rejects any non-loopback
|
|
1071
|
+
* `endpoint` host unless the operator explicitly sets `allow_remote: true`
|
|
1072
|
+
* (local OpenJev is the intended default; remote hosted Jev is opt-in).
|
|
1073
|
+
*
|
|
1074
|
+
* The schema never holds an API key value — only the name of the env var
|
|
1075
|
+
* (`api_key_env`) the adapter reads the key from at call time.
|
|
1076
|
+
*/
|
|
1077
|
+
export const TypedJudgmentConfigSchema = z
|
|
1078
|
+
.object({
|
|
1079
|
+
/** When false (default), no typed-judgment calls are made. */
|
|
1080
|
+
enabled: z.boolean().default(false),
|
|
1081
|
+
/** Judge endpoint. Loopback-only unless `allow_remote` is true. */
|
|
1082
|
+
endpoint: z.string().min(1).default('http://127.0.0.1:8080/v1/systemone'),
|
|
1083
|
+
/** Judge model id (hosted Jev ignores this; OpenJev uses it). */
|
|
1084
|
+
model: z.string().min(1).default('openjev-latest'),
|
|
1085
|
+
/**
|
|
1086
|
+
* Per-call timeout (ms), sized for user-prompt turns: local OpenJev runs
|
|
1087
|
+
* ~385–990 ms per judgment (epic #183), so 1500 ms covers p99 with
|
|
1088
|
+
* headroom without stalling interactive routing.
|
|
1089
|
+
*/
|
|
1090
|
+
timeout_ms: z.number().int().positive().default(1500),
|
|
1091
|
+
/**
|
|
1092
|
+
* Judge samples per question (OpenJev extension for self-consistency
|
|
1093
|
+
* voting). The adapter omits this field for hosted Jev.
|
|
1094
|
+
*/
|
|
1095
|
+
samples: z.number().int().positive().default(1),
|
|
1096
|
+
/**
|
|
1097
|
+
* Name of the env var holding the API key (never the key itself).
|
|
1098
|
+
* Domain code and config never read or store key material.
|
|
1099
|
+
*/
|
|
1100
|
+
api_key_env: z.string().min(1).default('TYPESAFE_API_KEY'),
|
|
1101
|
+
/**
|
|
1102
|
+
* Opt-in escape hatch allowing non-loopback endpoints (hosted Jev).
|
|
1103
|
+
* Default false = fail closed to local judge only.
|
|
1104
|
+
*/
|
|
1105
|
+
allow_remote: z.boolean().default(false),
|
|
1106
|
+
/**
|
|
1107
|
+
* Minimum answer confidence (0–1) for a judgment to influence routing.
|
|
1108
|
+
* Consumed by later trains (#188); defined now to stabilize the config
|
|
1109
|
+
* shape.
|
|
1110
|
+
*/
|
|
1111
|
+
min_confidence: z.number().min(0).max(1).default(0.6),
|
|
1112
|
+
})
|
|
1113
|
+
.superRefine((value, ctx) => {
|
|
1114
|
+
const hostname = parseEndpointHostname(value.endpoint);
|
|
1115
|
+
if (hostname === undefined) {
|
|
1116
|
+
ctx.addIssue({
|
|
1117
|
+
code: z.ZodIssueCode.custom,
|
|
1118
|
+
message: 'typed_judgment.endpoint must be a valid URL',
|
|
1119
|
+
path: ['endpoint'],
|
|
1120
|
+
});
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
if (!value.allow_remote && !LOOPBACK_HOSTS.has(hostname)) {
|
|
1124
|
+
ctx.addIssue({
|
|
1125
|
+
code: z.ZodIssueCode.custom,
|
|
1126
|
+
message:
|
|
1127
|
+
'typed_judgment.endpoint must be loopback (127.0.0.1, ::1, or localhost) ' +
|
|
1128
|
+
'unless allow_remote is true (fail closed: remote judges are opt-in)',
|
|
1129
|
+
path: ['endpoint'],
|
|
1130
|
+
});
|
|
1131
|
+
}
|
|
1132
|
+
});
|
|
1133
|
+
|
|
1134
|
+
export type TypedJudgmentConfig = z.infer<typeof TypedJudgmentConfigSchema>;
|
|
1135
|
+
|
|
1136
|
+
/** Typed judgment defaults per #184 (SP-304): disabled, loopback-only. */
|
|
1137
|
+
export const DEFAULT_TYPED_JUDGMENT_CONFIG: Readonly<TypedJudgmentConfig> = {
|
|
1138
|
+
enabled: false,
|
|
1139
|
+
endpoint: 'http://127.0.0.1:8080/v1/systemone',
|
|
1140
|
+
model: 'openjev-latest',
|
|
1141
|
+
timeout_ms: 1500,
|
|
1142
|
+
samples: 1,
|
|
1143
|
+
api_key_env: 'TYPESAFE_API_KEY',
|
|
1144
|
+
allow_remote: false,
|
|
1145
|
+
min_confidence: 0.6,
|
|
1146
|
+
} as const;
|
|
1147
|
+
|
|
1148
|
+
/** Env: SMART_ROUTER_TYPED_JUDGMENT_ENABLED — master switch (default false). */
|
|
1149
|
+
const ENV_TYPED_JUDGMENT_ENABLED = 'SMART_ROUTER_TYPED_JUDGMENT_ENABLED';
|
|
1150
|
+
/** Env: SMART_ROUTER_TYPED_JUDGMENT_ENDPOINT — judge endpoint (loopback-only unless allow_remote). */
|
|
1151
|
+
const ENV_TYPED_JUDGMENT_ENDPOINT = 'SMART_ROUTER_TYPED_JUDGMENT_ENDPOINT';
|
|
1152
|
+
/** Env: SMART_ROUTER_TYPED_JUDGMENT_MODEL — judge model id (default openjev-latest). */
|
|
1153
|
+
const ENV_TYPED_JUDGMENT_MODEL = 'SMART_ROUTER_TYPED_JUDGMENT_MODEL';
|
|
1154
|
+
/** Env: SMART_ROUTER_TYPED_JUDGMENT_TIMEOUT_MS — per-call timeout ms (default 1500). */
|
|
1155
|
+
const ENV_TYPED_JUDGMENT_TIMEOUT_MS = 'SMART_ROUTER_TYPED_JUDGMENT_TIMEOUT_MS';
|
|
1156
|
+
/** Env: SMART_ROUTER_TYPED_JUDGMENT_SAMPLES — OpenJev samples per question (default 1). */
|
|
1157
|
+
const ENV_TYPED_JUDGMENT_SAMPLES = 'SMART_ROUTER_TYPED_JUDGMENT_SAMPLES';
|
|
1158
|
+
/** Env: SMART_ROUTER_TYPED_JUDGMENT_API_KEY_ENV — name of the env var holding the API key. */
|
|
1159
|
+
const ENV_TYPED_JUDGMENT_API_KEY_ENV = 'SMART_ROUTER_TYPED_JUDGMENT_API_KEY_ENV';
|
|
1160
|
+
/** Env: SMART_ROUTER_TYPED_JUDGMENT_ALLOW_REMOTE — permit non-loopback endpoints (default false). */
|
|
1161
|
+
const ENV_TYPED_JUDGMENT_ALLOW_REMOTE = 'SMART_ROUTER_TYPED_JUDGMENT_ALLOW_REMOTE';
|
|
1162
|
+
/** Env: SMART_ROUTER_TYPED_JUDGMENT_MIN_CONFIDENCE — min answer confidence 0–1 (default 0.6). */
|
|
1163
|
+
const ENV_TYPED_JUDGMENT_MIN_CONFIDENCE = 'SMART_ROUTER_TYPED_JUDGMENT_MIN_CONFIDENCE';
|
|
1164
|
+
|
|
1165
|
+
function readNonEmptyStringEnv(name: string): string | undefined {
|
|
1166
|
+
const raw = process.env[name];
|
|
1167
|
+
if (raw === undefined) {
|
|
1168
|
+
return undefined;
|
|
1169
|
+
}
|
|
1170
|
+
const trimmed = raw.trim();
|
|
1171
|
+
return trimmed === '' ? undefined : trimmed;
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
/**
|
|
1175
|
+
* Merge typed judgment env overrides onto defaults (invalid scalar env values
|
|
1176
|
+
* are ignored). An env `endpoint` override is re-validated against the same
|
|
1177
|
+
* loopback rule as the schema: if the merged result fails validation (e.g. a
|
|
1178
|
+
* remote endpoint without `allow_remote`), fall back to the base config and
|
|
1179
|
+
* keep `enabled: false` (fail closed — never route judgments to an endpoint
|
|
1180
|
+
* the operator did not explicitly clear).
|
|
1181
|
+
*/
|
|
1182
|
+
export function resolveTypedJudgmentConfigFromEnv(
|
|
1183
|
+
base: TypedJudgmentConfig = DEFAULT_TYPED_JUDGMENT_CONFIG,
|
|
1184
|
+
): TypedJudgmentConfig {
|
|
1185
|
+
const candidate: TypedJudgmentConfig = {
|
|
1186
|
+
enabled: readBooleanEnv(ENV_TYPED_JUDGMENT_ENABLED) ?? base.enabled,
|
|
1187
|
+
endpoint: readNonEmptyStringEnv(ENV_TYPED_JUDGMENT_ENDPOINT) ?? base.endpoint,
|
|
1188
|
+
model: readNonEmptyStringEnv(ENV_TYPED_JUDGMENT_MODEL) ?? base.model,
|
|
1189
|
+
timeout_ms: readPositiveIntEnv(ENV_TYPED_JUDGMENT_TIMEOUT_MS) ?? base.timeout_ms,
|
|
1190
|
+
samples: readPositiveIntEnv(ENV_TYPED_JUDGMENT_SAMPLES) ?? base.samples,
|
|
1191
|
+
api_key_env: readNonEmptyStringEnv(ENV_TYPED_JUDGMENT_API_KEY_ENV) ?? base.api_key_env,
|
|
1192
|
+
allow_remote: readBooleanEnv(ENV_TYPED_JUDGMENT_ALLOW_REMOTE) ?? base.allow_remote,
|
|
1193
|
+
min_confidence:
|
|
1194
|
+
readUnitIntervalEnv(ENV_TYPED_JUDGMENT_MIN_CONFIDENCE) ?? base.min_confidence,
|
|
1195
|
+
};
|
|
1196
|
+
const parsed = TypedJudgmentConfigSchema.safeParse(candidate);
|
|
1197
|
+
if (parsed.success) {
|
|
1198
|
+
return parsed.data;
|
|
1199
|
+
}
|
|
1200
|
+
// Fail closed: invalid env override (e.g. remote endpoint without
|
|
1201
|
+
// allow_remote) leaves the base config intact and disabled.
|
|
1202
|
+
return { ...base, enabled: false };
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1001
1205
|
export const OperatorConfigSchema = z.object({
|
|
1002
1206
|
frugality: FrugalityConfigSchema,
|
|
1003
1207
|
loop_escalation: LoopEscalationConfigSchema,
|
|
@@ -1019,6 +1223,8 @@ export const OperatorConfigSchema = z.object({
|
|
|
1019
1223
|
speculative_prewarm: SpeculativePrewarmConfigSchema.optional(),
|
|
1020
1224
|
/** Adaptive reasoning (thinking-level) policy knobs (SP-246, #166). Default on. */
|
|
1021
1225
|
adaptive_reasoning: AdaptiveReasoningConfigSchema.optional(),
|
|
1226
|
+
/** Typed judgment (Jev systemone) knobs (SP-304, #184). Default off; loopback-only unless allow_remote. */
|
|
1227
|
+
typed_judgment: TypedJudgmentConfigSchema.optional(),
|
|
1022
1228
|
/**
|
|
1023
1229
|
* Emergency pin-on-first-turn fallback (#83, SP-161).
|
|
1024
1230
|
* When true, subsequent turns use the session pin only — multi-stage routing
|