pi-crew 0.11.11 → 0.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -0
- package/dist/index.mjs +551 -519
- package/package.json +1 -1
- package/src/config/config-validation.ts +9 -0
- package/src/config/defaults.ts +7 -0
- package/src/config/env-vars.ts +24 -0
- package/src/config/types.ts +11 -0
- package/src/extension/registration/tool-registration.ts +7 -0
- package/src/extension/registration/verify-gate-tool.ts +263 -0
- package/src/extension/team-tool/run-deadline.ts +22 -1
- package/src/extension/team-tool/run.ts +38 -2
- package/src/extension/team-tool-types.ts +4 -0
- package/src/extension/team-tool.ts +31 -3
- package/src/prompt/inbox-poll.ts +141 -3
- package/src/runtime/broker/crew-broker.ts +48 -2
- package/src/runtime/broker/protocol/lock-busy.ts +47 -1
- package/src/runtime/broker/protocol/request-parsers.ts +8 -1
- package/src/runtime/budget-enforcement.ts +36 -20
- package/src/runtime/child-pi/child-pi-constants.ts +24 -0
- package/src/runtime/child-pi/child-pi-kill.ts +59 -2
- package/src/runtime/child-pi/child-pi-rpc.ts +1021 -0
- package/src/runtime/child-pi/child-pi-steering.ts +10 -8
- package/src/runtime/child-pi/child-pi-streams.ts +69 -19
- package/src/runtime/child-pi/child-pi.ts +29 -30
- package/src/runtime/child-pi/owned-process.ts +434 -0
- package/src/runtime/child-pi/postmortem-registry.ts +134 -0
- package/src/runtime/classifier/ambient-noise-gate.ts +126 -0
- package/src/runtime/classifier/classifier-service.ts +277 -61
- package/src/runtime/dispatch-batch.ts +32 -0
- package/src/runtime/live-session/live-agent-control.ts +2 -0
- package/src/runtime/live-session/live-agent-manager.ts +72 -2
- package/src/runtime/live-session/live-session-runtime.ts +435 -40
- package/src/runtime/merge-loop.ts +63 -35
- package/src/runtime/recovery/crash-recovery.ts +141 -3
- package/src/runtime/recovery/retry-executor.ts +29 -0
- package/src/runtime/run-tracker.ts +28 -0
- package/src/runtime/stale-reconciler.ts +36 -1
- package/src/runtime/task-runner.ts +103 -0
- package/src/runtime/verification/classifier-pre-gate.ts +262 -0
- package/src/runtime/verification/verification-gates.ts +5 -1
- package/src/runtime/verification/verify-gate.ts +394 -0
- package/src/schema/config-schema.ts +13 -0
- package/src/state/contracts.ts +18 -0
- package/src/state/coordination/mailbox.ts +203 -55
- package/src/state/stores/sqlite-run-state.ts +378 -0
- package/src/state/stores/state-store.ts +134 -6
- package/src/ui/agents-jobs-browser.ts +154 -8
- package/src/ui/events-state-source.ts +358 -0
- package/src/ui/run-snapshot-cache.ts +87 -3
- package/src/ui/widget/widget-model.ts +13 -2
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* postmortem-registry.ts — U15: postmortem cleanup registry (pi-crew port).
|
|
3
|
+
*
|
|
4
|
+
* Ported from gajae-code `packages/utils/src/postmortem.ts` (fork pi-mono
|
|
5
|
+
* v0.8.2, spec U15 of pi-crew-upgrade-spec-2026-10-09.md), reduced to the
|
|
6
|
+
* ~80-line core pi-crew needs: reap still-live owned process groups on fatal
|
|
7
|
+
* or normal shutdown. Owners deregister from this registry only AFTER their
|
|
8
|
+
* teardown completed (owned-process.ts owns that discipline).
|
|
9
|
+
*
|
|
10
|
+
* Divergence from the gajae-code original (deliberate, reviewed against the
|
|
11
|
+
* spec's "guest in host process" constraint):
|
|
12
|
+
* - The original OWNS its process: after cleanup it calls
|
|
13
|
+
* `process.exit(128+sig)` / `process.exit(1)`. pi-crew runs as an EXTENSION
|
|
14
|
+
* inside the pi host process (and standalone inside background-runner),
|
|
15
|
+
* where other pi-crew modules already register SIGINT/SIGTERM/SIGHUP/
|
|
16
|
+
* uncaught/unhandled listeners (event-log.ts, atomic-write.ts,
|
|
17
|
+
* crew-cleanup.ts, background-runner.ts) without forcing exit. Forcing
|
|
18
|
+
* exit here would race the host's own shutdown path, so this port NEVER
|
|
19
|
+
* calls process.exit — hooks reap best-effort while the real exit path
|
|
20
|
+
* drains, and the 'exit' listener fires a last sync-effort kick.
|
|
21
|
+
* - "run once per process" is relaxed to "concurrent runs share one
|
|
22
|
+
* in-flight promise" because a host can survive a SIGTERM-like signal and
|
|
23
|
+
* must stay re-armable; per-process once-only semantics are the caller's
|
|
24
|
+
* (owned groups deregister themselves after teardown, so a second run has
|
|
25
|
+
* nothing left to reap).
|
|
26
|
+
*
|
|
27
|
+
* Listener model (installed lazily on first registration):
|
|
28
|
+
* - exit → fire-and-forget async run (cannot await; hooks
|
|
29
|
+
* whose sync prefix sends SIGTERM still deliver).
|
|
30
|
+
* - SIGINT/SIGTERM/SIGHUP → setImmediate kick (house pattern from
|
|
31
|
+
* event-log.ts) so sync flush handlers run first.
|
|
32
|
+
* - uncaughtException/unhandledRejection → setImmediate kick, never swallow:
|
|
33
|
+
* we add no output and no exit; existing pi-crew
|
|
34
|
+
* fatal handlers already own crash reporting (and
|
|
35
|
+
* already suppress Node's default, see event-log).
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
39
|
+
|
|
40
|
+
export type PostmortemReason = "exit" | "SIGINT" | "SIGTERM" | "SIGHUP" | "uncaughtException" | "unhandledRejection" | "manual";
|
|
41
|
+
|
|
42
|
+
export type PostmortemHook = (reason: PostmortemReason) => void | Promise<void>;
|
|
43
|
+
|
|
44
|
+
// Registered hooks, keyed by owner name. Last-wins on re-registration (same
|
|
45
|
+
// discipline as gajae-code's resource-owner map): a re-adopt with the same name
|
|
46
|
+
// replaces the prior hook instead of accumulating duplicates.
|
|
47
|
+
const postmortemHooks = new Map<string, PostmortemHook>();
|
|
48
|
+
|
|
49
|
+
let listenersInstalled = false;
|
|
50
|
+
let inFlightRun: Promise<void> | undefined;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Run every registered postmortem hook for `reason`. Concurrent invocations
|
|
54
|
+
* share the single in-flight run (recursion/reentry guard); failures are
|
|
55
|
+
* isolated per hook and logged, never propagated — one bad owner must not
|
|
56
|
+
* block the others from reaping.
|
|
57
|
+
*/
|
|
58
|
+
export function runPostmortemCleanup(reason: PostmortemReason): Promise<void> {
|
|
59
|
+
if (inFlightRun) return inFlightRun;
|
|
60
|
+
inFlightRun = (async () => {
|
|
61
|
+
try {
|
|
62
|
+
await Promise.allSettled(
|
|
63
|
+
[...postmortemHooks.entries()].map(async ([name, hook]) => {
|
|
64
|
+
try {
|
|
65
|
+
await hook(reason);
|
|
66
|
+
} catch (err) {
|
|
67
|
+
logInternalError(
|
|
68
|
+
"child-pi.postmortem-hook",
|
|
69
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
70
|
+
`name=${name} reason=${reason}`,
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
}),
|
|
74
|
+
);
|
|
75
|
+
} finally {
|
|
76
|
+
inFlightRun = undefined;
|
|
77
|
+
}
|
|
78
|
+
})();
|
|
79
|
+
return inFlightRun;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Register a postmortem cleanup hook. Idempotent by `name`: re-registering
|
|
84
|
+
* replaces the prior hook (last wins). Returns an unregister function that
|
|
85
|
+
* removes the owner only while it is still the active registration for that
|
|
86
|
+
* name — a stale unregister cannot remove a newer registration.
|
|
87
|
+
*/
|
|
88
|
+
export function registerPostmortemCleanup(name: string, hook: PostmortemHook): () => void {
|
|
89
|
+
postmortemHooks.set(name, hook);
|
|
90
|
+
ensurePostmortemListeners();
|
|
91
|
+
let unregistered = false;
|
|
92
|
+
return () => {
|
|
93
|
+
if (unregistered) return;
|
|
94
|
+
unregistered = true;
|
|
95
|
+
// Only remove when this exact hook is still the active registration —
|
|
96
|
+
// guards the "late unregister must not evict a newer adopter" case.
|
|
97
|
+
if (postmortemHooks.get(name) === hook) postmortemHooks.delete(name);
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Number of registered postmortem hooks. Exposed for leak assertions/tests. */
|
|
102
|
+
export function postmortemHookCount(): number {
|
|
103
|
+
return postmortemHooks.size;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Install the process-level listeners exactly once, on first registration. */
|
|
107
|
+
function ensurePostmortemListeners(): void {
|
|
108
|
+
if (listenersInstalled) return;
|
|
109
|
+
listenersInstalled = true;
|
|
110
|
+
// 'exit' is synchronous: an async hook can only run its sync prefix here
|
|
111
|
+
// (e.g. signalTree("SIGTERM") inside dispose()) before the process dies.
|
|
112
|
+
// The remaining escalation is covered owner-side while alive + child-side
|
|
113
|
+
// by the PI_CREW_PARENT_PID guard (complementary per spec U15).
|
|
114
|
+
process.on("exit", () => {
|
|
115
|
+
void runPostmortemCleanup("exit");
|
|
116
|
+
});
|
|
117
|
+
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
|
|
118
|
+
process.on(signal, () => {
|
|
119
|
+
setImmediate(() => {
|
|
120
|
+
void runPostmortemCleanup(signal);
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
process.on("uncaughtException", () => {
|
|
125
|
+
setImmediate(() => {
|
|
126
|
+
void runPostmortemCleanup("uncaughtException");
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
process.on("unhandledRejection", () => {
|
|
130
|
+
setImmediate(() => {
|
|
131
|
+
void runPostmortemCleanup("unhandledRejection");
|
|
132
|
+
});
|
|
133
|
+
});
|
|
134
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* U6A (upgrade spec 2026-10-09, §U6 Phase A): generalized "ambient-noise vs
|
|
3
|
+
* real-failure" classifier gate for the DISPATCH-BATCH retry loop —
|
|
4
|
+
* the sibling of child-executor's model-fallback `triageRetryWithClassifier`
|
|
5
|
+
* (retry-triage). Same never-rejects contract, same dormant default
|
|
6
|
+
* (`runtime.classifierEnabled`, default FALSE — resolveClassifierEnabled),
|
|
7
|
+
* one shared flag/model for both consumers.
|
|
8
|
+
*
|
|
9
|
+
* Semantics: after a failed task attempt and BEFORE the next (expensive)
|
|
10
|
+
* worker re-spawn is queued, ask the host classifier ONE bool question — is
|
|
11
|
+
* the failure AMBIENT NOISE (transient/environmental: rate limit, provider
|
|
12
|
+
* hiccup, timeout, network blip — a fresh attempt has a realistic chance)
|
|
13
|
+
* rather than a REAL FAILURE (deterministic: auth/permission denied, invalid
|
|
14
|
+
* request, missing resource — the retry will fail the same way)?
|
|
15
|
+
*
|
|
16
|
+
* SOFT-BRANCH CONTRACT (spec caveat, mandatory):
|
|
17
|
+
* - opencode/provider unconfigured (`getAvailableOfType("classifier")` →
|
|
18
|
+
* [], classify stopReason "error") → caller fallback;
|
|
19
|
+
* - classify throws / registry absent → caller fallback;
|
|
20
|
+
* - fallback on EVERY failure mode is `proceedWithRetry: true` — the
|
|
21
|
+
* pre-existing retry behavior. A wrong classifier verdict can only SKIP
|
|
22
|
+
* one queued retry (the task surfaces its original error — the same
|
|
23
|
+
* outcome as retry exhaustion); it can NEVER fail a task with a new
|
|
24
|
+
* error or block the loop.
|
|
25
|
+
*
|
|
26
|
+
* Diagnostics (confidence monitoring, spec Phase A):
|
|
27
|
+
* - `crew.classifier.calls_total{consumer="ambient_noise_gate",outcome}`
|
|
28
|
+
* + `crew.classifier.confidence` histogram via classifyBool;
|
|
29
|
+
* - `crew.classifier.decisions_total{consumer,decision}` and a
|
|
30
|
+
* `task.noise_gate` event ONLY when a real classifier answer informed
|
|
31
|
+
* the decision (fallback paths are already counted by the service's
|
|
32
|
+
* calls_total and must not double-report a decision they did not make).
|
|
33
|
+
*/
|
|
34
|
+
import type { MetricRegistry } from "../../observability/metric-registry.ts";
|
|
35
|
+
import { appendEventAsync } from "../../state/event-log/event-log.ts";
|
|
36
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
37
|
+
import { classifyBool } from "./classifier-service.ts";
|
|
38
|
+
|
|
39
|
+
export interface AmbientNoiseGateArgs {
|
|
40
|
+
/** Pre-resolved gate (resolveClassifierEnabled) — false short-circuits before ANY registry touch. */
|
|
41
|
+
enabled: boolean;
|
|
42
|
+
/** Host-process model registry (ctx.modelRegistry, threaded unknown). */
|
|
43
|
+
modelRegistry: unknown;
|
|
44
|
+
/** Pre-resolved classifier model id (resolveClassifierModel). */
|
|
45
|
+
classifierModel: string;
|
|
46
|
+
/** Failure summary the retry loop computed (the thrown attempt error message). */
|
|
47
|
+
failureSummary: string;
|
|
48
|
+
/** 1-based index of the attempt that just failed. */
|
|
49
|
+
attempt: number;
|
|
50
|
+
/** Retry policy maxAttempts (context for the classifier, not enforced here). */
|
|
51
|
+
maxAttempts: number;
|
|
52
|
+
/** Event-log identity for the task.noise_gate diagnostic event. */
|
|
53
|
+
eventsPath: string;
|
|
54
|
+
runId: string;
|
|
55
|
+
taskId: string;
|
|
56
|
+
/** Optional host metric registry (crew.classifier.* counters/histograms). */
|
|
57
|
+
metricRegistry?: MetricRegistry;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface AmbientNoiseGateResult {
|
|
61
|
+
/** false → skip the queued retry attempt (classifier judged the failure real/deterministic). */
|
|
62
|
+
proceedWithRetry: boolean;
|
|
63
|
+
/** true ONLY when a real classifier answer informed the decision (event + decision metric emitted). */
|
|
64
|
+
consulted: boolean;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Ask the ambient-noise gate. NEVER REJECTS and never throws up to the
|
|
69
|
+
* caller — the disabled path returns without touching the registry (zero
|
|
70
|
+
* behavior change), every classifier failure mode falls back to
|
|
71
|
+
* `proceedWithRetry: true`.
|
|
72
|
+
*/
|
|
73
|
+
export async function classifyAmbientNoiseGate(args: AmbientNoiseGateArgs): Promise<AmbientNoiseGateResult> {
|
|
74
|
+
if (!args.enabled) return { proceedWithRetry: true, consulted: false };
|
|
75
|
+
const result = await classifyBool({
|
|
76
|
+
modelRegistry: args.modelRegistry,
|
|
77
|
+
classifierModel: args.classifierModel,
|
|
78
|
+
questionKey: "ambient",
|
|
79
|
+
question: {
|
|
80
|
+
type: "bool",
|
|
81
|
+
instructions:
|
|
82
|
+
"A dispatched worker task attempt failed and an automatic retry is queued (the worker will be re-spawned from scratch). Read the failure state. Is the failure AMBIENT NOISE (transient or environmental — rate limit, provider hiccup, timeout, network error, stale lock — a fresh attempt has a realistic chance of success) rather than a REAL FAILURE (deterministic — auth or permission denied, invalid request, missing resource, code or prompt defect — the retry will fail the same way)?",
|
|
83
|
+
criteria: {
|
|
84
|
+
true: "Ambient noise — the queued retry may succeed",
|
|
85
|
+
false: "Real failure — the queued retry will fail the same way",
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
state: {
|
|
89
|
+
// Cap the summary: classifiers reason over small JSON state — the raw
|
|
90
|
+
// error can carry multi-KB stderr tails (same cap as retry-triage).
|
|
91
|
+
failureSummary: args.failureSummary.slice(0, 2000),
|
|
92
|
+
attempt: args.attempt,
|
|
93
|
+
maxAttempts: args.maxAttempts,
|
|
94
|
+
},
|
|
95
|
+
fallback: true,
|
|
96
|
+
metricRegistry: args.metricRegistry,
|
|
97
|
+
metricLabels: { consumer: "ambient_noise_gate" },
|
|
98
|
+
});
|
|
99
|
+
if (!result.fromClassifier) return { proceedWithRetry: true, consulted: false };
|
|
100
|
+
const decision = result.decision ? "noise_retry" : "real_giveup";
|
|
101
|
+
try {
|
|
102
|
+
args.metricRegistry?.counter("crew.classifier.decisions_total", "Classifier-informed decisions by consumer and verdict").inc({
|
|
103
|
+
consumer: "ambient_noise_gate",
|
|
104
|
+
decision,
|
|
105
|
+
});
|
|
106
|
+
} catch (metricError) {
|
|
107
|
+
logInternalError("ambient-noise-gate.metric", metricError, `decision=${decision}`, "warn");
|
|
108
|
+
}
|
|
109
|
+
void appendEventAsync(args.eventsPath, {
|
|
110
|
+
type: "task.noise_gate",
|
|
111
|
+
runId: args.runId,
|
|
112
|
+
taskId: args.taskId,
|
|
113
|
+
message: `Ambient-noise gate classified the attempt-${args.attempt} failure as ${decision} (classifier ${result.usedModel ?? args.classifierModel})`,
|
|
114
|
+
data: {
|
|
115
|
+
decision,
|
|
116
|
+
attempt: args.attempt,
|
|
117
|
+
classifierModel: result.usedModel,
|
|
118
|
+
configuredModel: args.classifierModel,
|
|
119
|
+
stopReason: result.stopReason,
|
|
120
|
+
...(result.confidence !== undefined ? { confidence: result.confidence } : {}),
|
|
121
|
+
},
|
|
122
|
+
}).catch(() => {
|
|
123
|
+
/* no-op: best-effort diagnostic append, ignore delivery errors */
|
|
124
|
+
});
|
|
125
|
+
return { proceedWithRetry: result.decision, consulted: true };
|
|
126
|
+
}
|
|
@@ -39,18 +39,36 @@ export interface ClassifierBoolQuestion {
|
|
|
39
39
|
criteria?: { true: string; false: string };
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
+
/** U6B (upgrade spec 2026-10-09 §U6 Phase B): one typed SCORE question — the
|
|
43
|
+
* graded sibling of {@link ClassifierBoolQuestion}, used by the verifier
|
|
44
|
+
* pre-gate. Mirrors pi-ai's ClassifierScoreQuestion (pi-ai dist/types.d.ts:
|
|
45
|
+
* {type:"score", instructions, criteria: string[]}); answers arrive as the
|
|
46
|
+
* ClassifierScoreAnswer shape {type:"score", score, confidence} — score is
|
|
47
|
+
* the graded 0..1 reading, confidence the provider-reported certainty. */
|
|
48
|
+
export interface ClassifierScoreQuestion {
|
|
49
|
+
type: "score";
|
|
50
|
+
instructions: string;
|
|
51
|
+
/** Rubric anchors for the 0..1 scale, ordered decisive (1.0) → useless (0.0). */
|
|
52
|
+
criteria: string[];
|
|
53
|
+
}
|
|
54
|
+
|
|
42
55
|
/** Request body for modelRegistry.classify(model, request). */
|
|
43
56
|
export interface ClassifyRequest {
|
|
44
57
|
state: Record<string, unknown>;
|
|
45
|
-
questions: Record<string, ClassifierBoolQuestion>;
|
|
58
|
+
questions: Record<string, ClassifierBoolQuestion | ClassifierScoreQuestion>;
|
|
46
59
|
}
|
|
47
60
|
|
|
48
61
|
/** Structured result shape returned by modelRegistry.classify (never throws
|
|
49
|
-
* per model-registry.d.ts — but we still guard a throwing implementation).
|
|
62
|
+
* per model-registry.d.ts — but we still guard a throwing implementation).
|
|
63
|
+
* U6A (spec 2026-10-09 §U6 Phase A): `confidence` is a passthrough — the
|
|
64
|
+
* classify API documents answers "kèm confidence" (research §8.2); when a
|
|
65
|
+
* provider reports it (top-level or per-answer `{value, confidence}`), we
|
|
66
|
+
* surface it on the result + `crew.classifier.confidence` histogram. */
|
|
50
67
|
export interface ClassifyCallResult {
|
|
51
68
|
answers?: Record<string, unknown>;
|
|
52
69
|
stopReason?: string;
|
|
53
70
|
errorMessage?: string;
|
|
71
|
+
confidence?: number;
|
|
54
72
|
}
|
|
55
73
|
|
|
56
74
|
/** Duck-typed host registry handle (mirrors ModelRegistryLike in
|
|
@@ -101,6 +119,47 @@ export interface ClassifyBoolResult {
|
|
|
101
119
|
/** Raw stopReason / errorMessage from the classify call, when one ran. */
|
|
102
120
|
stopReason: string | undefined;
|
|
103
121
|
errorMessage: string | undefined;
|
|
122
|
+
/** U6A: provider-reported answer confidence, when the registry supplied
|
|
123
|
+
* one (top-level `confidence` or per-answer `{value, confidence}`).
|
|
124
|
+
* Undefined on every fallback path and for bare-boolean answers. */
|
|
125
|
+
confidence: number | undefined;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export interface ClassifyScoreInput {
|
|
129
|
+
/** Host-process model registry handle (ctx.modelRegistry, threaded unknown). */
|
|
130
|
+
modelRegistry: unknown;
|
|
131
|
+
/** Configured classifier model id ("provider/id"). */
|
|
132
|
+
classifierModel: string;
|
|
133
|
+
/** Key under `questions` carrying the score question. */
|
|
134
|
+
questionKey: string;
|
|
135
|
+
question: ClassifierScoreQuestion;
|
|
136
|
+
/** JSON state the classifier reasons about (keep small + low-cardinality). */
|
|
137
|
+
state: Record<string, unknown>;
|
|
138
|
+
/** Caller-provided fallback score — returned on EVERY failure mode. */
|
|
139
|
+
fallback: number;
|
|
140
|
+
/** Optional metric registry (counter created on demand, dispatch-batch style). */
|
|
141
|
+
metricRegistry?: MetricRegistry;
|
|
142
|
+
/** Extra low-cardinality labels for the calls_total counter. */
|
|
143
|
+
metricLabels?: Record<string, string>;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export interface ClassifyScoreResult {
|
|
147
|
+
/** Effective graded reading: the classifier score, or the caller fallback. */
|
|
148
|
+
score: number;
|
|
149
|
+
/** True ONLY when a real classifier answer informed the score. */
|
|
150
|
+
fromClassifier: boolean;
|
|
151
|
+
/** Which branch produced the outcome (diagnostics + metric label). */
|
|
152
|
+
reason: ClassifierOutcomeReason;
|
|
153
|
+
/** Classifier model id actually used (undefined on every fallback path). */
|
|
154
|
+
usedModel: string | undefined;
|
|
155
|
+
/** Raw stopReason / errorMessage from the classify call, when one ran. */
|
|
156
|
+
stopReason: string | undefined;
|
|
157
|
+
errorMessage: string | undefined;
|
|
158
|
+
/** Provider-reported answer confidence (answer-level or top-level), when
|
|
159
|
+
* supplied. The pi-ai ClassifierScoreAnswer shape marks confidence
|
|
160
|
+
* REQUIRED, but a defensive read tolerates its absence. Undefined on
|
|
161
|
+
* every fallback path. */
|
|
162
|
+
confidence: number | undefined;
|
|
104
163
|
}
|
|
105
164
|
|
|
106
165
|
/**
|
|
@@ -144,6 +203,24 @@ function outcomeMetric(
|
|
|
144
203
|
}
|
|
145
204
|
}
|
|
146
205
|
|
|
206
|
+
/** U6A: observe the provider-reported confidence when one was supplied.
|
|
207
|
+
* Mirrors outcomeMetric's never-break contract. */
|
|
208
|
+
function confidenceMetric(
|
|
209
|
+
metricRegistry: MetricRegistry | undefined,
|
|
210
|
+
confidence: number | undefined,
|
|
211
|
+
labels: Record<string, string> | undefined,
|
|
212
|
+
): void {
|
|
213
|
+
if (confidence === undefined) return;
|
|
214
|
+
try {
|
|
215
|
+
metricRegistry
|
|
216
|
+
?.histogram("crew.classifier.confidence", "Classifier answer confidence when reported", [0.25, 0.5, 0.75, 0.9, 0.99])
|
|
217
|
+
.observe({ ...labels }, confidence);
|
|
218
|
+
} catch (error) {
|
|
219
|
+
// Metric plumbing must never break the never-rejects contract.
|
|
220
|
+
logInternalError("classifier-service.metric", error, `confidence=${confidence}`, "warn");
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
147
224
|
/** Normalize a registry handle into the duck-typed classifier surface. */
|
|
148
225
|
function asClassifierRegistry(modelRegistry: unknown): ClassifierModelRegistryLike | undefined {
|
|
149
226
|
if (!modelRegistry || typeof modelRegistry !== "object" || Array.isArray(modelRegistry)) return undefined;
|
|
@@ -173,25 +250,56 @@ export function resolveClassifierModel(explicit?: string): string {
|
|
|
173
250
|
}
|
|
174
251
|
|
|
175
252
|
/**
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
253
|
+
* Internal shared pipeline for the typed classify wrappers (U6B): resolves a
|
|
254
|
+
* usable classifier and runs ONE classify call, returning either the raw
|
|
255
|
+
* answer for the question key or a fallback marker with the outcome reason.
|
|
256
|
+
* Every registry-level failure mode (missing registry, zero credentialed
|
|
257
|
+
* classifiers, thrown call, non-"stop" stopReason) logs once + counts its
|
|
258
|
+
* metric here; answer interpretation (classified / answer_missing) stays
|
|
259
|
+
* with the typed wrappers so each keeps its own acceptance shape. The
|
|
260
|
+
* observable behavior of classifyBool is unchanged by the extraction (its
|
|
261
|
+
* landed tests are the equivalence guard).
|
|
181
262
|
*/
|
|
182
|
-
|
|
263
|
+
interface ClassifyPipelineInput {
|
|
264
|
+
/** Host-process model registry handle (ctx.modelRegistry, threaded unknown). */
|
|
265
|
+
modelRegistry: unknown;
|
|
266
|
+
/** Configured classifier model id ("provider/id"). */
|
|
267
|
+
classifierModel: string;
|
|
268
|
+
/** Key under `questions` carrying the typed question. */
|
|
269
|
+
questionKey: string;
|
|
270
|
+
question: ClassifierBoolQuestion | ClassifierScoreQuestion;
|
|
271
|
+
/** JSON state the classifier reasons about (keep small + low-cardinality). */
|
|
272
|
+
state: Record<string, unknown>;
|
|
273
|
+
/** Optional metric registry (counter created on demand, dispatch-batch style). */
|
|
274
|
+
metricRegistry?: MetricRegistry;
|
|
275
|
+
/** Extra low-cardinality labels for the calls_total counter. */
|
|
276
|
+
metricLabels?: Record<string, string>;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
interface ClassifyPipelineOk {
|
|
280
|
+
status: "ok";
|
|
281
|
+
/** Raw answers[questionKey] value — the wrapper decides the acceptance shape. */
|
|
282
|
+
rawAnswer: unknown;
|
|
283
|
+
/** Top-level result.confidence passthrough, when finite. */
|
|
284
|
+
topLevelConfidence: number | undefined;
|
|
285
|
+
stopReason: string | undefined;
|
|
286
|
+
errorMessage: string | undefined;
|
|
287
|
+
usedModel: string;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
interface ClassifyPipelineFallback {
|
|
291
|
+
status: "fallback";
|
|
292
|
+
reason: ClassifierOutcomeReason;
|
|
293
|
+
stopReason: string | undefined;
|
|
294
|
+
errorMessage: string | undefined;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
async function runClassifyQuestion(input: ClassifyPipelineInput): Promise<ClassifyPipelineOk | ClassifyPipelineFallback> {
|
|
183
298
|
const registry = asClassifierRegistry(input.modelRegistry);
|
|
184
299
|
if (!registry || typeof registry.getAvailableOfType !== "function" || typeof registry.classify !== "function") {
|
|
185
300
|
noteFailureMode("registry_missing", input.classifierModel, "model registry absent or lacks the classifier surface");
|
|
186
301
|
outcomeMetric(input.metricRegistry, "registry_missing", input.metricLabels);
|
|
187
|
-
return {
|
|
188
|
-
decision: input.fallback,
|
|
189
|
-
fromClassifier: false,
|
|
190
|
-
reason: "registry_missing",
|
|
191
|
-
usedModel: undefined,
|
|
192
|
-
stopReason: undefined,
|
|
193
|
-
errorMessage: undefined,
|
|
194
|
-
};
|
|
302
|
+
return { status: "fallback", reason: "registry_missing", stopReason: undefined, errorMessage: undefined };
|
|
195
303
|
}
|
|
196
304
|
// Credentialed classifiers only — the catalog (getModelsOfType) can list
|
|
197
305
|
// models whose provider has no key on this host (R2.3: 6 catalog entries,
|
|
@@ -203,14 +311,7 @@ export async function classifyBool(input: ClassifyBoolInput): Promise<ClassifyBo
|
|
|
203
311
|
} catch (error) {
|
|
204
312
|
noteFailureMode("classify_threw", input.classifierModel, `getAvailableOfType threw: ${String(error)}`);
|
|
205
313
|
outcomeMetric(input.metricRegistry, "classify_threw", input.metricLabels);
|
|
206
|
-
return {
|
|
207
|
-
decision: input.fallback,
|
|
208
|
-
fromClassifier: false,
|
|
209
|
-
reason: "classify_threw",
|
|
210
|
-
usedModel: undefined,
|
|
211
|
-
stopReason: undefined,
|
|
212
|
-
errorMessage: undefined,
|
|
213
|
-
};
|
|
314
|
+
return { status: "fallback", reason: "classify_threw", stopReason: undefined, errorMessage: undefined };
|
|
214
315
|
}
|
|
215
316
|
if (!Array.isArray(available) || available.length === 0) {
|
|
216
317
|
noteFailureMode(
|
|
@@ -219,14 +320,7 @@ export async function classifyBool(input: ClassifyBoolInput): Promise<ClassifyBo
|
|
|
219
320
|
"getAvailableOfType('classifier') is empty — no credentialed classifier",
|
|
220
321
|
);
|
|
221
322
|
outcomeMetric(input.metricRegistry, "no_classifier_available", input.metricLabels);
|
|
222
|
-
return {
|
|
223
|
-
decision: input.fallback,
|
|
224
|
-
fromClassifier: false,
|
|
225
|
-
reason: "no_classifier_available",
|
|
226
|
-
usedModel: undefined,
|
|
227
|
-
stopReason: undefined,
|
|
228
|
-
errorMessage: undefined,
|
|
229
|
-
};
|
|
323
|
+
return { status: "fallback", reason: "no_classifier_available", stopReason: undefined, errorMessage: undefined };
|
|
230
324
|
}
|
|
231
325
|
// Prefer the configured model id; fall back to the first available
|
|
232
326
|
// candidate so a renamed catalog id still gets a usable classifier.
|
|
@@ -250,49 +344,171 @@ export async function classifyBool(input: ClassifyBoolInput): Promise<ClassifyBo
|
|
|
250
344
|
`classify stopReason=${stopReason ?? "undefined"}${errorMessage ? ` errorMessage=${errorMessage}` : ""}`,
|
|
251
345
|
);
|
|
252
346
|
outcomeMetric(input.metricRegistry, "stop_reason_error", input.metricLabels);
|
|
253
|
-
return {
|
|
254
|
-
decision: input.fallback,
|
|
255
|
-
fromClassifier: false,
|
|
256
|
-
reason: "stop_reason_error",
|
|
257
|
-
usedModel: undefined,
|
|
258
|
-
stopReason,
|
|
259
|
-
errorMessage,
|
|
260
|
-
};
|
|
347
|
+
return { status: "fallback", reason: "stop_reason_error", stopReason, errorMessage };
|
|
261
348
|
}
|
|
262
|
-
const
|
|
263
|
-
|
|
264
|
-
noteFailureMode("answer_missing", input.classifierModel, `answers[${input.questionKey}] is ${typeof answer}, expected boolean`);
|
|
265
|
-
outcomeMetric(input.metricRegistry, "answer_missing", input.metricLabels);
|
|
266
|
-
return {
|
|
267
|
-
decision: input.fallback,
|
|
268
|
-
fromClassifier: false,
|
|
269
|
-
reason: "answer_missing",
|
|
270
|
-
usedModel: undefined,
|
|
271
|
-
stopReason,
|
|
272
|
-
errorMessage,
|
|
273
|
-
};
|
|
274
|
-
}
|
|
275
|
-
outcomeMetric(input.metricRegistry, "classified", input.metricLabels);
|
|
349
|
+
const topLevelConfidence =
|
|
350
|
+
typeof result?.confidence === "number" && Number.isFinite(result.confidence) ? result.confidence : undefined;
|
|
276
351
|
return {
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
usedModel,
|
|
352
|
+
status: "ok",
|
|
353
|
+
rawAnswer: result?.answers?.[input.questionKey],
|
|
354
|
+
topLevelConfidence,
|
|
281
355
|
stopReason,
|
|
282
356
|
errorMessage,
|
|
357
|
+
usedModel,
|
|
283
358
|
};
|
|
284
359
|
} catch (error) {
|
|
285
360
|
// The registry contract says classify never rejects, but a misbehaving
|
|
286
361
|
// implementation must still not take the orchestrator down with it.
|
|
287
362
|
noteFailureMode("classify_threw", input.classifierModel, `classify threw: ${String(error)}`);
|
|
288
363
|
outcomeMetric(input.metricRegistry, "classify_threw", input.metricLabels);
|
|
364
|
+
return { status: "fallback", reason: "classify_threw", stopReason: undefined, errorMessage: undefined };
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Ask ONE bool question of a host-process classifier. NEVER REJECTS — every
|
|
370
|
+
* failure mode (no registry, no credentialed classifier, thrown error,
|
|
371
|
+
* stopReason !== "stop", missing/non-boolean answer) returns the
|
|
372
|
+
* caller-provided fallback with {@link ClassifyBoolResult.fromClassifier}
|
|
373
|
+
* false, after a log-once note + metric counter increment.
|
|
374
|
+
*/
|
|
375
|
+
export async function classifyBool(input: ClassifyBoolInput): Promise<ClassifyBoolResult> {
|
|
376
|
+
const outcome = await runClassifyQuestion(input);
|
|
377
|
+
if (outcome.status === "fallback") {
|
|
289
378
|
return {
|
|
290
379
|
decision: input.fallback,
|
|
291
380
|
fromClassifier: false,
|
|
292
|
-
reason:
|
|
381
|
+
reason: outcome.reason,
|
|
382
|
+
usedModel: undefined,
|
|
383
|
+
stopReason: outcome.stopReason,
|
|
384
|
+
errorMessage: outcome.errorMessage,
|
|
385
|
+
confidence: undefined,
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
const rawAnswer = outcome.rawAnswer;
|
|
389
|
+
// U6A: bool answers arrive either as a bare boolean or wrapped with a
|
|
390
|
+
// confidence reading (`{value: boolean, confidence: number}` — research
|
|
391
|
+
// §8.2 "typed choice/score/bool ... answers carry confidence"). Accept
|
|
392
|
+
// both shapes; everything else stays the answer_missing fallback.
|
|
393
|
+
let answer: boolean | undefined;
|
|
394
|
+
let confidence: number | undefined;
|
|
395
|
+
if (typeof rawAnswer === "boolean") {
|
|
396
|
+
answer = rawAnswer;
|
|
397
|
+
} else if (rawAnswer && typeof rawAnswer === "object" && !Array.isArray(rawAnswer)) {
|
|
398
|
+
const wrapped = rawAnswer as { value?: unknown; confidence?: unknown };
|
|
399
|
+
if (typeof wrapped.value === "boolean") {
|
|
400
|
+
answer = wrapped.value;
|
|
401
|
+
if (typeof wrapped.confidence === "number" && Number.isFinite(wrapped.confidence)) {
|
|
402
|
+
confidence = wrapped.confidence;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
if (outcome.topLevelConfidence !== undefined && confidence === undefined) {
|
|
407
|
+
confidence = outcome.topLevelConfidence;
|
|
408
|
+
}
|
|
409
|
+
if (answer === undefined) {
|
|
410
|
+
noteFailureMode("answer_missing", input.classifierModel, `answers[${input.questionKey}] is ${typeof rawAnswer}, expected boolean`);
|
|
411
|
+
outcomeMetric(input.metricRegistry, "answer_missing", input.metricLabels);
|
|
412
|
+
return {
|
|
413
|
+
decision: input.fallback,
|
|
414
|
+
fromClassifier: false,
|
|
415
|
+
reason: "answer_missing",
|
|
416
|
+
usedModel: undefined,
|
|
417
|
+
stopReason: outcome.stopReason,
|
|
418
|
+
errorMessage: outcome.errorMessage,
|
|
419
|
+
confidence: undefined,
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
outcomeMetric(input.metricRegistry, "classified", input.metricLabels);
|
|
423
|
+
confidenceMetric(input.metricRegistry, confidence, input.metricLabels);
|
|
424
|
+
return {
|
|
425
|
+
decision: answer,
|
|
426
|
+
fromClassifier: true,
|
|
427
|
+
reason: "classified",
|
|
428
|
+
usedModel: outcome.usedModel,
|
|
429
|
+
stopReason: outcome.stopReason,
|
|
430
|
+
errorMessage: outcome.errorMessage,
|
|
431
|
+
confidence,
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* Ask ONE score question of a host-process classifier (U6B — the graded
|
|
437
|
+
* sibling of {@link classifyBool}, consumer: verifier pre-gate). NEVER
|
|
438
|
+
* REJECTS — every failure mode (no registry, no credentialed classifier,
|
|
439
|
+
* thrown error, stopReason !== "stop", missing/non-numeric answer) returns
|
|
440
|
+
* the caller-provided fallback score with
|
|
441
|
+
* {@link ClassifyScoreResult.fromClassifier} false, after a log-once note +
|
|
442
|
+
* metric counter increment. A malformed or unconfigured classifier can only
|
|
443
|
+
* degrade to the caller fallback — it can never fail a task.
|
|
444
|
+
*/
|
|
445
|
+
export async function classifyScore(input: ClassifyScoreInput): Promise<ClassifyScoreResult> {
|
|
446
|
+
const outcome = await runClassifyQuestion(input);
|
|
447
|
+
if (outcome.status === "fallback") {
|
|
448
|
+
return {
|
|
449
|
+
score: input.fallback,
|
|
450
|
+
fromClassifier: false,
|
|
451
|
+
reason: outcome.reason,
|
|
452
|
+
usedModel: undefined,
|
|
453
|
+
stopReason: outcome.stopReason,
|
|
454
|
+
errorMessage: outcome.errorMessage,
|
|
455
|
+
confidence: undefined,
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
const rawAnswer = outcome.rawAnswer;
|
|
459
|
+
// Real pi-ai shape (ClassifierScoreAnswer): {type:"score", score: number,
|
|
460
|
+
// confidence: number} — the type marks confidence REQUIRED, but a
|
|
461
|
+
// defensive read tolerates its absence. A bare-number answer and the U6A
|
|
462
|
+
// wrapped {value, confidence} shape are accepted for symmetry with
|
|
463
|
+
// classifyBool; everything else stays the answer_missing fallback.
|
|
464
|
+
let score: number | undefined;
|
|
465
|
+
let confidence: number | undefined;
|
|
466
|
+
if (typeof rawAnswer === "number" && Number.isFinite(rawAnswer)) {
|
|
467
|
+
score = rawAnswer;
|
|
468
|
+
} else if (rawAnswer && typeof rawAnswer === "object" && !Array.isArray(rawAnswer)) {
|
|
469
|
+
const wrapped = rawAnswer as { score?: unknown; value?: unknown; confidence?: unknown };
|
|
470
|
+
const candidate =
|
|
471
|
+
typeof wrapped.score === "number" && Number.isFinite(wrapped.score)
|
|
472
|
+
? wrapped.score
|
|
473
|
+
: typeof wrapped.value === "number" && Number.isFinite(wrapped.value)
|
|
474
|
+
? wrapped.value
|
|
475
|
+
: undefined;
|
|
476
|
+
if (candidate !== undefined) {
|
|
477
|
+
score = candidate;
|
|
478
|
+
if (typeof wrapped.confidence === "number" && Number.isFinite(wrapped.confidence)) {
|
|
479
|
+
confidence = wrapped.confidence;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
if (outcome.topLevelConfidence !== undefined && confidence === undefined) {
|
|
484
|
+
confidence = outcome.topLevelConfidence;
|
|
485
|
+
}
|
|
486
|
+
if (score === undefined) {
|
|
487
|
+
noteFailureMode(
|
|
488
|
+
"answer_missing",
|
|
489
|
+
input.classifierModel,
|
|
490
|
+
`answers[${input.questionKey}] is ${typeof rawAnswer}, expected score number`,
|
|
491
|
+
);
|
|
492
|
+
outcomeMetric(input.metricRegistry, "answer_missing", input.metricLabels);
|
|
493
|
+
return {
|
|
494
|
+
score: input.fallback,
|
|
495
|
+
fromClassifier: false,
|
|
496
|
+
reason: "answer_missing",
|
|
293
497
|
usedModel: undefined,
|
|
294
|
-
stopReason:
|
|
295
|
-
errorMessage:
|
|
498
|
+
stopReason: outcome.stopReason,
|
|
499
|
+
errorMessage: outcome.errorMessage,
|
|
500
|
+
confidence: undefined,
|
|
296
501
|
};
|
|
297
502
|
}
|
|
503
|
+
outcomeMetric(input.metricRegistry, "classified", input.metricLabels);
|
|
504
|
+
confidenceMetric(input.metricRegistry, confidence, input.metricLabels);
|
|
505
|
+
return {
|
|
506
|
+
score,
|
|
507
|
+
fromClassifier: true,
|
|
508
|
+
reason: "classified",
|
|
509
|
+
usedModel: outcome.usedModel,
|
|
510
|
+
stopReason: outcome.stopReason,
|
|
511
|
+
errorMessage: outcome.errorMessage,
|
|
512
|
+
confidence,
|
|
513
|
+
};
|
|
298
514
|
}
|