pi-crew 0.11.11 → 0.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -0
- package/dist/index.mjs +551 -519
- package/package.json +1 -1
- package/src/config/config-validation.ts +9 -0
- package/src/config/defaults.ts +7 -0
- package/src/config/env-vars.ts +24 -0
- package/src/config/types.ts +11 -0
- package/src/extension/registration/tool-registration.ts +7 -0
- package/src/extension/registration/verify-gate-tool.ts +263 -0
- package/src/extension/team-tool/run-deadline.ts +22 -1
- package/src/extension/team-tool/run.ts +38 -2
- package/src/extension/team-tool-types.ts +4 -0
- package/src/extension/team-tool.ts +31 -3
- package/src/prompt/inbox-poll.ts +141 -3
- package/src/runtime/broker/crew-broker.ts +48 -2
- package/src/runtime/broker/protocol/lock-busy.ts +47 -1
- package/src/runtime/broker/protocol/request-parsers.ts +8 -1
- package/src/runtime/budget-enforcement.ts +36 -20
- package/src/runtime/child-pi/child-pi-constants.ts +24 -0
- package/src/runtime/child-pi/child-pi-kill.ts +59 -2
- package/src/runtime/child-pi/child-pi-rpc.ts +1021 -0
- package/src/runtime/child-pi/child-pi-steering.ts +10 -8
- package/src/runtime/child-pi/child-pi-streams.ts +69 -19
- package/src/runtime/child-pi/child-pi.ts +29 -30
- package/src/runtime/child-pi/owned-process.ts +434 -0
- package/src/runtime/child-pi/postmortem-registry.ts +134 -0
- package/src/runtime/classifier/ambient-noise-gate.ts +126 -0
- package/src/runtime/classifier/classifier-service.ts +277 -61
- package/src/runtime/dispatch-batch.ts +32 -0
- package/src/runtime/live-session/live-agent-control.ts +2 -0
- package/src/runtime/live-session/live-agent-manager.ts +72 -2
- package/src/runtime/live-session/live-session-runtime.ts +435 -40
- package/src/runtime/merge-loop.ts +63 -35
- package/src/runtime/recovery/crash-recovery.ts +141 -3
- package/src/runtime/recovery/retry-executor.ts +29 -0
- package/src/runtime/run-tracker.ts +28 -0
- package/src/runtime/stale-reconciler.ts +36 -1
- package/src/runtime/task-runner.ts +103 -0
- package/src/runtime/verification/classifier-pre-gate.ts +262 -0
- package/src/runtime/verification/verification-gates.ts +5 -1
- package/src/runtime/verification/verify-gate.ts +394 -0
- package/src/schema/config-schema.ts +13 -0
- package/src/state/contracts.ts +18 -0
- package/src/state/coordination/mailbox.ts +203 -55
- package/src/state/stores/sqlite-run-state.ts +378 -0
- package/src/state/stores/state-store.ts +134 -6
- package/src/ui/agents-jobs-browser.ts +154 -8
- package/src/ui/events-state-source.ts +358 -0
- package/src/ui/run-snapshot-cache.ts +87 -3
- package/src/ui/widget/widget-model.ts +13 -2
package/package.json
CHANGED
|
@@ -381,6 +381,15 @@ function parseRuntimeConfig(value: unknown): CrewRuntimeConfig | undefined {
|
|
|
381
381
|
};
|
|
382
382
|
return Object.values(parsed).some((v) => v !== undefined) ? parsed : undefined;
|
|
383
383
|
})(),
|
|
384
|
+
liveSession: (() => {
|
|
385
|
+
const l = asRecord(obj.liveSession);
|
|
386
|
+
if (!l) return undefined;
|
|
387
|
+
const parsed: NonNullable<CrewRuntimeConfig["liveSession"]> = {
|
|
388
|
+
responseTimeoutMs: parsePositiveInteger(l.responseTimeoutMs, LIMIT_CEILINGS.runtimeTaskTimeoutMs),
|
|
389
|
+
hungToolGraceMs: parsePositiveInteger(l.hungToolGraceMs, LIMIT_CEILINGS.runtimeTaskTimeoutMs),
|
|
390
|
+
};
|
|
391
|
+
return Object.values(parsed).some((v) => v !== undefined) ? parsed : undefined;
|
|
392
|
+
})(),
|
|
384
393
|
excludeContextBash: parseWithSchema(Type.Boolean(), obj.excludeContextBash),
|
|
385
394
|
handoffBudgetTokens: parseHandoffBudgetTokens(obj.handoffBudgetTokens),
|
|
386
395
|
agentExtensions: parseStringList(obj.agentExtensions),
|
package/src/config/defaults.ts
CHANGED
|
@@ -46,6 +46,13 @@ export const DEFAULT_LIVE_SESSION = {
|
|
|
46
46
|
yieldPollIntervalMs: 500,
|
|
47
47
|
/** Maximum time to wait for session idle after prompt (ms). */
|
|
48
48
|
idleWaitTimeoutMs: 60_000,
|
|
49
|
+
/**
|
|
50
|
+
* U4 hung-tool escape hatch: after the response timeout fires and the
|
|
51
|
+
* cooperative abort() has been issued, how long to keep waiting for the
|
|
52
|
+
* prompt() promise to settle before declaring the session hung (tool kept
|
|
53
|
+
* the event loop / ignored the AbortSignal) and dropping the reference.
|
|
54
|
+
*/
|
|
55
|
+
hungToolGraceMs: 10_000,
|
|
49
56
|
};
|
|
50
57
|
|
|
51
58
|
export const DEFAULT_LOCKS = {
|
package/src/config/env-vars.ts
CHANGED
|
@@ -455,6 +455,19 @@ export const CREW_ENV_VARS: Record<string, CrewEnvVarSpec> = {
|
|
|
455
455
|
parser: "boolean",
|
|
456
456
|
doc: "R3-19/D5 hermetic worker spawn override: '1' forces --no-extensions on worker argv, '0' restores the old ambient-stack parity (discovered/configured/built-in extensions load); beats BuildPiWorkerArgsInput.hermeticWorkers / runtime.hermeticWorkers; default ON (pi-args.ts:resolveHermeticWorkers). Explicit -e prompt-runtime + agent extensions + --skill flags are unaffected (cli.md:186-191)",
|
|
457
457
|
},
|
|
458
|
+
PI_CREW_STATE_BACKEND: {
|
|
459
|
+
name: "PI_CREW_STATE_BACKEND",
|
|
460
|
+
doc: "U8 (upgrade-spec 2026-10-09) run-state store backend select: 'sqlite' stores manifest+tasks+worker-status in <runId>/state.sqlite via node:sqlite with ONE transaction per batch write (all-or-nothing; manifest.json/tasks.json stay written as read-side mirrors for legacy consumers — loads read the db); unset or any other value keeps the default JSON file backend. events.jsonl audit spine unchanged in both modes (state-store.ts / sqlite-run-state.ts isSqliteStateBackend)",
|
|
461
|
+
},
|
|
462
|
+
PI_CREW_CHILD_PI_MODE: {
|
|
463
|
+
name: "PI_CREW_CHILD_PI_MODE",
|
|
464
|
+
doc: "U14 (upgrade-spec 2026-10-09) child-pi protocol v2 transport select: 'rpc' spawns task workers over pi --mode rpc (SDK RpcClient wire — steer receipts, clear_queue hygiene, multi-prompt reuse; child-pi-rpc.ts), 'json' / unset keeps the --mode json -p stdio path (phase-1 default; both modes fully regression-tested). Surface TUI panes are unaffected",
|
|
465
|
+
},
|
|
466
|
+
PI_CREW_CHILD_PI_POOL: {
|
|
467
|
+
name: "PI_CREW_CHILD_PI_POOL",
|
|
468
|
+
parser: "boolean",
|
|
469
|
+
doc: "U14 rpc worker process pool (phase 1, default OFF): '1' lets consecutive tasks of the SAME agent reuse one live idle rpc process (warm context — 1 process = 1 session, many prompts). Phase-1 limitation: per-run env control vars (broker token, steering file, scratchpad gates) and argv flags apply only to the FIRST task on the process; entries evict on exit/abort (child-pi-rpc.ts)",
|
|
470
|
+
},
|
|
458
471
|
PI_CREW_WORKER_TRANSPORT: {
|
|
459
472
|
name: "PI_CREW_WORKER_TRANSPORT",
|
|
460
473
|
doc: "W7 EXPERIMENTAL worker transport override: 'rpc' selects the prototype RPC frame-client (src/runtime/rpc/) at the run-worker seam (wired e2728f68; D1/DR2 also plumbs config runtime.workerTransport to the same seam — env WINS over config when set, invalid env fails safe to stdio); anything else/unset → config value, default stdio child-pi path (rpc-worker.ts resolveWorkerTransport)",
|
|
@@ -468,6 +481,17 @@ export const CREW_ENV_VARS: Record<string, CrewEnvVarSpec> = {
|
|
|
468
481
|
name: "PI_CREW_CLASSIFIER_MODEL",
|
|
469
482
|
doc: "P2-1 classifier model id override ('provider/id', catalog ids only — availability depends on provider credentials at runtime): beats runtime.classifierModel; unset/blank → config value, default 'opencode/jev-1.13-free' (classifier-service.ts:resolveClassifierModel)",
|
|
470
483
|
},
|
|
484
|
+
PI_CREW_VERIFY_GATE: {
|
|
485
|
+
name: "PI_CREW_VERIFY_GATE",
|
|
486
|
+
parser: "boolean",
|
|
487
|
+
doc: "U13 deterministic verifier pre-gate override: '0' disables running the deterministic verify_gate checks (typecheck / test:critical via package.json scripts) before a verifier-role worker spawn — the LLM verifier then always spawns exactly as pre-U13; '1' forces it on. Default ON (gate PASS skips the LLM verifier spawn; FAILED/INCONCLUSIVE spawns it with the gate results in the prompt) (runtime/verification/verify-gate.ts resolveVerifyGateEnabled)",
|
|
488
|
+
},
|
|
489
|
+
PI_CREW_VERIFY_GATE_TIMEOUT_MS: {
|
|
490
|
+
name: "PI_CREW_VERIFY_GATE_TIMEOUT_MS",
|
|
491
|
+
parser: "int",
|
|
492
|
+
default: "120000",
|
|
493
|
+
doc: "U13 per-check timeout in ms for the deterministic verify_gate pre-gate commands. Invalid/≤0 → 120000, mirroring runPhaseGates (runtime/verification/verify-gate.ts resolveVerifyGateTimeoutMs)",
|
|
494
|
+
},
|
|
471
495
|
PI_CREW_RPC_DIALOG_ANSWER: {
|
|
472
496
|
name: "PI_CREW_RPC_DIALOG_ANSWER",
|
|
473
497
|
doc: "W7 EXPERIMENTAL dialog auto-answer policy for RPC-mode extension_ui_request dialogs (select/confirm/input/editor): 'cancel' (default) answers {cancelled:true} — universal safe answer; 'block' leaves the server promise pending (debug). Auto-confirm deliberately NOT implemented (security); invalid → 'cancel' (rpc-worker.ts resolveDialogAnswerPolicy)",
|
package/src/config/types.ts
CHANGED
|
@@ -123,6 +123,17 @@ export interface CrewRuntimeConfig {
|
|
|
123
123
|
maxReminders?: number;
|
|
124
124
|
reminderPrompt?: string;
|
|
125
125
|
};
|
|
126
|
+
/**
|
|
127
|
+
* U4 (live-session v2): tunables for the in-process live-session worker.
|
|
128
|
+
* `responseTimeoutMs` bounds one prompt() wall-clock (default
|
|
129
|
+
* DEFAULT_LIVE_SESSION.responseTimeoutMs); `hungToolGraceMs` is how long
|
|
130
|
+
* the escape hatch waits for the cooperatively-aborted prompt to settle
|
|
131
|
+
* before dropping the session reference and failing the task.
|
|
132
|
+
*/
|
|
133
|
+
liveSession?: {
|
|
134
|
+
responseTimeoutMs?: number;
|
|
135
|
+
hungToolGraceMs?: number;
|
|
136
|
+
};
|
|
126
137
|
/** Policy for per-role runtime selection. Not sensitive — safe to keep in project config. */
|
|
127
138
|
isolationPolicy?: {
|
|
128
139
|
/** Roles that should use child-process for crash isolation. Default: no roles. */
|
|
@@ -15,6 +15,7 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
|
15
15
|
import type { RegistrationContext } from "./registration-types.ts";
|
|
16
16
|
import { registerSubagentTools } from "./subagent-tools.ts";
|
|
17
17
|
import { registerTeamTool } from "./team-tool.ts";
|
|
18
|
+
import { registerVerifyGateTool } from "./verify-gate-tool.ts";
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* Register all pi-crew tools on the ExtensionAPI.
|
|
@@ -47,4 +48,10 @@ export function registerPiTools(pi: ExtensionAPI, ctx: RegistrationContext): voi
|
|
|
47
48
|
ctx.startForegroundRun(subCtx as Parameters<typeof ctx.startForegroundRun>[0], runner, runId),
|
|
48
49
|
batchBarrier: ctx.batchBarrier,
|
|
49
50
|
});
|
|
51
|
+
// U13 (upgrade spec 2026-10-09): deterministic verifier pre-gate tool —
|
|
52
|
+
// codemode exposure (hidden from the model, callable via ctx.executeTool
|
|
53
|
+
// by other tools); the verifier spawn site consumes the same engine.
|
|
54
|
+
registerVerifyGateTool(pi, {
|
|
55
|
+
getMetricRegistry: () => ctx.observabilityState.metricRegistry,
|
|
56
|
+
});
|
|
50
57
|
}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* U13 (upgrade spec 2026-10-09 §U13): the `verify_gate` TOOL — the
|
|
3
|
+
* extension-layer entry point of the deterministic verifier pre-gate.
|
|
4
|
+
*
|
|
5
|
+
* Runs the configured check commands (typecheck / test:critical / grep
|
|
6
|
+
* evidence lines) via `ctx.executeTool("bash", ...)` — the FIFTH parameter of
|
|
7
|
+
* ToolDefinition.execute. The round-10 probe (/tmp/pi-crew-verify10-xtool/,
|
|
8
|
+
* 13/13 checks PASS) proved this path: nested calls get id `<parent>/1` +
|
|
9
|
+
* parentToolCallId on events, errors arrive as isError VALUES (never throws),
|
|
10
|
+
* exit codes are readable in result.structuredContent.exit_code, calls go
|
|
11
|
+
* through the FULL pipeline (validation + tool_call/tool_result hooks +
|
|
12
|
+
* permissions), and the host enforces nested limits (256 calls / 8KB per
|
|
13
|
+
* call / 32KB total args) — the caps below stay far inside them.
|
|
14
|
+
*
|
|
15
|
+
* EXPOSURE — "codemode", not "model-only" (spec says "cân nhắc
|
|
16
|
+
* exposure:'model-only' (giấu khỏi model, chỉ tool khác gọi được — codemode
|
|
17
|
+
* pattern)"; the parenthetical is the INTENT, and per the SDK
|
|
18
|
+
* (core/extensions/types.d.ts ToolExposure) the literal names mean the
|
|
19
|
+
* opposite of the intent: `model-only` = "declared to the model while
|
|
20
|
+
* active, NEVER callable" — executeTool calls to it fail — while `codemode`
|
|
21
|
+
* = "callable whenever registered, not declared to the model unless
|
|
22
|
+
* explicitly activated". `codemode` IS the "hidden from the model, callable
|
|
23
|
+
* by other tools" pattern the spec wants (it is exactly what the codemode
|
|
24
|
+
* tool itself does), so that is what we register.
|
|
25
|
+
*
|
|
26
|
+
* The verifier SPAWN SITE (task-runner.ts) consumes the same pure engine via
|
|
27
|
+
* the subprocess executor; this tool is the in-host, zero-subprocess variant
|
|
28
|
+
* other tools (or codemode scripts) can call to get a structured gate
|
|
29
|
+
* verdict without any LLM involvement.
|
|
30
|
+
*/
|
|
31
|
+
import type { ExtensionAPI, ToolDefinition } from "@earendil-works/pi-coding-agent";
|
|
32
|
+
import { type Static, Type } from "@sinclair/typebox";
|
|
33
|
+
import type { MetricRegistry } from "../../observability/metric-registry.ts";
|
|
34
|
+
import {
|
|
35
|
+
DEFAULT_VERIFY_GATE_TIMEOUT_MS,
|
|
36
|
+
defaultVerifyGateChecks,
|
|
37
|
+
renderVerifyGateContext,
|
|
38
|
+
runVerifyGate,
|
|
39
|
+
type VerifyGateCheck,
|
|
40
|
+
type VerifyGateOutcome,
|
|
41
|
+
} from "../../runtime/verification/verify-gate.ts";
|
|
42
|
+
|
|
43
|
+
/** Nested-limit guards (host caps are 256 calls / 8KB call / 32KB args —
|
|
44
|
+
* these stay an order of magnitude under). */
|
|
45
|
+
export const VERIFY_GATE_TOOL_MAX_CHECKS = 8;
|
|
46
|
+
export const VERIFY_GATE_TOOL_MAX_COMMAND_CHARS = 2_000;
|
|
47
|
+
export const VERIFY_GATE_TOOL_MAX_TOTAL_COMMAND_CHARS = 8_000;
|
|
48
|
+
/** Bash tool timeout is in SECONDS. */
|
|
49
|
+
const BASH_TIMEOUT_MAX_SECONDS = 600;
|
|
50
|
+
|
|
51
|
+
const VerifyGateToolParams = Type.Object({
|
|
52
|
+
cwd: Type.Optional(Type.String({ description: "Working directory the checks run in. Defaults to the session cwd." })),
|
|
53
|
+
checks: Type.Optional(
|
|
54
|
+
Type.Array(
|
|
55
|
+
Type.Object({
|
|
56
|
+
name: Type.String({ description: "Short check name, e.g. 'typecheck'." }),
|
|
57
|
+
command: Type.String({ description: "Shell command; exit code 0 = pass, e.g. 'npm run --silent typecheck'." }),
|
|
58
|
+
critical: Type.Optional(Type.Boolean({ description: "Stop the gate on this check's failure (default true)." })),
|
|
59
|
+
}),
|
|
60
|
+
{
|
|
61
|
+
description:
|
|
62
|
+
"Explicit checks. Defaults to the repo preset: package.json scripts 'typecheck' + 'test:critical' when present. Add grep evidence lines here (e.g. `grep -c \"<marker>\" dist/index.mjs`).",
|
|
63
|
+
},
|
|
64
|
+
),
|
|
65
|
+
),
|
|
66
|
+
timeoutMs: Type.Optional(
|
|
67
|
+
Type.Number({
|
|
68
|
+
minimum: 1_000,
|
|
69
|
+
maximum: 600_000,
|
|
70
|
+
description: `Per-check timeout in ms (default ${DEFAULT_VERIFY_GATE_TIMEOUT_MS}).`,
|
|
71
|
+
}),
|
|
72
|
+
),
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
export interface VerifyGateToolDeps {
|
|
76
|
+
/** Host metric registry (lazily resolved per call, same as team-tool). */
|
|
77
|
+
getMetricRegistry?: () => MetricRegistry | undefined;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Quote a path for shell interpolation (single quotes; embedded quotes escaped). */
|
|
81
|
+
function shellQuote(value: string): string {
|
|
82
|
+
return `'${value.replaceAll("'", `'\\''`)}'`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Read the bash exit code from a nested executeTool outcome. The bash tool
|
|
86
|
+
* reports { exit_code, output, truncated, wall_time_seconds } in
|
|
87
|
+
* structuredContent; a NON-ZERO exit is an isError VALUE (no throw) per the
|
|
88
|
+
* probe, so the code stays readable on failure paths. */
|
|
89
|
+
function readNestedBashResult(outcome: { isError: boolean; result: { structuredContent?: unknown; content?: unknown } }): {
|
|
90
|
+
exitCode: number | null;
|
|
91
|
+
output: string;
|
|
92
|
+
} {
|
|
93
|
+
const structured = outcome.result.structuredContent;
|
|
94
|
+
let exitCode: number | null = null;
|
|
95
|
+
let output = "";
|
|
96
|
+
if (structured && typeof structured === "object" && !Array.isArray(structured)) {
|
|
97
|
+
const record = structured as Record<string, unknown>;
|
|
98
|
+
if (typeof record.exit_code === "number") exitCode = record.exit_code;
|
|
99
|
+
if (typeof record.output === "string") output = record.output;
|
|
100
|
+
}
|
|
101
|
+
if (!output) {
|
|
102
|
+
// Fallback: join text content parts (isError results keep content).
|
|
103
|
+
const content = outcome.result.content;
|
|
104
|
+
if (typeof content === "string") output = content;
|
|
105
|
+
else if (Array.isArray(content)) {
|
|
106
|
+
output = content
|
|
107
|
+
.map((part) =>
|
|
108
|
+
part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string"
|
|
109
|
+
? (part as { text: string }).text
|
|
110
|
+
: "",
|
|
111
|
+
)
|
|
112
|
+
.filter(Boolean)
|
|
113
|
+
.join("\n");
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
// isError without a structured exit code (blocked/thrown) → treat as a
|
|
117
|
+
// failed check with an unknown code; the verdict stays deterministic.
|
|
118
|
+
if (outcome.isError && exitCode === null) exitCode = -1;
|
|
119
|
+
return { exitCode, output };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function registerVerifyGateTool(pi: ExtensionAPI, deps: VerifyGateToolDeps): void {
|
|
123
|
+
const tool: ToolDefinition = {
|
|
124
|
+
name: "verify_gate",
|
|
125
|
+
label: "Verify Gate",
|
|
126
|
+
description: [
|
|
127
|
+
"Deterministic verification gate (U13): runs check commands (typecheck / test:critical / grep evidence lines) and returns a structured verdict {verdict: PASS|FAILED|INCONCLUSIVE, checks, evidenceLines} — exit codes only, no LLM judgment.",
|
|
128
|
+
"Registered codemode-style: callable by other tools via ctx.executeTool('verify_gate', {...}); not part of the model-facing active tool set.",
|
|
129
|
+
"A PASS verdict is positive deterministic evidence that an LLM verifier spawn can be skipped; FAILED/INCONCLUSIVE escalate with these results attached.",
|
|
130
|
+
].join("\n"),
|
|
131
|
+
// codemode = hidden from the model, callable via ctx.executeTool — the
|
|
132
|
+
// spec's "giấu khỏi model, chỉ tool khác gọi" pattern (see header note
|
|
133
|
+
// for why the literal 'model-only' value would defeat that intent).
|
|
134
|
+
exposure: "codemode",
|
|
135
|
+
defaultActive: false,
|
|
136
|
+
parameters: VerifyGateToolParams,
|
|
137
|
+
outputSchema: Type.Object({
|
|
138
|
+
verdict: Type.Union([Type.Literal("PASS"), Type.Literal("FAILED"), Type.Literal("INCONCLUSIVE")]),
|
|
139
|
+
evidenceLines: Type.Array(Type.String()),
|
|
140
|
+
allPassed: Type.Boolean(),
|
|
141
|
+
totalDurationMs: Type.Number(),
|
|
142
|
+
checks: Type.Array(
|
|
143
|
+
Type.Object({
|
|
144
|
+
name: Type.String(),
|
|
145
|
+
command: Type.String(),
|
|
146
|
+
exitCode: Type.Union([Type.Number(), Type.Null()]),
|
|
147
|
+
passed: Type.Boolean(),
|
|
148
|
+
durationMs: Type.Number(),
|
|
149
|
+
skipped: Type.Optional(Type.Boolean()),
|
|
150
|
+
}),
|
|
151
|
+
),
|
|
152
|
+
}),
|
|
153
|
+
async execute(_id, params: Static<typeof VerifyGateToolParams>, signal, _onUpdate, ctx) {
|
|
154
|
+
const cwd = params.cwd && params.cwd.trim().length > 0 ? params.cwd : ctx.cwd;
|
|
155
|
+
const checks: VerifyGateCheck[] = params.checks ?? defaultVerifyGateChecks(cwd);
|
|
156
|
+
// Nested-limit guards — validate BEFORE any executeTool call.
|
|
157
|
+
if (checks.length > VERIFY_GATE_TOOL_MAX_CHECKS) {
|
|
158
|
+
return {
|
|
159
|
+
content: [{ type: "text", text: `verify_gate: too many checks (${checks.length} > ${VERIFY_GATE_TOOL_MAX_CHECKS})` }],
|
|
160
|
+
details: undefined,
|
|
161
|
+
isError: true,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
let totalChars = 0;
|
|
165
|
+
for (const check of checks) {
|
|
166
|
+
if (check.command.length > VERIFY_GATE_TOOL_MAX_COMMAND_CHARS) {
|
|
167
|
+
return {
|
|
168
|
+
content: [
|
|
169
|
+
{
|
|
170
|
+
type: "text",
|
|
171
|
+
text: `verify_gate: check '${check.name}' command exceeds ${VERIFY_GATE_TOOL_MAX_COMMAND_CHARS} chars`,
|
|
172
|
+
},
|
|
173
|
+
],
|
|
174
|
+
details: undefined,
|
|
175
|
+
isError: true,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
totalChars += check.command.length;
|
|
179
|
+
}
|
|
180
|
+
if (totalChars > VERIFY_GATE_TOOL_MAX_TOTAL_COMMAND_CHARS) {
|
|
181
|
+
return {
|
|
182
|
+
content: [
|
|
183
|
+
{
|
|
184
|
+
type: "text",
|
|
185
|
+
text: `verify_gate: total command chars ${totalChars} > ${VERIFY_GATE_TOOL_MAX_TOTAL_COMMAND_CHARS}`,
|
|
186
|
+
},
|
|
187
|
+
],
|
|
188
|
+
details: undefined,
|
|
189
|
+
isError: true,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
const timeoutMs = params.timeoutMs ?? DEFAULT_VERIFY_GATE_TIMEOUT_MS;
|
|
193
|
+
// The probe-proven 5th parameter: ctx.executeTool runs the built-in
|
|
194
|
+
// bash tool through the full pipeline in the host process.
|
|
195
|
+
if (typeof ctx.executeTool !== "function") {
|
|
196
|
+
const outcome = inconclusiveOutcome(
|
|
197
|
+
"verify_gate: ctx.executeTool unavailable in this context (no tool-call context factory) — gate not run",
|
|
198
|
+
);
|
|
199
|
+
return gateResult(outcome, false);
|
|
200
|
+
}
|
|
201
|
+
const executeTool = ctx.executeTool;
|
|
202
|
+
const outcome = await runVerifyGate(
|
|
203
|
+
checks,
|
|
204
|
+
async (command, opts) => {
|
|
205
|
+
const bashCommand = `cd ${shellQuote(opts.cwd)} && ${command}`;
|
|
206
|
+
// executeTool never rejects for tool failures (probe): blocked
|
|
207
|
+
// calls and non-zero exits come back as isError values.
|
|
208
|
+
const nested = await executeTool(
|
|
209
|
+
"bash",
|
|
210
|
+
{ command: bashCommand, timeout: Math.min(Math.ceil(opts.timeoutMs / 1000), BASH_TIMEOUT_MAX_SECONDS) },
|
|
211
|
+
{ signal },
|
|
212
|
+
);
|
|
213
|
+
return readNestedBashResult(nested);
|
|
214
|
+
},
|
|
215
|
+
{ cwd, signal, timeoutMs },
|
|
216
|
+
);
|
|
217
|
+
try {
|
|
218
|
+
deps.getMetricRegistry?.()
|
|
219
|
+
?.counter("crew.verification.verify_gate_runs_total", "Deterministic verify_gate executions by verdict")
|
|
220
|
+
.inc({ verdict: outcome.verdict, entry: "tool" });
|
|
221
|
+
} catch {
|
|
222
|
+
/* metrics are best-effort */
|
|
223
|
+
}
|
|
224
|
+
return gateResult(outcome, false);
|
|
225
|
+
},
|
|
226
|
+
};
|
|
227
|
+
pi.registerTool(tool);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function inconclusiveOutcome(note: string): VerifyGateOutcome {
|
|
231
|
+
return {
|
|
232
|
+
verdict: "INCONCLUSIVE",
|
|
233
|
+
checks: [],
|
|
234
|
+
evidenceLines: [note],
|
|
235
|
+
allPassed: false,
|
|
236
|
+
totalDurationMs: 0,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function gateResult(outcome: VerifyGateOutcome, isError: boolean) {
|
|
241
|
+
const text = [renderVerifyGateContext(outcome).replace(/\n+$/, "")].join("\n");
|
|
242
|
+
return {
|
|
243
|
+
content: [{ type: "text" as const, text }],
|
|
244
|
+
// Gate verdicts (FAILED/INCONCLUSIVE) are legitimate OUTCOMES, not tool
|
|
245
|
+
// errors — callers (tools/codemode scripts) branch on structuredContent.
|
|
246
|
+
details: outcome,
|
|
247
|
+
structuredContent: {
|
|
248
|
+
verdict: outcome.verdict,
|
|
249
|
+
evidenceLines: outcome.evidenceLines,
|
|
250
|
+
allPassed: outcome.allPassed,
|
|
251
|
+
totalDurationMs: outcome.totalDurationMs,
|
|
252
|
+
checks: outcome.checks.map((check) => ({
|
|
253
|
+
name: check.name,
|
|
254
|
+
command: check.command,
|
|
255
|
+
exitCode: check.exitCode,
|
|
256
|
+
passed: check.passed,
|
|
257
|
+
durationMs: check.durationMs,
|
|
258
|
+
...(check.skipped ? { skipped: check.skipped } : {}),
|
|
259
|
+
})),
|
|
260
|
+
},
|
|
261
|
+
isError,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
@@ -22,6 +22,24 @@ export interface RunDeadline {
|
|
|
22
22
|
/** Fallback deadline when no config or param override is available: 1 hour. */
|
|
23
23
|
export const DEFAULT_RUN_DEADLINE_MS = 3_600_000;
|
|
24
24
|
|
|
25
|
+
/** Options for resolveRunDeadline. */
|
|
26
|
+
export interface RunDeadlineOptions {
|
|
27
|
+
/** "abort" (default): arm the deadline timer that ABORTS the controller —
|
|
28
|
+
* deadline expiry cancels the run (direct-await paths: DWF, goal-loop,
|
|
29
|
+
* inline scaffold — the deadline IS the execution bound).
|
|
30
|
+
*
|
|
31
|
+
* "watch-only": do NOT arm the abort timer. The deadline bounds only the
|
|
32
|
+
* caller's WATCH (waitForRun timeoutMs). Genuine caller cancels
|
|
33
|
+
* (ctx.signal abort outside a session switch, startForegroundRun callback
|
|
34
|
+
* abort) STILL abort the controller and cancel the run — deadline expiry
|
|
35
|
+
* alone never does. RELIABILITY FIX 2026-10-10 (wave-2 chain incident, run
|
|
36
|
+
* team_20261010045718): the foreground lane's armed timer was aborting
|
|
37
|
+
* executeTeamRun's signal at the 60-minute watch boundary — chain steps
|
|
38
|
+
* reported "partial" AND the still-healthy child run was cancelled. The
|
|
39
|
+
* watch is a timer; the cancel must be a caller decision. */
|
|
40
|
+
timer?: "abort" | "watch-only";
|
|
41
|
+
}
|
|
42
|
+
|
|
25
43
|
/**
|
|
26
44
|
* Resolve a unified run deadline with a shared AbortController.
|
|
27
45
|
*
|
|
@@ -49,6 +67,7 @@ export function resolveRunDeadline<T extends object>(
|
|
|
49
67
|
ctx: Pick<TeamContext, "cwd" | "signal">,
|
|
50
68
|
params: T,
|
|
51
69
|
config?: PiTeamsConfig,
|
|
70
|
+
options?: RunDeadlineOptions,
|
|
52
71
|
): RunDeadline {
|
|
53
72
|
const timeoutMs = (params as { timeoutMs?: number }).timeoutMs;
|
|
54
73
|
const effectiveConfig = config ?? loadConfig(ctx.cwd).config;
|
|
@@ -94,8 +113,10 @@ export function resolveRunDeadline<T extends object>(
|
|
|
94
113
|
// otherwise every run leaves a dangling 1h timer retaining ctx/params in closure.
|
|
95
114
|
// Same abort-after-exit guard as above: the timer may fire after workers
|
|
96
115
|
// already exited and their spawn-signal listeners are gone — Node can throw.
|
|
116
|
+
// "watch-only" (bug #2 fix): the timer is NOT armed — the deadline bounds the
|
|
117
|
+
// watch only, so the controller can only be aborted by a real caller cancel.
|
|
97
118
|
let timer: NodeJS.Timeout | undefined;
|
|
98
|
-
if (deadlineMs > 0) {
|
|
119
|
+
if (deadlineMs > 0 && options?.timer !== "watch-only") {
|
|
99
120
|
timer = setTimeout(() => safeAbort(controller, "run-deadline.timer"), deadlineMs);
|
|
100
121
|
timer.unref?.();
|
|
101
122
|
}
|
|
@@ -71,7 +71,7 @@ import * as path from "node:path";
|
|
|
71
71
|
import { t } from "../../i18n.ts";
|
|
72
72
|
import { hasAsyncStartMarker } from "../../runtime/async-marker.ts";
|
|
73
73
|
import { checkProcessLiveness, isActiveRunStatus } from "../../runtime/process-status.ts";
|
|
74
|
-
import { registerRunPromise, waitForRun } from "../../runtime/run-tracker.ts";
|
|
74
|
+
import { isWaitForRunTimeoutError, registerRunPromise, waitForRun } from "../../runtime/run-tracker.ts";
|
|
75
75
|
import { collectRunMetrics } from "../../state/stores/run-metrics.ts";
|
|
76
76
|
import type { PiTeamsToolResult } from "../tool-result.ts";
|
|
77
77
|
import { effectiveRunConfig } from "./config-patch.ts";
|
|
@@ -709,7 +709,14 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
709
709
|
const executeWorkers = runtime.kind !== "scaffold";
|
|
710
710
|
if (executeWorkers && ctx.startForegroundRun) {
|
|
711
711
|
// CORE-8: unified deadline — resolves params > config > 1h default.
|
|
712
|
-
|
|
712
|
+
// RELIABILITY FIX 2026-10-10 (bug #2, wave-2 chain incident run
|
|
713
|
+
// team_20261010045718): the foreground deadline is WATCH-ONLY. The armed
|
|
714
|
+
// abort timer used to cancel executeTeamRun's signal at the watch
|
|
715
|
+
// boundary (60m default) — chain steps reported "partial" AND the
|
|
716
|
+
// still-healthy child run was cancelled. waitForRun's own timeout still
|
|
717
|
+
// bounds the watch (partial-watch result below); real caller cancels
|
|
718
|
+
// (ctx.signal / foreground-run callback abort) still cancel the run.
|
|
719
|
+
const fgDeadline = resolveRunDeadline(ctx, params, executedConfig, { timer: "watch-only" });
|
|
713
720
|
// F1 register/await race fix (2026-09-12): the waitForRun below runs
|
|
714
721
|
// immediately after startForegroundRun returns (void), while
|
|
715
722
|
// executeTeamRunCore registers its promise only after several awaits —
|
|
@@ -842,6 +849,35 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
842
849
|
});
|
|
843
850
|
} catch (waitError: unknown) {
|
|
844
851
|
const waitErrMsg = errorMessage(waitError);
|
|
852
|
+
// RELIABILITY FIX 2026-10-10 (bug #2): a waitForRun WATCH expiry is NOT a
|
|
853
|
+
// run failure — with the watch-only deadline the run keeps executing in
|
|
854
|
+
// the foreground lane (exactly like the detached case) and the async
|
|
855
|
+
// notifier reports completion later. Report a partial-watch instead of an
|
|
856
|
+
// error so chain wrappers map the still-running step to "partial"
|
|
857
|
+
// without anyone reading this as "the run was cancelled/failed".
|
|
858
|
+
if (isWaitForRunTimeoutError(waitError)) {
|
|
859
|
+
return result(
|
|
860
|
+
[
|
|
861
|
+
`pi-crew run still running (watch window expired): ${updatedManifest.runId}`,
|
|
862
|
+
`Team: ${team.name}`,
|
|
863
|
+
`Workflow: ${workflow.name}`,
|
|
864
|
+
`Watch: ${waitErrMsg} — the run was NOT cancelled and keeps executing.`,
|
|
865
|
+
"",
|
|
866
|
+
"You will be notified when the run finishes.",
|
|
867
|
+
`Check status with: team action=status runId=${updatedManifest.runId}`,
|
|
868
|
+
`Wait again with: team action=wait runId=${updatedManifest.runId}`,
|
|
869
|
+
`State: ${updatedManifest.stateRoot}`,
|
|
870
|
+
].join("\n"),
|
|
871
|
+
{
|
|
872
|
+
action: "run",
|
|
873
|
+
status: "ok",
|
|
874
|
+
runId: updatedManifest.runId,
|
|
875
|
+
artifactsRoot: updatedManifest.artifactsRoot,
|
|
876
|
+
partialWatch: true,
|
|
877
|
+
},
|
|
878
|
+
false,
|
|
879
|
+
);
|
|
880
|
+
}
|
|
845
881
|
return result(
|
|
846
882
|
[
|
|
847
883
|
`pi-crew run timed out or failed: ${updatedManifest.runId}`,
|
|
@@ -29,6 +29,10 @@ export interface TeamToolDetails {
|
|
|
29
29
|
taskId?: string;
|
|
30
30
|
questionId?: string;
|
|
31
31
|
waiting?: boolean;
|
|
32
|
+
/** RELIABILITY FIX 2026-10-10 (bug #2): set when the foreground run's WATCH
|
|
33
|
+
* window expired (waitForRun timeout) — the run itself was NOT cancelled
|
|
34
|
+
* and keeps executing; the async notifier reports completion later. */
|
|
35
|
+
partialWatch?: boolean;
|
|
32
36
|
/** Structured data for programmatic consumption (e.g. TUI widgets). */
|
|
33
37
|
data?: Record<string, unknown>;
|
|
34
38
|
}
|
|
@@ -493,7 +493,7 @@ async function resumeDynamicWorkflowRun(
|
|
|
493
493
|
return result(
|
|
494
494
|
[
|
|
495
495
|
`Resumed dynamic-workflow run ${dwfResult.manifest.runId}.`,
|
|
496
|
-
`Status: ${dwfResult.manifest.status}`,
|
|
496
|
+
`Status: ${dwfResult.manifest.status} — resumed execution returned`,
|
|
497
497
|
dwfResult.manifest.summary ? `Result: ${dwfResult.manifest.summary}` : undefined,
|
|
498
498
|
]
|
|
499
499
|
.filter((line): line is string => line !== undefined)
|
|
@@ -819,6 +819,13 @@ export async function handleResume(params: TeamToolParamsValue, ctx: TeamContext
|
|
|
819
819
|
}
|
|
820
820
|
: task,
|
|
821
821
|
);
|
|
822
|
+
// RELIABILITY FIX 2026-10-10 (bug #3): count the tasks actually re-queued at
|
|
823
|
+
// resume so the result message can report what happened (work re-dispatched
|
|
824
|
+
// vs "run was already terminal, nothing to resume") instead of a bare
|
|
825
|
+
// "Status: completed" that reads as "everything finished instantly".
|
|
826
|
+
const requeuedCount = recovered.tasks.filter(
|
|
827
|
+
(task) => task.status === "failed" || task.status === "cancelled" || task.status === "skipped" || task.status === "running",
|
|
828
|
+
).length;
|
|
822
829
|
saveRunTasks(runtimeManifest, resetTasks);
|
|
823
830
|
const replay = replayPendingMailboxMessages(runtimeManifest);
|
|
824
831
|
await appendEventAsync(runtimeManifest.eventsPath, {
|
|
@@ -874,6 +881,7 @@ export async function handleResume(params: TeamToolParamsValue, ctx: TeamContext
|
|
|
874
881
|
kind: "execute" as const,
|
|
875
882
|
runtimeManifest: executingManifest,
|
|
876
883
|
resetTasks,
|
|
884
|
+
requeuedCount,
|
|
877
885
|
executeWorkers,
|
|
878
886
|
resumeSkillOverride,
|
|
879
887
|
runtime,
|
|
@@ -909,12 +917,32 @@ export async function handleResume(params: TeamToolParamsValue, ctx: TeamContext
|
|
|
909
917
|
// Finding 8: resume is the legitimate terminal-exit flow.
|
|
910
918
|
isResume: true,
|
|
911
919
|
});
|
|
920
|
+
// RELIABILITY FIX 2026-10-10 (bug #3, wave-2/wave-3 observations): the old
|
|
921
|
+
// message printed a bare "Status: completed" which read as "the whole run
|
|
922
|
+
// just finished" even when the resume had re-queued real work that kept
|
|
923
|
+
// executing. Report the truth: what the resumed execution returned, how
|
|
924
|
+
// much work was re-queued at resume, and the per-status task breakdown.
|
|
925
|
+
const statusLine =
|
|
926
|
+
executed.manifest.status === "completed"
|
|
927
|
+
? "completed — resumed execution finished (all tasks terminal)"
|
|
928
|
+
: executed.manifest.status === "running"
|
|
929
|
+
? "running — resumed dispatch is still executing"
|
|
930
|
+
: `${executed.manifest.status} — resumed execution ended in this state`;
|
|
931
|
+
const taskCounts = executed.tasks.reduce<Record<string, number>>((acc, task) => {
|
|
932
|
+
acc[task.status] = (acc[task.status] ?? 0) + 1;
|
|
933
|
+
return acc;
|
|
934
|
+
}, {});
|
|
935
|
+
const countsText = Object.entries(taskCounts)
|
|
936
|
+
.map(([status, count]) => `${count} ${status}`)
|
|
937
|
+
.join(", ");
|
|
912
938
|
return result(
|
|
913
939
|
[
|
|
914
940
|
`Resumed run ${executed.manifest.runId}.`,
|
|
915
|
-
`Status: ${
|
|
916
|
-
`
|
|
941
|
+
`Status: ${statusLine}`,
|
|
942
|
+
`Resumed work: ${decision.requeuedCount} task(s) re-queued${decision.requeuedCount === 0 ? " (run was already terminal — nothing to resume)" : ""}`,
|
|
943
|
+
`Tasks: ${executed.tasks.length} (${countsText})`,
|
|
917
944
|
`Artifacts: ${executed.manifest.artifactsRoot}`,
|
|
945
|
+
`Check live status with: team action=status runId=${executed.manifest.runId}`,
|
|
918
946
|
].join("\n"),
|
|
919
947
|
{
|
|
920
948
|
action: "resume",
|