taskplane 0.28.8 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/get-version.mjs +50 -0
- package/bin/taskplane.mjs +5 -8
- package/extensions/taskplane/agent-bridge-extension.ts +58 -3
- package/extensions/taskplane/agent-host.ts +11 -17
- package/extensions/taskplane/config-schema.ts +17 -12
- package/extensions/taskplane/diagnostics.ts +9 -0
- package/extensions/taskplane/engine-worker.ts +27 -0
- package/extensions/taskplane/engine.ts +235 -1
- package/extensions/taskplane/execution.ts +115 -6
- package/extensions/taskplane/extension.ts +303 -0
- package/extensions/taskplane/lane-runner.ts +74 -3
- package/extensions/taskplane/mailbox.ts +83 -0
- package/extensions/taskplane/messages.ts +19 -0
- package/extensions/taskplane/path-resolver.ts +63 -24
- package/extensions/taskplane/persistence.ts +378 -3
- package/extensions/taskplane/resume.ts +64 -7
- package/extensions/taskplane/tool-allowlist-constants.ts +37 -0
- package/extensions/taskplane/types.ts +60 -7
- package/extensions/taskplane/worktree.ts +5 -1
- package/package.json +1 -1
- package/skills/create-taskplane-task/SKILL.md +37 -0
- package/templates/agents/supervisor.md +62 -1
- package/templates/agents/task-worker.md +58 -6
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `getVersion` — capture a CLI's version string with stdout-precedence,
|
|
3
|
+
* stderr-fallback, and null-on-failure semantics.
|
|
4
|
+
*
|
|
5
|
+
* Extracted from the inline `getVersion()` in `bin/taskplane.mjs` so it
|
|
6
|
+
* can be unit-tested without subprocessing the whole CLI.
|
|
7
|
+
*
|
|
8
|
+
* Behavior:
|
|
9
|
+
* - Spawns `${cmd} ${flag}` with shell:true and stdio:['ignore','pipe','pipe']
|
|
10
|
+
* - Returns null if `spawnSync` itself throws (e.g., command not found)
|
|
11
|
+
* - Returns null if the subprocess errored OR exited with non-zero status
|
|
12
|
+
* (matches the prior `execSync`-throws-on-failure contract)
|
|
13
|
+
* - On success, returns stdout if non-empty, else stderr (some CLIs
|
|
14
|
+
* notably `pi` print version output to stderr)
|
|
15
|
+
* - Returns null if both streams are empty
|
|
16
|
+
*
|
|
17
|
+
* @since TP-189-C (extracted) / TP-185 follow-up (original fix scope)
|
|
18
|
+
*
|
|
19
|
+
* @param {string} cmd — command name (or already-formed token sequence)
|
|
20
|
+
* @param {string} [flag="--version"] — flag appended to cmd
|
|
21
|
+
* @returns {string | null} trimmed version string, or null on any failure
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { spawnSync } from "node:child_process";
|
|
25
|
+
|
|
26
|
+
export function getVersion(cmd, flag = "--version") {
|
|
27
|
+
let result;
|
|
28
|
+
try {
|
|
29
|
+
// shell:true matches the prior execSync behavior — accepts a
|
|
30
|
+
// space-joined command string and resolves via PATH lookup.
|
|
31
|
+
result = spawnSync(`${cmd} ${flag}`, [], {
|
|
32
|
+
shell: true,
|
|
33
|
+
encoding: "utf-8",
|
|
34
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
35
|
+
});
|
|
36
|
+
} catch {
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
// Match prior contract: any non-success path → null.
|
|
40
|
+
// `execSync` previously threw on either spawn failure or non-zero exit,
|
|
41
|
+
// and the caller's catch returned null. Replicate that here so a CLI
|
|
42
|
+
// that exits 1 with shell error text in stderr (e.g., "command not
|
|
43
|
+
// found") does NOT surface as a fake version string.
|
|
44
|
+
if (!result || result.error || result.status !== 0) return null;
|
|
45
|
+
const stdout = (result.stdout ?? "").toString().trim();
|
|
46
|
+
const stderr = (result.stderr ?? "").toString().trim();
|
|
47
|
+
if (stdout) return stdout;
|
|
48
|
+
if (stderr) return stderr;
|
|
49
|
+
return null;
|
|
50
|
+
}
|
package/bin/taskplane.mjs
CHANGED
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
ALL_GITIGNORE_PATTERNS,
|
|
37
37
|
patternToRegex,
|
|
38
38
|
} from "./gitignore-patterns.mjs";
|
|
39
|
+
import { getVersion } from "./get-version.mjs";
|
|
39
40
|
|
|
40
41
|
// ─── Paths ──────────────────────────────────────────────────────────────────
|
|
41
42
|
|
|
@@ -127,14 +128,10 @@ function commandExists(cmd) {
|
|
|
127
128
|
}
|
|
128
129
|
}
|
|
129
130
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
} catch {
|
|
135
|
-
return null;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
131
|
+
// `getVersion` lives in `./get-version.mjs` so it can be unit-tested
|
|
132
|
+
// without subprocessing the full CLI. Imported above. (TP-189-C / TP-185
|
|
133
|
+
// follow-up: capture both stdout and stderr because `pi --version`
|
|
134
|
+
// prints to stderr; null on failure preserves the original contract.)
|
|
138
135
|
|
|
139
136
|
/**
|
|
140
137
|
* Parse the tabular output from `pi --list-models` into structured model rows.
|
|
@@ -146,6 +146,12 @@ function writeSegmentExpansionRequest(request: SegmentExpansionRequest): string
|
|
|
146
146
|
* section. All-checkboxes-checked is also NOT a trigger — it is the normal
|
|
147
147
|
* pre-code-review state.
|
|
148
148
|
*
|
|
149
|
+
* Fenced code blocks (delimited by ``` or ~~~) inside a step's body are
|
|
150
|
+
* skipped during the scan (TP-189-A3). This avoids a false-positive
|
|
151
|
+
* refusal when a step documents the literal `**Status:** ✅ Complete`
|
|
152
|
+
* pattern as part of its own instructions — a legitimate authoring case
|
|
153
|
+
* that doesn't represent an actual completion claim.
|
|
154
|
+
*
|
|
149
155
|
* Designed to fail-open: any I/O error or a missing step heading returns
|
|
150
156
|
* `false` (the review proceeds). The prompt-side Recovery Recipe is the
|
|
151
157
|
* primary defense; this guard is a hard backstop, not a gatekeeper.
|
|
@@ -165,15 +171,64 @@ export function isStepMarkedComplete(statusPath: string, stepNum: number): boole
|
|
|
165
171
|
const lines = content.split(/\r?\n/);
|
|
166
172
|
const stepHeadingRe = new RegExp(`^###\\s+Step\\s+${stepNum}\\b`);
|
|
167
173
|
const nextStepHeadingRe = /^###\s+Step\s+\d+\b/;
|
|
168
|
-
|
|
174
|
+
// TP-189-A3: track fenced-code-block state per CommonMark semantics.
|
|
175
|
+
// A fence opens with 3+ backticks OR 3+ tildes optionally followed by
|
|
176
|
+
// an info string (e.g., ```javascript). A fence CLOSES only when a
|
|
177
|
+
// matching delimiter (same char, length >= opener length) is seen on
|
|
178
|
+
// a line by itself — the closer line MUST NOT contain trailing
|
|
179
|
+
// non-whitespace text. This distinction matters: ```javascript
|
|
180
|
+
// inside an open fence is content, not a closer; mistreating it as a
|
|
181
|
+
// closer would let `**Status:** ✅ Complete` later in the same code
|
|
182
|
+
// block trip the guard. Tracking opener char + length also avoids
|
|
183
|
+
// premature close on `~~~` inside a backtick fence (or vice versa).
|
|
184
|
+
const openerRe = /^\s*(`{3,}|~{3,})(.*)$/;
|
|
169
185
|
let inSection = false;
|
|
186
|
+
let fenceOpener: { char: string; length: number } | null = null;
|
|
170
187
|
for (const line of lines) {
|
|
171
188
|
if (!inSection) {
|
|
172
189
|
if (stepHeadingRe.test(line)) inSection = true;
|
|
173
190
|
continue;
|
|
174
191
|
}
|
|
175
|
-
//
|
|
176
|
-
|
|
192
|
+
// Step boundaries are recognized only OUTSIDE a fenced block.
|
|
193
|
+
// (A `### Step N:` line inside a code-fence sample is content,
|
|
194
|
+
// not a real heading.)
|
|
195
|
+
if (fenceOpener === null && nextStepHeadingRe.test(line)) break;
|
|
196
|
+
// Detect fence delimiter lines.
|
|
197
|
+
const fenceMatch = line.match(openerRe);
|
|
198
|
+
if (fenceMatch) {
|
|
199
|
+
const delim = fenceMatch[1];
|
|
200
|
+
const trailing = fenceMatch[2] ?? "";
|
|
201
|
+
const char = delim[0]; // "`" or "~"
|
|
202
|
+
const length = delim.length;
|
|
203
|
+
if (fenceOpener === null) {
|
|
204
|
+
// Opening: any trailing text is the info string — allowed.
|
|
205
|
+
// CommonMark forbids backticks in a backtick info string,
|
|
206
|
+
// but rejecting that case here only risks false negatives
|
|
207
|
+
// (i.e., not opening a fence we should have); the worst-
|
|
208
|
+
// case impact is a real Status line being inspected as if
|
|
209
|
+
// outside a fence — which is the safe default.
|
|
210
|
+
fenceOpener = { char, length };
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
// Already inside a fence — a line counts as a closer ONLY if:
|
|
214
|
+
// 1. delimiter char matches the opener,
|
|
215
|
+
// 2. delimiter length >= opener length,
|
|
216
|
+
// 3. nothing follows the delimiter except whitespace.
|
|
217
|
+
const trailingIsWhitespace = /^\s*$/.test(trailing);
|
|
218
|
+
if (
|
|
219
|
+
char === fenceOpener.char &&
|
|
220
|
+
length >= fenceOpener.length &&
|
|
221
|
+
trailingIsWhitespace
|
|
222
|
+
) {
|
|
223
|
+
fenceOpener = null;
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
// Else: this line is content INSIDE the open fence (e.g.,
|
|
227
|
+
// ```javascript inside a 4-backtick fence, or a non-matching
|
|
228
|
+
// tilde delimiter). Fall through to the inFence skip below.
|
|
229
|
+
}
|
|
230
|
+
// Skip lines inside an open fenced code block.
|
|
231
|
+
if (fenceOpener !== null) continue;
|
|
177
232
|
// Match a literal status line within this step's section.
|
|
178
233
|
// Examples that should match:
|
|
179
234
|
// **Status:** ✅ Complete
|
|
@@ -87,23 +87,17 @@ export const ENGINE_BRIDGE_TOOLS = [
|
|
|
87
87
|
"request_segment_expansion",
|
|
88
88
|
] as const;
|
|
89
89
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
* process, fs) into pure schema/type files. If you change the default
|
|
102
|
-
* here, update those copies too.
|
|
103
|
-
*
|
|
104
|
-
* @since TP-184
|
|
105
|
-
*/
|
|
106
|
-
export const DEFAULT_WORKER_USER_TOOLS = "read,write,edit,bash,grep,find,ls";
|
|
90
|
+
// TP-189 (Cluster B): `DEFAULT_WORKER_USER_TOOLS` now lives in the
|
|
91
|
+
// import-free `./tool-allowlist-constants.ts` module so that pure-data
|
|
92
|
+
// layers (`config-schema.ts`, `types.ts`) can import it without pulling
|
|
93
|
+
// agent-host's heavy `child_process`/`fs` imports into the schema/type
|
|
94
|
+
// graph. We re-export here so existing internal imports (e.g.,
|
|
95
|
+
// `execution.ts`, `worker-tools-allowlist.test.ts`) continue to work
|
|
96
|
+
// without churn.
|
|
97
|
+
//
|
|
98
|
+
// @since TP-184 (constant introduced) / TP-189 (moved to constants module)
|
|
99
|
+
export { DEFAULT_WORKER_USER_TOOLS } from "./tool-allowlist-constants.ts";
|
|
100
|
+
import { DEFAULT_WORKER_USER_TOOLS } from "./tool-allowlist-constants.ts";
|
|
107
101
|
|
|
108
102
|
/**
|
|
109
103
|
* Build the final worker `--tools` allowlist string by combining the
|
|
@@ -37,6 +37,12 @@
|
|
|
37
37
|
* @module config/schema
|
|
38
38
|
*/
|
|
39
39
|
|
|
40
|
+
// TP-189 (Cluster B): single source of truth for the worker user-tools
|
|
41
|
+
// default literal. This is a deliberately import-free module so we can
|
|
42
|
+
// import it here without pulling `agent-host.ts`'s `child_process`/`fs`
|
|
43
|
+
// imports into the schema layer.
|
|
44
|
+
import { DEFAULT_WORKER_USER_TOOLS } from "./tool-allowlist-constants.ts";
|
|
45
|
+
|
|
40
46
|
// ── Config Version ───────────────────────────────────────────────────
|
|
41
47
|
|
|
42
48
|
/**
|
|
@@ -594,14 +600,11 @@ export const DEFAULT_TASK_RUNNER_SECTION: TaskRunnerSection = {
|
|
|
594
600
|
testing: { commands: {} },
|
|
595
601
|
standards: { docs: [], rules: [] },
|
|
596
602
|
standardsOverrides: {},
|
|
597
|
-
//
|
|
598
|
-
// `
|
|
599
|
-
//
|
|
600
|
-
//
|
|
601
|
-
|
|
602
|
-
// default, update both copies. Engine bridge tools are appended at the
|
|
603
|
-
// lane-runner spawn site by `buildWorkerToolsAllowlist()`, not here.
|
|
604
|
-
worker: { model: "", tools: "read,write,edit,bash,grep,find,ls", thinking: "", excludeExtensions: [] },
|
|
603
|
+
// TP-189 (Cluster B): user-tools default sourced from
|
|
604
|
+
// `tool-allowlist-constants.ts` (single source of truth). Engine
|
|
605
|
+
// bridge tools are appended at the lane-runner spawn site by
|
|
606
|
+
// `buildWorkerToolsAllowlist()`, not here.
|
|
607
|
+
worker: { model: "", tools: DEFAULT_WORKER_USER_TOOLS, thinking: "", excludeExtensions: [] },
|
|
605
608
|
reviewer: { model: "", tools: "read,bash,grep,find,ls", thinking: "on", excludeExtensions: [] },
|
|
606
609
|
context: {
|
|
607
610
|
workerContextWindow: 0,
|
|
@@ -653,10 +656,12 @@ export const DEFAULT_ORCHESTRATOR_SECTION: OrchestratorSection = {
|
|
|
653
656
|
},
|
|
654
657
|
merge: {
|
|
655
658
|
model: "",
|
|
656
|
-
//
|
|
657
|
-
//
|
|
658
|
-
//
|
|
659
|
-
|
|
659
|
+
// TP-189 (Cluster B): merge default mirrors the worker user-tools
|
|
660
|
+
// constant. The merge agent does NOT run through
|
|
661
|
+
// `buildWorkerToolsAllowlist()` (no bridge-tool needs), so this
|
|
662
|
+
// reference is purely for default-value parity — not a hard
|
|
663
|
+
// coupling to the worker allowlist plumbing.
|
|
664
|
+
tools: DEFAULT_WORKER_USER_TOOLS,
|
|
660
665
|
thinking: "off",
|
|
661
666
|
verify: [],
|
|
662
667
|
order: "fewest-files-first",
|
|
@@ -52,7 +52,14 @@ export interface SessionTokenCounts {
|
|
|
52
52
|
* | `session_vanished` | Session disappeared without exit summary |
|
|
53
53
|
* | `stall_timeout` | No STATUS.md progress for stall_timeout minutes |
|
|
54
54
|
* | `user_killed` | User manually killed the session (e.g., forced process kill) |
|
|
55
|
+
* | `spawn_failure` | Worker process never spawned (e.g., Pi CLI not findable, worktree provisioning) |
|
|
55
56
|
* | `unknown` | Could not determine cause |
|
|
57
|
+
*
|
|
58
|
+
* Note: `spawn_failure` (TP-190, #561) is set BEFORE any agent process exists —
|
|
59
|
+
* it is produced synchronously when `spawnAgent()` throws (resolvePiCliPath
|
|
60
|
+
* miss, file-system error, etc.). It is intentionally NOT in
|
|
61
|
+
* `TIER0_RETRYABLE_CLASSIFICATIONS` because spawn-stage failures are never
|
|
62
|
+
* transient; retrying without operator intervention only burns budget.
|
|
56
63
|
*/
|
|
57
64
|
export type ExitClassification =
|
|
58
65
|
| "completed"
|
|
@@ -64,6 +71,7 @@ export type ExitClassification =
|
|
|
64
71
|
| "session_vanished"
|
|
65
72
|
| "stall_timeout"
|
|
66
73
|
| "user_killed"
|
|
74
|
+
| "spawn_failure"
|
|
67
75
|
| "unknown";
|
|
68
76
|
|
|
69
77
|
/**
|
|
@@ -79,6 +87,7 @@ export const EXIT_CLASSIFICATIONS: readonly ExitClassification[] = [
|
|
|
79
87
|
"session_vanished",
|
|
80
88
|
"stall_timeout",
|
|
81
89
|
"user_killed",
|
|
90
|
+
"spawn_failure",
|
|
82
91
|
"unknown",
|
|
83
92
|
] as const;
|
|
84
93
|
|
|
@@ -40,6 +40,17 @@ export type WorkerToMainMessage =
|
|
|
40
40
|
| { type: "monitor-update"; state: MonitorState }
|
|
41
41
|
| { type: "engine-event"; event: EngineEvent }
|
|
42
42
|
| { type: "supervisor-alert"; alert: SupervisorAlert }
|
|
43
|
+
/**
|
|
44
|
+
* TP-187 (#538): Lane has reached a terminal state. The supervisor process
|
|
45
|
+
* uses this to mark the lane terminated and filter any subsequently-arriving
|
|
46
|
+
* (zombie) alerts whose `context.laneNumber`/`context.agentId` matches.
|
|
47
|
+
*/
|
|
48
|
+
| { type: "lane-terminated"; info: import("./types.ts").LaneTerminatedInfo }
|
|
49
|
+
/**
|
|
50
|
+
* TP-187 (#538): Lane number has been re-allocated to a fresh task. The
|
|
51
|
+
* supervisor lifts the suppression so subsequent alerts pass through.
|
|
52
|
+
*/
|
|
53
|
+
| { type: "lane-respawned"; laneNumber: number; agentId: string; batchId: string }
|
|
43
54
|
| { type: "state-sync"; state: SerializedBatchState }
|
|
44
55
|
| { type: "complete"; state: SerializedBatchState }
|
|
45
56
|
| { type: "error"; message: string; stack?: string; source?: WorkerErrorSource };
|
|
@@ -325,6 +336,18 @@ if (process.env.TASKPLANE_ENGINE_FORK === "1" && typeof process.send === "functi
|
|
|
325
336
|
send({ type: "supervisor-alert", alert });
|
|
326
337
|
};
|
|
327
338
|
|
|
339
|
+
// TP-187 (#538): Lane termination callback — forwards lane-terminated to
|
|
340
|
+
// the supervisor process so it can suppress in-flight zombie alerts.
|
|
341
|
+
const onLaneTerminated = (info: import("./types.ts").LaneTerminatedInfo) => {
|
|
342
|
+
send({ type: "lane-terminated", info });
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
// TP-187 (#538): Lane respawn callback — forwards lane-respawned to
|
|
346
|
+
// the supervisor process so it can lift suppression for re-allocated lanes.
|
|
347
|
+
const onLaneRespawned = (laneNumber: number, agentId: string, batchId: string) => {
|
|
348
|
+
send({ type: "lane-respawned", laneNumber, agentId, batchId });
|
|
349
|
+
};
|
|
350
|
+
|
|
328
351
|
// ── Execute engine ───────────────────────────────────────────
|
|
329
352
|
const enginePromise = data.mode === "resume"
|
|
330
353
|
? resumeOrchBatch(
|
|
@@ -340,6 +363,8 @@ if (process.env.TASKPLANE_ENGINE_FORK === "1" && typeof process.send === "functi
|
|
|
340
363
|
data.force ?? false,
|
|
341
364
|
onSupervisorAlert,
|
|
342
365
|
data.supervisorAutonomy ?? "autonomous",
|
|
366
|
+
onLaneTerminated,
|
|
367
|
+
onLaneRespawned,
|
|
343
368
|
)
|
|
344
369
|
: executeOrchBatch(
|
|
345
370
|
data.args ?? "",
|
|
@@ -355,6 +380,8 @@ if (process.env.TASKPLANE_ENGINE_FORK === "1" && typeof process.send === "functi
|
|
|
355
380
|
onEngineEvent,
|
|
356
381
|
onSupervisorAlert,
|
|
357
382
|
data.supervisorAutonomy ?? "autonomous",
|
|
383
|
+
onLaneTerminated,
|
|
384
|
+
onLaneRespawned,
|
|
358
385
|
);
|
|
359
386
|
|
|
360
387
|
enginePromise
|
|
@@ -17,8 +17,9 @@ import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolic
|
|
|
17
17
|
import type { CleanupGateRepoFailure } from "./messages.ts";
|
|
18
18
|
import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-reports.ts";
|
|
19
19
|
import { resolveOperatorId } from "./naming.ts";
|
|
20
|
-
import { applyPartialProgressToOutcomes, buildTier0EventBase, deleteBatchState, emitEngineEvent, emitTier0Event, loadBatchHistory, loadBatchState, persistRuntimeState, saveBatchHistory, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
20
|
+
import { applyPartialProgressToOutcomes, buildTier0EventBase, deleteBatchState, emitEngineEvent, emitTier0Event, loadBatchHistory, loadBatchState, persistRuntimeState, saveBatchHistory, saveBatchMetaRuntimeArtifact, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
21
21
|
import { readRegistrySnapshot, isTerminalStatus, isProcessAlive as registryIsProcessAlive } from "./process-registry.ts";
|
|
22
|
+
import { drainAgentOutbox } from "./mailbox.ts";
|
|
22
23
|
import { buildBatchProgressSnapshot, buildEngineEventBase, buildSegmentId, buildSupervisorSegmentFrontierSnapshot, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
|
|
23
24
|
import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, ParsedTask, PersistedSegmentRecord, SegmentExpansionRequest, SupervisorAlert, SupervisorAlertCallback, TaskRunnerConfig, TaskSegmentPlan, TaskSegmentPlanMap, TaskSegmentNode, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
24
25
|
import { buildDependencyGraph, computeWaveAssignments, resolveBaseBranch, resolveRepoRoot, validateGraph } from "./waves.ts";
|
|
@@ -67,6 +68,114 @@ function emitTier0Escalation(
|
|
|
67
68
|
/** Zero-token sentinel used for task/wave/batch aggregation. */
|
|
68
69
|
const ZERO_TOKENS: TokenCounts = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, costUsd: 0 };
|
|
69
70
|
|
|
71
|
+
/**
|
|
72
|
+
* TP-190 (#561): Determine whether a wave's failures are entirely
|
|
73
|
+
* Runtime V2 spawn-stage failures.
|
|
74
|
+
*
|
|
75
|
+
* Returns `true` only when:
|
|
76
|
+
* - At least one task failed (`failedTaskIds.length > 0`)
|
|
77
|
+
* - No task succeeded — checked via BOTH the wave's projected
|
|
78
|
+
* `succeededTaskIds` (terminal task-level completion) AND a scan of
|
|
79
|
+
* `laneResults[].tasks[]` for any per-task outcome with
|
|
80
|
+
* `status === "succeeded"` (catches non-terminal segment successes
|
|
81
|
+
* on multi-segment tasks that schedule a continuation round and
|
|
82
|
+
* therefore don't appear in `succeededTaskIds`)
|
|
83
|
+
* - Every failed outcome carries
|
|
84
|
+
* `exitDiagnostic.classification === "spawn_failure"`
|
|
85
|
+
*
|
|
86
|
+
* The engine uses the result to transition `batchState.phase` to
|
|
87
|
+
* `"failed"` (not `"executing"` and not `"paused"`) so `orch_status()`
|
|
88
|
+
* and the dashboard surface an actionable answer for the operator.
|
|
89
|
+
* Spawn-stage errors (Pi CLI not findable, worktree provisioning
|
|
90
|
+
* failure, branch collision) are never transient — they require an
|
|
91
|
+
* external fix before re-running, so `"paused"` would be misleading.
|
|
92
|
+
*
|
|
93
|
+
* **Sage post-mortem note (multi-segment edge case, post-PR-#566):** the
|
|
94
|
+
* earlier version of this function checked only `succeededTaskIds.length
|
|
95
|
+
* !== 0` to gate the all-failed verdict. That field is the *projected*
|
|
96
|
+
* terminal-completion set: it's populated only when a task reaches its
|
|
97
|
+
* final segment. A wave that has a multi-segment task succeed on segment
|
|
98
|
+
* 1 (with a continuation segment in a later round) would have an empty
|
|
99
|
+
* `succeededTaskIds` even though work demonstrably succeeded. Combined
|
|
100
|
+
* with a single-segment spawn-failure on a different task, the wave
|
|
101
|
+
* would falsely trip the all-spawn-failed verdict. The added
|
|
102
|
+
* `laneResults` scan closes this gap by inspecting raw per-task outcome
|
|
103
|
+
* status before terminal projection.
|
|
104
|
+
*
|
|
105
|
+
* Pure function — exported for unit testing alongside the engine's
|
|
106
|
+
* post-wave handling logic.
|
|
107
|
+
*
|
|
108
|
+
* @since TP-190 (#561)
|
|
109
|
+
*/
|
|
110
|
+
export function isAllLanesSpawnFailedWave(
|
|
111
|
+
waveResult: {
|
|
112
|
+
failedTaskIds: string[];
|
|
113
|
+
succeededTaskIds: string[];
|
|
114
|
+
/**
|
|
115
|
+
* Optional per-lane outcomes. When provided, the function additionally
|
|
116
|
+
* checks whether ANY task outcome carried `status === "succeeded"`,
|
|
117
|
+
* which covers non-terminal segment successes that don't appear in the
|
|
118
|
+
* projected `succeededTaskIds`. Optional for backward compatibility
|
|
119
|
+
* with the v0.29.0 callers and the existing TP-190 unit tests; the
|
|
120
|
+
* production call site (engine.ts post-wave) always passes the full
|
|
121
|
+
* `WaveExecutionResult` and gets the stricter check.
|
|
122
|
+
*/
|
|
123
|
+
laneResults?: ReadonlyArray<{ tasks: ReadonlyArray<{ status: string }> }>;
|
|
124
|
+
},
|
|
125
|
+
allTaskOutcomes: LaneTaskOutcome[],
|
|
126
|
+
): boolean {
|
|
127
|
+
if (waveResult.failedTaskIds.length === 0) return false;
|
|
128
|
+
if (waveResult.succeededTaskIds.length !== 0) return false;
|
|
129
|
+
// TP-190 (#561) sage post-mortem: scan per-lane outcomes for any
|
|
130
|
+
// `status === "succeeded"`. This catches non-terminal segment successes
|
|
131
|
+
// that don't appear in `succeededTaskIds` (the latter is the terminal
|
|
132
|
+
// projection populated only when a multi-segment task reaches its final
|
|
133
|
+
// segment).
|
|
134
|
+
if (waveResult.laneResults) {
|
|
135
|
+
for (const laneResult of waveResult.laneResults) {
|
|
136
|
+
for (const taskOutcome of laneResult.tasks) {
|
|
137
|
+
if (taskOutcome.status === "succeeded") return false;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return waveResult.failedTaskIds.every((failedId) => {
|
|
142
|
+
const outcome = allTaskOutcomes.find((o) => o.taskId === failedId);
|
|
143
|
+
return outcome?.exitDiagnostic?.classification === "spawn_failure";
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* TP-190 (#561): Build the spawn-failure-specific extras layered onto a
|
|
149
|
+
* `task-failure` supervisor alert when the underlying outcome was a
|
|
150
|
+
* spawn-stage failure.
|
|
151
|
+
*
|
|
152
|
+
* Returns:
|
|
153
|
+
* - `exitCategory`: the structured `ExitClassification` that the
|
|
154
|
+
* supervisor playbook can branch on (e.g.,
|
|
155
|
+
* `"spawn_failure"` → escalate immediately rather than retry).
|
|
156
|
+
* - `summaryLine`: an extra ` Spawn failure: … escalate immediately…`
|
|
157
|
+
* line for human-readable display, blank string when the outcome
|
|
158
|
+
* is not a spawn failure (so the existing summary template renders
|
|
159
|
+
* unchanged for non-spawn cases).
|
|
160
|
+
*
|
|
161
|
+
* Pure function — exported for unit testing alongside the alert-emission
|
|
162
|
+
* logic in `executeOrchBatch` and `resumeOrchBatch`. Both call sites
|
|
163
|
+
* read from `outcome.exitDiagnostic?.classification` so the helper takes
|
|
164
|
+
* the optional classification directly.
|
|
165
|
+
*
|
|
166
|
+
* @since TP-190 (#561)
|
|
167
|
+
*/
|
|
168
|
+
export function buildSpawnFailureAlertExtras(
|
|
169
|
+
outcome: { exitDiagnostic?: { classification?: string } | undefined } | undefined,
|
|
170
|
+
): { exitCategory: import("./diagnostics.ts").ExitClassification | undefined; summaryLine: string } {
|
|
171
|
+
const raw = outcome?.exitDiagnostic?.classification;
|
|
172
|
+
const exitCategory = raw as import("./diagnostics.ts").ExitClassification | undefined;
|
|
173
|
+
const summaryLine = raw === "spawn_failure"
|
|
174
|
+
? ` Spawn failure: worker process never started — escalate immediately (do not retry)\n`
|
|
175
|
+
: "";
|
|
176
|
+
return { exitCategory, summaryLine };
|
|
177
|
+
}
|
|
178
|
+
|
|
70
179
|
/** Map embedded outcome telemetry to the batch-history TokenCounts shape. */
|
|
71
180
|
export function taskTokensFromOutcomeTelemetry(outcome: LaneTaskOutcome): TokenCounts {
|
|
72
181
|
const telemetry = outcome.telemetry;
|
|
@@ -1281,6 +1390,20 @@ async function attemptWorkerCrashRetry(
|
|
|
1281
1390
|
continue;
|
|
1282
1391
|
}
|
|
1283
1392
|
|
|
1393
|
+
// TP-190 (#561): Defense-in-depth — spawn-stage failures (Pi CLI not
|
|
1394
|
+
// findable, worktree provisioning failure, branch collision) are NEVER
|
|
1395
|
+
// transient. Retrying without operator intervention only burns the
|
|
1396
|
+
// retry budget and delays the supervisor alert. The generic
|
|
1397
|
+
// `TIER0_RETRYABLE_CLASSIFICATIONS.has()` gate below also catches this
|
|
1398
|
+
// (spawn_failure is not in the set), but the explicit early-return
|
|
1399
|
+
// here gives operators a clearer log message at the gate site.
|
|
1400
|
+
if (classification === "spawn_failure") {
|
|
1401
|
+
execLog("batch", batchState.batchId,
|
|
1402
|
+
`tier0: task ${taskId} spawn_failure — operator action required, NOT auto-retrying (TP-190)`,
|
|
1403
|
+
);
|
|
1404
|
+
continue;
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1284
1407
|
// Check if retryable
|
|
1285
1408
|
if (!TIER0_RETRYABLE_CLASSIFICATIONS.has(classification)) {
|
|
1286
1409
|
execLog("batch", batchState.batchId,
|
|
@@ -1781,6 +1904,8 @@ async function attemptStaleWorktreeRecovery(
|
|
|
1781
1904
|
onSupervisorAlert?: SupervisorAlertCallback,
|
|
1782
1905
|
supervisorAutonomy: "interactive" | "supervised" | "autonomous" = "autonomous",
|
|
1783
1906
|
runnerConfig?: TaskRunnerConfig,
|
|
1907
|
+
onLaneTerminated?: import("./types.ts").LaneTerminatedCallback,
|
|
1908
|
+
onLaneRespawned?: (laneNumber: number, agentId: string, batchId: string) => void,
|
|
1784
1909
|
): Promise<WaveExecutionResult | null> {
|
|
1785
1910
|
// Only attempt recovery for ALLOC_WORKTREE_FAILED
|
|
1786
1911
|
if (!waveResult.allocationError || waveResult.allocationError.code !== "ALLOC_WORKTREE_FAILED") {
|
|
@@ -1896,6 +2021,8 @@ async function attemptStaleWorktreeRecovery(
|
|
|
1896
2021
|
excludeExtensions: runnerConfig.worker.excludeExtensions ?? [],
|
|
1897
2022
|
} : undefined,
|
|
1898
2023
|
runnerConfig?.workerExcludeExtensions ?? [],
|
|
2024
|
+
onLaneTerminated,
|
|
2025
|
+
onLaneRespawned,
|
|
1899
2026
|
);
|
|
1900
2027
|
|
|
1901
2028
|
return retryResult;
|
|
@@ -1973,6 +2100,18 @@ export async function executeOrchBatch(
|
|
|
1973
2100
|
onEngineEvent?: EngineEventCallback | null,
|
|
1974
2101
|
onSupervisorAlert?: SupervisorAlertCallback | null,
|
|
1975
2102
|
supervisorAutonomy: "interactive" | "supervised" | "autonomous" = "autonomous",
|
|
2103
|
+
/**
|
|
2104
|
+
* TP-187 (#538): Optional callback fired when a lane reaches a terminal
|
|
2105
|
+
* state. The supervisor process forwards this over IPC and uses it to
|
|
2106
|
+
* suppress zombie alerts queued for the now-dead lane.
|
|
2107
|
+
*/
|
|
2108
|
+
onLaneTerminated?: import("./types.ts").LaneTerminatedCallback | null,
|
|
2109
|
+
/**
|
|
2110
|
+
* TP-187 (#538): Optional callback fired when a lane is freshly
|
|
2111
|
+
* (re-)allocated to a task. The supervisor process uses it to lift any
|
|
2112
|
+
* zombie-alert suppression carried over from a prior wave.
|
|
2113
|
+
*/
|
|
2114
|
+
onLaneRespawned?: ((laneNumber: number, agentId: string, batchId: string) => void) | null,
|
|
1976
2115
|
): Promise<void> {
|
|
1977
2116
|
const repoRoot = cwd;
|
|
1978
2117
|
// State files (.pi/batch-state.json, lane-state, etc.) belong in the workspace root,
|
|
@@ -1999,6 +2138,23 @@ export async function executeOrchBatch(
|
|
|
1999
2138
|
}
|
|
2000
2139
|
};
|
|
2001
2140
|
|
|
2141
|
+
// ── TP-187 (#538): Lane termination forwarding helper ──────
|
|
2142
|
+
// Forwards lane-terminated events through the same callback chain so the
|
|
2143
|
+
// supervisor process can suppress zombie alerts queued for a dead lane.
|
|
2144
|
+
const emitLaneTerminated = (info: import("./types.ts").LaneTerminatedInfo): void => {
|
|
2145
|
+
if (onLaneTerminated) {
|
|
2146
|
+
try {
|
|
2147
|
+
onLaneTerminated(info);
|
|
2148
|
+
} catch (err: unknown) {
|
|
2149
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2150
|
+
execLog("batch", batchState.batchId, `lane-terminated callback failed: ${msg}`, {
|
|
2151
|
+
laneNumber: info.laneNumber,
|
|
2152
|
+
reason: info.reason,
|
|
2153
|
+
});
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
};
|
|
2157
|
+
|
|
2002
2158
|
// ── TP-040 R002: Terminal event emission helper ──────────────
|
|
2003
2159
|
// Routes all early-return and terminal paths through consistent event
|
|
2004
2160
|
// emission so external consumers always receive a deterministic terminal
|
|
@@ -2312,6 +2468,22 @@ export async function executeOrchBatch(
|
|
|
2312
2468
|
// ── TS-009: Persist state on batch start (after wave computation) ──
|
|
2313
2469
|
persistRuntimeState("batch-start", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
2314
2470
|
|
|
2471
|
+
// ── TP-187 (#539): Persist batch-meta runtime artifact ──────────────
|
|
2472
|
+
// Captures the wave plan and core scalars to a runtime-side file that
|
|
2473
|
+
// survives `orch_abort()` (which deletes `.pi/batch-state.json`). Used by
|
|
2474
|
+
// `orch_resume(force=true)` to deterministically reconstruct state when
|
|
2475
|
+
// the main batch-state file is gone. Best-effort write: failures log only.
|
|
2476
|
+
saveBatchMetaRuntimeArtifact(stateRoot, {
|
|
2477
|
+
schemaVersion: 1,
|
|
2478
|
+
batchId: batchState.batchId,
|
|
2479
|
+
wavePlan: wavePlan.map(wave => [...wave]),
|
|
2480
|
+
baseBranch: batchState.baseBranch,
|
|
2481
|
+
orchBranch: batchState.orchBranch,
|
|
2482
|
+
mode: workspaceConfig ? "workspace" : "repo",
|
|
2483
|
+
startedAt: batchState.startedAt,
|
|
2484
|
+
totalWaves: wavePlan.length,
|
|
2485
|
+
});
|
|
2486
|
+
|
|
2315
2487
|
// ── TP-105: Runtime V2 backend selection ────────────────────
|
|
2316
2488
|
// Use Runtime V2 (no-TMUX lane-runner) when ALL conditions are met:
|
|
2317
2489
|
// 1. Exactly one task in the batch
|
|
@@ -2507,6 +2679,8 @@ export async function executeOrchBatch(
|
|
|
2507
2679
|
excludeExtensions: runnerConfig.worker.excludeExtensions ?? [],
|
|
2508
2680
|
} : undefined,
|
|
2509
2681
|
runnerConfig?.workerExcludeExtensions ?? [],
|
|
2682
|
+
emitLaneTerminated,
|
|
2683
|
+
onLaneRespawned ?? undefined,
|
|
2510
2684
|
);
|
|
2511
2685
|
|
|
2512
2686
|
// ── TP-039: Tier 0 — Stale worktree recovery ────────────
|
|
@@ -2530,6 +2704,8 @@ export async function executeOrchBatch(
|
|
|
2530
2704
|
emitAlert,
|
|
2531
2705
|
supervisorAutonomy,
|
|
2532
2706
|
runnerConfig,
|
|
2707
|
+
emitLaneTerminated,
|
|
2708
|
+
onLaneRespawned ?? undefined,
|
|
2533
2709
|
);
|
|
2534
2710
|
if (retryResult) {
|
|
2535
2711
|
const staleRecovered = !retryResult.allocationError;
|
|
@@ -3030,11 +3206,17 @@ export async function executeOrchBatch(
|
|
|
3030
3206
|
const frontierSummary = segmentFrontier
|
|
3031
3207
|
? ` Segment frontier: ${segmentFrontier.terminalSegments}/${segmentFrontier.totalSegments} terminal\n`
|
|
3032
3208
|
: "";
|
|
3209
|
+
// TP-190 (#561): Surface the structured exit category so the supervisor
|
|
3210
|
+
// playbook can branch deterministically. In particular,
|
|
3211
|
+
// `exitCategory === "spawn_failure"` signals an immediate-escalation
|
|
3212
|
+
// failure (not a retry candidate) — the worker process never spawned.
|
|
3213
|
+
const { exitCategory, summaryLine: spawnFailureLine } = buildSpawnFailureAlertExtras(outcome);
|
|
3033
3214
|
emitAlert({
|
|
3034
3215
|
category: "task-failure",
|
|
3035
3216
|
summary:
|
|
3036
3217
|
`⚠️ Task failure: ${taskId}\n` +
|
|
3037
3218
|
` Exit reason: ${exitReason}\n` +
|
|
3219
|
+
spawnFailureLine +
|
|
3038
3220
|
segmentSummary +
|
|
3039
3221
|
frontierSummary +
|
|
3040
3222
|
` Lane: ${laneForTask?.laneId ?? "unknown"} (lane ${laneForTask?.laneNumber ?? "?"})\n` +
|
|
@@ -3054,10 +3236,62 @@ export async function executeOrchBatch(
|
|
|
3054
3236
|
laneNumber: laneForTask?.laneNumber,
|
|
3055
3237
|
waveIndex: waveIdx,
|
|
3056
3238
|
exitReason,
|
|
3239
|
+
exitCategory,
|
|
3057
3240
|
partialProgress: hasPartialProgress,
|
|
3058
3241
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
3059
3242
|
},
|
|
3060
3243
|
});
|
|
3244
|
+
|
|
3245
|
+
// TP-187 (#538): Hard-fail termination — synchronously drain the
|
|
3246
|
+
// agent's outbox so stale escalations/replies don't get re-discovered
|
|
3247
|
+
// later, then emit lane-terminated so the supervisor process
|
|
3248
|
+
// suppresses any in-transit zombie alerts targeting this lane/agent.
|
|
3249
|
+
if (laneForTask) {
|
|
3250
|
+
const hardFailAgentId = outcome?.sessionName && outcome.sessionName.length > 0
|
|
3251
|
+
? outcome.sessionName
|
|
3252
|
+
: `${laneForTask.laneSessionId}-worker`;
|
|
3253
|
+
try {
|
|
3254
|
+
const drained = drainAgentOutbox(stateRoot, batchState.batchId, hardFailAgentId);
|
|
3255
|
+
if (drained > 0) {
|
|
3256
|
+
execLog("batch", batchState.batchId, `hard-fail outbox drain: ${drained} entr${drained === 1 ? "y" : "ies"} for ${hardFailAgentId}`);
|
|
3257
|
+
}
|
|
3258
|
+
} catch { /* best effort — do not block termination */ }
|
|
3259
|
+
emitLaneTerminated({
|
|
3260
|
+
laneNumber: laneForTask.laneNumber,
|
|
3261
|
+
agentId: hardFailAgentId,
|
|
3262
|
+
batchId: batchState.batchId,
|
|
3263
|
+
terminatedAt: Date.now(),
|
|
3264
|
+
reason: "hard-fail",
|
|
3265
|
+
});
|
|
3266
|
+
}
|
|
3267
|
+
}
|
|
3268
|
+
|
|
3269
|
+
// ── TP-190 (#561): All-lane spawn-failure phase transition ──
|
|
3270
|
+
// When every task in this wave failed AND every failure is a
|
|
3271
|
+
// `spawn_failure` (worker process never started), the operator cannot
|
|
3272
|
+
// recover without changing something external (Pi CLI install, file
|
|
3273
|
+
// permissions, branch state). Transition `phase` to `"failed"` so
|
|
3274
|
+
// `orch_status()` and the dashboard surface an actionable answer
|
|
3275
|
+
// (`failed`) instead of leaving the operator with `executing` while
|
|
3276
|
+
// every lane is dead. We use `"failed"` rather than `"paused"` (per
|
|
3277
|
+
// PROMPT design): `paused` implies an operator-flippable resume,
|
|
3278
|
+
// which is wrong here — spawn failures require an external fix first.
|
|
3279
|
+
// `isAllLanesSpawnFailedWave` is exported as a pure helper for unit
|
|
3280
|
+
// testing.
|
|
3281
|
+
const allFailedAreSpawnFailures = isAllLanesSpawnFailedWave(waveResult, allTaskOutcomes);
|
|
3282
|
+
if (allFailedAreSpawnFailures) {
|
|
3283
|
+
batchState.phase = "failed";
|
|
3284
|
+
execLog("batch", batchState.batchId,
|
|
3285
|
+
`phase → failed: every lane in wave ${waveIdx + 1} hit spawn_failure (TP-190 #561)`,
|
|
3286
|
+
{ failedTasks: waveResult.failedTaskIds.join(",") },
|
|
3287
|
+
);
|
|
3288
|
+
onNotify(
|
|
3289
|
+
ORCH_MESSAGES.orchBatchFailed(batchState.batchId, `all lanes in wave ${waveIdx + 1} failed to spawn (Runtime V2 spawn-failure — see task-failure alerts above)`),
|
|
3290
|
+
"error",
|
|
3291
|
+
);
|
|
3292
|
+
persistRuntimeState("wave-spawn-failure", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
3293
|
+
emitTerminalEvent(`All-lane spawn failure at wave ${waveIdx + 1}`);
|
|
3294
|
+
break;
|
|
3061
3295
|
}
|
|
3062
3296
|
|
|
3063
3297
|
// ── TS-009: Persist state after wave execution ──
|