taskplane 0.29.0 → 0.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -52,7 +52,14 @@ export interface SessionTokenCounts {
|
|
|
52
52
|
* | `session_vanished` | Session disappeared without exit summary |
|
|
53
53
|
* | `stall_timeout` | No STATUS.md progress for stall_timeout minutes |
|
|
54
54
|
* | `user_killed` | User manually killed the session (e.g., forced process kill) |
|
|
55
|
+
* | `spawn_failure` | Worker process never spawned (e.g., Pi CLI not findable, worktree provisioning) |
|
|
55
56
|
* | `unknown` | Could not determine cause |
|
|
57
|
+
*
|
|
58
|
+
* Note: `spawn_failure` (TP-190, #561) is set BEFORE any agent process exists —
|
|
59
|
+
* it is produced synchronously when `spawnAgent()` throws (resolvePiCliPath
|
|
60
|
+
* miss, file-system error, etc.). It is intentionally NOT in
|
|
61
|
+
* `TIER0_RETRYABLE_CLASSIFICATIONS` because spawn-stage failures are never
|
|
62
|
+
* transient; retrying without operator intervention only burns budget.
|
|
56
63
|
*/
|
|
57
64
|
export type ExitClassification =
|
|
58
65
|
| "completed"
|
|
@@ -64,6 +71,7 @@ export type ExitClassification =
|
|
|
64
71
|
| "session_vanished"
|
|
65
72
|
| "stall_timeout"
|
|
66
73
|
| "user_killed"
|
|
74
|
+
| "spawn_failure"
|
|
67
75
|
| "unknown";
|
|
68
76
|
|
|
69
77
|
/**
|
|
@@ -79,6 +87,7 @@ export const EXIT_CLASSIFICATIONS: readonly ExitClassification[] = [
|
|
|
79
87
|
"session_vanished",
|
|
80
88
|
"stall_timeout",
|
|
81
89
|
"user_killed",
|
|
90
|
+
"spawn_failure",
|
|
82
91
|
"unknown",
|
|
83
92
|
] as const;
|
|
84
93
|
|
|
@@ -68,6 +68,114 @@ function emitTier0Escalation(
|
|
|
68
68
|
/** Zero-token sentinel used for task/wave/batch aggregation. */
|
|
69
69
|
const ZERO_TOKENS: TokenCounts = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, costUsd: 0 };
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* TP-190 (#561): Determine whether a wave's failures are entirely
|
|
73
|
+
* Runtime V2 spawn-stage failures.
|
|
74
|
+
*
|
|
75
|
+
* Returns `true` only when:
|
|
76
|
+
* - At least one task failed (`failedTaskIds.length > 0`)
|
|
77
|
+
* - No task succeeded — checked via BOTH the wave's projected
|
|
78
|
+
* `succeededTaskIds` (terminal task-level completion) AND a scan of
|
|
79
|
+
* `laneResults[].tasks[]` for any per-task outcome with
|
|
80
|
+
* `status === "succeeded"` (catches non-terminal segment successes
|
|
81
|
+
* on multi-segment tasks that schedule a continuation round and
|
|
82
|
+
* therefore don't appear in `succeededTaskIds`)
|
|
83
|
+
* - Every failed outcome carries
|
|
84
|
+
* `exitDiagnostic.classification === "spawn_failure"`
|
|
85
|
+
*
|
|
86
|
+
* The engine uses the result to transition `batchState.phase` to
|
|
87
|
+
* `"failed"` (not `"executing"` and not `"paused"`) so `orch_status()`
|
|
88
|
+
* and the dashboard surface an actionable answer for the operator.
|
|
89
|
+
* Spawn-stage errors (Pi CLI not findable, worktree provisioning
|
|
90
|
+
* failure, branch collision) are never transient — they require an
|
|
91
|
+
* external fix before re-running, so `"paused"` would be misleading.
|
|
92
|
+
*
|
|
93
|
+
* **Sage post-mortem note (multi-segment edge case, post-PR-#566):** the
|
|
94
|
+
* earlier version of this function checked only `succeededTaskIds.length
|
|
95
|
+
* !== 0` to gate the all-failed verdict. That field is the *projected*
|
|
96
|
+
* terminal-completion set: it's populated only when a task reaches its
|
|
97
|
+
* final segment. A wave that has a multi-segment task succeed on segment
|
|
98
|
+
* 1 (with a continuation segment in a later round) would have an empty
|
|
99
|
+
* `succeededTaskIds` even though work demonstrably succeeded. Combined
|
|
100
|
+
* with a single-segment spawn-failure on a different task, the wave
|
|
101
|
+
* would falsely trip the all-spawn-failed verdict. The added
|
|
102
|
+
* `laneResults` scan closes this gap by inspecting raw per-task outcome
|
|
103
|
+
* status before terminal projection.
|
|
104
|
+
*
|
|
105
|
+
* Pure function — exported for unit testing alongside the engine's
|
|
106
|
+
* post-wave handling logic.
|
|
107
|
+
*
|
|
108
|
+
* @since TP-190 (#561)
|
|
109
|
+
*/
|
|
110
|
+
export function isAllLanesSpawnFailedWave(
|
|
111
|
+
waveResult: {
|
|
112
|
+
failedTaskIds: string[];
|
|
113
|
+
succeededTaskIds: string[];
|
|
114
|
+
/**
|
|
115
|
+
* Optional per-lane outcomes. When provided, the function additionally
|
|
116
|
+
* checks whether ANY task outcome carried `status === "succeeded"`,
|
|
117
|
+
* which covers non-terminal segment successes that don't appear in the
|
|
118
|
+
* projected `succeededTaskIds`. Optional for backward compatibility
|
|
119
|
+
* with the v0.29.0 callers and the existing TP-190 unit tests; the
|
|
120
|
+
* production call site (engine.ts post-wave) always passes the full
|
|
121
|
+
* `WaveExecutionResult` and gets the stricter check.
|
|
122
|
+
*/
|
|
123
|
+
laneResults?: ReadonlyArray<{ tasks: ReadonlyArray<{ status: string }> }>;
|
|
124
|
+
},
|
|
125
|
+
allTaskOutcomes: LaneTaskOutcome[],
|
|
126
|
+
): boolean {
|
|
127
|
+
if (waveResult.failedTaskIds.length === 0) return false;
|
|
128
|
+
if (waveResult.succeededTaskIds.length !== 0) return false;
|
|
129
|
+
// TP-190 (#561) sage post-mortem: scan per-lane outcomes for any
|
|
130
|
+
// `status === "succeeded"`. This catches non-terminal segment successes
|
|
131
|
+
// that don't appear in `succeededTaskIds` (the latter is the terminal
|
|
132
|
+
// projection populated only when a multi-segment task reaches its final
|
|
133
|
+
// segment).
|
|
134
|
+
if (waveResult.laneResults) {
|
|
135
|
+
for (const laneResult of waveResult.laneResults) {
|
|
136
|
+
for (const taskOutcome of laneResult.tasks) {
|
|
137
|
+
if (taskOutcome.status === "succeeded") return false;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return waveResult.failedTaskIds.every((failedId) => {
|
|
142
|
+
const outcome = allTaskOutcomes.find((o) => o.taskId === failedId);
|
|
143
|
+
return outcome?.exitDiagnostic?.classification === "spawn_failure";
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* TP-190 (#561): Build the spawn-failure-specific extras layered onto a
|
|
149
|
+
* `task-failure` supervisor alert when the underlying outcome was a
|
|
150
|
+
* spawn-stage failure.
|
|
151
|
+
*
|
|
152
|
+
* Returns:
|
|
153
|
+
* - `exitCategory`: the structured `ExitClassification` that the
|
|
154
|
+
* supervisor playbook can branch on (e.g.,
|
|
155
|
+
* `"spawn_failure"` → escalate immediately rather than retry).
|
|
156
|
+
* - `summaryLine`: an extra ` Spawn failure: … escalate immediately…`
|
|
157
|
+
* line for human-readable display, blank string when the outcome
|
|
158
|
+
* is not a spawn failure (so the existing summary template renders
|
|
159
|
+
* unchanged for non-spawn cases).
|
|
160
|
+
*
|
|
161
|
+
* Pure function — exported for unit testing alongside the alert-emission
|
|
162
|
+
* logic in `executeOrchBatch` and `resumeOrchBatch`. Both call sites
|
|
163
|
+
* read from `outcome.exitDiagnostic?.classification` so the helper takes
|
|
164
|
+
* the optional classification directly.
|
|
165
|
+
*
|
|
166
|
+
* @since TP-190 (#561)
|
|
167
|
+
*/
|
|
168
|
+
export function buildSpawnFailureAlertExtras(
|
|
169
|
+
outcome: { exitDiagnostic?: { classification?: string } | undefined } | undefined,
|
|
170
|
+
): { exitCategory: import("./diagnostics.ts").ExitClassification | undefined; summaryLine: string } {
|
|
171
|
+
const raw = outcome?.exitDiagnostic?.classification;
|
|
172
|
+
const exitCategory = raw as import("./diagnostics.ts").ExitClassification | undefined;
|
|
173
|
+
const summaryLine = raw === "spawn_failure"
|
|
174
|
+
? ` Spawn failure: worker process never started — escalate immediately (do not retry)\n`
|
|
175
|
+
: "";
|
|
176
|
+
return { exitCategory, summaryLine };
|
|
177
|
+
}
|
|
178
|
+
|
|
71
179
|
/** Map embedded outcome telemetry to the batch-history TokenCounts shape. */
|
|
72
180
|
export function taskTokensFromOutcomeTelemetry(outcome: LaneTaskOutcome): TokenCounts {
|
|
73
181
|
const telemetry = outcome.telemetry;
|
|
@@ -1282,6 +1390,20 @@ async function attemptWorkerCrashRetry(
|
|
|
1282
1390
|
continue;
|
|
1283
1391
|
}
|
|
1284
1392
|
|
|
1393
|
+
// TP-190 (#561): Defense-in-depth — spawn-stage failures (Pi CLI not
|
|
1394
|
+
// findable, worktree provisioning failure, branch collision) are NEVER
|
|
1395
|
+
// transient. Retrying without operator intervention only burns the
|
|
1396
|
+
// retry budget and delays the supervisor alert. The generic
|
|
1397
|
+
// `TIER0_RETRYABLE_CLASSIFICATIONS.has()` gate below also catches this
|
|
1398
|
+
// (spawn_failure is not in the set), but the explicit early-return
|
|
1399
|
+
// here gives operators a clearer log message at the gate site.
|
|
1400
|
+
if (classification === "spawn_failure") {
|
|
1401
|
+
execLog("batch", batchState.batchId,
|
|
1402
|
+
`tier0: task ${taskId} spawn_failure — operator action required, NOT auto-retrying (TP-190)`,
|
|
1403
|
+
);
|
|
1404
|
+
continue;
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1285
1407
|
// Check if retryable
|
|
1286
1408
|
if (!TIER0_RETRYABLE_CLASSIFICATIONS.has(classification)) {
|
|
1287
1409
|
execLog("batch", batchState.batchId,
|
|
@@ -3084,11 +3206,17 @@ export async function executeOrchBatch(
|
|
|
3084
3206
|
const frontierSummary = segmentFrontier
|
|
3085
3207
|
? ` Segment frontier: ${segmentFrontier.terminalSegments}/${segmentFrontier.totalSegments} terminal\n`
|
|
3086
3208
|
: "";
|
|
3209
|
+
// TP-190 (#561): Surface the structured exit category so the supervisor
|
|
3210
|
+
// playbook can branch deterministically. In particular,
|
|
3211
|
+
// `exitCategory === "spawn_failure"` signals an immediate-escalation
|
|
3212
|
+
// failure (not a retry candidate) — the worker process never spawned.
|
|
3213
|
+
const { exitCategory, summaryLine: spawnFailureLine } = buildSpawnFailureAlertExtras(outcome);
|
|
3087
3214
|
emitAlert({
|
|
3088
3215
|
category: "task-failure",
|
|
3089
3216
|
summary:
|
|
3090
3217
|
`⚠️ Task failure: ${taskId}\n` +
|
|
3091
3218
|
` Exit reason: ${exitReason}\n` +
|
|
3219
|
+
spawnFailureLine +
|
|
3092
3220
|
segmentSummary +
|
|
3093
3221
|
frontierSummary +
|
|
3094
3222
|
` Lane: ${laneForTask?.laneId ?? "unknown"} (lane ${laneForTask?.laneNumber ?? "?"})\n` +
|
|
@@ -3108,6 +3236,7 @@ export async function executeOrchBatch(
|
|
|
3108
3236
|
laneNumber: laneForTask?.laneNumber,
|
|
3109
3237
|
waveIndex: waveIdx,
|
|
3110
3238
|
exitReason,
|
|
3239
|
+
exitCategory,
|
|
3111
3240
|
partialProgress: hasPartialProgress,
|
|
3112
3241
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
3113
3242
|
},
|
|
@@ -3137,6 +3266,34 @@ export async function executeOrchBatch(
|
|
|
3137
3266
|
}
|
|
3138
3267
|
}
|
|
3139
3268
|
|
|
3269
|
+
// ── TP-190 (#561): All-lane spawn-failure phase transition ──
|
|
3270
|
+
// When every task in this wave failed AND every failure is a
|
|
3271
|
+
// `spawn_failure` (worker process never started), the operator cannot
|
|
3272
|
+
// recover without changing something external (Pi CLI install, file
|
|
3273
|
+
// permissions, branch state). Transition `phase` to `"failed"` so
|
|
3274
|
+
// `orch_status()` and the dashboard surface an actionable answer
|
|
3275
|
+
// (`failed`) instead of leaving the operator with `executing` while
|
|
3276
|
+
// every lane is dead. We use `"failed"` rather than `"paused"` (per
|
|
3277
|
+
// PROMPT design): `paused` implies an operator-flippable resume,
|
|
3278
|
+
// which is wrong here — spawn failures require an external fix first.
|
|
3279
|
+
// `isAllLanesSpawnFailedWave` is exported as a pure helper for unit
|
|
3280
|
+
// testing.
|
|
3281
|
+
const allFailedAreSpawnFailures = isAllLanesSpawnFailedWave(waveResult, allTaskOutcomes);
|
|
3282
|
+
if (allFailedAreSpawnFailures) {
|
|
3283
|
+
batchState.phase = "failed";
|
|
3284
|
+
execLog("batch", batchState.batchId,
|
|
3285
|
+
`phase → failed: every lane in wave ${waveIdx + 1} hit spawn_failure (TP-190 #561)`,
|
|
3286
|
+
{ failedTasks: waveResult.failedTaskIds.join(",") },
|
|
3287
|
+
);
|
|
3288
|
+
onNotify(
|
|
3289
|
+
ORCH_MESSAGES.orchBatchFailed(batchState.batchId, `all lanes in wave ${waveIdx + 1} failed to spawn (Runtime V2 spawn-failure — see task-failure alerts above)`),
|
|
3290
|
+
"error",
|
|
3291
|
+
);
|
|
3292
|
+
persistRuntimeState("wave-spawn-failure", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
3293
|
+
emitTerminalEvent(`All-lane spawn failure at wave ${waveIdx + 1}`);
|
|
3294
|
+
break;
|
|
3295
|
+
}
|
|
3296
|
+
|
|
3140
3297
|
// ── TS-009: Persist state after wave execution ──
|
|
3141
3298
|
persistRuntimeState("wave-execution-complete", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
3142
3299
|
|
|
@@ -8,9 +8,10 @@ import { join, dirname, basename, resolve, relative, delimiter as pathDelimiter
|
|
|
8
8
|
import { userInfo } from "os";
|
|
9
9
|
|
|
10
10
|
import { DONE_GRACE_MS, EXECUTION_POLL_INTERVAL_MS, ExecutionError, SESSION_SPAWN_RETRY_MAX } from "./types.ts";
|
|
11
|
-
import type { AllocatedLane, AllocatedTask, DependencyGraph, LaneExecutionResult, LaneMonitorSnapshot, LaneTaskOutcome, LaneTaskStatus, MonitorState, MtimeTracker, OrchestratorConfig, ParsedTask, TaskMonitorSnapshot, WaveExecutionResult, WorkspaceConfig, ExecutionUnit, PacketPaths, RuntimeAgentId, RuntimeAgentRole, SupervisorAlertCallback } from "./types.ts";
|
|
11
|
+
import type { AllocatedLane, AllocatedTask, DependencyGraph, LaneExecutionResult, LaneMonitorSnapshot, LaneTaskOutcome, LaneTaskStatus, MonitorState, MtimeTracker, OrchestratorConfig, ParsedTask, TaskMonitorSnapshot, WaveExecutionResult, WorkspaceConfig, ExecutionUnit, PacketPaths, RuntimeAgentId, RuntimeAgentRole, RuntimeLaneSnapshot, SupervisorAlertCallback } from "./types.ts";
|
|
12
12
|
import { resolvePacketPaths, buildRuntimeAgentId } from "./types.ts";
|
|
13
|
-
import {
|
|
13
|
+
import type { TaskExitDiagnostic } from "./diagnostics.ts";
|
|
14
|
+
import { readRegistrySnapshot, readLaneSnapshot, isTerminalStatus, isProcessAlive, detectOrphans, markOrphansCrashed, buildRegistrySnapshot, writeRegistrySnapshot, writeLaneSnapshot } from "./process-registry.ts";
|
|
14
15
|
import { allocateLanes } from "./waves.ts";
|
|
15
16
|
import { resolveOperatorId } from "./naming.ts";
|
|
16
17
|
import { runGit, runGitWithEnv } from "./git.ts";
|
|
@@ -829,18 +830,31 @@ export async function resolveTaskMonitorState(
|
|
|
829
830
|
// Assume alive initially, but if stale for >30s consult the registry
|
|
830
831
|
// to avoid indefinite false "running" if the lane-runner died.
|
|
831
832
|
const staleMs = snap?.updatedAt ? (now - snap.updatedAt) : 0;
|
|
833
|
+
const trackerAgeMs = now - tracker.firstObservedAt;
|
|
832
834
|
if (staleMs > 30_000) {
|
|
833
835
|
// Snapshot hasn't been updated for 30s+ — check registry as fallback.
|
|
834
836
|
// But also check if the tracker just started (firstObservedAt within
|
|
835
837
|
// last 60s) — wave transitions can leave stale snapshots from the
|
|
836
838
|
// prior wave/task while the new worker is still spawning.
|
|
837
|
-
const trackerAgeMs = now - tracker.firstObservedAt;
|
|
838
839
|
if (trackerAgeMs < 60_000) {
|
|
839
840
|
// New task, stale snapshot — give the worker startup grace period
|
|
840
841
|
sessionAlive = true;
|
|
841
842
|
} else {
|
|
842
843
|
sessionAlive = isV2AgentAlive(sessionName, runtimeBackend, v2Context?.laneNumber);
|
|
843
844
|
}
|
|
845
|
+
} else if (snap == null && trackerAgeMs >= 60_000) {
|
|
846
|
+
// TP-190 (#561 sage post-mortem): when NO snapshot exists at all
|
|
847
|
+
// (not even stale) and the tracker has been observing this task
|
|
848
|
+
// for >= 60s, fall back to the registry liveness check. Without
|
|
849
|
+
// this branch, a snapshot-write failure in the spawn-failure catch
|
|
850
|
+
// (disk full, permission error, transient I/O hiccup) leaves
|
|
851
|
+
// `snap == null` AND `staleMs == 0`, which previously hit the
|
|
852
|
+
// unconditional-alive default below — reintroducing the same
|
|
853
|
+
// monitor hang the spawn-failure catch was supposed to fix.
|
|
854
|
+
// 60s tracker-age threshold matches the existing startup-grace
|
|
855
|
+
// boundary so we don't false-fail a slow-starting worker that
|
|
856
|
+
// hasn't yet written its first snapshot.
|
|
857
|
+
sessionAlive = isV2AgentAlive(sessionName, runtimeBackend, v2Context?.laneNumber);
|
|
844
858
|
} else {
|
|
845
859
|
sessionAlive = true;
|
|
846
860
|
}
|
|
@@ -2722,17 +2736,90 @@ export async function executeLaneV2(
|
|
|
2722
2736
|
} catch (err: unknown) {
|
|
2723
2737
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
2724
2738
|
execLog(laneId, task.taskId, `Runtime V2 execution error: ${errMsg}`);
|
|
2739
|
+
|
|
2740
|
+
// TP-190 (#561): Spawn-stage failures (Pi CLI not findable, worktree
|
|
2741
|
+
// provisioning failure, etc.) reach this catch synchronously —
|
|
2742
|
+
// `spawnAgent()` calls `resolvePiCliPath()` and other resolvers that
|
|
2743
|
+
// throw before any process is registered. Tag the outcome with the
|
|
2744
|
+
// `spawn_failure` ExitClassification so:
|
|
2745
|
+
// 1. The retry classifier (TIER0_RETRYABLE_CLASSIFICATIONS) excludes
|
|
2746
|
+
// it deterministically — spawn errors are never transient.
|
|
2747
|
+
// 2. The supervisor `task-failure` IPC alert can carry
|
|
2748
|
+
// `context.exitCategory = "spawn_failure"` so the playbook can
|
|
2749
|
+
// escalate immediately rather than retrying.
|
|
2750
|
+
// 3. The engine's post-wave logic can transition `phase` to
|
|
2751
|
+
// `"failed"` when every lane in a wave spawn-failed.
|
|
2752
|
+
const spawnExitDiagnostic: TaskExitDiagnostic = {
|
|
2753
|
+
classification: "spawn_failure",
|
|
2754
|
+
exitCode: null,
|
|
2755
|
+
errorMessage: errMsg,
|
|
2756
|
+
tokensUsed: null,
|
|
2757
|
+
contextPct: null,
|
|
2758
|
+
partialProgressCommits: 0,
|
|
2759
|
+
partialProgressBranch: null,
|
|
2760
|
+
durationSec: 0,
|
|
2761
|
+
lastKnownStep: null,
|
|
2762
|
+
lastKnownCheckbox: null,
|
|
2763
|
+
repoId: lane.repoId ?? "default",
|
|
2764
|
+
};
|
|
2765
|
+
const workerAgentId = buildRuntimeAgentId(agentIdPrefix, lane.laneNumber, "worker");
|
|
2725
2766
|
outcomes.push({
|
|
2726
2767
|
taskId: task.taskId,
|
|
2727
2768
|
status: "failed",
|
|
2728
2769
|
segmentId: taskSegmentId,
|
|
2729
2770
|
startTime: Date.now(),
|
|
2730
2771
|
endTime: Date.now(),
|
|
2731
|
-
exitReason: `
|
|
2732
|
-
sessionName:
|
|
2772
|
+
exitReason: `spawn failure: ${errMsg}`,
|
|
2773
|
+
sessionName: workerAgentId,
|
|
2733
2774
|
doneFileFound: false,
|
|
2734
2775
|
laneNumber: lane.laneNumber,
|
|
2776
|
+
exitDiagnostic: spawnExitDiagnostic,
|
|
2735
2777
|
});
|
|
2778
|
+
|
|
2779
|
+
// TP-190 (#561): Write a synthetic terminal lane snapshot so the
|
|
2780
|
+
// monitor (`monitorLanes` → `resolveTaskMonitorState`) reads
|
|
2781
|
+
// `snap.taskId === taskId` AND `snap.status === "failed"`, which sets
|
|
2782
|
+
// `sessionAlive = false` and triggers Priority 3 ("Session exited
|
|
2783
|
+
// without .DONE → failed"). Without this, the monitor's
|
|
2784
|
+
// `snap == null` startup-grace branch keeps `sessionAlive = true`
|
|
2785
|
+
// indefinitely and `executeWave` blocks forever on `await
|
|
2786
|
+
// monitorPromise`. Use the full `RuntimeLaneSnapshot` shape so
|
|
2787
|
+
// dashboard consumers stay schema-consistent.
|
|
2788
|
+
try {
|
|
2789
|
+
const spawnFailureSnapshot: RuntimeLaneSnapshot = {
|
|
2790
|
+
batchId,
|
|
2791
|
+
laneNumber: lane.laneNumber,
|
|
2792
|
+
laneId: `lane-${lane.laneNumber}`,
|
|
2793
|
+
repoId: lane.repoId ?? "default",
|
|
2794
|
+
taskId: task.taskId,
|
|
2795
|
+
segmentId: taskSegmentId,
|
|
2796
|
+
status: "failed",
|
|
2797
|
+
worker: {
|
|
2798
|
+
agentId: workerAgentId,
|
|
2799
|
+
status: "crashed",
|
|
2800
|
+
elapsedMs: 0,
|
|
2801
|
+
toolCalls: 0,
|
|
2802
|
+
contextPct: 0,
|
|
2803
|
+
costUsd: 0,
|
|
2804
|
+
lastTool: "",
|
|
2805
|
+
inputTokens: 0,
|
|
2806
|
+
outputTokens: 0,
|
|
2807
|
+
cacheReadTokens: 0,
|
|
2808
|
+
cacheWriteTokens: 0,
|
|
2809
|
+
},
|
|
2810
|
+
reviewer: null,
|
|
2811
|
+
progress: null,
|
|
2812
|
+
updatedAt: Date.now(),
|
|
2813
|
+
};
|
|
2814
|
+
writeLaneSnapshot(stateRoot, batchId, lane.laneNumber, spawnFailureSnapshot as unknown as Record<string, unknown>);
|
|
2815
|
+
} catch (snapErr) {
|
|
2816
|
+
// Best effort — if the snapshot write fails, the monitor's
|
|
2817
|
+
// 30s-staleness fallback (snap with old updatedAt) eventually
|
|
2818
|
+
// kicks in via the registry liveness check. Log so this is
|
|
2819
|
+
// visible in operator diagnostics, but do NOT throw.
|
|
2820
|
+
execLog(laneId, task.taskId, `spawn-failure snapshot write failed (non-fatal): ${snapErr instanceof Error ? snapErr.message : String(snapErr)}`);
|
|
2821
|
+
}
|
|
2822
|
+
|
|
2736
2823
|
shouldSkipRemaining = true;
|
|
2737
2824
|
}
|
|
2738
2825
|
}
|
|
@@ -7,7 +7,7 @@ import { join } from "path";
|
|
|
7
7
|
|
|
8
8
|
import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-reports.ts";
|
|
9
9
|
import { runDiscovery } from "./discovery.ts";
|
|
10
|
-
import { executeOrchBatch, resolveDisplayWaveNumber } from "./engine.ts";
|
|
10
|
+
import { executeOrchBatch, resolveDisplayWaveNumber, buildSpawnFailureAlertExtras } from "./engine.ts";
|
|
11
11
|
import { buildReviewerEnv, buildWorkerEnv, buildWorkerExcludeEnv, computeTransitiveDependents, execLog, executeLaneV2, executeWave, resolveCanonicalTaskPaths } from "./execution.ts";
|
|
12
12
|
import type { MonitorUpdateCallback, RuntimeBackend } from "./execution.ts";
|
|
13
13
|
import { selectRuntimeBackend } from "./engine.ts";
|
|
@@ -2160,11 +2160,17 @@ export async function resumeOrchBatch(
|
|
|
2160
2160
|
const frontierSummary = segmentFrontier
|
|
2161
2161
|
? ` Segment frontier: ${segmentFrontier.terminalSegments}/${segmentFrontier.totalSegments} terminal\n`
|
|
2162
2162
|
: "";
|
|
2163
|
+
// TP-190 (#561): Mirror engine.ts emission — propagate the structured
|
|
2164
|
+
// exit category so /orch-resume task-failure alerts route through the
|
|
2165
|
+
// same supervisor playbook branches as /orch. Shared helper enforces
|
|
2166
|
+
// payload parity between the two emission sites.
|
|
2167
|
+
const { exitCategory, summaryLine: spawnFailureLine } = buildSpawnFailureAlertExtras(outcome);
|
|
2163
2168
|
emitAlert({
|
|
2164
2169
|
category: "task-failure",
|
|
2165
2170
|
summary:
|
|
2166
2171
|
`⚠️ Task failure: ${taskId}\n` +
|
|
2167
2172
|
` Exit reason: ${exitReason}\n` +
|
|
2173
|
+
spawnFailureLine +
|
|
2168
2174
|
segmentSummary +
|
|
2169
2175
|
frontierSummary +
|
|
2170
2176
|
` Lane: ${laneForTask?.laneId ?? "unknown"} (lane ${laneForTask?.laneNumber ?? "?"})\n` +
|
|
@@ -2184,6 +2190,7 @@ export async function resumeOrchBatch(
|
|
|
2184
2190
|
laneNumber: laneForTask?.laneNumber,
|
|
2185
2191
|
waveIndex: waveIdx,
|
|
2186
2192
|
exitReason,
|
|
2193
|
+
exitCategory,
|
|
2187
2194
|
partialProgress: hasPartialProgress,
|
|
2188
2195
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
2189
2196
|
},
|
|
@@ -1810,8 +1810,13 @@ export type Tier0RecoveryPattern =
|
|
|
1810
1810
|
*
|
|
1811
1811
|
* These are transient failures where re-running the task has a reasonable
|
|
1812
1812
|
* chance of success. Classifications NOT in this set (e.g., user_killed,
|
|
1813
|
-
* stall_timeout, context_overflow) indicate persistent
|
|
1814
|
-
* won't be fixed by retrying.
|
|
1813
|
+
* stall_timeout, context_overflow, spawn_failure) indicate persistent
|
|
1814
|
+
* problems that won't be fixed by retrying.
|
|
1815
|
+
*
|
|
1816
|
+
* **TP-190 (#561):** `spawn_failure` is intentionally excluded — spawn-stage
|
|
1817
|
+
* errors (Pi CLI not findable, worktree provisioning failure, branch
|
|
1818
|
+
* collision) are never transient and require operator action. Retrying
|
|
1819
|
+
* silently would just burn the retry budget and delay the alert.
|
|
1815
1820
|
*
|
|
1816
1821
|
* @since TP-039
|
|
1817
1822
|
*/
|
|
@@ -2132,6 +2137,25 @@ export interface SupervisorAlertContext {
|
|
|
2132
2137
|
waveIndex?: number;
|
|
2133
2138
|
/** Exit reason string (for task-failure alerts) */
|
|
2134
2139
|
exitReason?: string;
|
|
2140
|
+
/**
|
|
2141
|
+
* Structured exit category for task-failure alerts.
|
|
2142
|
+
*
|
|
2143
|
+
* Mirrors `LaneTaskOutcome.exitDiagnostic.classification` for IPC
|
|
2144
|
+
* consumption by the supervisor. Optional for backward compatibility
|
|
2145
|
+
* — absent when the engine produces a task-failure alert without
|
|
2146
|
+
* structured diagnostic data.
|
|
2147
|
+
*
|
|
2148
|
+
* Notable values consumed by the supervisor playbook:
|
|
2149
|
+
* - `"spawn_failure"` (TP-190, #561): worker process never spawned
|
|
2150
|
+
* (Pi CLI not findable, worktree provisioning error, etc.). Never
|
|
2151
|
+
* transient — the playbook MUST escalate immediately rather than
|
|
2152
|
+
* retry. When the post-wave phase-transition logic detects an
|
|
2153
|
+
* all-spawn-failed wave it also flips `batchState.phase` to
|
|
2154
|
+
* `"failed"`; that transition is independent of this alert.
|
|
2155
|
+
*
|
|
2156
|
+
* @since TP-190 (#561)
|
|
2157
|
+
*/
|
|
2158
|
+
exitCategory?: ExitClassification;
|
|
2135
2159
|
/** Segment frontier snapshot for task-failure diagnosis */
|
|
2136
2160
|
segmentFrontier?: SupervisorSegmentFrontierSnapshot;
|
|
2137
2161
|
/** Agent ID (for agent-message alerts) */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "taskplane",
|
|
3
|
-
"version": "0.29.
|
|
3
|
+
"version": "0.29.2",
|
|
4
4
|
"description": "AI agent orchestration for pi — parallel task execution with checkpoint discipline",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
|
@@ -35,11 +35,22 @@
|
|
|
35
35
|
"templates/"
|
|
36
36
|
],
|
|
37
37
|
"peerDependencies": {
|
|
38
|
-
"@
|
|
39
|
-
"@
|
|
40
|
-
"@
|
|
38
|
+
"@earendil-works/pi-coding-agent": "*",
|
|
39
|
+
"@earendil-works/pi-tui": "*",
|
|
40
|
+
"@earendil-works/pi-ai": "*",
|
|
41
41
|
"@sinclair/typebox": "*"
|
|
42
42
|
},
|
|
43
|
+
"peerDependenciesMeta": {
|
|
44
|
+
"@earendil-works/pi-coding-agent": {
|
|
45
|
+
"optional": true
|
|
46
|
+
},
|
|
47
|
+
"@earendil-works/pi-tui": {
|
|
48
|
+
"optional": true
|
|
49
|
+
},
|
|
50
|
+
"@earendil-works/pi-ai": {
|
|
51
|
+
"optional": true
|
|
52
|
+
}
|
|
53
|
+
},
|
|
43
54
|
"dependencies": {
|
|
44
55
|
"jiti": "^2.6.1",
|
|
45
56
|
"yaml": "^2.4.0"
|