taskplane 0.29.2 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/gitignore-patterns.mjs +11 -8
- package/bin/rpc-wrapper.mjs +410 -357
- package/bin/taskplane.mjs +533 -250
- package/extensions/reviewer-extension.ts +17 -11
- package/extensions/taskplane/abort.ts +50 -18
- package/extensions/taskplane/agent-bridge-extension.ts +232 -105
- package/extensions/taskplane/agent-host.ts +224 -97
- package/extensions/taskplane/cleanup.ts +71 -42
- package/extensions/taskplane/config-loader.ts +142 -58
- package/extensions/taskplane/config-schema.ts +6 -13
- package/extensions/taskplane/config.ts +10 -2
- package/extensions/taskplane/diagnostic-reports.ts +59 -47
- package/extensions/taskplane/diagnostics.ts +13 -13
- package/extensions/taskplane/discovery.ts +35 -61
- package/extensions/taskplane/engine-worker.ts +53 -46
- package/extensions/taskplane/engine.ts +1760 -602
- package/extensions/taskplane/execution.ts +426 -206
- package/extensions/taskplane/extension.ts +1073 -598
- package/extensions/taskplane/formatting.ts +136 -124
- package/extensions/taskplane/git.ts +0 -2
- package/extensions/taskplane/lane-runner.ts +542 -311
- package/extensions/taskplane/mailbox.ts +57 -49
- package/extensions/taskplane/merge.ts +662 -383
- package/extensions/taskplane/messages.ts +109 -51
- package/extensions/taskplane/migrations.ts +1 -1
- package/extensions/taskplane/path-resolver.ts +8 -9
- package/extensions/taskplane/persistence.ts +425 -262
- package/extensions/taskplane/process-registry.ts +36 -7
- package/extensions/taskplane/quality-gate.ts +107 -55
- package/extensions/taskplane/resume.ts +774 -267
- package/extensions/taskplane/sessions.ts +1 -1
- package/extensions/taskplane/settings-tui.ts +505 -164
- package/extensions/taskplane/sidecar-telemetry.ts +25 -10
- package/extensions/taskplane/supervisor.ts +477 -270
- package/extensions/taskplane/task-executor-core.ts +178 -53
- package/extensions/taskplane/types.ts +186 -108
- package/extensions/taskplane/verification.ts +27 -22
- package/extensions/taskplane/waves.ts +59 -43
- package/extensions/taskplane/workspace.ts +14 -12
- package/extensions/taskplane/worktree.ts +218 -196
- package/package.json +14 -2
|
@@ -6,25 +6,130 @@ import { existsSync, readdirSync, readFileSync, renameSync, unlinkSync } from "f
|
|
|
6
6
|
import { join, resolve } from "path";
|
|
7
7
|
|
|
8
8
|
import { formatDiscoveryResults, runDiscovery } from "./discovery.ts";
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
buildReviewerEnv,
|
|
11
|
+
buildWorkerEnv,
|
|
12
|
+
buildWorkerExcludeEnv,
|
|
13
|
+
computeTransitiveDependents,
|
|
14
|
+
execLog,
|
|
15
|
+
executeLaneV2,
|
|
16
|
+
executeWave,
|
|
17
|
+
killV2LaneAgents,
|
|
18
|
+
resolveCanonicalTaskPaths,
|
|
19
|
+
} from "./execution.ts";
|
|
10
20
|
import type { RuntimeBackend } from "./execution.ts";
|
|
11
21
|
import type { MonitorUpdateCallback } from "./execution.ts";
|
|
12
22
|
// classifyExit no longer called directly — Tier 0 uses exitDiagnostic.classification
|
|
13
23
|
// from the diagnostic-reports pipeline (populated by assembleDiagnosticInput).
|
|
14
24
|
import { getCurrentBranch, runGit } from "./git.ts";
|
|
15
25
|
import { killAllMergeAgentsV2, mergeWaveByRepo, MergeHealthMonitor } from "./merge.ts";
|
|
16
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
applyMergeRetryLoop,
|
|
28
|
+
computeCleanupGatePolicy,
|
|
29
|
+
computeMergeFailurePolicy,
|
|
30
|
+
extractFailedRepoId,
|
|
31
|
+
formatRepoMergeSummary,
|
|
32
|
+
ORCH_MESSAGES,
|
|
33
|
+
} from "./messages.ts";
|
|
17
34
|
import type { CleanupGateRepoFailure } from "./messages.ts";
|
|
18
35
|
import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-reports.ts";
|
|
19
36
|
import { resolveOperatorId } from "./naming.ts";
|
|
20
|
-
import {
|
|
21
|
-
|
|
37
|
+
import {
|
|
38
|
+
applyPartialProgressToOutcomes,
|
|
39
|
+
buildTier0EventBase,
|
|
40
|
+
deleteBatchState,
|
|
41
|
+
emitEngineEvent,
|
|
42
|
+
emitTier0Event,
|
|
43
|
+
loadBatchHistory,
|
|
44
|
+
loadBatchState,
|
|
45
|
+
persistRuntimeState,
|
|
46
|
+
saveBatchHistory,
|
|
47
|
+
saveBatchMetaRuntimeArtifact,
|
|
48
|
+
seedPendingOutcomesForAllocatedLanes,
|
|
49
|
+
syncTaskOutcomesFromMonitor,
|
|
50
|
+
upsertTaskOutcome,
|
|
51
|
+
} from "./persistence.ts";
|
|
52
|
+
import {
|
|
53
|
+
readRegistrySnapshot,
|
|
54
|
+
isTerminalStatus,
|
|
55
|
+
isProcessAlive as registryIsProcessAlive,
|
|
56
|
+
} from "./process-registry.ts";
|
|
22
57
|
import { drainAgentOutbox } from "./mailbox.ts";
|
|
23
|
-
import {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
58
|
+
import {
|
|
59
|
+
buildBatchProgressSnapshot,
|
|
60
|
+
buildEngineEventBase,
|
|
61
|
+
buildSegmentId,
|
|
62
|
+
buildSupervisorSegmentFrontierSnapshot,
|
|
63
|
+
defaultResilienceState,
|
|
64
|
+
FATAL_DISCOVERY_CODES,
|
|
65
|
+
generateBatchId,
|
|
66
|
+
TIER0_RETRYABLE_CLASSIFICATIONS,
|
|
67
|
+
TIER0_RETRY_BUDGETS,
|
|
68
|
+
tier0ScopeKey,
|
|
69
|
+
tier0WaveScopeKey,
|
|
70
|
+
} from "./types.ts";
|
|
71
|
+
import type {
|
|
72
|
+
AllocatedLane,
|
|
73
|
+
AllocatedTask,
|
|
74
|
+
BatchHistorySummary,
|
|
75
|
+
BatchTaskSummary,
|
|
76
|
+
BatchWaveSummary,
|
|
77
|
+
DiscoveryResult,
|
|
78
|
+
EngineEventCallback,
|
|
79
|
+
EscalationContext,
|
|
80
|
+
LaneExecutionResult,
|
|
81
|
+
LaneTaskOutcome,
|
|
82
|
+
MergeWaveResult,
|
|
83
|
+
OrchBatchPhase,
|
|
84
|
+
OrchBatchRuntimeState,
|
|
85
|
+
OrchestratorConfig,
|
|
86
|
+
ParsedTask,
|
|
87
|
+
PersistedSegmentRecord,
|
|
88
|
+
SegmentExpansionRequest,
|
|
89
|
+
SupervisorAlert,
|
|
90
|
+
SupervisorAlertCallback,
|
|
91
|
+
TaskRunnerConfig,
|
|
92
|
+
TaskSegmentPlan,
|
|
93
|
+
TaskSegmentPlanMap,
|
|
94
|
+
TaskSegmentNode,
|
|
95
|
+
Tier0EscalationPattern,
|
|
96
|
+
Tier0RecoveryPattern,
|
|
97
|
+
TokenCounts,
|
|
98
|
+
WaveExecutionResult,
|
|
99
|
+
WorkspaceConfig,
|
|
100
|
+
} from "./types.ts";
|
|
101
|
+
import {
|
|
102
|
+
buildDependencyGraph,
|
|
103
|
+
computeWaveAssignments,
|
|
104
|
+
resolveBaseBranch,
|
|
105
|
+
resolveRepoRoot,
|
|
106
|
+
validateGraph,
|
|
107
|
+
} from "./waves.ts";
|
|
108
|
+
import {
|
|
109
|
+
deleteBranchBestEffort,
|
|
110
|
+
forceCleanupWorktree,
|
|
111
|
+
formatPreflightResults,
|
|
112
|
+
listWorktrees,
|
|
113
|
+
preserveFailedLaneProgress,
|
|
114
|
+
preserveSkippedLaneProgress,
|
|
115
|
+
removeAllWorktrees,
|
|
116
|
+
removeWorktree,
|
|
117
|
+
runPreflight,
|
|
118
|
+
safeResetWorktree,
|
|
119
|
+
sleepSync,
|
|
120
|
+
} from "./worktree.ts";
|
|
121
|
+
import {
|
|
122
|
+
runPreflightCleanup,
|
|
123
|
+
formatPreflightCleanup,
|
|
124
|
+
sweepStaleArtifacts,
|
|
125
|
+
formatPreflightSweep,
|
|
126
|
+
rotateSupervisorLogs,
|
|
127
|
+
formatLogRotation,
|
|
128
|
+
enforceTelemetrySizeCap,
|
|
129
|
+
formatSizeCap,
|
|
130
|
+
cleanupPriorBatchArtifacts,
|
|
131
|
+
formatPriorBatchCleanup,
|
|
132
|
+
} from "./cleanup.ts";
|
|
28
133
|
|
|
29
134
|
// ── Tier 0: Automatic Recovery Helpers (TP-039) ─────────────────────
|
|
30
135
|
|
|
@@ -48,7 +153,12 @@ function emitTier0Escalation(
|
|
|
48
153
|
lastError: string,
|
|
49
154
|
affectedTasks: string[],
|
|
50
155
|
suggestion: string,
|
|
51
|
-
extra?: Partial<
|
|
156
|
+
extra?: Partial<
|
|
157
|
+
Pick<
|
|
158
|
+
import("./persistence.ts").Tier0Event,
|
|
159
|
+
"taskId" | "laneNumber" | "repoId" | "classification" | "scopeKey"
|
|
160
|
+
>
|
|
161
|
+
>,
|
|
52
162
|
): void {
|
|
53
163
|
const escalation: EscalationContext = {
|
|
54
164
|
pattern,
|
|
@@ -167,12 +277,16 @@ export function isAllLanesSpawnFailedWave(
|
|
|
167
277
|
*/
|
|
168
278
|
export function buildSpawnFailureAlertExtras(
|
|
169
279
|
outcome: { exitDiagnostic?: { classification?: string } | undefined } | undefined,
|
|
170
|
-
): {
|
|
280
|
+
): {
|
|
281
|
+
exitCategory: import("./diagnostics.ts").ExitClassification | undefined;
|
|
282
|
+
summaryLine: string;
|
|
283
|
+
} {
|
|
171
284
|
const raw = outcome?.exitDiagnostic?.classification;
|
|
172
285
|
const exitCategory = raw as import("./diagnostics.ts").ExitClassification | undefined;
|
|
173
|
-
const summaryLine =
|
|
174
|
-
|
|
175
|
-
|
|
286
|
+
const summaryLine =
|
|
287
|
+
raw === "spawn_failure"
|
|
288
|
+
? ` Spawn failure: worker process never started — escalate immediately (do not retry)\n`
|
|
289
|
+
: "";
|
|
176
290
|
return { exitCategory, summaryLine };
|
|
177
291
|
}
|
|
178
292
|
|
|
@@ -216,8 +330,9 @@ export function resolveBatchHistoryTaskTokens(
|
|
|
216
330
|
if (v2) return v2;
|
|
217
331
|
}
|
|
218
332
|
|
|
219
|
-
const bySession =
|
|
220
|
-
|
|
333
|
+
const bySession =
|
|
334
|
+
legacyLaneTokensByKey.get(outcome.sessionName) ||
|
|
335
|
+
legacyLaneTokensByKey.get(outcome.sessionName?.replace(/-(?:worker|reviewer)$/, ""));
|
|
221
336
|
if (bySession) return bySession;
|
|
222
337
|
|
|
223
338
|
if (laneNumber > 0) {
|
|
@@ -251,7 +366,10 @@ function buildSegmentDependencyMap(plan: TaskSegmentPlan): Map<string, string[]>
|
|
|
251
366
|
depsBySegmentId.get(edge.toSegmentId)!.push(edge.fromSegmentId);
|
|
252
367
|
}
|
|
253
368
|
for (const [segmentId, deps] of depsBySegmentId.entries()) {
|
|
254
|
-
depsBySegmentId.set(
|
|
369
|
+
depsBySegmentId.set(
|
|
370
|
+
segmentId,
|
|
371
|
+
[...new Set(deps)].sort((a, b) => a.localeCompare(b)),
|
|
372
|
+
);
|
|
255
373
|
}
|
|
256
374
|
return depsBySegmentId;
|
|
257
375
|
}
|
|
@@ -283,7 +401,11 @@ export function resolveTaskWorkerAgentId(
|
|
|
283
401
|
return `${lane.laneSessionId}-worker`;
|
|
284
402
|
}
|
|
285
403
|
|
|
286
|
-
function listPendingSegmentExpansionRequestFiles(
|
|
404
|
+
function listPendingSegmentExpansionRequestFiles(
|
|
405
|
+
stateRoot: string,
|
|
406
|
+
batchId: string,
|
|
407
|
+
agentId: string,
|
|
408
|
+
): string[] {
|
|
287
409
|
const outboxDir = join(stateRoot, ".pi", "mailbox", batchId, agentId, "outbox");
|
|
288
410
|
if (!existsSync(outboxDir)) return [];
|
|
289
411
|
let entries: string[] = [];
|
|
@@ -314,7 +436,12 @@ function parseSegmentExpansionRequestPayload(payload: unknown): SegmentExpansion
|
|
|
314
436
|
if (typeof candidate.requestId !== "string" || !candidate.requestId.trim()) return null;
|
|
315
437
|
if (typeof candidate.taskId !== "string" || !candidate.taskId.trim()) return null;
|
|
316
438
|
if (typeof candidate.fromSegmentId !== "string" || !candidate.fromSegmentId.trim()) return null;
|
|
317
|
-
if (
|
|
439
|
+
if (
|
|
440
|
+
!Array.isArray(candidate.requestedRepoIds) ||
|
|
441
|
+
candidate.requestedRepoIds.length === 0 ||
|
|
442
|
+
candidate.requestedRepoIds.some((repoId) => typeof repoId !== "string" || !repoId.trim())
|
|
443
|
+
)
|
|
444
|
+
return null;
|
|
318
445
|
if (typeof candidate.rationale !== "string") return null;
|
|
319
446
|
if (candidate.placement !== "after-current" && candidate.placement !== "end") return null;
|
|
320
447
|
if (!Array.isArray(candidate.edges)) return null;
|
|
@@ -385,7 +512,10 @@ function parseSegmentExpansionRequests(filePaths: string[]): {
|
|
|
385
512
|
return { valid, malformed };
|
|
386
513
|
}
|
|
387
514
|
|
|
388
|
-
function markSegmentExpansionRequestFile(
|
|
515
|
+
function markSegmentExpansionRequestFile(
|
|
516
|
+
filePath: string,
|
|
517
|
+
stateSuffix: "invalid" | "discarded" | "rejected" | "processed",
|
|
518
|
+
): boolean {
|
|
389
519
|
try {
|
|
390
520
|
renameSync(filePath, `${filePath}.${stateSuffix}`);
|
|
391
521
|
return true;
|
|
@@ -510,7 +640,13 @@ export function processSegmentExpansionRequestAtBoundary(
|
|
|
510
640
|
segmentState: SegmentFrontierTaskState,
|
|
511
641
|
workspaceConfig: WorkspaceConfig | null | undefined,
|
|
512
642
|
knownRequestIds: Set<string>,
|
|
513
|
-
|
|
643
|
+
// TP-195: `reason?: undefined` on the success branch makes this a
|
|
644
|
+
// well-formed discriminated union under `strict: false`. Without it,
|
|
645
|
+
// `if (!result.ok)` does not narrow `reason` because non-strict
|
|
646
|
+
// narrowing requires every member of the union to share the
|
|
647
|
+
// discriminating field. Runtime semantics are unchanged — the
|
|
648
|
+
// success branch never carries a reason.
|
|
649
|
+
): { ok: true; reason?: undefined } | { ok: false; reason: string } {
|
|
514
650
|
const validationFailure = validateSegmentExpansionRequestAtBoundary(
|
|
515
651
|
requestFile,
|
|
516
652
|
taskId,
|
|
@@ -536,7 +672,9 @@ export function processSegmentExpansionRequestAtBoundary(
|
|
|
536
672
|
return { ok: true };
|
|
537
673
|
}
|
|
538
674
|
|
|
539
|
-
function buildOutgoingBySegmentId(
|
|
675
|
+
function buildOutgoingBySegmentId(
|
|
676
|
+
dependsOnBySegmentId: Map<string, string[]>,
|
|
677
|
+
): Map<string, string[]> {
|
|
540
678
|
const outgoingBySegmentId = new Map<string, string[]>();
|
|
541
679
|
for (const segmentId of dependsOnBySegmentId.keys()) {
|
|
542
680
|
outgoingBySegmentId.set(segmentId, []);
|
|
@@ -549,12 +687,19 @@ function buildOutgoingBySegmentId(dependsOnBySegmentId: Map<string, string[]>):
|
|
|
549
687
|
}
|
|
550
688
|
}
|
|
551
689
|
for (const [segmentId, outgoing] of outgoingBySegmentId.entries()) {
|
|
552
|
-
outgoingBySegmentId.set(
|
|
690
|
+
outgoingBySegmentId.set(
|
|
691
|
+
segmentId,
|
|
692
|
+
[...new Set(outgoing)].sort((a, b) => a.localeCompare(b)),
|
|
693
|
+
);
|
|
553
694
|
}
|
|
554
695
|
return outgoingBySegmentId;
|
|
555
696
|
}
|
|
556
697
|
|
|
557
|
-
function addDependency(
|
|
698
|
+
function addDependency(
|
|
699
|
+
dependencyMap: Map<string, string[]>,
|
|
700
|
+
segmentId: string,
|
|
701
|
+
depSegmentId: string,
|
|
702
|
+
): void {
|
|
558
703
|
const deps = dependencyMap.get(segmentId) ?? [];
|
|
559
704
|
if (!deps.includes(depSegmentId)) {
|
|
560
705
|
deps.push(depSegmentId);
|
|
@@ -563,7 +708,11 @@ function addDependency(dependencyMap: Map<string, string[]>, segmentId: string,
|
|
|
563
708
|
}
|
|
564
709
|
}
|
|
565
710
|
|
|
566
|
-
function removeDependency(
|
|
711
|
+
function removeDependency(
|
|
712
|
+
dependencyMap: Map<string, string[]>,
|
|
713
|
+
segmentId: string,
|
|
714
|
+
depSegmentId: string,
|
|
715
|
+
): void {
|
|
567
716
|
const deps = dependencyMap.get(segmentId) ?? [];
|
|
568
717
|
const filtered = deps.filter((dep) => dep !== depSegmentId);
|
|
569
718
|
dependencyMap.set(segmentId, filtered);
|
|
@@ -573,12 +722,15 @@ function recomputeNextPendingSegmentIndex(segmentState: SegmentFrontierTaskState
|
|
|
573
722
|
const nextPendingIndex = segmentState.orderedSegments.findIndex((segment) => {
|
|
574
723
|
return segmentState.statusBySegmentId.get(segment.segmentId) === "pending";
|
|
575
724
|
});
|
|
576
|
-
segmentState.nextSegmentIndex =
|
|
577
|
-
? nextPendingIndex
|
|
578
|
-
: segmentState.orderedSegments.length;
|
|
725
|
+
segmentState.nextSegmentIndex =
|
|
726
|
+
nextPendingIndex >= 0 ? nextPendingIndex : segmentState.orderedSegments.length;
|
|
579
727
|
}
|
|
580
728
|
|
|
581
|
-
function hasTaskInFutureSegmentRounds(
|
|
729
|
+
function hasTaskInFutureSegmentRounds(
|
|
730
|
+
segmentRounds: string[][],
|
|
731
|
+
fromIndex: number,
|
|
732
|
+
taskId: string,
|
|
733
|
+
): boolean {
|
|
582
734
|
for (let idx = fromIndex; idx < segmentRounds.length; idx++) {
|
|
583
735
|
if (segmentRounds[idx]?.includes(taskId)) {
|
|
584
736
|
return true;
|
|
@@ -644,7 +796,10 @@ export function applySegmentExpansionMutation(
|
|
|
644
796
|
|
|
645
797
|
const dependencyMap = new Map<string, string[]>();
|
|
646
798
|
for (const [segmentId, deps] of segmentState.dependsOnBySegmentId.entries()) {
|
|
647
|
-
dependencyMap.set(
|
|
799
|
+
dependencyMap.set(
|
|
800
|
+
segmentId,
|
|
801
|
+
[...new Set(deps)].sort((a, b) => a.localeCompare(b)),
|
|
802
|
+
);
|
|
648
803
|
}
|
|
649
804
|
for (const segmentId of existingNodeById.keys()) {
|
|
650
805
|
if (!dependencyMap.has(segmentId)) {
|
|
@@ -659,8 +814,14 @@ export function applySegmentExpansionMutation(
|
|
|
659
814
|
|
|
660
815
|
const outgoingBeforeMutation = buildOutgoingBySegmentId(dependencyMap);
|
|
661
816
|
const anchorSuccessors = outgoingBeforeMutation.get(anchorSegmentId) ?? [];
|
|
662
|
-
const maxOrder = segmentState.orderedSegments.reduce(
|
|
663
|
-
|
|
817
|
+
const maxOrder = segmentState.orderedSegments.reduce(
|
|
818
|
+
(max, segment) => Math.max(max, segment.order),
|
|
819
|
+
-1,
|
|
820
|
+
);
|
|
821
|
+
const repoMaxSequenceByRepo = buildRepoMaxSequenceByRepo(
|
|
822
|
+
segmentState.orderedSegments,
|
|
823
|
+
request.taskId,
|
|
824
|
+
);
|
|
664
825
|
|
|
665
826
|
const newNodes: TaskSegmentNode[] = [];
|
|
666
827
|
const segmentIdByRequestedRepoId = new Map<string, string>();
|
|
@@ -777,10 +938,15 @@ export function applySegmentExpansionMutation(
|
|
|
777
938
|
if (nextOrderedSegmentIds.length !== dependencyMap.size) {
|
|
778
939
|
// Topological sort failed to cover all nodes — likely a cycle introduced
|
|
779
940
|
// by the expansion. Reject the mutation entirely and restore original state.
|
|
780
|
-
execLog(
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
941
|
+
execLog(
|
|
942
|
+
"batch",
|
|
943
|
+
request.taskId,
|
|
944
|
+
"segment expansion rejected: topological sort failed (possible cycle)",
|
|
945
|
+
{
|
|
946
|
+
expected: dependencyMap.size,
|
|
947
|
+
covered: nextOrderedSegmentIds.length,
|
|
948
|
+
},
|
|
949
|
+
);
|
|
784
950
|
// Full rollback to pre-mutation state
|
|
785
951
|
for (const node of newNodes) {
|
|
786
952
|
segmentState.statusBySegmentId.delete(node.segmentId);
|
|
@@ -865,7 +1031,9 @@ export function upsertPendingExpandedSegmentRecords(
|
|
|
865
1031
|
let changed = false;
|
|
866
1032
|
|
|
867
1033
|
for (const segmentId of pendingSegmentIds) {
|
|
868
|
-
const segment = segmentState.orderedSegments.find(
|
|
1034
|
+
const segment = segmentState.orderedSegments.find(
|
|
1035
|
+
(candidate) => candidate.segmentId === segmentId,
|
|
1036
|
+
);
|
|
869
1037
|
if (!segment) continue;
|
|
870
1038
|
const existing = segmentRecords.find((record) => record.segmentId === segmentId);
|
|
871
1039
|
if (!existing && !insertedSegmentIdSet.has(segmentId)) {
|
|
@@ -904,21 +1072,23 @@ export function upsertPendingExpandedSegmentRecords(
|
|
|
904
1072
|
}
|
|
905
1073
|
|
|
906
1074
|
const recordChanged =
|
|
907
|
-
existing.taskId !== next.taskId
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
||
|
|
1075
|
+
existing.taskId !== next.taskId ||
|
|
1076
|
+
existing.repoId !== next.repoId ||
|
|
1077
|
+
existing.status !== next.status ||
|
|
1078
|
+
existing.laneId !== next.laneId ||
|
|
1079
|
+
existing.sessionName !== next.sessionName ||
|
|
1080
|
+
existing.worktreePath !== next.worktreePath ||
|
|
1081
|
+
existing.branch !== next.branch ||
|
|
1082
|
+
existing.startedAt !== next.startedAt ||
|
|
1083
|
+
existing.endedAt !== next.endedAt ||
|
|
1084
|
+
existing.retries !== next.retries ||
|
|
1085
|
+
existing.exitReason !== next.exitReason ||
|
|
1086
|
+
existing.dependsOnSegmentIds.length !== next.dependsOnSegmentIds.length ||
|
|
1087
|
+
existing.dependsOnSegmentIds.some(
|
|
1088
|
+
(depSegmentId, idx) => depSegmentId !== next.dependsOnSegmentIds[idx],
|
|
1089
|
+
) ||
|
|
1090
|
+
existing.expandedFrom !== next.expandedFrom ||
|
|
1091
|
+
existing.expansionRequestId !== next.expansionRequestId;
|
|
922
1092
|
|
|
923
1093
|
if (recordChanged) {
|
|
924
1094
|
Object.assign(existing, next);
|
|
@@ -952,7 +1122,9 @@ function recordProcessedSegmentExpansionRequestId(
|
|
|
952
1122
|
batchState.resilience = defaultResilienceState();
|
|
953
1123
|
}
|
|
954
1124
|
const history = batchState.resilience.repairHistory;
|
|
955
|
-
if (
|
|
1125
|
+
if (
|
|
1126
|
+
history.some((entry) => entry.strategy === "segment-expansion-request" && entry.id === requestId)
|
|
1127
|
+
) {
|
|
956
1128
|
return false;
|
|
957
1129
|
}
|
|
958
1130
|
const now = Date.now();
|
|
@@ -975,7 +1147,9 @@ function upsertRunningSegmentRecord(
|
|
|
975
1147
|
const activeSegmentId = task.activeSegmentId;
|
|
976
1148
|
if (!activeSegmentId) return false;
|
|
977
1149
|
|
|
978
|
-
const activeSegment = segmentState.orderedSegments.find(
|
|
1150
|
+
const activeSegment = segmentState.orderedSegments.find(
|
|
1151
|
+
(segment) => segment.segmentId === activeSegmentId,
|
|
1152
|
+
);
|
|
979
1153
|
if (!activeSegment) return false;
|
|
980
1154
|
|
|
981
1155
|
const segmentRecords = ensureSegmentRecords(batchState);
|
|
@@ -983,9 +1157,7 @@ function upsertRunningSegmentRecord(
|
|
|
983
1157
|
const existing = segmentRecords.find((record) => record.segmentId === activeSegmentId);
|
|
984
1158
|
const now = Date.now();
|
|
985
1159
|
|
|
986
|
-
const restarted = !!existing
|
|
987
|
-
&& existing.status !== "running"
|
|
988
|
-
&& existing.startedAt !== null;
|
|
1160
|
+
const restarted = !!existing && existing.status !== "running" && existing.startedAt !== null;
|
|
989
1161
|
|
|
990
1162
|
const next: PersistedSegmentRecord = {
|
|
991
1163
|
segmentId: activeSegmentId,
|
|
@@ -996,20 +1168,12 @@ function upsertRunningSegmentRecord(
|
|
|
996
1168
|
sessionName: lane.laneSessionId,
|
|
997
1169
|
worktreePath: lane.worktreePath,
|
|
998
1170
|
branch: lane.branch,
|
|
999
|
-
startedAt: existing?.status === "running"
|
|
1000
|
-
? existing.startedAt
|
|
1001
|
-
: (existing?.startedAt ?? now),
|
|
1171
|
+
startedAt: existing?.status === "running" ? existing.startedAt : (existing?.startedAt ?? now),
|
|
1002
1172
|
endedAt: null,
|
|
1003
|
-
retries: existing
|
|
1004
|
-
|
|
1005
|
-
: 0,
|
|
1006
|
-
exitReason: existing?.status === "running"
|
|
1007
|
-
? existing.exitReason
|
|
1008
|
-
: "Segment running",
|
|
1173
|
+
retries: existing ? existing.retries + (restarted ? 1 : 0) : 0,
|
|
1174
|
+
exitReason: existing?.status === "running" ? existing.exitReason : "Segment running",
|
|
1009
1175
|
dependsOnSegmentIds,
|
|
1010
|
-
exitDiagnostic: existing?.status === "running"
|
|
1011
|
-
? existing.exitDiagnostic
|
|
1012
|
-
: undefined,
|
|
1176
|
+
exitDiagnostic: existing?.status === "running" ? existing.exitDiagnostic : undefined,
|
|
1013
1177
|
expandedFrom: existing?.expandedFrom,
|
|
1014
1178
|
expansionRequestId: existing?.expansionRequestId,
|
|
1015
1179
|
};
|
|
@@ -1020,22 +1184,24 @@ function upsertRunningSegmentRecord(
|
|
|
1020
1184
|
}
|
|
1021
1185
|
|
|
1022
1186
|
const changed =
|
|
1023
|
-
existing.taskId !== next.taskId
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
||
|
|
1038
|
-
|
|
1187
|
+
existing.taskId !== next.taskId ||
|
|
1188
|
+
existing.repoId !== next.repoId ||
|
|
1189
|
+
existing.status !== next.status ||
|
|
1190
|
+
existing.laneId !== next.laneId ||
|
|
1191
|
+
existing.sessionName !== next.sessionName ||
|
|
1192
|
+
existing.worktreePath !== next.worktreePath ||
|
|
1193
|
+
existing.branch !== next.branch ||
|
|
1194
|
+
existing.startedAt !== next.startedAt ||
|
|
1195
|
+
existing.endedAt !== next.endedAt ||
|
|
1196
|
+
existing.retries !== next.retries ||
|
|
1197
|
+
existing.exitReason !== next.exitReason ||
|
|
1198
|
+
existing.dependsOnSegmentIds.length !== next.dependsOnSegmentIds.length ||
|
|
1199
|
+
existing.dependsOnSegmentIds.some(
|
|
1200
|
+
(segmentId, idx) => segmentId !== next.dependsOnSegmentIds[idx],
|
|
1201
|
+
) ||
|
|
1202
|
+
existing.exitDiagnostic !== next.exitDiagnostic ||
|
|
1203
|
+
existing.expandedFrom !== next.expandedFrom ||
|
|
1204
|
+
existing.expansionRequestId !== next.expansionRequestId;
|
|
1039
1205
|
|
|
1040
1206
|
if (changed) {
|
|
1041
1207
|
Object.assign(existing, next);
|
|
@@ -1052,16 +1218,17 @@ function upsertTerminalSegmentRecord(
|
|
|
1052
1218
|
outcome: LaneTaskOutcome | undefined,
|
|
1053
1219
|
lane: AllocatedLane | undefined,
|
|
1054
1220
|
): boolean {
|
|
1055
|
-
const segment = segmentState.orderedSegments.find(
|
|
1221
|
+
const segment = segmentState.orderedSegments.find(
|
|
1222
|
+
(candidate) => candidate.segmentId === segmentId,
|
|
1223
|
+
);
|
|
1056
1224
|
if (!segment) return false;
|
|
1057
1225
|
|
|
1058
1226
|
const segmentRecords = ensureSegmentRecords(batchState);
|
|
1059
1227
|
const existing = segmentRecords.find((record) => record.segmentId === segmentId);
|
|
1060
1228
|
const now = Date.now();
|
|
1061
1229
|
const dependsOnSegmentIds = segmentState.dependsOnBySegmentId.get(segmentId) ?? [];
|
|
1062
|
-
const nextExitDiagnostic =
|
|
1063
|
-
? (outcome?.exitDiagnostic ?? existing?.exitDiagnostic)
|
|
1064
|
-
: undefined;
|
|
1230
|
+
const nextExitDiagnostic =
|
|
1231
|
+
status === "failed" ? (outcome?.exitDiagnostic ?? existing?.exitDiagnostic) : undefined;
|
|
1065
1232
|
|
|
1066
1233
|
const next: PersistedSegmentRecord = {
|
|
1067
1234
|
segmentId,
|
|
@@ -1075,11 +1242,13 @@ function upsertTerminalSegmentRecord(
|
|
|
1075
1242
|
startedAt: existing?.startedAt ?? outcome?.startTime ?? now,
|
|
1076
1243
|
endedAt: outcome?.endTime ?? now,
|
|
1077
1244
|
retries: existing?.retries ?? 0,
|
|
1078
|
-
exitReason:
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1245
|
+
exitReason:
|
|
1246
|
+
outcome?.exitReason ??
|
|
1247
|
+
(status === "succeeded"
|
|
1248
|
+
? "Segment completed"
|
|
1249
|
+
: status === "failed"
|
|
1250
|
+
? "Segment failed"
|
|
1251
|
+
: "Segment skipped"),
|
|
1083
1252
|
dependsOnSegmentIds,
|
|
1084
1253
|
exitDiagnostic: nextExitDiagnostic,
|
|
1085
1254
|
expandedFrom: existing?.expandedFrom,
|
|
@@ -1092,22 +1261,24 @@ function upsertTerminalSegmentRecord(
|
|
|
1092
1261
|
}
|
|
1093
1262
|
|
|
1094
1263
|
const changed =
|
|
1095
|
-
existing.taskId !== next.taskId
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
||
|
|
1110
|
-
|
|
1264
|
+
existing.taskId !== next.taskId ||
|
|
1265
|
+
existing.repoId !== next.repoId ||
|
|
1266
|
+
existing.status !== next.status ||
|
|
1267
|
+
existing.laneId !== next.laneId ||
|
|
1268
|
+
existing.sessionName !== next.sessionName ||
|
|
1269
|
+
existing.worktreePath !== next.worktreePath ||
|
|
1270
|
+
existing.branch !== next.branch ||
|
|
1271
|
+
existing.startedAt !== next.startedAt ||
|
|
1272
|
+
existing.endedAt !== next.endedAt ||
|
|
1273
|
+
existing.retries !== next.retries ||
|
|
1274
|
+
existing.exitReason !== next.exitReason ||
|
|
1275
|
+
existing.dependsOnSegmentIds.length !== next.dependsOnSegmentIds.length ||
|
|
1276
|
+
existing.dependsOnSegmentIds.some(
|
|
1277
|
+
(depSegmentId, idx) => depSegmentId !== next.dependsOnSegmentIds[idx],
|
|
1278
|
+
) ||
|
|
1279
|
+
existing.exitDiagnostic !== next.exitDiagnostic ||
|
|
1280
|
+
existing.expandedFrom !== next.expandedFrom ||
|
|
1281
|
+
existing.expansionRequestId !== next.expansionRequestId;
|
|
1111
1282
|
|
|
1112
1283
|
if (changed) {
|
|
1113
1284
|
Object.assign(existing, next);
|
|
@@ -1165,7 +1336,7 @@ export function linearizeTaskSegmentPlan(plan: TaskSegmentPlan): TaskSegmentNode
|
|
|
1165
1336
|
|
|
1166
1337
|
const ready: TaskSegmentNode[] = plan.segments
|
|
1167
1338
|
.filter((segment) => (indegree.get(segment.segmentId) ?? 0) === 0)
|
|
1168
|
-
.sort((a, b) =>
|
|
1339
|
+
.sort((a, b) => a.order - b.order || a.segmentId.localeCompare(b.segmentId));
|
|
1169
1340
|
|
|
1170
1341
|
const ordered: TaskSegmentNode[] = [];
|
|
1171
1342
|
while (ready.length > 0) {
|
|
@@ -1178,7 +1349,7 @@ export function linearizeTaskSegmentPlan(plan: TaskSegmentPlan): TaskSegmentNode
|
|
|
1178
1349
|
const depNode = nodeById.get(dep);
|
|
1179
1350
|
if (depNode) {
|
|
1180
1351
|
ready.push(depNode);
|
|
1181
|
-
ready.sort((a, b) =>
|
|
1352
|
+
ready.sort((a, b) => a.order - b.order || a.segmentId.localeCompare(b.segmentId));
|
|
1182
1353
|
}
|
|
1183
1354
|
}
|
|
1184
1355
|
}
|
|
@@ -1186,7 +1357,9 @@ export function linearizeTaskSegmentPlan(plan: TaskSegmentPlan): TaskSegmentNode
|
|
|
1186
1357
|
|
|
1187
1358
|
// Defensive fallback: malformed/cyclic plans retain deterministic segment order.
|
|
1188
1359
|
if (ordered.length !== plan.segments.length) {
|
|
1189
|
-
return [...plan.segments].sort(
|
|
1360
|
+
return [...plan.segments].sort(
|
|
1361
|
+
(a, b) => a.order - b.order || a.segmentId.localeCompare(b.segmentId),
|
|
1362
|
+
);
|
|
1190
1363
|
}
|
|
1191
1364
|
|
|
1192
1365
|
return ordered;
|
|
@@ -1236,8 +1409,8 @@ export function resolveDisplayWaveNumber(
|
|
|
1236
1409
|
fallbackTotal?: number,
|
|
1237
1410
|
): { displayWave: number; displayTotal: number } {
|
|
1238
1411
|
const taskWaveIdx = roundToTaskWave?.[roundIdx];
|
|
1239
|
-
const displayWave =
|
|
1240
|
-
const displayTotal = taskLevelWaveCount ?? fallbackTotal ??
|
|
1412
|
+
const displayWave = taskWaveIdx != null ? taskWaveIdx + 1 : roundIdx + 1;
|
|
1413
|
+
const displayTotal = taskLevelWaveCount ?? fallbackTotal ?? roundIdx + 1;
|
|
1241
1414
|
return { displayWave, displayTotal };
|
|
1242
1415
|
}
|
|
1243
1416
|
|
|
@@ -1272,16 +1445,16 @@ export function buildSegmentFrontierWaves(
|
|
|
1272
1445
|
// Resolve packetTaskPath to absolute so it works from any repo's worktree.
|
|
1273
1446
|
// task.taskFolder is relative to workspace root (e.g., "shared-libs/task-management/.../TP-004").
|
|
1274
1447
|
// When a segment executes in a different repo, the lane worktree won't contain this path.
|
|
1275
|
-
task.packetTaskPath = workspaceRoot
|
|
1276
|
-
? resolve(workspaceRoot, task.taskFolder)
|
|
1277
|
-
: task.taskFolder;
|
|
1448
|
+
task.packetTaskPath = workspaceRoot ? resolve(workspaceRoot, task.taskFolder) : task.taskFolder;
|
|
1278
1449
|
}
|
|
1279
1450
|
|
|
1280
1451
|
taskStateById.set(taskId, {
|
|
1281
1452
|
taskId,
|
|
1282
1453
|
orderedSegments,
|
|
1283
1454
|
nextSegmentIndex: 0,
|
|
1284
|
-
statusBySegmentId: new Map(
|
|
1455
|
+
statusBySegmentId: new Map(
|
|
1456
|
+
orderedSegments.map((segment) => [segment.segmentId, "pending" as SegmentLifecycleStatus]),
|
|
1457
|
+
),
|
|
1285
1458
|
dependsOnBySegmentId,
|
|
1286
1459
|
terminalStatus: "pending",
|
|
1287
1460
|
});
|
|
@@ -1373,7 +1546,7 @@ async function attemptWorkerCrashRetry(
|
|
|
1373
1546
|
if (!lane) continue;
|
|
1374
1547
|
|
|
1375
1548
|
// Find the task outcome to get exit info
|
|
1376
|
-
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
1549
|
+
const outcome = allTaskOutcomes.find((o) => o.taskId === taskId);
|
|
1377
1550
|
if (!outcome) continue;
|
|
1378
1551
|
|
|
1379
1552
|
// Use the canonical exit diagnostic classification when available.
|
|
@@ -1384,7 +1557,9 @@ async function attemptWorkerCrashRetry(
|
|
|
1384
1557
|
const classification = outcome.exitDiagnostic?.classification;
|
|
1385
1558
|
|
|
1386
1559
|
if (!classification) {
|
|
1387
|
-
execLog(
|
|
1560
|
+
execLog(
|
|
1561
|
+
"batch",
|
|
1562
|
+
batchState.batchId,
|
|
1388
1563
|
`tier0: task ${taskId} has no exit diagnostic classification — skipping auto-retry (conservative)`,
|
|
1389
1564
|
);
|
|
1390
1565
|
continue;
|
|
@@ -1398,7 +1573,9 @@ async function attemptWorkerCrashRetry(
|
|
|
1398
1573
|
// (spawn_failure is not in the set), but the explicit early-return
|
|
1399
1574
|
// here gives operators a clearer log message at the gate site.
|
|
1400
1575
|
if (classification === "spawn_failure") {
|
|
1401
|
-
execLog(
|
|
1576
|
+
execLog(
|
|
1577
|
+
"batch",
|
|
1578
|
+
batchState.batchId,
|
|
1402
1579
|
`tier0: task ${taskId} spawn_failure — operator action required, NOT auto-retrying (TP-190)`,
|
|
1403
1580
|
);
|
|
1404
1581
|
continue;
|
|
@@ -1406,7 +1583,9 @@ async function attemptWorkerCrashRetry(
|
|
|
1406
1583
|
|
|
1407
1584
|
// Check if retryable
|
|
1408
1585
|
if (!TIER0_RETRYABLE_CLASSIFICATIONS.has(classification)) {
|
|
1409
|
-
execLog(
|
|
1586
|
+
execLog(
|
|
1587
|
+
"batch",
|
|
1588
|
+
batchState.batchId,
|
|
1410
1589
|
`tier0: task ${taskId} exit classification "${classification}" is not retryable — skipping`,
|
|
1411
1590
|
);
|
|
1412
1591
|
continue;
|
|
@@ -1414,7 +1593,9 @@ async function attemptWorkerCrashRetry(
|
|
|
1414
1593
|
|
|
1415
1594
|
// model_access_error is handled by attemptModelFallbackRetry() — skip here
|
|
1416
1595
|
if (classification === "model_access_error") {
|
|
1417
|
-
execLog(
|
|
1596
|
+
execLog(
|
|
1597
|
+
"batch",
|
|
1598
|
+
batchState.batchId,
|
|
1418
1599
|
`tier0: task ${taskId} classified as model_access_error — deferring to model fallback handler`,
|
|
1419
1600
|
);
|
|
1420
1601
|
continue;
|
|
@@ -1424,13 +1605,22 @@ async function attemptWorkerCrashRetry(
|
|
|
1424
1605
|
const scopeKey = tier0ScopeKey("worker_crash", taskId, waveIdx);
|
|
1425
1606
|
const currentCount = batchState.resilience.retryCountByScope[scopeKey] ?? 0;
|
|
1426
1607
|
if (currentCount >= budget.maxRetries) {
|
|
1427
|
-
execLog(
|
|
1608
|
+
execLog(
|
|
1609
|
+
"batch",
|
|
1610
|
+
batchState.batchId,
|
|
1428
1611
|
`tier0: task ${taskId} retry budget exhausted (${currentCount}/${budget.maxRetries}) — skipping`,
|
|
1429
1612
|
{ scopeKey },
|
|
1430
1613
|
);
|
|
1431
1614
|
// Emit exhausted event
|
|
1432
1615
|
emitTier0Event(stateRoot, {
|
|
1433
|
-
...buildTier0EventBase(
|
|
1616
|
+
...buildTier0EventBase(
|
|
1617
|
+
"tier0_recovery_exhausted",
|
|
1618
|
+
batchState.batchId,
|
|
1619
|
+
waveIdx,
|
|
1620
|
+
"worker_crash",
|
|
1621
|
+
currentCount,
|
|
1622
|
+
budget.maxRetries,
|
|
1623
|
+
),
|
|
1434
1624
|
taskId,
|
|
1435
1625
|
laneNumber: lane.laneNumber,
|
|
1436
1626
|
repoId: lane.repoId ?? null,
|
|
@@ -1440,8 +1630,15 @@ async function attemptWorkerCrashRetry(
|
|
|
1440
1630
|
affectedTaskIds: [taskId],
|
|
1441
1631
|
suggestion: `Task ${taskId} failed with ${classification} and exhausted ${budget.maxRetries} retry attempt(s). Consider investigating the root cause or manually re-running the task.`,
|
|
1442
1632
|
});
|
|
1443
|
-
emitTier0Escalation(
|
|
1444
|
-
|
|
1633
|
+
emitTier0Escalation(
|
|
1634
|
+
stateRoot,
|
|
1635
|
+
batchState.batchId,
|
|
1636
|
+
waveIdx,
|
|
1637
|
+
"worker_crash",
|
|
1638
|
+
currentCount,
|
|
1639
|
+
budget.maxRetries,
|
|
1640
|
+
`Retry budget exhausted for task ${taskId} (${classification})`,
|
|
1641
|
+
[taskId],
|
|
1445
1642
|
`Task ${taskId} failed with ${classification} and exhausted ${budget.maxRetries} retry attempt(s). Consider investigating the root cause or manually re-running the task.`,
|
|
1446
1643
|
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
1447
1644
|
);
|
|
@@ -1452,7 +1649,9 @@ async function attemptWorkerCrashRetry(
|
|
|
1452
1649
|
batchState.resilience.retryCountByScope[scopeKey] = currentCount + 1;
|
|
1453
1650
|
retriedCount++;
|
|
1454
1651
|
|
|
1455
|
-
execLog(
|
|
1652
|
+
execLog(
|
|
1653
|
+
"batch",
|
|
1654
|
+
batchState.batchId,
|
|
1456
1655
|
`tier0: retrying task ${taskId} (worker_crash, attempt ${currentCount + 1}/${budget.maxRetries}, classification=${classification})`,
|
|
1457
1656
|
{ scopeKey, classification },
|
|
1458
1657
|
);
|
|
@@ -1463,7 +1662,14 @@ async function attemptWorkerCrashRetry(
|
|
|
1463
1662
|
|
|
1464
1663
|
// Emit attempt event
|
|
1465
1664
|
emitTier0Event(stateRoot, {
|
|
1466
|
-
...buildTier0EventBase(
|
|
1665
|
+
...buildTier0EventBase(
|
|
1666
|
+
"tier0_recovery_attempt",
|
|
1667
|
+
batchState.batchId,
|
|
1668
|
+
waveIdx,
|
|
1669
|
+
"worker_crash",
|
|
1670
|
+
currentCount + 1,
|
|
1671
|
+
budget.maxRetries,
|
|
1672
|
+
),
|
|
1467
1673
|
taskId,
|
|
1468
1674
|
laneNumber: lane.laneNumber,
|
|
1469
1675
|
repoId: lane.repoId ?? null,
|
|
@@ -1478,7 +1684,7 @@ async function attemptWorkerCrashRetry(
|
|
|
1478
1684
|
}
|
|
1479
1685
|
|
|
1480
1686
|
// Find the specific AllocatedTask
|
|
1481
|
-
const allocatedTask = lane.tasks.find(t => t.taskId === taskId);
|
|
1687
|
+
const allocatedTask = lane.tasks.find((t) => t.taskId === taskId);
|
|
1482
1688
|
if (!allocatedTask) continue;
|
|
1483
1689
|
|
|
1484
1690
|
// Re-execute: create a single-task lane config for executeLane
|
|
@@ -1488,9 +1694,7 @@ async function attemptWorkerCrashRetry(
|
|
|
1488
1694
|
};
|
|
1489
1695
|
|
|
1490
1696
|
const isWsMode = !!workspaceConfig;
|
|
1491
|
-
const wsRoot = workspaceConfig
|
|
1492
|
-
? resolve(workspaceConfig.configPath, "..", "..")
|
|
1493
|
-
: undefined;
|
|
1697
|
+
const wsRoot = workspaceConfig ? resolve(workspaceConfig.configPath, "..", "..") : undefined;
|
|
1494
1698
|
|
|
1495
1699
|
try {
|
|
1496
1700
|
// Use a fresh pause signal for the retry — the batch pauseSignal
|
|
@@ -1504,7 +1708,12 @@ async function attemptWorkerCrashRetry(
|
|
|
1504
1708
|
retryPauseSignal,
|
|
1505
1709
|
wsRoot,
|
|
1506
1710
|
isWsMode,
|
|
1507
|
-
{
|
|
1711
|
+
{
|
|
1712
|
+
ORCH_BATCH_ID: batchState.batchId,
|
|
1713
|
+
...buildWorkerEnv(runnerConfig?.worker),
|
|
1714
|
+
...buildReviewerEnv(runnerConfig?.reviewer),
|
|
1715
|
+
...buildWorkerExcludeEnv(runnerConfig?.workerExcludeExtensions),
|
|
1716
|
+
}, // TP-089: ensure mailbox works for retries
|
|
1508
1717
|
);
|
|
1509
1718
|
|
|
1510
1719
|
const retryOutcome = retryResult.tasks[0];
|
|
@@ -1518,7 +1727,7 @@ async function attemptWorkerCrashRetry(
|
|
|
1518
1727
|
|
|
1519
1728
|
// Update lane results — replace the failed task outcome
|
|
1520
1729
|
for (const lr of waveResult.laneResults) {
|
|
1521
|
-
const taskIdx = lr.tasks.findIndex(t => t.taskId === taskId);
|
|
1730
|
+
const taskIdx = lr.tasks.findIndex((t) => t.taskId === taskId);
|
|
1522
1731
|
if (taskIdx !== -1) {
|
|
1523
1732
|
lr.tasks[taskIdx] = retryOutcome;
|
|
1524
1733
|
break;
|
|
@@ -1528,18 +1737,19 @@ async function attemptWorkerCrashRetry(
|
|
|
1528
1737
|
// Update allTaskOutcomes
|
|
1529
1738
|
upsertTaskOutcome(allTaskOutcomes, retryOutcome);
|
|
1530
1739
|
|
|
1531
|
-
execLog("batch", batchState.batchId,
|
|
1532
|
-
|
|
1533
|
-
{ scopeKey },
|
|
1534
|
-
);
|
|
1535
|
-
onNotify(
|
|
1536
|
-
`✅ Tier 0: Task ${taskId} retry succeeded`,
|
|
1537
|
-
"info",
|
|
1538
|
-
);
|
|
1740
|
+
execLog("batch", batchState.batchId, `tier0: task ${taskId} retry succeeded`, { scopeKey });
|
|
1741
|
+
onNotify(`✅ Tier 0: Task ${taskId} retry succeeded`, "info");
|
|
1539
1742
|
|
|
1540
1743
|
// Emit success event
|
|
1541
1744
|
emitTier0Event(stateRoot, {
|
|
1542
|
-
...buildTier0EventBase(
|
|
1745
|
+
...buildTier0EventBase(
|
|
1746
|
+
"tier0_recovery_success",
|
|
1747
|
+
batchState.batchId,
|
|
1748
|
+
waveIdx,
|
|
1749
|
+
"worker_crash",
|
|
1750
|
+
currentCount + 1,
|
|
1751
|
+
budget.maxRetries,
|
|
1752
|
+
),
|
|
1543
1753
|
taskId,
|
|
1544
1754
|
laneNumber: lane.laneNumber,
|
|
1545
1755
|
repoId: lane.repoId ?? null,
|
|
@@ -1552,16 +1762,23 @@ async function attemptWorkerCrashRetry(
|
|
|
1552
1762
|
if (retryOutcome) {
|
|
1553
1763
|
upsertTaskOutcome(allTaskOutcomes, retryOutcome);
|
|
1554
1764
|
}
|
|
1555
|
-
execLog("batch", batchState.batchId,
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
);
|
|
1765
|
+
execLog("batch", batchState.batchId, `tier0: task ${taskId} retry failed again`, {
|
|
1766
|
+
scopeKey,
|
|
1767
|
+
exitReason: retryOutcome?.exitReason,
|
|
1768
|
+
});
|
|
1559
1769
|
|
|
1560
1770
|
// Emit exhausted event (retry failed and budget now consumed)
|
|
1561
1771
|
const retryFailError = retryOutcome?.exitReason ?? `Task ${taskId} retry failed again`;
|
|
1562
1772
|
const retryFailSuggestion = `Task ${taskId} failed again after retry (${classification}). The failure may be persistent — investigate task logs.`;
|
|
1563
1773
|
emitTier0Event(stateRoot, {
|
|
1564
|
-
...buildTier0EventBase(
|
|
1774
|
+
...buildTier0EventBase(
|
|
1775
|
+
"tier0_recovery_exhausted",
|
|
1776
|
+
batchState.batchId,
|
|
1777
|
+
waveIdx,
|
|
1778
|
+
"worker_crash",
|
|
1779
|
+
currentCount + 1,
|
|
1780
|
+
budget.maxRetries,
|
|
1781
|
+
),
|
|
1565
1782
|
taskId,
|
|
1566
1783
|
laneNumber: lane.laneNumber,
|
|
1567
1784
|
repoId: lane.repoId ?? null,
|
|
@@ -1571,23 +1788,37 @@ async function attemptWorkerCrashRetry(
|
|
|
1571
1788
|
affectedTaskIds: [taskId],
|
|
1572
1789
|
suggestion: retryFailSuggestion,
|
|
1573
1790
|
});
|
|
1574
|
-
emitTier0Escalation(
|
|
1575
|
-
|
|
1791
|
+
emitTier0Escalation(
|
|
1792
|
+
stateRoot,
|
|
1793
|
+
batchState.batchId,
|
|
1794
|
+
waveIdx,
|
|
1795
|
+
"worker_crash",
|
|
1796
|
+
currentCount + 1,
|
|
1797
|
+
budget.maxRetries,
|
|
1798
|
+
retryFailError,
|
|
1799
|
+
[taskId],
|
|
1800
|
+
retryFailSuggestion,
|
|
1576
1801
|
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
1577
1802
|
);
|
|
1578
1803
|
}
|
|
1579
1804
|
} catch (err: unknown) {
|
|
1580
1805
|
failedRetries.push(taskId);
|
|
1581
1806
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
1582
|
-
execLog("batch", batchState.batchId,
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
);
|
|
1807
|
+
execLog("batch", batchState.batchId, `tier0: task ${taskId} retry threw error: ${errMsg}`, {
|
|
1808
|
+
scopeKey,
|
|
1809
|
+
});
|
|
1586
1810
|
|
|
1587
1811
|
// Emit exhausted event for exception during retry
|
|
1588
1812
|
const exceptionSuggestion = `Task ${taskId} retry threw an exception: ${errMsg}. Investigate the execution environment.`;
|
|
1589
1813
|
emitTier0Event(stateRoot, {
|
|
1590
|
-
...buildTier0EventBase(
|
|
1814
|
+
...buildTier0EventBase(
|
|
1815
|
+
"tier0_recovery_exhausted",
|
|
1816
|
+
batchState.batchId,
|
|
1817
|
+
waveIdx,
|
|
1818
|
+
"worker_crash",
|
|
1819
|
+
currentCount + 1,
|
|
1820
|
+
budget.maxRetries,
|
|
1821
|
+
),
|
|
1591
1822
|
taskId,
|
|
1592
1823
|
laneNumber: lane.laneNumber,
|
|
1593
1824
|
repoId: lane.repoId ?? null,
|
|
@@ -1597,8 +1828,16 @@ async function attemptWorkerCrashRetry(
|
|
|
1597
1828
|
affectedTaskIds: [taskId],
|
|
1598
1829
|
suggestion: exceptionSuggestion,
|
|
1599
1830
|
});
|
|
1600
|
-
emitTier0Escalation(
|
|
1601
|
-
|
|
1831
|
+
emitTier0Escalation(
|
|
1832
|
+
stateRoot,
|
|
1833
|
+
batchState.batchId,
|
|
1834
|
+
waveIdx,
|
|
1835
|
+
"worker_crash",
|
|
1836
|
+
currentCount + 1,
|
|
1837
|
+
budget.maxRetries,
|
|
1838
|
+
errMsg,
|
|
1839
|
+
[taskId],
|
|
1840
|
+
exceptionSuggestion,
|
|
1602
1841
|
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
1603
1842
|
);
|
|
1604
1843
|
}
|
|
@@ -1680,7 +1919,7 @@ async function attemptModelFallbackRetry(
|
|
|
1680
1919
|
const lane = taskToLane.get(taskId);
|
|
1681
1920
|
if (!lane) continue;
|
|
1682
1921
|
|
|
1683
|
-
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
1922
|
+
const outcome = allTaskOutcomes.find((o) => o.taskId === taskId);
|
|
1684
1923
|
if (!outcome) continue;
|
|
1685
1924
|
|
|
1686
1925
|
const classification = outcome.exitDiagnostic?.classification;
|
|
@@ -1690,12 +1929,21 @@ async function attemptModelFallbackRetry(
|
|
|
1690
1929
|
const scopeKey = tier0ScopeKey("model_fallback", taskId, waveIdx);
|
|
1691
1930
|
const currentCount = batchState.resilience.retryCountByScope[scopeKey] ?? 0;
|
|
1692
1931
|
if (currentCount >= budget.maxRetries) {
|
|
1693
|
-
execLog(
|
|
1932
|
+
execLog(
|
|
1933
|
+
"batch",
|
|
1934
|
+
batchState.batchId,
|
|
1694
1935
|
`tier0: task ${taskId} model fallback retry budget exhausted (${currentCount}/${budget.maxRetries})`,
|
|
1695
1936
|
{ scopeKey },
|
|
1696
1937
|
);
|
|
1697
1938
|
emitTier0Event(stateRoot, {
|
|
1698
|
-
...buildTier0EventBase(
|
|
1939
|
+
...buildTier0EventBase(
|
|
1940
|
+
"tier0_recovery_exhausted",
|
|
1941
|
+
batchState.batchId,
|
|
1942
|
+
waveIdx,
|
|
1943
|
+
"model_fallback",
|
|
1944
|
+
currentCount,
|
|
1945
|
+
budget.maxRetries,
|
|
1946
|
+
),
|
|
1699
1947
|
taskId,
|
|
1700
1948
|
laneNumber: lane.laneNumber,
|
|
1701
1949
|
repoId: lane.repoId ?? null,
|
|
@@ -1705,8 +1953,15 @@ async function attemptModelFallbackRetry(
|
|
|
1705
1953
|
affectedTaskIds: [taskId],
|
|
1706
1954
|
suggestion: `Task ${taskId} failed with model_access_error and model fallback retry exhausted. Check API key validity and model availability.`,
|
|
1707
1955
|
});
|
|
1708
|
-
emitTier0Escalation(
|
|
1709
|
-
|
|
1956
|
+
emitTier0Escalation(
|
|
1957
|
+
stateRoot,
|
|
1958
|
+
batchState.batchId,
|
|
1959
|
+
waveIdx,
|
|
1960
|
+
"model_fallback",
|
|
1961
|
+
currentCount,
|
|
1962
|
+
budget.maxRetries,
|
|
1963
|
+
`Model fallback retry budget exhausted for task ${taskId}`,
|
|
1964
|
+
[taskId],
|
|
1710
1965
|
`Task ${taskId} failed with model_access_error and model fallback retry exhausted. Check API key validity and model availability.`,
|
|
1711
1966
|
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
1712
1967
|
);
|
|
@@ -1718,7 +1973,9 @@ async function attemptModelFallbackRetry(
|
|
|
1718
1973
|
retriedCount++;
|
|
1719
1974
|
|
|
1720
1975
|
const failedModel = outcome.exitDiagnostic?.errorMessage || "configured model";
|
|
1721
|
-
execLog(
|
|
1976
|
+
execLog(
|
|
1977
|
+
"batch",
|
|
1978
|
+
batchState.batchId,
|
|
1722
1979
|
`tier0: model fallback — retrying task ${taskId} without explicit model (${failedModel} unavailable)`,
|
|
1723
1980
|
{ scopeKey, classification },
|
|
1724
1981
|
);
|
|
@@ -1729,7 +1986,14 @@ async function attemptModelFallbackRetry(
|
|
|
1729
1986
|
|
|
1730
1987
|
// Emit attempt event
|
|
1731
1988
|
emitTier0Event(stateRoot, {
|
|
1732
|
-
...buildTier0EventBase(
|
|
1989
|
+
...buildTier0EventBase(
|
|
1990
|
+
"tier0_recovery_attempt",
|
|
1991
|
+
batchState.batchId,
|
|
1992
|
+
waveIdx,
|
|
1993
|
+
"model_fallback",
|
|
1994
|
+
currentCount + 1,
|
|
1995
|
+
budget.maxRetries,
|
|
1996
|
+
),
|
|
1733
1997
|
taskId,
|
|
1734
1998
|
laneNumber: lane.laneNumber,
|
|
1735
1999
|
repoId: lane.repoId ?? null,
|
|
@@ -1744,7 +2008,7 @@ async function attemptModelFallbackRetry(
|
|
|
1744
2008
|
}
|
|
1745
2009
|
|
|
1746
2010
|
// Find the specific AllocatedTask
|
|
1747
|
-
const allocatedTask = lane.tasks.find(t => t.taskId === taskId);
|
|
2011
|
+
const allocatedTask = lane.tasks.find((t) => t.taskId === taskId);
|
|
1748
2012
|
if (!allocatedTask) continue;
|
|
1749
2013
|
|
|
1750
2014
|
// Re-execute with model fallback env var
|
|
@@ -1754,16 +2018,19 @@ async function attemptModelFallbackRetry(
|
|
|
1754
2018
|
};
|
|
1755
2019
|
|
|
1756
2020
|
const isWsMode = !!workspaceConfig;
|
|
1757
|
-
const wsRoot = workspaceConfig
|
|
1758
|
-
? resolve(workspaceConfig.configPath, "..", "..")
|
|
1759
|
-
: undefined;
|
|
2021
|
+
const wsRoot = workspaceConfig ? resolve(workspaceConfig.configPath, "..", "..") : undefined;
|
|
1760
2022
|
|
|
1761
2023
|
try {
|
|
1762
2024
|
const retryPauseSignal = { paused: false };
|
|
1763
2025
|
// Pass TASKPLANE_MODEL_FALLBACK=1 as extra env var to signal
|
|
1764
2026
|
// the task-runner to use the session model instead of configured model.
|
|
1765
2027
|
// TP-089: Also include ORCH_BATCH_ID so mailbox steering works for retries.
|
|
1766
|
-
const modelFallbackEnv = {
|
|
2028
|
+
const modelFallbackEnv = {
|
|
2029
|
+
TASKPLANE_MODEL_FALLBACK: "1",
|
|
2030
|
+
ORCH_BATCH_ID: batchState.batchId,
|
|
2031
|
+
...buildReviewerEnv(runnerConfig?.reviewer),
|
|
2032
|
+
...buildWorkerExcludeEnv(runnerConfig?.workerExcludeExtensions),
|
|
2033
|
+
};
|
|
1767
2034
|
const retryResult = await executeLaneV2(
|
|
1768
2035
|
retryLane,
|
|
1769
2036
|
orchConfig,
|
|
@@ -1785,7 +2052,7 @@ async function attemptModelFallbackRetry(
|
|
|
1785
2052
|
|
|
1786
2053
|
// Update lane results
|
|
1787
2054
|
for (const lr of waveResult.laneResults) {
|
|
1788
|
-
const taskIdx = lr.tasks.findIndex(t => t.taskId === taskId);
|
|
2055
|
+
const taskIdx = lr.tasks.findIndex((t) => t.taskId === taskId);
|
|
1789
2056
|
if (taskIdx !== -1) {
|
|
1790
2057
|
lr.tasks[taskIdx] = retryOutcome;
|
|
1791
2058
|
break;
|
|
@@ -1794,17 +2061,20 @@ async function attemptModelFallbackRetry(
|
|
|
1794
2061
|
|
|
1795
2062
|
upsertTaskOutcome(allTaskOutcomes, retryOutcome);
|
|
1796
2063
|
|
|
1797
|
-
execLog("batch", batchState.batchId,
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
);
|
|
1801
|
-
onNotify(
|
|
1802
|
-
`✅ Model fallback: Task ${taskId} succeeded with session model`,
|
|
1803
|
-
"info",
|
|
1804
|
-
);
|
|
2064
|
+
execLog("batch", batchState.batchId, `tier0: task ${taskId} model fallback retry succeeded`, {
|
|
2065
|
+
scopeKey,
|
|
2066
|
+
});
|
|
2067
|
+
onNotify(`✅ Model fallback: Task ${taskId} succeeded with session model`, "info");
|
|
1805
2068
|
|
|
1806
2069
|
emitTier0Event(stateRoot, {
|
|
1807
|
-
...buildTier0EventBase(
|
|
2070
|
+
...buildTier0EventBase(
|
|
2071
|
+
"tier0_recovery_success",
|
|
2072
|
+
batchState.batchId,
|
|
2073
|
+
waveIdx,
|
|
2074
|
+
"model_fallback",
|
|
2075
|
+
currentCount + 1,
|
|
2076
|
+
budget.maxRetries,
|
|
2077
|
+
),
|
|
1808
2078
|
taskId,
|
|
1809
2079
|
laneNumber: lane.laneNumber,
|
|
1810
2080
|
repoId: lane.repoId ?? null,
|
|
@@ -1817,14 +2087,21 @@ async function attemptModelFallbackRetry(
|
|
|
1817
2087
|
if (retryOutcome) {
|
|
1818
2088
|
upsertTaskOutcome(allTaskOutcomes, retryOutcome);
|
|
1819
2089
|
}
|
|
1820
|
-
execLog("batch", batchState.batchId,
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
);
|
|
2090
|
+
execLog("batch", batchState.batchId, `tier0: task ${taskId} model fallback retry failed`, {
|
|
2091
|
+
scopeKey,
|
|
2092
|
+
exitReason: retryOutcome?.exitReason,
|
|
2093
|
+
});
|
|
1824
2094
|
|
|
1825
2095
|
const retryFailError = retryOutcome?.exitReason ?? `Task ${taskId} model fallback retry failed`;
|
|
1826
2096
|
emitTier0Event(stateRoot, {
|
|
1827
|
-
...buildTier0EventBase(
|
|
2097
|
+
...buildTier0EventBase(
|
|
2098
|
+
"tier0_recovery_exhausted",
|
|
2099
|
+
batchState.batchId,
|
|
2100
|
+
waveIdx,
|
|
2101
|
+
"model_fallback",
|
|
2102
|
+
currentCount + 1,
|
|
2103
|
+
budget.maxRetries,
|
|
2104
|
+
),
|
|
1828
2105
|
taskId,
|
|
1829
2106
|
laneNumber: lane.laneNumber,
|
|
1830
2107
|
repoId: lane.repoId ?? null,
|
|
@@ -1834,8 +2111,15 @@ async function attemptModelFallbackRetry(
|
|
|
1834
2111
|
affectedTaskIds: [taskId],
|
|
1835
2112
|
suggestion: `Task ${taskId} failed even with session model fallback. Investigate task logs.`,
|
|
1836
2113
|
});
|
|
1837
|
-
emitTier0Escalation(
|
|
1838
|
-
|
|
2114
|
+
emitTier0Escalation(
|
|
2115
|
+
stateRoot,
|
|
2116
|
+
batchState.batchId,
|
|
2117
|
+
waveIdx,
|
|
2118
|
+
"model_fallback",
|
|
2119
|
+
currentCount + 1,
|
|
2120
|
+
budget.maxRetries,
|
|
2121
|
+
retryFailError,
|
|
2122
|
+
[taskId],
|
|
1839
2123
|
`Task ${taskId} failed even with session model fallback. Investigate task logs.`,
|
|
1840
2124
|
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
1841
2125
|
);
|
|
@@ -1843,12 +2127,21 @@ async function attemptModelFallbackRetry(
|
|
|
1843
2127
|
} catch (err: unknown) {
|
|
1844
2128
|
failedRetries.push(taskId);
|
|
1845
2129
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
1846
|
-
execLog(
|
|
2130
|
+
execLog(
|
|
2131
|
+
"batch",
|
|
2132
|
+
batchState.batchId,
|
|
1847
2133
|
`tier0: task ${taskId} model fallback retry threw error: ${errMsg}`,
|
|
1848
2134
|
{ scopeKey },
|
|
1849
2135
|
);
|
|
1850
2136
|
emitTier0Event(stateRoot, {
|
|
1851
|
-
...buildTier0EventBase(
|
|
2137
|
+
...buildTier0EventBase(
|
|
2138
|
+
"tier0_recovery_exhausted",
|
|
2139
|
+
batchState.batchId,
|
|
2140
|
+
waveIdx,
|
|
2141
|
+
"model_fallback",
|
|
2142
|
+
currentCount + 1,
|
|
2143
|
+
budget.maxRetries,
|
|
2144
|
+
),
|
|
1852
2145
|
taskId,
|
|
1853
2146
|
laneNumber: lane.laneNumber,
|
|
1854
2147
|
repoId: lane.repoId ?? null,
|
|
@@ -1858,8 +2151,15 @@ async function attemptModelFallbackRetry(
|
|
|
1858
2151
|
affectedTaskIds: [taskId],
|
|
1859
2152
|
suggestion: `Model fallback retry for task ${taskId} threw an exception: ${errMsg}`,
|
|
1860
2153
|
});
|
|
1861
|
-
emitTier0Escalation(
|
|
1862
|
-
|
|
2154
|
+
emitTier0Escalation(
|
|
2155
|
+
stateRoot,
|
|
2156
|
+
batchState.batchId,
|
|
2157
|
+
waveIdx,
|
|
2158
|
+
"model_fallback",
|
|
2159
|
+
currentCount + 1,
|
|
2160
|
+
budget.maxRetries,
|
|
2161
|
+
errMsg,
|
|
2162
|
+
[taskId],
|
|
1863
2163
|
`Model fallback retry for task ${taskId} threw an exception: ${errMsg}`,
|
|
1864
2164
|
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
1865
2165
|
);
|
|
@@ -1921,22 +2221,39 @@ async function attemptStaleWorktreeRecovery(
|
|
|
1921
2221
|
const currentCount = batchState.resilience.retryCountByScope[scopeKey] ?? 0;
|
|
1922
2222
|
|
|
1923
2223
|
if (currentCount >= budget.maxRetries) {
|
|
1924
|
-
execLog(
|
|
2224
|
+
execLog(
|
|
2225
|
+
"batch",
|
|
2226
|
+
batchState.batchId,
|
|
1925
2227
|
`tier0: stale worktree retry budget exhausted (${currentCount}/${budget.maxRetries})`,
|
|
1926
2228
|
{ scopeKey },
|
|
1927
2229
|
);
|
|
1928
2230
|
const staleExhaustedError = waveResult.allocationError.message;
|
|
1929
2231
|
const staleExhaustedSuggestion = `Stale worktree cleanup exhausted ${budget.maxRetries} retry(s). Manually remove worktrees and prune git state.`;
|
|
1930
2232
|
emitTier0Event(stateRoot, {
|
|
1931
|
-
...buildTier0EventBase(
|
|
2233
|
+
...buildTier0EventBase(
|
|
2234
|
+
"tier0_recovery_exhausted",
|
|
2235
|
+
batchState.batchId,
|
|
2236
|
+
waveIdx,
|
|
2237
|
+
"stale_worktree",
|
|
2238
|
+
currentCount,
|
|
2239
|
+
budget.maxRetries,
|
|
2240
|
+
),
|
|
1932
2241
|
repoId: null, // wave-scoped
|
|
1933
2242
|
error: staleExhaustedError,
|
|
1934
2243
|
scopeKey,
|
|
1935
2244
|
affectedTaskIds: waveTasks,
|
|
1936
2245
|
suggestion: staleExhaustedSuggestion,
|
|
1937
2246
|
});
|
|
1938
|
-
emitTier0Escalation(
|
|
1939
|
-
|
|
2247
|
+
emitTier0Escalation(
|
|
2248
|
+
stateRoot,
|
|
2249
|
+
batchState.batchId,
|
|
2250
|
+
waveIdx,
|
|
2251
|
+
"stale_worktree",
|
|
2252
|
+
currentCount,
|
|
2253
|
+
budget.maxRetries,
|
|
2254
|
+
staleExhaustedError,
|
|
2255
|
+
waveTasks,
|
|
2256
|
+
staleExhaustedSuggestion,
|
|
1940
2257
|
{ repoId: null, scopeKey },
|
|
1941
2258
|
);
|
|
1942
2259
|
return null;
|
|
@@ -1944,14 +2261,23 @@ async function attemptStaleWorktreeRecovery(
|
|
|
1944
2261
|
|
|
1945
2262
|
batchState.resilience.retryCountByScope[scopeKey] = currentCount + 1;
|
|
1946
2263
|
|
|
1947
|
-
execLog(
|
|
2264
|
+
execLog(
|
|
2265
|
+
"batch",
|
|
2266
|
+
batchState.batchId,
|
|
1948
2267
|
`tier0: attempting stale worktree recovery (attempt ${currentCount + 1}/${budget.maxRetries})`,
|
|
1949
2268
|
{ scopeKey, allocationError: waveResult.allocationError.message },
|
|
1950
2269
|
);
|
|
1951
2270
|
|
|
1952
2271
|
// Emit attempt event
|
|
1953
2272
|
emitTier0Event(stateRoot, {
|
|
1954
|
-
...buildTier0EventBase(
|
|
2273
|
+
...buildTier0EventBase(
|
|
2274
|
+
"tier0_recovery_attempt",
|
|
2275
|
+
batchState.batchId,
|
|
2276
|
+
waveIdx,
|
|
2277
|
+
"stale_worktree",
|
|
2278
|
+
currentCount + 1,
|
|
2279
|
+
budget.maxRetries,
|
|
2280
|
+
),
|
|
1955
2281
|
repoId: null, // wave-scoped: allocation failure may span multiple repos
|
|
1956
2282
|
classification: waveResult.allocationError.code,
|
|
1957
2283
|
cooldownMs: budget.cooldownMs,
|
|
@@ -1988,7 +2314,9 @@ async function attemptStaleWorktreeRecovery(
|
|
|
1988
2314
|
}
|
|
1989
2315
|
|
|
1990
2316
|
// Retry the wave execution
|
|
1991
|
-
execLog(
|
|
2317
|
+
execLog(
|
|
2318
|
+
"batch",
|
|
2319
|
+
batchState.batchId,
|
|
1992
2320
|
`tier0: retrying wave ${waveIdx + 1} after stale worktree cleanup`,
|
|
1993
2321
|
);
|
|
1994
2322
|
|
|
@@ -2014,12 +2342,14 @@ async function attemptStaleWorktreeRecovery(
|
|
|
2014
2342
|
tools: runnerConfig?.reviewer?.tools || "",
|
|
2015
2343
|
excludeExtensions: runnerConfig?.reviewer?.excludeExtensions ?? [],
|
|
2016
2344
|
},
|
|
2017
|
-
runnerConfig?.worker
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2345
|
+
runnerConfig?.worker
|
|
2346
|
+
? {
|
|
2347
|
+
model: runnerConfig.worker.model || "",
|
|
2348
|
+
thinking: runnerConfig.worker.thinking || "",
|
|
2349
|
+
tools: runnerConfig.worker.tools || "",
|
|
2350
|
+
excludeExtensions: runnerConfig.worker.excludeExtensions ?? [],
|
|
2351
|
+
}
|
|
2352
|
+
: undefined,
|
|
2023
2353
|
runnerConfig?.workerExcludeExtensions ?? [],
|
|
2024
2354
|
onLaneTerminated,
|
|
2025
2355
|
onLaneRespawned,
|
|
@@ -2028,7 +2358,6 @@ async function attemptStaleWorktreeRecovery(
|
|
|
2028
2358
|
return retryResult;
|
|
2029
2359
|
}
|
|
2030
2360
|
|
|
2031
|
-
|
|
2032
2361
|
export interface RuntimeBackendSelection {
|
|
2033
2362
|
backend: RuntimeBackend;
|
|
2034
2363
|
isSingleTask: boolean;
|
|
@@ -2050,8 +2379,7 @@ export function selectRuntimeBackend(
|
|
|
2050
2379
|
const isSingleTask = rawWaves.length === 1 && rawWaves[0]?.length === 1;
|
|
2051
2380
|
const isRepoMode = !workspaceConfig;
|
|
2052
2381
|
const argTokens = args.trim().split(/\s+/).filter(Boolean);
|
|
2053
|
-
const isDirectPromptTarget =
|
|
2054
|
-
argTokens.length === 1 && /PROMPT\.md$/i.test(argTokens[0]);
|
|
2382
|
+
const isDirectPromptTarget = argTokens.length === 1 && /PROMPT\.md$/i.test(argTokens[0]);
|
|
2055
2383
|
|
|
2056
2384
|
// TP-108: Runtime V2 for all repo-mode batches.
|
|
2057
2385
|
// TP-109: Workspace mode also uses V2 now that packet-home paths are
|
|
@@ -2166,20 +2494,40 @@ export async function executeOrchBatch(
|
|
|
2166
2494
|
if (terminalEventEmitted) return;
|
|
2167
2495
|
terminalEventEmitted = true;
|
|
2168
2496
|
if (batchState.phase === "completed" || batchState.phase === "failed") {
|
|
2169
|
-
emitEvent(
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2497
|
+
emitEvent(
|
|
2498
|
+
stateRoot,
|
|
2499
|
+
{
|
|
2500
|
+
...buildEngineEventBase(
|
|
2501
|
+
"batch_complete",
|
|
2502
|
+
batchState.batchId,
|
|
2503
|
+
batchState.currentWaveIndex,
|
|
2504
|
+
batchState.phase,
|
|
2505
|
+
),
|
|
2506
|
+
succeededTasks: batchState.succeededTasks,
|
|
2507
|
+
failedTasks: batchState.failedTasks,
|
|
2508
|
+
skippedTasks: batchState.skippedTasks,
|
|
2509
|
+
blockedTasks: batchState.blockedTasks,
|
|
2510
|
+
batchDurationMs: batchState.endedAt ? batchState.endedAt - batchState.startedAt : undefined,
|
|
2511
|
+
},
|
|
2512
|
+
onEngineEvent,
|
|
2513
|
+
);
|
|
2177
2514
|
} else if (batchState.phase === "paused" || batchState.phase === "stopped") {
|
|
2178
|
-
emitEvent(
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2515
|
+
emitEvent(
|
|
2516
|
+
stateRoot,
|
|
2517
|
+
{
|
|
2518
|
+
...buildEngineEventBase(
|
|
2519
|
+
"batch_paused",
|
|
2520
|
+
batchState.batchId,
|
|
2521
|
+
batchState.currentWaveIndex,
|
|
2522
|
+
batchState.phase,
|
|
2523
|
+
),
|
|
2524
|
+
reason:
|
|
2525
|
+
reason ||
|
|
2526
|
+
(batchState.errors.length > 0 ? batchState.errors[batchState.errors.length - 1] : "paused"),
|
|
2527
|
+
failedTasks: batchState.failedTasks,
|
|
2528
|
+
},
|
|
2529
|
+
onEngineEvent,
|
|
2530
|
+
);
|
|
2183
2531
|
}
|
|
2184
2532
|
};
|
|
2185
2533
|
|
|
@@ -2200,7 +2548,10 @@ export async function executeOrchBatch(
|
|
|
2200
2548
|
batchState.phase = "failed";
|
|
2201
2549
|
batchState.endedAt = Date.now();
|
|
2202
2550
|
batchState.errors.push("Cannot determine current branch (detached HEAD or not a git repo)");
|
|
2203
|
-
onNotify(
|
|
2551
|
+
onNotify(
|
|
2552
|
+
"❌ Cannot determine current branch. Ensure HEAD is on a branch (not detached).",
|
|
2553
|
+
"error",
|
|
2554
|
+
);
|
|
2204
2555
|
emitTerminalEvent();
|
|
2205
2556
|
return;
|
|
2206
2557
|
}
|
|
@@ -2251,6 +2602,13 @@ export async function executeOrchBatch(
|
|
|
2251
2602
|
// Sweep stale artifacts, rotate oversized logs, enforce size cap,
|
|
2252
2603
|
// and clean prior batch artifacts before batch starts.
|
|
2253
2604
|
// Always non-fatal — failures warn but never block batch execution.
|
|
2605
|
+
//
|
|
2606
|
+
// TP-195: imported `sweepStaleArtifacts`, `formatPreflightSweep`,
|
|
2607
|
+
// `rotateSupervisorLogs`, and `formatLogRotation` from `./cleanup.ts`
|
|
2608
|
+
// (they were referenced here but never imported, so the try/catch was
|
|
2609
|
+
// swallowing a ReferenceError on every batch and Layers 2–5 had been
|
|
2610
|
+
// silently a no-op since TP-065 ~2024-09). With the imports added,
|
|
2611
|
+
// the preflight cleanup feature now works as advertised.
|
|
2254
2612
|
try {
|
|
2255
2613
|
// Layer 2: Age-based sweep of stale telemetry/merge/verification/conversation artifacts (>3 days)
|
|
2256
2614
|
const sweepResult = sweepStaleArtifacts(stateRoot, {
|
|
@@ -2258,10 +2616,17 @@ export async function executeOrchBatch(
|
|
|
2258
2616
|
// Check persisted state — a prior batch may still be active
|
|
2259
2617
|
try {
|
|
2260
2618
|
const state = loadBatchState(stateRoot);
|
|
2261
|
-
if (
|
|
2619
|
+
if (
|
|
2620
|
+
state &&
|
|
2621
|
+
state.phase !== "completed" &&
|
|
2622
|
+
state.phase !== "failed" &&
|
|
2623
|
+
state.phase !== "stopped"
|
|
2624
|
+
) {
|
|
2262
2625
|
return true;
|
|
2263
2626
|
}
|
|
2264
|
-
} catch {
|
|
2627
|
+
} catch {
|
|
2628
|
+
/* state unreadable — safe to sweep */
|
|
2629
|
+
}
|
|
2265
2630
|
return false;
|
|
2266
2631
|
},
|
|
2267
2632
|
now: () => Date.now(),
|
|
@@ -2324,14 +2689,12 @@ export async function executeOrchBatch(
|
|
|
2324
2689
|
"info",
|
|
2325
2690
|
);
|
|
2326
2691
|
}
|
|
2327
|
-
const hasStrictErrors = fatalErrors.some(
|
|
2328
|
-
(e) => e.code === "TASK_ROUTING_STRICT",
|
|
2329
|
-
);
|
|
2692
|
+
const hasStrictErrors = fatalErrors.some((e) => e.code === "TASK_ROUTING_STRICT");
|
|
2330
2693
|
if (hasStrictErrors) {
|
|
2331
2694
|
onNotify(
|
|
2332
2695
|
"💡 Strict routing is enabled (routing.strict: true). Every task must declare an explicit execution target.\n" +
|
|
2333
|
-
|
|
2334
|
-
|
|
2696
|
+
" Add a `## Execution Target` section with `Repo: <id>` to each task's PROMPT.md.\n" +
|
|
2697
|
+
" To disable strict routing, set `routing.strict: false` in workspace config.",
|
|
2335
2698
|
"info",
|
|
2336
2699
|
);
|
|
2337
2700
|
}
|
|
@@ -2356,7 +2719,7 @@ export async function executeOrchBatch(
|
|
|
2356
2719
|
if (!validation.valid) {
|
|
2357
2720
|
batchState.phase = "failed";
|
|
2358
2721
|
batchState.endedAt = Date.now();
|
|
2359
|
-
const errMsgs = validation.errors.map(e => `[${e.code}] ${e.message}`).join("\n");
|
|
2722
|
+
const errMsgs = validation.errors.map((e) => `[${e.code}] ${e.message}`).join("\n");
|
|
2360
2723
|
batchState.errors.push(`Graph validation failed:\n${errMsgs}`);
|
|
2361
2724
|
onNotify(`❌ Dependency graph errors:\n${errMsgs}`, "error");
|
|
2362
2725
|
emitTerminalEvent();
|
|
@@ -2375,14 +2738,16 @@ export async function executeOrchBatch(
|
|
|
2375
2738
|
if (waveComputation.errors.length > 0) {
|
|
2376
2739
|
batchState.phase = "failed";
|
|
2377
2740
|
batchState.endedAt = Date.now();
|
|
2378
|
-
const errMsgs = waveComputation.errors.map(e => `[${e.code}] ${e.message}`).join("\n");
|
|
2741
|
+
const errMsgs = waveComputation.errors.map((e) => `[${e.code}] ${e.message}`).join("\n");
|
|
2379
2742
|
batchState.errors.push(`Wave computation failed:\n${errMsgs}`);
|
|
2380
2743
|
onNotify(`❌ Wave computation errors:\n${errMsgs}`, "error");
|
|
2381
2744
|
emitTerminalEvent();
|
|
2382
2745
|
return;
|
|
2383
2746
|
}
|
|
2384
2747
|
|
|
2385
|
-
const taskWaves = waveComputation.waves.map((wave) =>
|
|
2748
|
+
const taskWaves = waveComputation.waves.map((wave) =>
|
|
2749
|
+
wave.tasks.map((assignment) => assignment.taskId),
|
|
2750
|
+
);
|
|
2386
2751
|
const packetRepoId = workspaceConfig?.routing?.taskPacketRepo;
|
|
2387
2752
|
const frontier = buildSegmentFrontierWaves(
|
|
2388
2753
|
taskWaves,
|
|
@@ -2428,19 +2793,27 @@ export async function executeOrchBatch(
|
|
|
2428
2793
|
const repoBranch = getCurrentBranch(rRoot) || "HEAD";
|
|
2429
2794
|
const result = runGit(["branch", orchBranch, repoBranch], rRoot);
|
|
2430
2795
|
if (result.ok) {
|
|
2431
|
-
execLog("batch", batchState.batchId, `created orch branch in ${repoId}`, {
|
|
2796
|
+
execLog("batch", batchState.batchId, `created orch branch in ${repoId}`, {
|
|
2797
|
+
orchBranch,
|
|
2798
|
+
base: repoBranch,
|
|
2799
|
+
});
|
|
2432
2800
|
} else {
|
|
2433
2801
|
const errDetail = result.stderr || result.stdout || "unknown error";
|
|
2434
2802
|
execLog("batch", batchState.batchId, `failed to create orch branch in ${repoId}: ${errDetail}`);
|
|
2435
2803
|
batchState.phase = "failed";
|
|
2436
2804
|
batchState.endedAt = Date.now();
|
|
2437
|
-
batchState.errors.push(
|
|
2805
|
+
batchState.errors.push(
|
|
2806
|
+
`Failed to create orch branch '${orchBranch}' in ${repoId}: ${errDetail}`,
|
|
2807
|
+
);
|
|
2438
2808
|
onNotify(`❌ Failed to create orch branch '${orchBranch}' in ${repoId}: ${errDetail}`, "error");
|
|
2439
2809
|
orchBranchFailed = true;
|
|
2440
2810
|
break;
|
|
2441
2811
|
}
|
|
2442
2812
|
}
|
|
2443
|
-
if (orchBranchFailed) {
|
|
2813
|
+
if (orchBranchFailed) {
|
|
2814
|
+
emitTerminalEvent();
|
|
2815
|
+
return;
|
|
2816
|
+
}
|
|
2444
2817
|
} else {
|
|
2445
2818
|
const branchResult = runGit(["branch", orchBranch, batchState.baseBranch], repoRoot);
|
|
2446
2819
|
if (!branchResult.ok) {
|
|
@@ -2452,7 +2825,10 @@ export async function executeOrchBatch(
|
|
|
2452
2825
|
emitTerminalEvent();
|
|
2453
2826
|
return;
|
|
2454
2827
|
}
|
|
2455
|
-
execLog("batch", batchState.batchId, "created orch branch", {
|
|
2828
|
+
execLog("batch", batchState.batchId, "created orch branch", {
|
|
2829
|
+
orchBranch,
|
|
2830
|
+
baseBranch: batchState.baseBranch,
|
|
2831
|
+
});
|
|
2456
2832
|
}
|
|
2457
2833
|
batchState.orchBranch = orchBranch;
|
|
2458
2834
|
|
|
@@ -2466,7 +2842,15 @@ export async function executeOrchBatch(
|
|
|
2466
2842
|
batchState.phase = "executing";
|
|
2467
2843
|
|
|
2468
2844
|
// ── TS-009: Persist state on batch start (after wave computation) ──
|
|
2469
|
-
persistRuntimeState(
|
|
2845
|
+
persistRuntimeState(
|
|
2846
|
+
"batch-start",
|
|
2847
|
+
batchState,
|
|
2848
|
+
wavePlan,
|
|
2849
|
+
latestAllocatedLanes,
|
|
2850
|
+
allTaskOutcomes,
|
|
2851
|
+
discoveryRef,
|
|
2852
|
+
stateRoot,
|
|
2853
|
+
);
|
|
2470
2854
|
|
|
2471
2855
|
// ── TP-187 (#539): Persist batch-meta runtime artifact ──────────────
|
|
2472
2856
|
// Captures the wave plan and core scalars to a runtime-side file that
|
|
@@ -2476,7 +2860,7 @@ export async function executeOrchBatch(
|
|
|
2476
2860
|
saveBatchMetaRuntimeArtifact(stateRoot, {
|
|
2477
2861
|
schemaVersion: 1,
|
|
2478
2862
|
batchId: batchState.batchId,
|
|
2479
|
-
wavePlan: wavePlan.map(wave => [...wave]),
|
|
2863
|
+
wavePlan: wavePlan.map((wave) => [...wave]),
|
|
2480
2864
|
baseBranch: batchState.baseBranch,
|
|
2481
2865
|
orchBranch: batchState.orchBranch,
|
|
2482
2866
|
mode: workspaceConfig ? "workspace" : "repo",
|
|
@@ -2506,10 +2890,21 @@ export async function executeOrchBatch(
|
|
|
2506
2890
|
execLog("batch", batchState.batchId, `batch paused before wave ${waveIdx + 1}`);
|
|
2507
2891
|
{
|
|
2508
2892
|
const { displayWave } = resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount);
|
|
2509
|
-
onNotify(
|
|
2893
|
+
onNotify(
|
|
2894
|
+
`⏸️ Batch paused before wave ${displayWave}. Resume not yet implemented (TS-009).`,
|
|
2895
|
+
"warning",
|
|
2896
|
+
);
|
|
2510
2897
|
}
|
|
2511
2898
|
// ── TS-009: Persist state on pause ──
|
|
2512
|
-
persistRuntimeState(
|
|
2899
|
+
persistRuntimeState(
|
|
2900
|
+
"pause-before-wave",
|
|
2901
|
+
batchState,
|
|
2902
|
+
wavePlan,
|
|
2903
|
+
latestAllocatedLanes,
|
|
2904
|
+
allTaskOutcomes,
|
|
2905
|
+
discoveryRef,
|
|
2906
|
+
stateRoot,
|
|
2907
|
+
);
|
|
2513
2908
|
// TP-040: Emit batch_paused event (via terminal helper for dedup)
|
|
2514
2909
|
emitTerminalEvent(`Paused before wave ${waveIdx + 1}`);
|
|
2515
2910
|
break;
|
|
@@ -2518,7 +2913,15 @@ export async function executeOrchBatch(
|
|
|
2518
2913
|
batchState.currentWaveIndex = waveIdx;
|
|
2519
2914
|
|
|
2520
2915
|
// ── TS-009: Persist state on wave index change ──
|
|
2521
|
-
persistRuntimeState(
|
|
2916
|
+
persistRuntimeState(
|
|
2917
|
+
"wave-index-change",
|
|
2918
|
+
batchState,
|
|
2919
|
+
wavePlan,
|
|
2920
|
+
latestAllocatedLanes,
|
|
2921
|
+
allTaskOutcomes,
|
|
2922
|
+
discoveryRef,
|
|
2923
|
+
stateRoot,
|
|
2924
|
+
);
|
|
2522
2925
|
|
|
2523
2926
|
// Filter wave tasks against blocked + terminal task sets, then bind the
|
|
2524
2927
|
// next active segment for each surviving task.
|
|
@@ -2563,26 +2966,48 @@ export async function executeOrchBatch(
|
|
|
2563
2966
|
}
|
|
2564
2967
|
|
|
2565
2968
|
if (blockedInWave.length > 0) {
|
|
2566
|
-
execLog(
|
|
2567
|
-
|
|
2568
|
-
|
|
2969
|
+
execLog(
|
|
2970
|
+
"batch",
|
|
2971
|
+
batchState.batchId,
|
|
2972
|
+
`wave ${waveIdx + 1}: skipping ${blockedInWave.length} blocked task(s)`,
|
|
2973
|
+
{
|
|
2974
|
+
blocked: blockedInWave.join(","),
|
|
2975
|
+
},
|
|
2976
|
+
);
|
|
2569
2977
|
batchState.blockedTasks += blockedInWave.length;
|
|
2570
2978
|
}
|
|
2571
2979
|
if (terminalInWave.length > 0) {
|
|
2572
|
-
execLog(
|
|
2573
|
-
|
|
2574
|
-
|
|
2980
|
+
execLog(
|
|
2981
|
+
"batch",
|
|
2982
|
+
batchState.batchId,
|
|
2983
|
+
`wave ${waveIdx + 1}: skipping ${terminalInWave.length} terminal task(s)`,
|
|
2984
|
+
{
|
|
2985
|
+
terminal: terminalInWave.join(","),
|
|
2986
|
+
},
|
|
2987
|
+
);
|
|
2575
2988
|
}
|
|
2576
2989
|
|
|
2577
2990
|
if (waveTasks.length === 0) {
|
|
2578
|
-
execLog(
|
|
2991
|
+
execLog(
|
|
2992
|
+
"batch",
|
|
2993
|
+
batchState.batchId,
|
|
2994
|
+
`wave ${waveIdx + 1}: no tasks to execute (all blocked or terminal)`,
|
|
2995
|
+
);
|
|
2579
2996
|
continue;
|
|
2580
2997
|
}
|
|
2581
2998
|
|
|
2582
2999
|
const handleWaveMonitorUpdate: MonitorUpdateCallback = (monitorState) => {
|
|
2583
3000
|
const changed = syncTaskOutcomesFromMonitor(monitorState, allTaskOutcomes);
|
|
2584
3001
|
if (changed) {
|
|
2585
|
-
persistRuntimeState(
|
|
3002
|
+
persistRuntimeState(
|
|
3003
|
+
"task-transition",
|
|
3004
|
+
batchState,
|
|
3005
|
+
wavePlan,
|
|
3006
|
+
latestAllocatedLanes,
|
|
3007
|
+
allTaskOutcomes,
|
|
3008
|
+
discoveryRef,
|
|
3009
|
+
stateRoot,
|
|
3010
|
+
);
|
|
2586
3011
|
}
|
|
2587
3012
|
onMonitorUpdate?.(monitorState);
|
|
2588
3013
|
};
|
|
@@ -2593,13 +3018,23 @@ export async function executeOrchBatch(
|
|
|
2593
3018
|
batchState.currentLanes = lanes;
|
|
2594
3019
|
|
|
2595
3020
|
// TP-166: Use task-level wave number for operator display
|
|
2596
|
-
const { displayWave, displayTotal } = resolveDisplayWaveNumber(
|
|
3021
|
+
const { displayWave, displayTotal } = resolveDisplayWaveNumber(
|
|
3022
|
+
waveIdx,
|
|
3023
|
+
roundToTaskWave,
|
|
3024
|
+
taskLevelWaveCount,
|
|
3025
|
+
);
|
|
2597
3026
|
onNotify(
|
|
2598
3027
|
ORCH_MESSAGES.orchWaveStart(displayWave, displayTotal, waveTasks.length, lanes.length),
|
|
2599
3028
|
"info",
|
|
2600
3029
|
);
|
|
2601
3030
|
// TP-148: Build per-task segment context for the wave_start event
|
|
2602
|
-
const waveSegmentContext: Array<{
|
|
3031
|
+
const waveSegmentContext: Array<{
|
|
3032
|
+
taskId: string;
|
|
3033
|
+
segmentIndex: number;
|
|
3034
|
+
totalSegments: number;
|
|
3035
|
+
repoId: string;
|
|
3036
|
+
segmentId: string;
|
|
3037
|
+
}> = [];
|
|
2603
3038
|
for (const taskId of waveTasks) {
|
|
2604
3039
|
const segState = segmentStateByTask.get(taskId);
|
|
2605
3040
|
if (segState && segState.orderedSegments.length > 1) {
|
|
@@ -2616,12 +3051,16 @@ export async function executeOrchBatch(
|
|
|
2616
3051
|
}
|
|
2617
3052
|
}
|
|
2618
3053
|
}
|
|
2619
|
-
emitEvent(
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
3054
|
+
emitEvent(
|
|
3055
|
+
stateRoot,
|
|
3056
|
+
{
|
|
3057
|
+
...buildEngineEventBase("wave_start", batchState.batchId, waveIdx, batchState.phase),
|
|
3058
|
+
taskIds: waveTasks,
|
|
3059
|
+
laneCount: lanes.length,
|
|
3060
|
+
...(waveSegmentContext.length > 0 ? { segmentContext: waveSegmentContext } : {}),
|
|
3061
|
+
},
|
|
3062
|
+
onEngineEvent,
|
|
3063
|
+
);
|
|
2625
3064
|
// TP-029: Track repos from newly allocated lanes for cleanup coverage
|
|
2626
3065
|
for (const lane of lanes) {
|
|
2627
3066
|
const laneRepoRoot = resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig);
|
|
@@ -2634,7 +3073,8 @@ export async function executeOrchBatch(
|
|
|
2634
3073
|
const task = discovery.pending.get(laneTask.taskId);
|
|
2635
3074
|
const segmentState = segmentStateByTask.get(laneTask.taskId);
|
|
2636
3075
|
if (!task || !segmentState) continue;
|
|
2637
|
-
startedSegments =
|
|
3076
|
+
startedSegments =
|
|
3077
|
+
upsertRunningSegmentRecord(batchState, task, segmentState, lane) || startedSegments;
|
|
2638
3078
|
}
|
|
2639
3079
|
}
|
|
2640
3080
|
if (seededPendingOutcomes || startedSegments) {
|
|
@@ -2672,12 +3112,14 @@ export async function executeOrchBatch(
|
|
|
2672
3112
|
tools: runnerConfig?.reviewer?.tools || "",
|
|
2673
3113
|
excludeExtensions: runnerConfig?.reviewer?.excludeExtensions ?? [],
|
|
2674
3114
|
},
|
|
2675
|
-
runnerConfig?.worker
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
3115
|
+
runnerConfig?.worker
|
|
3116
|
+
? {
|
|
3117
|
+
model: runnerConfig.worker.model || "",
|
|
3118
|
+
thinking: runnerConfig.worker.thinking || "",
|
|
3119
|
+
tools: runnerConfig.worker.tools || "",
|
|
3120
|
+
excludeExtensions: runnerConfig.worker.excludeExtensions ?? [],
|
|
3121
|
+
}
|
|
3122
|
+
: undefined,
|
|
2681
3123
|
runnerConfig?.workerExcludeExtensions ?? [],
|
|
2682
3124
|
emitLaneTerminated,
|
|
2683
3125
|
onLaneRespawned ?? undefined,
|
|
@@ -2719,24 +3161,48 @@ export async function executeOrchBatch(
|
|
|
2719
3161
|
const staleCount = batchState.resilience?.retryCountByScope[staleScopeKey] ?? 1;
|
|
2720
3162
|
if (staleRecovered) {
|
|
2721
3163
|
emitTier0Event(stateRoot, {
|
|
2722
|
-
...buildTier0EventBase(
|
|
3164
|
+
...buildTier0EventBase(
|
|
3165
|
+
"tier0_recovery_success",
|
|
3166
|
+
batchState.batchId,
|
|
3167
|
+
waveIdx,
|
|
3168
|
+
"stale_worktree",
|
|
3169
|
+
staleCount,
|
|
3170
|
+
TIER0_RETRY_BUDGETS.stale_worktree.maxRetries,
|
|
3171
|
+
),
|
|
2723
3172
|
repoId: null, // wave-scoped
|
|
2724
3173
|
resolution: `Stale worktree cleanup succeeded — wave ${waveIdx + 1} re-executed successfully`,
|
|
2725
3174
|
scopeKey: staleScopeKey,
|
|
2726
3175
|
});
|
|
2727
3176
|
} else {
|
|
2728
|
-
const staleRetryError =
|
|
2729
|
-
|
|
3177
|
+
const staleRetryError =
|
|
3178
|
+
retryResult.allocationError?.message ?? "Allocation failed again after cleanup";
|
|
3179
|
+
const staleRetrySuggestion =
|
|
3180
|
+
"Stale worktree cleanup did not resolve the allocation failure. Manually inspect and remove worktrees.";
|
|
2730
3181
|
emitTier0Event(stateRoot, {
|
|
2731
|
-
...buildTier0EventBase(
|
|
3182
|
+
...buildTier0EventBase(
|
|
3183
|
+
"tier0_recovery_exhausted",
|
|
3184
|
+
batchState.batchId,
|
|
3185
|
+
waveIdx,
|
|
3186
|
+
"stale_worktree",
|
|
3187
|
+
staleCount,
|
|
3188
|
+
TIER0_RETRY_BUDGETS.stale_worktree.maxRetries,
|
|
3189
|
+
),
|
|
2732
3190
|
repoId: null, // wave-scoped
|
|
2733
3191
|
error: staleRetryError,
|
|
2734
3192
|
scopeKey: staleScopeKey,
|
|
2735
3193
|
affectedTaskIds: waveTasks,
|
|
2736
3194
|
suggestion: staleRetrySuggestion,
|
|
2737
3195
|
});
|
|
2738
|
-
emitTier0Escalation(
|
|
2739
|
-
|
|
3196
|
+
emitTier0Escalation(
|
|
3197
|
+
stateRoot,
|
|
3198
|
+
batchState.batchId,
|
|
3199
|
+
waveIdx,
|
|
3200
|
+
"stale_worktree",
|
|
3201
|
+
staleCount,
|
|
3202
|
+
TIER0_RETRY_BUDGETS.stale_worktree.maxRetries,
|
|
3203
|
+
staleRetryError,
|
|
3204
|
+
waveTasks,
|
|
3205
|
+
staleRetrySuggestion,
|
|
2740
3206
|
{ repoId: null, scopeKey: staleScopeKey },
|
|
2741
3207
|
);
|
|
2742
3208
|
}
|
|
@@ -2777,17 +3243,22 @@ export async function executeOrchBatch(
|
|
|
2777
3243
|
if (modelFallbackOutcome.succeededRetries.length > 0) {
|
|
2778
3244
|
// Recompute blocked tasks after model fallback successes
|
|
2779
3245
|
if (waveResult.policyApplied === "skip-dependents" && waveResult.failedTaskIds.length > 0) {
|
|
2780
|
-
const recomputed = computeTransitiveDependents(
|
|
2781
|
-
new Set(waveResult.failedTaskIds),
|
|
2782
|
-
depGraph,
|
|
2783
|
-
);
|
|
3246
|
+
const recomputed = computeTransitiveDependents(new Set(waveResult.failedTaskIds), depGraph);
|
|
2784
3247
|
waveResult.blockedTaskIds = [...recomputed].sort();
|
|
2785
3248
|
} else if (waveResult.failedTaskIds.length === 0) {
|
|
2786
3249
|
waveResult.blockedTaskIds = [];
|
|
2787
3250
|
}
|
|
2788
3251
|
}
|
|
2789
3252
|
if (modelFallbackOutcome.retriedCount > 0) {
|
|
2790
|
-
persistRuntimeState(
|
|
3253
|
+
persistRuntimeState(
|
|
3254
|
+
"tier0-model-fallback",
|
|
3255
|
+
batchState,
|
|
3256
|
+
wavePlan,
|
|
3257
|
+
latestAllocatedLanes,
|
|
3258
|
+
allTaskOutcomes,
|
|
3259
|
+
discoveryRef,
|
|
3260
|
+
stateRoot,
|
|
3261
|
+
);
|
|
2791
3262
|
}
|
|
2792
3263
|
}
|
|
2793
3264
|
|
|
@@ -2814,10 +3285,7 @@ export async function executeOrchBatch(
|
|
|
2814
3285
|
// attemptWorkerCrashRetry already updated waveResult.failedTaskIds
|
|
2815
3286
|
// and waveResult.succeededTaskIds in-place.
|
|
2816
3287
|
if (waveResult.policyApplied === "skip-dependents" && waveResult.failedTaskIds.length > 0) {
|
|
2817
|
-
const recomputed = computeTransitiveDependents(
|
|
2818
|
-
new Set(waveResult.failedTaskIds),
|
|
2819
|
-
depGraph,
|
|
2820
|
-
);
|
|
3288
|
+
const recomputed = computeTransitiveDependents(new Set(waveResult.failedTaskIds), depGraph);
|
|
2821
3289
|
waveResult.blockedTaskIds = [...recomputed].sort();
|
|
2822
3290
|
} else if (waveResult.failedTaskIds.length === 0) {
|
|
2823
3291
|
// All failures recovered — no blocked tasks
|
|
@@ -2826,7 +3294,15 @@ export async function executeOrchBatch(
|
|
|
2826
3294
|
}
|
|
2827
3295
|
if (retryOutcome.retriedCount > 0) {
|
|
2828
3296
|
// Persist updated state after retries
|
|
2829
|
-
persistRuntimeState(
|
|
3297
|
+
persistRuntimeState(
|
|
3298
|
+
"tier0-worker-retry",
|
|
3299
|
+
batchState,
|
|
3300
|
+
wavePlan,
|
|
3301
|
+
latestAllocatedLanes,
|
|
3302
|
+
allTaskOutcomes,
|
|
3303
|
+
discoveryRef,
|
|
3304
|
+
stateRoot,
|
|
3305
|
+
);
|
|
2830
3306
|
}
|
|
2831
3307
|
|
|
2832
3308
|
// If stop-wave had paused the batch but Tier 0 retry recovered all
|
|
@@ -2834,18 +3310,17 @@ export async function executeOrchBatch(
|
|
|
2834
3310
|
// proceed. attemptWorkerCrashRetry already set stoppedEarly=false
|
|
2835
3311
|
// and overallStatus="succeeded" on the waveResult (R002-4 fix).
|
|
2836
3312
|
if (
|
|
2837
|
-
waveResult.failedTaskIds.length === 0
|
|
2838
|
-
|
|
2839
|
-
|
|
3313
|
+
waveResult.failedTaskIds.length === 0 &&
|
|
3314
|
+
batchState.pauseSignal.paused &&
|
|
3315
|
+
waveResult.policyApplied === "stop-wave"
|
|
2840
3316
|
) {
|
|
2841
3317
|
batchState.pauseSignal.paused = false;
|
|
2842
|
-
execLog(
|
|
3318
|
+
execLog(
|
|
3319
|
+
"batch",
|
|
3320
|
+
batchState.batchId,
|
|
2843
3321
|
`tier0: all failed tasks recovered — clearing stop-wave pause`,
|
|
2844
3322
|
);
|
|
2845
|
-
onNotify(
|
|
2846
|
-
`✅ Tier 0: All failed tasks recovered — batch continuing past stop-wave`,
|
|
2847
|
-
"info",
|
|
2848
|
-
);
|
|
3323
|
+
onNotify(`✅ Tier 0: All failed tasks recovered — batch continuing past stop-wave`, "info");
|
|
2849
3324
|
}
|
|
2850
3325
|
}
|
|
2851
3326
|
|
|
@@ -2873,28 +3348,53 @@ export async function executeOrchBatch(
|
|
|
2873
3348
|
const activeSegmentId = outcome?.segmentId ?? task.activeSegmentId;
|
|
2874
3349
|
if (activeSegmentId) {
|
|
2875
3350
|
segmentState.statusBySegmentId.set(activeSegmentId, "succeeded");
|
|
2876
|
-
upsertTerminalSegmentRecord(
|
|
3351
|
+
upsertTerminalSegmentRecord(
|
|
3352
|
+
batchState,
|
|
3353
|
+
task,
|
|
3354
|
+
segmentState,
|
|
3355
|
+
activeSegmentId,
|
|
3356
|
+
"succeeded",
|
|
3357
|
+
outcome,
|
|
3358
|
+
laneByTaskId.get(taskId),
|
|
3359
|
+
);
|
|
2877
3360
|
|
|
2878
|
-
const workerAgentId = resolveTaskWorkerAgentId(
|
|
3361
|
+
const workerAgentId = resolveTaskWorkerAgentId(
|
|
3362
|
+
taskId,
|
|
3363
|
+
allTaskOutcomes,
|
|
3364
|
+
laneByTaskId,
|
|
3365
|
+
agentIdPrefix,
|
|
3366
|
+
);
|
|
2879
3367
|
if (workerAgentId) {
|
|
2880
|
-
const pendingExpansionFiles = listPendingSegmentExpansionRequestFiles(
|
|
3368
|
+
const pendingExpansionFiles = listPendingSegmentExpansionRequestFiles(
|
|
3369
|
+
stateRoot,
|
|
3370
|
+
batchState.batchId,
|
|
3371
|
+
workerAgentId,
|
|
3372
|
+
);
|
|
2881
3373
|
if (pendingExpansionFiles.length > 0) {
|
|
2882
3374
|
const parsedRequests = parseSegmentExpansionRequests(pendingExpansionFiles);
|
|
2883
3375
|
for (const malformed of parsedRequests.malformed) {
|
|
2884
3376
|
const renamed = markSegmentExpansionRequestFile(malformed.filePath, "invalid");
|
|
2885
|
-
execLog(
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
3377
|
+
execLog(
|
|
3378
|
+
"batch",
|
|
3379
|
+
batchState.batchId,
|
|
3380
|
+
`segment expansion request malformed (${renamed ? "renamed to .invalid" : "rename failed"})`,
|
|
3381
|
+
{
|
|
3382
|
+
taskId,
|
|
3383
|
+
agentId: workerAgentId,
|
|
3384
|
+
segmentId: activeSegmentId,
|
|
3385
|
+
filePath: malformed.filePath,
|
|
3386
|
+
reason: malformed.reason,
|
|
3387
|
+
},
|
|
3388
|
+
);
|
|
2892
3389
|
}
|
|
2893
|
-
const orderedRequests = [...parsedRequests.valid].sort((a, b) =>
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
3390
|
+
const orderedRequests = [...parsedRequests.valid].sort((a, b) =>
|
|
3391
|
+
a.request.requestId.localeCompare(b.request.requestId),
|
|
3392
|
+
);
|
|
3393
|
+
const scopedRequests = orderedRequests.filter(
|
|
3394
|
+
(pendingRequest) =>
|
|
3395
|
+
pendingRequest.request.taskId === taskId &&
|
|
3396
|
+
pendingRequest.request.fromSegmentId === activeSegmentId,
|
|
3397
|
+
);
|
|
2898
3398
|
let rejectedCount = 0;
|
|
2899
3399
|
let acceptedCount = 0;
|
|
2900
3400
|
for (const pendingRequest of scopedRequests) {
|
|
@@ -2912,11 +3412,26 @@ export async function executeOrchBatch(
|
|
|
2912
3412
|
if (!processingResult.ok) {
|
|
2913
3413
|
rejectedCount += 1;
|
|
2914
3414
|
processedSegmentExpansionRequestIds.add(requestId);
|
|
2915
|
-
const recordedRequestId = recordProcessedSegmentExpansionRequestId(
|
|
3415
|
+
const recordedRequestId = recordProcessedSegmentExpansionRequestId(
|
|
3416
|
+
batchState,
|
|
3417
|
+
requestId,
|
|
3418
|
+
"failed",
|
|
3419
|
+
);
|
|
2916
3420
|
if (recordedRequestId) {
|
|
2917
|
-
persistRuntimeState(
|
|
3421
|
+
persistRuntimeState(
|
|
3422
|
+
"segment-expansion-rejected",
|
|
3423
|
+
batchState,
|
|
3424
|
+
wavePlan,
|
|
3425
|
+
latestAllocatedLanes,
|
|
3426
|
+
allTaskOutcomes,
|
|
3427
|
+
discoveryRef,
|
|
3428
|
+
stateRoot,
|
|
3429
|
+
);
|
|
2918
3430
|
}
|
|
2919
|
-
const renamedRejected = markSegmentExpansionRequestFile(
|
|
3431
|
+
const renamedRejected = markSegmentExpansionRequestFile(
|
|
3432
|
+
pendingRequest.filePath,
|
|
3433
|
+
"rejected",
|
|
3434
|
+
);
|
|
2920
3435
|
emitAlert({
|
|
2921
3436
|
category: "segment-expansion-rejected",
|
|
2922
3437
|
summary:
|
|
@@ -2957,7 +3472,11 @@ export async function executeOrchBatch(
|
|
|
2957
3472
|
requestId,
|
|
2958
3473
|
batchState.orchBranch,
|
|
2959
3474
|
);
|
|
2960
|
-
const recordedRequestId = recordProcessedSegmentExpansionRequestId(
|
|
3475
|
+
const recordedRequestId = recordProcessedSegmentExpansionRequestId(
|
|
3476
|
+
batchState,
|
|
3477
|
+
requestId,
|
|
3478
|
+
"succeeded",
|
|
3479
|
+
);
|
|
2961
3480
|
|
|
2962
3481
|
// TP-145 hardening: if .DONE was prematurely created by the
|
|
2963
3482
|
// completing segment (because it was the last segment at that
|
|
@@ -2973,11 +3492,11 @@ export async function executeOrchBatch(
|
|
|
2973
3492
|
const lane = laneByTaskId.get(taskId);
|
|
2974
3493
|
const doneDir = lane
|
|
2975
3494
|
? resolveCanonicalTaskPaths(
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
3495
|
+
task.taskFolder,
|
|
3496
|
+
lane.worktreePath,
|
|
3497
|
+
repoRoot,
|
|
3498
|
+
!!workspaceConfig,
|
|
3499
|
+
).taskFolderResolved
|
|
2981
3500
|
: task.packetTaskPath || task.taskFolder;
|
|
2982
3501
|
if (doneDir) {
|
|
2983
3502
|
const donePath = join(doneDir, ".DONE");
|
|
@@ -2985,17 +3504,36 @@ export async function executeOrchBatch(
|
|
|
2985
3504
|
try {
|
|
2986
3505
|
unlinkSync(donePath);
|
|
2987
3506
|
execLog("batch", batchState.batchId, "removed premature .DONE after segment expansion", {
|
|
2988
|
-
taskId,
|
|
3507
|
+
taskId,
|
|
3508
|
+
donePath,
|
|
3509
|
+
requestId,
|
|
2989
3510
|
});
|
|
2990
|
-
} catch {
|
|
3511
|
+
} catch {
|
|
3512
|
+
/* non-fatal */
|
|
3513
|
+
}
|
|
2991
3514
|
}
|
|
2992
3515
|
}
|
|
2993
3516
|
}
|
|
2994
3517
|
|
|
2995
|
-
if (
|
|
2996
|
-
|
|
3518
|
+
if (
|
|
3519
|
+
persistedInsertedSegments ||
|
|
3520
|
+
recordedRequestId ||
|
|
3521
|
+
mutation.insertedSegmentIds.length > 0
|
|
3522
|
+
) {
|
|
3523
|
+
persistRuntimeState(
|
|
3524
|
+
"segment-expansion-approved",
|
|
3525
|
+
batchState,
|
|
3526
|
+
wavePlan,
|
|
3527
|
+
latestAllocatedLanes,
|
|
3528
|
+
allTaskOutcomes,
|
|
3529
|
+
discoveryRef,
|
|
3530
|
+
stateRoot,
|
|
3531
|
+
);
|
|
2997
3532
|
}
|
|
2998
|
-
const renamedProcessed = markSegmentExpansionRequestFile(
|
|
3533
|
+
const renamedProcessed = markSegmentExpansionRequestFile(
|
|
3534
|
+
pendingRequest.filePath,
|
|
3535
|
+
"processed",
|
|
3536
|
+
);
|
|
2999
3537
|
emitAlert({
|
|
3000
3538
|
category: "segment-expansion-approved",
|
|
3001
3539
|
summary:
|
|
@@ -3016,17 +3554,22 @@ export async function executeOrchBatch(
|
|
|
3016
3554
|
});
|
|
3017
3555
|
acceptedCount += 1;
|
|
3018
3556
|
}
|
|
3019
|
-
execLog(
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3557
|
+
execLog(
|
|
3558
|
+
"batch",
|
|
3559
|
+
batchState.batchId,
|
|
3560
|
+
`segment ${activeSegmentId} completed with ${pendingExpansionFiles.length} pending expansion request(s)`,
|
|
3561
|
+
{
|
|
3562
|
+
taskId,
|
|
3563
|
+
agentId: workerAgentId,
|
|
3564
|
+
segmentId: activeSegmentId,
|
|
3565
|
+
acceptedCount,
|
|
3566
|
+
rejectedCount,
|
|
3567
|
+
validRequests: parsedRequests.valid.length,
|
|
3568
|
+
scopedRequests: scopedRequests.length,
|
|
3569
|
+
ignoredRequests: orderedRequests.length - scopedRequests.length,
|
|
3570
|
+
malformedRequests: parsedRequests.malformed.length,
|
|
3571
|
+
},
|
|
3572
|
+
);
|
|
3030
3573
|
}
|
|
3031
3574
|
}
|
|
3032
3575
|
}
|
|
@@ -3042,17 +3585,26 @@ export async function executeOrchBatch(
|
|
|
3042
3585
|
}
|
|
3043
3586
|
}
|
|
3044
3587
|
if (continuationTaskIds.size > 0) {
|
|
3045
|
-
const continuationWave = scheduleContinuationSegmentRound(
|
|
3588
|
+
const continuationWave = scheduleContinuationSegmentRound(
|
|
3589
|
+
runtimeSegmentRounds,
|
|
3590
|
+
waveIdx,
|
|
3591
|
+
continuationTaskIds,
|
|
3592
|
+
);
|
|
3046
3593
|
// TP-166: Maintain roundToTaskWave mapping for the inserted continuation round.
|
|
3047
3594
|
// The continuation belongs to the same task-level wave as the current round.
|
|
3048
3595
|
const parentTaskWave = roundToTaskWave[waveIdx] ?? 0;
|
|
3049
3596
|
roundToTaskWave.splice(waveIdx + 1, 0, parentTaskWave);
|
|
3050
3597
|
batchState.roundToTaskWave = [...roundToTaskWave];
|
|
3051
|
-
execLog(
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3598
|
+
execLog(
|
|
3599
|
+
"batch",
|
|
3600
|
+
batchState.batchId,
|
|
3601
|
+
"scheduled continuation segment round for expanded task frontier",
|
|
3602
|
+
{
|
|
3603
|
+
waveIndex: waveIdx,
|
|
3604
|
+
taskIds: continuationWave.join(","),
|
|
3605
|
+
runtimeSegmentRoundCount: runtimeSegmentRounds.length,
|
|
3606
|
+
},
|
|
3607
|
+
);
|
|
3056
3608
|
}
|
|
3057
3609
|
|
|
3058
3610
|
for (const taskId of waveResult.failedTaskIds) {
|
|
@@ -3063,11 +3615,28 @@ export async function executeOrchBatch(
|
|
|
3063
3615
|
const activeSegmentId = failOutcome?.segmentId ?? task.activeSegmentId;
|
|
3064
3616
|
if (activeSegmentId) {
|
|
3065
3617
|
segmentState.statusBySegmentId.set(activeSegmentId, "failed");
|
|
3066
|
-
upsertTerminalSegmentRecord(
|
|
3618
|
+
upsertTerminalSegmentRecord(
|
|
3619
|
+
batchState,
|
|
3620
|
+
task,
|
|
3621
|
+
segmentState,
|
|
3622
|
+
activeSegmentId,
|
|
3623
|
+
"failed",
|
|
3624
|
+
failOutcome,
|
|
3625
|
+
laneByTaskId.get(taskId),
|
|
3626
|
+
);
|
|
3067
3627
|
|
|
3068
|
-
const workerAgentId = resolveTaskWorkerAgentId(
|
|
3628
|
+
const workerAgentId = resolveTaskWorkerAgentId(
|
|
3629
|
+
taskId,
|
|
3630
|
+
allTaskOutcomes,
|
|
3631
|
+
laneByTaskId,
|
|
3632
|
+
agentIdPrefix,
|
|
3633
|
+
);
|
|
3069
3634
|
if (workerAgentId) {
|
|
3070
|
-
const pendingExpansionFiles = listPendingSegmentExpansionRequestFiles(
|
|
3635
|
+
const pendingExpansionFiles = listPendingSegmentExpansionRequestFiles(
|
|
3636
|
+
stateRoot,
|
|
3637
|
+
batchState.batchId,
|
|
3638
|
+
workerAgentId,
|
|
3639
|
+
);
|
|
3071
3640
|
if (pendingExpansionFiles.length > 0) {
|
|
3072
3641
|
const parsedRequests = parseSegmentExpansionRequests(pendingExpansionFiles);
|
|
3073
3642
|
for (const malformed of parsedRequests.malformed) {
|
|
@@ -3077,12 +3646,27 @@ export async function executeOrchBatch(
|
|
|
3077
3646
|
let discardedCount = 0;
|
|
3078
3647
|
let ignoredCount = 0;
|
|
3079
3648
|
for (const requestFile of parsedRequests.valid) {
|
|
3080
|
-
if (
|
|
3649
|
+
if (
|
|
3650
|
+
requestFile.request.taskId === taskId &&
|
|
3651
|
+
requestFile.request.fromSegmentId === activeSegmentId
|
|
3652
|
+
) {
|
|
3081
3653
|
const requestId = requestFile.request.requestId;
|
|
3082
3654
|
processedSegmentExpansionRequestIds.add(requestId);
|
|
3083
|
-
const recordedRequestId = recordProcessedSegmentExpansionRequestId(
|
|
3655
|
+
const recordedRequestId = recordProcessedSegmentExpansionRequestId(
|
|
3656
|
+
batchState,
|
|
3657
|
+
requestId,
|
|
3658
|
+
"skipped",
|
|
3659
|
+
);
|
|
3084
3660
|
if (recordedRequestId) {
|
|
3085
|
-
persistRuntimeState(
|
|
3661
|
+
persistRuntimeState(
|
|
3662
|
+
"segment-expansion-discarded",
|
|
3663
|
+
batchState,
|
|
3664
|
+
wavePlan,
|
|
3665
|
+
latestAllocatedLanes,
|
|
3666
|
+
allTaskOutcomes,
|
|
3667
|
+
discoveryRef,
|
|
3668
|
+
stateRoot,
|
|
3669
|
+
);
|
|
3086
3670
|
}
|
|
3087
3671
|
if (markSegmentExpansionRequestFile(requestFile.filePath, "discarded")) {
|
|
3088
3672
|
discardedCount += 1;
|
|
@@ -3091,14 +3675,19 @@ export async function executeOrchBatch(
|
|
|
3091
3675
|
}
|
|
3092
3676
|
ignoredCount += 1;
|
|
3093
3677
|
}
|
|
3094
|
-
execLog(
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3678
|
+
execLog(
|
|
3679
|
+
"batch",
|
|
3680
|
+
batchState.batchId,
|
|
3681
|
+
`segment ${activeSegmentId} failed with ${pendingExpansionFiles.length} pending expansion request(s)`,
|
|
3682
|
+
{
|
|
3683
|
+
taskId,
|
|
3684
|
+
agentId: workerAgentId,
|
|
3685
|
+
segmentId: activeSegmentId,
|
|
3686
|
+
discardedCount,
|
|
3687
|
+
ignoredCount,
|
|
3688
|
+
malformedCount: parsedRequests.malformed.length,
|
|
3689
|
+
},
|
|
3690
|
+
);
|
|
3102
3691
|
if (discardedCount > 0) {
|
|
3103
3692
|
emitAlert({
|
|
3104
3693
|
category: "segment-expansion-rejected",
|
|
@@ -3133,7 +3722,15 @@ export async function executeOrchBatch(
|
|
|
3133
3722
|
if (activeSegmentId) {
|
|
3134
3723
|
segmentState.statusBySegmentId.set(activeSegmentId, "skipped");
|
|
3135
3724
|
const outcome = allTaskOutcomes.find((candidate) => candidate.taskId === taskId);
|
|
3136
|
-
upsertTerminalSegmentRecord(
|
|
3725
|
+
upsertTerminalSegmentRecord(
|
|
3726
|
+
batchState,
|
|
3727
|
+
task,
|
|
3728
|
+
segmentState,
|
|
3729
|
+
activeSegmentId,
|
|
3730
|
+
"skipped",
|
|
3731
|
+
outcome,
|
|
3732
|
+
laneByTaskId.get(taskId),
|
|
3733
|
+
);
|
|
3137
3734
|
}
|
|
3138
3735
|
task.activeSegmentId = null;
|
|
3139
3736
|
segmentState.terminalStatus = "skipped";
|
|
@@ -3159,31 +3756,37 @@ export async function executeOrchBatch(
|
|
|
3159
3756
|
// ── TP-040: Emit task_complete / task_failed events ──────
|
|
3160
3757
|
// Emitted after Tier 0 retry so events reflect final status.
|
|
3161
3758
|
for (const taskId of waveResult.succeededTaskIds) {
|
|
3162
|
-
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
3163
|
-
emitEvent(
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
:
|
|
3169
|
-
|
|
3170
|
-
|
|
3759
|
+
const outcome = allTaskOutcomes.find((o) => o.taskId === taskId);
|
|
3760
|
+
emitEvent(
|
|
3761
|
+
stateRoot,
|
|
3762
|
+
{
|
|
3763
|
+
...buildEngineEventBase("task_complete", batchState.batchId, waveIdx, batchState.phase),
|
|
3764
|
+
taskId,
|
|
3765
|
+
durationMs:
|
|
3766
|
+
outcome?.startTime && outcome?.endTime ? outcome.endTime - outcome.startTime : undefined,
|
|
3767
|
+
outcome: "succeeded",
|
|
3768
|
+
},
|
|
3769
|
+
onEngineEvent,
|
|
3770
|
+
);
|
|
3171
3771
|
}
|
|
3172
3772
|
for (const taskId of waveResult.failedTaskIds) {
|
|
3173
|
-
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
3174
|
-
emitEvent(
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
:
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3773
|
+
const outcome = allTaskOutcomes.find((o) => o.taskId === taskId);
|
|
3774
|
+
emitEvent(
|
|
3775
|
+
stateRoot,
|
|
3776
|
+
{
|
|
3777
|
+
...buildEngineEventBase("task_failed", batchState.batchId, waveIdx, batchState.phase),
|
|
3778
|
+
taskId,
|
|
3779
|
+
durationMs:
|
|
3780
|
+
outcome?.startTime && outcome?.endTime ? outcome.endTime - outcome.startTime : undefined,
|
|
3781
|
+
reason: outcome?.exitReason || "unknown",
|
|
3782
|
+
partialProgress: (outcome?.partialProgressCommits ?? 0) > 0,
|
|
3783
|
+
},
|
|
3784
|
+
onEngineEvent,
|
|
3785
|
+
);
|
|
3183
3786
|
|
|
3184
3787
|
// ── TP-076: Emit supervisor alert for task failure ──────
|
|
3185
|
-
const laneForTask = latestAllocatedLanes.find(l => l.tasks.some(t => t.taskId === taskId));
|
|
3186
|
-
const allocatedTask = laneForTask?.tasks.find(t => t.taskId === taskId)?.task;
|
|
3788
|
+
const laneForTask = latestAllocatedLanes.find((l) => l.tasks.some((t) => t.taskId === taskId));
|
|
3789
|
+
const allocatedTask = laneForTask?.tasks.find((t) => t.taskId === taskId)?.task;
|
|
3187
3790
|
const exitReason = outcome?.exitReason || "unknown";
|
|
3188
3791
|
const hasPartialProgress = (outcome?.partialProgressCommits ?? 0) > 0;
|
|
3189
3792
|
const segmentFrontier = buildSupervisorSegmentFrontierSnapshot(
|
|
@@ -3193,12 +3796,14 @@ export async function executeOrchBatch(
|
|
|
3193
3796
|
batchState.segments,
|
|
3194
3797
|
outcome?.segmentId,
|
|
3195
3798
|
);
|
|
3196
|
-
const segmentId =
|
|
3197
|
-
??
|
|
3198
|
-
|
|
3199
|
-
??
|
|
3799
|
+
const segmentId =
|
|
3800
|
+
outcome?.segmentId ??
|
|
3801
|
+
allocatedTask?.activeSegmentId ??
|
|
3802
|
+
segmentFrontier?.activeSegmentId ??
|
|
3803
|
+
undefined;
|
|
3200
3804
|
const repoId = segmentId
|
|
3201
|
-
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ??
|
|
3805
|
+
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ??
|
|
3806
|
+
laneForTask?.repoId)
|
|
3202
3807
|
: laneForTask?.repoId;
|
|
3203
3808
|
const segmentSummary = segmentId
|
|
3204
3809
|
? ` Segment: ${segmentId}${repoId ? ` (repo: ${repoId})` : ""}\n`
|
|
@@ -3247,15 +3852,22 @@ export async function executeOrchBatch(
|
|
|
3247
3852
|
// later, then emit lane-terminated so the supervisor process
|
|
3248
3853
|
// suppresses any in-transit zombie alerts targeting this lane/agent.
|
|
3249
3854
|
if (laneForTask) {
|
|
3250
|
-
const hardFailAgentId =
|
|
3251
|
-
|
|
3252
|
-
|
|
3855
|
+
const hardFailAgentId =
|
|
3856
|
+
outcome?.sessionName && outcome.sessionName.length > 0
|
|
3857
|
+
? outcome.sessionName
|
|
3858
|
+
: `${laneForTask.laneSessionId}-worker`;
|
|
3253
3859
|
try {
|
|
3254
3860
|
const drained = drainAgentOutbox(stateRoot, batchState.batchId, hardFailAgentId);
|
|
3255
3861
|
if (drained > 0) {
|
|
3256
|
-
execLog(
|
|
3862
|
+
execLog(
|
|
3863
|
+
"batch",
|
|
3864
|
+
batchState.batchId,
|
|
3865
|
+
`hard-fail outbox drain: ${drained} entr${drained === 1 ? "y" : "ies"} for ${hardFailAgentId}`,
|
|
3866
|
+
);
|
|
3257
3867
|
}
|
|
3258
|
-
} catch {
|
|
3868
|
+
} catch {
|
|
3869
|
+
/* best effort — do not block termination */
|
|
3870
|
+
}
|
|
3259
3871
|
emitLaneTerminated({
|
|
3260
3872
|
laneNumber: laneForTask.laneNumber,
|
|
3261
3873
|
agentId: hardFailAgentId,
|
|
@@ -3281,25 +3893,50 @@ export async function executeOrchBatch(
|
|
|
3281
3893
|
const allFailedAreSpawnFailures = isAllLanesSpawnFailedWave(waveResult, allTaskOutcomes);
|
|
3282
3894
|
if (allFailedAreSpawnFailures) {
|
|
3283
3895
|
batchState.phase = "failed";
|
|
3284
|
-
execLog(
|
|
3896
|
+
execLog(
|
|
3897
|
+
"batch",
|
|
3898
|
+
batchState.batchId,
|
|
3285
3899
|
`phase → failed: every lane in wave ${waveIdx + 1} hit spawn_failure (TP-190 #561)`,
|
|
3286
3900
|
{ failedTasks: waveResult.failedTaskIds.join(",") },
|
|
3287
3901
|
);
|
|
3288
3902
|
onNotify(
|
|
3289
|
-
ORCH_MESSAGES.orchBatchFailed(
|
|
3903
|
+
ORCH_MESSAGES.orchBatchFailed(
|
|
3904
|
+
batchState.batchId,
|
|
3905
|
+
`all lanes in wave ${waveIdx + 1} failed to spawn (Runtime V2 spawn-failure — see task-failure alerts above)`,
|
|
3906
|
+
),
|
|
3290
3907
|
"error",
|
|
3291
3908
|
);
|
|
3292
|
-
persistRuntimeState(
|
|
3909
|
+
persistRuntimeState(
|
|
3910
|
+
"wave-spawn-failure",
|
|
3911
|
+
batchState,
|
|
3912
|
+
wavePlan,
|
|
3913
|
+
latestAllocatedLanes,
|
|
3914
|
+
allTaskOutcomes,
|
|
3915
|
+
discoveryRef,
|
|
3916
|
+
stateRoot,
|
|
3917
|
+
);
|
|
3293
3918
|
emitTerminalEvent(`All-lane spawn failure at wave ${waveIdx + 1}`);
|
|
3294
3919
|
break;
|
|
3295
3920
|
}
|
|
3296
3921
|
|
|
3297
3922
|
// ── TS-009: Persist state after wave execution ──
|
|
3298
|
-
persistRuntimeState(
|
|
3923
|
+
persistRuntimeState(
|
|
3924
|
+
"wave-execution-complete",
|
|
3925
|
+
batchState,
|
|
3926
|
+
wavePlan,
|
|
3927
|
+
latestAllocatedLanes,
|
|
3928
|
+
allTaskOutcomes,
|
|
3929
|
+
discoveryRef,
|
|
3930
|
+
stateRoot,
|
|
3931
|
+
);
|
|
3299
3932
|
|
|
3300
3933
|
const elapsedSec = Math.round((waveResult.endedAt - waveResult.startedAt) / 1000);
|
|
3301
3934
|
{
|
|
3302
|
-
const { displayWave: completeDisplayWave } = resolveDisplayWaveNumber(
|
|
3935
|
+
const { displayWave: completeDisplayWave } = resolveDisplayWaveNumber(
|
|
3936
|
+
waveIdx,
|
|
3937
|
+
roundToTaskWave,
|
|
3938
|
+
taskLevelWaveCount,
|
|
3939
|
+
);
|
|
3303
3940
|
onNotify(
|
|
3304
3941
|
ORCH_MESSAGES.orchWaveComplete(
|
|
3305
3942
|
completeDisplayWave,
|
|
@@ -3321,7 +3958,15 @@ export async function executeOrchBatch(
|
|
|
3321
3958
|
if (waveResult.policyApplied === "stop-all") {
|
|
3322
3959
|
batchState.phase = "stopped";
|
|
3323
3960
|
// ── TS-009: Persist state on stop-all ──
|
|
3324
|
-
persistRuntimeState(
|
|
3961
|
+
persistRuntimeState(
|
|
3962
|
+
"stop-all",
|
|
3963
|
+
batchState,
|
|
3964
|
+
wavePlan,
|
|
3965
|
+
latestAllocatedLanes,
|
|
3966
|
+
allTaskOutcomes,
|
|
3967
|
+
discoveryRef,
|
|
3968
|
+
stateRoot,
|
|
3969
|
+
);
|
|
3325
3970
|
onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-all"), "error");
|
|
3326
3971
|
// TP-040: Emit batch_paused event (via terminal helper for dedup)
|
|
3327
3972
|
emitTerminalEvent(`Stopped by stop-all policy at wave ${waveIdx + 1}`);
|
|
@@ -3330,7 +3975,15 @@ export async function executeOrchBatch(
|
|
|
3330
3975
|
if (waveResult.policyApplied === "stop-wave") {
|
|
3331
3976
|
batchState.phase = "stopped";
|
|
3332
3977
|
// ── TS-009: Persist state on stop-wave ──
|
|
3333
|
-
persistRuntimeState(
|
|
3978
|
+
persistRuntimeState(
|
|
3979
|
+
"stop-wave",
|
|
3980
|
+
batchState,
|
|
3981
|
+
wavePlan,
|
|
3982
|
+
latestAllocatedLanes,
|
|
3983
|
+
allTaskOutcomes,
|
|
3984
|
+
discoveryRef,
|
|
3985
|
+
stateRoot,
|
|
3986
|
+
);
|
|
3334
3987
|
onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-wave"), "error");
|
|
3335
3988
|
// TP-040: Emit batch_paused event (via terminal helper for dedup)
|
|
3336
3989
|
emitTerminalEvent(`Stopped by stop-wave policy at wave ${waveIdx + 1}`);
|
|
@@ -3348,11 +4001,9 @@ export async function executeOrchBatch(
|
|
|
3348
4001
|
for (const lr of waveResult.laneResults) {
|
|
3349
4002
|
laneOutcomeByNumber.set(lr.laneNumber, lr);
|
|
3350
4003
|
}
|
|
3351
|
-
const mixedOutcomeLanes = waveResult.laneResults.filter(lr => {
|
|
3352
|
-
const hasSucceeded = lr.tasks.some(t => t.status === "succeeded");
|
|
3353
|
-
const hasHardFailure = lr.tasks.some(
|
|
3354
|
-
t => t.status === "failed" || t.status === "stalled",
|
|
3355
|
-
);
|
|
4004
|
+
const mixedOutcomeLanes = waveResult.laneResults.filter((lr) => {
|
|
4005
|
+
const hasSucceeded = lr.tasks.some((t) => t.status === "succeeded");
|
|
4006
|
+
const hasHardFailure = lr.tasks.some((t) => t.status === "failed" || t.status === "stalled");
|
|
3356
4007
|
return hasSucceeded && hasHardFailure;
|
|
3357
4008
|
});
|
|
3358
4009
|
|
|
@@ -3367,44 +4018,55 @@ export async function executeOrchBatch(
|
|
|
3367
4018
|
if (!lane.worktreePath || !existsSync(lane.worktreePath)) continue;
|
|
3368
4019
|
const laneOutcome = laneOutcomeByNumber.get(lane.laneNumber);
|
|
3369
4020
|
if (!laneOutcome) continue;
|
|
3370
|
-
const hasSucceeded = laneOutcome.tasks.some(t => t.status === "succeeded");
|
|
3371
|
-
const hasSkipped = laneOutcome.tasks.some(t => t.status === "skipped");
|
|
4021
|
+
const hasSucceeded = laneOutcome.tasks.some((t) => t.status === "succeeded");
|
|
4022
|
+
const hasSkipped = laneOutcome.tasks.some((t) => t.status === "skipped");
|
|
3372
4023
|
// Auto-commit merge candidates (succeeded) and skipped-task lanes
|
|
3373
4024
|
if (!hasSucceeded && !hasSkipped) continue;
|
|
3374
4025
|
try {
|
|
3375
4026
|
const addResult = runGit(["add", "-A"], lane.worktreePath);
|
|
3376
4027
|
if (!addResult.ok) {
|
|
3377
|
-
execLog("merge", batchState.batchId, `safety-net: git add failed in ${lane.laneId}`, {
|
|
4028
|
+
execLog("merge", batchState.batchId, `safety-net: git add failed in ${lane.laneId}`, {
|
|
4029
|
+
stderr: addResult.stderr,
|
|
4030
|
+
});
|
|
3378
4031
|
continue;
|
|
3379
4032
|
}
|
|
3380
4033
|
const statusResult = runGit(["status", "--porcelain"], lane.worktreePath);
|
|
3381
4034
|
if (!statusResult.ok || !statusResult.stdout?.trim()) continue;
|
|
3382
|
-
const taskIds = lane.tasks.map(t => t.taskId).join(", ");
|
|
4035
|
+
const taskIds = lane.tasks.map((t) => t.taskId).join(", ");
|
|
3383
4036
|
const commitResult = runGit(
|
|
3384
4037
|
["commit", "-m", `safety-net: uncommitted artifacts for ${taskIds}`],
|
|
3385
4038
|
lane.worktreePath,
|
|
3386
4039
|
);
|
|
3387
4040
|
if (commitResult.ok) {
|
|
3388
|
-
execLog(
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
|
|
3392
|
-
|
|
4041
|
+
execLog(
|
|
4042
|
+
"merge",
|
|
4043
|
+
batchState.batchId,
|
|
4044
|
+
`safety-net: auto-committed uncommitted files in ${lane.laneId}`,
|
|
4045
|
+
{
|
|
4046
|
+
worktree: lane.worktreePath,
|
|
4047
|
+
taskIds,
|
|
4048
|
+
files: statusResult.stdout.trim(),
|
|
4049
|
+
},
|
|
4050
|
+
);
|
|
3393
4051
|
} else {
|
|
3394
|
-
execLog("merge", batchState.batchId, `safety-net: commit failed in ${lane.laneId}`, {
|
|
4052
|
+
execLog("merge", batchState.batchId, `safety-net: commit failed in ${lane.laneId}`, {
|
|
4053
|
+
stderr: commitResult.stderr,
|
|
4054
|
+
});
|
|
3395
4055
|
}
|
|
3396
4056
|
} catch (err: any) {
|
|
3397
|
-
execLog("merge", batchState.batchId, `safety-net: unexpected error in ${lane.laneId}`, {
|
|
4057
|
+
execLog("merge", batchState.batchId, `safety-net: unexpected error in ${lane.laneId}`, {
|
|
4058
|
+
error: err?.message,
|
|
4059
|
+
});
|
|
3398
4060
|
}
|
|
3399
4061
|
}
|
|
3400
4062
|
|
|
3401
4063
|
if (succeededSegmentTaskIdsForMerge.length > 0) {
|
|
3402
|
-
const mergeableLaneCount = waveResult.allocatedLanes.filter(lane => {
|
|
4064
|
+
const mergeableLaneCount = waveResult.allocatedLanes.filter((lane) => {
|
|
3403
4065
|
const outcome = laneOutcomeByNumber.get(lane.laneNumber);
|
|
3404
4066
|
if (!outcome) return false;
|
|
3405
|
-
const hasSucceeded = outcome.tasks.some(t => t.status === "succeeded");
|
|
4067
|
+
const hasSucceeded = outcome.tasks.some((t) => t.status === "succeeded");
|
|
3406
4068
|
const hasHardFailure = outcome.tasks.some(
|
|
3407
|
-
t => t.status === "failed" || t.status === "stalled",
|
|
4069
|
+
(t) => t.status === "failed" || t.status === "stalled",
|
|
3408
4070
|
);
|
|
3409
4071
|
return hasSucceeded && !hasHardFailure;
|
|
3410
4072
|
}).length;
|
|
@@ -3412,13 +4074,31 @@ export async function executeOrchBatch(
|
|
|
3412
4074
|
if (mergeableLaneCount > 0) {
|
|
3413
4075
|
batchState.phase = "merging";
|
|
3414
4076
|
// ── TS-009: Persist state on executing→merging transition ──
|
|
3415
|
-
persistRuntimeState(
|
|
3416
|
-
|
|
4077
|
+
persistRuntimeState(
|
|
4078
|
+
"merge-start",
|
|
4079
|
+
batchState,
|
|
4080
|
+
wavePlan,
|
|
4081
|
+
latestAllocatedLanes,
|
|
4082
|
+
allTaskOutcomes,
|
|
4083
|
+
discoveryRef,
|
|
4084
|
+
stateRoot,
|
|
4085
|
+
);
|
|
4086
|
+
onNotify(
|
|
4087
|
+
ORCH_MESSAGES.orchMergeStart(
|
|
4088
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
4089
|
+
mergeableLaneCount,
|
|
4090
|
+
),
|
|
4091
|
+
"info",
|
|
4092
|
+
);
|
|
3417
4093
|
// TP-040: Emit merge_start event
|
|
3418
|
-
emitEvent(
|
|
3419
|
-
|
|
3420
|
-
|
|
3421
|
-
|
|
4094
|
+
emitEvent(
|
|
4095
|
+
stateRoot,
|
|
4096
|
+
{
|
|
4097
|
+
...buildEngineEventBase("merge_start", batchState.batchId, waveIdx, batchState.phase),
|
|
4098
|
+
laneCount: mergeableLaneCount,
|
|
4099
|
+
},
|
|
4100
|
+
onEngineEvent,
|
|
4101
|
+
);
|
|
3422
4102
|
|
|
3423
4103
|
// TP-056: Start merge health monitor during merge phase
|
|
3424
4104
|
const mergeHealthMonitor = new MergeHealthMonitor({
|
|
@@ -3461,7 +4141,15 @@ export async function executeOrchBatch(
|
|
|
3461
4141
|
batchState.mergeResults.push(mergeResult);
|
|
3462
4142
|
|
|
3463
4143
|
// Persist state after merge so dashboard shows wave merge results
|
|
3464
|
-
persistRuntimeState(
|
|
4144
|
+
persistRuntimeState(
|
|
4145
|
+
"merge-complete",
|
|
4146
|
+
batchState,
|
|
4147
|
+
wavePlan,
|
|
4148
|
+
latestAllocatedLanes,
|
|
4149
|
+
allTaskOutcomes,
|
|
4150
|
+
discoveryRef,
|
|
4151
|
+
stateRoot,
|
|
4152
|
+
);
|
|
3465
4153
|
|
|
3466
4154
|
// Emit per-lane merge notifications
|
|
3467
4155
|
for (const lr of mergeResult.laneResults) {
|
|
@@ -3471,10 +4159,23 @@ export async function executeOrchBatch(
|
|
|
3471
4159
|
if (lr.error) {
|
|
3472
4160
|
onNotify(ORCH_MESSAGES.orchMergeLaneFailed(lr.laneNumber, lr.error), "error");
|
|
3473
4161
|
} else if (lr.result?.status === "SUCCESS") {
|
|
3474
|
-
onNotify(
|
|
4162
|
+
onNotify(
|
|
4163
|
+
ORCH_MESSAGES.orchMergeLaneSuccess(lr.laneNumber, lr.result.merge_commit, durationSec),
|
|
4164
|
+
"info",
|
|
4165
|
+
);
|
|
3475
4166
|
} else if (lr.result?.status === "CONFLICT_RESOLVED") {
|
|
3476
|
-
onNotify(
|
|
3477
|
-
|
|
4167
|
+
onNotify(
|
|
4168
|
+
ORCH_MESSAGES.orchMergeLaneConflictResolved(
|
|
4169
|
+
lr.laneNumber,
|
|
4170
|
+
lr.result.conflicts.length,
|
|
4171
|
+
durationSec,
|
|
4172
|
+
),
|
|
4173
|
+
"info",
|
|
4174
|
+
);
|
|
4175
|
+
} else if (
|
|
4176
|
+
lr.result?.status === "CONFLICT_UNRESOLVED" ||
|
|
4177
|
+
lr.result?.status === "BUILD_FAILURE"
|
|
4178
|
+
) {
|
|
3478
4179
|
onNotify(ORCH_MESSAGES.orchMergeLaneFailed(lr.laneNumber, lr.result.status), "error");
|
|
3479
4180
|
}
|
|
3480
4181
|
}
|
|
@@ -3482,13 +4183,18 @@ export async function executeOrchBatch(
|
|
|
3482
4183
|
// If any lane has mixed outcomes, do not silently discard succeeded work.
|
|
3483
4184
|
// Force merge failure handling so state is preserved for manual resolution.
|
|
3484
4185
|
if (mixedOutcomeLanes.length > 0) {
|
|
3485
|
-
const mixedIds = mixedOutcomeLanes.map(l => `lane-${l.laneNumber}`).join(", ");
|
|
4186
|
+
const mixedIds = mixedOutcomeLanes.map((l) => `lane-${l.laneNumber}`).join(", ");
|
|
3486
4187
|
const failureReason =
|
|
3487
4188
|
`Lane(s) ${mixedIds} contain both succeeded and failed tasks. ` +
|
|
3488
4189
|
`Automatic partial-branch merge is disabled to avoid dropping succeeded commits.`;
|
|
3489
|
-
execLog(
|
|
3490
|
-
|
|
3491
|
-
|
|
4190
|
+
execLog(
|
|
4191
|
+
"merge",
|
|
4192
|
+
`W${waveIdx + 1}`,
|
|
4193
|
+
"mixed-outcome lanes detected — escalating to merge failure handling",
|
|
4194
|
+
{
|
|
4195
|
+
mixedLaneIds: mixedIds,
|
|
4196
|
+
},
|
|
4197
|
+
);
|
|
3492
4198
|
mergeResult = {
|
|
3493
4199
|
...mergeResult,
|
|
3494
4200
|
status: "partial",
|
|
@@ -3503,33 +4209,53 @@ export async function executeOrchBatch(
|
|
|
3503
4209
|
// Emit overall merge result notification
|
|
3504
4210
|
// TP-032 R006-3: Exclude verification_new_failure lanes from success count
|
|
3505
4211
|
const mergedCount = mergeResult.laneResults.filter(
|
|
3506
|
-
r =>
|
|
4212
|
+
(r) =>
|
|
4213
|
+
!r.error && (r.result?.status === "SUCCESS" || r.result?.status === "CONFLICT_RESOLVED"),
|
|
3507
4214
|
).length;
|
|
3508
4215
|
const mergeTotalSec = Math.round(mergeResult.totalDurationMs / 1000);
|
|
3509
4216
|
|
|
3510
4217
|
if (mergeResult.status === "succeeded") {
|
|
3511
|
-
const { displayWave: mergeDisplayWave } = resolveDisplayWaveNumber(
|
|
3512
|
-
|
|
4218
|
+
const { displayWave: mergeDisplayWave } = resolveDisplayWaveNumber(
|
|
4219
|
+
waveIdx,
|
|
4220
|
+
roundToTaskWave,
|
|
4221
|
+
taskLevelWaveCount,
|
|
4222
|
+
);
|
|
4223
|
+
onNotify(
|
|
4224
|
+
ORCH_MESSAGES.orchMergeComplete(mergeDisplayWave, mergedCount, mergeTotalSec),
|
|
4225
|
+
"info",
|
|
4226
|
+
);
|
|
3513
4227
|
|
|
3514
4228
|
// TP-040: Emit merge_success event
|
|
3515
|
-
emitEvent(
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
4229
|
+
emitEvent(
|
|
4230
|
+
stateRoot,
|
|
4231
|
+
{
|
|
4232
|
+
...buildEngineEventBase("merge_success", batchState.batchId, waveIdx, batchState.phase),
|
|
4233
|
+
laneCount: mergedCount,
|
|
4234
|
+
durationMs: mergeResult.totalDurationMs,
|
|
4235
|
+
totalWaves: taskLevelWaveCount,
|
|
4236
|
+
},
|
|
4237
|
+
onEngineEvent,
|
|
4238
|
+
);
|
|
3521
4239
|
} else {
|
|
3522
4240
|
onNotify(
|
|
3523
|
-
ORCH_MESSAGES.orchMergeFailed(
|
|
4241
|
+
ORCH_MESSAGES.orchMergeFailed(
|
|
4242
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
4243
|
+
mergeResult.failedLane ?? 0,
|
|
4244
|
+
mergeResult.failureReason || "unknown",
|
|
4245
|
+
),
|
|
3524
4246
|
"error",
|
|
3525
4247
|
);
|
|
3526
4248
|
|
|
3527
4249
|
// TP-040: Emit merge_failed event
|
|
3528
|
-
emitEvent(
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
4250
|
+
emitEvent(
|
|
4251
|
+
stateRoot,
|
|
4252
|
+
{
|
|
4253
|
+
...buildEngineEventBase("merge_failed", batchState.batchId, waveIdx, batchState.phase),
|
|
4254
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
4255
|
+
error: mergeResult.failureReason || "unknown",
|
|
4256
|
+
},
|
|
4257
|
+
onEngineEvent,
|
|
4258
|
+
);
|
|
3533
4259
|
|
|
3534
4260
|
// Emit repo-divergence summary when partial is caused by cross-repo outcome differences
|
|
3535
4261
|
if (mergeResult.status === "partial") {
|
|
@@ -3543,9 +4269,17 @@ export async function executeOrchBatch(
|
|
|
3543
4269
|
// Restore phase to executing (may be overridden below by failure handling)
|
|
3544
4270
|
batchState.phase = "executing";
|
|
3545
4271
|
// ── TS-009: Persist state after merge (merging→executing) ──
|
|
3546
|
-
persistRuntimeState(
|
|
4272
|
+
persistRuntimeState(
|
|
4273
|
+
"merge-complete",
|
|
4274
|
+
batchState,
|
|
4275
|
+
wavePlan,
|
|
4276
|
+
latestAllocatedLanes,
|
|
4277
|
+
allTaskOutcomes,
|
|
4278
|
+
discoveryRef,
|
|
4279
|
+
stateRoot,
|
|
4280
|
+
);
|
|
3547
4281
|
} else if (mixedOutcomeLanes.length > 0) {
|
|
3548
|
-
const mixedIds = mixedOutcomeLanes.map(l => `lane-${l.laneNumber}`).join(", ");
|
|
4282
|
+
const mixedIds = mixedOutcomeLanes.map((l) => `lane-${l.laneNumber}`).join(", ");
|
|
3549
4283
|
mergeResult = {
|
|
3550
4284
|
waveIndex: waveIdx + 1,
|
|
3551
4285
|
status: "partial",
|
|
@@ -3561,23 +4295,41 @@ export async function executeOrchBatch(
|
|
|
3561
4295
|
allMergeResults.push(mergeResult);
|
|
3562
4296
|
batchState.mergeResults.push(mergeResult);
|
|
3563
4297
|
onNotify(
|
|
3564
|
-
ORCH_MESSAGES.orchMergeFailed(
|
|
4298
|
+
ORCH_MESSAGES.orchMergeFailed(
|
|
4299
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
4300
|
+
mergeResult.failedLane,
|
|
4301
|
+
mergeResult.failureReason || "unknown",
|
|
4302
|
+
),
|
|
3565
4303
|
"error",
|
|
3566
4304
|
);
|
|
3567
4305
|
|
|
3568
4306
|
// TP-040 R002: Emit merge_failed for mixed-outcome/no-mergeable-lane path
|
|
3569
|
-
emitEvent(
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
4307
|
+
emitEvent(
|
|
4308
|
+
stateRoot,
|
|
4309
|
+
{
|
|
4310
|
+
...buildEngineEventBase("merge_failed", batchState.batchId, waveIdx, batchState.phase),
|
|
4311
|
+
laneNumber: mergeResult.failedLane,
|
|
4312
|
+
error: mergeResult.failureReason,
|
|
4313
|
+
},
|
|
4314
|
+
onEngineEvent,
|
|
4315
|
+
);
|
|
3574
4316
|
} else {
|
|
3575
4317
|
// No mergeable lanes and no mixed outcomes (e.g., only skipped tasks)
|
|
3576
|
-
onNotify(
|
|
4318
|
+
onNotify(
|
|
4319
|
+
ORCH_MESSAGES.orchMergeSkipped(
|
|
4320
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
4321
|
+
),
|
|
4322
|
+
"info",
|
|
4323
|
+
);
|
|
3577
4324
|
}
|
|
3578
4325
|
} else {
|
|
3579
4326
|
// No succeeded tasks — skip merge entirely
|
|
3580
|
-
onNotify(
|
|
4327
|
+
onNotify(
|
|
4328
|
+
ORCH_MESSAGES.orchMergeSkipped(
|
|
4329
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
4330
|
+
),
|
|
4331
|
+
"info",
|
|
4332
|
+
);
|
|
3581
4333
|
}
|
|
3582
4334
|
|
|
3583
4335
|
// ── TP-033: Safe-stop on rollback failure ─────────────────
|
|
@@ -3587,30 +4339,44 @@ export async function executeOrchBatch(
|
|
|
3587
4339
|
if (mergeResult?.rollbackFailed) {
|
|
3588
4340
|
// TP-033 R004-2: Include persistence error warning when transaction
|
|
3589
4341
|
// record files may be missing, so operator knows to inspect manually
|
|
3590
|
-
const hasPersistErrors =
|
|
4342
|
+
const hasPersistErrors =
|
|
4343
|
+
mergeResult.persistenceErrors && mergeResult.persistenceErrors.length > 0;
|
|
3591
4344
|
const persistWarning = hasPersistErrors
|
|
3592
4345
|
? ` WARNING: ${mergeResult.persistenceErrors!.length} transaction record(s) failed to persist — recovery file(s) may be missing.`
|
|
3593
4346
|
: "";
|
|
3594
4347
|
|
|
3595
|
-
execLog(
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
4348
|
+
execLog(
|
|
4349
|
+
"batch",
|
|
4350
|
+
batchState.batchId,
|
|
4351
|
+
"SAFE-STOP: verification rollback failed — forcing paused regardless of policy",
|
|
4352
|
+
{
|
|
4353
|
+
waveIndex: waveIdx,
|
|
4354
|
+
configPolicy: orchConfig.failure.on_merge_failure,
|
|
4355
|
+
...(hasPersistErrors ? { persistenceErrors: mergeResult.persistenceErrors } : {}),
|
|
4356
|
+
},
|
|
4357
|
+
);
|
|
3600
4358
|
|
|
3601
4359
|
batchState.phase = "paused";
|
|
3602
4360
|
batchState.errors.push(
|
|
3603
4361
|
`Safe-stop at wave ${waveIdx + 1}: verification rollback failed. ` +
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
4362
|
+
`Merge worktree and temp branch preserved for recovery. ` +
|
|
4363
|
+
`Check transaction records in .pi/verification/ for recovery commands.` +
|
|
4364
|
+
persistWarning,
|
|
4365
|
+
);
|
|
4366
|
+
persistRuntimeState(
|
|
4367
|
+
"merge-rollback-safe-stop",
|
|
4368
|
+
batchState,
|
|
4369
|
+
wavePlan,
|
|
4370
|
+
latestAllocatedLanes,
|
|
4371
|
+
allTaskOutcomes,
|
|
4372
|
+
discoveryRef,
|
|
4373
|
+
stateRoot,
|
|
3607
4374
|
);
|
|
3608
|
-
persistRuntimeState("merge-rollback-safe-stop", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
3609
4375
|
onNotify(
|
|
3610
4376
|
`🛑 Safe-stop: verification rollback failed at wave ${waveIdx + 1}. ` +
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
4377
|
+
`Batch force-paused. Merge worktree preserved for manual recovery. ` +
|
|
4378
|
+
`See .pi/verification/ transaction records for recovery commands.` +
|
|
4379
|
+
persistWarning,
|
|
3614
4380
|
"error",
|
|
3615
4381
|
);
|
|
3616
4382
|
|
|
@@ -3678,7 +4444,16 @@ export async function executeOrchBatch(
|
|
|
3678
4444
|
selectedBackend,
|
|
3679
4445
|
);
|
|
3680
4446
|
},
|
|
3681
|
-
persist: (trigger) =>
|
|
4447
|
+
persist: (trigger) =>
|
|
4448
|
+
persistRuntimeState(
|
|
4449
|
+
trigger,
|
|
4450
|
+
batchState,
|
|
4451
|
+
wavePlan,
|
|
4452
|
+
latestAllocatedLanes,
|
|
4453
|
+
allTaskOutcomes,
|
|
4454
|
+
discoveryRef,
|
|
4455
|
+
stateRoot,
|
|
4456
|
+
),
|
|
3682
4457
|
log: (message, details) => execLog("batch", batchState.batchId, message, details),
|
|
3683
4458
|
notify: (message, level) => onNotify(message, level),
|
|
3684
4459
|
updateMergeResult: (result) => {
|
|
@@ -3691,7 +4466,14 @@ export async function executeOrchBatch(
|
|
|
3691
4466
|
// with accurate classification/attempt data from the retry decision.
|
|
3692
4467
|
onRetryAttempt: (decision) => {
|
|
3693
4468
|
emitTier0Event(stateRoot, {
|
|
3694
|
-
...buildTier0EventBase(
|
|
4469
|
+
...buildTier0EventBase(
|
|
4470
|
+
"tier0_recovery_attempt",
|
|
4471
|
+
batchState.batchId,
|
|
4472
|
+
waveIdx,
|
|
4473
|
+
"merge_timeout",
|
|
4474
|
+
decision.currentAttempt,
|
|
4475
|
+
decision.maxAttempts,
|
|
4476
|
+
),
|
|
3695
4477
|
laneNumber: mergeFailedLane,
|
|
3696
4478
|
repoId: mergeRepoId,
|
|
3697
4479
|
classification: decision.classification,
|
|
@@ -3704,11 +4486,26 @@ export async function executeOrchBatch(
|
|
|
3704
4486
|
if (retryOutcome.kind === "retry_succeeded") {
|
|
3705
4487
|
mergeResult = retryOutcome.mergeResult;
|
|
3706
4488
|
batchState.phase = "executing";
|
|
3707
|
-
persistRuntimeState(
|
|
4489
|
+
persistRuntimeState(
|
|
4490
|
+
"merge-retry-succeeded",
|
|
4491
|
+
batchState,
|
|
4492
|
+
wavePlan,
|
|
4493
|
+
latestAllocatedLanes,
|
|
4494
|
+
allTaskOutcomes,
|
|
4495
|
+
discoveryRef,
|
|
4496
|
+
stateRoot,
|
|
4497
|
+
);
|
|
3708
4498
|
|
|
3709
4499
|
// Emit merge retry success event
|
|
3710
4500
|
emitTier0Event(stateRoot, {
|
|
3711
|
-
...buildTier0EventBase(
|
|
4501
|
+
...buildTier0EventBase(
|
|
4502
|
+
"tier0_recovery_success",
|
|
4503
|
+
batchState.batchId,
|
|
4504
|
+
waveIdx,
|
|
4505
|
+
"merge_timeout",
|
|
4506
|
+
retryOutcome.lastDecision.currentAttempt,
|
|
4507
|
+
retryOutcome.lastDecision.maxAttempts,
|
|
4508
|
+
),
|
|
3712
4509
|
laneNumber: mergeFailedLane,
|
|
3713
4510
|
repoId: mergeRepoId,
|
|
3714
4511
|
classification: retryOutcome.classification ?? undefined,
|
|
@@ -3721,7 +4518,15 @@ export async function executeOrchBatch(
|
|
|
3721
4518
|
mergeResult = retryOutcome.mergeResult;
|
|
3722
4519
|
batchState.phase = "paused";
|
|
3723
4520
|
batchState.errors.push(retryOutcome.errorMessage);
|
|
3724
|
-
persistRuntimeState(
|
|
4521
|
+
persistRuntimeState(
|
|
4522
|
+
"merge-rollback-safe-stop",
|
|
4523
|
+
batchState,
|
|
4524
|
+
wavePlan,
|
|
4525
|
+
latestAllocatedLanes,
|
|
4526
|
+
allTaskOutcomes,
|
|
4527
|
+
discoveryRef,
|
|
4528
|
+
stateRoot,
|
|
4529
|
+
);
|
|
3725
4530
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
3726
4531
|
|
|
3727
4532
|
// ── TP-076: Emit supervisor alert for merge safe-stop ──
|
|
@@ -3746,9 +4551,17 @@ export async function executeOrchBatch(
|
|
|
3746
4551
|
});
|
|
3747
4552
|
|
|
3748
4553
|
// Emit merge safe-stop event (treated as exhausted — no further automatic recovery possible)
|
|
3749
|
-
const mergeSafeStopSuggestion =
|
|
4554
|
+
const mergeSafeStopSuggestion =
|
|
4555
|
+
"Merge rollback failed — batch force-paused for manual recovery. Check .pi/verification/ for recovery commands.";
|
|
3750
4556
|
emitTier0Event(stateRoot, {
|
|
3751
|
-
...buildTier0EventBase(
|
|
4557
|
+
...buildTier0EventBase(
|
|
4558
|
+
"tier0_recovery_exhausted",
|
|
4559
|
+
batchState.batchId,
|
|
4560
|
+
waveIdx,
|
|
4561
|
+
"merge_timeout",
|
|
4562
|
+
retryOutcome.lastDecision.currentAttempt,
|
|
4563
|
+
retryOutcome.lastDecision.maxAttempts,
|
|
4564
|
+
),
|
|
3752
4565
|
laneNumber: mergeFailedLane,
|
|
3753
4566
|
repoId: mergeRepoId,
|
|
3754
4567
|
classification: retryOutcome.classification ?? undefined,
|
|
@@ -3756,10 +4569,22 @@ export async function executeOrchBatch(
|
|
|
3756
4569
|
scopeKey: retryOutcome.scopeKey,
|
|
3757
4570
|
suggestion: mergeSafeStopSuggestion,
|
|
3758
4571
|
});
|
|
3759
|
-
emitTier0Escalation(
|
|
3760
|
-
|
|
3761
|
-
|
|
3762
|
-
|
|
4572
|
+
emitTier0Escalation(
|
|
4573
|
+
stateRoot,
|
|
4574
|
+
batchState.batchId,
|
|
4575
|
+
waveIdx,
|
|
4576
|
+
"merge_timeout",
|
|
4577
|
+
retryOutcome.lastDecision.currentAttempt,
|
|
4578
|
+
retryOutcome.lastDecision.maxAttempts,
|
|
4579
|
+
retryOutcome.errorMessage,
|
|
4580
|
+
[],
|
|
4581
|
+
mergeSafeStopSuggestion,
|
|
4582
|
+
{
|
|
4583
|
+
laneNumber: mergeFailedLane,
|
|
4584
|
+
repoId: mergeRepoId,
|
|
4585
|
+
classification: retryOutcome.classification ?? undefined,
|
|
4586
|
+
scopeKey: retryOutcome.scopeKey,
|
|
4587
|
+
},
|
|
3763
4588
|
);
|
|
3764
4589
|
|
|
3765
4590
|
preserveWorktreesForResume = true;
|
|
@@ -3768,7 +4593,8 @@ export async function executeOrchBatch(
|
|
|
3768
4593
|
// TP-033 R006-2: Force paused regardless of on_merge_failure config.
|
|
3769
4594
|
// Retry exhaustion takes precedence over config policy.
|
|
3770
4595
|
mergeResult = retryOutcome.mergeResult;
|
|
3771
|
-
const exhaustionMsg =
|
|
4596
|
+
const exhaustionMsg =
|
|
4597
|
+
retryOutcome.errorMessage +
|
|
3772
4598
|
` [${retryOutcome.classification ?? "unknown"} ${retryOutcome.lastDecision.currentAttempt}/${retryOutcome.lastDecision.maxAttempts}, scope=${retryOutcome.scopeKey}]`;
|
|
3773
4599
|
|
|
3774
4600
|
execLog("batch", batchState.batchId, `merge retry exhausted — forcing paused`, {
|
|
@@ -3781,7 +4607,14 @@ export async function executeOrchBatch(
|
|
|
3781
4607
|
// Emit merge retry exhausted event
|
|
3782
4608
|
const mergeExhaustedSuggestion = `Merge retry exhausted (${retryOutcome.classification ?? "unknown"}) after ${retryOutcome.lastDecision.currentAttempt} attempt(s). Investigate merge failure and retry manually.`;
|
|
3783
4609
|
emitTier0Event(stateRoot, {
|
|
3784
|
-
...buildTier0EventBase(
|
|
4610
|
+
...buildTier0EventBase(
|
|
4611
|
+
"tier0_recovery_exhausted",
|
|
4612
|
+
batchState.batchId,
|
|
4613
|
+
waveIdx,
|
|
4614
|
+
"merge_timeout",
|
|
4615
|
+
retryOutcome.lastDecision.currentAttempt,
|
|
4616
|
+
retryOutcome.lastDecision.maxAttempts,
|
|
4617
|
+
),
|
|
3785
4618
|
laneNumber: mergeFailedLane,
|
|
3786
4619
|
repoId: mergeRepoId,
|
|
3787
4620
|
classification: retryOutcome.classification ?? undefined,
|
|
@@ -3789,15 +4622,35 @@ export async function executeOrchBatch(
|
|
|
3789
4622
|
scopeKey: retryOutcome.scopeKey,
|
|
3790
4623
|
suggestion: mergeExhaustedSuggestion,
|
|
3791
4624
|
});
|
|
3792
|
-
emitTier0Escalation(
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
4625
|
+
emitTier0Escalation(
|
|
4626
|
+
stateRoot,
|
|
4627
|
+
batchState.batchId,
|
|
4628
|
+
waveIdx,
|
|
4629
|
+
"merge_timeout",
|
|
4630
|
+
retryOutcome.lastDecision.currentAttempt,
|
|
4631
|
+
retryOutcome.lastDecision.maxAttempts,
|
|
4632
|
+
exhaustionMsg,
|
|
4633
|
+
[],
|
|
4634
|
+
mergeExhaustedSuggestion,
|
|
4635
|
+
{
|
|
4636
|
+
laneNumber: mergeFailedLane,
|
|
4637
|
+
repoId: mergeRepoId,
|
|
4638
|
+
classification: retryOutcome.classification ?? undefined,
|
|
4639
|
+
scopeKey: retryOutcome.scopeKey,
|
|
4640
|
+
},
|
|
3796
4641
|
);
|
|
3797
4642
|
|
|
3798
4643
|
batchState.phase = "paused";
|
|
3799
4644
|
batchState.errors.push(exhaustionMsg);
|
|
3800
|
-
persistRuntimeState(
|
|
4645
|
+
persistRuntimeState(
|
|
4646
|
+
"merge-retry-exhausted",
|
|
4647
|
+
batchState,
|
|
4648
|
+
wavePlan,
|
|
4649
|
+
latestAllocatedLanes,
|
|
4650
|
+
allTaskOutcomes,
|
|
4651
|
+
discoveryRef,
|
|
4652
|
+
stateRoot,
|
|
4653
|
+
);
|
|
3801
4654
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
3802
4655
|
|
|
3803
4656
|
// ── TP-076: Emit supervisor alert for merge retry exhausted ──
|
|
@@ -3832,11 +4685,24 @@ export async function executeOrchBatch(
|
|
|
3832
4685
|
? ` [not retriable: ${retryOutcome.classification}, scope=${retryOutcome.scopeKey}]`
|
|
3833
4686
|
: "";
|
|
3834
4687
|
|
|
3835
|
-
execLog(
|
|
4688
|
+
execLog(
|
|
4689
|
+
"batch",
|
|
4690
|
+
batchState.batchId,
|
|
4691
|
+
`merge failure — applying ${policyResult.policy} policy${classNote}`,
|
|
4692
|
+
policyResult.logDetails,
|
|
4693
|
+
);
|
|
3836
4694
|
|
|
3837
4695
|
batchState.phase = policyResult.targetPhase;
|
|
3838
4696
|
batchState.errors.push(policyResult.errorMessage + classNote);
|
|
3839
|
-
persistRuntimeState(
|
|
4697
|
+
persistRuntimeState(
|
|
4698
|
+
policyResult.persistTrigger,
|
|
4699
|
+
batchState,
|
|
4700
|
+
wavePlan,
|
|
4701
|
+
latestAllocatedLanes,
|
|
4702
|
+
allTaskOutcomes,
|
|
4703
|
+
discoveryRef,
|
|
4704
|
+
stateRoot,
|
|
4705
|
+
);
|
|
3840
4706
|
onNotify(policyResult.notifyMessage + classNote, policyResult.notifyLevel);
|
|
3841
4707
|
|
|
3842
4708
|
// ── TP-076: Emit supervisor alert for merge failure (no-retry policy) ──
|
|
@@ -3888,30 +4754,46 @@ export async function executeOrchBatch(
|
|
|
3888
4754
|
let targetBranch = batchState.orchBranch;
|
|
3889
4755
|
if (repoId && perRepoRoot !== repoRoot) {
|
|
3890
4756
|
try {
|
|
3891
|
-
targetBranch = resolveBaseBranch(
|
|
3892
|
-
|
|
4757
|
+
targetBranch = resolveBaseBranch(
|
|
4758
|
+
repoId,
|
|
4759
|
+
perRepoRoot,
|
|
4760
|
+
batchState.orchBranch,
|
|
4761
|
+
workspaceConfig,
|
|
4762
|
+
);
|
|
4763
|
+
} catch {
|
|
4764
|
+
/* fall back to orchBranch */
|
|
4765
|
+
}
|
|
3893
4766
|
}
|
|
3894
4767
|
return { repoRoot: perRepoRoot, targetBranch };
|
|
3895
4768
|
},
|
|
3896
4769
|
);
|
|
3897
4770
|
ppUnsafeBranches = ppResult.unsafeBranches;
|
|
3898
|
-
if (ppResult.results.some(r => r.saved)) {
|
|
3899
|
-
execLog(
|
|
3900
|
-
|
|
4771
|
+
if (ppResult.results.some((r) => r.saved)) {
|
|
4772
|
+
execLog(
|
|
4773
|
+
"batch",
|
|
4774
|
+
batchState.batchId,
|
|
4775
|
+
`preserved partial progress for ${ppResult.results.filter((r) => r.saved).length} failed task(s) before inter-wave reset`,
|
|
4776
|
+
);
|
|
3901
4777
|
}
|
|
3902
4778
|
// Log per-task warnings for failed preservation attempts
|
|
3903
4779
|
for (const r of ppResult.results) {
|
|
3904
4780
|
if (!r.saved && (r.commitCount > 0 || r.error)) {
|
|
3905
|
-
execLog(
|
|
4781
|
+
execLog(
|
|
4782
|
+
"batch",
|
|
4783
|
+
batchState.batchId,
|
|
3906
4784
|
`WARNING: Failed to preserve partial progress for task ${r.taskId} ` +
|
|
3907
|
-
|
|
3908
|
-
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" }
|
|
4785
|
+
`(${r.commitCount} commit(s) at risk on lane branch)`,
|
|
4786
|
+
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" },
|
|
4787
|
+
);
|
|
3909
4788
|
}
|
|
3910
4789
|
}
|
|
3911
4790
|
if (ppUnsafeBranches.size > 0) {
|
|
3912
|
-
execLog(
|
|
4791
|
+
execLog(
|
|
4792
|
+
"batch",
|
|
4793
|
+
batchState.batchId,
|
|
3913
4794
|
`WARNING: ${ppUnsafeBranches.size} lane branch(es) could not be preserved — skipping reset for those lanes to prevent commit loss`,
|
|
3914
|
-
{ unsafeBranches: [...ppUnsafeBranches] }
|
|
4795
|
+
{ unsafeBranches: [...ppUnsafeBranches] },
|
|
4796
|
+
);
|
|
3915
4797
|
}
|
|
3916
4798
|
// TP-028: Stamp task outcomes with partial progress data for persistence
|
|
3917
4799
|
applyPartialProgressToOutcomes(ppResult, allTaskOutcomes);
|
|
@@ -3927,8 +4809,15 @@ export async function executeOrchBatch(
|
|
|
3927
4809
|
let targetBranch = batchState.orchBranch;
|
|
3928
4810
|
if (repoId && perRepoRoot !== repoRoot) {
|
|
3929
4811
|
try {
|
|
3930
|
-
targetBranch = resolveBaseBranch(
|
|
3931
|
-
|
|
4812
|
+
targetBranch = resolveBaseBranch(
|
|
4813
|
+
repoId,
|
|
4814
|
+
perRepoRoot,
|
|
4815
|
+
batchState.orchBranch,
|
|
4816
|
+
workspaceConfig,
|
|
4817
|
+
);
|
|
4818
|
+
} catch {
|
|
4819
|
+
/* fall back to orchBranch */
|
|
4820
|
+
}
|
|
3932
4821
|
}
|
|
3933
4822
|
return { repoRoot: perRepoRoot, targetBranch };
|
|
3934
4823
|
},
|
|
@@ -3937,9 +4826,12 @@ export async function executeOrchBatch(
|
|
|
3937
4826
|
for (const branch of skippedPpResult.unsafeBranches) {
|
|
3938
4827
|
ppUnsafeBranches.add(branch);
|
|
3939
4828
|
}
|
|
3940
|
-
if (skippedPpResult.results.some(r => r.saved)) {
|
|
3941
|
-
execLog(
|
|
3942
|
-
|
|
4829
|
+
if (skippedPpResult.results.some((r) => r.saved)) {
|
|
4830
|
+
execLog(
|
|
4831
|
+
"batch",
|
|
4832
|
+
batchState.batchId,
|
|
4833
|
+
`preserved partial progress for ${skippedPpResult.results.filter((r) => r.saved).length} skipped task(s) before inter-wave reset`,
|
|
4834
|
+
);
|
|
3943
4835
|
}
|
|
3944
4836
|
// Stamp skipped task outcomes with partial progress data
|
|
3945
4837
|
applyPartialProgressToOutcomes(skippedPpResult, allTaskOutcomes);
|
|
@@ -3957,10 +4849,18 @@ export async function executeOrchBatch(
|
|
|
3957
4849
|
// TP-029 R006: Track worktrees that failed reset AND removal
|
|
3958
4850
|
// so the cleanup gate only fires on true stale state, not
|
|
3959
4851
|
// successfully-reset reusable worktrees.
|
|
3960
|
-
const failedRemovalWorktrees = new Map<
|
|
4852
|
+
const failedRemovalWorktrees = new Map<
|
|
4853
|
+
string,
|
|
4854
|
+
{ repoId: string | undefined; paths: string[] }
|
|
4855
|
+
>();
|
|
3961
4856
|
|
|
3962
4857
|
for (const [perRepoRoot, perRepoId] of encounteredRepoRoots) {
|
|
3963
|
-
const existingWorktrees = listWorktrees(
|
|
4858
|
+
const existingWorktrees = listWorktrees(
|
|
4859
|
+
resetPrefix,
|
|
4860
|
+
perRepoRoot,
|
|
4861
|
+
resetOpId,
|
|
4862
|
+
batchState.batchId,
|
|
4863
|
+
);
|
|
3964
4864
|
if (existingWorktrees.length === 0) continue;
|
|
3965
4865
|
totalResetWorktrees += existingWorktrees.length;
|
|
3966
4866
|
|
|
@@ -3971,7 +4871,12 @@ export async function executeOrchBatch(
|
|
|
3971
4871
|
targetBranch = batchState.orchBranch;
|
|
3972
4872
|
} else {
|
|
3973
4873
|
try {
|
|
3974
|
-
targetBranch = resolveBaseBranch(
|
|
4874
|
+
targetBranch = resolveBaseBranch(
|
|
4875
|
+
perRepoId,
|
|
4876
|
+
perRepoRoot,
|
|
4877
|
+
batchState.orchBranch,
|
|
4878
|
+
workspaceConfig,
|
|
4879
|
+
);
|
|
3975
4880
|
} catch {
|
|
3976
4881
|
// If resolution fails, fall back to orchBranch (reset will
|
|
3977
4882
|
// fail gracefully and trigger worktree removal)
|
|
@@ -3983,9 +4888,12 @@ export async function executeOrchBatch(
|
|
|
3983
4888
|
// TP-028: Skip reset for worktrees whose lane branch has
|
|
3984
4889
|
// unsaved partial progress (preservation failed with commits)
|
|
3985
4890
|
if (ppUnsafeBranches.has(wt.branch)) {
|
|
3986
|
-
execLog(
|
|
4891
|
+
execLog(
|
|
4892
|
+
"batch",
|
|
4893
|
+
batchState.batchId,
|
|
3987
4894
|
`skipping worktree reset for lane ${wt.laneNumber} — branch "${wt.branch}" has unsaved partial progress`,
|
|
3988
|
-
{ path: wt.path, branch: wt.branch }
|
|
4895
|
+
{ path: wt.path, branch: wt.branch },
|
|
4896
|
+
);
|
|
3989
4897
|
continue;
|
|
3990
4898
|
}
|
|
3991
4899
|
|
|
@@ -3999,12 +4907,21 @@ export async function executeOrchBatch(
|
|
|
3999
4907
|
// If reset fails, remove this worktree so the next wave can recreate it cleanly.
|
|
4000
4908
|
try {
|
|
4001
4909
|
removeWorktree(wt, perRepoRoot);
|
|
4002
|
-
execLog(
|
|
4910
|
+
execLog(
|
|
4911
|
+
"batch",
|
|
4912
|
+
batchState.batchId,
|
|
4913
|
+
`removed unrecoverable worktree for lane ${wt.laneNumber}`,
|
|
4914
|
+
);
|
|
4003
4915
|
} catch (removeErr: unknown) {
|
|
4004
|
-
execLog(
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
4916
|
+
execLog(
|
|
4917
|
+
"batch",
|
|
4918
|
+
batchState.batchId,
|
|
4919
|
+
`removeWorktree failed for lane ${wt.laneNumber}, attempting force cleanup`,
|
|
4920
|
+
{
|
|
4921
|
+
error: removeErr instanceof Error ? removeErr.message : String(removeErr),
|
|
4922
|
+
path: wt.path,
|
|
4923
|
+
},
|
|
4924
|
+
);
|
|
4008
4925
|
// Last resort: force-remove the directory and prune git worktree state.
|
|
4009
4926
|
forceCleanupWorktree(wt, perRepoRoot, batchState.batchId);
|
|
4010
4927
|
// Track this worktree for the cleanup gate — it may still be registered
|
|
@@ -4021,7 +4938,10 @@ export async function executeOrchBatch(
|
|
|
4021
4938
|
|
|
4022
4939
|
if (totalResetWorktrees > 0) {
|
|
4023
4940
|
onNotify(
|
|
4024
|
-
ORCH_MESSAGES.orchWorktreeReset(
|
|
4941
|
+
ORCH_MESSAGES.orchWorktreeReset(
|
|
4942
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
4943
|
+
totalResetWorktrees,
|
|
4944
|
+
),
|
|
4025
4945
|
"info",
|
|
4026
4946
|
);
|
|
4027
4947
|
}
|
|
@@ -4036,9 +4956,9 @@ export async function executeOrchBatch(
|
|
|
4036
4956
|
if (failedRemovalWorktrees.size > 0) {
|
|
4037
4957
|
for (const [perRepoRoot, { repoId: perRepoId, paths: failedPaths }] of failedRemovalWorktrees) {
|
|
4038
4958
|
const remaining = listWorktrees(resetPrefix, perRepoRoot, resetOpId, batchState.batchId);
|
|
4039
|
-
const remainingPaths = new Set(remaining.map(wt => wt.path));
|
|
4959
|
+
const remainingPaths = new Set(remaining.map((wt) => wt.path));
|
|
4040
4960
|
// Only report worktrees that were targeted for removal but are still registered
|
|
4041
|
-
const stale = failedPaths.filter(p => remainingPaths.has(p));
|
|
4961
|
+
const stale = failedPaths.filter((p) => remainingPaths.has(p));
|
|
4042
4962
|
if (stale.length > 0) {
|
|
4043
4963
|
cleanupGateFailures.push({
|
|
4044
4964
|
repoRoot: perRepoRoot,
|
|
@@ -4066,15 +4986,30 @@ export async function executeOrchBatch(
|
|
|
4066
4986
|
if (cleanupRetryCount < cleanupBudget.maxRetries) {
|
|
4067
4987
|
batchState.resilience.retryCountByScope[cleanupScopeKey] = cleanupRetryCount + 1;
|
|
4068
4988
|
|
|
4069
|
-
execLog(
|
|
4989
|
+
execLog(
|
|
4990
|
+
"batch",
|
|
4991
|
+
batchState.batchId,
|
|
4070
4992
|
`tier0: retrying cleanup gate (attempt ${cleanupRetryCount + 1}/${cleanupBudget.maxRetries})`,
|
|
4071
|
-
{
|
|
4993
|
+
{
|
|
4994
|
+
cleanupScopeKey,
|
|
4995
|
+
staleCount: cleanupGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0),
|
|
4996
|
+
},
|
|
4072
4997
|
);
|
|
4073
4998
|
|
|
4074
4999
|
// Emit attempt event
|
|
4075
|
-
const staleWorktreeCount = cleanupGateFailures.reduce(
|
|
5000
|
+
const staleWorktreeCount = cleanupGateFailures.reduce(
|
|
5001
|
+
(n, f) => n + f.staleWorktrees.length,
|
|
5002
|
+
0,
|
|
5003
|
+
);
|
|
4076
5004
|
emitTier0Event(stateRoot, {
|
|
4077
|
-
...buildTier0EventBase(
|
|
5005
|
+
...buildTier0EventBase(
|
|
5006
|
+
"tier0_recovery_attempt",
|
|
5007
|
+
batchState.batchId,
|
|
5008
|
+
waveIdx,
|
|
5009
|
+
"cleanup_gate",
|
|
5010
|
+
cleanupRetryCount + 1,
|
|
5011
|
+
cleanupBudget.maxRetries,
|
|
5012
|
+
),
|
|
4078
5013
|
repoId: null, // wave-scoped: cleanup gate spans all repos
|
|
4079
5014
|
classification: `stale_worktrees:${staleWorktreeCount}`,
|
|
4080
5015
|
cooldownMs: cleanupBudget.cooldownMs,
|
|
@@ -4101,8 +5036,8 @@ export async function executeOrchBatch(
|
|
|
4101
5036
|
const retriedGateFailures: CleanupGateRepoFailure[] = [];
|
|
4102
5037
|
for (const failure of cleanupGateFailures) {
|
|
4103
5038
|
const remaining = listWorktrees(resetPrefix, failure.repoRoot, resetOpId, batchState.batchId);
|
|
4104
|
-
const remainingPaths = new Set(remaining.map(wt => wt.path));
|
|
4105
|
-
const stillStale = failure.staleWorktrees.filter(p => remainingPaths.has(p));
|
|
5039
|
+
const remainingPaths = new Set(remaining.map((wt) => wt.path));
|
|
5040
|
+
const stillStale = failure.staleWorktrees.filter((p) => remainingPaths.has(p));
|
|
4106
5041
|
if (stillStale.length > 0) {
|
|
4107
5042
|
retriedGateFailures.push({
|
|
4108
5043
|
repoRoot: failure.repoRoot,
|
|
@@ -4113,7 +5048,9 @@ export async function executeOrchBatch(
|
|
|
4113
5048
|
}
|
|
4114
5049
|
|
|
4115
5050
|
if (retriedGateFailures.length === 0) {
|
|
4116
|
-
execLog(
|
|
5051
|
+
execLog(
|
|
5052
|
+
"batch",
|
|
5053
|
+
batchState.batchId,
|
|
4117
5054
|
`tier0: cleanup gate retry succeeded — all stale worktrees removed`,
|
|
4118
5055
|
{ cleanupScopeKey },
|
|
4119
5056
|
);
|
|
@@ -4124,19 +5061,36 @@ export async function executeOrchBatch(
|
|
|
4124
5061
|
|
|
4125
5062
|
// Emit success event
|
|
4126
5063
|
emitTier0Event(stateRoot, {
|
|
4127
|
-
...buildTier0EventBase(
|
|
5064
|
+
...buildTier0EventBase(
|
|
5065
|
+
"tier0_recovery_success",
|
|
5066
|
+
batchState.batchId,
|
|
5067
|
+
waveIdx,
|
|
5068
|
+
"cleanup_gate",
|
|
5069
|
+
cleanupRetryCount + 1,
|
|
5070
|
+
cleanupBudget.maxRetries,
|
|
5071
|
+
),
|
|
4128
5072
|
repoId: null, // wave-scoped
|
|
4129
5073
|
resolution: `Cleanup gate retry succeeded — all stale worktrees removed at wave ${waveIdx + 1}`,
|
|
4130
5074
|
scopeKey: cleanupScopeKey,
|
|
4131
5075
|
});
|
|
4132
5076
|
|
|
4133
|
-
persistRuntimeState(
|
|
5077
|
+
persistRuntimeState(
|
|
5078
|
+
"tier0-cleanup-retry-success",
|
|
5079
|
+
batchState,
|
|
5080
|
+
wavePlan,
|
|
5081
|
+
latestAllocatedLanes,
|
|
5082
|
+
allTaskOutcomes,
|
|
5083
|
+
discoveryRef,
|
|
5084
|
+
stateRoot,
|
|
5085
|
+
);
|
|
4134
5086
|
// Fall through to continue the wave loop (don't break)
|
|
4135
5087
|
} else {
|
|
4136
5088
|
// Retry failed — fall through to pausing
|
|
4137
5089
|
const gatePolicyResult = computeCleanupGatePolicy(waveIdx, retriedGateFailures);
|
|
4138
5090
|
|
|
4139
|
-
execLog(
|
|
5091
|
+
execLog(
|
|
5092
|
+
"batch",
|
|
5093
|
+
batchState.batchId,
|
|
4140
5094
|
`tier0: cleanup gate retry failed — still ${retriedGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0)} stale worktree(s), pausing batch`,
|
|
4141
5095
|
gatePolicyResult.logDetails,
|
|
4142
5096
|
);
|
|
@@ -4144,24 +5098,47 @@ export async function executeOrchBatch(
|
|
|
4144
5098
|
const stillStaleCount = retriedGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0);
|
|
4145
5099
|
const cleanupRetryError = `Cleanup gate retry failed — ${stillStaleCount} stale worktree(s) remain`;
|
|
4146
5100
|
const cleanupRetrySuggestion = `Post-merge cleanup retry did not remove all stale worktrees. Manually remove the remaining ${stillStaleCount} worktree(s) and prune git state.`;
|
|
4147
|
-
const cleanupRetryAffected = retriedGateFailures.flatMap(f => f.staleWorktrees);
|
|
5101
|
+
const cleanupRetryAffected = retriedGateFailures.flatMap((f) => f.staleWorktrees);
|
|
4148
5102
|
// Emit exhausted event (retry attempted but failed)
|
|
4149
5103
|
emitTier0Event(stateRoot, {
|
|
4150
|
-
...buildTier0EventBase(
|
|
5104
|
+
...buildTier0EventBase(
|
|
5105
|
+
"tier0_recovery_exhausted",
|
|
5106
|
+
batchState.batchId,
|
|
5107
|
+
waveIdx,
|
|
5108
|
+
"cleanup_gate",
|
|
5109
|
+
cleanupRetryCount + 1,
|
|
5110
|
+
cleanupBudget.maxRetries,
|
|
5111
|
+
),
|
|
4151
5112
|
repoId: null, // wave-scoped
|
|
4152
5113
|
error: cleanupRetryError,
|
|
4153
5114
|
scopeKey: cleanupScopeKey,
|
|
4154
5115
|
affectedTaskIds: cleanupRetryAffected,
|
|
4155
5116
|
suggestion: cleanupRetrySuggestion,
|
|
4156
5117
|
});
|
|
4157
|
-
emitTier0Escalation(
|
|
4158
|
-
|
|
5118
|
+
emitTier0Escalation(
|
|
5119
|
+
stateRoot,
|
|
5120
|
+
batchState.batchId,
|
|
5121
|
+
waveIdx,
|
|
5122
|
+
"cleanup_gate",
|
|
5123
|
+
cleanupRetryCount + 1,
|
|
5124
|
+
cleanupBudget.maxRetries,
|
|
5125
|
+
cleanupRetryError,
|
|
5126
|
+
cleanupRetryAffected,
|
|
5127
|
+
cleanupRetrySuggestion,
|
|
4159
5128
|
{ repoId: null, scopeKey: cleanupScopeKey },
|
|
4160
5129
|
);
|
|
4161
5130
|
|
|
4162
5131
|
batchState.phase = gatePolicyResult.targetPhase;
|
|
4163
5132
|
batchState.errors.push(gatePolicyResult.errorMessage);
|
|
4164
|
-
persistRuntimeState(
|
|
5133
|
+
persistRuntimeState(
|
|
5134
|
+
gatePolicyResult.persistTrigger,
|
|
5135
|
+
batchState,
|
|
5136
|
+
wavePlan,
|
|
5137
|
+
latestAllocatedLanes,
|
|
5138
|
+
allTaskOutcomes,
|
|
5139
|
+
discoveryRef,
|
|
5140
|
+
stateRoot,
|
|
5141
|
+
);
|
|
4165
5142
|
onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
|
|
4166
5143
|
preserveWorktreesForResume = true;
|
|
4167
5144
|
break;
|
|
@@ -4170,28 +5147,56 @@ export async function executeOrchBatch(
|
|
|
4170
5147
|
// Cleanup retry budget exhausted — pause immediately
|
|
4171
5148
|
const gatePolicyResult = computeCleanupGatePolicy(waveIdx, cleanupGateFailures);
|
|
4172
5149
|
|
|
4173
|
-
execLog(
|
|
5150
|
+
execLog(
|
|
5151
|
+
"batch",
|
|
5152
|
+
batchState.batchId,
|
|
5153
|
+
`cleanup gate failed — pausing batch (retry budget exhausted)`,
|
|
5154
|
+
gatePolicyResult.logDetails,
|
|
5155
|
+
);
|
|
4174
5156
|
|
|
4175
5157
|
// Emit exhausted event (budget already consumed from prior waves)
|
|
4176
5158
|
const cleanupBudgetError = `Cleanup gate retry budget exhausted (${cleanupRetryCount}/${cleanupBudget.maxRetries})`;
|
|
4177
5159
|
const cleanupBudgetSuggestion = `Cleanup gate retry budget was already consumed. Manually remove stale worktrees and prune git state.`;
|
|
4178
|
-
const cleanupBudgetAffected = cleanupGateFailures.flatMap(f => f.staleWorktrees);
|
|
5160
|
+
const cleanupBudgetAffected = cleanupGateFailures.flatMap((f) => f.staleWorktrees);
|
|
4179
5161
|
emitTier0Event(stateRoot, {
|
|
4180
|
-
...buildTier0EventBase(
|
|
5162
|
+
...buildTier0EventBase(
|
|
5163
|
+
"tier0_recovery_exhausted",
|
|
5164
|
+
batchState.batchId,
|
|
5165
|
+
waveIdx,
|
|
5166
|
+
"cleanup_gate",
|
|
5167
|
+
cleanupRetryCount,
|
|
5168
|
+
cleanupBudget.maxRetries,
|
|
5169
|
+
),
|
|
4181
5170
|
repoId: null, // wave-scoped
|
|
4182
5171
|
error: cleanupBudgetError,
|
|
4183
5172
|
scopeKey: cleanupScopeKey,
|
|
4184
5173
|
affectedTaskIds: cleanupBudgetAffected,
|
|
4185
5174
|
suggestion: cleanupBudgetSuggestion,
|
|
4186
5175
|
});
|
|
4187
|
-
emitTier0Escalation(
|
|
4188
|
-
|
|
5176
|
+
emitTier0Escalation(
|
|
5177
|
+
stateRoot,
|
|
5178
|
+
batchState.batchId,
|
|
5179
|
+
waveIdx,
|
|
5180
|
+
"cleanup_gate",
|
|
5181
|
+
cleanupRetryCount,
|
|
5182
|
+
cleanupBudget.maxRetries,
|
|
5183
|
+
cleanupBudgetError,
|
|
5184
|
+
cleanupBudgetAffected,
|
|
5185
|
+
cleanupBudgetSuggestion,
|
|
4189
5186
|
{ repoId: null, scopeKey: cleanupScopeKey },
|
|
4190
5187
|
);
|
|
4191
5188
|
|
|
4192
5189
|
batchState.phase = gatePolicyResult.targetPhase;
|
|
4193
5190
|
batchState.errors.push(gatePolicyResult.errorMessage);
|
|
4194
|
-
persistRuntimeState(
|
|
5191
|
+
persistRuntimeState(
|
|
5192
|
+
gatePolicyResult.persistTrigger,
|
|
5193
|
+
batchState,
|
|
5194
|
+
wavePlan,
|
|
5195
|
+
latestAllocatedLanes,
|
|
5196
|
+
allTaskOutcomes,
|
|
5197
|
+
discoveryRef,
|
|
5198
|
+
stateRoot,
|
|
5199
|
+
);
|
|
4195
5200
|
onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
|
|
4196
5201
|
preserveWorktreesForResume = true;
|
|
4197
5202
|
break;
|
|
@@ -4212,7 +5217,7 @@ export async function executeOrchBatch(
|
|
|
4212
5217
|
try {
|
|
4213
5218
|
const lanesDir = join(piDir, "runtime", batchState.batchId, "lanes");
|
|
4214
5219
|
if (existsSync(lanesDir)) {
|
|
4215
|
-
const files = readdirSync(lanesDir).filter(f => f.startsWith("lane-") && f.endsWith(".json"));
|
|
5220
|
+
const files = readdirSync(lanesDir).filter((f) => f.startsWith("lane-") && f.endsWith(".json"));
|
|
4216
5221
|
for (const f of files) {
|
|
4217
5222
|
try {
|
|
4218
5223
|
const snap = JSON.parse(readFileSync(join(lanesDir, f), "utf-8"));
|
|
@@ -4227,14 +5232,20 @@ export async function executeOrchBatch(
|
|
|
4227
5232
|
cacheWrite: (w.cacheWriteTokens || 0) + (r.cacheWriteTokens || 0),
|
|
4228
5233
|
costUsd: (w.costUsd || 0) + (r.costUsd || 0),
|
|
4229
5234
|
});
|
|
4230
|
-
} catch {
|
|
5235
|
+
} catch {
|
|
5236
|
+
/* skip invalid files */
|
|
5237
|
+
}
|
|
4231
5238
|
}
|
|
4232
5239
|
}
|
|
4233
|
-
} catch {
|
|
5240
|
+
} catch {
|
|
5241
|
+
/* runtime dir may not exist */
|
|
5242
|
+
}
|
|
4234
5243
|
|
|
4235
5244
|
// Legacy fallback: lane-state-*.json sidecars (pre-V2).
|
|
4236
5245
|
try {
|
|
4237
|
-
const files = readdirSync(piDir).filter(
|
|
5246
|
+
const files = readdirSync(piDir).filter(
|
|
5247
|
+
(f) => f.startsWith("lane-state-") && f.endsWith(".json"),
|
|
5248
|
+
);
|
|
4238
5249
|
for (const f of files) {
|
|
4239
5250
|
try {
|
|
4240
5251
|
const raw = readFileSync(join(piDir, f), "utf-8").trim();
|
|
@@ -4249,25 +5260,33 @@ export async function executeOrchBatch(
|
|
|
4249
5260
|
costUsd: data.workerCostUsd || 0,
|
|
4250
5261
|
});
|
|
4251
5262
|
}
|
|
4252
|
-
} catch {
|
|
5263
|
+
} catch {
|
|
5264
|
+
/* skip invalid files */
|
|
5265
|
+
}
|
|
4253
5266
|
}
|
|
4254
|
-
} catch {
|
|
5267
|
+
} catch {
|
|
5268
|
+
/* .pi dir may not exist */
|
|
5269
|
+
}
|
|
4255
5270
|
|
|
4256
5271
|
// Build per-task summaries from allTaskOutcomes + wave plan
|
|
4257
5272
|
const taskSummaries: BatchTaskSummary[] = allTaskOutcomes.map((to) => {
|
|
4258
5273
|
// Find which wave and lane this task ran in
|
|
4259
5274
|
let wave = 0;
|
|
4260
5275
|
for (let wi = 0; wi < wavePlan.length; wi++) {
|
|
4261
|
-
if (wavePlan[wi].includes(to.taskId)) {
|
|
5276
|
+
if (wavePlan[wi].includes(to.taskId)) {
|
|
5277
|
+
wave = wi + 1;
|
|
5278
|
+
break;
|
|
5279
|
+
}
|
|
4262
5280
|
}
|
|
4263
|
-
const lane =
|
|
4264
|
-
??
|
|
5281
|
+
const lane =
|
|
5282
|
+
to.laneNumber ??
|
|
5283
|
+
(() => {
|
|
4265
5284
|
const laneMatch = to.sessionName?.match(/lane-(\d+)/);
|
|
4266
5285
|
return laneMatch ? parseInt(laneMatch[1], 10) : 0;
|
|
4267
5286
|
})();
|
|
4268
5287
|
|
|
4269
5288
|
// Compute duration from start/end times
|
|
4270
|
-
const durationMs =
|
|
5289
|
+
const durationMs = to.startTime && to.endTime ? to.endTime - to.startTime : 0;
|
|
4271
5290
|
|
|
4272
5291
|
// TP-116: Resolve tokens from outcome telemetry first; only fallback for legacy outcomes.
|
|
4273
5292
|
const tokens = resolveBatchHistoryTaskTokens(
|
|
@@ -4280,7 +5299,14 @@ export async function executeOrchBatch(
|
|
|
4280
5299
|
// TP-171: Map outcome status to valid BatchTaskSummary status.
|
|
4281
5300
|
// Non-terminal statuses ("running", "pending") can appear if batch
|
|
4282
5301
|
// was paused/aborted mid-wave. Map them to appropriate history values.
|
|
4283
|
-
const validStatuses: Set<string> = new Set([
|
|
5302
|
+
const validStatuses: Set<string> = new Set([
|
|
5303
|
+
"succeeded",
|
|
5304
|
+
"failed",
|
|
5305
|
+
"skipped",
|
|
5306
|
+
"blocked",
|
|
5307
|
+
"stalled",
|
|
5308
|
+
"pending",
|
|
5309
|
+
]);
|
|
4284
5310
|
const historyStatus: BatchTaskSummary["status"] = validStatuses.has(to.status)
|
|
4285
5311
|
? (to.status as BatchTaskSummary["status"])
|
|
4286
5312
|
: "pending"; // "running" or unknown → "pending" in history
|
|
@@ -4300,7 +5326,7 @@ export async function executeOrchBatch(
|
|
|
4300
5326
|
// TP-147: Ensure ALL tasks from the wave plan are represented in history.
|
|
4301
5327
|
// Tasks that never got allocated (blocked by upstream failures, never started)
|
|
4302
5328
|
// won't have entries in allTaskOutcomes. Add them with appropriate status.
|
|
4303
|
-
const coveredTaskIds = new Set(taskSummaries.map(t => t.taskId));
|
|
5329
|
+
const coveredTaskIds = new Set(taskSummaries.map((t) => t.taskId));
|
|
4304
5330
|
for (let wi = 0; wi < wavePlan.length; wi++) {
|
|
4305
5331
|
for (const taskId of wavePlan[wi]) {
|
|
4306
5332
|
if (coveredTaskIds.has(taskId)) continue;
|
|
@@ -4323,8 +5349,8 @@ export async function executeOrchBatch(
|
|
|
4323
5349
|
|
|
4324
5350
|
// Build per-wave summaries
|
|
4325
5351
|
const waveSummaries: BatchWaveSummary[] = wavePlan.map((taskIds, wi) => {
|
|
4326
|
-
const waveTasks = taskSummaries.filter(t => t.wave === wi + 1);
|
|
4327
|
-
const mergeResult = batchState.mergeResults.find(mr => mr.waveIndex === wi + 1);
|
|
5352
|
+
const waveTasks = taskSummaries.filter((t) => t.wave === wi + 1);
|
|
5353
|
+
const mergeResult = batchState.mergeResults.find((mr) => mr.waveIndex === wi + 1);
|
|
4328
5354
|
const waveTokens: TokenCounts = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, costUsd: 0 };
|
|
4329
5355
|
for (const t of waveTasks) {
|
|
4330
5356
|
waveTokens.input += t.tokens.input;
|
|
@@ -4357,7 +5383,9 @@ export async function executeOrchBatch(
|
|
|
4357
5383
|
// (phase hasn't been set to "completed" yet at this point in the flow).
|
|
4358
5384
|
const historyStatus: "completed" | "partial" | "failed" | "aborted" =
|
|
4359
5385
|
batchState.failedTasks > 0
|
|
4360
|
-
?
|
|
5386
|
+
? batchState.succeededTasks > 0
|
|
5387
|
+
? "partial"
|
|
5388
|
+
: "failed"
|
|
4361
5389
|
: batchState.succeededTasks > 0
|
|
4362
5390
|
? "completed"
|
|
4363
5391
|
: "aborted";
|
|
@@ -4367,9 +5395,12 @@ export async function executeOrchBatch(
|
|
|
4367
5395
|
// and log a warning if it diverges from batchState.totalTasks.
|
|
4368
5396
|
const actualTotalTasks = taskSummaries.length;
|
|
4369
5397
|
if (actualTotalTasks !== batchState.totalTasks) {
|
|
4370
|
-
execLog(
|
|
5398
|
+
execLog(
|
|
5399
|
+
"batch",
|
|
5400
|
+
batchState.batchId,
|
|
4371
5401
|
`WARNING: totalTasks mismatch — batchState.totalTasks=${batchState.totalTasks}, ` +
|
|
4372
|
-
|
|
5402
|
+
`taskSummaries.length=${actualTotalTasks}. Using taskSummaries.length for history.`,
|
|
5403
|
+
);
|
|
4373
5404
|
}
|
|
4374
5405
|
|
|
4375
5406
|
const summary: BatchHistorySummary = {
|
|
@@ -4398,18 +5429,29 @@ export async function executeOrchBatch(
|
|
|
4398
5429
|
// TP-031 (R006): This check MUST run before cleanup so that worktrees
|
|
4399
5430
|
// survive when failedTasks > 0. Without this, cleanup deletes worktrees
|
|
4400
5431
|
// before the batch is marked "paused", breaking resumability.
|
|
4401
|
-
if (
|
|
4402
|
-
|
|
4403
|
-
batchState.
|
|
5432
|
+
if (
|
|
5433
|
+
!preserveWorktreesForResume &&
|
|
5434
|
+
((batchState.phase as OrchBatchPhase) === "executing" ||
|
|
5435
|
+
(batchState.phase as OrchBatchPhase) === "merging") &&
|
|
5436
|
+
batchState.failedTasks > 0
|
|
5437
|
+
) {
|
|
4404
5438
|
preserveWorktreesForResume = true;
|
|
4405
|
-
execLog(
|
|
5439
|
+
execLog(
|
|
5440
|
+
"batch",
|
|
5441
|
+
batchState.batchId,
|
|
5442
|
+
"pre-cleanup: failedTasks > 0 detected, preserving worktrees for resume",
|
|
5443
|
+
);
|
|
4406
5444
|
}
|
|
4407
5445
|
|
|
4408
5446
|
// ── Phase 3: Cleanup ─────────────────────────────────────────
|
|
4409
5447
|
const prefix = orchConfig.orchestrator.worktree_prefix;
|
|
4410
5448
|
|
|
4411
5449
|
if (preserveWorktreesForResume) {
|
|
4412
|
-
execLog(
|
|
5450
|
+
execLog(
|
|
5451
|
+
"batch",
|
|
5452
|
+
batchState.batchId,
|
|
5453
|
+
"skipping final cleanup to preserve worktrees/branches for resume",
|
|
5454
|
+
);
|
|
4413
5455
|
} else {
|
|
4414
5456
|
// Kill lingering Runtime V2 agents BEFORE removing worktrees.
|
|
4415
5457
|
// On Windows, lingering processes with cwd inside the worktree can lock
|
|
@@ -4426,7 +5468,11 @@ export async function executeOrchBatch(
|
|
|
4426
5468
|
|
|
4427
5469
|
let performedAgentCleanup = false;
|
|
4428
5470
|
if (lingeringLaneSessions.size > 0) {
|
|
4429
|
-
execLog(
|
|
5471
|
+
execLog(
|
|
5472
|
+
"batch",
|
|
5473
|
+
batchState.batchId,
|
|
5474
|
+
`killing ${lingeringLaneSessions.size} lingering lane agent session(s) before cleanup`,
|
|
5475
|
+
);
|
|
4430
5476
|
for (const sessionName of lingeringLaneSessions) {
|
|
4431
5477
|
killV2LaneAgents(sessionName, {
|
|
4432
5478
|
stateRoot,
|
|
@@ -4439,7 +5485,11 @@ export async function executeOrchBatch(
|
|
|
4439
5485
|
|
|
4440
5486
|
const killedMergeAgents = killAllMergeAgentsV2();
|
|
4441
5487
|
if (killedMergeAgents > 0) {
|
|
4442
|
-
execLog(
|
|
5488
|
+
execLog(
|
|
5489
|
+
"batch",
|
|
5490
|
+
batchState.batchId,
|
|
5491
|
+
`killed ${killedMergeAgents} lingering merge agent(s) before cleanup`,
|
|
5492
|
+
);
|
|
4443
5493
|
performedAgentCleanup = true;
|
|
4444
5494
|
}
|
|
4445
5495
|
|
|
@@ -4451,18 +5501,25 @@ export async function executeOrchBatch(
|
|
|
4451
5501
|
const piDir = join(stateRoot, ".pi");
|
|
4452
5502
|
try {
|
|
4453
5503
|
const sidecarFiles = readdirSync(piDir).filter(
|
|
4454
|
-
f =>
|
|
5504
|
+
(f) =>
|
|
5505
|
+
f.startsWith("lane-state-") ||
|
|
4455
5506
|
f.startsWith("worker-conversation-") ||
|
|
4456
5507
|
f.startsWith("merge-result-") ||
|
|
4457
5508
|
f.startsWith("merge-request-"),
|
|
4458
5509
|
);
|
|
4459
5510
|
for (const f of sidecarFiles) {
|
|
4460
|
-
try {
|
|
5511
|
+
try {
|
|
5512
|
+
unlinkSync(join(piDir, f));
|
|
5513
|
+
} catch {
|
|
5514
|
+
/* best effort */
|
|
5515
|
+
}
|
|
4461
5516
|
}
|
|
4462
5517
|
if (sidecarFiles.length > 0) {
|
|
4463
5518
|
execLog("batch", batchState.batchId, `cleaned up ${sidecarFiles.length} sidecar file(s)`);
|
|
4464
5519
|
}
|
|
4465
|
-
} catch {
|
|
5520
|
+
} catch {
|
|
5521
|
+
/* .pi dir may not exist */
|
|
5522
|
+
}
|
|
4466
5523
|
|
|
4467
5524
|
// ── TP-028: Preserve partial progress before terminal cleanup ──
|
|
4468
5525
|
// Save failed task commits as named branches before worktree removal
|
|
@@ -4480,25 +5537,38 @@ export async function executeOrchBatch(
|
|
|
4480
5537
|
let targetBranch = batchState.orchBranch;
|
|
4481
5538
|
if (repoId && perRepoRoot !== repoRoot) {
|
|
4482
5539
|
try {
|
|
4483
|
-
targetBranch = resolveBaseBranch(
|
|
4484
|
-
|
|
5540
|
+
targetBranch = resolveBaseBranch(
|
|
5541
|
+
repoId,
|
|
5542
|
+
perRepoRoot,
|
|
5543
|
+
batchState.orchBranch,
|
|
5544
|
+
workspaceConfig,
|
|
5545
|
+
);
|
|
5546
|
+
} catch {
|
|
5547
|
+
/* fall back to orchBranch */
|
|
5548
|
+
}
|
|
4485
5549
|
}
|
|
4486
5550
|
return { repoRoot: perRepoRoot, targetBranch };
|
|
4487
5551
|
},
|
|
4488
5552
|
);
|
|
4489
|
-
if (ppResult.results.some(r => r.saved)) {
|
|
4490
|
-
execLog(
|
|
4491
|
-
|
|
5553
|
+
if (ppResult.results.some((r) => r.saved)) {
|
|
5554
|
+
execLog(
|
|
5555
|
+
"batch",
|
|
5556
|
+
batchState.batchId,
|
|
5557
|
+
`preserved partial progress for ${ppResult.results.filter((r) => r.saved).length} failed task(s) before terminal cleanup`,
|
|
5558
|
+
);
|
|
4492
5559
|
}
|
|
4493
5560
|
// Log warnings for failed preservation attempts — at terminal cleanup
|
|
4494
5561
|
// we cannot skip deletion (batch is ending), but operators need to know
|
|
4495
5562
|
// that commits may become unreachable via reflog only.
|
|
4496
5563
|
for (const r of ppResult.results) {
|
|
4497
5564
|
if (!r.saved && (r.commitCount > 0 || r.error)) {
|
|
4498
|
-
execLog(
|
|
5565
|
+
execLog(
|
|
5566
|
+
"batch",
|
|
5567
|
+
batchState.batchId,
|
|
4499
5568
|
`WARNING: Failed to preserve partial progress for task ${r.taskId} ` +
|
|
4500
|
-
|
|
4501
|
-
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" }
|
|
5569
|
+
`(${r.commitCount} commit(s) may become unreachable after cleanup)`,
|
|
5570
|
+
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" },
|
|
5571
|
+
);
|
|
4502
5572
|
}
|
|
4503
5573
|
}
|
|
4504
5574
|
// TP-028: Stamp task outcomes with partial progress data for persistence
|
|
@@ -4515,22 +5585,35 @@ export async function executeOrchBatch(
|
|
|
4515
5585
|
let targetBranch = batchState.orchBranch;
|
|
4516
5586
|
if (repoId && perRepoRoot !== repoRoot) {
|
|
4517
5587
|
try {
|
|
4518
|
-
targetBranch = resolveBaseBranch(
|
|
4519
|
-
|
|
5588
|
+
targetBranch = resolveBaseBranch(
|
|
5589
|
+
repoId,
|
|
5590
|
+
perRepoRoot,
|
|
5591
|
+
batchState.orchBranch,
|
|
5592
|
+
workspaceConfig,
|
|
5593
|
+
);
|
|
5594
|
+
} catch {
|
|
5595
|
+
/* fall back to orchBranch */
|
|
5596
|
+
}
|
|
4520
5597
|
}
|
|
4521
5598
|
return { repoRoot: perRepoRoot, targetBranch };
|
|
4522
5599
|
},
|
|
4523
5600
|
);
|
|
4524
|
-
if (skippedPpResult.results.some(r => r.saved)) {
|
|
4525
|
-
execLog(
|
|
4526
|
-
|
|
5601
|
+
if (skippedPpResult.results.some((r) => r.saved)) {
|
|
5602
|
+
execLog(
|
|
5603
|
+
"batch",
|
|
5604
|
+
batchState.batchId,
|
|
5605
|
+
`preserved partial progress for ${skippedPpResult.results.filter((r) => r.saved).length} skipped task(s) before terminal cleanup`,
|
|
5606
|
+
);
|
|
4527
5607
|
}
|
|
4528
5608
|
for (const r of skippedPpResult.results) {
|
|
4529
5609
|
if (!r.saved && (r.commitCount > 0 || r.error)) {
|
|
4530
|
-
execLog(
|
|
5610
|
+
execLog(
|
|
5611
|
+
"batch",
|
|
5612
|
+
batchState.batchId,
|
|
4531
5613
|
`WARNING: Failed to preserve partial progress for skipped task ${r.taskId} ` +
|
|
4532
|
-
|
|
4533
|
-
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" }
|
|
5614
|
+
`(${r.commitCount} commit(s) may become unreachable after cleanup)`,
|
|
5615
|
+
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" },
|
|
5616
|
+
);
|
|
4534
5617
|
}
|
|
4535
5618
|
}
|
|
4536
5619
|
applyPartialProgressToOutcomes(skippedPpResult, allTaskOutcomes);
|
|
@@ -4551,14 +5634,26 @@ export async function executeOrchBatch(
|
|
|
4551
5634
|
} else {
|
|
4552
5635
|
// Secondary repo (workspace mode): resolve the repo's own branch
|
|
4553
5636
|
try {
|
|
4554
|
-
targetBranch = resolveBaseBranch(
|
|
5637
|
+
targetBranch = resolveBaseBranch(
|
|
5638
|
+
perRepoId,
|
|
5639
|
+
perRepoRoot,
|
|
5640
|
+
batchState.orchBranch,
|
|
5641
|
+
workspaceConfig,
|
|
5642
|
+
);
|
|
4555
5643
|
} catch {
|
|
4556
5644
|
// Fall back to undefined — skips branch protection
|
|
4557
5645
|
// (safe because successfully merged branches were already cleaned)
|
|
4558
5646
|
targetBranch = undefined;
|
|
4559
5647
|
}
|
|
4560
5648
|
}
|
|
4561
|
-
const removeResult = removeAllWorktrees(
|
|
5649
|
+
const removeResult = removeAllWorktrees(
|
|
5650
|
+
prefix,
|
|
5651
|
+
perRepoRoot,
|
|
5652
|
+
cleanupOpId,
|
|
5653
|
+
targetBranch,
|
|
5654
|
+
batchState.batchId,
|
|
5655
|
+
orchConfig,
|
|
5656
|
+
);
|
|
4562
5657
|
|
|
4563
5658
|
// Log preserved branches
|
|
4564
5659
|
for (const p of removeResult.preserved) {
|
|
@@ -4573,15 +5668,25 @@ export async function executeOrchBatch(
|
|
|
4573
5668
|
}
|
|
4574
5669
|
|
|
4575
5670
|
if (removeResult.failed.length > 0) {
|
|
4576
|
-
const failedPaths = removeResult.failed.map(f => f.worktree.path).join(", ");
|
|
4577
|
-
execLog(
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
5671
|
+
const failedPaths = removeResult.failed.map((f) => f.worktree.path).join(", ");
|
|
5672
|
+
execLog(
|
|
5673
|
+
"batch",
|
|
5674
|
+
batchState.batchId,
|
|
5675
|
+
`worktree cleanup: ${removeResult.removed.length} removed, ${removeResult.failed.length} failed, ${removeResult.preserved.length} preserved`,
|
|
5676
|
+
{
|
|
5677
|
+
failedPaths,
|
|
5678
|
+
repoId: perRepoId ?? "(default)",
|
|
5679
|
+
},
|
|
5680
|
+
);
|
|
4581
5681
|
} else if (removeResult.totalAttempted > 0) {
|
|
4582
|
-
execLog(
|
|
4583
|
-
|
|
4584
|
-
|
|
5682
|
+
execLog(
|
|
5683
|
+
"batch",
|
|
5684
|
+
batchState.batchId,
|
|
5685
|
+
`worktree cleanup: ${removeResult.removed.length} removed, ${removeResult.preserved.length} preserved`,
|
|
5686
|
+
{
|
|
5687
|
+
repoId: perRepoId ?? "(default)",
|
|
5688
|
+
},
|
|
5689
|
+
);
|
|
4585
5690
|
}
|
|
4586
5691
|
}
|
|
4587
5692
|
|
|
@@ -4598,7 +5703,10 @@ export async function executeOrchBatch(
|
|
|
4598
5703
|
for (const lr of mergeResult.laneResults) {
|
|
4599
5704
|
// TP-032 R006-3: Exclude verification_new_failure lanes from branch cleanup
|
|
4600
5705
|
// (their merge commits were rolled back, so the branch is NOT merged)
|
|
4601
|
-
if (
|
|
5706
|
+
if (
|
|
5707
|
+
!lr.error &&
|
|
5708
|
+
(lr.result?.status === "SUCCESS" || lr.result?.status === "CONFLICT_RESOLVED")
|
|
5709
|
+
) {
|
|
4602
5710
|
const laneRepoRoot = resolveRepoRoot(lr.repoId, repoRoot, workspaceConfig);
|
|
4603
5711
|
const ancestorCheck = runGit(
|
|
4604
5712
|
["merge-base", "--is-ancestor", lr.sourceBranch, lr.targetBranch],
|
|
@@ -4611,14 +5719,24 @@ export async function executeOrchBatch(
|
|
|
4611
5719
|
repoId: lr.repoId ?? "(default)",
|
|
4612
5720
|
});
|
|
4613
5721
|
} else {
|
|
4614
|
-
execLog(
|
|
4615
|
-
|
|
4616
|
-
|
|
5722
|
+
execLog(
|
|
5723
|
+
"batch",
|
|
5724
|
+
batchState.batchId,
|
|
5725
|
+
`warning: failed to delete merged branch ${lr.sourceBranch} — retained for manual cleanup`,
|
|
5726
|
+
{
|
|
5727
|
+
repoId: lr.repoId ?? "(default)",
|
|
5728
|
+
},
|
|
5729
|
+
);
|
|
4617
5730
|
}
|
|
4618
5731
|
} else {
|
|
4619
|
-
execLog(
|
|
4620
|
-
|
|
4621
|
-
|
|
5732
|
+
execLog(
|
|
5733
|
+
"batch",
|
|
5734
|
+
batchState.batchId,
|
|
5735
|
+
`warning: branch ${lr.sourceBranch} not fully merged into ${lr.targetBranch} — retained`,
|
|
5736
|
+
{
|
|
5737
|
+
repoId: lr.repoId ?? "(default)",
|
|
5738
|
+
},
|
|
5739
|
+
);
|
|
4622
5740
|
}
|
|
4623
5741
|
}
|
|
4624
5742
|
}
|
|
@@ -4633,7 +5751,10 @@ export async function executeOrchBatch(
|
|
|
4633
5751
|
// Determine final batch state. Cast to OrchBatchPhase to bypass control-flow
|
|
4634
5752
|
// narrowing — mergeWave() could leave phase as "merging" if an unexpected
|
|
4635
5753
|
// throw occurs between setting "merging" and restoring "executing".
|
|
4636
|
-
if (
|
|
5754
|
+
if (
|
|
5755
|
+
(batchState.phase as OrchBatchPhase) === "executing" ||
|
|
5756
|
+
(batchState.phase as OrchBatchPhase) === "merging"
|
|
5757
|
+
) {
|
|
4637
5758
|
// Normal completion (not stopped, paused, or aborted)
|
|
4638
5759
|
if (batchState.failedTasks > 0) {
|
|
4639
5760
|
// TP-031: Default to "paused" so the batch is resumable without --force.
|
|
@@ -4661,30 +5782,55 @@ export async function executeOrchBatch(
|
|
|
4661
5782
|
// always handles integration.
|
|
4662
5783
|
const mergedTaskCount = batchState.succeededTasks;
|
|
4663
5784
|
const isTerminalPhase = batchState.phase === "completed" || batchState.phase === "failed";
|
|
4664
|
-
if (
|
|
4665
|
-
|
|
5785
|
+
if (
|
|
5786
|
+
isTerminalPhase &&
|
|
5787
|
+
!preserveWorktreesForResume &&
|
|
5788
|
+
batchState.orchBranch &&
|
|
5789
|
+
mergedTaskCount > 0
|
|
5790
|
+
) {
|
|
5791
|
+
if (
|
|
5792
|
+
orchConfig.orchestrator.integration === "supervised" ||
|
|
5793
|
+
orchConfig.orchestrator.integration === "auto"
|
|
5794
|
+
) {
|
|
4666
5795
|
// TP-043: Supervisor-managed integration modes. The supervisor
|
|
4667
5796
|
// agent handles integration after batch_complete event. The engine
|
|
4668
5797
|
// does NOT perform legacy fast-forward here — defer to supervisor.
|
|
4669
|
-
execLog(
|
|
5798
|
+
execLog(
|
|
5799
|
+
"batch",
|
|
5800
|
+
batchState.batchId,
|
|
5801
|
+
`integration deferred to supervisor (mode: ${orchConfig.orchestrator.integration})`,
|
|
5802
|
+
);
|
|
4670
5803
|
} else {
|
|
4671
5804
|
// Manual mode (default): show integration guidance
|
|
4672
5805
|
onNotify(
|
|
4673
|
-
ORCH_MESSAGES.orchIntegrationManual(
|
|
5806
|
+
ORCH_MESSAGES.orchIntegrationManual(
|
|
5807
|
+
batchState.orchBranch,
|
|
5808
|
+
batchState.baseBranch,
|
|
5809
|
+
mergedTaskCount,
|
|
5810
|
+
),
|
|
4674
5811
|
"info",
|
|
4675
5812
|
);
|
|
4676
5813
|
}
|
|
4677
5814
|
}
|
|
4678
5815
|
|
|
4679
5816
|
// ── TS-009: Persist terminal state ──
|
|
4680
|
-
persistRuntimeState(
|
|
5817
|
+
persistRuntimeState(
|
|
5818
|
+
"batch-terminal",
|
|
5819
|
+
batchState,
|
|
5820
|
+
wavePlan,
|
|
5821
|
+
latestAllocatedLanes,
|
|
5822
|
+
allTaskOutcomes,
|
|
5823
|
+
discoveryRef,
|
|
5824
|
+
stateRoot,
|
|
5825
|
+
);
|
|
4681
5826
|
|
|
4682
5827
|
// ── TP-076: Emit supervisor alert for batch completion ──────
|
|
4683
5828
|
if (batchState.phase === "completed" || batchState.phase === "failed") {
|
|
4684
5829
|
const batchDurationMs = batchState.endedAt ? batchState.endedAt - batchState.startedAt : 0;
|
|
4685
|
-
const durationStr =
|
|
4686
|
-
|
|
4687
|
-
|
|
5830
|
+
const durationStr =
|
|
5831
|
+
batchDurationMs > 0
|
|
5832
|
+
? `${Math.floor(batchDurationMs / 60000)}m ${Math.round((batchDurationMs % 60000) / 1000)}s`
|
|
5833
|
+
: "unknown";
|
|
4688
5834
|
if (batchState.phase === "completed" && batchState.failedTasks === 0) {
|
|
4689
5835
|
emitAlert({
|
|
4690
5836
|
category: "batch-complete",
|
|
@@ -4724,12 +5870,26 @@ export async function executeOrchBatch(
|
|
|
4724
5870
|
|
|
4725
5871
|
// ── TP-031: Emit diagnostic reports (JSONL + markdown) ──
|
|
4726
5872
|
// Non-fatal: errors are logged but never crash batch finalization.
|
|
4727
|
-
emitDiagnosticReports(
|
|
5873
|
+
emitDiagnosticReports(
|
|
5874
|
+
assembleDiagnosticInput(
|
|
5875
|
+
orchConfig,
|
|
5876
|
+
batchState,
|
|
5877
|
+
wavePlan,
|
|
5878
|
+
latestAllocatedLanes,
|
|
5879
|
+
allTaskOutcomes,
|
|
5880
|
+
stateRoot,
|
|
5881
|
+
),
|
|
5882
|
+
);
|
|
4728
5883
|
|
|
4729
5884
|
if (batchState.phase === "paused" || batchState.phase === "stopped") {
|
|
4730
|
-
execLog(
|
|
4731
|
-
|
|
4732
|
-
|
|
5885
|
+
execLog(
|
|
5886
|
+
"batch",
|
|
5887
|
+
batchState.batchId,
|
|
5888
|
+
"batch ended in non-terminal execution state; completion banner suppressed",
|
|
5889
|
+
{
|
|
5890
|
+
phase: batchState.phase,
|
|
5891
|
+
},
|
|
5892
|
+
);
|
|
4733
5893
|
} else {
|
|
4734
5894
|
onNotify(
|
|
4735
5895
|
ORCH_MESSAGES.orchBatchComplete(
|
|
@@ -4768,6 +5928,4 @@ export async function executeOrchBatch(
|
|
|
4768
5928
|
}
|
|
4769
5929
|
}
|
|
4770
5930
|
|
|
4771
|
-
|
|
4772
5931
|
// ── Dashboard Widget (Step 6) ────────────────────────────────────────
|
|
4773
|
-
|