taskplane 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dashboard/public/app.js +227 -0
- package/dashboard/public/index.html +19 -0
- package/dashboard/public/style.css +319 -0
- package/dashboard/server.cjs +219 -1
- package/extensions/taskplane/config-loader.ts +7 -1
- package/extensions/taskplane/config-schema.ts +19 -2
- package/extensions/taskplane/config.ts +23 -1
- package/extensions/taskplane/engine.ts +913 -46
- package/extensions/taskplane/execution.ts +1 -0
- package/extensions/taskplane/extension.ts +975 -54
- package/extensions/taskplane/formatting.ts +713 -712
- package/extensions/taskplane/index.ts +1 -0
- package/extensions/taskplane/merge.ts +236 -48
- package/extensions/taskplane/messages.ts +10 -0
- package/extensions/taskplane/persistence.ts +183 -3
- package/extensions/taskplane/resume.ts +243 -24
- package/extensions/taskplane/settings-tui.ts +10 -3
- package/extensions/taskplane/supervisor-primer.md +626 -0
- package/extensions/taskplane/supervisor.ts +3659 -0
- package/extensions/taskplane/types.ts +330 -3
- package/package.json +1 -1
|
@@ -8,12 +8,13 @@ import { join, dirname, resolve, relative } from "path";
|
|
|
8
8
|
|
|
9
9
|
import { buildLaneEnvVars, buildTmuxSpawnArgs, execLog, tmuxHasSession, tmuxKillSession, toTmuxPath } from "./execution.ts";
|
|
10
10
|
import { resolveOperatorId } from "./naming.ts";
|
|
11
|
-
import { MERGE_POLL_INTERVAL_MS, MERGE_RESULT_GRACE_MS, MERGE_RESULT_READ_RETRIES, MERGE_RESULT_READ_RETRY_DELAY_MS, MERGE_SPAWN_RETRY_MAX, MERGE_TIMEOUT_MS, MergeError, VALID_MERGE_STATUSES } from "./types.ts";
|
|
11
|
+
import { MERGE_POLL_INTERVAL_MS, MERGE_RESULT_GRACE_MS, MERGE_RESULT_READ_RETRIES, MERGE_RESULT_READ_RETRY_DELAY_MS, MERGE_SPAWN_RETRY_MAX, MERGE_TIMEOUT_MAX_RETRIES, MERGE_TIMEOUT_MS, MergeError, VALID_MERGE_STATUSES } from "./types.ts";
|
|
12
12
|
import type { AllocatedLane, LaneExecutionResult, MergeLaneResult, MergeResult, MergeResultStatus, MergeWaveResult, OrchestratorConfig, RepoMergeOutcome, TaskRunnerConfig, TransactionRecord, TransactionStatus, VerificationBaselineResult, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
13
13
|
import { resolveBaseBranch, resolveRepoRoot } from "./waves.ts";
|
|
14
14
|
import { generateMergeWorktreePath, sleepSync } from "./worktree.ts";
|
|
15
15
|
import { getCurrentBranch, runGit } from "./git.ts";
|
|
16
16
|
import { ORCH_MESSAGES } from "./messages.ts";
|
|
17
|
+
import { loadOrchestratorConfig } from "./config.ts";
|
|
17
18
|
import { captureBaseline, diffFingerprints, runVerificationCommands, parseTestOutput, deduplicateFingerprints } from "./verification.ts";
|
|
18
19
|
import type { VerificationBaseline, FingerprintDiff, TestFingerprint } from "./verification.ts";
|
|
19
20
|
|
|
@@ -43,6 +44,66 @@ export function parseMergeResult(resultPath: string): MergeResult {
|
|
|
43
44
|
);
|
|
44
45
|
}
|
|
45
46
|
|
|
47
|
+
const pickString = (obj: Record<string, unknown>, ...keys: string[]): string | null => {
|
|
48
|
+
for (const key of keys) {
|
|
49
|
+
const value = obj[key];
|
|
50
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
51
|
+
return value;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return null;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const hasFlatVerification = (obj: Record<string, unknown>): boolean =>
|
|
58
|
+
typeof obj.verification_passed === "boolean"
|
|
59
|
+
|| Array.isArray(obj.verification_commands)
|
|
60
|
+
|| typeof obj.verification_output === "string"
|
|
61
|
+
|| typeof obj.verification_exit_code === "number";
|
|
62
|
+
|
|
63
|
+
const normalizeVerification = (obj: Record<string, unknown>): MergeResult["verification"] | null => {
|
|
64
|
+
const nested = (obj.verification && typeof obj.verification === "object")
|
|
65
|
+
? obj.verification as Record<string, unknown>
|
|
66
|
+
: null;
|
|
67
|
+
|
|
68
|
+
if (!nested && !hasFlatVerification(obj)) {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const passedFromBool =
|
|
73
|
+
(nested && typeof nested.passed === "boolean" ? nested.passed : undefined)
|
|
74
|
+
?? (nested && typeof nested.all_passed === "boolean" ? nested.all_passed : undefined)
|
|
75
|
+
?? (typeof obj.verification_passed === "boolean" ? obj.verification_passed : undefined);
|
|
76
|
+
|
|
77
|
+
const exitCode =
|
|
78
|
+
(nested && typeof nested.exitCode === "number" ? nested.exitCode : undefined)
|
|
79
|
+
?? (nested && typeof nested.exit_code === "number" ? nested.exit_code : undefined)
|
|
80
|
+
?? (typeof obj.verification_exit_code === "number" ? obj.verification_exit_code : undefined);
|
|
81
|
+
|
|
82
|
+
const passed = typeof passedFromBool === "boolean"
|
|
83
|
+
? passedFromBool
|
|
84
|
+
: (typeof exitCode === "number" ? exitCode === 0 : false);
|
|
85
|
+
|
|
86
|
+
const ran = (nested && typeof nested.ran === "boolean")
|
|
87
|
+
? nested.ran
|
|
88
|
+
: (
|
|
89
|
+
typeof passedFromBool === "boolean"
|
|
90
|
+
|| typeof exitCode === "number"
|
|
91
|
+
|| (nested && typeof nested.command === "string")
|
|
92
|
+
|| (nested && typeof nested.summary === "string")
|
|
93
|
+
|| typeof obj.verification_output === "string"
|
|
94
|
+
|| Array.isArray(obj.verification_commands)
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
const output = (
|
|
98
|
+
(nested && typeof nested.output === "string" ? nested.output : undefined)
|
|
99
|
+
?? (nested && typeof nested.summary === "string" ? nested.summary : undefined)
|
|
100
|
+
?? (nested && typeof nested.notes === "string" ? nested.notes : undefined)
|
|
101
|
+
?? (typeof obj.verification_output === "string" ? obj.verification_output : "")
|
|
102
|
+
).slice(0, 2000);
|
|
103
|
+
|
|
104
|
+
return { ran, passed, output };
|
|
105
|
+
};
|
|
106
|
+
|
|
46
107
|
// Retry-read loop for partially-written files
|
|
47
108
|
let lastParseError = "";
|
|
48
109
|
for (let attempt = 1; attempt <= MERGE_RESULT_READ_RETRIES; attempt++) {
|
|
@@ -60,7 +121,7 @@ export function parseMergeResult(resultPath: string): MergeResult {
|
|
|
60
121
|
);
|
|
61
122
|
}
|
|
62
123
|
|
|
63
|
-
const parsed = JSON.parse(raw)
|
|
124
|
+
const parsed = JSON.parse(raw) as Record<string, unknown>;
|
|
64
125
|
|
|
65
126
|
// Validate required fields
|
|
66
127
|
if (typeof parsed.status !== "string") {
|
|
@@ -69,29 +130,23 @@ export function parseMergeResult(resultPath: string): MergeResult {
|
|
|
69
130
|
`Merge result missing required field "status": ${resultPath}`,
|
|
70
131
|
);
|
|
71
132
|
}
|
|
72
|
-
|
|
133
|
+
|
|
134
|
+
// Accept known source-field variants written by different merge agents.
|
|
135
|
+
// Canonical field remains source_branch.
|
|
136
|
+
const sourceBranch = pickString(parsed, "source_branch", "sourceBranch", "source");
|
|
137
|
+
if (!sourceBranch) {
|
|
73
138
|
throw new MergeError(
|
|
74
139
|
"MERGE_RESULT_MISSING_FIELDS",
|
|
75
|
-
`Merge result missing required field "source_branch": ${resultPath}`,
|
|
140
|
+
`Merge result missing required field "source_branch" (accepted aliases: sourceBranch, source): ${resultPath}`,
|
|
76
141
|
);
|
|
77
142
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
all_passed: parsed.verification_passed !== false,
|
|
86
|
-
output: "",
|
|
87
|
-
notes: "",
|
|
88
|
-
};
|
|
89
|
-
} else {
|
|
90
|
-
throw new MergeError(
|
|
91
|
-
"MERGE_RESULT_MISSING_FIELDS",
|
|
92
|
-
`Merge result missing required field "verification": ${resultPath}`,
|
|
93
|
-
);
|
|
94
|
-
}
|
|
143
|
+
|
|
144
|
+
const verification = normalizeVerification(parsed);
|
|
145
|
+
if (!verification) {
|
|
146
|
+
throw new MergeError(
|
|
147
|
+
"MERGE_RESULT_MISSING_FIELDS",
|
|
148
|
+
`Merge result missing required field "verification": ${resultPath}`,
|
|
149
|
+
);
|
|
95
150
|
}
|
|
96
151
|
|
|
97
152
|
// Normalize status to uppercase (merge agents may write lowercase)
|
|
@@ -105,20 +160,33 @@ export function parseMergeResult(resultPath: string): MergeResult {
|
|
|
105
160
|
parsed.status = "BUILD_FAILURE";
|
|
106
161
|
}
|
|
107
162
|
|
|
163
|
+
const targetBranch = pickString(parsed, "target_branch", "targetBranch", "target") ?? "";
|
|
164
|
+
const mergeCommit = pickString(parsed, "merge_commit", "mergeCommit") ?? "";
|
|
165
|
+
const conflicts = Array.isArray(parsed.conflicts)
|
|
166
|
+
? parsed.conflicts
|
|
167
|
+
.filter((c): c is { file: string; type: string; resolved: boolean; resolution?: string } => (
|
|
168
|
+
typeof c === "object"
|
|
169
|
+
&& c !== null
|
|
170
|
+
&& typeof (c as { file?: unknown }).file === "string"
|
|
171
|
+
&& typeof (c as { type?: unknown }).type === "string"
|
|
172
|
+
&& typeof (c as { resolved?: unknown }).resolved === "boolean"
|
|
173
|
+
))
|
|
174
|
+
.map(c => ({
|
|
175
|
+
file: c.file,
|
|
176
|
+
type: c.type,
|
|
177
|
+
resolved: c.resolved,
|
|
178
|
+
...(typeof c.resolution === "string" ? { resolution: c.resolution } : {}),
|
|
179
|
+
}))
|
|
180
|
+
: [];
|
|
181
|
+
|
|
108
182
|
// Normalize optional fields with defaults
|
|
109
183
|
return {
|
|
110
184
|
status: parsed.status as MergeResultStatus,
|
|
111
|
-
source_branch:
|
|
112
|
-
target_branch:
|
|
113
|
-
merge_commit:
|
|
114
|
-
conflicts
|
|
115
|
-
verification
|
|
116
|
-
ran: !!parsed.verification.ran,
|
|
117
|
-
passed: !!parsed.verification.passed,
|
|
118
|
-
output: typeof parsed.verification.output === "string"
|
|
119
|
-
? parsed.verification.output.slice(0, 2000)
|
|
120
|
-
: "",
|
|
121
|
-
},
|
|
185
|
+
source_branch: sourceBranch,
|
|
186
|
+
target_branch: targetBranch,
|
|
187
|
+
merge_commit: mergeCommit,
|
|
188
|
+
conflicts,
|
|
189
|
+
verification,
|
|
122
190
|
};
|
|
123
191
|
} catch (err: unknown) {
|
|
124
192
|
if (err instanceof MergeError) throw err;
|
|
@@ -237,7 +305,23 @@ export function buildMergeRequest(
|
|
|
237
305
|
`result_file: ${resultFilePath}`,
|
|
238
306
|
`Write your JSON result to: ${resultFilePath}`,
|
|
239
307
|
"",
|
|
240
|
-
|
|
308
|
+
"## Result JSON Schema (required)",
|
|
309
|
+
"Use EXACT snake_case keys shown below. Do not use camelCase or shortened keys.",
|
|
310
|
+
"",
|
|
311
|
+
"```json",
|
|
312
|
+
"{",
|
|
313
|
+
" \"status\": \"SUCCESS\" | \"CONFLICT_RESOLVED\" | \"CONFLICT_UNRESOLVED\" | \"BUILD_FAILURE\",",
|
|
314
|
+
" \"source_branch\": \"<source branch name>\",",
|
|
315
|
+
" \"target_branch\": \"<target branch name>\",",
|
|
316
|
+
" \"merge_commit\": \"<merge commit sha or empty string>\",",
|
|
317
|
+
" \"conflicts\": [{ \"file\": \"...\", \"type\": \"...\", \"resolved\": true|false }],",
|
|
318
|
+
" \"verification\": { \"ran\": true|false, \"passed\": true|false, \"output\": \"...\" }",
|
|
319
|
+
"}",
|
|
320
|
+
"```",
|
|
321
|
+
"",
|
|
322
|
+
"Do NOT use keys like source/sourceBranch/target/mergeCommit.",
|
|
323
|
+
"Write valid JSON only (no markdown around the final file).",
|
|
324
|
+
"",
|
|
241
325
|
"## Important",
|
|
242
326
|
"- You are working in an ISOLATED MERGE WORKTREE (not the user's main repo)",
|
|
243
327
|
"- The correct branch is ALREADY checked out — do NOT checkout any other branch",
|
|
@@ -351,6 +435,33 @@ export function spawnMergeAgent(
|
|
|
351
435
|
);
|
|
352
436
|
}
|
|
353
437
|
|
|
438
|
+
/**
|
|
439
|
+
* Re-read merge timeout from config on disk.
|
|
440
|
+
*
|
|
441
|
+
* TP-038: Allows the operator to increase `merge.timeoutMinutes` without
|
|
442
|
+
* restarting the pi session. Called before each retry attempt so the
|
|
443
|
+
* retry loop picks up any config changes made while the batch was running.
|
|
444
|
+
*
|
|
445
|
+
* @param configRoot - The directory containing `.pi/taskplane-config.json`
|
|
446
|
+
* @param pointerConfigRoot - Optional pointer config root (workspace mode)
|
|
447
|
+
* @returns Fresh timeout in milliseconds
|
|
448
|
+
*/
|
|
449
|
+
export function reloadMergeTimeoutMs(configRoot: string, pointerConfigRoot?: string): number {
|
|
450
|
+
try {
|
|
451
|
+
const freshConfig = loadOrchestratorConfig(configRoot, pointerConfigRoot);
|
|
452
|
+
const minutes = freshConfig.merge.timeout_minutes ?? 10;
|
|
453
|
+
return minutes * 60 * 1000;
|
|
454
|
+
} catch (err: unknown) {
|
|
455
|
+
// Config re-read is best-effort — fall back to default on failure
|
|
456
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
457
|
+
execLog("merge", "config-reload", `failed to re-read merge timeout from config: ${errMsg} — using default`);
|
|
458
|
+
return MERGE_TIMEOUT_MS;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/** Merge result statuses that indicate the merge agent completed successfully. */
|
|
463
|
+
const SUCCESSFUL_MERGE_STATUSES = new Set<string>(["SUCCESS", "CONFLICT_RESOLVED"]);
|
|
464
|
+
|
|
354
465
|
/**
|
|
355
466
|
* Wait for merge agent to produce a result file.
|
|
356
467
|
*
|
|
@@ -358,7 +469,10 @@ export function spawnMergeAgent(
|
|
|
358
469
|
* 1. Check if result file exists → parse and return
|
|
359
470
|
* 2. Check if TMUX session is still alive
|
|
360
471
|
* 3. If session died without result → grace period → check again → fail
|
|
361
|
-
* 4. If timeout exceeded →
|
|
472
|
+
* 4. If timeout exceeded → check result before killing:
|
|
473
|
+
* a. If result exists with SUCCESS/CONFLICT_RESOLVED: accept it
|
|
474
|
+
* (merge agent slow but succeeded)
|
|
475
|
+
* b. If result missing or non-success: kill session → fail
|
|
362
476
|
*
|
|
363
477
|
* @param resultPath - Path to the expected result JSON file
|
|
364
478
|
* @param sessionName - TMUX session name for liveness checking
|
|
@@ -384,21 +498,41 @@ export function waitForMergeResult(
|
|
|
384
498
|
|
|
385
499
|
// Check timeout
|
|
386
500
|
if (elapsed >= timeoutMs) {
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
});
|
|
391
|
-
tmuxKillSession(sessionName);
|
|
392
|
-
|
|
393
|
-
// One final check for result file (agent may have written it just before timeout)
|
|
501
|
+
// TP-038: Check result file BEFORE killing the session.
|
|
502
|
+
// The merge may have actually succeeded — the verification tests
|
|
503
|
+
// just pushed past the timeout. Accept successful results without killing.
|
|
394
504
|
if (existsSync(resultPath)) {
|
|
395
505
|
try {
|
|
396
|
-
|
|
506
|
+
const lateResult = parseMergeResult(resultPath);
|
|
507
|
+
if (SUCCESSFUL_MERGE_STATUSES.has(lateResult.status)) {
|
|
508
|
+
execLog("merge", sessionName, "merge agent slow but succeeded — accepting result at timeout", {
|
|
509
|
+
status: lateResult.status,
|
|
510
|
+
elapsed,
|
|
511
|
+
timeoutMs,
|
|
512
|
+
});
|
|
513
|
+
// Clean up session (agent may still be running post-write)
|
|
514
|
+
if (tmuxHasSession(sessionName)) {
|
|
515
|
+
tmuxKillSession(sessionName);
|
|
516
|
+
}
|
|
517
|
+
return lateResult;
|
|
518
|
+
}
|
|
519
|
+
// Non-success result at timeout — fall through to kill
|
|
520
|
+
execLog("merge", sessionName, "merge result exists at timeout but non-success — killing session", {
|
|
521
|
+
status: lateResult.status,
|
|
522
|
+
elapsed,
|
|
523
|
+
timeoutMs,
|
|
524
|
+
});
|
|
397
525
|
} catch {
|
|
398
|
-
//
|
|
526
|
+
// Result file unreadable — fall through to kill
|
|
399
527
|
}
|
|
400
528
|
}
|
|
401
529
|
|
|
530
|
+
execLog("merge", sessionName, "merge timeout — killing session", {
|
|
531
|
+
elapsed,
|
|
532
|
+
timeoutMs,
|
|
533
|
+
});
|
|
534
|
+
tmuxKillSession(sessionName);
|
|
535
|
+
|
|
402
536
|
throw new MergeError(
|
|
403
537
|
"MERGE_TIMEOUT",
|
|
404
538
|
`Merge agent '${sessionName}' did not produce a result within ` +
|
|
@@ -1077,12 +1211,66 @@ export function mergeWave(
|
|
|
1077
1211
|
// Write merge request to temp file
|
|
1078
1212
|
writeFileSync(requestFilePath, mergeRequestContent, "utf-8");
|
|
1079
1213
|
|
|
1080
|
-
//
|
|
1081
|
-
|
|
1214
|
+
// ── TP-038: Spawn + wait with retry-on-timeout ──────────────
|
|
1215
|
+
// On MERGE_TIMEOUT, retry with 2× the previous timeout (up to
|
|
1216
|
+
// MERGE_TIMEOUT_MAX_RETRIES). Before each retry, re-read config
|
|
1217
|
+
// from disk so operators can increase merge.timeoutMinutes without
|
|
1218
|
+
// restarting the session.
|
|
1219
|
+
let mergeResult: MergeResult;
|
|
1220
|
+
{
|
|
1221
|
+
const configRoot = stateRoot ?? repoRoot;
|
|
1222
|
+
let currentTimeoutMs = (config.merge.timeout_minutes ?? 10) * 60 * 1000;
|
|
1223
|
+
let lastTimeoutError: MergeError | null = null;
|
|
1224
|
+
|
|
1225
|
+
for (let attempt = 0; attempt <= MERGE_TIMEOUT_MAX_RETRIES; attempt++) {
|
|
1226
|
+
// On retry: clean up stale result, re-read config, apply backoff
|
|
1227
|
+
if (attempt > 0) {
|
|
1228
|
+
// Re-read config from disk (TP-038: allows operator to adjust timeout)
|
|
1229
|
+
const freshTimeoutMs = reloadMergeTimeoutMs(configRoot);
|
|
1230
|
+
// Apply 2× backoff: double the timeout for each retry attempt
|
|
1231
|
+
currentTimeoutMs = freshTimeoutMs * Math.pow(2, attempt);
|
|
1232
|
+
|
|
1233
|
+
execLog("merge", sessionName, `retry ${attempt}/${MERGE_TIMEOUT_MAX_RETRIES} after timeout — respawning merge agent`, {
|
|
1234
|
+
newTimeoutMs: currentTimeoutMs,
|
|
1235
|
+
newTimeoutMin: Math.round(currentTimeoutMs / 60_000),
|
|
1236
|
+
attempt,
|
|
1237
|
+
});
|
|
1238
|
+
|
|
1239
|
+
// Clean up stale result file from prior attempt
|
|
1240
|
+
if (existsSync(resultFilePath)) {
|
|
1241
|
+
try { unlinkSync(resultFilePath); } catch { /* best effort */ }
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
// Re-spawn merge agent for the retry
|
|
1245
|
+
spawnMergeAgent(sessionName, repoRoot, mergeWorkDir, requestFilePath, config, stateRoot, agentRoot);
|
|
1246
|
+
} else {
|
|
1247
|
+
// First attempt: spawn merge agent
|
|
1248
|
+
spawnMergeAgent(sessionName, repoRoot, mergeWorkDir, requestFilePath, config, stateRoot, agentRoot);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
try {
|
|
1252
|
+
mergeResult = waitForMergeResult(resultFilePath, sessionName, currentTimeoutMs);
|
|
1253
|
+
lastTimeoutError = null;
|
|
1254
|
+
break; // Success — exit retry loop
|
|
1255
|
+
} catch (waitErr: unknown) {
|
|
1256
|
+
if (
|
|
1257
|
+
waitErr instanceof MergeError &&
|
|
1258
|
+
waitErr.code === "MERGE_TIMEOUT" &&
|
|
1259
|
+
attempt < MERGE_TIMEOUT_MAX_RETRIES
|
|
1260
|
+
) {
|
|
1261
|
+
// Timeout — will retry on next loop iteration
|
|
1262
|
+
lastTimeoutError = waitErr;
|
|
1263
|
+
continue;
|
|
1264
|
+
}
|
|
1265
|
+
// Non-timeout error or final retry exhausted — propagate
|
|
1266
|
+
throw waitErr;
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1082
1269
|
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1270
|
+
// TypeScript: mergeResult is guaranteed to be assigned here
|
|
1271
|
+
// (either break from loop or throw propagated the error)
|
|
1272
|
+
mergeResult = mergeResult!;
|
|
1273
|
+
}
|
|
1086
1274
|
|
|
1087
1275
|
// Clean up request file (leave result file for debugging)
|
|
1088
1276
|
try {
|
|
@@ -755,6 +755,10 @@ export function applyMergeRetryLoop(
|
|
|
755
755
|
});
|
|
756
756
|
|
|
757
757
|
callbacks.persist("merge-retry-increment");
|
|
758
|
+
|
|
759
|
+
// Emit Tier 0 attempt event via callback (TP-039 R004: emit only when retry is scheduled)
|
|
760
|
+
callbacks.onRetryAttempt?.(lastDecision);
|
|
761
|
+
|
|
758
762
|
callbacks.notify(
|
|
759
763
|
`🔄 Merge retry (${lastDecision.reason}) at wave ${waveIdx + 1}. ` +
|
|
760
764
|
(lastDecision.cooldownMs > 0 ? `Waiting ${lastDecision.cooldownMs}ms before retry...` : "Retrying immediately..."),
|
|
@@ -777,6 +781,9 @@ export function applyMergeRetryLoop(
|
|
|
777
781
|
return {
|
|
778
782
|
kind: "retry_succeeded",
|
|
779
783
|
mergeResult: currentResult,
|
|
784
|
+
classification,
|
|
785
|
+
scopeKey,
|
|
786
|
+
lastDecision,
|
|
780
787
|
};
|
|
781
788
|
}
|
|
782
789
|
|
|
@@ -790,6 +797,9 @@ export function applyMergeRetryLoop(
|
|
|
790
797
|
return {
|
|
791
798
|
kind: "safe_stop",
|
|
792
799
|
mergeResult: currentResult,
|
|
800
|
+
classification,
|
|
801
|
+
scopeKey,
|
|
802
|
+
lastDecision,
|
|
793
803
|
errorMessage:
|
|
794
804
|
`Safe-stop at wave ${waveIdx + 1}: verification rollback failed after retry. ` +
|
|
795
805
|
`Merge worktree and temp branch preserved for recovery.` + persistWarning,
|
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
* State persistence, serialization, orphan detection
|
|
3
3
|
* @module orch/persistence
|
|
4
4
|
*/
|
|
5
|
-
import { readFileSync, writeFileSync, existsSync, unlinkSync, renameSync, mkdirSync } from "fs";
|
|
5
|
+
import { readFileSync, writeFileSync, existsSync, unlinkSync, renameSync, mkdirSync, appendFileSync } from "fs";
|
|
6
6
|
import { execSync } from "child_process";
|
|
7
7
|
import { join, dirname, basename } from "path";
|
|
8
8
|
|
|
9
9
|
import { execLog } from "./execution.ts";
|
|
10
10
|
import { BATCH_STATE_SCHEMA_VERSION, StateFileError, batchStatePath, BATCH_HISTORY_MAX_ENTRIES, defaultResilienceState, defaultBatchDiagnostics } from "./types.ts";
|
|
11
11
|
import type { BatchHistorySummary } from "./types.ts";
|
|
12
|
-
import type { AllocatedLane, DiscoveryResult, LaneTaskOutcome, LaneTaskStatus, MonitorState, OrchBatchPhase, OrchBatchRuntimeState, PersistedBatchState, PersistedLaneRecord, PersistedMergeResult, PersistedTaskRecord, TaskMonitorSnapshot, WorkspaceMode } from "./types.ts";
|
|
12
|
+
import type { AllocatedLane, DiscoveryResult, EngineEvent, EscalationContext, LaneTaskOutcome, LaneTaskStatus, MonitorState, OrchBatchPhase, OrchBatchRuntimeState, PersistedBatchState, PersistedLaneRecord, PersistedMergeResult, PersistedTaskRecord, TaskMonitorSnapshot, Tier0RecoveryPattern, WorkspaceMode } from "./types.ts";
|
|
13
13
|
import { sleepSync } from "./worktree.ts";
|
|
14
14
|
import type { PreserveFailedLaneProgressResult } from "./worktree.ts";
|
|
15
15
|
|
|
@@ -310,7 +310,7 @@ export function persistRuntimeState(
|
|
|
310
310
|
|
|
311
311
|
/** All valid OrchBatchPhase values for validation. */
|
|
312
312
|
export const VALID_BATCH_PHASES: ReadonlySet<string> = new Set([
|
|
313
|
-
"idle", "planning", "executing", "merging", "paused", "stopped", "completed", "failed",
|
|
313
|
+
"idle", "launching", "planning", "executing", "merging", "paused", "stopped", "completed", "failed",
|
|
314
314
|
]);
|
|
315
315
|
|
|
316
316
|
/** All valid LaneTaskStatus values for validation. */
|
|
@@ -1620,3 +1620,183 @@ export function saveBatchHistory(repoRoot: string, summary: BatchHistorySummary)
|
|
|
1620
1620
|
}
|
|
1621
1621
|
}
|
|
1622
1622
|
|
|
1623
|
+
|
|
1624
|
+
// ── Tier 0 Supervisor Event Logging (TP-039 Step 2) ─────────────────
|
|
1625
|
+
|
|
1626
|
+
/**
|
|
1627
|
+
* Event types emitted by Tier 0 recovery actions.
|
|
1628
|
+
*
|
|
1629
|
+
* - `tier0_recovery_attempt` — A recovery action is being tried
|
|
1630
|
+
* - `tier0_recovery_success` — Recovery succeeded
|
|
1631
|
+
* - `tier0_recovery_exhausted` — Retry budget exhausted, escalation needed
|
|
1632
|
+
* - `tier0_escalation` — Escalation to supervisor (emitted alongside exhausted)
|
|
1633
|
+
*
|
|
1634
|
+
* @since TP-039
|
|
1635
|
+
*/
|
|
1636
|
+
export type Tier0EventType =
|
|
1637
|
+
| "tier0_recovery_attempt"
|
|
1638
|
+
| "tier0_recovery_success"
|
|
1639
|
+
| "tier0_recovery_exhausted"
|
|
1640
|
+
| "tier0_escalation";
|
|
1641
|
+
|
|
1642
|
+
/**
|
|
1643
|
+
* Structured event written to `.pi/supervisor/events.jsonl`.
|
|
1644
|
+
*
|
|
1645
|
+
* Each event contains enough context for the supervisor agent (Tier 1)
|
|
1646
|
+
* to understand what happened and decide next actions.
|
|
1647
|
+
*
|
|
1648
|
+
* @since TP-039
|
|
1649
|
+
*/
|
|
1650
|
+
export interface Tier0Event {
|
|
1651
|
+
/** ISO 8601 timestamp */
|
|
1652
|
+
timestamp: string;
|
|
1653
|
+
/** Event type */
|
|
1654
|
+
type: Tier0EventType;
|
|
1655
|
+
/** Batch identifier */
|
|
1656
|
+
batchId: string;
|
|
1657
|
+
/** Wave index (0-based) */
|
|
1658
|
+
waveIndex: number;
|
|
1659
|
+
/** Recovery pattern being applied */
|
|
1660
|
+
pattern: Tier0RecoveryPattern | "merge_timeout";
|
|
1661
|
+
/** Current attempt number (1-based) */
|
|
1662
|
+
attempt: number;
|
|
1663
|
+
/** Maximum attempts allowed */
|
|
1664
|
+
maxAttempts: number;
|
|
1665
|
+
/** Affected task ID (for task-scoped patterns like worker_crash) */
|
|
1666
|
+
taskId?: string;
|
|
1667
|
+
/** Lane number (for lane-scoped patterns) */
|
|
1668
|
+
laneNumber?: number;
|
|
1669
|
+
/** Repo ID (for workspace-mode attribution; null/undefined for repo-mode) */
|
|
1670
|
+
repoId?: string | null;
|
|
1671
|
+
/** Exit classification or error type */
|
|
1672
|
+
classification?: string;
|
|
1673
|
+
/** Error message (for exhausted events) */
|
|
1674
|
+
error?: string;
|
|
1675
|
+
/** Resolution description (for success events) */
|
|
1676
|
+
resolution?: string;
|
|
1677
|
+
/** Cooldown/timeout in milliseconds before retry (for attempt events) */
|
|
1678
|
+
cooldownMs?: number;
|
|
1679
|
+
/** Scope key used for retry counter tracking */
|
|
1680
|
+
scopeKey?: string;
|
|
1681
|
+
/** Affected task IDs (for escalation context in exhausted events) */
|
|
1682
|
+
affectedTaskIds?: string[];
|
|
1683
|
+
/** Suggested remediation (for exhausted events) */
|
|
1684
|
+
suggestion?: string;
|
|
1685
|
+
/** Typed escalation payload (present only on `tier0_escalation` events) */
|
|
1686
|
+
escalation?: EscalationContext;
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
/**
|
|
1690
|
+
* Build the required base fields for a Tier 0 event.
|
|
1691
|
+
*
|
|
1692
|
+
* Ensures consistent field population across all emit sites so
|
|
1693
|
+
* supervisor consumers get a deterministic event shape.
|
|
1694
|
+
*
|
|
1695
|
+
* @since TP-039 R004
|
|
1696
|
+
*/
|
|
1697
|
+
export function buildTier0EventBase(
|
|
1698
|
+
type: Tier0EventType,
|
|
1699
|
+
batchId: string,
|
|
1700
|
+
waveIndex: number,
|
|
1701
|
+
pattern: Tier0RecoveryPattern | "merge_timeout",
|
|
1702
|
+
attempt: number,
|
|
1703
|
+
maxAttempts: number,
|
|
1704
|
+
): Pick<Tier0Event, "timestamp" | "type" | "batchId" | "waveIndex" | "pattern" | "attempt" | "maxAttempts"> {
|
|
1705
|
+
return {
|
|
1706
|
+
timestamp: new Date().toISOString(),
|
|
1707
|
+
type,
|
|
1708
|
+
batchId,
|
|
1709
|
+
waveIndex,
|
|
1710
|
+
pattern,
|
|
1711
|
+
attempt,
|
|
1712
|
+
maxAttempts,
|
|
1713
|
+
};
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
/**
|
|
1717
|
+
* Emit a Tier 0 event to `.pi/supervisor/events.jsonl`.
|
|
1718
|
+
*
|
|
1719
|
+
* Best-effort: creates the directory if needed, appends the event as a
|
|
1720
|
+
* single JSONL line. Failures are logged but never crash the batch.
|
|
1721
|
+
*
|
|
1722
|
+
* @param stateRoot - Root directory for state files (workspace root or repo root)
|
|
1723
|
+
* @param event - The event to emit
|
|
1724
|
+
*
|
|
1725
|
+
* @since TP-039
|
|
1726
|
+
*/
|
|
1727
|
+
export function emitTier0Event(stateRoot: string, event: Tier0Event): void {
|
|
1728
|
+
try {
|
|
1729
|
+
const supervisorDir = join(stateRoot, ".pi", "supervisor");
|
|
1730
|
+
if (!existsSync(supervisorDir)) {
|
|
1731
|
+
mkdirSync(supervisorDir, { recursive: true });
|
|
1732
|
+
}
|
|
1733
|
+
const eventsPath = join(supervisorDir, "events.jsonl");
|
|
1734
|
+
const line = JSON.stringify(event) + "\n";
|
|
1735
|
+
appendFileSync(eventsPath, line);
|
|
1736
|
+
} catch (err: unknown) {
|
|
1737
|
+
// Best-effort: log but don't crash the batch
|
|
1738
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1739
|
+
execLog("batch", event.batchId, `tier0 event write failed: ${msg}`, {
|
|
1740
|
+
eventType: event.type,
|
|
1741
|
+
pattern: event.pattern,
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1746
|
+
|
|
1747
|
+
// ── Engine Event Logging (TP-040) ───────────────────────────────────
|
|
1748
|
+
|
|
1749
|
+
/**
|
|
1750
|
+
* Emit an engine lifecycle event to `.pi/supervisor/events.jsonl`.
|
|
1751
|
+
*
|
|
1752
|
+
* Shares the same JSONL file as Tier 0 events for unified consumption
|
|
1753
|
+
* by the supervisor agent. Engine events cover batch lifecycle transitions
|
|
1754
|
+
* (wave start/end, task completion, merge phases, batch terminal states).
|
|
1755
|
+
*
|
|
1756
|
+
* Best-effort: creates the directory if needed, appends the event as a
|
|
1757
|
+
* single JSONL line. Failures are logged but never crash the batch.
|
|
1758
|
+
*
|
|
1759
|
+
* Also invokes the optional event callback for in-process consumers
|
|
1760
|
+
* (command handler, dashboard).
|
|
1761
|
+
*
|
|
1762
|
+
* @param stateRoot - Root directory for state files (workspace root or repo root)
|
|
1763
|
+
* @param event - The engine event to emit
|
|
1764
|
+
* @param callback - Optional in-process event callback
|
|
1765
|
+
*
|
|
1766
|
+
* @since TP-040
|
|
1767
|
+
*/
|
|
1768
|
+
export function emitEngineEvent(
|
|
1769
|
+
stateRoot: string,
|
|
1770
|
+
event: EngineEvent,
|
|
1771
|
+
callback?: ((event: EngineEvent) => void) | null,
|
|
1772
|
+
): void {
|
|
1773
|
+
// Write to JSONL file (same path as Tier 0 events)
|
|
1774
|
+
try {
|
|
1775
|
+
const supervisorDir = join(stateRoot, ".pi", "supervisor");
|
|
1776
|
+
if (!existsSync(supervisorDir)) {
|
|
1777
|
+
mkdirSync(supervisorDir, { recursive: true });
|
|
1778
|
+
}
|
|
1779
|
+
const eventsPath = join(supervisorDir, "events.jsonl");
|
|
1780
|
+
const line = JSON.stringify(event) + "\n";
|
|
1781
|
+
appendFileSync(eventsPath, line);
|
|
1782
|
+
} catch (err: unknown) {
|
|
1783
|
+
// Best-effort: log but don't crash the batch
|
|
1784
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1785
|
+
execLog("batch", event.batchId, `engine event write failed: ${msg}`, {
|
|
1786
|
+
eventType: event.type,
|
|
1787
|
+
});
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
// Invoke in-process callback
|
|
1791
|
+
if (callback) {
|
|
1792
|
+
try {
|
|
1793
|
+
callback(event);
|
|
1794
|
+
} catch (err: unknown) {
|
|
1795
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1796
|
+
execLog("batch", event.batchId, `engine event callback failed: ${msg}`, {
|
|
1797
|
+
eventType: event.type,
|
|
1798
|
+
});
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
|