pi-better-background-tasks 0.2.16 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -0
- package/package.json +1 -1
- package/src/conditions.ts +7 -0
- package/src/failures.ts +99 -0
- package/src/maintenance.ts +2 -0
- package/src/navigator-provider.ts +9 -3
- package/src/runtime.ts +76 -21
- package/src/sandbox.ts +74 -14
- package/src/shared-callback-batcher.ts +42 -16
- package/src/shared-failure-observations.ts +205 -0
- package/src/shared-sandbox-core.ts +283 -11
- package/src/tools.ts +30 -9
package/README.md
CHANGED
|
@@ -80,6 +80,25 @@ timeout can terminate the local SSH client but the remote process may still be
|
|
|
80
80
|
running. See the detailed usage notes for bootstrap policy, watch conditions,
|
|
81
81
|
timeouts, and v1 non-goals.
|
|
82
82
|
|
|
83
|
+
## Watch conditions
|
|
84
|
+
|
|
85
|
+
JSON conditions require a root-prefixed path, such as `$.status` or
|
|
86
|
+
`$.terminalFailure`; bare keys such as `status` are rejected before the command
|
|
87
|
+
starts. For a command that emits `{"status":"FAILURE","terminalFailure":true}`,
|
|
88
|
+
use:
|
|
89
|
+
|
|
90
|
+
```json
|
|
91
|
+
{
|
|
92
|
+
"success_when": { "type": "json_path_equals", "path": "$.status", "value": "SUCCESS" },
|
|
93
|
+
"failure_when": { "type": "json_path_equals", "path": "$.terminalFailure", "value": true }
|
|
94
|
+
}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Persisted watchers with unsupported paths fail explicitly on their next poll.
|
|
98
|
+
Missing JSON fields or invalid JSON output remain retryable; task status shows
|
|
99
|
+
the condition evaluation error until a subsequent poll recovers. Keep a finite
|
|
100
|
+
timeout to bound watches whose output never becomes evaluable.
|
|
101
|
+
|
|
83
102
|
## Install
|
|
84
103
|
|
|
85
104
|
```sh
|
|
@@ -92,6 +111,27 @@ Try it for one run:
|
|
|
92
111
|
pi -e npm:pi-better-background-tasks
|
|
93
112
|
```
|
|
94
113
|
|
|
114
|
+
## Failure observations
|
|
115
|
+
|
|
116
|
+
Task lifecycle and failure evidence are reported separately. A watch can remain
|
|
117
|
+
`running` while a poll or condition evaluator has failed. Status, list, log, and
|
|
118
|
+
navigator views show unresolved observations before ordinary progress. A success
|
|
119
|
+
match cannot finish a watch while evaluation of its failure condition is broken;
|
|
120
|
+
a definite failure match still terminates it.
|
|
121
|
+
|
|
122
|
+
Observations live in `failures.jsonl` beside task metadata. Recovery requires a
|
|
123
|
+
successful evaluation of the same operation. Repeated failures are grouped, and
|
|
124
|
+
an explicitly configured nonzero success exit is treated as expected. Verbose
|
|
125
|
+
status includes observation details and the journal path. Corrupt or unreadable
|
|
126
|
+
evidence is reported as **observation incomplete**.
|
|
127
|
+
|
|
128
|
+
Unresolved running failures become eligible for attention after 60 seconds;
|
|
129
|
+
observation gaps are eligible immediately. Terminal failures use the normal
|
|
130
|
+
completion notification. A delivery receipt is stored only after handoff;
|
|
131
|
+
notification delivery does not clear the failure. `callback:false` stays quiet
|
|
132
|
+
while all inspection surfaces retain the evidence. Journals follow the task's
|
|
133
|
+
existing retention and explicit-clear behavior.
|
|
134
|
+
|
|
95
135
|
## When To Use
|
|
96
136
|
|
|
97
137
|
Use this package for shell commands that need logs, status, cancellation, or completion notifications across a Pi turn.
|
package/package.json
CHANGED
package/src/conditions.ts
CHANGED
|
@@ -6,6 +6,13 @@ export interface ConditionMatch {
|
|
|
6
6
|
error?: string;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
export function validateCondition(condition: Condition): string | undefined {
|
|
10
|
+
if ((condition.type === "json_path_equals" || condition.type === "json_path_exists") && !parseJsonPath(condition.path)) {
|
|
11
|
+
return `unsupported JSON path: ${condition.path}. Use a root-prefixed path such as $.status or $.terminalFailure.`;
|
|
12
|
+
}
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
|
|
9
16
|
export function evaluateCondition(condition: Condition, result: CommandResult): ConditionMatch {
|
|
10
17
|
switch (condition.type) {
|
|
11
18
|
case "exit_code":
|
package/src/failures.ts
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
3
|
+
import {
|
|
4
|
+
failureAttentionHandled, failureIdentity, formatFailureSummary, markFailureAttentionDelivered,
|
|
5
|
+
observeFailures, pendingFailureAttention, readFailureState,
|
|
6
|
+
} from "./shared-failure-observations.js";
|
|
7
|
+
import { getCallbackBatcher } from "./shared-callback-batcher.js";
|
|
8
|
+
import { readMeta, taskDir } from "./registry.js";
|
|
9
|
+
import type { ActiveSessionProvider } from "./runtime.js";
|
|
10
|
+
import type { BackgroundTaskMeta } from "./types.js";
|
|
11
|
+
|
|
12
|
+
export const failurePath = (id: string): string => join(taskDir(id), "failures.jsonl");
|
|
13
|
+
export const failureSummary = (id: string): string => formatFailureSummary(readFailureState(failurePath(id)));
|
|
14
|
+
|
|
15
|
+
export function recordFailure(meta: BackgroundTaskMeta, operation: string, summary: string, eventKey: unknown,
|
|
16
|
+
options: { category?: string; expected?: boolean; incomplete?: boolean; evidence?: string; at?: number } = {}): void {
|
|
17
|
+
observeFailures(failurePath(meta.id), [{
|
|
18
|
+
id: failureIdentity(meta.id, operation, eventKey), operation,
|
|
19
|
+
kind: options.incomplete ? "incomplete" : "failure", summary,
|
|
20
|
+
category: options.category, expected: options.expected, evidence: options.evidence,
|
|
21
|
+
at: options.at,
|
|
22
|
+
}]);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function recoverFailure(meta: BackgroundTaskMeta, operation: string, eventKey: unknown, at = Date.now()): void {
|
|
26
|
+
const path = failurePath(meta.id);
|
|
27
|
+
const active = readFailureState(path).observations[failureIdentity(operation)];
|
|
28
|
+
if (!active || active.status === "resolved") return;
|
|
29
|
+
observeFailures(path, [{
|
|
30
|
+
id: failureIdentity(meta.id, operation, "recovered", eventKey), operation,
|
|
31
|
+
kind: "recovered", incidents: [active.id], at,
|
|
32
|
+
}]);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const attentionTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
36
|
+
export function stopFailureAttention(id: string): void {
|
|
37
|
+
const timer = attentionTimers.get(id);
|
|
38
|
+
if (timer) clearTimeout(timer);
|
|
39
|
+
attentionTimers.delete(id);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Running incidents get one grace wake. Terminal incidents ride the completion callback. */
|
|
43
|
+
export function scheduleFailureAttention(pi: ExtensionAPI, id: string, getActiveSession?: ActiveSessionProvider): void {
|
|
44
|
+
stopFailureAttention(id);
|
|
45
|
+
const meta = readMeta(id);
|
|
46
|
+
if (!meta) {
|
|
47
|
+
const timer = setTimeout(() => scheduleFailureAttention(pi, id, getActiveSession), 1_000);
|
|
48
|
+
timer.unref();
|
|
49
|
+
attentionTimers.set(id, timer);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (meta.status !== "running" || meta.callback === false || meta.stopRequestedAt) return;
|
|
53
|
+
const state = readFailureState(failurePath(id));
|
|
54
|
+
const pending = pendingFailureAttention(state, Date.now());
|
|
55
|
+
if (pending) {
|
|
56
|
+
const delivery = getCallbackBatcher(pi).deliverUrgent({
|
|
57
|
+
source: "background-task", id: `failure:${id}:${pending.key}`, label: meta.name ?? id,
|
|
58
|
+
status: "failure", customType: "background-task-failure",
|
|
59
|
+
content: `Background task ${id}: ${pending.summary}\nInspect: bg_task_status id=${id}`,
|
|
60
|
+
isDelivered: () => {
|
|
61
|
+
const current = readMeta(id);
|
|
62
|
+
if (!current) throw new Error("Task metadata is unavailable; defer failure notification");
|
|
63
|
+
const now = readFailureState(failurePath(id));
|
|
64
|
+
return current.status !== "running" || failureAttentionHandled(now, pending.incidents);
|
|
65
|
+
},
|
|
66
|
+
getSuppressionReason: () => {
|
|
67
|
+
const current = readMeta(id);
|
|
68
|
+
if (!current) throw new Error("Task metadata is unavailable; defer failure notification");
|
|
69
|
+
if (current.status !== "running" || current.callback === false || current.stopRequestedAt) return "task is no longer running";
|
|
70
|
+
const origin = current.callbackOrigin;
|
|
71
|
+
const active = getActiveSession?.();
|
|
72
|
+
if (origin && (!active || origin.cwd !== active.cwd || (origin.sessionId && origin.sessionId !== active.sessionId))) return "callback origin is not active";
|
|
73
|
+
if (!origin && active && active.cwd !== current.cwd) return "callback cwd is not active";
|
|
74
|
+
return undefined;
|
|
75
|
+
},
|
|
76
|
+
onDelivered: (at) => { markFailureAttentionDelivered(failurePath(id), pending, at); },
|
|
77
|
+
});
|
|
78
|
+
void Promise.resolve(delivery).then((sent) => {
|
|
79
|
+
if (!sent && (!readMeta(id) || readMeta(id)?.status === "running")) {
|
|
80
|
+
const timer = setTimeout(() => scheduleFailureAttention(pi, id, getActiveSession), 1_000);
|
|
81
|
+
timer.unref();
|
|
82
|
+
attentionTimers.set(id, timer);
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
const due = Object.values(state.observations)
|
|
88
|
+
.filter((x) => x.status === "unresolved" && state.delivered[x.id] === undefined)
|
|
89
|
+
.map((x) => x.firstObservedAt + 60_000 - Date.now());
|
|
90
|
+
if (due.length) {
|
|
91
|
+
const timer = setTimeout(() => scheduleFailureAttention(pi, id, getActiveSession), Math.max(1, Math.min(...due)));
|
|
92
|
+
timer.unref();
|
|
93
|
+
attentionTimers.set(id, timer);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function terminalFailureAttention(id: string) {
|
|
98
|
+
return pendingFailureAttention(readFailureState(failurePath(id)), Date.now(), { terminal: true });
|
|
99
|
+
}
|
package/src/maintenance.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
+
import { recordFailure } from "./failures.js";
|
|
3
4
|
import { baseDir, listMetas, removeMeta, writeMeta } from "./registry.js";
|
|
4
5
|
import { processIdentityAlive as defaultProcessIdentityAlive } from "./process-identity.js";
|
|
5
6
|
import type { BackgroundTaskCallbackOrigin, BackgroundTaskMeta } from "./types.js";
|
|
@@ -49,6 +50,7 @@ export function runTaskMaintenance(options: TaskMaintenanceOptions = {}): TaskMa
|
|
|
49
50
|
meta.endedAt = now;
|
|
50
51
|
meta.error = "task supervisor is no longer alive; execution result is unavailable";
|
|
51
52
|
meta.result = { reason: meta.error };
|
|
53
|
+
recordFailure(meta, "execution", meta.error, "supervisor-lost", { incomplete: true, at: now });
|
|
52
54
|
writeMeta(meta);
|
|
53
55
|
reconciled += 1;
|
|
54
56
|
}
|
|
@@ -9,6 +9,8 @@ import {
|
|
|
9
9
|
} from "./shared-navigator.ts";
|
|
10
10
|
import { CustomEditor } from "@earendil-works/pi-coding-agent";
|
|
11
11
|
import { Key, matchesKey, truncateToWidth } from "@earendil-works/pi-tui";
|
|
12
|
+
import { activeFailures, readFailureState } from "./shared-failure-observations.js";
|
|
13
|
+
import { failurePath, failureSummary } from "./failures.js";
|
|
12
14
|
import { readLog } from "./logs.js";
|
|
13
15
|
import { listMetasForOrigin, onMetaChanged, readMeta, writeMeta } from "./registry.js";
|
|
14
16
|
import { stopTask } from "./runtime.js";
|
|
@@ -102,6 +104,7 @@ function isExpiredTerminalNavigatorRow(meta: BackgroundTaskMeta, now: number): b
|
|
|
102
104
|
}
|
|
103
105
|
|
|
104
106
|
function rowFromMeta(meta: BackgroundTaskMeta, now: number): BackgroundWorkRow {
|
|
107
|
+
const failure = failureSummary(meta.id);
|
|
105
108
|
const elapsed = formatDuration((meta.endedAt ?? now) - meta.startedAt);
|
|
106
109
|
return {
|
|
107
110
|
providerId: "background-tasks",
|
|
@@ -111,7 +114,7 @@ function rowFromMeta(meta: BackgroundTaskMeta, now: number): BackgroundWorkRow {
|
|
|
111
114
|
statusTone: toneForStatus(meta.status),
|
|
112
115
|
kind: meta.kind === "command_watch" ? "watch" : "process",
|
|
113
116
|
elapsed,
|
|
114
|
-
primary: compactCommandLabel(meta),
|
|
117
|
+
primary: failure ? failure.split("\n")[0]! : compactCommandLabel(meta),
|
|
115
118
|
command: commandLabel(meta),
|
|
116
119
|
tool: compactCommandLabel(meta),
|
|
117
120
|
secondary: secondaryLabel(meta),
|
|
@@ -125,6 +128,7 @@ function rowFromMeta(meta: BackgroundTaskMeta, now: number): BackgroundWorkRow {
|
|
|
125
128
|
|
|
126
129
|
function detailFromMeta(meta: BackgroundTaskMeta | undefined, now: number, options?: { logTailLines?: number }): BackgroundWorkDetail | null {
|
|
127
130
|
if (!meta) return null;
|
|
131
|
+
const failure = failureSummary(meta.id);
|
|
128
132
|
const log = readLog(meta.logPath, options?.logTailLines ?? 10);
|
|
129
133
|
const command = commandLabel(meta);
|
|
130
134
|
const metadata = [
|
|
@@ -159,7 +163,7 @@ function detailFromMeta(meta: BackgroundTaskMeta | undefined, now: number, optio
|
|
|
159
163
|
title: meta.name || meta.id,
|
|
160
164
|
status: meta.status,
|
|
161
165
|
statusTone: toneForStatus(meta.status),
|
|
162
|
-
subtitle: compactCommandLabel(meta),
|
|
166
|
+
subtitle: failure ? failure.split("\n")[0]! : compactCommandLabel(meta),
|
|
163
167
|
metadata,
|
|
164
168
|
foldedSections: [{
|
|
165
169
|
id: "command",
|
|
@@ -170,7 +174,7 @@ function detailFromMeta(meta: BackgroundTaskMeta | undefined, now: number, optio
|
|
|
170
174
|
}],
|
|
171
175
|
evidence: {
|
|
172
176
|
label: log.truncated ? "log tail" : "log",
|
|
173
|
-
text: log.text || "(log is empty)",
|
|
177
|
+
text: [failure, log.text || "(log is empty)"].filter(Boolean).join("\n"),
|
|
174
178
|
},
|
|
175
179
|
footerActions: [meta.status === "running" ? "x stop" : "x dismiss"],
|
|
176
180
|
};
|
|
@@ -212,6 +216,8 @@ function secondaryLabel(meta: BackgroundTaskMeta): string | undefined {
|
|
|
212
216
|
|
|
213
217
|
function factsForMeta(meta: BackgroundTaskMeta, now: number): string[] {
|
|
214
218
|
const facts: string[] = [];
|
|
219
|
+
const incident = activeFailures(readFailureState(failurePath(meta.id)))[0];
|
|
220
|
+
if (incident) facts.push(`${incident.category === "observation-incomplete" ? "Observation incomplete" : incident.status === "expected" ? "Expected failure" : "Unresolved failure"}: ${incident.summary}`);
|
|
215
221
|
if (meta.status === "running") {
|
|
216
222
|
const stall = observeBackgroundTaskStall(meta, now);
|
|
217
223
|
if (stall.state === "stalled") facts.push("stalled");
|
package/src/runtime.ts
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { statSync } from "node:fs";
|
|
2
2
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
3
3
|
import { appendLine, appendTaskOutput, appendWatchResult, retainLogTail, resolveMaxLogBytes } from "./logs.js";
|
|
4
|
-
import { evaluateCondition } from "./conditions.js";
|
|
4
|
+
import { evaluateCondition, validateCondition } from "./conditions.js";
|
|
5
5
|
import { processExists, runCommandOnce, spawnCommand, stopProcessGroup } from "./process.js";
|
|
6
6
|
import { currentProcessStartToken, readProcessStartToken } from "./process-identity.js";
|
|
7
7
|
import { DEFAULT_TMUX_BOOTSTRAP_TIMEOUT_MS, expandSshRemoteTaskPreset } from "./remote-task-preset.js";
|
|
8
8
|
import type { RemoteRunner, ResolvedSshRemoteTask } from "./remote-task-preset.js";
|
|
9
9
|
import { ensureTaskDir, logPathFor, nextTaskId, readMeta, sandboxProfilePathFor, writeMeta } from "./registry.js";
|
|
10
10
|
import { confineCommandSpec, resolveForegroundSandboxPlan } from "./sandbox.js";
|
|
11
|
+
import { failurePath, recordFailure, recoverFailure, scheduleFailureAttention, stopFailureAttention, terminalFailureAttention } from "./failures.js";
|
|
12
|
+
import { markFailureAttentionDelivered } from "./shared-failure-observations.js";
|
|
11
13
|
import { getCallbackBatcher } from "./shared-callback-batcher.js";
|
|
12
14
|
import type {
|
|
13
15
|
BackgroundTaskCallbackOrigin,
|
|
@@ -34,8 +36,7 @@ const REMOTE_SESSION_POLL_MS = 100;
|
|
|
34
36
|
|
|
35
37
|
export const DEFAULT_WATCH_TIMEOUT_SECONDS = 15 * 60;
|
|
36
38
|
|
|
37
|
-
|
|
38
|
-
const UNCONFINED_LAUNCH = { confined: false } as const;
|
|
39
|
+
|
|
39
40
|
|
|
40
41
|
export type ActiveSessionProvider = () => BackgroundTaskCallbackOrigin | undefined;
|
|
41
42
|
|
|
@@ -75,9 +76,8 @@ export function spawnTask(
|
|
|
75
76
|
dependencies: TaskRuntimeDependencies = {},
|
|
76
77
|
): BackgroundTaskMeta {
|
|
77
78
|
// Resolved before any task directory, log, or metadata exists so a blocked
|
|
78
|
-
// launch leaves nothing behind.
|
|
79
|
-
|
|
80
|
-
const sandboxPlan = params.ssh ? UNCONFINED_LAUNCH : resolveForegroundSandboxPlan(pi);
|
|
79
|
+
// launch leaves nothing behind. Structured SSH must also honor launch restrictions.
|
|
80
|
+
const sandboxPlan = resolveForegroundSandboxPlan(pi, !!params.ssh);
|
|
81
81
|
const id = nextTaskId();
|
|
82
82
|
const cwd = params.cwd ?? defaultCwd;
|
|
83
83
|
const logPath = logPathFor(id);
|
|
@@ -145,6 +145,8 @@ export function spawnTask(
|
|
|
145
145
|
if (!latest) return;
|
|
146
146
|
enforceLogRetention(latest);
|
|
147
147
|
if (isTerminalStatus(latest.status)) return;
|
|
148
|
+
if (exitCode !== 0) recordFailure(latest, "execution", `Process exited with code ${exitCode ?? "unknown"}${signal ? ` (${signal})` : ""}`, "close", { category: "exit", at: Date.now() });
|
|
149
|
+
else recoverFailure(latest, "execution", "close");
|
|
148
150
|
latest.status = exitCode === 0 ? "succeeded" : "failed";
|
|
149
151
|
latest.endedAt = Date.now();
|
|
150
152
|
latest.lastExitCode = exitCode;
|
|
@@ -181,11 +183,13 @@ async function launchRemoteTmux(
|
|
|
181
183
|
appendLine(latest.logPath, `--- remote setup: ${bootstrap.message} ---`);
|
|
182
184
|
writeMeta(latest);
|
|
183
185
|
if (bootstrap.status !== "present" && bootstrap.status !== "installed") {
|
|
186
|
+
recordFailure(latest, "remote-bootstrap", bootstrap.message, "bootstrap", { category: "ssh" });
|
|
184
187
|
latest.error = bootstrap.message;
|
|
185
188
|
finalize(latest, { status: "failed", reason: bootstrap.message }, pi, getActiveSession);
|
|
186
189
|
return;
|
|
187
190
|
}
|
|
188
191
|
|
|
192
|
+
recoverFailure(latest, "remote-bootstrap", "bootstrap");
|
|
189
193
|
const startAttempt = remoteTask.startTmuxSession(bootstrap.tmuxPath);
|
|
190
194
|
remoteSessionStarts.set(id, startAttempt);
|
|
191
195
|
let started: CommandResult;
|
|
@@ -200,9 +204,11 @@ async function launchRemoteTmux(
|
|
|
200
204
|
const detail = started.stderr.trim() || started.stdout.trim() || "remote tmux returned no diagnostic";
|
|
201
205
|
const reason = `Could not create remote tmux session ${afterStart.remote?.sessionName} on ${afterStart.ssh?.target} (exit ${started.exitCode ?? "unknown"}): ${detail}`;
|
|
202
206
|
afterStart.error = reason;
|
|
207
|
+
recordFailure(afterStart, "remote-start", reason, "start", { category: "ssh", at: started.endedAt });
|
|
203
208
|
finalize(afterStart, { status: "failed", reason, commandResult: started }, pi, getActiveSession);
|
|
204
209
|
return;
|
|
205
210
|
}
|
|
211
|
+
recoverFailure(afterStart, "remote-start", "start");
|
|
206
212
|
appendLine(afterStart.logPath, `--- remote tmux session ${afterStart.remote?.sessionName} started on ${afterStart.ssh?.target} ---`);
|
|
207
213
|
afterStart.remote = { ...afterStart.remote!, sessionStarted: true };
|
|
208
214
|
afterStart.lastProgressAt = Date.now();
|
|
@@ -266,12 +272,15 @@ async function pollRemoteSession(
|
|
|
266
272
|
}
|
|
267
273
|
if (poll.status === "missing") {
|
|
268
274
|
const reason = `Remote tmux session ${latest.remote?.sessionName} disappeared on ${latest.ssh?.target} before an exit status was captured.`;
|
|
275
|
+
recordFailure(latest, "remote-session", reason, "missing", { incomplete: true });
|
|
269
276
|
latest.error = reason;
|
|
270
277
|
finalize(latest, { status: "failed", reason }, pi, getActiveSession);
|
|
271
278
|
return;
|
|
272
279
|
}
|
|
273
280
|
const commandResult = { ...poll.commandResult, exitCode: poll.status, stdout: poll.output };
|
|
274
281
|
latest.lastExitCode = poll.status;
|
|
282
|
+
if (poll.status !== 0) recordFailure(latest, "execution", `Remote command exited with code ${poll.status}`, "exit", { category: "exit" });
|
|
283
|
+
else recoverFailure(latest, "execution", "exit");
|
|
275
284
|
finalize(latest, {
|
|
276
285
|
status: poll.status === 0 ? "succeeded" : "failed",
|
|
277
286
|
reason: `remote command exited with code ${poll.status}`,
|
|
@@ -300,6 +309,7 @@ function failRemoteTask(
|
|
|
300
309
|
if (!meta || meta.status !== "running" || meta.stopRequestedAt) return;
|
|
301
310
|
const reason = error instanceof Error ? error.message : String(error);
|
|
302
311
|
meta.error = reason;
|
|
312
|
+
recordFailure(meta, "remote-control", reason, "error", { category: "ssh" });
|
|
303
313
|
finalize(meta, { status: "failed", reason }, pi, getActiveSession);
|
|
304
314
|
}
|
|
305
315
|
|
|
@@ -311,7 +321,11 @@ export function startWatchTask(
|
|
|
311
321
|
getActiveSession?: ActiveSessionProvider,
|
|
312
322
|
dependencies: TaskRuntimeDependencies = {},
|
|
313
323
|
): BackgroundTaskMeta {
|
|
314
|
-
const
|
|
324
|
+
for (const [name, condition] of [["success_when", params.success_when], ["failure_when", params.failure_when]] as const) {
|
|
325
|
+
const error = condition && validateCondition(condition);
|
|
326
|
+
if (error) throw new Error(`${name}: ${error}`);
|
|
327
|
+
}
|
|
328
|
+
const sandboxPlan = resolveForegroundSandboxPlan(pi, !!params.ssh);
|
|
315
329
|
const id = nextTaskId();
|
|
316
330
|
const cwd = params.cwd ?? defaultCwd;
|
|
317
331
|
const now = Date.now();
|
|
@@ -382,6 +396,7 @@ export function resumeRunningTask(
|
|
|
382
396
|
void notifyTerminal(pi, meta, getActiveSession);
|
|
383
397
|
return meta;
|
|
384
398
|
}
|
|
399
|
+
scheduleFailureAttention(pi, meta.id, getActiveSession);
|
|
385
400
|
|
|
386
401
|
if (meta.spawnPid !== process.pid || meta.spawnPidStartTime !== currentProcessStartToken()) {
|
|
387
402
|
meta.spawnPid = process.pid;
|
|
@@ -409,6 +424,7 @@ export function resumeRunningTask(
|
|
|
409
424
|
meta.endedAt = Date.now();
|
|
410
425
|
meta.error = "process is no longer alive; exit result was not captured by this pi session";
|
|
411
426
|
meta.result = { reason: meta.error };
|
|
427
|
+
recordFailure(meta, "execution", meta.error, "lost", { incomplete: true });
|
|
412
428
|
writeMeta(meta);
|
|
413
429
|
void notifyTerminal(pi, meta, getActiveSession);
|
|
414
430
|
return meta;
|
|
@@ -510,6 +526,7 @@ export async function stopTask(
|
|
|
510
526
|
}
|
|
511
527
|
}
|
|
512
528
|
|
|
529
|
+
stopFailureAttention(id);
|
|
513
530
|
meta.status = "cancelled";
|
|
514
531
|
meta.endedAt = Date.now();
|
|
515
532
|
meta.result = {
|
|
@@ -573,35 +590,68 @@ async function pollWatch(
|
|
|
573
590
|
latest.lastSignal = result.signal;
|
|
574
591
|
latest.lastState = extractLastState(result);
|
|
575
592
|
|
|
593
|
+
const pollKey = result.startedAt;
|
|
576
594
|
if (result.timedOut) {
|
|
577
595
|
finalize(latest, { status: "timed_out", reason: watchTimeoutReason(latest), commandResult: result }, pi, getActiveSession);
|
|
578
596
|
return;
|
|
579
597
|
}
|
|
580
598
|
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
599
|
+
const conditionErrors: string[] = [];
|
|
600
|
+
delete latest.error;
|
|
601
|
+
for (const [name, condition] of [["success_when", latest.successWhen], ["failure_when", latest.failureWhen]] as const) {
|
|
602
|
+
const error = condition && validateCondition(condition);
|
|
603
|
+
if (error) {
|
|
604
|
+
latest.error = `${name}: ${error}`;
|
|
605
|
+
if (result.exitCode !== 0) recordFailure(latest, "watch-poll", `Watch poll exited with code ${result.exitCode ?? "unknown"}`, pollKey,
|
|
606
|
+
{ category: "exit", at: result.endedAt });
|
|
607
|
+
recordFailure(latest, name, latest.error, pollKey, { incomplete: true, at: result.endedAt });
|
|
608
|
+
finalize(latest, { status: "failed", reason: latest.error, commandResult: result }, pi, getActiveSession);
|
|
585
609
|
return;
|
|
586
610
|
}
|
|
587
611
|
}
|
|
588
612
|
|
|
613
|
+
// Evaluate both conditions before deciding whether either can terminate the watch.
|
|
614
|
+
const failure = latest.failureWhen ? evaluateCondition(latest.failureWhen, result) : undefined;
|
|
615
|
+
const success = latest.successWhen ? evaluateCondition(latest.successWhen, result) : undefined;
|
|
616
|
+
for (const [name, match] of [["failure_when", failure], ["success_when", success]] as const) {
|
|
617
|
+
if (match?.error) {
|
|
618
|
+
conditionErrors.push(`${name}: ${match.error}`);
|
|
619
|
+
recordFailure(latest, name, `${name}: ${match.error}`, pollKey, { incomplete: true, at: result.endedAt });
|
|
620
|
+
} else if (match) {
|
|
621
|
+
recoverFailure(latest, name, pollKey, result.endedAt);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
589
624
|
const transportFailure = sshTransportFailure(latest, result);
|
|
625
|
+
const expectedPollExit = !transportFailure && !conditionErrors.length && failure?.matched !== true &&
|
|
626
|
+
success?.matched === true && latest.successWhen?.type === "exit_code";
|
|
627
|
+
if (expectedPollExit) recoverFailure(latest, "watch-poll", pollKey, result.endedAt);
|
|
628
|
+
if (transportFailure) recordFailure(latest, "watch-poll", transportFailure, pollKey, { category: "ssh", at: result.endedAt });
|
|
629
|
+
else if (result.exitCode !== 0) recordFailure(latest, "watch-poll", `Watch poll exited with code ${result.exitCode ?? "unknown"}`, pollKey,
|
|
630
|
+
{ category: "exit", expected: expectedPollExit, at: result.endedAt });
|
|
631
|
+
else recoverFailure(latest, "watch-poll", pollKey, result.endedAt);
|
|
632
|
+
if (conditionErrors.length) latest.error = conditionErrors.join("; ");
|
|
633
|
+
if (failure?.matched) {
|
|
634
|
+
recordFailure(latest, "failure_when", "failure condition matched", pollKey, { category: "condition", at: result.endedAt });
|
|
635
|
+
finalize(latest, { status: "failed", reason: "failure condition matched", matchedCondition: latest.failureWhen, commandResult: result }, pi, getActiveSession);
|
|
636
|
+
return;
|
|
637
|
+
}
|
|
590
638
|
if (transportFailure) {
|
|
591
639
|
latest.error = transportFailure;
|
|
592
640
|
finalize(latest, { status: "failed", reason: transportFailure, commandResult: result }, pi, getActiveSession);
|
|
593
641
|
return;
|
|
594
642
|
}
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
643
|
+
if (conditionErrors.length) {
|
|
644
|
+
writeMeta(latest);
|
|
645
|
+
scheduleFailureAttention(pi, id, getActiveSession);
|
|
646
|
+
scheduleWatch(pi, id, nextWatchDelayMs(latest), getActiveSession, runOnce);
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
if (success?.matched) {
|
|
650
|
+
finalize(latest, { status: "succeeded", reason: "success condition matched", matchedCondition: latest.successWhen, commandResult: result }, pi, getActiveSession);
|
|
651
|
+
return;
|
|
602
652
|
}
|
|
603
|
-
|
|
604
653
|
writeMeta(latest);
|
|
654
|
+
scheduleFailureAttention(pi, id, getActiveSession);
|
|
605
655
|
scheduleWatch(pi, id, nextWatchDelayMs(latest), getActiveSession, runOnce);
|
|
606
656
|
} catch (error) {
|
|
607
657
|
const meta = readMeta(id);
|
|
@@ -612,6 +662,7 @@ async function pollWatch(
|
|
|
612
662
|
meta.error = reason;
|
|
613
663
|
appendLine(meta.logPath, `--- poll error ${new Date().toISOString()} ---\n${reason}`);
|
|
614
664
|
}
|
|
665
|
+
recordFailure(meta, "watch-poll", reason, `throw:${meta.lastCheckedAt ?? meta.startedAt}`, { category: meta.ssh ? "ssh" : "execution" });
|
|
615
666
|
finalize(meta, { status: "failed", reason }, pi, getActiveSession);
|
|
616
667
|
}
|
|
617
668
|
} finally {
|
|
@@ -625,6 +676,8 @@ function finalize(
|
|
|
625
676
|
pi: ExtensionAPI,
|
|
626
677
|
getActiveSession?: ActiveSessionProvider,
|
|
627
678
|
): void {
|
|
679
|
+
stopFailureAttention(meta.id);
|
|
680
|
+
if (terminal.status === "timed_out") recordFailure(meta, "timeout", terminal.reason, "deadline", { category: "timeout" });
|
|
628
681
|
meta.status = terminal.status;
|
|
629
682
|
meta.endedAt = Date.now();
|
|
630
683
|
meta.result = {
|
|
@@ -782,12 +835,13 @@ async function notifyTerminal(
|
|
|
782
835
|
writeMeta(latest);
|
|
783
836
|
return;
|
|
784
837
|
}
|
|
838
|
+
const pending = terminalFailureAttention(latest.id);
|
|
785
839
|
const label = latest.name ? `${latest.name} (${latest.id})` : latest.id;
|
|
786
840
|
getCallbackBatcher(pi).enqueue({
|
|
787
841
|
source: "background-task",
|
|
788
842
|
id: latest.id,
|
|
789
843
|
label,
|
|
790
|
-
status: latest.status,
|
|
844
|
+
status: pending ? `${latest.status}: ${pending.summary}` : latest.status,
|
|
791
845
|
detailTool: "bg_task_status",
|
|
792
846
|
callback: true,
|
|
793
847
|
isDelivered: () => {
|
|
@@ -796,7 +850,7 @@ async function notifyTerminal(
|
|
|
796
850
|
},
|
|
797
851
|
getSuppressionReason: () => {
|
|
798
852
|
const current = readMeta(latest.id);
|
|
799
|
-
if (!current)
|
|
853
|
+
if (!current) throw new Error("Background task metadata is unavailable; defer completion");
|
|
800
854
|
return getCallbackSuppressionReason(current, getActiveSession?.());
|
|
801
855
|
},
|
|
802
856
|
onDelivered: (at) => {
|
|
@@ -804,6 +858,7 @@ async function notifyTerminal(
|
|
|
804
858
|
if (!current || current.callbackSentAt !== undefined || current.callbackSuppressedAt !== undefined) return;
|
|
805
859
|
current.callbackSentAt = at;
|
|
806
860
|
writeMeta(current);
|
|
861
|
+
if (pending) markFailureAttentionDelivered(failurePath(latest.id), pending, at);
|
|
807
862
|
},
|
|
808
863
|
onSuppressed: (reason, at) => {
|
|
809
864
|
const current = readMeta(latest.id);
|