pi-better-background-tasks 0.2.20 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -2
- package/package.json +1 -1
- package/src/failures.ts +101 -6
- package/src/index.ts +5 -0
- package/src/logs.ts +51 -7
- package/src/navigator-provider.ts +4 -3
- package/src/output.ts +783 -0
- package/src/process.ts +95 -8
- package/src/registry.ts +84 -7
- package/src/runtime.ts +228 -33
- package/src/sandbox.ts +6 -4
- package/src/shared-callback-batcher.ts +358 -29
- package/src/shared-failure-observations.ts +833 -23
- package/src/shared-log-utils.ts +1139 -2
- package/src/shared-navigator.ts +12 -10
- package/src/shared-sandbox-core.ts +660 -17
- package/src/tools.ts +201 -156
- package/src/types.ts +23 -0
package/src/runtime.ts
CHANGED
|
@@ -8,9 +8,11 @@ import { DEFAULT_TMUX_BOOTSTRAP_TIMEOUT_MS, expandSshRemoteTaskPreset } from "./
|
|
|
8
8
|
import type { RemoteRunner, ResolvedSshRemoteTask } from "./remote-task-preset.js";
|
|
9
9
|
import { ensureTaskDir, logPathFor, nextTaskId, readMeta, sandboxProfilePathFor, writeMeta } from "./registry.js";
|
|
10
10
|
import { confineCommandSpec, resolveForegroundSandboxPlan } from "./sandbox.js";
|
|
11
|
-
import {
|
|
11
|
+
import { takeRecoverySnapshot } from "./shared-sandbox-core.js";
|
|
12
|
+
import { failurePath, readTaskIntent, recordExitFailure, recordFailure, recoverDeclaredOperation, recoverFailure, resumeFailureAttention, scheduleFailureAttention, stopFailureAttention, suspendFailureAttention, terminalFailureAttention } from "./failures.js";
|
|
12
13
|
import { markFailureAttentionDelivered } from "./shared-failure-observations.js";
|
|
13
14
|
import { getCallbackBatcher } from "./shared-callback-batcher.js";
|
|
15
|
+
import { formatCallbackFacts } from "./output.js";
|
|
14
16
|
import type {
|
|
15
17
|
BackgroundTaskCallbackOrigin,
|
|
16
18
|
BackgroundTaskMeta,
|
|
@@ -32,16 +34,68 @@ const remoteSessionStarts = new Map<string, Promise<CommandResult>>();
|
|
|
32
34
|
const activePolls = new Set<string>();
|
|
33
35
|
const logRetentionTimers = new Map<string, ReturnType<typeof setInterval>>();
|
|
34
36
|
const LOG_RETENTION_CHECK_MS = 1000;
|
|
37
|
+
const handoffTimers = new Map<string, ReturnType<typeof setInterval>>();
|
|
38
|
+
const HANDOFF_CHECK_MS = 250;
|
|
39
|
+
const HANDOFF_LOST_GRACE_MS = 5_000;
|
|
40
|
+
let scheduledWorkSuspended = false;
|
|
35
41
|
const REMOTE_SESSION_POLL_MS = 100;
|
|
36
42
|
|
|
37
43
|
export const DEFAULT_WATCH_TIMEOUT_SECONDS = 15 * 60;
|
|
38
44
|
|
|
45
|
+
/**
|
|
46
|
+
* Stop this extension instance's scheduled work when its session shuts down (#324).
|
|
47
|
+
*
|
|
48
|
+
* Pi emits session_shutdown for /reload, /new, /resume, fork, session switch,
|
|
49
|
+
* and quit, and the next session loads a fresh extension instance. Work left in
|
|
50
|
+
* the old instance runs with no active session, so a task finishing there used to
|
|
51
|
+
* have its completion callback durably suppressed ("active session identity is
|
|
52
|
+
* unavailable") and the next session never heard about it.
|
|
53
|
+
*
|
|
54
|
+
* After suspension this instance schedules nothing and notifies nothing. Work
|
|
55
|
+
* already in flight (a child's close listener, a remote tmux bootstrap/start, a
|
|
56
|
+
* timeout kill) still records its result in metadata, but leaves the callback to
|
|
57
|
+
* the instance that resumes the task. Processes are not stopped.
|
|
58
|
+
*
|
|
59
|
+
* Resume is per callback origin: session_start resumes only tasks whose origin
|
|
60
|
+
* (cwd and session id) matches the new session. After /reload that is every task
|
|
61
|
+
* of the session. After switching to another session, the previous session's
|
|
62
|
+
* tasks keep running, but their watches do not poll, remote tmux output is not
|
|
63
|
+
* collected, and their `timeout_seconds` deadlines are not enforced until that
|
|
64
|
+
* session is active again; an overdue deadline is enforced immediately on resume.
|
|
65
|
+
* A local process that exits meanwhile is recorded as terminal and its callback
|
|
66
|
+
* is delivered when its session resumes.
|
|
67
|
+
*/
|
|
68
|
+
export function suspendScheduledWork(): void {
|
|
69
|
+
scheduledWorkSuspended = true;
|
|
70
|
+
for (const timer of watcherTimers.values()) clearTimeout(timer);
|
|
71
|
+
for (const timer of remoteSessionTimers.values()) clearTimeout(timer);
|
|
72
|
+
for (const timer of processTimeoutTimers.values()) clearTimeout(timer);
|
|
73
|
+
for (const timer of logRetentionTimers.values()) clearInterval(timer);
|
|
74
|
+
for (const timer of handoffTimers.values()) clearInterval(timer);
|
|
75
|
+
handoffTimers.clear();
|
|
76
|
+
watcherTimers.clear();
|
|
77
|
+
remoteSessionTimers.clear();
|
|
78
|
+
processTimeoutTimers.clear();
|
|
79
|
+
logRetentionTimers.clear();
|
|
80
|
+
suspendFailureAttention();
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Allow scheduling again; called at session_start before running tasks are resumed. */
|
|
84
|
+
export function resumeScheduledWork(): void {
|
|
85
|
+
scheduledWorkSuspended = false;
|
|
86
|
+
resumeFailureAttention();
|
|
87
|
+
}
|
|
88
|
+
|
|
39
89
|
|
|
40
90
|
|
|
41
91
|
export type ActiveSessionProvider = () => BackgroundTaskCallbackOrigin | undefined;
|
|
42
92
|
|
|
43
93
|
export interface SpawnTaskParams extends CommandSpec {
|
|
44
94
|
name?: string;
|
|
95
|
+
/** Structured intent (#325): stable id shared by modified retries of one operation. */
|
|
96
|
+
operation_id?: string | null;
|
|
97
|
+
/** Structured intent (#325): non-zero exit codes declared intentional before launch. */
|
|
98
|
+
expected_exit_codes?: number[] | null;
|
|
45
99
|
callback?: boolean;
|
|
46
100
|
timeout_seconds?: number;
|
|
47
101
|
max_log_bytes?: number;
|
|
@@ -51,6 +105,10 @@ export interface SpawnTaskParams extends CommandSpec {
|
|
|
51
105
|
|
|
52
106
|
export interface WatchTaskParams extends CommandSpec {
|
|
53
107
|
name?: string;
|
|
108
|
+
/** Structured intent (#325): stable id shared by modified retries of one operation. */
|
|
109
|
+
operation_id?: string | null;
|
|
110
|
+
/** Structured intent (#325): non-zero exit codes declared intentional before launch. */
|
|
111
|
+
expected_exit_codes?: number[] | null;
|
|
54
112
|
callback?: boolean;
|
|
55
113
|
interval_seconds?: number;
|
|
56
114
|
timeout_seconds?: number;
|
|
@@ -77,6 +135,7 @@ export function spawnTask(
|
|
|
77
135
|
): BackgroundTaskMeta {
|
|
78
136
|
// Resolved before any task directory, log, or metadata exists so a blocked
|
|
79
137
|
// launch leaves nothing behind. Structured SSH must also honor launch restrictions.
|
|
138
|
+
const intent = readTaskIntent(params);
|
|
80
139
|
const sandboxPlan = resolveForegroundSandboxPlan(pi, !!params.ssh);
|
|
81
140
|
const id = nextTaskId();
|
|
82
141
|
const cwd = params.cwd ?? defaultCwd;
|
|
@@ -98,11 +157,21 @@ export function spawnTask(
|
|
|
98
157
|
? commandSpec
|
|
99
158
|
: confineCommandSpec(commandSpec, sandboxPlan, sandboxProfilePathFor(id));
|
|
100
159
|
const tmuxBacked = remoteTask?.metadata.remote.session === "tmux";
|
|
160
|
+
// Outside project = Write or Write & delete can change files outside the
|
|
161
|
+
// project: start an APFS local snapshot (macOS, background, never blocks).
|
|
162
|
+
const snapshot = !remoteTask && sandboxPlan.confined ? takeRecoverySnapshot(sandboxPlan.permissions) : undefined;
|
|
101
163
|
const spawned = tmuxBacked
|
|
102
164
|
? undefined
|
|
103
165
|
: remoteTask
|
|
104
166
|
? remoteTask.spawn(logPath, true)
|
|
105
167
|
: spawnCommand(launchSpec, logPath, true);
|
|
168
|
+
if (snapshot?.started) {
|
|
169
|
+
void snapshot.done.then((outcome) => {
|
|
170
|
+
if (!outcome.ok) {
|
|
171
|
+
try { appendLine(logPath, `--- recovery snapshot failed (task continues): ${outcome.detail} ---`); } catch { /* log gone */ }
|
|
172
|
+
}
|
|
173
|
+
});
|
|
174
|
+
}
|
|
106
175
|
const now = Date.now();
|
|
107
176
|
const meta: BackgroundTaskMeta = {
|
|
108
177
|
id,
|
|
@@ -129,6 +198,7 @@ export function spawnTask(
|
|
|
129
198
|
spawnPidStartTime: currentProcessStartToken(),
|
|
130
199
|
ssh: remoteTask?.metadata.ssh,
|
|
131
200
|
remote: remoteTask?.metadata.remote,
|
|
201
|
+
...intent,
|
|
132
202
|
};
|
|
133
203
|
writeMeta(meta);
|
|
134
204
|
scheduleLogRetention(id);
|
|
@@ -145,8 +215,11 @@ export function spawnTask(
|
|
|
145
215
|
if (!latest) return;
|
|
146
216
|
enforceLogRetention(latest);
|
|
147
217
|
if (isTerminalStatus(latest.status)) return;
|
|
148
|
-
if (exitCode !== 0)
|
|
149
|
-
else
|
|
218
|
+
if (exitCode !== 0) recordExitFailure(latest, "execution", `Process exited with code ${exitCode ?? "unknown"}${signal ? ` (${signal})` : ""}`, exitCode, "close", { at: Date.now() });
|
|
219
|
+
else {
|
|
220
|
+
recoverFailure(latest, "execution", "close");
|
|
221
|
+
recoverDeclaredOperation(latest);
|
|
222
|
+
}
|
|
150
223
|
latest.status = exitCode === 0 ? "succeeded" : "failed";
|
|
151
224
|
latest.endedAt = Date.now();
|
|
152
225
|
latest.lastExitCode = exitCode;
|
|
@@ -226,6 +299,7 @@ function scheduleRemoteSessionPoll(
|
|
|
226
299
|
getActiveSession?: ActiveSessionProvider,
|
|
227
300
|
): void {
|
|
228
301
|
clearRemoteSessionTimer(id);
|
|
302
|
+
if (scheduledWorkSuspended) return;
|
|
229
303
|
const timer = setTimeout(() => void pollRemoteSession(pi, id, getActiveSession), delayMs);
|
|
230
304
|
timer.unref();
|
|
231
305
|
remoteSessionTimers.set(id, timer);
|
|
@@ -249,6 +323,7 @@ async function pollRemoteSession(
|
|
|
249
323
|
const remoteTask = activeRemoteTasks.get(id);
|
|
250
324
|
if (!meta || meta.status !== "running" || meta.remote?.session !== "tmux" || !remoteTask) return;
|
|
251
325
|
const poll = await remoteTask.pollTmuxSession(meta.remote.logOffset ?? 0, remainingDeadlineMs(meta.deadlineAt));
|
|
326
|
+
if (scheduledWorkSuspended) return;
|
|
252
327
|
appendTaskOutput(meta.logPath, poll.output);
|
|
253
328
|
if (poll.status === "timed_out") {
|
|
254
329
|
clearProcessTimeout(id);
|
|
@@ -279,7 +354,7 @@ async function pollRemoteSession(
|
|
|
279
354
|
}
|
|
280
355
|
const commandResult = { ...poll.commandResult, exitCode: poll.status, stdout: poll.output };
|
|
281
356
|
latest.lastExitCode = poll.status;
|
|
282
|
-
if (poll.status !== 0)
|
|
357
|
+
if (poll.status !== 0) recordExitFailure(latest, "execution", `Remote command exited with code ${poll.status}`, poll.status, "exit");
|
|
283
358
|
else recoverFailure(latest, "execution", "exit");
|
|
284
359
|
finalize(latest, {
|
|
285
360
|
status: poll.status === 0 ? "succeeded" : "failed",
|
|
@@ -325,6 +400,7 @@ export function startWatchTask(
|
|
|
325
400
|
const error = condition && validateCondition(condition);
|
|
326
401
|
if (error) throw new Error(`${name}: ${error}`);
|
|
327
402
|
}
|
|
403
|
+
const intent = readTaskIntent(params);
|
|
328
404
|
const sandboxPlan = resolveForegroundSandboxPlan(pi, !!params.ssh);
|
|
329
405
|
const id = nextTaskId();
|
|
330
406
|
const cwd = params.cwd ?? defaultCwd;
|
|
@@ -370,6 +446,7 @@ export function startWatchTask(
|
|
|
370
446
|
notifyOn: "terminal",
|
|
371
447
|
ssh: remoteTask?.metadata.ssh,
|
|
372
448
|
remote: remoteTask?.metadata.remote,
|
|
449
|
+
...intent,
|
|
373
450
|
};
|
|
374
451
|
ensureTaskDir(id);
|
|
375
452
|
appendLine(meta.logPath, `--- watch ${new Date(now).toISOString()} interval_ms=${meta.intervalMs} ---`);
|
|
@@ -398,6 +475,9 @@ export function resumeRunningTask(
|
|
|
398
475
|
}
|
|
399
476
|
scheduleFailureAttention(pi, meta.id, getActiveSession);
|
|
400
477
|
|
|
478
|
+
// Same Pi process: an earlier (reloaded or switched-away) extension instance may
|
|
479
|
+
// still hold this task's child or in-flight remote work and record its result.
|
|
480
|
+
const ownedByThisProcess = meta.spawnPid === process.pid && meta.spawnPidStartTime === currentProcessStartToken();
|
|
401
481
|
if (meta.spawnPid !== process.pid || meta.spawnPidStartTime !== currentProcessStartToken()) {
|
|
402
482
|
meta.spawnPid = process.pid;
|
|
403
483
|
meta.spawnPidStartTime = currentProcessStartToken();
|
|
@@ -417,22 +497,86 @@ export function resumeRunningTask(
|
|
|
417
497
|
activeRemoteTasks.set(meta.id, remoteTask);
|
|
418
498
|
scheduleRemoteSessionPoll(pi, meta.id, 0, getActiveSession);
|
|
419
499
|
if (meta.deadlineAt) scheduleProcessTimeout(pi, meta.id, meta.deadlineAt, getActiveSession);
|
|
500
|
+
// An earlier instance's in-flight timeout kill may still finalize it.
|
|
501
|
+
if (ownedByThisProcess) scheduleHandoff(pi, meta.id, getActiveSession);
|
|
420
502
|
return meta;
|
|
421
503
|
}
|
|
422
|
-
if (
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
meta.
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
writeMeta(meta);
|
|
429
|
-
void notifyTerminal(pi, meta, getActiveSession);
|
|
504
|
+
if (ownedByThisProcess) {
|
|
505
|
+
// A dead pid is not yet "lost" here: the earlier instance's close listener may
|
|
506
|
+
// still record the real exit. The handoff marks it lost after a grace period.
|
|
507
|
+
scheduleHandoff(pi, meta.id, getActiveSession, remoteTask);
|
|
508
|
+
} else if (meta.pid && !processExists(meta.pid)) {
|
|
509
|
+
markProcessLost(pi, meta, getActiveSession);
|
|
430
510
|
return meta;
|
|
431
511
|
}
|
|
432
512
|
if (meta.deadlineAt) scheduleProcessTimeout(pi, meta.id, meta.deadlineAt, getActiveSession);
|
|
433
513
|
return meta;
|
|
434
514
|
}
|
|
435
515
|
|
|
516
|
+
/**
|
|
517
|
+
* Watch a task whose driver may still be an earlier instance in this process: a
|
|
518
|
+
* local child's close listener, or a remote tmux launch that was mid-bootstrap.
|
|
519
|
+
* Deliver the completion once it is recorded terminal, and take over remote
|
|
520
|
+
* polling once the tmux session is recorded as started (#324).
|
|
521
|
+
*/
|
|
522
|
+
function scheduleHandoff(
|
|
523
|
+
pi: ExtensionAPI,
|
|
524
|
+
id: string,
|
|
525
|
+
getActiveSession?: ActiveSessionProvider,
|
|
526
|
+
remoteTask?: ResolvedSshRemoteTask,
|
|
527
|
+
): void {
|
|
528
|
+
stopHandoff(id);
|
|
529
|
+
if (scheduledWorkSuspended) return;
|
|
530
|
+
let deadSince: number | undefined;
|
|
531
|
+
const timer = setInterval(() => {
|
|
532
|
+
const meta = readMeta(id);
|
|
533
|
+
if (!meta) {
|
|
534
|
+
stopHandoff(id);
|
|
535
|
+
return;
|
|
536
|
+
}
|
|
537
|
+
if (meta.status !== "running") {
|
|
538
|
+
stopHandoff(id);
|
|
539
|
+
clearProcessTimeout(id);
|
|
540
|
+
stopLogRetention(id);
|
|
541
|
+
void notifyTerminal(pi, meta, getActiveSession);
|
|
542
|
+
return;
|
|
543
|
+
}
|
|
544
|
+
if (meta.remote?.session === "tmux" && meta.remote.sessionStarted === true && remoteTask && !activeRemoteTasks.has(id)) {
|
|
545
|
+
stopHandoff(id);
|
|
546
|
+
activeRemoteTasks.set(id, remoteTask);
|
|
547
|
+
scheduleRemoteSessionPoll(pi, id, 0, getActiveSession);
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
if (meta.remote?.session !== "tmux" && meta.pid && !processExists(meta.pid)) {
|
|
551
|
+
deadSince ??= Date.now();
|
|
552
|
+
if (Date.now() - deadSince >= HANDOFF_LOST_GRACE_MS) {
|
|
553
|
+
stopHandoff(id);
|
|
554
|
+
clearProcessTimeout(id);
|
|
555
|
+
stopLogRetention(id);
|
|
556
|
+
markProcessLost(pi, meta, getActiveSession);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}, HANDOFF_CHECK_MS);
|
|
560
|
+
timer.unref();
|
|
561
|
+
handoffTimers.set(id, timer);
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
function stopHandoff(id: string): void {
|
|
565
|
+
const timer = handoffTimers.get(id);
|
|
566
|
+
if (timer) clearInterval(timer);
|
|
567
|
+
handoffTimers.delete(id);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
function markProcessLost(pi: ExtensionAPI, meta: BackgroundTaskMeta, getActiveSession?: ActiveSessionProvider): void {
|
|
571
|
+
meta.status = "failed";
|
|
572
|
+
meta.endedAt = Date.now();
|
|
573
|
+
meta.error = "process is no longer alive; exit result was not captured by this pi session";
|
|
574
|
+
meta.result = { reason: meta.error };
|
|
575
|
+
recordFailure(meta, "execution", meta.error, "lost", { incomplete: true });
|
|
576
|
+
writeMeta(meta);
|
|
577
|
+
void notifyTerminal(pi, meta, getActiveSession);
|
|
578
|
+
}
|
|
579
|
+
|
|
436
580
|
function resolvePersistedRemoteTask(
|
|
437
581
|
meta: BackgroundTaskMeta,
|
|
438
582
|
remoteRunner?: RemoteRunner,
|
|
@@ -487,9 +631,7 @@ export async function stopTask(
|
|
|
487
631
|
if (remoteSessionMayExist) {
|
|
488
632
|
const remoteTask = activeRemoteTasks.get(id);
|
|
489
633
|
if (!remoteTask) {
|
|
490
|
-
meta.
|
|
491
|
-
meta.error = `Cannot stop remote tmux session ${remote.sessionName}: its active SSH controller is unavailable.`;
|
|
492
|
-
writeMeta(meta);
|
|
634
|
+
recordStopError(meta, `Cannot stop remote tmux session ${remote.sessionName}: its active SSH controller is unavailable.`);
|
|
493
635
|
scheduleRemoteSessionPoll(pi, id, REMOTE_SESSION_POLL_MS, getActiveSession);
|
|
494
636
|
return meta;
|
|
495
637
|
}
|
|
@@ -497,18 +639,14 @@ export async function stopTask(
|
|
|
497
639
|
const stopped = await remoteTask.killTmuxSession();
|
|
498
640
|
if (stopped.exitCode !== 0) {
|
|
499
641
|
const detail = stopped.stderr.trim() || stopped.stdout.trim() || "remote tmux returned no diagnostic";
|
|
500
|
-
meta.
|
|
501
|
-
meta.error = `Could not kill remote tmux session ${remote.sessionName} on ${meta.ssh?.target} (exit ${stopped.exitCode ?? "unknown"}): ${detail}`;
|
|
502
|
-
writeMeta(meta);
|
|
642
|
+
recordStopError(meta, `Could not kill remote tmux session ${remote.sessionName} on ${meta.ssh?.target} (exit ${stopped.exitCode ?? "unknown"}): ${detail}`);
|
|
503
643
|
scheduleRemoteSessionPoll(pi, id, REMOTE_SESSION_POLL_MS, getActiveSession);
|
|
504
644
|
return meta;
|
|
505
645
|
}
|
|
506
646
|
remote.stopMessage = `Killed remote tmux session ${remote.sessionName} on ${meta.ssh?.target}.`;
|
|
507
647
|
appendLine(meta.logPath, `--- ${remote.stopMessage} ---`);
|
|
508
648
|
} catch (error) {
|
|
509
|
-
meta.
|
|
510
|
-
meta.error = error instanceof Error ? error.message : String(error);
|
|
511
|
-
writeMeta(meta);
|
|
649
|
+
recordStopError(meta, error instanceof Error ? error.message : String(error));
|
|
512
650
|
scheduleRemoteSessionPoll(pi, id, REMOTE_SESSION_POLL_MS, getActiveSession);
|
|
513
651
|
return meta;
|
|
514
652
|
}
|
|
@@ -516,9 +654,7 @@ export async function stopTask(
|
|
|
516
654
|
try {
|
|
517
655
|
stopProcessGroup(meta.pid, meta.pgid);
|
|
518
656
|
} catch (error) {
|
|
519
|
-
meta.
|
|
520
|
-
meta.error = error instanceof Error ? error.message : String(error);
|
|
521
|
-
writeMeta(meta);
|
|
657
|
+
recordStopError(meta, error instanceof Error ? error.message : String(error));
|
|
522
658
|
if (meta.deadlineAt && meta.deadlineAt > Date.now()) {
|
|
523
659
|
scheduleProcessTimeout(pi, id, meta.deadlineAt, getActiveSession);
|
|
524
660
|
}
|
|
@@ -528,6 +664,7 @@ export async function stopTask(
|
|
|
528
664
|
|
|
529
665
|
stopFailureAttention(id);
|
|
530
666
|
meta.status = "cancelled";
|
|
667
|
+
meta.stopError = undefined;
|
|
531
668
|
meta.endedAt = Date.now();
|
|
532
669
|
meta.result = {
|
|
533
670
|
reason: meta.remote?.session === "direct"
|
|
@@ -541,6 +678,13 @@ export async function stopTask(
|
|
|
541
678
|
return meta;
|
|
542
679
|
}
|
|
543
680
|
|
|
681
|
+
function recordStopError(meta: BackgroundTaskMeta, message: string): void {
|
|
682
|
+
meta.stopRequestedAt = undefined;
|
|
683
|
+
meta.stopError = message;
|
|
684
|
+
meta.error = message;
|
|
685
|
+
writeMeta(meta);
|
|
686
|
+
}
|
|
687
|
+
|
|
544
688
|
function scheduleWatch(
|
|
545
689
|
pi: ExtensionAPI,
|
|
546
690
|
id: string,
|
|
@@ -549,6 +693,7 @@ function scheduleWatch(
|
|
|
549
693
|
runOnce?: WatchPollRunner,
|
|
550
694
|
): void {
|
|
551
695
|
clearWatchTimer(id);
|
|
696
|
+
if (scheduledWorkSuspended) return;
|
|
552
697
|
const timer = setTimeout(() => void pollWatch(pi, id, getActiveSession, runOnce), delayMs);
|
|
553
698
|
timer.unref();
|
|
554
699
|
watcherTimers.set(id, timer);
|
|
@@ -580,9 +725,12 @@ async function pollWatch(
|
|
|
580
725
|
const result = runOnce
|
|
581
726
|
? await runOnce(timeoutMs)
|
|
582
727
|
: await runCommandOnce(commandSpecFromMeta(meta), undefined, timeoutMs);
|
|
728
|
+
// A poll that was in flight when the session shut down belongs to a stale instance.
|
|
729
|
+
if (scheduledWorkSuspended) return;
|
|
583
730
|
appendWatchResult(meta.logPath, result);
|
|
584
731
|
const latest = readMeta(id);
|
|
585
732
|
if (!latest || latest.status !== "running") return;
|
|
733
|
+
applyCaptureOverflow(latest, result);
|
|
586
734
|
enforceLogRetention(latest);
|
|
587
735
|
latest.lastCheckedAt = Date.now();
|
|
588
736
|
latest.lastProgressAt = latest.lastCheckedAt;
|
|
@@ -602,8 +750,8 @@ async function pollWatch(
|
|
|
602
750
|
const error = condition && validateCondition(condition);
|
|
603
751
|
if (error) {
|
|
604
752
|
latest.error = `${name}: ${error}`;
|
|
605
|
-
if (result.exitCode !== 0)
|
|
606
|
-
{
|
|
753
|
+
if (result.exitCode !== 0) recordExitFailure(latest, "watch-poll", `Watch poll exited with code ${result.exitCode ?? "unknown"}`, result.exitCode, pollKey,
|
|
754
|
+
{ at: result.endedAt });
|
|
607
755
|
recordFailure(latest, name, latest.error, pollKey, { incomplete: true, at: result.endedAt });
|
|
608
756
|
finalize(latest, { status: "failed", reason: latest.error, commandResult: result }, pi, getActiveSession);
|
|
609
757
|
return;
|
|
@@ -626,13 +774,19 @@ async function pollWatch(
|
|
|
626
774
|
success?.matched === true && latest.successWhen?.type === "exit_code";
|
|
627
775
|
if (expectedPollExit) recoverFailure(latest, "watch-poll", pollKey, result.endedAt);
|
|
628
776
|
if (transportFailure) recordFailure(latest, "watch-poll", transportFailure, pollKey, { category: "ssh", at: result.endedAt });
|
|
629
|
-
else if (result.exitCode !== 0)
|
|
630
|
-
{
|
|
777
|
+
else if (result.exitCode !== 0) recordExitFailure(latest, "watch-poll", `Watch poll exited with code ${result.exitCode ?? "unknown"}`, result.exitCode, pollKey,
|
|
778
|
+
{ expected: expectedPollExit, at: result.endedAt });
|
|
631
779
|
else recoverFailure(latest, "watch-poll", pollKey, result.endedAt);
|
|
632
780
|
if (conditionErrors.length) latest.error = conditionErrors.join("; ");
|
|
633
781
|
if (failure?.matched) {
|
|
634
782
|
recordFailure(latest, "failure_when", "failure condition matched", pollKey, { category: "condition", at: result.endedAt });
|
|
635
|
-
finalize(latest, {
|
|
783
|
+
finalize(latest, {
|
|
784
|
+
status: "failed",
|
|
785
|
+
reason: "failure condition matched",
|
|
786
|
+
matchedCondition: latest.failureWhen,
|
|
787
|
+
matchedValue: failure.value,
|
|
788
|
+
commandResult: result,
|
|
789
|
+
}, pi, getActiveSession);
|
|
636
790
|
return;
|
|
637
791
|
}
|
|
638
792
|
if (transportFailure) {
|
|
@@ -647,7 +801,13 @@ async function pollWatch(
|
|
|
647
801
|
return;
|
|
648
802
|
}
|
|
649
803
|
if (success?.matched) {
|
|
650
|
-
finalize(latest, {
|
|
804
|
+
finalize(latest, {
|
|
805
|
+
status: "succeeded",
|
|
806
|
+
reason: "success condition matched",
|
|
807
|
+
matchedCondition: latest.successWhen,
|
|
808
|
+
matchedValue: success.value,
|
|
809
|
+
commandResult: result,
|
|
810
|
+
}, pi, getActiveSession);
|
|
651
811
|
return;
|
|
652
812
|
}
|
|
653
813
|
writeMeta(latest);
|
|
@@ -678,15 +838,18 @@ function finalize(
|
|
|
678
838
|
): void {
|
|
679
839
|
stopFailureAttention(meta.id);
|
|
680
840
|
if (terminal.status === "timed_out") recordFailure(meta, "timeout", terminal.reason, "deadline", { category: "timeout" });
|
|
841
|
+
if (terminal.status === "succeeded") recoverDeclaredOperation(meta);
|
|
681
842
|
meta.status = terminal.status;
|
|
682
843
|
meta.endedAt = Date.now();
|
|
683
844
|
meta.result = {
|
|
684
845
|
reason: terminal.reason,
|
|
685
846
|
matchedCondition: terminal.matchedCondition,
|
|
847
|
+
matchedValue: terminal.matchedValue,
|
|
686
848
|
exitCode: terminal.commandResult?.exitCode,
|
|
687
849
|
signal: terminal.commandResult?.signal,
|
|
688
850
|
};
|
|
689
851
|
if (terminal.commandResult) {
|
|
852
|
+
applyCaptureOverflow(meta, terminal.commandResult);
|
|
690
853
|
meta.lastExitCode = terminal.commandResult.exitCode;
|
|
691
854
|
meta.lastSignal = terminal.commandResult.signal;
|
|
692
855
|
meta.lastCheckedAt = terminal.commandResult.endedAt;
|
|
@@ -734,6 +897,7 @@ function scheduleProcessTimeout(
|
|
|
734
897
|
getActiveSession?: ActiveSessionProvider,
|
|
735
898
|
): void {
|
|
736
899
|
clearProcessTimeout(id);
|
|
900
|
+
if (scheduledWorkSuspended) return;
|
|
737
901
|
const delay = Math.max(0, deadlineAt - Date.now());
|
|
738
902
|
const timer = setTimeout(() => void timeoutProcess(pi, id, getActiveSession), delay);
|
|
739
903
|
timer.unref();
|
|
@@ -799,8 +963,8 @@ async function finalizeProcessTimeout(
|
|
|
799
963
|
stopProcessGroup(meta.pid, meta.pgid);
|
|
800
964
|
} catch (error) {
|
|
801
965
|
reason = `timeout; could not terminate local process tree: ${readableError(error)}`;
|
|
802
|
-
|
|
803
|
-
|
|
966
|
+
// The task is still running: this is a stop failure, reported as such.
|
|
967
|
+
recordStopError(meta, reason);
|
|
804
968
|
return;
|
|
805
969
|
}
|
|
806
970
|
}
|
|
@@ -823,6 +987,9 @@ async function notifyTerminal(
|
|
|
823
987
|
meta: BackgroundTaskMeta,
|
|
824
988
|
getActiveSession?: ActiveSessionProvider,
|
|
825
989
|
): Promise<void> {
|
|
990
|
+
// A suspended instance has no active session: delivering or suppressing here would
|
|
991
|
+
// be wrong either way. The instance that resumes the task delivers it (#324).
|
|
992
|
+
if (scheduledWorkSuspended) return;
|
|
826
993
|
if (meta.callback === false || meta.callbackSentAt || meta.callbackSuppressedAt) return;
|
|
827
994
|
const latest = readMeta(meta.id) ?? meta;
|
|
828
995
|
if (latest.callback === false || latest.callbackSentAt || latest.callbackSuppressedAt) return;
|
|
@@ -837,12 +1004,21 @@ async function notifyTerminal(
|
|
|
837
1004
|
}
|
|
838
1005
|
const pending = terminalFailureAttention(latest.id);
|
|
839
1006
|
const label = latest.name ? `${latest.name} (${latest.id})` : latest.id;
|
|
1007
|
+
const facts = formatCallbackFacts(latest);
|
|
840
1008
|
getCallbackBatcher(pi).enqueue({
|
|
841
1009
|
source: "background-task",
|
|
842
1010
|
id: latest.id,
|
|
843
1011
|
label,
|
|
844
|
-
|
|
1012
|
+
// Lifecycle plus an attention count; the incident text rides failureRows
|
|
1013
|
+
// with exact counts, so the status field is never a clipped summary (#323).
|
|
1014
|
+
status: pending
|
|
1015
|
+
? `${latest.status}; ${pending.incidents.length} incident${pending.incidents.length === 1 ? "" : "s"} need${pending.incidents.length === 1 ? "s" : ""} attention`
|
|
1016
|
+
: latest.status,
|
|
845
1017
|
detailTool: "bg_task_status",
|
|
1018
|
+
outcome: facts.outcome,
|
|
1019
|
+
failureRows: facts.failureRows,
|
|
1020
|
+
decision: facts.decision,
|
|
1021
|
+
incidentCount: facts.incidentCount,
|
|
846
1022
|
callback: true,
|
|
847
1023
|
isDelivered: () => {
|
|
848
1024
|
const current = readMeta(latest.id);
|
|
@@ -918,6 +1094,7 @@ function launchArgvOf(commandSpec: CommandSpec, launchSpec: CommandSpec): string
|
|
|
918
1094
|
|
|
919
1095
|
function scheduleLogRetention(id: string): void {
|
|
920
1096
|
stopLogRetention(id);
|
|
1097
|
+
if (scheduledWorkSuspended) return;
|
|
921
1098
|
const timer = setInterval(() => {
|
|
922
1099
|
const meta = readMeta(id);
|
|
923
1100
|
if (!meta || meta.status !== "running" || meta.kind !== "process") {
|
|
@@ -950,9 +1127,27 @@ function enforceLogRetention(meta: BackgroundTaskMeta): void {
|
|
|
950
1127
|
if (!compacted) return;
|
|
951
1128
|
meta.logDiscardedBytes = (meta.logDiscardedBytes ?? 0) + compacted.discardedBytes;
|
|
952
1129
|
meta.logRetentionEvents = (meta.logRetentionEvents ?? 0) + 1;
|
|
1130
|
+
meta.logGeneration = (meta.logGeneration ?? 0) + 1;
|
|
953
1131
|
writeMeta(meta);
|
|
954
1132
|
}
|
|
955
1133
|
|
|
1134
|
+
/** Results whose capture loss is already counted; one poll result is counted once. */
|
|
1135
|
+
const countedCaptureResults = new WeakSet<CommandResult>();
|
|
1136
|
+
|
|
1137
|
+
function applyCaptureOverflow(meta: BackgroundTaskMeta, result: CommandResult): void {
|
|
1138
|
+
const stdout = result.stdoutDiscardedBytes ?? 0;
|
|
1139
|
+
const stderr = result.stderrDiscardedBytes ?? 0;
|
|
1140
|
+
if (!result.captureTruncated && stdout === 0 && stderr === 0) return;
|
|
1141
|
+
// pollWatch counts a result before evaluating conditions, and finalize sees
|
|
1142
|
+
// the same result again when that poll ends the watch.
|
|
1143
|
+
if (countedCaptureResults.has(result)) return;
|
|
1144
|
+
countedCaptureResults.add(result);
|
|
1145
|
+
meta.stdoutDiscardedBytes = (meta.stdoutDiscardedBytes ?? 0) + stdout;
|
|
1146
|
+
meta.stderrDiscardedBytes = (meta.stderrDiscardedBytes ?? 0) + stderr;
|
|
1147
|
+
meta.captureDiscardedBytes = (meta.captureDiscardedBytes ?? 0) + stdout + stderr;
|
|
1148
|
+
meta.captureOverflowEvents = (meta.captureOverflowEvents ?? 0) + 1;
|
|
1149
|
+
}
|
|
1150
|
+
|
|
956
1151
|
function extractLastState(result: { stdout: string }): unknown {
|
|
957
1152
|
try {
|
|
958
1153
|
return JSON.parse(result.stdout);
|
package/src/sandbox.ts
CHANGED
|
@@ -32,8 +32,9 @@ import { baseDir } from "./registry.js";
|
|
|
32
32
|
import { compileWritePolicy, maybeBuildSandboxCommand, type SandboxSeams } from "./shared-sandbox-core.js";
|
|
33
33
|
|
|
34
34
|
type SandboxPermissions = {
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
/** `write` = read, create and overwrite in place; `read-write` = Write & delete. */
|
|
36
|
+
projectFiles: "off" | "read" | "write" | "read-write";
|
|
37
|
+
outsideProject: "off" | "read" | "write" | "read-write";
|
|
37
38
|
storedCredentials: "off" | "read" | "read-write";
|
|
38
39
|
commands: boolean;
|
|
39
40
|
network: boolean;
|
|
@@ -160,10 +161,11 @@ function readPolicy(data: unknown): ForegroundSandboxPolicy | undefined {
|
|
|
160
161
|
function readPermissions(value: unknown): ForegroundSandboxPolicy["permissions"] | undefined {
|
|
161
162
|
if (!value || typeof value !== "object") return undefined;
|
|
162
163
|
const p = value as Record<string, unknown>;
|
|
163
|
-
const
|
|
164
|
+
const credential = (v: unknown) => v === "off" || v === "read" || v === "read-write";
|
|
165
|
+
const access = (v: unknown) => credential(v) || v === "write";
|
|
164
166
|
if (typeof p.enabled !== "boolean" || typeof p.commands !== "boolean" ||
|
|
165
167
|
typeof p.network !== "boolean" || !access(p.projectFiles) ||
|
|
166
|
-
!access(p.outsideProject) || !
|
|
168
|
+
!access(p.outsideProject) || !credential(p.storedCredentials)) return undefined;
|
|
167
169
|
return Object.freeze({
|
|
168
170
|
enabled: p.enabled, commands: p.commands, network: p.network,
|
|
169
171
|
projectFiles: p.projectFiles as SandboxPermissions["projectFiles"],
|