pi-better-background-tasks 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -4
- package/package.json +1 -1
- package/src/failures.ts +9 -1
- package/src/navigator-provider.ts +9 -6
- package/src/output.ts +101 -8
- package/src/runtime.ts +189 -21
- package/src/sandbox.ts +3 -0
- package/src/shared-failure-observations.ts +70 -21
- package/src/shared-log-utils.ts +6 -12
- package/src/shared-navigator.ts +224 -125
- package/src/shared-sandbox-core.ts +194 -22
- package/src/tools.ts +53 -11
- package/src/types.ts +20 -0
package/src/runtime.ts
CHANGED
|
@@ -19,6 +19,7 @@ import type {
|
|
|
19
19
|
CommandResult,
|
|
20
20
|
CommandSpec,
|
|
21
21
|
Condition,
|
|
22
|
+
FirstWatchCheck,
|
|
22
23
|
RemoteTaskParams,
|
|
23
24
|
SshConnectionParams,
|
|
24
25
|
TerminalResult,
|
|
@@ -31,16 +32,78 @@ const processTimeoutTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
|
31
32
|
const activeProcessTimeouts = new Set<string>();
|
|
32
33
|
const activeRemoteTasks = new Map<string, ResolvedSshRemoteTask>();
|
|
33
34
|
const remoteSessionStarts = new Map<string, Promise<CommandResult>>();
|
|
35
|
+
/** Tasks whose stop in this instance is waiting on an in-flight tmux start, and kills the session itself. */
|
|
36
|
+
const remoteStopsAwaitingStart = new Set<string>();
|
|
34
37
|
const activePolls = new Set<string>();
|
|
35
38
|
const logRetentionTimers = new Map<string, ReturnType<typeof setInterval>>();
|
|
36
39
|
const LOG_RETENTION_CHECK_MS = 1000;
|
|
37
|
-
const handoffTimers = new Map<string, ReturnType<typeof
|
|
40
|
+
const handoffTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
41
|
+
// The handoff check backs off from 250 ms to 4 s while the task keeps running,
|
|
42
|
+
// and returns to 250 ms once its process is gone so the grace period is timed closely.
|
|
38
43
|
const HANDOFF_CHECK_MS = 250;
|
|
44
|
+
const HANDOFF_MAX_CHECK_MS = 4_000;
|
|
39
45
|
const HANDOFF_LOST_GRACE_MS = 5_000;
|
|
40
46
|
let scheduledWorkSuspended = false;
|
|
41
47
|
const REMOTE_SESSION_POLL_MS = 100;
|
|
42
48
|
|
|
43
49
|
export const DEFAULT_WATCH_TIMEOUT_SECONDS = 15 * 60;
|
|
50
|
+
/** Consecutive blind checks (exit 0, stderr, no condition matched) before a watch is flagged (#359). */
|
|
51
|
+
export const DEFAULT_BLIND_CHECKS = 3;
|
|
52
|
+
/** How long bg_task_watch waits for the first check before returning (#359). */
|
|
53
|
+
export const FIRST_WATCH_CHECK_WAIT_MS = 15_000;
|
|
54
|
+
const BLIND_OPERATION = "watch-blind";
|
|
55
|
+
/** How to fix a false alarm: some tools write progress or warnings to stderr on success. */
|
|
56
|
+
export const BLIND_CHECK_HINT = "If the stderr is expected (progress or warnings), redirect it (2>/dev/null) or set blind_checks:0.";
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Why the launch stopped waiting before the first check finished (#359): the bounded wait
|
|
60
|
+
* ran out, the tool call was aborted (Esc), or the session shut down.
|
|
61
|
+
*/
|
|
62
|
+
export type FirstCheckWaitEnd = "timeout" | "aborted" | "suspended";
|
|
63
|
+
/** A first check's outcome, or why it is not known yet; undefined when the watch ended without one. */
|
|
64
|
+
export type FirstCheckOutcome = FirstWatchCheck | { pending: FirstCheckWaitEnd; waitedMs: number } | undefined;
|
|
65
|
+
|
|
66
|
+
/** In-flight first checks of watches launched by this instance, keyed by task id. */
|
|
67
|
+
const firstCheckWaiters = new Map<string, { promise: Promise<FirstWatchCheck | "suspended" | undefined>; resolve: (check: FirstWatchCheck | "suspended" | undefined) => void }>();
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Wait for a watch's first check, bounded by `timeoutMs` and `signal` (#359). Resolves a pending
|
|
71
|
+
* outcome when the wait ends first, and undefined when this instance did not launch the watch
|
|
72
|
+
* or the watch ended without a check.
|
|
73
|
+
*/
|
|
74
|
+
export async function awaitFirstWatchCheck(id: string, timeoutMs = FIRST_WATCH_CHECK_WAIT_MS, signal?: AbortSignal): Promise<FirstCheckOutcome> {
|
|
75
|
+
const waiter = firstCheckWaiters.get(id);
|
|
76
|
+
if (!waiter) return undefined;
|
|
77
|
+
const started = Date.now();
|
|
78
|
+
const pending = (reason: FirstCheckWaitEnd) => ({ pending: reason, waitedMs: Date.now() - started });
|
|
79
|
+
if (signal?.aborted) return pending("aborted");
|
|
80
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
81
|
+
let onAbort: (() => void) | undefined;
|
|
82
|
+
const timeout = new Promise<"timeout">((resolve) => {
|
|
83
|
+
// Kept referenced: the launching tool call is waiting on it, and the watch timers are
|
|
84
|
+
// unref'd, so an unref'd wait could let the event loop drain before the first check.
|
|
85
|
+
timer = setTimeout(() => resolve("timeout"), Math.max(0, timeoutMs));
|
|
86
|
+
});
|
|
87
|
+
const aborted = new Promise<"aborted">((resolve) => {
|
|
88
|
+
onAbort = () => resolve("aborted");
|
|
89
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
90
|
+
});
|
|
91
|
+
try {
|
|
92
|
+
const outcome = await Promise.race([waiter.promise, timeout, aborted]);
|
|
93
|
+
if (outcome === "timeout" || outcome === "aborted" || outcome === "suspended") return pending(outcome);
|
|
94
|
+
return outcome;
|
|
95
|
+
} finally {
|
|
96
|
+
if (timer) clearTimeout(timer);
|
|
97
|
+
if (onAbort) signal?.removeEventListener("abort", onAbort);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function settleFirstCheck(id: string, check: FirstWatchCheck | "suspended" | undefined): void {
|
|
102
|
+
const waiter = firstCheckWaiters.get(id);
|
|
103
|
+
if (!waiter) return;
|
|
104
|
+
firstCheckWaiters.delete(id);
|
|
105
|
+
waiter.resolve(check);
|
|
106
|
+
}
|
|
44
107
|
|
|
45
108
|
/**
|
|
46
109
|
* Stop this extension instance's scheduled work when its session shuts down (#324).
|
|
@@ -62,8 +125,9 @@ export const DEFAULT_WATCH_TIMEOUT_SECONDS = 15 * 60;
|
|
|
62
125
|
* tasks keep running, but their watches do not poll, remote tmux output is not
|
|
63
126
|
* collected, and their `timeout_seconds` deadlines are not enforced until that
|
|
64
127
|
* session is active again; an overdue deadline is enforced immediately on resume.
|
|
65
|
-
* A local process that exits meanwhile
|
|
66
|
-
* is delivered when its session resumes
|
|
128
|
+
* A local process that exits meanwhile, while this Pi process is still running,
|
|
129
|
+
* is recorded as terminal and its callback is delivered when its session resumes;
|
|
130
|
+
* after Pi quits, a resumed task whose process is gone is marked lost instead.
|
|
67
131
|
*/
|
|
68
132
|
export function suspendScheduledWork(): void {
|
|
69
133
|
scheduledWorkSuspended = true;
|
|
@@ -71,12 +135,14 @@ export function suspendScheduledWork(): void {
|
|
|
71
135
|
for (const timer of remoteSessionTimers.values()) clearTimeout(timer);
|
|
72
136
|
for (const timer of processTimeoutTimers.values()) clearTimeout(timer);
|
|
73
137
|
for (const timer of logRetentionTimers.values()) clearInterval(timer);
|
|
74
|
-
for (const timer of handoffTimers.values())
|
|
138
|
+
for (const timer of handoffTimers.values()) clearTimeout(timer);
|
|
75
139
|
handoffTimers.clear();
|
|
76
140
|
watcherTimers.clear();
|
|
77
141
|
remoteSessionTimers.clear();
|
|
78
142
|
processTimeoutTimers.clear();
|
|
79
143
|
logRetentionTimers.clear();
|
|
144
|
+
// A launch still waiting on a first check that will not run here reports it as still running.
|
|
145
|
+
for (const id of [...firstCheckWaiters.keys()]) settleFirstCheck(id, "suspended");
|
|
80
146
|
suspendFailureAttention();
|
|
81
147
|
}
|
|
82
148
|
|
|
@@ -90,12 +156,16 @@ export function resumeScheduledWork(): void {
|
|
|
90
156
|
|
|
91
157
|
export type ActiveSessionProvider = () => BackgroundTaskCallbackOrigin | undefined;
|
|
92
158
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
/**
|
|
159
|
+
/** Structured intent (#325), declared by the launching agent and shared by spawns and watchers. */
|
|
160
|
+
export interface TaskIntentParams {
|
|
161
|
+
/** Stable id shared by modified retries of one operation. */
|
|
96
162
|
operation_id?: string | null;
|
|
97
|
-
/**
|
|
163
|
+
/** Non-zero exit codes declared intentional before launch. */
|
|
98
164
|
expected_exit_codes?: number[] | null;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export interface SpawnTaskParams extends CommandSpec, TaskIntentParams {
|
|
168
|
+
name?: string;
|
|
99
169
|
callback?: boolean;
|
|
100
170
|
timeout_seconds?: number;
|
|
101
171
|
max_log_bytes?: number;
|
|
@@ -103,18 +173,16 @@ export interface SpawnTaskParams extends CommandSpec {
|
|
|
103
173
|
remote?: RemoteTaskParams;
|
|
104
174
|
}
|
|
105
175
|
|
|
106
|
-
export interface WatchTaskParams extends CommandSpec {
|
|
176
|
+
export interface WatchTaskParams extends CommandSpec, TaskIntentParams {
|
|
107
177
|
name?: string;
|
|
108
|
-
/** Structured intent (#325): stable id shared by modified retries of one operation. */
|
|
109
|
-
operation_id?: string | null;
|
|
110
|
-
/** Structured intent (#325): non-zero exit codes declared intentional before launch. */
|
|
111
|
-
expected_exit_codes?: number[] | null;
|
|
112
178
|
callback?: boolean;
|
|
113
179
|
interval_seconds?: number;
|
|
114
180
|
timeout_seconds?: number;
|
|
115
181
|
max_log_bytes?: number;
|
|
116
182
|
success_when: Condition;
|
|
117
183
|
failure_when?: Condition;
|
|
184
|
+
/** Consecutive blind checks before the watch is flagged (#359). Default 3; 0 turns it off. */
|
|
185
|
+
blind_checks?: number;
|
|
118
186
|
ssh?: SshConnectionParams;
|
|
119
187
|
remote?: RemoteTaskParams;
|
|
120
188
|
}
|
|
@@ -153,9 +221,10 @@ export function spawnTask(
|
|
|
153
221
|
}, dependencies.remoteRunner)
|
|
154
222
|
: undefined;
|
|
155
223
|
const commandSpec: CommandSpec = remoteTask?.commandSpec ?? { ...params, cwd, shell: params.shell ?? true };
|
|
224
|
+
const sandboxNotices: string[] = [];
|
|
156
225
|
const launchSpec = remoteTask
|
|
157
226
|
? commandSpec
|
|
158
|
-
: confineCommandSpec(commandSpec, sandboxPlan, sandboxProfilePathFor(id));
|
|
227
|
+
: confineCommandSpec(commandSpec, sandboxPlan, sandboxProfilePathFor(id), {}, (line) => sandboxNotices.push(line));
|
|
159
228
|
const tmuxBacked = remoteTask?.metadata.remote.session === "tmux";
|
|
160
229
|
// Outside project = Write or Write & delete can change files outside the
|
|
161
230
|
// project: start an APFS local snapshot (macOS, background, never blocks).
|
|
@@ -165,6 +234,9 @@ export function spawnTask(
|
|
|
165
234
|
: remoteTask
|
|
166
235
|
? remoteTask.spawn(logPath, true)
|
|
167
236
|
: spawnCommand(launchSpec, logPath, true);
|
|
237
|
+
for (const line of sandboxNotices) {
|
|
238
|
+
try { appendLine(logPath, `--- ${line} ---`); } catch { /* log gone */ }
|
|
239
|
+
}
|
|
168
240
|
if (snapshot?.started) {
|
|
169
241
|
void snapshot.done.then((outcome) => {
|
|
170
242
|
if (!outcome.ok) {
|
|
@@ -272,7 +344,15 @@ async function launchRemoteTmux(
|
|
|
272
344
|
if (remoteSessionStarts.get(id) === startAttempt) remoteSessionStarts.delete(id);
|
|
273
345
|
}
|
|
274
346
|
const afterStart = readMeta(id);
|
|
275
|
-
if (!afterStart
|
|
347
|
+
if (!afterStart) return;
|
|
348
|
+
if (afterStart.status !== "running" || afterStart.stopRequestedAt) {
|
|
349
|
+
// Stopped (or timed out) while the start was in flight. A stop in this instance awaited the
|
|
350
|
+
// attempt and kills the session itself. Anything else could not see the attempt: a stop from
|
|
351
|
+
// an instance loaded by /reload, or a deadline that found the session not yet started. Only
|
|
352
|
+
// this launch can still reach the session it may just have created, so it kills it (#332).
|
|
353
|
+
if (!remoteStopsAwaitingStart.has(id)) await killSessionStartedAfterStop(afterStart, remoteTask);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
276
356
|
if (started.exitCode !== 0) {
|
|
277
357
|
const detail = started.stderr.trim() || started.stdout.trim() || "remote tmux returned no diagnostic";
|
|
278
358
|
const reason = `Could not create remote tmux session ${afterStart.remote?.sessionName} on ${afterStart.ssh?.target} (exit ${started.exitCode ?? "unknown"}): ${detail}`;
|
|
@@ -292,6 +372,19 @@ async function launchRemoteTmux(
|
|
|
292
372
|
}
|
|
293
373
|
}
|
|
294
374
|
|
|
375
|
+
/** Kill a tmux session whose start completed after its task stopped; a failed start may still have left it running. */
|
|
376
|
+
async function killSessionStartedAfterStop(meta: BackgroundTaskMeta, remoteTask: ResolvedSshRemoteTask): Promise<void> {
|
|
377
|
+
const session = `remote tmux session ${meta.remote?.sessionName} on ${meta.ssh?.target}`;
|
|
378
|
+
try {
|
|
379
|
+
const stopped = await remoteTask.killTmuxSession();
|
|
380
|
+
appendLine(meta.logPath, stopped.exitCode === 0
|
|
381
|
+
? `--- Killed ${session}: it started after the task was ${meta.status === "running" ? "stopped" : meta.status} ---`
|
|
382
|
+
: `--- Could not kill ${session} that started after the task stopped (exit ${stopped.exitCode ?? "unknown"}) ---`);
|
|
383
|
+
} catch (error) {
|
|
384
|
+
appendLine(meta.logPath, `--- Could not kill ${session} that started after the task stopped: ${error instanceof Error ? error.message : String(error)} ---`);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
295
388
|
function scheduleRemoteSessionPoll(
|
|
296
389
|
pi: ExtensionAPI,
|
|
297
390
|
id: string,
|
|
@@ -400,6 +493,7 @@ export function startWatchTask(
|
|
|
400
493
|
const error = condition && validateCondition(condition);
|
|
401
494
|
if (error) throw new Error(`${name}: ${error}`);
|
|
402
495
|
}
|
|
496
|
+
const blindChecks = readBlindChecks(params.blind_checks);
|
|
403
497
|
const intent = readTaskIntent(params);
|
|
404
498
|
const sandboxPlan = resolveForegroundSandboxPlan(pi, !!params.ssh);
|
|
405
499
|
const id = nextTaskId();
|
|
@@ -417,9 +511,10 @@ export function startWatchTask(
|
|
|
417
511
|
}, dependencies.remoteRunner)
|
|
418
512
|
: undefined;
|
|
419
513
|
const commandSpec: CommandSpec = remoteTask?.commandSpec ?? { ...params, cwd, shell: params.shell ?? true };
|
|
514
|
+
const sandboxNotices: string[] = [];
|
|
420
515
|
const launchSpec = remoteTask
|
|
421
516
|
? commandSpec
|
|
422
|
-
: confineCommandSpec(commandSpec, sandboxPlan, sandboxProfilePathFor(id));
|
|
517
|
+
: confineCommandSpec(commandSpec, sandboxPlan, sandboxProfilePathFor(id), {}, (line) => sandboxNotices.push(line));
|
|
423
518
|
const meta: BackgroundTaskMeta = {
|
|
424
519
|
id,
|
|
425
520
|
name: params.name,
|
|
@@ -443,6 +538,7 @@ export function startWatchTask(
|
|
|
443
538
|
spawnPidStartTime: currentProcessStartToken(),
|
|
444
539
|
successWhen: params.success_when,
|
|
445
540
|
failureWhen: params.failure_when,
|
|
541
|
+
...(blindChecks !== undefined ? { blindChecks } : {}),
|
|
446
542
|
notifyOn: "terminal",
|
|
447
543
|
ssh: remoteTask?.metadata.ssh,
|
|
448
544
|
remote: remoteTask?.metadata.remote,
|
|
@@ -450,13 +546,25 @@ export function startWatchTask(
|
|
|
450
546
|
};
|
|
451
547
|
ensureTaskDir(id);
|
|
452
548
|
appendLine(meta.logPath, `--- watch ${new Date(now).toISOString()} interval_ms=${meta.intervalMs} ---`);
|
|
549
|
+
for (const line of sandboxNotices) appendLine(meta.logPath, `--- ${line} ---`);
|
|
453
550
|
writeMeta(meta);
|
|
551
|
+
let resolveFirst!: (check: FirstWatchCheck | "suspended" | undefined) => void;
|
|
552
|
+
const firstCheck = new Promise<FirstWatchCheck | "suspended" | undefined>((resolve) => { resolveFirst = resolve; });
|
|
553
|
+
firstCheckWaiters.set(id, { promise: firstCheck, resolve: resolveFirst });
|
|
454
554
|
scheduleWatch(pi, id, 0, getActiveSession, remoteTask
|
|
455
555
|
? (timeoutMs) => remoteTask.runOnce(undefined, timeoutMs)
|
|
456
556
|
: undefined);
|
|
457
557
|
return meta;
|
|
458
558
|
}
|
|
459
559
|
|
|
560
|
+
function readBlindChecks(value: unknown): number | undefined {
|
|
561
|
+
if (value === undefined || value === null) return undefined;
|
|
562
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
563
|
+
throw new Error("blind_checks must be a non-negative integer (0 turns the blind-check rule off).");
|
|
564
|
+
}
|
|
565
|
+
return value;
|
|
566
|
+
}
|
|
567
|
+
|
|
460
568
|
function resolveWatchTimeoutSeconds(timeoutSeconds: number | undefined): number | undefined {
|
|
461
569
|
if (timeoutSeconds === undefined) return DEFAULT_WATCH_TIMEOUT_SECONDS;
|
|
462
570
|
if (timeoutSeconds <= 0) return undefined;
|
|
@@ -528,7 +636,9 @@ function scheduleHandoff(
|
|
|
528
636
|
stopHandoff(id);
|
|
529
637
|
if (scheduledWorkSuspended) return;
|
|
530
638
|
let deadSince: number | undefined;
|
|
531
|
-
|
|
639
|
+
let delayMs = HANDOFF_CHECK_MS;
|
|
640
|
+
const check = () => {
|
|
641
|
+
handoffTimers.delete(id);
|
|
532
642
|
const meta = readMeta(id);
|
|
533
643
|
if (!meta) {
|
|
534
644
|
stopHandoff(id);
|
|
@@ -554,16 +664,26 @@ function scheduleHandoff(
|
|
|
554
664
|
clearProcessTimeout(id);
|
|
555
665
|
stopLogRetention(id);
|
|
556
666
|
markProcessLost(pi, meta, getActiveSession);
|
|
667
|
+
return;
|
|
557
668
|
}
|
|
669
|
+
delayMs = HANDOFF_CHECK_MS;
|
|
670
|
+
} else {
|
|
671
|
+
delayMs = Math.min(HANDOFF_MAX_CHECK_MS, delayMs * 2);
|
|
558
672
|
}
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
673
|
+
arm();
|
|
674
|
+
};
|
|
675
|
+
const arm = () => {
|
|
676
|
+
if (scheduledWorkSuspended) return;
|
|
677
|
+
const timer = setTimeout(check, delayMs);
|
|
678
|
+
timer.unref();
|
|
679
|
+
handoffTimers.set(id, timer);
|
|
680
|
+
};
|
|
681
|
+
arm();
|
|
562
682
|
}
|
|
563
683
|
|
|
564
684
|
function stopHandoff(id: string): void {
|
|
565
685
|
const timer = handoffTimers.get(id);
|
|
566
|
-
if (timer)
|
|
686
|
+
if (timer) clearTimeout(timer);
|
|
567
687
|
handoffTimers.delete(id);
|
|
568
688
|
}
|
|
569
689
|
|
|
@@ -625,7 +745,9 @@ export async function stopTask(
|
|
|
625
745
|
const remoteSessionMayExist = remote?.session === "tmux"
|
|
626
746
|
&& (remote.sessionStarted !== false || remoteStartAttempt !== undefined);
|
|
627
747
|
if (remoteStartAttempt) {
|
|
748
|
+
remoteStopsAwaitingStart.add(id);
|
|
628
749
|
try { await remoteStartAttempt; } catch { /* A failed SSH result can still leave the detached session running. */ }
|
|
750
|
+
finally { remoteStopsAwaitingStart.delete(id); }
|
|
629
751
|
}
|
|
630
752
|
|
|
631
753
|
if (remoteSessionMayExist) {
|
|
@@ -713,6 +835,7 @@ async function pollWatch(
|
|
|
713
835
|
): Promise<void> {
|
|
714
836
|
if (activePolls.has(id)) return;
|
|
715
837
|
activePolls.add(id);
|
|
838
|
+
let checked: FirstWatchCheck | undefined;
|
|
716
839
|
try {
|
|
717
840
|
const meta = readMeta(id);
|
|
718
841
|
if (!meta || meta.status !== "running" || meta.kind !== "command_watch") return;
|
|
@@ -725,6 +848,10 @@ async function pollWatch(
|
|
|
725
848
|
const result = runOnce
|
|
726
849
|
? await runOnce(timeoutMs)
|
|
727
850
|
: await runCommandOnce(commandSpecFromMeta(meta), undefined, timeoutMs);
|
|
851
|
+
checked = {
|
|
852
|
+
exitCode: result.exitCode, signal: result.signal, durationMs: Math.max(0, result.endedAt - result.startedAt),
|
|
853
|
+
...(result.timedOut ? { timedOut: true } : {}), stdout: result.stdout, stderr: result.stderr,
|
|
854
|
+
};
|
|
728
855
|
// A poll that was in flight when the session shut down belongs to a stale instance.
|
|
729
856
|
if (scheduledWorkSuspended) return;
|
|
730
857
|
appendWatchResult(meta.logPath, result);
|
|
@@ -777,6 +904,10 @@ async function pollWatch(
|
|
|
777
904
|
else if (result.exitCode !== 0) recordExitFailure(latest, "watch-poll", `Watch poll exited with code ${result.exitCode ?? "unknown"}`, result.exitCode, pollKey,
|
|
778
905
|
{ expected: expectedPollExit, at: result.endedAt });
|
|
779
906
|
else recoverFailure(latest, "watch-poll", pollKey, result.endedAt);
|
|
907
|
+
observeBlindCheck(latest, result, pollKey, {
|
|
908
|
+
clean: !transportFailure && !conditionErrors.length,
|
|
909
|
+
matched: failure?.matched === true || success?.matched === true,
|
|
910
|
+
});
|
|
780
911
|
if (conditionErrors.length) latest.error = conditionErrors.join("; ");
|
|
781
912
|
if (failure?.matched) {
|
|
782
913
|
recordFailure(latest, "failure_when", "failure condition matched", pollKey, { category: "condition", at: result.endedAt });
|
|
@@ -824,10 +955,47 @@ async function pollWatch(
|
|
|
824
955
|
}
|
|
825
956
|
recordFailure(meta, "watch-poll", reason, `throw:${meta.lastCheckedAt ?? meta.startedAt}`, { category: meta.ssh ? "ssh" : "execution" });
|
|
826
957
|
finalize(meta, { status: "failed", reason }, pi, getActiveSession);
|
|
958
|
+
checked ??= { exitCode: null, signal: null, durationMs: 0, stdout: "", stderr: "", error: reason };
|
|
827
959
|
}
|
|
828
960
|
} finally {
|
|
829
961
|
activePolls.delete(id);
|
|
962
|
+
if (firstCheckWaiters.has(id)) {
|
|
963
|
+
if (checked) settleFirstCheck(id, checked);
|
|
964
|
+
else if (readMeta(id)?.status !== "running") settleFirstCheck(id, undefined);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
/**
|
|
970
|
+
* Blind-check rule (#359). A check that exits 0, writes stderr, and matches neither condition
|
|
971
|
+
* is probably broken: it prints an error and then reports "still pending" forever (the real
|
|
972
|
+
* incident was a gcloud --format error followed by `exit 0`). After `blindChecks` such checks
|
|
973
|
+
* in a row, record one actionable incident; the existing attention path wakes the parent once.
|
|
974
|
+
* The watch keeps running. A later check with empty stderr, whatever its exit code (including a
|
|
975
|
+
* non-zero, SSH-transport-failed or condition-error check), or one matching a condition,
|
|
976
|
+
* recovers it; those failures are recorded as their own incidents. Any other check (non-zero
|
|
977
|
+
* with stderr) resets the count and leaves the incident open. A clean pending check (exit 0,
|
|
978
|
+
* no stderr) never counts.
|
|
979
|
+
*/
|
|
980
|
+
function observeBlindCheck(meta: BackgroundTaskMeta, result: CommandResult, pollKey: unknown,
|
|
981
|
+
check: { clean: boolean; matched: boolean }): void {
|
|
982
|
+
const threshold = meta.blindChecks ?? DEFAULT_BLIND_CHECKS;
|
|
983
|
+
const stderr = result.stderr.trim();
|
|
984
|
+
const blind = threshold > 0 && check.clean && !check.matched && result.exitCode === 0 && stderr.length > 0;
|
|
985
|
+
if (!blind) {
|
|
986
|
+
meta.blindCheckStreak = 0;
|
|
987
|
+
if (!stderr || check.matched) recoverFailure(meta, BLIND_OPERATION, pollKey, result.endedAt);
|
|
988
|
+
return;
|
|
830
989
|
}
|
|
990
|
+
const streak = (meta.blindCheckStreak ?? 0) + 1;
|
|
991
|
+
meta.blindCheckStreak = streak;
|
|
992
|
+
if (streak !== threshold) return;
|
|
993
|
+
const lastLine = stderr.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).at(-1) ?? "";
|
|
994
|
+
const clipped = lastLine.length > 240 ? `${lastLine.slice(0, 239)}…` : lastLine;
|
|
995
|
+
// The summary is capped in compact rows; the stderr line rides as evidence, which rows show whole.
|
|
996
|
+
recordFailure(meta, BLIND_OPERATION,
|
|
997
|
+
`Blind watch check: ${streak} checks in a row exited 0 with stderr and matched no condition; the check may be broken. The watch keeps running.`,
|
|
998
|
+
pollKey, { category: "blind-check", at: result.endedAt, evidence: `latest stderr: ${clipped} · ${BLIND_CHECK_HINT}` });
|
|
831
999
|
}
|
|
832
1000
|
|
|
833
1001
|
function finalize(
|
package/src/sandbox.ts
CHANGED
|
@@ -279,6 +279,8 @@ export function confineCommandSpec(
|
|
|
279
279
|
plan: ForegroundSandboxPlan,
|
|
280
280
|
profilePath: string,
|
|
281
281
|
seams: SandboxSeams = {},
|
|
282
|
+
/** Receives lines to show with the launch (e.g. a placeholder left in the user's files). */
|
|
283
|
+
onNotice: (line: string) => void = () => {},
|
|
282
284
|
): CommandSpec {
|
|
283
285
|
if (!plan.confined) return spec;
|
|
284
286
|
|
|
@@ -331,6 +333,7 @@ export function confineCommandSpec(
|
|
|
331
333
|
throw blocked(plan, error instanceof Error ? error.message : String(error));
|
|
332
334
|
}
|
|
333
335
|
if (!command) throw blocked(plan, "no sandbox backend was applied");
|
|
336
|
+
for (const line of command.notices ?? []) onNotice(line);
|
|
334
337
|
|
|
335
338
|
return { ...spec, argv: [command.file, ...command.fileArgs], shell: false };
|
|
336
339
|
}
|
|
@@ -75,7 +75,8 @@ export function failureIdentity(...parts: unknown[]): string {
|
|
|
75
75
|
return createHash("sha256").update(JSON.stringify(parts)).digest("hex").slice(0, 32);
|
|
76
76
|
}
|
|
77
77
|
function text(value: string | undefined, fallback: string): string {
|
|
78
|
-
|
|
78
|
+
// The 400-unit cut can split a surrogate pair; a lone half would render as U+FFFD (#332).
|
|
79
|
+
return dropLoneSurrogates((value || fallback).replace(/[\x00-\x1f\x7f]/g, " ").slice(0, 400));
|
|
79
80
|
}
|
|
80
81
|
|
|
81
82
|
/** Agent tool attempts: the agent handles its own tool errors, so one failure is not yet actionable. */
|
|
@@ -104,24 +105,60 @@ export interface CommandIntent {
|
|
|
104
105
|
export type CommandIntentField = keyof CommandIntent;
|
|
105
106
|
const INTENT_FIELDS: readonly CommandIntentField[] = ["operationId", "attemptId", "expectedExitCodes"];
|
|
106
107
|
/**
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
108
|
+
* Pi's own argument coercion for the intent fields (pi-ai `validateToolArguments` against the confined
|
|
109
|
+
* bash schema, where each field is `anyOf: [field schema, null]`). Pi applies it before execute, but the
|
|
110
|
+
* process log and session record keep the raw arguments; mirroring it lets the parent's replay read
|
|
111
|
+
* the declaration the child executed with (#332).
|
|
112
|
+
* - ids: a number or boolean becomes its string; `""` fails the pattern and falls through to null.
|
|
113
|
+
* - exit codes: array items that read as integers become integers (null -> 0, true -> 1);
|
|
114
|
+
* a scalar `0`, `""`, or `false` falls through to null.
|
|
115
|
+
*/
|
|
116
|
+
function coerceInteger(value: unknown): unknown {
|
|
117
|
+
if (value === null) return 0;
|
|
118
|
+
if (typeof value === "string" && value.trim() !== "" && Number.isInteger(Number(value))) return Number(value);
|
|
119
|
+
if (typeof value === "boolean") return value ? 1 : 0;
|
|
120
|
+
return value;
|
|
121
|
+
}
|
|
122
|
+
function normalizeIntentField(key: CommandIntentField, value: unknown): unknown {
|
|
123
|
+
if (value == null) return undefined;
|
|
124
|
+
if (key === "expectedExitCodes") {
|
|
125
|
+
if (value === 0 || value === "" || value === false) return undefined;
|
|
126
|
+
if (!Array.isArray(value)) return value;
|
|
127
|
+
// 0 is a no-op declaration: exit 0 is already success. A list of only zeros declares nothing.
|
|
128
|
+
const codes = value.map(coerceInteger).filter((code) => code !== 0);
|
|
129
|
+
if (codes.length === 0 && value.length > 0) return undefined;
|
|
130
|
+
return codes.length === value.length && codes.every((code, index) => code === value[index]) ? value : codes;
|
|
131
|
+
}
|
|
132
|
+
if (value === "") return undefined;
|
|
133
|
+
return typeof value === "number" || typeof value === "boolean" ? String(value) : value;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* The one normalization of intent-bearing arguments. An explicit `null` intent field means "not
|
|
137
|
+
* declared", exactly like an omitted one: models routinely send optional fields as null, and Pi's
|
|
138
|
+
* argument validation drops or nulls them before a tool runs, while the process log and session
|
|
139
|
+
* record keep the raw arguments. Values Pi would coerce are coerced the same way, and `0` is dropped
|
|
140
|
+
* from `expectedExitCodes`. Every reader of intent, the executing tool and the replay alike, and the
|
|
141
|
+
* exact operation identity go through this, so all see the same declaration from the same input.
|
|
142
|
+
* Malformed values are kept as they are for the validator to refuse.
|
|
112
143
|
*/
|
|
113
144
|
export function withoutAbsentIntent<T>(args: T): T {
|
|
114
145
|
if (!args || typeof args !== "object" || Array.isArray(args)) return args;
|
|
115
146
|
const input = args as Record<string, unknown>;
|
|
116
|
-
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
147
|
+
let out: Record<string, unknown> | undefined;
|
|
148
|
+
for (const key of INTENT_FIELDS) {
|
|
149
|
+
if (!(key in input)) continue;
|
|
150
|
+
const value = normalizeIntentField(key, input[key]);
|
|
151
|
+
if (value === input[key]) continue;
|
|
152
|
+
out ??= { ...input };
|
|
153
|
+
if (value === undefined) delete out[key];
|
|
154
|
+
else out[key] = value;
|
|
155
|
+
}
|
|
156
|
+
return (out ?? args) as T;
|
|
120
157
|
}
|
|
121
158
|
/**
|
|
122
|
-
* Validate structured intent fields
|
|
159
|
+
* Validate structured intent fields after `withoutAbsentIntent`. Absent fields are fine; malformed
|
|
123
160
|
* ones are an error, and the caller must not run the command. `names` renames fields in the error
|
|
124
|
-
* (e.g. snake_case parameters).
|
|
161
|
+
* (e.g. snake_case parameters). `0` in `expectedExitCodes` is accepted and dropped (#332).
|
|
125
162
|
*/
|
|
126
163
|
export function readCommandIntent(args: unknown, names: Partial<Record<CommandIntentField, string>> = {}): { intent: CommandIntent; error?: string } {
|
|
127
164
|
const input = withoutAbsentIntent((args && typeof args === "object" ? args : {}) as Record<string, unknown>);
|
|
@@ -138,7 +175,7 @@ export function readCommandIntent(args: unknown, names: Partial<Record<CommandIn
|
|
|
138
175
|
if (codes !== undefined) {
|
|
139
176
|
if (!Array.isArray(codes) || codes.length === 0 || codes.length > MAX_EXPECTED_EXIT_CODES ||
|
|
140
177
|
!codes.every((code) => Number.isInteger(code) && code >= 1 && code <= 255) || new Set(codes).size !== codes.length) {
|
|
141
|
-
return { intent: {}, error: `${names.expectedExitCodes ?? "expectedExitCodes"} must be 1-${MAX_EXPECTED_EXIT_CODES} distinct integers from 1 to 255` };
|
|
178
|
+
return { intent: {}, error: `${names.expectedExitCodes ?? "expectedExitCodes"} must be 1-${MAX_EXPECTED_EXIT_CODES} distinct integers from 1 to 255 (0 is allowed and ignored)` };
|
|
142
179
|
}
|
|
143
180
|
intent.expectedExitCodes = [...codes] as number[];
|
|
144
181
|
}
|
|
@@ -481,6 +518,8 @@ export interface TerminalFailureParts {
|
|
|
481
518
|
rows: string[];
|
|
482
519
|
/** Counts and separate facts: earlier-reported, unclassified, expected, closed history, correctness. */
|
|
483
520
|
notes: string[];
|
|
521
|
+
/** The unclassified-count note, when it is among `notes`: a byte budget keeps it longer than the others. */
|
|
522
|
+
unclassifiedNote?: string;
|
|
484
523
|
}
|
|
485
524
|
/**
|
|
486
525
|
* Terminal/completion facts: actionable and incomplete incidents from `incidents` as rows,
|
|
@@ -494,12 +533,14 @@ export function terminalFailureParts(state: FailureState, incidents: readonly st
|
|
|
494
533
|
const earlier = activeFailures(state).filter((x) => !wanted.has(x.id) && reportable(x)).length;
|
|
495
534
|
if (earlier) notes.push(`${earlier} actionable incident${earlier === 1 ? " was" : "s were"} reported earlier; not repeated here.`);
|
|
496
535
|
const counts = failureCounts(state);
|
|
497
|
-
|
|
536
|
+
const unclassifiedNote = counts.unclassified
|
|
537
|
+
? `${counts.unclassified} earlier tool failure${counts.unclassified === 1 ? "" : "s"} remain${counts.unclassified === 1 ? "s" : ""} unclassified.` : undefined;
|
|
538
|
+
if (unclassifiedNote) notes.push(unclassifiedNote);
|
|
498
539
|
if (counts.expected) notes.push(`${counts.expected} expected failure${counts.expected === 1 ? "" : "s"} recorded.`);
|
|
499
540
|
const history = closedHistoryLine(state);
|
|
500
541
|
if (history) notes.push(history);
|
|
501
542
|
if (counts.unclassified || counts.actionRequired) notes.push(CORRECTNESS_NOTE);
|
|
502
|
-
return { rows, notes };
|
|
543
|
+
return { rows, notes, ...(unclassifiedNote ? { unclassifiedNote } : {}) };
|
|
503
544
|
}
|
|
504
545
|
export function formatTerminalFailureFacts(state: FailureState, incidents: readonly string[] = []): string {
|
|
505
546
|
const { rows, notes } = terminalFailureParts(state, incidents);
|
|
@@ -607,7 +648,10 @@ export interface IncidentPageRequest extends IncidentRowOptions {
|
|
|
607
648
|
* Caller-owned incident pages. Whole rows are preferred; a row larger than the
|
|
608
649
|
* page is split at a code-point boundary and resumes at that byte, so pages
|
|
609
650
|
* reconstruct formatFailureLines() exactly (join with "\n" except after a page
|
|
610
|
-
* that `endsPartial`). A page never exceeds `maxBytes
|
|
651
|
+
* that `endsPartial`). A page never exceeds `maxBytes`, so one smaller than the
|
|
652
|
+
* next code point (up to 4 bytes) returns no text, `hasMore`, and a `nextCursor`
|
|
653
|
+
* equal to `cursor`: the caller retries with a larger page, as with
|
|
654
|
+
* pageVerbatimText. Consumers never ask for fewer than 4 bytes.
|
|
611
655
|
*/
|
|
612
656
|
export function pageFailureIncidents(state: FailureState, request: IncidentPageRequest = {}): FailureIncidentPage {
|
|
613
657
|
// A cursor carries its view: following a history cursor stays in history without the flag.
|
|
@@ -725,14 +769,19 @@ export function incidentVerbatimPage(state: FailureState, request: IncidentPageR
|
|
|
725
769
|
* it, so a cache keyed by it never serves stale incident counts.
|
|
726
770
|
*/
|
|
727
771
|
export function failureJournalFingerprint(path: string): string {
|
|
728
|
-
|
|
772
|
+
return `${fileChangeIdentity(path)}|${existsSync(`${path}.observed`) ? 1 : 0}|${pendingWrites.get(path)?.length ?? 0}`;
|
|
773
|
+
}
|
|
774
|
+
/**
|
|
775
|
+
* A file's identity and change stamp (device, inode, size, and nanosecond mtime/ctime), or its read
|
|
776
|
+
* error. Any append, rewrite, replacement, or deletion changes it; for cache keys, never for trust.
|
|
777
|
+
*/
|
|
778
|
+
export function fileChangeIdentity(path: string): string {
|
|
729
779
|
try {
|
|
730
780
|
const stats = statSync(path, { bigint: true });
|
|
731
|
-
|
|
781
|
+
return `${stats.dev}:${stats.ino}:${stats.size}:${stats.mtimeNs}:${stats.ctimeNs}`;
|
|
732
782
|
} catch (error) {
|
|
733
|
-
|
|
783
|
+
return `unreadable:${(error as NodeJS.ErrnoException).code ?? "error"}`;
|
|
734
784
|
}
|
|
735
|
-
return `${file}|${existsSync(`${path}.observed`) ? 1 : 0}|${pendingWrites.get(path)?.length ?? 0}`;
|
|
736
785
|
}
|
|
737
786
|
|
|
738
787
|
export function incidentCursorAt(state: FailureState, offset: number, resource?: string, options: IncidentRowOptions = {}): string {
|
|
@@ -867,7 +916,7 @@ export function formatTerminalIncidentSummary(state: FailureState, options: Inci
|
|
|
867
916
|
// cursor's retrieval hint, the unclassified count, the cursor, and the count line. The
|
|
868
917
|
// correctness note goes last, and only when it alone does not fit.
|
|
869
918
|
const correctness: string[] = notes.filter((x) => x === CORRECTNESS_NOTE);
|
|
870
|
-
const unclassified = notes.filter((x) => x
|
|
919
|
+
const unclassified = notes.filter((x) => x === parts.unclassifiedNote);
|
|
871
920
|
const low = notes.filter((x) => !correctness.includes(x) && !unclassified.includes(x));
|
|
872
921
|
const ladder: Array<{ mode: Header | undefined; notes: string[] }> = [];
|
|
873
922
|
for (let keep = low.length - 1; keep >= 0; keep -= 1) ladder.push({ mode: "full", notes: [...low.slice(0, keep), ...unclassified, ...correctness] });
|
package/src/shared-log-utils.ts
CHANGED
|
@@ -819,29 +819,22 @@ export interface OutputControls {
|
|
|
819
819
|
maxBytes?: unknown;
|
|
820
820
|
/** Raw requested line count (canonical `lines`, else deprecated `tail_lines`). */
|
|
821
821
|
lines?: unknown;
|
|
822
|
-
/** Deprecated alias names the caller used (whether or not the canonical name won). */
|
|
823
|
-
deprecated: string[];
|
|
824
822
|
}
|
|
825
823
|
|
|
826
824
|
/**
|
|
827
825
|
* Resolve output-control parameters. Both spellings work; when both are given
|
|
828
|
-
* the canonical name wins.
|
|
829
|
-
*
|
|
826
|
+
* the canonical name wins. An explicit `null` is "not given", so a null
|
|
827
|
+
* canonical value never hides an alias value (#332). Values are returned
|
|
828
|
+
* unvalidated so each surface keeps its own defaults and hard caps.
|
|
830
829
|
*/
|
|
831
830
|
export function readOutputControls(params: unknown): OutputControls {
|
|
832
831
|
const p = (params && typeof params === "object" ? params : {}) as Record<string, unknown>;
|
|
833
|
-
const
|
|
834
|
-
const pick = (canonical: keyof typeof OUTPUT_CONTROL_ALIASES): unknown => {
|
|
835
|
-
const alias = OUTPUT_CONTROL_ALIASES[canonical];
|
|
836
|
-
if (p[alias] !== undefined) deprecated.push(alias);
|
|
837
|
-
return p[canonical] !== undefined ? p[canonical] : p[alias];
|
|
838
|
-
};
|
|
832
|
+
const pick = (canonical: keyof typeof OUTPUT_CONTROL_ALIASES): unknown => p[canonical] ?? p[OUTPUT_CONTROL_ALIASES[canonical]] ?? undefined;
|
|
839
833
|
const maxBytes = pick("max_bytes");
|
|
840
834
|
const lines = pick("lines");
|
|
841
835
|
return {
|
|
842
836
|
...(maxBytes !== undefined ? { maxBytes } : {}),
|
|
843
837
|
...(lines !== undefined ? { lines } : {}),
|
|
844
|
-
deprecated,
|
|
845
838
|
};
|
|
846
839
|
}
|
|
847
840
|
|
|
@@ -851,7 +844,8 @@ export type OutputInclude = (typeof OUTPUT_INCLUDE_VALUES)[number];
|
|
|
851
844
|
|
|
852
845
|
/** Parse an `include` request into known values and unknown ones (both deduplicated). */
|
|
853
846
|
export function readOutputInclude(value: unknown): { include: Set<OutputInclude>; unknown: string[] } {
|
|
854
|
-
|
|
847
|
+
// Any other non-null value (a number, boolean, or object) is one unknown entry, never silently nothing (#332).
|
|
848
|
+
const raw = Array.isArray(value) ? value : typeof value === "string" ? value.split(",") : value == null ? [] : [value];
|
|
855
849
|
const include = new Set<OutputInclude>();
|
|
856
850
|
const unknown: string[] = [];
|
|
857
851
|
for (const entry of raw) {
|