pi-better-subagents 0.26.0 → 0.26.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/child-dependencies.mjs +161 -0
- package/child-preflight.mjs +22 -0
- package/child-runtime.mjs +57 -0
- package/cleanup.ts +58 -2
- package/config.ts +8 -4
- package/continuation.ts +58 -7
- package/docs/usage.md +76 -9
- package/extensions.mjs +1 -0
- package/finalization.ts +1 -1
- package/index.ts +613 -291
- package/output-payload.ts +40 -3
- package/package.json +1 -1
- package/registry.ts +215 -4
- package/shared-callback-batcher.ts +18 -1
- package/shared-failure-observations.ts +78 -24
- package/shared-log-utils.ts +277 -31
- package/stop.ts +60 -2
- package/task-runtime.mjs +2 -1
- package/timing.ts +45 -4
- package/tools.ts +29 -6
package/output-payload.ts
CHANGED
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
} from "./lifecycle.ts";
|
|
38
38
|
import { parseRun, parseRunForLifecycle, type ParsedRun } from "./parse.ts";
|
|
39
39
|
import {
|
|
40
|
+
ABANDONED_LAUNCH_GRACE_MS,
|
|
40
41
|
belongsToOrigin,
|
|
41
42
|
effectiveStatus,
|
|
42
43
|
inspectRunMeta,
|
|
@@ -83,6 +84,11 @@ export interface PayloadRequest {
|
|
|
83
84
|
* those that need action, as an incident page. Default output lists only what needs action.
|
|
84
85
|
*/
|
|
85
86
|
history?: unknown;
|
|
87
|
+
/**
|
|
88
|
+
* Raw mode only: start from the newest retained byte and page toward the
|
|
89
|
+
* oldest one (#618). Without it, raw pages start at byte 0.
|
|
90
|
+
*/
|
|
91
|
+
tail?: unknown;
|
|
86
92
|
}
|
|
87
93
|
|
|
88
94
|
export type LoadedRun =
|
|
@@ -227,7 +233,7 @@ function parserDiagnostics(run: ParsedRun): string | undefined {
|
|
|
227
233
|
}
|
|
228
234
|
|
|
229
235
|
function retrievalHint(id: string): string {
|
|
230
|
-
return `Retrieve retained evidence with subagent_output id="${id}" mode="raw" (16 KiB default, up to 64 KiB, caller cursor).`;
|
|
236
|
+
return `Retrieve retained evidence with subagent_output id="${id}" mode="raw" (16 KiB default, up to 64 KiB, caller cursor; tail:true starts at the newest bytes).`;
|
|
231
237
|
}
|
|
232
238
|
|
|
233
239
|
function bestParsed(run: ParsedRun): string {
|
|
@@ -386,12 +392,22 @@ function isContinuationPage(cursor: string | undefined): boolean {
|
|
|
386
392
|
return cursorKind(cursor) === "t";
|
|
387
393
|
}
|
|
388
394
|
|
|
395
|
+
/** An explicit end-anchored raw read: `tail: true`. */
|
|
396
|
+
function wantsTail(request: PayloadRequest): boolean {
|
|
397
|
+
return isAll(request.tail);
|
|
398
|
+
}
|
|
399
|
+
|
|
389
400
|
function rawPage(ctx: CallContext, cursor: string | undefined, remaining: number): VerbatimPage {
|
|
390
|
-
|
|
401
|
+
const page = pageRetainedFile(logPathFor(ctx.id), {
|
|
391
402
|
cursor,
|
|
392
403
|
maxBytes: remaining,
|
|
393
404
|
resource: `raw:${ctx.scopeKey}:${ctx.id}`,
|
|
405
|
+
...(wantsTail(ctx.request) ? { direction: "backward" as const } : {}),
|
|
394
406
|
});
|
|
407
|
+
// A forward raw page points at the end-anchored read: a multi-MB retained
|
|
408
|
+
// log must not cost one round-trip per page to reach its newest bytes (#618).
|
|
409
|
+
if (page.direction === "backward" || !page.hasMore) return page;
|
|
410
|
+
return { ...page, via: "tail:true reads the newest bytes first" };
|
|
395
411
|
}
|
|
396
412
|
|
|
397
413
|
function answerPage(ctx: CallContext, text: string, cursor: string | undefined, remaining: number, maxLines?: number): VerbatimPage {
|
|
@@ -546,7 +562,7 @@ function assembleRaw(
|
|
|
546
562
|
const continuing = cursorKind(request.cursor) === "f";
|
|
547
563
|
return envelopeText({
|
|
548
564
|
maxBytes,
|
|
549
|
-
identity: `${identity} raw retained log`,
|
|
565
|
+
identity: `${identity} raw retained log${wantsTail(request) ? " · newest first" : ""}`,
|
|
550
566
|
// Raw evidence keeps whole excerpts and evidence paths.
|
|
551
567
|
failure: failureSection(ctx, continuing ? CONTINUATION_FAILURE_BYTES : 1024, "full"),
|
|
552
568
|
diagnostics: joinSections([changeDiagnostics(ctx), optInFacts(run, request), parserDiagnostics(run)]),
|
|
@@ -854,6 +870,12 @@ export function assembleSubagentListPayload(input: {
|
|
|
854
870
|
statusesKey?: string;
|
|
855
871
|
sessionWarning?: string;
|
|
856
872
|
unreadable?: Array<{ id: string; detail: string }>;
|
|
873
|
+
/**
|
|
874
|
+
* Launch attempts that never produced a supervised run (no metadata, no
|
|
875
|
+
* log). They are not run evidence, so they are reported separately from
|
|
876
|
+
* `unreadable` and never counted as runs (#623).
|
|
877
|
+
*/
|
|
878
|
+
abandonedLaunches?: string[];
|
|
857
879
|
indexError?: string;
|
|
858
880
|
incidentRuns?: number;
|
|
859
881
|
}): string {
|
|
@@ -877,6 +899,8 @@ export function assembleSubagentListPayload(input: {
|
|
|
877
899
|
});
|
|
878
900
|
}
|
|
879
901
|
const unreadable = input.unreadable ?? [];
|
|
902
|
+
const abandonedLaunches = input.abandonedLaunches ?? [];
|
|
903
|
+
const abandonedGrace = describeAbandonedLaunchGrace();
|
|
880
904
|
const pageCursor = input.cursor && !statusCursorInput ? input.cursor : undefined;
|
|
881
905
|
const identity = input.indexError
|
|
882
906
|
? "subagent_list · run index unreadable"
|
|
@@ -890,6 +914,9 @@ export function assembleSubagentListPayload(input: {
|
|
|
890
914
|
unreadable.length
|
|
891
915
|
? `${unreadable.length} run record(s) with missing or unreadable metadata (ownership unknown; not counted as absent or healthy): ${unreadable.slice(0, 3).map((item) => item.id).join(", ")}${unreadable.length > 3 ? ", …" : ""}`
|
|
892
916
|
: undefined,
|
|
917
|
+
abandonedLaunches.length
|
|
918
|
+
? `${abandonedLaunches.length} abandoned launch record(s), not run evidence and not counted: the launch stopped before any child started (prompt written, no metadata, no log): ${abandonedLaunches.slice(0, 3).join(", ")}${abandonedLaunches.length > 3 ? ", …" : ""}. Cleanup reclaims them after ${abandonedGrace}; remove one now with subagent_stop <id>.`
|
|
919
|
+
: undefined,
|
|
893
920
|
statusCursorInput && status.change === "failure" ? "change=failure" : undefined,
|
|
894
921
|
statusCursorInput && status.change === "reset" ? "reset=stale-cursor (status cursor from another scope or filter)" : undefined,
|
|
895
922
|
]);
|
|
@@ -925,4 +952,14 @@ export function unknownRunError(id: string): Error {
|
|
|
925
952
|
return new Error(`Unknown run id: ${id}`);
|
|
926
953
|
}
|
|
927
954
|
|
|
955
|
+
/** Human-readable grace period for abandoned launch records. */
|
|
956
|
+
function describeAbandonedLaunchGrace(): string {
|
|
957
|
+
const minutes = Math.round(ABANDONED_LAUNCH_GRACE_MS / 60_000);
|
|
958
|
+
if (minutes % 60 === 0) {
|
|
959
|
+
const hours = minutes / 60;
|
|
960
|
+
return `${hours} hour${hours === 1 ? "" : "s"}`;
|
|
961
|
+
}
|
|
962
|
+
return `${minutes} minutes`;
|
|
963
|
+
}
|
|
964
|
+
|
|
928
965
|
export { originOf };
|
package/package.json
CHANGED
package/registry.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import type { RunTiming } from "./timing.ts";
|
|
16
16
|
import { execFileSync } from "node:child_process";
|
|
17
17
|
import { createHash, randomBytes } from "node:crypto";
|
|
18
|
-
import { closeSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
18
|
+
import { closeSync, existsSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
19
19
|
import { createRequire } from "node:module";
|
|
20
20
|
import { tmpdir } from "node:os";
|
|
21
21
|
import { basename, dirname, join } from "node:path";
|
|
@@ -176,6 +176,18 @@ export interface RunMeta {
|
|
|
176
176
|
* diagnostic: it explains a `lost` record nobody's session reported.
|
|
177
177
|
*/
|
|
178
178
|
adoptedFromLostParentAt?: number;
|
|
179
|
+
/**
|
|
180
|
+
* When a session shutdown ended this extension instance's direct supervision
|
|
181
|
+
* of the run while its detached child was left running (#625). The child is
|
|
182
|
+
* never signalled for a reload/quit: a later session of the same pi process
|
|
183
|
+
* re-adopts the record, and because the instance that held the in-memory
|
|
184
|
+
* exit observer is gone, that session finalizes the run from the child's own
|
|
185
|
+
* coherent log when its process later disappears. Additive and diagnostic;
|
|
186
|
+
* terminal records keep the marker as history.
|
|
187
|
+
*/
|
|
188
|
+
supervisionInterruptedAt?: number;
|
|
189
|
+
/** Pi's `session_shutdown` reason that interrupted supervision (`reload`, `quit`, …). */
|
|
190
|
+
supervisionInterruptedReason?: string;
|
|
179
191
|
model?: string;
|
|
180
192
|
/** Reasoning effort passed to the child via Pi's --thinking option. */
|
|
181
193
|
effort?: string;
|
|
@@ -286,6 +298,14 @@ function metaPathFor(id: string): string {
|
|
|
286
298
|
return join(runDir(id), "meta.json");
|
|
287
299
|
}
|
|
288
300
|
|
|
301
|
+
/**
|
|
302
|
+
* How long a launch attempt that never produced a child is kept before a sweep
|
|
303
|
+
* reclaims it. Launch preparation takes seconds; the grace only exists so a
|
|
304
|
+
* paused or unusually slow launch in another process is never reclaimed under
|
|
305
|
+
* it. A record this old has no metadata and no log, so it can never become a run.
|
|
306
|
+
*/
|
|
307
|
+
export const ABANDONED_LAUNCH_GRACE_MS = 60 * 60 * 1000;
|
|
308
|
+
|
|
289
309
|
/**
|
|
290
310
|
* Parent-authored provenance for a run launched on the trusted task runtime (#325).
|
|
291
311
|
*
|
|
@@ -1135,6 +1155,12 @@ export interface RunRecordIndex {
|
|
|
1135
1155
|
metas: RunMeta[];
|
|
1136
1156
|
/** Run directories whose metadata is missing, unreadable, or corrupt. */
|
|
1137
1157
|
unreadable: Array<{ id: string; detail: string }>;
|
|
1158
|
+
/**
|
|
1159
|
+
* Launch attempts that never produced a supervised run (no metadata, no
|
|
1160
|
+
* log). They are not runs, cannot be read or finalized, and are reclaimed
|
|
1161
|
+
* by the launch-debris sweep or an explicit `subagent_stop`.
|
|
1162
|
+
*/
|
|
1163
|
+
abandoned: string[];
|
|
1138
1164
|
/** The runs directory itself could not be read (not "no runs"). */
|
|
1139
1165
|
indexError?: string;
|
|
1140
1166
|
}
|
|
@@ -1149,6 +1175,187 @@ export function runMetaShapeProblem(meta: unknown, id: string): string | undefin
|
|
|
1149
1175
|
return undefined;
|
|
1150
1176
|
}
|
|
1151
1177
|
|
|
1178
|
+
/** A run directory a launch created but never turned into a supervised run. */
|
|
1179
|
+
export interface AbandonedLaunchRecord {
|
|
1180
|
+
id: string;
|
|
1181
|
+
/** Directory mtime (ms), used as the launch attempt's age. */
|
|
1182
|
+
mtimeMs: number;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
/**
|
|
1186
|
+
* Durable claim a launch writes into its run directory before it prepares
|
|
1187
|
+
* anything (#623). It states WHO owns the attempt while no metadata exists, so a
|
|
1188
|
+
* concurrent purge can tell a live launch from the debris of a dead one.
|
|
1189
|
+
*
|
|
1190
|
+
* It is a plain file rather than the metadata lock because the claim must cover
|
|
1191
|
+
* the whole preparation (workspace, sandbox control dir, session dir, spawn and
|
|
1192
|
+
* metadata publication), not only the metadata write. It is removed at the same
|
|
1193
|
+
* moment `meta.json` becomes the record of the run, and it is never a second
|
|
1194
|
+
* registry: nothing reads it as run evidence.
|
|
1195
|
+
*/
|
|
1196
|
+
export interface LaunchAttemptClaim {
|
|
1197
|
+
version: 1;
|
|
1198
|
+
id: string;
|
|
1199
|
+
pid: number;
|
|
1200
|
+
/** Start-identity token of the claiming pi; only equality is meaningful. */
|
|
1201
|
+
start?: string;
|
|
1202
|
+
at: number;
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
function launchAttemptClaimPath(id: string): string {
|
|
1206
|
+
return join(runDir(id), ".launch-attempt.json");
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
/** Claim this run directory for a launch in progress. The directory must exist. */
|
|
1210
|
+
export function recordLaunchAttempt(id: string): void {
|
|
1211
|
+
const claim: LaunchAttemptClaim = { version: 1, id, pid: process.pid, at: Date.now() };
|
|
1212
|
+
const start = processStartToken(process.pid);
|
|
1213
|
+
if (start) claim.start = start;
|
|
1214
|
+
writeFileSync(launchAttemptClaimPath(id), JSON.stringify(claim), { mode: 0o600 });
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
/** Drop the claim once `meta.json` is the run's record. Best effort. */
|
|
1218
|
+
export function clearLaunchAttempt(id: string): void {
|
|
1219
|
+
try {
|
|
1220
|
+
unlinkSync(launchAttemptClaimPath(id));
|
|
1221
|
+
} catch { /* no claim, or the attempt's directory is already gone */ }
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
/**
|
|
1225
|
+
* Whether a launch claim is still owned by a live process. Conservative for one
|
|
1226
|
+
* direction only: an unreadable, malformed, or token-less claim counts as live,
|
|
1227
|
+
* so the attempt's artifacts are preserved. A dead pid, or a pid whose start
|
|
1228
|
+
* token no longer matches (recycled), is a dead attempt.
|
|
1229
|
+
*/
|
|
1230
|
+
function launchAttemptClaimIsLive(id: string): boolean {
|
|
1231
|
+
let raw: string;
|
|
1232
|
+
try {
|
|
1233
|
+
raw = readFileSync(launchAttemptClaimPath(id), "utf-8");
|
|
1234
|
+
} catch (error) {
|
|
1235
|
+
return errnoOf(error) !== "ENOENT";
|
|
1236
|
+
}
|
|
1237
|
+
let parsed: Partial<LaunchAttemptClaim> | undefined;
|
|
1238
|
+
try {
|
|
1239
|
+
parsed = JSON.parse(raw) as Partial<LaunchAttemptClaim>;
|
|
1240
|
+
} catch {
|
|
1241
|
+
return true;
|
|
1242
|
+
}
|
|
1243
|
+
if (!parsed || typeof parsed !== "object") return true;
|
|
1244
|
+
const pid = parsed.pid;
|
|
1245
|
+
if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return true;
|
|
1246
|
+
if (!processExists(pid)) return false;
|
|
1247
|
+
const start = parsed.start;
|
|
1248
|
+
if (typeof start !== "string" || start.length === 0) return true;
|
|
1249
|
+
const current = cachedStartToken(pid);
|
|
1250
|
+
// An unavailable token does not prove the pid is dead, so the claim stays live.
|
|
1251
|
+
return current === undefined || current === start;
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
/**
|
|
1255
|
+
* True when a launch is preparing this run id right now, so its directory must
|
|
1256
|
+
* not be treated as debris. Unreadable state counts as in progress.
|
|
1257
|
+
*/
|
|
1258
|
+
export function launchAttemptInProgress(id: string): boolean {
|
|
1259
|
+
if (!/^sa_[a-z0-9]+_[a-z0-9]+$/i.test(id)) return false;
|
|
1260
|
+
return launchAttemptClaimIsLive(id);
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
/**
|
|
1264
|
+
* Read one run directory as an abandoned launch attempt, or `undefined`.
|
|
1265
|
+
*
|
|
1266
|
+
* A launch creates the directory, claims it, writes `prompt.md`, and prepares
|
|
1267
|
+
* the sandbox `control/` dir before the child starts and `meta.json` is
|
|
1268
|
+
* committed. When that preparation throws, or pi dies in that window, only
|
|
1269
|
+
* those artifacts remain: no metadata (nothing was committed) and no
|
|
1270
|
+
* `output.log` (the log is created at spawn and removed when the spawn itself
|
|
1271
|
+
* fails). Such a directory was never a run: it has no owner, no status, no log
|
|
1272
|
+
* and no result, and it can never gain any, so it must not be reported as
|
|
1273
|
+
* unreadable run evidence forever.
|
|
1274
|
+
*
|
|
1275
|
+
* The rule is deliberately narrow in the other direction too. A live launch
|
|
1276
|
+
* claim means the attempt is still being prepared and is never debris; a
|
|
1277
|
+
* directory WITH a log is not debris either, because a child may have started
|
|
1278
|
+
* and its output is real evidence; and a directory without the launch artifacts
|
|
1279
|
+
* is left to the age sweep rather than guessed at.
|
|
1280
|
+
*/
|
|
1281
|
+
export function abandonedLaunchRecord(id: string): AbandonedLaunchRecord | undefined {
|
|
1282
|
+
if (!/^sa_[a-z0-9]+_[a-z0-9]+$/i.test(id)) return undefined;
|
|
1283
|
+
let mtimeMs: number;
|
|
1284
|
+
try {
|
|
1285
|
+
mtimeMs = statSync(runDir(id)).mtimeMs;
|
|
1286
|
+
} catch {
|
|
1287
|
+
return undefined;
|
|
1288
|
+
}
|
|
1289
|
+
// Metadata absence is the claim of debris, so it is re-established as late as
|
|
1290
|
+
// possible: the launch shape first, then every piece of run evidence.
|
|
1291
|
+
if (!existsSync(promptPathFor(id)) && !existsSync(join(runDir(id), "control"))) return undefined;
|
|
1292
|
+
if (launchAttemptClaimIsLive(id)) return undefined;
|
|
1293
|
+
try {
|
|
1294
|
+
statSync(metaPathFor(id));
|
|
1295
|
+
return undefined;
|
|
1296
|
+
} catch (error) {
|
|
1297
|
+
if (errnoOf(error) !== "ENOENT") return undefined;
|
|
1298
|
+
}
|
|
1299
|
+
try {
|
|
1300
|
+
statSync(logPathFor(id));
|
|
1301
|
+
return undefined;
|
|
1302
|
+
} catch (error) {
|
|
1303
|
+
if (errnoOf(error) !== "ENOENT") return undefined;
|
|
1304
|
+
}
|
|
1305
|
+
return { id, mtimeMs };
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
export type AbandonedLaunchPurge =
|
|
1309
|
+
/** The attempt's artifacts were removed. */
|
|
1310
|
+
| "purged"
|
|
1311
|
+
/** Not launch debris (a run record, a live launch, or something else entirely). */
|
|
1312
|
+
| "not-debris"
|
|
1313
|
+
/** It looked like debris, but a writer or an unavailable lock made removal unsafe. */
|
|
1314
|
+
| "retained";
|
|
1315
|
+
|
|
1316
|
+
/**
|
|
1317
|
+
* Reclaim one abandoned launch attempt: its run directory, the per-run session
|
|
1318
|
+
* directory a confined launch prepared, and the clone workspace it owned. Only
|
|
1319
|
+
* paths named by the attempt's own id are removed (shared sessions and a
|
|
1320
|
+
* caller-chosen `sandbox_dir` are never touched), exactly like a failed launch.
|
|
1321
|
+
*
|
|
1322
|
+
* Classification runs again inside the same per-run lock `writeMeta` publishes
|
|
1323
|
+
* under, and the removal happens there, so a launch that commits metadata while
|
|
1324
|
+
* the purger waits is preserved instead of deleted. A held or unavailable lock
|
|
1325
|
+
* is a live writer, so it is preserved too. This closes the race the earlier
|
|
1326
|
+
* shape-only rule lost (#623 review R1): no deletion of a live attempt's
|
|
1327
|
+
* metadata or log, and no orphaned child left behind by a purge.
|
|
1328
|
+
*/
|
|
1329
|
+
export function purgeAbandonedLaunch(id: string): AbandonedLaunchPurge {
|
|
1330
|
+
if (!abandonedLaunchRecord(id)) return "not-debris";
|
|
1331
|
+
let outcome: AbandonedLaunchPurge = "retained";
|
|
1332
|
+
try {
|
|
1333
|
+
withRunMetaLock(id, (owns) => {
|
|
1334
|
+
if (!owns()) throw new MetadataLockLost();
|
|
1335
|
+
// Re-establish the negative evidence under the publication exclusion.
|
|
1336
|
+
if (!abandonedLaunchRecord(id)) {
|
|
1337
|
+
outcome = "not-debris";
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
discardFailedLaunch(id, { sessionDir: join(sessionsDir(), id), workspaceDir: taskWorkspaceDir(id) });
|
|
1341
|
+
outcome = "purged";
|
|
1342
|
+
});
|
|
1343
|
+
} catch {
|
|
1344
|
+
// A live writer (or an unreadable run directory) is never safe to delete.
|
|
1345
|
+
// A directory that disappeared meanwhile is no longer debris.
|
|
1346
|
+
return abandonedLaunchRecord(id) ? "retained" : "not-debris";
|
|
1347
|
+
}
|
|
1348
|
+
if (outcome === "purged") {
|
|
1349
|
+
try {
|
|
1350
|
+
statSync(runDir(id));
|
|
1351
|
+
outcome = "retained";
|
|
1352
|
+
} catch (error) {
|
|
1353
|
+
if (errnoOf(error) !== "ENOENT") outcome = "retained";
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
return outcome;
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1152
1359
|
/**
|
|
1153
1360
|
* Inspect one run's metadata without collapsing "missing" and "corrupt":
|
|
1154
1361
|
* `missing` only when neither metadata nor the run directory exists.
|
|
@@ -1196,10 +1403,11 @@ export function listRunRecords(): RunRecordIndex {
|
|
|
1196
1403
|
ids = readdirSync(join(baseDir(), "runs"));
|
|
1197
1404
|
} catch (error) {
|
|
1198
1405
|
const code = (error as NodeJS.ErrnoException).code;
|
|
1199
|
-
if (code !== "ENOENT") return { metas: [], unreadable: [], indexError: error instanceof Error ? error.message : String(error) };
|
|
1406
|
+
if (code !== "ENOENT") return { metas: [], unreadable: [], abandoned: [], indexError: error instanceof Error ? error.message : String(error) };
|
|
1200
1407
|
}
|
|
1201
1408
|
const metas: RunMeta[] = [];
|
|
1202
1409
|
const unreadable: Array<{ id: string; detail: string }> = [];
|
|
1410
|
+
const abandoned: string[] = [];
|
|
1203
1411
|
for (const id of ids) {
|
|
1204
1412
|
const cached = metaCache.get(id);
|
|
1205
1413
|
if (cached && cached.status !== "running" && cached.status !== "orphaned") {
|
|
@@ -1208,14 +1416,17 @@ export function listRunRecords(): RunRecordIndex {
|
|
|
1208
1416
|
}
|
|
1209
1417
|
const inspected = inspectRunMeta(id);
|
|
1210
1418
|
if (inspected.kind === "ok") metas.push(inspected.meta);
|
|
1211
|
-
else if (inspected.kind === "unreadable")
|
|
1419
|
+
else if (inspected.kind === "unreadable") {
|
|
1420
|
+
if (abandonedLaunchRecord(id)) abandoned.push(id);
|
|
1421
|
+
else unreadable.push({ id, detail: inspected.detail });
|
|
1422
|
+
}
|
|
1212
1423
|
}
|
|
1213
1424
|
const volatileIds = new Set(ids);
|
|
1214
1425
|
for (const archived of listArchivedAttempts()) {
|
|
1215
1426
|
if (!volatileIds.has(archived.id)) metas.push(archived);
|
|
1216
1427
|
}
|
|
1217
1428
|
metas.sort((a, b) => b.startedAt - a.startedAt);
|
|
1218
|
-
return { metas, unreadable };
|
|
1429
|
+
return { metas, unreadable, abandoned };
|
|
1219
1430
|
}
|
|
1220
1431
|
|
|
1221
1432
|
export function listMetasForParent(parentPid: number): RunMeta[] {
|
|
@@ -604,7 +604,7 @@ export function createCallbackBatcher(
|
|
|
604
604
|
|
|
605
605
|
const enqueue = (event: CallbackBatchEvent): boolean => {
|
|
606
606
|
if (event.callback === false) return false;
|
|
607
|
-
const key =
|
|
607
|
+
const key = completionKey(event);
|
|
608
608
|
if (pending.has(key) || inFlight.has(key)) return false;
|
|
609
609
|
pending.set(key, { event, sequence: sequence++ });
|
|
610
610
|
schedule(windowMs);
|
|
@@ -884,6 +884,23 @@ function eventKey(event: Pick<CallbackBatchEvent, "source" | "id" | "status">):
|
|
|
884
884
|
return `${event.source}\u0000${event.id}\u0000${event.status}`;
|
|
885
885
|
}
|
|
886
886
|
|
|
887
|
+
/**
|
|
888
|
+
* Ordinary completion identity is the run/task id, not the status wording (#624).
|
|
889
|
+
*
|
|
890
|
+
* The status field is a snapshot taken when a notification is enqueued (failure
|
|
891
|
+
* notes, observation gaps, timing reason). Including it in the key meant a
|
|
892
|
+
* not-yet-delivered run whose status changed between enqueues entered the
|
|
893
|
+
* pending map twice; one flush then rendered both rows and the caller saw one
|
|
894
|
+
* completion twice. Urgent events keep the status in their key: those are
|
|
895
|
+
* distinct transitions (orphaned, lost, a timing wake, a failure key), not
|
|
896
|
+
* restatements of a single ordinary completion. The first queued snapshot is the
|
|
897
|
+
* one delivered; a later status wording for the same id is dropped rather than
|
|
898
|
+
* re-queued, and the durable result stays the authority for failure detail.
|
|
899
|
+
*/
|
|
900
|
+
function completionKey(event: Pick<CallbackBatchEvent, "source" | "id">): string {
|
|
901
|
+
return `${event.source}\u0000${event.id}`;
|
|
902
|
+
}
|
|
903
|
+
|
|
887
904
|
function eventDisposition(
|
|
888
905
|
event: Pick<CallbackBatchEvent, "isDelivered" | "getSuppressionReason">,
|
|
889
906
|
): | { kind: "deliver" } | { kind: "delivered" } | { kind: "deferred" } | { kind: "suppressed"; reason: string } {
|
|
@@ -414,7 +414,15 @@ export function historyTailLine(state: FailureState): string | undefined {
|
|
|
414
414
|
return history ? `Also in history: ${history}` : undefined;
|
|
415
415
|
}
|
|
416
416
|
|
|
417
|
-
|
|
417
|
+
/**
|
|
418
|
+
* Current incident cursor prefix. The prefix is the cursor format: it replaced
|
|
419
|
+
* the in-payload `x` version field so that two position witnesses fit in the
|
|
420
|
+
* same space (these cursors are printed inside 1 KiB incident summaries, where
|
|
421
|
+
* every byte competes with the count line). `i1.` cursors are still recognised
|
|
422
|
+
* for routing but decode as stale.
|
|
423
|
+
*/
|
|
424
|
+
const INCIDENT_CURSOR_PREFIX = "i2.";
|
|
425
|
+
const LEGACY_INCIDENT_CURSOR_PREFIXES = ["i1."];
|
|
418
426
|
const encoder = new TextEncoder();
|
|
419
427
|
|
|
420
428
|
/** Longest summary or disposition-reason excerpt a compact row shows, in UTF-8 bytes. */
|
|
@@ -599,7 +607,7 @@ export interface FailureIncidentPage {
|
|
|
599
607
|
cursor: string;
|
|
600
608
|
nextCursor: string;
|
|
601
609
|
hasMore: boolean;
|
|
602
|
-
reset?: "stale-cursor" | "source-replaced";
|
|
610
|
+
reset?: "stale-cursor" | "source-replaced" | "cursor-corrupt";
|
|
603
611
|
/** The first line continues a row begun on an earlier page. */
|
|
604
612
|
startsPartial: boolean;
|
|
605
613
|
/** The last line is the start of a row that continues on the next page. */
|
|
@@ -630,39 +638,67 @@ function resourceTag(resource: string | undefined): string | undefined {
|
|
|
630
638
|
}
|
|
631
639
|
|
|
632
640
|
/**
|
|
633
|
-
* Incident cursor
|
|
634
|
-
*
|
|
635
|
-
*
|
|
641
|
+
* Incident cursor (`i2.`). `h`: the explicit history view; `f`: full rows; `q` is an integrity tag
|
|
642
|
+
* over the other fields; `v` and `w` are two independent witnesses of the same fact — this page
|
|
643
|
+
* set at this row and byte offset (#617). A hand-copied cursor used to decode into a still
|
|
644
|
+
* parseable payload whose row digest no longer matched, which was reported as a replaced source and
|
|
645
|
+
* replayed the first page forever; it also used to resume at its decoded offset even when the page
|
|
646
|
+
* set had moved, skipping a failure that had just been inserted ahead of the cursor. A corrupt
|
|
647
|
+
* cursor now resumes only when a witness recomputed from the current rows proves its offset, and
|
|
648
|
+
* otherwise restarts at the first row. Both witnesses are short because these cursors are printed
|
|
649
|
+
* inside 1 KiB incident summaries.
|
|
636
650
|
*/
|
|
637
|
-
|
|
638
|
-
interface
|
|
651
|
+
interface IncidentCursor { o: number; b: number; v: string; w: string; n: number; r?: string; h?: 1; f?: 1 }
|
|
652
|
+
interface DecodedIncidentCursor { cursor: IncidentCursor; corrupt: boolean }
|
|
653
|
+
|
|
654
|
+
function incidentDigest(domain: string, revision: string, o: number, b: number): string {
|
|
655
|
+
return createHash("sha256").update(`${domain}\0${revision}\0${o}\0${b}`).digest("base64url");
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/** Digest of this page set at this row and byte offset. */
|
|
659
|
+
function incidentPositionDigest(revision: string, o: number, b: number): string {
|
|
660
|
+
return incidentDigest("v", revision, o, b).slice(0, 16);
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
/** Independent witness of the same position, so damage to either digest alone stays usable. */
|
|
664
|
+
function incidentPositionWitness(revision: string, o: number, b: number): string {
|
|
665
|
+
return incidentDigest("w", revision, b, o).slice(0, 4);
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
function incidentCursorTag(payload: Record<string, unknown>): string {
|
|
669
|
+
const { q: _ignored, ...rest } = payload;
|
|
670
|
+
return createHash("sha256").update(JSON.stringify(rest)).digest("base64url").slice(0, 4);
|
|
671
|
+
}
|
|
639
672
|
|
|
640
673
|
function encodeIncidentCursor(cursor: IncidentCursor): string {
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
...(cursor.f ? { f: 1 } : {})
|
|
674
|
+
const payload = { o: cursor.o, ...(cursor.b ? { b: cursor.b } : {}), v: cursor.v, n: cursor.n,
|
|
675
|
+
...(cursor.r !== undefined ? { r: cursor.r } : {}), ...(cursor.h ? { h: 1 } : {}),
|
|
676
|
+
...(cursor.f ? { f: 1 } : {}), w: cursor.w };
|
|
677
|
+
return INCIDENT_CURSOR_PREFIX + Buffer.from(JSON.stringify({ ...payload, q: incidentCursorTag(payload) }), "utf8").toString("base64url");
|
|
644
678
|
}
|
|
645
679
|
|
|
646
|
-
function decodeIncidentCursor(cursor: string | undefined):
|
|
680
|
+
function decodeIncidentCursor(cursor: string | undefined): DecodedIncidentCursor | undefined {
|
|
647
681
|
if (!cursor || !cursor.startsWith(INCIDENT_CURSOR_PREFIX)) return undefined;
|
|
648
682
|
try {
|
|
649
|
-
const parsed = JSON.parse(Buffer.from(cursor.slice(INCIDENT_CURSOR_PREFIX.length), "base64url").toString("utf8")) as {
|
|
650
|
-
if (parsed?.
|
|
651
|
-
return {
|
|
683
|
+
const parsed = JSON.parse(Buffer.from(cursor.slice(INCIDENT_CURSOR_PREFIX.length), "base64url").toString("utf8")) as { o?: number; b?: number; v?: string; n?: number; r?: string; h?: number; f?: number; q?: string; w?: string };
|
|
684
|
+
if (typeof parsed?.v === "string" && typeof parsed.w === "string") {
|
|
685
|
+
return { corrupt: typeof parsed.q !== "string" || parsed.q !== incidentCursorTag(parsed), cursor: {
|
|
686
|
+
o: Math.max(0, Math.floor(parsed.o ?? 0)), b: Math.max(0, Math.floor(parsed.b ?? 0)), v: parsed.v, w: parsed.w,
|
|
652
687
|
n: Math.max(0, Math.floor(parsed.n ?? 0)), ...(typeof parsed.r === "string" ? { r: parsed.r } : {}),
|
|
653
|
-
...(parsed.h === 1 ? { h: 1 as const } : {}), ...(parsed.f === 1 ? { f: 1 as const } : {}) };
|
|
688
|
+
...(parsed.h === 1 ? { h: 1 as const } : {}), ...(parsed.f === 1 ? { f: 1 as const } : {}) } };
|
|
654
689
|
}
|
|
655
690
|
} catch { /* stale */ }
|
|
656
691
|
return undefined;
|
|
657
692
|
}
|
|
658
693
|
|
|
659
694
|
export function isIncidentCursor(cursor: string | undefined): boolean {
|
|
660
|
-
|
|
695
|
+
if (!cursor) return false;
|
|
696
|
+
return cursor.startsWith(INCIDENT_CURSOR_PREFIX) || LEGACY_INCIDENT_CURSOR_PREFIXES.some((prefix) => cursor.startsWith(prefix));
|
|
661
697
|
}
|
|
662
698
|
|
|
663
699
|
/** The scope an incident cursor pages, or undefined for anything that is not an incident cursor. */
|
|
664
700
|
export function incidentCursorScope(cursor: string | undefined): IncidentScope | undefined {
|
|
665
|
-
const parsed = decodeIncidentCursor(cursor);
|
|
701
|
+
const parsed = decodeIncidentCursor(cursor)?.cursor;
|
|
666
702
|
return parsed ? (parsed.h ? "all" : "actionable") : undefined;
|
|
667
703
|
}
|
|
668
704
|
|
|
@@ -697,7 +733,8 @@ export interface IncidentPageRequest extends IncidentRowOptions {
|
|
|
697
733
|
*/
|
|
698
734
|
export function pageFailureIncidents(state: FailureState, request: IncidentPageRequest = {}): FailureIncidentPage {
|
|
699
735
|
// A cursor carries its view: following a history cursor stays in history without the flag.
|
|
700
|
-
const
|
|
736
|
+
const decoded = request.cursor ? decodeIncidentCursor(request.cursor) : undefined;
|
|
737
|
+
const parsed = decoded?.cursor;
|
|
701
738
|
const scope: IncidentScope = request.scope ?? (parsed?.h ? "all" : "actionable");
|
|
702
739
|
const detail: IncidentDetail = request.detail ?? (parsed?.f ? "full" : "compact");
|
|
703
740
|
const lines = formatFailureLines(state, { scope, detail });
|
|
@@ -708,14 +745,28 @@ export function pageFailureIncidents(state: FailureState, request: IncidentPageR
|
|
|
708
745
|
let byte = 0;
|
|
709
746
|
let reset: FailureIncidentPage["reset"];
|
|
710
747
|
if (request.cursor) {
|
|
711
|
-
|
|
712
|
-
|
|
748
|
+
const corrupt = decoded?.corrupt === true;
|
|
749
|
+
if (!parsed || parsed.r !== resource || Boolean(parsed.h) !== (scope === "all") || Boolean(parsed.f) !== (detail === "full")) {
|
|
750
|
+
reset = corrupt ? "cursor-corrupt" : "stale-cursor";
|
|
751
|
+
} else if (corrupt) {
|
|
752
|
+
// A hand-copied cursor is corruption, not a replaced source (#617), and
|
|
753
|
+
// its offset is never trusted on the payload's word. It resumes only
|
|
754
|
+
// when a witness recomputed from the current rows proves this exact
|
|
755
|
+
// position in this exact page set; otherwise it restarts at the first
|
|
756
|
+
// row, so a failure inserted ahead of the cursor is not skipped.
|
|
757
|
+
reset = "cursor-corrupt";
|
|
758
|
+
const witnessed = parsed.o <= total
|
|
759
|
+
&& (parsed.v === incidentPositionDigest(revision, parsed.o, parsed.b)
|
|
760
|
+
|| parsed.w === incidentPositionWitness(revision, parsed.o, parsed.b));
|
|
761
|
+
if (witnessed) { offset = parsed.o; byte = offset < total ? parsed.b : 0; }
|
|
762
|
+
} else if (parsed.v !== incidentPositionDigest(revision, parsed.o, parsed.b)) reset = "source-replaced";
|
|
713
763
|
else { offset = Math.min(total, parsed.o); byte = offset < total ? parsed.b : 0; }
|
|
714
764
|
}
|
|
715
765
|
const maxBytes = Number.isFinite(request.maxBytes) && (request.maxBytes ?? -1) >= 0
|
|
716
766
|
? Math.floor(request.maxBytes as number)
|
|
717
767
|
: 2 * 1024;
|
|
718
|
-
const mint = (o: number, b: number) => encodeIncidentCursor({ o, b, v: revision,
|
|
768
|
+
const mint = (o: number, b: number) => encodeIncidentCursor({ o, b, v: incidentPositionDigest(revision, o, b),
|
|
769
|
+
w: incidentPositionWitness(revision, o, b), n: total, ...(resource !== undefined ? { r: resource } : {}),
|
|
719
770
|
...(scope === "all" ? { h: 1 as const } : {}), ...(detail === "full" ? { f: 1 as const } : {}) });
|
|
720
771
|
const parts: string[] = [];
|
|
721
772
|
let used = 0;
|
|
@@ -786,7 +837,7 @@ export function incidentVerbatimPage(state: FailureState, request: IncidentPageR
|
|
|
786
837
|
nextCursor: string;
|
|
787
838
|
omittedBytes: number;
|
|
788
839
|
omittedRows: number;
|
|
789
|
-
reset?: "stale-cursor" | "source-replaced";
|
|
840
|
+
reset?: "stale-cursor" | "source-replaced" | "cursor-corrupt";
|
|
790
841
|
} {
|
|
791
842
|
const page = pageFailureIncidents(state, request);
|
|
792
843
|
const scope = request.scope ?? incidentCursorScope(request.cursor) ?? "actionable";
|
|
@@ -830,9 +881,12 @@ export function incidentCursorAt(state: FailureState, offset: number, resource?:
|
|
|
830
881
|
const scope = options.scope ?? "actionable";
|
|
831
882
|
const detail = options.detail ?? "compact";
|
|
832
883
|
const lines = formatFailureLines(state, { scope, detail });
|
|
884
|
+
const revision = incidentRevision(state, scope, detail);
|
|
885
|
+
const o = Math.max(0, Math.floor(offset));
|
|
833
886
|
const tag = resourceTag(resource);
|
|
834
|
-
return encodeIncidentCursor({ o
|
|
835
|
-
...(tag !== undefined ? { r: tag } : {}), ...(scope === "all" ? { h: 1 as const } : {}),
|
|
887
|
+
return encodeIncidentCursor({ o, b: 0, v: incidentPositionDigest(revision, o, 0), w: incidentPositionWitness(revision, o, 0),
|
|
888
|
+
n: lines.length, ...(tag !== undefined ? { r: tag } : {}), ...(scope === "all" ? { h: 1 as const } : {}),
|
|
889
|
+
...(options.detail === "full" ? { f: 1 as const } : {}) });
|
|
836
890
|
}
|
|
837
891
|
|
|
838
892
|
const MIN_PARTIAL_ROW_BYTES = 96;
|