pi-crew 0.11.21 → 0.11.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/index.mjs +537 -526
- package/package.json +2 -1
- package/src/errors.ts +14 -1
- package/src/extension/async-notifier.ts +42 -17
- package/src/extension/cross-extension-rpc.ts +42 -5
- package/src/extension/notification-router.ts +15 -2
- package/src/extension/registration/lifecycle-handlers.ts +36 -13
- package/src/extension/registration/runtime-cleanup.ts +17 -0
- package/src/extension/registration/wire-cross-extension.ts +26 -0
- package/src/extension/rpc-hmac.ts +53 -2
- package/src/extension/team-tool/cancel.ts +34 -21
- package/src/extension/team-tool/dispatch/index.ts +6 -5
- package/src/extension/team-tool/dispatch/status.ts +42 -2
- package/src/extension/team-tool.ts +24 -2
- package/src/prompt/inbox-poll.ts +77 -25
- package/src/prompt/message-tool.ts +62 -4
- package/src/prompt/prompt-runtime.ts +82 -32
- package/src/runtime/broker/crew-broker-client.ts +74 -16
- package/src/runtime/broker/crew-broker.ts +111 -77
- package/src/runtime/broker/protocol/msg-send-targets.ts +74 -0
- package/src/runtime/broker/wait-status-cache.ts +14 -5
- package/src/runtime/child-pi/child-pi-spawn.ts +47 -9
- package/src/runtime/child-pi/child-pi.ts +20 -2
- package/src/runtime/child-pi/mock-fixtures.ts +23 -0
- package/src/runtime/compaction/compact-stages/truncation-stage.ts +45 -3
- package/src/runtime/compaction/important-line-classifier.ts +85 -11
- package/src/runtime/compaction/tool-output-pruner.ts +35 -11
- package/src/runtime/crew-agent-records.ts +7 -1
- package/src/runtime/delivery-coordinator.ts +30 -0
- package/src/runtime/dwf-state-store.ts +41 -7
- package/src/runtime/finalize-run.ts +13 -2
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +192 -64
- package/src/runtime/goal-workflow/goal-loop-runner.ts +130 -3
- package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
- package/src/runtime/live-session/embedded-tier.ts +3 -1
- package/src/runtime/live-session/live-session-runtime.ts +54 -1
- package/src/runtime/model/model-fallback.ts +23 -1
- package/src/runtime/output/pi-json-output.ts +26 -1
- package/src/runtime/policy-engine.ts +18 -0
- package/src/runtime/process/zombie-scanner.ts +78 -14
- package/src/runtime/process-status.ts +6 -1
- package/src/runtime/recovery/crash-recovery.ts +170 -28
- package/src/runtime/rpc/rpc-worker.ts +44 -1
- package/src/runtime/scheduling/run-coalesced-task-group.ts +64 -3
- package/src/runtime/scheduling/scheduler.ts +74 -11
- package/src/runtime/scratchpad/engine.ts +26 -3
- package/src/runtime/settings-store.ts +22 -2
- package/src/runtime/surface/tmux-provider.ts +24 -3
- package/src/runtime/task-health.ts +16 -1
- package/src/runtime/task-output-context.ts +143 -71
- package/src/runtime/task-packet.ts +70 -17
- package/src/runtime/task-runner/child-executor.ts +27 -0
- package/src/runtime/task-runner/output-splitter.ts +79 -16
- package/src/runtime/task-runner/post-execution.ts +75 -15
- package/src/runtime/task-runner/state-helpers.ts +92 -2
- package/src/runtime/usage-tracker.ts +11 -0
- package/src/runtime/verification/green-contract.ts +17 -0
- package/src/runtime/verification/verification-gates.ts +24 -6
- package/src/runtime/workspace-lock.ts +53 -0
- package/src/schema/team-tool-schema.ts +10 -4
- package/src/state/coordination/mailbox.ts +133 -38
- package/src/state/decision-ledger.ts +56 -8
- package/src/state/event-log/event-log-rotation.ts +83 -5
- package/src/state/event-log/event-log.ts +181 -12
- package/src/state/stores/active-run-registry.ts +10 -3
- package/src/state/types.ts +9 -1
- package/src/ui/agents-jobs-browser.ts +22 -8
- package/src/ui/dashboard-panes/agents-pane.ts +5 -2
- package/src/ui/events-state-source.ts +67 -28
- package/src/ui/inline-panel/agent-pane.ts +13 -6
- package/src/ui/inline-panel/agent-transcript.ts +33 -21
- package/src/ui/inline-panel/agent-view-overlay.ts +18 -3
- package/src/ui/inline-panel/index.ts +21 -1
- package/src/ui/live-run-sidebar.ts +14 -3
- package/src/ui/overlays/mailbox-compose-overlay.ts +10 -4
- package/src/ui/powerbar-publisher.ts +6 -3
- package/src/ui/rail.ts +7 -1
- package/src/ui/render-scheduler.ts +53 -1
- package/src/ui/run-action-dispatcher.ts +68 -20
- package/src/ui/status-colors.ts +9 -0
- package/src/utils/env-filter.ts +8 -0
|
@@ -28,6 +28,22 @@
|
|
|
28
28
|
* Granularity: per-goal, held for the goal's lifetime (release() on goal end).
|
|
29
29
|
* Contention: default QUEUE (poll until released or stale);
|
|
30
30
|
* opts.failOnWorkspaceBusy:true → THROW instead of queue.
|
|
31
|
+
*
|
|
32
|
+
* ─── HEARTBEAT REFRESH CONTRACT (WI-2 / DR-02 R4-3, 2026-10-15) ───
|
|
33
|
+
* The heartbeat is written ONCE at acquire. Clause (b) above marks a lock
|
|
34
|
+
* stale after HEARTBEAT_STALE_MS REGARDLESS of holder pid liveness — so a
|
|
35
|
+
* live goal whose loop runs longer than 60s would be force-stolen by a
|
|
36
|
+
* second goal on the same cwd (multi-goal clobber, audit R4-3). Holders that
|
|
37
|
+
* stay in a long-running loop MUST keep the heartbeat fresh via
|
|
38
|
+
* refreshWorkspaceLockHeartbeat(lockPath, goalId) — the goal-loop runner does
|
|
39
|
+
* this on a 15s unref-ed timer (startWorkspaceLockHeartbeatRefresh in
|
|
40
|
+
* goal-loop-runner.ts), stopped in the same finally that releases the lock.
|
|
41
|
+
* The refresh re-stamps ONLY `heartbeat` (all other fields + file format
|
|
42
|
+
* preserved) and ONLY while the lock still belongs to (goalId, pid) — a lock
|
|
43
|
+
* reclaimed by another goal is never clobbered. Decision note: the interval
|
|
44
|
+
* refresh was chosen over "drop clause (b) / PID-liveness-only staleness"
|
|
45
|
+
* because clause (b) is the only crash signal for kill -9'd holders on
|
|
46
|
+
* non-Linux platforms where /proc startTime is unavailable.
|
|
31
47
|
*/
|
|
32
48
|
|
|
33
49
|
import { createHash } from "node:crypto";
|
|
@@ -149,6 +165,43 @@ function writeLock(lockPath: string, contents: WorkspaceLockContents): void {
|
|
|
149
165
|
atomicWriteJson(lockPath, contents);
|
|
150
166
|
}
|
|
151
167
|
|
|
168
|
+
export interface RefreshWorkspaceLockHeartbeatOptions {
|
|
169
|
+
/** Test injection: override current time (ms). Default Date.now(). */
|
|
170
|
+
now?: () => number;
|
|
171
|
+
/** Test injection: override the current pid. Default process.pid. */
|
|
172
|
+
pid?: number;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Refresh the heartbeat of a lock this goal still owns (WI-2 / DR-02 R4-3).
|
|
177
|
+
*
|
|
178
|
+
* Reads the lockfile and rewrites it with `heartbeat: now()` — preserving
|
|
179
|
+
* every other field (pid, startTime, goalId, acquiredAt) and the file format
|
|
180
|
+
* — ONLY when the lock still belongs to (goalId, pid). Returns false —
|
|
181
|
+
* leaving the file untouched — when the lockfile is missing, was reclaimed /
|
|
182
|
+
* re-acquired by another goal, or the write failed: refreshing must NEVER
|
|
183
|
+
* clobber a lock that someone else owns (same ownership guard philosophy as
|
|
184
|
+
* safeRelease). This is what keeps a >60s-running goal's lock from being
|
|
185
|
+
* force-stolen via isLockStale clause (b) despite a live pid.
|
|
186
|
+
*
|
|
187
|
+
* Non-throwing by contract: failures surface as false, never as an exception
|
|
188
|
+
* into the caller's loop (the goal-loop still wraps calls in try/catch →
|
|
189
|
+
* logInternalError warn as belt-and-braces, see goal-loop-runner.ts).
|
|
190
|
+
*/
|
|
191
|
+
export function refreshWorkspaceLockHeartbeat(lockPath: string, goalId: string, opts: RefreshWorkspaceLockHeartbeatOptions = {}): boolean {
|
|
192
|
+
try {
|
|
193
|
+
const current = readLock(lockPath);
|
|
194
|
+
if (!current) return false;
|
|
195
|
+
const pid = opts.pid ?? process.pid;
|
|
196
|
+
if (current.goalId !== goalId || current.pid !== pid) return false;
|
|
197
|
+
const now = opts.now ?? Date.now;
|
|
198
|
+
writeLock(lockPath, { ...current, heartbeat: now() });
|
|
199
|
+
return true;
|
|
200
|
+
} catch {
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
152
205
|
/**
|
|
153
206
|
* CLAIM the lockfile atomically via O_EXCL (cold-review #3 NIT #N1 fix).
|
|
154
207
|
* The previous claim used temp+rename (writeLock), which is NOT cross-process atomic —
|
|
@@ -74,7 +74,7 @@ export function buildStringEnum(values: readonly string[], description: string,
|
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
// ───────────────────────────────────────────────────────────────────────────
|
|
77
|
-
// API-5 facade split: the
|
|
77
|
+
// API-5 facade split: the 57-action mega-tool schema is split into 5 domain
|
|
78
78
|
// schemas. Each domain owns a subset of the action union; all parameter fields
|
|
79
79
|
// are shared across every variant so Phase 1 validation is identical to the
|
|
80
80
|
// pre-split flat Object (additionalProperties: true → no tightening).
|
|
@@ -226,14 +226,20 @@ const sharedFields = {
|
|
|
226
226
|
// Also accept stringified numbers ("123456") — pi-ai coercion stringifies
|
|
227
227
|
// numeric values when a Union has a string-literal branch. handleTeamTool
|
|
228
228
|
// coerces strings back to numbers before handlers run.
|
|
229
|
+
// R7-5 (docs-truth): this value is ADVISORY metadata — the enforced
|
|
230
|
+
// deadline is the broker-clamped task.waiting.deadline, not this field.
|
|
229
231
|
Type.Union(
|
|
230
232
|
[
|
|
231
233
|
Type.Literal(""),
|
|
232
|
-
Type.Integer({
|
|
234
|
+
Type.Integer({
|
|
235
|
+
description:
|
|
236
|
+
"Advisory metadata only. The enforced deadline is auto-clamped by the broker (task.waiting.deadline); this value is recorded, not enforced.",
|
|
237
|
+
}),
|
|
233
238
|
Type.String({ pattern: NUMERIC_STRING_RE }),
|
|
234
239
|
],
|
|
235
240
|
{
|
|
236
|
-
description:
|
|
241
|
+
description:
|
|
242
|
+
"Advisory metadata only. The enforced deadline is auto-clamped by the broker (task.waiting.deadline); this value is recorded, not enforced.",
|
|
237
243
|
},
|
|
238
244
|
),
|
|
239
245
|
),
|
|
@@ -589,7 +595,7 @@ export interface TeamToolParamsValue {
|
|
|
589
595
|
replyTo?: string;
|
|
590
596
|
/** Task ID sending the reply. */
|
|
591
597
|
replyFrom?: string;
|
|
592
|
-
/**
|
|
598
|
+
/** Advisory metadata only. The enforced deadline is auto-clamped by the broker (task.waiting.deadline); this value is recorded, not enforced. */
|
|
593
599
|
replyDeadline?: number;
|
|
594
600
|
/** Path to a markdown plan document for orchestration. */
|
|
595
601
|
planPath?: string;
|
|
@@ -375,9 +375,27 @@ function cachedMailboxRead(filePath: string, direction: MailboxDirection): Mailb
|
|
|
375
375
|
return messages.slice();
|
|
376
376
|
}
|
|
377
377
|
|
|
378
|
+
/** R7-3 (2026-10-16): rotated archive siblings of a live mailbox file,
|
|
379
|
+
* sorted lexicographically. The archive name embeds an ISO timestamp, so
|
|
380
|
+
* lexicographic order ≈ chronological (oldest first). */
|
|
381
|
+
function listMailboxArchives(filePath: string): string[] {
|
|
382
|
+
try {
|
|
383
|
+
const dir = path.dirname(filePath);
|
|
384
|
+
const base = path.basename(filePath);
|
|
385
|
+
return fs
|
|
386
|
+
.readdirSync(dir)
|
|
387
|
+
.filter((entry) => entry.startsWith(`${base}.`) && entry.endsWith(".archive.jsonl"))
|
|
388
|
+
.sort()
|
|
389
|
+
.map((entry) => path.join(dir, entry));
|
|
390
|
+
} catch {
|
|
391
|
+
// Directory missing — nothing to read.
|
|
392
|
+
return [];
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
378
396
|
function safeReadMailboxFile(filePath: string, direction: MailboxDirection): MailboxMessage[] {
|
|
379
397
|
// PERF: stat-gated primary read — ENOENT is handled inside cachedMailboxRead.
|
|
380
|
-
const
|
|
398
|
+
const live = cachedMailboxRead(filePath, direction);
|
|
381
399
|
// 3.3 — also include any rotated archive files alongside the live file.
|
|
382
400
|
// Archive naming: `<filename>.<isoTimestamp>.archive.jsonl`.
|
|
383
401
|
// PERF: archives are immutable once written (rotation only ever creates
|
|
@@ -385,18 +403,37 @@ function safeReadMailboxFile(filePath: string, direction: MailboxDirection): Mai
|
|
|
385
403
|
// sequence), so they route through cachedMailboxRead too and permanently
|
|
386
404
|
// hit after the first parse. pruneOldMailboxArchives deleting an archive
|
|
387
405
|
// is covered by cachedMailboxRead's ENOENT → cache-delete branch.
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
406
|
+
const archives = listMailboxArchives(filePath);
|
|
407
|
+
if (archives.length === 0) return live;
|
|
408
|
+
// R7-3 (2026-10-16) — same-id LAST-WINS merge, live file applied LAST.
|
|
409
|
+
//
|
|
410
|
+
// DESIGN: rotation moves a message into an archive and recreates an empty
|
|
411
|
+
// live file, so a message normally exists in exactly ONE place and a plain
|
|
412
|
+
// concat was duplicate-free. The exception is updateMailboxMessageReply:
|
|
413
|
+
// archives are IMMUTABLE history and are never rewritten, so when the
|
|
414
|
+
// original message lives only in an archive the reply is recorded as a
|
|
415
|
+
// COMPENSATING record — same id, reply metadata set — appended to the LIVE
|
|
416
|
+
// file. Merging by id (archives oldest→newest first, live file last)
|
|
417
|
+
// collapses the pair into a single reply-marked message, so readMailbox /
|
|
418
|
+
// readMailboxMessage expose ONE copy with repliedAt set; consumers
|
|
419
|
+
// (replayPendingMailboxMessages, listMailboxByKind, …) never see the
|
|
420
|
+
// duplicate reply-less archived original.
|
|
421
|
+
const merged: MailboxMessage[] = [];
|
|
422
|
+
const indexById = new Map<string, number>();
|
|
423
|
+
const upsert = (message: MailboxMessage): void => {
|
|
424
|
+
const existing = indexById.get(message.id);
|
|
425
|
+
if (existing === undefined) {
|
|
426
|
+
indexById.set(message.id, merged.length);
|
|
427
|
+
merged.push(message);
|
|
428
|
+
} else {
|
|
429
|
+
merged[existing] = message;
|
|
395
430
|
}
|
|
396
|
-
}
|
|
397
|
-
|
|
431
|
+
};
|
|
432
|
+
for (const archivePath of archives) {
|
|
433
|
+
for (const message of cachedMailboxRead(archivePath, direction)) upsert(message);
|
|
398
434
|
}
|
|
399
|
-
|
|
435
|
+
for (const message of live) upsert(message);
|
|
436
|
+
return merged;
|
|
400
437
|
}
|
|
401
438
|
|
|
402
439
|
/**
|
|
@@ -1138,6 +1175,15 @@ export function acknowledgeMailboxMessage(manifest: TeamRunManifest, messageId:
|
|
|
1138
1175
|
* Update an original mailbox message with reply metadata.
|
|
1139
1176
|
* Rewrites the mailbox file line containing the original message
|
|
1140
1177
|
* to include `repliedAt` and `replyContent`.
|
|
1178
|
+
*
|
|
1179
|
+
* R7-3 (2026-10-16): the original may have been ROTATED out of the live file
|
|
1180
|
+
* into a `*.archive.jsonl` sibling (rotateMailboxFileIfNeeded). Archives are
|
|
1181
|
+
* IMMUTABLE history — they are never rewritten here. When the original is
|
|
1182
|
+
* found only in an archive, a COMPENSATING record (same id + reply metadata)
|
|
1183
|
+
* is appended to the LIVE file instead; safeReadMailboxFile merges by id
|
|
1184
|
+
* last-wins (live overrides archive), so every reader sees the reply-marked
|
|
1185
|
+
* message. Global run mailboxes AND every task mailbox (inbox + outbox, live
|
|
1186
|
+
* + archives) are scanned.
|
|
1141
1187
|
*/
|
|
1142
1188
|
export function updateMailboxMessageReply(manifest: TeamRunManifest, originalMessageId: string, replyContent: string): void {
|
|
1143
1189
|
const directions: MailboxDirection[] = ["inbox", "outbox"];
|
|
@@ -1167,35 +1213,64 @@ export function updateMailboxMessageReply(manifest: TeamRunManifest, originalMes
|
|
|
1167
1213
|
}
|
|
1168
1214
|
|
|
1169
1215
|
for (const { filePath, direction } of filesToSearch) {
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
const
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
msg
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1216
|
+
const archives = listMailboxArchives(filePath);
|
|
1217
|
+
if (fs.existsSync(filePath)) {
|
|
1218
|
+
// Path 1 (original behavior): the message still lives in the LIVE
|
|
1219
|
+
// file — rewrite its line in place.
|
|
1220
|
+
// FIX: Wrap read-modify-write in withFileLockSync to prevent concurrent
|
|
1221
|
+
// updates from clobbering each other (each reply rewrites the whole file).
|
|
1222
|
+
const found = withFileLockSync(filePath, () => {
|
|
1223
|
+
const lines = fs.readFileSync(filePath, "utf-8").split(/\r?\n/).filter(Boolean);
|
|
1224
|
+
let localFound = false;
|
|
1225
|
+
const updatedLines: string[] = [];
|
|
1226
|
+
for (const line of lines) {
|
|
1227
|
+
try {
|
|
1228
|
+
const parsed = JSON.parse(line) as unknown;
|
|
1229
|
+
const msg = parseMailboxMessage(parsed, direction);
|
|
1230
|
+
if (msg && msg.id === originalMessageId) {
|
|
1231
|
+
msg.repliedAt = new Date().toISOString();
|
|
1232
|
+
msg.replyContent = replyContent;
|
|
1233
|
+
updatedLines.push(JSON.stringify(redactSecrets(msg)));
|
|
1234
|
+
localFound = true;
|
|
1235
|
+
} else {
|
|
1236
|
+
updatedLines.push(line);
|
|
1237
|
+
}
|
|
1238
|
+
} catch {
|
|
1187
1239
|
updatedLines.push(line);
|
|
1188
1240
|
}
|
|
1189
|
-
} catch {
|
|
1190
|
-
updatedLines.push(line);
|
|
1191
1241
|
}
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1242
|
+
if (localFound) {
|
|
1243
|
+
atomicWriteFile(filePath, `${updatedLines.join("\n")}\n`);
|
|
1244
|
+
}
|
|
1245
|
+
return localFound;
|
|
1246
|
+
});
|
|
1247
|
+
if (found) return;
|
|
1248
|
+
}
|
|
1249
|
+
if (archives.length === 0) continue;
|
|
1250
|
+
// Path 2 (R7-3): the original was rotated into an archive. Archives are
|
|
1251
|
+
// immutable (see safeReadMailboxFile) — do NOT rewrite them; append a
|
|
1252
|
+
// compensating record carrying the reply metadata to the LIVE file.
|
|
1253
|
+
// The id-based last-wins merge makes it win over the archived original.
|
|
1254
|
+
const archived = archives
|
|
1255
|
+
.flatMap((archivePath) => cachedMailboxRead(archivePath, direction))
|
|
1256
|
+
.find((message) => message.id === originalMessageId);
|
|
1257
|
+
if (!archived) continue;
|
|
1258
|
+
const compensating: MailboxMessage = {
|
|
1259
|
+
...archived,
|
|
1260
|
+
repliedAt: new Date().toISOString(),
|
|
1261
|
+
replyContent,
|
|
1262
|
+
};
|
|
1263
|
+
withFileLockSync(filePath, () => {
|
|
1264
|
+
// Recreate the live file if it went missing — the archives in this
|
|
1265
|
+
// directory prove the mailbox itself is valid.
|
|
1266
|
+
if (!fs.existsSync(filePath)) atomicWriteFile(filePath, "");
|
|
1267
|
+
fs.appendFileSync(filePath, `${JSON.stringify(redactSecrets(compensating))}\n`, "utf-8");
|
|
1268
|
+
// The append may itself trigger rotation — safe: the compensating
|
|
1269
|
+
// record then lands in the NEWEST archive, which sorts after the
|
|
1270
|
+
// original's archive in the merge order, so it still wins by id.
|
|
1271
|
+
rotateMailboxFileIfNeeded(filePath);
|
|
1197
1272
|
});
|
|
1198
|
-
|
|
1273
|
+
return;
|
|
1199
1274
|
}
|
|
1200
1275
|
// Not finding the original is non-fatal; the reply is still delivered.
|
|
1201
1276
|
}
|
|
@@ -1224,9 +1299,27 @@ export function validateMailbox(
|
|
|
1224
1299
|
ensureRunMailbox(manifest);
|
|
1225
1300
|
const issues: MailboxValidationIssue[] = [];
|
|
1226
1301
|
const repaired: string[] = [];
|
|
1302
|
+
// L-9 (2026-10-16): validate + repair the TASK mailboxes
|
|
1303
|
+
// (`mailbox/tasks/<id>/<direction>.jsonl`) with the same line-parse
|
|
1304
|
+
// semantics as the run-level files — enumerate the tasks dir the same way
|
|
1305
|
+
// updateMailboxMessageReply does. Rotated `*.archive.jsonl` files are
|
|
1306
|
+
// immutable history and are deliberately NOT repaired.
|
|
1307
|
+
const filesToValidate: Array<{ filePath: string; direction: MailboxDirection }> = [];
|
|
1227
1308
|
for (const direction of ["inbox", "outbox"] as const) {
|
|
1309
|
+
filesToValidate.push({ filePath: mailboxFile(manifest, direction), direction });
|
|
1310
|
+
}
|
|
1311
|
+
const tasksDir = safeMailboxTasksRoot(manifest);
|
|
1312
|
+
if (fs.existsSync(tasksDir)) {
|
|
1313
|
+
for (const entry of fs.readdirSync(tasksDir, { withFileTypes: true })) {
|
|
1314
|
+
if (!entry.isDirectory()) continue;
|
|
1315
|
+
for (const direction of ["inbox", "outbox"] as const) {
|
|
1316
|
+
filesToValidate.push({ filePath: mailboxFile(manifest, direction, entry.name), direction });
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
for (const { filePath, direction } of filesToValidate) {
|
|
1228
1321
|
if (options.signal?.aborted) break;
|
|
1229
|
-
|
|
1322
|
+
if (!fs.existsSync(filePath)) continue;
|
|
1230
1323
|
// FIX: Wrap read + optional repair in withFileLockSync so concurrent appends
|
|
1231
1324
|
// don't race with the read-modify-write. Mailbox files are capped at 10MB
|
|
1232
1325
|
// (MAILBOX_ARCHIVE_THRESHOLD_BYTES), so the per-call memory is bounded.
|
|
@@ -1254,7 +1347,9 @@ export function validateMailbox(
|
|
|
1254
1347
|
});
|
|
1255
1348
|
}
|
|
1256
1349
|
const delivery = readDeliveryState(manifest);
|
|
1257
|
-
|
|
1350
|
+
// L-9: delivery-entry coverage now includes task mailbox messages
|
|
1351
|
+
// (readAllMessages walks the global files AND every task mailbox).
|
|
1352
|
+
const allMessages = [...readAllMessages(manifest, "inbox"), ...readAllMessages(manifest, "outbox")];
|
|
1258
1353
|
for (const message of allMessages) {
|
|
1259
1354
|
if (options.signal?.aborted) break;
|
|
1260
1355
|
if (!delivery.messages[message.id])
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
|
+
import { logInternalError } from "../utils/internal-error.ts";
|
|
3
4
|
import { projectCrewRoot } from "../utils/paths.ts";
|
|
4
5
|
import { assertSafePathId } from "../utils/safe-paths.ts";
|
|
5
6
|
import { withFileLockSync } from "./coordination/locks.ts";
|
|
@@ -145,6 +146,12 @@ export function appendEntry(runId: string, entry: RolloutEntry): RolloutEntry {
|
|
|
145
146
|
|
|
146
147
|
/**
|
|
147
148
|
* Read all entries from the decision ledger.
|
|
149
|
+
*
|
|
150
|
+
* P-3 (DR-04 round-7): tolerant parse — a corrupt line (partial write, manual
|
|
151
|
+
* edit, disk garbage) is SKIPPED, not thrown: one bad tail line must not take
|
|
152
|
+
* down every reader of the ledger. Unparseable lines are logged ONCE per read
|
|
153
|
+
* (with the count) via logInternalError; valid lines are preserved. The reader
|
|
154
|
+
* never throws.
|
|
148
155
|
*/
|
|
149
156
|
export function getLedger(runId: string): RolloutEntry[] {
|
|
150
157
|
assertSafePathId("runId", runId);
|
|
@@ -154,15 +161,47 @@ export function getLedger(runId: string): RolloutEntry[] {
|
|
|
154
161
|
return [];
|
|
155
162
|
}
|
|
156
163
|
|
|
157
|
-
|
|
164
|
+
let content: string;
|
|
165
|
+
try {
|
|
166
|
+
content = readFileSync(ledgerPath, "utf-8");
|
|
167
|
+
} catch (error) {
|
|
168
|
+
// P-3: read failure (EACCES, transient EAGAIN, …) — warn and treat as
|
|
169
|
+
// empty rather than throwing out of a status/summary reader.
|
|
170
|
+
logInternalError("decision-ledger.getLedger", error, `run=${runId}`, "warn");
|
|
171
|
+
return [];
|
|
172
|
+
}
|
|
158
173
|
if (!content.trim()) {
|
|
159
174
|
return [];
|
|
160
175
|
}
|
|
161
176
|
|
|
162
|
-
return content
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
177
|
+
return parseLedgerLines(content, runId, "decision-ledger.getLedger");
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* P-3 (DR-04 round-7): parse JSONL ledger lines tolerantly — valid lines are
|
|
182
|
+
* kept, unparseable lines are skipped and reported ONCE per call with a count
|
|
183
|
+
* (plus a bounded preview) so a corrupt tail is visible without flooding.
|
|
184
|
+
*/
|
|
185
|
+
function parseLedgerLines(content: string, runId: string, scope: string): RolloutEntry[] {
|
|
186
|
+
const entries: RolloutEntry[] = [];
|
|
187
|
+
const skipped: string[] = [];
|
|
188
|
+
for (const line of content.split("\n")) {
|
|
189
|
+
if (!line.trim()) continue;
|
|
190
|
+
try {
|
|
191
|
+
entries.push(JSON.parse(line) as RolloutEntry);
|
|
192
|
+
} catch {
|
|
193
|
+
skipped.push(line.slice(0, 80));
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (skipped.length > 0) {
|
|
197
|
+
logInternalError(
|
|
198
|
+
scope,
|
|
199
|
+
new Error(`skipped ${skipped.length} unparseable decision-ledger line(s)`),
|
|
200
|
+
`run=${runId} lines=${JSON.stringify(skipped)}`,
|
|
201
|
+
"warn",
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
return entries;
|
|
166
205
|
}
|
|
167
206
|
|
|
168
207
|
/**
|
|
@@ -181,13 +220,22 @@ function readTailEntries(runId: string, n: number = 10): RolloutEntry[] {
|
|
|
181
220
|
return [];
|
|
182
221
|
}
|
|
183
222
|
|
|
184
|
-
|
|
223
|
+
let content: string;
|
|
224
|
+
try {
|
|
225
|
+
content = readFileSync(ledgerPath, "utf-8");
|
|
226
|
+
} catch (error) {
|
|
227
|
+
// P-3: same tolerant-read contract as getLedger — the append path must
|
|
228
|
+
// not throw out of the file-lock callback because the tail was unreadable.
|
|
229
|
+
logInternalError("decision-ledger.readTailEntries", error, `run=${runId}`, "warn");
|
|
230
|
+
return [];
|
|
231
|
+
}
|
|
185
232
|
if (!content.trim()) {
|
|
186
233
|
return [];
|
|
187
234
|
}
|
|
188
235
|
|
|
189
|
-
|
|
190
|
-
|
|
236
|
+
// P-3: skip unparseable lines first, THEN take the tail — otherwise a
|
|
237
|
+
// single corrupt tail line would slice garbage into the coherence window.
|
|
238
|
+
return parseLedgerLines(content, runId, "decision-ledger.readTailEntries").slice(-n);
|
|
191
239
|
}
|
|
192
240
|
|
|
193
241
|
/**
|
|
@@ -2,7 +2,7 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
4
4
|
import { atomicWriteFile } from "../atomic-write.ts";
|
|
5
|
-
import { type TeamEvent, withEventLogLockSync } from "./event-log.ts";
|
|
5
|
+
import { type TeamEvent, withEventLogAppendLockSync, withEventLogLockSync } from "./event-log.ts";
|
|
6
6
|
import { poisonMarkerPath } from "./poison-marker.ts";
|
|
7
7
|
|
|
8
8
|
/**
|
|
@@ -169,7 +169,11 @@ export interface CompactionResult {
|
|
|
169
169
|
* 5. If events were lost, append them
|
|
170
170
|
* 6. Return compaction stats
|
|
171
171
|
*/
|
|
172
|
-
export function compactEventLog(
|
|
172
|
+
export function compactEventLog(
|
|
173
|
+
eventsPath: string,
|
|
174
|
+
config?: Partial<RotationConfig>,
|
|
175
|
+
options?: CompactionLockOptions,
|
|
176
|
+
): CompactionResult | undefined {
|
|
173
177
|
// US-001 (2026-09-22): lock-scope reduction. The expensive part is the READ +
|
|
174
178
|
// parse of the whole log (prepareCompaction). It is read-only, so it does NOT
|
|
175
179
|
// need the append lock: hold the lock ONLY for the write+recover phase
|
|
@@ -196,7 +200,15 @@ export function compactEventLog(eventsPath: string, config?: Partial<RotationCon
|
|
|
196
200
|
// log grew unbounded until events were silently dropped past 50MB).
|
|
197
201
|
const prepared = prepareCompaction(eventsPath, config);
|
|
198
202
|
if (!prepared) return undefined;
|
|
199
|
-
return compactPreparedEventLog(eventsPath, prepared);
|
|
203
|
+
return compactPreparedEventLog(eventsPath, prepared, options);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** Options for the locked write phase of compaction. `lock` is forwarded to
|
|
207
|
+
* `withEventLogAppendLockSync` (and transitively `withEventLogLockSync`) so
|
|
208
|
+
* callers/tests can bound the wait — the defaults (5s timeout / 10s stale)
|
|
209
|
+
* mirror both existing lock families. */
|
|
210
|
+
export interface CompactionLockOptions {
|
|
211
|
+
lock?: { timeoutMs?: number; staleMs?: number };
|
|
200
212
|
}
|
|
201
213
|
|
|
202
214
|
/**
|
|
@@ -212,15 +224,81 @@ export function compactEventLog(eventsPath: string, config?: Partial<RotationCon
|
|
|
212
224
|
* ones it never saw. So under the lock we read the bytes appended after the
|
|
213
225
|
* snapshot offset and append them to the write. Appenders hold the same lock,
|
|
214
226
|
* so this read is a complete tail snapshot.
|
|
227
|
+
*
|
|
228
|
+
* R4-2 (DR-02 Wave 2, 2026-10-15): the write phase now ADDITIONALLY acquires the
|
|
229
|
+
* async family's `.alock` (sync-mode, via `withEventLogAppendLockSync`) BEFORE
|
|
230
|
+
* `readTailSince`/the `atomicWriteFile` rename, releasing it only after the C2
|
|
231
|
+
* recovery check completes. Rationale: async-family appends hold `.alock`
|
|
232
|
+
* ACROSS await gaps (`open(eventsPath, "a")` → `appendFile`); running the rename
|
|
233
|
+
* under `.mkdirlock` alone let it interleave with that gap, orphaning the
|
|
234
|
+
* appender's fd — its event landed on the old inode and was silently lost (C2
|
|
235
|
+
* only re-splices the `kept` window). Holding `.alock` across the whole write
|
|
236
|
+
* phase excludes in-flight async appends cross-process (same on-disk lock dir
|
|
237
|
+
* + pid-file protocol). Lock order stays strictly `.mkdirlock` → `.alock` (the
|
|
238
|
+
* async family never takes `.mkdirlock`), so no deadlock. NOT archive-style by
|
|
239
|
+
* lead decision: rotation's rename-to-archive already preserves in-flight fds
|
|
240
|
+
* (ST-8), so only compaction's inode-replacing write needed the guard.
|
|
215
241
|
*/
|
|
216
242
|
export function compactPreparedEventLog(
|
|
217
243
|
eventsPath: string,
|
|
218
244
|
prepared: { lines: string; originalSize: number; originalCount: number; kept: TeamEvent[] },
|
|
245
|
+
options?: CompactionLockOptions,
|
|
219
246
|
): CompactionResult | undefined {
|
|
220
|
-
|
|
247
|
+
// R7-2 (DR-04, 2026-10-16): the write phase is now `applyCompactionAppendGuarded`
|
|
248
|
+
// (below) — `.alock` nested inside the `.mkdirlock` held here, plus the
|
|
249
|
+
// mandatory `readTailSince` splice. This wrapper keeps the historical
|
|
250
|
+
// locked entry point (public API + tests) with identical semantics.
|
|
251
|
+
return withEventLogLockSync(eventsPath, () => applyCompactionAppendGuarded(eventsPath, prepared, options), options?.lock);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** R7-2 (DR-04, 2026-10-16): options for `applyCompactionAppendGuarded`.
|
|
255
|
+
* `lock` is forwarded to `withEventLogAppendLockSync` so callers/tests can
|
|
256
|
+
* bound the wait; `holdsAppendLock` declares the caller's lock state (see the
|
|
257
|
+
* helper docblock). */
|
|
258
|
+
export interface ApplyCompactionGuardOptions extends CompactionLockOptions {
|
|
259
|
+
/** Pass `true` ONLY when the caller ALREADY holds the async family's
|
|
260
|
+
* `.alock` (i.e. the guarded apply runs inside `withEventLogLockAsync`).
|
|
261
|
+
* The on-disk lock is NOT re-entrant — re-acquiring our own `.alock` in
|
|
262
|
+
* sync mode can only end in the bounded timeout (sleepSync starves the
|
|
263
|
+
* very in-process holder that must release it). With `true` the helper
|
|
264
|
+
* only tail-merges; with `false`/omitted it acquires `.alock` itself. */
|
|
265
|
+
holdsAppendLock?: boolean;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* R7-2 (DR-04, 2026-10-16): the guarded write phase for INLINE compaction
|
|
270
|
+
* call sites (the overflow/rotation-tick paths inside event-log.ts) that
|
|
271
|
+
* previously called `prepareCompaction` + `applyCompactionUnlocked` bare —
|
|
272
|
+
* no `.alock` and no tail-merge, so the inode-replacing write could interleave
|
|
273
|
+
* with an in-flight async-family append parked between `open()` and
|
|
274
|
+
* `appendFile()` (its fd kept pointing at the renamed-away inode → silent
|
|
275
|
+
* loss; C2 recovery only re-splices the `kept` window), and sync-family bytes
|
|
276
|
+
* appended between the prepare-read and the write were clobbered.
|
|
277
|
+
*
|
|
278
|
+
* Contract:
|
|
279
|
+
* - ALWAYS merges `readTailSince(eventsPath, prepared.originalSize)` into
|
|
280
|
+
* `prepared.lines` before `applyCompactionUnlocked` — sync-family appenders
|
|
281
|
+
* never hold `.alock`, so they can land in the prepare→apply window even
|
|
282
|
+
* while an async-family caller holds `.alock`; the splice rescues them.
|
|
283
|
+
* - `opts.holdsAppendLock === true` (caller is inside `withEventLogLockAsync`):
|
|
284
|
+
* tail-merge ONLY — never re-acquire `.alock` (non-reentrant → self-timeout).
|
|
285
|
+
* - `opts.holdsAppendLock` false/omitted (caller holds `.mkdirlock` — sync
|
|
286
|
+
* family — or no lock): acquire `.alock` in sync mode nested inside the
|
|
287
|
+
* held `.mkdirlock`, per the R4-2 lock order (`.mkdirlock` → `.alock`; the
|
|
288
|
+
* async family never takes `.mkdirlock`, so no inversion). NEVER call this
|
|
289
|
+
* with `false` while `.alock` is already held by this process.
|
|
290
|
+
*/
|
|
291
|
+
export function applyCompactionAppendGuarded(
|
|
292
|
+
eventsPath: string,
|
|
293
|
+
prepared: { lines: string; originalSize: number; originalCount: number; kept: TeamEvent[] },
|
|
294
|
+
options?: ApplyCompactionGuardOptions,
|
|
295
|
+
): CompactionResult | undefined {
|
|
296
|
+
const applyWithTailMerge = (): CompactionResult | undefined => {
|
|
221
297
|
const merged = { ...prepared, lines: prepared.lines + readTailSince(eventsPath, prepared.originalSize) };
|
|
222
298
|
return applyCompactionUnlocked(eventsPath, merged);
|
|
223
|
-
}
|
|
299
|
+
};
|
|
300
|
+
if (options?.holdsAppendLock) return applyWithTailMerge();
|
|
301
|
+
return withEventLogAppendLockSync(eventsPath, applyWithTailMerge, options?.lock);
|
|
224
302
|
}
|
|
225
303
|
|
|
226
304
|
/**
|