pi-crew 0.9.44 → 0.9.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/README.md +22 -1
- package/dist/build-meta.json +68 -47
- package/dist/index.mjs +486 -237
- package/dist/index.mjs.map +3 -3
- package/package.json +1 -1
- package/src/agents/discover-agents.ts +1 -1
- package/src/extension/registration/lifecycle-handlers.ts +34 -3
- package/src/extension/registration/subagent-manager-setup.ts +178 -59
- package/src/extension/run-import.ts +21 -1
- package/src/extension/team-tool/api.ts +4 -2
- package/src/runtime/async-runner.ts +9 -1
- package/src/runtime/manifest-cache.ts +30 -0
- package/src/runtime/resilient-edit.ts +16 -15
- package/src/runtime/role-permission.ts +27 -2
- package/src/runtime/run-coalesced-task-group.ts +72 -15
- package/src/runtime/task-packet.ts +1 -1
- package/src/state/event-log.ts +88 -34
- package/src/state/locks.ts +53 -0
- package/src/state/mailbox.ts +165 -4
- package/src/state/run-metrics.ts +40 -12
- package/src/ui/live-run-sidebar.ts +1 -9
- package/src/ui/run-dashboard.ts +1 -9
- package/src/utils/incremental-reader.ts +105 -0
- package/src/utils/visual.ts +27 -91
- package/src/runtime/auto-resume.ts +0 -100
- package/src/runtime/notebook-helpers.ts +0 -88
- package/src/runtime/orphan-sentinel.ts +0 -7
|
@@ -1,20 +1,45 @@
|
|
|
1
1
|
export type RolePermissionMode = "read_only" | "workspace_write" | "danger_full_access" | "explicit_confirm";
|
|
2
2
|
|
|
3
|
+
// (FIND-12 R1 review: the AgentPermissionOptions opt-in was dropped — no caller
|
|
4
|
+
// passed `options` and AgentConfig has no `permissions` field, so it was dead
|
|
5
|
+
// code. Custom write-capable roles must instead be added to WRITE_ROLES below.)
|
|
6
|
+
|
|
3
7
|
// Read-only roles: cannot mutate files/source. `verifier` is NOT here — it runs
|
|
4
8
|
// tests (bash + cache writes) so it is a WRITE role (F4). `planner` stays
|
|
5
9
|
// read-only to preserve the plan-approval gate boundary (F3).
|
|
6
10
|
const READ_ONLY_ROLES = new Set(["explorer", "reviewer", "security-reviewer", "analyst", "critic", "planner"]);
|
|
7
|
-
|
|
11
|
+
// Write-capable roles: can mutate files/source within the workspace.
|
|
12
|
+
// FIND-12 (R1 review): this is an EXPLICIT allowlist — every shipped write-capable
|
|
13
|
+
// role must be listed here, otherwise default-deny demotes it to read-only.
|
|
14
|
+
// `agent` is the default direct-agent role (run.ts); `cold-verifier` runs bash/
|
|
15
|
+
// cache writes; `chain-executor` is the chain-workflow executor variant;
|
|
16
|
+
// `worker` is the autonomous goal-loop + dynamic-workflow executor role
|
|
17
|
+
// (goal-loop-runner.ts, run.ts dynamic workflow synthesis).
|
|
18
|
+
const WRITE_ROLES = new Set(["executor", "test-engineer", "writer", "verifier", "agent", "cold-verifier", "chain-executor", "worker"]);
|
|
8
19
|
export interface PermissionCheckResult {
|
|
9
20
|
allowed: boolean;
|
|
10
21
|
mode: RolePermissionMode;
|
|
11
22
|
reason?: string;
|
|
12
23
|
}
|
|
13
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the permission mode for a given role.
|
|
27
|
+
*
|
|
28
|
+
* FIND-12 (2026-07-20): **default-deny** — unknown/unrecognized roles now
|
|
29
|
+
* receive `"read_only"` instead of the previous permissive `"workspace_write"`.
|
|
30
|
+
* This prevents privilege escalation via typo'd or custom agent names that
|
|
31
|
+
* accidentally inherit write capabilities. Write-capable roles are an EXPLICIT
|
|
32
|
+
* allowlist (`WRITE_ROLES`); to add a custom write-capable role, add it there.
|
|
33
|
+
* (R1 review: a per-agent-config `permissions.workspaceWrite` opt-in was
|
|
34
|
+
* considered but dropped — it was unreachable from all 8 call sites.)
|
|
35
|
+
*
|
|
36
|
+
* @param role the role name (case-sensitive, must match exactly)
|
|
37
|
+
*/
|
|
14
38
|
export function permissionForRole(role: string): RolePermissionMode {
|
|
15
39
|
if (READ_ONLY_ROLES.has(role)) return "read_only";
|
|
16
40
|
if (WRITE_ROLES.has(role)) return "workspace_write";
|
|
17
|
-
|
|
41
|
+
// FIND-12: default-deny for unknown/unrecognized roles.
|
|
42
|
+
return "read_only";
|
|
18
43
|
}
|
|
19
44
|
|
|
20
45
|
export function currentCrewRole(env: NodeJS.ProcessEnv = process.env): string | undefined {
|
|
@@ -80,6 +80,18 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
80
80
|
|
|
81
81
|
let rawOutput = "";
|
|
82
82
|
let success = false;
|
|
83
|
+
// FIND-06: serialize heartbeat saves and retain the active save so terminal
|
|
84
|
+
// results can drain it before their final write.
|
|
85
|
+
let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
|
86
|
+
let heartbeatInFlight = false;
|
|
87
|
+
let heartbeatPromise: Promise<void> | null = null;
|
|
88
|
+
// FIND-06 P1 fix (R1 review): a heartbeat save that exceeds the 5s drain
|
|
89
|
+
// timeout continues in the background and can rename its temp file AFTER
|
|
90
|
+
// the terminal write, clobbering terminal state with the pre-terminal
|
|
91
|
+
// snapshot. finalWriteStarted lets the IIFE repair that by re-writing the
|
|
92
|
+
// (now-terminal) updatedTasks AFTER its own save resolves, so the terminal
|
|
93
|
+
// state always lands last regardless of disk timing.
|
|
94
|
+
let finalWriteStarted = false;
|
|
83
95
|
if (!executeWorkers) {
|
|
84
96
|
rawOutput = buildScaffoldOutput(groupTasks);
|
|
85
97
|
success = true;
|
|
@@ -87,20 +99,41 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
87
99
|
// Heartbeat refresher: touch every task's heartbeat every 15s while the
|
|
88
100
|
// worker is alive. Set `alive: true` explicitly so post-completion
|
|
89
101
|
// staleness checks immediately recognize liveness.
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
102
|
+
heartbeatTimer = setInterval(() => {
|
|
103
|
+
// FIND-06: setInterval does not await async callbacks. Do not start a
|
|
104
|
+
// second read/modify/write while the preceding heartbeat save is active.
|
|
105
|
+
if (heartbeatInFlight) return;
|
|
106
|
+
heartbeatInFlight = true;
|
|
107
|
+
heartbeatPromise = (async () => {
|
|
108
|
+
try {
|
|
109
|
+
updatedTasks = updatedTasks.map((t) => {
|
|
110
|
+
if (!taskIds.includes(t.id)) return t;
|
|
111
|
+
// FIND-06 belt-and-suspenders: never replace terminal state or its
|
|
112
|
+
// resultArtifact with a heartbeat-only snapshot.
|
|
113
|
+
if (t.status === "completed" || t.status === "failed") return t;
|
|
114
|
+
return {
|
|
115
|
+
...t,
|
|
116
|
+
heartbeat: touchWorkerHeartbeat(t.heartbeat ?? createWorkerHeartbeat(t.id), { alive: true }),
|
|
117
|
+
};
|
|
118
|
+
});
|
|
119
|
+
await saveRunTasksAsync(manifest, updatedTasks);
|
|
120
|
+
} catch {
|
|
121
|
+
// Run may have been pruned mid-dispatch — best-effort only.
|
|
122
|
+
} finally {
|
|
123
|
+
heartbeatInFlight = false;
|
|
124
|
+
// FIND-06 P1 fix: if the terminal write started while our heartbeat
|
|
125
|
+
// save was in flight, re-write the (now-terminal) updatedTasks so
|
|
126
|
+
// our late snapshot cannot leave stale pre-terminal state on disk.
|
|
127
|
+
// Runs after our own save resolves, so it always lands last.
|
|
128
|
+
if (finalWriteStarted) {
|
|
129
|
+
try {
|
|
130
|
+
await saveRunTasksAsync(manifest, updatedTasks);
|
|
131
|
+
} catch {
|
|
132
|
+
// best-effort repair — same swallow policy as the heartbeat.
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
})();
|
|
104
137
|
}, 15_000);
|
|
105
138
|
try {
|
|
106
139
|
const result = await executeWithRetry(
|
|
@@ -124,7 +157,7 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
124
157
|
rawOutput = `Worker dispatch failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
125
158
|
success = false;
|
|
126
159
|
} finally {
|
|
127
|
-
clearInterval(heartbeatTimer);
|
|
160
|
+
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
|
128
161
|
}
|
|
129
162
|
}
|
|
130
163
|
|
|
@@ -163,6 +196,30 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
163
196
|
resultArtifact,
|
|
164
197
|
};
|
|
165
198
|
});
|
|
199
|
+
|
|
200
|
+
// FIND-06: stop ticks and drain the snapshot captured by any active
|
|
201
|
+
// heartbeat before persisting terminal results. Bound the wait so a stuck
|
|
202
|
+
// filesystem operation cannot block dispatch completion indefinitely.
|
|
203
|
+
// P1 fix (R1 review): set finalWriteStarted BEFORE the drain so any pending
|
|
204
|
+
// heartbeat IIFE observes it and re-writes the terminal state after its
|
|
205
|
+
// own (possibly late) save resolves — preventing a late heartbeat snapshot
|
|
206
|
+
// from clobbering the terminal write.
|
|
207
|
+
finalWriteStarted = true;
|
|
208
|
+
if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
|
|
209
|
+
const pendingHeartbeat = heartbeatPromise;
|
|
210
|
+
if (pendingHeartbeat) {
|
|
211
|
+
let drainTimeout: ReturnType<typeof setTimeout> | undefined;
|
|
212
|
+
try {
|
|
213
|
+
await Promise.race([
|
|
214
|
+
pendingHeartbeat,
|
|
215
|
+
new Promise<void>((resolve) => {
|
|
216
|
+
drainTimeout = setTimeout(resolve, 5_000);
|
|
217
|
+
}),
|
|
218
|
+
]);
|
|
219
|
+
} finally {
|
|
220
|
+
if (drainTimeout !== undefined) clearTimeout(drainTimeout);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
166
223
|
await saveRunTasksAsync(manifest, updatedTasks);
|
|
167
224
|
let updatedManifest: TeamRunManifest = {
|
|
168
225
|
...manifest,
|
|
@@ -85,7 +85,7 @@ export function buildTaskPacket(input: BuildTaskPacketInput): TaskPacket {
|
|
|
85
85
|
|
|
86
86
|
// Generate a deterministic hash-based task ID for traceability and logging.
|
|
87
87
|
// Uses goal + step ID + run ID as content parts.
|
|
88
|
-
// TODO:
|
|
88
|
+
// TODO(hashId): tracked — add hashId to TaskPacket when the schema supports it.
|
|
89
89
|
const _taskHashId = generateTaskHashId([input.manifest.goal, input.step.id, input.manifest.runId]);
|
|
90
90
|
|
|
91
91
|
return {
|
package/src/state/event-log.ts
CHANGED
|
@@ -4,7 +4,7 @@ import * as path from "node:path";
|
|
|
4
4
|
import { DEFAULT_EVENT_LOG } from "../config/defaults.ts";
|
|
5
5
|
import { errors } from "../errors.ts";
|
|
6
6
|
import { emitFromTeamEvent } from "../ui/run-event-bus.ts";
|
|
7
|
-
import { type IncrementalReadState, readJsonlSince } from "../utils/incremental-reader.ts";
|
|
7
|
+
import { type IncrementalReadState, readJsonlSince, readJsonlTail } from "../utils/incremental-reader.ts";
|
|
8
8
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
9
9
|
import { redactSecrets } from "../utils/redaction.ts";
|
|
10
10
|
import { sleepSync } from "../utils/sleep.ts";
|
|
@@ -543,9 +543,13 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
543
543
|
} catch {
|
|
544
544
|
/* file does not exist */
|
|
545
545
|
}
|
|
546
|
+
// FIND-10: track whether overflow handling modified the file so we can
|
|
547
|
+
// reuse fileStat for the post-overflow size check (avoids redundant stat).
|
|
548
|
+
let overflowHandled = false;
|
|
546
549
|
if (!isTerminal && fileStat) {
|
|
547
550
|
const stat = fileStat;
|
|
548
551
|
if (stat.size > MAX_EVENTS_BYTES) {
|
|
552
|
+
overflowHandled = true;
|
|
549
553
|
try {
|
|
550
554
|
compactEventLog(eventsPath);
|
|
551
555
|
} catch (error) {
|
|
@@ -564,11 +568,17 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
564
568
|
}
|
|
565
569
|
}
|
|
566
570
|
}
|
|
571
|
+
// FIND-10: collapse redundant stat. If no overflow handling occurred,
|
|
572
|
+
// the file hasn't changed since fileStat — reuse it instead of re-stat'ing.
|
|
567
573
|
let sizeCheckStat: fs.Stats | undefined;
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
574
|
+
if (overflowHandled) {
|
|
575
|
+
try {
|
|
576
|
+
sizeCheckStat = await fs.promises.stat(eventsPath).catch(() => undefined);
|
|
577
|
+
} catch {
|
|
578
|
+
/* file does not exist */
|
|
579
|
+
}
|
|
580
|
+
} else {
|
|
581
|
+
sizeCheckStat = fileStat;
|
|
572
582
|
}
|
|
573
583
|
try {
|
|
574
584
|
if (sizeCheckStat && sizeCheckStat.size > MAX_EVENTS_BYTES) {
|
|
@@ -583,25 +593,44 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
583
593
|
logInternalError("event-log.size-check", error, `eventsPath=${eventsPath}`);
|
|
584
594
|
}
|
|
585
595
|
|
|
596
|
+
// FIND-10: post-append stat captured from the same fd (non-worker path)
|
|
597
|
+
// for reuse in the cache update below, avoiding a redundant path stat.
|
|
598
|
+
let postAppendStat: fs.Stats | undefined;
|
|
586
599
|
if (!skippedDueToSize) {
|
|
587
600
|
const line = JSON.stringify(redactSecrets(fullEvent)) + "\n";
|
|
588
601
|
// Phase 1.5: when worker atomic writer is enabled, append via worker.
|
|
589
602
|
if (isWorkerAtomicWriterEnabled()) {
|
|
590
603
|
await appendFileViaWorker(eventsPath, line);
|
|
604
|
+
// Worker path: fsync via a separate open (worker manages its own fd).
|
|
605
|
+
const fd = await fs.promises.open(eventsPath, "r+");
|
|
606
|
+
try {
|
|
607
|
+
await fd.sync();
|
|
608
|
+
} finally {
|
|
609
|
+
await fd.close();
|
|
610
|
+
}
|
|
591
611
|
} else {
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
612
|
+
// FIND-10: single-fd append+fsync. Opens in append mode, writes,
|
|
613
|
+
// fsyncs on the SAME fd, then closes — eliminating the separate
|
|
614
|
+
// open("r+") + sync that previously doubled the fd count. The
|
|
615
|
+
// fsync (seq-integrity protection) is preserved exactly: it still
|
|
616
|
+
// closes the crash window between append and persistSequence.
|
|
617
|
+
const fd = await fs.promises.open(eventsPath, "a");
|
|
618
|
+
try {
|
|
619
|
+
await fd.appendFile(line, "utf-8");
|
|
620
|
+
await fd.sync();
|
|
621
|
+
// FIND-10 R1 fix: the cache-optimization fd.stat() must NOT sit in the
|
|
622
|
+
// seq-durability critical path. If it threw (rare — fd invalidated),
|
|
623
|
+
// it would skip persistSequence below and reopen the seq-reuse
|
|
624
|
+
// window the fsync just closed. Guard it; fall back to undefined
|
|
625
|
+
// (the later cache-update takes a path stat instead).
|
|
626
|
+
try {
|
|
627
|
+
postAppendStat = await fd.stat();
|
|
628
|
+
} catch {
|
|
629
|
+
postAppendStat = undefined;
|
|
630
|
+
}
|
|
631
|
+
} finally {
|
|
632
|
+
await fd.close();
|
|
633
|
+
}
|
|
605
634
|
}
|
|
606
635
|
// FIX: Persist sequence AFTER successful appendFile to ensure sidecar
|
|
607
636
|
// is only updated when the event is definitively written. If appendFile
|
|
@@ -609,7 +638,11 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
609
638
|
// preventing sequence reuse on restart.
|
|
610
639
|
persistSequence(eventsPath, seq);
|
|
611
640
|
}
|
|
641
|
+
// FIND-10: track whether compaction happened after the append so the
|
|
642
|
+
// cache-update stat can safely reuse postAppendStat (file unchanged).
|
|
643
|
+
let compactedAfterAppend = false;
|
|
612
644
|
if (appendCounter % 100 === 0 && needsRotation(eventsPath)) {
|
|
645
|
+
compactedAfterAppend = true;
|
|
613
646
|
try {
|
|
614
647
|
compactEventLog(eventsPath);
|
|
615
648
|
} catch (error) {
|
|
@@ -626,11 +659,18 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
626
659
|
// Only update the cache here (the sidecar persist is already done).
|
|
627
660
|
const finalSeq = fullEvent.metadata?.seq ?? 0;
|
|
628
661
|
try {
|
|
662
|
+
// FIND-10: reuse post-append fd stat when available and no compaction
|
|
663
|
+
// happened after the append (file unchanged). Falls back to path stat
|
|
664
|
+
// for the worker path, skipped events, or post-compaction cases.
|
|
629
665
|
let statResult: fs.Stats | undefined;
|
|
630
|
-
|
|
631
|
-
statResult =
|
|
632
|
-
}
|
|
633
|
-
|
|
666
|
+
if (postAppendStat && !compactedAfterAppend) {
|
|
667
|
+
statResult = postAppendStat;
|
|
668
|
+
} else {
|
|
669
|
+
try {
|
|
670
|
+
statResult = await fs.promises.stat(eventsPath).catch(() => undefined);
|
|
671
|
+
} catch {
|
|
672
|
+
/* file may not exist */
|
|
673
|
+
}
|
|
634
674
|
}
|
|
635
675
|
if (statResult) {
|
|
636
676
|
if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
|
|
@@ -1217,24 +1257,38 @@ export function readEventsCursor(eventsPath: string, options: EventCursorOptions
|
|
|
1217
1257
|
};
|
|
1218
1258
|
}
|
|
1219
1259
|
|
|
1220
|
-
//
|
|
1221
|
-
//
|
|
1222
|
-
//
|
|
1223
|
-
//
|
|
1224
|
-
//
|
|
1260
|
+
// FIND-05 default path: byte-level tail read (last 4MB) instead of
|
|
1261
|
+
// full-file read. Bounds CPU to O(tail bytes) instead of O(total
|
|
1262
|
+
// events). The legacy readEvents() full parse path is preserved for
|
|
1263
|
+
// callers that explicitly need the full history (e.g. tests that
|
|
1264
|
+
// assert exact contents) and as a small-file fallback.
|
|
1265
|
+
//
|
|
1266
|
+
// The 5000-event tail cap and the "event-log.cursor-full-read"
|
|
1267
|
+
// warning are preserved. A separate cursor-tail-truncated warning is
|
|
1268
|
+
// emitted whenever the file exceeds the 4MB tail budget, signalling
|
|
1269
|
+
// that a prefix was dropped and callers should pass fromByteOffset for
|
|
1270
|
+
// streaming reads.
|
|
1271
|
+
const TAIL_BYTES = 4 * 1024 * 1024; // 4 MB
|
|
1272
|
+
const TAIL_EVENT_CAP = 5000;
|
|
1225
1273
|
const sinceSeq = positiveInteger(options.sinceSeq) ?? 0;
|
|
1226
1274
|
const limit = positiveInteger(options.limit);
|
|
1227
|
-
|
|
1228
|
-
const
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1275
|
+
|
|
1276
|
+
const tail = readJsonlTail<TeamEvent>(eventsPath, TAIL_BYTES);
|
|
1277
|
+
let all = tail.items;
|
|
1278
|
+
if (tail.truncated) {
|
|
1279
|
+
logInternalError("event-log.cursor-tail-truncated", {
|
|
1280
|
+
eventsPath,
|
|
1281
|
+
returned: all.length,
|
|
1282
|
+
tailBytes: TAIL_BYTES,
|
|
1283
|
+
});
|
|
1284
|
+
}
|
|
1285
|
+
if (all.length > TAIL_EVENT_CAP) {
|
|
1232
1286
|
logInternalError(
|
|
1233
1287
|
"event-log.cursor-full-read",
|
|
1234
|
-
new Error(`readEventsCursor read
|
|
1288
|
+
new Error(`readEventsCursor tail read dropped events from a larger log; pass fromByteOffset for incremental reads`),
|
|
1235
1289
|
`eventsPath=${eventsPath}`,
|
|
1236
1290
|
);
|
|
1237
|
-
all = all.slice(-
|
|
1291
|
+
all = all.slice(-TAIL_EVENT_CAP);
|
|
1238
1292
|
}
|
|
1239
1293
|
const filtered = all.filter((event) => (event.metadata?.seq ?? 0) > sinceSeq);
|
|
1240
1294
|
const events = limit !== undefined ? filtered.slice(0, limit) : filtered;
|
package/src/state/locks.ts
CHANGED
|
@@ -447,6 +447,59 @@ const lockCtx = new AsyncLocalStorage<Set<string>>();
|
|
|
447
447
|
// at the top of withFileLockSync for the full deadlock mechanism.
|
|
448
448
|
const fileLockHeldByUs = new Map<string, string>(); // lockFile -> token
|
|
449
449
|
|
|
450
|
+
// --- Async file lock (non-blocking alternative to withFileLockSync) ---
|
|
451
|
+
// Uses a promise-chain pattern to serialize per-path access without blocking
|
|
452
|
+
// the Node.js event loop. Unlike withFileLockSync (which uses O_EXCL +
|
|
453
|
+
// sleepSync for cross-process safety), this is **in-process only** —
|
|
454
|
+
// sufficient for single-process mailbox writes (team-runner is single-process).
|
|
455
|
+
// Mirrors the structure of withEventLogLockAsync in event-log.ts.
|
|
456
|
+
const fileAsyncLocks = new Map<string, Promise<unknown>>();
|
|
457
|
+
|
|
458
|
+
// FIND-02 follow-up (P3): re-entrance guard for the async file lock, mirroring
|
|
459
|
+
// the lockCtx pattern used by withRunLock. A future caller doing same-path
|
|
460
|
+
// nested withFileLockAsync(path, ...) inside another withFileLockAsync(path,
|
|
461
|
+
// ...) in the SAME async context would otherwise deadlock (the nested call
|
|
462
|
+
// chains after the outer's still-pending promise). AsyncLocalStorage scopes
|
|
463
|
+
// the held set to the current async context so cross-context callers still
|
|
464
|
+
// serialize via the promise chain (Phase 1 mailbox path is unaffected).
|
|
465
|
+
const fileAsyncLockCtx = new AsyncLocalStorage<Set<string>>();
|
|
466
|
+
|
|
467
|
+
export async function withFileLockAsync<T>(filePath: string, fn: () => Promise<T>): Promise<T> {
|
|
468
|
+
// Re-entrant within the same async context — run fn() directly (no chaining).
|
|
469
|
+
// Same semantics as the sync guard in withFileLockSync (fileLockHeldByUs)
|
|
470
|
+
// and the async guard in withRunLock (lockCtx).
|
|
471
|
+
if (fileAsyncLockCtx.getStore()?.has(filePath)) {
|
|
472
|
+
return await fn();
|
|
473
|
+
}
|
|
474
|
+
// Merge with the parent context's held set so nested DIFFERENT-path locks
|
|
475
|
+
// also bypass correctly (prevents cross-path deadlock, matching withRunLock).
|
|
476
|
+
const prevHeld = fileAsyncLockCtx.getStore() ?? new Set<string>();
|
|
477
|
+
const held = new Set(prevHeld);
|
|
478
|
+
held.add(filePath);
|
|
479
|
+
const prev = fileAsyncLocks.get(filePath) ?? Promise.resolve();
|
|
480
|
+
// Chain fn after the previous holder. `next` may reject (propagating to the
|
|
481
|
+
// caller), but `stored` never rejects so subsequent waiters aren't blocked.
|
|
482
|
+
// fn() is wrapped in fileAsyncLockCtx.run so nested same-context calls see
|
|
483
|
+
// `held` and bypass the promise chain (re-entrant), while cross-context
|
|
484
|
+
// callers chain normally via `prev`.
|
|
485
|
+
const next = prev.then(() => fileAsyncLockCtx.run(held, () => fn()));
|
|
486
|
+
const stored = next.then(
|
|
487
|
+
() => undefined,
|
|
488
|
+
() => undefined,
|
|
489
|
+
);
|
|
490
|
+
fileAsyncLocks.set(filePath, stored);
|
|
491
|
+
try {
|
|
492
|
+
return await next;
|
|
493
|
+
} finally {
|
|
494
|
+
// Compare-and-delete: only remove our entry if it still points at `stored`.
|
|
495
|
+
// With 3+ overlapping callers, an earlier caller's finally would otherwise
|
|
496
|
+
// delete a later caller's promise, breaking mutual exclusion.
|
|
497
|
+
if (fileAsyncLocks.get(filePath) === stored) {
|
|
498
|
+
fileAsyncLocks.delete(filePath);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
450
503
|
export function withRunLockSync<T>(manifest: TeamRunManifest, fn: () => T, options: RunLockOptions = {}): T {
|
|
451
504
|
const filePath = lockPath(manifest);
|
|
452
505
|
const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
|
package/src/state/mailbox.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { logInternalError } from "../utils/internal-error.ts";
|
|
|
5
5
|
import { redactSecrets } from "../utils/redaction.ts";
|
|
6
6
|
import { atomicWriteFile } from "./atomic-write.ts";
|
|
7
7
|
import { withEventLogLockSync } from "./event-log.ts";
|
|
8
|
-
import { withFileLockSync } from "./locks.ts";
|
|
8
|
+
import { withFileLockAsync, withFileLockSync } from "./locks.ts";
|
|
9
9
|
import type { TeamRunManifest } from "./types.ts";
|
|
10
10
|
|
|
11
11
|
export type MailboxDirection = "inbox" | "outbox";
|
|
@@ -377,19 +377,61 @@ function readAllInboxMessages(manifest: TeamRunManifest): MailboxMessage[] {
|
|
|
377
377
|
return readAllMessages(manifest, "inbox");
|
|
378
378
|
}
|
|
379
379
|
|
|
380
|
+
// FIND-01: in-process delivery cache to avoid O(N²) re-reads on every append.
|
|
381
|
+
// Keyed by delivery file path; invalidated by mtime check on read + updated on
|
|
382
|
+
// write. Team-runner is single-process so in-process caching is sufficient.
|
|
383
|
+
const deliveryCache = new Map<string, { mtimeMs: number; state: MailboxDeliveryState }>();
|
|
384
|
+
const MAX_DELIVERY_CACHE_ENTRIES = 256;
|
|
385
|
+
// R1 review fix: setDeliveryCacheEntry stores an immutable snapshot (deep
|
|
386
|
+
// copy of `messages`) so callers mutating the returned state cannot corrupt
|
|
387
|
+
// the cache (TOCTOU race), and bounds the map size with FIFO eviction to
|
|
388
|
+
// prevent unbounded growth across runs.
|
|
389
|
+
function setDeliveryCacheEntry(filePath: string, entry: { mtimeMs: number; state: MailboxDeliveryState }): void {
|
|
390
|
+
if (deliveryCache.size >= MAX_DELIVERY_CACHE_ENTRIES) {
|
|
391
|
+
const oldest = deliveryCache.keys().next().value;
|
|
392
|
+
if (oldest !== undefined) deliveryCache.delete(oldest);
|
|
393
|
+
}
|
|
394
|
+
deliveryCache.set(filePath, {
|
|
395
|
+
mtimeMs: entry.mtimeMs,
|
|
396
|
+
state: { ...entry.state, messages: { ...entry.state.messages } },
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
|
|
380
400
|
export function readDeliveryState(manifest: TeamRunManifest): MailboxDeliveryState {
|
|
401
|
+
const filePath = deliveryFile(manifest);
|
|
402
|
+
let stat: fs.Stats;
|
|
381
403
|
try {
|
|
382
|
-
|
|
404
|
+
stat = fs.statSync(filePath);
|
|
405
|
+
} catch (e) {
|
|
406
|
+
// R1 review fix: narrow to ENOENT so permission errors aren't silently
|
|
407
|
+
// treated as "missing file" (would wipe a valid cache entry).
|
|
408
|
+
if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e;
|
|
409
|
+
deliveryCache.delete(filePath);
|
|
410
|
+
return { messages: {}, updatedAt: new Date().toISOString() };
|
|
411
|
+
}
|
|
412
|
+
const cached = deliveryCache.get(filePath);
|
|
413
|
+
if (cached && cached.mtimeMs === stat.mtimeMs) {
|
|
414
|
+
// R2 review fix: return a copy so callers mutating the result cannot
|
|
415
|
+
// leak into the cached snapshot (residual TOCTOU: the cache holds the
|
|
416
|
+
// snapshot until the next write replaces it; without this copy, a
|
|
417
|
+
// pre-write mutation by one caller would be persisted into the
|
|
418
|
+
// post-write snapshot by the next writer's setDeliveryCacheEntry).
|
|
419
|
+
return { ...cached.state, messages: { ...cached.state.messages } };
|
|
420
|
+
}
|
|
421
|
+
try {
|
|
422
|
+
const raw = JSON.parse(fs.readFileSync(filePath, "utf-8")) as unknown;
|
|
383
423
|
if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error("Invalid delivery state.");
|
|
384
424
|
const obj = raw as Record<string, unknown>;
|
|
385
425
|
const messages: Record<string, MailboxMessageStatus> = {};
|
|
386
426
|
if (obj.messages && typeof obj.messages === "object" && !Array.isArray(obj.messages)) {
|
|
387
427
|
for (const [id, status] of Object.entries(obj.messages)) if (isStatus(status)) messages[id] = status;
|
|
388
428
|
}
|
|
389
|
-
|
|
429
|
+
const state: MailboxDeliveryState = {
|
|
390
430
|
messages,
|
|
391
431
|
updatedAt: typeof obj.updatedAt === "string" ? obj.updatedAt : new Date().toISOString(),
|
|
392
432
|
};
|
|
433
|
+
setDeliveryCacheEntry(filePath, { mtimeMs: stat.mtimeMs, state });
|
|
434
|
+
return state;
|
|
393
435
|
} catch {
|
|
394
436
|
return { messages: {}, updatedAt: new Date().toISOString() };
|
|
395
437
|
}
|
|
@@ -414,9 +456,20 @@ function writeDeliveryState(
|
|
|
414
456
|
// F4: mailbox delivery is informational — accept losing the very last write on
|
|
415
457
|
// a hard crash (the next message will overwrite it on disk). Cheaper fsync on
|
|
416
458
|
// the hot path; terminal/reply paths still pass full durability below.
|
|
417
|
-
|
|
459
|
+
const filePath = deliveryFile(manifest, true);
|
|
460
|
+
atomicWriteFile(filePath, `${JSON.stringify(redactSecrets(state), null, 2)}\n`, {
|
|
418
461
|
durability: options?.durability ?? "best-effort",
|
|
419
462
|
});
|
|
463
|
+
// FIND-01: update cache with post-write mtime so subsequent reads get a hit.
|
|
464
|
+
try {
|
|
465
|
+
const postStat = fs.statSync(filePath);
|
|
466
|
+
// setDeliveryCacheEntry stores an immutable snapshot (deep copy of
|
|
467
|
+
// messages) so subsequent read-modify-write callers mutating the
|
|
468
|
+
// returned state cannot corrupt the cache.
|
|
469
|
+
setDeliveryCacheEntry(filePath, { mtimeMs: postStat.mtimeMs, state });
|
|
470
|
+
} catch {
|
|
471
|
+
deliveryCache.delete(filePath);
|
|
472
|
+
}
|
|
420
473
|
}
|
|
421
474
|
|
|
422
475
|
/**
|
|
@@ -541,6 +594,114 @@ export function appendFollowUpMessage(
|
|
|
541
594
|
});
|
|
542
595
|
}
|
|
543
596
|
|
|
597
|
+
/**
|
|
598
|
+
* FIND-02: Async variant of appendMailboxMessage for the live-session path.
|
|
599
|
+
* Uses withFileLockAsync (promise-chain, no sleepSync) instead of
|
|
600
|
+
* withEventLogLockSync/withFileLockSync, preventing event-loop stalls during
|
|
601
|
+
* steering/follow-up delivery. readDeliveryState/writeDeliveryState remain
|
|
602
|
+
* sync but are cheap thanks to the FIND-01 delivery cache.
|
|
603
|
+
*/
|
|
604
|
+
export async function appendMailboxMessageAsync(
|
|
605
|
+
manifest: TeamRunManifest,
|
|
606
|
+
message: Omit<MailboxMessage, "id" | "runId" | "createdAt" | "status"> & {
|
|
607
|
+
id?: string;
|
|
608
|
+
status?: MailboxMessageStatus;
|
|
609
|
+
},
|
|
610
|
+
): Promise<MailboxMessage> {
|
|
611
|
+
if (message.taskId) ensureTaskMailbox(manifest, message.taskId);
|
|
612
|
+
else ensureRunMailbox(manifest);
|
|
613
|
+
const createdAt = new Date().toISOString();
|
|
614
|
+
const complete: MailboxMessage = {
|
|
615
|
+
id: message.id ?? `msg_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`,
|
|
616
|
+
runId: manifest.runId,
|
|
617
|
+
direction: message.direction,
|
|
618
|
+
from: message.from,
|
|
619
|
+
to: message.to,
|
|
620
|
+
body: message.body,
|
|
621
|
+
createdAt,
|
|
622
|
+
status: message.status ?? "queued",
|
|
623
|
+
kind: message.kind,
|
|
624
|
+
priority: message.priority,
|
|
625
|
+
deliveryMode: message.deliveryMode,
|
|
626
|
+
taskId: message.taskId,
|
|
627
|
+
data: message.data,
|
|
628
|
+
replyTo: message.replyTo,
|
|
629
|
+
replyFrom: message.replyFrom,
|
|
630
|
+
replyDeadline: message.replyDeadline,
|
|
631
|
+
repliedAt: message.repliedAt,
|
|
632
|
+
replyContent: message.replyContent,
|
|
633
|
+
};
|
|
634
|
+
const mbFile = mailboxFile(manifest, complete.direction, complete.taskId);
|
|
635
|
+
await withFileLockAsync(mbFile, async () => {
|
|
636
|
+
await fs.promises.appendFile(mbFile, `${JSON.stringify(redactSecrets(complete))}\n`, "utf-8");
|
|
637
|
+
rotateMailboxFileIfNeeded(mbFile);
|
|
638
|
+
});
|
|
639
|
+
// R1 review fix / plan §5 #6: delivery RMW uses the cross-process sync
|
|
640
|
+
// lock (withFileLockSync) so sync callers (acknowledgeMailboxMessage,
|
|
641
|
+
// replayPendingMailboxMessages) serialize against this async path. The
|
|
642
|
+
// body has no await, so sync locking is safe and restores the
|
|
643
|
+
// cross-process safety net that the async lock cannot provide.
|
|
644
|
+
withFileLockSync(deliveryFile(manifest, true), () => {
|
|
645
|
+
const delivery = readDeliveryState(manifest);
|
|
646
|
+
delivery.messages[complete.id] = complete.status;
|
|
647
|
+
delivery.updatedAt = createdAt;
|
|
648
|
+
writeDeliveryState(manifest, delivery, { durability: "full" });
|
|
649
|
+
});
|
|
650
|
+
return complete;
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
export async function appendSteeringMessageAsync(
|
|
654
|
+
manifest: TeamRunManifest,
|
|
655
|
+
input: {
|
|
656
|
+
taskId: string;
|
|
657
|
+
body: string;
|
|
658
|
+
from?: string;
|
|
659
|
+
to?: string;
|
|
660
|
+
priority?: MailboxMessagePriority;
|
|
661
|
+
status?: MailboxMessageStatus;
|
|
662
|
+
data?: Record<string, unknown>;
|
|
663
|
+
},
|
|
664
|
+
): Promise<MailboxMessage> {
|
|
665
|
+
return appendMailboxMessageAsync(manifest, {
|
|
666
|
+
direction: "inbox",
|
|
667
|
+
from: input.from ?? "leader",
|
|
668
|
+
to: input.to ?? input.taskId,
|
|
669
|
+
taskId: input.taskId,
|
|
670
|
+
body: input.body,
|
|
671
|
+
kind: "steer",
|
|
672
|
+
priority: input.priority ?? "urgent",
|
|
673
|
+
deliveryMode: "interrupt",
|
|
674
|
+
status: input.status,
|
|
675
|
+
data: { ...(input.data ?? {}), kind: "steer" },
|
|
676
|
+
});
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
export async function appendFollowUpMessageAsync(
|
|
680
|
+
manifest: TeamRunManifest,
|
|
681
|
+
input: {
|
|
682
|
+
taskId: string;
|
|
683
|
+
body: string;
|
|
684
|
+
from?: string;
|
|
685
|
+
to?: string;
|
|
686
|
+
priority?: MailboxMessagePriority;
|
|
687
|
+
status?: MailboxMessageStatus;
|
|
688
|
+
data?: Record<string, unknown>;
|
|
689
|
+
},
|
|
690
|
+
): Promise<MailboxMessage> {
|
|
691
|
+
return appendMailboxMessageAsync(manifest, {
|
|
692
|
+
direction: "inbox",
|
|
693
|
+
from: input.from ?? "leader",
|
|
694
|
+
to: input.to ?? input.taskId,
|
|
695
|
+
taskId: input.taskId,
|
|
696
|
+
body: input.body,
|
|
697
|
+
kind: "follow-up",
|
|
698
|
+
priority: input.priority ?? "normal",
|
|
699
|
+
deliveryMode: "next_turn",
|
|
700
|
+
status: input.status,
|
|
701
|
+
data: { ...(input.data ?? {}), kind: "follow-up" },
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
|
|
544
705
|
export function listMailboxByKind(manifest: TeamRunManifest, kind: MailboxMessageKind, direction?: MailboxDirection): MailboxMessage[] {
|
|
545
706
|
const messages = direction
|
|
546
707
|
? readAllMessages(manifest, direction)
|