pi-crew 0.11.53 → 0.11.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +152 -0
- package/dist/index.mjs +523 -523
- package/package.json +1 -1
- package/src/config/env-allowlist-registry.ts +4 -0
- package/src/config/env-vars.ts +4 -0
- package/src/extension/team-tool/parallel-dispatch.ts +5 -0
- package/src/extension/team-tool/respond.ts +65 -2
- package/src/extension/team-tool/run.ts +8 -0
- package/src/runtime/broker/protocol/msg-inbox.ts +27 -3
- package/src/runtime/child-pi/child-pi-spawn.ts +25 -1
- package/src/runtime/child-pi/child-pi-timers.ts +25 -0
- package/src/runtime/child-pi/child-pi.ts +9 -0
- package/src/runtime/foreground-control.ts +164 -116
- package/src/runtime/goal-workflow/goal-loop-runner.ts +6 -0
- package/src/runtime/pi-spawn.ts +112 -2
- package/src/runtime/task-runner/child-executor.ts +7 -0
- package/src/runtime/task-runner.ts +17 -0
- package/src/runtime/verification/verification-gates.ts +103 -7
- package/src/runtime/verification/verify-gate.ts +209 -21
- package/src/state/atomic-write.ts +59 -0
- package/src/state/contracts.ts +7 -0
- package/src/state/coordination/mailbox.ts +52 -6
- package/src/state/event-log/event-log.ts +108 -27
- package/src/state/event-log/event-reconstructor.ts +5 -0
- package/src/state/stores/state-store.ts +8 -0
- package/src/state/types.ts +16 -0
- package/src/worktree/worktree-manager.ts +12 -0
|
@@ -987,6 +987,12 @@ interface CoalescedAtomicWrite {
|
|
|
987
987
|
generation: number;
|
|
988
988
|
/** F4: durability knob carried into the underlying atomicWriteFile flush. */
|
|
989
989
|
durability: WriteDurability;
|
|
990
|
+
/** ASK-PARK CLOBBER FIX (2026-10-31): on-disk file identity at QUEUE time
|
|
991
|
+
* for the flush-time staleness CAS. null = file did not exist yet. */
|
|
992
|
+
queuedMtimeMs: number | null;
|
|
993
|
+
/** ASK-PARK CLOBBER FIX: file size at queue time — backstop for
|
|
994
|
+
* coarse-mtime filesystems (FAT 1-2s granularity). */
|
|
995
|
+
queuedSize: number | null;
|
|
990
996
|
}
|
|
991
997
|
const MAX_FLUSH_RETRIES = 5;
|
|
992
998
|
const pendingAtomicWrites = new Map<string, CoalescedAtomicWrite>();
|
|
@@ -1095,6 +1101,24 @@ export function atomicWriteJsonCoalesced<T>(
|
|
|
1095
1101
|
// future caller mutates after queueing, that mutation persists.
|
|
1096
1102
|
const previous = pendingAtomicWrites.get(filePath);
|
|
1097
1103
|
if (previous) clearTimeout(previous.timer);
|
|
1104
|
+
// ASK-PARK CLOBBER FIX (2026-10-31): snapshot the on-disk file identity at
|
|
1105
|
+
// QUEUE time so the flush can compare-and-swap against it.
|
|
1106
|
+
// cancelPendingCoalescedWrite only sees SAME-process supersessions — a sync
|
|
1107
|
+
// write from ANOTHER process (e.g. the detached broker's waiting-park via
|
|
1108
|
+
// saveRunTasks) cannot cancel this buffer, and a blind flush would then
|
|
1109
|
+
// clobber that fresher data (live incident team_20261031084714: tasks.json
|
|
1110
|
+
// waiting→running, every leader respond rejected, 3/3 ask timeouts).
|
|
1111
|
+
// stat throw = file absent → null, so a flush-time APPEARANCE of the file
|
|
1112
|
+
// is detected as an external write.
|
|
1113
|
+
let queuedMtimeMs: number | null = null;
|
|
1114
|
+
let queuedSize: number | null = null;
|
|
1115
|
+
try {
|
|
1116
|
+
const queuedStat = fs.statSync(filePath);
|
|
1117
|
+
queuedMtimeMs = queuedStat.mtimeMs;
|
|
1118
|
+
queuedSize = queuedStat.size;
|
|
1119
|
+
} catch {
|
|
1120
|
+
/* file does not exist yet — null is the queue-time identity */
|
|
1121
|
+
}
|
|
1098
1122
|
const timer = setTimeout(() => flushOnePendingAtomicWrite(filePath), coalesceMs);
|
|
1099
1123
|
timer.unref();
|
|
1100
1124
|
// Issue 2 fix: increment generation for each new entry
|
|
@@ -1107,6 +1131,8 @@ export function atomicWriteJsonCoalesced<T>(
|
|
|
1107
1131
|
retryCount: 0,
|
|
1108
1132
|
generation,
|
|
1109
1133
|
durability: normalized.durability,
|
|
1134
|
+
queuedMtimeMs,
|
|
1135
|
+
queuedSize,
|
|
1110
1136
|
});
|
|
1111
1137
|
}
|
|
1112
1138
|
|
|
@@ -1118,6 +1144,39 @@ function flushOnePendingAtomicWrite(filePath: string): void {
|
|
|
1118
1144
|
// and we will NOT delete the newer entry after the flush completes.
|
|
1119
1145
|
const savedGeneration = entry.generation;
|
|
1120
1146
|
clearTimeout(entry.timer);
|
|
1147
|
+
// ASK-PARK CLOBBER FIX (2026-10-31): flush-time staleness CAS. Contract: a
|
|
1148
|
+
// coalesced write is a CHECKPOINT-class snapshot — last-writer-wins applies
|
|
1149
|
+
// only WITHIN this process (same-process supersession cancels the buffer
|
|
1150
|
+
// via cancelPendingCoalescedWrite inside atomicWriteFile). If an EXTERNAL
|
|
1151
|
+
// write landed between queue and flush — detected as an mtime/size identity
|
|
1152
|
+
// change, or as the file APPEARING after a queue-time absence — the
|
|
1153
|
+
// buffered snapshot is stale and is DISCARDED, never clobbered over the
|
|
1154
|
+
// fresher data. A stat throw here (file vanished since queue, e.g. an
|
|
1155
|
+
// external delete) keeps the pre-fix flush behavior: write proceeds.
|
|
1156
|
+
// Order matters: this MUST run BEFORE atomicWriteFile — its first
|
|
1157
|
+
// statement cancels the pending entry, so on discard WE delete the entry
|
|
1158
|
+
// explicitly. The retry/backoff path re-adds the SAME entry object, so the
|
|
1159
|
+
// queue-time identity survives retries. Residual window: coarse-mtime
|
|
1160
|
+
// filesystems (FAT 1-2s) are narrowed by the size check; what remains is
|
|
1161
|
+
// acceptable — the next checkpoint re-queues fresh data, and respond.ts
|
|
1162
|
+
// carries a defense-in-depth self-heal for the ask park (2026-10-31).
|
|
1163
|
+
try {
|
|
1164
|
+
const currentStat = fs.statSync(filePath);
|
|
1165
|
+
if (entry.queuedMtimeMs === null || entry.queuedMtimeMs !== currentStat.mtimeMs || entry.queuedSize !== currentStat.size) {
|
|
1166
|
+
pendingAtomicWrites.delete(filePath);
|
|
1167
|
+
logInternalError(
|
|
1168
|
+
"atomic-write.coalesced-stale-drop",
|
|
1169
|
+
new Error(
|
|
1170
|
+
`discarded stale coalesced write: external write landed between queue and flush (queued mtimeMs=${entry.queuedMtimeMs} size=${entry.queuedSize}, current mtimeMs=${currentStat.mtimeMs} size=${currentStat.size})`,
|
|
1171
|
+
),
|
|
1172
|
+
filePath,
|
|
1173
|
+
"warn",
|
|
1174
|
+
);
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
} catch {
|
|
1178
|
+
/* file vanished since queue — keep existing flush behavior (write proceeds) */
|
|
1179
|
+
}
|
|
1121
1180
|
try {
|
|
1122
1181
|
// PERF (2026-08-24): stringify happens HERE (flush time), not at queue
|
|
1123
1182
|
// time — see the note in atomicWriteJsonCoalesced. No defensive deep-copy
|
package/src/state/contracts.ts
CHANGED
|
@@ -94,6 +94,13 @@ export const TEAM_EVENT_TYPES = [
|
|
|
94
94
|
"async.stale",
|
|
95
95
|
"task.waiting",
|
|
96
96
|
"task.resumed",
|
|
97
|
+
// ASK-PARK CLOBBER FIX (2026-10-31): respond-side self-heal trace — fired
|
|
98
|
+
// when the no-waiting-tasks fallback finds manifest.waitState still live
|
|
99
|
+
// while tasks.json lost the park to the cross-process coalesce race (host
|
|
100
|
+
// coalesced flush vs broker sync park) and restores `waiting` from the
|
|
101
|
+
// waitState pointer before running the answer path. Participates in
|
|
102
|
+
// event-log reconstruction (→ "waiting", mirrors task.waiting).
|
|
103
|
+
"task.waiting_repaired",
|
|
97
104
|
// G5 (deep-review 2026-10-01): respond-delivery observability. The leader
|
|
98
105
|
// previously had NO confirmation that a mailbox respond reached the worker:
|
|
99
106
|
// `respond_delivered` fires at the broker's wait.resolve ack when the parked
|
|
@@ -37,8 +37,9 @@ export type MailboxReason = "need_decision" | "progress_update" | "interview_req
|
|
|
37
37
|
// Fix: the parent/broker process SIGNS every row it writes with an Ed25519
|
|
38
38
|
// per-process keypair. The PRIVATE key is heap-only in the writer process
|
|
39
39
|
// (never persisted, never serialized, never placed in any child env). Children
|
|
40
|
-
// receive ONLY
|
|
41
|
-
//
|
|
40
|
+
// receive ONLY public verify keys (PI_CREW_MAILBOX_VERIFY_KEY and/or the
|
|
41
|
+
// FINDING-B plural set PI_CREW_MAILBOX_VERIFY_KEYS — base64 SPKI DER,
|
|
42
|
+
// public keys, safe in env) set by child-pi-spawn at spawn time.
|
|
42
43
|
// Delivery-side readers (parseMailboxFile — the shared path under readMailbox /
|
|
43
44
|
// readAllMailboxMessages / pollWorkerInbox / replay) verify the signature over
|
|
44
45
|
// the identity-bearing fields and treat unverifiable rows as NOISE (dropped,
|
|
@@ -69,6 +70,25 @@ export type MailboxReason = "need_decision" | "progress_update" | "interview_req
|
|
|
69
70
|
* key (set by child-pi-spawn). A PUBLIC key — non-secret, env-safe. */
|
|
70
71
|
export const COORDINATION_VERIFY_KEY_ENV = "PI_CREW_MAILBOX_VERIFY_KEY";
|
|
71
72
|
|
|
73
|
+
/** Child env var carrying a COMMA-SEPARATED list of base64 SPKI DER Ed25519
|
|
74
|
+
* public verify keys (set by child-pi-spawn). A PUBLIC key — non-secret,
|
|
75
|
+
* env-safe.
|
|
76
|
+
*
|
|
77
|
+
* FINDING B (2026-10-31): the singular var above carries exactly ONE key —
|
|
78
|
+
* the key of the process that SPAWNED the child. But legitimate coordination
|
|
79
|
+
* rows are written by MORE processes than the spawner: async runs have the
|
|
80
|
+
* leader session (the dispatcher) write `respond` rows and steering
|
|
81
|
+
* fs-fallback lines while the detached runner spawns the workers — those
|
|
82
|
+
* leader-signed rows failed the single-key verify and were dropped as noise
|
|
83
|
+
* (ask poll 480s timeout). This plural var lets the spawner hand the child
|
|
84
|
+
* the FULL verify set: spawner-relayed keys (nested same-run grandchild case)
|
|
85
|
+
* plus the run's dispatcher key threaded via the manifest
|
|
86
|
+
* (TeamRunManifest.dispatcherVerifyKey → ChildPiRunInput.dispatcherVerifyKey
|
|
87
|
+
* → child-pi-spawn env build). The singular var keeps its exact semantics
|
|
88
|
+
* (backward-compatible); the resolver merges + dedupes both plus the
|
|
89
|
+
* in-process keypair. */
|
|
90
|
+
export const COORDINATION_VERIFY_KEYS_ENV = "PI_CREW_MAILBOX_VERIFY_KEYS";
|
|
91
|
+
|
|
72
92
|
interface MailboxSigningKeypair {
|
|
73
93
|
privateKey: KeyObject;
|
|
74
94
|
publicKey: KeyObject;
|
|
@@ -97,16 +117,42 @@ export function getMailboxVerifyKeyPublic(): string {
|
|
|
97
117
|
return getMailboxSigningKeypair().publicBase64;
|
|
98
118
|
}
|
|
99
119
|
|
|
120
|
+
/** Dispatch-path alias of {@link getMailboxVerifyKeyPublic}: get-or-create
|
|
121
|
+
* this process's coordination public verify key (base64 SPKI DER). The
|
|
122
|
+
* dispatcher (leader session building a run — team-tool run/parallel) stamps
|
|
123
|
+
* the returned key onto TeamRunManifest.dispatcherVerifyKey so the spawner
|
|
124
|
+
* can hand it to workers via PI_CREW_MAILBOX_VERIFY_KEYS; rows this process
|
|
125
|
+
* signs (respond, steering fs-fallback) then verify in those children.
|
|
126
|
+
* PUBLIC material only — the private key never leaves this heap. */
|
|
127
|
+
export function ensureCoordinationPublicVerifyKey(): string {
|
|
128
|
+
return getMailboxVerifyKeyPublic();
|
|
129
|
+
}
|
|
130
|
+
|
|
100
131
|
/** Whether any verify key is resolvable in THIS process (child env key or an
|
|
101
132
|
* already-created in-process keypair). Keyless processes are inert readers. */
|
|
102
133
|
export function hasCoordinationVerifyKeys(): boolean {
|
|
103
134
|
return resolveCoordinationVerifyKeys().length > 0;
|
|
104
135
|
}
|
|
105
|
-
|
|
136
|
+
/** Resolve the verify-key set for THIS process: MERGE + dedupe of
|
|
137
|
+
* [PI_CREW_MAILBOX_VERIFY_KEYS (plural env), PI_CREW_MAILBOX_VERIFY_KEY
|
|
138
|
+
* (singular env, backward-compatible), the in-process keypair when it
|
|
139
|
+
* exists]. No source shadows another (FINDING B shadow bug: the old
|
|
140
|
+
* `if (envKey) return [envKey]` early-return made the env key HIDE the
|
|
141
|
+
* in-process keypair, so a writer process that also carried a child env key
|
|
142
|
+
* dropped its own freshly-signed rows on re-read). Empty entries and
|
|
143
|
+
* duplicates are skipped; malformed key material is tolerated here and
|
|
144
|
+
* skipped per-key at verify time. */
|
|
106
145
|
function resolveCoordinationVerifyKeys(): string[] {
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
146
|
+
const keys: string[] = [];
|
|
147
|
+
const pushKey = (key: string): void => {
|
|
148
|
+
if (key.length > 0 && !keys.includes(key)) keys.push(key);
|
|
149
|
+
};
|
|
150
|
+
const pluralEnv = getCrewEnv(COORDINATION_VERIFY_KEYS_ENV);
|
|
151
|
+
if (pluralEnv !== undefined) for (const key of pluralEnv.split(",")) pushKey(key.trim());
|
|
152
|
+
const singularEnv = getCrewEnv(COORDINATION_VERIFY_KEY_ENV);
|
|
153
|
+
if (singularEnv !== undefined) pushKey(singularEnv.trim());
|
|
154
|
+
if (signingKeypair) pushKey(signingKeypair.publicBase64);
|
|
155
|
+
return keys;
|
|
110
156
|
}
|
|
111
157
|
|
|
112
158
|
/** Canonical payload string over the identity-bearing mailbox-row fields —
|
|
@@ -1209,11 +1209,35 @@ interface BufferedAppend {
|
|
|
1209
1209
|
reject: (error: unknown) => void;
|
|
1210
1210
|
}
|
|
1211
1211
|
const bufferedQueues = new Map<string, BufferedAppend[]>();
|
|
1212
|
-
|
|
1212
|
+
/**
|
|
1213
|
+
* Armed flush timer per path. The stored `bufferMs` is the delay the queue was
|
|
1214
|
+
* armed with — reused when a flush re-arms for leftovers appended mid-flight
|
|
1215
|
+
* (F3, 2026-10-31).
|
|
1216
|
+
*/
|
|
1217
|
+
interface ArmedFlushTimer {
|
|
1218
|
+
handle: ReturnType<typeof setTimeout>;
|
|
1219
|
+
bufferMs: number;
|
|
1220
|
+
}
|
|
1221
|
+
const bufferedTimers = new Map<string, ArmedFlushTimer>();
|
|
1213
1222
|
/** Per-path in-flight flush (started, not yet settled) — see flushOneEventLogBuffer. */
|
|
1214
1223
|
const inflightBufferFlushes = new Map<string, Promise<void>>();
|
|
1215
1224
|
const DEFAULT_BUFFER_MS = 20;
|
|
1216
1225
|
|
|
1226
|
+
/** Arm the per-path flush timer (single armer for appendEventBuffered and the F3 finally re-arm). */
|
|
1227
|
+
function armBufferedFlushTimer(eventsPath: string, bufferMs: number): void {
|
|
1228
|
+
// Wrap flush in async IIFE so the returned Promise is awaited (avoids
|
|
1229
|
+
// "floating promise" warnings under --test-force-exit and prevents
|
|
1230
|
+
// the timer from being treated as done before the flush actually
|
|
1231
|
+
// completes its async work).
|
|
1232
|
+
const handle = setTimeout(() => {
|
|
1233
|
+
flushOneEventLogBuffer(eventsPath).catch((error) => {
|
|
1234
|
+
logInternalError("event-log.buffered-flush", error, `eventsPath=${eventsPath}`);
|
|
1235
|
+
});
|
|
1236
|
+
}, bufferMs);
|
|
1237
|
+
handle.unref();
|
|
1238
|
+
bufferedTimers.set(eventsPath, { handle, bufferMs });
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1217
1241
|
export function appendEventBuffered(eventsPath: string, event: AppendTeamEvent, bufferMs = DEFAULT_BUFFER_MS): Promise<TeamEvent> {
|
|
1218
1242
|
// FIX: Terminal events must bypass buffer to ensure they're written immediately.
|
|
1219
1243
|
// Previously, terminal events like task.failed could be lost on process crash.
|
|
@@ -1235,26 +1259,28 @@ export function appendEventBuffered(eventsPath: string, event: AppendTeamEvent,
|
|
|
1235
1259
|
const queue = bufferedQueues.get(eventsPath) ?? [];
|
|
1236
1260
|
queue.push({ event, resolve, reject });
|
|
1237
1261
|
bufferedQueues.set(eventsPath, queue);
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
const timer = setTimeout(() => {
|
|
1244
|
-
flushOneEventLogBuffer(eventsPath).catch((error) => {
|
|
1245
|
-
logInternalError("event-log.buffered-flush", error, `eventsPath=${eventsPath}`);
|
|
1246
|
-
});
|
|
1247
|
-
}, bufferMs);
|
|
1248
|
-
bufferedTimers.set(eventsPath, timer);
|
|
1249
|
-
timer.unref();
|
|
1250
|
-
}
|
|
1262
|
+
// Arm-once gate: while a timer entry exists (armed OR mid-flight),
|
|
1263
|
+
// appends piggyback on it. A flush started for this path clears the
|
|
1264
|
+
// handle at grab (F5) and re-arms for leftovers in its finally (F3), so
|
|
1265
|
+
// events appended during an in-flight flush are never lost from the gate.
|
|
1266
|
+
if (!bufferedTimers.has(eventsPath)) armBufferedFlushTimer(eventsPath, bufferMs);
|
|
1251
1267
|
});
|
|
1252
1268
|
}
|
|
1253
1269
|
|
|
1254
1270
|
async function flushOneEventLogBuffer(eventsPath: string): Promise<void> {
|
|
1255
1271
|
const queue = bufferedQueues.get(eventsPath);
|
|
1256
1272
|
bufferedQueues.delete(eventsPath);
|
|
1257
|
-
|
|
1273
|
+
// F5 (2026-10-31): clear the armed handle NOW, at grab time. When the flush
|
|
1274
|
+
// is triggered by anything other than the timer itself (terminal-event
|
|
1275
|
+
// flush, flushEventLogBuffer, drainEventLogWriters) a timer may still be
|
|
1276
|
+
// pending — without this clearTimeout the stale timer fires later and runs
|
|
1277
|
+
// a redundant flush. When the timer itself triggered this flush, the
|
|
1278
|
+
// clearTimeout is a harmless no-op on the already-fired handle. The entry
|
|
1279
|
+
// itself stays until the finally below: appends landing mid-flight must see
|
|
1280
|
+
// it (arm-once gate) so they cannot double-arm; the finally then re-arms for
|
|
1281
|
+
// them (F3) or deletes the entry.
|
|
1282
|
+
const armedTimer = bufferedTimers.get(eventsPath);
|
|
1283
|
+
if (armedTimer) clearTimeout(armedTimer.handle);
|
|
1258
1284
|
// PERF-CORRECTNESS (2026-10-31, macos CI run 41147573288): track the
|
|
1259
1285
|
// in-flight flush so flushEventLogBuffer()/drainEventLogWriters() can AWAIT
|
|
1260
1286
|
// it. Previously those helpers were no-ops for a path whose flush had just
|
|
@@ -1265,7 +1291,9 @@ async function flushOneEventLogBuffer(eventsPath: string): Promise<void> {
|
|
|
1265
1291
|
// Same window on the exit path: flushBufferedQueuesSync sees an empty map
|
|
1266
1292
|
// and cannot recover an in-flight batch.
|
|
1267
1293
|
const doFlush = (async (): Promise<void> => {
|
|
1268
|
-
//
|
|
1294
|
+
// The armed timer HANDLE was cleared at grab time (F5 above); the ENTRY
|
|
1295
|
+
// lifecycle is owned by the finally below (re-arm for leftovers, delete
|
|
1296
|
+
// when drained).
|
|
1269
1297
|
try {
|
|
1270
1298
|
if (!queue || queue.length === 0) return;
|
|
1271
1299
|
|
|
@@ -1313,7 +1341,26 @@ async function flushOneEventLogBuffer(eventsPath: string): Promise<void> {
|
|
|
1313
1341
|
// Lock acquire failed — fail every queued item so callers can fall back.
|
|
1314
1342
|
if (queue) for (const item of queue) item.reject(error);
|
|
1315
1343
|
} finally {
|
|
1316
|
-
|
|
1344
|
+
// F3 (2026-10-31): events appended while this flush was in flight saw
|
|
1345
|
+
// the stale timer entry and could NOT arm one of their own (arm-once
|
|
1346
|
+
// gate) — previously the finally just deleted the entry, leaving them
|
|
1347
|
+
// un-timered until the next append or the exit-path flush. Re-arm with
|
|
1348
|
+
// the queue's last-used delay when leftovers exist; else just delete.
|
|
1349
|
+
// The identity check re-arms only while the entry is still OURS (the
|
|
1350
|
+
// mid-flight stale one): when a prior flush already deleted its entry,
|
|
1351
|
+
// an append during this window armed a FRESH timer — overwriting it
|
|
1352
|
+
// would orphan that handle (double flush later).
|
|
1353
|
+
if (bufferedQueues.has(eventsPath)) {
|
|
1354
|
+
if (bufferedTimers.get(eventsPath) === armedTimer) {
|
|
1355
|
+
armBufferedFlushTimer(eventsPath, armedTimer?.bufferMs ?? DEFAULT_BUFFER_MS);
|
|
1356
|
+
}
|
|
1357
|
+
} else {
|
|
1358
|
+
// Drain case: also clear any handle another flush's re-arm may have
|
|
1359
|
+
// left behind (ours was already cleared at grab — no-op then).
|
|
1360
|
+
const leftoverEntry = bufferedTimers.get(eventsPath);
|
|
1361
|
+
if (leftoverEntry) clearTimeout(leftoverEntry.handle);
|
|
1362
|
+
bufferedTimers.delete(eventsPath);
|
|
1363
|
+
}
|
|
1317
1364
|
}
|
|
1318
1365
|
})();
|
|
1319
1366
|
// Chain per-path: a second flush must wait for the in-flight one (keeps the
|
|
@@ -1323,18 +1370,30 @@ async function flushOneEventLogBuffer(eventsPath: string): Promise<void> {
|
|
|
1323
1370
|
() => doFlush,
|
|
1324
1371
|
() => doFlush,
|
|
1325
1372
|
);
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
)
|
|
1373
|
+
// F4 (2026-10-31): the map stores the `.finally()` WRAPPER, so the
|
|
1374
|
+
// compare-and-delete on settle must compare against the WRAPPER reference.
|
|
1375
|
+
// The old code compared against the pre-`.finally()` `tracked` promise — an
|
|
1376
|
+
// identity that never matches what the map holds — so entries were never
|
|
1377
|
+
// deleted (leak, and the "may already have self-deleted" no-op path of the
|
|
1378
|
+
// drain helpers was dead code).
|
|
1379
|
+
const wrapper = tracked.finally(() => {
|
|
1380
|
+
if (inflightBufferFlushes.get(eventsPath) === wrapper) inflightBufferFlushes.delete(eventsPath);
|
|
1381
|
+
});
|
|
1382
|
+
inflightBufferFlushes.set(eventsPath, wrapper);
|
|
1332
1383
|
await tracked;
|
|
1333
1384
|
}
|
|
1334
1385
|
|
|
1335
1386
|
/** Asynchronously flush every queued buffered event across all paths. */
|
|
1336
1387
|
export async function flushEventLogBuffer(): Promise<void> {
|
|
1337
|
-
|
|
1388
|
+
// PERF-CORRECTNESS follow-up (2026-10-31): iterate the UNION of queued and
|
|
1389
|
+
// in-flight paths. Iterating only bufferedQueues keys skipped a path whose
|
|
1390
|
+
// queue an in-flight flush had ALREADY grabbed (removed from the map at
|
|
1391
|
+
// flush start) — exactly the case the await-inflight below exists for — so a
|
|
1392
|
+
// caller awaiting this during an in-flight flush could still observe a log
|
|
1393
|
+
// without the in-flight events (the deterministic-read pin in
|
|
1394
|
+
// event-log-buffered.test.ts flaked ~1 run in 10 on slow fs via that hole).
|
|
1395
|
+
const paths = new Set<string>([...bufferedQueues.keys(), ...inflightBufferFlushes.keys()]);
|
|
1396
|
+
for (const eventsPath of paths) {
|
|
1338
1397
|
// PERF-CORRECTNESS (2026-10-31): await any IN-FLIGHT flush for this path
|
|
1339
1398
|
// first — the timer-led flush may hold events that are no longer in
|
|
1340
1399
|
// bufferedQueues (removed at flush start). Without this, a caller that
|
|
@@ -1380,7 +1439,14 @@ export function flushBufferedQueuesSync(): void {
|
|
|
1380
1439
|
logInternalError("event-log.sync-flush", error, eventsPath);
|
|
1381
1440
|
}
|
|
1382
1441
|
}
|
|
1383
|
-
|
|
1442
|
+
// Adjacent (2026-10-31): clear each pending timer HANDLE before dropping the
|
|
1443
|
+
// entry — an unref'd handle that survives the sync flush would fire later
|
|
1444
|
+
// (tests keep the process alive) and run a redundant no-op flush.
|
|
1445
|
+
for (const eventsPath of [...bufferedTimers.keys()]) {
|
|
1446
|
+
const armedTimer = bufferedTimers.get(eventsPath);
|
|
1447
|
+
if (armedTimer) clearTimeout(armedTimer.handle);
|
|
1448
|
+
bufferedTimers.delete(eventsPath);
|
|
1449
|
+
}
|
|
1384
1450
|
}
|
|
1385
1451
|
|
|
1386
1452
|
// Exported only for the buffered-batch unit tests. NOT part of the public API —
|
|
@@ -1402,6 +1468,18 @@ export function __test__appendBatchForUnitTest(
|
|
|
1402
1468
|
return appendEventBatchInsideLock(eventsPath, queue, locks);
|
|
1403
1469
|
}
|
|
1404
1470
|
|
|
1471
|
+
/**
|
|
1472
|
+
* Exposed only for the buffered-flush unit tests (F4 pin, 2026-10-31): how many
|
|
1473
|
+
* per-path in-flight flush trackers are currently registered. NOT part of the
|
|
1474
|
+
* public API — same `__test__` convention as `__test__appendBatchForUnitTest`
|
|
1475
|
+
* above. Pins that the tracker entry SELF-DELETES on settle (the pre-F4 code
|
|
1476
|
+
* stored a `.finally()` wrapper but compare-deleted against the pre-wrapper
|
|
1477
|
+
* promise, so entries leaked for the process lifetime).
|
|
1478
|
+
*/
|
|
1479
|
+
export function __test__inflightBufferFlushCount(): number {
|
|
1480
|
+
return inflightBufferFlushes.size;
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1405
1483
|
/**
|
|
1406
1484
|
* Schedule an async event append without waiting for the result.
|
|
1407
1485
|
* Uses the non-blocking async queue to avoid blocking the event loop.
|
|
@@ -1423,8 +1501,11 @@ export function appendEventFireAndForget(eventsPath: string, event: AppendTeamEv
|
|
|
1423
1501
|
* write chain (tail-chaining guarantees the tail completes last), then
|
|
1424
1502
|
* flushes that path's buffered tier via the internal `flushOneEventLogBuffer`
|
|
1425
1503
|
* (per-path flush is a simple direct call — no complexity caveat).
|
|
1426
|
-
* The
|
|
1427
|
-
* settle) — `?? Promise.resolve()` makes that a clean no-op.
|
|
1504
|
+
* The asyncQueues tail entry may already have self-deleted (compare-and-delete
|
|
1505
|
+
* on settle) — `?? Promise.resolve()` makes that a clean no-op. The
|
|
1506
|
+
* inflight-buffer tracker below it self-deletes on settle the same way
|
|
1507
|
+
* (F4 2026-10-31: it previously stored a `.finally()` wrapper but compared
|
|
1508
|
+
* against the pre-wrapper promise, so the entry never matched and leaked).
|
|
1428
1509
|
* - Without `eventsPath`: awaits every queue tail + `flushEventLogBuffer()`.
|
|
1429
1510
|
*
|
|
1430
1511
|
* Never throws: a rejected tail is absorbed by `allSettled` (a teardown
|
|
@@ -34,6 +34,9 @@ const TASK_LIFECYCLE_EVENT_TYPES = new Set([
|
|
|
34
34
|
"task.needs_attention",
|
|
35
35
|
"task.waiting",
|
|
36
36
|
"task.resumed",
|
|
37
|
+
// ASK-PARK CLOBBER FIX (2026-10-31): respond's clobbered-park self-heal —
|
|
38
|
+
// reconstructs to "waiting" exactly like the task.waiting park it repairs.
|
|
39
|
+
"task.waiting_repaired",
|
|
37
40
|
"task.retried",
|
|
38
41
|
"task.blocked",
|
|
39
42
|
"task.progress",
|
|
@@ -55,6 +58,8 @@ const EVENT_STATUS_MAP: Readonly<Record<string, string>> = {
|
|
|
55
58
|
"task.needs_attention": "needs_attention",
|
|
56
59
|
"task.waiting": "waiting",
|
|
57
60
|
"task.resumed": "running",
|
|
61
|
+
// ASK-PARK CLOBBER FIX (2026-10-31): see TASK_LIFECYCLE_EVENT_TYPES above.
|
|
62
|
+
"task.waiting_repaired": "waiting",
|
|
58
63
|
"task.retried": "queued",
|
|
59
64
|
};
|
|
60
65
|
|
|
@@ -485,6 +485,13 @@ export function createRunManifest(params: {
|
|
|
485
485
|
* catalogue must be persisted on the manifest at creation time.
|
|
486
486
|
* Omitted when undefined so old manifests stay byte-identical. */
|
|
487
487
|
modelContext?: RunModelContext;
|
|
488
|
+
/** FINDING B (2026-10-31): PUBLIC base64 SPKI verify key of the DISPATCHING
|
|
489
|
+
* session process, threaded to every worker's
|
|
490
|
+
* PI_CREW_MAILBOX_VERIFY_KEYS env so the dispatcher's signed rows
|
|
491
|
+
* (respond / steering fs-fallback) verify cross-process. See
|
|
492
|
+
* TeamRunManifest.dispatcherVerifyKey. Omitted when undefined so old
|
|
493
|
+
* manifests stay byte-identical. */
|
|
494
|
+
dispatcherVerifyKey?: string;
|
|
488
495
|
}): { manifest: TeamRunManifest; tasks: TeamTaskState[]; paths: RunPaths } {
|
|
489
496
|
const paths = createRunPaths(params.cwd, params.runId);
|
|
490
497
|
const now = (params.now ? params.now() : new Date()).toISOString();
|
|
@@ -532,6 +539,7 @@ export function createRunManifest(params: {
|
|
|
532
539
|
runKind: params.runKind ?? "team-run",
|
|
533
540
|
...(params.args !== undefined ? { args: params.args } : {}),
|
|
534
541
|
...(params.modelContext ? { modelContext: params.modelContext } : {}),
|
|
542
|
+
...(params.dispatcherVerifyKey ? { dispatcherVerifyKey: params.dispatcherVerifyKey } : {}),
|
|
535
543
|
};
|
|
536
544
|
fs.mkdirSync(paths.stateRoot, { recursive: true });
|
|
537
545
|
fs.mkdirSync(paths.artifactsRoot, { recursive: true });
|
package/src/state/types.ts
CHANGED
|
@@ -455,6 +455,22 @@ export interface TeamRunManifest {
|
|
|
455
455
|
* models.json happens to list first.
|
|
456
456
|
*/
|
|
457
457
|
modelContext?: RunModelContext;
|
|
458
|
+
/** FINDING B (2026-10-31) — w1a coordination verify-key distribution: the
|
|
459
|
+
* base64 SPKI DER PUBLIC verify key of the DISPATCHING session process
|
|
460
|
+
* (the leader that created this run; stamped via
|
|
461
|
+
* ensureCoordinationPublicVerifyKey() at team-tool run/parallel dispatch).
|
|
462
|
+
* Async runs execute in a detached runner whose workers verify
|
|
463
|
+
* coordination rows against the SPAWNER's key — without this field the
|
|
464
|
+
* leader's respond rows and steering fs-fallback lines (signed with the
|
|
465
|
+
* LEADER's keypair) fail verify and drop as noise (ask poll timeout).
|
|
466
|
+
* The runner threads it into every worker's
|
|
467
|
+
* PI_CREW_MAILBOX_VERIFY_KEYS env (plural set). PUBLIC material — safe to
|
|
468
|
+
* persist (same-uid readable); possession cannot forge rows. Optional:
|
|
469
|
+
* absent on every pre-FINDING-B manifest → workers keep verifying against
|
|
470
|
+
* the singular spawner key only (pre-fix behavior). goal-loop turns
|
|
471
|
+
* inherit the OUTER manifest's key so the session that dispatched the
|
|
472
|
+
* loop stays the verifiable responder across every turn. */
|
|
473
|
+
dispatcherVerifyKey?: string;
|
|
458
474
|
/** MuxSurface A1 (spec §8.3): pane/pid/lockout state of this run. Absent on
|
|
459
475
|
* runs that never booted a surface worker and on older manifests. */
|
|
460
476
|
surface?: ManifestSurfaceState;
|
|
@@ -1133,6 +1133,18 @@ export function prepareTaskWorkspace(
|
|
|
1133
1133
|
}
|
|
1134
1134
|
try {
|
|
1135
1135
|
const syntheticPaths = runSetupHook(manifest, task, repoRoot, worktreePath, branch);
|
|
1136
|
+
// TRUST ASSUMPTION (S-1 shared-cwd-untracked-harness review, 2026-10-31):
|
|
1137
|
+
// linking the leader's node_modules into a task worktree means those
|
|
1138
|
+
// files are NOT covered by the verify-gate's dispatch-pinned bindings
|
|
1139
|
+
// (treeOid/cleanTree/memo key bind TRACKED content only — a gitignored
|
|
1140
|
+
// shim inside the leader node_modules would execute in the worktree's
|
|
1141
|
+
// gate too, outside every binding). Accepted residual: worktree-mode
|
|
1142
|
+
// executors are trusted not to write outside their own worktree (their
|
|
1143
|
+
// uncommitted work there is separately guarded by the W2 probes), and
|
|
1144
|
+
// the leader node_modules is host-managed. The shared-cwd variant of
|
|
1145
|
+
// this residual IS closed: the gate refuses the deterministic skip when
|
|
1146
|
+
// a write-role task executed in the gate cwd (reason
|
|
1147
|
+
// "shared-cwd-untracked-harness", verify-gate.ts). See docs/trust-model.md.
|
|
1136
1148
|
const nodeModulesLinked =
|
|
1137
1149
|
loadedConfig.config.worktree?.linkNodeModules === true ? linkNodeModulesIfPresent(repoRoot, worktreePath) : false;
|
|
1138
1150
|
// Overlay seed paths from config + step-level seedPaths
|