pi-crew 0.11.22 → 0.11.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/agents/explorer.md +3 -3
- package/agents/librarian.md +3 -3
- package/dist/index.mjs +523 -520
- package/docs/dynamic-workflows.md +14 -2
- package/package.json +4 -4
- package/src/agents/discover-agents.ts +8 -2
- package/src/config/config-validation.ts +3 -1
- package/src/config/env-allowlist-registry.ts +5 -0
- package/src/config/env-vars.ts +9 -6
- package/src/config/role-tools.ts +7 -6
- package/src/extension/management.ts +7 -0
- package/src/extension/register.ts +15 -0
- package/src/extension/run-index.ts +24 -1
- package/src/extension/team-tool/destructive-gate.ts +208 -0
- package/src/extension/team-tool/goal-wrap.ts +11 -1
- package/src/extension/team-tool/goal.ts +13 -1
- package/src/extension/team-tool/respond.ts +8 -6
- package/src/extension/team-tool/run.ts +4 -4
- package/src/extension/team-tool.ts +10 -0
- package/src/observability/exporters/otlp-exporter.ts +11 -1
- package/src/prompt/prompt-runtime.ts +102 -18
- package/src/prompt/surface-worker.ts +13 -1
- package/src/runtime/broker/crew-broker-client.ts +7 -0
- package/src/runtime/broker/crew-broker.ts +113 -152
- package/src/runtime/broker/protocol/connection-state.ts +159 -1
- package/src/runtime/broker/protocol/events-replay.ts +50 -10
- package/src/runtime/broker/protocol/request-parsers.ts +68 -0
- package/src/runtime/child-pi/child-pi-constants.ts +9 -0
- package/src/runtime/child-pi/child-pi-rpc.ts +72 -7
- package/src/runtime/child-pi/child-pi-streams.ts +19 -3
- package/src/runtime/child-pi/postmortem-registry.ts +8 -2
- package/src/runtime/classifier/ambient-noise-gate.ts +5 -1
- package/src/runtime/compaction/tool-output-pruner.ts +20 -2
- package/src/runtime/crew-agent-records.ts +92 -17
- package/src/runtime/custom-tools/submit-result-tool.ts +17 -0
- package/src/runtime/deadletter.ts +5 -1
- package/src/runtime/dispatch-batch.ts +60 -16
- package/src/runtime/effectiveness.ts +24 -1
- package/src/runtime/finalize-run.ts +12 -1
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +12 -3
- package/src/runtime/goal-workflow/goal-evaluator.ts +30 -5
- package/src/runtime/goal-workflow/goal-loop-runner.ts +60 -3
- package/src/runtime/live-session/live-agent-control.ts +34 -14
- package/src/runtime/live-session/live-agent-manager.ts +20 -5
- package/src/runtime/live-session/live-session-runtime.ts +66 -81
- package/src/runtime/live-session/usage-bridge.ts +66 -2
- package/src/runtime/mcp-proxy.ts +10 -0
- package/src/runtime/model/model-fallback.ts +68 -0
- package/src/runtime/model/runtime-policy.ts +20 -6
- package/src/runtime/model/runtime-resolver.ts +5 -3
- package/src/runtime/orphan-worker-registry.ts +64 -24
- package/src/runtime/pi-spawn.ts +4 -1
- package/src/runtime/process/zombie-scanner.ts +6 -2
- package/src/runtime/recovery/crash-recovery.ts +35 -2
- package/src/runtime/scheduling/batch-barrier.ts +42 -5
- package/src/runtime/scheduling/run-coalesced-task-group.ts +9 -3
- package/src/runtime/scheduling/scheduler.ts +74 -29
- package/src/runtime/stale-reconciler.ts +45 -6
- package/src/runtime/surface/herdr-provider.ts +54 -1
- package/src/runtime/surface/launch-script.ts +7 -1
- package/src/runtime/surface/resolve-surface.ts +4 -1
- package/src/runtime/surface/tmux-provider.ts +32 -2
- package/src/runtime/task-runner/child-executor.ts +3 -2
- package/src/runtime/workspace-lock.ts +52 -4
- package/src/state/contracts.ts +2 -0
- package/src/state/coordination/process-lock.ts +50 -0
- package/src/state/event-log/cursor.ts +24 -3
- package/src/state/event-log/event-log-rotation.ts +26 -2
- package/src/state/event-log/worker-atomic-writer.ts +5 -0
- package/src/state/stores/active-run-registry.ts +56 -5
- package/src/state/stores/ownership-map.ts +26 -7
- package/src/state/stores/plan-store.ts +30 -1
- package/src/state/stores/sqlite-run-state.ts +91 -1
- package/src/state/types.ts +5 -0
- package/src/ui/dashboard-panes/agents-pane.ts +4 -3
- package/src/ui/dashboard-panes/plan-pane.ts +5 -3
- package/src/ui/dashboard-panes/transcript-pane.ts +2 -1
- package/src/ui/overlays/mailbox-detail-overlay.ts +5 -3
- package/src/ui/run-dashboard.ts +6 -4
- package/src/ui/run-snapshot-cache.ts +3 -2
- package/src/ui/settings-overlay.ts +68 -10
- package/src/ui/theme-discovery.ts +26 -7
- package/src/ui/transcript-viewer.ts +5 -3
- package/src/utils/visual.ts +84 -0
|
@@ -174,31 +174,46 @@ function isValidId(id: string): boolean {
|
|
|
174
174
|
return true;
|
|
175
175
|
}
|
|
176
176
|
|
|
177
|
-
|
|
177
|
+
// FAMILY-2 (2026-10-17): strict read variant — ENOENT still maps to [] (a
|
|
178
|
+
// missing registry is a legitimate empty state), but any OTHER fault
|
|
179
|
+
// (EACCES/EBUSY/parse error/shape mismatch) THROWS. RMW callers
|
|
180
|
+
// (registerWorker/unregisterWorker) must skip their write on a strict-read
|
|
181
|
+
// fault: persisting the resulting [] would WIPE every sibling entry and
|
|
182
|
+
// orphan those workers until host exit.
|
|
183
|
+
function readRegistryStrict(): OrphanWorkerEntry[] {
|
|
178
184
|
const p = getRegistryPath();
|
|
185
|
+
// Atomic read: if file doesn't exist, readFileSync throws ENOENT
|
|
186
|
+
// which we handle explicitly. This eliminates the TOCTOU window
|
|
187
|
+
// between existsSync and readFileSync.
|
|
188
|
+
let raw: string;
|
|
179
189
|
try {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
190
|
+
raw = fs.readFileSync(p, "utf-8");
|
|
191
|
+
} catch (error) {
|
|
192
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
193
|
+
throw error;
|
|
194
|
+
}
|
|
195
|
+
const parsed: unknown = JSON.parse(raw); // corrupt JSON throws (strict)
|
|
196
|
+
if (!Array.isArray(parsed)) {
|
|
197
|
+
throw new Error(`orphan-worker-registry: expected array, got ${typeof parsed}`);
|
|
198
|
+
}
|
|
199
|
+
return parsed.filter(
|
|
200
|
+
(e): e is OrphanWorkerEntry =>
|
|
201
|
+
typeof e === "object" &&
|
|
202
|
+
e !== null &&
|
|
203
|
+
typeof e.pid === "number" &&
|
|
204
|
+
typeof e.sessionId === "string" &&
|
|
205
|
+
typeof e.runId === "string" &&
|
|
206
|
+
typeof e.registeredAt === "number" &&
|
|
207
|
+
typeof (e as OrphanWorkerEntry).parentPid === "number" &&
|
|
208
|
+
typeof (e as OrphanWorkerEntry).parentPidStartTime === "number" &&
|
|
209
|
+
typeof (e as OrphanWorkerEntry).startTime === "number",
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function readRegistry(): OrphanWorkerEntry[] {
|
|
214
|
+
try {
|
|
215
|
+
return readRegistryStrict();
|
|
198
216
|
} catch (error) {
|
|
199
|
-
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
|
200
|
-
return [];
|
|
201
|
-
}
|
|
202
217
|
// Silent failure is deliberate for robustness (registry read failures
|
|
203
218
|
// shouldn't crash the process), but log at warning level to aid troubleshooting.
|
|
204
219
|
logInternalError("orphan-worker-registry", new Error(`readRegistry failed: ${error}`), undefined, "warn");
|
|
@@ -278,7 +293,20 @@ export function registerWorker(
|
|
|
278
293
|
const startTime = getProcessStartTime(pid) ?? 0;
|
|
279
294
|
const parentPidStartTime = Number.isFinite(parentPid) && parentPid > 0 ? (getProcessStartTime(parentPid) ?? 0) : 0;
|
|
280
295
|
withFileLockSync(getRegistryPath(), () => {
|
|
281
|
-
|
|
296
|
+
// FAMILY-2: strict read — a read fault must SKIP the write (persisting
|
|
297
|
+
// the [] rebuild would wipe every sibling entry), not mask as empty.
|
|
298
|
+
let entries: OrphanWorkerEntry[];
|
|
299
|
+
try {
|
|
300
|
+
entries = readRegistryStrict();
|
|
301
|
+
} catch (error) {
|
|
302
|
+
logInternalError(
|
|
303
|
+
"orphan-worker-registry.register",
|
|
304
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
305
|
+
`pid=${pid} runId=${runId} — skipping write (registry read fault)`,
|
|
306
|
+
"warn",
|
|
307
|
+
);
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
282
310
|
// Dedupe by PID
|
|
283
311
|
const filtered = entries.filter((e) => e.pid !== pid);
|
|
284
312
|
filtered.push({
|
|
@@ -301,7 +329,19 @@ export function registerWorker(
|
|
|
301
329
|
export function unregisterWorker(pid: number): void {
|
|
302
330
|
if (!Number.isFinite(pid) || pid <= 0) return;
|
|
303
331
|
withFileLockSync(getRegistryPath(), () => {
|
|
304
|
-
|
|
332
|
+
// FAMILY-2: strict read — skip the write on a read fault (see registerWorker).
|
|
333
|
+
let entries: OrphanWorkerEntry[];
|
|
334
|
+
try {
|
|
335
|
+
entries = readRegistryStrict();
|
|
336
|
+
} catch (error) {
|
|
337
|
+
logInternalError(
|
|
338
|
+
"orphan-worker-registry.unregister",
|
|
339
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
340
|
+
`pid=${pid} — skipping write (registry read fault)`,
|
|
341
|
+
"warn",
|
|
342
|
+
);
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
305
345
|
const filtered = entries.filter((e) => e.pid !== pid);
|
|
306
346
|
if (filtered.length !== entries.length) {
|
|
307
347
|
writeRegistry(filtered);
|
package/src/runtime/pi-spawn.ts
CHANGED
|
@@ -96,7 +96,10 @@ function isWithinAllowedPrefixes(resolvedPath: string): boolean {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
-
|
|
99
|
+
// R2-L-A9 (wave-2b): a bare startsWith accepts sibling-name escapes
|
|
100
|
+
// (prefix "/opt/pi" also matches "/opt/pwned"). The prefix must match the
|
|
101
|
+
// path itself or a path-separator-bounded descendant.
|
|
102
|
+
return allowedPrefixes.some((prefix) => normalized === prefix || normalized.startsWith(prefix + path.sep));
|
|
100
103
|
}
|
|
101
104
|
|
|
102
105
|
function resolvePiPackageRoot(): string | undefined {
|
|
@@ -179,8 +179,12 @@ function isPidAlive(pid: number): boolean {
|
|
|
179
179
|
// process.kill(pid, 0) throws if the pid is not alive (or not ours).
|
|
180
180
|
process.kill(pid, 0);
|
|
181
181
|
return true;
|
|
182
|
-
} catch {
|
|
183
|
-
|
|
182
|
+
} catch (error) {
|
|
183
|
+
// R7-A2 (2026-10-17): only ESRCH/ENOENT mean the process is gone.
|
|
184
|
+
// EPERM means the process is alive but protected (different security
|
|
185
|
+
// context); treat as alive — mirror of active-run-registry.ts fix.
|
|
186
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
187
|
+
return code !== "ESRCH" && code !== "ENOENT";
|
|
184
188
|
}
|
|
185
189
|
}
|
|
186
190
|
|
|
@@ -700,6 +700,7 @@ export function purgeStaleActiveRunIndex(
|
|
|
700
700
|
// is fire-and-forget so it stays OUTSIDE the lock to avoid blocking the
|
|
701
701
|
// critical section on async IPC.
|
|
702
702
|
let abortedFreshLiveness = false;
|
|
703
|
+
let abortedIntentionalWait = false; // R2-M-A3: keep intentional-wait runs
|
|
703
704
|
try {
|
|
704
705
|
const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
|
|
705
706
|
if (fullLoaded) {
|
|
@@ -709,6 +710,14 @@ export function purgeStaleActiveRunIndex(
|
|
|
709
710
|
const fresh = loadRunManifestById(entry.cwd, entry.runId);
|
|
710
711
|
if (!fresh) return;
|
|
711
712
|
if (fresh.manifest.status !== "running") return;
|
|
713
|
+
// R2-M-A3 (2026-10-17): mirror :135/:329/:937 — purgeStaleActiveRunIndex
|
|
714
|
+
// was the only recovery path without an isIntentionalWait guard: a run
|
|
715
|
+
// parked on a plan approval / pending ask was purge-CANCELLED here even
|
|
716
|
+
// though every sibling recovery path preserves it. KEEP the entry.
|
|
717
|
+
if (isIntentionalWait(fresh.manifest, now)) {
|
|
718
|
+
abortedIntentionalWait = true;
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
712
721
|
// R7-6 (2026-10-16): under-lock liveness revalidation — the pre-lock
|
|
713
722
|
// gate used inputs read BEFORE the lock; a concurrent `team resume`
|
|
714
723
|
// adoption may have rewritten them in that window (fresh
|
|
@@ -747,6 +756,14 @@ export function purgeStaleActiveRunIndex(
|
|
|
747
756
|
}
|
|
748
757
|
}
|
|
749
758
|
updateRunStatus(fresh.manifest, "cancelled", "Orphaned run: worker process dead and no recent activity");
|
|
759
|
+
// R2-M-A3: clear waitState on cancel (mirror cancelOrphanedRuns :431-438) —
|
|
760
|
+
// an orphan-cancelled run must not keep the park pointer shielding it
|
|
761
|
+
// from staleness repair for the full waiting TTL.
|
|
762
|
+
if (fresh.manifest.waitState) {
|
|
763
|
+
const cleared = { ...fresh.manifest, waitState: undefined, updatedAt: new Date(now).toISOString() };
|
|
764
|
+
saveRunManifest(cleared);
|
|
765
|
+
fresh.manifest = cleared;
|
|
766
|
+
}
|
|
750
767
|
});
|
|
751
768
|
// R7-6: only terminate live agents when the cancel actually landed —
|
|
752
769
|
// an aborted (fresh-evidence) run is alive; terminating it would kill
|
|
@@ -767,7 +784,9 @@ export function purgeStaleActiveRunIndex(
|
|
|
767
784
|
}
|
|
768
785
|
// R7-6: a fresh-evidence abort means the run is alive again — KEEP the
|
|
769
786
|
// registry entry (live-claim signal) instead of unregistering it.
|
|
770
|
-
|
|
787
|
+
// R2-M-A3: an intentional-wait abort keeps the entry too (same reasoning —
|
|
788
|
+
// the run is parked on a human decision, not crashed).
|
|
789
|
+
if (abortedFreshLiveness || abortedIntentionalWait) {
|
|
771
790
|
kept.push(entry.runId);
|
|
772
791
|
continue;
|
|
773
792
|
}
|
|
@@ -794,12 +813,19 @@ export function purgeStaleActiveRunIndex(
|
|
|
794
813
|
if (!hasRecentLifeEvidence(entry, manifest.updatedAt, now, staleThresholdMs)) {
|
|
795
814
|
let cancelledUnderLock = false;
|
|
796
815
|
let abortedFreshLiveness = false;
|
|
816
|
+
let abortedIntentionalWait = false; // R2-M-A3: keep intentional-wait runs
|
|
797
817
|
try {
|
|
798
818
|
const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
|
|
799
819
|
if (fullLoaded && fullLoaded.manifest.status === "running") {
|
|
800
820
|
withRunLockSync(fullLoaded.manifest, () => {
|
|
801
821
|
const fresh = loadRunManifestById(entry.cwd, entry.runId);
|
|
802
822
|
if (fresh?.manifest.status !== "running") return;
|
|
823
|
+
// R2-M-A3 (2026-10-17): mirror the step-5 guard (and :135/:329/:937) — a
|
|
824
|
+
// parked run (plan approval / pending ask) is not an orphan. KEEP it.
|
|
825
|
+
if (isIntentionalWait(fresh.manifest, now)) {
|
|
826
|
+
abortedIntentionalWait = true;
|
|
827
|
+
return;
|
|
828
|
+
}
|
|
803
829
|
// R7-6 (2026-10-16): under-lock liveness revalidation — the pre-lock
|
|
804
830
|
// gate used inputs read BEFORE the lock; a concurrent `team resume`
|
|
805
831
|
// adoption may have rewritten them in that window (fresh
|
|
@@ -846,6 +872,12 @@ export function purgeStaleActiveRunIndex(
|
|
|
846
872
|
Math.round(staleThresholdMs / 60000) +
|
|
847
873
|
" minutes",
|
|
848
874
|
);
|
|
875
|
+
// R2-M-A3: clear waitState on cancel (mirror :431-438 + step 5 above).
|
|
876
|
+
if (fresh.manifest.waitState) {
|
|
877
|
+
const cleared = { ...fresh.manifest, waitState: undefined, updatedAt: new Date(now).toISOString() };
|
|
878
|
+
saveRunManifest(cleared);
|
|
879
|
+
fresh.manifest = cleared;
|
|
880
|
+
}
|
|
849
881
|
cancelledUnderLock = true;
|
|
850
882
|
});
|
|
851
883
|
// R7-1/R7-6: only terminate live agents when the cancel actually landed
|
|
@@ -867,7 +899,8 @@ export function purgeStaleActiveRunIndex(
|
|
|
867
899
|
}
|
|
868
900
|
// R7-1/R7-6: a fresh-evidence abort means the run is alive again — KEEP
|
|
869
901
|
// the registry entry (live-claim signal) instead of unregistering it.
|
|
870
|
-
|
|
902
|
+
// R2-M-A3: an intentional-wait abort keeps the entry too.
|
|
903
|
+
if (abortedFreshLiveness || abortedIntentionalWait) {
|
|
871
904
|
kept.push(entry.runId);
|
|
872
905
|
continue;
|
|
873
906
|
}
|
|
@@ -55,8 +55,19 @@ export class BatchBarrier {
|
|
|
55
55
|
}
|
|
56
56
|
>();
|
|
57
57
|
|
|
58
|
-
/**
|
|
58
|
+
/** R2-L-B6 (SEV-10): consolidated-notified batches are FREED (member maps
|
|
59
|
+
* dropped) and remembered here as tombstones so `alreadyNotified` stays
|
|
60
|
+
* true and late `markTerminal`/`register` calls never re-notify. Bounded
|
|
61
|
+
* (evict-oldest) — previously every batchId leaked its member maps for the
|
|
62
|
+
* session lifetime (zero production callers of dispose()). */
|
|
63
|
+
private readonly notifiedTombstones = new Map<string, number>();
|
|
64
|
+
private static readonly TOMBSTONE_CAP = 256;
|
|
65
|
+
|
|
66
|
+
/** Register a member at spawn time. Idempotent per (batchId, agentId).
|
|
67
|
+
* R2-L-B6: no-op for a tombstoned (already notified) batch — the member
|
|
68
|
+
* map is intentionally gone and must not resurrect a re-notifiable batch. */
|
|
59
69
|
register(batchId: string, agentId: string, meta?: { description?: string; type?: string }): void {
|
|
70
|
+
if (this.notifiedTombstones.has(batchId)) return;
|
|
60
71
|
let batch = this.batches.get(batchId);
|
|
61
72
|
if (!batch) {
|
|
62
73
|
batch = {
|
|
@@ -83,6 +94,18 @@ export class BatchBarrier {
|
|
|
83
94
|
* on-the-fly as a batch-of-one so its terminal state is not silently lost.
|
|
84
95
|
*/
|
|
85
96
|
markTerminal(batchId: string, member: BatchMember): BatchSnapshot {
|
|
97
|
+
// R2-L-B6: the batch already fired its consolidated notification and was
|
|
98
|
+
// freed — report notified:true so the caller NEVER re-notifies a late
|
|
99
|
+
// member (documented late-spawn semantics), without resurrecting the map.
|
|
100
|
+
if (this.notifiedTombstones.has(batchId)) {
|
|
101
|
+
return {
|
|
102
|
+
batchId,
|
|
103
|
+
members: [{ ...member }],
|
|
104
|
+
terminal: [{ ...member }],
|
|
105
|
+
allDone: true,
|
|
106
|
+
notified: true,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
86
109
|
let batch = this.batches.get(batchId);
|
|
87
110
|
if (!batch) {
|
|
88
111
|
batch = {
|
|
@@ -117,13 +140,22 @@ export class BatchBarrier {
|
|
|
117
140
|
|
|
118
141
|
/** Has the consolidated notification already been emitted for this batch? */
|
|
119
142
|
alreadyNotified(batchId: string): boolean {
|
|
120
|
-
return this.batches.get(batchId)?.notified ??
|
|
143
|
+
return this.batches.get(batchId)?.notified ?? this.notifiedTombstones.has(batchId);
|
|
121
144
|
}
|
|
122
145
|
|
|
123
|
-
/** Mark the consolidated notification as emitted. No-op if already set.
|
|
146
|
+
/** Mark the consolidated notification as emitted. No-op if already set.
|
|
147
|
+
* R2-L-B6: DISPOSE the batch here — once the consolidated notify fired, the
|
|
148
|
+
* member maps are dead weight; keep only a bounded tombstone so
|
|
149
|
+
* `alreadyNotified`/late `markTerminal` preserve their answers. */
|
|
124
150
|
markNotified(batchId: string): void {
|
|
125
151
|
const batch = this.batches.get(batchId);
|
|
126
152
|
if (batch) batch.notified = true;
|
|
153
|
+
this.batches.delete(batchId);
|
|
154
|
+
this.notifiedTombstones.set(batchId, Date.now());
|
|
155
|
+
if (this.notifiedTombstones.size > BatchBarrier.TOMBSTONE_CAP) {
|
|
156
|
+
const oldest = this.notifiedTombstones.keys().next().value;
|
|
157
|
+
if (oldest !== undefined) this.notifiedTombstones.delete(oldest);
|
|
158
|
+
}
|
|
127
159
|
}
|
|
128
160
|
|
|
129
161
|
/** Read-only snapshot (for tests / debugging). */
|
|
@@ -141,7 +173,12 @@ export class BatchBarrier {
|
|
|
141
173
|
|
|
142
174
|
/** Drop a batch (used on cleanup / test reset). */
|
|
143
175
|
dispose(batchId?: string): void {
|
|
144
|
-
if (batchId === undefined)
|
|
145
|
-
|
|
176
|
+
if (batchId === undefined) {
|
|
177
|
+
this.batches.clear();
|
|
178
|
+
this.notifiedTombstones.clear();
|
|
179
|
+
} else {
|
|
180
|
+
this.batches.delete(batchId);
|
|
181
|
+
this.notifiedTombstones.delete(batchId);
|
|
182
|
+
}
|
|
146
183
|
}
|
|
147
184
|
}
|
|
@@ -61,7 +61,9 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
61
61
|
}
|
|
62
62
|
return t;
|
|
63
63
|
});
|
|
64
|
-
|
|
64
|
+
// R3-H1 (fix): fresh-read group-scoped merge — never persist the dispatch-time
|
|
65
|
+
// full-array snapshot over concurrent sibling state changes.
|
|
66
|
+
await repairGroupTerminalWrite(manifest, taskIds, updatedTasks);
|
|
65
67
|
await appendEventAsync(manifest.eventsPath, {
|
|
66
68
|
type: "task.coalesced_dispatch_start",
|
|
67
69
|
runId: manifest.runId,
|
|
@@ -86,7 +88,9 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
86
88
|
heartbeat: t.heartbeat ?? createWorkerHeartbeat(t.id),
|
|
87
89
|
};
|
|
88
90
|
});
|
|
89
|
-
|
|
91
|
+
// R3-H1 (fix): fresh-read group-scoped merge — never persist the dispatch-time
|
|
92
|
+
// full-array snapshot over concurrent sibling state changes.
|
|
93
|
+
await repairGroupTerminalWrite(manifest, taskIds, updatedTasks);
|
|
90
94
|
|
|
91
95
|
let rawOutput = "";
|
|
92
96
|
let success = false;
|
|
@@ -398,7 +402,9 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
|
|
|
398
402
|
if (drainTimeout !== undefined) clearTimeout(drainTimeout);
|
|
399
403
|
}
|
|
400
404
|
}
|
|
401
|
-
|
|
405
|
+
// R3-H1 (fix): fresh-read group-scoped merge — never persist the dispatch-time
|
|
406
|
+
// full-array snapshot over concurrent sibling state changes.
|
|
407
|
+
await repairGroupTerminalWrite(manifest, taskIds, updatedTasks);
|
|
402
408
|
let updatedManifest: TeamRunManifest = {
|
|
403
409
|
...manifest,
|
|
404
410
|
artifacts: mergeArtifacts([...manifest.artifacts, ...newArtifacts]),
|
|
@@ -36,6 +36,10 @@ export interface CrewSchedulerOptions {
|
|
|
36
36
|
* waits (e.g. a yearly cron) must be chained in hops below this ceiling. */
|
|
37
37
|
const MAX_TIMER_DELAY_MS = 2_147_483_000;
|
|
38
38
|
|
|
39
|
+
/** R2-M-B4 (SEV-4): retention cap for job.spawnedRunIds (bounded persisted JSON;
|
|
40
|
+
* remove() cancels at most this most-recent window). */
|
|
41
|
+
const SPAWNED_RUN_IDS_CAP = 50;
|
|
42
|
+
|
|
39
43
|
export class CrewScheduler {
|
|
40
44
|
private jobs = new Map<string, ScheduledJob>();
|
|
41
45
|
private timers = new Map<string, ReturnType<typeof setInterval | typeof setTimeout>>();
|
|
@@ -81,6 +85,12 @@ export class CrewScheduler {
|
|
|
81
85
|
|
|
82
86
|
add(job: ScheduledJob): void {
|
|
83
87
|
this.jobs.set(job.id, job);
|
|
88
|
+
// R2-M-B6 (SEV-11): a persisted nextRun in the past = occurrences missed
|
|
89
|
+
// while the host was down. Classic cron leaves mail; we leave a one-time
|
|
90
|
+
// `skipped` event + log line on RELOAD (this path). No catch-up — firing a
|
|
91
|
+
// stale queue at load would storm the host. Runtime re-arms go through
|
|
92
|
+
// update() (never add()), so a normal fire's stale nextRun cannot reach here.
|
|
93
|
+
this.reportMissedCronOccurrences(job);
|
|
84
94
|
if (job.enabled) this.arm(job);
|
|
85
95
|
this.emit?.({ type: "added", job });
|
|
86
96
|
}
|
|
@@ -123,7 +133,13 @@ export class CrewScheduler {
|
|
|
123
133
|
recordSpawnedRun(jobId: string, runId: string): void {
|
|
124
134
|
const job = this.jobs.get(jobId);
|
|
125
135
|
if (!job) return;
|
|
126
|
-
|
|
136
|
+
// R2-M-B4 (SEV-4): cap spawnedRunIds. The unbounded append persisted the
|
|
137
|
+
// FULL array per fire (O(n²) writes — a 5-min job ≈ 8.6k entries/30d,
|
|
138
|
+
// ~300-350 KB) and remove() then cancelled ALL historical runIds. Keep the
|
|
139
|
+
// most recent SPAWNED_RUN_IDS_CAP; runs beyond the cap are long terminal,
|
|
140
|
+
// so not cancelling them is the natural no-op.
|
|
141
|
+
const appended = [...(job.spawnedRunIds ?? []), runId];
|
|
142
|
+
const spawnedRunIds = appended.length > SPAWNED_RUN_IDS_CAP ? appended.slice(-SPAWNED_RUN_IDS_CAP) : appended;
|
|
127
143
|
this.jobs.set(jobId, { ...job, spawnedRunIds });
|
|
128
144
|
}
|
|
129
145
|
|
|
@@ -150,6 +166,26 @@ export class CrewScheduler {
|
|
|
150
166
|
return { ok: true };
|
|
151
167
|
}
|
|
152
168
|
|
|
169
|
+
/** R2-M-B6: one-time missed-occurrence report for a reloaded cron job whose
|
|
170
|
+
* persisted nextRun is in the past (bounded occurrence walk — no catch-up). */
|
|
171
|
+
private reportMissedCronOccurrences(job: ScheduledJob): void {
|
|
172
|
+
if (job.scheduleType !== "cron" || !job.nextRun) return;
|
|
173
|
+
const nowMs = this.now().getTime();
|
|
174
|
+
const persistedMs = new Date(job.nextRun).getTime();
|
|
175
|
+
if (!Number.isFinite(persistedMs) || persistedMs >= nowMs) return;
|
|
176
|
+
let missed = 0;
|
|
177
|
+
let cursor = new Date(persistedMs);
|
|
178
|
+
for (let i = 0; i < 10_000; i++) {
|
|
179
|
+
const next = nextRunTime({ kind: "cron", spec: job.schedule }, cursor);
|
|
180
|
+
if (!(next instanceof Date) || next.getTime() > nowMs) break;
|
|
181
|
+
missed += 1;
|
|
182
|
+
cursor = next;
|
|
183
|
+
}
|
|
184
|
+
const reason = `missed ${missed} occurrence(s) while host down (persisted nextRun ${job.nextRun} < now ${new Date(nowMs).toISOString()})`;
|
|
185
|
+
this.emit?.({ type: "skipped", jobId: job.id, reason });
|
|
186
|
+
console.warn(`[pi-crew scheduler] job ${job.id} (${job.name ?? job.id}) skipped: ${reason}`);
|
|
187
|
+
}
|
|
188
|
+
|
|
153
189
|
private arm(job: ScheduledJob): void {
|
|
154
190
|
if (this.timers.has(job.id)) return;
|
|
155
191
|
if (job.scheduleType === "interval" && job.intervalMs) {
|
|
@@ -445,46 +481,55 @@ function cronFieldCandidates(field: string, min: number, max: number, names?: Re
|
|
|
445
481
|
return matched;
|
|
446
482
|
}
|
|
447
483
|
|
|
448
|
-
/**
|
|
449
|
-
* Match a single cron field value against a cron field expression.
|
|
450
|
-
* Supports the standard cron grammar: wildcard, single N, range a-b, list
|
|
451
|
-
* a,b,c, step syntax (wildcard-step, range-step, from-step), and named tokens
|
|
452
|
-
* (MON, JAN) when a `names` map is passed. Fixes the prior matcher that
|
|
453
|
-
* rejected step values and named DOW (parseInt returned NaN).
|
|
454
|
-
*/
|
|
455
|
-
function cronFieldMatches(value: number, field: string, min: number, max: number, names?: Record<string, number>): boolean {
|
|
456
|
-
const candidates = cronFieldCandidates(field, min, max, names);
|
|
457
|
-
return candidates?.has(value) === true;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
484
|
function nextCronDate(spec: string, from: Date): Date | { error: string } | null {
|
|
461
485
|
const parts = spec.split(/\s+/);
|
|
462
486
|
if (parts.length < 5) return { error: "Invalid cron expression" };
|
|
463
487
|
const [minStr, hourStr, domStr, monthStr, dowStr] = parts;
|
|
464
488
|
|
|
489
|
+
// R2-L-B5 (SEV-9): hoist the candidate Sets OUT of the scan loop. The old
|
|
490
|
+
// minute-by-minute scan rebuilt 5 Sets per step (sparse crons walked up to
|
|
491
|
+
// 527k iterations ≈ 309ms event-loop stall, re-run after every fire). The
|
|
492
|
+
// scan below keeps EXACTLY the same semantics (first matching minute in
|
|
493
|
+
// chronological order) but hops by hour/day when the coarse fields mismatch.
|
|
494
|
+
const minuteCandidates = cronFieldCandidates(minStr, 0, 59);
|
|
495
|
+
const hourCandidates = cronFieldCandidates(hourStr, 0, 23);
|
|
496
|
+
const domCandidates = cronFieldCandidates(domStr, 1, 31);
|
|
497
|
+
const monthCandidates = cronFieldCandidates(monthStr, 1, 12, CRON_MONTH_NAMES);
|
|
498
|
+
const dowCandidates = cronFieldCandidates(dowStr, 0, 6, CRON_DOW_NAMES);
|
|
499
|
+
if (!minuteCandidates || !hourCandidates || !domCandidates || !monthCandidates || !dowCandidates) {
|
|
500
|
+
return { error: "Invalid cron expression" };
|
|
501
|
+
}
|
|
502
|
+
|
|
465
503
|
let cursor = new Date(from.getTime());
|
|
466
504
|
cursor.setSeconds(0, 0);
|
|
467
505
|
cursor = new Date(cursor.getTime() + 60_000);
|
|
468
506
|
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
const dom = cursor.getUTCDate();
|
|
474
|
-
const month = cursor.getUTCMonth() + 1;
|
|
475
|
-
const dow = cursor.getUTCDay();
|
|
476
|
-
|
|
507
|
+
// Search window: 366 days (the old 366*24*60 minute-step bound).
|
|
508
|
+
const deadlineMs = cursor.getTime() + 366 * 24 * 60 * 60_000;
|
|
509
|
+
while (cursor.getTime() <= deadlineMs) {
|
|
510
|
+
// Day-level filter: month/dom/dow — skip whole days on mismatch.
|
|
477
511
|
if (
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
cronFieldMatches(month, monthStr, 1, 12, CRON_MONTH_NAMES) &&
|
|
482
|
-
cronFieldMatches(dow, dowStr, 0, 6, CRON_DOW_NAMES)
|
|
512
|
+
!monthCandidates.has(cursor.getUTCMonth() + 1) ||
|
|
513
|
+
!domCandidates.has(cursor.getUTCDate()) ||
|
|
514
|
+
!dowCandidates.has(cursor.getUTCDay())
|
|
483
515
|
) {
|
|
484
|
-
|
|
516
|
+
// Hop to 00:00 of the next UTC day (Date.UTC normalizes overflow).
|
|
517
|
+
cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth(), cursor.getUTCDate() + 1, 0, 0));
|
|
518
|
+
continue;
|
|
485
519
|
}
|
|
486
|
-
|
|
487
|
-
|
|
520
|
+
// Hour-level filter: skip the rest of the hour on mismatch.
|
|
521
|
+
if (!hourCandidates.has(cursor.getUTCHours())) {
|
|
522
|
+
cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth(), cursor.getUTCDate(), cursor.getUTCHours() + 1, 0));
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
// Matching day + hour: scan the remaining minutes of THIS hour only.
|
|
526
|
+
for (let minute = cursor.getUTCMinutes(); minute <= 59; minute++) {
|
|
527
|
+
if (minuteCandidates.has(minute)) {
|
|
528
|
+
return new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth(), cursor.getUTCDate(), cursor.getUTCHours(), minute));
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
// No matching minute left in this hour → next hour boundary.
|
|
532
|
+
cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth(), cursor.getUTCDate(), cursor.getUTCHours() + 1, 0));
|
|
488
533
|
}
|
|
489
534
|
|
|
490
535
|
return { error: "No next cron occurrence found within search window" };
|
|
@@ -108,10 +108,28 @@ const WAITING_TTL_MS = 24 * 60 * 60 * 1000;
|
|
|
108
108
|
* normal staleness repair paths apply (leak guard).
|
|
109
109
|
*/
|
|
110
110
|
export function isWaitAnswerPending(manifest: TeamRunManifest, now = Date.now()): boolean {
|
|
111
|
-
const
|
|
112
|
-
|
|
111
|
+
const waitState = manifest.waitState;
|
|
112
|
+
if (!waitState?.askedAt) return false;
|
|
113
|
+
const askedAtMs = new Date(waitState.askedAt).getTime();
|
|
114
|
+
if (!Number.isFinite(askedAtMs) || now - askedAtMs > WAITING_TTL_MS) return false;
|
|
115
|
+
// R2-M-A2 (2026-10-17): the parked-ask shield outlived the ask itself —
|
|
116
|
+
// WAITING_TTL_MS (24h) ≫ the server-clamped ask deadline (≤1h,
|
|
117
|
+
// request-parsers.ts WAIT_REQUEST_TIMEOUT_SEC_MAX), so a runner that died
|
|
118
|
+
// parked stayed shielded ~23h past its park deadline (dead parked runner
|
|
119
|
+
// blocked verdict tree/cancel/U9 fast-path). When the broker stamped a
|
|
120
|
+
// deadline on the park, the shield expires at deadline + grace; the 24h
|
|
121
|
+
// TTL above stays ONLY as the no-deadline fallback (pre-fix manifests).
|
|
122
|
+
if (typeof waitState.deadline === "number" && Number.isFinite(waitState.deadline)) {
|
|
123
|
+
return now <= waitState.deadline + PARKED_ASK_GRACE_MS;
|
|
124
|
+
}
|
|
125
|
+
return true;
|
|
113
126
|
}
|
|
114
127
|
|
|
128
|
+
/** R2-M-A2: grace past the ask deadline before a parked run loses
|
|
129
|
+
* reconciliation protection (buffer for the sweep cadence / in-tool
|
|
130
|
+
* surfacing — the deadline-owner sweep in dispatch-batch co-owns this). */
|
|
131
|
+
const PARKED_ASK_GRACE_MS = 60 * 60 * 1000;
|
|
132
|
+
|
|
115
133
|
/**
|
|
116
134
|
* Generalized intentional-wait predicate (ADR-0 item 9, extended by ADR-4 §2):
|
|
117
135
|
* the run is blocked on a human decision — plan approval (record-first with
|
|
@@ -415,10 +433,22 @@ export function hasLiveRunClaim(runId: string, manifest: TeamRunManifest): boole
|
|
|
415
433
|
return false;
|
|
416
434
|
}
|
|
417
435
|
|
|
418
|
-
function repairStaleRun(
|
|
436
|
+
function repairStaleRun(
|
|
437
|
+
manifest: TeamRunManifest,
|
|
438
|
+
tasks: TeamTaskState[],
|
|
439
|
+
reason: string,
|
|
440
|
+
// R2-M-A1 (2026-10-17): when set, ONLY these task ids are cancelled — used
|
|
441
|
+
// by the no_pid_individual_stale_task branch, whose contract (and comment)
|
|
442
|
+
// says "repair only the zombie subset, not the whole run": cancelling
|
|
443
|
+
// pid-ALIVE workers there destroyed healthy siblings in the SAME run.
|
|
444
|
+
onlyTaskIds?: Set<string>,
|
|
445
|
+
): TeamTaskState[] {
|
|
419
446
|
const now = new Date().toISOString();
|
|
420
447
|
const repairedTasks = tasks.map((task) => {
|
|
421
|
-
if (
|
|
448
|
+
if (
|
|
449
|
+
(onlyTaskIds === undefined || onlyTaskIds.has(task.id)) &&
|
|
450
|
+
(task.status === "running" || task.status === "queued" || task.status === "waiting")
|
|
451
|
+
) {
|
|
422
452
|
return {
|
|
423
453
|
...task,
|
|
424
454
|
status: "cancelled" as const,
|
|
@@ -571,7 +601,13 @@ export function reconcileStaleRun(manifest: TeamRunManifest, tasks: TeamTaskStat
|
|
|
571
601
|
// ALL tasks are zombies, while this path repairs only the zombie
|
|
572
602
|
// subset when some tasks are still healthy.
|
|
573
603
|
if (staleTaskIds.length > 0) {
|
|
574
|
-
|
|
604
|
+
// R2-M-A1 (2026-10-17): cancel ONLY the stale subset. The pre-fix code
|
|
605
|
+
// called repairStaleRun unscoped, which cancelled EVERY running/queued/
|
|
606
|
+
// waiting task in the run — including pid-ALIVE workers the branch was
|
|
607
|
+
// supposed to leave alone (pinned by stale-reconciler-r2h1-retention,
|
|
608
|
+
// updated in the same wave: task_a now stays running).
|
|
609
|
+
const staleSet = new Set(staleTaskIds);
|
|
610
|
+
const repaired = repairStaleRun(manifest, tasks, "no_pid_individual_stale_task", staleSet);
|
|
575
611
|
// Only return the individually repaired tasks in detail
|
|
576
612
|
return {
|
|
577
613
|
runId,
|
|
@@ -713,7 +749,10 @@ export function reconcileOrphanedTempWorkspaces(
|
|
|
713
749
|
const result = reconcileStaleRun(manifest, tasks, now);
|
|
714
750
|
if (result.repaired && result.repairedTasks) {
|
|
715
751
|
// Persist repaired tasks (atomic — temp+rename to survive mid-write crash)
|
|
716
|
-
|
|
752
|
+
// R2-H1 (fix): persist the FULL array (NEW-1 persistTasks), not the filtered
|
|
753
|
+
// stale subset — the subset write destroyed every sibling record (mirror
|
|
754
|
+
// crash-recovery.ts:1066).
|
|
755
|
+
atomicWriteJson(tasksPath, result.persistTasks ?? result.repairedTasks);
|
|
717
756
|
// Update manifest status
|
|
718
757
|
const updated = {
|
|
719
758
|
...manifest,
|