pi-crew 0.11.38 → 0.11.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,6 +4,7 @@ import * as path from "node:path";
4
4
  import { getCrewEnv } from "../config/env-vars.ts";
5
5
  import { errors } from "../errors.ts";
6
6
  import { atomicWriteFile, atomicWriteJson } from "../state/atomic-write.ts";
7
+ import { isTerminalTaskStatus } from "../state/contracts.ts";
7
8
  import { type ActiveRunRegistryEntry, activeRunEntries } from "../state/stores/active-run-registry.ts";
8
9
  import { getCurrentPlanRecord } from "../state/stores/plan-store.ts";
9
10
  import { loadManifestWithRecovery, loadTasksWithRecovery, saveRunManifest } from "../state/stores/state-store.ts";
@@ -11,9 +12,19 @@ import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
11
12
  import { logInternalError } from "../utils/internal-error.ts";
12
13
  import { findRunStateDir } from "../utils/paths.ts";
13
14
  import { recordFromTask, upsertCrewAgent } from "./crew-agent-records.ts";
15
+ import { currentCrewDepth } from "./model/pi-args.ts";
14
16
  import { reapRecordedWorkerOrphans } from "./process/orphan-reaper.ts";
17
+ import { type ProcSnapshot, reapRunDescendants, resolveRunDescendantSweepEnabled } from "./process/run-descendant-sweep.ts";
15
18
  import { checkProcessLiveness } from "./process-status.ts";
16
19
 
20
+ /**
21
+ * Synchronous shutdown flush — RE-EXPORTED so every caller of the escalation
22
+ * machinery (`drainReapEscalations` in this module, `flushReapEscalationsSync`
23
+ * in `process/reap-group.ts`) imports from one family. `reap-group.ts` owns the
24
+ * armed registry because it is the only module that may signal a group.
25
+ */
26
+ export { flushReapEscalationsSync } from "./process/reap-group.ts";
27
+
17
28
  /** Age threshold for orphaned temp directory cleanup: 1 hour. */
18
29
  const ORPHAN_TEMP_DIR_AGE_THRESHOLD_MS = 60 * 60 * 1000;
19
30
  /** A `.cleanup-in-progress` sentinel older than this is abandoned — its owner
@@ -449,6 +460,93 @@ export function hasLiveRunClaim(runId: string, manifest: TeamRunManifest): boole
449
460
  return false;
450
461
  }
451
462
 
463
+ // ── armed group escalations (wave 11a review, MAJOR #3) ────────────────────
464
+ //
465
+ // `repairStaleRun` is SYNCHRONOUS by contract: it is reached from
466
+ // `withRunLockSync` (crash-recovery) and from the sync
467
+ // `reconcileOrphanedTempWorkspaces`, and 50+ tests pin its sync return value
468
+ // (see the OPT-02 note in checkResultFile). It therefore CANNOT await the
469
+ // SIGKILL escalation it arms when it reaps a worker process group. The pre-fix
470
+ // code dropped the promise outright — `void Promise.allSettled(...).catch(...)`,
471
+ // whose catch could never fire because allSettled never rejects — so a host
472
+ // that returned and exited here could leave the escalation undelivered.
473
+ //
474
+ // Every armed escalation is now REGISTERED, and an async owner awaits
475
+ // `drainReapEscalations()`: `reconcileAllStaleRuns` (crash-recovery.ts) does it
476
+ // for the background sweep, and any other caller that wants the guarantee can.
477
+ //
478
+ // Why the keep-alive pin below: the escalation itself is a chain of UNREF'd
479
+ // timers (reap-group.ts must never keep an idle run alive), so the event loop
480
+ // would drain underneath a still-in-flight escalation and abandon the SIGKILL
481
+ // (measured on Node 22: an unsettled top-level await over unref'd timers exits
482
+ // 13). The drain therefore holds ONE ref'd interval of its own — and only while
483
+ // it actually has something to await.
484
+ const inflightReapEscalations = new Set<Promise<unknown>>();
485
+ /** Keep-alive cadence for the drain (ms). Any value works: it only has to exist. */
486
+ const DRAIN_KEEPALIVE_MS = 200;
487
+
488
+ /** Register an armed escalation; the entry clears once it settles. Never throws. */
489
+ function trackReapEscalation(promise: Promise<unknown>): void {
490
+ const settled = promise.catch((err) => {
491
+ logInternalError("stale-reconciler.reap-escalation", err, "group escalation rejected", "warn");
492
+ });
493
+ inflightReapEscalations.add(settled);
494
+ void settled.finally(() => inflightReapEscalations.delete(settled));
495
+ }
496
+
497
+ /**
498
+ * Await every group escalation this module armed (SIGKILL rung included) and
499
+ * clear the registry. Never throws; resolves only AFTER the in-flight
500
+ * escalations settled, and holds a ref'd keep-alive for exactly that window (an
501
+ * empty drain is an immediate no-op that creates no keep-alive at all).
502
+ */
503
+ export async function drainReapEscalations(): Promise<void> {
504
+ const snapshot = [...inflightReapEscalations];
505
+ if (snapshot.length === 0) return;
506
+ // REF'd (not unref'd) on purpose: this is the only thing that keeps the
507
+ // process alive while an escalation is mid-flight. Cleared in `finally`, so
508
+ // it lives exactly as long as the wait.
509
+ const pin = setInterval(() => {
510
+ /* ref'd no-op: its ONLY job is to keep the loop alive */
511
+ }, DRAIN_KEEPALIVE_MS);
512
+ pin.ref();
513
+ try {
514
+ await Promise.allSettled(snapshot);
515
+ } catch (err) {
516
+ // allSettled never rejects — belt-and-braces only.
517
+ logInternalError("stale-reconciler.drainReapEscalations", err, undefined, "warn");
518
+ } finally {
519
+ clearInterval(pin);
520
+ for (const promise of snapshot) inflightReapEscalations.delete(promise);
521
+ }
522
+ }
523
+
524
+ // Wave 11a review test seam (same shape as child-pi-kill.ts `__test`): swap the
525
+ // orphan reaper so the escalation registration can be driven by a deferred
526
+ // promise — no real /proc read and no real signal.
527
+ let reapRecordedWorkerOrphansImpl = reapRecordedWorkerOrphans;
528
+ // Wave 11b.1 test seam (same shape as the orphan-reaper seam above): swap the
529
+ // run-descendant sweep so its REACHABILITY can be pinned at reconcile level — a
530
+ // real /proc scan and a real signal must never be part of a reconcile test.
531
+ let reapRunDescendantsImpl: typeof reapRunDescendants = reapRunDescendants;
532
+ export const __test = {
533
+ setReapRecordedWorkerOrphans(fn: typeof reapRecordedWorkerOrphans | undefined): void {
534
+ reapRecordedWorkerOrphansImpl = fn ?? reapRecordedWorkerOrphans;
535
+ },
536
+ setReapRunDescendants(fn: typeof reapRunDescendants | undefined): void {
537
+ reapRunDescendantsImpl = fn ?? reapRunDescendants;
538
+ },
539
+ /** Register a synthetic escalation (event-loop pin test — no real reap). */
540
+ trackEscalation(promise: Promise<unknown>): void {
541
+ trackReapEscalation(promise);
542
+ },
543
+ reset(): void {
544
+ reapRecordedWorkerOrphansImpl = reapRecordedWorkerOrphans;
545
+ reapRunDescendantsImpl = reapRunDescendants;
546
+ inflightReapEscalations.clear();
547
+ },
548
+ };
549
+
452
550
  function repairStaleRun(
453
551
  manifest: TeamRunManifest,
454
552
  tasks: TeamTaskState[],
@@ -488,9 +586,9 @@ function repairStaleRun(
488
586
  // worker re-parented to PID 1 stayed alive and burned a slot forever. This
489
587
  // reaps only pids confirmed by the AUTHORITATIVE zombie signal (crew-subagent
490
588
  // marker + dead crew-parent), so a live worker is never touched. Best-effort.
589
+ const reapTargets = onlyTaskIds ? tasks.filter((task) => onlyTaskIds.has(task.id)) : tasks;
491
590
  try {
492
- const reapTargets = onlyTaskIds ? tasks.filter((task) => onlyTaskIds.has(task.id)) : tasks;
493
- const reapResult = reapRecordedWorkerOrphans(reapTargets);
591
+ const reapResult = reapRecordedWorkerOrphansImpl(reapTargets);
494
592
  if (reapResult.reaped.length > 0) {
495
593
  logInternalError(
496
594
  "stale-reconciler",
@@ -499,12 +597,237 @@ function repairStaleRun(
499
597
  "warn",
500
598
  );
501
599
  }
600
+ // Wave 11a (F2) + review MAJOR #3: the group path delivers SIGTERM
601
+ // synchronously and returns the SIGKILL escalation as a promise. This
602
+ // function is sync by contract (see the block above `repairStaleRun`), so
603
+ // the escalations are REGISTERED here and the async owner awaits
604
+ // `drainReapEscalations()` — nothing is dropped, nothing dangles.
605
+ for (const escalation of reapResult.pending) trackReapEscalation(escalation);
502
606
  } catch (reapError) {
503
607
  logInternalError("stale-reconciler", reapError, `orphan reap failed for ${manifest.runId}`, "warn");
504
608
  }
609
+ // Wave 11b (F7): the worker-level reap above cannot reach pi's bash
610
+ // grandchildren — the bash tool spawns every command `detached`, so they own
611
+ // a group nothing rooted at the worker can signal (the 390 MB / 24 h leak
612
+ // after team_20261022110510 completed). The run-scoped sweep matches the env
613
+ // markers they INHERITED instead, and only once this run is terminal.
614
+ // NOTE (11b.1): the sweep is handed the POST-repair array, never the input
615
+ // `tasks` and never `reapTargets`. The terminal guard inside the helper is
616
+ // `tasks.every(isTerminalTaskStatus)`, and the input array still carries the
617
+ // `running`/`queued` statuses this function just cancelled — so the pre-11b.1
618
+ // call site passed a guard that could NEVER pass and the sweep was dead code
619
+ // on this path too. `repairedTasks` is exactly "what is left to run": all
620
+ // terminal when the whole run was repaired, still non-terminal (⇒ no sweep)
621
+ // on the individual-stale-subset branch, which is the behaviour we want.
622
+ sweepTerminalRunDescendants(manifest, repairedTasks);
505
623
  return repairedTasks;
506
624
  }
507
625
 
626
+ /**
627
+ * Wave 11b.2: persist "this run has been swept" so no later reconcile pass
628
+ * rescans /proc for it. `saveRunManifest` is the same helper every other writer
629
+ * in this module uses (sync by contract, atomic write, cache-repopulating), and
630
+ * the caller already holds the run lock on every path that reaches the sweep.
631
+ * The field is also set IN MEMORY so a caller that saves the manifest right
632
+ * after (updateRunStatus on the repair path) carries it along even when the
633
+ * write below fails. Never throws: the sweep already ran.
634
+ *
635
+ * 11b.4 (HIGH-2): `descendantSweepAt` records WHEN the last attempt ran, not
636
+ * that no work is left. When the attempt left a RETRYABLE skip behind (the same
637
+ * leak is still out there, we just could not signal it yet) `pending` is set so
638
+ * the startup lane re-selects the run after `TERMINAL_SWEEP_RETRY_MS` instead of
639
+ * never looking at it again — which is exactly how 11b.2/11b.3 could bury a real
640
+ * leak for good. A clean attempt clears the flag (and the field, so the manifest
641
+ * keeps its pre-11b.4 shape).
642
+ */
643
+ function stampDescendantSweep(manifest: TeamRunManifest, pending: boolean): void {
644
+ try {
645
+ manifest.descendantSweepAt = new Date().toISOString();
646
+ if (pending) manifest.descendantSweepPending = true;
647
+ else delete manifest.descendantSweepPending;
648
+ saveRunManifest(manifest);
649
+ } catch (stampError) {
650
+ // Warn, never throw: the sweep is done; the worst case is one redundant
651
+ // (idempotent) rescan on the next reconcile pass.
652
+ logInternalError("stale-reconciler.run-descendant-sweep", stampError, `marker write failed for ${manifest.runId}`, "warn");
653
+ }
654
+ }
655
+
656
+ /**
657
+ * Skip reasons that mean "the leak may still be there, try again later" (11b.4).
658
+ *
659
+ * The complement is what must NOT be retried: `dry-run` (report only — the
660
+ * caller returns before stamping at all), `already-swept` (this incarnation was
661
+ * already signalled; the idempotency registry remembers it) and `group-refused`
662
+ * reasons that are permanent for the run.
663
+ */
664
+ function isRetryableSweepSkip(reason: string): boolean {
665
+ return (
666
+ reason === "leader-alive-unmatched" || reason === "no-pgid" || reason === "interactive-tty" || reason.startsWith("group-refused:")
667
+ );
668
+ }
669
+
670
+ /** Recorded worker pids of a run's tasks (heartbeat / checkpoint), deduped. */
671
+ function collectRecordedWorkerPids(tasks: TeamTaskState[]): number[] {
672
+ const pids = new Set<number>();
673
+ for (const task of tasks) {
674
+ const pid = task.heartbeat?.pid ?? task.checkpoint?.childPid;
675
+ if (typeof pid === "number" && Number.isInteger(pid) && pid > 1) pids.add(pid);
676
+ }
677
+ return [...pids];
678
+ }
679
+
680
+ /**
681
+ * Wave 11b.3: pids the run's mux-surface controller recorded per task
682
+ * (`manifest.surface.workerPids`, `ManifestSurfaceState` in state/types.ts —
683
+ * verified field, never invented). A pane pid that is still alive is the pane
684
+ * the operator may be watching, so the sweep must not treat it as a descendant.
685
+ */
686
+ function collectSurfacePids(manifest: TeamRunManifest): number[] {
687
+ const pids = new Set<number>();
688
+ for (const pid of Object.values(manifest.surface?.workerPids ?? {})) {
689
+ if (typeof pid === "number" && Number.isInteger(pid) && pid > 1) pids.add(pid);
690
+ }
691
+ return [...pids];
692
+ }
693
+
694
+ /**
695
+ * Wave 11b (F7) + 11b.1 (reachability): run-scoped descendant sweep for a
696
+ * TERMINAL run. Two preconditions, in order:
697
+ *
698
+ * 1. TERMINAL only — a run that still has a live task may legitimately own
699
+ * live descendants; sweeping it would kill in-flight work. `repairStaleRun`
700
+ * passes its POST-repair array (the pre-repair one is never all-terminal,
701
+ * which is what made the guard unsatisfiable before 11b.1); the
702
+ * `result_exists` path passes the tasks checkResultFile just proved
703
+ * terminal.
704
+ * 2. NEVER inside a worker — a worker process carries `PI_CREW_KIND=subagent`
705
+ * and/or a non-zero crew depth, and every descendant it can see may belong
706
+ * to its OWN (still-running) run. Only the host session / detached runner
707
+ * may sweep, which is also why the kill switch is forwarded there.
708
+ *
709
+ * The pre-11b.1 third precondition ("every recorded ancestor must be dead") is
710
+ * GONE: it was all-or-nothing, so ONE surviving recorded worker aborted the
711
+ * sweep for the entire run and the leaked grandchildren — the incident this wave
712
+ * exists for — were never reaped. Descent is now proven PER CANDIDATE inside
713
+ * run-descendant-sweep.ts: a recorded ancestor (runner/worker) is never a
714
+ * candidate, and a `task-id` match additionally requires its recorded parent to
715
+ * be dead. A live recorded pid therefore no longer vetoes the sweep; it simply
716
+ * is not swept.
717
+ *
718
+ * Best-effort: never throws. When something was reaped the result is logged at
719
+ * `warn` (logInternalError has no `info` level; this mirrors the F2 reap above,
720
+ * whose success log is also `warn`), otherwise at `debug`. Escalations are
721
+ * registered with the existing drain owners.
722
+ *
723
+ * Wave 11b.2 — the SWEEP MARKER. A leak lives LONGER than its run (the incident
724
+ * run was `completed` a full day before its grandchildren died), so the sweep is
725
+ * not a run-lifecycle step at all: it has to be reachable from any later
726
+ * reconcile pass. `manifest.descendantSweepAt` is that reachability record —
727
+ * startup reconciliation (`reconcileAllStaleRuns`) enumerates TERMINAL runs that
728
+ * lack it, and this function stamps it after a REAL sweep so the /proc scan runs
729
+ * at most once per run. Stamped only when a sweep was actually attempted:
730
+ * - never on a dry-run report (nothing was signalled ⇒ not swept),
731
+ * - never when the kill switch (`PI_CREW_REAP_RUN_DESCENDANTS=0`) is off, so
732
+ * turning it back on still finds the run (surface-runtime forwarded kill
733
+ * switches must not silently burn the one-shot marker),
734
+ * - never when the sweep throws (the failure stays retryable next pass).
735
+ * The persist is warn-only: the sweep already happened, so a write error must
736
+ * not fail the reconcile pass — at worst the run is swept again next pass, which
737
+ * is idempotent (its descendants are gone by then).
738
+ *
739
+ * Wave 11b.4 (HIGH-2) — the marker is no longer one-shot. An attempt that could
740
+ * not SIGNAL anything it had proven (a live unmatched group leader, a candidate
741
+ * with no usable pgid, a refused group reap, an interactive tty) used to stamp
742
+ * the run as done anyway, so the very leak the sweep exists for was never looked
743
+ * at again. Now the stamp always records the attempt time, and
744
+ * `descendantSweepPending` marks the run for a bounded retry after
745
+ * `TERMINAL_SWEEP_RETRY_MS`.
746
+ */
747
+ export function sweepTerminalRunDescendants(
748
+ manifest: TeamRunManifest,
749
+ tasks: TeamTaskState[],
750
+ opts: { snapshot?: ProcSnapshot } = {},
751
+ ): void {
752
+ try {
753
+ if (!tasks.every((task) => isTerminalTaskStatus(task.status))) return;
754
+ if (getCrewEnv("PI_CREW_KIND") === "subagent") return;
755
+ if (currentCrewDepth() !== 0) return;
756
+ // Kill switch OFF ⇒ no scan, and NO marker: the run must stay eligible so
757
+ // that flipping the switch back on still reaps its leak.
758
+ if (!resolveRunDescendantSweepEnabled()) return;
759
+ const runnerPid = manifest.async?.pid;
760
+ const workerPids = collectRecordedWorkerPids(tasks);
761
+ // Wave 11c (battery 2026-10-24 F2): the pid that CREATED the run. A SYNC
762
+ // (foreground) run records no `async.pid`, so its leaked descendants carry
763
+ // `PI_CREW_PARENT_PID=<session pi pid>` — a pid no other field records, and
764
+ // the descent proof could never hold, so the leak was skipped forever.
765
+ // The sweep accepts it as a recorded PROOF parent only; the dead-only
766
+ // guard inside the sweep keeps a LIVE session from ever serving as proof
767
+ // (the operator may still be using that very pi — never reap under it).
768
+ const sessionPid = manifest.sessionPid;
769
+ // 11b.3: recorded surface pane pids — alive ones are excluded from the
770
+ // candidate set inside the sweep (safety rung 10), so a run that just went
771
+ // terminal cannot lose a pane the operator is still watching.
772
+ const surfacePids = collectSurfacePids(manifest);
773
+ const recordedPids = [
774
+ ...(sessionPid !== undefined ? [sessionPid] : []),
775
+ ...(runnerPid !== undefined ? [runnerPid] : []),
776
+ ...workerPids,
777
+ ];
778
+ // No recorded process ⇒ no marker can prove descent for the shapes this
779
+ // wave targets (the parent-pid / task-id branches both need a recorded
780
+ // parent), so skip the /proc scan entirely.
781
+ if (recordedPids.length === 0) return;
782
+ const startedAtMs = Date.parse(manifest.async?.spawnedAt ?? manifest.createdAt);
783
+ const result = reapRunDescendantsImpl(
784
+ {
785
+ runId: manifest.runId,
786
+ ...(runnerPid !== undefined ? { runnerPid } : {}),
787
+ workerPids,
788
+ ...(sessionPid !== undefined ? { sessionPid } : {}),
789
+ ...(surfacePids.length > 0 ? { surfacePids } : {}),
790
+ taskIds: tasks.map((task) => task.id),
791
+ startedAtMs: Number.isFinite(startedAtMs) ? startedAtMs : 0,
792
+ },
793
+ // 11b.4 (LOW-4): a caller sweeping several runs in one pass shares ONE
794
+ // /proc snapshot; a lone call builds its own inside the sweep.
795
+ opts.snapshot === undefined ? {} : { snapshot: opts.snapshot },
796
+ );
797
+ // Same owner as the F2 escalations above — one drain mechanism, never two.
798
+ for (const escalation of result.pending) trackReapEscalation(escalation);
799
+ // A dry-run pass signalled nothing (every candidate carries reason
800
+ // "dry-run"), so it must not consume the one-shot marker.
801
+ if (result.skipped.some((skip) => skip.reason === "dry-run")) return;
802
+ // 11b.4 (HIGH-2): the marker records the attempt; `pending` records that
803
+ // something PROVEN could not be signalled yet, so the startup lane retries.
804
+ stampDescendantSweep(
805
+ manifest,
806
+ result.skipped.some((skip) => isRetryableSweepSkip(skip.reason)),
807
+ );
808
+ if (result.reaped.length > 0) {
809
+ logInternalError(
810
+ "stale-reconciler.run-descendant-sweep",
811
+ new Error(
812
+ `swept ${result.reaped.length} descendant process(es) of terminal run: ${result.reaped.map((descendant) => `${descendant.pid}(${descendant.matchedBy})`).join(",")}`,
813
+ ),
814
+ manifest.runId,
815
+ "warn",
816
+ );
817
+ } else if (result.skipped.length > 0) {
818
+ logInternalError(
819
+ "stale-reconciler.run-descendant-sweep",
820
+ new Error(`no descendant reaped (${result.skipped.length} skipped)`),
821
+ manifest.runId,
822
+ "debug",
823
+ );
824
+ }
825
+ } catch (sweepError) {
826
+ // The sweep is cleanup, never a reason to fail a reconcile pass.
827
+ logInternalError("stale-reconciler.run-descendant-sweep", sweepError, manifest.runId, "debug");
828
+ }
829
+ }
830
+
508
831
  /**
509
832
  * Three-phase stale run reconciliation.
510
833
  *
@@ -588,6 +911,18 @@ export function reconcileStaleRun(manifest: TeamRunManifest, tasks: TeamTaskStat
588
911
  // NOTE (OPT-02): Remains sync — reconcileStaleRun is called from withRunLockSync in
589
912
  // crash-recovery.ts and from sync reconcileOrphanedTempWorkspaces; cannot await here.
590
913
  saveRunManifest(manifest);
914
+ // Wave 11b.1 (F7 reachability): the sweep MUST run on THIS path too.
915
+ // `result_exists` is the most common way a run becomes terminal, and the
916
+ // only pre-11b.1 caller was `repairStaleRun` — unreachable once a result
917
+ // file exists. That is exactly why the leak behind
918
+ // team_20261022110510 (6 grandchildren, 399 MB, 24 h) survived a run that
919
+ // was `completed` while the sweep's unit tests were green: a leak lives
920
+ // LONGER than its run, so "this run is finished" is the reason to sweep,
921
+ // never the reason to skip. Every task is terminal here (checkResultFile
922
+ // just proved it); the helper keeps its own guards (all tasks terminal,
923
+ // never inside a worker, kill switch) and this branch RETURNS, so the
924
+ // sweep still runs at most once per run per reconcile pass.
925
+ sweepTerminalRunDescendants(manifest, tasks);
591
926
  return {
592
927
  runId,
593
928
  verdict: "result_exists",
@@ -1002,6 +1337,12 @@ export function reconcileOrphanedTempWorkspaces(
1002
1337
  } catch {
1003
1338
  /* skip if tmpdir unreadable */
1004
1339
  }
1340
+ // Wave 11a review (MAJOR #3): this scan repairs runs too, so it can arm the
1341
+ // same group escalation. The function is sync (its callers are sync configurers
1342
+ // and 5+ tests), so the best it can do is START the drain — the escalations are
1343
+ // already registered with the module and any later `drainReapEscalations()`
1344
+ // caller awaits them. The promise never rejects (allSettled + catch inside).
1345
+ void drainReapEscalations();
1005
1346
  return { repaired, cleanedDirs };
1006
1347
  }
1007
1348
 
@@ -68,6 +68,7 @@ import { readEnabledModelsPatterns } from "../model/model-scope.ts";
68
68
  import { type ParsedPiJsonOutput, parsePiJsonOutput } from "../output/pi-json-output.ts";
69
69
  import { type ProgressEventSummary, shouldAppendProgressEventUpdate } from "../output/progress-event-coalescer.ts";
70
70
  import { buildSyntheticTerminalEvidence, cancellationReasonFromSignal } from "../process/cancellation.ts";
71
+ import { readProcIdentity } from "../process/reap-group.ts";
71
72
  import { checkProcessLiveness } from "../process-status.ts";
72
73
  import { DEFAULT_RETRY_POLICY } from "../recovery/retry-executor.ts";
73
74
  import { runWorker } from "../run-worker.ts";
@@ -866,7 +867,10 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
866
867
  } catch (err) {
867
868
  logInternalError("task-runner.ownership-write", err, `pid=${pid}, taskId=${task.id}`);
868
869
  }
869
- ({ task, tasks } = checkpointTask(manifest, tasks, task, "child-spawned", pid));
870
+ // Wave 11a (F3): persist the spawn-time identity alongside the pid so a
871
+ // later reaper can prove the process (starttime/pgid/uid) instead of
872
+ // guessing from a pid that may already have been recycled.
873
+ ({ task, tasks } = checkpointTask(manifest, tasks, task, "child-spawned", pid, readProcIdentity(pid)));
870
874
  // Security review (T1/WP-1 round 1, security-1 replay fix): scope the
871
875
  // steering file to ONE worker incarnation on EVERY spawn. Steering
872
876
  // lines are id-less; each new worker process reads the file from
@@ -1035,10 +1039,17 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
1035
1039
  // Surface pid was previously never persisted into task state - the
1036
1040
  // heartbeat-watcher PID-liveness gate (alive -> downgrade dead->stale)
1037
1041
  // had nothing to check. Pin it once from the worker.started self-report.
1042
+ // Wave 11a (F3): also pin the spawn-time identity (pgid/starttime/uid)
1043
+ // so the orphan reaper can prove this pid later; a /proc read that
1044
+ // races the exit simply contributes nothing (all fields optional).
1045
+ const surfaceIdentity = readProcIdentity(pidFromStart);
1038
1046
  task = {
1039
1047
  ...task,
1040
1048
  heartbeat: touchWorkerHeartbeat(task.heartbeat ?? createWorkerHeartbeat(task.id), {
1041
1049
  pid: pidFromStart,
1050
+ ...(surfaceIdentity?.pgid !== undefined ? { pgid: surfaceIdentity.pgid } : {}),
1051
+ ...(surfaceIdentity?.starttime !== undefined ? { starttime: surfaceIdentity.starttime } : {}),
1052
+ ...(surfaceIdentity?.uid !== undefined ? { uid: surfaceIdentity.uid } : {}),
1042
1053
  }),
1043
1054
  };
1044
1055
  }
@@ -7,6 +7,7 @@ import { loadRunManifestById, saveRunTasksCoalesced } from "../../state/stores/s
7
7
  import type { TaskCheckpointState, TeamRunManifest, TeamTaskState } from "../../state/types.ts";
8
8
  import { logInternalError } from "../../utils/internal-error.ts";
9
9
  import { recordFromTask, upsertCrewAgent } from "../crew-agent-records.ts";
10
+ import type { ProcIdentity } from "../process/reap-group.ts";
10
11
 
11
12
  export function updateTask(tasks: TeamTaskState[], updated: TeamTaskState): TeamTaskState[] {
12
13
  return tasks.map((task) => (task.id === updated.id ? updated : task));
@@ -283,11 +284,37 @@ export function checkpointTask(
283
284
  task: TeamTaskState,
284
285
  phase: TaskCheckpointState["phase"],
285
286
  childPid?: number,
287
+ identity?: ProcIdentity,
286
288
  ): { task: TeamTaskState; tasks: TeamTaskState[] } {
289
+ // Wave 11a review (MAJOR #4): CARRY the spawn-time identity over exactly like
290
+ // `childPid`. This function is also called on phase transitions that know
291
+ // nothing about the process ("child-stdout-final", "artifact-written"); those
292
+ // calls used to REBUILD the checkpoint without the three identity fields and
293
+ // silently erase them, so a later reaper lost the pid proof F3 exists to
294
+ // provide. Only a DEFINED argument overrides the carried value, so a caller
295
+ // that passes nothing keeps the pre-wave record shape byte-for-byte.
296
+ const previous = task.checkpoint;
297
+ const carriedPid = childPid ?? previous?.childPid;
298
+ // MINOR #6 (review): the identity belongs to ONE pid. Carrying it over a
299
+ // CHANGE of `childPid` (retry / re-spawn) would pair a fresh pid with the old
300
+ // incarnation's starttime — the reaper's reuse-proof field would then be a
301
+ // lie that REFUSES the real child (or, worse, matches an unrelated pid). So
302
+ // carry identity only while the child stays the same one.
303
+ const sameChild = childPid === undefined || childPid === previous?.childPid;
304
+ const carriedPgid = identity?.pgid ?? (sameChild ? previous?.childPgid : undefined);
305
+ const carriedStartTime = identity?.starttime ?? (sameChild ? previous?.childStartTime : undefined);
306
+ const carriedUid = identity?.uid ?? (sameChild ? previous?.childUid : undefined);
287
307
  const checkpoint: TaskCheckpointState = {
288
308
  phase,
289
309
  updatedAt: new Date().toISOString(),
290
- ...(childPid ? { childPid } : task.checkpoint?.childPid ? { childPid: task.checkpoint.childPid } : {}),
310
+ ...(carriedPid !== undefined ? { childPid: carriedPid } : {}),
311
+ // Wave 11a (F3): persist the spawn-time identity so a later reaper can
312
+ // PROVE the pid (starttime) instead of guessing. Only defined fields are
313
+ // spread, so a caller that passes nothing keeps the pre-wave record shape
314
+ // and old records/state files keep loading unchanged.
315
+ ...(carriedPgid !== undefined ? { childPgid: carriedPgid } : {}),
316
+ ...(carriedStartTime !== undefined ? { childStartTime: carriedStartTime } : {}),
317
+ ...(carriedUid !== undefined ? { childUid: carriedUid } : {}),
291
318
  };
292
319
  const nextTask = { ...task, checkpoint };
293
320
  const nextTasks = persistSingleTaskUpdate(manifest, updateTask(tasks, nextTask), nextTask);
@@ -475,6 +475,14 @@ export function createRunManifest(params: {
475
475
  tasksPath: paths.tasksPath,
476
476
  eventsPath: paths.eventsPath,
477
477
  artifacts: [],
478
+ // Wave 11c (battery F2): record the SPAWNING process pid at creation — for
479
+ // a SYNC/foreground run (no async runner ever spawned) this is the only
480
+ // recorded parent its leaked descendants can name in PI_CREW_PARENT_PID,
481
+ // and without it the descendant sweep could never prove their descent.
482
+ // Sync-vs-async is NOT known at creation time (the async dispatch happens
483
+ // after this), so every run records it; async runs keep using
484
+ // `manifest.async.pid` as their primary recorded runner.
485
+ sessionPid: process.pid,
478
486
  ...(params.ownerSessionId ? { ownerSessionId: params.ownerSessionId } : {}),
479
487
  runKind: params.runKind ?? "team-run",
480
488
  ...(params.args !== undefined ? { args: params.args } : {}),
@@ -368,6 +368,18 @@ export interface TeamRunManifest {
368
368
  eventsPath: string;
369
369
  artifacts: ArtifactDescriptor[];
370
370
  async?: AsyncRunState;
371
+ /** Wave 11c (battery 2026-10-24 F2): pid of the process that CREATED the
372
+ * run — `process.pid` at createRunManifest time (the session pi for a
373
+ * foreground team-tool dispatch; the detached runner itself when it
374
+ * creates sub-runs). A SYNC (foreground) run records no `async.pid`, so a
375
+ * leaked descendant carrying `PI_CREW_PARENT_PID=<session pi pid>` could
376
+ * never satisfy the run-descendant sweep's descent proof and was skipped
377
+ * forever; `sweepTerminalRunDescendants` now accepts this pid as a
378
+ * recorded proof parent (DEAD-only, like every other proof parent — a
379
+ * live session is never proof, the operator may still be using it).
380
+ * Purely additive/optional: absent on every manifest written before 11c,
381
+ * and async runs keep `async.pid` as their primary recorded runner. */
382
+ sessionPid?: number;
371
383
  /** @deprecated-plan-pointer T2/R4 (ADR-4 §2): dual-write era — PlanRecord at
372
384
  * `<stateRoot>/plans/plans.json` is authoritative; this field is kept (never
373
385
  * dropped) as the pre-v2 fallback and legacy UI surface. Deprecate-only. */
@@ -437,6 +449,26 @@ export interface TeamRunManifest {
437
449
  /** MuxSurface A1 (spec §8.3): pane/pid/lockout state of this run. Absent on
438
450
  * runs that never booted a surface worker and on older manifests. */
439
451
  surface?: ManifestSurfaceState;
452
+ /** Wave 11b.2 (F7 reachability): ISO timestamp of the LAST completed
453
+ * run-descendant sweep for this run (`sweepTerminalRunDescendants`).
454
+ * Purely additive/optional — absent on every manifest written before
455
+ * 11b.2, and absent means "never swept" (⇒ still eligible for the
456
+ * startup sweep). Startup reconciliation enumerates terminal runs that
457
+ * lack it, because a leak OUTLIVES its run: the incident run was
458
+ * `completed` the day before its 6 grandchildren were finally reaped.
459
+ * Stamped only after a REAL sweep (never on dry-run, kill switch off, or
460
+ * a throwing sweep) so a failed attempt is retried on the next pass. */
461
+ descendantSweepAt?: string;
462
+ /** Wave 11b.4 (HIGH-2): the last sweep attempt PROVED descendants it could
463
+ * not signal (a live unmatched group leader, a candidate with no usable
464
+ * pgid, a refused group reap, an interactive tty). Without this the run
465
+ * would be stamped as done and the leak never looked at again — the exact
466
+ * failure mode 11b.2 introduced. Purely additive/optional: absent (or
467
+ * `false`) means "nothing left to retry". Startup reconciliation re-selects
468
+ * a pending run once `TERMINAL_SWEEP_RETRY_MS` has elapsed since
469
+ * `descendantSweepAt`, so a permanently un-signallable leak does not burn a
470
+ * /proc scan on every pass. */
471
+ descendantSweepPending?: boolean;
440
472
  }
441
473
 
442
474
  export interface RunModelContext {
@@ -621,6 +653,16 @@ export interface TaskCheckpointState {
621
653
  phase: "started" | "child-spawned" | "child-stdout-final" | "artifact-written";
622
654
  updatedAt: string;
623
655
  childPid?: number;
656
+ /**
657
+ * Wave 11a (F3): spawn-time process identity of `childPid`, written from
658
+ * `readProcIdentity(pid)` right after the spawn. `starttime` is the
659
+ * PID-reuse-proof field and `pgid` is only meaningful when the child leads
660
+ * its own group (detached spawn). All three are OPTIONAL: records written
661
+ * before this wave keep loading unchanged.
662
+ */
663
+ childPgid?: number;
664
+ childStartTime?: string;
665
+ childUid?: number;
624
666
  }
625
667
 
626
668
  export interface TaskAttemptState {