session-orchestrator 5.2.0 → 5.4.0

package/scripts/lib/project-hygiene.mjs

@@ -462,7 +462,9 @@ export function checkStaleArtifacts(repoRoot, ageDays = DEFAULT_ARTIFACT_AGE_DAY
  * H4 — CI configuration hygiene.
  *
  * Measured 5/6. Checks locally declared executable commands without evaluating
- * pipeline conditions or external includes. The dependency-audit gap is the load-bearing one: three
+ * pipeline conditions or external includes; when no local audit step is found
+ * but `.gitlab-ci.yml` declares `include:`, the finding is downgraded to an
+ * unverifiable, non-fixable one. The dependency-audit gap is the load-bearing one: three
  * of the tested repos carried known-vulnerable dependencies that no pipeline
  * would ever surface.
  *
@@ -506,6 +508,24 @@ export function checkCiConfig(repoRoot) {
   }
 
   if (ciConfigs.some(({ text }) => text) && !ciConfigs.some(hasAuditStep)) {
+    // Jobs supplied via include: (component/local/project/remote/template) are
+    // never resolved, so an audit may exist there — report it as unverifiable.
+    let includes = 0;
+    try {
+      const gitlabText = hasGitlab ? ciConfigs[0].text : '';
+      const include = yaml.loadAll(gitlabText, undefined, { schema: GITLAB_SCHEMA }).filter(isConfigMap).at(-1)?.include;
+      includes = Array.isArray(include) ? include.length : include ? 1 : 0;
+    } catch {
+      /* unparseable config — keep the plain missing-audit finding */
+    }
+    if (includes > 0) {
+      findings.push({
+        check: 'ci-audit-job',
+        fixable: false,
+        message: `CI dependency-audit step could not be verified: ${includes} include(s) not evaluated (.gitlab-ci.yml include:)`,
+      });
+      return findings;
+    }
     findings.push({
       check: 'ci-audit-job',
       fixable: true,

package/scripts/lib/quality-gate.mjs

@@ -26,7 +26,9 @@
  *   - Output collection: each gate captures the last ~50 lines of combined
  *     stdout+stderr (vs. `gate-helpers.mjs::runCheck` which truncates to 5).
  *     The longer tail flows into the diagnostics bundle and the fixer's
- *     failureContext.
+ *     failureContext. Since #1427 the tail is cut by `process-group.mjs`'s own
+ *     `OUTPUT_TAIL_LINES`, which is the same 50, and the full capture is
+ *     byte-capped there instead of by `spawnSync`'s `maxBuffer`.
  *
  *   - `last-green-sha.txt` lives at `.orchestrator/runtime/last-green-sha.txt`
  *     and is updated atomically after every successful gate. `changedFiles` is
@@ -61,12 +63,17 @@
 
 import { spawnSync } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync } from 'node:fs';
-import { dirname, join } from 'node:path';
+import { dirname, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { emitEvent, sessionAttribution } from './events.mjs';
 import { parsePorcelainZ } from './git-porcelain.mjs';
 import { admitSuiteCounts, extractTestCounts } from './gates/gate-helpers.mjs';
+import {
+  DEFAULT_GATE_TIMEOUT_MS,
+  buildCommandSignature,
+  spawnInGroup,
+} from './process-group.mjs';
 import { redactDiagnosticsBundle } from './quality-gate/diagnostics.mjs';
 import { readProcessLocalSessionIds } from './session-identity/own-session.mjs';
 
@@ -86,17 +93,20 @@ const DEFAULT_COMMANDS = {
   test: 'npm test',
 };
 
-/** Max lines of combined stdout+stderr retained per failure. */
-const OUTPUT_TAIL_LINES = 50;
-
 /** Max corrective_context entries forwarded to the fixer (most-recent). */
 const CORRECTIVE_CONTEXT_TAIL = 5;
 
 /** Hard ceiling on retries — defensive coercion. */
 const MAX_RETRIES_HARD_CAP = 10;
 
-/** Per-gate command timeout (15 min). Wave gates can be long; never infinite. */
-const GATE_TIMEOUT_MS = 15 * 60 * 1000;
+/**
+ * Per-gate command timeout (15 min). Wave gates can be long; never infinite.
+ *
+ * Re-exported from `process-group.mjs` rather than re-spelled here: both gate
+ * paths are allowed exactly as long (PRD parameter `gate.timeout-path-b-ms`),
+ * and a second literal would drift from it silently.
+ */
+const GATE_TIMEOUT_MS = DEFAULT_GATE_TIMEOUT_MS;
 
 // ---------------------------------------------------------------------------
 // Internal helpers
@@ -110,7 +120,8 @@ const GATE_TIMEOUT_MS = 15 * 60 * 1000;
  * @returns {string}
  */
 function resolveRepoRoot(explicit) {
-  if (typeof explicit === 'string' && explicit.trim()) return explicit;
+  // Absolute on purpose: emitEvent refuses a relative root (#1468).
+  if (typeof explicit === 'string' && explicit.trim()) return resolve(explicit);
   return process.cwd();
 }
 
@@ -274,29 +285,48 @@ function resolveCommands(override, repoRoot) {
 }
 
 /**
- * Run a shell command, capture stdout+stderr, return last ~50 lines plus exit code.
- *
- * Does NOT throw — failures are encoded in the return value. Honours
- * GATE_TIMEOUT_MS as a hard ceiling.
+ * Run a gate command as the leader of its OWN process group, capture
+ * stdout+stderr, return the last ~50 lines plus the exit code.
+ *
+ * ASYNC since #1427 A1/A2, and that is the point rather than an incidental
+ * refactor. The previous `spawnSync(cmd, { shell: true, timeout })` made the
+ * SHELL the child, so Node's timeout signalled the shell alone and every
+ * grandchild — `tsgo --noEmit`, Vitest workers — was reparented to PID 1 and
+ * kept running. That is the 2026-09-20 incident: four orphaned `tsgo`
+ * processes, two at PPID 1, up to 8.0 GB RSS each, host at 13% free memory.
+ * {@link spawnInGroup} spawns `detached: true` (the child becomes a process-GROUP
+ * leader) and signals `-pgid` along the SIGTERM → grace → SIGKILL → verify
+ * ladder, so the whole group is gone when this resolves — or reported as a
+ * survivor, never booked as success (PRD B6).
+ *
+ * Does NOT throw — failures are encoded in the return value. The three fields
+ * the pre-#1427 contract carried (`exitCode`, `output`, `timedOut`) are
+ * unchanged; `overflow`, `killSignals`, `survivors` and `pgid` are additive.
+ *
+ * Exit-code contract, preserved verbatim from the synchronous path:
+ *   - timeout → `124`
+ *   - output over the byte cap → `1` (what `spawnSync`'s ENOBUFS produced)
+ *   - otherwise the child's own code.
  *
  * @param {string} cmd
  * @param {string} cwd
- * @returns {{ exitCode: number, output: string, timedOut: boolean }}
+ * @param {object} [opts]
+ * @param {number} [opts.timeoutMs]   — wall-clock ceiling. Defaults to GATE_TIMEOUT_MS.
+ * @param {string|null} [opts.sessionId] — owner recorded in the gate-process ledger.
+ * @param {object} [opts.seams]       — forwarded verbatim to {@link spawnInGroup}
+ *   (`spawnFn`, `killFn`, `isAliveFn`, `sleepFn`, `killGraceMs`, `verifyWaitMs`).
+ *   Test seam only; production passes nothing and gets the module defaults.
+ * @returns {Promise<{ exitCode: number, output: string, timedOut: boolean,
+ *   overflow: boolean, killSignals: string[], survivors: number[], pgid: number }>}
  */
-function runGate(cmd, cwd) {
+async function runGate(cmd, cwd, opts = {}) {
   try {
     // Gate commands are executable configuration from the caller or local
     // Session Config, not data interpolated into a command template. Shell
     // syntax is intentional; callers must trust both command sources (including
     // uncommitted config). See security.md: Session Config Command Trust.
-    // nosemgrep: unsafe-shell-spawn
-    const result = spawnSync(cmd, {
+    const result = await spawnInGroup(cmd, {
       cwd,
-      shell: true,
-      encoding: 'utf8',
-      stdio: ['ignore', 'pipe', 'pipe'],
-      timeout: GATE_TIMEOUT_MS,
-      maxBuffer: 16 * 1024 * 1024, // 16 MiB cap
       // #1360: gate runs are where worker over-subscription was measured to
       // cause timeout failures (integration fixtures spawn their own Node/npm/
       // git children on top of Vitest's workers). `vitest.config.mjs` reads this
@@ -305,19 +335,40 @@ function runGate(cmd, cwd) {
       // Inherited by every gate subprocess, not only the test one — typecheck
       // and lint ignore it, so setting it unconditionally costs nothing.
       env: { ...process.env, SO_BOUNDED_WORKERS: '1' },
+      timeoutMs: typeof opts.timeoutMs === 'number' ? opts.timeoutMs : GATE_TIMEOUT_MS,
+      // The ledger (A4) is what lets the orphan-reaper tell a descendant of OUR
+      // gate from a foreign process before it signals anything.
+      repoRoot: cwd,
+      commandSignature: buildCommandSignature(cmd),
+      sessionId: opts.sessionId ?? null,
+      ...(opts.seams ?? {}),
     });
-    const combined = (result.stdout ?? '') + (result.stderr ?? '');
-    const tail = combined.split('\n').slice(-OUTPUT_TAIL_LINES).join('\n').trim();
-    const timedOut = result.signal === 'SIGTERM' && result.error?.code === 'ETIMEDOUT';
-    const exitCode = typeof result.status === 'number'
-      ? result.status
-      : (timedOut ? 124 : 1);
-    return { exitCode, output: tail, timedOut };
+
+    // A process that outlived SIGKILL (one that `setsid`-ed out of the group) is
+    // SAID OUT LOUD on the channel the operator and the diagnostics bundle both
+    // read. Silence here would be the exact failure the ladder exists to expose.
+    const output = result.survivors.length > 0
+      ? `${result.output}\ngate: ${result.survivors.length} process(es) survived SIGKILL: ${result.survivors.join(', ')}`.trim()
+      : result.output;
+
+    return {
+      exitCode: result.exitCode,
+      output,
+      timedOut: result.timedOut,
+      overflow: result.overflow,
+      killSignals: result.killSignals,
+      survivors: result.survivors,
+      pgid: result.pgid,
+    };
   } catch (err) {
     return {
       exitCode: 1,
       output: `quality-gate: failed to spawn command "${cmd}": ${err?.message ?? String(err)}`,
       timedOut: false,
+      overflow: false,
+      killSignals: [],
+      survivors: [],
+      pgid: -1,
     };
   }
 }
@@ -723,13 +774,44 @@ function coerceMaxRetries(n) {
 // Public API
 // ---------------------------------------------------------------------------
 
+/**
+ * The kill-ladder fields of a `quality_gate.passed/.failed` record (#1439).
+ *
+ * ONE builder for BOTH emitters — this library's {@link emitGateEvent} and the
+ * `scripts/run-quality-gate.mjs` CLI — so the two can never publish different
+ * shapes under the same keys. From THIS builder all three keys are always
+ * present: `timed_out` `false`, `survivors` `0` and `kill_signals` `[]` are
+ * measurements ("not killed"), unlike `counts`, whose absence means "not
+ * measured". The CLI then passes the result through `joinGateKillFields()`
+ * (`scripts/lib/gates/gate-helpers.mjs`, #1457), which OR-joins the inner
+ * per-command ladders and MAY omit the three keys when that inner side was
+ * not measured and the outer ladder did not fire.
+ *
+ * - `timed_out` — the run hit its wall-clock ceiling and was killed (exit 124).
+ * - `survivors` — how many pids were still alive after the SIGKILL step; the
+ *   pids themselves stay in the gate output / diagnostics bundle.
+ * - `kill_signals` — the signals actually sent, in order (`SIGTERM`, `SIGKILL`).
+ *   Copied, because `spawnInGroup` records them at send time into a live array.
+ *
+ * @param {{timedOut?: boolean, killSignals?: string[], survivors?: number[]}|null|undefined} run
+ * @returns {{timed_out: boolean, survivors: number, kill_signals: string[]}}
+ */
+export function gateKillFields(run) {
+  return {
+    timed_out: run?.timedOut === true,
+    survivors: Array.isArray(run?.survivors) ? run.survivors.length : 0,
+    kill_signals: Array.isArray(run?.killSignals) ? [...run.killSignals] : [],
+  };
+}
+
 /**
  * Emit exactly one `orchestrator.quality_gate.{passed,failed}` event per
  * `runQualityGateWithRetry` CALL (#928b).
  *
  * NO DOUBLE-COUNTING. The two gate paths never nest:
  *   - `scripts/run-quality-gate.mjs` spawns `scripts/lib/gates/gate-*.mjs`
- *     directly and does not import this module;
+ *     directly; it imports only the pure `gateKillFields` helper from this
+ *     module, never `runQualityGateWithRetry`, so it never reaches this emitter;
  *   - this module spawns the resolved gate COMMANDS (and `parse-config.mjs`)
  *     directly and does not invoke that wrapper.
  * A single run therefore passes through exactly one emitter.
@@ -783,7 +865,7 @@ function coerceMaxRetries(n) {
  * wrote it.
  *
  * Extraction margin: {@link suiteCountsFromOutput} sees only the
- * `OUTPUT_TAIL_LINES` (50) tail `runCheck` retains. Measured on `npm test`
+ * 50-line tail `process-group.mjs` retains (`OUTPUT_TAIL_LINES` there). Measured on `npm test`
  * (vitest 2026-07-31), the `Tests` summary line sits 5 lines from the end — 45
  * lines of headroom. A runner epilogue longer than that (coverage table, long
  * unhandled-error dump) pushes the summary out of the window; `counts` is then
@@ -796,8 +878,13 @@ function coerceMaxRetries(n) {
  * @param {number} attempts
  * @param {string|null} gate
  * @param {{passed: number, failed: number, total: number}|null} [counts]
+ * @param {{timedOut?: boolean, killSignals?: string[], survivors?: number[]}|null} [killRun]
+ *   the run that decided the verdict — the last attempt's failing gate; `null`
+ *   on the passing path, where every gate exited 0 and so none was killed
+ *   (`spawnInGroup` only starts its kill ladder on timeout or overflow, and
+ *   both force a non-zero exit code).
  */
-async function emitGateEvent(repoRoot, ok, attempts, gate, counts) {
+async function emitGateEvent(repoRoot, ok, attempts, gate, counts, killRun = null) {
   try {
     await emitEvent(
       `orchestrator.quality_gate.${ok ? 'passed' : 'failed'}`,
@@ -807,6 +894,7 @@ async function emitGateEvent(repoRoot, ok, attempts, gate, counts) {
         attempts,
         ...(gate ? { gate } : {}),
         ...(counts ? { counts } : {}),
+        ...gateKillFields(killRun),
         ...sessionAttribution(repoRoot),
       },
       { repoRoot },
@@ -833,6 +921,14 @@ async function emitGateEvent(repoRoot, ok, attempts, gate, counts) {
  * @param {string} [opts.repoRoot]        — defaults to process.cwd().
  * @param {{lint?: string, typecheck?: string, test?: string}} [opts.commands]
  *                                          — override individual gate commands.
+ * @param {number} [opts.gateTimeoutMs]   — per-gate wall-clock ceiling. Defaults to
+ *                                          GATE_TIMEOUT_MS (15 min). On expiry the gate's
+ *                                          whole process GROUP gets SIGTERM → grace →
+ *                                          SIGKILL and the gate reports exit 124.
+ * @param {object} [opts._processSeams]   — TEST SEAM, forwarded verbatim to
+ *                                          `spawnInGroup` (`spawnFn`, `killFn`,
+ *                                          `isAliveFn`, `sleepFn`, `killGraceMs`,
+ *                                          `verifyWaitMs`). Production passes nothing.
  *
  * @returns {Promise<{
  *   ok: boolean,
@@ -851,6 +947,24 @@ export async function runQualityGateWithRetry(opts) {
     : async () => {};
   const repoRoot = resolveRepoRoot(safeOpts.repoRoot);
   const commands = resolveCommands(safeOpts.commands, repoRoot);
+  /**
+   * Per-gate spawn options, resolved ONCE per call.
+   *
+   * `sessionId` comes from the PROCESS-LOCAL witness only
+   * ({@link readOwnSessionIds} → `CLAUDE_CODE_SESSION_ID`), never from
+   * `.orchestrator/session.lock`: the lock is a repo-GLOBAL artefact any
+   * session in this working copy may hold, so reading it here would stamp a
+   * PEER's id onto our own gate processes and point the reaper at the wrong
+   * owner (`.claude/rules/identity-and-locks.md` — rank witnesses, never union
+   * them). No witness → `null`, which the ledger records as "owner unknown".
+   */
+  const gateOpts = {
+    timeoutMs: typeof safeOpts.gateTimeoutMs === 'number' ? safeOpts.gateTimeoutMs : undefined,
+    sessionId: [...readOwnSessionIds()][0] ?? null,
+    seams: (typeof safeOpts._processSeams === 'object' && safeOpts._processSeams !== null)
+      ? safeOpts._processSeams
+      : undefined,
+  };
 
   // Accumulate per-attempt failure info for the diagnostics bundle.
   const allFailures = [];
@@ -889,7 +1003,10 @@ export async function runQualityGateWithRetry(opts) {
 
     for (const gate of GATE_ORDER) {
       const cmd = commands[gate];
-      const result = runGate(cmd, repoRoot);
+      // `await` inside `for…of` is deliberate: the gates are fail-fast and
+      // SEQUENTIAL (lint → typecheck → test), exactly as the synchronous path
+      // ran them. Parallelising them would change the contract, not the plumbing.
+      const result = await runGate(cmd, repoRoot, gateOpts);
       if (gate === 'test') {
         // The numbers are in hand right here — capture them at the seam rather
         // than letting them travel as prose (#954).
@@ -906,6 +1023,8 @@ export async function runQualityGateWithRetry(opts) {
         exitCode: result.exitCode,
         output: result.output,
         timedOut: result.timedOut,
+        killSignals: result.killSignals,
+        survivors: result.survivors,
         command: cmd,
         attempt,
       };
@@ -984,7 +1103,7 @@ export async function runQualityGateWithRetry(opts) {
     `❌ quality-gate exhausted retries (${attempt}), writing diagnostics to ${bundlePath ?? '<unwritable>'}\n`,
   );
 
-  await emitGateEvent(repoRoot, false, attempt, lastFailure?.gate ?? null, testCounts);
+  await emitGateEvent(repoRoot, false, attempt, lastFailure?.gate ?? null, testCounts, lastFailure);
 
   const out = {
     ok: false,