start-command 0.35.0 → 0.35.2

package/src/lib/execution-control.js

@@ -302,6 +302,23 @@ function restoreStopRequested(store, record, previous) {
   }
 }
 
+/**
+ * Whether a failed docker control command hit a container that has already
+ * exited while launch-time recovery is configured: the stop marker alone then
+ * cancels the resume the watcher may still be waiting to make.
+ * @param {object} record - Execution record
+ * @param {object} result - Failed control command result
+ * @returns {boolean} Whether the stop marker should stand
+ */
+function isPendingRecoveryStop(record, result) {
+  const options = record.options || {};
+  if (options.isolated !== 'docker' || !options.onKillResume) {
+    return false;
+  }
+  const detail = `${result.stderr || ''} ${result.error || ''}`;
+  return /is not running/i.test(detail);
+}
+
 function controlExecution(store, identifier, action, runner = runCommand) {
   if (!store) {
     return { success: false, error: 'Execution tracking is disabled.' };
@@ -325,6 +342,25 @@ function controlExecution(store, identifier, action, runner = runCommand) {
   // which exits 137 just like the OOM killer.
   const previousStopRequestedAt = markStopRequested(store, record);
   const result = runner(control.command, control.args);
+  if (!result.success && isPendingRecoveryStop(record, result)) {
+    // The container already exited and its watcher is waiting out
+    // `--on-kill-resume-delay` (issue #181): `docker kill` has nothing to
+    // signal, but the marker it left cancels the pending resume.
+    return {
+      success: true,
+      output: formatControlResultAsLinksNotation({
+        action,
+        identifier,
+        uuid: record.uuid,
+        status: 'recovery-cancelled',
+        backend: record.options.isolated,
+        sessionName: record.options.sessionName,
+        method: control.method,
+        processIds: null,
+        message: `Cancelled the pending recovery of detached docker container: ${record.options.sessionName}`,
+      }),
+    };
+  }
   if (!result.success) {
     restoreStopRequested(store, record, previousStopRequestedAt);
     const detail =

package/src/lib/execution-recovery.js

@@ -2,11 +2,12 @@
  * Launch-time recovery for killed detached docker sessions (issue #176).
  *
  * `$ --isolated docker --detached --on-kill-resume 3 --recovery-command B -- A`
- * runs `A`; when the main process is killed (exit 137 / `OOMKilled`), the
- * detached completion watcher hands the inspected facts to this module, which
- * restarts the *same* container so `B` continues on the same filesystem, with
- * the same resource limits (they live in the container's HostConfig), under
- * the same execution UUID and appending to the same log file.
+ * runs `A`; when the main process is killed (exit 137, or `OOMKilled` with no
+ * exit status of its own), the detached completion watcher hands the inspected
+ * facts to this module, which restarts the *same* container so `B` continues
+ * on the same filesystem, with the same resource limits (they live in the
+ * container's HostConfig), under the same execution UUID and appending to the
+ * same log file.
  *
  * How `B` replaces `A` inside the same container: a container launched with a
  * recovery command runs a tiny selector as its entrypoint command. The
@@ -38,6 +39,17 @@ const {
   readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { shellQuote } = require('./isolation-log-utils');
+const {
+  CGROUP_SHELL_VARS,
+  formatCgroupMemoryLogLine,
+  parseCgroupMemorySample,
+} = require('./cgroup-memory');
+const {
+  formatRecoveryDelay,
+  getOnKillResumeDelay,
+  pickRecoveryDelayMs,
+  waitForRecoveryDelay,
+} = require('./recovery-delay');
 
 /** File whose presence makes the selector run the recovery command. */
 const RECOVERY_MARKER_PATH = '/.start-command-recovery';
@@ -81,37 +93,51 @@ function buildRecoverySelectorArgs(mainArgs, params) {
 }
 
 /**
+ * Whether the main process was killed.
+ *
+ * Exit 137 (SIGKILL, the OOM killer's signal) always counts. `OOMKilled` alone
+ * does not: Docker sets it when *any* process in the container's cgroup was
+ * OOM-killed (a compiler, a test runner, a child `node`), and it stays set
+ * until the container is started again. A main process that survived that and
+ * then exited 0-127 on its own ran to completion, so the flag only counts when
+ * there is no usable exit status (the watcher's `-1`, or nothing at all)
+ * (issue #178).
+ *
  * @param {number|string|null} exitCode - `.State.ExitCode`
  * @param {boolean|string|null} oomKilled - `.State.OOMKilled`
  * @returns {boolean} Whether the main process was killed
  */
 function isKilledExit(exitCode, oomKilled) {
-  return (
-    describeExitCode(exitCode).code === KILLED_EXIT_CODE ||
-    oomKilled === true ||
-    oomKilled === 'true'
-  );
+  const { code } = describeExitCode(exitCode);
+  if (code === KILLED_EXIT_CODE) {
+    return true;
+  }
+  const oom = oomKilled === true || oomKilled === 'true';
+  return oom && (code === null || code < 0);
 }
 
 /**
  * Shell condition run by the completion watcher: true when the container was
- * killed and the recovery entry point resumed it.
+ * killed and the recovery entry point resumed it. Mirrors `isKilledExit()`:
+ * `OOMKilled` only counts without a non-negative exit code (issue #178).
  * @param {string} executionId - Execution UUID
  * @returns {string} POSIX shell condition
  */
 function buildRecoverySnippet(executionId) {
   const v = SHELL_VARS;
   return (
-    `{ [ "$${v.exit}" = ${KILLED_EXIT_CODE} ] || [ "$${v.oom}" = true ]; } && ` +
+    `{ [ "$${v.exit}" = ${KILLED_EXIT_CODE} ] || ` +
+    `{ [ "$${v.oom}" = true ] && ! [ "$${v.exit}" -ge 0 ] 2>/dev/null; }; } && ` +
     `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
     `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
-    `"$${v.started}" "$${v.finished}" "$${v.error}" >/dev/null 2>&1`
+    `"$${v.started}" "$${v.finished}" "$${v.error}" ` +
+    `"$${CGROUP_SHELL_VARS.sample}" >/dev/null 2>&1`
   );
 }
 
 /**
  * The `[Recovery k/N]` separator written into the session log.
- * @param {object} params - {attempt, maxAttempts, exitCode, oomKilled, containerName, command}
+ * @param {object} params - {attempt, maxAttempts, exitCode, oomKilled, containerName, command, delayMs}
  * @returns {string} Separator line, newline terminated
  */
 function formatRecoverySeparator(params) {
@@ -120,14 +146,29 @@ function formatRecoverySeparator(params) {
   const what = params.command
     ? `running recovery command: ${params.command}`
     : 'running the original command again';
+  const delay =
+    params.delayMs > 0
+      ? ` after a ${formatRecoveryDelay(params.delayMs)} delay`
+      : '';
   return (
     `\n[Recovery ${params.attempt}/${params.maxAttempts}] ` +
     `Main process was killed (exit ${described.code}${signal}, ` +
     `oomKilled=${params.oomKilled === true || params.oomKilled === 'true'}); ` +
-    `resuming container ${params.containerName}, ${what}\n`
+    `resuming container ${params.containerName}${delay}, ${what}\n`
   );
 }
 
+function isStopRequested(store, executionId) {
+  try {
+    const current = store.get(executionId);
+    return Boolean(
+      current && current.options && current.options.stopRequestedAt
+    );
+  } catch {
+    return false;
+  }
+}
+
 function appendToLog(logPath, text) {
   if (!logPath) {
     return;
@@ -162,10 +203,13 @@ function failureDetail(result) {
 }
 
 /**
- * Resume a killed execution in its own container.
+ * Resume a killed execution in its own container, after the random
+ * `--on-kill-resume-delay` wait (issue #181) when one was requested.
  * @param {object} params - {store, executionId, exitCode, oomKilled, startedAt,
- *   finishedAt, containerError, runner, startWatcher, now}
- * @returns {{recovered: boolean, reason: string, attempt?: number}} Outcome
+ *   finishedAt, containerError, cgroupMemory, runner, startWatcher, now,
+ *   random, sleep}; `cgroupMemory` is the watcher's last cgroup v2 sample
+ *   (issue #182)
+ * @returns {{recovered: boolean, reason: string, attempt?: number, delayMs?: number}} Outcome
  */
 function recoverKilledExecution(params = {}) {
   const { store, executionId } = params;
@@ -201,6 +245,8 @@ function recoverKilledExecution(params = {}) {
   const runner = params.runner || require('./execution-control').runCommand;
   const now = params.now || (() => new Date());
   const attempt = used + 1;
+  const delayRange = getOnKillResumeDelay(opts);
+  const delayMs = pickRecoveryDelayMs(delayRange, params.random);
   const giveUp = (reason) => {
     log(`[Recovery ${attempt}/${maxAttempts}] Failed: ${reason}\n`);
     return { recovered: false, reason: 'resume-failed', attempt };
@@ -216,6 +262,10 @@ function recoverKilledExecution(params = {}) {
       error: params.containerError,
     })}`
   );
+  const memoryLine = formatCgroupMemoryLogLine(params.cgroupMemory);
+  if (memoryLine) {
+    log(`${memoryLine}\n`);
+  }
   log(
     formatRecoverySeparator({
       attempt,
@@ -224,9 +274,28 @@ function recoverKilledExecution(params = {}) {
       oomKilled: params.oomKilled,
       containerName,
       command: opts.recoveryCommand || null,
+      delayMs,
     })
   );
 
+  // Executions killed by one host-wide OOM event must not all come back in
+  // the same second (issue #181). `--stop` during the wait cancels the resume:
+  // the container has already exited, so `docker stop` only leaves the
+  // `stopRequestedAt` marker this loop polls for.
+  if (delayMs > 0) {
+    const cancelled = waitForRecoveryDelay({
+      delayMs,
+      sleep: params.sleep,
+      shouldCancel: () => isStopRequested(store, executionId),
+    });
+    if (cancelled) {
+      log(
+        `[Recovery ${attempt}/${maxAttempts}] Not resuming: the session was stopped on request during the delay.\n`
+      );
+      return { recovered: false, reason: 'stop-requested', attempt, delayMs };
+    }
+  }
+
   // Docker keeps `docker update` limits in the HostConfig across restarts;
   // they are read back so the log and `--status` show what the resumed run
   // is held to, and so a later relaunch can re-apply them.
@@ -255,14 +324,24 @@ function recoverKilledExecution(params = {}) {
   }
 
   const described = describeExitCode(params.exitCode);
+  const delayed = delayRange ? `, delayMs=${delayMs}` : '';
+  // The killed run's own counters: the resumed run starts a fresh cgroup.
+  const memory = parseCgroupMemorySample(params.cgroupMemory);
+  const counted = memory
+    ? ['oomEvents', 'oomKills']
+        .filter((field) => memory[field] !== null)
+        .map((field) => `, ${field}=${memory[field]}`)
+        .join('')
+    : '';
   record.options = {
     ...opts,
     recoveryAttempts: attempt,
     recoveryHistory: [
       ...(Array.isArray(opts.recoveryHistory) ? opts.recoveryHistory : []),
-      `${attempt}: exit ${described.code}, oomKilled=${params.oomKilled === true || params.oomKilled === 'true'}, resumed at ${since}`,
+      `${attempt}: exit ${described.code}, oomKilled=${params.oomKilled === true || params.oomKilled === 'true'}${counted}${delayed}, resumed at ${since}`,
     ],
     lastRecoveryAt: since,
+    ...(delayRange ? { lastRecoveryDelayMs: delayMs } : {}),
     ...(resourceLimits && resourceLimits.length > 0 ? { resourceLimits } : {}),
   };
   record.status = 'executing';
@@ -270,6 +349,7 @@ function recoverKilledExecution(params = {}) {
   record.endTime = null;
   record.exitReason = undefined;
   record.oomKilled = undefined;
+  record.cgroupMemory = undefined;
   try {
     store.save(record);
   } catch {
@@ -286,19 +366,32 @@ function recoverKilledExecution(params = {}) {
     record.uuid,
     { since, recoverOnKill: true }
   );
-  return { recovered: true, reason: 'resumed', attempt };
+  return {
+    recovered: true,
+    reason: 'resumed',
+    attempt,
+    ...(delayRange ? { delayMs } : {}),
+  };
 }
 
 /**
  * Entry point used by the detached watcher:
- *   <runtime> execution-recovery.js <uuid> <exit> <oom> <started> <finished> <error>
+ *   <runtime> execution-recovery.js <uuid> <exit> <oom> <started> <finished> <error> <cgroup>
  * Exits 0 only when the container was resumed; any other outcome lets the
  * watcher continue with its normal cleanup, footer and finalization.
  * @param {string[]} argv - Positional arguments
  * @returns {number} Process exit code
  */
 function main(argv) {
-  const [executionId, exitCode, oomKilled, startedAt, finishedAt, error] = argv;
+  const [
+    executionId,
+    exitCode,
+    oomKilled,
+    startedAt,
+    finishedAt,
+    error,
+    cgroupMemory,
+  ] = argv;
   if (!executionId || process.env.START_DISABLE_TRACKING === 'true') {
     return 1;
   }
@@ -317,6 +410,7 @@ function main(argv) {
       startedAt,
       finishedAt,
       containerError: error,
+      cgroupMemory,
     });
     return outcome.recovered ? 0 : 1;
   } catch (err) {

package/src/lib/execution-store.js

@@ -116,6 +116,12 @@ class ExecutionRecord {
       options.memoryExhaustedReason !== undefined
         ? options.memoryExhaustedReason
         : undefined;
+    // cgroup v2 memory counters the detached watcher sampled for the last run
+    // ({limitBytes, peakBytes, oomEvents, oomKills}, issue #182). Unlike the
+    // sticky `oomKilled`, they say how many processes the OOM killer took and
+    // whether the container hit its own limit.
+    this.cgroupMemory =
+      options.cgroupMemory !== undefined ? options.cgroupMemory : undefined;
     this.workingDirectory = options.workingDirectory || process.cwd();
     this.shell = options.shell || process.env.SHELL || '/bin/sh';
     this.platform = options.platform || process.platform;
@@ -176,6 +182,9 @@ class ExecutionRecord {
     ) {
       obj.memoryExhaustedReason = this.memoryExhaustedReason;
     }
+    if (this.cgroupMemory !== undefined && this.cgroupMemory !== null) {
+      obj.cgroupMemory = this.cgroupMemory;
+    }
     Object.assign(obj, {
       workingDirectory: this.workingDirectory,
       shell: this.shell,

package/src/lib/exit-reason.js

@@ -25,6 +25,11 @@ const os = require('os');
  */
 const MEMORY_EXHAUSTION_PREFIX = 'memory-exhaustion';
 
+/**
+ * Reason reported when the container-wide OOM observation explains the exit.
+ */
+const CGROUP_OOM_EXIT_REASON = `${MEMORY_EXHAUSTION_PREFIX} (cgroup-oom-killer)`;
+
 /**
  * Fatal markers, most specific first. The first matching entry wins.
  * @type {{reason: string, pattern: RegExp}[]}
@@ -155,7 +160,12 @@ function detectMemoryMarker(text) {
  * a fatal marker (a test fixture, an `rg` dump) cannot turn a clean run into a
  * reported memory failure.
  *
- * @param {{exitCode?: number|null, logTail?: string|null, oomKilled?: boolean}|null} input
+ * The container-wide OOM signals (`State.OOMKilled`, the cgroup `oom_kill`
+ * counter) only count when the exit fits an OOM kill of the command itself:
+ * a child OOM-killed under a command that later exits 1 is not the reason the
+ * command ended (#180).
+ *
+ * @param {{exitCode?: number|null, logTail?: string|null, oomKilled?: boolean, cgroupMemory?: object|null}|null} input
  * @returns {{memoryExhausted: true, memoryExhaustedReason: string}|null}
  */
 function resolveMemoryExhaustion(input) {
@@ -175,15 +185,63 @@ function resolveMemoryExhaustion(input) {
   if (marker) {
     return { memoryExhausted: true, memoryExhaustedReason: marker.line };
   }
-  if (input.oomKilled === true) {
+  if (input.oomKilled === true && isOomKillOfCommand(input)) {
     return {
       memoryExhausted: true,
       memoryExhaustedReason: 'Docker reported State.OOMKilled=true',
     };
   }
+  const oomKills = cgroupOomKills(input.cgroupMemory);
+  if (oomKills > 0 && isOomKillOfCommand(input)) {
+    return {
+      memoryExhausted: true,
+      memoryExhaustedReason: `cgroup memory.events reported oom_kill=${oomKills}`,
+    };
+  }
   return null;
 }
 
+/**
+ * `oom_kill` counter sampled from the container cgroup (issue #182).
+ * @param {{oomKills?: number|null}|null|undefined} cgroupMemory - Sampled counters
+ * @returns {number} Number of OOM-killed processes, 0 when unknown
+ */
+function cgroupOomKills(cgroupMemory) {
+  const value = cgroupMemory ? cgroupMemory.oomKills : null;
+  return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+
+/**
+ * Can a container-wide OOM observation be blamed on the command itself?
+ *
+ * Docker's `State.OOMKilled` and the cgroup `oom_kill` counter are
+ * container-wide and sticky (moby/moby#43564): they turn on as soon as *any*
+ * process in the container is OOM-killed, e.g. one `rustc` child under
+ * `cargo test`, while the command keeps running and later exits on its own
+ * (issue #180). The OOM killer always sends SIGKILL, so the observation only
+ * explains the command's exit when that exit is 137 (`128 + SIGKILL`) or there
+ * is no usable exit code at all (null, or the `-1` of an unknown exit). This is
+ * the rule `isKilledExit()` already applies to `--on-kill-resume` (#178).
+ *
+ * @param {{exitCode?: number|null, oomKilled?: boolean, cgroupMemory?: object|null}|null} input
+ * @returns {boolean} True when an OOM was observed and the exit fits an OOM kill
+ */
+function isOomKillOfCommand(input) {
+  if (!input) {
+    return false;
+  }
+  const observed =
+    input.oomKilled === true || cgroupOomKills(input.cgroupMemory) > 0;
+  if (!observed) {
+    return false;
+  }
+  const { exitCode } = input;
+  if (typeof exitCode !== 'number' || !Number.isFinite(exitCode)) {
+    return true;
+  }
+  return exitCode < 0 || signalNameForExitCode(exitCode) === 'SIGKILL';
+}
+
 /**
  * Exit code of a child process that has finished.
  *
@@ -249,9 +307,10 @@ function describeExitCode(exitCode) {
  * Resolve the best available hint for why an execution ended.
  *
  * Precedence: the log marker (evidence written by the command itself), then the
- * cgroup OOM observation, then the signal encoded in the exit code.
+ * cgroup OOM observation when the exit fits an OOM kill of the command (#180),
+ * then the signal encoded in the exit code.
  *
- * @param {{exitCode?: number|null, logTail?: string|null, oomKilled?: boolean}|null} input
+ * @param {{exitCode?: number|null, logTail?: string|null, oomKilled?: boolean, cgroupMemory?: object|null}|null} input
  * @returns {string|null} Exit reason hint, or null when nothing is known
  */
 function resolveExitReason(input) {
@@ -264,8 +323,8 @@ function resolveExitReason(input) {
     return fromLog;
   }
 
-  if (input.oomKilled === true) {
-    return 'memory-exhaustion (cgroup-oom-killer)';
+  if (isOomKillOfCommand(input)) {
+    return CGROUP_OOM_EXIT_REASON;
   }
 
   const signalName = signalNameForExitCode(input.exitCode);
@@ -273,6 +332,7 @@ function resolveExitReason(input) {
 }
 
 module.exports = {
+  CGROUP_OOM_EXIT_REASON,
   EXIT_REASON_MARKERS,
   UNKNOWN_EXIT_CODE,
   MEMORY_EXHAUSTION_PREFIX,
@@ -280,6 +340,7 @@ module.exports = {
   detectExitReason,
   detectMemoryMarker,
   findExitReasonMarker,
+  isOomKillOfCommand,
   resolveMemoryExhaustion,
   describeExitCode,
   resolveChildExitCode,

package/src/lib/recovery-delay.js

@@ -0,0 +1,151 @@
+/**
+ * Random delay before each launch-time kill recovery (issue #181):
+ *   --on-kill-resume-delay <min[-max]>   seconds; one number is a fixed delay,
+ *                                        0 (the default) resumes at once
+ *
+ * One host-wide OOM event can kill several detached docker executions at the
+ * same moment. Without a delay every watcher resumes its container in the same
+ * second, all of them rebuild their working sets at once, and the next OOM
+ * event follows. A uniformly random delay spreads the recoveries out.
+ */
+
+const DELAY_PATTERN = /^(\d+(?:\.\d+)?)(?:-(\d+(?:\.\d+)?))?$/;
+
+/** How often a pending recovery checks whether a stop was requested. */
+const CANCEL_CHECK_INTERVAL_MS = 1000;
+
+/**
+ * Parse a delay range in seconds.
+ * @param {string|number|null|undefined} value - `30-90`, `45`, `0`, ...
+ * @returns {?{minSeconds: number, maxSeconds: number}} Range, or null when
+ *   the value is missing or malformed
+ */
+function parseRecoveryDelayRange(value) {
+  if (value === null || value === undefined) {
+    return null;
+  }
+  const match = DELAY_PATTERN.exec(String(value).trim());
+  if (!match) {
+    return null;
+  }
+  const minSeconds = Number(match[1]);
+  const maxSeconds = match[2] === undefined ? minSeconds : Number(match[2]);
+  if (
+    !Number.isFinite(minSeconds) ||
+    !Number.isFinite(maxSeconds) ||
+    maxSeconds < minSeconds
+  ) {
+    return null;
+  }
+  return { minSeconds, maxSeconds };
+}
+
+/**
+ * Validate and normalize the `--on-kill-resume-delay` argument.
+ * @param {string} value - Raw argument
+ * @returns {string} Normalized range: `30-90`, `45` or `0`
+ * @throws {Error} If the value is not `<min>[-<max>]` seconds with max >= min
+ */
+function parseOnKillResumeDelayValue(value) {
+  const range = parseRecoveryDelayRange(value);
+  if (!range) {
+    throw new Error(
+      `Invalid --on-kill-resume-delay value: "${value}". ` +
+        'Expected seconds as <min>[-<max>] with max >= min, e.g. 30-90.'
+    );
+  }
+  return formatRecoveryDelayRange(range);
+}
+
+/**
+ * @param {{minSeconds: number, maxSeconds: number}} range - Delay range
+ * @returns {string} `30-90` for a range, `45` for a fixed delay
+ */
+function formatRecoveryDelayRange(range) {
+  return range.minSeconds === range.maxSeconds
+    ? `${range.minSeconds}`
+    : `${range.minSeconds}-${range.maxSeconds}`;
+}
+
+/**
+ * The delay range stored in the record, or null when there is no delay.
+ * @param {object} options - Options or record options (onKillResumeDelay)
+ * @returns {?string} Normalized non-zero range
+ */
+function getOnKillResumeDelay(options = {}) {
+  const range = parseRecoveryDelayRange(options.onKillResumeDelay);
+  if (!range || range.maxSeconds <= 0) {
+    return null;
+  }
+  return formatRecoveryDelayRange(range);
+}
+
+/**
+ * Pick a uniformly random delay from the range.
+ * @param {string|null} value - Stored range (`30-90`)
+ * @param {function(): number} [random] - Source in [0, 1), for tests
+ * @returns {number} Delay in whole milliseconds, 0 without a range
+ */
+function pickRecoveryDelayMs(value, random = Math.random) {
+  const range = parseRecoveryDelayRange(value);
+  if (!range) {
+    return 0;
+  }
+  const minMs = range.minSeconds * 1000;
+  const maxMs = range.maxSeconds * 1000;
+  return Math.round(minMs + (maxMs - minMs) * random());
+}
+
+/**
+ * @param {number} delayMs - Delay in milliseconds
+ * @returns {string} `42s`, `42.5s`
+ */
+function formatRecoveryDelay(delayMs) {
+  return `${Math.round(delayMs / 100) / 10}s`;
+}
+
+/**
+ * Block the current thread. The recovery entry point is a short-lived
+ * synchronous child of the watcher shell, so there is no event loop to keep
+ * responsive; `Atomics.wait` works on the main thread of node and bun.
+ * @param {number} ms - Milliseconds
+ */
+function sleepSync(ms) {
+  if (ms > 0) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+  }
+}
+
+/**
+ * Wait out the delay in short steps, giving up as soon as `shouldCancel`
+ * returns true (a `--stop` during the wait).
+ * @param {object} params - {delayMs, shouldCancel, sleep, intervalMs}
+ * @returns {boolean} True when the wait was cancelled
+ */
+function waitForRecoveryDelay(params) {
+  const sleep = params.sleep || sleepSync;
+  const shouldCancel = params.shouldCancel || (() => false);
+  const intervalMs = params.intervalMs || CANCEL_CHECK_INTERVAL_MS;
+  let remaining = Math.max(0, Number(params.delayMs) || 0);
+  while (remaining > 0) {
+    if (shouldCancel()) {
+      return true;
+    }
+    const step = Math.min(intervalMs, remaining);
+    sleep(step);
+    remaining -= step;
+  }
+  return shouldCancel();
+}
+
+module.exports = {
+  CANCEL_CHECK_INTERVAL_MS,
+  formatRecoveryDelay,
+  formatRecoveryDelayRange,
+  getOnKillResumeDelay,
+  parseOnKillResumeDelayValue,
+  parseRecoveryDelayRange,
+  pickRecoveryDelayMs,
+  sleepSync,
+  waitForRecoveryDelay,
+};

package/src/lib/status-formatter.js

@@ -16,6 +16,7 @@ const { collectProcessIds } = require('./execution-control');
 const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
 const {
   describeExitCode,
+  CGROUP_OOM_EXIT_REASON,
   resolveExitReason,
   resolveMemoryExhaustion,
 } = require('./exit-reason');
@@ -25,6 +26,7 @@ const {
 } = require('./isolation-log-utils');
 const { normalizeDockerTimestamp } = require('./docker-post-mortem');
 const { END_TIME_SOURCE } = require('./detached-finalize');
+const { formatCgroupMemory } = require('./cgroup-memory');
 
 /**
  * Inspect the live state of a detached docker container by name.
@@ -433,9 +435,18 @@ function attachExitReason(record, logTail) {
     exitCode: record.exitCode,
     logTail,
     oomKilled: record.oomKilled,
+    cgroupMemory: record.cgroupMemory,
   });
 
   if (!exitReason) {
+    // A record finalized before #180 may carry the cgroup OOM reason for an
+    // ordinary exit (the sticky flag was blamed for exit 0/1): drop it.
+    if (record.exitReason === CGROUP_OOM_EXIT_REASON) {
+      const cleared = Object.create(Object.getPrototypeOf(record));
+      Object.assign(cleared, record);
+      cleared.exitReason = undefined;
+      return cleared;
+    }
     return record;
   }
 
@@ -470,6 +481,7 @@ function attachMemoryExhaustion(record, logTail) {
     exitCode: record.exitCode,
     logTail,
     oomKilled: record.oomKilled,
+    cgroupMemory: record.cgroupMemory,
   });
 
   if (!memory) {
@@ -655,6 +667,10 @@ function formatRecordAsText(record) {
     ...(obj.memoryExhaustedReason !== undefined
       ? [`Memory Evidence:   ${obj.memoryExhaustedReason}`]
       : []),
+    // cgroup v2 counters the detached watcher sampled (issue #182).
+    ...(formatCgroupMemory(obj.cgroupMemory)
+      ? [`Cgroup Memory:     ${formatCgroupMemory(obj.cgroupMemory)}`]
+      : []),
     `PID:               ${obj.pid !== null ? obj.pid : 'N/A'}`,
     `Working Directory: ${obj.workingDirectory}`,
     `Shell:             ${obj.shell}`,