start-command 0.35.1 → 0.35.2

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # start-command
 
+## 0.35.2
+
+### Patch Changes
+
+- e234c05: Stop reporting `exitReason: memory-exhaustion (cgroup-oom-killer)` and `memoryExhausted: true` (`Docker reported State.OOMKilled=true`) for executions that exited 0, 1 or any other ordinary code while Docker's sticky `OOMKilled` flag was set. The flag is container-wide (moby/moby#43564): it turns on when _any_ process in the container is OOM-killed, such as a `rustc` child under `cargo test`, and stays on until the next start. It now counts as the exit reason only when the command itself died by SIGKILL (exit 137) or its exit code is unknown, the same rule `--on-kill-resume` uses since #178. `oomKilled: true` is still reported. A stale cgroup reason stored by an older version is dropped by `--status`. An attached container kept after a child OOM kill now says `Docker reports a process in it was OOM-killed`.
+
+  Add `--on-kill-resume-delay <min[-max]>` for `--on-kill-resume`/`--recovery-command`: wait a uniformly random number of seconds (e.g. `30-90`; one number is a fixed delay, the default `0` resumes at once) before each recovery, so executions killed by the same host-wide OOM event do not all restart in the same second and trigger the next one. The chosen delay is printed in the `[Recovery k/N]` line, stored as `delayMs` in `recoveryHistory` and as `lastRecoveryDelayMs`, and shown in the `[Isolation] On kill:` line. A `--stop` or `--terminate` during the wait cancels the pending resume; `--terminate` of the already exited container now reports `recovery-cancelled` instead of failing.
+
+  Record the container's cgroup v2 memory counters for detached Docker executions: the completion watcher samples `memory.max`, `memory.peak` and the `memory.events` `oom`/`oom_kill` counters every second (and once more after the container exits, since the cgroup disappears with it). They are written as a `Memory:` line after the container post-mortem, stored as `cgroupMemory` (`limitBytes`, `peakBytes`, `oomEvents`, `oomKills`) and shown by `--status` as `Cgroup Memory:` with a note telling a container-limit OOM (`oom_kill == oom`) from a host-wide or parent-cgroup OOM (`oom_kill > oom`). A non-zero `oomKills` explains a SIGKILL or unknown exit as a cgroup OOM kill, and each `--on-kill-resume` `recoveryHistory` entry notes `oomEvents`/`oomKills` of the killed run. Hosts without cgroup v2 or with a remote `DOCKER_HOST` record nothing.
+
 ## 0.35.1
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.35.1",
+  "version": "0.35.2",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/lib/args-parser.js

@@ -197,6 +197,7 @@ function parseArgs(args) {
     networkAliases: [], // Docker network-scoped aliases
     onKillResume: null, // Resume up to N times when the docker main process is killed (issue #176)
     recoveryCommand: null, // Command run in the same container on such a resume (issue #176)
+    onKillResumeDelay: null, // Random <min>[-<max>] seconds to wait before each such resume (issue #181)
     endpoint: null, // SSH endpoint (current level, e.g., user@host)
     endpointStack: null, // SSH endpoints for each level (with nulls for non-ssh levels)
     user: false, // Create isolated user

package/src/lib/cgroup-memory.js

@@ -0,0 +1,283 @@
+/**
+ * cgroup v2 memory counters of a detached docker container (issue #182).
+ *
+ * Docker's `State.OOMKilled` is container-wide and sticky (moby/moby#43564):
+ * it says that *some* process in the container was OOM-killed at some point,
+ * but not how many, not whether the container hit its own `--memory` limit or
+ * the whole host ran out, and not how close the execution came to its limit.
+ * The kernel keeps all three per cgroup
+ * (https://docs.kernel.org/admin-guide/cgroup-v2.html#memory-interface-files):
+ *   memory.events `oom`       the cgroup hit its own limit
+ *   memory.events `oom_kill`  processes killed here by *any* OOM killer, so
+ *                             `oom_kill > oom` points to a host-wide (or
+ *                             parent cgroup) OOM
+ *   memory.peak / memory.max  peak usage (Linux 5.19+) and limit (`max` when
+ *                             unlimited)
+ *
+ * The cgroup is removed when the container stops, so the values cannot be read
+ * afterwards. The detached completion watcher therefore samples them while the
+ * container runs and hands the last sample to the post-mortem, the finalizer
+ * and the recovery step. Best effort by design: no cgroup v2, a remote docker
+ * daemon or a hidden `/proc` leave the counters unknown, and a kill in the last
+ * sampling interval can be missed when the cgroup is gone before the final
+ * read.
+ */
+
+const { shellQuote } = require('./isolation-log-utils');
+
+/** Seconds between two samples of a running container's cgroup. */
+const CGROUP_SAMPLE_INTERVAL_SECONDS = 1;
+
+/** Shell variables used by the sampler snippets. */
+const CGROUP_SHELL_VARS = {
+  /** Last sample: `<memory.max> <memory.peak> <oom> <oom_kill>`. */
+  sample: '__start_command_cgroup',
+  sampler: '__start_command_cgroup_sampler',
+  file: '__start_command_cgroup_file',
+};
+
+/**
+ * Shell functions shared by the sampler and its final read:
+ *   __start_command_cgroup_dir NAME  print the container's cgroup v2 directory
+ *   __start_command_cgroup_read DIR  print `<max> <peak> <oom> <oom_kill> DIR`
+ *
+ * The directory comes from `/proc/<.State.Pid>/cgroup` (`0::<path>`), which
+ * covers the systemd and cgroupfs drivers, `--cgroup-parent` and rootless
+ * docker alike; the two default layouts are fallbacks. A candidate must name
+ * the container ID, so a PID reported by a remote daemon can never select an
+ * unrelated local cgroup. `START_COMMAND_CGROUP_ROOT` and
+ * `START_COMMAND_PROC_ROOT` replace `/sys/fs/cgroup` and `/proc` (tests).
+ *
+ * @returns {string} Shell function definitions
+ */
+function buildCgroupFunctionsSnippet() {
+  return [
+    '__start_command_cgroup_dir() { __scd_id=$(docker inspect -f \'{{.Id}}\' "$1" 2>/dev/null); ' +
+      '[ -n "$__scd_id" ] || return 1; ' +
+      "__scd_pid=$(docker inspect -f '{{.State.Pid}}' \"$1\" 2>/dev/null); __scd_rel=''; " +
+      'case "$__scd_pid" in \'\'|0|*[!0-9]*) ;; ' +
+      '*) __scd_rel=$(sed -n \'s/^0:://p\' "${START_COMMAND_PROC_ROOT:-/proc}/$__scd_pid/cgroup" 2>/dev/null);; esac; ' +
+      '__scd_root=${START_COMMAND_CGROUP_ROOT:-/sys/fs/cgroup}; ' +
+      'for __scd_dir in ${__scd_rel:+"$__scd_root$__scd_rel"} ' +
+      '"$__scd_root/system.slice/docker-$__scd_id.scope" "$__scd_root/docker/$__scd_id"; do ' +
+      'case "$__scd_dir" in *"$__scd_id"*) if [ -r "$__scd_dir/memory.events" ]; then ' +
+      'printf \'%s\' "$__scd_dir"; return 0; fi;; esac; done; return 1; }',
+    '__start_command_cgroup_read() { __scr_events=$(cat "$1/memory.events" 2>/dev/null) || return 1; ' +
+      "__scr_oom=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom //p'); " +
+      "__scr_kill=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom_kill //p'); " +
+      '__scr_max=$(cat "$1/memory.max" 2>/dev/null); __scr_peak=$(cat "$1/memory.peak" 2>/dev/null); ' +
+      'printf \'%s %s %s %s %s\\n\' "${__scr_max:--}" "${__scr_peak:--}" ' +
+      '"${__scr_oom:--}" "${__scr_kill:--}" "$1"; }',
+  ].join('; ');
+}
+
+/**
+ * Shell fragment starting the background sampler. It writes each sample to a
+ * temp file with `mv`, so the watcher never reads a half-written line, and
+ * ends by itself once the cgroup is gone.
+ * @param {string} containerName - Container to sample
+ * @returns {string} Shell command
+ */
+function buildCgroupSamplerStartSnippet(containerName) {
+  const v = CGROUP_SHELL_VARS;
+  return [
+    buildCgroupFunctionsSnippet(),
+    `${v.file}="\${TMPDIR:-/tmp}/start-command-cgroup.$$"`,
+    `rm -f "$${v.file}" "$${v.file}.tmp"`,
+    `( __scs_dir=$(__start_command_cgroup_dir ${shellQuote(containerName)}) || exit 0; ` +
+      'while __scs_line=$(__start_command_cgroup_read "$__scs_dir"); do ' +
+      `printf '%s\\n' "$__scs_line" > "$${v.file}.tmp" && mv -f "$${v.file}.tmp" "$${v.file}"; ` +
+      `sleep ${CGROUP_SAMPLE_INTERVAL_SECONDS}; done ) >/dev/null 2>&1 & ${v.sampler}=$!`,
+  ].join('; ');
+}
+
+/**
+ * Shell fragment stopping the sampler once the container has exited and
+ * leaving the last sample (`<max> <peak> <oom> <oom_kill>`, or empty) in
+ * `$__start_command_cgroup`. The cgroup can outlive the main process for a
+ * moment, so it is read one last time when it still exists.
+ * @returns {string} Shell command
+ */
+function buildCgroupSamplerStopSnippet() {
+  const v = CGROUP_SHELL_VARS;
+  return [
+    `kill "$${v.sampler}" 2>/dev/null`,
+    `wait "$${v.sampler}" 2>/dev/null`,
+    `${v.sample}=$(cat "$${v.file}" 2>/dev/null)`,
+    `__scs_dir=$(printf '%s' "$${v.sample}" | cut -s -d ' ' -f 5-)`,
+    `if [ -n "$__scs_dir" ] && __scs_line=$(__start_command_cgroup_read "$__scs_dir"); then ${v.sample}=$__scs_line; fi`,
+    `rm -f "$${v.file}" "$${v.file}.tmp"`,
+    `${v.sample}=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 1-4)`,
+  ].join('; ');
+}
+
+/**
+ * Shell fragment appending the `Memory:` line of the post-mortem when a
+ * sample exists.
+ * @param {string} quotedLogPath - Already shell-quoted log path
+ * @returns {string} Shell command
+ */
+function buildCgroupMemoryLogSnippet(quotedLogPath) {
+  const v = CGROUP_SHELL_VARS;
+  // A function, so `$1`.. of the watcher script itself stay untouched.
+  return (
+    `__start_command_cgroup_log() { __scm_note=''; ` +
+    `if [ "$4" -gt 0 ] 2>/dev/null; then if [ "$4" -gt "$3" ] 2>/dev/null; then ` +
+    `__scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.HOST_OR_PARENT]})'; ` +
+    `else __scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.CONTAINER_LIMIT]})'; fi; fi; ` +
+    `printf 'Memory:     memory.max=%s memory.peak=%s oom=%s oom_kill=%s%s\\n' ` +
+    `"$1" "$2" "$3" "$4" "$__scm_note"; }; ` +
+    `if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath}; fi`
+  );
+}
+
+/**
+ * The `Memory:` line for a sample, exactly as the watcher's shell writes it.
+ * Used where JavaScript writes the post-mortem itself (the recovery step).
+ * @param {string|null|undefined} text - `<memory.max> <memory.peak> <oom> <oom_kill>`
+ * @returns {?string} Line without a trailing newline, or null without a sample
+ */
+function formatCgroupMemoryLogLine(text) {
+  const counters = parseCgroupMemorySample(text);
+  if (!counters) {
+    return null;
+  }
+  const [max, peak, oom, kill] = String(text).trim().split(/\s+/);
+  const scope = describeCgroupOomScope(counters);
+  return `Memory:     memory.max=${max} memory.peak=${peak} oom=${oom} oom_kill=${kill}${
+    scope ? ` (${OOM_SCOPE_NOTES[scope]})` : ''
+  }`;
+}
+
+/** Where the OOM killer that killed processes in the container came from. */
+const OOM_SCOPE = {
+  CONTAINER_LIMIT: 'container-limit',
+  HOST_OR_PARENT: 'host-or-parent',
+};
+
+const OOM_SCOPE_NOTES = {
+  [OOM_SCOPE.CONTAINER_LIMIT]: 'the container hit its own memory limit',
+  [OOM_SCOPE.HOST_OR_PARENT]:
+    'oom_kill > oom: a host-wide or parent cgroup OOM killed processes here',
+};
+
+function parseCounter(text) {
+  return /^\d+$/.test(text || '') ? Number(text) : null;
+}
+
+/**
+ * Parse a sample written by the watcher.
+ * @param {string|null|undefined} text - `<memory.max> <memory.peak> <oom> <oom_kill>`
+ * @returns {?{limitBytes: ?number, peakBytes: ?number, oomEvents: ?number, oomKills: ?number}}
+ *   Counters (`limitBytes` is null without a limit), or null without a sample
+ */
+function parseCgroupMemorySample(text) {
+  const fields = String(text === null || text === undefined ? '' : text)
+    .trim()
+    .split(/\s+/);
+  if (fields.length < 4) {
+    return null;
+  }
+  const [max, peak, oom, kill] = fields;
+  const counters = {
+    limitBytes: parseCounter(max),
+    peakBytes: parseCounter(peak),
+    oomEvents: parseCounter(oom),
+    oomKills: parseCounter(kill),
+  };
+  if (counters.oomEvents === null && counters.oomKills === null) {
+    // memory.events was unreadable: the line carries no information.
+    return null;
+  }
+  return counters;
+}
+
+/**
+ * Normalize counters read back from an execution record.
+ * @param {*} value - `record.cgroupMemory`
+ * @returns {?{limitBytes: ?number, peakBytes: ?number, oomEvents: ?number, oomKills: ?number}}
+ */
+function normalizeCgroupMemory(value) {
+  if (!value || typeof value !== 'object') {
+    return null;
+  }
+  const counter = (field) => {
+    const raw = value[field];
+    const number = typeof raw === 'string' && raw !== '' ? Number(raw) : raw;
+    return Number.isSafeInteger(number) && number >= 0 ? number : null;
+  };
+  return {
+    limitBytes: counter('limitBytes'),
+    peakBytes: counter('peakBytes'),
+    oomEvents: counter('oomEvents'),
+    oomKills: counter('oomKills'),
+  };
+}
+
+/**
+ * @param {?object} memory - Counters
+ * @returns {?string} One of OOM_SCOPE, or null when nothing was OOM-killed
+ */
+function describeCgroupOomScope(memory) {
+  const counters = normalizeCgroupMemory(memory);
+  if (!counters || !(counters.oomKills > 0)) {
+    return null;
+  }
+  return counters.oomKills > (counters.oomEvents || 0)
+    ? OOM_SCOPE.HOST_OR_PARENT
+    : OOM_SCOPE.CONTAINER_LIMIT;
+}
+
+function formatBytes(bytes) {
+  if (bytes === null) {
+    return 'unknown';
+  }
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit++;
+  }
+  return unit === 0 ? `${bytes} B` : `${value.toFixed(1)} ${units[unit]}`;
+}
+
+/**
+ * Human-readable counters for `--status`.
+ * @param {?object} memory - Counters
+ * @returns {?string} e.g. `peak 255.9 MiB of 256.0 MiB limit, oom 0, oom_kill 3 (...)`
+ */
+function formatCgroupMemory(memory) {
+  const counters = normalizeCgroupMemory(memory);
+  if (!counters) {
+    return null;
+  }
+  const limit =
+    counters.limitBytes === null
+      ? 'no limit'
+      : `${formatBytes(counters.limitBytes)} limit`;
+  const count = (value) => (value === null ? 'unknown' : value);
+  const scope = describeCgroupOomScope(counters);
+  return (
+    `peak ${formatBytes(counters.peakBytes)} of ${limit}, ` +
+    `oom ${count(counters.oomEvents)}, oom_kill ${count(counters.oomKills)}${
+      scope ? ` (${OOM_SCOPE_NOTES[scope]})` : ''
+    }`
+  );
+}
+
+module.exports = {
+  CGROUP_SAMPLE_INTERVAL_SECONDS,
+  CGROUP_SHELL_VARS,
+  OOM_SCOPE,
+  OOM_SCOPE_NOTES,
+  buildCgroupFunctionsSnippet,
+  buildCgroupMemoryLogSnippet,
+  buildCgroupSamplerStartSnippet,
+  buildCgroupSamplerStopSnippet,
+  describeCgroupOomScope,
+  formatCgroupMemory,
+  formatCgroupMemoryLogLine,
+  normalizeCgroupMemory,
+  parseCgroupMemorySample,
+};

package/src/lib/detached-finalize.js

@@ -23,6 +23,10 @@ const {
   normalizeDockerTimestamp,
 } = require('./docker-post-mortem');
 const { shellQuote } = require('./isolation-log-utils');
+const {
+  CGROUP_SHELL_VARS,
+  parseCgroupMemorySample,
+} = require('./cgroup-memory');
 
 /** Provenance markers for `endTime` (issue #170.2). */
 const END_TIME_SOURCE = {
@@ -57,6 +61,8 @@ const WATCHER_LOST_CONTAINER = 'watcher-lost-container';
  * @param {string|null} [options.finishedAt] - `.State.FinishedAt`
  * @param {string|null} [options.containerError] - `.State.Error`
  * @param {boolean|string|null} [options.running] - `.State.Running`
+ * @param {string|null} [options.cgroupMemory] - Last cgroup v2 sample of the
+ *   watcher, `<memory.max> <memory.peak> <oom> <oom_kill>` (issue #182)
  * @returns {{updated: boolean, reason?: string, record?: Object}} Outcome
  */
 function finalizeDetachedExecution(options = {}) {
@@ -123,6 +129,12 @@ function finalizeDetachedExecution(options = {}) {
     record.oomKilled = oomKilled;
   }
 
+  // Counted by the kernel for this run only, unlike the sticky `OOMKilled`.
+  const cgroupMemory = parseCgroupMemorySample(options.cgroupMemory);
+  if (cgroupMemory) {
+    record.cgroupMemory = cgroupMemory;
+  }
+
   const exitReason =
     resolveReason(record) || (finishedAt ? null : WATCHER_LOST_CONTAINER);
   if (exitReason) {
@@ -162,6 +174,7 @@ function resolveReason(record) {
       exitCode: record.exitCode,
       logTail,
       oomKilled: record.oomKilled,
+      cgroupMemory: record.cgroupMemory,
     });
   } catch {
     return null;
@@ -244,13 +257,13 @@ function buildDetachedFinalizeSnippet(executionId) {
     `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
     `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
     `"$${v.started}" "$${v.finished}" "$${v.error}" "$${v.running}" ` +
-    `>/dev/null 2>&1 || true`
+    `"$${CGROUP_SHELL_VARS.sample}" >/dev/null 2>&1 || true`
   );
 }
 
 /**
  * Entry point used by the detached watcher:
- *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error> <running>
+ *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error> <running> <cgroup>
  * Always exits 0 — a bookkeeping failure must never turn into a visible error
  * in a log the user is reading for the command's own output.
  * @param {string[]} argv - Positional arguments
@@ -265,6 +278,7 @@ function main(argv) {
     finishedAt,
     containerError,
     running,
+    cgroupMemory,
   ] = argv;
   if (!executionId || process.env.START_DISABLE_TRACKING === 'true') {
     return;
@@ -285,6 +299,7 @@ function main(argv) {
       finishedAt,
       containerError,
       running,
+      cgroupMemory,
     });
   } catch {
     // Deliberately silent: see the exit-0 contract above.

package/src/lib/docker-cleanup.js

@@ -6,7 +6,10 @@ const {
   readLogTail,
   shellQuote,
 } = require('./isolation-log-utils');
-const { resolveMemoryExhaustion } = require('./exit-reason');
+const {
+  isOomKillOfCommand,
+  resolveMemoryExhaustion,
+} = require('./exit-reason');
 const {
   buildDockerPostMortemSnippet,
   buildDockerRemovalNoteSnippet,
@@ -20,6 +23,11 @@ const {
   SHELL_VARS,
 } = require('./docker-post-mortem');
 const { buildDetachedFinalizeSnippet } = require('./detached-finalize');
+const {
+  buildCgroupMemoryLogSnippet,
+  buildCgroupSamplerStartSnippet,
+  buildCgroupSamplerStopSnippet,
+} = require('./cgroup-memory');
 
 const DOCKER_CONTAINER_CLEANUP_POLICY = {
   DEFAULT: 'default',
@@ -266,10 +274,16 @@ function buildAttachedDockerKeptMessage({
   oomKilled,
   logPath,
 }) {
-  let message =
-    oomKilled === true
-      ? `\nContainer kept because Docker reports it was OOM-killed.`
-      : `\nContainer kept because the command failed.`;
+  let message;
+  if (oomKilled !== true) {
+    message = `\nContainer kept because the command failed.`;
+  } else if (isOomKillOfCommand({ exitCode, oomKilled })) {
+    message = `\nContainer kept because Docker reports it was OOM-killed.`;
+  } else {
+    // The flag is container-wide (#180): a child was OOM-killed, the command
+    // itself exited on its own.
+    message = `\nContainer kept because Docker reports a process in it was OOM-killed.`;
+  }
   const memory = resolveMemoryExhaustion({
     exitCode,
     logTail: logPath ? readLogTail(logPath, FATAL_MARKER_TAIL_BYTES) : null,
@@ -343,7 +357,9 @@ function buildDetachedDockerCompletionScript(
   watcherOptions = {}
 ) {
   const quotedName = shellQuote(containerName);
-  const parts = [];
+  // The container's cgroup disappears with it, so its memory counters are
+  // sampled while it runs (issue #182).
+  const parts = [buildCgroupSamplerStartSnippet(containerName)];
   // Everything that assumes the container has exited: cleanup, footer and
   // finalization. Guarded as a whole by `.State.Running` below.
   const exited = [];
@@ -358,14 +374,16 @@ function buildDetachedDockerCompletionScript(
       containerName,
       quotedLogPath
     );
-    const remove = `docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true; ${removalNote}`;
-    const keep = `${postMortem}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
+    const memory = buildCgroupMemoryLogSnippet(quotedLogPath);
+    const remove = `docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true; ${removalNote}; ${memory}`;
+    const keep = `${postMortem}; ${memory}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
 
     const since = watcherOptions.since
       ? ` --since ${shellQuote(watcherOptions.since)}`
       : '';
     parts.push(`docker logs -f${since} ${quotedName} >> ${quotedLogPath} 2>&1`);
     parts.push(buildDockerWaitForExitSnippet(containerName, quotedLogPath));
+    parts.push(buildCgroupSamplerStopSnippet());
     parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
       exited.push(remove);
@@ -386,6 +404,7 @@ function buildDetachedDockerCompletionScript(
   } else {
     parts.push(`docker wait ${quotedName} >/dev/null 2>&1`);
     parts.push(buildDockerWaitForExitSnippet(containerName));
+    parts.push(buildCgroupSamplerStopSnippet());
     parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
       exited.push(`docker rm -f ${quotedName} >/dev/null 2>&1 || true`);

package/src/lib/docker-recovery-options.js

@@ -4,8 +4,12 @@
  *                               killed (exit 137 / OOMKilled)
  *   --recovery-command <cmd>    command to run in the same container on resume
  *                               (without it, the original command is re-run)
+ *   --on-kill-resume-delay <s>  wait a random <min>[-<max>] seconds before
+ *                               each resume (issue #181)
  */
 
+const { parseOnKillResumeDelayValue } = require('./recovery-delay');
+
 function parseOnKillResumeValue(value) {
   const text = String(value).trim();
   const count = Number(text);
@@ -19,18 +23,34 @@ function parseOnKillResumeValue(value) {
 
 function parseDockerRecoveryOption(args, index, options) {
   const arg = args[index];
-  if (arg === '--on-kill-resume' || arg === '--recovery-command') {
+  if (
+    arg === '--on-kill-resume' ||
+    arg === '--recovery-command' ||
+    arg === '--on-kill-resume-delay'
+  ) {
     if (index + 1 >= args.length || args[index + 1].startsWith('-')) {
-      const value = arg === '--on-kill-resume' ? 'count' : 'command';
+      const value = {
+        '--on-kill-resume': 'count',
+        '--recovery-command': 'command',
+        '--on-kill-resume-delay': 'seconds',
+      }[arg];
       throw new Error(`Option ${arg} requires a ${value} argument`);
     }
     if (arg === '--on-kill-resume') {
       options.onKillResume = parseOnKillResumeValue(args[index + 1]);
+    } else if (arg === '--on-kill-resume-delay') {
+      options.onKillResumeDelay = parseOnKillResumeDelayValue(args[index + 1]);
     } else {
       options.recoveryCommand = args[index + 1];
     }
     return 2;
   }
+  if (arg.startsWith('--on-kill-resume-delay=')) {
+    options.onKillResumeDelay = parseOnKillResumeDelayValue(
+      arg.slice('--on-kill-resume-delay='.length)
+    );
+    return 1;
+  }
   if (arg.startsWith('--on-kill-resume=')) {
     options.onKillResume = parseOnKillResumeValue(
       arg.slice('--on-kill-resume='.length)
@@ -47,7 +67,7 @@ function parseDockerRecoveryOption(args, index, options) {
 /**
  * Recovery is driven by the detached docker completion watcher, so it needs a
  * detached session whose only isolation level is docker. A recovery command on
- * its own implies one attempt.
+ * its own implies one attempt; a delay without either has nothing to delay.
  * @param {object} options - Parsed options
  * @throws {Error} If the recovery options are used where they cannot work
  */
@@ -62,6 +82,14 @@ function validateDockerRecoveryOptions(options) {
     options.onKillResume ??= 1;
   }
   if (!options.onKillResume) {
+    if (
+      options.onKillResumeDelay !== null &&
+      options.onKillResumeDelay !== undefined
+    ) {
+      throw new Error(
+        '--on-kill-resume-delay requires --on-kill-resume or --recovery-command'
+      );
+    }
     return;
   }
   const flag = options.recoveryCommand

package/src/lib/docker-runtime-args.js

@@ -9,11 +9,12 @@ const {
   buildResourceLimitsStatusLine,
   normalizeResourceLimits,
 } = require('./docker-resource-limits');
+const { getOnKillResumeDelay } = require('./recovery-delay');
 
 /**
  * Number of kill recoveries requested at launch; a recovery command on its own
  * implies one (issue #176).
- * @param {object} options - Options (onKillResume, recoveryCommand)
+ * @param {object} options - Options (onKillResume, recoveryCommand, onKillResumeDelay)
  * @returns {?number} Attempt limit, or null when recovery is off
  */
 function getOnKillResume(options = {}) {
@@ -63,7 +64,7 @@ function buildDockerRuntimeArgs(options = {}) {
  * options (volumes, mounts, env, privileged, networks, resource limits and
  * the launch-time recovery command). Empty collections and a falsy privileged
  * flag contribute no lines.
- * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases, resourceLimits, onKillResume, recoveryCommand)
+ * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases, resourceLimits, onKillResume, recoveryCommand, onKillResumeDelay)
  * @returns {string[]} Status lines for the start block / log header
  */
 function buildDockerRuntimeStatusLines(options = {}) {
@@ -99,8 +100,10 @@ function buildDockerRuntimeStatusLines(options = {}) {
   const onKillResume = getOnKillResume(options);
   if (onKillResume) {
     const what = options.recoveryCommand || 'the original command';
+    const delay = getOnKillResumeDelay(options);
+    const after = delay ? ` after a random ${delay}s delay` : '';
     lines.push(
-      `[Isolation] On kill: resume up to ${onKillResume} time(s) with ${what}`
+      `[Isolation] On kill: resume up to ${onKillResume} time(s)${after} with ${what}`
     );
   }
   return lines;
@@ -109,7 +112,7 @@ function buildDockerRuntimeStatusLines(options = {}) {
 /**
  * Build the execution-record metadata for docker runtime options, normalizing
  * empty collections and a falsy privileged flag to `null`.
- * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases, resourceLimits, onKillResume, recoveryCommand)
+ * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases, resourceLimits, onKillResume, recoveryCommand, onKillResumeDelay)
  * @returns {object} Record metadata
  */
 function buildDockerRuntimeMetadata(options = {}) {
@@ -130,6 +133,9 @@ function buildDockerRuntimeMetadata(options = {}) {
     resourceLimits: resourceLimits.length > 0 ? resourceLimits : null,
     onKillResume: getOnKillResume(options),
     recoveryCommand: options.recoveryCommand || null,
+    onKillResumeDelay: getOnKillResume(options)
+      ? getOnKillResumeDelay(options)
+      : null,
   };
 }
 

package/src/lib/execution-control.js

@@ -302,6 +302,23 @@ function restoreStopRequested(store, record, previous) {
   }
 }
 
+/**
+ * Whether a failed docker control command hit a container that has already
+ * exited while launch-time recovery is configured: the stop marker alone then
+ * cancels the resume the watcher may still be waiting to make.
+ * @param {object} record - Execution record
+ * @param {object} result - Failed control command result
+ * @returns {boolean} Whether the stop marker should stand
+ */
+function isPendingRecoveryStop(record, result) {
+  const options = record.options || {};
+  if (options.isolated !== 'docker' || !options.onKillResume) {
+    return false;
+  }
+  const detail = `${result.stderr || ''} ${result.error || ''}`;
+  return /is not running/i.test(detail);
+}
+
 function controlExecution(store, identifier, action, runner = runCommand) {
   if (!store) {
     return { success: false, error: 'Execution tracking is disabled.' };
@@ -325,6 +342,25 @@ function controlExecution(store, identifier, action, runner = runCommand) {
   // which exits 137 just like the OOM killer.
   const previousStopRequestedAt = markStopRequested(store, record);
   const result = runner(control.command, control.args);
+  if (!result.success && isPendingRecoveryStop(record, result)) {
+    // The container already exited and its watcher is waiting out
+    // `--on-kill-resume-delay` (issue #181): `docker kill` has nothing to
+    // signal, but the marker it left cancels the pending resume.
+    return {
+      success: true,
+      output: formatControlResultAsLinksNotation({
+        action,
+        identifier,
+        uuid: record.uuid,
+        status: 'recovery-cancelled',
+        backend: record.options.isolated,
+        sessionName: record.options.sessionName,
+        method: control.method,
+        processIds: null,
+        message: `Cancelled the pending recovery of detached docker container: ${record.options.sessionName}`,
+      }),
+    };
+  }
   if (!result.success) {
     restoreStopRequested(store, record, previousStopRequestedAt);
     const detail =