start-command 0.34.1 → 0.35.0

package/src/lib/execution-recovery.js

@@ -0,0 +1,350 @@
+/**
+ * Launch-time recovery for killed detached docker sessions (issue #176).
+ *
+ * `$ --isolated docker --detached --on-kill-resume 3 --recovery-command B -- A`
+ * runs `A`; when the main process is killed (exit 137 / `OOMKilled`), the
+ * detached completion watcher hands the inspected facts to this module, which
+ * restarts the *same* container so `B` continues on the same filesystem, with
+ * the same resource limits (they live in the container's HostConfig), under
+ * the same execution UUID and appending to the same log file.
+ *
+ * How `B` replaces `A` inside the same container: a container launched with a
+ * recovery command runs a tiny selector as its entrypoint command. The
+ * selector runs `A` normally, but runs `B` when the marker file exists. Before
+ * `docker start`, the marker (holding the attempt number) is copied into the
+ * stopped container with `docker cp`. Without a recovery command the original
+ * command is simply started again.
+ *
+ * Like `detached-finalize.js`, the entry point runs as a short-lived child of
+ * the watcher on a host whose CLI invocation is long gone, so it never throws.
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const {
+  getDockerCommand,
+  getDockerContainerCleanupPolicy,
+  startDetachedDockerCompletionWatcher,
+} = require('./docker-cleanup');
+const {
+  SHELL_VARS,
+  describeExitCode,
+  formatContainerPostMortem,
+} = require('./docker-post-mortem');
+const {
+  buildResourceLimitsStatusLine,
+  readDockerResourceLimits,
+} = require('./docker-resource-limits');
+const { shellQuote } = require('./isolation-log-utils');
+
+/** File whose presence makes the selector run the recovery command. */
+const RECOVERY_MARKER_PATH = '/.start-command-recovery';
+
+/** Environment variable carrying the attempt number into the recovery command. */
+const RECOVERY_ATTEMPT_ENV = 'START_COMMAND_RECOVERY_ATTEMPT';
+
+/** Exit code of a process killed with SIGKILL (the OOM killer's signal). */
+const KILLED_EXIT_CODE = 137;
+
+/**
+ * POSIX selector run as the container command when a recovery command is set:
+ *   sh -c SELECTOR start-command <shell> <flag> <recovery> <main argv...>
+ */
+const RECOVERY_SELECTOR =
+  's=$1; f=$2; r=$3; shift 3; ' +
+  `if [ -e ${RECOVERY_MARKER_PATH} ]; then ` +
+  `${RECOVERY_ATTEMPT_ENV}=$(cat ${RECOVERY_MARKER_PATH} 2>/dev/null); ` +
+  `export ${RECOVERY_ATTEMPT_ENV}; ` +
+  'if [ -n "$f" ]; then exec "$s" "$f" -c "$r"; fi; exec "$s" -c "$r"; fi; ' +
+  'exec "$@"';
+
+/**
+ * Wrap the container command so a later `docker start` can switch to the
+ * recovery command.
+ * @param {string[]} mainArgs - Container command for the main run
+ * @param {object} params - {shell, shellFlag, recoveryCommand}
+ * @returns {string[]} Container command
+ */
+function buildRecoverySelectorArgs(mainArgs, params) {
+  return [
+    'sh',
+    '-c',
+    RECOVERY_SELECTOR,
+    'start-command',
+    params.shell || 'sh',
+    params.shellFlag || '',
+    params.recoveryCommand,
+    ...mainArgs,
+  ];
+}
+
+/**
+ * @param {number|string|null} exitCode - `.State.ExitCode`
+ * @param {boolean|string|null} oomKilled - `.State.OOMKilled`
+ * @returns {boolean} Whether the main process was killed
+ */
+function isKilledExit(exitCode, oomKilled) {
+  return (
+    describeExitCode(exitCode).code === KILLED_EXIT_CODE ||
+    oomKilled === true ||
+    oomKilled === 'true'
+  );
+}
+
+/**
+ * Shell condition run by the completion watcher: true when the container was
+ * killed and the recovery entry point resumed it.
+ * @param {string} executionId - Execution UUID
+ * @returns {string} POSIX shell condition
+ */
+function buildRecoverySnippet(executionId) {
+  const v = SHELL_VARS;
+  return (
+    `{ [ "$${v.exit}" = ${KILLED_EXIT_CODE} ] || [ "$${v.oom}" = true ]; } && ` +
+    `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
+    `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
+    `"$${v.started}" "$${v.finished}" "$${v.error}" >/dev/null 2>&1`
+  );
+}
+
+/**
+ * The `[Recovery k/N]` separator written into the session log.
+ * @param {object} params - {attempt, maxAttempts, exitCode, oomKilled, containerName, command}
+ * @returns {string} Separator line, newline terminated
+ */
+function formatRecoverySeparator(params) {
+  const described = describeExitCode(params.exitCode);
+  const signal = described.signal ? `, ${described.signal}` : '';
+  const what = params.command
+    ? `running recovery command: ${params.command}`
+    : 'running the original command again';
+  return (
+    `\n[Recovery ${params.attempt}/${params.maxAttempts}] ` +
+    `Main process was killed (exit ${described.code}${signal}, ` +
+    `oomKilled=${params.oomKilled === true || params.oomKilled === 'true'}); ` +
+    `resuming container ${params.containerName}, ${what}\n`
+  );
+}
+
+function appendToLog(logPath, text) {
+  if (!logPath) {
+    return;
+  }
+  try {
+    fs.appendFileSync(logPath, text);
+  } catch {
+    // The log is best-effort here: recovery itself must still proceed.
+  }
+}
+
+function writeRecoveryMarker(containerName, attempt, runner) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'start-recovery-'));
+  const file = path.join(dir, 'marker');
+  try {
+    fs.writeFileSync(file, `${attempt}\n`, { mode: 0o644 });
+    fs.chmodSync(file, 0o644);
+    return runner(getDockerCommand(), [
+      'cp',
+      file,
+      `${containerName}:${RECOVERY_MARKER_PATH}`,
+    ]);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+function failureDetail(result) {
+  return (
+    (result.stderr || '').trim() || result.error || `exit code ${result.status}`
+  );
+}
+
+/**
+ * Resume a killed execution in its own container.
+ * @param {object} params - {store, executionId, exitCode, oomKilled, startedAt,
+ *   finishedAt, containerError, runner, startWatcher, now}
+ * @returns {{recovered: boolean, reason: string, attempt?: number}} Outcome
+ */
+function recoverKilledExecution(params = {}) {
+  const { store, executionId } = params;
+  if (!store || !executionId) {
+    return { recovered: false, reason: 'missing-arguments' };
+  }
+  const record = store.get(executionId);
+  if (!record) {
+    return { recovered: false, reason: 'record-not-found' };
+  }
+  const opts = record.options || {};
+  const maxAttempts = Number(opts.onKillResume) || 0;
+  const containerName = opts.sessionName;
+  if (opts.isolated !== 'docker' || !containerName || maxAttempts < 1) {
+    return { recovered: false, reason: 'not-configured' };
+  }
+  if (!isKilledExit(params.exitCode, params.oomKilled)) {
+    return { recovered: false, reason: 'not-killed' };
+  }
+  const log = (text) => appendToLog(record.logPath, text);
+  if (opts.stopRequestedAt) {
+    log('\n[Recovery] Not resuming: the session was stopped on request.\n');
+    return { recovered: false, reason: 'stop-requested' };
+  }
+  const used = Number(opts.recoveryAttempts) || 0;
+  if (used >= maxAttempts) {
+    log(
+      `\n[Recovery] Not resuming: all ${maxAttempts} recovery attempt(s) used.\n`
+    );
+    return { recovered: false, reason: 'attempts-exhausted' };
+  }
+
+  const runner = params.runner || require('./execution-control').runCommand;
+  const now = params.now || (() => new Date());
+  const attempt = used + 1;
+  const giveUp = (reason) => {
+    log(`[Recovery ${attempt}/${maxAttempts}] Failed: ${reason}\n`);
+    return { recovered: false, reason: 'resume-failed', attempt };
+  };
+
+  log(
+    `\n${formatContainerPostMortem({
+      containerName,
+      exitCode: params.exitCode,
+      oomKilled: params.oomKilled,
+      startedAt: params.startedAt,
+      finishedAt: params.finishedAt,
+      error: params.containerError,
+    })}`
+  );
+  log(
+    formatRecoverySeparator({
+      attempt,
+      maxAttempts,
+      exitCode: params.exitCode,
+      oomKilled: params.oomKilled,
+      containerName,
+      command: opts.recoveryCommand || null,
+    })
+  );
+
+  // Docker keeps `docker update` limits in the HostConfig across restarts;
+  // they are read back so the log and `--status` show what the resumed run
+  // is held to, and so a later relaunch can re-apply them.
+  const resourceLimits = readDockerResourceLimits(containerName, runner);
+  const limitsLine = buildResourceLimitsStatusLine(resourceLimits);
+  if (limitsLine) {
+    log(`${limitsLine}\n`);
+  }
+
+  if (opts.recoveryCommand) {
+    let copied;
+    try {
+      copied = writeRecoveryMarker(containerName, attempt, runner);
+    } catch (err) {
+      copied = { success: false, error: err.message };
+    }
+    if (!copied.success) {
+      return giveUp(`could not mark the container: ${failureDetail(copied)}`);
+    }
+  }
+
+  const since = now().toISOString();
+  const started = runner(getDockerCommand(), ['start', containerName]);
+  if (!started.success) {
+    return giveUp(`docker start failed: ${failureDetail(started)}`);
+  }
+
+  const described = describeExitCode(params.exitCode);
+  record.options = {
+    ...opts,
+    recoveryAttempts: attempt,
+    recoveryHistory: [
+      ...(Array.isArray(opts.recoveryHistory) ? opts.recoveryHistory : []),
+      `${attempt}: exit ${described.code}, oomKilled=${params.oomKilled === true || params.oomKilled === 'true'}, resumed at ${since}`,
+    ],
+    lastRecoveryAt: since,
+    ...(resourceLimits && resourceLimits.length > 0 ? { resourceLimits } : {}),
+  };
+  record.status = 'executing';
+  record.exitCode = null;
+  record.endTime = null;
+  record.exitReason = undefined;
+  record.oomKilled = undefined;
+  try {
+    store.save(record);
+  } catch {
+    // The container is already running again; the new watcher still follows
+    // it and finalizes the record when it ends.
+  }
+
+  const startWatcher =
+    params.startWatcher || startDetachedDockerCompletionWatcher;
+  startWatcher(
+    containerName,
+    getDockerContainerCleanupPolicy(opts),
+    record.logPath || null,
+    record.uuid,
+    { since, recoverOnKill: true }
+  );
+  return { recovered: true, reason: 'resumed', attempt };
+}
+
+/**
+ * Entry point used by the detached watcher:
+ *   <runtime> execution-recovery.js <uuid> <exit> <oom> <started> <finished> <error>
+ * Exits 0 only when the container was resumed; any other outcome lets the
+ * watcher continue with its normal cleanup, footer and finalization.
+ * @param {string[]} argv - Positional arguments
+ * @returns {number} Process exit code
+ */
+function main(argv) {
+  const [executionId, exitCode, oomKilled, startedAt, finishedAt, error] = argv;
+  if (!executionId || process.env.START_DISABLE_TRACKING === 'true') {
+    return 1;
+  }
+  try {
+    const { ExecutionStore } = require('./execution-store');
+    const store = new ExecutionStore(
+      process.env.START_APP_FOLDER
+        ? { appFolder: path.resolve(process.env.START_APP_FOLDER) }
+        : {}
+    );
+    const outcome = recoverKilledExecution({
+      store,
+      executionId,
+      exitCode,
+      oomKilled,
+      startedAt,
+      finishedAt,
+      containerError: error,
+    });
+    return outcome.recovered ? 0 : 1;
+  } catch (err) {
+    // The watcher discards this output; START_DEBUG=1 shows it when the
+    // entry point is run by hand.
+    if (process.env.START_DEBUG === '1' || process.env.START_DEBUG === 'true') {
+      console.error(`[DEBUG] execution-recovery failed: ${err.stack || err}`);
+    }
+    return 1;
+  }
+}
+
+module.exports = {
+  KILLED_EXIT_CODE,
+  RECOVERY_ATTEMPT_ENV,
+  RECOVERY_MARKER_PATH,
+  RECOVERY_SELECTOR,
+  buildRecoverySelectorArgs,
+  buildRecoverySnippet,
+  formatRecoverySeparator,
+  isKilledExit,
+  main,
+  recoverKilledExecution,
+};
+
+// After `module.exports`: run as the watcher's entry point, recovery starts the
+// next watcher through `docker-cleanup.js`, which requires this module back
+// and needs its exports complete.
+if (require.main === module) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/src/lib/execution-resume.js

@@ -25,6 +25,12 @@ const {
   startDetachedDockerCompletionWatcher,
 } = require('./docker-cleanup');
 const { escapeForLinksNotation } = require('./output-blocks');
+const {
+  buildResourceLimitsStatusLine,
+  normalizeResourceLimits,
+  readDockerResourceLimits,
+} = require('./docker-resource-limits');
+const { appendLogFile } = require('./isolation-log-utils');
 const { runCommand } = require('./execution-control');
 const { SessionState, probeSession } = require('./session-probe');
 const { getDockerNetworks } = require('./docker-network-lifecycle');
@@ -95,6 +101,8 @@ function buildLaunchOptions(record) {
     mounts: opts.mounts || [],
     networks: opts.networks || [],
     networkAliases: opts.networkAliases || [],
+    // Limits captured from the container on an earlier resume (issue #176).
+    resourceLimits: normalizeResourceLimits(opts.resourceLimits),
     // Append to the same log so one logical session keeps one gap-free record.
     logPath: record.logPath || null,
   };
@@ -152,9 +160,16 @@ function buildSnapshotStartSteps({
  * @param {object} record - Execution record
  * @param {?string} newCommand - Replacement command (from `-- <command>`)
  * @param {object} probe - Result of `probeSession`
+ * @param {?string[]} [liveResourceLimits] - Limits read from the stopped
+ *   container with `docker inspect`; falls back to the stored ones when null
  * @returns {object} Resume plan, or {error}
  */
-function buildResumePlan(record, newCommand, probe = {}) {
+function buildResumePlan(
+  record,
+  newCommand,
+  probe = {},
+  liveResourceLimits = null
+) {
   const opts = (record && record.options) || {};
   const backend = opts.isolated;
   const sessionName = opts.sessionName;
@@ -208,6 +223,11 @@ function buildResumePlan(record, newCommand, probe = {}) {
     // Lazily required: isolation.js pulls in this module's siblings, so a
     // top-level require would create a cycle.
     const { buildDockerRuntimeArgs } = require('./isolation');
+    // `docker commit` keeps the filesystem but not the HostConfig, so the
+    // limits must be passed to the new container explicitly (issue #176).
+    const resourceLimits = normalizeResourceLimits(
+      liveResourceLimits || opts.resourceLimits
+    );
     const snapshotImage = buildSnapshotImageName(sessionName, attempt);
     const newSessionName = buildResumedSessionName(sessionName, attempt);
     return {
@@ -218,6 +238,7 @@ function buildResumePlan(record, newCommand, probe = {}) {
       snapshotImage,
       command,
       attempt,
+      resourceLimits,
       steps: [
         {
           command: getDockerCommand(),
@@ -229,7 +250,7 @@ function buildResumePlan(record, newCommand, probe = {}) {
             '--name',
             newSessionName,
             ...(opts.user ? ['--user', opts.user] : []),
-            ...buildDockerRuntimeArgs(opts),
+            ...buildDockerRuntimeArgs({ ...opts, resourceLimits }),
             snapshotImage,
             'sh',
             '-c',
@@ -280,6 +301,11 @@ function formatResumeResultAsLinksNotation(result) {
       `  snapshotImage ${escapeForLinksNotation(result.snapshotImage)}`
     );
   }
+  if (result.resourceLimits && result.resourceLimits.length > 0) {
+    lines.push(
+      `  resourceLimits ${escapeForLinksNotation(result.resourceLimits.join(' '))}`
+    );
+  }
   lines.push(`  command ${escapeForLinksNotation(result.command)}`);
   lines.push(`  message ${escapeForLinksNotation(result.message)}`);
   return lines.join('\n');
@@ -296,6 +322,9 @@ function formatResumeResult(result, outputFormat) {
       `Backend:       ${result.backend}`,
       `Session Name:  ${result.sessionName}`,
       `Command:       ${result.command}`,
+      ...(result.resourceLimits && result.resourceLimits.length > 0
+        ? [`Resource Limits: ${result.resourceLimits.join(' ')}`]
+        : []),
       result.message,
     ].join('\n');
   }
@@ -314,6 +343,11 @@ function applyResumeToRecord(record, plan, containerId) {
   const options = { ...(record.options || {}) };
   options.resumeCount = plan.attempt;
   options.resumedAt = new Date().toISOString();
+  // A resume is a new deliberate start: launch-time recovery applies again.
+  delete options.stopRequestedAt;
+  if (plan.resourceLimits && plan.resourceLimits.length > 0) {
+    options.resourceLimits = plan.resourceLimits;
+  }
 
   if (plan.newSessionName) {
     options.sessionNameHistory = [
@@ -365,7 +399,25 @@ async function resumeExecution(store, identifier, deps = {}) {
 
   const runner = deps.runner || runCommand;
   const probeFn = deps.probe || ((r) => probeSession(r, runner));
-  const plan = buildResumePlan(record, deps.command || null, probeFn(record));
+  const probe = probeFn(record);
+  const opts = record.options || {};
+  // Only a snapshot resume creates a new container, so only it needs the old
+  // container's limits (issue #176).
+  const liveResourceLimits =
+    deps.command &&
+    opts.isolated === 'docker' &&
+    opts.sessionName &&
+    probe &&
+    !probe.alive &&
+    probe.state === SessionState.STOPPED
+      ? readDockerResourceLimits(opts.sessionName, runner)
+      : null;
+  const plan = buildResumePlan(
+    record,
+    deps.command || null,
+    probe,
+    liveResourceLimits
+  );
   if (plan.error) {
     return { success: false, error: plan.error };
   }
@@ -404,6 +456,11 @@ async function resumeExecution(store, identifier, deps = {}) {
       containerId = (result.stdout || '').trim() || containerId;
     }
 
+    const limitsLine = buildResourceLimitsStatusLine(plan.resourceLimits);
+    if (limitsLine && record.logPath) {
+      appendLogFile(record.logPath, `${limitsLine}\n`);
+    }
+
     // The completion watcher died with the previous run (or with the
     // supervisor), so a new one must follow the resumed container.
     const startWatcher =
@@ -412,7 +469,13 @@ async function resumeExecution(store, identifier, deps = {}) {
       activeSessionName(plan),
       getDockerContainerCleanupPolicy(record.options || {}),
       record.logPath || null,
-      record.uuid || null
+      record.uuid || null,
+      // `docker start` re-runs the launch-time selector, so the session keeps
+      // its kill recovery (issue #176); a snapshot container has no selector.
+      {
+        recoverOnKill:
+          plan.mode === ResumeMode.DOCKER_START && Boolean(opts.onKillResume),
+      }
     );
   }
 
@@ -431,6 +494,7 @@ async function resumeExecution(store, identifier, deps = {}) {
         sessionName: updated.options.sessionName,
         previousSessionName,
         snapshotImage: plan.snapshotImage || null,
+        resourceLimits: plan.resourceLimits || null,
         command: plan.command,
         message: plan.message,
       },

package/src/lib/isolation.js

@@ -505,96 +505,12 @@ const {
   canRunLinuxDockerImages,
 } = require('./docker-utils');
 
-/**
- * Build the docker run runtime argument list contributed by configurable
- * container options: privileged mode, environment variables, volumes/bind
- * mounts, --mount specs, and network configuration. Returned in a stable order so they can be spliced
- * into the `docker run` argv before the image name.
- * @param {object} options - Options (privileged, env, volumes, mounts, network, networkAliases)
- * @returns {string[]} Docker CLI arguments
- */
-function buildDockerRuntimeArgs(options = {}) {
-  const args = [];
-  if (options.privileged) {
-    args.push('--privileged');
-  }
-  for (const envVar of options.env || []) {
-    args.push('-e', envVar);
-  }
-  for (const volume of options.volumes || []) {
-    args.push('-v', volume);
-  }
-  for (const mount of options.mounts || []) {
-    args.push('--mount', mount);
-  }
-  const [firstNetwork] = getDockerNetworks(options);
-  if (firstNetwork) {
-    args.push('--network', firstNetwork);
-  }
-  for (const alias of options.networkAliases || []) {
-    args.push('--network-alias', alias);
-  }
-  return args;
-}
-
-/**
- * Build the human-readable `[Isolation]` status lines for docker runtime
- * options (volumes, mounts, env, privileged). Empty collections and a falsy
- * privileged flag contribute no lines.
- * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases)
- * @returns {string[]} Status lines for the start block / log header
- */
-function buildDockerRuntimeStatusLines(options = {}) {
-  const lines = [];
-  if (options.volumes && options.volumes.length > 0) {
-    lines.push(`[Isolation] Volumes: ${options.volumes.join(', ')}`);
-  }
-  if (options.mounts && options.mounts.length > 0) {
-    lines.push(`[Isolation] Mounts: ${options.mounts.join(', ')}`);
-  }
-  if (options.env && options.env.length > 0) {
-    lines.push(`[Isolation] Env: ${options.env.join(', ')}`);
-  }
-  if (options.privileged) {
-    lines.push(`[Isolation] Privileged: true`);
-  }
-  const networks = getDockerNetworks(options);
-  if (networks.length > 0) {
-    lines.push(`[Isolation] Network: ${networks[0]}`);
-  }
-  if (networks.length > 1) {
-    lines.push(`[Isolation] Networks: ${networks.join(', ')}`);
-  }
-  if (options.networkAliases && options.networkAliases.length > 0) {
-    lines.push(
-      `[Isolation] Network aliases: ${options.networkAliases.join(', ')}`
-    );
-  }
-  return lines;
-}
-
-/**
- * Build the execution-record metadata for docker runtime options, normalizing
- * empty collections and a falsy privileged flag to `null`.
- * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases)
- * @returns {{volumes: ?string[], mounts: ?string[], env: ?string[], privileged: ?boolean, network: ?string, networks: ?string[], networkAliases: ?string[]}}
- */
-function buildDockerRuntimeMetadata(options = {}) {
-  const networks = getDockerNetworks(options);
-  return {
-    volumes:
-      options.volumes && options.volumes.length > 0 ? options.volumes : null,
-    mounts: options.mounts && options.mounts.length > 0 ? options.mounts : null,
-    env: options.env && options.env.length > 0 ? options.env : null,
-    privileged: options.privileged || null,
-    network: networks[0] || null,
-    networks: networks.length > 0 ? networks : null,
-    networkAliases:
-      options.networkAliases && options.networkAliases.length > 0
-        ? options.networkAliases
-        : null,
-  };
-}
+const {
+  buildDockerRuntimeArgs,
+  buildDockerRuntimeStatusLines,
+  buildDockerRuntimeMetadata,
+} = require('./docker-runtime-args');
+const { buildRecoverySelectorArgs } = require('./execution-recovery');
 
 /**
  * Run command in Docker container
@@ -679,11 +595,22 @@ function runInDocker(command, options = {}) {
       const shellArgs = shellInteractiveFlag
         ? [shellToUse, shellInteractiveFlag]
         : [shellToUse];
-      const cmdArgs = isBareShell
+      const mainCmdArgs = isBareShell
         ? toShellWords(command)
         : isShellInvocationWithArgs(command)
           ? buildShellWithArgsCmdArgs(effectiveCommand)
           : [...shellArgs, '-c', effectiveCommand];
+      // A recovery command is selected by a marker that `docker cp` drops
+      // into the container before it is started again (issue #176).
+      const cmdArgs = options.recoveryCommand
+        ? buildRecoverySelectorArgs(mainCmdArgs, {
+            shell: shellToUse,
+            shellFlag: shellInteractiveFlag,
+            recoveryCommand: options.keepAlive
+              ? `${options.recoveryCommand}; exec ${shellToUse}`
+              : options.recoveryCommand,
+          })
+        : mainCmdArgs;
       dockerArgs.push(options.image, ...cmdArgs);
 
       if (DEBUG) {
@@ -736,7 +663,8 @@ function runInDocker(command, options = {}) {
         containerName,
         cleanupPolicy,
         options.logPath,
-        options.executionId || null
+        options.executionId || null,
+        { recoverOnKill: Boolean(options.onKillResume) }
       );
 
       let message = `Command started in detached docker container: ${containerName}`;

package/src/lib/usage.js

@@ -16,6 +16,8 @@ Options:
   --privileged          Run docker container in privileged mode (docker only)
   --network <name>      Connect to a named network (repeatable, docker only)
   --network-alias <alias> Add alias to the first network (repeatable, docker only)
+  --on-kill-resume <N>  Resume a killed (exit 137 / OOM) detached docker session up to N times
+  --recovery-command <cmd>  Command to run in the same container on such a resume
   --endpoint <endpoint> SSH endpoint (required for ssh isolation, e.g., user@host)
   --isolated-user, -u [name]  Create isolated user with same permissions
   --keep-user           Keep isolated user after command completes
@@ -66,6 +68,7 @@ Examples:
   $ --resume my-docker-session             # Restart the stored command in the same container
   $ --resume my-docker-session -- bash     # Run a different command in the same container
   $ --resume-all                           # Re-attach every still-running execution
+  $ -i docker -d --on-kill-resume 3 --recovery-command 'solve --resume' -- solve
   $ --use-command-stream echo "Hello"      # Use command-stream library`);
   console.log('');
   console.log('Piping with $:');

package/test/docker-runtime-options.js

@@ -304,6 +304,9 @@ describe('buildDockerRuntimeMetadata', () => {
       network: null,
       networks: null,
       networkAliases: null,
+      resourceLimits: null,
+      onKillResume: null,
+      recoveryCommand: null,
     });
   });
 
@@ -326,6 +329,9 @@ describe('buildDockerRuntimeMetadata', () => {
         network: 'hive-formal-ai',
         networks: ['hive-formal-ai', 'public-egress'],
         networkAliases: ['formal-ai', 'checker'],
+        resourceLimits: null,
+        onKillResume: null,
+        recoveryCommand: null,
       }
     );
   });