start-command 0.34.0 → 0.35.0

package/src/lib/execution-recovery.js

@@ -0,0 +1,350 @@
+/**
+ * Launch-time recovery for killed detached docker sessions (issue #176).
+ *
+ * `$ --isolated docker --detached --on-kill-resume 3 --recovery-command B -- A`
+ * runs `A`; when the main process is killed (exit 137 / `OOMKilled`), the
+ * detached completion watcher hands the inspected facts to this module, which
+ * restarts the *same* container so `B` continues on the same filesystem, with
+ * the same resource limits (they live in the container's HostConfig), under
+ * the same execution UUID and appending to the same log file.
+ *
+ * How `B` replaces `A` inside the same container: a container launched with a
+ * recovery command runs a tiny selector as its entrypoint command. The
+ * selector runs `A` normally, but runs `B` when the marker file exists. Before
+ * `docker start`, the marker (holding the attempt number) is copied into the
+ * stopped container with `docker cp`. Without a recovery command the original
+ * command is simply started again.
+ *
+ * Like `detached-finalize.js`, the entry point runs as a short-lived child of
+ * the watcher on a host whose CLI invocation is long gone, so it never throws.
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const {
+  getDockerCommand,
+  getDockerContainerCleanupPolicy,
+  startDetachedDockerCompletionWatcher,
+} = require('./docker-cleanup');
+const {
+  SHELL_VARS,
+  describeExitCode,
+  formatContainerPostMortem,
+} = require('./docker-post-mortem');
+const {
+  buildResourceLimitsStatusLine,
+  readDockerResourceLimits,
+} = require('./docker-resource-limits');
+const { shellQuote } = require('./isolation-log-utils');
+
+/** File whose presence makes the selector run the recovery command. */
+const RECOVERY_MARKER_PATH = '/.start-command-recovery';
+
+/** Environment variable carrying the attempt number into the recovery command. */
+const RECOVERY_ATTEMPT_ENV = 'START_COMMAND_RECOVERY_ATTEMPT';
+
+/** Exit code of a process killed with SIGKILL (the OOM killer's signal). */
+const KILLED_EXIT_CODE = 137;
+
+/**
+ * POSIX selector run as the container command when a recovery command is set:
+ *   sh -c SELECTOR start-command <shell> <flag> <recovery> <main argv...>
+ */
+const RECOVERY_SELECTOR =
+  's=$1; f=$2; r=$3; shift 3; ' +
+  `if [ -e ${RECOVERY_MARKER_PATH} ]; then ` +
+  `${RECOVERY_ATTEMPT_ENV}=$(cat ${RECOVERY_MARKER_PATH} 2>/dev/null); ` +
+  `export ${RECOVERY_ATTEMPT_ENV}; ` +
+  'if [ -n "$f" ]; then exec "$s" "$f" -c "$r"; fi; exec "$s" -c "$r"; fi; ' +
+  'exec "$@"';
+
+/**
+ * Wrap the container command so a later `docker start` can switch to the
+ * recovery command.
+ * @param {string[]} mainArgs - Container command for the main run
+ * @param {object} params - {shell, shellFlag, recoveryCommand}
+ * @returns {string[]} Container command
+ */
+function buildRecoverySelectorArgs(mainArgs, params) {
+  return [
+    'sh',
+    '-c',
+    RECOVERY_SELECTOR,
+    'start-command',
+    params.shell || 'sh',
+    params.shellFlag || '',
+    params.recoveryCommand,
+    ...mainArgs,
+  ];
+}
+
+/**
+ * @param {number|string|null} exitCode - `.State.ExitCode`
+ * @param {boolean|string|null} oomKilled - `.State.OOMKilled`
+ * @returns {boolean} Whether the main process was killed
+ */
+function isKilledExit(exitCode, oomKilled) {
+  return (
+    describeExitCode(exitCode).code === KILLED_EXIT_CODE ||
+    oomKilled === true ||
+    oomKilled === 'true'
+  );
+}
+
+/**
+ * Shell condition run by the completion watcher: true when the container was
+ * killed and the recovery entry point resumed it.
+ * @param {string} executionId - Execution UUID
+ * @returns {string} POSIX shell condition
+ */
+function buildRecoverySnippet(executionId) {
+  const v = SHELL_VARS;
+  return (
+    `{ [ "$${v.exit}" = ${KILLED_EXIT_CODE} ] || [ "$${v.oom}" = true ]; } && ` +
+    `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
+    `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
+    `"$${v.started}" "$${v.finished}" "$${v.error}" >/dev/null 2>&1`
+  );
+}
+
+/**
+ * The `[Recovery k/N]` separator written into the session log.
+ * @param {object} params - {attempt, maxAttempts, exitCode, oomKilled, containerName, command}
+ * @returns {string} Separator line, newline terminated
+ */
+function formatRecoverySeparator(params) {
+  const described = describeExitCode(params.exitCode);
+  const signal = described.signal ? `, ${described.signal}` : '';
+  const what = params.command
+    ? `running recovery command: ${params.command}`
+    : 'running the original command again';
+  return (
+    `\n[Recovery ${params.attempt}/${params.maxAttempts}] ` +
+    `Main process was killed (exit ${described.code}${signal}, ` +
+    `oomKilled=${params.oomKilled === true || params.oomKilled === 'true'}); ` +
+    `resuming container ${params.containerName}, ${what}\n`
+  );
+}
+
+function appendToLog(logPath, text) {
+  if (!logPath) {
+    return;
+  }
+  try {
+    fs.appendFileSync(logPath, text);
+  } catch {
+    // The log is best-effort here: recovery itself must still proceed.
+  }
+}
+
+function writeRecoveryMarker(containerName, attempt, runner) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'start-recovery-'));
+  const file = path.join(dir, 'marker');
+  try {
+    fs.writeFileSync(file, `${attempt}\n`, { mode: 0o644 });
+    fs.chmodSync(file, 0o644);
+    return runner(getDockerCommand(), [
+      'cp',
+      file,
+      `${containerName}:${RECOVERY_MARKER_PATH}`,
+    ]);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+function failureDetail(result) {
+  return (
+    (result.stderr || '').trim() || result.error || `exit code ${result.status}`
+  );
+}
+
+/**
+ * Resume a killed execution in its own container.
+ * @param {object} params - {store, executionId, exitCode, oomKilled, startedAt,
+ *   finishedAt, containerError, runner, startWatcher, now}
+ * @returns {{recovered: boolean, reason: string, attempt?: number}} Outcome
+ */
+function recoverKilledExecution(params = {}) {
+  const { store, executionId } = params;
+  if (!store || !executionId) {
+    return { recovered: false, reason: 'missing-arguments' };
+  }
+  const record = store.get(executionId);
+  if (!record) {
+    return { recovered: false, reason: 'record-not-found' };
+  }
+  const opts = record.options || {};
+  const maxAttempts = Number(opts.onKillResume) || 0;
+  const containerName = opts.sessionName;
+  if (opts.isolated !== 'docker' || !containerName || maxAttempts < 1) {
+    return { recovered: false, reason: 'not-configured' };
+  }
+  if (!isKilledExit(params.exitCode, params.oomKilled)) {
+    return { recovered: false, reason: 'not-killed' };
+  }
+  const log = (text) => appendToLog(record.logPath, text);
+  if (opts.stopRequestedAt) {
+    log('\n[Recovery] Not resuming: the session was stopped on request.\n');
+    return { recovered: false, reason: 'stop-requested' };
+  }
+  const used = Number(opts.recoveryAttempts) || 0;
+  if (used >= maxAttempts) {
+    log(
+      `\n[Recovery] Not resuming: all ${maxAttempts} recovery attempt(s) used.\n`
+    );
+    return { recovered: false, reason: 'attempts-exhausted' };
+  }
+
+  const runner = params.runner || require('./execution-control').runCommand;
+  const now = params.now || (() => new Date());
+  const attempt = used + 1;
+  const giveUp = (reason) => {
+    log(`[Recovery ${attempt}/${maxAttempts}] Failed: ${reason}\n`);
+    return { recovered: false, reason: 'resume-failed', attempt };
+  };
+
+  log(
+    `\n${formatContainerPostMortem({
+      containerName,
+      exitCode: params.exitCode,
+      oomKilled: params.oomKilled,
+      startedAt: params.startedAt,
+      finishedAt: params.finishedAt,
+      error: params.containerError,
+    })}`
+  );
+  log(
+    formatRecoverySeparator({
+      attempt,
+      maxAttempts,
+      exitCode: params.exitCode,
+      oomKilled: params.oomKilled,
+      containerName,
+      command: opts.recoveryCommand || null,
+    })
+  );
+
+  // Docker keeps `docker update` limits in the HostConfig across restarts;
+  // they are read back so the log and `--status` show what the resumed run
+  // is held to, and so a later relaunch can re-apply them.
+  const resourceLimits = readDockerResourceLimits(containerName, runner);
+  const limitsLine = buildResourceLimitsStatusLine(resourceLimits);
+  if (limitsLine) {
+    log(`${limitsLine}\n`);
+  }
+
+  if (opts.recoveryCommand) {
+    let copied;
+    try {
+      copied = writeRecoveryMarker(containerName, attempt, runner);
+    } catch (err) {
+      copied = { success: false, error: err.message };
+    }
+    if (!copied.success) {
+      return giveUp(`could not mark the container: ${failureDetail(copied)}`);
+    }
+  }
+
+  const since = now().toISOString();
+  const started = runner(getDockerCommand(), ['start', containerName]);
+  if (!started.success) {
+    return giveUp(`docker start failed: ${failureDetail(started)}`);
+  }
+
+  const described = describeExitCode(params.exitCode);
+  record.options = {
+    ...opts,
+    recoveryAttempts: attempt,
+    recoveryHistory: [
+      ...(Array.isArray(opts.recoveryHistory) ? opts.recoveryHistory : []),
+      `${attempt}: exit ${described.code}, oomKilled=${params.oomKilled === true || params.oomKilled === 'true'}, resumed at ${since}`,
+    ],
+    lastRecoveryAt: since,
+    ...(resourceLimits && resourceLimits.length > 0 ? { resourceLimits } : {}),
+  };
+  record.status = 'executing';
+  record.exitCode = null;
+  record.endTime = null;
+  record.exitReason = undefined;
+  record.oomKilled = undefined;
+  try {
+    store.save(record);
+  } catch {
+    // The container is already running again; the new watcher still follows
+    // it and finalizes the record when it ends.
+  }
+
+  const startWatcher =
+    params.startWatcher || startDetachedDockerCompletionWatcher;
+  startWatcher(
+    containerName,
+    getDockerContainerCleanupPolicy(opts),
+    record.logPath || null,
+    record.uuid,
+    { since, recoverOnKill: true }
+  );
+  return { recovered: true, reason: 'resumed', attempt };
+}
+
+/**
+ * Entry point used by the detached watcher:
+ *   <runtime> execution-recovery.js <uuid> <exit> <oom> <started> <finished> <error>
+ * Exits 0 only when the container was resumed; any other outcome lets the
+ * watcher continue with its normal cleanup, footer and finalization.
+ * @param {string[]} argv - Positional arguments
+ * @returns {number} Process exit code
+ */
+function main(argv) {
+  const [executionId, exitCode, oomKilled, startedAt, finishedAt, error] = argv;
+  if (!executionId || process.env.START_DISABLE_TRACKING === 'true') {
+    return 1;
+  }
+  try {
+    const { ExecutionStore } = require('./execution-store');
+    const store = new ExecutionStore(
+      process.env.START_APP_FOLDER
+        ? { appFolder: path.resolve(process.env.START_APP_FOLDER) }
+        : {}
+    );
+    const outcome = recoverKilledExecution({
+      store,
+      executionId,
+      exitCode,
+      oomKilled,
+      startedAt,
+      finishedAt,
+      containerError: error,
+    });
+    return outcome.recovered ? 0 : 1;
+  } catch (err) {
+    // The watcher discards this output; START_DEBUG=1 shows it when the
+    // entry point is run by hand.
+    if (process.env.START_DEBUG === '1' || process.env.START_DEBUG === 'true') {
+      console.error(`[DEBUG] execution-recovery failed: ${err.stack || err}`);
+    }
+    return 1;
+  }
+}
+
+module.exports = {
+  KILLED_EXIT_CODE,
+  RECOVERY_ATTEMPT_ENV,
+  RECOVERY_MARKER_PATH,
+  RECOVERY_SELECTOR,
+  buildRecoverySelectorArgs,
+  buildRecoverySnippet,
+  formatRecoverySeparator,
+  isKilledExit,
+  main,
+  recoverKilledExecution,
+};
+
+// After `module.exports`: run as the watcher's entry point, recovery starts the
+// next watcher through `docker-cleanup.js`, which requires this module back
+// and needs its exports complete.
+if (require.main === module) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/src/lib/execution-resume.js

@@ -25,6 +25,12 @@ const {
   startDetachedDockerCompletionWatcher,
 } = require('./docker-cleanup');
 const { escapeForLinksNotation } = require('./output-blocks');
+const {
+  buildResourceLimitsStatusLine,
+  normalizeResourceLimits,
+  readDockerResourceLimits,
+} = require('./docker-resource-limits');
+const { appendLogFile } = require('./isolation-log-utils');
 const { runCommand } = require('./execution-control');
 const { SessionState, probeSession } = require('./session-probe');
 const { getDockerNetworks } = require('./docker-network-lifecycle');
@@ -95,6 +101,8 @@ function buildLaunchOptions(record) {
     mounts: opts.mounts || [],
     networks: opts.networks || [],
     networkAliases: opts.networkAliases || [],
+    // Limits captured from the container on an earlier resume (issue #176).
+    resourceLimits: normalizeResourceLimits(opts.resourceLimits),
     // Append to the same log so one logical session keeps one gap-free record.
     logPath: record.logPath || null,
   };
@@ -152,9 +160,16 @@ function buildSnapshotStartSteps({
  * @param {object} record - Execution record
  * @param {?string} newCommand - Replacement command (from `-- <command>`)
  * @param {object} probe - Result of `probeSession`
+ * @param {?string[]} [liveResourceLimits] - Limits read from the stopped
+ *   container with `docker inspect`; falls back to the stored ones when null
  * @returns {object} Resume plan, or {error}
  */
-function buildResumePlan(record, newCommand, probe = {}) {
+function buildResumePlan(
+  record,
+  newCommand,
+  probe = {},
+  liveResourceLimits = null
+) {
   const opts = (record && record.options) || {};
   const backend = opts.isolated;
   const sessionName = opts.sessionName;
@@ -208,6 +223,11 @@ function buildResumePlan(record, newCommand, probe = {}) {
     // Lazily required: isolation.js pulls in this module's siblings, so a
     // top-level require would create a cycle.
     const { buildDockerRuntimeArgs } = require('./isolation');
+    // `docker commit` keeps the filesystem but not the HostConfig, so the
+    // limits must be passed to the new container explicitly (issue #176).
+    const resourceLimits = normalizeResourceLimits(
+      liveResourceLimits || opts.resourceLimits
+    );
     const snapshotImage = buildSnapshotImageName(sessionName, attempt);
     const newSessionName = buildResumedSessionName(sessionName, attempt);
     return {
@@ -218,6 +238,7 @@ function buildResumePlan(record, newCommand, probe = {}) {
       snapshotImage,
       command,
       attempt,
+      resourceLimits,
       steps: [
         {
           command: getDockerCommand(),
@@ -229,7 +250,7 @@ function buildResumePlan(record, newCommand, probe = {}) {
             '--name',
             newSessionName,
             ...(opts.user ? ['--user', opts.user] : []),
-            ...buildDockerRuntimeArgs(opts),
+            ...buildDockerRuntimeArgs({ ...opts, resourceLimits }),
             snapshotImage,
             'sh',
             '-c',
@@ -280,6 +301,11 @@ function formatResumeResultAsLinksNotation(result) {
       `  snapshotImage ${escapeForLinksNotation(result.snapshotImage)}`
     );
   }
+  if (result.resourceLimits && result.resourceLimits.length > 0) {
+    lines.push(
+      `  resourceLimits ${escapeForLinksNotation(result.resourceLimits.join(' '))}`
+    );
+  }
   lines.push(`  command ${escapeForLinksNotation(result.command)}`);
   lines.push(`  message ${escapeForLinksNotation(result.message)}`);
   return lines.join('\n');
@@ -296,6 +322,9 @@ function formatResumeResult(result, outputFormat) {
       `Backend:       ${result.backend}`,
       `Session Name:  ${result.sessionName}`,
       `Command:       ${result.command}`,
+      ...(result.resourceLimits && result.resourceLimits.length > 0
+        ? [`Resource Limits: ${result.resourceLimits.join(' ')}`]
+        : []),
       result.message,
     ].join('\n');
   }
@@ -314,6 +343,11 @@ function applyResumeToRecord(record, plan, containerId) {
   const options = { ...(record.options || {}) };
   options.resumeCount = plan.attempt;
   options.resumedAt = new Date().toISOString();
+  // A resume is a new deliberate start: launch-time recovery applies again.
+  delete options.stopRequestedAt;
+  if (plan.resourceLimits && plan.resourceLimits.length > 0) {
+    options.resourceLimits = plan.resourceLimits;
+  }
 
   if (plan.newSessionName) {
     options.sessionNameHistory = [
@@ -365,7 +399,25 @@ async function resumeExecution(store, identifier, deps = {}) {
 
   const runner = deps.runner || runCommand;
   const probeFn = deps.probe || ((r) => probeSession(r, runner));
-  const plan = buildResumePlan(record, deps.command || null, probeFn(record));
+  const probe = probeFn(record);
+  const opts = record.options || {};
+  // Only a snapshot resume creates a new container, so only it needs the old
+  // container's limits (issue #176).
+  const liveResourceLimits =
+    deps.command &&
+    opts.isolated === 'docker' &&
+    opts.sessionName &&
+    probe &&
+    !probe.alive &&
+    probe.state === SessionState.STOPPED
+      ? readDockerResourceLimits(opts.sessionName, runner)
+      : null;
+  const plan = buildResumePlan(
+    record,
+    deps.command || null,
+    probe,
+    liveResourceLimits
+  );
   if (plan.error) {
     return { success: false, error: plan.error };
   }
@@ -404,6 +456,11 @@ async function resumeExecution(store, identifier, deps = {}) {
       containerId = (result.stdout || '').trim() || containerId;
     }
 
+    const limitsLine = buildResourceLimitsStatusLine(plan.resourceLimits);
+    if (limitsLine && record.logPath) {
+      appendLogFile(record.logPath, `${limitsLine}\n`);
+    }
+
     // The completion watcher died with the previous run (or with the
     // supervisor), so a new one must follow the resumed container.
     const startWatcher =
@@ -412,7 +469,13 @@ async function resumeExecution(store, identifier, deps = {}) {
       activeSessionName(plan),
       getDockerContainerCleanupPolicy(record.options || {}),
       record.logPath || null,
-      record.uuid || null
+      record.uuid || null,
+      // `docker start` re-runs the launch-time selector, so the session keeps
+      // its kill recovery (issue #176); a snapshot container has no selector.
+      {
+        recoverOnKill:
+          plan.mode === ResumeMode.DOCKER_START && Boolean(opts.onKillResume),
+      }
     );
   }
 
@@ -431,6 +494,7 @@ async function resumeExecution(store, identifier, deps = {}) {
         sessionName: updated.options.sessionName,
         previousSessionName,
         snapshotImage: plan.snapshotImage || null,
+        resourceLimits: plan.resourceLimits || null,
         command: plan.command,
         message: plan.message,
       },

package/src/lib/exit-reason.js

@@ -17,6 +17,8 @@
  * `exitCode` or `oomKilled`.
  */
 
+const os = require('os');
+
 /**
  * Reasons carrying this prefix report memory exhaustion, whatever the mechanism
  * (runtime self-abort, kernel OOM killer, failed allocation).
@@ -182,6 +184,27 @@ function resolveMemoryExhaustion(input) {
   return null;
 }
 
+/**
+ * Exit code of a child process that has finished.
+ *
+ * Node reports `code === null` for a child killed by a signal, and the
+ * `code || 0` this replaces turned that into a successful exit 0 (issue #174).
+ * A signal death follows the shell's `128 + n` convention, so the result
+ * decodes back to the signal through `describeExitCode()`; a missing code with
+ * no known signal is a failure (1), never a success.
+ *
+ * @param {number|null|undefined} code - `code` from `exit`/`close`
+ * @param {string|null} [signal] - `signal` from `exit`/`close`, e.g. `SIGKILL`
+ * @returns {number} Exit code
+ */
+function resolveChildExitCode(code, signal = null) {
+  if (typeof code === 'number' && Number.isFinite(code)) {
+    return code;
+  }
+  const number = signal ? os.constants.signals[signal] : undefined;
+  return typeof number === 'number' ? 128 + number : 1;
+}
+
 /**
  * Map a shell exit code to the signal name it encodes.
  * @param {number|null|undefined} exitCode - Terminal exit code
@@ -259,6 +282,7 @@ module.exports = {
   findExitReasonMarker,
   resolveMemoryExhaustion,
   describeExitCode,
+  resolveChildExitCode,
   resolveExitReason,
   signalNameForExitCode,
 };

package/src/lib/isolation-log-utils.js

@@ -4,7 +4,7 @@ const { spawn } = require('child_process');
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
-const { describeExitCode } = require('./exit-reason');
+const { describeExitCode, resolveChildExitCode } = require('./exit-reason');
 
 function getTempRoot() {
   return process.env.START_TEMP_ROOT || path.join(os.tmpdir(), 'start-command');
@@ -190,11 +190,15 @@ function runAsIsolatedUser(cmd, username) {
       stdio: 'inherit',
     });
 
-    child.on('exit', (code) => {
+    child.on('exit', (code, signal) => {
+      const exitCode = resolveChildExitCode(code, signal);
       resolve({
-        success: code === 0,
-        message: `Command completed as user "${username}" with exit code ${code}`,
-        exitCode: code || 0,
+        success: exitCode === 0,
+        message: signal
+          ? `Command as user "${username}" was killed by ${signal} (exit code ${exitCode})`
+          : `Command completed as user "${username}" with exit code ${exitCode}`,
+        exitCode,
+        signal: signal || null,
       });
     });