start-command 0.34.1 → 0.35.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # start-command
 
+## 0.35.0
+
+### Minor Changes
+
+- 9679323: Keep Docker resource limits across `--resume <id> -- <command>`, and add `--on-kill-resume <N>` / `--recovery-command <cmd>`. `docker commit` does not capture HostConfig, so the snapshot resume now reads the stopped container's limits with `docker inspect` (including ones applied later with `docker update`) and re-applies the non-default ones (`--memory`, `--memory-swap`, `--memory-reservation`, `--cpus`/`--cpu-quota`/`--cpu-period`, `--cpu-shares`, `--cpuset-cpus`, `--cpuset-mems`, `--pids-limit`, `--shm-size`, `--storage-opt`, `--ulimit`) to the `-resume-N` container, prints them as `[Isolation] Resource limits: ...` and stores them as `resourceLimits`. With `--on-kill-resume <N>` a detached Docker execution whose main process is killed (exit 137 or `OOMKilled`) is resumed in the same container up to N times, running `--recovery-command` (or the original command) with `START_COMMAND_RECOVERY_ATTEMPT` set. The UUID and log file are kept, each attempt is separated by a `[Recovery k/N]` line, and `--status` shows `recoveryAttempts` and `recoveryHistory`. `--stop` cancels further recovery.
+
 ## 0.34.1
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.34.1",
+  "version": "0.35.0",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {
@@ -72,7 +72,7 @@
     "lino-objects-codec": "^0.4.0"
   },
   "overrides": {
-    "brace-expansion": "^5.0.9",
+    "brace-expansion": "^5.0.12",
     "flatted": "^3.4.2",
     "js-yaml": "^4.3.2"
   }

package/src/lib/args-parser.js

@@ -43,6 +43,7 @@
 
 const { getDefaultDockerImage } = require('./docker-utils');
 const dockerNetworkOptions = require('./docker-network-options');
+const dockerRecoveryOptions = require('./docker-recovery-options');
 const { parseSequence, isSequence } = require('./sequence-parser');
 const { buildCommandString } = require('./shell-utils');
 const {
@@ -194,6 +195,8 @@ function parseArgs(args) {
     network: null, // First Docker network name (compatibility accessor)
     networks: [], // Ordered Docker network names
     networkAliases: [], // Docker network-scoped aliases
+    onKillResume: null, // Resume up to N times when the docker main process is killed (issue #176)
+    recoveryCommand: null, // Command run in the same container on such a resume (issue #176)
     endpoint: null, // SSH endpoint (current level, e.g., user@host)
     endpointStack: null, // SSH endpoints for each level (with nulls for non-ssh levels)
     user: false, // Create isolated user
@@ -410,6 +413,11 @@ function parseOption(args, index, options) {
     return networkOption;
   }
 
+  const recoveryOption = dockerRecoveryOptions.parse(args, index, options);
+  if (recoveryOption) {
+    return recoveryOption;
+  }
+
   // --endpoint (for ssh) - supports sequence for stacked isolation
   if (arg === '--endpoint') {
     if (index + 1 < args.length && !args[index + 1].startsWith('-')) {
@@ -746,6 +754,8 @@ function validateOptions(options) {
     validateDockerRuntimeOptionsRequireDocker(options);
   }
 
+  dockerRecoveryOptions.validate(options);
+
   // Session name is only valid with isolation
   if (options.session && !options.isolated) {
     throw new Error('--session option is only valid with --isolated');

package/src/lib/docker-cleanup.js

@@ -320,17 +320,27 @@ function buildSuccessfulNonOomCondition() {
  * container that is somehow still running afterwards is never removed, gets no
  * `Exit Code:` footer and is never finalized — its record stays `executing`.
  *
+ * With `recoverOnKill` (issue #176), a killed main process (exit 137 or
+ * `OOMKilled`) is first handed to the recovery entry point. When it resumes
+ * the container, this watcher stops here — a new one follows the resumed run —
+ * and cleanup, footer and finalization are left to whichever watcher sees the
+ * last run end.
+ *
  * @param {string} containerName - Docker container name
  * @param {string} policy - One of DOCKER_CONTAINER_CLEANUP_POLICY
  * @param {string|null} logPath - Log file to append to, or null
  * @param {string|null} [executionId] - Execution UUID to finalize, or null
+ * @param {{since?: string, recoverOnKill?: boolean}} [watcherOptions] -
+ *   `since` limits `docker logs` to output after a restart, so a resumed run
+ *   does not copy the previous run's output into the log again
  * @returns {string} The shell script
  */
 function buildDetachedDockerCompletionScript(
   containerName,
   policy,
   logPath,
-  executionId = null
+  executionId = null,
+  watcherOptions = {}
 ) {
   const quotedName = shellQuote(containerName);
   const parts = [];
@@ -351,7 +361,10 @@ function buildDetachedDockerCompletionScript(
     const remove = `docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true; ${removalNote}`;
     const keep = `${postMortem}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
 
-    parts.push(`docker logs -f ${quotedName} >> ${quotedLogPath} 2>&1`);
+    const since = watcherOptions.since
+      ? ` --since ${shellQuote(watcherOptions.since)}`
+      : '';
+    parts.push(`docker logs -f${since} ${quotedName} >> ${quotedLogPath} 2>&1`);
     parts.push(buildDockerWaitForExitSnippet(containerName, quotedLogPath));
     parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
@@ -394,9 +407,15 @@ function buildDetachedDockerCompletionScript(
   const stillRunning = logPath
     ? buildDockerStillRunningNoteSnippet(containerName, shellQuote(logPath))
     : ':';
+  let recovery = '';
+  if (executionId && watcherOptions.recoverOnKill) {
+    // Lazily required: the recovery module starts watchers itself.
+    const { buildRecoverySnippet } = require('./execution-recovery');
+    recovery = `elif ${buildRecoverySnippet(executionId)}; then :; `;
+  }
   parts.push(
     `if [ "$${SHELL_VARS.running}" = true ]; then ${stillRunning}; ` +
-      `else ${exited.length ? exited.join('; ') : ':'}; fi`
+      `${recovery}else ${exited.length ? exited.join('; ') : ':'}; fi`
   );
 
   return parts.join('; ');
@@ -408,13 +427,16 @@ function buildDetachedDockerCompletionScript(
  * @param {string} policy - One of DOCKER_CONTAINER_CLEANUP_POLICY
  * @param {string|null} logPath - Log file to append to, or null
  * @param {string|null} [executionId] - Execution UUID to finalize, or null
+ * @param {{since?: string, recoverOnKill?: boolean}} [watcherOptions] - See
+ *   `buildDetachedDockerCompletionScript`
  * @returns {void}
  */
 function startDetachedDockerCompletionWatcher(
   containerName,
   policy,
   logPath,
-  executionId = null
+  executionId = null,
+  watcherOptions = {}
 ) {
   const watcher = spawn(
     'sh',
@@ -424,7 +446,8 @@ function startDetachedDockerCompletionWatcher(
         containerName,
         policy,
         logPath,
-        executionId
+        executionId,
+        watcherOptions
       ),
     ],
     {

package/src/lib/docker-recovery-options.js

@@ -0,0 +1,84 @@
+/**
+ * Parse and validate the launch-time recovery options (issue #176):
+ *   --on-kill-resume <N>        resume up to N times when the main process is
+ *                               killed (exit 137 / OOMKilled)
+ *   --recovery-command <cmd>    command to run in the same container on resume
+ *                               (without it, the original command is re-run)
+ */
+
+function parseOnKillResumeValue(value) {
+  const text = String(value).trim();
+  const count = Number(text);
+  if (!/^\d+$/.test(text) || !Number.isSafeInteger(count) || count < 1) {
+    throw new Error(
+      `Invalid --on-kill-resume value: "${value}". Expected a positive integer.`
+    );
+  }
+  return count;
+}
+
+function parseDockerRecoveryOption(args, index, options) {
+  const arg = args[index];
+  if (arg === '--on-kill-resume' || arg === '--recovery-command') {
+    if (index + 1 >= args.length || args[index + 1].startsWith('-')) {
+      const value = arg === '--on-kill-resume' ? 'count' : 'command';
+      throw new Error(`Option ${arg} requires a ${value} argument`);
+    }
+    if (arg === '--on-kill-resume') {
+      options.onKillResume = parseOnKillResumeValue(args[index + 1]);
+    } else {
+      options.recoveryCommand = args[index + 1];
+    }
+    return 2;
+  }
+  if (arg.startsWith('--on-kill-resume=')) {
+    options.onKillResume = parseOnKillResumeValue(
+      arg.slice('--on-kill-resume='.length)
+    );
+    return 1;
+  }
+  if (arg.startsWith('--recovery-command=')) {
+    options.recoveryCommand = arg.slice('--recovery-command='.length);
+    return 1;
+  }
+  return 0;
+}
+
+/**
+ * Recovery is driven by the detached docker completion watcher, so it needs a
+ * detached session whose only isolation level is docker. A recovery command on
+ * its own implies one attempt.
+ * @param {object} options - Parsed options
+ * @throws {Error} If the recovery options are used where they cannot work
+ */
+function validateDockerRecoveryOptions(options) {
+  if (
+    options.recoveryCommand !== null &&
+    options.recoveryCommand !== undefined
+  ) {
+    if (String(options.recoveryCommand).trim() === '') {
+      throw new Error('--recovery-command requires a non-empty command');
+    }
+    options.onKillResume ??= 1;
+  }
+  if (!options.onKillResume) {
+    return;
+  }
+  const flag = options.recoveryCommand
+    ? '--recovery-command'
+    : '--on-kill-resume';
+  const stack = options.isolatedStack || [options.isolated];
+  if (!options.isolated || stack.length !== 1 || stack[0] !== 'docker') {
+    throw new Error(
+      `${flag} option is only valid with --isolated docker as the only isolation level`
+    );
+  }
+  if (!options.detached) {
+    throw new Error(`${flag} option requires --detached`);
+  }
+}
+
+module.exports = {
+  parse: parseDockerRecoveryOption,
+  validate: validateDockerRecoveryOptions,
+};

package/src/lib/docker-resource-limits.js

@@ -0,0 +1,211 @@
+/**
+ * Docker resource limits that must survive a resume (issue #176).
+ *
+ * `docker commit` captures a container's filesystem and image config, but not
+ * its HostConfig. A snapshot-based resume (`$ --resume <id> -- <cmd>`) used to
+ * start the replacement container without the memory, CPU and PIDs limits the
+ * original one had — limits a supervisor typically applied with
+ * `docker update`, so they were never part of the stored launch options.
+ *
+ * This module reads the limits back out of `docker inspect` and translates the
+ * non-default ones into `docker run` flags. The flags are kept in the
+ * `--flag=value` form, so the same list is stored in the execution record,
+ * shown in `[Isolation]` status lines, and spliced into `docker run` as is.
+ */
+
+const { getDockerCommand } = require('./docker-cleanup');
+
+/** Docker's default `/dev/shm` size; only a different size is re-applied. */
+const DEFAULT_SHM_SIZE = 64 * 1024 * 1024;
+
+const KIB = 1024;
+const MIB = KIB * 1024;
+const GIB = MIB * 1024;
+
+/**
+ * Format a byte count with the largest binary unit `docker run` accepts that
+ * represents it exactly (`268435456` → `256m`).
+ * @param {number} bytes - Byte count
+ * @returns {string} Docker memory size
+ */
+function formatDockerBytes(bytes) {
+  for (const [unit, size] of [
+    ['g', GIB],
+    ['m', MIB],
+    ['k', KIB],
+  ]) {
+    if (bytes >= size && bytes % size === 0) {
+      return `${bytes / size}${unit}`;
+    }
+  }
+  return String(bytes);
+}
+
+/**
+ * Format `NanoCpus` as the decimal `--cpus` value (`500000000` → `0.5`).
+ * @param {number} nanoCpus - CPU quota in units of 1e-9 CPUs
+ * @returns {string} Docker CPU count
+ */
+function formatDockerCpus(nanoCpus) {
+  return String(Number((nanoCpus / 1e9).toFixed(9)));
+}
+
+function positive(value) {
+  const number = Number(value);
+  return Number.isFinite(number) && number > 0 ? number : null;
+}
+
+function nonEmpty(value) {
+  return typeof value === 'string' && value.trim() !== '' ? value.trim() : null;
+}
+
+/**
+ * Translate an inspected `HostConfig` into `docker run` flags.
+ *
+ * Only limits that differ from Docker's defaults are emitted, and only in
+ * combinations `docker run` accepts: `--memory-swap` needs `--memory`, and
+ * `--cpus` cannot be combined with `--cpu-quota`/`--cpu-period`.
+ *
+ * @param {?object} hostConfig - `docker inspect` `.HostConfig`
+ * @returns {string[]} Flags in `--flag=value` form
+ */
+function parseDockerResourceLimits(hostConfig) {
+  if (!hostConfig || typeof hostConfig !== 'object') {
+    return [];
+  }
+  const h = hostConfig;
+  const limits = [];
+
+  const memory = positive(h.Memory);
+  if (memory) {
+    limits.push(`--memory=${formatDockerBytes(memory)}`);
+    const swap = Number(h.MemorySwap);
+    if (swap === -1) {
+      limits.push('--memory-swap=-1');
+    } else if (positive(swap)) {
+      limits.push(`--memory-swap=${formatDockerBytes(swap)}`);
+    }
+  }
+  const reservation = positive(h.MemoryReservation);
+  if (reservation) {
+    limits.push(`--memory-reservation=${formatDockerBytes(reservation)}`);
+  }
+
+  const nanoCpus = positive(h.NanoCpus);
+  if (nanoCpus) {
+    limits.push(`--cpus=${formatDockerCpus(nanoCpus)}`);
+  } else {
+    const quota = positive(h.CpuQuota);
+    if (quota) {
+      limits.push(`--cpu-quota=${quota}`);
+    }
+    const period = positive(h.CpuPeriod);
+    if (period) {
+      limits.push(`--cpu-period=${period}`);
+    }
+  }
+  const shares = positive(h.CpuShares);
+  if (shares) {
+    limits.push(`--cpu-shares=${shares}`);
+  }
+  const cpusetCpus = nonEmpty(h.CpusetCpus);
+  if (cpusetCpus) {
+    limits.push(`--cpuset-cpus=${cpusetCpus}`);
+  }
+  const cpusetMems = nonEmpty(h.CpusetMems);
+  if (cpusetMems) {
+    limits.push(`--cpuset-mems=${cpusetMems}`);
+  }
+
+  const pids = positive(h.PidsLimit);
+  if (pids) {
+    limits.push(`--pids-limit=${pids}`);
+  }
+
+  const shm = positive(h.ShmSize);
+  if (shm && shm !== DEFAULT_SHM_SIZE) {
+    limits.push(`--shm-size=${formatDockerBytes(shm)}`);
+  }
+
+  if (h.StorageOpt && typeof h.StorageOpt === 'object') {
+    for (const key of Object.keys(h.StorageOpt).sort()) {
+      const value = nonEmpty(String(h.StorageOpt[key] ?? ''));
+      if (value) {
+        limits.push(`--storage-opt=${key}=${value}`);
+      }
+    }
+  }
+
+  for (const ulimit of Array.isArray(h.Ulimits) ? h.Ulimits : []) {
+    if (ulimit && nonEmpty(ulimit.Name)) {
+      limits.push(`--ulimit=${ulimit.Name}=${ulimit.Soft}:${ulimit.Hard}`);
+    }
+  }
+
+  return limits;
+}
+
+/**
+ * Read a container's resource limits with `docker inspect`.
+ * @param {string} containerName - Container to inspect
+ * @param {Function} runner - `(command, args) => {success, stdout}`
+ * @returns {?string[]} Flags, or null when the container could not be read
+ */
+function readDockerResourceLimits(containerName, runner) {
+  let result;
+  try {
+    result = runner(getDockerCommand(), [
+      'inspect',
+      '--format',
+      '{{json .HostConfig}}',
+      containerName,
+    ]);
+  } catch {
+    return null;
+  }
+  if (!result || !result.success) {
+    return null;
+  }
+  try {
+    return parseDockerResourceLimits(JSON.parse(String(result.stdout).trim()));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Normalize a stored `resourceLimits` value to a list of flags.
+ * @param {*} value - Value from an execution record or options object
+ * @returns {string[]} Flags
+ */
+function normalizeResourceLimits(value) {
+  if (Array.isArray(value)) {
+    return value.map(String).filter((flag) => flag.startsWith('--'));
+  }
+  if (typeof value === 'string' && value.trim() !== '') {
+    return value.split(/\s+/).filter((flag) => flag.startsWith('--'));
+  }
+  return [];
+}
+
+/**
+ * `[Isolation]` status line for resource limits, or null when there are none.
+ * @param {*} value - Stored `resourceLimits`
+ * @returns {?string} Status line
+ */
+function buildResourceLimitsStatusLine(value) {
+  const limits = normalizeResourceLimits(value);
+  return limits.length > 0
+    ? `[Isolation] Resource limits: ${limits.join(' ')}`
+    : null;
+}
+
+module.exports = {
+  DEFAULT_SHM_SIZE,
+  buildResourceLimitsStatusLine,
+  formatDockerBytes,
+  formatDockerCpus,
+  normalizeResourceLimits,
+  parseDockerResourceLimits,
+  readDockerResourceLimits,
+};

package/src/lib/docker-runtime-args.js

@@ -0,0 +1,141 @@
+/**
+ * Docker runtime options shared by `docker run`, `[Isolation]` status lines
+ * and execution-record metadata: privileged mode, environment, volumes,
+ * mounts, networks (issue #150) and resource limits (issue #176).
+ */
+
+const { getDockerNetworks } = require('./docker-network-lifecycle');
+const {
+  buildResourceLimitsStatusLine,
+  normalizeResourceLimits,
+} = require('./docker-resource-limits');
+
+/**
+ * Number of kill recoveries requested at launch; a recovery command on its own
+ * implies one (issue #176).
+ * @param {object} options - Options (onKillResume, recoveryCommand)
+ * @returns {?number} Attempt limit, or null when recovery is off
+ */
+function getOnKillResume(options = {}) {
+  const count = Number(options.onKillResume) || 0;
+  if (count > 0) {
+    return count;
+  }
+  return options.recoveryCommand ? 1 : null;
+}
+
+/**
+ * Build the docker run runtime argument list contributed by configurable
+ * container options: privileged mode, environment variables, volumes/bind
+ * mounts, --mount specs, network configuration and resource limits. Returned
+ * in a stable order so they can be spliced into the `docker run` argv before
+ * the image name.
+ * @param {object} options - Options (privileged, env, volumes, mounts, network, networkAliases, resourceLimits)
+ * @returns {string[]} Docker CLI arguments
+ */
+function buildDockerRuntimeArgs(options = {}) {
+  const args = [];
+  if (options.privileged) {
+    args.push('--privileged');
+  }
+  for (const envVar of options.env || []) {
+    args.push('-e', envVar);
+  }
+  for (const volume of options.volumes || []) {
+    args.push('-v', volume);
+  }
+  for (const mount of options.mounts || []) {
+    args.push('--mount', mount);
+  }
+  const [firstNetwork] = getDockerNetworks(options);
+  if (firstNetwork) {
+    args.push('--network', firstNetwork);
+  }
+  for (const alias of options.networkAliases || []) {
+    args.push('--network-alias', alias);
+  }
+  args.push(...normalizeResourceLimits(options.resourceLimits));
+  return args;
+}
+
+/**
+ * Build the human-readable `[Isolation]` status lines for docker runtime
+ * options (volumes, mounts, env, privileged, networks, resource limits and
+ * the launch-time recovery command). Empty collections and a falsy privileged
+ * flag contribute no lines.
+ * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases, resourceLimits, onKillResume, recoveryCommand)
+ * @returns {string[]} Status lines for the start block / log header
+ */
+function buildDockerRuntimeStatusLines(options = {}) {
+  const lines = [];
+  if (options.volumes && options.volumes.length > 0) {
+    lines.push(`[Isolation] Volumes: ${options.volumes.join(', ')}`);
+  }
+  if (options.mounts && options.mounts.length > 0) {
+    lines.push(`[Isolation] Mounts: ${options.mounts.join(', ')}`);
+  }
+  if (options.env && options.env.length > 0) {
+    lines.push(`[Isolation] Env: ${options.env.join(', ')}`);
+  }
+  if (options.privileged) {
+    lines.push(`[Isolation] Privileged: true`);
+  }
+  const networks = getDockerNetworks(options);
+  if (networks.length > 0) {
+    lines.push(`[Isolation] Network: ${networks[0]}`);
+  }
+  if (networks.length > 1) {
+    lines.push(`[Isolation] Networks: ${networks.join(', ')}`);
+  }
+  if (options.networkAliases && options.networkAliases.length > 0) {
+    lines.push(
+      `[Isolation] Network aliases: ${options.networkAliases.join(', ')}`
+    );
+  }
+  const limitsLine = buildResourceLimitsStatusLine(options.resourceLimits);
+  if (limitsLine) {
+    lines.push(limitsLine);
+  }
+  const onKillResume = getOnKillResume(options);
+  if (onKillResume) {
+    const what = options.recoveryCommand || 'the original command';
+    lines.push(
+      `[Isolation] On kill: resume up to ${onKillResume} time(s) with ${what}`
+    );
+  }
+  return lines;
+}
+
+/**
+ * Build the execution-record metadata for docker runtime options, normalizing
+ * empty collections and a falsy privileged flag to `null`.
+ * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases, resourceLimits, onKillResume, recoveryCommand)
+ * @returns {object} Record metadata
+ */
+function buildDockerRuntimeMetadata(options = {}) {
+  const networks = getDockerNetworks(options);
+  const resourceLimits = normalizeResourceLimits(options.resourceLimits);
+  return {
+    volumes:
+      options.volumes && options.volumes.length > 0 ? options.volumes : null,
+    mounts: options.mounts && options.mounts.length > 0 ? options.mounts : null,
+    env: options.env && options.env.length > 0 ? options.env : null,
+    privileged: options.privileged || null,
+    network: networks[0] || null,
+    networks: networks.length > 0 ? networks : null,
+    networkAliases:
+      options.networkAliases && options.networkAliases.length > 0
+        ? options.networkAliases
+        : null,
+    resourceLimits: resourceLimits.length > 0 ? resourceLimits : null,
+    onKillResume: getOnKillResume(options),
+    recoveryCommand: options.recoveryCommand || null,
+  };
+}
+
+module.exports = {
+  buildDockerRuntimeArgs,
+  buildDockerRuntimeStatusLines,
+  buildDockerRuntimeMetadata,
+  getOnKillResume,
+};

package/src/lib/execution-control.js

@@ -268,6 +268,40 @@ function formatControlResultAsLinksNotation(result) {
   return lines.join('\n');
 }
 
+/**
+ * Record that a stop was requested, for sessions with launch-time kill
+ * recovery (issue #176).
+ * @param {object} store - Execution store
+ * @param {object} record - Execution record
+ * @returns {string|undefined} The previous marker, to restore on failure
+ */
+function markStopRequested(store, record) {
+  const options = record.options || {};
+  if (!options.onKillResume) {
+    return undefined;
+  }
+  const previous = options.stopRequestedAt;
+  record.options = { ...options, stopRequestedAt: new Date().toISOString() };
+  try {
+    store.save(record);
+  } catch {
+    // Best effort: without the marker the session may be recovered once more.
+  }
+  return previous;
+}
+
+function restoreStopRequested(store, record, previous) {
+  if (!record.options || !record.options.onKillResume) {
+    return;
+  }
+  record.options = { ...record.options, stopRequestedAt: previous };
+  try {
+    store.save(record);
+  } catch {
+    // Best effort, see markStopRequested.
+  }
+}
+
 function controlExecution(store, identifier, action, runner = runCommand) {
   if (!store) {
     return { success: false, error: 'Execution tracking is disabled.' };
@@ -286,8 +320,13 @@ function controlExecution(store, identifier, action, runner = runCommand) {
     return { success: false, error: control.error };
   }
 
+  // A deliberate stop must not be mistaken for a kill that launch-time
+  // recovery should undo (issue #176): `docker stop` escalates to SIGKILL,
+  // which exits 137 just like the OOM killer.
+  const previousStopRequestedAt = markStopRequested(store, record);
   const result = runner(control.command, control.args);
   if (!result.success) {
+    restoreStopRequested(store, record, previousStopRequestedAt);
     const detail =
       result.stderr || result.error || `exit code ${result.status}`;
     return {