start-command 0.37.8 → 0.37.9

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # start-command
 
+## 0.37.9
+
+### Patch Changes
+
+- 55ed66d: A Docker snapshot resume can now be supervised safely. `--snapshot-wait <seconds>` (or `START_SNAPSHOT_LOCK_WAIT`) waits for another resume's snapshot on the host and says once which container it is waiting for. `--snapshot-disk-wait <seconds>` (or `START_SNAPSHOT_DISK_WAIT`) re-checks free disk instead of refusing at once. A busy lock and disk that is still short now exit 75 (`EX_TEMPFAIL`), so a caller can tell "retry later" from a real failure. `--remove-original-after <seconds>` removes the original only if the replacement is still running after the delay. `--snapshot-size-hint <bytes>` skips the `docker inspect --size` walk, and `--snapshot-prepare <command>` runs a cleanup command in the started original before the commit. A long `docker commit` writes its elapsed time to the log every minute. The `start-command.snapshot-image` label is documented as a stable interface, and `$ --cleanup <uuid-or-session-name>` removes an execution's containers and then their snapshot images, never forcing a removal (#224).
+
+  `$ --status` now reports `containerState removed` and `containerRemovedObservedAt` when the docker container of a finished detached execution no longer exists, instead of showing the old `containerId` as if the container still held its disk. The removal is recorded when start's own cleanup policy removes the container and when `--status` finds it gone; `containerId` is kept for history, and a docker daemon that cannot be reached is not mistaken for a removed container (#225).
+
+  `$ --status` no longer prints `[object Object]` for nested option values such as `launchOwner`, `exitEvidence`, `resolvedLimits` and `resourceLimitSpecs`: the links-notation output nests them like other objects and the text output prints them as compact JSON (#226).
+
 ## 0.37.8
 
 ### Patch Changes

package/README.md

@@ -40,6 +40,11 @@ checked examples.
 containers support copy-free command handoff; legacy snapshots require capacity
 preflight. Log uploads are private and sanitized; manual `--no-sanitize` opts out.
 
+Snapshot resumes take `--snapshot-wait`, `--snapshot-disk-wait`,
+`--remove-original-after`, `--snapshot-size-hint` and `--snapshot-prepare`, and
+exit 75 when they should be retried later; `--cleanup <id>` removes one
+execution's containers and snapshot images without forcing.
+
 ## Development
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.37.8",
+  "version": "0.37.9",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/bin/cli.js

@@ -549,6 +549,10 @@ async function runWithIsolation(
     if (mode !== 'detached') {
       executionRecord.complete(exitCode);
     }
+    if (result.containerRemoved) {
+      // The cleanup policy removed the container (issue #225).
+      require('../lib/container-removal').markContainerRemoved(executionRecord);
+    }
     try {
       store.save(
         mode === 'detached'

package/src/lib/args-parser-queries.js

@@ -7,7 +7,8 @@
  *
  * Query modes:   --status, --list [--running]
  * Control modes: --upload-log, --stop, --terminate, --attach [--read-only],
- *                --resume [-- <command>], --resume-all, --cleanup
+ *                --resume [-- <command>], --resume-all,
+ *                --cleanup [<uuid-or-session-name>]
  */
 
 /**
@@ -37,6 +38,7 @@ function createQueryOptionDefaults() {
     resumeAll: false, // Re-attach/resume every execution still marked running
     cleanup: false, // Clean up stale "executing" records
     cleanupDryRun: false, // Show what would be cleaned without cleaning
+    cleanupTarget: null, // UUID/session name whose containers and images --cleanup removes
   };
 }
 
@@ -75,15 +77,45 @@ const FLAG_OPTIONS = {
   '--resume-all': (options) => {
     options.resumeAll = true;
   },
-  '--cleanup': (options) => {
-    options.cleanup = true;
-  },
-  '--cleanup-dry-run': (options) => {
-    options.cleanup = true;
-    options.cleanupDryRun = true;
-  },
 };
 
+/**
+ * `--cleanup` and `--cleanup-dry-run` take an optional execution: without one
+ * they clean up stale records, with one they remove that execution's Docker
+ * containers and snapshot images (issue #224).
+ */
+const CLEANUP_OPTIONS = {
+  '--cleanup': false,
+  '--cleanup-dry-run': true,
+};
+
+function parseCleanupOption(args, index, options) {
+  const arg = args[index];
+  const equals = arg.indexOf('=');
+  const name = equals === -1 ? arg : arg.slice(0, equals);
+  if (!(name in CLEANUP_OPTIONS)) {
+    return 0;
+  }
+  options.cleanup = true;
+  options.cleanupDryRun = options.cleanupDryRun || CLEANUP_OPTIONS[name];
+  if (equals !== -1) {
+    const value = arg.slice(equals + 1);
+    if (!value) {
+      throw new Error(
+        `Option ${name} requires a UUID or session name argument`
+      );
+    }
+    options.cleanupTarget = value;
+    return 1;
+  }
+  const next = args[index + 1];
+  if (next !== undefined && !next.startsWith('-')) {
+    options.cleanupTarget = next;
+    return 2;
+  }
+  return 1;
+}
+
 function hasValue(value) {
   return value !== null && value !== undefined;
 }
@@ -98,6 +130,11 @@ function hasValue(value) {
 function parseQueryOption(args, index, options) {
   const arg = args[index];
 
+  const cleanupConsumed = parseCleanupOption(args, index, options);
+  if (cleanupConsumed > 0) {
+    return cleanupConsumed;
+  }
+
   const flag = FLAG_OPTIONS[arg];
   if (flag) {
     flag(options);

package/src/lib/args-parser.js

@@ -37,7 +37,7 @@
  * --read-only                      Follow --attach output without forwarding stdin
  * --resume <uuid-or-session> [-- cmd] Resume a stored execution (optionally with a new command)
  * --resume-all                     Re-attach or reconcile every execution still marked running
- * --cleanup                        Clean up stale "executing" records (processes that crashed or were killed)
+ * --cleanup [<uuid-or-session>]    Clean up stale "executing" records, or one execution's containers and snapshot images
  * --cleanup-dry-run                Show stale records that would be cleaned up (without actually cleaning)
  */
 
@@ -46,6 +46,7 @@ const dockerNetworkOptions = require('./docker-network-options');
 const dockerRecoveryOptions = require('./docker-recovery-options');
 const cpuPenalty = require('./cpu-penalty');
 const dockerResourceOptions = require('./docker-resource-options');
+const snapshotResumeOptions = require('./snapshot-resume-options');
 const { parseSequence, isSequence } = require('./sequence-parser');
 const { buildCommandString } = require('./shell-utils');
 const {
@@ -215,6 +216,7 @@ function parseArgs(args) {
     shell: 'auto', // Shell to use in isolation environments: auto, bash, zsh, sh
     useCommandStream: false, // Use command-stream library for command execution
     ...createQueryOptionDefaults(),
+    ...snapshotResumeOptions.createSnapshotOptionDefaults(),
   };
 
   let commandArgs = [];
@@ -566,6 +568,12 @@ function parseOption(args, index, options) {
     return 1;
   }
 
+  // Snapshot resume options (--snapshot-wait, --remove-original-after, ...)
+  const snapshotConsumed = snapshotResumeOptions.parse(args, index, options);
+  if (snapshotConsumed > 0) {
+    return snapshotConsumed;
+  }
+
   // Query/control options (--status, --list, --attach, --resume, ...)
   const queryConsumed = parseQueryOption(args, index, options);
   if (queryConsumed > 0) {
@@ -820,6 +828,7 @@ function validateOptions(options) {
     throw new Error('--keep-user option is only valid with --isolated-user');
   }
 
+  snapshotResumeOptions.validate(options);
   validateQueryOptions(options);
 
   // Validate shell option

package/src/lib/container-removal.js

@@ -0,0 +1,163 @@
+/**
+ * Removed-container bookkeeping for `--status` (issue #225).
+ *
+ * A kept docker container can be removed later (by hand, by an orchestrator,
+ * or by start-command's own cleanup policy). `docker inspect` then exits 1
+ * with `No such object` / `No such container`, which used to be folded into
+ * the same `null` as "docker could not be queried at all", so `--status` kept
+ * reporting the old `containerId` as if the container still held its disk.
+ *
+ * This module tells the two failures apart and records the removal on the
+ * execution record as `containerState: removed` plus
+ * `containerRemovedObservedAt`. `containerId` is kept for history.
+ */
+
+const { spawnSync } = require('child_process');
+
+/** `containerState` of a record whose container no longer exists. */
+const CONTAINER_STATE_REMOVED = 'removed';
+
+/**
+ * Docker's answer for a container that does not exist:
+ * `Error: No such object: <name>` (`docker inspect`, Docker 29 prints it
+ * lower-case) and `Error response from daemon: No such container: <name>`
+ * (`docker container inspect`). A daemon that cannot be reached prints
+ * "Cannot connect" / "failed to connect", which must not match.
+ */
+const DOCKER_NOT_FOUND_PATTERN = /no such (object|container)/i;
+
+/**
+ * Environment variable the detached watcher sets for the finalizer when its
+ * own `docker rm -f` removed the container.
+ */
+const CONTAINER_REMOVED_ENV = 'START_INTERNAL_CONTAINER_REMOVED';
+
+/**
+ * @param {{status?: number|null, stderr?: string|Buffer, error?: Error}} result
+ *   `spawnSync` result of a `docker inspect`
+ * @returns {boolean} True only when docker answered that the object is unknown
+ */
+function isDockerNotFound(result) {
+  if (!result || result.error || result.status === 0) {
+    return false;
+  }
+  return DOCKER_NOT_FOUND_PATTERN.test(String(result.stderr || ''));
+}
+
+/**
+ * Run `docker inspect -f <format> <target>`.
+ * @returns {{stdout: string|null, notFound: boolean}} `stdout` is null when
+ *   the inspect failed; `notFound` says the failure was "no such container"
+ */
+function dockerInspect(dockerCommand, spawnOptions, format, target) {
+  const result = spawnSync(
+    dockerCommand,
+    ['inspect', '-f', format, target],
+    spawnOptions
+  );
+  if (result.error || result.status !== 0 || !result.stdout) {
+    return { stdout: null, notFound: isDockerNotFound(result) };
+  }
+  return { stdout: String(result.stdout), notFound: false };
+}
+
+/**
+ * Set `containerState: removed` on a record, keeping an earlier observation
+ * time when there is one.
+ * @param {Object} record - Execution record (mutated)
+ * @param {string} [observedAt] - When the removal was observed
+ * @returns {Object} The same record
+ */
+function markContainerRemoved(record, observedAt = new Date().toISOString()) {
+  record.containerState = CONTAINER_STATE_REMOVED;
+  record.containerRemovedObservedAt =
+    record.containerRemovedObservedAt || observedAt;
+  return record;
+}
+
+/**
+ * Whether a finished detached docker record should be checked for removal.
+ * Executing records are never touched: a container that is not visible yet
+ * on a slow Docker-in-Docker host is not a removed one (issue #136).
+ */
+function shouldProbeRemoval(record) {
+  const opts = (record && record.options) || {};
+  return (
+    record.status === 'executed' &&
+    opts.isolated === 'docker' &&
+    opts.isolationMode === 'detached' &&
+    Boolean(opts.sessionName || opts.containerId) &&
+    record.containerState !== CONTAINER_STATE_REMOVED
+  );
+}
+
+/**
+ * Attach `containerState: removed` to a finished detached docker record whose
+ * container no longer exists. The container id is preferred over the session
+ * name, so a later container that reuses the name is not mistaken for it.
+ *
+ * @param {Object} record - Enriched execution record
+ * @param {{dockerCommand: string, spawnOptions: Object,
+ *   sessionNotFound?: boolean}} probe - How to run docker, and whether the
+ *   status probe by session name already answered "no such container"
+ * @returns {Object} The record, or a copy carrying the removal
+ */
+function attachContainerRemoval(record, probe) {
+  if (!shouldProbeRemoval(record)) {
+    return record;
+  }
+  const containerId = record.options.containerId;
+  const notFound = containerId
+    ? dockerInspect(
+        probe.dockerCommand,
+        probe.spawnOptions,
+        '{{.Id}}',
+        containerId
+      ).notFound
+    : probe.sessionNotFound === true;
+  if (!notFound) {
+    return record;
+  }
+  const copy = Object.assign(
+    Object.create(Object.getPrototypeOf(record)),
+    record
+  );
+  return markContainerRemoved(copy);
+}
+
+/**
+ * Persist a removal `--status` observed, best effort: a read-only query must
+ * never fail because the store could not be written.
+ * @param {Object} store - ExecutionStore
+ * @param {Object} stored - Record as read from the store
+ * @param {Object} enriched - Record as reported
+ */
+function persistContainerRemoval(store, stored, enriched) {
+  if (
+    !store ||
+    enriched.containerState !== CONTAINER_STATE_REMOVED ||
+    stored.containerState === CONTAINER_STATE_REMOVED ||
+    typeof store.markContainerRemoved !== 'function'
+  ) {
+    return;
+  }
+  try {
+    store.markContainerRemoved(
+      stored.uuid,
+      enriched.containerRemovedObservedAt
+    );
+  } catch {
+    // The removal is still reported; it is simply observed again next time.
+  }
+}
+
+module.exports = {
+  CONTAINER_REMOVED_ENV,
+  CONTAINER_STATE_REMOVED,
+  DOCKER_NOT_FOUND_PATTERN,
+  attachContainerRemoval,
+  dockerInspect,
+  isDockerNotFound,
+  markContainerRemoved,
+  persistContainerRemoval,
+};

package/src/lib/detached-finalize.js

@@ -63,6 +63,8 @@ const WATCHER_LOST_CONTAINER = 'watcher-lost-container';
  * @param {boolean|string|null} [options.running] - `.State.Running`
  * @param {string|null} [options.cgroupMemory] - Last cgroup v2 sample of the
  *   watcher, `<memory.max> <memory.peak> <oom> <oom_kill>` (issue #182)
+ * @param {boolean|string|null} [options.containerRemoved] - The watcher's own
+ *   `docker rm -f` removed the container (issue #225)
  * @returns {{updated: boolean, reason?: string, record?: Object}} Outcome
  */
 function finalizeDetachedExecution(options = {}) {
@@ -96,9 +98,23 @@ function finalizeDetachedExecution(options = {}) {
   ) {
     return { updated: false, reason: 'stale-attempt' };
   }
+  const containerRemoved = normalizeBoolean(options.containerRemoved) === true;
   if (record.status === 'executed' && record.endTime) {
     // Already finalized (e.g. a `--status` query got there first, or the
-    // watcher ran twice after a resume). Nothing to correct.
+    // watcher ran twice after a resume). Only the removal can still be new.
+    if (containerRemoved && record.containerState !== 'removed') {
+      try {
+        const marked = store.markContainerRemoved(
+          record.uuid,
+          new Date().toISOString()
+        );
+        if (marked) {
+          return { updated: true, reason: 'container-removed', record: marked };
+        }
+      } catch {
+        // Bookkeeping only: see the exit-0 contract of `main`.
+      }
+    }
     return { updated: false, reason: 'already-final', record };
   }
 
@@ -171,6 +187,11 @@ function finalizeDetachedExecution(options = {}) {
   if (containerError) {
     record.options = { ...(record.options || {}), containerError };
   }
+  if (containerRemoved) {
+    // The cleanup policy removed the container: say so, and keep its id as
+    // history (issue #225).
+    require('./container-removal').markContainerRemoved(record);
+  }
 
   const {
     appendLifecycle,
@@ -291,7 +312,9 @@ function reconcileFinalizedRecord(store, record) {
  */
 function buildDetachedFinalizeSnippet(executionId, attemptNumber = null) {
   const v = SHELL_VARS;
+  const { CONTAINER_REMOVED_ENV } = require('./container-removal');
   return (
+    `${CONTAINER_REMOVED_ENV}="$${v.removed}" ` +
     `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
     `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
     `"$${v.started}" "$${v.finished}" "$${v.error}" "$${v.running}" ` +
@@ -301,7 +324,9 @@ function buildDetachedFinalizeSnippet(executionId, attemptNumber = null) {
 
 /**
  * Entry point used by the detached watcher:
- *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error> <running> <cgroup>
+ *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error> <running> <cgroup> [attempt]
+ * with `START_INTERNAL_CONTAINER_REMOVED=true` when the watcher removed the
+ * container (issue #225).
  * Always exits 0 — a bookkeeping failure must never turn into a visible error
  * in a log the user is reading for the command's own output.
  * @param {string[]} argv - Positional arguments
@@ -340,6 +365,8 @@ function main(argv) {
       running,
       cgroupMemory,
       attemptNumber: attemptNumber || null,
+      containerRemoved:
+        process.env[require('./container-removal').CONTAINER_REMOVED_ENV],
     });
   } catch {
     // Deliberately silent: see the exit-0 contract above.

package/src/lib/docker-cleanup.js

@@ -381,7 +381,10 @@ function buildDetachedDockerCompletionScript(
   }
   // Everything that assumes the container has exited: cleanup, footer and
   // finalization. Guarded as a whole by `.State.Running` below.
-  const exited = [];
+  // `removed` tells the finalizer whether the cleanup policy removed the
+  // container (issue #225).
+  const removed = SHELL_VARS.removed;
+  const exited = [`${removed}=false`];
 
   if (logPath) {
     const quotedLogPath = shellQuote(logPath);
@@ -394,7 +397,7 @@ function buildDetachedDockerCompletionScript(
       quotedLogPath
     );
     const memory = buildCgroupMemoryLogSnippet(quotedLogPath);
-    const remove = `${imageCleanup.capture}; if docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1; then ${imageCleanup.remove}; fi; ${removalNote}; ${memory}`;
+    const remove = `${imageCleanup.capture}; if docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi; ${removalNote}; ${memory}`;
     const keep = `${postMortem}; ${memory}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
 
     const since = watcherOptions.since
@@ -444,14 +447,14 @@ function buildDetachedDockerCompletionScript(
     parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
       exited.push(
-        `${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi`
+        `${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi`
       );
     } else if (
       policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT ||
       policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
     ) {
       exited.push(
-        `if ${buildSuccessfulNonOomCondition()}; then ${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi; fi`
+        `if ${buildSuccessfulNonOomCondition()}; then ${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi; fi`
       );
     }
   }

package/src/lib/docker-post-mortem.js

@@ -139,6 +139,8 @@ const SHELL_VARS = {
   exitText: '__start_command_exit_text',
   lifetime: '__start_command_lifetime',
   running: '__start_command_running',
+  /** `true` once the watcher's own `docker rm -f` succeeded (issue #225). */
+  removed: '__start_command_removed',
 };
 
 /** `docker inspect -f` template collecting every post-mortem fact at once. */

package/src/lib/docker-snapshot-resume.js

@@ -0,0 +1,222 @@
+/**
+ * Steps around the `docker commit` of a snapshot resume (issue #224):
+ *
+ * - `--snapshot-prepare <command>` runs in the stopped original before the
+ *   commit: `docker start`, `docker exec ... sh -c <command>`, and always a
+ *   `docker kill` afterwards. Starting the original runs its own command
+ *   again, so this is only for containers whose start is known to be safe.
+ * - Progress lines while `docker commit` runs, which prints nothing itself.
+ * - `--remove-original-after <seconds>`: remove the stopped original only if
+ *   the replacement is still running after the delay.
+ */
+const { spawn } = require('child_process');
+const { setInterval, clearInterval, setTimeout } = globalThis;
+const { appendLogFile } = require('./isolation-log-utils');
+const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
+
+const DEFAULT_PROGRESS_INTERVAL_SECONDS = 60;
+/** Upper bound for one `--snapshot-prepare`; the original is killed after. */
+const DEFAULT_PREPARE_TIMEOUT_SECONDS = 600;
+
+/**
+ * Format a duration the way the progress lines print it: "45s", "12 min".
+ * @param {number} seconds - Elapsed seconds
+ * @returns {string} Human-readable duration
+ */
+function formatElapsed(seconds) {
+  const whole = Math.max(0, Math.floor(seconds));
+  if (whole < 60) {
+    return `${whole}s`;
+  }
+  const minutes = Math.floor(whole / 60);
+  const rest = whole % 60;
+  return rest ? `${minutes} min ${rest}s` : `${minutes} min`;
+}
+
+function positiveSeconds(value, fallback) {
+  const seconds = Number(value);
+  return Number.isFinite(seconds) && seconds > 0 ? seconds : fallback;
+}
+
+function progressIntervalSeconds(env = process.env) {
+  return positiveSeconds(
+    env.START_SNAPSHOT_PROGRESS_INTERVAL,
+    DEFAULT_PROGRESS_INTERVAL_SECONDS
+  );
+}
+
+/** Report one line to the execution log and through `notify`. */
+function report(context, line) {
+  if (context.logPath) {
+    appendLogFile(context.logPath, `${line}\n`);
+  }
+  context.notify(line);
+}
+
+/**
+ * Run a command without blocking the event loop.
+ * @returns {Promise<{success, stdout, stderr, status, error}>}
+ */
+function runCommandAsync(command, args) {
+  return new Promise((resolve) => {
+    const options = { stdio: ['ignore', 'pipe', 'pipe'] };
+    const child = spawn(
+      command,
+      args,
+      command === getDockerCommand() ? getDockerSpawnOptions(options) : options
+    );
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    child.on('error', (error) => {
+      resolve({
+        success: false,
+        stdout,
+        stderr,
+        status: null,
+        error: error.message,
+      });
+    });
+    child.on('close', (status) => {
+      resolve({ success: status === 0, stdout, stderr, status, error: null });
+    });
+  });
+}
+
+/**
+ * Run `docker commit` and report the elapsed time every interval.
+ * @param {{command: string, args: string[]}} step - The commit step
+ * @param {object} context - {sessionName, logPath, notify, runAsync,
+ *   intervalSeconds}
+ * @returns {Promise<object>} The runner result
+ */
+async function commitWithProgress(step, context) {
+  const started = Date.now();
+  const intervalSeconds = context.intervalSeconds ?? progressIntervalSeconds();
+  const timer = setInterval(() => {
+    const elapsed = formatElapsed((Date.now() - started) / 1000);
+    report(
+      context,
+      `[Resume] snapshotting ${context.sessionName}: ${elapsed} elapsed (docker commit reports no progress)`
+    );
+  }, intervalSeconds * 1000);
+  let result;
+  try {
+    result = await context.runAsync(step.command, step.args);
+  } finally {
+    clearInterval(timer);
+  }
+  if (result && result.success) {
+    report(
+      context,
+      `[Resume] snapshot of ${context.sessionName} committed after ${formatElapsed((Date.now() - started) / 1000)}`
+    );
+  }
+  return result;
+}
+
+function detail(result, fallback) {
+  return (
+    String(result?.stderr || '').trim() ||
+    result?.error ||
+    (result?.status !== undefined && result?.status !== null
+      ? `exit code ${result.status}`
+      : fallback)
+  );
+}
+
+/**
+ * Run `--snapshot-prepare` in the stopped original. A failure is reported and
+ * the snapshot continues with the files as they are.
+ * @param {string} sessionName - Original container
+ * @param {string} prepare - Shell command run with `sh -c`
+ * @param {Function} runner - Sync command runner
+ * @param {object} context - {logPath, notify}
+ * @returns {{success: boolean, message: string}}
+ */
+function runSnapshotPrepare(sessionName, prepare, runner, context) {
+  const docker = getDockerCommand();
+  const started = runner(docker, ['start', sessionName]);
+  if (!started.success) {
+    const message = `[Resume] snapshot prepare skipped: could not start ${sessionName}: ${detail(started, 'docker start failed')}`;
+    report(context, message);
+    return { success: false, message };
+  }
+  let result;
+  try {
+    const timeout =
+      positiveSeconds(
+        process.env.START_SNAPSHOT_PREPARE_TIMEOUT,
+        DEFAULT_PREPARE_TIMEOUT_SECONDS
+      ) * 1000;
+    result = runner(docker, ['exec', sessionName, 'sh', '-c', prepare], {
+      timeout,
+    });
+  } catch (error) {
+    result = { success: false, error: error.message };
+  } finally {
+    // Never commit a running original, and never leave it running.
+    runner(docker, ['kill', sessionName]);
+    runner(docker, ['wait', sessionName]);
+  }
+  const success = Boolean(result.success);
+  const message = success
+    ? `[Resume] snapshot prepare finished in ${sessionName}`
+    : `[Resume] snapshot prepare failed in ${sessionName} (${detail(result, 'docker exec failed')}); snapshotting the files as they are`;
+  report(context, message);
+  return { success, message };
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Remove the stopped original once the replacement has survived the delay.
+ * @param {object} plan - Resume plan with sessionName and newSessionName
+ * @param {number} delaySeconds - Startup window
+ * @param {Function} runner - Sync command runner
+ * @param {object} context - {logPath, notify, sleep}
+ * @returns {Promise<string>} The note describing what happened
+ */
+async function removeOriginalAfter(plan, delaySeconds, runner, context) {
+  report(
+    context,
+    `[Resume] Checking ${plan.newSessionName} again in ${formatElapsed(delaySeconds)} before removing ${plan.sessionName}`
+  );
+  await (context.sleep || sleep)(delaySeconds * 1000);
+  const docker = getDockerCommand();
+  const running = runner(docker, [
+    'inspect',
+    '-f',
+    '{{.State.Running}}',
+    plan.newSessionName,
+  ]);
+  let note;
+  if (running.success && String(running.stdout).trim() === 'true') {
+    const removed = runner(docker, ['rm', plan.sessionName]);
+    note = removed.success
+      ? `Stopped original container removed: ${plan.sessionName}`
+      : `Original container retained: ${detail(removed, 'docker rm failed')}`;
+  } else {
+    note = `Original container retained: the snapshot container is not running ${formatElapsed(delaySeconds)} after launch.`;
+  }
+  if (context.logPath) {
+    appendLogFile(context.logPath, `${note}\n`);
+  }
+  return note;
+}
+
+module.exports = {
+  commitWithProgress,
+  formatElapsed,
+  progressIntervalSeconds,
+  removeOriginalAfter,
+  runCommandAsync,
+  runSnapshotPrepare,
+};