start-command 0.37.8 → 0.38.0

package/src/lib/detached-finalize.js

@@ -63,6 +63,8 @@ const WATCHER_LOST_CONTAINER = 'watcher-lost-container';
  * @param {boolean|string|null} [options.running] - `.State.Running`
  * @param {string|null} [options.cgroupMemory] - Last cgroup v2 sample of the
  *   watcher, `<memory.max> <memory.peak> <oom> <oom_kill>` (issue #182)
+ * @param {boolean|string|null} [options.containerRemoved] - The watcher's own
+ *   `docker rm -f` removed the container (issue #225)
  * @returns {{updated: boolean, reason?: string, record?: Object}} Outcome
  */
 function finalizeDetachedExecution(options = {}) {
@@ -96,9 +98,23 @@ function finalizeDetachedExecution(options = {}) {
   ) {
     return { updated: false, reason: 'stale-attempt' };
   }
+  const containerRemoved = normalizeBoolean(options.containerRemoved) === true;
   if (record.status === 'executed' && record.endTime) {
     // Already finalized (e.g. a `--status` query got there first, or the
-    // watcher ran twice after a resume). Nothing to correct.
+    // watcher ran twice after a resume). Only the removal can still be new.
+    if (containerRemoved && record.containerState !== 'removed') {
+      try {
+        const marked = store.markContainerRemoved(
+          record.uuid,
+          new Date().toISOString()
+        );
+        if (marked) {
+          return { updated: true, reason: 'container-removed', record: marked };
+        }
+      } catch {
+        // Bookkeeping only: see the exit-0 contract of `main`.
+      }
+    }
     return { updated: false, reason: 'already-final', record };
   }
 
@@ -171,6 +187,11 @@ function finalizeDetachedExecution(options = {}) {
   if (containerError) {
     record.options = { ...(record.options || {}), containerError };
   }
+  if (containerRemoved) {
+    // The cleanup policy removed the container: say so, and keep its id as
+    // history (issue #225).
+    require('./container-removal').markContainerRemoved(record);
+  }
 
   const {
     appendLifecycle,
@@ -291,7 +312,9 @@ function reconcileFinalizedRecord(store, record) {
  */
 function buildDetachedFinalizeSnippet(executionId, attemptNumber = null) {
   const v = SHELL_VARS;
+  const { CONTAINER_REMOVED_ENV } = require('./container-removal');
   return (
+    `${CONTAINER_REMOVED_ENV}="$${v.removed}" ` +
     `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
     `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
     `"$${v.started}" "$${v.finished}" "$${v.error}" "$${v.running}" ` +
@@ -301,7 +324,9 @@ function buildDetachedFinalizeSnippet(executionId, attemptNumber = null) {
 
 /**
  * Entry point used by the detached watcher:
- *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error> <running> <cgroup>
+ *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error> <running> <cgroup> [attempt]
+ * with `START_INTERNAL_CONTAINER_REMOVED=true` when the watcher removed the
+ * container (issue #225).
  * Always exits 0 — a bookkeeping failure must never turn into a visible error
  * in a log the user is reading for the command's own output.
  * @param {string[]} argv - Positional arguments
@@ -340,6 +365,8 @@ function main(argv) {
       running,
       cgroupMemory,
       attemptNumber: attemptNumber || null,
+      containerRemoved:
+        process.env[require('./container-removal').CONTAINER_REMOVED_ENV],
     });
   } catch {
     // Deliberately silent: see the exit-0 contract above.

package/src/lib/docker-cleanup.js

@@ -381,7 +381,10 @@ function buildDetachedDockerCompletionScript(
   }
   // Everything that assumes the container has exited: cleanup, footer and
   // finalization. Guarded as a whole by `.State.Running` below.
-  const exited = [];
+  // `removed` tells the finalizer whether the cleanup policy removed the
+  // container (issue #225).
+  const removed = SHELL_VARS.removed;
+  const exited = [`${removed}=false`];
 
   if (logPath) {
     const quotedLogPath = shellQuote(logPath);
@@ -394,7 +397,7 @@ function buildDetachedDockerCompletionScript(
       quotedLogPath
     );
     const memory = buildCgroupMemoryLogSnippet(quotedLogPath);
-    const remove = `${imageCleanup.capture}; if docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1; then ${imageCleanup.remove}; fi; ${removalNote}; ${memory}`;
+    const remove = `${imageCleanup.capture}; if docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi; ${removalNote}; ${memory}`;
     const keep = `${postMortem}; ${memory}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
 
     const since = watcherOptions.since
@@ -444,14 +447,14 @@ function buildDetachedDockerCompletionScript(
     parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
       exited.push(
-        `${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi`
+        `${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi`
       );
     } else if (
       policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT ||
       policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
     ) {
       exited.push(
-        `if ${buildSuccessfulNonOomCondition()}; then ${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi; fi`
+        `if ${buildSuccessfulNonOomCondition()}; then ${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi; fi`
       );
     }
   }

package/src/lib/docker-command-handoff.js

@@ -12,7 +12,9 @@ function buildCommandHandoffArgs(mainArgs, markerPath) {
   return [
     'sh',
     '-c',
-    'p=$1; shift; if [ -f "$p" ]; then exec sh "$p"; fi; exec "$@"',
+    // `$p.env` refreshes the START_* variables a `docker start` cannot
+    // change (issue #230) before either command runs.
+    'p=$1; shift; if [ -f "$p.env" ]; then . "$p.env"; fi; if [ -f "$p" ]; then exec sh "$p"; fi; exec "$@"',
     'start-command-handoff',
     markerPath,
     ...mainArgs,
@@ -49,27 +51,53 @@ function buildCommandHandoffScript(command, options = {}) {
   return `exec ${shellQuote(shell)}${interactive} -c ${shellQuote(effective)}\n`;
 }
 
-function writeCommandHandoff(containerName, command, runner, options = {}) {
+function copyIntoContainer(containerName, target, content, runner) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'start-handoff-'));
   const file = path.join(dir, 'command');
   try {
-    fs.writeFileSync(file, buildCommandHandoffScript(command, options), {
-      mode: 0o644,
-    });
+    fs.writeFileSync(file, content, { mode: 0o644 });
     fs.chmodSync(file, 0o644);
     return runner(require('./docker-cleanup').getDockerCommand(), [
       'cp',
       file,
-      `${containerName}:${commandHandoffPath(containerName)}`,
+      `${containerName}:${target}`,
     ]);
   } finally {
     fs.rmSync(dir, { recursive: true, force: true });
   }
 }
 
+function writeCommandHandoff(containerName, command, runner, options = {}) {
+  return copyIntoContainer(
+    containerName,
+    commandHandoffPath(containerName),
+    buildCommandHandoffScript(command, options),
+    runner
+  );
+}
+
+/**
+ * Copy the START_* variables of the next attempt into a stopped container
+ * that was created with the handoff selector (issue #230).
+ * @param {string} containerName - Container name
+ * @param {object} env - Variables from `buildExecutionEnv`
+ * @param {Function} runner - Command runner
+ * @returns {object} Runner result
+ */
+function writeExecutionEnvHandoff(containerName, env, runner) {
+  const statement = require('./execution-env').buildExecutionEnvExport(env);
+  return copyIntoContainer(
+    containerName,
+    `${commandHandoffPath(containerName)}.env`,
+    statement ? `${statement}\n` : '',
+    runner
+  );
+}
+
 module.exports = {
   commandHandoffPath,
   buildCommandHandoffArgs,
   buildCommandHandoffScript,
   writeCommandHandoff,
+  writeExecutionEnvHandoff,
 };

package/src/lib/docker-post-mortem.js

@@ -139,6 +139,8 @@ const SHELL_VARS = {
   exitText: '__start_command_exit_text',
   lifetime: '__start_command_lifetime',
   running: '__start_command_running',
+  /** `true` once the watcher's own `docker rm -f` succeeded (issue #225). */
+  removed: '__start_command_removed',
 };
 
 /** `docker inspect -f` template collecting every post-mortem fact at once. */

package/src/lib/docker-runtime-args.js

@@ -42,6 +42,13 @@ function buildDockerRuntimeArgs(options = {}) {
   for (const envVar of options.env || []) {
     args.push('-e', envVar);
   }
+  // Execution identity (issue #230); last, so a user `--env` cannot spoof it.
+  const executionEnv = require('./execution-env');
+  for (const envVar of executionEnv.executionEnvAssignments(
+    executionEnv.executionEnvFromOptions(options)
+  )) {
+    args.push('-e', envVar);
+  }
   for (const label of require('./docker-labels').dockerLabels(options)) {
     args.push('--label', label);
   }

package/src/lib/docker-snapshot-resume.js

@@ -0,0 +1,222 @@
+/**
+ * Steps around the `docker commit` of a snapshot resume (issue #224):
+ *
+ * - `--snapshot-prepare <command>` runs in the stopped original before the
+ *   commit: `docker start`, `docker exec ... sh -c <command>`, and always a
+ *   `docker kill` afterwards. Starting the original runs its own command
+ *   again, so this is only for containers whose start is known to be safe.
+ * - Progress lines while `docker commit` runs, which prints nothing itself.
+ * - `--remove-original-after <seconds>`: remove the stopped original only if
+ *   the replacement is still running after the delay.
+ */
+const { spawn } = require('child_process');
+const { setInterval, clearInterval, setTimeout } = globalThis;
+const { appendLogFile } = require('./isolation-log-utils');
+const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
+
+const DEFAULT_PROGRESS_INTERVAL_SECONDS = 60;
+/** Upper bound for one `--snapshot-prepare`; the original is killed after. */
+const DEFAULT_PREPARE_TIMEOUT_SECONDS = 600;
+
+/**
+ * Format a duration the way the progress lines print it: "45s", "12 min".
+ * @param {number} seconds - Elapsed seconds
+ * @returns {string} Human-readable duration
+ */
+function formatElapsed(seconds) {
+  const whole = Math.max(0, Math.floor(seconds));
+  if (whole < 60) {
+    return `${whole}s`;
+  }
+  const minutes = Math.floor(whole / 60);
+  const rest = whole % 60;
+  return rest ? `${minutes} min ${rest}s` : `${minutes} min`;
+}
+
+function positiveSeconds(value, fallback) {
+  const seconds = Number(value);
+  return Number.isFinite(seconds) && seconds > 0 ? seconds : fallback;
+}
+
+function progressIntervalSeconds(env = process.env) {
+  return positiveSeconds(
+    env.START_SNAPSHOT_PROGRESS_INTERVAL,
+    DEFAULT_PROGRESS_INTERVAL_SECONDS
+  );
+}
+
+/** Report one line to the execution log and through `notify`. */
+function report(context, line) {
+  if (context.logPath) {
+    appendLogFile(context.logPath, `${line}\n`);
+  }
+  context.notify(line);
+}
+
+/**
+ * Run a command without blocking the event loop.
+ * @returns {Promise<{success, stdout, stderr, status, error}>}
+ */
+function runCommandAsync(command, args) {
+  return new Promise((resolve) => {
+    const options = { stdio: ['ignore', 'pipe', 'pipe'] };
+    const child = spawn(
+      command,
+      args,
+      command === getDockerCommand() ? getDockerSpawnOptions(options) : options
+    );
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    child.on('error', (error) => {
+      resolve({
+        success: false,
+        stdout,
+        stderr,
+        status: null,
+        error: error.message,
+      });
+    });
+    child.on('close', (status) => {
+      resolve({ success: status === 0, stdout, stderr, status, error: null });
+    });
+  });
+}
+
+/**
+ * Run `docker commit` and report the elapsed time every interval.
+ * @param {{command: string, args: string[]}} step - The commit step
+ * @param {object} context - {sessionName, logPath, notify, runAsync,
+ *   intervalSeconds}
+ * @returns {Promise<object>} The runner result
+ */
+async function commitWithProgress(step, context) {
+  const started = Date.now();
+  const intervalSeconds = context.intervalSeconds ?? progressIntervalSeconds();
+  const timer = setInterval(() => {
+    const elapsed = formatElapsed((Date.now() - started) / 1000);
+    report(
+      context,
+      `[Resume] snapshotting ${context.sessionName}: ${elapsed} elapsed (docker commit reports no progress)`
+    );
+  }, intervalSeconds * 1000);
+  let result;
+  try {
+    result = await context.runAsync(step.command, step.args);
+  } finally {
+    clearInterval(timer);
+  }
+  if (result && result.success) {
+    report(
+      context,
+      `[Resume] snapshot of ${context.sessionName} committed after ${formatElapsed((Date.now() - started) / 1000)}`
+    );
+  }
+  return result;
+}
+
+function detail(result, fallback) {
+  return (
+    String(result?.stderr || '').trim() ||
+    result?.error ||
+    (result?.status !== undefined && result?.status !== null
+      ? `exit code ${result.status}`
+      : fallback)
+  );
+}
+
+/**
+ * Run `--snapshot-prepare` in the stopped original. A failure is reported and
+ * the snapshot continues with the files as they are.
+ * @param {string} sessionName - Original container
+ * @param {string} prepare - Shell command run with `sh -c`
+ * @param {Function} runner - Sync command runner
+ * @param {object} context - {logPath, notify}
+ * @returns {{success: boolean, message: string}}
+ */
+function runSnapshotPrepare(sessionName, prepare, runner, context) {
+  const docker = getDockerCommand();
+  const started = runner(docker, ['start', sessionName]);
+  if (!started.success) {
+    const message = `[Resume] snapshot prepare skipped: could not start ${sessionName}: ${detail(started, 'docker start failed')}`;
+    report(context, message);
+    return { success: false, message };
+  }
+  let result;
+  try {
+    const timeout =
+      positiveSeconds(
+        process.env.START_SNAPSHOT_PREPARE_TIMEOUT,
+        DEFAULT_PREPARE_TIMEOUT_SECONDS
+      ) * 1000;
+    result = runner(docker, ['exec', sessionName, 'sh', '-c', prepare], {
+      timeout,
+    });
+  } catch (error) {
+    result = { success: false, error: error.message };
+  } finally {
+    // Never commit a running original, and never leave it running.
+    runner(docker, ['kill', sessionName]);
+    runner(docker, ['wait', sessionName]);
+  }
+  const success = Boolean(result.success);
+  const message = success
+    ? `[Resume] snapshot prepare finished in ${sessionName}`
+    : `[Resume] snapshot prepare failed in ${sessionName} (${detail(result, 'docker exec failed')}); snapshotting the files as they are`;
+  report(context, message);
+  return { success, message };
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Remove the stopped original once the replacement has survived the delay.
+ * @param {object} plan - Resume plan with sessionName and newSessionName
+ * @param {number} delaySeconds - Startup window
+ * @param {Function} runner - Sync command runner
+ * @param {object} context - {logPath, notify, sleep}
+ * @returns {Promise<string>} The note describing what happened
+ */
+async function removeOriginalAfter(plan, delaySeconds, runner, context) {
+  report(
+    context,
+    `[Resume] Checking ${plan.newSessionName} again in ${formatElapsed(delaySeconds)} before removing ${plan.sessionName}`
+  );
+  await (context.sleep || sleep)(delaySeconds * 1000);
+  const docker = getDockerCommand();
+  const running = runner(docker, [
+    'inspect',
+    '-f',
+    '{{.State.Running}}',
+    plan.newSessionName,
+  ]);
+  let note;
+  if (running.success && String(running.stdout).trim() === 'true') {
+    const removed = runner(docker, ['rm', plan.sessionName]);
+    note = removed.success
+      ? `Stopped original container removed: ${plan.sessionName}`
+      : `Original container retained: ${detail(removed, 'docker rm failed')}`;
+  } else {
+    note = `Original container retained: the snapshot container is not running ${formatElapsed(delaySeconds)} after launch.`;
+  }
+  if (context.logPath) {
+    appendLogFile(context.logPath, `${note}\n`);
+  }
+  return note;
+}
+
+module.exports = {
+  commitWithProgress,
+  formatElapsed,
+  progressIntervalSeconds,
+  removeOriginalAfter,
+  runCommandAsync,
+  runSnapshotPrepare,
+};

package/src/lib/docker-snapshot-safety.js

@@ -8,6 +8,30 @@ const GiB = 1024 ** 3;
 const SNAPSHOT_RESERVE_BYTES = 10 * GiB;
 const SNAPSHOT_IMAGE_LABEL = 'start-command.snapshot-image';
 
+/** sysexits.h EX_TEMPFAIL: the resume can succeed when retried later. */
+const EX_TEMPFAIL = 75;
+
+function tempfail(message) {
+  const error = new Error(message);
+  error.exitCode = EX_TEMPFAIL;
+  return error;
+}
+
+function formatSeconds(seconds) {
+  return `${Number(seconds.toFixed(3))}s`;
+}
+
+function notifier(options) {
+  return options.notify || ((line) => process.stderr.write(`${line}\n`));
+}
+
+/**
+ * Take the host-wide snapshot lock. With `waitSeconds` above 0 a busy lock is
+ * waited for (issue #224); the wait is announced once with the holder's name.
+ * @param {object} [options] - {lockPath, holder, waitSeconds, notify}
+ * @returns {LockManager} The held lock
+ * @throws {Error} With `exitCode` 75 when the lock stays busy
+ */
 function acquireSnapshotLock(options = {}) {
   const lockPath =
     options.lockPath ||
@@ -17,6 +41,9 @@ function acquireSnapshotLock(options = {}) {
       'start-command-docker-snapshot.lock'
     );
   const lock = new LockManager(lockPath);
+  if (options.holder) {
+    lock.metadata = { holder: options.holder };
+  }
   // The ordinary store lock is held only for a brief write. A snapshot can
   // legitimately take hours, so never reclaim this lock merely for its age.
   lock.isLockStale = (data) => {
@@ -35,12 +62,28 @@ function acquireSnapshotLock(options = {}) {
       return error.code === 'ESRCH';
     }
   };
-  if (!lock.acquire(1000)) {
-    throw new Error(
-      'A Docker snapshot resume is in progress on this host. Retry after it finishes.'
+  const waitMs = Math.max(0, (options.waitSeconds || 0) * 1000);
+  if (lock.acquire(Math.min(1000, waitMs || 1000))) {
+    return lock;
+  }
+  const holder = lock.readLockFile()?.holder;
+  const busy = holder
+    ? `A Docker snapshot resume of ${holder} is in progress on this host.`
+    : 'A Docker snapshot resume is in progress on this host.';
+  if (waitMs <= 1000) {
+    throw tempfail(
+      `${busy} Retry after it finishes, or wait for it with --snapshot-wait <seconds>.`
     );
   }
-  return lock;
+  notifier(options)(
+    `[Resume] Waiting up to ${formatSeconds(waitMs / 1000)} for the snapshot of ${holder || 'another container'} to finish before snapshotting ${options.holder || 'this container'}.`
+  );
+  if (lock.acquire(waitMs - 1000)) {
+    return lock;
+  }
+  throw tempfail(
+    `${busy} Still busy after waiting ${formatSeconds(waitMs / 1000)}; retry after it finishes.`
+  );
 }
 
 function containerdRoot(config = '') {
@@ -73,8 +116,10 @@ function readFreeBytes(root) {
   return Number(stats.bavail * stats.bsize);
 }
 
-function preflightSnapshot(containerName, runner, options = {}) {
-  const docker = require('./docker-cleanup').getDockerCommand();
+function measureWritableLayer(containerName, runner, docker, options) {
+  if (options.sizeHint !== null && options.sizeHint !== undefined) {
+    return { size: options.sizeHint, source: 'size hint' };
+  }
   const sizeResult = runner(docker, [
     'inspect',
     '--size',
@@ -93,6 +138,10 @@ function preflightSnapshot(containerName, runner, options = {}) {
       'Cannot determine the writable layer size; refusing Docker snapshot resume.'
     );
   }
+  return { size, source: null };
+}
+
+function imageStoreRoots(runner, docker, options) {
   const infoResult = runner(docker, ['info', '--format', '{{json .}}']);
   let info;
   try {
@@ -113,9 +162,12 @@ function preflightSnapshot(containerName, runner, options = {}) {
   ) {
     roots.push(options.containerdRoot || defaultContainerdRoot());
   }
-  const required = 2 * size + (options.reserveBytes ?? SNAPSHOT_RESERVE_BYTES);
-  const free = options.freeBytes || readFreeBytes;
-  for (const root of [...new Set(roots)]) {
+  return [...new Set(roots)];
+}
+
+/** @returns {?object} The first root without enough space, or null */
+function findShortRoot(roots, required, free) {
+  for (const root of roots) {
     let available;
     try {
       available = free(root);
@@ -126,16 +178,75 @@ function preflightSnapshot(containerName, runner, options = {}) {
       );
     }
     if (!Number.isFinite(available) || available < required) {
-      throw new Error(
-        `Insufficient disk at ${root}: need ${(required / GiB).toFixed(2)} GiB (2 × writable layer plus reserve), available ${(available / GiB).toFixed(2)} GiB. Docker snapshot resume refused.`
+      return { root, available };
+    }
+  }
+  return null;
+}
+
+function sleepSync(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * Refuse a snapshot that could fill the disk. The writable layer is measured
+ * once (or taken from `sizeHint`); with `diskWaitSeconds` the free space is
+ * re-checked every `diskPollSeconds` until it suffices (issue #224).
+ * @param {object} [options] - {sizeHint, diskWaitSeconds, diskPollSeconds,
+ *   reserveBytes, freeBytes, containerdRoot, notify, sleep, now}
+ * @throws {Error} With `exitCode` 75 when the disk stays short
+ */
+function preflightSnapshot(containerName, runner, options = {}) {
+  const docker = require('./docker-cleanup').getDockerCommand();
+  const { size, source } = measureWritableLayer(
+    containerName,
+    runner,
+    docker,
+    options
+  );
+  const roots = imageStoreRoots(runner, docker, options);
+  const required = 2 * size + (options.reserveBytes ?? SNAPSHOT_RESERVE_BYTES);
+  const free = options.freeBytes || readFreeBytes;
+  const waitMs = Math.max(0, (options.diskWaitSeconds || 0) * 1000);
+  const pollMs = Math.max(
+    1,
+    (options.diskPollSeconds ??
+      Number(process.env.START_SNAPSHOT_DISK_POLL || 10)) * 1000
+  );
+  const now = options.now || Date.now;
+  const sleep = options.sleep || sleepSync;
+  const started = now();
+  let waited = false;
+  for (;;) {
+    const short = findShortRoot(roots, required, free);
+    if (!short) {
+      break;
+    }
+    const detail = `Insufficient disk at ${short.root}: need ${(required / GiB).toFixed(2)} GiB (2 × writable layer plus reserve), available ${(short.available / GiB).toFixed(2)} GiB.`;
+    const left = waitMs - (now() - started);
+    if (left <= 0) {
+      throw tempfail(
+        waited
+          ? `${detail} Still short after waiting ${formatSeconds(waitMs / 1000)}; Docker snapshot resume refused.`
+          : `${detail} Docker snapshot resume refused.`
+      );
+    }
+    if (!waited) {
+      notifier(options)(
+        `[Resume] Waiting up to ${formatSeconds(waitMs / 1000)} for disk: ${detail}`
       );
+      waited = true;
     }
+    sleep(Math.min(pollMs, left));
   }
+  const measured = source
+    ? `${(size / GiB).toFixed(2)} GiB (${source})`
+    : `${(size / GiB).toFixed(2)} GiB`;
   return {
     size,
     required,
     roots,
-    message: `[Resume] snapshotting ${(size / GiB).toFixed(2)} GiB; disk preflight passed (${(required / GiB).toFixed(2)} GiB required).`,
+    message: `[Resume] snapshotting ${measured}; disk preflight passed (${(required / GiB).toFixed(2)} GiB required).`,
   };
 }
 
@@ -155,6 +266,7 @@ function snapshotImageCleanupSnippet(
 }
 
 module.exports = {
+  EX_TEMPFAIL,
   acquireSnapshotLock,
   containerdRoot,
   preflightSnapshot,

package/src/lib/execution-attempt.js

@@ -7,6 +7,8 @@ const TERMINAL_FIELDS = [
   'observedAt',
   'staleDetectedAt',
   'containerStartedAt',
+  'containerState',
+  'containerRemovedObservedAt',
   'exitReason',
   'oomKilled',
   'memoryExhausted',
@@ -44,6 +46,21 @@ function readAttemptActivity(record) {
   return record;
 }
 
+/** Number of the record's current attempt: 1 until it is resumed. */
+function currentAttemptNumber(record) {
+  return (
+    record?.attempt?.number ||
+    (Number(record?.options?.resumeCount) || 0) +
+      (Number(record?.options?.recoveryAttempts) || 0) +
+      1
+  );
+}
+
+/** Number the next attempt of `record` gets (`START_ATTEMPT`, issue #230). */
+function nextAttemptNumber(record) {
+  return currentAttemptNumber(record) + 1;
+}
+
 function createAttempt(record, plan) {
   let logOffset = null;
   if (record.logPath) {
@@ -56,11 +73,7 @@ function createAttempt(record, plan) {
     }
   }
   return {
-    number:
-      (record.attempt?.number ||
-        (Number(record.options?.resumeCount) || 0) +
-          (Number(record.options?.recoveryAttempts) || 0) +
-          1) + 1,
+    number: nextAttemptNumber(record),
     startedAt: new Date().toISOString(),
     logOffset,
     lastOutputAt: null,
@@ -76,10 +89,7 @@ function archiveAttempt(record) {
   readAttemptActivity(record);
   const snapshot = {
     ...(record.attempt || {
-      number:
-        (Number(record.options?.resumeCount) || 0) +
-        (Number(record.options?.recoveryAttempts) || 0) +
-        1,
+      number: currentAttemptNumber(record),
       startedAt: record.options?.resumedAt || record.startTime,
       logOffset: null,
     }),
@@ -174,6 +184,8 @@ module.exports = {
   appendLifecycle,
   archiveAttempt,
   createAttempt,
+  currentAttemptNumber,
+  nextAttemptNumber,
   patchAttempt,
   readAttemptActivity,
   readAttemptLogTail,