start-command 0.37.7 → 0.37.9

package/src/lib/docker-snapshot-resume.js

@@ -0,0 +1,222 @@
+/**
+ * Steps around the `docker commit` of a snapshot resume (issue #224):
+ *
+ * - `--snapshot-prepare <command>` runs in the stopped original before the
+ *   commit: `docker start`, `docker exec ... sh -c <command>`, and always a
+ *   `docker kill` afterwards. Starting the original runs its own command
+ *   again, so this is only for containers whose start is known to be safe.
+ * - Progress lines while `docker commit` runs, which prints nothing itself.
+ * - `--remove-original-after <seconds>`: remove the stopped original only if
+ *   the replacement is still running after the delay.
+ */
+const { spawn } = require('child_process');
+const { setInterval, clearInterval, setTimeout } = globalThis;
+const { appendLogFile } = require('./isolation-log-utils');
+const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
+
+const DEFAULT_PROGRESS_INTERVAL_SECONDS = 60;
+/** Upper bound for one `--snapshot-prepare`; the original is killed after. */
+const DEFAULT_PREPARE_TIMEOUT_SECONDS = 600;
+
+/**
+ * Format a duration the way the progress lines print it: "45s", "12 min".
+ * @param {number} seconds - Elapsed seconds
+ * @returns {string} Human-readable duration
+ */
+function formatElapsed(seconds) {
+  const whole = Math.max(0, Math.floor(seconds));
+  if (whole < 60) {
+    return `${whole}s`;
+  }
+  const minutes = Math.floor(whole / 60);
+  const rest = whole % 60;
+  return rest ? `${minutes} min ${rest}s` : `${minutes} min`;
+}
+
+function positiveSeconds(value, fallback) {
+  const seconds = Number(value);
+  return Number.isFinite(seconds) && seconds > 0 ? seconds : fallback;
+}
+
+function progressIntervalSeconds(env = process.env) {
+  return positiveSeconds(
+    env.START_SNAPSHOT_PROGRESS_INTERVAL,
+    DEFAULT_PROGRESS_INTERVAL_SECONDS
+  );
+}
+
+/** Report one line to the execution log and through `notify`. */
+function report(context, line) {
+  if (context.logPath) {
+    appendLogFile(context.logPath, `${line}\n`);
+  }
+  context.notify(line);
+}
+
+/**
+ * Run a command without blocking the event loop.
+ * @returns {Promise<{success, stdout, stderr, status, error}>}
+ */
+function runCommandAsync(command, args) {
+  return new Promise((resolve) => {
+    const options = { stdio: ['ignore', 'pipe', 'pipe'] };
+    const child = spawn(
+      command,
+      args,
+      command === getDockerCommand() ? getDockerSpawnOptions(options) : options
+    );
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    child.on('error', (error) => {
+      resolve({
+        success: false,
+        stdout,
+        stderr,
+        status: null,
+        error: error.message,
+      });
+    });
+    child.on('close', (status) => {
+      resolve({ success: status === 0, stdout, stderr, status, error: null });
+    });
+  });
+}
+
+/**
+ * Run `docker commit` and report the elapsed time every interval.
+ * @param {{command: string, args: string[]}} step - The commit step
+ * @param {object} context - {sessionName, logPath, notify, runAsync,
+ *   intervalSeconds}
+ * @returns {Promise<object>} The runner result
+ */
+async function commitWithProgress(step, context) {
+  const started = Date.now();
+  const intervalSeconds = context.intervalSeconds ?? progressIntervalSeconds();
+  const timer = setInterval(() => {
+    const elapsed = formatElapsed((Date.now() - started) / 1000);
+    report(
+      context,
+      `[Resume] snapshotting ${context.sessionName}: ${elapsed} elapsed (docker commit reports no progress)`
+    );
+  }, intervalSeconds * 1000);
+  let result;
+  try {
+    result = await context.runAsync(step.command, step.args);
+  } finally {
+    clearInterval(timer);
+  }
+  if (result && result.success) {
+    report(
+      context,
+      `[Resume] snapshot of ${context.sessionName} committed after ${formatElapsed((Date.now() - started) / 1000)}`
+    );
+  }
+  return result;
+}
+
+function detail(result, fallback) {
+  return (
+    String(result?.stderr || '').trim() ||
+    result?.error ||
+    (result?.status !== undefined && result?.status !== null
+      ? `exit code ${result.status}`
+      : fallback)
+  );
+}
+
+/**
+ * Run `--snapshot-prepare` in the stopped original. A failure is reported and
+ * the snapshot continues with the files as they are.
+ * @param {string} sessionName - Original container
+ * @param {string} prepare - Shell command run with `sh -c`
+ * @param {Function} runner - Sync command runner
+ * @param {object} context - {logPath, notify}
+ * @returns {{success: boolean, message: string}}
+ */
+function runSnapshotPrepare(sessionName, prepare, runner, context) {
+  const docker = getDockerCommand();
+  const started = runner(docker, ['start', sessionName]);
+  if (!started.success) {
+    const message = `[Resume] snapshot prepare skipped: could not start ${sessionName}: ${detail(started, 'docker start failed')}`;
+    report(context, message);
+    return { success: false, message };
+  }
+  let result;
+  try {
+    const timeout =
+      positiveSeconds(
+        process.env.START_SNAPSHOT_PREPARE_TIMEOUT,
+        DEFAULT_PREPARE_TIMEOUT_SECONDS
+      ) * 1000;
+    result = runner(docker, ['exec', sessionName, 'sh', '-c', prepare], {
+      timeout,
+    });
+  } catch (error) {
+    result = { success: false, error: error.message };
+  } finally {
+    // Never commit a running original, and never leave it running.
+    runner(docker, ['kill', sessionName]);
+    runner(docker, ['wait', sessionName]);
+  }
+  const success = Boolean(result.success);
+  const message = success
+    ? `[Resume] snapshot prepare finished in ${sessionName}`
+    : `[Resume] snapshot prepare failed in ${sessionName} (${detail(result, 'docker exec failed')}); snapshotting the files as they are`;
+  report(context, message);
+  return { success, message };
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Remove the stopped original once the replacement has survived the delay.
+ * @param {object} plan - Resume plan with sessionName and newSessionName
+ * @param {number} delaySeconds - Startup window
+ * @param {Function} runner - Sync command runner
+ * @param {object} context - {logPath, notify, sleep}
+ * @returns {Promise<string>} The note describing what happened
+ */
+async function removeOriginalAfter(plan, delaySeconds, runner, context) {
+  report(
+    context,
+    `[Resume] Checking ${plan.newSessionName} again in ${formatElapsed(delaySeconds)} before removing ${plan.sessionName}`
+  );
+  await (context.sleep || sleep)(delaySeconds * 1000);
+  const docker = getDockerCommand();
+  const running = runner(docker, [
+    'inspect',
+    '-f',
+    '{{.State.Running}}',
+    plan.newSessionName,
+  ]);
+  let note;
+  if (running.success && String(running.stdout).trim() === 'true') {
+    const removed = runner(docker, ['rm', plan.sessionName]);
+    note = removed.success
+      ? `Stopped original container removed: ${plan.sessionName}`
+      : `Original container retained: ${detail(removed, 'docker rm failed')}`;
+  } else {
+    note = `Original container retained: the snapshot container is not running ${formatElapsed(delaySeconds)} after launch.`;
+  }
+  if (context.logPath) {
+    appendLogFile(context.logPath, `${note}\n`);
+  }
+  return note;
+}
+
+module.exports = {
+  commitWithProgress,
+  formatElapsed,
+  progressIntervalSeconds,
+  removeOriginalAfter,
+  runCommandAsync,
+  runSnapshotPrepare,
+};

package/src/lib/docker-snapshot-safety.js

@@ -8,6 +8,30 @@ const GiB = 1024 ** 3;
 const SNAPSHOT_RESERVE_BYTES = 10 * GiB;
 const SNAPSHOT_IMAGE_LABEL = 'start-command.snapshot-image';
 
+/** sysexits.h EX_TEMPFAIL: the resume can succeed when retried later. */
+const EX_TEMPFAIL = 75;
+
+function tempfail(message) {
+  const error = new Error(message);
+  error.exitCode = EX_TEMPFAIL;
+  return error;
+}
+
+function formatSeconds(seconds) {
+  return `${Number(seconds.toFixed(3))}s`;
+}
+
+function notifier(options) {
+  return options.notify || ((line) => process.stderr.write(`${line}\n`));
+}
+
+/**
+ * Take the host-wide snapshot lock. With `waitSeconds` above 0 a busy lock is
+ * waited for (issue #224); the wait is announced once with the holder's name.
+ * @param {object} [options] - {lockPath, holder, waitSeconds, notify}
+ * @returns {LockManager} The held lock
+ * @throws {Error} With `exitCode` 75 when the lock stays busy
+ */
 function acquireSnapshotLock(options = {}) {
   const lockPath =
     options.lockPath ||
@@ -17,6 +41,9 @@ function acquireSnapshotLock(options = {}) {
       'start-command-docker-snapshot.lock'
     );
   const lock = new LockManager(lockPath);
+  if (options.holder) {
+    lock.metadata = { holder: options.holder };
+  }
   // The ordinary store lock is held only for a brief write. A snapshot can
   // legitimately take hours, so never reclaim this lock merely for its age.
   lock.isLockStale = (data) => {
@@ -35,12 +62,28 @@ function acquireSnapshotLock(options = {}) {
       return error.code === 'ESRCH';
     }
   };
-  if (!lock.acquire(1000)) {
-    throw new Error(
-      'A Docker snapshot resume is in progress on this host. Retry after it finishes.'
+  const waitMs = Math.max(0, (options.waitSeconds || 0) * 1000);
+  if (lock.acquire(Math.min(1000, waitMs || 1000))) {
+    return lock;
+  }
+  const holder = lock.readLockFile()?.holder;
+  const busy = holder
+    ? `A Docker snapshot resume of ${holder} is in progress on this host.`
+    : 'A Docker snapshot resume is in progress on this host.';
+  if (waitMs <= 1000) {
+    throw tempfail(
+      `${busy} Retry after it finishes, or wait for it with --snapshot-wait <seconds>.`
     );
   }
-  return lock;
+  notifier(options)(
+    `[Resume] Waiting up to ${formatSeconds(waitMs / 1000)} for the snapshot of ${holder || 'another container'} to finish before snapshotting ${options.holder || 'this container'}.`
+  );
+  if (lock.acquire(waitMs - 1000)) {
+    return lock;
+  }
+  throw tempfail(
+    `${busy} Still busy after waiting ${formatSeconds(waitMs / 1000)}; retry after it finishes.`
+  );
 }
 
 function containerdRoot(config = '') {
@@ -73,8 +116,10 @@ function readFreeBytes(root) {
   return Number(stats.bavail * stats.bsize);
 }
 
-function preflightSnapshot(containerName, runner, options = {}) {
-  const docker = require('./docker-cleanup').getDockerCommand();
+function measureWritableLayer(containerName, runner, docker, options) {
+  if (options.sizeHint !== null && options.sizeHint !== undefined) {
+    return { size: options.sizeHint, source: 'size hint' };
+  }
   const sizeResult = runner(docker, [
     'inspect',
     '--size',
@@ -93,6 +138,10 @@ function preflightSnapshot(containerName, runner, options = {}) {
       'Cannot determine the writable layer size; refusing Docker snapshot resume.'
     );
   }
+  return { size, source: null };
+}
+
+function imageStoreRoots(runner, docker, options) {
   const infoResult = runner(docker, ['info', '--format', '{{json .}}']);
   let info;
   try {
@@ -113,9 +162,12 @@ function preflightSnapshot(containerName, runner, options = {}) {
   ) {
     roots.push(options.containerdRoot || defaultContainerdRoot());
   }
-  const required = 2 * size + (options.reserveBytes ?? SNAPSHOT_RESERVE_BYTES);
-  const free = options.freeBytes || readFreeBytes;
-  for (const root of [...new Set(roots)]) {
+  return [...new Set(roots)];
+}
+
+/** @returns {?object} The first root without enough space, or null */
+function findShortRoot(roots, required, free) {
+  for (const root of roots) {
     let available;
     try {
       available = free(root);
@@ -126,16 +178,75 @@ function preflightSnapshot(containerName, runner, options = {}) {
       );
     }
     if (!Number.isFinite(available) || available < required) {
-      throw new Error(
-        `Insufficient disk at ${root}: need ${(required / GiB).toFixed(2)} GiB (2 × writable layer plus reserve), available ${(available / GiB).toFixed(2)} GiB. Docker snapshot resume refused.`
+      return { root, available };
+    }
+  }
+  return null;
+}
+
+function sleepSync(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * Refuse a snapshot that could fill the disk. The writable layer is measured
+ * once (or taken from `sizeHint`); with `diskWaitSeconds` the free space is
+ * re-checked every `diskPollSeconds` until it suffices (issue #224).
+ * @param {object} [options] - {sizeHint, diskWaitSeconds, diskPollSeconds,
+ *   reserveBytes, freeBytes, containerdRoot, notify, sleep, now}
+ * @throws {Error} With `exitCode` 75 when the disk stays short
+ */
+function preflightSnapshot(containerName, runner, options = {}) {
+  const docker = require('./docker-cleanup').getDockerCommand();
+  const { size, source } = measureWritableLayer(
+    containerName,
+    runner,
+    docker,
+    options
+  );
+  const roots = imageStoreRoots(runner, docker, options);
+  const required = 2 * size + (options.reserveBytes ?? SNAPSHOT_RESERVE_BYTES);
+  const free = options.freeBytes || readFreeBytes;
+  const waitMs = Math.max(0, (options.diskWaitSeconds || 0) * 1000);
+  const pollMs = Math.max(
+    1,
+    (options.diskPollSeconds ??
+      Number(process.env.START_SNAPSHOT_DISK_POLL || 10)) * 1000
+  );
+  const now = options.now || Date.now;
+  const sleep = options.sleep || sleepSync;
+  const started = now();
+  let waited = false;
+  for (;;) {
+    const short = findShortRoot(roots, required, free);
+    if (!short) {
+      break;
+    }
+    const detail = `Insufficient disk at ${short.root}: need ${(required / GiB).toFixed(2)} GiB (2 × writable layer plus reserve), available ${(short.available / GiB).toFixed(2)} GiB.`;
+    const left = waitMs - (now() - started);
+    if (left <= 0) {
+      throw tempfail(
+        waited
+          ? `${detail} Still short after waiting ${formatSeconds(waitMs / 1000)}; Docker snapshot resume refused.`
+          : `${detail} Docker snapshot resume refused.`
+      );
+    }
+    if (!waited) {
+      notifier(options)(
+        `[Resume] Waiting up to ${formatSeconds(waitMs / 1000)} for disk: ${detail}`
       );
+      waited = true;
     }
+    sleep(Math.min(pollMs, left));
   }
+  const measured = source
+    ? `${(size / GiB).toFixed(2)} GiB (${source})`
+    : `${(size / GiB).toFixed(2)} GiB`;
   return {
     size,
     required,
     roots,
-    message: `[Resume] snapshotting ${(size / GiB).toFixed(2)} GiB; disk preflight passed (${(required / GiB).toFixed(2)} GiB required).`,
+    message: `[Resume] snapshotting ${measured}; disk preflight passed (${(required / GiB).toFixed(2)} GiB required).`,
   };
 }
 
@@ -155,6 +266,7 @@ function snapshotImageCleanupSnippet(
 }
 
 module.exports = {
+  EX_TEMPFAIL,
   acquireSnapshotLock,
   containerdRoot,
   preflightSnapshot,

package/src/lib/execution-attempt.js

@@ -7,6 +7,8 @@ const TERMINAL_FIELDS = [
   'observedAt',
   'staleDetectedAt',
   'containerStartedAt',
+  'containerState',
+  'containerRemovedObservedAt',
   'exitReason',
   'oomKilled',
   'memoryExhausted',

package/src/lib/execution-cleanup.js

@@ -0,0 +1,193 @@
+/**
+ * `$ --cleanup <uuid-or-session-name>` (issue #224): remove the Docker
+ * containers of one execution and the snapshot images they were derived from.
+ *
+ * For callers that remove containers themselves: start-command's completion
+ * watcher removes a snapshot image only when it removes the container. Nothing
+ * here is forced, so Docker keeps any container that is still running and any
+ * image that something else still uses; those are reported as retained.
+ *
+ * The containers are the current one and every earlier container of a
+ * `--resume` chain (`sessionNameHistory`). The images are the
+ * `start-command-resume/*` images named by the `start-command.snapshot-image`
+ * label of those containers, by the record, and by the `-resume-N` names of
+ * the chain, removed newest first.
+ */
+const { getDockerCommand } = require('./docker-cleanup');
+const { runCommand } = require('./execution-control');
+const { SNAPSHOT_IMAGE_LABEL } = require('./docker-snapshot-safety');
+
+const SNAPSHOT_IMAGE_PATTERN = /^start-command-resume\/[a-z0-9._-]+:\d+$/;
+const NOT_FOUND_PATTERN = /no such (container|image|object)/i;
+
+/** Docker image name for snapshot attempt `attempt` of `sessionName`. */
+function snapshotImageName(sessionName, attempt) {
+  const { buildSnapshotImageName } = require('./execution-resume');
+  return buildSnapshotImageName(sessionName, attempt);
+}
+
+/** Snapshot images implied by a `<name>-resume-<N>` chain, newest first. */
+function chainImages(containerName) {
+  const images = [];
+  let current = String(containerName || '');
+  for (
+    let match = /^(.+)-resume-(\d+)$/.exec(current);
+    match;
+    match = /^(.+)-resume-(\d+)$/.exec(current)
+  ) {
+    images.push(snapshotImageName(match[1], match[2]));
+    current = match[1];
+  }
+  return images;
+}
+
+function attemptOf(image) {
+  return Number(/:(\d+)$/.exec(image)?.[1] || 0);
+}
+
+function errorText(result) {
+  return (
+    String(result?.stderr || '').trim() ||
+    result?.error ||
+    `exit code ${result?.status}`
+  );
+}
+
+/**
+ * The containers and images `--cleanup <execution>` would remove.
+ * @param {object} record - Execution record
+ * @param {Function} runner - Sync command runner
+ * @returns {{containers: string[], images: string[]}}
+ */
+function planExecutionCleanup(record, runner) {
+  const options = record.options || {};
+  const current = options.sessionName || options.containerId || null;
+  const containers = [
+    ...new Set(
+      [current, ...[...(options.sessionNameHistory || [])].reverse()].filter(
+        Boolean
+      )
+    ),
+  ];
+  const images = [options.snapshotImage];
+  for (const container of containers) {
+    const labelled = runner(getDockerCommand(), [
+      'inspect',
+      '-f',
+      `{{index .Config.Labels "${SNAPSHOT_IMAGE_LABEL}"}}`,
+      container,
+    ]);
+    if (labelled.success) {
+      images.push(String(labelled.stdout || '').trim());
+    }
+    images.push(...chainImages(container));
+  }
+  return {
+    containers,
+    images: [...new Set(images)]
+      .filter((image) => SNAPSHOT_IMAGE_PATTERN.test(image || ''))
+      .sort((a, b) => attemptOf(b) - attemptOf(a)),
+  };
+}
+
+/**
+ * Remove (or, with `dryRun`, list) one object without force.
+ * @returns {{name: string, kind: string, state: string, detail?: string}}
+ *   `state` is removed, absent, retained or present (dry run)
+ */
+function removeObject(kind, name, runner, dryRun) {
+  const docker = getDockerCommand();
+  const args = dryRun
+    ? kind === 'image'
+      ? ['image', 'inspect', '--format', '{{.Id}}', name]
+      : ['container', 'inspect', '--format', '{{.Id}}', name]
+    : kind === 'image'
+      ? ['rmi', name]
+      : ['rm', name];
+  const result = runner(docker, args);
+  if (result.success) {
+    return { kind, name, state: dryRun ? 'present' : 'removed' };
+  }
+  const detail = errorText(result);
+  if (!result.error && NOT_FOUND_PATTERN.test(detail)) {
+    return { kind, name, state: 'absent' };
+  }
+  return { kind, name, state: 'retained', detail };
+}
+
+const LABELS = {
+  removed: 'Removed',
+  absent: 'Already gone:',
+  retained: 'Retained',
+  present: 'Would remove',
+};
+
+function formatOutcome(outcome) {
+  const line = `${LABELS[outcome.state]} ${outcome.kind} ${outcome.name}`;
+  return outcome.detail ? `${line} (${outcome.detail})` : line;
+}
+
+/**
+ * Remove one execution's containers, then its snapshot images.
+ * @param {?object} store - Execution store
+ * @param {string} identifier - UUID or session name
+ * @param {object} [deps] - {dryRun, runner}
+ * @returns {{success: boolean, output?: string, error?: string}}
+ */
+function cleanupExecution(store, identifier, deps = {}) {
+  if (!store) {
+    return { success: false, error: 'Execution tracking is disabled.' };
+  }
+  const record = store.get(identifier);
+  if (!record) {
+    return {
+      success: false,
+      error: `No execution found with UUID or session name: ${identifier}`,
+    };
+  }
+  if (record.options?.isolated !== 'docker') {
+    return {
+      success: false,
+      error: `--cleanup <execution> only removes Docker containers and images; ${identifier} is not a Docker execution.`,
+    };
+  }
+  const runner = deps.runner || runCommand;
+  const dryRun = deps.dryRun === true;
+  const plan = planExecutionCleanup(record, runner);
+  const outcomes = plan.containers.map((name) =>
+    removeObject('container', name, runner, dryRun)
+  );
+  // Images go only after their containers; Docker refuses a used image.
+  for (const image of plan.images) {
+    outcomes.push(removeObject('image', image, runner, dryRun));
+  }
+  const current = outcomes[0];
+  if (
+    !dryRun &&
+    current &&
+    ['removed', 'absent'].includes(current.state) &&
+    typeof store.markContainerRemoved === 'function'
+  ) {
+    store.markContainerRemoved(record.uuid, new Date().toISOString());
+  }
+  const retained = outcomes.filter((outcome) => outcome.state === 'retained');
+  const lines = outcomes.length
+    ? outcomes.map(formatOutcome)
+    : ['No containers or snapshot images recorded for this execution.'];
+  const output = lines.join('\n');
+  if (retained.length > 0) {
+    return {
+      success: false,
+      output,
+      error: `${retained.length} Docker object(s) retained; nothing was forced.`,
+    };
+  }
+  return { success: true, output };
+}
+
+module.exports = {
+  SNAPSHOT_IMAGE_PATTERN,
+  chainImages,
+  cleanupExecution,
+  planExecutionCleanup,
+};

package/src/lib/execution-control.js

@@ -17,10 +17,11 @@ const ControlAction = {
   TERMINATE: 'terminate',
 };
 
-function runCommand(command, args) {
+function runCommand(command, args, extra = {}) {
   const options = {
     encoding: 'utf8',
     stdio: ['ignore', 'pipe', 'pipe'],
+    ...(extra.timeout ? { timeout: extra.timeout } : {}),
   };
   const result = spawnSync(
     command,

package/src/lib/execution-recovery.js

@@ -458,6 +458,8 @@ function recoverKilledExecution(params = {}) {
   record.observedAt = undefined;
   record.staleDetectedAt = undefined;
   record.containerStartedAt = undefined;
+  record.containerState = undefined;
+  record.containerRemovedObservedAt = undefined;
   const previousData = globalThis.structuredClone(store.get(executionId));
   const previous = record.constructor.fromObject
     ? record.constructor.fromObject(previousData)