start-command 0.37.8 → 0.37.9

package/src/lib/docker-snapshot-safety.js

@@ -8,6 +8,30 @@ const GiB = 1024 ** 3;
 const SNAPSHOT_RESERVE_BYTES = 10 * GiB;
 const SNAPSHOT_IMAGE_LABEL = 'start-command.snapshot-image';
 
+/** sysexits.h EX_TEMPFAIL: the resume can succeed when retried later. */
+const EX_TEMPFAIL = 75;
+
+function tempfail(message) {
+  const error = new Error(message);
+  error.exitCode = EX_TEMPFAIL;
+  return error;
+}
+
+function formatSeconds(seconds) {
+  return `${Number(seconds.toFixed(3))}s`;
+}
+
+function notifier(options) {
+  return options.notify || ((line) => process.stderr.write(`${line}\n`));
+}
+
+/**
+ * Take the host-wide snapshot lock. With `waitSeconds` above 0 a busy lock is
+ * waited for (issue #224); the wait is announced once with the holder's name.
+ * @param {object} [options] - {lockPath, holder, waitSeconds, notify}
+ * @returns {LockManager} The held lock
+ * @throws {Error} With `exitCode` 75 when the lock stays busy
+ */
 function acquireSnapshotLock(options = {}) {
   const lockPath =
     options.lockPath ||
@@ -17,6 +41,9 @@ function acquireSnapshotLock(options = {}) {
       'start-command-docker-snapshot.lock'
     );
   const lock = new LockManager(lockPath);
+  if (options.holder) {
+    lock.metadata = { holder: options.holder };
+  }
   // The ordinary store lock is held only for a brief write. A snapshot can
   // legitimately take hours, so never reclaim this lock merely for its age.
   lock.isLockStale = (data) => {
@@ -35,12 +62,28 @@ function acquireSnapshotLock(options = {}) {
       return error.code === 'ESRCH';
     }
   };
-  if (!lock.acquire(1000)) {
-    throw new Error(
-      'A Docker snapshot resume is in progress on this host. Retry after it finishes.'
+  const waitMs = Math.max(0, (options.waitSeconds || 0) * 1000);
+  if (lock.acquire(Math.min(1000, waitMs || 1000))) {
+    return lock;
+  }
+  const holder = lock.readLockFile()?.holder;
+  const busy = holder
+    ? `A Docker snapshot resume of ${holder} is in progress on this host.`
+    : 'A Docker snapshot resume is in progress on this host.';
+  if (waitMs <= 1000) {
+    throw tempfail(
+      `${busy} Retry after it finishes, or wait for it with --snapshot-wait <seconds>.`
     );
   }
-  return lock;
+  notifier(options)(
+    `[Resume] Waiting up to ${formatSeconds(waitMs / 1000)} for the snapshot of ${holder || 'another container'} to finish before snapshotting ${options.holder || 'this container'}.`
+  );
+  if (lock.acquire(waitMs - 1000)) {
+    return lock;
+  }
+  throw tempfail(
+    `${busy} Still busy after waiting ${formatSeconds(waitMs / 1000)}; retry after it finishes.`
+  );
 }
 
 function containerdRoot(config = '') {
@@ -73,8 +116,10 @@ function readFreeBytes(root) {
   return Number(stats.bavail * stats.bsize);
 }
 
-function preflightSnapshot(containerName, runner, options = {}) {
-  const docker = require('./docker-cleanup').getDockerCommand();
+function measureWritableLayer(containerName, runner, docker, options) {
+  if (options.sizeHint !== null && options.sizeHint !== undefined) {
+    return { size: options.sizeHint, source: 'size hint' };
+  }
   const sizeResult = runner(docker, [
     'inspect',
     '--size',
@@ -93,6 +138,10 @@ function preflightSnapshot(containerName, runner, options = {}) {
       'Cannot determine the writable layer size; refusing Docker snapshot resume.'
     );
   }
+  return { size, source: null };
+}
+
+function imageStoreRoots(runner, docker, options) {
   const infoResult = runner(docker, ['info', '--format', '{{json .}}']);
   let info;
   try {
@@ -113,9 +162,12 @@ function preflightSnapshot(containerName, runner, options = {}) {
   ) {
     roots.push(options.containerdRoot || defaultContainerdRoot());
   }
-  const required = 2 * size + (options.reserveBytes ?? SNAPSHOT_RESERVE_BYTES);
-  const free = options.freeBytes || readFreeBytes;
-  for (const root of [...new Set(roots)]) {
+  return [...new Set(roots)];
+}
+
+/** @returns {?object} The first root without enough space, or null */
+function findShortRoot(roots, required, free) {
+  for (const root of roots) {
     let available;
     try {
       available = free(root);
@@ -126,16 +178,75 @@ function preflightSnapshot(containerName, runner, options = {}) {
       );
     }
     if (!Number.isFinite(available) || available < required) {
-      throw new Error(
-        `Insufficient disk at ${root}: need ${(required / GiB).toFixed(2)} GiB (2 × writable layer plus reserve), available ${(available / GiB).toFixed(2)} GiB. Docker snapshot resume refused.`
+      return { root, available };
+    }
+  }
+  return null;
+}
+
+function sleepSync(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * Refuse a snapshot that could fill the disk. The writable layer is measured
+ * once (or taken from `sizeHint`); with `diskWaitSeconds` the free space is
+ * re-checked every `diskPollSeconds` until it suffices (issue #224).
+ * @param {object} [options] - {sizeHint, diskWaitSeconds, diskPollSeconds,
+ *   reserveBytes, freeBytes, containerdRoot, notify, sleep, now}
+ * @throws {Error} With `exitCode` 75 when the disk stays short
+ */
+function preflightSnapshot(containerName, runner, options = {}) {
+  const docker = require('./docker-cleanup').getDockerCommand();
+  const { size, source } = measureWritableLayer(
+    containerName,
+    runner,
+    docker,
+    options
+  );
+  const roots = imageStoreRoots(runner, docker, options);
+  const required = 2 * size + (options.reserveBytes ?? SNAPSHOT_RESERVE_BYTES);
+  const free = options.freeBytes || readFreeBytes;
+  const waitMs = Math.max(0, (options.diskWaitSeconds || 0) * 1000);
+  const pollMs = Math.max(
+    1,
+    (options.diskPollSeconds ??
+      Number(process.env.START_SNAPSHOT_DISK_POLL || 10)) * 1000
+  );
+  const now = options.now || Date.now;
+  const sleep = options.sleep || sleepSync;
+  const started = now();
+  let waited = false;
+  for (;;) {
+    const short = findShortRoot(roots, required, free);
+    if (!short) {
+      break;
+    }
+    const detail = `Insufficient disk at ${short.root}: need ${(required / GiB).toFixed(2)} GiB (2 × writable layer plus reserve), available ${(short.available / GiB).toFixed(2)} GiB.`;
+    const left = waitMs - (now() - started);
+    if (left <= 0) {
+      throw tempfail(
+        waited
+          ? `${detail} Still short after waiting ${formatSeconds(waitMs / 1000)}; Docker snapshot resume refused.`
+          : `${detail} Docker snapshot resume refused.`
+      );
+    }
+    if (!waited) {
+      notifier(options)(
+        `[Resume] Waiting up to ${formatSeconds(waitMs / 1000)} for disk: ${detail}`
       );
+      waited = true;
     }
+    sleep(Math.min(pollMs, left));
   }
+  const measured = source
+    ? `${(size / GiB).toFixed(2)} GiB (${source})`
+    : `${(size / GiB).toFixed(2)} GiB`;
   return {
     size,
     required,
     roots,
-    message: `[Resume] snapshotting ${(size / GiB).toFixed(2)} GiB; disk preflight passed (${(required / GiB).toFixed(2)} GiB required).`,
+    message: `[Resume] snapshotting ${measured}; disk preflight passed (${(required / GiB).toFixed(2)} GiB required).`,
   };
 }
 
@@ -155,6 +266,7 @@ function snapshotImageCleanupSnippet(
 }
 
 module.exports = {
+  EX_TEMPFAIL,
   acquireSnapshotLock,
   containerdRoot,
   preflightSnapshot,

package/src/lib/execution-attempt.js

@@ -7,6 +7,8 @@ const TERMINAL_FIELDS = [
   'observedAt',
   'staleDetectedAt',
   'containerStartedAt',
+  'containerState',
+  'containerRemovedObservedAt',
   'exitReason',
   'oomKilled',
   'memoryExhausted',

package/src/lib/execution-cleanup.js

@@ -0,0 +1,193 @@
+/**
+ * `$ --cleanup <uuid-or-session-name>` (issue #224): remove the Docker
+ * containers of one execution and the snapshot images they were derived from.
+ *
+ * For callers that remove containers themselves: start-command's completion
+ * watcher removes a snapshot image only when it removes the container. Nothing
+ * here is forced, so Docker keeps any container that is still running and any
+ * image that something else still uses; those are reported as retained.
+ *
+ * The containers are the current one and every earlier container of a
+ * `--resume` chain (`sessionNameHistory`). The images are the
+ * `start-command-resume/*` images named by the `start-command.snapshot-image`
+ * label of those containers, by the record, and by the `-resume-N` names of
+ * the chain, removed newest first.
+ */
+const { getDockerCommand } = require('./docker-cleanup');
+const { runCommand } = require('./execution-control');
+const { SNAPSHOT_IMAGE_LABEL } = require('./docker-snapshot-safety');
+
+const SNAPSHOT_IMAGE_PATTERN = /^start-command-resume\/[a-z0-9._-]+:\d+$/;
+const NOT_FOUND_PATTERN = /no such (container|image|object)/i;
+
+/** Docker image name for snapshot attempt `attempt` of `sessionName`. */
+function snapshotImageName(sessionName, attempt) {
+  const { buildSnapshotImageName } = require('./execution-resume');
+  return buildSnapshotImageName(sessionName, attempt);
+}
+
+/** Snapshot images implied by a `<name>-resume-<N>` chain, newest first. */
+function chainImages(containerName) {
+  const images = [];
+  let current = String(containerName || '');
+  for (
+    let match = /^(.+)-resume-(\d+)$/.exec(current);
+    match;
+    match = /^(.+)-resume-(\d+)$/.exec(current)
+  ) {
+    images.push(snapshotImageName(match[1], match[2]));
+    current = match[1];
+  }
+  return images;
+}
+
+function attemptOf(image) {
+  return Number(/:(\d+)$/.exec(image)?.[1] || 0);
+}
+
+function errorText(result) {
+  return (
+    String(result?.stderr || '').trim() ||
+    result?.error ||
+    `exit code ${result?.status}`
+  );
+}
+
+/**
+ * The containers and images `--cleanup <execution>` would remove.
+ * @param {object} record - Execution record
+ * @param {Function} runner - Sync command runner
+ * @returns {{containers: string[], images: string[]}}
+ */
+function planExecutionCleanup(record, runner) {
+  const options = record.options || {};
+  const current = options.sessionName || options.containerId || null;
+  const containers = [
+    ...new Set(
+      [current, ...[...(options.sessionNameHistory || [])].reverse()].filter(
+        Boolean
+      )
+    ),
+  ];
+  const images = [options.snapshotImage];
+  for (const container of containers) {
+    const labelled = runner(getDockerCommand(), [
+      'inspect',
+      '-f',
+      `{{index .Config.Labels "${SNAPSHOT_IMAGE_LABEL}"}}`,
+      container,
+    ]);
+    if (labelled.success) {
+      images.push(String(labelled.stdout || '').trim());
+    }
+    images.push(...chainImages(container));
+  }
+  return {
+    containers,
+    images: [...new Set(images)]
+      .filter((image) => SNAPSHOT_IMAGE_PATTERN.test(image || ''))
+      .sort((a, b) => attemptOf(b) - attemptOf(a)),
+  };
+}
+
+/**
+ * Remove (or, with `dryRun`, list) one object without force.
+ * @returns {{name: string, kind: string, state: string, detail?: string}}
+ *   `state` is removed, absent, retained or present (dry run)
+ */
+function removeObject(kind, name, runner, dryRun) {
+  const docker = getDockerCommand();
+  const args = dryRun
+    ? kind === 'image'
+      ? ['image', 'inspect', '--format', '{{.Id}}', name]
+      : ['container', 'inspect', '--format', '{{.Id}}', name]
+    : kind === 'image'
+      ? ['rmi', name]
+      : ['rm', name];
+  const result = runner(docker, args);
+  if (result.success) {
+    return { kind, name, state: dryRun ? 'present' : 'removed' };
+  }
+  const detail = errorText(result);
+  if (!result.error && NOT_FOUND_PATTERN.test(detail)) {
+    return { kind, name, state: 'absent' };
+  }
+  return { kind, name, state: 'retained', detail };
+}
+
+const LABELS = {
+  removed: 'Removed',
+  absent: 'Already gone:',
+  retained: 'Retained',
+  present: 'Would remove',
+};
+
+function formatOutcome(outcome) {
+  const line = `${LABELS[outcome.state]} ${outcome.kind} ${outcome.name}`;
+  return outcome.detail ? `${line} (${outcome.detail})` : line;
+}
+
+/**
+ * Remove one execution's containers, then its snapshot images.
+ * @param {?object} store - Execution store
+ * @param {string} identifier - UUID or session name
+ * @param {object} [deps] - {dryRun, runner}
+ * @returns {{success: boolean, output?: string, error?: string}}
+ */
+function cleanupExecution(store, identifier, deps = {}) {
+  if (!store) {
+    return { success: false, error: 'Execution tracking is disabled.' };
+  }
+  const record = store.get(identifier);
+  if (!record) {
+    return {
+      success: false,
+      error: `No execution found with UUID or session name: ${identifier}`,
+    };
+  }
+  if (record.options?.isolated !== 'docker') {
+    return {
+      success: false,
+      error: `--cleanup <execution> only removes Docker containers and images; ${identifier} is not a Docker execution.`,
+    };
+  }
+  const runner = deps.runner || runCommand;
+  const dryRun = deps.dryRun === true;
+  const plan = planExecutionCleanup(record, runner);
+  const outcomes = plan.containers.map((name) =>
+    removeObject('container', name, runner, dryRun)
+  );
+  // Images go only after their containers; Docker refuses a used image.
+  for (const image of plan.images) {
+    outcomes.push(removeObject('image', image, runner, dryRun));
+  }
+  const current = outcomes[0];
+  if (
+    !dryRun &&
+    current &&
+    ['removed', 'absent'].includes(current.state) &&
+    typeof store.markContainerRemoved === 'function'
+  ) {
+    store.markContainerRemoved(record.uuid, new Date().toISOString());
+  }
+  const retained = outcomes.filter((outcome) => outcome.state === 'retained');
+  const lines = outcomes.length
+    ? outcomes.map(formatOutcome)
+    : ['No containers or snapshot images recorded for this execution.'];
+  const output = lines.join('\n');
+  if (retained.length > 0) {
+    return {
+      success: false,
+      output,
+      error: `${retained.length} Docker object(s) retained; nothing was forced.`,
+    };
+  }
+  return { success: true, output };
+}
+
+module.exports = {
+  SNAPSHOT_IMAGE_PATTERN,
+  chainImages,
+  cleanupExecution,
+  planExecutionCleanup,
+};

package/src/lib/execution-control.js

@@ -17,10 +17,11 @@ const ControlAction = {
   TERMINATE: 'terminate',
 };
 
-function runCommand(command, args) {
+function runCommand(command, args, extra = {}) {
   const options = {
     encoding: 'utf8',
     stdio: ['ignore', 'pipe', 'pipe'],
+    ...(extra.timeout ? { timeout: extra.timeout } : {}),
   };
   const result = spawnSync(
     command,

package/src/lib/execution-recovery.js

@@ -458,6 +458,8 @@ function recoverKilledExecution(params = {}) {
   record.observedAt = undefined;
   record.staleDetectedAt = undefined;
   record.containerStartedAt = undefined;
+  record.containerState = undefined;
+  record.containerRemovedObservedAt = undefined;
   const previousData = globalThis.structuredClone(store.get(executionId));
   const previous = record.constructor.fromObject
     ? record.constructor.fromObject(previousData)

package/src/lib/execution-resume.js

@@ -54,6 +54,7 @@ const {
   preflightSnapshot,
   SNAPSHOT_IMAGE_LABEL,
 } = require('./docker-snapshot-safety');
+const snapshotResume = require('./docker-snapshot-resume');
 const {
   ResumeAllAction,
   resumeAllExecutions,
@@ -441,8 +442,10 @@ function activeSessionName(plan) {
  * Resume a tracked execution by UUID or session name.
  * @param {?object} store - Execution store
  * @param {string} identifier - UUID or session name
- * @param {object} deps - {command, outputFormat, probe, runner, startWatcher, runIsolated}
- * @returns {Promise<{success: boolean, output?: string, error?: string}>}
+ * @param {object} deps - {command, outputFormat, probe, runner, runAsync,
+ *   startWatcher, runIsolated, removeOriginal, removeOriginalAfter,
+ *   snapshotOptions, notify}
+ * @returns {Promise<{success: boolean, output?: string, error?: string, exitCode?: number}>}
  */
 async function resumeExecution(store, identifier, deps = {}) {
   if (!store) {
@@ -605,16 +608,34 @@ async function resumeExecution(store, identifier, deps = {}) {
     }
     containerId = launchResult.containerId || null;
   } else {
+    const notify = deps.notify || ((line) => process.stderr.write(`${line}\n`));
+    const snapshotOptions = { notify, ...deps.snapshotOptions };
+    const progress = {
+      sessionName: plan.sessionName,
+      logPath: record.logPath,
+      notify,
+    };
     let snapshotLock;
     let snapshotCreated = false;
     let derivedCreated = false;
     try {
       if (plan.mode === ResumeMode.DOCKER_SNAPSHOT) {
-        snapshotLock = acquireSnapshotLock(deps.snapshotOptions);
+        snapshotLock = acquireSnapshotLock({
+          ...snapshotOptions,
+          holder: plan.sessionName,
+        });
+        if (snapshotOptions.prepare) {
+          snapshotResume.runSnapshotPrepare(
+            plan.sessionName,
+            snapshotOptions.prepare,
+            runner,
+            progress
+          );
+        }
         const preflight = preflightSnapshot(
           plan.sessionName,
           runner,
-          deps.snapshotOptions
+          snapshotOptions
         );
         plan.message += `\n${preflight.message}`;
         if (record.logPath) {
@@ -639,7 +660,18 @@ async function resumeExecution(store, identifier, deps = {}) {
       for (const step of plan.steps) {
         let result;
         try {
-          result = runner(step.command, step.args);
+          result =
+            step.args[0] === 'commit'
+              ? await snapshotResume.commitWithProgress(step, {
+                  ...progress,
+                  runAsync:
+                    deps.runAsync ||
+                    (deps.runner
+                      ? (command, args) =>
+                          Promise.resolve().then(() => runner(command, args))
+                      : snapshotResume.runCommandAsync),
+                })
+              : runner(step.command, step.args);
         } catch (error) {
           result = { success: false, error: error.message };
         }
@@ -661,7 +693,11 @@ async function resumeExecution(store, identifier, deps = {}) {
           containerId = (result.stdout || '').trim() || containerId;
         }
       }
-      if (plan.mode === ResumeMode.DOCKER_SNAPSHOT && deps.removeOriginal) {
+      if (
+        plan.mode === ResumeMode.DOCKER_SNAPSHOT &&
+        deps.removeOriginal &&
+        !(deps.removeOriginalAfter > 0)
+      ) {
         const running = runner(getDockerCommand(), [
           'inspect',
           '-f',
@@ -704,6 +740,7 @@ async function resumeExecution(store, identifier, deps = {}) {
       return {
         success: false,
         error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${error.message}`,
+        ...(error.exitCode ? { exitCode: error.exitCode } : {}),
       };
     } finally {
       snapshotLock?.release();
@@ -793,6 +830,23 @@ async function resumeExecution(store, identifier, deps = {}) {
     };
   }
 
+  if (
+    plan.mode === ResumeMode.DOCKER_SNAPSHOT &&
+    deps.removeOriginal &&
+    deps.removeOriginalAfter > 0
+  ) {
+    plan.message += `\n${await snapshotResume.removeOriginalAfter(
+      plan,
+      deps.removeOriginalAfter,
+      runner,
+      {
+        logPath: record.logPath,
+        notify: deps.notify || ((line) => process.stderr.write(`${line}\n`)),
+        sleep: deps.sleep,
+      }
+    )}`;
+  }
+
   return {
     success: true,
     output: formatResumeResult(

package/src/lib/execution-store.js

@@ -101,6 +101,14 @@ class ExecutionRecord {
       options.containerStartedAt !== undefined
         ? options.containerStartedAt
         : undefined;
+    // `removed` once the container is known to be gone (issue #225), with
+    // when that was observed; `options.containerId` is kept as history.
+    this.containerState =
+      options.containerState !== undefined ? options.containerState : undefined;
+    this.containerRemovedObservedAt =
+      options.containerRemovedObservedAt !== undefined
+        ? options.containerRemovedObservedAt
+        : undefined;
     this.oomKilled =
       options.oomKilled !== undefined ? options.oomKilled : undefined;
     // Query-time hint explaining an opaque exit code (issue #162). Derived from
@@ -179,6 +187,15 @@ class ExecutionRecord {
     ) {
       obj.containerStartedAt = this.containerStartedAt;
     }
+    if (this.containerState !== undefined && this.containerState !== null) {
+      obj.containerState = this.containerState;
+    }
+    if (
+      this.containerRemovedObservedAt !== undefined &&
+      this.containerRemovedObservedAt !== null
+    ) {
+      obj.containerRemovedObservedAt = this.containerRemovedObservedAt;
+    }
     if (this.oomKilled !== undefined && this.oomKilled !== null) {
       obj.oomKilled = this.oomKilled;
     }
@@ -489,6 +506,41 @@ class ExecutionStore {
     }
   }
 
+  /**
+   * Record that a finished execution's container no longer exists
+   * (issue #225). Applied under the database lock to the stored record, and
+   * only while it is still finished, so a concurrent resume is never undone.
+   * @param {string} uuid - Execution UUID
+   * @param {string} observedAt - When the removal was observed
+   * @returns {ExecutionRecord|null} The updated record, or null
+   */
+  markContainerRemoved(uuid, observedAt) {
+    const lock = new LockManager(this.lockFilePath);
+    if (!lock.acquire()) {
+      throw new Error('Failed to acquire lock for container state');
+    }
+    try {
+      const records = this.readLinoRecords();
+      const current = records.find((record) => record.uuid === uuid);
+      if (
+        !current ||
+        current.status !== ExecutionStatus.EXECUTED ||
+        current.containerState === 'removed'
+      ) {
+        return null;
+      }
+      current.containerState = 'removed';
+      current.containerRemovedObservedAt = observedAt;
+      this.writeLinoRecords(records);
+      if (this.useLinks) {
+        this.writeLinksRecord(current);
+      }
+      return current;
+    } finally {
+      lock.release();
+    }
+  }
+
   /**
    * Get an execution record by UUID or session name
    * First tries exact UUID match, then the current session name, then any

package/src/lib/isolation.js

@@ -813,8 +813,10 @@ function runInDocker(command, options = {}) {
             !containerExistedBeforeLaunch &&
             readDockerContainerStatus(containerName) === 'created';
 
+          let containerRemoved = false;
           if (launchFailed) {
             if (removeDockerContainer(containerName, options.logPath)) {
+              containerRemoved = true;
               message += `\nContainer removed after launch failure.`;
             } else {
               message += `\nWarning: failed to remove container after launch failure.`;
@@ -823,6 +825,7 @@ function runInDocker(command, options = {}) {
             shouldCleanupDockerContainer(cleanupPolicy, exitCode, oomKilled)
           ) {
             if (removeDockerContainer(containerName, options.logPath)) {
+              containerRemoved = true;
               message += `\nContainer removed after completion.${postMortem(true)}`;
             } else {
               message += `\nWarning: failed to remove container automatically.`;
@@ -848,6 +851,7 @@ function runInDocker(command, options = {}) {
             diagnostics: diagnosticFacts,
             message,
             exitCode,
+            containerRemoved,
           });
         });