start-command 0.37.7 → 0.37.9

package/src/lib/execution-resume.js

@@ -54,6 +54,7 @@ const {
   preflightSnapshot,
   SNAPSHOT_IMAGE_LABEL,
 } = require('./docker-snapshot-safety');
+const snapshotResume = require('./docker-snapshot-resume');
 const {
   ResumeAllAction,
   resumeAllExecutions,
@@ -441,8 +442,10 @@ function activeSessionName(plan) {
  * Resume a tracked execution by UUID or session name.
  * @param {?object} store - Execution store
  * @param {string} identifier - UUID or session name
- * @param {object} deps - {command, outputFormat, probe, runner, startWatcher, runIsolated}
- * @returns {Promise<{success: boolean, output?: string, error?: string}>}
+ * @param {object} deps - {command, outputFormat, probe, runner, runAsync,
+ *   startWatcher, runIsolated, removeOriginal, removeOriginalAfter,
+ *   snapshotOptions, notify}
+ * @returns {Promise<{success: boolean, output?: string, error?: string, exitCode?: number}>}
  */
 async function resumeExecution(store, identifier, deps = {}) {
   if (!store) {
@@ -605,16 +608,34 @@ async function resumeExecution(store, identifier, deps = {}) {
     }
     containerId = launchResult.containerId || null;
   } else {
+    const notify = deps.notify || ((line) => process.stderr.write(`${line}\n`));
+    const snapshotOptions = { notify, ...deps.snapshotOptions };
+    const progress = {
+      sessionName: plan.sessionName,
+      logPath: record.logPath,
+      notify,
+    };
     let snapshotLock;
     let snapshotCreated = false;
     let derivedCreated = false;
     try {
       if (plan.mode === ResumeMode.DOCKER_SNAPSHOT) {
-        snapshotLock = acquireSnapshotLock(deps.snapshotOptions);
+        snapshotLock = acquireSnapshotLock({
+          ...snapshotOptions,
+          holder: plan.sessionName,
+        });
+        if (snapshotOptions.prepare) {
+          snapshotResume.runSnapshotPrepare(
+            plan.sessionName,
+            snapshotOptions.prepare,
+            runner,
+            progress
+          );
+        }
         const preflight = preflightSnapshot(
           plan.sessionName,
           runner,
-          deps.snapshotOptions
+          snapshotOptions
         );
         plan.message += `\n${preflight.message}`;
         if (record.logPath) {
@@ -639,7 +660,18 @@ async function resumeExecution(store, identifier, deps = {}) {
       for (const step of plan.steps) {
         let result;
         try {
-          result = runner(step.command, step.args);
+          result =
+            step.args[0] === 'commit'
+              ? await snapshotResume.commitWithProgress(step, {
+                  ...progress,
+                  runAsync:
+                    deps.runAsync ||
+                    (deps.runner
+                      ? (command, args) =>
+                          Promise.resolve().then(() => runner(command, args))
+                      : snapshotResume.runCommandAsync),
+                })
+              : runner(step.command, step.args);
         } catch (error) {
           result = { success: false, error: error.message };
         }
@@ -661,7 +693,11 @@ async function resumeExecution(store, identifier, deps = {}) {
           containerId = (result.stdout || '').trim() || containerId;
         }
       }
-      if (plan.mode === ResumeMode.DOCKER_SNAPSHOT && deps.removeOriginal) {
+      if (
+        plan.mode === ResumeMode.DOCKER_SNAPSHOT &&
+        deps.removeOriginal &&
+        !(deps.removeOriginalAfter > 0)
+      ) {
         const running = runner(getDockerCommand(), [
           'inspect',
           '-f',
@@ -704,6 +740,7 @@ async function resumeExecution(store, identifier, deps = {}) {
       return {
         success: false,
         error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${error.message}`,
+        ...(error.exitCode ? { exitCode: error.exitCode } : {}),
       };
     } finally {
       snapshotLock?.release();
@@ -793,6 +830,23 @@ async function resumeExecution(store, identifier, deps = {}) {
     };
   }
 
+  if (
+    plan.mode === ResumeMode.DOCKER_SNAPSHOT &&
+    deps.removeOriginal &&
+    deps.removeOriginalAfter > 0
+  ) {
+    plan.message += `\n${await snapshotResume.removeOriginalAfter(
+      plan,
+      deps.removeOriginalAfter,
+      runner,
+      {
+        logPath: record.logPath,
+        notify: deps.notify || ((line) => process.stderr.write(`${line}\n`)),
+        sleep: deps.sleep,
+      }
+    )}`;
+  }
+
   return {
     success: true,
     output: formatResumeResult(

package/src/lib/execution-store.js

@@ -101,6 +101,14 @@ class ExecutionRecord {
       options.containerStartedAt !== undefined
         ? options.containerStartedAt
         : undefined;
+    // `removed` once the container is known to be gone (issue #225), with
+    // when that was observed; `options.containerId` is kept as history.
+    this.containerState =
+      options.containerState !== undefined ? options.containerState : undefined;
+    this.containerRemovedObservedAt =
+      options.containerRemovedObservedAt !== undefined
+        ? options.containerRemovedObservedAt
+        : undefined;
     this.oomKilled =
       options.oomKilled !== undefined ? options.oomKilled : undefined;
     // Query-time hint explaining an opaque exit code (issue #162). Derived from
@@ -179,6 +187,15 @@ class ExecutionRecord {
     ) {
       obj.containerStartedAt = this.containerStartedAt;
     }
+    if (this.containerState !== undefined && this.containerState !== null) {
+      obj.containerState = this.containerState;
+    }
+    if (
+      this.containerRemovedObservedAt !== undefined &&
+      this.containerRemovedObservedAt !== null
+    ) {
+      obj.containerRemovedObservedAt = this.containerRemovedObservedAt;
+    }
     if (this.oomKilled !== undefined && this.oomKilled !== null) {
       obj.oomKilled = this.oomKilled;
     }
@@ -489,6 +506,41 @@ class ExecutionStore {
     }
   }
 
+  /**
+   * Record that a finished execution's container no longer exists
+   * (issue #225). Applied under the database lock to the stored record, and
+   * only while it is still finished, so a concurrent resume is never undone.
+   * @param {string} uuid - Execution UUID
+   * @param {string} observedAt - When the removal was observed
+   * @returns {ExecutionRecord|null} The updated record, or null
+   */
+  markContainerRemoved(uuid, observedAt) {
+    const lock = new LockManager(this.lockFilePath);
+    if (!lock.acquire()) {
+      throw new Error('Failed to acquire lock for container state');
+    }
+    try {
+      const records = this.readLinoRecords();
+      const current = records.find((record) => record.uuid === uuid);
+      if (
+        !current ||
+        current.status !== ExecutionStatus.EXECUTED ||
+        current.containerState === 'removed'
+      ) {
+        return null;
+      }
+      current.containerState = 'removed';
+      current.containerRemovedObservedAt = observedAt;
+      this.writeLinoRecords(records);
+      if (this.useLinks) {
+        this.writeLinksRecord(current);
+      }
+      return current;
+    } finally {
+      lock.release();
+    }
+  }
+
   /**
    * Get an execution record by UUID or session name
    * First tries exact UUID match, then the current session name, then any

package/src/lib/isolation.js

@@ -813,8 +813,10 @@ function runInDocker(command, options = {}) {
             !containerExistedBeforeLaunch &&
             readDockerContainerStatus(containerName) === 'created';
 
+          let containerRemoved = false;
           if (launchFailed) {
             if (removeDockerContainer(containerName, options.logPath)) {
+              containerRemoved = true;
               message += `\nContainer removed after launch failure.`;
             } else {
               message += `\nWarning: failed to remove container after launch failure.`;
@@ -823,6 +825,7 @@ function runInDocker(command, options = {}) {
             shouldCleanupDockerContainer(cleanupPolicy, exitCode, oomKilled)
           ) {
             if (removeDockerContainer(containerName, options.logPath)) {
+              containerRemoved = true;
               message += `\nContainer removed after completion.${postMortem(true)}`;
             } else {
               message += `\nWarning: failed to remove container automatically.`;
@@ -848,6 +851,7 @@ function runInDocker(command, options = {}) {
             diagnostics: diagnosticFacts,
             message,
             exitCode,
+            containerRemoved,
           });
         });
 

package/src/lib/query-commands.js

@@ -13,6 +13,8 @@ const { ControlAction, controlExecution } = require('./execution-control');
 const { attachExecution } = require('./execution-attach');
 const { resumeAllExecutions, resumeExecution } = require('./execution-resume');
 const { uploadExecutionLog } = require('./log-uploader');
+const { snapshotResumeSettings } = require('./snapshot-resume-options');
+const { cleanupExecution } = require('./execution-cleanup');
 
 /**
  * Print a handler result, returning the process exit code.
@@ -68,6 +70,8 @@ async function handleResumeQuery(
       outputFormat,
       resourceOptions,
       removeOriginal: resourceOptions.removeOriginal === true,
+      removeOriginalAfter: resourceOptions.removeOriginalAfter ?? 0,
+      snapshotOptions: snapshotResumeSettings(resourceOptions),
     })
   );
 }
@@ -78,12 +82,21 @@ function handleResumeAllQuery(store, outputFormat) {
 
 /**
  * Handle --cleanup / --cleanup-dry-run.
- * Cleans up stale "executing" records (processes that crashed or were killed).
+ * Cleans up stale "executing" records (processes that crashed or were killed),
+ * or with a target the Docker containers and snapshot images of one execution.
  * @param {?object} store - Execution store
  * @param {boolean} dryRun - If true, just report what would be cleaned
+ * @param {?string} [target] - UUID or session name of one execution
  * @returns {number} Exit code
  */
-function handleCleanup(store, dryRun) {
+function handleCleanup(store, dryRun, target = null) {
+  if (hasValue(target)) {
+    const result = cleanupExecution(store, target, { dryRun });
+    if (!result.success && result.output) {
+      console.log(result.output);
+    }
+    return reportResult(result);
+  }
   if (!store) {
     console.error('Error: Execution tracking is disabled.');
     return 1;
@@ -193,7 +206,7 @@ async function dispatchQueryCommand(options, context = {}) {
     return handleResumeAllQuery(store, outputFormat);
   }
   if (options.cleanup) {
-    return handleCleanup(store, options.cleanupDryRun);
+    return handleCleanup(store, options.cleanupDryRun, options.cleanupTarget);
   }
 
   return null;

package/src/lib/snapshot-resume-options.js

@@ -0,0 +1,174 @@
+/**
+ * Options for the guarded Docker snapshot resume (issue #224):
+ *   --snapshot-wait <seconds>          wait for the host snapshot lock
+ *   --snapshot-disk-wait <seconds>     wait for free disk instead of refusing
+ *   --remove-original-after <seconds>  remove the original only if the
+ *                                      replacement still runs after the delay
+ *   --snapshot-size-hint <bytes>       skip the `docker inspect --size` walk
+ *   --snapshot-prepare <command>       `docker exec` a command in the original
+ *                                      before it is committed
+ *
+ * They only apply to `--resume`. The two waits also read
+ * START_SNAPSHOT_LOCK_WAIT and START_SNAPSHOT_DISK_WAIT when the flags are
+ * absent, so a supervisor can set them once for every resume it runs.
+ */
+
+const FIELDS = {
+  '--snapshot-wait': 'snapshotWait',
+  '--snapshot-disk-wait': 'snapshotDiskWait',
+  '--remove-original-after': 'removeOriginalAfter',
+  '--snapshot-size-hint': 'snapshotSizeHint',
+  '--snapshot-prepare': 'snapshotPrepare',
+};
+
+const ENV_DEFAULTS = {
+  snapshotWait: ['START_SNAPSHOT_LOCK_WAIT', '--snapshot-wait'],
+  snapshotDiskWait: ['START_SNAPSHOT_DISK_WAIT', '--snapshot-disk-wait'],
+};
+
+const UNITS = {
+  '': 1,
+  b: 1,
+  k: 1024,
+  m: 1024 ** 2,
+  g: 1024 ** 3,
+  t: 1024 ** 4,
+};
+
+function createSnapshotOptionDefaults() {
+  return {
+    snapshotWait: null,
+    snapshotDiskWait: null,
+    removeOriginalAfter: null,
+    snapshotSizeHint: null,
+    snapshotPrepare: null,
+  };
+}
+
+/**
+ * Parse a non-negative number of seconds; fractions are allowed.
+ * @param {string} value - Raw value
+ * @param {string} name - Option or variable name for the error message
+ * @returns {number} Seconds
+ */
+function parseSeconds(value, name) {
+  const text = String(value).trim();
+  const seconds = Number(text);
+  if (!/^\d+(?:\.\d+)?$/.test(text) || !Number.isFinite(seconds)) {
+    throw new Error(
+      `Invalid ${name} value: "${value}". Expected a non-negative number of seconds.`
+    );
+  }
+  return seconds;
+}
+
+/**
+ * Parse a size in bytes; k, m, g and t suffixes are 1024-based.
+ * @param {string} value - Raw value
+ * @returns {number} Bytes
+ */
+function parseSizeHint(value) {
+  const match = /^(\d+)([bkmgt])?$/.exec(String(value).trim().toLowerCase());
+  const bytes = match ? Number(match[1]) * UNITS[match[2] || ''] : NaN;
+  if (!Number.isSafeInteger(bytes)) {
+    throw new Error(
+      `Invalid --snapshot-size-hint value: "${value}". Expected a number of bytes (k, m, g, t suffixes allowed).`
+    );
+  }
+  return bytes;
+}
+
+function parseValue(flag, value) {
+  if (flag === '--snapshot-size-hint') {
+    return parseSizeHint(value);
+  }
+  if (flag === '--snapshot-prepare') {
+    if (String(value).trim() === '') {
+      throw new Error('--snapshot-prepare requires a non-empty command');
+    }
+    return value;
+  }
+  return parseSeconds(value, flag);
+}
+
+/**
+ * Parse one snapshot option.
+ * @returns {number} Arguments consumed, 0 when `args[index]` is not one
+ */
+function parse(args, index, options) {
+  const arg = args[index];
+  const equals = arg.indexOf('=');
+  const flag = equals === -1 ? arg : arg.slice(0, equals);
+  const field = FIELDS[flag];
+  if (!field) {
+    return 0;
+  }
+  if (equals !== -1) {
+    options[field] = parseValue(flag, arg.slice(equals + 1));
+    return 1;
+  }
+  const value = args[index + 1];
+  // A prepare command may itself start with a dash only through `=`.
+  if (value === undefined || value.startsWith('-')) {
+    const kind = {
+      '--snapshot-size-hint': 'bytes',
+      '--snapshot-prepare': 'command',
+    }[flag];
+    throw new Error(`Option ${flag} requires a ${kind || 'seconds'} argument`);
+  }
+  options[field] = parseValue(flag, value);
+  return 2;
+}
+
+function hasValue(value) {
+  return value !== null && value !== undefined;
+}
+
+/**
+ * Reject snapshot options outside `--resume` and fill the waits from the
+ * environment.
+ * @param {object} options - Parsed wrapper options
+ * @param {object} [env] - Environment
+ */
+function validate(options, env = process.env) {
+  if (!hasValue(options.resume)) {
+    for (const [flag, field] of Object.entries(FIELDS)) {
+      if (hasValue(options[field])) {
+        throw new Error(`${flag} is only valid with --resume`);
+      }
+    }
+    return;
+  }
+  for (const [field, [name, flag]] of Object.entries(ENV_DEFAULTS)) {
+    if (!hasValue(options[field]) && hasValue(env[name]) && env[name] !== '') {
+      options[field] = parseSeconds(env[name], `${name} (${flag})`);
+    }
+  }
+  if (hasValue(options.removeOriginalAfter)) {
+    options.removeOriginal = true;
+  }
+}
+
+/**
+ * The `snapshotOptions` `resumeExecution` reads from the wrapper options.
+ * @param {object} options - Parsed wrapper options
+ * @returns {object} {waitSeconds, diskWaitSeconds, sizeHint, prepare}
+ */
+function snapshotResumeSettings(options = {}) {
+  return {
+    waitSeconds: options.snapshotWait ?? 0,
+    diskWaitSeconds: options.snapshotDiskWait ?? 0,
+    sizeHint: options.snapshotSizeHint ?? null,
+    prepare: options.snapshotPrepare ?? null,
+  };
+}
+
+module.exports = {
+  FIELDS,
+  createSnapshotOptionDefaults,
+  parse,
+  parseSeconds,
+  parseSizeHint,
+  snapshotResumeSettings,
+  validate,
+};

package/src/lib/status-formatter.js

@@ -7,11 +7,12 @@
  * - Text: Human-readable text format
  */
 
-const { execSync, spawnSync } = require('child_process');
+const { execSync } = require('child_process');
+const { escapeForLinksNotation } = require('./output-blocks');
 const {
-  escapeForLinksNotation,
-  formatAsNestedLinksNotation,
-} = require('./output-blocks');
+  appendLinksValue,
+  formatTextOptionValue,
+} = require('./status-links-notation');
 const { collectProcessIds } = require('./execution-control');
 const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
 const {
@@ -27,6 +28,11 @@ const {
 const { normalizeDockerTimestamp } = require('./docker-post-mortem');
 const { END_TIME_SOURCE } = require('./detached-finalize');
 const { formatCgroupMemory } = require('./cgroup-memory');
+const {
+  attachContainerRemoval,
+  dockerInspect,
+  persistContainerRemoval,
+} = require('./container-removal');
 
 /**
  * Inspect the live state of a detached docker container by name.
@@ -43,32 +49,31 @@ const { formatCgroupMemory } = require('./cgroup-memory');
  * instead of fabricating a terminal `-1` result.
  *
  * @param {string} sessionName - Container name
+ * @param {Object|null} [probe] - Receives `sessionNotFound` (issue #225)
  * @returns {{running: boolean, exitCode: number|null, oomKilled: boolean|null}|null} State,
  *   or null when the container cannot be inspected (not found yet, removed, or docker error)
  */
-function inspectDockerState(sessionName) {
-  const result = spawnSync(
+function inspectDockerState(sessionName, probe = null) {
+  const { stdout, notFound } = dockerInspect(
     getDockerCommand(),
-    [
-      'inspect',
-      '-f',
-      '{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ' +
-        '{{.State.StartedAt}} {{.State.FinishedAt}}',
-      sessionName,
-    ],
-    getDockerSpawnOptions({
-      encoding: 'utf8',
-      env: process.env,
-      stdio: ['pipe', 'pipe', 'pipe'],
-    })
+    dockerProbeSpawnOptions(),
+    '{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ' +
+      '{{.State.StartedAt}} {{.State.FinishedAt}}',
+    sessionName
   );
-  if (result.error || result.status !== 0 || !result.stdout) {
+  if (probe) {
+    // "No such container" is a fact, unlike a docker that cannot be reached
+    // (issue #225); it still reads as unknown liveness here (issue #136).
+    probe.sessionNotFound = notFound;
+  }
+  if (stdout === null) {
     return null;
   }
   // Parsed positionally and defensively: an older docker (or a stubbed one)
   // may answer with fewer fields than the template asks for.
-  const [runningRaw, exitRaw, oomKilledRaw, startedRaw, finishedRaw] =
-    result.stdout.trim().split(/\s+/);
+  const [runningRaw, exitRaw, oomKilledRaw, startedRaw, finishedRaw] = stdout
+    .trim()
+    .split(/\s+/);
   const exitCode = Number.parseInt(exitRaw, 10);
   return {
     running: runningRaw === 'true',
@@ -89,12 +94,20 @@ function isDetachedDockerRecord(record) {
   );
 }
 
-function readDockerState(record) {
+function dockerProbeSpawnOptions() {
+  return getDockerSpawnOptions({
+    encoding: 'utf8',
+    env: process.env,
+    stdio: ['pipe', 'pipe', 'pipe'],
+  });
+}
+
+function readDockerState(record, probe = null) {
   const opts = record.options || {};
   if (opts.isolated !== 'docker' || !opts.sessionName) {
     return null;
   }
-  const state = inspectDockerState(opts.sessionName);
+  const state = inspectDockerState(opts.sessionName, probe);
   if (
     record.attempt &&
     state?.startedAt &&
@@ -325,14 +338,14 @@ function applyEndTime(enriched, dockerState, footerFinishedAt) {
   enriched.observedAt = enriched.endTime;
 }
 
-function resolveDetachedStatus(record, logTail) {
+function resolveDetachedStatus(record, logTail, probe = null) {
   if (require('./launch-owner').hasActiveLaunch(record)) {
     return record;
   }
   const footer = parseFooterFromTail(logTail);
   const footerExit = footer.exitCode;
   const dockerState = isDetachedDockerRecord(record)
-    ? readDockerState(record)
+    ? readDockerState(record, probe)
     : null;
   const alive = isDetachedDockerRecord(record)
     ? dockerState === null
@@ -550,7 +563,14 @@ function enrichDetachedStatus(record) {
   const logTail = record
     ? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
     : null;
-  const resolved = resolveDetachedStatus(record, logTail);
+  const probe = {
+    dockerCommand: getDockerCommand(),
+    spawnOptions: dockerProbeSpawnOptions(),
+  };
+  const resolved = attachContainerRemoval(
+    resolveDetachedStatus(record, logTail, probe),
+    probe
+  );
   const enriched = attachMemoryExhaustion(
     attachExitReason(resolved, logTail),
     logTail
@@ -671,22 +691,21 @@ function formatRecordAsLinksNotation(record) {
   for (const [key, value] of Object.entries(obj)) {
     if (value !== null && value !== undefined) {
       if (key === 'options' && typeof value === 'object') {
-        // Format options as nested Links notation
+        // Nested option values (launchOwner, exitEvidence, resolvedLimits…)
+        // are rendered as nested Links notation, never "[object Object]"
+        // (issue #226).
         const optionEntries = Object.entries(value).filter(
           ([, v]) => v !== null && v !== undefined
         );
         if (optionEntries.length > 0) {
           lines.push('  options');
           for (const [optKey, optValue] of optionEntries) {
-            const formattedOptValue = escapeForLinksNotation(optValue);
-            lines.push(`    ${optKey} ${formattedOptValue}`);
+            appendLinksValue(lines, optKey, optValue, 4);
           }
         }
       } else if (typeof value === 'object') {
-        // For other objects, still format as nested Links notation
-        lines.push(`  ${key}`);
-        const nested = formatAsNestedLinksNotation(value, 2, 2);
-        lines.push(nested);
+        // Other objects use the same nested layout as the Rust formatter.
+        appendLinksValue(lines, key, value, 2);
       } else {
         const formattedValue = escapeForLinksNotation(value);
         lines.push(`  ${key} ${formattedValue}`);
@@ -751,6 +770,11 @@ function formatRecordAsText(record) {
   if (obj.staleDetectedAt) {
     lines.push(`Stale Detected At: ${obj.staleDetectedAt}`);
   }
+  if (obj.containerState) {
+    // The container no longer exists; `containerId` is history (issue #225).
+    lines.push(`Container State:   ${obj.containerState}`);
+    lines.push(`Container Removed: ${obj.containerRemovedObservedAt || 'N/A'}`);
+  }
   lines.push(`Log Path:          ${obj.logPath}`);
   if (obj.attempt) {
     lines.push(
@@ -770,7 +794,7 @@ function formatRecordAsText(record) {
   if (optionEntries.length > 0) {
     lines.push(`Options:`);
     for (const [key, value] of optionEntries) {
-      lines.push(`  ${key}: ${value}`);
+      lines.push(`  ${key}: ${formatTextOptionValue(value)}`);
     }
   }
 
@@ -897,6 +921,7 @@ function queryStatus(store, identifier, outputFormat) {
   try {
     // Enrich detached execution status with live session check
     const enrichedRecord = enrichDetachedStatus(record);
+    persistContainerRemoval(store, record, enrichedRecord);
     const withProcessIds = attachProcessIds(enrichedRecord);
     // Attach currentTime so callers can see how long an executing command has been running
     const withCurrentTime = attachCurrentTime(withProcessIds);