start-command 0.35.4 → 0.37.0

package/src/lib/execution-resume.js

@@ -7,13 +7,13 @@
  * addressing one logical session across restarts.
  *
  * Three strategies, chosen from the probed session state:
- * - DOCKER_START:    the container still exists and the stored command is
- *                    re-run by `docker start` (its original entrypoint).
- * - DOCKER_SNAPSHOT: the container still exists but a new command was given,
- *                    so its filesystem is committed to an image and a derived
- *                    container runs the new command. This avoids
- *                    `docker start -ai`, which would re-run the original
- *                    entrypoint from scratch.
+ * - DOCKER_START:    the container still exists. Newly managed containers
+ *                    accept replacement commands through a small handoff
+ *                    script, then restart in place without a snapshot.
+ * - DOCKER_SNAPSHOT: a legacy container needs a replacement command, or
+ *                    immutable labels need changing. After disk preflight,
+ *                    its filesystem is committed under a host-wide lock and
+ *                    a derived container runs the command.
  * - RELAUNCH:        nothing is left of the session, so the command is
  *                    launched again through the stored isolation options.
  */
@@ -28,7 +28,6 @@ const { escapeForLinksNotation } = require('./output-blocks');
 const {
   buildResourceLimitsStatusLine,
   normalizeResourceLimits,
-  readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { appendLogFile } = require('./isolation-log-utils');
 const {
@@ -38,8 +37,23 @@ const {
   patchAttempt,
 } = require('./execution-attempt');
 const { runCommand } = require('./execution-control');
+const {
+  reserveLaunch,
+  rollbackLaunch,
+  failLaunchedPersistence,
+} = require('./launch-persistence');
 const { SessionState, probeSession } = require('./session-probe');
 const { getDockerNetworks } = require('./docker-network-lifecycle');
+const {
+  writeCommandHandoff,
+  buildCommandHandoffArgs,
+  commandHandoffPath,
+} = require('./docker-command-handoff');
+const {
+  acquireSnapshotLock,
+  preflightSnapshot,
+  SNAPSHOT_IMAGE_LABEL,
+} = require('./docker-snapshot-safety');
 const {
   ResumeAllAction,
   resumeAllExecutions,
@@ -89,6 +103,12 @@ function buildResumedSessionName(sessionName, attempt) {
 function buildLaunchOptions(record) {
   const opts = record.options || {};
   return {
+    ...opts,
+    labels: opts.labels || [],
+    sessionId: record.uuid,
+    rootSession:
+      opts.rootSession || opts.sessionNameHistory?.[0] || opts.sessionName,
+    resumeCount: (Number(opts.resumeCount) || 0) + 1,
     image: opts.image || null,
     session: opts.sessionName,
     detached: true,
@@ -209,12 +229,13 @@ function buildResumePlan(
   const attempt = (Number(opts.resumeCount) || 0) + 1;
 
   if (backend === 'docker' && probe.state === SessionState.STOPPED) {
-    if (!newCommand) {
+    if (!opts.forceSnapshot && (!newCommand || opts.commandHandoff)) {
       return {
         mode: ResumeMode.DOCKER_START,
         backend,
         sessionName,
         command,
+        ...(newCommand ? { handoffCommand: command } : {}),
         attempt,
         steps: [
           {
@@ -223,7 +244,9 @@ function buildResumePlan(
             description: `Start stopped container ${sessionName}`,
           },
         ],
-        message: `Resumed detached docker container: ${sessionName}`,
+        message: newCommand
+          ? `Resumed new command in the same detached docker container: ${sessionName} (no filesystem snapshot)`
+          : `Resumed detached docker container: ${sessionName}`,
       };
     }
 
@@ -256,12 +279,24 @@ function buildResumePlan(
           containerArgs: [
             '--name',
             newSessionName,
+            '--label',
+            `${SNAPSHOT_IMAGE_LABEL}=${snapshotImage}`,
             ...(opts.user ? ['--user', opts.user] : []),
-            ...buildDockerRuntimeArgs({ ...opts, resourceLimits }),
+            ...buildDockerRuntimeArgs({
+              ...opts,
+              resourceLimits,
+              session: newSessionName,
+              containerName: newSessionName,
+              sessionId: record.uuid,
+              rootSession:
+                opts.rootSession || opts.sessionNameHistory?.[0] || sessionName,
+              resumeCount: attempt,
+            }),
             snapshotImage,
-            'sh',
-            '-c',
-            command,
+            ...buildCommandHandoffArgs(
+              ['sh', '-c', command],
+              commandHandoffPath(newSessionName)
+            ),
           ],
           extraNetworks: getDockerNetworks(opts).slice(1),
           newSessionName,
@@ -355,10 +390,15 @@ function applyResumeToRecord(
   archiveAttempt(record);
   record.attempt = attempt;
   const options = { ...(record.options || {}) };
+  options.rootSession =
+    options.rootSession ||
+    options.sessionNameHistory?.[0] ||
+    options.sessionName;
   options.resumeCount = plan.attempt;
   options.resumedAt = attempt.startedAt;
   // A resume is a new deliberate start: launch-time recovery applies again.
   delete options.stopRequestedAt;
+  delete options.exitEvidence;
   if (plan.resourceLimits && plan.resourceLimits.length > 0) {
     options.resourceLimits = plan.resourceLimits;
   }
@@ -372,6 +412,12 @@ function applyResumeToRecord(
   }
   if (plan.snapshotImage) {
     options.image = plan.snapshotImage;
+    options.snapshotImage = plan.snapshotImage;
+    // Every new detached container, including a snapshot successor, supports handoff.
+    options.commandHandoff = true;
+  }
+  if (plan.mode === ResumeMode.RELAUNCH && plan.backend === 'docker') {
+    options.commandHandoff = true;
   }
   if (containerId) {
     options.containerId = containerId;
@@ -411,27 +457,83 @@ async function resumeExecution(store, identifier, deps = {}) {
     };
   }
 
+  if (require('./launch-owner').hasActiveLaunch(record)) {
+    return {
+      success: false,
+      error: 'An execution launch is already reserved.',
+    };
+  }
+  const previousData = globalThis.structuredClone(
+    record.toObject ? record.toObject() : record
+  );
+  const previous = record.constructor.fromObject
+    ? record.constructor.fromObject(previousData)
+    : previousData;
   const runner = deps.runner || runCommand;
   const probeFn = deps.probe || ((r) => probeSession(r, runner));
   const probe = probeFn(record);
   const opts = record.options || {};
-  // Only a snapshot resume creates a new container, so only it needs the old
-  // container's limits (issue #176).
-  const liveResourceLimits =
-    deps.command &&
-    opts.isolated === 'docker' &&
-    opts.sessionName &&
-    probe &&
-    !probe.alive &&
-    probe.state === SessionState.STOPPED
-      ? readDockerResourceLimits(opts.sessionName, runner)
-      : null;
+  let resources;
+  try {
+    resources = require('./resume-resources').prepareResumeResources(
+      record,
+      probe,
+      deps.resourceOptions || {},
+      runner,
+      deps.random
+    );
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+  const requestedLabels = deps.resourceOptions?.labels || [];
+  const labels = [...(opts.labels || [])];
+  for (const label of requestedLabels) {
+    const key = label.split('=')[0];
+    const existing = labels.findIndex((value) => value.split('=')[0] === key);
+    if (existing >= 0) {
+      labels[existing] = label;
+    } else {
+      labels.push(label);
+    }
+  }
+  if (requestedLabels.length && opts.isolated !== 'docker') {
+    return {
+      success: false,
+      error: 'Docker labels require a Docker execution when resuming.',
+    };
+  }
+  const labelsChanged =
+    JSON.stringify(labels) !== JSON.stringify(opts.labels || []);
+  const planningRecord = {
+    ...record,
+    options: {
+      ...opts,
+      ...(resources ? { resourceLimits: resources.resourceLimits } : {}),
+      ...(requestedLabels.length ? { labels } : {}),
+      ...(labelsChanged ? { forceSnapshot: true } : {}),
+    },
+  };
   const plan = buildResumePlan(
-    record,
+    planningRecord,
     deps.command || null,
     probe,
-    liveResourceLimits
+    resources?.resourceLimits
   );
+  if (resources && !plan.error) {
+    plan.resourceLimits = resources.resourceLimits;
+    if (plan.mode === ResumeMode.DOCKER_START && resources.update) {
+      plan.steps.unshift({
+        command: getDockerCommand(),
+        args: [
+          'update',
+          ...resources.resourceLimits.filter((s) =>
+            /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
+          ),
+          plan.sessionName,
+        ],
+      });
+    }
+  }
   if (plan.error) {
     return { success: false, error: plan.error };
   }
@@ -444,15 +546,53 @@ async function resumeExecution(store, identifier, deps = {}) {
     attempt,
   };
   appendLifecycle(lifecycleRecord, 'resume-started');
+  const updated = applyResumeToRecord(record, plan, null, attempt);
+  if (requestedLabels.length) {
+    updated.options.labels = labels;
+  }
+  if (resources) {
+    updated.options.resolvedLimits = resources.resolvedLimits;
+    updated.options.resourceLimitSpecs = resources.resourceLimitSpecs;
+    if (
+      opts.cpuPenaltyConfig &&
+      deps.resourceOptions?.cpus !== null &&
+      deps.resourceOptions?.cpus !== undefined
+    ) {
+      updated.options.baseResourceLimits = resources.resourceLimits;
+    }
+  }
+  try {
+    reserveLaunch(store, updated, previous);
+  } catch (error) {
+    appendLifecycle(lifecycleRecord, 'launch-failed', { error: error.message });
+    return {
+      success: false,
+      error: `Launch reservation failed: ${error.message}`,
+    };
+  }
+
+  if (resources && deps.resourceOptions?.memory && record.logPath) {
+    const oldMemory = resources.oldMemory;
+    appendLogFile(
+      record.logPath,
+      `Resume memory limit: ${oldMemory} -> ${resources.resolvedLimits.memory} (${deps.resourceOptions.memory})\n`
+    );
+  }
 
   if (plan.mode === ResumeMode.RELAUNCH) {
     const runIsolated = deps.runIsolated || require('./isolation').runIsolated;
-    const launchResult = await runIsolated(
-      plan.backend,
-      plan.command,
-      plan.launchOptions
-    );
+    let launchResult;
+    try {
+      launchResult = await runIsolated(
+        plan.backend,
+        plan.command,
+        plan.launchOptions
+      );
+    } catch (error) {
+      launchResult = { success: false, message: error.message };
+    }
     if (!launchResult || !launchResult.success) {
+      rollbackLaunch(store, previous);
       appendLifecycle(lifecycleRecord, 'launch-failed', {
         error: launchResult?.message || 'unknown error',
       });
@@ -465,20 +605,108 @@ async function resumeExecution(store, identifier, deps = {}) {
     }
     containerId = launchResult.containerId || null;
   } else {
-    for (const step of plan.steps) {
-      const result = runner(step.command, step.args);
-      if (!result.success) {
-        const detail =
-          (result.stderr || '').trim() ||
-          result.error ||
-          `exit code ${result.status}`;
-        appendLifecycle(lifecycleRecord, 'launch-failed', { error: detail });
-        return {
-          success: false,
-          error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${detail}`,
-        };
+    let snapshotLock;
+    let snapshotCreated = false;
+    let derivedCreated = false;
+    try {
+      if (plan.mode === ResumeMode.DOCKER_SNAPSHOT) {
+        snapshotLock = acquireSnapshotLock(deps.snapshotOptions);
+        const preflight = preflightSnapshot(
+          plan.sessionName,
+          runner,
+          deps.snapshotOptions
+        );
+        plan.message += `\n${preflight.message}`;
+        if (record.logPath) {
+          appendLogFile(record.logPath, `${preflight.message}\n`);
+        }
+      }
+      if (plan.handoffCommand) {
+        const copied = writeCommandHandoff(
+          plan.sessionName,
+          plan.handoffCommand,
+          runner,
+          opts
+        );
+        if (!copied.success) {
+          throw new Error(
+            (copied.stderr || '').trim() ||
+              copied.error ||
+              'Could not copy replacement command into the stopped container'
+          );
+        }
+      }
+      for (const step of plan.steps) {
+        let result;
+        try {
+          result = runner(step.command, step.args);
+        } catch (error) {
+          result = { success: false, error: error.message };
+        }
+        if (!result.success) {
+          const detail =
+            (result.stderr || '').trim() ||
+            result.error ||
+            `exit code ${result.status}`;
+          throw new Error(detail);
+        }
+        if (step.args[0] === 'commit') {
+          snapshotCreated = true;
+        }
+        if (step.args[0] === 'create') {
+          derivedCreated = true;
+        }
+        // commit returns an image ID; cp/update/network commands are not IDs.
+        if (['run', 'create'].includes(step.args[0])) {
+          containerId = (result.stdout || '').trim() || containerId;
+        }
+      }
+      if (plan.mode === ResumeMode.DOCKER_SNAPSHOT && deps.removeOriginal) {
+        const running = runner(getDockerCommand(), [
+          'inspect',
+          '-f',
+          '{{.State.Running}}',
+          plan.newSessionName,
+        ]);
+        if (running.success && String(running.stdout).trim() === 'true') {
+          const removed = runner(getDockerCommand(), ['rm', plan.sessionName]);
+          const note = removed.success
+            ? `Stopped original container removed: ${plan.sessionName}`
+            : `Original container retained: ${(removed.stderr || removed.error || 'docker rm failed').trim()}`;
+          plan.message += `\n${note}`;
+          if (record.logPath) {
+            appendLogFile(record.logPath, `${note}\n`);
+          }
+        } else {
+          plan.message +=
+            '\nOriginal container retained: the snapshot container is not running.';
+        }
+      }
+    } catch (error) {
+      if (derivedCreated) {
+        runner(getDockerCommand(), ['rm', plan.newSessionName]);
+      }
+      if (snapshotCreated) {
+        runner(getDockerCommand(), ['rmi', plan.snapshotImage]);
       }
-      containerId = (result.stdout || '').trim() || containerId;
+      if (plan.handoffCommand) {
+        // A failed start must not leave a replacement hidden behind the old record.
+        try {
+          writeCommandHandoff(plan.sessionName, previous.command, runner, opts);
+        } catch {
+          /* Next explicit command can still repair the handoff. */
+        }
+      }
+      rollbackLaunch(store, previous);
+      appendLifecycle(lifecycleRecord, 'launch-failed', {
+        error: error.message,
+      });
+      return {
+        success: false,
+        error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${error.message}`,
+      };
+    } finally {
+      snapshotLock?.release();
     }
 
     const limitsLine = buildResourceLimitsStatusLine(plan.resourceLimits);
@@ -489,8 +717,16 @@ async function resumeExecution(store, identifier, deps = {}) {
 
   const previousSessionName = plan.newSessionName ? plan.sessionName : null;
   attempt.launchAcceptedAt = new Date().toISOString();
-  const updated = applyResumeToRecord(record, plan, containerId, attempt);
-  store.save(updated);
+  updated.options.launchPending = false;
+  if (containerId) {
+    updated.options.containerId = containerId;
+  }
+  let persistenceError = null;
+  try {
+    store.save(updated);
+  } catch (error) {
+    persistenceError = error;
+  }
   appendLifecycle(updated, 'launch-accepted');
 
   if (plan.backend === 'docker') {
@@ -513,26 +749,50 @@ async function resumeExecution(store, identifier, deps = {}) {
       if (watcher === false) {
         throw new Error('Watcher process could not be started');
       }
-      const current = patchAttempt(store, updated, {
-        watcherAttachedAt: new Date().toISOString(),
-      });
+      const current = persistenceError
+        ? null
+        : patchAttempt(store, updated, {
+            watcherAttachedAt: new Date().toISOString(),
+          });
       if (current) {
         appendLifecycle(current, 'watcher-attached');
       }
     } catch (error) {
-      const current = patchAttempt(store, updated, {
-        watcherError: error.message,
-      });
+      if (persistenceError) {
+        return {
+          success: false,
+          error: failLaunchedPersistence(
+            store,
+            updated,
+            runner,
+            persistenceError
+          ),
+        };
+      }
+      const current = persistenceError
+        ? null
+        : patchAttempt(store, updated, {
+            watcherError: error.message,
+          });
       appendLifecycle(current || updated, 'watcher-attachment-failed', {
         error: error.message,
       });
       return {
         success: false,
-        error: `Launch accepted, but completion watcher attachment failed: ${error.message}`,
+        error: `Launch accepted for container ${activeSessionName(plan)}, but completion watcher attachment failed: ${error.message}`,
       };
     }
   }
 
+  if (persistenceError) {
+    // The reservation remains durable and the watcher is attached. Stop the
+    // new container rather than allow an untracked command to keep running.
+    return {
+      success: false,
+      error: failLaunchedPersistence(store, updated, runner, persistenceError),
+    };
+  }
+
   return {
     success: true,
     output: formatResumeResult(