start-command 0.35.4 → 0.36.0

package/src/lib/execution-recovery.js

@@ -39,6 +39,11 @@ const {
   readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { shellQuote } = require('./isolation-log-utils');
+const {
+  reserveLaunch,
+  rollbackLaunch,
+  failLaunchedPersistence,
+} = require('./launch-persistence');
 const {
   CGROUP_SHELL_VARS,
   formatCgroupMemoryLogLine,
@@ -299,7 +304,51 @@ function recoverKilledExecution(params = {}) {
   // Docker keeps `docker update` limits in the HostConfig across restarts;
   // they are read back so the log and `--status` show what the resumed run
   // is held to, and so a later relaunch can re-apply them.
-  const resourceLimits = readDockerResourceLimits(containerName, runner);
+  let resourceLimits =
+    readDockerResourceLimits(containerName, runner) ||
+    opts.resourceLimits ||
+    [];
+  resourceLimits = require('./resume-resources').restoreBaseCpu(
+    resourceLimits,
+    opts
+  );
+  if (opts.cpuPenaltyConfig) {
+    try {
+      resourceLimits = require('./resume-resources').clampCpuToDaemon(
+        resourceLimits,
+        runner
+      );
+    } catch (error) {
+      return giveUp(error.message);
+    }
+  }
+  const evidence = require('./exit-evidence');
+  const terminalOom =
+    (params.oomKilled === true || params.oomKilled === 'true') &&
+    evidence.recentOomDelta(params.cgroupMemory, params.finishedAt) &&
+    !evidence.fromLog(
+      record.logPath
+        ? require('./execution-attempt').readAttemptLogTail(record, 65536)
+        : null
+    ).daemonRestart;
+  let recoveryLimits = null;
+  if (opts.onKillResumeMemory && terminalOom) {
+    try {
+      recoveryLimits =
+        require('./docker-resource-options').resolveResourceOptions(
+          { memory: opts.onKillResumeMemory, resourceLimits },
+          runner,
+          params.random
+        );
+    } catch (error) {
+      return giveUp(`recovery limit failed: ${error.message}`);
+    }
+    log(
+      `Recovery memory limit: ${resourceLimits.find((s) => s.startsWith('--memory=')) || 'unlimited'} -> ${recoveryLimits.resolvedLimits.memory} (${opts.onKillResumeMemory})\n`
+    );
+    resourceLimits = recoveryLimits.resourceLimits;
+  }
+
   const limitsLine = buildResourceLimitsStatusLine(resourceLimits);
   if (limitsLine) {
     log(`${limitsLine}\n`);
@@ -337,18 +386,6 @@ function recoverKilledExecution(params = {}) {
       'resume-started'
     );
   }
-  const started = runner(getDockerCommand(), ['start', containerName]);
-  if (!started.success) {
-    if (nextAttempt) {
-      appendLifecycle(
-        { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
-        'launch-failed',
-        { error: failureDetail(started) }
-      );
-    }
-    return giveUp(`docker start failed: ${failureDetail(started)}`);
-  }
-
   const described = describeExitCode(params.exitCode);
   const delayed = delayRange ? `, delayMs=${delayMs}` : '';
   // The killed run's own counters: the resumed run starts a fresh cgroup.
@@ -363,7 +400,7 @@ function recoverKilledExecution(params = {}) {
       cgroupMemory: memory || undefined,
     });
     archiveAttempt(record);
-    nextAttempt.launchAcceptedAt = new Date().toISOString();
+
     record.attempt = nextAttempt;
   }
   const counted = memory
@@ -374,6 +411,18 @@ function recoverKilledExecution(params = {}) {
     : '';
   record.options = {
     ...opts,
+    ...(recoveryLimits
+      ? {
+          resolvedLimits: {
+            ...opts.resolvedLimits,
+            ...recoveryLimits.resolvedLimits,
+          },
+          resourceLimitSpecs: {
+            ...opts.resourceLimitSpecs,
+            memory: opts.onKillResumeMemory,
+          },
+        }
+      : {}),
     recoveryAttempts: attempt,
     recoveryHistory: [
       ...(Array.isArray(opts.recoveryHistory) ? opts.recoveryHistory : []),
@@ -383,6 +432,7 @@ function recoverKilledExecution(params = {}) {
     ...(delayRange ? { lastRecoveryDelayMs: delayMs } : {}),
     ...(resourceLimits && resourceLimits.length > 0 ? { resourceLimits } : {}),
   };
+  delete record.options.exitEvidence;
   record.status = 'executing';
   record.exitCode = null;
   record.endTime = null;
@@ -395,11 +445,56 @@ function recoverKilledExecution(params = {}) {
   record.observedAt = undefined;
   record.staleDetectedAt = undefined;
   record.containerStartedAt = undefined;
+  const previousData = globalThis.structuredClone(store.get(executionId));
+  const previous = record.constructor.fromObject
+    ? record.constructor.fromObject(previousData)
+    : previousData;
+  try {
+    reserveLaunch(store, record, previous);
+  } catch (error) {
+    return giveUp(`launch reservation failed: ${error.message}`);
+  }
+  if (recoveryLimits || opts.cpuPenaltyConfig) {
+    const updated = runner(getDockerCommand(), [
+      'update',
+      ...resourceLimits.filter((s) =>
+        /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
+      ),
+      containerName,
+    ]);
+    if (!updated.success) {
+      rollbackLaunch(store, previous);
+      return giveUp(`docker update failed: ${failureDetail(updated)}`);
+    }
+  }
+  const started = runner(getDockerCommand(), ['start', containerName]);
+  if (!started.success) {
+    if (nextAttempt) {
+      appendLifecycle(
+        { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
+        'launch-failed',
+        { error: failureDetail(started) }
+      );
+    }
+    rollbackLaunch(store, previous);
+    return giveUp(`docker start failed: ${failureDetail(started)}`);
+  }
+
+  if (opts.cpuPenaltyConfig) {
+    record.options.resolvedLimits = {
+      ...record.options.resolvedLimits,
+      cpus: require('./docker-resource-limits').cpuCount(resourceLimits),
+    };
+  }
+  record.options.launchPending = false;
+  if (nextAttempt) {
+    nextAttempt.launchAcceptedAt = new Date().toISOString();
+  }
+  let persistenceError = null;
   try {
     store.save(record);
-  } catch {
-    // The container is already running again; the new watcher still follows
-    // it and finalizes the record when it ends.
+  } catch (error) {
+    persistenceError = error;
   }
 
   const startWatcher =
@@ -407,18 +502,29 @@ function recoverKilledExecution(params = {}) {
   if (nextAttempt) {
     appendLifecycle(record, 'launch-accepted');
   }
-  startWatcher(
-    containerName,
-    getDockerContainerCleanupPolicy(opts),
-    record.logPath || null,
-    record.uuid,
-    {
-      since,
-      recoverOnKill: true,
-      ...(nextAttempt ? { attemptNumber: nextAttempt.number } : {}),
-    }
-  );
-  if (nextAttempt) {
+  try {
+    startWatcher(
+      containerName,
+      getDockerContainerCleanupPolicy(opts),
+      record.logPath || null,
+      record.uuid,
+      {
+        since,
+        recoverOnKill: true,
+        ...(nextAttempt ? { attemptNumber: nextAttempt.number } : {}),
+      }
+    );
+  } catch (error) {
+    const warning = failLaunchedPersistence(
+      store,
+      record,
+      runner,
+      persistenceError || error
+    );
+    log(`[Recovery] ${warning}\n`);
+    return { recovered: false, reason: warning, attempt };
+  }
+  if (nextAttempt && !persistenceError) {
     const current = patchAttempt(store, record, {
       watcherAttachedAt: new Date().toISOString(),
     });
@@ -426,6 +532,16 @@ function recoverKilledExecution(params = {}) {
       appendLifecycle(current, 'watcher-attached');
     }
   }
+  if (persistenceError) {
+    const warning = failLaunchedPersistence(
+      store,
+      record,
+      runner,
+      persistenceError
+    );
+    log(`[Recovery] ${warning}\n`);
+    return { recovered: false, reason: warning, attempt };
+  }
   return {
     recovered: true,
     reason: 'resumed',

package/src/lib/execution-resume-all.js

@@ -125,6 +125,17 @@ function resumeAllExecutions(store, deps = {}) {
   for (const record of records) {
     const probe = probeFn(record);
     const sessionName = probe.sessionName;
+    if (require('./launch-owner').hasActiveLaunch(record)) {
+      executions.push(
+        describeRecord(
+          record,
+          probe,
+          ResumeAllAction.RUNNING,
+          'Launch is reserved; no watcher or duplicate launch was started.'
+        )
+      );
+      continue;
+    }
 
     if (probe.alive && probe.backend === 'docker') {
       startWatcher(

package/src/lib/execution-resume.js

@@ -28,7 +28,6 @@ const { escapeForLinksNotation } = require('./output-blocks');
 const {
   buildResourceLimitsStatusLine,
   normalizeResourceLimits,
-  readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { appendLogFile } = require('./isolation-log-utils');
 const {
@@ -38,6 +37,11 @@ const {
   patchAttempt,
 } = require('./execution-attempt');
 const { runCommand } = require('./execution-control');
+const {
+  reserveLaunch,
+  rollbackLaunch,
+  failLaunchedPersistence,
+} = require('./launch-persistence');
 const { SessionState, probeSession } = require('./session-probe');
 const { getDockerNetworks } = require('./docker-network-lifecycle');
 const {
@@ -89,6 +93,7 @@ function buildResumedSessionName(sessionName, attempt) {
 function buildLaunchOptions(record) {
   const opts = record.options || {};
   return {
+    ...opts,
     image: opts.image || null,
     session: opts.sessionName,
     detached: true,
@@ -359,6 +364,7 @@ function applyResumeToRecord(
   options.resumedAt = attempt.startedAt;
   // A resume is a new deliberate start: launch-time recovery applies again.
   delete options.stopRequestedAt;
+  delete options.exitEvidence;
   if (plan.resourceLimits && plan.resourceLimits.length > 0) {
     options.resourceLimits = plan.resourceLimits;
   }
@@ -411,27 +417,61 @@ async function resumeExecution(store, identifier, deps = {}) {
     };
   }
 
+  if (require('./launch-owner').hasActiveLaunch(record)) {
+    return {
+      success: false,
+      error: 'An execution launch is already reserved.',
+    };
+  }
+  const previousData = globalThis.structuredClone(
+    record.toObject ? record.toObject() : record
+  );
+  const previous = record.constructor.fromObject
+    ? record.constructor.fromObject(previousData)
+    : previousData;
   const runner = deps.runner || runCommand;
   const probeFn = deps.probe || ((r) => probeSession(r, runner));
   const probe = probeFn(record);
   const opts = record.options || {};
-  // Only a snapshot resume creates a new container, so only it needs the old
-  // container's limits (issue #176).
-  const liveResourceLimits =
-    deps.command &&
-    opts.isolated === 'docker' &&
-    opts.sessionName &&
-    probe &&
-    !probe.alive &&
-    probe.state === SessionState.STOPPED
-      ? readDockerResourceLimits(opts.sessionName, runner)
-      : null;
+  let resources;
+  try {
+    resources = require('./resume-resources').prepareResumeResources(
+      record,
+      probe,
+      deps.resourceOptions || {},
+      runner,
+      deps.random
+    );
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+  const planningRecord = resources
+    ? {
+        ...record,
+        options: { ...opts, resourceLimits: resources.resourceLimits },
+      }
+    : record;
   const plan = buildResumePlan(
-    record,
+    planningRecord,
     deps.command || null,
     probe,
-    liveResourceLimits
+    resources?.resourceLimits
   );
+  if (resources && !plan.error) {
+    plan.resourceLimits = resources.resourceLimits;
+    if (plan.mode === ResumeMode.DOCKER_START && resources.update) {
+      plan.steps.unshift({
+        command: getDockerCommand(),
+        args: [
+          'update',
+          ...resources.resourceLimits.filter((s) =>
+            /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
+          ),
+          plan.sessionName,
+        ],
+      });
+    }
+  }
   if (plan.error) {
     return { success: false, error: plan.error };
   }
@@ -444,15 +484,50 @@ async function resumeExecution(store, identifier, deps = {}) {
     attempt,
   };
   appendLifecycle(lifecycleRecord, 'resume-started');
+  const updated = applyResumeToRecord(record, plan, null, attempt);
+  if (resources) {
+    updated.options.resolvedLimits = resources.resolvedLimits;
+    updated.options.resourceLimitSpecs = resources.resourceLimitSpecs;
+    if (
+      opts.cpuPenaltyConfig &&
+      deps.resourceOptions?.cpus !== null &&
+      deps.resourceOptions?.cpus !== undefined
+    ) {
+      updated.options.baseResourceLimits = resources.resourceLimits;
+    }
+  }
+  try {
+    reserveLaunch(store, updated, previous);
+  } catch (error) {
+    appendLifecycle(lifecycleRecord, 'launch-failed', { error: error.message });
+    return {
+      success: false,
+      error: `Launch reservation failed: ${error.message}`,
+    };
+  }
+
+  if (resources && deps.resourceOptions?.memory && record.logPath) {
+    const oldMemory = resources.oldMemory;
+    appendLogFile(
+      record.logPath,
+      `Resume memory limit: ${oldMemory} -> ${resources.resolvedLimits.memory} (${deps.resourceOptions.memory})\n`
+    );
+  }
 
   if (plan.mode === ResumeMode.RELAUNCH) {
     const runIsolated = deps.runIsolated || require('./isolation').runIsolated;
-    const launchResult = await runIsolated(
-      plan.backend,
-      plan.command,
-      plan.launchOptions
-    );
+    let launchResult;
+    try {
+      launchResult = await runIsolated(
+        plan.backend,
+        plan.command,
+        plan.launchOptions
+      );
+    } catch (error) {
+      launchResult = { success: false, message: error.message };
+    }
     if (!launchResult || !launchResult.success) {
+      rollbackLaunch(store, previous);
       appendLifecycle(lifecycleRecord, 'launch-failed', {
         error: launchResult?.message || 'unknown error',
       });
@@ -466,8 +541,14 @@ async function resumeExecution(store, identifier, deps = {}) {
     containerId = launchResult.containerId || null;
   } else {
     for (const step of plan.steps) {
-      const result = runner(step.command, step.args);
+      let result;
+      try {
+        result = runner(step.command, step.args);
+      } catch (error) {
+        result = { success: false, error: error.message };
+      }
       if (!result.success) {
+        rollbackLaunch(store, previous);
         const detail =
           (result.stderr || '').trim() ||
           result.error ||
@@ -489,8 +570,16 @@ async function resumeExecution(store, identifier, deps = {}) {
 
   const previousSessionName = plan.newSessionName ? plan.sessionName : null;
   attempt.launchAcceptedAt = new Date().toISOString();
-  const updated = applyResumeToRecord(record, plan, containerId, attempt);
-  store.save(updated);
+  updated.options.launchPending = false;
+  if (containerId) {
+    updated.options.containerId = containerId;
+  }
+  let persistenceError = null;
+  try {
+    store.save(updated);
+  } catch (error) {
+    persistenceError = error;
+  }
   appendLifecycle(updated, 'launch-accepted');
 
   if (plan.backend === 'docker') {
@@ -513,26 +602,50 @@ async function resumeExecution(store, identifier, deps = {}) {
       if (watcher === false) {
         throw new Error('Watcher process could not be started');
       }
-      const current = patchAttempt(store, updated, {
-        watcherAttachedAt: new Date().toISOString(),
-      });
+      const current = persistenceError
+        ? null
+        : patchAttempt(store, updated, {
+            watcherAttachedAt: new Date().toISOString(),
+          });
       if (current) {
         appendLifecycle(current, 'watcher-attached');
       }
     } catch (error) {
-      const current = patchAttempt(store, updated, {
-        watcherError: error.message,
-      });
+      if (persistenceError) {
+        return {
+          success: false,
+          error: failLaunchedPersistence(
+            store,
+            updated,
+            runner,
+            persistenceError
+          ),
+        };
+      }
+      const current = persistenceError
+        ? null
+        : patchAttempt(store, updated, {
+            watcherError: error.message,
+          });
       appendLifecycle(current || updated, 'watcher-attachment-failed', {
         error: error.message,
       });
       return {
         success: false,
-        error: `Launch accepted, but completion watcher attachment failed: ${error.message}`,
+        error: `Launch accepted for container ${activeSessionName(plan)}, but completion watcher attachment failed: ${error.message}`,
       };
     }
   }
 
+  if (persistenceError) {
+    // The reservation remains durable and the watcher is attached. Stop the
+    // new container rather than allow an untracked command to keep running.
+    return {
+      success: false,
+      error: failLaunchedPersistence(store, updated, runner, persistenceError),
+    };
+  }
+
   return {
     success: true,
     output: formatResumeResult(