start-command 0.35.3 → 0.36.0

package/src/lib/docker-runtime-args.js

@@ -97,6 +97,12 @@ function buildDockerRuntimeStatusLines(options = {}) {
   if (limitsLine) {
     lines.push(limitsLine);
   }
+  const limitsSummary = require('./docker-resource-options').limitsLogLine(
+    options
+  );
+  if (limitsSummary) {
+    lines.push(limitsSummary);
+  }
   const onKillResume = getOnKillResume(options);
   if (onKillResume) {
     const what = options.recoveryCommand || 'the original command';
@@ -131,6 +137,15 @@ function buildDockerRuntimeMetadata(options = {}) {
         ? options.networkAliases
         : null,
     resourceLimits: resourceLimits.length > 0 ? resourceLimits : null,
+    resolvedLimits: options.resolvedLimits || null,
+    resourceLimitSpecs: options.resourceLimitSpecs || null,
+    onKillResumeMemory: options.onKillResumeMemory || null,
+    ...(options.cpuPenaltyConfig
+      ? {
+          cpuPenaltyConfig: options.cpuPenaltyConfig,
+          baseResourceLimits: resourceLimits,
+        }
+      : {}),
     onKillResume: getOnKillResume(options),
     recoveryCommand: options.recoveryCommand || null,
     onKillResumeDelay: getOnKillResume(options)

package/src/lib/execution-attempt.js

@@ -0,0 +1,153 @@
+/** Attempt-scoped evidence for logical executions sharing a UUID and log. */
+const fs = require('fs');
+const { appendLogFile, readLogTail } = require('./isolation-log-utils');
+
+const TERMINAL_FIELDS = [
+  'endTimeSource',
+  'observedAt',
+  'staleDetectedAt',
+  'containerStartedAt',
+  'exitReason',
+  'oomKilled',
+  'memoryExhausted',
+  'memoryExhaustedReason',
+  'cgroupMemory',
+];
+
+function activityPath(record) {
+  return `${record.logPath}.attempt-${record.attempt.number}.activity`;
+}
+
+function readAttemptActivity(record) {
+  if (!record.attempt || !record.logPath) {
+    return record;
+  }
+  let fd;
+  try {
+    fd = fs.openSync(activityPath(record), 'r');
+    const buffer = Buffer.alloc(128);
+    const length = fs.readSync(fd, buffer, 0, buffer.length, 0);
+    const at = buffer.subarray(0, length).toString().trim();
+    if (
+      Number.isFinite(Date.parse(at)) &&
+      Date.parse(at) >= Date.parse(record.attempt.startedAt)
+    ) {
+      record.attempt = { ...record.attempt, lastOutputAt: at };
+    }
+  } catch {
+    // Quiet, missing, or unreadable output is not evidence of task progress.
+  } finally {
+    if (fd !== undefined) {
+      fs.closeSync(fd);
+    }
+  }
+  return record;
+}
+
+function createAttempt(record, plan) {
+  let logOffset = null;
+  if (record.logPath) {
+    try {
+      logOffset = fs.statSync(record.logPath).size;
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        logOffset = 0;
+      }
+    }
+  }
+  return {
+    number:
+      (record.attempt?.number ||
+        (Number(record.options?.resumeCount) || 0) +
+          (Number(record.options?.recoveryAttempts) || 0) +
+          1) + 1,
+    startedAt: new Date().toISOString(),
+    logOffset,
+    lastOutputAt: null,
+    launchAcceptedAt: null,
+    watcherAttachedAt: null,
+    mode: plan.mode,
+    previousSessionName: record.options?.sessionName || null,
+    sessionName: plan.newSessionName || plan.sessionName,
+  };
+}
+
+function archiveAttempt(record) {
+  readAttemptActivity(record);
+  const snapshot = {
+    ...(record.attempt || {
+      number:
+        (Number(record.options?.resumeCount) || 0) +
+        (Number(record.options?.recoveryAttempts) || 0) +
+        1,
+      startedAt: record.options?.resumedAt || record.startTime,
+      logOffset: null,
+    }),
+    command: record.command,
+    sessionName: record.options?.sessionName || null,
+    containerId: record.options?.containerId || null,
+    containerError: record.options?.containerError || null,
+    status: record.status,
+    exitCode: record.exitCode,
+    endTime: record.endTime,
+  };
+  for (const field of TERMINAL_FIELDS) {
+    if (record[field] !== undefined) {
+      snapshot[field] = record[field];
+    }
+    record[field] = undefined;
+  }
+  record.attemptHistory = [...(record.attemptHistory || []), snapshot];
+  delete record.options.containerError;
+}
+
+function appendLifecycle(record, event, details = {}) {
+  if (!record.logPath || !record.attempt) {
+    return;
+  }
+  appendLogFile(
+    record.logPath,
+    `\n[Start Command Lifecycle] ${JSON.stringify({
+      event,
+      at: new Date().toISOString(),
+      uuid: record.uuid,
+      attemptNumber: record.attempt.number,
+      ...record.attempt,
+      ...details,
+    })}\n`
+  );
+}
+
+function patchAttempt(store, record, fields) {
+  if (typeof store.patchAttempt === 'function') {
+    return store.patchAttempt(record.uuid, record.attempt.number, fields);
+  }
+  // Compatibility for consumers injecting a minimal in-memory store.
+  const current = store.get(record.uuid) || record;
+  if (current.attempt?.number !== record.attempt.number) {
+    return null;
+  }
+  current.attempt = { ...current.attempt, ...fields };
+  store.save(current);
+  return current;
+}
+
+function readAttemptLogTail(record, bytes) {
+  if (
+    !record.logPath ||
+    (record.attempt && record.attempt.logOffset === null)
+  ) {
+    return null;
+  }
+  return readLogTail(record.logPath, bytes, record.attempt?.logOffset || 0);
+}
+
+module.exports = {
+  activityPath,
+  appendLifecycle,
+  archiveAttempt,
+  createAttempt,
+  patchAttempt,
+  readAttemptActivity,
+  readAttemptLogTail,
+};

package/src/lib/execution-recovery.js

@@ -39,6 +39,11 @@ const {
   readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { shellQuote } = require('./isolation-log-utils');
+const {
+  reserveLaunch,
+  rollbackLaunch,
+  failLaunchedPersistence,
+} = require('./launch-persistence');
 const {
   CGROUP_SHELL_VARS,
   formatCgroupMemoryLogLine,
@@ -299,7 +304,51 @@ function recoverKilledExecution(params = {}) {
   // Docker keeps `docker update` limits in the HostConfig across restarts;
   // they are read back so the log and `--status` show what the resumed run
   // is held to, and so a later relaunch can re-apply them.
-  const resourceLimits = readDockerResourceLimits(containerName, runner);
+  let resourceLimits =
+    readDockerResourceLimits(containerName, runner) ||
+    opts.resourceLimits ||
+    [];
+  resourceLimits = require('./resume-resources').restoreBaseCpu(
+    resourceLimits,
+    opts
+  );
+  if (opts.cpuPenaltyConfig) {
+    try {
+      resourceLimits = require('./resume-resources').clampCpuToDaemon(
+        resourceLimits,
+        runner
+      );
+    } catch (error) {
+      return giveUp(error.message);
+    }
+  }
+  const evidence = require('./exit-evidence');
+  const terminalOom =
+    (params.oomKilled === true || params.oomKilled === 'true') &&
+    evidence.recentOomDelta(params.cgroupMemory, params.finishedAt) &&
+    !evidence.fromLog(
+      record.logPath
+        ? require('./execution-attempt').readAttemptLogTail(record, 65536)
+        : null
+    ).daemonRestart;
+  let recoveryLimits = null;
+  if (opts.onKillResumeMemory && terminalOom) {
+    try {
+      recoveryLimits =
+        require('./docker-resource-options').resolveResourceOptions(
+          { memory: opts.onKillResumeMemory, resourceLimits },
+          runner,
+          params.random
+        );
+    } catch (error) {
+      return giveUp(`recovery limit failed: ${error.message}`);
+    }
+    log(
+      `Recovery memory limit: ${resourceLimits.find((s) => s.startsWith('--memory=')) || 'unlimited'} -> ${recoveryLimits.resolvedLimits.memory} (${opts.onKillResumeMemory})\n`
+    );
+    resourceLimits = recoveryLimits.resourceLimits;
+  }
+
   const limitsLine = buildResourceLimitsStatusLine(resourceLimits);
   if (limitsLine) {
     log(`${limitsLine}\n`);
@@ -318,15 +367,42 @@ function recoverKilledExecution(params = {}) {
   }
 
   const since = now().toISOString();
-  const started = runner(getDockerCommand(), ['start', containerName]);
-  if (!started.success) {
-    return giveUp(`docker start failed: ${failureDetail(started)}`);
+  const {
+    appendLifecycle,
+    archiveAttempt,
+    createAttempt,
+    patchAttempt,
+  } = require('./execution-attempt');
+  const nextAttempt = record.attempt
+    ? createAttempt(record, {
+        mode: 'automatic-recovery',
+        sessionName: containerName,
+      })
+    : null;
+  if (nextAttempt) {
+    nextAttempt.startedAt = since;
+    appendLifecycle(
+      { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
+      'resume-started'
+    );
   }
-
   const described = describeExitCode(params.exitCode);
   const delayed = delayRange ? `, delayMs=${delayMs}` : '';
   // The killed run's own counters: the resumed run starts a fresh cgroup.
   const memory = parseCgroupMemorySample(params.cgroupMemory);
+  if (nextAttempt) {
+    Object.assign(record, {
+      status: 'executed',
+      exitCode: described.code,
+      endTime: params.finishedAt || null,
+      containerStartedAt: params.startedAt || undefined,
+      oomKilled: params.oomKilled === true || params.oomKilled === 'true',
+      cgroupMemory: memory || undefined,
+    });
+    archiveAttempt(record);
+
+    record.attempt = nextAttempt;
+  }
   const counted = memory
     ? ['oomEvents', 'oomKills']
         .filter((field) => memory[field] !== null)
@@ -335,6 +411,18 @@ function recoverKilledExecution(params = {}) {
     : '';
   record.options = {
     ...opts,
+    ...(recoveryLimits
+      ? {
+          resolvedLimits: {
+            ...opts.resolvedLimits,
+            ...recoveryLimits.resolvedLimits,
+          },
+          resourceLimitSpecs: {
+            ...opts.resourceLimitSpecs,
+            memory: opts.onKillResumeMemory,
+          },
+        }
+      : {}),
     recoveryAttempts: attempt,
     recoveryHistory: [
       ...(Array.isArray(opts.recoveryHistory) ? opts.recoveryHistory : []),
@@ -344,28 +432,116 @@ function recoverKilledExecution(params = {}) {
     ...(delayRange ? { lastRecoveryDelayMs: delayMs } : {}),
     ...(resourceLimits && resourceLimits.length > 0 ? { resourceLimits } : {}),
   };
+  delete record.options.exitEvidence;
   record.status = 'executing';
   record.exitCode = null;
   record.endTime = null;
   record.exitReason = undefined;
   record.oomKilled = undefined;
   record.cgroupMemory = undefined;
+  record.memoryExhausted = undefined;
+  record.memoryExhaustedReason = undefined;
+  record.endTimeSource = undefined;
+  record.observedAt = undefined;
+  record.staleDetectedAt = undefined;
+  record.containerStartedAt = undefined;
+  const previousData = globalThis.structuredClone(store.get(executionId));
+  const previous = record.constructor.fromObject
+    ? record.constructor.fromObject(previousData)
+    : previousData;
+  try {
+    reserveLaunch(store, record, previous);
+  } catch (error) {
+    return giveUp(`launch reservation failed: ${error.message}`);
+  }
+  if (recoveryLimits || opts.cpuPenaltyConfig) {
+    const updated = runner(getDockerCommand(), [
+      'update',
+      ...resourceLimits.filter((s) =>
+        /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
+      ),
+      containerName,
+    ]);
+    if (!updated.success) {
+      rollbackLaunch(store, previous);
+      return giveUp(`docker update failed: ${failureDetail(updated)}`);
+    }
+  }
+  const started = runner(getDockerCommand(), ['start', containerName]);
+  if (!started.success) {
+    if (nextAttempt) {
+      appendLifecycle(
+        { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
+        'launch-failed',
+        { error: failureDetail(started) }
+      );
+    }
+    rollbackLaunch(store, previous);
+    return giveUp(`docker start failed: ${failureDetail(started)}`);
+  }
+
+  if (opts.cpuPenaltyConfig) {
+    record.options.resolvedLimits = {
+      ...record.options.resolvedLimits,
+      cpus: require('./docker-resource-limits').cpuCount(resourceLimits),
+    };
+  }
+  record.options.launchPending = false;
+  if (nextAttempt) {
+    nextAttempt.launchAcceptedAt = new Date().toISOString();
+  }
+  let persistenceError = null;
   try {
     store.save(record);
-  } catch {
-    // The container is already running again; the new watcher still follows
-    // it and finalizes the record when it ends.
+  } catch (error) {
+    persistenceError = error;
   }
 
   const startWatcher =
     params.startWatcher || startDetachedDockerCompletionWatcher;
-  startWatcher(
-    containerName,
-    getDockerContainerCleanupPolicy(opts),
-    record.logPath || null,
-    record.uuid,
-    { since, recoverOnKill: true }
-  );
+  if (nextAttempt) {
+    appendLifecycle(record, 'launch-accepted');
+  }
+  try {
+    startWatcher(
+      containerName,
+      getDockerContainerCleanupPolicy(opts),
+      record.logPath || null,
+      record.uuid,
+      {
+        since,
+        recoverOnKill: true,
+        ...(nextAttempt ? { attemptNumber: nextAttempt.number } : {}),
+      }
+    );
+  } catch (error) {
+    const warning = failLaunchedPersistence(
+      store,
+      record,
+      runner,
+      persistenceError || error
+    );
+    log(`[Recovery] ${warning}\n`);
+    return { recovered: false, reason: warning, attempt };
+  }
+  if (nextAttempt && !persistenceError) {
+    const current = patchAttempt(store, record, {
+      watcherAttachedAt: new Date().toISOString(),
+    });
+    if (current) {
+      appendLifecycle(current, 'watcher-attached');
+    }
+  }
+  if (persistenceError) {
+    const warning = failLaunchedPersistence(
+      store,
+      record,
+      runner,
+      persistenceError
+    );
+    log(`[Recovery] ${warning}\n`);
+    return { recovered: false, reason: warning, attempt };
+  }
   return {
     recovered: true,
     reason: 'resumed',

package/src/lib/execution-resume-all.js

@@ -125,13 +125,35 @@ function resumeAllExecutions(store, deps = {}) {
   for (const record of records) {
     const probe = probeFn(record);
     const sessionName = probe.sessionName;
+    if (require('./launch-owner').hasActiveLaunch(record)) {
+      executions.push(
+        describeRecord(
+          record,
+          probe,
+          ResumeAllAction.RUNNING,
+          'Launch is reserved; no watcher or duplicate launch was started.'
+        )
+      );
+      continue;
+    }
 
     if (probe.alive && probe.backend === 'docker') {
       startWatcher(
         sessionName,
         getDockerContainerCleanupPolicy(record.options || {}),
         record.logPath || null,
-        record.uuid || null
+        record.uuid || null,
+        record.attempt
+          ? {
+              since: record.attempt.startedAt,
+              attemptNumber: record.attempt.number,
+              recoverOnKill:
+                Boolean(record.options?.onKillResume) &&
+                ['docker-start', 'automatic-recovery', 'relaunch'].includes(
+                  record.attempt.mode
+                ),
+            }
+          : {}
       );
       executions.push(
         describeRecord(