start-command 0.35.4 → 0.37.0

package/src/lib/docker-resource-options.js

@@ -0,0 +1,180 @@
+/** Launch and recovery resource specifications, resolved against the daemon. */
+const { getDockerCommand } = require('./docker-cleanup');
+const { normalizeResourceLimits } = require('./docker-resource-limits');
+const FIELDS = {
+  '--memory': 'memory',
+  '--memory-swap': 'memorySwap',
+  '--cpus': 'cpus',
+  '--on-kill-resume-memory': 'onKillResumeMemory',
+};
+
+function parseResourceSpec(value, cpu = false) {
+  const text = String(value).trim().toLowerCase();
+  const percent = /^(\d+(?:\.\d+)?)%(?:-(\d+(?:\.\d+)?)%)?$/.exec(text);
+  if (percent) {
+    const min = Number(percent[1]),
+      max = Number(percent[2] || percent[1]);
+    if (min > 0 && max <= 100 && min <= max) {
+      return { min, max, percent: true };
+    }
+  } else {
+    const absolute = /^(\d+(?:\.\d+)?)(b|k|m|g|t|kb|mb|gb|tb)?$/.exec(text);
+    if (absolute && (!cpu || !absolute[2])) {
+      const power = ['', 'k', 'm', 'g', 't'].indexOf(
+        (absolute[2] || '').replace('b', '')
+      );
+      const amount =
+        Number(absolute[1]) * (cpu ? 1 : 1024 ** Math.max(0, power));
+      if (
+        Number.isFinite(amount) &&
+        amount > 0 &&
+        amount <= Number.MAX_SAFE_INTEGER
+      ) {
+        return { amount };
+      }
+    }
+  }
+  throw new Error(
+    `Invalid resource limit: "${value}". Expected a positive ${cpu ? 'CPU count' : 'Docker size'}, N% or MIN%-MAX% within (0, 100].`
+  );
+}
+
+function parse(args, index, options) {
+  const [flag, inline] = args[index].split('=', 2);
+  if (!FIELDS[flag]) {
+    return 0;
+  }
+  const value = inline ?? args[index + 1];
+  if (!value || value.startsWith('-')) {
+    throw new Error(`Option ${flag} requires a resource limit`);
+  }
+  parseResourceSpec(value, flag === '--cpus');
+  options[FIELDS[flag]] = value;
+  return inline === undefined ? 2 : 1;
+}
+
+function validate(options) {
+  if (
+    !Object.values(FIELDS).some(
+      (field) => options[field] !== null && options[field] !== undefined
+    )
+  ) {
+    return;
+  }
+  if (
+    !options.resume &&
+    !(options.isolatedStack || [options.isolated]).includes('docker')
+  ) {
+    throw new Error('Resource options require Docker isolation or --resume');
+  }
+  if (options.memorySwap && !options.memory && !options.resume) {
+    throw new Error('--memory-swap requires --memory');
+  }
+  if (
+    options.onKillResumeMemory &&
+    !options.onKillResume &&
+    !options.recoveryCommand
+  ) {
+    throw new Error(
+      '--on-kill-resume-memory requires --on-kill-resume or --recovery-command'
+    );
+  }
+}
+
+function readDaemonCapacity(runner) {
+  const result = runner(getDockerCommand(), ['info', '--format', '{{json .}}']);
+  if (!result?.success) {
+    throw new Error('Cannot read resource capacity from Docker daemon');
+  }
+  const info = JSON.parse(result.stdout);
+  if (!(info.MemTotal > 0) || !(info.NCPU > 0)) {
+    throw new Error('Docker daemon did not report MemTotal/NCPU');
+  }
+  return { memory: info.MemTotal, cpus: info.NCPU };
+}
+
+function resolveResourceOptions(options, runner, random = Math.random) {
+  const specs = {};
+  for (const field of ['memory', 'memorySwap', 'cpus']) {
+    if (options[field] !== null && options[field] !== undefined) {
+      specs[field] = parseResourceSpec(options[field], field === 'cpus');
+    }
+  }
+  const capacity = Object.values(specs).some((s) => s.percent)
+    ? readDaemonCapacity(runner)
+    : null;
+  const resolved = {};
+  const limits = normalizeResourceLimits(options.resourceLimits);
+  for (const [field, spec] of Object.entries(specs)) {
+    const amount = spec.percent
+      ? (capacity[field === 'cpus' ? 'cpus' : 'memory'] *
+          (spec.min + (spec.max - spec.min) * random())) /
+        100
+      : spec.amount;
+    resolved[field] =
+      field === 'cpus' ? Math.floor(amount * 1e9) / 1e9 : Math.floor(amount);
+    if (resolved[field] < (field === 'cpus' ? 0.01 : 6 * 1024 * 1024)) {
+      throw new Error(`Resolved ${field} is below Docker's minimum`);
+    }
+  }
+  if (
+    resolved.memory !== null &&
+    resolved.memory !== undefined &&
+    (resolved.memorySwap === null || resolved.memorySwap === undefined)
+  ) {
+    resolved.memorySwap = resolved.memory;
+  }
+  const existingMemory = limits
+    .find((flag) => flag.startsWith('--memory='))
+    ?.slice(9);
+  const memory =
+    resolved.memory ??
+    (existingMemory ? parseResourceSpec(existingMemory).amount : null);
+  if (
+    resolved.memorySwap !== null &&
+    resolved.memorySwap !== undefined &&
+    (memory === null || memory === undefined || resolved.memorySwap < memory)
+  ) {
+    throw new Error('memory-swap must be at least memory');
+  }
+  for (const [field, amount] of Object.entries(resolved)) {
+    const flag = field === 'memorySwap' ? '--memory-swap' : `--${field}`;
+    for (let i = limits.length - 1; i >= 0; i--) {
+      if (
+        limits[i].startsWith(`${flag}=`) ||
+        (field === 'cpus' && /^--cpu-(quota|period)=/.test(limits[i]))
+      ) {
+        limits.splice(i, 1);
+      }
+    }
+    limits.push(`${flag}=${amount}`);
+  }
+  return {
+    resourceLimits: limits,
+    resolvedLimits: resolved,
+    resourceLimitSpecs: Object.fromEntries(
+      Object.keys(specs).map((field) => [field, options[field]])
+    ),
+  };
+}
+
+function limitsLogLine(options) {
+  const values = options.resolvedLimits || {};
+  return Object.keys(values).length
+    ? `Limits: ${Object.entries(values)
+        .map(
+          ([field, value]) =>
+            `${field}=${value}${options.resourceLimitSpecs?.[field] ? ` (${options.resourceLimitSpecs[field]} -> ${value})` : ''}`
+        )
+        .join(' ')}`
+    : null;
+}
+
+module.exports = {
+  parse,
+  validate,
+  parseResourceSpec,
+  readDaemonCapacity,
+  resolveResourceOptions,
+  limitsLogLine,
+};

package/src/lib/docker-runtime-args.js

@@ -42,6 +42,9 @@ function buildDockerRuntimeArgs(options = {}) {
   for (const envVar of options.env || []) {
     args.push('-e', envVar);
   }
+  for (const label of require('./docker-labels').dockerLabels(options)) {
+    args.push('--label', label);
+  }
   for (const volume of options.volumes || []) {
     args.push('-v', volume);
   }
@@ -69,6 +72,9 @@ function buildDockerRuntimeArgs(options = {}) {
  */
 function buildDockerRuntimeStatusLines(options = {}) {
   const lines = [];
+  if (options.labels && options.labels.length > 0) {
+    lines.push(`[Isolation] Labels: ${options.labels.join(', ')}`);
+  }
   if (options.volumes && options.volumes.length > 0) {
     lines.push(`[Isolation] Volumes: ${options.volumes.join(', ')}`);
   }
@@ -97,6 +103,12 @@ function buildDockerRuntimeStatusLines(options = {}) {
   if (limitsLine) {
     lines.push(limitsLine);
   }
+  const limitsSummary = require('./docker-resource-options').limitsLogLine(
+    options
+  );
+  if (limitsSummary) {
+    lines.push(limitsSummary);
+  }
   const onKillResume = getOnKillResume(options);
   if (onKillResume) {
     const what = options.recoveryCommand || 'the original command';
@@ -123,6 +135,8 @@ function buildDockerRuntimeMetadata(options = {}) {
       options.volumes && options.volumes.length > 0 ? options.volumes : null,
     mounts: options.mounts && options.mounts.length > 0 ? options.mounts : null,
     env: options.env && options.env.length > 0 ? options.env : null,
+    labels: options.labels && options.labels.length > 0 ? options.labels : null,
+    commandHandoff: options.detached ? true : null,
     privileged: options.privileged || null,
     network: networks[0] || null,
     networks: networks.length > 0 ? networks : null,
@@ -131,6 +145,15 @@ function buildDockerRuntimeMetadata(options = {}) {
         ? options.networkAliases
         : null,
     resourceLimits: resourceLimits.length > 0 ? resourceLimits : null,
+    resolvedLimits: options.resolvedLimits || null,
+    resourceLimitSpecs: options.resourceLimitSpecs || null,
+    onKillResumeMemory: options.onKillResumeMemory || null,
+    ...(options.cpuPenaltyConfig
+      ? {
+          cpuPenaltyConfig: options.cpuPenaltyConfig,
+          baseResourceLimits: resourceLimits,
+        }
+      : {}),
     onKillResume: getOnKillResume(options),
     recoveryCommand: options.recoveryCommand || null,
     onKillResumeDelay: getOnKillResume(options)

package/src/lib/docker-snapshot-safety.js

@@ -0,0 +1,164 @@
+/** Serialize costly snapshot resumes and check every image-store filesystem. */
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { LockManager } = require('./store-lock');
+
+const GiB = 1024 ** 3;
+const SNAPSHOT_RESERVE_BYTES = 10 * GiB;
+const SNAPSHOT_IMAGE_LABEL = 'start-command.snapshot-image';
+
+function acquireSnapshotLock(options = {}) {
+  const lockPath =
+    options.lockPath ||
+    process.env.START_SNAPSHOT_LOCK ||
+    path.join(
+      process.platform === 'win32' ? os.tmpdir() : '/tmp',
+      'start-command-docker-snapshot.lock'
+    );
+  const lock = new LockManager(lockPath);
+  // The ordinary store lock is held only for a brief write. A snapshot can
+  // legitimately take hours, so never reclaim this lock merely for its age.
+  lock.isLockStale = (data) => {
+    if (
+      !data ||
+      data.hostname !== os.hostname() ||
+      !Number.isInteger(data.pid) ||
+      data.pid <= 0
+    ) {
+      return false;
+    }
+    try {
+      process.kill(data.pid, 0);
+      return false;
+    } catch (error) {
+      return error.code === 'ESRCH';
+    }
+  };
+  if (!lock.acquire(1000)) {
+    throw new Error(
+      'A Docker snapshot resume is in progress on this host. Retry after it finishes.'
+    );
+  }
+  return lock;
+}
+
+function containerdRoot(config = '') {
+  // Only the root-level TOML key configures containerd's storage root.
+  const topLevel = config.split(/^\s*\[/m)[0];
+  return (
+    topLevel.match(/^\s*root\s*=\s*["']([^"']+)["']/m)?.[1] ||
+    '/var/lib/containerd'
+  );
+}
+
+function defaultContainerdRoot() {
+  if (process.env.START_CONTAINERD_ROOT) {
+    return process.env.START_CONTAINERD_ROOT;
+  }
+  try {
+    return containerdRoot(
+      fs.readFileSync('/etc/containerd/config.toml', 'utf8')
+    );
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return containerdRoot();
+    }
+    throw error;
+  }
+}
+
+function readFreeBytes(root) {
+  const stats = fs.statfsSync(root, { bigint: true });
+  return Number(stats.bavail * stats.bsize);
+}
+
+function preflightSnapshot(containerName, runner, options = {}) {
+  const docker = require('./docker-cleanup').getDockerCommand();
+  const sizeResult = runner(docker, [
+    'inspect',
+    '--size',
+    '--format',
+    '{{.SizeRw}}',
+    containerName,
+  ]);
+  const size = Number(String(sizeResult.stdout || '').trim());
+  if (
+    !sizeResult.success ||
+    !/^\d+$/.test(String(sizeResult.stdout || '').trim()) ||
+    !Number.isSafeInteger(size) ||
+    size < 0
+  ) {
+    throw new Error(
+      'Cannot determine the writable layer size; refusing Docker snapshot resume.'
+    );
+  }
+  const infoResult = runner(docker, ['info', '--format', '{{json .}}']);
+  let info;
+  try {
+    info = JSON.parse(infoResult.stdout);
+  } catch {
+    /* Refuse unknown storage. */
+  }
+  if (!infoResult.success || !info?.DockerRootDir) {
+    throw new Error(
+      'Cannot determine Docker data root; refusing Docker snapshot resume.'
+    );
+  }
+  const roots = [info.DockerRootDir];
+  if (
+    JSON.stringify(info.DriverStatus || []).includes(
+      'io.containerd.snapshotter'
+    )
+  ) {
+    roots.push(options.containerdRoot || defaultContainerdRoot());
+  }
+  const required = 2 * size + (options.reserveBytes ?? SNAPSHOT_RESERVE_BYTES);
+  const free = options.freeBytes || readFreeBytes;
+  for (const root of [...new Set(roots)]) {
+    let available;
+    try {
+      available = free(root);
+    } catch (error) {
+      throw new Error(
+        `Cannot check free disk at ${root}: ${error.message}; refusing Docker snapshot resume.`,
+        { cause: error }
+      );
+    }
+    if (!Number.isFinite(available) || available < required) {
+      throw new Error(
+        `Insufficient disk at ${root}: need ${(required / GiB).toFixed(2)} GiB (2 × writable layer plus reserve), available ${(available / GiB).toFixed(2)} GiB. Docker snapshot resume refused.`
+      );
+    }
+  }
+  return {
+    size,
+    required,
+    roots,
+    message: `[Resume] snapshotting ${(size / GiB).toFixed(2)} GiB; disk preflight passed (${(required / GiB).toFixed(2)} GiB required).`,
+  };
+}
+
+function snapshotImageCleanupSnippet(
+  containerName,
+  redirection = '>/dev/null 2>&1'
+) {
+  const { shellQuote } = require('./isolation-log-utils');
+  const name = shellQuote(containerName);
+  const format = shellQuote(
+    `{{index .Config.Labels "${SNAPSHOT_IMAGE_LABEL}"}}`
+  );
+  return {
+    capture: `__start_snapshot_image=$(docker inspect -f ${format} ${name} 2>/dev/null || true)`,
+    remove: `case "$__start_snapshot_image" in start-command-resume/*) docker rmi "$__start_snapshot_image" ${redirection} || true;; esac`,
+  };
+}
+
+module.exports = {
+  acquireSnapshotLock,
+  containerdRoot,
+  preflightSnapshot,
+  SNAPSHOT_RESERVE_BYTES,
+  SNAPSHOT_IMAGE_LABEL,
+  snapshotImageCleanupSnippet,
+};

package/src/lib/execution-recovery.js

@@ -39,6 +39,11 @@ const {
   readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { shellQuote } = require('./isolation-log-utils');
+const {
+  reserveLaunch,
+  rollbackLaunch,
+  failLaunchedPersistence,
+} = require('./launch-persistence');
 const {
   CGROUP_SHELL_VARS,
   formatCgroupMemoryLogLine,
@@ -299,7 +304,51 @@ function recoverKilledExecution(params = {}) {
   // Docker keeps `docker update` limits in the HostConfig across restarts;
   // they are read back so the log and `--status` show what the resumed run
   // is held to, and so a later relaunch can re-apply them.
-  const resourceLimits = readDockerResourceLimits(containerName, runner);
+  let resourceLimits =
+    readDockerResourceLimits(containerName, runner) ||
+    opts.resourceLimits ||
+    [];
+  resourceLimits = require('./resume-resources').restoreBaseCpu(
+    resourceLimits,
+    opts
+  );
+  if (opts.cpuPenaltyConfig) {
+    try {
+      resourceLimits = require('./resume-resources').clampCpuToDaemon(
+        resourceLimits,
+        runner
+      );
+    } catch (error) {
+      return giveUp(error.message);
+    }
+  }
+  const evidence = require('./exit-evidence');
+  const terminalOom =
+    (params.oomKilled === true || params.oomKilled === 'true') &&
+    evidence.recentOomDelta(params.cgroupMemory, params.finishedAt) &&
+    !evidence.fromLog(
+      record.logPath
+        ? require('./execution-attempt').readAttemptLogTail(record, 65536)
+        : null
+    ).daemonRestart;
+  let recoveryLimits = null;
+  if (opts.onKillResumeMemory && terminalOom) {
+    try {
+      recoveryLimits =
+        require('./docker-resource-options').resolveResourceOptions(
+          { memory: opts.onKillResumeMemory, resourceLimits },
+          runner,
+          params.random
+        );
+    } catch (error) {
+      return giveUp(`recovery limit failed: ${error.message}`);
+    }
+    log(
+      `Recovery memory limit: ${resourceLimits.find((s) => s.startsWith('--memory=')) || 'unlimited'} -> ${recoveryLimits.resolvedLimits.memory} (${opts.onKillResumeMemory})\n`
+    );
+    resourceLimits = recoveryLimits.resourceLimits;
+  }
+
   const limitsLine = buildResourceLimitsStatusLine(resourceLimits);
   if (limitsLine) {
     log(`${limitsLine}\n`);
@@ -308,7 +357,14 @@ function recoverKilledExecution(params = {}) {
   if (opts.recoveryCommand) {
     let copied;
     try {
-      copied = writeRecoveryMarker(containerName, attempt, runner);
+      copied = opts.commandHandoff
+        ? require('./docker-command-handoff').writeCommandHandoff(
+            containerName,
+            `export ${RECOVERY_ATTEMPT_ENV}=${attempt}; ${opts.recoveryCommand}`,
+            runner,
+            opts
+          )
+        : writeRecoveryMarker(containerName, attempt, runner);
     } catch (err) {
       copied = { success: false, error: err.message };
     }
@@ -337,18 +393,6 @@ function recoverKilledExecution(params = {}) {
       'resume-started'
     );
   }
-  const started = runner(getDockerCommand(), ['start', containerName]);
-  if (!started.success) {
-    if (nextAttempt) {
-      appendLifecycle(
-        { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
-        'launch-failed',
-        { error: failureDetail(started) }
-      );
-    }
-    return giveUp(`docker start failed: ${failureDetail(started)}`);
-  }
-
   const described = describeExitCode(params.exitCode);
   const delayed = delayRange ? `, delayMs=${delayMs}` : '';
   // The killed run's own counters: the resumed run starts a fresh cgroup.
@@ -363,7 +407,7 @@ function recoverKilledExecution(params = {}) {
       cgroupMemory: memory || undefined,
     });
     archiveAttempt(record);
-    nextAttempt.launchAcceptedAt = new Date().toISOString();
+
     record.attempt = nextAttempt;
   }
   const counted = memory
@@ -374,6 +418,18 @@ function recoverKilledExecution(params = {}) {
     : '';
   record.options = {
     ...opts,
+    ...(recoveryLimits
+      ? {
+          resolvedLimits: {
+            ...opts.resolvedLimits,
+            ...recoveryLimits.resolvedLimits,
+          },
+          resourceLimitSpecs: {
+            ...opts.resourceLimitSpecs,
+            memory: opts.onKillResumeMemory,
+          },
+        }
+      : {}),
     recoveryAttempts: attempt,
     recoveryHistory: [
       ...(Array.isArray(opts.recoveryHistory) ? opts.recoveryHistory : []),
@@ -383,6 +439,7 @@ function recoverKilledExecution(params = {}) {
     ...(delayRange ? { lastRecoveryDelayMs: delayMs } : {}),
     ...(resourceLimits && resourceLimits.length > 0 ? { resourceLimits } : {}),
   };
+  delete record.options.exitEvidence;
   record.status = 'executing';
   record.exitCode = null;
   record.endTime = null;
@@ -395,11 +452,56 @@ function recoverKilledExecution(params = {}) {
   record.observedAt = undefined;
   record.staleDetectedAt = undefined;
   record.containerStartedAt = undefined;
+  const previousData = globalThis.structuredClone(store.get(executionId));
+  const previous = record.constructor.fromObject
+    ? record.constructor.fromObject(previousData)
+    : previousData;
+  try {
+    reserveLaunch(store, record, previous);
+  } catch (error) {
+    return giveUp(`launch reservation failed: ${error.message}`);
+  }
+  if (recoveryLimits || opts.cpuPenaltyConfig) {
+    const updated = runner(getDockerCommand(), [
+      'update',
+      ...resourceLimits.filter((s) =>
+        /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
+      ),
+      containerName,
+    ]);
+    if (!updated.success) {
+      rollbackLaunch(store, previous);
+      return giveUp(`docker update failed: ${failureDetail(updated)}`);
+    }
+  }
+  const started = runner(getDockerCommand(), ['start', containerName]);
+  if (!started.success) {
+    if (nextAttempt) {
+      appendLifecycle(
+        { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
+        'launch-failed',
+        { error: failureDetail(started) }
+      );
+    }
+    rollbackLaunch(store, previous);
+    return giveUp(`docker start failed: ${failureDetail(started)}`);
+  }
+
+  if (opts.cpuPenaltyConfig) {
+    record.options.resolvedLimits = {
+      ...record.options.resolvedLimits,
+      cpus: require('./docker-resource-limits').cpuCount(resourceLimits),
+    };
+  }
+  record.options.launchPending = false;
+  if (nextAttempt) {
+    nextAttempt.launchAcceptedAt = new Date().toISOString();
+  }
+  let persistenceError = null;
   try {
     store.save(record);
-  } catch {
-    // The container is already running again; the new watcher still follows
-    // it and finalizes the record when it ends.
+  } catch (error) {
+    persistenceError = error;
   }
 
   const startWatcher =
@@ -407,18 +509,29 @@ function recoverKilledExecution(params = {}) {
   if (nextAttempt) {
     appendLifecycle(record, 'launch-accepted');
   }
-  startWatcher(
-    containerName,
-    getDockerContainerCleanupPolicy(opts),
-    record.logPath || null,
-    record.uuid,
-    {
-      since,
-      recoverOnKill: true,
-      ...(nextAttempt ? { attemptNumber: nextAttempt.number } : {}),
-    }
-  );
-  if (nextAttempt) {
+  try {
+    startWatcher(
+      containerName,
+      getDockerContainerCleanupPolicy(opts),
+      record.logPath || null,
+      record.uuid,
+      {
+        since,
+        recoverOnKill: true,
+        ...(nextAttempt ? { attemptNumber: nextAttempt.number } : {}),
+      }
+    );
+  } catch (error) {
+    const warning = failLaunchedPersistence(
+      store,
+      record,
+      runner,
+      persistenceError || error
+    );
+    log(`[Recovery] ${warning}\n`);
+    return { recovered: false, reason: warning, attempt };
+  }
+  if (nextAttempt && !persistenceError) {
     const current = patchAttempt(store, record, {
       watcherAttachedAt: new Date().toISOString(),
     });
@@ -426,6 +539,16 @@ function recoverKilledExecution(params = {}) {
       appendLifecycle(current, 'watcher-attached');
     }
   }
+  if (persistenceError) {
+    const warning = failLaunchedPersistence(
+      store,
+      record,
+      runner,
+      persistenceError
+    );
+    log(`[Recovery] ${warning}\n`);
+    return { recovered: false, reason: warning, attempt };
+  }
   return {
     recovered: true,
     reason: 'resumed',

package/src/lib/execution-resume-all.js

@@ -125,6 +125,17 @@ function resumeAllExecutions(store, deps = {}) {
   for (const record of records) {
     const probe = probeFn(record);
     const sessionName = probe.sessionName;
+    if (require('./launch-owner').hasActiveLaunch(record)) {
+      executions.push(
+        describeRecord(
+          record,
+          probe,
+          ResumeAllAction.RUNNING,
+          'Launch is reserved; no watcher or duplicate launch was started.'
+        )
+      );
+      continue;
+    }
 
     if (probe.alive && probe.backend === 'docker') {
       startWatcher(