start-command 0.35.4 → 0.36.0

package/src/lib/execution-store.js

@@ -24,6 +24,7 @@ const path = require('path');
 const os = require('os');
 const { execFileSync, spawnSync } = require('child_process');
 const crypto = require('crypto');
+const { atomicWrite } = require('./atomic-write');
 
 // Synchronous wrapper using Bun's native ESM support
 // This works because Bun handles ESM/CJS interop
@@ -51,8 +52,7 @@ const DEFAULT_APP_FOLDER = path.join(os.homedir(), '.start-command');
 const LINO_DB_FILE = 'executions.lino';
 const LINKS_DB_FILE = 'executions.links';
 const LOCK_FILE = 'executions.lock';
-const LOCK_TIMEOUT_MS = 30000; // 30 second timeout for lock acquisition
-const LOCK_STALE_MS = 60000; // Consider lock stale after 60 seconds
+const { LockManager } = require('./store-lock');
 
 /**
  * Execution status enumeration
@@ -128,6 +128,7 @@ class ExecutionRecord {
     this.shell = options.shell || process.env.SHELL || '/bin/sh';
     this.platform = options.platform || process.platform;
     this.options = options.options || {};
+    this.cpuPenalty = options.cpuPenalty;
   }
 
   /**
@@ -154,6 +155,9 @@ class ExecutionRecord {
       startTime: this.startTime,
       endTime: this.endTime,
     };
+    if (this.cpuPenalty) {
+      obj.cpuPenalty = this.cpuPenalty;
+    }
     if (this.attempt) {
       obj.attempt = this.attempt;
     }
@@ -210,123 +214,6 @@ class ExecutionRecord {
   }
 }
 
-/**
- * File-based lock manager
- */
-class LockManager {
-  constructor(lockFilePath) {
-    this.lockFilePath = lockFilePath;
-    this.lockAcquired = false;
-  }
-
-  /**
-   * Acquire an exclusive lock
-   * @param {number} timeout - Maximum time to wait for lock in ms
-   * @returns {boolean} True if lock acquired
-   */
-  acquire(timeout = LOCK_TIMEOUT_MS) {
-    const startTime = Date.now();
-
-    while (Date.now() - startTime < timeout) {
-      try {
-        // Check if existing lock is stale
-        if (fs.existsSync(this.lockFilePath)) {
-          const lockData = this.readLockFile();
-          if (lockData && this.isLockStale(lockData)) {
-            // Remove stale lock
-            fs.unlinkSync(this.lockFilePath);
-          }
-        }
-
-        // Try to create lock file exclusively
-        const lockData = {
-          pid: process.pid,
-          timestamp: Date.now(),
-          hostname: os.hostname(),
-        };
-
-        fs.writeFileSync(this.lockFilePath, JSON.stringify(lockData), {
-          flag: 'wx', // Fail if file exists
-        });
-
-        this.lockAcquired = true;
-        return true;
-      } catch (err) {
-        if (err.code === 'EEXIST') {
-          // Lock file exists, wait and retry
-          this.sleep(100);
-          continue;
-        }
-        throw err;
-      }
-    }
-
-    return false;
-  }
-
-  /**
-   * Release the lock
-   */
-  release() {
-    if (this.lockAcquired) {
-      try {
-        fs.unlinkSync(this.lockFilePath);
-      } catch {
-        // Ignore errors during release
-      }
-      this.lockAcquired = false;
-    }
-  }
-
-  /**
-   * Read lock file data
-   */
-  readLockFile() {
-    try {
-      const content = fs.readFileSync(this.lockFilePath, 'utf8');
-      return JSON.parse(content);
-    } catch {
-      return null;
-    }
-  }
-
-  /**
-   * Check if lock is stale
-   */
-  isLockStale(lockData) {
-    if (!lockData || !lockData.timestamp) {
-      return true;
-    }
-
-    // Check if lock is too old
-    if (Date.now() - lockData.timestamp > LOCK_STALE_MS) {
-      return true;
-    }
-
-    // Check if the process that holds the lock is still running
-    if (lockData.pid && lockData.hostname === os.hostname()) {
-      try {
-        process.kill(lockData.pid, 0); // Signal 0 just checks if process exists
-        return false; // Process exists, lock is valid
-      } catch {
-        return true; // Process doesn't exist, lock is stale
-      }
-    }
-
-    return false;
-  }
-
-  /**
-   * Simple sleep function
-   */
-  sleep(ms) {
-    const end = Date.now() + ms;
-    while (Date.now() < end) {
-      // Busy wait (not ideal but works for short durations)
-    }
-  }
-}
-
 /**
  * Check if clink is installed
  * @returns {boolean}
@@ -412,7 +299,7 @@ class ExecutionStore {
   writeLinoRecords(records) {
     const data = records.map((r) => r.toObject());
     const content = encodeSync(data);
-    fs.writeFileSync(this.linoDbPath, content, 'utf8');
+    atomicWrite(this.linoDbPath, content);
     this.log(`Wrote ${records.length} records to lino file`);
   }
 
@@ -529,7 +416,7 @@ class ExecutionStore {
    * @param {ExecutionRecord} record
    * @returns {boolean}
    */
-  save(record) {
+  save(record, previous = null) {
     const lock = new LockManager(this.lockFilePath);
 
     if (!lock.acquire()) {
@@ -542,6 +429,16 @@ class ExecutionStore {
 
       // Find existing record index
       const existingIndex = records.findIndex((r) => r.uuid === record.uuid);
+      if (previous) {
+        const current = records[existingIndex];
+        if (
+          !current ||
+          require('./launch-owner').hasActiveLaunch(current) ||
+          (current.attempt?.number || 0) !== (previous.attempt?.number || 0)
+        ) {
+          throw new Error('A different resume already reserved this execution');
+        }
+      }
 
       if (existingIndex >= 0) {
         // Update existing record
@@ -565,6 +462,10 @@ class ExecutionStore {
     }
   }
 
+  reserveLaunch(record, previous) {
+    return this.save(record, previous);
+  }
+
   // Merge watcher metadata under the database lock without replacing terminal state.
   patchAttempt(uuid, number, fields) {
     const lock = new LockManager(this.lockFilePath);
@@ -689,6 +590,9 @@ class ExecutionStore {
     const staleRecords = [];
 
     for (const record of executingRecords) {
+      if (require('./launch-owner').hasActiveLaunch(record)) {
+        continue;
+      }
       let isStale = false;
       let reason = '';
 

package/src/lib/exit-evidence.js

@@ -0,0 +1,123 @@
+/** Attempt-scoped evidence, independent of Docker's sticky OOMKilled flag. */
+const { spawnSync } = require('child_process');
+const { shellQuote } = require('./isolation-log-utils');
+const MAIN_OOM = 'Exit evidence: main-oom (recent cgroup oom_kill delta)';
+const DAEMON_RESTART =
+  'Exit evidence: docker-daemon-restart (Docker service journal)';
+
+function recentOomDelta(sample, finishedAt) {
+  const fields = String(sample || '')
+    .trim()
+    .split(/\s+/);
+  const changed = Number(fields[4]),
+    observed = Number(fields[5]);
+  const finished = Date.parse(finishedAt) / 1000;
+  return (
+    changed > 0 &&
+    Number.isFinite(finished) &&
+    changed <= observed &&
+    Math.abs(finished - changed) <= 3 &&
+    Math.abs(finished - observed) <= 3
+  );
+}
+
+function daemonRestartEvidence(journal, containerId) {
+  const lines = String(journal || '').split('\n');
+  const exited = lines.some((line) =>
+    /Main process exited|Stopping Docker|Starting Docker|Daemon shutdown complete/.test(
+      line
+    )
+  );
+  const forced =
+    containerId &&
+    lines.some(
+      (line) =>
+        line.includes(containerId) &&
+        /failed to exit|using the force|force.kill/i.test(line)
+    );
+  return Boolean(exited && forced);
+}
+
+function collectExitEvidence(
+  { containerName, sample, finishedAt, exitCode, oomKilled },
+  runner
+) {
+  const run =
+    runner ||
+    ((cmd, args) => {
+      const r = spawnSync(cmd, args, { encoding: 'utf8' });
+      return { success: r.status === 0, stdout: r.stdout || '' };
+    });
+  // A local journal cannot diagnose a remote daemon. Require a local socket.
+  const local =
+    !process.env.DOCKER_HOST || process.env.DOCKER_HOST.startsWith('unix://');
+  if (
+    local &&
+    Number(exitCode) === 137 &&
+    Number.isFinite(Date.parse(finishedAt))
+  ) {
+    const docker = require('./docker-cleanup').getDockerCommand();
+    const id = run(docker, ['inspect', '-f', '{{.Id}}', containerName]);
+    if (id.success && /^[a-f0-9]{64}$/.test(id.stdout.trim())) {
+      const end = Date.parse(finishedAt);
+      const journal = run('journalctl', [
+        '-u',
+        'docker.service',
+        '--since',
+        new Date(end - 60000).toISOString(),
+        '--until',
+        new Date(end + 1000).toISOString(),
+        '--no-pager',
+        '-o',
+        'cat',
+      ]);
+      if (
+        journal.success &&
+        daemonRestartEvidence(journal.stdout, id.stdout.trim())
+      ) {
+        return DAEMON_RESTART;
+      }
+    }
+  }
+  if (
+    (oomKilled === true || oomKilled === 'true') &&
+    (Number(exitCode) === 137 || Number(exitCode) < 0) &&
+    recentOomDelta(sample, finishedAt)
+  ) {
+    return MAIN_OOM;
+  }
+  return 'Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)';
+}
+
+function fromLog(log) {
+  const lines = String(log || '').split('\n');
+  const last = lines.filter((line) => line.startsWith('Exit evidence:')).at(-1);
+  return { mainOom: last === MAIN_OOM, daemonRestart: last === DAEMON_RESTART };
+}
+
+function buildExitEvidenceSnippet(containerName, quotedLogPath) {
+  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} 2>/dev/null`;
+}
+
+module.exports = {
+  MAIN_OOM,
+  DAEMON_RESTART,
+  recentOomDelta,
+  daemonRestartEvidence,
+  collectExitEvidence,
+  fromLog,
+  buildExitEvidenceSnippet,
+};
+if (require.main === module) {
+  const [containerName, sample, finishedAt, exitCode, oomKilled] =
+    process.argv.slice(2);
+  console.log(
+    collectExitEvidence({
+      containerName,
+      sample,
+      finishedAt,
+      exitCode,
+      oomKilled,
+    })
+  );
+}

package/src/lib/exit-reason.js

@@ -181,8 +181,16 @@ function resolveMemoryExhaustion(input) {
     return null;
   }
 
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (evidence.daemonRestart) {
+    return null;
+  }
   const marker = detectMemoryMarker(input.logTail);
-  if (marker) {
+  if (
+    marker &&
+    !(marker.reason.includes('kernel-oom-killer') && !evidence.mainOom)
+  ) {
     return { memoryExhausted: true, memoryExhaustedReason: marker.line };
   }
   if (input.oomKilled === true && isOomKillOfCommand(input)) {
@@ -230,9 +238,9 @@ function isOomKillOfCommand(input) {
   if (!input) {
     return false;
   }
-  const observed =
-    input.oomKilled === true || cgroupOomKills(input.cgroupMemory) > 0;
-  if (!observed) {
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (!evidence.mainOom || evidence.daemonRestart) {
     return false;
   }
   const { exitCode } = input;
@@ -318,8 +326,16 @@ function resolveExitReason(input) {
     return null;
   }
 
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (evidence.daemonRestart) {
+    return 'killed (docker daemon restart)';
+  }
   const fromLog = detectExitReason(input.logTail);
-  if (fromLog) {
+  if (
+    fromLog &&
+    !(fromLog.includes('kernel-oom-killer') && !evidence.mainOom)
+  ) {
     return fromLog;
   }
 
@@ -328,7 +344,9 @@ function resolveExitReason(input) {
   }
 
   const signalName = signalNameForExitCode(input.exitCode);
-  return signalName ? `signal (${signalName})` : null;
+  return signalName
+    ? `signal (${signalName}${signalName === 'SIGKILL' ? '; cause unknown' : ''})`
+    : null;
 }
 
 module.exports = {

package/src/lib/isolation.js

@@ -547,6 +547,23 @@ function runInDocker(command, options = {}) {
     });
   }
 
+  if (!options.resolvedLimits) {
+    try {
+      Object.assign(
+        options,
+        require('./docker-resource-options').resolveResourceOptions(
+          options,
+          require('./execution-control').runCommand
+        )
+      );
+    } catch (error) {
+      return Promise.resolve({
+        success: false,
+        containerName: null,
+        message: error.message,
+      });
+    }
+  }
   const containerName = options.session || generateSessionName('docker');
   const containerExistedBeforeLaunch =
     readDockerContainerStatus(containerName) !== null;
@@ -665,7 +682,7 @@ function runInDocker(command, options = {}) {
           cleanupPolicy,
           options.logPath,
           options.executionId || null,
-          { recoverOnKill: Boolean(options.onKillResume) }
+          { recoverOnKill: Boolean(options.onKillResume), cpuOptions: options }
         );
       }
 
@@ -745,9 +762,14 @@ function runInDocker(command, options = {}) {
         const launchArgs = needsNetworkSetup
           ? ['start', '-a', '-i', containerName]
           : dockerArgs;
+        const diagnostics =
+          require('./attached-diagnostics').startAttachedDiagnostics(
+            containerName,
+            options
+          );
         const child = spawnAttachedDocker(launchArgs, options.logPath);
 
-        child.on('exit', (code) => {
+        child.on('exit', async (code) => {
           const durationMs = Date.now() - startTime;
           const exitCode = code ?? 1;
           let message = `Docker container "${containerName}" exited with code ${exitCode}`;
@@ -762,6 +784,7 @@ function runInDocker(command, options = {}) {
           }
 
           const dockerState = readDockerContainerState(containerName);
+          const diagnosticFacts = await diagnostics.finish(dockerState);
           const oomKilled = dockerState ? dockerState.oomKilled : null;
           const postMortem = (removed) =>
             recordAttachedDockerPostMortem({
@@ -806,12 +829,14 @@ function runInDocker(command, options = {}) {
           resolve({
             success: exitCode === 0,
             containerName,
+            diagnostics: diagnosticFacts,
             message,
             exitCode,
           });
         });
 
-        child.on('error', (err) => {
+        child.on('error', async (err) => {
+          await diagnostics.finish(null);
           resolve({
             success: false,
             containerName,

package/src/lib/launch-owner.js

@@ -0,0 +1,25 @@
+/** A dead launcher leaves a usable reservation, rather than a permanent gate. */
+const os = require('os');
+
+function markLaunch(record) {
+  record.options.launchPending = true;
+  record.options.launchOwner = { pid: process.pid, hostname: os.hostname() };
+}
+
+function hasActiveLaunch(record) {
+  if (!record?.options?.launchPending) {
+    return false;
+  }
+  const owner = record.options.launchOwner;
+  if (!owner || owner.hostname !== os.hostname()) {
+    return true;
+  }
+  try {
+    process.kill(owner.pid, 0);
+    return true;
+  } catch (error) {
+    return error.code !== 'ESRCH';
+  }
+}
+
+module.exports = { markLaunch, hasActiveLaunch };

package/src/lib/launch-persistence.js

@@ -0,0 +1,54 @@
+/** Reserve before launch, and identify/stop a launch whose final save failed. */
+const { getDockerCommand } = require('./docker-cleanup');
+
+function reserveLaunch(store, record, previous) {
+  require('./launch-owner').markLaunch(record);
+  if (typeof store.reserveLaunch === 'function') {
+    store.reserveLaunch(record, previous);
+  } else {
+    store.save(record);
+  }
+}
+
+function rollbackLaunch(store, previous) {
+  try {
+    store.save(previous);
+  } catch {
+    /* The durable reservation still names the session. */
+  }
+}
+
+function failLaunchedPersistence(store, record, runner, error) {
+  const name = record.options.sessionName;
+  let stopped;
+  try {
+    stopped =
+      record.options.isolated === 'docker'
+        ? runner(getDockerCommand(), ['stop', name])
+        : null;
+  } catch {
+    stopped = null;
+  }
+  const warning = {
+    code: 'LAUNCH_PERSISTENCE_FAILED',
+    uuid: record.uuid,
+    containerName: name,
+    running: !stopped?.success,
+    error: error.message,
+  };
+  if (stopped?.success) {
+    record.status = 'executed';
+    record.exitCode = -1;
+    record.endTime = new Date().toISOString();
+    record.endTimeSource = 'observed-at';
+    record.options.launchPending = false;
+    try {
+      store.save(record);
+    } catch {
+      /* Completion watcher remains attached. */
+    }
+  }
+  return JSON.stringify(warning);
+}
+
+module.exports = { reserveLaunch, rollbackLaunch, failLaunchedPersistence };

package/src/lib/query-commands.js

@@ -55,9 +55,19 @@ function handleAttachQuery(store, identifier, readOnly) {
   return reportResult(attachExecution(store, identifier, { readOnly }));
 }
 
-async function handleResumeQuery(store, identifier, command, outputFormat) {
+async function handleResumeQuery(
+  store,
+  identifier,
+  command,
+  outputFormat,
+  resourceOptions = {}
+) {
   return reportResult(
-    await resumeExecution(store, identifier, { command, outputFormat })
+    await resumeExecution(store, identifier, {
+      command,
+      outputFormat,
+      resourceOptions,
+    })
   );
 }
 
@@ -174,7 +184,8 @@ async function dispatchQueryCommand(options, context = {}) {
       store,
       options.resume,
       context.command || null,
-      outputFormat
+      outputFormat,
+      options
     );
   }
   if (options.resumeAll) {

package/src/lib/resume-resources.js

@@ -0,0 +1,88 @@
+const {
+  normalizeResourceLimits,
+  cpuCount,
+  withCpuLimit,
+  readDockerResourceLimits,
+} = require('./docker-resource-limits');
+const { resolveResourceOptions } = require('./docker-resource-options');
+
+// Temporary penalty quotas must never become a resumed execution's base quota.
+function restoreBaseCpu(limits, options) {
+  if (!options.cpuPenaltyConfig) {
+    return limits;
+  }
+  const base = normalizeResourceLimits(
+    options.baseResourceLimits || options.resourceLimits
+  );
+  return [
+    ...limits.filter((s) => !/^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
+    ...base.filter((s) => /^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
+  ];
+}
+
+function prepareResumeResources(record, probe, overrides, runner, random) {
+  const options = record.options || {};
+  if (options.isolated !== 'docker') {
+    if (
+      ['memory', 'memorySwap', 'cpus'].some(
+        (k) => overrides[k] !== null && overrides[k] !== undefined
+      )
+    ) {
+      throw new Error('Resource overrides require a Docker execution');
+    }
+    return null;
+  }
+  const live =
+    probe.state === 'stopped'
+      ? readDockerResourceLimits(options.sessionName, runner)
+      : null;
+  const base = restoreBaseCpu(
+    normalizeResourceLimits(live || options.resourceLimits),
+    options
+  );
+  const oldMemory = base.find((s) => s.startsWith('--memory='))?.slice(9);
+  const result = resolveResourceOptions(
+    { ...overrides, resourceLimits: base },
+    runner,
+    random
+  );
+  if (overrides.cpus && probe.state === 'stopped') {
+    // Docker cannot switch an existing legacy quota to NanoCPUs in place.
+    const cpu = withCpuLimit(base, result.resolvedLimits.cpus);
+    result.resourceLimits = [
+      ...result.resourceLimits.filter(
+        (s) => !/^--(?:cpus|cpu-quota|cpu-period)=/.test(s)
+      ),
+      ...cpu.filter((s) => /^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
+    ];
+  }
+  result.oldMemory = oldMemory
+    ? require('./docker-resource-options').parseResourceSpec(oldMemory).amount
+    : 'unlimited';
+  result.resolvedLimits = {
+    ...options.resolvedLimits,
+    ...result.resolvedLimits,
+  };
+  result.resourceLimitSpecs = {
+    ...options.resourceLimitSpecs,
+    ...result.resourceLimitSpecs,
+  };
+  result.update =
+    probe.state === 'stopped' &&
+    (['memory', 'memorySwap', 'cpus'].some(
+      (k) => overrides[k] !== null && overrides[k] !== undefined
+    ) ||
+      Boolean(options.cpuPenaltyConfig));
+  if (options.cpuPenaltyConfig && !overrides.cpus) {
+    result.resourceLimits = clampCpuToDaemon(result.resourceLimits, runner);
+    result.resolvedLimits.cpus = cpuCount(result.resourceLimits);
+  }
+  return result;
+}
+
+function clampCpuToDaemon(limits, runner) {
+  const { readDaemonCapacity } = require('./docker-resource-options');
+  const capacity = readDaemonCapacity(runner).cpus;
+  return withCpuLimit(limits, Math.min(cpuCount(limits) || capacity, capacity));
+}
+module.exports = { prepareResumeResources, restoreBaseCpu, clampCpuToDaemon };