start-command 0.35.4 → 0.37.0

package/src/lib/execution-store.js

@@ -24,6 +24,7 @@ const path = require('path');
 const os = require('os');
 const { execFileSync, spawnSync } = require('child_process');
 const crypto = require('crypto');
+const { atomicWrite } = require('./atomic-write');
 
 // Synchronous wrapper using Bun's native ESM support
 // This works because Bun handles ESM/CJS interop
@@ -51,8 +52,7 @@ const DEFAULT_APP_FOLDER = path.join(os.homedir(), '.start-command');
 const LINO_DB_FILE = 'executions.lino';
 const LINKS_DB_FILE = 'executions.links';
 const LOCK_FILE = 'executions.lock';
-const LOCK_TIMEOUT_MS = 30000; // 30 second timeout for lock acquisition
-const LOCK_STALE_MS = 60000; // Consider lock stale after 60 seconds
+const { LockManager } = require('./store-lock');
 
 /**
  * Execution status enumeration
@@ -128,6 +128,7 @@ class ExecutionRecord {
     this.shell = options.shell || process.env.SHELL || '/bin/sh';
     this.platform = options.platform || process.platform;
     this.options = options.options || {};
+    this.cpuPenalty = options.cpuPenalty;
   }
 
   /**
@@ -154,6 +155,9 @@ class ExecutionRecord {
       startTime: this.startTime,
       endTime: this.endTime,
     };
+    if (this.cpuPenalty) {
+      obj.cpuPenalty = this.cpuPenalty;
+    }
     if (this.attempt) {
       obj.attempt = this.attempt;
     }
@@ -210,123 +214,6 @@ class ExecutionRecord {
   }
 }
 
-/**
- * File-based lock manager
- */
-class LockManager {
-  constructor(lockFilePath) {
-    this.lockFilePath = lockFilePath;
-    this.lockAcquired = false;
-  }
-
-  /**
-   * Acquire an exclusive lock
-   * @param {number} timeout - Maximum time to wait for lock in ms
-   * @returns {boolean} True if lock acquired
-   */
-  acquire(timeout = LOCK_TIMEOUT_MS) {
-    const startTime = Date.now();
-
-    while (Date.now() - startTime < timeout) {
-      try {
-        // Check if existing lock is stale
-        if (fs.existsSync(this.lockFilePath)) {
-          const lockData = this.readLockFile();
-          if (lockData && this.isLockStale(lockData)) {
-            // Remove stale lock
-            fs.unlinkSync(this.lockFilePath);
-          }
-        }
-
-        // Try to create lock file exclusively
-        const lockData = {
-          pid: process.pid,
-          timestamp: Date.now(),
-          hostname: os.hostname(),
-        };
-
-        fs.writeFileSync(this.lockFilePath, JSON.stringify(lockData), {
-          flag: 'wx', // Fail if file exists
-        });
-
-        this.lockAcquired = true;
-        return true;
-      } catch (err) {
-        if (err.code === 'EEXIST') {
-          // Lock file exists, wait and retry
-          this.sleep(100);
-          continue;
-        }
-        throw err;
-      }
-    }
-
-    return false;
-  }
-
-  /**
-   * Release the lock
-   */
-  release() {
-    if (this.lockAcquired) {
-      try {
-        fs.unlinkSync(this.lockFilePath);
-      } catch {
-        // Ignore errors during release
-      }
-      this.lockAcquired = false;
-    }
-  }
-
-  /**
-   * Read lock file data
-   */
-  readLockFile() {
-    try {
-      const content = fs.readFileSync(this.lockFilePath, 'utf8');
-      return JSON.parse(content);
-    } catch {
-      return null;
-    }
-  }
-
-  /**
-   * Check if lock is stale
-   */
-  isLockStale(lockData) {
-    if (!lockData || !lockData.timestamp) {
-      return true;
-    }
-
-    // Check if lock is too old
-    if (Date.now() - lockData.timestamp > LOCK_STALE_MS) {
-      return true;
-    }
-
-    // Check if the process that holds the lock is still running
-    if (lockData.pid && lockData.hostname === os.hostname()) {
-      try {
-        process.kill(lockData.pid, 0); // Signal 0 just checks if process exists
-        return false; // Process exists, lock is valid
-      } catch {
-        return true; // Process doesn't exist, lock is stale
-      }
-    }
-
-    return false;
-  }
-
-  /**
-   * Simple sleep function
-   */
-  sleep(ms) {
-    const end = Date.now() + ms;
-    while (Date.now() < end) {
-      // Busy wait (not ideal but works for short durations)
-    }
-  }
-}
-
 /**
  * Check if clink is installed
  * @returns {boolean}
@@ -412,7 +299,7 @@ class ExecutionStore {
   writeLinoRecords(records) {
     const data = records.map((r) => r.toObject());
     const content = encodeSync(data);
-    fs.writeFileSync(this.linoDbPath, content, 'utf8');
+    atomicWrite(this.linoDbPath, content);
     this.log(`Wrote ${records.length} records to lino file`);
   }
 
@@ -529,7 +416,7 @@ class ExecutionStore {
    * @param {ExecutionRecord} record
    * @returns {boolean}
    */
-  save(record) {
+  save(record, previous = null) {
     const lock = new LockManager(this.lockFilePath);
 
     if (!lock.acquire()) {
@@ -542,6 +429,16 @@ class ExecutionStore {
 
       // Find existing record index
       const existingIndex = records.findIndex((r) => r.uuid === record.uuid);
+      if (previous) {
+        const current = records[existingIndex];
+        if (
+          !current ||
+          require('./launch-owner').hasActiveLaunch(current) ||
+          (current.attempt?.number || 0) !== (previous.attempt?.number || 0)
+        ) {
+          throw new Error('A different resume already reserved this execution');
+        }
+      }
 
       if (existingIndex >= 0) {
         // Update existing record
@@ -565,6 +462,10 @@ class ExecutionStore {
     }
   }
 
+  reserveLaunch(record, previous) {
+    return this.save(record, previous);
+  }
+
   // Merge watcher metadata under the database lock without replacing terminal state.
   patchAttempt(uuid, number, fields) {
     const lock = new LockManager(this.lockFilePath);
@@ -689,6 +590,9 @@ class ExecutionStore {
     const staleRecords = [];
 
     for (const record of executingRecords) {
+      if (require('./launch-owner').hasActiveLaunch(record)) {
+        continue;
+      }
       let isStale = false;
       let reason = '';
 

package/src/lib/exit-evidence.js

@@ -0,0 +1,123 @@
+/** Attempt-scoped evidence, independent of Docker's sticky OOMKilled flag. */
+const { spawnSync } = require('child_process');
+const { shellQuote } = require('./isolation-log-utils');
+const MAIN_OOM = 'Exit evidence: main-oom (recent cgroup oom_kill delta)';
+const DAEMON_RESTART =
+  'Exit evidence: docker-daemon-restart (Docker service journal)';
+
+function recentOomDelta(sample, finishedAt) {
+  const fields = String(sample || '')
+    .trim()
+    .split(/\s+/);
+  const changed = Number(fields[4]),
+    observed = Number(fields[5]);
+  const finished = Date.parse(finishedAt) / 1000;
+  return (
+    changed > 0 &&
+    Number.isFinite(finished) &&
+    changed <= observed &&
+    Math.abs(finished - changed) <= 3 &&
+    Math.abs(finished - observed) <= 3
+  );
+}
+
+function daemonRestartEvidence(journal, containerId) {
+  const lines = String(journal || '').split('\n');
+  const exited = lines.some((line) =>
+    /Main process exited|Stopping Docker|Starting Docker|Daemon shutdown complete/.test(
+      line
+    )
+  );
+  const forced =
+    containerId &&
+    lines.some(
+      (line) =>
+        line.includes(containerId) &&
+        /failed to exit|using the force|force.kill/i.test(line)
+    );
+  return Boolean(exited && forced);
+}
+
+function collectExitEvidence(
+  { containerName, sample, finishedAt, exitCode, oomKilled },
+  runner
+) {
+  const run =
+    runner ||
+    ((cmd, args) => {
+      const r = spawnSync(cmd, args, { encoding: 'utf8' });
+      return { success: r.status === 0, stdout: r.stdout || '' };
+    });
+  // A local journal cannot diagnose a remote daemon. Require a local socket.
+  const local =
+    !process.env.DOCKER_HOST || process.env.DOCKER_HOST.startsWith('unix://');
+  if (
+    local &&
+    Number(exitCode) === 137 &&
+    Number.isFinite(Date.parse(finishedAt))
+  ) {
+    const docker = require('./docker-cleanup').getDockerCommand();
+    const id = run(docker, ['inspect', '-f', '{{.Id}}', containerName]);
+    if (id.success && /^[a-f0-9]{64}$/.test(id.stdout.trim())) {
+      const end = Date.parse(finishedAt);
+      const journal = run('journalctl', [
+        '-u',
+        'docker.service',
+        '--since',
+        new Date(end - 60000).toISOString(),
+        '--until',
+        new Date(end + 1000).toISOString(),
+        '--no-pager',
+        '-o',
+        'cat',
+      ]);
+      if (
+        journal.success &&
+        daemonRestartEvidence(journal.stdout, id.stdout.trim())
+      ) {
+        return DAEMON_RESTART;
+      }
+    }
+  }
+  if (
+    (oomKilled === true || oomKilled === 'true') &&
+    (Number(exitCode) === 137 || Number(exitCode) < 0) &&
+    recentOomDelta(sample, finishedAt)
+  ) {
+    return MAIN_OOM;
+  }
+  return 'Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)';
+}
+
+function fromLog(log) {
+  const lines = String(log || '').split('\n');
+  const last = lines.filter((line) => line.startsWith('Exit evidence:')).at(-1);
+  return { mainOom: last === MAIN_OOM, daemonRestart: last === DAEMON_RESTART };
+}
+
+function buildExitEvidenceSnippet(containerName, quotedLogPath) {
+  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} 2>/dev/null`;
+}
+
+module.exports = {
+  MAIN_OOM,
+  DAEMON_RESTART,
+  recentOomDelta,
+  daemonRestartEvidence,
+  collectExitEvidence,
+  fromLog,
+  buildExitEvidenceSnippet,
+};
+if (require.main === module) {
+  const [containerName, sample, finishedAt, exitCode, oomKilled] =
+    process.argv.slice(2);
+  console.log(
+    collectExitEvidence({
+      containerName,
+      sample,
+      finishedAt,
+      exitCode,
+      oomKilled,
+    })
+  );
+}

package/src/lib/exit-reason.js

@@ -181,8 +181,16 @@ function resolveMemoryExhaustion(input) {
     return null;
   }
 
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (evidence.daemonRestart) {
+    return null;
+  }
   const marker = detectMemoryMarker(input.logTail);
-  if (marker) {
+  if (
+    marker &&
+    !(marker.reason.includes('kernel-oom-killer') && !evidence.mainOom)
+  ) {
     return { memoryExhausted: true, memoryExhaustedReason: marker.line };
   }
   if (input.oomKilled === true && isOomKillOfCommand(input)) {
@@ -230,9 +238,9 @@ function isOomKillOfCommand(input) {
   if (!input) {
     return false;
   }
-  const observed =
-    input.oomKilled === true || cgroupOomKills(input.cgroupMemory) > 0;
-  if (!observed) {
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (!evidence.mainOom || evidence.daemonRestart) {
     return false;
   }
   const { exitCode } = input;
@@ -318,8 +326,16 @@ function resolveExitReason(input) {
     return null;
   }
 
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (evidence.daemonRestart) {
+    return 'killed (docker daemon restart)';
+  }
   const fromLog = detectExitReason(input.logTail);
-  if (fromLog) {
+  if (
+    fromLog &&
+    !(fromLog.includes('kernel-oom-killer') && !evidence.mainOom)
+  ) {
     return fromLog;
   }
 
@@ -328,7 +344,9 @@ function resolveExitReason(input) {
   }
 
   const signalName = signalNameForExitCode(input.exitCode);
-  return signalName ? `signal (${signalName})` : null;
+  return signalName
+    ? `signal (${signalName}${signalName === 'SIGKILL' ? '; cause unknown' : ''})`
+    : null;
 }
 
 module.exports = {

package/src/lib/failure-handler.js

@@ -9,6 +9,7 @@ const os = require('os');
 const fs = require('fs');
 const path = require('path');
 const { getTimestamp } = require('./isolation');
+const { sanitizeLogToTemp, StreamingSanitizer } = require('./log-sanitizer');
 
 /**
  * Handle command failure - detect repository, upload log, create issue
@@ -55,7 +56,7 @@ function handleFailure(config, cmdName, fullCommand, exitCode, logPath) {
       console.log('Log upload disabled via START_DISABLE_LOG_UPLOAD');
     }
   } else if (isGhUploadLogAvailable()) {
-    logUrl = uploadLog(logPath);
+    logUrl = uploadLog(logPath, { verbose: config.verbose });
     if (logUrl) {
       console.log(`Log uploaded: ${logUrl}`);
     }
@@ -273,9 +274,11 @@ function isGhUploadLogAvailable() {
  * @param {string} logPath - Path to the log file
  * @returns {string|null} URL of the uploaded log or null
  */
-function uploadLog(logPath) {
+function uploadLog(logPath, options = {}) {
+  let prepared;
   try {
-    const result = execFileSync('gh-upload-log', [logPath, '--public'], {
+    prepared = sanitizeLogToTemp(logPath, { verbose: options.verbose });
+    const result = execFileSync('gh-upload-log', [prepared.path, '--private'], {
       encoding: 'utf8',
       stdio: ['pipe', 'pipe', 'pipe'],
     });
@@ -293,9 +296,13 @@ function uploadLog(logPath) {
     }
 
     return null;
-  } catch (err) {
-    console.log(`Warning: Log upload failed - ${err.message}`);
+  } catch {
+    console.log('Warning: Log preparation or upload failed; upload blocked.');
     return null;
+  } finally {
+    if (prepared) {
+      prepared.cleanup();
+    }
   }
 }
 
@@ -327,6 +334,9 @@ function canCreateIssue(owner, repo) {
  */
 function createIssue(repoInfo, fullCommand, exitCode, logUrl) {
   try {
+    fullCommand = new StreamingSanitizer()
+      .push(Buffer.from(fullCommand), true)
+      .toString();
     const title = `Command failed with exit code ${exitCode}: ${fullCommand.substring(0, 50)}${fullCommand.length > 50 ? '...' : ''}`;
 
     // Get runtime information

package/src/lib/isolation.js

@@ -547,7 +547,32 @@ function runInDocker(command, options = {}) {
     });
   }
 
+  if (!options.resolvedLimits) {
+    try {
+      Object.assign(
+        options,
+        require('./docker-resource-options').resolveResourceOptions(
+          options,
+          require('./execution-control').runCommand
+        )
+      );
+    } catch (error) {
+      return Promise.resolve({
+        success: false,
+        containerName: null,
+        message: error.message,
+      });
+    }
+  }
   const containerName = options.session || generateSessionName('docker');
+  const attribution = {
+    ...options,
+    containerName,
+    sessionId:
+      options.sessionId ||
+      options.executionId ||
+      require('crypto').randomUUID(),
+  };
   const containerExistedBeforeLaunch =
     readDockerContainerStatus(containerName) !== null;
   const cleanupPolicy = getDockerContainerCleanupPolicy(options);
@@ -587,7 +612,7 @@ function runInDocker(command, options = {}) {
         dockerArgs.push('--user', options.user);
       }
 
-      dockerArgs.push(...buildDockerRuntimeArgs(options));
+      dockerArgs.push(...buildDockerRuntimeArgs(attribution));
 
       const effectiveCommand = options.keepAlive
         ? `${command}; exec ${shellToUse}`
@@ -597,12 +622,12 @@ function runInDocker(command, options = {}) {
         : [shellToUse];
       const mainCmdArgs = isBareShell
         ? toShellWords(command)
-        : isShellInvocationWithArgs(command)
+        : isShellInvocationWithArgs(effectiveCommand)
           ? buildShellWithArgsCmdArgs(effectiveCommand)
           : [...shellArgs, '-c', effectiveCommand];
       // A recovery command is selected by a marker that `docker cp` drops
       // into the container before it is started again (issue #176).
-      const cmdArgs = options.recoveryCommand
+      const recoveryArgs = options.recoveryCommand
         ? buildRecoverySelectorArgs(mainCmdArgs, {
             shell: shellToUse,
             shellFlag: shellInteractiveFlag,
@@ -611,6 +636,14 @@ function runInDocker(command, options = {}) {
               : options.recoveryCommand,
           })
         : mainCmdArgs;
+      const {
+        buildCommandHandoffArgs,
+        commandHandoffPath,
+      } = require('./docker-command-handoff');
+      const cmdArgs = buildCommandHandoffArgs(
+        recoveryArgs,
+        commandHandoffPath(containerName)
+      );
       dockerArgs.push(options.image, ...cmdArgs);
 
       if (DEBUG) {
@@ -665,7 +698,7 @@ function runInDocker(command, options = {}) {
           cleanupPolicy,
           options.logPath,
           options.executionId || null,
-          { recoverOnKill: Boolean(options.onKillResume) }
+          { recoverOnKill: Boolean(options.onKillResume), cpuOptions: options }
         );
       }
 
@@ -700,7 +733,7 @@ function runInDocker(command, options = {}) {
       if (options.user) {
         dockerArgs.push('--user', options.user);
       }
-      dockerArgs.push(...buildDockerRuntimeArgs(options));
+      dockerArgs.push(...buildDockerRuntimeArgs(attribution));
       if (DEBUG) {
         console.log(`[DEBUG] shell: ${shellToUse}`);
       }
@@ -745,9 +778,14 @@ function runInDocker(command, options = {}) {
         const launchArgs = needsNetworkSetup
           ? ['start', '-a', '-i', containerName]
           : dockerArgs;
+        const diagnostics =
+          require('./attached-diagnostics').startAttachedDiagnostics(
+            containerName,
+            options
+          );
         const child = spawnAttachedDocker(launchArgs, options.logPath);
 
-        child.on('exit', (code) => {
+        child.on('exit', async (code) => {
           const durationMs = Date.now() - startTime;
           const exitCode = code ?? 1;
           let message = `Docker container "${containerName}" exited with code ${exitCode}`;
@@ -762,6 +800,7 @@ function runInDocker(command, options = {}) {
           }
 
           const dockerState = readDockerContainerState(containerName);
+          const diagnosticFacts = await diagnostics.finish(dockerState);
           const oomKilled = dockerState ? dockerState.oomKilled : null;
           const postMortem = (removed) =>
             recordAttachedDockerPostMortem({
@@ -806,12 +845,14 @@ function runInDocker(command, options = {}) {
           resolve({
             success: exitCode === 0,
             containerName,
+            diagnostics: diagnosticFacts,
             message,
             exitCode,
           });
         });
 
-        child.on('error', (err) => {
+        child.on('error', async (err) => {
+          await diagnostics.finish(null);
           resolve({
             success: false,
             containerName,

package/src/lib/launch-owner.js

@@ -0,0 +1,25 @@
+/** A dead launcher leaves a usable reservation, rather than a permanent gate. */
+const os = require('os');
+
+function markLaunch(record) {
+  record.options.launchPending = true;
+  record.options.launchOwner = { pid: process.pid, hostname: os.hostname() };
+}
+
+function hasActiveLaunch(record) {
+  if (!record?.options?.launchPending) {
+    return false;
+  }
+  const owner = record.options.launchOwner;
+  if (!owner || owner.hostname !== os.hostname()) {
+    return true;
+  }
+  try {
+    process.kill(owner.pid, 0);
+    return true;
+  } catch (error) {
+    return error.code !== 'ESRCH';
+  }
+}
+
+module.exports = { markLaunch, hasActiveLaunch };

package/src/lib/launch-persistence.js

@@ -0,0 +1,54 @@
+/** Reserve before launch, and identify/stop a launch whose final save failed. */
+const { getDockerCommand } = require('./docker-cleanup');
+
+function reserveLaunch(store, record, previous) {
+  require('./launch-owner').markLaunch(record);
+  if (typeof store.reserveLaunch === 'function') {
+    store.reserveLaunch(record, previous);
+  } else {
+    store.save(record);
+  }
+}
+
+function rollbackLaunch(store, previous) {
+  try {
+    store.save(previous);
+  } catch {
+    /* The durable reservation still names the session. */
+  }
+}
+
+function failLaunchedPersistence(store, record, runner, error) {
+  const name = record.options.sessionName;
+  let stopped;
+  try {
+    stopped =
+      record.options.isolated === 'docker'
+        ? runner(getDockerCommand(), ['stop', name])
+        : null;
+  } catch {
+    stopped = null;
+  }
+  const warning = {
+    code: 'LAUNCH_PERSISTENCE_FAILED',
+    uuid: record.uuid,
+    containerName: name,
+    running: !stopped?.success,
+    error: error.message,
+  };
+  if (stopped?.success) {
+    record.status = 'executed';
+    record.exitCode = -1;
+    record.endTime = new Date().toISOString();
+    record.endTimeSource = 'observed-at';
+    record.options.launchPending = false;
+    try {
+      store.save(record);
+    } catch {
+      /* Completion watcher remains attached. */
+    }
+  }
+  return JSON.stringify(warning);
+}
+
+module.exports = { reserveLaunch, rollbackLaunch, failLaunchedPersistence };