start-command 0.35.3 → 0.36.0

package/src/lib/execution-resume.js

@@ -28,10 +28,20 @@ const { escapeForLinksNotation } = require('./output-blocks');
 const {
   buildResourceLimitsStatusLine,
   normalizeResourceLimits,
-  readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { appendLogFile } = require('./isolation-log-utils');
+const {
+  appendLifecycle,
+  archiveAttempt,
+  createAttempt,
+  patchAttempt,
+} = require('./execution-attempt');
 const { runCommand } = require('./execution-control');
+const {
+  reserveLaunch,
+  rollbackLaunch,
+  failLaunchedPersistence,
+} = require('./launch-persistence');
 const { SessionState, probeSession } = require('./session-probe');
 const { getDockerNetworks } = require('./docker-network-lifecycle');
 const {
@@ -83,6 +93,7 @@ function buildResumedSessionName(sessionName, attempt) {
 function buildLaunchOptions(record) {
   const opts = record.options || {};
   return {
+    ...opts,
     image: opts.image || null,
     session: opts.sessionName,
     detached: true,
@@ -105,6 +116,7 @@ function buildLaunchOptions(record) {
     resourceLimits: normalizeResourceLimits(opts.resourceLimits),
     // Append to the same log so one logical session keeps one gap-free record.
     logPath: record.logPath || null,
+    deferCompletionWatcher: true,
   };
 }
 
@@ -339,12 +351,20 @@ function formatResumeResult(result, outputFormat) {
  * @param {?string} containerId - New container id, when one was created
  * @returns {object} The updated record
  */
-function applyResumeToRecord(record, plan, containerId) {
+function applyResumeToRecord(
+  record,
+  plan,
+  containerId,
+  attempt = createAttempt(record, plan)
+) {
+  archiveAttempt(record);
+  record.attempt = attempt;
   const options = { ...(record.options || {}) };
   options.resumeCount = plan.attempt;
-  options.resumedAt = new Date().toISOString();
+  options.resumedAt = attempt.startedAt;
   // A resume is a new deliberate start: launch-time recovery applies again.
   delete options.stopRequestedAt;
+  delete options.exitEvidence;
   if (plan.resourceLimits && plan.resourceLimits.length > 0) {
     options.resourceLimits = plan.resourceLimits;
   }
@@ -397,41 +417,120 @@ async function resumeExecution(store, identifier, deps = {}) {
     };
   }
 
+  if (require('./launch-owner').hasActiveLaunch(record)) {
+    return {
+      success: false,
+      error: 'An execution launch is already reserved.',
+    };
+  }
+  const previousData = globalThis.structuredClone(
+    record.toObject ? record.toObject() : record
+  );
+  const previous = record.constructor.fromObject
+    ? record.constructor.fromObject(previousData)
+    : previousData;
   const runner = deps.runner || runCommand;
   const probeFn = deps.probe || ((r) => probeSession(r, runner));
   const probe = probeFn(record);
   const opts = record.options || {};
-  // Only a snapshot resume creates a new container, so only it needs the old
-  // container's limits (issue #176).
-  const liveResourceLimits =
-    deps.command &&
-    opts.isolated === 'docker' &&
-    opts.sessionName &&
-    probe &&
-    !probe.alive &&
-    probe.state === SessionState.STOPPED
-      ? readDockerResourceLimits(opts.sessionName, runner)
-      : null;
+  let resources;
+  try {
+    resources = require('./resume-resources').prepareResumeResources(
+      record,
+      probe,
+      deps.resourceOptions || {},
+      runner,
+      deps.random
+    );
+  } catch (error) {
+    return { success: false, error: error.message };
+  }
+  const planningRecord = resources
+    ? {
+        ...record,
+        options: { ...opts, resourceLimits: resources.resourceLimits },
+      }
+    : record;
   const plan = buildResumePlan(
-    record,
+    planningRecord,
     deps.command || null,
     probe,
-    liveResourceLimits
+    resources?.resourceLimits
   );
+  if (resources && !plan.error) {
+    plan.resourceLimits = resources.resourceLimits;
+    if (plan.mode === ResumeMode.DOCKER_START && resources.update) {
+      plan.steps.unshift({
+        command: getDockerCommand(),
+        args: [
+          'update',
+          ...resources.resourceLimits.filter((s) =>
+            /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
+          ),
+          plan.sessionName,
+        ],
+      });
+    }
+  }
   if (plan.error) {
     return { success: false, error: plan.error };
   }
 
   let containerId = null;
+  const attempt = createAttempt(record, plan);
+  const lifecycleRecord = {
+    uuid: record.uuid,
+    logPath: record.logPath,
+    attempt,
+  };
+  appendLifecycle(lifecycleRecord, 'resume-started');
+  const updated = applyResumeToRecord(record, plan, null, attempt);
+  if (resources) {
+    updated.options.resolvedLimits = resources.resolvedLimits;
+    updated.options.resourceLimitSpecs = resources.resourceLimitSpecs;
+    if (
+      opts.cpuPenaltyConfig &&
+      deps.resourceOptions?.cpus !== null &&
+      deps.resourceOptions?.cpus !== undefined
+    ) {
+      updated.options.baseResourceLimits = resources.resourceLimits;
+    }
+  }
+  try {
+    reserveLaunch(store, updated, previous);
+  } catch (error) {
+    appendLifecycle(lifecycleRecord, 'launch-failed', { error: error.message });
+    return {
+      success: false,
+      error: `Launch reservation failed: ${error.message}`,
+    };
+  }
+
+  if (resources && deps.resourceOptions?.memory && record.logPath) {
+    const oldMemory = resources.oldMemory;
+    appendLogFile(
+      record.logPath,
+      `Resume memory limit: ${oldMemory} -> ${resources.resolvedLimits.memory} (${deps.resourceOptions.memory})\n`
+    );
+  }
 
   if (plan.mode === ResumeMode.RELAUNCH) {
     const runIsolated = deps.runIsolated || require('./isolation').runIsolated;
-    const launchResult = await runIsolated(
-      plan.backend,
-      plan.command,
-      plan.launchOptions
-    );
+    let launchResult;
+    try {
+      launchResult = await runIsolated(
+        plan.backend,
+        plan.command,
+        plan.launchOptions
+      );
+    } catch (error) {
+      launchResult = { success: false, message: error.message };
+    }
     if (!launchResult || !launchResult.success) {
+      rollbackLaunch(store, previous);
+      appendLifecycle(lifecycleRecord, 'launch-failed', {
+        error: launchResult?.message || 'unknown error',
+      });
       return {
         success: false,
         error: `Failed to relaunch ${plan.backend} session "${plan.sessionName}": ${
@@ -442,12 +541,19 @@ async function resumeExecution(store, identifier, deps = {}) {
     containerId = launchResult.containerId || null;
   } else {
     for (const step of plan.steps) {
-      const result = runner(step.command, step.args);
+      let result;
+      try {
+        result = runner(step.command, step.args);
+      } catch (error) {
+        result = { success: false, error: error.message };
+      }
       if (!result.success) {
+        rollbackLaunch(store, previous);
         const detail =
           (result.stderr || '').trim() ||
           result.error ||
           `exit code ${result.status}`;
+        appendLifecycle(lifecycleRecord, 'launch-failed', { error: detail });
         return {
           success: false,
           error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${detail}`,
@@ -460,28 +566,85 @@ async function resumeExecution(store, identifier, deps = {}) {
     if (limitsLine && record.logPath) {
       appendLogFile(record.logPath, `${limitsLine}\n`);
     }
+  }
 
-    // The completion watcher died with the previous run (or with the
-    // supervisor), so a new one must follow the resumed container.
-    const startWatcher =
-      deps.startWatcher || startDetachedDockerCompletionWatcher;
-    startWatcher(
-      activeSessionName(plan),
-      getDockerContainerCleanupPolicy(record.options || {}),
-      record.logPath || null,
-      record.uuid || null,
-      // `docker start` re-runs the launch-time selector, so the session keeps
-      // its kill recovery (issue #176); a snapshot container has no selector.
-      {
-        recoverOnKill:
-          plan.mode === ResumeMode.DOCKER_START && Boolean(opts.onKillResume),
+  const previousSessionName = plan.newSessionName ? plan.sessionName : null;
+  attempt.launchAcceptedAt = new Date().toISOString();
+  updated.options.launchPending = false;
+  if (containerId) {
+    updated.options.containerId = containerId;
+  }
+  let persistenceError = null;
+  try {
+    store.save(updated);
+  } catch (error) {
+    persistenceError = error;
+  }
+  appendLifecycle(updated, 'launch-accepted');
+
+  if (plan.backend === 'docker') {
+    try {
+      const startWatcher =
+        deps.startWatcher || startDetachedDockerCompletionWatcher;
+      const watcher = startWatcher(
+        activeSessionName(plan),
+        getDockerContainerCleanupPolicy(updated.options),
+        updated.logPath || null,
+        updated.uuid || null,
+        {
+          since: attempt.startedAt,
+          attemptNumber: attempt.number,
+          recoverOnKill:
+            plan.mode !== ResumeMode.DOCKER_SNAPSHOT &&
+            Boolean(opts.onKillResume),
+        }
+      );
+      if (watcher === false) {
+        throw new Error('Watcher process could not be started');
       }
-    );
+      const current = persistenceError
+        ? null
+        : patchAttempt(store, updated, {
+            watcherAttachedAt: new Date().toISOString(),
+          });
+      if (current) {
+        appendLifecycle(current, 'watcher-attached');
+      }
+    } catch (error) {
+      if (persistenceError) {
+        return {
+          success: false,
+          error: failLaunchedPersistence(
+            store,
+            updated,
+            runner,
+            persistenceError
+          ),
+        };
+      }
+      const current = persistenceError
+        ? null
+        : patchAttempt(store, updated, {
+            watcherError: error.message,
+          });
+      appendLifecycle(current || updated, 'watcher-attachment-failed', {
+        error: error.message,
+      });
+      return {
+        success: false,
+        error: `Launch accepted for container ${activeSessionName(plan)}, but completion watcher attachment failed: ${error.message}`,
+      };
+    }
   }
 
-  const previousSessionName = plan.newSessionName ? plan.sessionName : null;
-  const updated = applyResumeToRecord(record, plan, containerId);
-  store.save(updated);
+  if (persistenceError) {
+    // The reservation remains durable and the watcher is attached. Stop the
+    // new container rather than allow an untracked command to keep running.
+    return {
+      success: false,
+      error: failLaunchedPersistence(store, updated, runner, persistenceError),
+    };
+  }
 
   return {
     success: true,

package/src/lib/execution-store.js

@@ -24,6 +24,7 @@ const path = require('path');
 const os = require('os');
 const { execFileSync, spawnSync } = require('child_process');
 const crypto = require('crypto');
+const { atomicWrite } = require('./atomic-write');
 
 // Synchronous wrapper using Bun's native ESM support
 // This works because Bun handles ESM/CJS interop
@@ -51,8 +52,7 @@ const DEFAULT_APP_FOLDER = path.join(os.homedir(), '.start-command');
 const LINO_DB_FILE = 'executions.lino';
 const LINKS_DB_FILE = 'executions.links';
 const LOCK_FILE = 'executions.lock';
-const LOCK_TIMEOUT_MS = 30000; // 30 second timeout for lock acquisition
-const LOCK_STALE_MS = 60000; // Consider lock stale after 60 seconds
+const { LockManager } = require('./store-lock');
 
 /**
  * Execution status enumeration
@@ -75,6 +75,8 @@ class ExecutionRecord {
     this.logPath = options.logPath || '';
     this.startTime = options.startTime || new Date().toISOString();
     this.endTime = options.endTime || null;
+    this.attempt = options.attempt;
+    this.attemptHistory = options.attemptHistory;
     // Provenance of `endTime` (issue #170.2). `--status` used to fabricate a
     // finish time with `new Date()` whenever the record had none, which made an
     // observation indistinguishable from a real finish time. One of
@@ -126,6 +128,7 @@ class ExecutionRecord {
     this.shell = options.shell || process.env.SHELL || '/bin/sh';
     this.platform = options.platform || process.platform;
     this.options = options.options || {};
+    this.cpuPenalty = options.cpuPenalty;
   }
 
   /**
@@ -152,6 +155,15 @@ class ExecutionRecord {
       startTime: this.startTime,
       endTime: this.endTime,
     };
+    if (this.cpuPenalty) {
+      obj.cpuPenalty = this.cpuPenalty;
+    }
+    if (this.attempt) {
+      obj.attempt = this.attempt;
+    }
+    if (this.attemptHistory) {
+      obj.attemptHistory = this.attemptHistory;
+    }
     if (this.endTimeSource !== undefined && this.endTimeSource !== null) {
       obj.endTimeSource = this.endTimeSource;
     }
@@ -202,123 +214,6 @@ class ExecutionRecord {
   }
 }
 
-/**
- * File-based lock manager
- */
-class LockManager {
-  constructor(lockFilePath) {
-    this.lockFilePath = lockFilePath;
-    this.lockAcquired = false;
-  }
-
-  /**
-   * Acquire an exclusive lock
-   * @param {number} timeout - Maximum time to wait for lock in ms
-   * @returns {boolean} True if lock acquired
-   */
-  acquire(timeout = LOCK_TIMEOUT_MS) {
-    const startTime = Date.now();
-
-    while (Date.now() - startTime < timeout) {
-      try {
-        // Check if existing lock is stale
-        if (fs.existsSync(this.lockFilePath)) {
-          const lockData = this.readLockFile();
-          if (lockData && this.isLockStale(lockData)) {
-            // Remove stale lock
-            fs.unlinkSync(this.lockFilePath);
-          }
-        }
-
-        // Try to create lock file exclusively
-        const lockData = {
-          pid: process.pid,
-          timestamp: Date.now(),
-          hostname: os.hostname(),
-        };
-
-        fs.writeFileSync(this.lockFilePath, JSON.stringify(lockData), {
-          flag: 'wx', // Fail if file exists
-        });
-
-        this.lockAcquired = true;
-        return true;
-      } catch (err) {
-        if (err.code === 'EEXIST') {
-          // Lock file exists, wait and retry
-          this.sleep(100);
-          continue;
-        }
-        throw err;
-      }
-    }
-
-    return false;
-  }
-
-  /**
-   * Release the lock
-   */
-  release() {
-    if (this.lockAcquired) {
-      try {
-        fs.unlinkSync(this.lockFilePath);
-      } catch {
-        // Ignore errors during release
-      }
-      this.lockAcquired = false;
-    }
-  }
-
-  /**
-   * Read lock file data
-   */
-  readLockFile() {
-    try {
-      const content = fs.readFileSync(this.lockFilePath, 'utf8');
-      return JSON.parse(content);
-    } catch {
-      return null;
-    }
-  }
-
-  /**
-   * Check if lock is stale
-   */
-  isLockStale(lockData) {
-    if (!lockData || !lockData.timestamp) {
-      return true;
-    }
-
-    // Check if lock is too old
-    if (Date.now() - lockData.timestamp > LOCK_STALE_MS) {
-      return true;
-    }
-
-    // Check if the process that holds the lock is still running
-    if (lockData.pid && lockData.hostname === os.hostname()) {
-      try {
-        process.kill(lockData.pid, 0); // Signal 0 just checks if process exists
-        return false; // Process exists, lock is valid
-      } catch {
-        return true; // Process doesn't exist, lock is stale
-      }
-    }
-
-    return false;
-  }
-
-  /**
-   * Simple sleep function
-   */
-  sleep(ms) {
-    const end = Date.now() + ms;
-    while (Date.now() < end) {
-      // Busy wait (not ideal but works for short durations)
-    }
-  }
-}
-
 /**
  * Check if clink is installed
  * @returns {boolean}
@@ -404,7 +299,7 @@ class ExecutionStore {
   writeLinoRecords(records) {
     const data = records.map((r) => r.toObject());
     const content = encodeSync(data);
-    fs.writeFileSync(this.linoDbPath, content, 'utf8');
+    atomicWrite(this.linoDbPath, content);
     this.log(`Wrote ${records.length} records to lino file`);
   }
 
@@ -521,7 +416,7 @@ class ExecutionStore {
    * @param {ExecutionRecord} record
    * @returns {boolean}
    */
-  save(record) {
+  save(record, previous = null) {
     const lock = new LockManager(this.lockFilePath);
 
     if (!lock.acquire()) {
@@ -534,6 +429,16 @@ class ExecutionStore {
 
       // Find existing record index
       const existingIndex = records.findIndex((r) => r.uuid === record.uuid);
+      if (previous) {
+        const current = records[existingIndex];
+        if (
+          !current ||
+          require('./launch-owner').hasActiveLaunch(current) ||
+          (current.attempt?.number || 0) !== (previous.attempt?.number || 0)
+        ) {
+          throw new Error('A different resume already reserved this execution');
+        }
+      }
 
       if (existingIndex >= 0) {
         // Update existing record
@@ -557,6 +462,33 @@ class ExecutionStore {
     }
   }
 
+  reserveLaunch(record, previous) {
+    return this.save(record, previous);
+  }
+
+  // Merge watcher metadata under the database lock without replacing terminal state.
+  patchAttempt(uuid, number, fields) {
+    const lock = new LockManager(this.lockFilePath);
+    if (!lock.acquire()) {
+      throw new Error('Failed to acquire lock for attempt metadata');
+    }
+    try {
+      const records = this.readLinoRecords();
+      const current = records.find((record) => record.uuid === uuid);
+      if (current?.attempt?.number !== number) {
+        return null;
+      }
+      current.attempt = { ...current.attempt, ...fields };
+      this.writeLinoRecords(records);
+      if (this.useLinks) {
+        this.writeLinksRecord(current);
+      }
+      return current;
+    } finally {
+      lock.release();
+    }
+  }
+
   /**
    * Get an execution record by UUID or session name
    * First tries exact UUID match, then the current session name, then any
@@ -658,6 +590,9 @@ class ExecutionStore {
     const staleRecords = [];
 
     for (const record of executingRecords) {
+      if (require('./launch-owner').hasActiveLaunch(record)) {
+        continue;
+      }
       let isStale = false;
       let reason = '';
 

package/src/lib/exit-evidence.js

@@ -0,0 +1,123 @@
+/** Attempt-scoped evidence, independent of Docker's sticky OOMKilled flag. */
+const { spawnSync } = require('child_process');
+const { shellQuote } = require('./isolation-log-utils');
+const MAIN_OOM = 'Exit evidence: main-oom (recent cgroup oom_kill delta)';
+const DAEMON_RESTART =
+  'Exit evidence: docker-daemon-restart (Docker service journal)';
+
+function recentOomDelta(sample, finishedAt) {
+  const fields = String(sample || '')
+    .trim()
+    .split(/\s+/);
+  const changed = Number(fields[4]),
+    observed = Number(fields[5]);
+  const finished = Date.parse(finishedAt) / 1000;
+  return (
+    changed > 0 &&
+    Number.isFinite(finished) &&
+    changed <= observed &&
+    Math.abs(finished - changed) <= 3 &&
+    Math.abs(finished - observed) <= 3
+  );
+}
+
+function daemonRestartEvidence(journal, containerId) {
+  const lines = String(journal || '').split('\n');
+  const exited = lines.some((line) =>
+    /Main process exited|Stopping Docker|Starting Docker|Daemon shutdown complete/.test(
+      line
+    )
+  );
+  const forced =
+    containerId &&
+    lines.some(
+      (line) =>
+        line.includes(containerId) &&
+        /failed to exit|using the force|force.kill/i.test(line)
+    );
+  return Boolean(exited && forced);
+}
+
+function collectExitEvidence(
+  { containerName, sample, finishedAt, exitCode, oomKilled },
+  runner
+) {
+  const run =
+    runner ||
+    ((cmd, args) => {
+      const r = spawnSync(cmd, args, { encoding: 'utf8' });
+      return { success: r.status === 0, stdout: r.stdout || '' };
+    });
+  // A local journal cannot diagnose a remote daemon. Require a local socket.
+  const local =
+    !process.env.DOCKER_HOST || process.env.DOCKER_HOST.startsWith('unix://');
+  if (
+    local &&
+    Number(exitCode) === 137 &&
+    Number.isFinite(Date.parse(finishedAt))
+  ) {
+    const docker = require('./docker-cleanup').getDockerCommand();
+    const id = run(docker, ['inspect', '-f', '{{.Id}}', containerName]);
+    if (id.success && /^[a-f0-9]{64}$/.test(id.stdout.trim())) {
+      const end = Date.parse(finishedAt);
+      const journal = run('journalctl', [
+        '-u',
+        'docker.service',
+        '--since',
+        new Date(end - 60000).toISOString(),
+        '--until',
+        new Date(end + 1000).toISOString(),
+        '--no-pager',
+        '-o',
+        'cat',
+      ]);
+      if (
+        journal.success &&
+        daemonRestartEvidence(journal.stdout, id.stdout.trim())
+      ) {
+        return DAEMON_RESTART;
+      }
+    }
+  }
+  if (
+    (oomKilled === true || oomKilled === 'true') &&
+    (Number(exitCode) === 137 || Number(exitCode) < 0) &&
+    recentOomDelta(sample, finishedAt)
+  ) {
+    return MAIN_OOM;
+  }
+  return 'Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)';
+}
+
+function fromLog(log) {
+  const lines = String(log || '').split('\n');
+  const last = lines.filter((line) => line.startsWith('Exit evidence:')).at(-1);
+  return { mainOom: last === MAIN_OOM, daemonRestart: last === DAEMON_RESTART };
+}
+
+function buildExitEvidenceSnippet(containerName, quotedLogPath) {
+  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} 2>/dev/null`;
+}
+
+module.exports = {
+  MAIN_OOM,
+  DAEMON_RESTART,
+  recentOomDelta,
+  daemonRestartEvidence,
+  collectExitEvidence,
+  fromLog,
+  buildExitEvidenceSnippet,
+};
+if (require.main === module) {
+  const [containerName, sample, finishedAt, exitCode, oomKilled] =
+    process.argv.slice(2);
+  console.log(
+    collectExitEvidence({
+      containerName,
+      sample,
+      finishedAt,
+      exitCode,
+      oomKilled,
+    })
+  );
+}