start-command 0.35.3 → 0.36.0

package/src/lib/exit-reason.js

@@ -181,8 +181,16 @@ function resolveMemoryExhaustion(input) {
     return null;
   }
 
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (evidence.daemonRestart) {
+    return null;
+  }
   const marker = detectMemoryMarker(input.logTail);
-  if (marker) {
+  if (
+    marker &&
+    !(marker.reason.includes('kernel-oom-killer') && !evidence.mainOom)
+  ) {
     return { memoryExhausted: true, memoryExhaustedReason: marker.line };
   }
   if (input.oomKilled === true && isOomKillOfCommand(input)) {
@@ -230,9 +238,9 @@ function isOomKillOfCommand(input) {
   if (!input) {
     return false;
   }
-  const observed =
-    input.oomKilled === true || cgroupOomKills(input.cgroupMemory) > 0;
-  if (!observed) {
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (!evidence.mainOom || evidence.daemonRestart) {
     return false;
   }
   const { exitCode } = input;
@@ -318,8 +326,16 @@ function resolveExitReason(input) {
     return null;
   }
 
+  const evidence =
+    input.exitEvidence || require('./exit-evidence').fromLog(input.logTail);
+  if (evidence.daemonRestart) {
+    return 'killed (docker daemon restart)';
+  }
   const fromLog = detectExitReason(input.logTail);
-  if (fromLog) {
+  if (
+    fromLog &&
+    !(fromLog.includes('kernel-oom-killer') && !evidence.mainOom)
+  ) {
     return fromLog;
   }
 
@@ -328,7 +344,9 @@ function resolveExitReason(input) {
   }
 
   const signalName = signalNameForExitCode(input.exitCode);
-  return signalName ? `signal (${signalName})` : null;
+  return signalName
+    ? `signal (${signalName}${signalName === 'SIGKILL' ? '; cause unknown' : ''})`
+    : null;
 }
 
 module.exports = {

package/src/lib/isolation-log-utils.js

@@ -240,16 +240,16 @@ const FATAL_MARKER_TAIL_BYTES = 64 * 1024;
  * @param {number} [bytes] - Maximum number of trailing bytes to read
  * @returns {string|null} Tail content, or null when the file cannot be read
  */
-function readLogTail(logPath, bytes = LOG_TAIL_BYTES) {
+function readLogTail(logPath, bytes = LOG_TAIL_BYTES, offset = 0) {
   let fd;
   try {
     fd = fs.openSync(logPath, 'r');
     const size = fs.fstatSync(fd).size;
-    const length = Math.min(size, bytes);
+    const length = Math.min(Math.max(0, size - offset), bytes);
     const buffer = Buffer.alloc(length);
     fs.readSync(fd, buffer, 0, length, size - length);
     const tail = buffer.toString('utf8');
-    if (size <= bytes) {
+    if (size - length === offset || length === 0) {
       return tail;
     }
     const firstNewline = tail.indexOf('\n');

package/src/lib/isolation.js

@@ -547,6 +547,23 @@ function runInDocker(command, options = {}) {
     });
   }
 
+  if (!options.resolvedLimits) {
+    try {
+      Object.assign(
+        options,
+        require('./docker-resource-options').resolveResourceOptions(
+          options,
+          require('./execution-control').runCommand
+        )
+      );
+    } catch (error) {
+      return Promise.resolve({
+        success: false,
+        containerName: null,
+        message: error.message,
+      });
+    }
+  }
   const containerName = options.session || generateSessionName('docker');
   const containerExistedBeforeLaunch =
     readDockerContainerStatus(containerName) !== null;
@@ -659,13 +676,15 @@ function runInDocker(command, options = {}) {
         }
       }
 
-      startDetachedDockerCompletionWatcher(
-        containerName,
-        cleanupPolicy,
-        options.logPath,
-        options.executionId || null,
-        { recoverOnKill: Boolean(options.onKillResume) }
-      );
+      if (!options.deferCompletionWatcher) {
+        startDetachedDockerCompletionWatcher(
+          containerName,
+          cleanupPolicy,
+          options.logPath,
+          options.executionId || null,
+          { recoverOnKill: Boolean(options.onKillResume), cpuOptions: options }
+        );
+      }
 
       let message = `Command started in detached docker container: ${containerName}`;
       message += `\nContainer ID: ${containerId.substring(0, 12)}`;
@@ -743,9 +762,14 @@ function runInDocker(command, options = {}) {
         const launchArgs = needsNetworkSetup
           ? ['start', '-a', '-i', containerName]
           : dockerArgs;
+        const diagnostics =
+          require('./attached-diagnostics').startAttachedDiagnostics(
+            containerName,
+            options
+          );
         const child = spawnAttachedDocker(launchArgs, options.logPath);
 
-        child.on('exit', (code) => {
+        child.on('exit', async (code) => {
           const durationMs = Date.now() - startTime;
           const exitCode = code ?? 1;
           let message = `Docker container "${containerName}" exited with code ${exitCode}`;
@@ -760,6 +784,7 @@ function runInDocker(command, options = {}) {
           }
 
           const dockerState = readDockerContainerState(containerName);
+          const diagnosticFacts = await diagnostics.finish(dockerState);
           const oomKilled = dockerState ? dockerState.oomKilled : null;
           const postMortem = (removed) =>
             recordAttachedDockerPostMortem({
@@ -804,12 +829,14 @@ function runInDocker(command, options = {}) {
           resolve({
             success: exitCode === 0,
             containerName,
+            diagnostics: diagnosticFacts,
             message,
             exitCode,
           });
         });
 
-        child.on('error', (err) => {
+        child.on('error', async (err) => {
+          await diagnostics.finish(null);
           resolve({
             success: false,
             containerName,

package/src/lib/launch-owner.js

@@ -0,0 +1,25 @@
+/** A dead launcher leaves a usable reservation, rather than a permanent gate. */
+const os = require('os');
+
+function markLaunch(record) {
+  record.options.launchPending = true;
+  record.options.launchOwner = { pid: process.pid, hostname: os.hostname() };
+}
+
+function hasActiveLaunch(record) {
+  if (!record?.options?.launchPending) {
+    return false;
+  }
+  const owner = record.options.launchOwner;
+  if (!owner || owner.hostname !== os.hostname()) {
+    return true;
+  }
+  try {
+    process.kill(owner.pid, 0);
+    return true;
+  } catch (error) {
+    return error.code !== 'ESRCH';
+  }
+}
+
+module.exports = { markLaunch, hasActiveLaunch };

package/src/lib/launch-persistence.js

@@ -0,0 +1,54 @@
+/** Reserve before launch, and identify/stop a launch whose final save failed. */
+const { getDockerCommand } = require('./docker-cleanup');
+
+function reserveLaunch(store, record, previous) {
+  require('./launch-owner').markLaunch(record);
+  if (typeof store.reserveLaunch === 'function') {
+    store.reserveLaunch(record, previous);
+  } else {
+    store.save(record);
+  }
+}
+
+function rollbackLaunch(store, previous) {
+  try {
+    store.save(previous);
+  } catch {
+    /* The durable reservation still names the session. */
+  }
+}
+
+function failLaunchedPersistence(store, record, runner, error) {
+  const name = record.options.sessionName;
+  let stopped;
+  try {
+    stopped =
+      record.options.isolated === 'docker'
+        ? runner(getDockerCommand(), ['stop', name])
+        : null;
+  } catch {
+    stopped = null;
+  }
+  const warning = {
+    code: 'LAUNCH_PERSISTENCE_FAILED',
+    uuid: record.uuid,
+    containerName: name,
+    running: !stopped?.success,
+    error: error.message,
+  };
+  if (stopped?.success) {
+    record.status = 'executed';
+    record.exitCode = -1;
+    record.endTime = new Date().toISOString();
+    record.endTimeSource = 'observed-at';
+    record.options.launchPending = false;
+    try {
+      store.save(record);
+    } catch {
+      /* Completion watcher remains attached. */
+    }
+  }
+  return JSON.stringify(warning);
+}
+
+module.exports = { reserveLaunch, rollbackLaunch, failLaunchedPersistence };

package/src/lib/query-commands.js

@@ -55,9 +55,19 @@ function handleAttachQuery(store, identifier, readOnly) {
   return reportResult(attachExecution(store, identifier, { readOnly }));
 }
 
-async function handleResumeQuery(store, identifier, command, outputFormat) {
+async function handleResumeQuery(
+  store,
+  identifier,
+  command,
+  outputFormat,
+  resourceOptions = {}
+) {
   return reportResult(
-    await resumeExecution(store, identifier, { command, outputFormat })
+    await resumeExecution(store, identifier, {
+      command,
+      outputFormat,
+      resourceOptions,
+    })
   );
 }
 
@@ -174,7 +184,8 @@ async function dispatchQueryCommand(options, context = {}) {
       store,
       options.resume,
       context.command || null,
-      outputFormat
+      outputFormat,
+      options
     );
   }
   if (options.resumeAll) {

package/src/lib/resume-resources.js

@@ -0,0 +1,88 @@
+const {
+  normalizeResourceLimits,
+  cpuCount,
+  withCpuLimit,
+  readDockerResourceLimits,
+} = require('./docker-resource-limits');
+const { resolveResourceOptions } = require('./docker-resource-options');
+
+// Temporary penalty quotas must never become a resumed execution's base quota.
+function restoreBaseCpu(limits, options) {
+  if (!options.cpuPenaltyConfig) {
+    return limits;
+  }
+  const base = normalizeResourceLimits(
+    options.baseResourceLimits || options.resourceLimits
+  );
+  return [
+    ...limits.filter((s) => !/^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
+    ...base.filter((s) => /^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
+  ];
+}
+
+function prepareResumeResources(record, probe, overrides, runner, random) {
+  const options = record.options || {};
+  if (options.isolated !== 'docker') {
+    if (
+      ['memory', 'memorySwap', 'cpus'].some(
+        (k) => overrides[k] !== null && overrides[k] !== undefined
+      )
+    ) {
+      throw new Error('Resource overrides require a Docker execution');
+    }
+    return null;
+  }
+  const live =
+    probe.state === 'stopped'
+      ? readDockerResourceLimits(options.sessionName, runner)
+      : null;
+  const base = restoreBaseCpu(
+    normalizeResourceLimits(live || options.resourceLimits),
+    options
+  );
+  const oldMemory = base.find((s) => s.startsWith('--memory='))?.slice(9);
+  const result = resolveResourceOptions(
+    { ...overrides, resourceLimits: base },
+    runner,
+    random
+  );
+  if (overrides.cpus && probe.state === 'stopped') {
+    // Docker cannot switch an existing legacy quota to NanoCPUs in place.
+    const cpu = withCpuLimit(base, result.resolvedLimits.cpus);
+    result.resourceLimits = [
+      ...result.resourceLimits.filter(
+        (s) => !/^--(?:cpus|cpu-quota|cpu-period)=/.test(s)
+      ),
+      ...cpu.filter((s) => /^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
+    ];
+  }
+  result.oldMemory = oldMemory
+    ? require('./docker-resource-options').parseResourceSpec(oldMemory).amount
+    : 'unlimited';
+  result.resolvedLimits = {
+    ...options.resolvedLimits,
+    ...result.resolvedLimits,
+  };
+  result.resourceLimitSpecs = {
+    ...options.resourceLimitSpecs,
+    ...result.resourceLimitSpecs,
+  };
+  result.update =
+    probe.state === 'stopped' &&
+    (['memory', 'memorySwap', 'cpus'].some(
+      (k) => overrides[k] !== null && overrides[k] !== undefined
+    ) ||
+      Boolean(options.cpuPenaltyConfig));
+  if (options.cpuPenaltyConfig && !overrides.cpus) {
+    result.resourceLimits = clampCpuToDaemon(result.resourceLimits, runner);
+    result.resolvedLimits.cpus = cpuCount(result.resourceLimits);
+  }
+  return result;
+}
+
+function clampCpuToDaemon(limits, runner) {
+  const { readDaemonCapacity } = require('./docker-resource-options');
+  const capacity = readDaemonCapacity(runner).cpus;
+  return withCpuLimit(limits, Math.min(cpuCount(limits) || capacity, capacity));
+}
+module.exports = { prepareResumeResources, restoreBaseCpu, clampCpuToDaemon };

package/src/lib/status-formatter.js

@@ -94,7 +94,15 @@ function readDockerState(record) {
   if (opts.isolated !== 'docker' || !opts.sessionName) {
     return null;
   }
-  return inspectDockerState(opts.sessionName);
+  const state = inspectDockerState(opts.sessionName);
+  if (
+    record.attempt &&
+    state?.startedAt &&
+    Date.parse(state.startedAt) < Date.parse(record.attempt.startedAt)
+  ) {
+    return null;
+  }
+  return state;
 }
 
 /**
@@ -318,6 +326,9 @@ function applyEndTime(enriched, dockerState, footerFinishedAt) {
 }
 
 function resolveDetachedStatus(record, logTail) {
+  if (require('./launch-owner').hasActiveLaunch(record)) {
+    return record;
+  }
   const footer = parseFooterFromTail(logTail);
   const footerExit = footer.exitCode;
   const dockerState = isDetachedDockerRecord(record)
@@ -436,6 +447,7 @@ function attachExitReason(record, logTail) {
     logTail,
     oomKilled: record.oomKilled,
     cgroupMemory: record.cgroupMemory,
+    exitEvidence: record.options?.exitEvidence,
   });
 
   if (!exitReason) {
@@ -482,9 +494,22 @@ function attachMemoryExhaustion(record, logTail) {
     logTail,
     oomKilled: record.oomKilled,
     cgroupMemory: record.cgroupMemory,
+    exitEvidence: record.options?.exitEvidence,
   });
 
   if (!memory) {
+    if (
+      record.memoryExhaustedReason?.startsWith('Docker reported') ||
+      record.memoryExhaustedReason?.startsWith('cgroup memory.events')
+    ) {
+      const cleared = Object.assign(
+        Object.create(Object.getPrototypeOf(record)),
+        record
+      );
+      delete cleared.memoryExhausted;
+      delete cleared.memoryExhaustedReason;
+      return cleared;
+    }
     return record;
   }
 
@@ -506,12 +531,34 @@ function attachMemoryExhaustion(record, logTail) {
  * @returns {Object} Possibly updated execution record
  */
 function enrichDetachedStatus(record) {
-  const logTail =
-    record && record.logPath
-      ? readLogTail(record.logPath, FATAL_MARKER_TAIL_BYTES)
-      : null;
+  if (!record) {
+    return record;
+  }
+  const cpuPenalty = require('./cpu-penalty-monitor').readState(record);
+  if (cpuPenalty) {
+    record = Object.assign(
+      Object.create(Object.getPrototypeOf(record)),
+      record
+    );
+    record.cpuPenalty = cpuPenalty;
+  }
+  const {
+    readAttemptActivity,
+    readAttemptLogTail,
+  } = require('./execution-attempt');
+  const logTail = record
+    ? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
+    : null;
   const resolved = resolveDetachedStatus(record, logTail);
-  return attachMemoryExhaustion(attachExitReason(resolved, logTail), logTail);
+  const enriched = attachMemoryExhaustion(
+    attachExitReason(resolved, logTail),
+    logTail
+  );
+  const copy = Object.assign(
+    Object.create(Object.getPrototypeOf(enriched)),
+    enriched
+  );
+  return readAttemptActivity(copy);
 }
 
 /**
@@ -671,6 +718,9 @@ function formatRecordAsText(record) {
     ...(formatCgroupMemory(obj.cgroupMemory)
       ? [`Cgroup Memory:     ${formatCgroupMemory(obj.cgroupMemory)}`]
       : []),
+    ...(obj.cpuPenalty
+      ? [`CPU Penalty:       ${JSON.stringify(obj.cpuPenalty)}`]
+      : []),
     `PID:               ${obj.pid !== null ? obj.pid : 'N/A'}`,
     `Working Directory: ${obj.workingDirectory}`,
     `Shell:             ${obj.shell}`,
@@ -693,6 +743,16 @@ function formatRecordAsText(record) {
     lines.push(`Stale Detected At: ${obj.staleDetectedAt}`);
   }
   lines.push(`Log Path:          ${obj.logPath}`);
+  if (obj.attempt) {
+    lines.push(
+      `Attempt:           ${obj.attempt.number}`,
+      `Attempt Started:   ${obj.attempt.startedAt}`,
+      `Attempt Log Offset: ${obj.attempt.logOffset ?? 'N/A'}`,
+      `Launch Accepted:   ${obj.attempt.launchAcceptedAt || 'N/A'}`,
+      `Watcher Attached:  ${obj.attempt.watcherAttachedAt || 'N/A'}`,
+      `Last Output:       ${obj.attempt.lastOutputAt || 'N/A'}`
+    );
+  }
 
   // Format options as nested list instead of JSON
   const optionEntries = Object.entries(obj.options || {}).filter(

package/src/lib/store-lock.js

@@ -0,0 +1,133 @@
+/** Populate before publishing: a failed write must never poison the shared lock. */
+const fs = require('fs');
+const os = require('os');
+const crypto = require('crypto');
+const LOCK_TIMEOUT_MS = 30000;
+const LOCK_STALE_MS = 60000;
+const MALFORMED_LOCK_GRACE_MS = 3000;
+
+class LockManager {
+  constructor(lockFilePath, filesystem = fs) {
+    this.lockFilePath = lockFilePath;
+    this.fs = filesystem;
+    this.lockAcquired = false;
+    this.identity = null;
+  }
+
+  acquire(timeout = LOCK_TIMEOUT_MS) {
+    const started = Date.now();
+    const debug = (message) => {
+      if (process.env.START_DEBUG === '1') {
+        console.error(`[DEBUG] Store lock ${this.lockFilePath}: ${message}`);
+      }
+    };
+    while (Date.now() - started < timeout) {
+      const temp = `${this.lockFilePath}.${process.pid}.${crypto.randomUUID()}`;
+      try {
+        try {
+          const stat = this.fs.statSync(this.lockFilePath);
+          if (this.isLockStale(this.readLockFile(), stat)) {
+            const current = this.fs.statSync(this.lockFilePath);
+            if (current.ino === stat.ino && current.mtimeMs === stat.mtimeMs) {
+              debug('reclaiming stale or malformed lock');
+              this.fs.unlinkSync(this.lockFilePath);
+            }
+          }
+        } catch (error) {
+          if (error.code !== 'ENOENT') {
+            throw error;
+          }
+        }
+        this.fs.writeFileSync(
+          temp,
+          JSON.stringify({
+            pid: process.pid,
+            timestamp: Date.now(),
+            hostname: os.hostname(),
+          }),
+          { flag: 'wx' }
+        );
+        const handle = this.fs.openSync(temp, 'r+');
+        try {
+          this.fs.fsyncSync(handle);
+        } finally {
+          this.fs.closeSync(handle);
+        }
+        const identity = this.fs.statSync(temp);
+        this.fs.linkSync(temp, this.lockFilePath);
+        this.identity = identity;
+        this.lockAcquired = true;
+        debug(`acquired after ${Date.now() - started}ms`);
+        return true;
+      } catch (error) {
+        if (error.code !== 'EEXIST') {
+          throw error;
+        }
+        this.sleep(100);
+      } finally {
+        try {
+          this.fs.unlinkSync(temp);
+        } catch {
+          /* May not have been created. */
+        }
+      }
+    }
+    debug('acquisition timed out');
+    return false;
+  }
+
+  release() {
+    if (!this.lockAcquired) {
+      return;
+    }
+    try {
+      const current = this.fs.statSync(this.lockFilePath);
+      if (
+        current.dev === this.identity.dev &&
+        current.ino === this.identity.ino
+      ) {
+        this.fs.unlinkSync(this.lockFilePath);
+      }
+    } catch {
+      /* The lock may already be gone. */
+    }
+    this.lockAcquired = false;
+  }
+
+  readLockFile() {
+    try {
+      return JSON.parse(this.fs.readFileSync(this.lockFilePath, 'utf8'));
+    } catch {
+      return null;
+    }
+  }
+
+  isLockStale(data, stat) {
+    if (
+      !data ||
+      !Number.isFinite(data.timestamp) ||
+      !Number.isInteger(data.pid) ||
+      data.pid <= 0 ||
+      data.pid > 2147483647
+    ) {
+      return Boolean(
+        stat && Date.now() - stat.mtimeMs >= MALFORMED_LOCK_GRACE_MS
+      );
+    }
+    if (data.hostname === os.hostname()) {
+      try {
+        process.kill(data.pid, 0);
+        return false;
+      } catch (error) {
+        return error.code === 'ESRCH';
+      }
+    }
+    return Date.now() - data.timestamp > LOCK_STALE_MS;
+  }
+
+  sleep(ms) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+  }
+}
+
+module.exports = { LockManager, LOCK_TIMEOUT_MS, MALFORMED_LOCK_GRACE_MS };

package/src/lib/usage.js

@@ -19,6 +19,17 @@ Options:
   --on-kill-resume <N>  Resume a killed (exit 137 / OOM) detached docker session up to N times
   --recovery-command <cmd>  Command to run in the same container on such a resume
   --on-kill-resume-delay <min[-max]>  Wait a random number of seconds before each such resume (default 0)
+  --memory <size|N%|MIN%-MAX%>  Docker RAM cap; percentages use daemon capacity
+  --memory-swap <spec>  Combined memory+swap cap (defaults to memory, no extra swap)
+  --cpus <n|N%|MIN%-MAX%>  Docker CPU cap; ranges draw once per creation
+  --on-kill-resume-memory <spec>  New memory cap before a qualifying OOM recovery
+  --cpu-penalty        Enable delayed Docker CPU penalty (default off)
+  --cpu-penalty-cpus <n>  Penalty cap (default 2 CPUs)
+  --cpu-penalty-trigger <p%>  Busy threshold (default 95% of usable CPUs)
+  --cpu-penalty-trigger-window <d>  Fully covered busy window (default 15m)
+  --cpu-penalty-release <p%>  Quiet threshold (default 65% of penalty cap)
+  --cpu-penalty-release-window <d>  Minimum quiet/capped window (default 15m)
+                        Durations accept ms, s, m, h; --status reports penalty state
   --endpoint <endpoint> SSH endpoint (required for ssh isolation, e.g., user@host)
   --isolated-user, -u [name]  Create isolated user with same permissions
   --keep-user           Keep isolated user after command completes

package/test/docker-runtime-options.js

@@ -305,6 +305,9 @@ describe('buildDockerRuntimeMetadata', () => {
       networks: null,
       networkAliases: null,
       resourceLimits: null,
+      resolvedLimits: null,
+      resourceLimitSpecs: null,
+      onKillResumeMemory: null,
       onKillResume: null,
       recoveryCommand: null,
       onKillResumeDelay: null,
@@ -331,6 +334,9 @@ describe('buildDockerRuntimeMetadata', () => {
         networks: ['hive-formal-ai', 'public-egress'],
         networkAliases: ['formal-ai', 'checker'],
         resourceLimits: null,
+        resolvedLimits: null,
+        resourceLimitSpecs: null,
+        onKillResumeMemory: null,
         onKillResume: null,
         recoveryCommand: null,
         onKillResumeDelay: null,