start-command 0.35.0 → 0.35.2

package/src/lib/usage.js

@@ -18,6 +18,7 @@ Options:
   --network-alias <alias> Add alias to the first network (repeatable, docker only)
   --on-kill-resume <N>  Resume a killed (exit 137 / OOM) detached docker session up to N times
   --recovery-command <cmd>  Command to run in the same container on such a resume
+  --on-kill-resume-delay <min[-max]>  Wait a random number of seconds before each such resume (default 0)
   --endpoint <endpoint> SSH endpoint (required for ssh isolation, e.g., user@host)
   --isolated-user, -u [name]  Create isolated user with same permissions
   --keep-user           Keep isolated user after command completes
@@ -69,6 +70,7 @@ Examples:
   $ --resume my-docker-session -- bash     # Run a different command in the same container
   $ --resume-all                           # Re-attach every still-running execution
   $ -i docker -d --on-kill-resume 3 --recovery-command 'solve --resume' -- solve
+  $ -i docker -d --on-kill-resume 3 --on-kill-resume-delay 30-90 -- cargo test
   $ --use-command-stream echo "Hello"      # Use command-stream library`);
   console.log('');
   console.log('Piping with $:');

package/test/docker-runtime-options.js

@@ -307,6 +307,7 @@ describe('buildDockerRuntimeMetadata', () => {
       resourceLimits: null,
       onKillResume: null,
       recoveryCommand: null,
+      onKillResumeDelay: null,
     });
   });
 
@@ -332,6 +333,7 @@ describe('buildDockerRuntimeMetadata', () => {
         resourceLimits: null,
         onKillResume: null,
         recoveryCommand: null,
+        onKillResumeDelay: null,
       }
     );
   });

package/test/regression-174.js

@@ -113,6 +113,8 @@ function runWatcher({ policy, running, state, withLog = true }) {
       '        echo "$answer" > "$dir/current"',
       '        echo "$answer" ;;',
       '      *State.Error*) echo "" ;;',
+      // The cgroup sampler (issue #182): no cgroup to find here.
+      "      *'{{.Id}}'*|*State.Pid*) echo 0 ;;",
       '      *) echo "inspect-state" >> "$dir/calls"; cat "$dir/state" ;;',
       '    esac ;;',
       '  logs) echo logs >> "$dir/calls"; echo work; exit 1 ;;',

package/test/regression-176.js

@@ -490,10 +490,10 @@ describe('issue #176: resuming a killed session', () => {
     return { outcome, runner, watchers };
   }
 
-  it('detects a kill by exit 137 or OOMKilled', () => {
+  it('detects a kill by exit 137, or OOMKilled without an exit code', () => {
     expect(isKilledExit(137, false)).toBe(true);
     expect(isKilledExit('137', 'false')).toBe(true);
-    expect(isKilledExit('0', 'true')).toBe(true);
+    expect(isKilledExit('-1', 'true')).toBe(true);
     expect(isKilledExit('1', 'false')).toBe(false);
   });
 

package/test/regression-178.js

@@ -0,0 +1,232 @@
+/**
+ * Regression tests for issue #178:
+ *
+ *   `--on-kill-resume N` resumed an execution whenever Docker reported
+ *   `State.OOMKilled=true`, whatever the exit code. Docker sets that flag when
+ *   *any* process in the container's cgroup is OOM-killed (a compiler, a test
+ *   runner, a child `node`) and keeps it set until the container is started
+ *   again. A main process that survived that and then exited 0 (success) or 1
+ *   (a deliberate failure) on its own was resumed as if it had been killed.
+ *
+ *   A container counts as killed only when the main process was: exit 137,
+ *   or `OOMKilled=true` with no usable exit code (the watcher's `-1`).
+ */
+
+const { describe, it, expect, afterEach } = require('bun:test');
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const {
+  ExecutionStore,
+  ExecutionRecord,
+  ExecutionStatus,
+} = require('../src/lib/execution-store');
+const {
+  buildDetachedDockerCompletionScript,
+  DOCKER_CONTAINER_CLEANUP_POLICY,
+} = require('../src/lib/docker-cleanup');
+const { SHELL_VARS } = require('../src/lib/docker-post-mortem');
+const {
+  buildRecoverySnippet,
+  isKilledExit,
+  recoverKilledExecution,
+} = require('../src/lib/execution-recovery');
+
+const tempDirs = [];
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+const needsPosixShell = () => {
+  if (process.platform !== 'win32') {
+    return false;
+  }
+  console.log('  Skipping: the test needs a POSIX /bin/sh');
+  return true;
+};
+
+/** `(exitCode, oomKilled)` pairs from the issue and whether they are kills. */
+const CASES = [
+  ['137', 'false', true],
+  ['137', 'true', true],
+  ['0', 'true', false],
+  ['1', 'true', false],
+  ['1', 'false', false],
+  ['0', 'false', false],
+  ['-1', 'true', true],
+  ['-1', 'false', false],
+];
+
+describe('issue #178: sticky OOMKilled is not a kill of the main process', () => {
+  it('isKilledExit only trusts OOMKilled without an exit code', () => {
+    for (const [exitCode, oomKilled, killed] of CASES) {
+      expect([exitCode, oomKilled, isKilledExit(exitCode, oomKilled)]).toEqual([
+        exitCode,
+        oomKilled,
+        killed,
+      ]);
+    }
+    expect(isKilledExit(137, false)).toBe(true);
+    expect(isKilledExit(0, true)).toBe(false);
+    expect(isKilledExit(1, true)).toBe(false);
+    expect(isKilledExit(null, true)).toBe(true);
+    expect(isKilledExit(undefined, 'true')).toBe(true);
+    expect(isKilledExit('', 'true')).toBe(true);
+    expect(isKilledExit(null, false)).toBe(false);
+  });
+
+  it('the watcher shell condition agrees with isKilledExit', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    // Only the condition: the command after it would start a real recovery.
+    const snippet = buildRecoverySnippet('uuid');
+    const condition = snippet.slice(0, snippet.indexOf('; }; } && ') + 6);
+    expect(condition.endsWith('; }; }')).toBe(true);
+    for (const [exitCode, oomKilled, killed] of [
+      ...CASES,
+      ['', 'true', true],
+      ['unknown', 'true', true],
+    ]) {
+      const result = spawnSync(
+        '/bin/sh',
+        [
+          '-c',
+          `${SHELL_VARS.exit}='${exitCode}'; ${SHELL_VARS.oom}='${oomKilled}'; ` +
+            `if ${condition}; then echo recover; else echo keep; fi`,
+        ],
+        { encoding: 'utf8' }
+      );
+      expect([exitCode, oomKilled, result.stdout.trim()]).toEqual([
+        exitCode,
+        oomKilled,
+        killed ? 'recover' : 'keep',
+      ]);
+      expect(result.stderr).toBe('');
+    }
+  });
+
+  function makeRecord(dir) {
+    const appFolder = path.join(dir, 'app');
+    const logPath = path.join(dir, 'run.log');
+    fs.writeFileSync(logPath, 'main output\n');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const record = new ExecutionRecord({
+      status: ExecutionStatus.EXECUTING,
+      command: 'solve',
+      logPath,
+      options: {
+        isolated: 'docker',
+        isolationMode: 'detached',
+        sessionName: 'box',
+        onKillResume: 1,
+      },
+    });
+    store.save(record);
+    return { appFolder, logPath, store, record };
+  }
+
+  it('recoverKilledExecution does not resume an exit 0/1 with OOMKilled', () => {
+    const { store, record } = makeRecord(makeTempDir('recovery-178-'));
+    for (const exitCode of ['0', '1']) {
+      const calls = [];
+      const outcome = recoverKilledExecution({
+        store,
+        executionId: record.uuid,
+        exitCode,
+        oomKilled: 'true',
+        runner: (...args) => {
+          calls.push(args);
+          return { success: true, stdout: '', stderr: '', status: 0 };
+        },
+        startWatcher: () => calls.push('watcher'),
+      });
+      expect(outcome).toEqual({ recovered: false, reason: 'not-killed' });
+      expect(calls).toEqual([]);
+    }
+    expect(store.get(record.uuid).options.recoveryAttempts).toBeUndefined();
+  });
+
+  /** Run the real watcher against a fake docker that reports `state`. */
+  function runWatcher(state) {
+    const dir = makeTempDir('watcher-178-');
+    const binDir = path.join(dir, 'bin');
+    fs.mkdirSync(binDir);
+    const dockerPath = path.join(binDir, 'docker');
+    fs.writeFileSync(
+      dockerPath,
+      [
+        '#!/bin/sh',
+        `dir='${dir}'`,
+        'echo "$*" >> "$dir/calls"',
+        'case "$1" in',
+        '  inspect)',
+        '    case "$3" in',
+        '      *State.Running*) echo false ;;',
+        '      *State.Error*) echo "" ;;',
+        '      *HostConfig*) echo "{}" ;;',
+        `      *) echo '${state}' ;;`,
+        '    esac ;;',
+        '  logs) echo work ;;',
+        'esac',
+        '',
+      ].join('\n')
+    );
+    fs.chmodSync(dockerPath, 0o755);
+    const { appFolder, logPath, record } = makeRecord(dir);
+    const script = buildDetachedDockerCompletionScript(
+      'box',
+      DOCKER_CONTAINER_CLEANUP_POLICY.KEEP,
+      logPath,
+      record.uuid,
+      { recoverOnKill: true }
+    );
+    spawnSync('/bin/sh', ['-c', script], {
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        PATH: `${binDir}${path.delimiter}${process.env.PATH}`,
+        START_APP_FOLDER: appFolder,
+        START_DISABLE_TRACKING: '',
+      },
+      timeout: 20000,
+    });
+    return {
+      calls: fs.readFileSync(path.join(dir, 'calls'), 'utf8'),
+      log: fs.readFileSync(logPath, 'utf8'),
+      record: new ExecutionStore({ appFolder, useLinks: false }).get(
+        record.uuid
+      ),
+    };
+  }
+
+  const TIMES = '2026-10-03T12:00:00.000000000Z 2026-10-03T16:55:20.000000000Z';
+
+  for (const exitCode of [0, 1]) {
+    it(`the watcher finalizes exit ${exitCode} with OOMKilled=true without resuming`, () => {
+      if (needsPosixShell()) {
+        return;
+      }
+      const { calls, log, record } = runWatcher(`${exitCode} true ${TIMES}`);
+      expect(calls).not.toContain('start box');
+      expect(log).not.toContain('[Recovery');
+      // Still reported as an OOM event in the post-mortem and the record.
+      expect(log).toContain('OOMKilled:  true');
+      expect(record.status).toBe(ExecutionStatus.EXECUTED);
+      expect(record.exitCode).toBe(exitCode);
+      expect(record.oomKilled).toBe(true);
+      expect(record.options.recoveryAttempts).toBeUndefined();
+    }, 30000);
+  }
+});

package/test/regression-180.js

@@ -0,0 +1,196 @@
+/**
+ * Regression tests for issue #180:
+ *
+ *   `$ --status` reported `exitReason memory-exhaustion (cgroup-oom-killer)`
+ *   and `memoryExhausted true` for an ordinary exit 0/1 whenever Docker's
+ *   `State.OOMKilled` flag was set. Since moby/moby#43564 that flag is
+ *   container-wide and sticky: it turns on as soon as *any* process in the
+ *   container is OOM-killed (a `rustc` child under `cargo test`) and stays on
+ *   until the container starts again. A command that survived that and later
+ *   exited 1 on its own was reported as having run out of memory.
+ *
+ *   The flag only explains the command's exit when that exit is 137
+ *   (`128 + SIGKILL`) or unknown: the rule `isKilledExit()` applies since #178.
+ */
+
+const { describe, it, expect, afterEach } = require('bun:test');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const {
+  CGROUP_OOM_EXIT_REASON,
+  isOomKillOfCommand,
+  resolveExitReason,
+  resolveMemoryExhaustion,
+} = require('../src/lib/exit-reason');
+const {
+  attachExitReason,
+  attachMemoryExhaustion,
+} = require('../src/lib/status-formatter');
+const { finalizeDetachedExecution } = require('../src/lib/detached-finalize');
+const { buildAttachedDockerKeptMessage } = require('../src/lib/docker-cleanup');
+const {
+  ExecutionStore,
+  ExecutionRecord,
+  ExecutionStatus,
+} = require('../src/lib/execution-store');
+
+const OOM_REASON = 'Docker reported State.OOMKilled=true';
+
+const tempDirs = [];
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+function finishedRecord(fields) {
+  const record = new ExecutionRecord({
+    status: ExecutionStatus.EXECUTED,
+    command: 'cargo test',
+    logPath: null,
+  });
+  return Object.assign(record, fields);
+}
+
+describe('issue #180: the sticky OOMKilled flag is not an exit reason', () => {
+  // The reproduction table from the issue, plus the watcher's unknown exit.
+  const CASES = [
+    { exitCode: 0, oomKilled: true, reason: null, memory: false },
+    { exitCode: 1, oomKilled: true, reason: null, memory: false },
+    { exitCode: 1, oomKilled: false, reason: null, memory: false },
+    {
+      exitCode: 137,
+      oomKilled: true,
+      reason: CGROUP_OOM_EXIT_REASON,
+      memory: true,
+    },
+    {
+      exitCode: 137,
+      oomKilled: false,
+      reason: 'signal (SIGKILL)',
+      memory: false,
+    },
+    {
+      exitCode: -1,
+      oomKilled: true,
+      reason: CGROUP_OOM_EXIT_REASON,
+      memory: true,
+    },
+    {
+      exitCode: 139,
+      oomKilled: true,
+      reason: 'signal (SIGSEGV)',
+      memory: false,
+    },
+  ];
+
+  for (const { exitCode, oomKilled, reason, memory } of CASES) {
+    it(`exit ${exitCode} with oomKilled=${oomKilled}`, () => {
+      expect(resolveExitReason({ exitCode, oomKilled })).toBe(reason);
+      expect(resolveMemoryExhaustion({ exitCode, oomKilled })).toEqual(
+        memory
+          ? { memoryExhausted: true, memoryExhaustedReason: OOM_REASON }
+          : null
+      );
+    });
+  }
+
+  it('blames the flag on the command only for SIGKILL or an unknown exit', () => {
+    expect(isOomKillOfCommand({ exitCode: 137, oomKilled: true })).toBe(true);
+    expect(isOomKillOfCommand({ exitCode: null, oomKilled: true })).toBe(true);
+    expect(isOomKillOfCommand({ exitCode: -1, oomKilled: true })).toBe(true);
+    expect(isOomKillOfCommand({ exitCode: 0, oomKilled: true })).toBe(false);
+    expect(isOomKillOfCommand({ exitCode: 1, oomKilled: true })).toBe(false);
+    expect(isOomKillOfCommand({ exitCode: 137, oomKilled: false })).toBe(false);
+    expect(isOomKillOfCommand(null)).toBe(false);
+  });
+
+  it('still trusts a memory marker the command printed itself', () => {
+    const logTail =
+      'FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory';
+    expect(resolveExitReason({ exitCode: 1, oomKilled: true, logTail })).toBe(
+      'memory-exhaustion (v8-heap-limit)'
+    );
+    expect(
+      resolveMemoryExhaustion({ exitCode: 1, oomKilled: true, logTail })
+        .memoryExhausted
+    ).toBe(true);
+  });
+
+  it('reports the incident record without a memory verdict', () => {
+    const record = finishedRecord({ exitCode: 1, oomKilled: true });
+    const enriched = attachMemoryExhaustion(
+      attachExitReason(record, 'Failed to authenticate: OAuth session expired'),
+      'Failed to authenticate: OAuth session expired'
+    );
+    expect(enriched.oomKilled).toBe(true);
+    expect(enriched.exitReason).toBeUndefined();
+    expect(enriched.memoryExhausted).toBeUndefined();
+  });
+
+  it('drops a stale cgroup reason stored before the fix', () => {
+    const record = finishedRecord({
+      exitCode: 1,
+      oomKilled: true,
+      exitReason: CGROUP_OOM_EXIT_REASON,
+    });
+    const enriched = attachExitReason(record, null);
+    expect(enriched.exitReason).toBeUndefined();
+    expect(record.exitReason).toBe(CGROUP_OOM_EXIT_REASON);
+  });
+
+  it('the finalizer does not record the flag as the reason for exit 1', () => {
+    const appFolder = makeTempDir('finalize-180-');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const record = new ExecutionRecord({
+      status: ExecutionStatus.EXECUTING,
+      command: 'cargo test',
+      logPath: null,
+      options: { isolated: 'docker', isolationMode: 'detached' },
+    });
+    store.save(record);
+
+    finalizeDetachedExecution({
+      store,
+      executionId: record.uuid,
+      exitCode: '1',
+      oomKilled: 'true',
+      startedAt: '2026-10-01T19:00:00Z',
+      finishedAt: '2026-10-01T19:41:00Z',
+      running: 'false',
+    });
+
+    const stored = store.get(record.uuid);
+    expect(stored.exitCode).toBe(1);
+    expect(stored.oomKilled).toBe(true);
+    expect(stored.exitReason).toBeUndefined();
+  });
+
+  it('the attached kept message names a child OOM kill, not the command', () => {
+    const child = buildAttachedDockerKeptMessage({
+      containerName: 'c',
+      exitCode: 1,
+      oomKilled: true,
+      logPath: null,
+    });
+    expect(child).toContain('a process in it was OOM-killed');
+    expect(child).not.toContain('Memory exhaustion detected');
+
+    const main = buildAttachedDockerKeptMessage({
+      containerName: 'c',
+      exitCode: 137,
+      oomKilled: true,
+      logPath: null,
+    });
+    expect(main).toContain('Docker reports it was OOM-killed.');
+  });
+});