start-command 0.35.0 → 0.35.1

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # start-command
 
+## 0.35.1
+
+### Patch Changes
+
+- 27737e3: Do not resume `--on-kill-resume` executions whose main process exited 0–127 on its own while Docker's sticky `OOMKilled` flag was set. Docker sets `OOMKilled` when _any_ process in the container was OOM-killed (a compiler, a test runner, a child `node`) and keeps it until the next start, so a run that survived that and later exited 0 or 1 was resumed as if it had been killed. A run now counts as killed only on exit 137, or on `OOMKilled` without a usable exit code; `oomKilled: true` is still reported in `--status` and the post-mortem.
+
 ## 0.35.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.35.0",
+  "version": "0.35.1",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {
@@ -47,7 +47,7 @@
   },
   "homepage": "https://github.com/link-foundation/start#readme",
   "devDependencies": {
-    "@changesets/cli": "^2.31.0",
+    "@changesets/cli": "^3.0.3",
     "@eslint/js": "^10.0.1",
     "eslint": "^10.3.0",
     "eslint-config-prettier": "^10.1.8",

package/src/lib/docker-cleanup.js

@@ -320,11 +320,11 @@ function buildSuccessfulNonOomCondition() {
  * container that is somehow still running afterwards is never removed, gets no
  * `Exit Code:` footer and is never finalized — its record stays `executing`.
  *
- * With `recoverOnKill` (issue #176), a killed main process (exit 137 or
- * `OOMKilled`) is first handed to the recovery entry point. When it resumes
- * the container, this watcher stops here — a new one follows the resumed run —
- * and cleanup, footer and finalization are left to whichever watcher sees the
- * last run end.
+ * With `recoverOnKill` (issue #176), a killed main process (exit 137, or
+ * `OOMKilled` without a usable exit code, issue #178) is first handed to the
+ * recovery entry point. When it resumes the container, this watcher stops
+ * here — a new one follows the resumed run — and cleanup, footer and
+ * finalization are left to whichever watcher sees the last run end.
  *
  * @param {string} containerName - Docker container name
  * @param {string} policy - One of DOCKER_CONTAINER_CLEANUP_POLICY

package/src/lib/execution-recovery.js

@@ -2,11 +2,12 @@
  * Launch-time recovery for killed detached docker sessions (issue #176).
  *
  * `$ --isolated docker --detached --on-kill-resume 3 --recovery-command B -- A`
- * runs `A`; when the main process is killed (exit 137 / `OOMKilled`), the
- * detached completion watcher hands the inspected facts to this module, which
- * restarts the *same* container so `B` continues on the same filesystem, with
- * the same resource limits (they live in the container's HostConfig), under
- * the same execution UUID and appending to the same log file.
+ * runs `A`; when the main process is killed (exit 137, or `OOMKilled` with no
+ * exit status of its own), the detached completion watcher hands the inspected
+ * facts to this module, which restarts the *same* container so `B` continues
+ * on the same filesystem, with the same resource limits (they live in the
+ * container's HostConfig), under the same execution UUID and appending to the
+ * same log file.
  *
  * How `B` replaces `A` inside the same container: a container launched with a
  * recovery command runs a tiny selector as its entrypoint command. The
@@ -81,28 +82,41 @@ function buildRecoverySelectorArgs(mainArgs, params) {
 }
 
 /**
+ * Whether the main process was killed.
+ *
+ * Exit 137 (SIGKILL, the OOM killer's signal) always counts. `OOMKilled` alone
+ * does not: Docker sets it when *any* process in the container's cgroup was
+ * OOM-killed (a compiler, a test runner, a child `node`), and it stays set
+ * until the container is started again. A main process that survived that and
+ * then exited 0-127 on its own ran to completion, so the flag only counts when
+ * there is no usable exit status (the watcher's `-1`, or nothing at all)
+ * (issue #178).
+ *
  * @param {number|string|null} exitCode - `.State.ExitCode`
  * @param {boolean|string|null} oomKilled - `.State.OOMKilled`
  * @returns {boolean} Whether the main process was killed
  */
 function isKilledExit(exitCode, oomKilled) {
-  return (
-    describeExitCode(exitCode).code === KILLED_EXIT_CODE ||
-    oomKilled === true ||
-    oomKilled === 'true'
-  );
+  const { code } = describeExitCode(exitCode);
+  if (code === KILLED_EXIT_CODE) {
+    return true;
+  }
+  const oom = oomKilled === true || oomKilled === 'true';
+  return oom && (code === null || code < 0);
 }
 
 /**
  * Shell condition run by the completion watcher: true when the container was
- * killed and the recovery entry point resumed it.
+ * killed and the recovery entry point resumed it. Mirrors `isKilledExit()`:
+ * `OOMKilled` only counts without a non-negative exit code (issue #178).
  * @param {string} executionId - Execution UUID
  * @returns {string} POSIX shell condition
  */
 function buildRecoverySnippet(executionId) {
   const v = SHELL_VARS;
   return (
-    `{ [ "$${v.exit}" = ${KILLED_EXIT_CODE} ] || [ "$${v.oom}" = true ]; } && ` +
+    `{ [ "$${v.exit}" = ${KILLED_EXIT_CODE} ] || ` +
+    `{ [ "$${v.oom}" = true ] && ! [ "$${v.exit}" -ge 0 ] 2>/dev/null; }; } && ` +
     `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
     `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
     `"$${v.started}" "$${v.finished}" "$${v.error}" >/dev/null 2>&1`

package/test/regression-176.js

@@ -490,10 +490,10 @@ describe('issue #176: resuming a killed session', () => {
     return { outcome, runner, watchers };
   }
 
-  it('detects a kill by exit 137 or OOMKilled', () => {
+  it('detects a kill by exit 137, or OOMKilled without an exit code', () => {
     expect(isKilledExit(137, false)).toBe(true);
     expect(isKilledExit('137', 'false')).toBe(true);
-    expect(isKilledExit('0', 'true')).toBe(true);
+    expect(isKilledExit('-1', 'true')).toBe(true);
     expect(isKilledExit('1', 'false')).toBe(false);
   });
 

package/test/regression-178.js

@@ -0,0 +1,232 @@
+/**
+ * Regression tests for issue #178:
+ *
+ *   `--on-kill-resume N` resumed an execution whenever Docker reported
+ *   `State.OOMKilled=true`, whatever the exit code. Docker sets that flag when
+ *   *any* process in the container's cgroup is OOM-killed (a compiler, a test
+ *   runner, a child `node`) and keeps it set until the container is started
+ *   again. A main process that survived that and then exited 0 (success) or 1
+ *   (a deliberate failure) on its own was resumed as if it had been killed.
+ *
+ *   A container counts as killed only when the main process was: exit 137,
+ *   or `OOMKilled=true` with no usable exit code (the watcher's `-1`).
+ */
+
+const { describe, it, expect, afterEach } = require('bun:test');
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const {
+  ExecutionStore,
+  ExecutionRecord,
+  ExecutionStatus,
+} = require('../src/lib/execution-store');
+const {
+  buildDetachedDockerCompletionScript,
+  DOCKER_CONTAINER_CLEANUP_POLICY,
+} = require('../src/lib/docker-cleanup');
+const { SHELL_VARS } = require('../src/lib/docker-post-mortem');
+const {
+  buildRecoverySnippet,
+  isKilledExit,
+  recoverKilledExecution,
+} = require('../src/lib/execution-recovery');
+
+const tempDirs = [];
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+const needsPosixShell = () => {
+  if (process.platform !== 'win32') {
+    return false;
+  }
+  console.log('  Skipping: the test needs a POSIX /bin/sh');
+  return true;
+};
+
+/** `(exitCode, oomKilled)` pairs from the issue and whether they are kills. */
+const CASES = [
+  ['137', 'false', true],
+  ['137', 'true', true],
+  ['0', 'true', false],
+  ['1', 'true', false],
+  ['1', 'false', false],
+  ['0', 'false', false],
+  ['-1', 'true', true],
+  ['-1', 'false', false],
+];
+
+describe('issue #178: sticky OOMKilled is not a kill of the main process', () => {
+  it('isKilledExit only trusts OOMKilled without an exit code', () => {
+    for (const [exitCode, oomKilled, killed] of CASES) {
+      expect([exitCode, oomKilled, isKilledExit(exitCode, oomKilled)]).toEqual([
+        exitCode,
+        oomKilled,
+        killed,
+      ]);
+    }
+    expect(isKilledExit(137, false)).toBe(true);
+    expect(isKilledExit(0, true)).toBe(false);
+    expect(isKilledExit(1, true)).toBe(false);
+    expect(isKilledExit(null, true)).toBe(true);
+    expect(isKilledExit(undefined, 'true')).toBe(true);
+    expect(isKilledExit('', 'true')).toBe(true);
+    expect(isKilledExit(null, false)).toBe(false);
+  });
+
+  it('the watcher shell condition agrees with isKilledExit', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    // Only the condition: the command after it would start a real recovery.
+    const snippet = buildRecoverySnippet('uuid');
+    const condition = snippet.slice(0, snippet.indexOf('; }; } && ') + 6);
+    expect(condition.endsWith('; }; }')).toBe(true);
+    for (const [exitCode, oomKilled, killed] of [
+      ...CASES,
+      ['', 'true', true],
+      ['unknown', 'true', true],
+    ]) {
+      const result = spawnSync(
+        '/bin/sh',
+        [
+          '-c',
+          `${SHELL_VARS.exit}='${exitCode}'; ${SHELL_VARS.oom}='${oomKilled}'; ` +
+            `if ${condition}; then echo recover; else echo keep; fi`,
+        ],
+        { encoding: 'utf8' }
+      );
+      expect([exitCode, oomKilled, result.stdout.trim()]).toEqual([
+        exitCode,
+        oomKilled,
+        killed ? 'recover' : 'keep',
+      ]);
+      expect(result.stderr).toBe('');
+    }
+  });
+
+  function makeRecord(dir) {
+    const appFolder = path.join(dir, 'app');
+    const logPath = path.join(dir, 'run.log');
+    fs.writeFileSync(logPath, 'main output\n');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const record = new ExecutionRecord({
+      status: ExecutionStatus.EXECUTING,
+      command: 'solve',
+      logPath,
+      options: {
+        isolated: 'docker',
+        isolationMode: 'detached',
+        sessionName: 'box',
+        onKillResume: 1,
+      },
+    });
+    store.save(record);
+    return { appFolder, logPath, store, record };
+  }
+
+  it('recoverKilledExecution does not resume an exit 0/1 with OOMKilled', () => {
+    const { store, record } = makeRecord(makeTempDir('recovery-178-'));
+    for (const exitCode of ['0', '1']) {
+      const calls = [];
+      const outcome = recoverKilledExecution({
+        store,
+        executionId: record.uuid,
+        exitCode,
+        oomKilled: 'true',
+        runner: (...args) => {
+          calls.push(args);
+          return { success: true, stdout: '', stderr: '', status: 0 };
+        },
+        startWatcher: () => calls.push('watcher'),
+      });
+      expect(outcome).toEqual({ recovered: false, reason: 'not-killed' });
+      expect(calls).toEqual([]);
+    }
+    expect(store.get(record.uuid).options.recoveryAttempts).toBeUndefined();
+  });
+
+  /** Run the real watcher against a fake docker that reports `state`. */
+  function runWatcher(state) {
+    const dir = makeTempDir('watcher-178-');
+    const binDir = path.join(dir, 'bin');
+    fs.mkdirSync(binDir);
+    const dockerPath = path.join(binDir, 'docker');
+    fs.writeFileSync(
+      dockerPath,
+      [
+        '#!/bin/sh',
+        `dir='${dir}'`,
+        'echo "$*" >> "$dir/calls"',
+        'case "$1" in',
+        '  inspect)',
+        '    case "$3" in',
+        '      *State.Running*) echo false ;;',
+        '      *State.Error*) echo "" ;;',
+        '      *HostConfig*) echo "{}" ;;',
+        `      *) echo '${state}' ;;`,
+        '    esac ;;',
+        '  logs) echo work ;;',
+        'esac',
+        '',
+      ].join('\n')
+    );
+    fs.chmodSync(dockerPath, 0o755);
+    const { appFolder, logPath, record } = makeRecord(dir);
+    const script = buildDetachedDockerCompletionScript(
+      'box',
+      DOCKER_CONTAINER_CLEANUP_POLICY.KEEP,
+      logPath,
+      record.uuid,
+      { recoverOnKill: true }
+    );
+    spawnSync('/bin/sh', ['-c', script], {
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        PATH: `${binDir}${path.delimiter}${process.env.PATH}`,
+        START_APP_FOLDER: appFolder,
+        START_DISABLE_TRACKING: '',
+      },
+      timeout: 20000,
+    });
+    return {
+      calls: fs.readFileSync(path.join(dir, 'calls'), 'utf8'),
+      log: fs.readFileSync(logPath, 'utf8'),
+      record: new ExecutionStore({ appFolder, useLinks: false }).get(
+        record.uuid
+      ),
+    };
+  }
+
+  const TIMES = '2026-10-03T12:00:00.000000000Z 2026-10-03T16:55:20.000000000Z';
+
+  for (const exitCode of [0, 1]) {
+    it(`the watcher finalizes exit ${exitCode} with OOMKilled=true without resuming`, () => {
+      if (needsPosixShell()) {
+        return;
+      }
+      const { calls, log, record } = runWatcher(`${exitCode} true ${TIMES}`);
+      expect(calls).not.toContain('start box');
+      expect(log).not.toContain('[Recovery');
+      // Still reported as an OOM event in the post-mortem and the record.
+      expect(log).toContain('OOMKilled:  true');
+      expect(record.status).toBe(ExecutionStatus.EXECUTED);
+      expect(record.exitCode).toBe(exitCode);
+      expect(record.oomKilled).toBe(true);
+      expect(record.options.recoveryAttempts).toBeUndefined();
+    }, 30000);
+  }
+});