start-command 0.35.2 → 0.35.3

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # start-command
 
+## 0.35.3
+
+### Patch Changes
+
+- ec0a9cf: Report OOM kill scope as unknown when only cgroup memory counters are available. Allocation events and killed processes have different units, so their comparison cannot establish container, parent or host scope. Preserve raw counters and the exit-code guard for earlier child OOM kills.
+
 ## 0.35.2
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.35.2",
+  "version": "0.35.3",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/lib/cgroup-memory.js

@@ -5,14 +5,16 @@
  * it says that *some* process in the container was OOM-killed at some point,
  * but not how many, not whether the container hit its own `--memory` limit or
  * the whole host ran out, and not how close the execution came to its limit.
- * The kernel keeps all three per cgroup
+ * The kernel provides raw counters per cgroup
  * (https://docs.kernel.org/admin-guide/cgroup-v2.html#memory-interface-files):
- *   memory.events `oom`       the cgroup hit its own limit
- *   memory.events `oom_kill`  processes killed here by *any* OOM killer, so
- *                             `oom_kill > oom` points to a host-wide (or
- *                             parent cgroup) OOM
+ *   memory.events `oom`       allocation events reaching the memory limit
+ *   memory.events `oom_kill`  processes killed here by *any* OOM killer
  *   memory.peak / memory.max  peak usage (Linux 5.19+) and limit (`max` when
  *                             unlimited)
+ * These counters have different units and are hierarchical. A group OOM can
+ * kill several processes for one allocation event; earlier allocation events
+ * can also coexist with a later host OOM. Their comparison cannot establish
+ * container, parent or host scope (issue #185).
  *
  * The cgroup is removed when the container stops, so the values cannot be read
  * afterwards. The detached completion watcher therefore samples them while the
@@ -122,9 +124,8 @@ function buildCgroupMemoryLogSnippet(quotedLogPath) {
   // A function, so `$1`.. of the watcher script itself stay untouched.
   return (
     `__start_command_cgroup_log() { __scm_note=''; ` +
-    `if [ "$4" -gt 0 ] 2>/dev/null; then if [ "$4" -gt "$3" ] 2>/dev/null; then ` +
-    `__scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.HOST_OR_PARENT]})'; ` +
-    `else __scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.CONTAINER_LIMIT]})'; fi; fi; ` +
+    `if [ "$4" -gt 0 ] 2>/dev/null; then ` +
+    `__scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.UNKNOWN]})'; fi; ` +
     `printf 'Memory:     memory.max=%s memory.peak=%s oom=%s oom_kill=%s%s\\n' ` +
     `"$1" "$2" "$3" "$4" "$__scm_note"; }; ` +
     `if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath}; fi`
@@ -151,14 +152,17 @@ function formatCgroupMemoryLogLine(text) {
 
 /** Where the OOM killer that killed processes in the container came from. */
 const OOM_SCOPE = {
+  UNKNOWN: 'unknown',
+  // Retained scope names; raw counters alone cannot establish either one.
   CONTAINER_LIMIT: 'container-limit',
   HOST_OR_PARENT: 'host-or-parent',
 };
 
 const OOM_SCOPE_NOTES = {
+  [OOM_SCOPE.UNKNOWN]: 'OOM kill scope unknown',
   [OOM_SCOPE.CONTAINER_LIMIT]: 'the container hit its own memory limit',
   [OOM_SCOPE.HOST_OR_PARENT]:
-    'oom_kill > oom: a host-wide or parent cgroup OOM killed processes here',
+    'a host-wide or parent cgroup OOM killed processes here',
 };
 
 function parseCounter(text) {
@@ -216,16 +220,17 @@ function normalizeCgroupMemory(value) {
 
 /**
  * @param {?object} memory - Counters
- * @returns {?string} One of OOM_SCOPE, or null when nothing was OOM-killed
+ * Raw counters do not attribute the OOM killer's scope. Separately attributed
+ * kernel/cgroup evidence would be needed to establish container, parent or
+ * host scope. An unknown oomEvents count remains unknown, never zero.
+ * @returns {?string} OOM_SCOPE.UNKNOWN for observed kills, otherwise null
  */
 function describeCgroupOomScope(memory) {
   const counters = normalizeCgroupMemory(memory);
   if (!counters || !(counters.oomKills > 0)) {
     return null;
   }
-  return counters.oomKills > (counters.oomEvents || 0)
-    ? OOM_SCOPE.HOST_OR_PARENT
-    : OOM_SCOPE.CONTAINER_LIMIT;
+  return OOM_SCOPE.UNKNOWN;
 }
 
 function formatBytes(bytes) {

package/test/regression-182.js

@@ -4,7 +4,7 @@
  *   Docker's `State.OOMKilled` is container-wide and sticky (moby/moby#43564):
  *   it cannot say how many processes the OOM killer took, whether the
  *   container hit its own `--memory` limit or the whole host ran out, or how
- *   close the run came to its limit. The kernel counts all of that per cgroup
+ *   close the run came to its limit. The kernel provides raw counters per cgroup
  *   (`memory.events` `oom`/`oom_kill`, `memory.peak`, `memory.max`), but the
  *   cgroup is gone once the container stops.
  *
@@ -73,8 +73,8 @@ const needsPosixShell = () => {
 };
 
 const CONTAINER_ID = `182c${'0'.repeat(60)}`;
-const HOST_OOM_SAMPLE = '268435456 268300000 0 3';
-const HOST_OOM = {
+const OOM_SAMPLE = '268435456 268300000 0 3';
+const OOM_COUNTERS = {
   limitBytes: 268435456,
   peakBytes: 268300000,
   oomEvents: 0,
@@ -196,7 +196,7 @@ function runWatcher({ events, during, policy, state = KILLED_STATE, max }) {
 
 describe('issue #182: reading cgroup v2 memory counters', () => {
   it('parses the sample the watcher writes', () => {
-    expect(parseCgroupMemorySample(HOST_OOM_SAMPLE)).toEqual(HOST_OOM);
+    expect(parseCgroupMemorySample(OOM_SAMPLE)).toEqual(OOM_COUNTERS);
     expect(parseCgroupMemorySample('max - 1 1\n')).toEqual({
       limitBytes: null,
       peakBytes: null,
@@ -208,19 +208,19 @@ describe('issue #182: reading cgroup v2 memory counters', () => {
     }
   });
 
-  it('tells a container limit from a host-wide or parent OOM', () => {
-    expect(describeCgroupOomScope(HOST_OOM)).toBe(OOM_SCOPE.HOST_OR_PARENT);
+  it('leaves OOM scope unknown with only raw counters', () => {
+    expect(describeCgroupOomScope(OOM_COUNTERS)).toBe(OOM_SCOPE.UNKNOWN);
     expect(describeCgroupOomScope({ oomEvents: 2, oomKills: 2 })).toBe(
-      OOM_SCOPE.CONTAINER_LIMIT
+      OOM_SCOPE.UNKNOWN
     );
     expect(describeCgroupOomScope({ oomEvents: 4, oomKills: 0 })).toBeNull();
     expect(describeCgroupOomScope(null)).toBeNull();
   });
 
   it('formats the counters for --status and for the log', () => {
-    expect(formatCgroupMemory(HOST_OOM)).toBe(
+    expect(formatCgroupMemory(OOM_COUNTERS)).toBe(
       'peak 255.9 MiB of 256.0 MiB limit, oom 0, oom_kill 3 ' +
-        '(oom_kill > oom: a host-wide or parent cgroup OOM killed processes here)'
+        '(OOM kill scope unknown)'
     );
     expect(
       formatCgroupMemory({
@@ -230,7 +230,7 @@ describe('issue #182: reading cgroup v2 memory counters', () => {
         oomKills: 1,
       })
     ).toBe(
-      'peak unknown of no limit, oom 1, oom_kill 1 (the container hit its own memory limit)'
+      'peak unknown of no limit, oom 1, oom_kill 1 (OOM kill scope unknown)'
     );
     expect(formatCgroupMemoryLogLine('max 1024 0 0')).toBe(
       'Memory:     memory.max=max memory.peak=1024 oom=0 oom_kill=0'
@@ -286,10 +286,10 @@ describe('issue #182: the watcher samples the cgroup while the container runs',
     });
     expect(result.status).toBe(0);
     expect(fs.readFileSync(path.join(host.dir, 'sample'), 'utf8')).toBe(
-      HOST_OOM_SAMPLE
+      OOM_SAMPLE
     );
     expect(fs.readFileSync(logPath, 'utf8')).toBe(
-      `${formatCgroupMemoryLogLine(HOST_OOM_SAMPLE)}\n`
+      `${formatCgroupMemoryLogLine(OOM_SAMPLE)}\n`
     );
   });
 
@@ -309,7 +309,7 @@ describe('issue #182: the watcher samples the cgroup while the container runs',
     expect(run.log).toContain('Exit Code:  137 (SIGKILL - 128+9)');
     expect(run.log).toContain(
       'Memory:     memory.max=268435456 memory.peak=268300000 oom=1 oom_kill=3 ' +
-        '(oom_kill > oom: a host-wide or parent cgroup OOM killed processes here)'
+        '(OOM kill scope unknown)'
     );
     expect(run.record.status).toBe(ExecutionStatus.EXECUTED);
     expect(run.record.cgroupMemory).toEqual({
@@ -378,9 +378,9 @@ describe('issue #182: the watcher samples the cgroup while the container runs',
       exitCode: '137',
       oomKilled: 'false',
       finishedAt: '2026-10-01T10:05:00Z',
-      cgroupMemory: HOST_OOM_SAMPLE,
+      cgroupMemory: OOM_SAMPLE,
     });
-    expect(finalized.cgroupMemory).toEqual(HOST_OOM);
+    expect(finalized.cgroupMemory).toEqual(OOM_COUNTERS);
     expect(finalized.exitReason).toBeTruthy();
   });
 });
@@ -402,7 +402,7 @@ describe('issue #182: the kill recovery keeps the killed run counters', () => {
       status: ExecutionStatus.EXECUTING,
       command: 'cargo test',
       logPath,
-      cgroupMemory: HOST_OOM,
+      cgroupMemory: OOM_COUNTERS,
       options: {
         isolated: 'docker',
         isolationMode: 'detached',
@@ -419,14 +419,14 @@ describe('issue #182: the kill recovery keeps the killed run counters', () => {
       startedAt: '2026-10-01T10:00:00Z',
       finishedAt: '2026-10-01T10:05:00Z',
       containerError: '',
-      cgroupMemory: HOST_OOM_SAMPLE,
+      cgroupMemory: OOM_SAMPLE,
       runner: (command, args) => ok(args[0] === 'inspect' ? '{}\n' : 'cid\n'),
       startWatcher: () => {},
       now: () => new Date('2026-10-01T10:06:00.000Z'),
     });
     expect(outcome.recovered).toBe(true);
     expect(fs.readFileSync(logPath, 'utf8')).toContain(
-      `${formatCgroupMemoryLogLine(HOST_OOM_SAMPLE)}\n\n[Recovery 1/2]`
+      `${formatCgroupMemoryLogLine(OOM_SAMPLE)}\n\n[Recovery 1/2]`
     );
     const stored = store.get(record.uuid);
     expect(stored.options.recoveryHistory).toEqual([

package/test/regression-185.js

@@ -0,0 +1,111 @@
+/** Issue #185: allocation events and killed processes cannot establish OOM scope. */
+const { describe, it, expect } = require('bun:test');
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const {
+  buildCgroupMemoryLogSnippet,
+  describeCgroupOomScope,
+  formatCgroupMemory,
+  formatCgroupMemoryLogLine,
+  parseCgroupMemorySample,
+} = require('../src/lib/cgroup-memory');
+const {
+  CGROUP_OOM_EXIT_REASON,
+  resolveExitReason,
+  resolveMemoryExhaustion,
+} = require('../src/lib/exit-reason');
+
+// Group kills, equal counts, earlier allocation failures, zero and unknown oom.
+const SAMPLES = [
+  '3135373312 3135373312 1 3',
+  '3135373312 3135373312 2 2',
+  '3135373312 3135373312 4 1',
+  '3135373312 3135373312 0 3',
+  '3135373312 3135373312 - 3',
+];
+const NOTE = 'OOM kill scope unknown';
+
+describe('issue #185: counter-only OOM scope is unknown', () => {
+  for (const sample of SAMPLES) {
+    it(`preserves raw counters without attributing scope: ${sample}`, () => {
+      const memory = parseCgroupMemorySample(sample);
+      const [, , oom, kills] = sample.split(' ');
+      expect(memory).toEqual({
+        limitBytes: 3135373312,
+        peakBytes: 3135373312,
+        oomEvents: oom === '-' ? null : Number(oom),
+        oomKills: Number(kills),
+      });
+      expect(describeCgroupOomScope(memory)).toBe('unknown');
+      expect(formatCgroupMemory(memory)).toBe(
+        `peak 2.9 GiB of 2.9 GiB limit, oom ${oom === '-' ? 'unknown' : oom}, oom_kill ${kills} (${NOTE})`
+      );
+      expect(formatCgroupMemoryLogLine(sample)).toBe(
+        `Memory:     memory.max=3135373312 memory.peak=3135373312 oom=${oom} oom_kill=${kills} (${NOTE})`
+      );
+    });
+  }
+
+  it('does not report an OOM scope when kills are zero or unknown', () => {
+    for (const memory of [null, {}, { oomKills: 0 }, { oomKills: null }]) {
+      expect(describeCgroupOomScope(memory)).toBeNull();
+    }
+    for (const sample of ['max - 4 0', 'max - 4 -']) {
+      expect(formatCgroupMemoryLogLine(sample)).not.toContain(NOTE);
+    }
+  });
+
+  it('writes the same unknown scope from the POSIX watcher shell', () => {
+    if (process.platform === 'win32') {
+      return;
+    }
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cgroup-185-'));
+    try {
+      const logPath = path.join(dir, 'memory.log');
+      for (const sample of [...SAMPLES, 'max - 4 0', 'max - 4 -']) {
+        fs.writeFileSync(logPath, '');
+        const result = spawnSync(
+          '/bin/sh',
+          [
+            '-c',
+            `__start_command_cgroup='${sample}'; ${buildCgroupMemoryLogSnippet('"$1"')}`,
+            'watcher',
+            logPath,
+          ],
+          { encoding: 'utf8', timeout: 5000 }
+        );
+        expect(result.status).toBe(0);
+        const [max, peak, oom, kills] = sample.split(' ');
+        const note = Number(kills) > 0 ? ` (${NOTE})` : '';
+        expect(fs.readFileSync(logPath, 'utf8')).toBe(
+          `Memory:     memory.max=${max} memory.peak=${peak} oom=${oom} oom_kill=${kills}${note}\n`
+        );
+      }
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it('retains the exit-code guard for earlier child OOM kills', () => {
+    const cgroupMemory = parseCgroupMemorySample(SAMPLES[0]);
+    for (const exitCode of [0, 1, 134, 139, 143]) {
+      const input = { exitCode, oomKilled: false, cgroupMemory };
+      expect(resolveExitReason(input)).not.toBe(CGROUP_OOM_EXIT_REASON);
+      expect(resolveMemoryExhaustion(input)).toBeNull();
+    }
+    for (const exitCode of [137, null, -1]) {
+      const input = { exitCode, oomKilled: false, cgroupMemory };
+      expect(resolveExitReason(input)).toBe(CGROUP_OOM_EXIT_REASON);
+      expect(resolveMemoryExhaustion(input)).toEqual(
+        exitCode === null
+          ? null
+          : {
+              memoryExhausted: true,
+              memoryExhaustedReason: 'cgroup memory.events reported oom_kill=3',
+            }
+      );
+    }
+  });
+});