start-command 0.37.1 → 0.37.2

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # start-command
 
+## 0.37.2
+
+### Patch Changes
+
+- 65314a3: Attribute exit 137 to recent Docker OOM events when the last cgroup sample misses the final kill. Preserve daemon restart and ordinary-exit safeguards, apply attributed evidence to recovery memory limits, and recompute each archived attempt's status diagnostics independently.
+
 ## 0.37.1
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.37.1",
+  "version": "0.37.2",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/lib/execution-attempt.js

@@ -90,6 +90,9 @@ function archiveAttempt(record) {
     status: record.status,
     exitCode: record.exitCode,
     endTime: record.endTime,
+    ...(record.options?.exitEvidence
+      ? { exitEvidence: { ...record.options.exitEvidence } }
+      : {}),
   };
   for (const field of TERMINAL_FIELDS) {
     if (record[field] !== undefined) {
@@ -142,7 +145,31 @@ function readAttemptLogTail(record, bytes) {
   return readLogTail(record.logPath, bytes, record.attempt?.logOffset || 0);
 }
 
+function enrichAttemptHistory(record, enrich, bytes) {
+  return (record.attemptHistory || []).map((attempt, index, history) => {
+    const next = history[index + 1];
+    const end = next ? next.logOffset : record.attempt?.logOffset;
+    const bounded =
+      Number.isSafeInteger(attempt.logOffset) &&
+      attempt.logOffset >= 0 &&
+      Number.isSafeInteger(end) &&
+      end >= attempt.logOffset;
+    const tail =
+      record.logPath && bounded
+        ? readLogTail(record.logPath, bytes, attempt.logOffset, end)
+        : null;
+    const input = {
+      ...attempt,
+      options: { exitEvidence: attempt.exitEvidence },
+    };
+    const output = enrich(input, tail);
+    delete output.options;
+    return output;
+  });
+}
+
 module.exports = {
+  enrichAttemptHistory,
   activityPath,
   appendLifecycle,
   archiveAttempt,

package/src/lib/execution-recovery.js

@@ -323,14 +323,20 @@ function recoverKilledExecution(params = {}) {
     }
   }
   const evidence = require('./exit-evidence');
+  const logged = evidence.fromLog(
+    record.logPath
+      ? require('./execution-attempt').readAttemptLogTail(record, 65536)
+      : null
+  );
   const terminalOom =
-    (params.oomKilled === true || params.oomKilled === 'true') &&
-    evidence.recentOomDelta(params.cgroupMemory, params.finishedAt) &&
-    !evidence.fromLog(
-      record.logPath
-        ? require('./execution-attempt').readAttemptLogTail(record, 65536)
-        : null
-    ).daemonRestart;
+    !logged.daemonRestart &&
+    (logged.mainOom ||
+      ((params.oomKilled === true || params.oomKilled === 'true') &&
+        evidence.recentOomDelta(params.cgroupMemory, params.finishedAt)));
+  record.options.exitEvidence = {
+    mainOom: terminalOom,
+    daemonRestart: logged.daemonRestart,
+  };
   let recoveryLimits = null;
   if (opts.onKillResumeMemory && terminalOom) {
     try {

package/src/lib/exit-evidence.js

@@ -2,6 +2,7 @@
 const { spawnSync } = require('child_process');
 const { shellQuote } = require('./isolation-log-utils');
 const MAIN_OOM = 'Exit evidence: main-oom (recent cgroup oom_kill delta)';
+const DOCKER_OOM = 'Exit evidence: main-oom (recent Docker oom event)';
 const DAEMON_RESTART =
   'Exit evidence: docker-daemon-restart (Docker service journal)';
 
@@ -38,6 +39,22 @@ function daemonRestartEvidence(journal, containerId) {
   return Boolean(exited && forced);
 }
 
+function recentDockerOom(events, finishedAt) {
+  const finished = Date.parse(finishedAt) / 1000;
+  return (
+    Number.isFinite(finished) &&
+    String(events || '')
+      .split('\n')
+      .some((line) => {
+        if (!/^\d+$/.test(line.trim())) {
+          return false;
+        }
+        const at = Number(line.trim());
+        return at > 0 && finished - at >= -1 && finished - at <= 3;
+      })
+  );
+}
+
 function collectExitEvidence(
   { containerName, sample, finishedAt, exitCode, oomKilled },
   runner
@@ -52,31 +69,56 @@ function collectExitEvidence(
   const local =
     !process.env.DOCKER_HOST || process.env.DOCKER_HOST.startsWith('unix://');
   if (
-    local &&
     Number(exitCode) === 137 &&
+    containerName &&
     Number.isFinite(Date.parse(finishedAt))
   ) {
     const docker = require('./docker-cleanup').getDockerCommand();
     const id = run(docker, ['inspect', '-f', '{{.Id}}', containerName]);
     if (id.success && /^[a-f0-9]{64}$/.test(id.stdout.trim())) {
       const end = Date.parse(finishedAt);
-      const journal = run('journalctl', [
-        '-u',
-        'docker.service',
-        '--since',
-        new Date(end - 60000).toISOString(),
-        '--until',
-        new Date(end + 1000).toISOString(),
-        '--no-pager',
-        '-o',
-        'cat',
-      ]);
+      const journal = local
+        ? run('journalctl', [
+            '-u',
+            'docker.service',
+            '--since',
+            new Date(end - 60000).toISOString(),
+            '--until',
+            new Date(end + 1000).toISOString(),
+            '--no-pager',
+            '-o',
+            'cat',
+          ])
+        : { success: false, stdout: '' };
       if (
         journal.success &&
         daemonRestartEvidence(journal.stdout, id.stdout.trim())
       ) {
         return DAEMON_RESTART;
       }
+      // Query the daemon's bounded event history: works with DooD and remote
+      // sockets even after the final kill destroys the sampled cgroup.
+      const events = run(docker, [
+        'events',
+        '--since',
+        new Date(end - 60000).toISOString(),
+        '--until',
+        new Date(end + 1000).toISOString(),
+        '--filter',
+        `container=${id.stdout.trim()}`,
+        '--filter',
+        'event=oom',
+        '--format',
+        '{{.Time}}',
+      ]);
+      if (events.success && recentDockerOom(events.stdout, finishedAt)) {
+        return DOCKER_OOM;
+      }
+      if (process.env.START_DEBUG === '1') {
+        console.error(
+          `[DEBUG] Exit evidence: container=${id.stdout.trim()} finishedAt=${finishedAt} eventsAvailable=${events.success} oomEvents=${JSON.stringify(events.stdout)} sample=${JSON.stringify(sample || '')}`
+        );
+      }
     }
   }
   if (
@@ -92,17 +134,23 @@ function collectExitEvidence(
 function fromLog(log) {
   const lines = String(log || '').split('\n');
   const last = lines.filter((line) => line.startsWith('Exit evidence:')).at(-1);
-  return { mainOom: last === MAIN_OOM, daemonRestart: last === DAEMON_RESTART };
+  return {
+    mainOom: last === MAIN_OOM || last === DOCKER_OOM,
+    daemonRestart: last === DAEMON_RESTART,
+  };
 }
 
 function buildExitEvidenceSnippet(containerName, quotedLogPath) {
-  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} 2>/dev/null`;
+  const stderr = process.env.START_DEBUG === '1' ? '2>&1' : '2>/dev/null';
+  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} ${stderr}`;
 }
 
 module.exports = {
   MAIN_OOM,
+  DOCKER_OOM,
   DAEMON_RESTART,
   recentOomDelta,
+  recentDockerOom,
   daemonRestartEvidence,
   collectExitEvidence,
   fromLog,

package/src/lib/exit-reason.js

@@ -206,6 +206,13 @@ function resolveMemoryExhaustion(input) {
       memoryExhaustedReason: `cgroup memory.events reported oom_kill=${oomKills}`,
     };
   }
+  if (isOomKillOfCommand(input)) {
+    return {
+      memoryExhausted: true,
+      memoryExhaustedReason:
+        'Exit-time evidence reported a main-process OOM kill',
+    };
+  }
   return null;
 }
 

package/src/lib/isolation-log-utils.js

@@ -240,11 +240,16 @@ const FATAL_MARKER_TAIL_BYTES = 64 * 1024;
  * @param {number} [bytes] - Maximum number of trailing bytes to read
  * @returns {string|null} Tail content, or null when the file cannot be read
  */
-function readLogTail(logPath, bytes = LOG_TAIL_BYTES, offset = 0) {
+function readLogTail(
+  logPath,
+  bytes = LOG_TAIL_BYTES,
+  offset = 0,
+  end = Infinity
+) {
   let fd;
   try {
     fd = fs.openSync(logPath, 'r');
-    const size = fs.fstatSync(fd).size;
+    const size = Math.min(fs.fstatSync(fd).size, end);
     const length = Math.min(Math.max(0, size - offset), bytes);
     const buffer = Buffer.alloc(length);
     fs.readSync(fd, buffer, 0, length, size - length);

package/src/lib/status-formatter.js

@@ -500,7 +500,8 @@ function attachMemoryExhaustion(record, logTail) {
   if (!memory) {
     if (
       record.memoryExhaustedReason?.startsWith('Docker reported') ||
-      record.memoryExhaustedReason?.startsWith('cgroup memory.events')
+      record.memoryExhaustedReason?.startsWith('cgroup memory.events') ||
+      record.memoryExhaustedReason?.startsWith('Exit-time evidence')
     ) {
       const cleared = Object.assign(
         Object.create(Object.getPrototypeOf(record)),
@@ -558,6 +559,14 @@ function enrichDetachedStatus(record) {
     Object.create(Object.getPrototypeOf(enriched)),
     enriched
   );
+  if (record.attemptHistory) {
+    copy.attemptHistory = require('./execution-attempt').enrichAttemptHistory(
+      record,
+      (attempt, tail) =>
+        attachMemoryExhaustion(attachExitReason(attempt, tail), tail),
+      FATAL_MARKER_TAIL_BYTES
+    );
+  }
   return readAttemptActivity(copy);
 }
 

package/test/regression-191.js

@@ -97,15 +97,32 @@ it('manual snapshot override applies limits to the new container at creation', a
     fs.rmSync(f.dir, { recursive: true, force: true });
   }
 });
-for (const [exitCode, oomKilled, sample, change] of [
+for (const [exitCode, oomKilled, sample, change, marker] of [
   [137, true, '67108864 66000000 1 1 100 100', true],
   [137, true, '67108864 66000000 1 1 0 100', false],
   [137, false, '67108864 66000000 0 0 0 100', false],
   [1, true, '67108864 66000000 1 1 100 100', false],
+  [
+    137,
+    true,
+    '67108864 66000000 1 1 62 62',
+    true,
+    'Exit evidence: main-oom (recent Docker oom event)',
+  ],
+  [
+    137,
+    true,
+    '67108864 66000000 1 1 62 62',
+    false,
+    'Exit evidence: docker-daemon-restart (Docker service journal)',
+  ],
 ]) {
   it(`automatic memory change requires a qualifying OOM (exit=${exitCode}, fresh=${change})`, () => {
     const f = fixture();
     try {
+      if (marker) {
+        fs.writeFileSync(f.record.logPath, `${marker}\n`);
+      }
       const result = recoverKilledExecution({
         store: f.store,
         executionId: f.record.uuid,

package/test/regression-209.js

@@ -0,0 +1,240 @@
+const { it, expect } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const evidence = require('../src/lib/exit-evidence');
+const {
+  resolveExitReason,
+  resolveMemoryExhaustion,
+} = require('../src/lib/exit-reason');
+const { enrichDetachedStatus } = require('../src/lib/status-formatter');
+const { archiveAttempt } = require('../src/lib/execution-attempt');
+const { ExecutionRecord } = require('../src/lib/execution-store');
+
+const finishedAt = '1970-01-01T00:01:40.592Z';
+const id = 'a'.repeat(64);
+const facts = {
+  containerName: 'task',
+  sample: '3135369216 3135373312 1 1 62 62',
+  finishedAt,
+  exitCode: 137,
+  oomKilled: true,
+};
+function collect(events, overrides = {}, journal = '') {
+  const calls = [];
+  const line = evidence.collectExitEvidence(
+    { ...facts, ...overrides },
+    (cmd, args) => {
+      calls.push([cmd, args]);
+      if (args[0] === 'inspect') {
+        return { success: true, stdout: id };
+      }
+      if (args[0] === 'events') {
+        return { success: true, stdout: events };
+      }
+      return { success: Boolean(journal), stdout: journal };
+    }
+  );
+  return { line, calls };
+}
+
+it('attributes the final Docker OOM event despite a 38-second-old child sample', () => {
+  const { line, calls } = collect('62\n99\n');
+  expect(line).toBe('Exit evidence: main-oom (recent Docker oom event)');
+  const args = calls.find(([, args]) => args[0] === 'events')[1];
+  expect(args).toContain(`container=${id}`);
+  expect(args).toContain('event=oom');
+  expect(args).toContain('1970-01-01T00:00:40.592Z');
+  expect(args).toContain('1970-01-01T00:01:41.592Z');
+  const input = { exitCode: 137, oomKilled: true, logTail: line };
+  expect(resolveExitReason(input)).toBe(
+    'memory-exhaustion (cgroup-oom-killer)'
+  );
+  expect(resolveMemoryExhaustion(input).memoryExhausted).toBe(true);
+  expect(
+    resolveMemoryExhaustion({ ...input, oomKilled: false }).memoryExhausted
+  ).toBe(true);
+});
+
+it('uses daemon events over remote Docker connections without a host journal', () => {
+  const previous = process.env.DOCKER_HOST;
+  process.env.DOCKER_HOST = 'tcp://remote:2376';
+  try {
+    const { line, calls } = collect('100');
+    expect(evidence.fromLog(line).mainOom).toBe(true);
+    expect(calls.some(([cmd]) => cmd === 'journalctl')).toBe(false);
+  } finally {
+    if (previous === undefined) {
+      delete process.env.DOCKER_HOST;
+    } else {
+      process.env.DOCKER_HOST = previous;
+    }
+  }
+});
+
+it('rejects old, future and malformed events and ordinary exits', () => {
+  for (const events of ['62', '96', '102', 'nope', '', '0', 'Infinity']) {
+    expect(evidence.fromLog(collect(events).line).mainOom).toBe(false);
+  }
+  for (const exitCode of [0, 1, 139, -1]) {
+    const { line, calls } = collect('99', { exitCode });
+    expect(evidence.fromLog(line).mainOom).toBe(false);
+    expect(calls.some(([, args]) => args[0] === 'events')).toBe(false);
+  }
+});
+
+it('keeps daemon restart evidence ahead of even a recent OOM event', () => {
+  const journal = `Main process exited\nContainer failed to exit - using the force container=${id}`;
+  const { line } = collect('99', {}, journal);
+  expect(line).toBe(evidence.DAEMON_RESTART);
+  expect(
+    resolveMemoryExhaustion({ exitCode: 137, oomKilled: true, logTail: line })
+  ).toBeNull();
+});
+
+it('archives attributed evidence before a resume clears current options', () => {
+  const record = new ExecutionRecord({
+    command: 'work',
+    status: 'executed',
+    options: {
+      exitEvidence: { mainOom: true, daemonRestart: false },
+    },
+  });
+  archiveAttempt(record);
+  expect(record.attemptHistory[0].exitEvidence).toEqual(
+    record.options.exitEvidence
+  );
+});
+
+it('recomputes history independently while the current attempt is executing', () => {
+  const record = new ExecutionRecord({
+    command: 'work',
+    status: 'executing',
+    options: {},
+  });
+  record.attemptHistory = [
+    {
+      status: 'executed',
+      exitCode: 137,
+      oomKilled: true,
+      exitReason: 'signal (SIGKILL; cause unknown)',
+      exitEvidence: { mainOom: true, daemonRestart: false },
+    },
+    {
+      status: 'executed',
+      exitCode: 1,
+      oomKilled: true,
+      exitReason: 'memory-exhaustion (cgroup-oom-killer)',
+      memoryExhausted: true,
+      memoryExhaustedReason: 'Docker reported State.OOMKilled=true',
+    },
+  ];
+  const result = enrichDetachedStatus(record);
+  expect(result.attemptHistory[0].exitReason).toBe(
+    'memory-exhaustion (cgroup-oom-killer)'
+  );
+  expect(result.attemptHistory[0].memoryExhausted).toBe(true);
+  expect(result.attemptHistory[1].exitReason).toBeUndefined();
+  expect(result.attemptHistory[1].memoryExhausted).toBeUndefined();
+  expect(record.attemptHistory[0].exitReason).toBe(
+    'signal (SIGKILL; cause unknown)'
+  );
+});
+
+it('reads only each archived attempt log range and never leaks current fatal output', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'issue-209-'));
+  try {
+    const first = 'Exit evidence: main-oom (recent Docker oom event)\n';
+    const second = 'ordinary exit\n';
+    const logPath = path.join(dir, 'run.log');
+    fs.writeFileSync(
+      logPath,
+      `${first + second}FATAL ERROR: JavaScript heap out of memory\n`
+    );
+    const record = new ExecutionRecord({
+      command: 'work',
+      status: 'executed',
+      logPath,
+      options: {},
+    });
+    record.exitCode = 1;
+    record.attempt = {
+      number: 3,
+      logOffset: Buffer.byteLength(first + second),
+    };
+    record.attemptHistory = [
+      {
+        status: 'executed',
+        number: 1,
+        logOffset: 0,
+        exitCode: 137,
+        oomKilled: true,
+      },
+      {
+        status: 'executed',
+        number: 2,
+        logOffset: Buffer.byteLength(first),
+        exitCode: 1,
+        oomKilled: true,
+      },
+    ];
+    const result = enrichDetachedStatus(record);
+    expect(result.attemptHistory[0].exitReason).toBe(
+      'memory-exhaustion (cgroup-oom-killer)'
+    );
+    expect(result.attemptHistory[1].memoryExhausted).toBeUndefined();
+    expect(result.exitReason).toBe('memory-exhaustion (v8-heap-limit)');
+    record.attemptHistory[1].logOffset = null;
+    expect(enrichDetachedStatus(record).attemptHistory[0].exitReason).toBe(
+      'signal (SIGKILL; cause unknown)'
+    );
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+it('finalizes and preserves Docker event attribution after the log is removed', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'issue-209-finalize-'));
+  try {
+    const { ExecutionStore } = require('../src/lib/execution-store');
+    const {
+      finalizeDetachedExecution,
+    } = require('../src/lib/detached-finalize');
+    const store = new ExecutionStore({ appFolder: dir, useLinks: false });
+    const logPath = path.join(dir, 'run.log');
+    fs.writeFileSync(logPath, `${evidence.DOCKER_OOM}\n`);
+    const record = new ExecutionRecord({
+      command: 'work',
+      logPath,
+      options: {},
+    });
+    store.save(record);
+    finalizeDetachedExecution({
+      store,
+      executionId: record.uuid,
+      ...facts,
+      running: false,
+    });
+    const current = store.get(record.uuid);
+    expect(current.options.exitEvidence.mainOom).toBe(true);
+    expect(current.exitReason).toBe('memory-exhaustion (cgroup-oom-killer)');
+    const stored = current.options.exitEvidence;
+    current.options.exitEvidence = null;
+    expect(enrichDetachedStatus(current).memoryExhausted).toBe(true);
+    current.options.exitEvidence = { mainOom: false, daemonRestart: false };
+    expect(enrichDetachedStatus(current).memoryExhausted).toBeUndefined();
+    current.options.exitEvidence = stored;
+    const message =
+      require('../src/lib/docker-cleanup').buildAttachedDockerKeptMessage({
+        containerName: 'task',
+        exitCode: 137,
+        oomKilled: true,
+        logPath,
+      });
+    expect(message).toContain('reports it was OOM-killed');
+    fs.unlinkSync(logPath);
+    expect(enrichDetachedStatus(current).memoryExhausted).toBe(true);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});

package/test/shared-golden.js

@@ -5,12 +5,36 @@ const { resolve } = require('node:path');
 const { parseArgs } = require('../src/lib/args-parser');
 const { buildDockerRuntimeArgs } = require('../src/lib/docker-runtime-args');
 const { ExecutionRecord } = require('../src/lib/execution-store');
-const { formatRecord } = require('../src/lib/status-formatter');
+const {
+  formatRecord,
+  enrichDetachedStatus,
+} = require('../src/lib/status-formatter');
 const fixtures = JSON.parse(
   readFileSync(resolve(__dirname, '../../parity/fixtures/contracts.json'))
 );
 
 describe('shared JavaScript and Rust golden contracts', () => {
+  for (const fixture of fixtures['attempt-status-output']) {
+    it(`attempt diagnostics: ${fixture.name}`, () => {
+      const record = ExecutionRecord.fromObject(fixture.record);
+      const output = JSON.parse(
+        formatRecord(enrichDetachedStatus(record), 'json')
+      );
+      assert.deepEqual(output.attemptHistory, fixture.attemptHistory);
+      assert.deepEqual(record.toObject(), fixture.record);
+    });
+  }
+  for (const fixture of fixtures['docker-oom-event']) {
+    it(`Docker OOM event: ${fixture.name}`, () => {
+      assert.equal(
+        require('../src/lib/exit-evidence').recentDockerOom(
+          fixture.events,
+          fixture.finishedAt
+        ),
+        fixture.recent
+      );
+    });
+  }
   for (const fixture of fixtures['cli-parsing']) {
     it(`CLI parsing: ${fixture.name}`, () => {
       if (fixture.error) {