start-command 0.35.2 → 0.35.4

package/src/lib/execution-resume.js

@@ -31,6 +31,12 @@ const {
   readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { appendLogFile } = require('./isolation-log-utils');
+const {
+  appendLifecycle,
+  archiveAttempt,
+  createAttempt,
+  patchAttempt,
+} = require('./execution-attempt');
 const { runCommand } = require('./execution-control');
 const { SessionState, probeSession } = require('./session-probe');
 const { getDockerNetworks } = require('./docker-network-lifecycle');
@@ -105,6 +111,7 @@ function buildLaunchOptions(record) {
     resourceLimits: normalizeResourceLimits(opts.resourceLimits),
     // Append to the same log so one logical session keeps one gap-free record.
     logPath: record.logPath || null,
+    deferCompletionWatcher: true,
   };
 }
 
@@ -339,10 +346,17 @@ function formatResumeResult(result, outputFormat) {
  * @param {?string} containerId - New container id, when one was created
  * @returns {object} The updated record
  */
-function applyResumeToRecord(record, plan, containerId) {
+function applyResumeToRecord(
+  record,
+  plan,
+  containerId,
+  attempt = createAttempt(record, plan)
+) {
+  archiveAttempt(record);
+  record.attempt = attempt;
   const options = { ...(record.options || {}) };
   options.resumeCount = plan.attempt;
-  options.resumedAt = new Date().toISOString();
+  options.resumedAt = attempt.startedAt;
   // A resume is a new deliberate start: launch-time recovery applies again.
   delete options.stopRequestedAt;
   if (plan.resourceLimits && plan.resourceLimits.length > 0) {
@@ -423,6 +437,13 @@ async function resumeExecution(store, identifier, deps = {}) {
   }
 
   let containerId = null;
+  const attempt = createAttempt(record, plan);
+  const lifecycleRecord = {
+    uuid: record.uuid,
+    logPath: record.logPath,
+    attempt,
+  };
+  appendLifecycle(lifecycleRecord, 'resume-started');
 
   if (plan.mode === ResumeMode.RELAUNCH) {
     const runIsolated = deps.runIsolated || require('./isolation').runIsolated;
@@ -432,6 +453,9 @@ async function resumeExecution(store, identifier, deps = {}) {
       plan.launchOptions
     );
     if (!launchResult || !launchResult.success) {
+      appendLifecycle(lifecycleRecord, 'launch-failed', {
+        error: launchResult?.message || 'unknown error',
+      });
       return {
         success: false,
         error: `Failed to relaunch ${plan.backend} session "${plan.sessionName}": ${
@@ -448,6 +472,7 @@ async function resumeExecution(store, identifier, deps = {}) {
           (result.stderr || '').trim() ||
           result.error ||
           `exit code ${result.status}`;
+        appendLifecycle(lifecycleRecord, 'launch-failed', { error: detail });
         return {
           success: false,
           error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${detail}`,
@@ -460,28 +485,53 @@ async function resumeExecution(store, identifier, deps = {}) {
     if (limitsLine && record.logPath) {
       appendLogFile(record.logPath, `${limitsLine}\n`);
     }
-
-    // The completion watcher died with the previous run (or with the
-    // supervisor), so a new one must follow the resumed container.
-    const startWatcher =
-      deps.startWatcher || startDetachedDockerCompletionWatcher;
-    startWatcher(
-      activeSessionName(plan),
-      getDockerContainerCleanupPolicy(record.options || {}),
-      record.logPath || null,
-      record.uuid || null,
-      // `docker start` re-runs the launch-time selector, so the session keeps
-      // its kill recovery (issue #176); a snapshot container has no selector.
-      {
-        recoverOnKill:
-          plan.mode === ResumeMode.DOCKER_START && Boolean(opts.onKillResume),
-      }
-    );
   }
 
   const previousSessionName = plan.newSessionName ? plan.sessionName : null;
-  const updated = applyResumeToRecord(record, plan, containerId);
+  attempt.launchAcceptedAt = new Date().toISOString();
+  const updated = applyResumeToRecord(record, plan, containerId, attempt);
   store.save(updated);
+  appendLifecycle(updated, 'launch-accepted');
+
+  if (plan.backend === 'docker') {
+    try {
+      const startWatcher =
+        deps.startWatcher || startDetachedDockerCompletionWatcher;
+      const watcher = startWatcher(
+        activeSessionName(plan),
+        getDockerContainerCleanupPolicy(updated.options),
+        updated.logPath || null,
+        updated.uuid || null,
+        {
+          since: attempt.startedAt,
+          attemptNumber: attempt.number,
+          recoverOnKill:
+            plan.mode !== ResumeMode.DOCKER_SNAPSHOT &&
+            Boolean(opts.onKillResume),
+        }
+      );
+      if (watcher === false) {
+        throw new Error('Watcher process could not be started');
+      }
+      const current = patchAttempt(store, updated, {
+        watcherAttachedAt: new Date().toISOString(),
+      });
+      if (current) {
+        appendLifecycle(current, 'watcher-attached');
+      }
+    } catch (error) {
+      const current = patchAttempt(store, updated, {
+        watcherError: error.message,
+      });
+      appendLifecycle(current || updated, 'watcher-attachment-failed', {
+        error: error.message,
+      });
+      return {
+        success: false,
+        error: `Launch accepted, but completion watcher attachment failed: ${error.message}`,
+      };
+    }
+  }
 
   return {
     success: true,

package/src/lib/execution-store.js

@@ -75,6 +75,8 @@ class ExecutionRecord {
     this.logPath = options.logPath || '';
     this.startTime = options.startTime || new Date().toISOString();
     this.endTime = options.endTime || null;
+    this.attempt = options.attempt;
+    this.attemptHistory = options.attemptHistory;
     // Provenance of `endTime` (issue #170.2). `--status` used to fabricate a
     // finish time with `new Date()` whenever the record had none, which made an
     // observation indistinguishable from a real finish time. One of
@@ -152,6 +154,12 @@ class ExecutionRecord {
       startTime: this.startTime,
       endTime: this.endTime,
     };
+    if (this.attempt) {
+      obj.attempt = this.attempt;
+    }
+    if (this.attemptHistory) {
+      obj.attemptHistory = this.attemptHistory;
+    }
     if (this.endTimeSource !== undefined && this.endTimeSource !== null) {
       obj.endTimeSource = this.endTimeSource;
     }
@@ -557,6 +565,29 @@ class ExecutionStore {
     }
   }
 
+  // Merge watcher metadata under the database lock without replacing terminal state.
+  patchAttempt(uuid, number, fields) {
+    const lock = new LockManager(this.lockFilePath);
+    if (!lock.acquire()) {
+      throw new Error('Failed to acquire lock for attempt metadata');
+    }
+    try {
+      const records = this.readLinoRecords();
+      const current = records.find((record) => record.uuid === uuid);
+      if (current?.attempt?.number !== number) {
+        return null;
+      }
+      current.attempt = { ...current.attempt, ...fields };
+      this.writeLinoRecords(records);
+      if (this.useLinks) {
+        this.writeLinksRecord(current);
+      }
+      return current;
+    } finally {
+      lock.release();
+    }
+  }
+
   /**
    * Get an execution record by UUID or session name
    * First tries exact UUID match, then the current session name, then any

package/src/lib/isolation-log-utils.js

@@ -240,16 +240,16 @@ const FATAL_MARKER_TAIL_BYTES = 64 * 1024;
  * @param {number} [bytes] - Maximum number of trailing bytes to read
  * @returns {string|null} Tail content, or null when the file cannot be read
  */
-function readLogTail(logPath, bytes = LOG_TAIL_BYTES) {
+function readLogTail(logPath, bytes = LOG_TAIL_BYTES, offset = 0) {
   let fd;
   try {
     fd = fs.openSync(logPath, 'r');
     const size = fs.fstatSync(fd).size;
-    const length = Math.min(size, bytes);
+    const length = Math.min(Math.max(0, size - offset), bytes);
     const buffer = Buffer.alloc(length);
     fs.readSync(fd, buffer, 0, length, size - length);
     const tail = buffer.toString('utf8');
-    if (size <= bytes) {
+    if (size - length === offset || length === 0) {
       return tail;
     }
     const firstNewline = tail.indexOf('\n');

package/src/lib/isolation.js

@@ -659,13 +659,15 @@ function runInDocker(command, options = {}) {
         }
       }
 
-      startDetachedDockerCompletionWatcher(
-        containerName,
-        cleanupPolicy,
-        options.logPath,
-        options.executionId || null,
-        { recoverOnKill: Boolean(options.onKillResume) }
-      );
+      if (!options.deferCompletionWatcher) {
+        startDetachedDockerCompletionWatcher(
+          containerName,
+          cleanupPolicy,
+          options.logPath,
+          options.executionId || null,
+          { recoverOnKill: Boolean(options.onKillResume) }
+        );
+      }
 
       let message = `Command started in detached docker container: ${containerName}`;
       message += `\nContainer ID: ${containerId.substring(0, 12)}`;

package/src/lib/status-formatter.js

@@ -94,7 +94,15 @@ function readDockerState(record) {
   if (opts.isolated !== 'docker' || !opts.sessionName) {
     return null;
   }
-  return inspectDockerState(opts.sessionName);
+  const state = inspectDockerState(opts.sessionName);
+  if (
+    record.attempt &&
+    state?.startedAt &&
+    Date.parse(state.startedAt) < Date.parse(record.attempt.startedAt)
+  ) {
+    return null;
+  }
+  return state;
 }
 
 /**
@@ -506,12 +514,26 @@ function attachMemoryExhaustion(record, logTail) {
  * @returns {Object} Possibly updated execution record
  */
 function enrichDetachedStatus(record) {
-  const logTail =
-    record && record.logPath
-      ? readLogTail(record.logPath, FATAL_MARKER_TAIL_BYTES)
-      : null;
+  if (!record) {
+    return record;
+  }
+  const {
+    readAttemptActivity,
+    readAttemptLogTail,
+  } = require('./execution-attempt');
+  const logTail = record
+    ? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
+    : null;
   const resolved = resolveDetachedStatus(record, logTail);
-  return attachMemoryExhaustion(attachExitReason(resolved, logTail), logTail);
+  const enriched = attachMemoryExhaustion(
+    attachExitReason(resolved, logTail),
+    logTail
+  );
+  const copy = Object.assign(
+    Object.create(Object.getPrototypeOf(enriched)),
+    enriched
+  );
+  return readAttemptActivity(copy);
 }
 
 /**
@@ -693,6 +715,16 @@ function formatRecordAsText(record) {
     lines.push(`Stale Detected At: ${obj.staleDetectedAt}`);
   }
   lines.push(`Log Path:          ${obj.logPath}`);
+  if (obj.attempt) {
+    lines.push(
+      `Attempt:           ${obj.attempt.number}`,
+      `Attempt Started:   ${obj.attempt.startedAt}`,
+      `Attempt Log Offset: ${obj.attempt.logOffset ?? 'N/A'}`,
+      `Launch Accepted:   ${obj.attempt.launchAcceptedAt || 'N/A'}`,
+      `Watcher Attached:  ${obj.attempt.watcherAttachedAt || 'N/A'}`,
+      `Last Output:       ${obj.attempt.lastOutputAt || 'N/A'}`
+    );
+  }
 
   // Format options as nested list instead of JSON
   const optionEntries = Object.entries(obj.options || {}).filter(

package/test/execution-resume.js

@@ -274,7 +274,7 @@ describe('resumeExecution', () => {
     assert.strictEqual(watchers.length, 1);
     assert.strictEqual(watchers[0][0], 'box');
     assert.strictEqual(watchers[0][2], '/tmp/session.log');
-    assert.strictEqual(store.saved.length, 1);
+    assert.strictEqual(store.saved.length, 2);
     assert.strictEqual(store.saved[0].uuid, record.uuid);
     assert.strictEqual(store.saved[0].status, 'executing');
     assert.strictEqual(store.saved[0].options.resumeCount, 1);
@@ -329,6 +329,7 @@ describe('resumeExecution', () => {
     const launches = [];
     const result = await resumeExecution(store, 'box', {
       probe: () => probeWith(SessionState.MISSING),
+      startWatcher: () => {},
       runIsolated: (backend, command, options) => {
         launches.push({ backend, command, options });
         return Promise.resolve({ success: true, containerId: 'newid' });

package/test/regression-176.js

@@ -313,7 +313,11 @@ describe('issue #176: snapshot resume keeps the resource limits', () => {
     expect(saved[0].options.sessionName).toBe('box-resume-1');
     expect(result.output).toContain('--pids-limit=64');
     // A snapshot container has no recovery selector.
-    expect(watchers[0][4]).toEqual({ recoverOnKill: false });
+    expect(watchers[0][4]).toEqual({
+      recoverOnKill: false,
+      since: saved[0].attempt.startedAt,
+      attemptNumber: saved[0].attempt.number,
+    });
   });
 
   it('shows the limits in [Isolation] lines and stores them as metadata', () => {

package/test/regression-182.js

@@ -4,7 +4,7 @@
  *   Docker's `State.OOMKilled` is container-wide and sticky (moby/moby#43564):
  *   it cannot say how many processes the OOM killer took, whether the
  *   container hit its own `--memory` limit or the whole host ran out, or how
- *   close the run came to its limit. The kernel counts all of that per cgroup
+ *   close the run came to its limit. The kernel provides raw counters per cgroup
  *   (`memory.events` `oom`/`oom_kill`, `memory.peak`, `memory.max`), but the
  *   cgroup is gone once the container stops.
  *
@@ -73,8 +73,8 @@ const needsPosixShell = () => {
 };
 
 const CONTAINER_ID = `182c${'0'.repeat(60)}`;
-const HOST_OOM_SAMPLE = '268435456 268300000 0 3';
-const HOST_OOM = {
+const OOM_SAMPLE = '268435456 268300000 0 3';
+const OOM_COUNTERS = {
   limitBytes: 268435456,
   peakBytes: 268300000,
   oomEvents: 0,
@@ -196,7 +196,7 @@ function runWatcher({ events, during, policy, state = KILLED_STATE, max }) {
 
 describe('issue #182: reading cgroup v2 memory counters', () => {
   it('parses the sample the watcher writes', () => {
-    expect(parseCgroupMemorySample(HOST_OOM_SAMPLE)).toEqual(HOST_OOM);
+    expect(parseCgroupMemorySample(OOM_SAMPLE)).toEqual(OOM_COUNTERS);
     expect(parseCgroupMemorySample('max - 1 1\n')).toEqual({
       limitBytes: null,
       peakBytes: null,
@@ -208,19 +208,19 @@ describe('issue #182: reading cgroup v2 memory counters', () => {
     }
   });
 
-  it('tells a container limit from a host-wide or parent OOM', () => {
-    expect(describeCgroupOomScope(HOST_OOM)).toBe(OOM_SCOPE.HOST_OR_PARENT);
+  it('leaves OOM scope unknown with only raw counters', () => {
+    expect(describeCgroupOomScope(OOM_COUNTERS)).toBe(OOM_SCOPE.UNKNOWN);
     expect(describeCgroupOomScope({ oomEvents: 2, oomKills: 2 })).toBe(
-      OOM_SCOPE.CONTAINER_LIMIT
+      OOM_SCOPE.UNKNOWN
     );
     expect(describeCgroupOomScope({ oomEvents: 4, oomKills: 0 })).toBeNull();
     expect(describeCgroupOomScope(null)).toBeNull();
   });
 
   it('formats the counters for --status and for the log', () => {
-    expect(formatCgroupMemory(HOST_OOM)).toBe(
+    expect(formatCgroupMemory(OOM_COUNTERS)).toBe(
       'peak 255.9 MiB of 256.0 MiB limit, oom 0, oom_kill 3 ' +
-        '(oom_kill > oom: a host-wide or parent cgroup OOM killed processes here)'
+        '(OOM kill scope unknown)'
     );
     expect(
       formatCgroupMemory({
@@ -230,7 +230,7 @@ describe('issue #182: reading cgroup v2 memory counters', () => {
         oomKills: 1,
       })
     ).toBe(
-      'peak unknown of no limit, oom 1, oom_kill 1 (the container hit its own memory limit)'
+      'peak unknown of no limit, oom 1, oom_kill 1 (OOM kill scope unknown)'
     );
     expect(formatCgroupMemoryLogLine('max 1024 0 0')).toBe(
       'Memory:     memory.max=max memory.peak=1024 oom=0 oom_kill=0'
@@ -286,10 +286,10 @@ describe('issue #182: the watcher samples the cgroup while the container runs',
     });
     expect(result.status).toBe(0);
     expect(fs.readFileSync(path.join(host.dir, 'sample'), 'utf8')).toBe(
-      HOST_OOM_SAMPLE
+      OOM_SAMPLE
     );
     expect(fs.readFileSync(logPath, 'utf8')).toBe(
-      `${formatCgroupMemoryLogLine(HOST_OOM_SAMPLE)}\n`
+      `${formatCgroupMemoryLogLine(OOM_SAMPLE)}\n`
     );
   });
 
@@ -309,7 +309,7 @@ describe('issue #182: the watcher samples the cgroup while the container runs',
     expect(run.log).toContain('Exit Code:  137 (SIGKILL - 128+9)');
     expect(run.log).toContain(
       'Memory:     memory.max=268435456 memory.peak=268300000 oom=1 oom_kill=3 ' +
-        '(oom_kill > oom: a host-wide or parent cgroup OOM killed processes here)'
+        '(OOM kill scope unknown)'
     );
     expect(run.record.status).toBe(ExecutionStatus.EXECUTED);
     expect(run.record.cgroupMemory).toEqual({
@@ -378,9 +378,9 @@ describe('issue #182: the watcher samples the cgroup while the container runs',
       exitCode: '137',
       oomKilled: 'false',
       finishedAt: '2026-10-01T10:05:00Z',
-      cgroupMemory: HOST_OOM_SAMPLE,
+      cgroupMemory: OOM_SAMPLE,
     });
-    expect(finalized.cgroupMemory).toEqual(HOST_OOM);
+    expect(finalized.cgroupMemory).toEqual(OOM_COUNTERS);
     expect(finalized.exitReason).toBeTruthy();
   });
 });
@@ -402,7 +402,7 @@ describe('issue #182: the kill recovery keeps the killed run counters', () => {
       status: ExecutionStatus.EXECUTING,
       command: 'cargo test',
       logPath,
-      cgroupMemory: HOST_OOM,
+      cgroupMemory: OOM_COUNTERS,
       options: {
         isolated: 'docker',
         isolationMode: 'detached',
@@ -419,14 +419,14 @@ describe('issue #182: the kill recovery keeps the killed run counters', () => {
       startedAt: '2026-10-01T10:00:00Z',
       finishedAt: '2026-10-01T10:05:00Z',
       containerError: '',
-      cgroupMemory: HOST_OOM_SAMPLE,
+      cgroupMemory: OOM_SAMPLE,
       runner: (command, args) => ok(args[0] === 'inspect' ? '{}\n' : 'cid\n'),
       startWatcher: () => {},
       now: () => new Date('2026-10-01T10:06:00.000Z'),
     });
     expect(outcome.recovered).toBe(true);
     expect(fs.readFileSync(logPath, 'utf8')).toContain(
-      `${formatCgroupMemoryLogLine(HOST_OOM_SAMPLE)}\n\n[Recovery 1/2]`
+      `${formatCgroupMemoryLogLine(OOM_SAMPLE)}\n\n[Recovery 1/2]`
     );
     const stored = store.get(record.uuid);
     expect(stored.options.recoveryHistory).toEqual([

package/test/regression-185.js

@@ -0,0 +1,111 @@
+/** Issue #185: allocation events and killed processes cannot establish OOM scope. */
+const { describe, it, expect } = require('bun:test');
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const {
+  buildCgroupMemoryLogSnippet,
+  describeCgroupOomScope,
+  formatCgroupMemory,
+  formatCgroupMemoryLogLine,
+  parseCgroupMemorySample,
+} = require('../src/lib/cgroup-memory');
+const {
+  CGROUP_OOM_EXIT_REASON,
+  resolveExitReason,
+  resolveMemoryExhaustion,
+} = require('../src/lib/exit-reason');
+
+// Group kills, equal counts, earlier allocation failures, zero and unknown oom.
+const SAMPLES = [
+  '3135373312 3135373312 1 3',
+  '3135373312 3135373312 2 2',
+  '3135373312 3135373312 4 1',
+  '3135373312 3135373312 0 3',
+  '3135373312 3135373312 - 3',
+];
+const NOTE = 'OOM kill scope unknown';
+
+describe('issue #185: counter-only OOM scope is unknown', () => {
+  for (const sample of SAMPLES) {
+    it(`preserves raw counters without attributing scope: ${sample}`, () => {
+      const memory = parseCgroupMemorySample(sample);
+      const [, , oom, kills] = sample.split(' ');
+      expect(memory).toEqual({
+        limitBytes: 3135373312,
+        peakBytes: 3135373312,
+        oomEvents: oom === '-' ? null : Number(oom),
+        oomKills: Number(kills),
+      });
+      expect(describeCgroupOomScope(memory)).toBe('unknown');
+      expect(formatCgroupMemory(memory)).toBe(
+        `peak 2.9 GiB of 2.9 GiB limit, oom ${oom === '-' ? 'unknown' : oom}, oom_kill ${kills} (${NOTE})`
+      );
+      expect(formatCgroupMemoryLogLine(sample)).toBe(
+        `Memory:     memory.max=3135373312 memory.peak=3135373312 oom=${oom} oom_kill=${kills} (${NOTE})`
+      );
+    });
+  }
+
+  it('does not report an OOM scope when kills are zero or unknown', () => {
+    for (const memory of [null, {}, { oomKills: 0 }, { oomKills: null }]) {
+      expect(describeCgroupOomScope(memory)).toBeNull();
+    }
+    for (const sample of ['max - 4 0', 'max - 4 -']) {
+      expect(formatCgroupMemoryLogLine(sample)).not.toContain(NOTE);
+    }
+  });
+
+  it('writes the same unknown scope from the POSIX watcher shell', () => {
+    if (process.platform === 'win32') {
+      return;
+    }
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cgroup-185-'));
+    try {
+      const logPath = path.join(dir, 'memory.log');
+      for (const sample of [...SAMPLES, 'max - 4 0', 'max - 4 -']) {
+        fs.writeFileSync(logPath, '');
+        const result = spawnSync(
+          '/bin/sh',
+          [
+            '-c',
+            `__start_command_cgroup='${sample}'; ${buildCgroupMemoryLogSnippet('"$1"')}`,
+            'watcher',
+            logPath,
+          ],
+          { encoding: 'utf8', timeout: 5000 }
+        );
+        expect(result.status).toBe(0);
+        const [max, peak, oom, kills] = sample.split(' ');
+        const note = Number(kills) > 0 ? ` (${NOTE})` : '';
+        expect(fs.readFileSync(logPath, 'utf8')).toBe(
+          `Memory:     memory.max=${max} memory.peak=${peak} oom=${oom} oom_kill=${kills}${note}\n`
+        );
+      }
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it('retains the exit-code guard for earlier child OOM kills', () => {
+    const cgroupMemory = parseCgroupMemorySample(SAMPLES[0]);
+    for (const exitCode of [0, 1, 134, 139, 143]) {
+      const input = { exitCode, oomKilled: false, cgroupMemory };
+      expect(resolveExitReason(input)).not.toBe(CGROUP_OOM_EXIT_REASON);
+      expect(resolveMemoryExhaustion(input)).toBeNull();
+    }
+    for (const exitCode of [137, null, -1]) {
+      const input = { exitCode, oomKilled: false, cgroupMemory };
+      expect(resolveExitReason(input)).toBe(CGROUP_OOM_EXIT_REASON);
+      expect(resolveMemoryExhaustion(input)).toEqual(
+        exitCode === null
+          ? null
+          : {
+              memoryExhausted: true,
+              memoryExhaustedReason: 'cgroup memory.events reported oom_kill=3',
+            }
+      );
+    }
+  });
+});