start-command 0.37.0 → 0.37.2

package/src/lib/isolation-log-utils.js

@@ -240,11 +240,16 @@ const FATAL_MARKER_TAIL_BYTES = 64 * 1024;
  * @param {number} [bytes] - Maximum number of trailing bytes to read
  * @returns {string|null} Tail content, or null when the file cannot be read
  */
-function readLogTail(logPath, bytes = LOG_TAIL_BYTES, offset = 0) {
+function readLogTail(
+  logPath,
+  bytes = LOG_TAIL_BYTES,
+  offset = 0,
+  end = Infinity
+) {
   let fd;
   try {
     fd = fs.openSync(logPath, 'r');
-    const size = fs.fstatSync(fd).size;
+    const size = Math.min(fs.fstatSync(fd).size, end);
     const length = Math.min(Math.max(0, size - offset), bytes);
     const buffer = Buffer.alloc(length);
     fs.readSync(fd, buffer, 0, length, size - length);

package/src/lib/resume-resources.js

@@ -7,13 +7,23 @@ const {
 const { resolveResourceOptions } = require('./docker-resource-options');
 
 // Temporary penalty quotas must never become a resumed execution's base quota.
-function restoreBaseCpu(limits, options) {
+function restoreBaseCpu(limits, options, saved = null, inspected = false) {
   if (!options.cpuPenaltyConfig) {
     return limits;
   }
-  const base = normalizeResourceLimits(
+  if (
+    saved &&
+    inspected &&
+    !(saved.phase === 'penalized' && cpuCount(limits) === saved.limitCpus)
+  ) {
+    return limits;
+  }
+  let base = normalizeResourceLimits(
     options.baseResourceLimits || options.resourceLimits
   );
+  if (saved && Object.hasOwn(saved, 'baseCpus')) {
+    base = saved.baseCpus > 0 ? withCpuLimit(limits, saved.baseCpus) : [];
+  }
   return [
     ...limits.filter((s) => !/^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
     ...base.filter((s) => /^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
@@ -38,7 +48,9 @@ function prepareResumeResources(record, probe, overrides, runner, random) {
       : null;
   const base = restoreBaseCpu(
     normalizeResourceLimits(live || options.resourceLimits),
-    options
+    options,
+    require('./cpu-penalty-monitor').readState(record),
+    live !== null
   );
   const oldMemory = base.find((s) => s.startsWith('--memory='))?.slice(9);
   const result = resolveResourceOptions(

package/src/lib/status-formatter.js

@@ -500,7 +500,8 @@ function attachMemoryExhaustion(record, logTail) {
   if (!memory) {
     if (
       record.memoryExhaustedReason?.startsWith('Docker reported') ||
-      record.memoryExhaustedReason?.startsWith('cgroup memory.events')
+      record.memoryExhaustedReason?.startsWith('cgroup memory.events') ||
+      record.memoryExhaustedReason?.startsWith('Exit-time evidence')
     ) {
       const cleared = Object.assign(
         Object.create(Object.getPrototypeOf(record)),
@@ -558,6 +559,14 @@ function enrichDetachedStatus(record) {
     Object.create(Object.getPrototypeOf(enriched)),
     enriched
   );
+  if (record.attemptHistory) {
+    copy.attemptHistory = require('./execution-attempt').enrichAttemptHistory(
+      record,
+      (attempt, tail) =>
+        attachMemoryExhaustion(attachExitReason(attempt, tail), tail),
+      FATAL_MARKER_TAIL_BYTES
+    );
+  }
   return readAttemptActivity(copy);
 }
 

package/src/lib/store-lock.js

@@ -7,11 +7,13 @@ const LOCK_STALE_MS = 60000;
 const MALFORMED_LOCK_GRACE_MS = 3000;
 
 class LockManager {
-  constructor(lockFilePath, filesystem = fs) {
+  constructor(lockFilePath, filesystem = fs, monitorOwner = false) {
     this.lockFilePath = lockFilePath;
     this.fs = filesystem;
     this.lockAcquired = false;
     this.identity = null;
+    this.token = null;
+    this.monitorOwner = monitorOwner;
   }
 
   acquire(timeout = LOCK_TIMEOUT_MS) {
@@ -22,7 +24,8 @@ class LockManager {
       }
     };
     while (Date.now() - started < timeout) {
-      const temp = `${this.lockFilePath}.${process.pid}.${crypto.randomUUID()}`;
+      const token = crypto.randomUUID();
+      const temp = `${this.lockFilePath}.${process.pid}.${token}`;
       try {
         try {
           const stat = this.fs.statSync(this.lockFilePath);
@@ -44,6 +47,8 @@ class LockManager {
             pid: process.pid,
             timestamp: Date.now(),
             hostname: os.hostname(),
+            processStart: this.processInfo(process.pid)?.start,
+            token,
           }),
           { flag: 'wx' }
         );
@@ -56,6 +61,7 @@ class LockManager {
         const identity = this.fs.statSync(temp);
         this.fs.linkSync(temp, this.lockFilePath);
         this.identity = identity;
+        this.token = token;
         this.lockAcquired = true;
         debug(`acquired after ${Date.now() - started}ms`);
         return true;
@@ -84,7 +90,8 @@ class LockManager {
       const current = this.fs.statSync(this.lockFilePath);
       if (
         current.dev === this.identity.dev &&
-        current.ino === this.identity.ino
+        current.ino === this.identity.ino &&
+        this.readLockFile()?.token === this.token
       ) {
         this.fs.unlinkSync(this.lockFilePath);
       }
@@ -92,6 +99,7 @@ class LockManager {
       /* The lock may already be gone. */
     }
     this.lockAcquired = false;
+    this.token = null;
   }
 
   readLockFile() {
@@ -117,6 +125,32 @@ class LockManager {
     if (data.hostname === os.hostname()) {
       try {
         process.kill(data.pid, 0);
+        const info = this.processInfo(data.pid);
+        if (
+          info &&
+          (['Z', 'X', 'x'].includes(info.state) ||
+            (data.processStart && data.processStart !== info.start))
+        ) {
+          return true;
+        }
+        if (
+          this.monitorOwner &&
+          !data.processStart &&
+          process.platform === 'linux'
+        ) {
+          try {
+            const command = this.fs.readFileSync(
+              `/proc/${data.pid}/cmdline`,
+              'utf8'
+            );
+            return (
+              !command.includes('cpu-penalty-monitor') &&
+              !command.includes('__start-cpu-monitor')
+            );
+          } catch {
+            /* Proc may be restricted; do not evict an unverified live owner. */
+          }
+        }
         return false;
       } catch (error) {
         return error.code === 'ESRCH';
@@ -125,6 +159,23 @@ class LockManager {
     return Date.now() - data.timestamp > LOCK_STALE_MS;
   }
 
+  processInfo(pid) {
+    if (process.platform !== 'linux') {
+      return null;
+    }
+    try {
+      const stat = this.fs.readFileSync(`/proc/${pid}/stat`, 'utf8');
+      // comm is parenthesized and can contain spaces and closing parentheses.
+      const fields = stat
+        .slice(stat.lastIndexOf(')') + 2)
+        .trim()
+        .split(/\s+/);
+      return { state: fields[0], start: fields[19] };
+    } catch {
+      return null;
+    }
+  }
+
   sleep(ms) {
     Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
   }

package/test/regression-191.js

@@ -97,15 +97,32 @@ it('manual snapshot override applies limits to the new container at creation', a
     fs.rmSync(f.dir, { recursive: true, force: true });
   }
 });
-for (const [exitCode, oomKilled, sample, change] of [
+for (const [exitCode, oomKilled, sample, change, marker] of [
   [137, true, '67108864 66000000 1 1 100 100', true],
   [137, true, '67108864 66000000 1 1 0 100', false],
   [137, false, '67108864 66000000 0 0 0 100', false],
   [1, true, '67108864 66000000 1 1 100 100', false],
+  [
+    137,
+    true,
+    '67108864 66000000 1 1 62 62',
+    true,
+    'Exit evidence: main-oom (recent Docker oom event)',
+  ],
+  [
+    137,
+    true,
+    '67108864 66000000 1 1 62 62',
+    false,
+    'Exit evidence: docker-daemon-restart (Docker service journal)',
+  ],
 ]) {
   it(`automatic memory change requires a qualifying OOM (exit=${exitCode}, fresh=${change})`, () => {
     const f = fixture();
     try {
+      if (marker) {
+        fs.writeFileSync(f.record.logPath, `${marker}\n`);
+      }
       const result = recoverKilledExecution({
         store: f.store,
         executionId: f.record.uuid,

package/test/regression-207.js

@@ -0,0 +1,249 @@
+const { it, expect } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { initialState, transitionLog } = require('../src/lib/cpu-penalty');
+const cpu = require('../src/lib/cpu-penalty-monitor');
+
+it('restarted monitors retain an observed base absent from launch flags', () => {
+  const saved = { ...initialState(4, 0), phase: 'penalized', limitCpus: 2 };
+  expect(initialState(null, 100, saved).baseCpus).toBe(4);
+});
+
+it('saved observing limits preserve the base after a daemon-clamped lift', () => {
+  const saved = { ...initialState(4, 0), limitCpus: 1 };
+  const { observeBase } = require('../src/lib/cpu-penalty');
+  expect(observeBase(initialState(null, 100, saved), 1, 100).baseCpus).toBe(4);
+});
+
+for (const mode of [
+  'nano',
+  'quota',
+  'restart',
+  'zombie',
+  'reused',
+  'supervise',
+  'publish-fault',
+  'graceful',
+  'override',
+]) {
+  it(`CPU monitor recovers external base and idle release: ${mode}`, () => {
+    if (
+      process.platform === 'win32' ||
+      (['zombie', 'reused'].includes(mode) && process.platform !== 'linux')
+    ) {
+      return;
+    }
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cpu-monitor-207-'));
+    try {
+      const logPath = path.join(dir, 'task.log');
+      fs.writeFileSync(logPath, '');
+      const record = {
+        logPath,
+        options: {
+          sessionName: 'cpu-task',
+          isolated: 'docker',
+          cpuPenaltyConfig: {
+            cpus: 2,
+            trigger: 95,
+            release: 65,
+            triggerWindowMs: 400,
+            releaseWindowMs: 400,
+          },
+          baseResourceLimits: [],
+        },
+      };
+      const command = [
+        process.execPath,
+        require.resolve('../src/lib/cpu-penalty-monitor'),
+        '',
+        'cpu-task',
+        '1',
+        JSON.stringify(record),
+      ];
+      if (mode === 'supervise') {
+        const script = `${cpu.startSnippet(null, 'cpu-task', 1, { ...record.options, logPath })}; read done; ${cpu.stopSnippet()}`;
+        command.splice(0, command.length, 'sh', '-c', script);
+      }
+      const result = spawnSync(
+        'python3',
+        [
+          path.resolve(__dirname, '../../experiments/issue-207-monitor.py'),
+          mode,
+          logPath,
+          ...command,
+        ],
+        { encoding: 'utf8' }
+      );
+      expect(result.status, result.stderr).toBe(0);
+      const state = cpu.readState(record);
+      const expected = mode === 'override' ? 3 : 4;
+      expect(state.baseCpus).toBe(expected);
+      expect(state.phase).toBe('observing');
+      expect(state.limitCpus).toBe(expected);
+      if (mode !== 'override') {
+        expect(fs.readFileSync(logPath, 'utf8')).toContain(
+          'CPU penalty lifted: 4'
+        );
+      }
+      const { ExecutionRecord } = require('../src/lib/execution-store');
+      const {
+        enrichDetachedStatus,
+        formatRecord,
+      } = require('../src/lib/status-formatter');
+      const statusRecord = new ExecutionRecord({
+        ...record,
+        options: { ...record.options, isolated: undefined },
+      });
+      const status = JSON.parse(
+        formatRecord(enrichDetachedStatus(statusRecord), 'json')
+      );
+      expect(status.cpuPenalty.baseCpus).toBe(expected);
+      expect(status.cpuPenalty.limitCpus).toBe(expected);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  }, 15000);
+}
+
+for (const failure of ['inspect', 'stats', 'update', 'publish']) {
+  it(`an exception in ${failure} resets coverage and monitoring continues`, async () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cpu-tick-207-'));
+    const logPath = path.join(dir, 'task.log');
+    const record = {
+      logPath,
+      options: {
+        sessionName: 'cpu-task',
+        cpuPenaltyConfig: {
+          cpus: 2,
+          trigger: 95,
+          release: 65,
+          triggerWindowMs: 100,
+          releaseWindowMs: 100,
+        },
+      },
+    };
+    let ticks = 0;
+    let injected = false,
+      limit = 4,
+      lifted = false;
+    const filesystem = Object.create(fs);
+    filesystem.renameSync = (from, to) => {
+      if (
+        failure === 'publish' &&
+        !injected &&
+        JSON.parse(fs.readFileSync(from)).phase === 'penalized'
+      ) {
+        injected = true;
+        throw new Error('injected ENOSPC');
+      }
+      fs.renameSync(from, to);
+    };
+    try {
+      await cpu.monitor('', 'cpu-task', 1, JSON.stringify(record), {
+        fs: filesystem,
+        run: (_command, args) => {
+          if (args[0] === failure && !injected) {
+            injected = true;
+            throw new Error(`injected ${failure}`);
+          }
+          const result = (stdout) => ({ success: true, stdout });
+          if (args[0] === 'inspect') {
+            if (!args[2].includes('HostConfig') && ++ticks > 60) {
+              lifted = false;
+              return result('false');
+            }
+            return result(
+              args[2].includes('HostConfig')
+                ? JSON.stringify({ NanoCpus: limit * 1e9 })
+                : String(!lifted)
+            );
+          }
+          if (args[0] === 'info') {
+            return result('{"NCPU":6}');
+          }
+          if (args[0] === 'stats') {
+            return result(
+              JSON.stringify({
+                Name: 'cpu-task',
+                CPUPerc: limit === 2 ? '20%' : '400%',
+              })
+            );
+          }
+          if (args[0] === 'update') {
+            limit = Number(args[2]);
+            lifted = limit === 4;
+            return result('');
+          }
+          throw new Error('unexpected command');
+        },
+      });
+      expect(injected).toBe(true);
+      expect(lifted).toBe(true);
+      expect(cpu.readState(record).baseCpus).toBe(4);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  }, 5000);
+}
+
+it('stopped resume uses the observed base rather than the temporary quota', () => {
+  const { restoreBaseCpu } = require('../src/lib/resume-resources');
+  expect(
+    restoreBaseCpu(
+      ['--memory=512m', '--cpus=2'],
+      { cpuPenaltyConfig: {}, baseResourceLimits: [] },
+      { baseCpus: 4 }
+    )
+  ).toEqual(['--memory=512m', '--cpus=4']);
+});
+
+it('stopped resume preserves a later supervisor override instead of the stale cap', () => {
+  const { restoreBaseCpu } = require('../src/lib/resume-resources');
+  const options = { cpuPenaltyConfig: {}, baseResourceLimits: [] };
+  const saved = { phase: 'penalized', baseCpus: 4, limitCpus: 2 };
+  expect(restoreBaseCpu(['--cpus=2'], options, saved, true)).toEqual([
+    '--cpus=4',
+  ]);
+  expect(restoreBaseCpu(['--cpus=3'], options, saved, true)).toEqual([
+    '--cpus=3',
+  ]);
+});
+
+it('transition logs explain the window and phase denominator', () => {
+  const config = { cpus: 2, triggerWindowMs: 900000, releaseWindowMs: 400 };
+  expect(
+    transitionLog({ kind: 'apply', cpus: 2, capacity: 4, average: 100 }, config)
+  ).toContain('of 4 CPUs for 15m');
+  expect(
+    transitionLog({ kind: 'lift', cpus: 4, capacity: 4, average: 10 }, config)
+  ).toContain('of 2 CPUs for 400ms');
+});
+
+it('process birth identity reclaims reused PIDs without evicting a live lock', () => {
+  if (process.platform !== 'linux') {
+    return;
+  }
+  const { LockManager } = require('../src/lib/store-lock');
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cpu-lock-207-'));
+  const file = path.join(dir, 'lock');
+  const first = new LockManager(file);
+  const replacement = new LockManager(file);
+  try {
+    expect(first.acquire(100)).toBe(true);
+    const data = JSON.parse(fs.readFileSync(file));
+    expect(data.processStart).toBeTruthy();
+    expect(replacement.acquire(100)).toBe(false);
+    fs.writeFileSync(
+      file,
+      JSON.stringify({ ...data, processStart: 'different-birth' })
+    );
+    expect(replacement.acquire(100)).toBe(true);
+    first.release();
+    expect(fs.existsSync(file)).toBe(true);
+  } finally {
+    replacement.release();
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});