start-command 0.37.0 → 0.37.2

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # start-command
 
+## 0.37.2
+
+### Patch Changes
+
+- 65314a3: Attribute exit 137 to recent Docker OOM events when the last cgroup sample misses the final kill. Preserve daemon restart and ordinary-exit safeguards, apply attributed evidence to recovery memory limits, and recompute each archived attempt's status diagnostics independently.
+
+## 0.37.1
+
+### Patch Changes
+
+- c67c92f: Observe Docker CPU limits applied after launch, restore the remembered base on lift and resume, and recover CPU penalty monitors after crashes, zombie/reused-PID locks and transient tick failures.
+
 ## 0.37.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.37.0",
+  "version": "0.37.2",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/lib/cpu-penalty-monitor.js

@@ -2,10 +2,20 @@
 const fs = require('fs');
 const path = require('path');
 const { spawnSync } = require('child_process');
-const { initialState, evaluate } = require('./cpu-penalty');
+const {
+  initialState,
+  evaluate,
+  observeBase,
+  transitionLog,
+} = require('./cpu-penalty');
 const { LockManager } = require('./store-lock');
 const { shellQuote } = require('./isolation-log-utils');
-const { cpuCount, cpuUpdateArgs } = require('./docker-resource-limits');
+const {
+  cpuCount,
+  cpuUpdateArgs,
+  normalizeResourceLimits,
+  readDockerResourceLimits,
+} = require('./docker-resource-limits');
 function statePath(record) {
   return `${record.logPath}.cpu-penalty.json`;
 }
@@ -40,9 +50,21 @@ function publicState(state) {
 }
 function run(command, args) {
   const result = spawnSync(command, args, { encoding: 'utf8' });
-  return { success: result.status === 0, stdout: result.stdout || '' };
+  return {
+    success: result.status === 0,
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+  };
 }
-async function monitor(uuid, name, attemptNumber, standalone) {
+async function monitor(
+  uuid,
+  name,
+  attemptNumber,
+  standalone,
+  dependencies = {}
+) {
+  const filesystem = dependencies.fs || fs;
+  const runner = dependencies.run || run;
   const { ExecutionStore } = require('./execution-store');
   const store = uuid
     ? new ExecutionStore(
@@ -60,36 +82,40 @@ async function monitor(uuid, name, attemptNumber, standalone) {
     return;
   }
   const config = record.options.cpuPenaltyConfig;
-  const lock = new LockManager(`${statePath(record)}.lock`);
+  if ((record.attempt?.number || 1) !== Number(attemptNumber)) {
+    return;
+  }
+  const lock = new LockManager(`${statePath(record)}.lock`, filesystem, true);
   if (!lock.acquire(100)) {
     return;
   }
   const docker = require('./docker-cleanup').getDockerCommand();
-  const baseFlags =
-    record.options.baseResourceLimits || record.options.resourceLimits || [];
+  let baseFlags = normalizeResourceLimits(
+    record.options.baseResourceLimits || record.options.resourceLimits
+  );
   const baseCpus = cpuCount(baseFlags);
   let state = initialState(baseCpus, Date.now(), readState(record));
   const interval = Math.min(
     30000,
     Math.max(50, Math.min(config.triggerWindowMs, config.releaseWindowMs) / 8)
   );
-  const publish = () => {
+  const publish = (value = state) => {
     const file = statePath(record),
       tmp = `${file}.${process.pid}.tmp`;
     try {
-      fs.writeFileSync(
+      filesystem.writeFileSync(
         tmp,
         JSON.stringify({
-          ...publicState(state),
+          ...publicState(value),
           containerName: name,
           attemptNumber: Number(attemptNumber),
           updatedAt: Date.now(),
         })
       );
-      fs.renameSync(tmp, file);
+      filesystem.renameSync(tmp, file);
     } finally {
       try {
-        fs.unlinkSync(tmp);
+        filesystem.unlinkSync(tmp);
       } catch {
         /* Optional diagnostics are best effort. */
       }
@@ -97,95 +123,188 @@ async function monitor(uuid, name, attemptNumber, standalone) {
   };
   const log = (text) => {
     try {
-      fs.appendFileSync(record.logPath, `${text}\n`);
+      filesystem.appendFileSync(record.logPath, `${text}\n`);
     } catch {
       /* Optional diagnostics are best effort. */
     }
   };
-  const started = Date.now();
+  const debug = (error) => {
+    if (process.env.START_DEBUG === '1') {
+      const message = `[DEBUG] CPU monitor: ${error.stack || error}`;
+      console.error(message);
+      log(message);
+    }
+  };
+  let stopping = false,
+    wake;
+  const stop = () => {
+    stopping = true;
+    wake?.();
+  };
+  process.on('SIGTERM', stop);
+  process.on('SIGINT', stop);
   try {
-    publish();
-    while (true) {
-      const current = store ? store.get(uuid) : record;
-      if (
-        !current ||
-        current.options.sessionName !== name ||
-        (current.attempt?.number || 1) !== Number(attemptNumber)
-      ) {
-        break;
-      }
-      const live = run(docker, ['inspect', '-f', '{{.State.Running}}', name]);
-      if (live.success && live.stdout.trim() === 'false') {
-        break;
-      }
-      if (!live.success && Date.now() - started > 30000) {
-        break;
-      }
-      const info = run(docker, ['info', '--format', '{{json .}}']);
-      const stats = run(docker, [
-        'stats',
-        '--no-stream',
-        '--format',
-        '{{json .}}',
-      ]);
-      let daemonCpus = NaN,
-        cores = NaN;
+    while (!stopping) {
       try {
-        if (info.success) {
-          daemonCpus = JSON.parse(info.stdout).NCPU;
+        const current = store ? store.get(uuid) : record;
+        if (
+          !current ||
+          current.options.sessionName !== name ||
+          (current.attempt?.number || 1) !== Number(attemptNumber)
+        ) {
+          break;
+        }
+        const live = runner(docker, [
+          'inspect',
+          '-f',
+          '{{.State.Running}}',
+          name,
+        ]);
+        if (live.success && live.stdout.trim() === 'false') {
+          break;
+        }
+        if (
+          !live.success &&
+          /No such (object|container)/i.test(live.stderr || '')
+        ) {
+          break;
+        }
+        const observed = readDockerResourceLimits(name, runner);
+        if (observed !== null) {
+          baseFlags = observed;
+          state = observeBase(state, cpuCount(observed), Date.now());
         }
-        if (stats.success) {
-          const row = stats.stdout
-            .trim()
-            .split('\n')
-            .map((s) => JSON.parse(s))
-            .find((s) => s.Name === name);
-          if (row && /^\d+(?:\.\d+)?%$/.test(row.CPUPerc)) {
-            cores = Number(row.CPUPerc.slice(0, -1)) / 100;
+        const info = runner(docker, ['info', '--format', '{{json .}}']);
+        const stats = runner(docker, [
+          'stats',
+          '--no-stream',
+          '--format',
+          '{{json .}}',
+        ]);
+        let daemonCpus = NaN,
+          cores = NaN;
+        try {
+          if (info.success) {
+            daemonCpus = JSON.parse(info.stdout).NCPU;
+          }
+          if (stats.success && live.success && observed !== null) {
+            const row = stats.stdout
+              .trim()
+              .split('\n')
+              .map((s) => JSON.parse(s))
+              .find((s) => s.Name === name);
+            if (row && /^\d+(?:\.\d+)?%$/.test(row.CPUPerc)) {
+              cores = Number(row.CPUPerc.slice(0, -1)) / 100;
+            }
           }
+        } catch (error) {
+          debug(error);
         }
-      } catch {
-        /* Optional diagnostics are best effort. */
+        debug(
+          `sample: phase=${state.phase} base=${state.baseCpus} limit=${state.limitCpus} cores=${cores} daemon=${daemonCpus}`
+        );
+        const decision = evaluate(
+          state,
+          {
+            now: Date.now(),
+            cores,
+            daemonCpus,
+            maxGapMs: Math.max(5000, interval * 3),
+          },
+          config
+        );
+        if (decision.action) {
+          const action = decision.action;
+          // Persist the base and intent before applying a cap. If this fails,
+          // leave Docker alone; a killed monitor can always recover its base.
+          if (action.kind === 'apply') {
+            publish(decision.state);
+          }
+          const update = runner(docker, [
+            'update',
+            ...cpuUpdateArgs(baseFlags, action.cpus),
+            name,
+          ]);
+          if (update.success) {
+            state = decision.state;
+            log(transitionLog(action, config));
+          } else {
+            debug(update.stderr || 'docker update failed');
+            state = {
+              ...decision.state,
+              phase: state.phase,
+              since: state.since,
+              limitCpus: state.limitCpus,
+              penaltyCount: state.penaltyCount,
+              samples: [],
+            };
+          }
+        } else {
+          state = decision.state;
+        }
+        publish();
+      } catch (error) {
+        state.samples = [];
+        debug(`tick failed: ${error.stack || error}`);
       }
-      const decision = evaluate(
-        state,
-        {
-          now: Date.now(),
-          cores,
-          daemonCpus,
-          maxGapMs: Math.max(5000, interval * 3),
-        },
-        config
-      );
-      if (decision.action) {
-        const action = decision.action;
-        const update = run(docker, [
-          'update',
-          ...cpuUpdateArgs(baseFlags, action.cpus),
+      await new Promise((resolve) => {
+        const timer = globalThis.setTimeout(resolve, interval);
+        wake = () => {
+          globalThis.clearTimeout(timer);
+          resolve();
+        };
+        if (stopping) {
+          wake();
+        }
+      });
+    }
+  } finally {
+    try {
+      const current = store ? store.get(uuid) : record;
+      if (
+        state.phase === 'penalized' &&
+        current?.options.sessionName === name &&
+        (current.attempt?.number || 1) === Number(attemptNumber)
+      ) {
+        const live = runner(docker, [
+          'inspect',
+          '-f',
+          '{{.State.Running}}',
           name,
         ]);
-        if (update.success) {
-          state = decision.state;
-          log(
-            `[start] CPU penalty ${action.kind === 'apply' ? 'applied' : 'lifted'}: ${action.cpus} CPUs (${action.average === null || action.average === undefined ? 'capacity changed' : `${action.average.toFixed(2)} of ${action.capacity} CPUs`})`
-          );
-        } else {
-          state = {
-            ...decision.state,
-            phase: state.phase,
-            since: state.since,
-            limitCpus: state.limitCpus,
-            penaltyCount: state.penaltyCount,
-            samples: [],
-          };
+        const observed = readDockerResourceLimits(name, runner);
+        const info = runner(docker, ['info', '--format', '{{json .}}']);
+        const capacity = info.success ? JSON.parse(info.stdout).NCPU : NaN;
+        if (
+          live.success &&
+          live.stdout.trim() === 'true' &&
+          observed !== null &&
+          cpuCount(observed) === state.limitCpus &&
+          capacity > 0
+        ) {
+          const cpus = Math.min(state.baseCpus || capacity, capacity);
+          const update = runner(docker, [
+            'update',
+            ...cpuUpdateArgs(observed, cpus),
+            name,
+          ]);
+          if (update.success) {
+            state = {
+              ...state,
+              phase: 'observing',
+              limitCpus: cpus,
+              since: Date.now(),
+            };
+            log(`[start] CPU penalty lifted: ${cpus} CPUs (monitor stopped)`);
+            publish();
+          }
         }
-      } else {
-        state = decision.state;
       }
-      publish();
-      await new Promise((resolve) => globalThis.setTimeout(resolve, interval));
+    } catch (error) {
+      debug(error);
     }
-  } finally {
+    process.off('SIGTERM', stop);
+    process.off('SIGINT', stop);
     lock.release();
   }
 }
@@ -204,7 +323,9 @@ function startSnippet(uuid, name, attempt = 1, options = null) {
   if (!uuid && !standalone) {
     return '';
   }
-  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(uuid || '')} ${shellQuote(name)} ${Number(attempt)} ${shellQuote(standalone || '')} >/dev/null 2>&1 & __start_cpu_monitor=$!`;
+  const command = `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(uuid || '')} ${shellQuote(name)} ${Number(attempt)} ${shellQuote(standalone || '')}`;
+  // wait reaps a crashed child even in containers whose PID 1 never reaps.
+  return `( __start_cpu_child=; trap 'if [ -n "$__start_cpu_child" ]; then kill "$__start_cpu_child" 2>/dev/null; wait "$__start_cpu_child" 2>/dev/null; fi; exit 0' TERM INT HUP; while :; do ${command} >/dev/null 2>&1 & __start_cpu_child=$!; wait "$__start_cpu_child"; __start_cpu_code=$?; [ "$__start_cpu_code" -eq 0 ] && break; sleep 1 & __start_cpu_child=$!; wait "$__start_cpu_child"; done ) & __start_cpu_monitor=$!`;
 }
 function stopSnippet() {
   return 'if [ -n "$__start_cpu_monitor" ]; then kill "$__start_cpu_monitor" 2>/dev/null; wait "$__start_cpu_monitor" 2>/dev/null; fi';
@@ -219,6 +340,7 @@ module.exports = {
 };
 if (require.main === module) {
   monitor(...process.argv.slice(2)).catch((error) => {
+    process.exitCode = 1;
     if (process.env.START_DEBUG === '1') {
       console.error(`[DEBUG] CPU monitor: ${error.stack}`);
     }

package/src/lib/cpu-penalty.js

@@ -70,10 +70,13 @@ function validate(options) {
   options.cpuPenaltyConfig = { ...DEFAULTS, ...options.cpuPenaltyConfig };
 }
 function initialState(baseCpus = null, now = Date.now(), saved = null) {
+  baseCpus =
+    saved && Object.hasOwn(saved, 'baseCpus') ? saved.baseCpus : baseCpus;
   return {
     phase: saved?.phase === 'penalized' ? 'penalized' : 'observing',
     since: saved?.phase === 'penalized' ? saved.since : now,
-    limitCpus: saved?.phase === 'penalized' ? saved.limitCpus : baseCpus,
+    limitCpus:
+      saved && Object.hasOwn(saved, 'limitCpus') ? saved.limitCpus : baseCpus,
     baseCpus,
     penaltyCount: saved?.penaltyCount || 0,
     penalizedMs: saved?.penalizedMs || 0,
@@ -82,6 +85,30 @@ function initialState(baseCpus = null, now = Date.now(), saved = null) {
     capacity: null,
   };
 }
+// A quota different from our cap is a supervisor override, not our penalty.
+function observeBase(state, baseCpus, now) {
+  if (baseCpus === state.limitCpus) {
+    return state;
+  }
+  return {
+    ...state,
+    phase: 'observing',
+    baseCpus,
+    limitCpus: baseCpus,
+    since: now,
+    samples: [],
+  };
+}
+function transitionLog(action, config) {
+  const applying = action.kind === 'apply';
+  const window = applying ? config.triggerWindowMs : config.releaseWindowMs;
+  const duration = window % 60000 === 0 ? `${window / 60000}m` : `${window}ms`;
+  const measured =
+    action.average === null || action.average === undefined
+      ? 'capacity changed'
+      : `${action.average.toFixed(2)} of ${applying ? action.capacity : config.cpus} CPUs for ${duration}`;
+  return `[start] CPU penalty ${applying ? 'applied' : 'lifted'}: ${action.cpus} CPUs (${measured})`;
+}
 function average(samples, now, window) {
   const start = now - window;
   if (
@@ -170,6 +197,8 @@ module.exports = {
   validate,
   duration,
   initialState,
+  observeBase,
+  transitionLog,
   average,
   evaluate,
 };

package/src/lib/execution-attempt.js

@@ -90,6 +90,9 @@ function archiveAttempt(record) {
     status: record.status,
     exitCode: record.exitCode,
     endTime: record.endTime,
+    ...(record.options?.exitEvidence
+      ? { exitEvidence: { ...record.options.exitEvidence } }
+      : {}),
   };
   for (const field of TERMINAL_FIELDS) {
     if (record[field] !== undefined) {
@@ -142,7 +145,31 @@ function readAttemptLogTail(record, bytes) {
   return readLogTail(record.logPath, bytes, record.attempt?.logOffset || 0);
 }
 
+function enrichAttemptHistory(record, enrich, bytes) {
+  return (record.attemptHistory || []).map((attempt, index, history) => {
+    const next = history[index + 1];
+    const end = next ? next.logOffset : record.attempt?.logOffset;
+    const bounded =
+      Number.isSafeInteger(attempt.logOffset) &&
+      attempt.logOffset >= 0 &&
+      Number.isSafeInteger(end) &&
+      end >= attempt.logOffset;
+    const tail =
+      record.logPath && bounded
+        ? readLogTail(record.logPath, bytes, attempt.logOffset, end)
+        : null;
+    const input = {
+      ...attempt,
+      options: { exitEvidence: attempt.exitEvidence },
+    };
+    const output = enrich(input, tail);
+    delete output.options;
+    return output;
+  });
+}
+
 module.exports = {
+  enrichAttemptHistory,
   activityPath,
   appendLifecycle,
   archiveAttempt,

package/src/lib/execution-recovery.js

@@ -323,14 +323,20 @@ function recoverKilledExecution(params = {}) {
     }
   }
   const evidence = require('./exit-evidence');
+  const logged = evidence.fromLog(
+    record.logPath
+      ? require('./execution-attempt').readAttemptLogTail(record, 65536)
+      : null
+  );
   const terminalOom =
-    (params.oomKilled === true || params.oomKilled === 'true') &&
-    evidence.recentOomDelta(params.cgroupMemory, params.finishedAt) &&
-    !evidence.fromLog(
-      record.logPath
-        ? require('./execution-attempt').readAttemptLogTail(record, 65536)
-        : null
-    ).daemonRestart;
+    !logged.daemonRestart &&
+    (logged.mainOom ||
+      ((params.oomKilled === true || params.oomKilled === 'true') &&
+        evidence.recentOomDelta(params.cgroupMemory, params.finishedAt)));
+  record.options.exitEvidence = {
+    mainOom: terminalOom,
+    daemonRestart: logged.daemonRestart,
+  };
   let recoveryLimits = null;
   if (opts.onKillResumeMemory && terminalOom) {
     try {

package/src/lib/exit-evidence.js

@@ -2,6 +2,7 @@
 const { spawnSync } = require('child_process');
 const { shellQuote } = require('./isolation-log-utils');
 const MAIN_OOM = 'Exit evidence: main-oom (recent cgroup oom_kill delta)';
+const DOCKER_OOM = 'Exit evidence: main-oom (recent Docker oom event)';
 const DAEMON_RESTART =
   'Exit evidence: docker-daemon-restart (Docker service journal)';
 
@@ -38,6 +39,22 @@ function daemonRestartEvidence(journal, containerId) {
   return Boolean(exited && forced);
 }
 
+function recentDockerOom(events, finishedAt) {
+  const finished = Date.parse(finishedAt) / 1000;
+  return (
+    Number.isFinite(finished) &&
+    String(events || '')
+      .split('\n')
+      .some((line) => {
+        if (!/^\d+$/.test(line.trim())) {
+          return false;
+        }
+        const at = Number(line.trim());
+        return at > 0 && finished - at >= -1 && finished - at <= 3;
+      })
+  );
+}
+
 function collectExitEvidence(
   { containerName, sample, finishedAt, exitCode, oomKilled },
   runner
@@ -52,31 +69,56 @@ function collectExitEvidence(
   const local =
     !process.env.DOCKER_HOST || process.env.DOCKER_HOST.startsWith('unix://');
   if (
-    local &&
     Number(exitCode) === 137 &&
+    containerName &&
     Number.isFinite(Date.parse(finishedAt))
   ) {
     const docker = require('./docker-cleanup').getDockerCommand();
     const id = run(docker, ['inspect', '-f', '{{.Id}}', containerName]);
     if (id.success && /^[a-f0-9]{64}$/.test(id.stdout.trim())) {
       const end = Date.parse(finishedAt);
-      const journal = run('journalctl', [
-        '-u',
-        'docker.service',
-        '--since',
-        new Date(end - 60000).toISOString(),
-        '--until',
-        new Date(end + 1000).toISOString(),
-        '--no-pager',
-        '-o',
-        'cat',
-      ]);
+      const journal = local
+        ? run('journalctl', [
+            '-u',
+            'docker.service',
+            '--since',
+            new Date(end - 60000).toISOString(),
+            '--until',
+            new Date(end + 1000).toISOString(),
+            '--no-pager',
+            '-o',
+            'cat',
+          ])
+        : { success: false, stdout: '' };
       if (
         journal.success &&
         daemonRestartEvidence(journal.stdout, id.stdout.trim())
       ) {
         return DAEMON_RESTART;
       }
+      // Query the daemon's bounded event history: works with DooD and remote
+      // sockets even after the final kill destroys the sampled cgroup.
+      const events = run(docker, [
+        'events',
+        '--since',
+        new Date(end - 60000).toISOString(),
+        '--until',
+        new Date(end + 1000).toISOString(),
+        '--filter',
+        `container=${id.stdout.trim()}`,
+        '--filter',
+        'event=oom',
+        '--format',
+        '{{.Time}}',
+      ]);
+      if (events.success && recentDockerOom(events.stdout, finishedAt)) {
+        return DOCKER_OOM;
+      }
+      if (process.env.START_DEBUG === '1') {
+        console.error(
+          `[DEBUG] Exit evidence: container=${id.stdout.trim()} finishedAt=${finishedAt} eventsAvailable=${events.success} oomEvents=${JSON.stringify(events.stdout)} sample=${JSON.stringify(sample || '')}`
+        );
+      }
     }
   }
   if (
@@ -92,17 +134,23 @@ function collectExitEvidence(
 function fromLog(log) {
   const lines = String(log || '').split('\n');
   const last = lines.filter((line) => line.startsWith('Exit evidence:')).at(-1);
-  return { mainOom: last === MAIN_OOM, daemonRestart: last === DAEMON_RESTART };
+  return {
+    mainOom: last === MAIN_OOM || last === DOCKER_OOM,
+    daemonRestart: last === DAEMON_RESTART,
+  };
 }
 
 function buildExitEvidenceSnippet(containerName, quotedLogPath) {
-  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} 2>/dev/null`;
+  const stderr = process.env.START_DEBUG === '1' ? '2>&1' : '2>/dev/null';
+  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} ${stderr}`;
 }
 
 module.exports = {
   MAIN_OOM,
+  DOCKER_OOM,
   DAEMON_RESTART,
   recentOomDelta,
+  recentDockerOom,
   daemonRestartEvidence,
   collectExitEvidence,
   fromLog,

package/src/lib/exit-reason.js

@@ -206,6 +206,13 @@ function resolveMemoryExhaustion(input) {
       memoryExhaustedReason: `cgroup memory.events reported oom_kill=${oomKills}`,
     };
   }
+  if (isOomKillOfCommand(input)) {
+    return {
+      memoryExhausted: true,
+      memoryExhaustedReason:
+        'Exit-time evidence reported a main-process OOM kill',
+    };
+  }
   return null;
 }