start-command 0.35.4 → 0.36.0

package/src/lib/cpu-penalty.js

@@ -0,0 +1,175 @@
+/** Docker CPU hysteresis with timestamp-weighted, fully covered windows. */
+const DEFAULTS = {
+  cpus: 2,
+  trigger: 95,
+  triggerWindowMs: 900000,
+  release: 65,
+  releaseWindowMs: 900000,
+};
+const FIELDS = {
+  '--cpu-penalty-cpus': 'cpus',
+  '--cpu-penalty-trigger': 'trigger',
+  '--cpu-penalty-trigger-window': 'triggerWindowMs',
+  '--cpu-penalty-release': 'release',
+  '--cpu-penalty-release-window': 'releaseWindowMs',
+};
+function duration(value) {
+  const m = /^(\d+(?:\.\d+)?)(ms|s|m|h)$/.exec(value);
+  const n = m
+    ? Number(m[1]) * { ms: 1, s: 1000, m: 60000, h: 3600000 }[m[2]]
+    : NaN;
+  if (!Number.isSafeInteger(n) || n <= 0) {
+    throw Error(`Invalid CPU penalty duration: ${value}`);
+  }
+  return n;
+}
+function parse(args, index, options) {
+  const [flag, inline] = args[index].split('=', 2);
+  if (flag === '--cpu-penalty') {
+    if (inline !== undefined) {
+      throw Error('--cpu-penalty does not take a value');
+    }
+    options.cpuPenalty = true;
+    return 1;
+  }
+  const field = FIELDS[flag];
+  if (!field) {
+    return 0;
+  }
+  const value = inline ?? args[index + 1];
+  if (!value) {
+    throw Error(`${flag} requires a value`);
+  }
+  const number = field.endsWith('Ms')
+    ? duration(value)
+    : Number(field === 'cpus' ? value : value.replace(/%$/, ''));
+  if (
+    !(number > 0) ||
+    !Number.isFinite(number) ||
+    (field !== 'cpus' && !field.endsWith('Ms') && number > 100)
+  ) {
+    throw Error(`Invalid ${flag}: ${value}`);
+  }
+  options.cpuPenaltyConfig = {
+    ...DEFAULTS,
+    ...options.cpuPenaltyConfig,
+    [field]: number,
+  };
+  return inline === undefined ? 2 : 1;
+}
+function validate(options) {
+  if (!options.cpuPenalty && !options.cpuPenaltyConfig) {
+    return;
+  }
+  if (!options.cpuPenalty) {
+    throw Error('CPU penalty settings require --cpu-penalty');
+  }
+  if (!(options.isolatedStack || [options.isolated]).includes('docker')) {
+    throw Error('--cpu-penalty requires Docker isolation');
+  }
+  options.cpuPenaltyConfig = { ...DEFAULTS, ...options.cpuPenaltyConfig };
+}
+function initialState(baseCpus = null, now = Date.now(), saved = null) {
+  return {
+    phase: saved?.phase === 'penalized' ? 'penalized' : 'observing',
+    since: saved?.phase === 'penalized' ? saved.since : now,
+    limitCpus: saved?.phase === 'penalized' ? saved.limitCpus : baseCpus,
+    baseCpus,
+    penaltyCount: saved?.penaltyCount || 0,
+    penalizedMs: saved?.penalizedMs || 0,
+    samples: [],
+    lastAt: now,
+    capacity: null,
+  };
+}
+function average(samples, now, window) {
+  const start = now - window;
+  if (
+    samples.length < 2 ||
+    samples[0].at > start ||
+    samples.at(-1).at !== now
+  ) {
+    return null;
+  }
+  let weighted = 0;
+  for (let i = 1; i < samples.length; i++) {
+    const left = Math.max(start, samples[i - 1].at),
+      right = samples[i].at;
+    if (right > left) {
+      weighted += (right - left) * samples[i - 1].cores;
+    }
+  }
+  return weighted / window;
+}
+function evaluate(
+  state,
+  { now, cores, daemonCpus, maxGapMs },
+  config = DEFAULTS
+) {
+  const s = { ...state, samples: [...state.samples] };
+  if (s.phase === 'penalized') {
+    s.penalizedMs += Math.max(0, now - s.lastAt);
+  }
+  s.lastAt = now;
+  const capacity = Number.isFinite(daemonCpus)
+    ? Math.min(s.baseCpus || Infinity, daemonCpus)
+    : NaN;
+  if (
+    !(cores >= 0) ||
+    !Number.isFinite(cores) ||
+    !(capacity > 0) ||
+    now <= (s.samples.at(-1)?.at ?? -Infinity) ||
+    (s.samples.length && now - s.samples.at(-1).at > maxGapMs) ||
+    s.capacity !== capacity
+  ) {
+    s.samples = [];
+  }
+  s.capacity = capacity;
+  if (!(cores >= 0) || !Number.isFinite(cores) || !(capacity > 0)) {
+    return { state: s, action: null };
+  }
+  s.samples.push({ at: now, cores });
+  const window =
+    s.phase === 'penalized' ? config.releaseWindowMs : config.triggerWindowMs;
+  while (s.samples.length > 2 && s.samples[1].at <= now - window) {
+    s.samples.shift();
+  }
+  const avg = average(s.samples, now, window);
+  let action = null;
+  if (
+    s.phase === 'observing' &&
+    config.cpus < capacity &&
+    avg !== null &&
+    avg >= (config.trigger / 100) * capacity
+  ) {
+    action = { kind: 'apply', cpus: config.cpus, average: avg, capacity };
+    s.phase = 'penalized';
+    s.since = now;
+    s.limitCpus = config.cpus;
+    s.penaltyCount++;
+  } else if (
+    s.phase === 'penalized' &&
+    (config.cpus >= capacity ||
+      (now - s.since >= config.releaseWindowMs &&
+        avg !== null &&
+        avg < (config.release / 100) * config.cpus))
+  ) {
+    action = { kind: 'lift', cpus: capacity, average: avg, capacity };
+    s.phase = 'observing';
+    s.since = now;
+    s.limitCpus = capacity;
+  }
+  if (action) {
+    s.samples = [];
+  }
+  return { state: s, action };
+}
+module.exports = {
+  DEFAULTS,
+  parse,
+  validate,
+  duration,
+  initialState,
+  average,
+  evaluate,
+};

package/src/lib/detached-finalize.js

@@ -146,6 +146,21 @@ function finalizeDetachedExecution(options = {}) {
     record.cgroupMemory = cgroupMemory;
   }
 
+  const evidence = require('./exit-evidence');
+  const logged = evidence.fromLog(
+    record.logPath
+      ? require('./execution-attempt').readAttemptLogTail(record, 65536)
+      : null
+  );
+  record.options.exitEvidence = {
+    daemonRestart: logged.daemonRestart,
+    mainOom:
+      !logged.daemonRestart &&
+      (logged.mainOom ||
+        (evidence.recentOomDelta(options.cgroupMemory, options.finishedAt) &&
+          record.oomKilled === true &&
+          (record.exitCode === 137 || record.exitCode < 0))),
+  };
   const exitReason =
     resolveReason(record) || (finishedAt ? null : WATCHER_LOST_CONTAINER);
   if (exitReason) {
@@ -197,6 +212,7 @@ function resolveReason(record) {
       logTail,
       oomKilled: record.oomKilled,
       cgroupMemory: record.cgroupMemory,
+      exitEvidence: record.options?.exitEvidence,
     });
   } catch {
     return null;

package/src/lib/docker-cleanup.js

@@ -275,10 +275,13 @@ function buildAttachedDockerKeptMessage({
   oomKilled,
   logPath,
 }) {
+  const logTail = logPath
+    ? readLogTail(logPath, FATAL_MARKER_TAIL_BYTES)
+    : null;
   let message;
   if (oomKilled !== true) {
     message = `\nContainer kept because the command failed.`;
-  } else if (isOomKillOfCommand({ exitCode, oomKilled })) {
+  } else if (isOomKillOfCommand({ exitCode, oomKilled, logTail })) {
     message = `\nContainer kept because Docker reports it was OOM-killed.`;
   } else {
     // The flag is container-wide (#180): a child was OOM-killed, the command
@@ -287,7 +290,7 @@ function buildAttachedDockerKeptMessage({
   }
   const memory = resolveMemoryExhaustion({
     exitCode,
-    logTail: logPath ? readLogTail(logPath, FATAL_MARKER_TAIL_BYTES) : null,
+    logTail,
     oomKilled,
   });
   if (memory) {
@@ -360,7 +363,17 @@ function buildDetachedDockerCompletionScript(
   const quotedName = shellQuote(containerName);
   // The container's cgroup disappears with it, so its memory counters are
   // sampled while it runs (issue #182).
+  const cpuMonitor = require('./cpu-penalty-monitor');
   const parts = [buildCgroupSamplerStartSnippet(containerName)];
+  const cpuStart = cpuMonitor.startSnippet(
+    executionId,
+    containerName,
+    watcherOptions.attemptNumber || 1,
+    watcherOptions.cpuOptions
+  );
+  if (cpuStart) {
+    parts.push(cpuStart);
+  }
   // Everything that assumes the container has exited: cleanup, footer and
   // finalization. Guarded as a whole by `.State.Running` below.
   const exited = [];
@@ -393,8 +406,15 @@ function buildDetachedDockerCompletionScript(
       );
     }
     parts.push(buildDockerWaitForExitSnippet(containerName, quotedLogPath));
+    parts.push(cpuMonitor.stopSnippet());
     parts.push(buildCgroupSamplerStopSnippet());
     parts.push(buildDockerStateSnippet(containerName));
+    parts.push(
+      require('./exit-evidence').buildExitEvidenceSnippet(
+        containerName,
+        quotedLogPath
+      )
+    );
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
       exited.push(remove);
     } else if (
@@ -414,6 +434,7 @@ function buildDetachedDockerCompletionScript(
   } else {
     parts.push(`docker wait ${quotedName} >/dev/null 2>&1`);
     parts.push(buildDockerWaitForExitSnippet(containerName));
+    parts.push(cpuMonitor.stopSnippet());
     parts.push(buildCgroupSamplerStopSnippet());
     parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {

package/src/lib/docker-resource-limits.js

@@ -200,8 +200,36 @@ function buildResourceLimitsStatusLine(value) {
     : null;
 }
 
+/** Preserve Docker's CPU representation: an existing quota cannot become NanoCPUs in place. */
+function cpuCount(limits) {
+  const get = (prefix) =>
+    positive(limits.find((s) => s.startsWith(prefix))?.slice(prefix.length));
+  return (
+    get('--cpus=') ||
+    (get('--cpu-quota=')
+      ? get('--cpu-quota=') / (get('--cpu-period=') || 100000)
+      : null)
+  );
+}
+function cpuUpdateArgs(limits, cpus) {
+  return !limits.some((s) => s.startsWith('--cpus=')) &&
+    limits.some((s) => /^--cpu-(quota|period)=/.test(s))
+    ? ['--cpu-period=100000', `--cpu-quota=${Math.ceil(cpus * 100000)}`]
+    : ['--cpus', String(cpus)];
+}
+function withCpuLimit(limits, cpus) {
+  const args = cpuUpdateArgs(limits, cpus);
+  return [
+    ...limits.filter((s) => !/^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
+    ...(args[0] === '--cpus' ? [`--cpus=${args[1]}`] : args),
+  ];
+}
+
 module.exports = {
   DEFAULT_SHM_SIZE,
+  cpuCount,
+  cpuUpdateArgs,
+  withCpuLimit,
   buildResourceLimitsStatusLine,
   formatDockerBytes,
   formatDockerCpus,

package/src/lib/docker-resource-options.js

@@ -0,0 +1,180 @@
+/** Launch and recovery resource specifications, resolved against the daemon. */
+const { getDockerCommand } = require('./docker-cleanup');
+const { normalizeResourceLimits } = require('./docker-resource-limits');
+const FIELDS = {
+  '--memory': 'memory',
+  '--memory-swap': 'memorySwap',
+  '--cpus': 'cpus',
+  '--on-kill-resume-memory': 'onKillResumeMemory',
+};
+
+function parseResourceSpec(value, cpu = false) {
+  const text = String(value).trim().toLowerCase();
+  const percent = /^(\d+(?:\.\d+)?)%(?:-(\d+(?:\.\d+)?)%)?$/.exec(text);
+  if (percent) {
+    const min = Number(percent[1]),
+      max = Number(percent[2] || percent[1]);
+    if (min > 0 && max <= 100 && min <= max) {
+      return { min, max, percent: true };
+    }
+  } else {
+    const absolute = /^(\d+(?:\.\d+)?)(b|k|m|g|t|kb|mb|gb|tb)?$/.exec(text);
+    if (absolute && (!cpu || !absolute[2])) {
+      const power = ['', 'k', 'm', 'g', 't'].indexOf(
+        (absolute[2] || '').replace('b', '')
+      );
+      const amount =
+        Number(absolute[1]) * (cpu ? 1 : 1024 ** Math.max(0, power));
+      if (
+        Number.isFinite(amount) &&
+        amount > 0 &&
+        amount <= Number.MAX_SAFE_INTEGER
+      ) {
+        return { amount };
+      }
+    }
+  }
+  throw new Error(
+    `Invalid resource limit: "${value}". Expected a positive ${cpu ? 'CPU count' : 'Docker size'}, N% or MIN%-MAX% within (0, 100].`
+  );
+}
+
+function parse(args, index, options) {
+  const [flag, inline] = args[index].split('=', 2);
+  if (!FIELDS[flag]) {
+    return 0;
+  }
+  const value = inline ?? args[index + 1];
+  if (!value || value.startsWith('-')) {
+    throw new Error(`Option ${flag} requires a resource limit`);
+  }
+  parseResourceSpec(value, flag === '--cpus');
+  options[FIELDS[flag]] = value;
+  return inline === undefined ? 2 : 1;
+}
+
+function validate(options) {
+  if (
+    !Object.values(FIELDS).some(
+      (field) => options[field] !== null && options[field] !== undefined
+    )
+  ) {
+    return;
+  }
+  if (
+    !options.resume &&
+    !(options.isolatedStack || [options.isolated]).includes('docker')
+  ) {
+    throw new Error('Resource options require Docker isolation or --resume');
+  }
+  if (options.memorySwap && !options.memory && !options.resume) {
+    throw new Error('--memory-swap requires --memory');
+  }
+  if (
+    options.onKillResumeMemory &&
+    !options.onKillResume &&
+    !options.recoveryCommand
+  ) {
+    throw new Error(
+      '--on-kill-resume-memory requires --on-kill-resume or --recovery-command'
+    );
+  }
+}
+
+function readDaemonCapacity(runner) {
+  const result = runner(getDockerCommand(), ['info', '--format', '{{json .}}']);
+  if (!result?.success) {
+    throw new Error('Cannot read resource capacity from Docker daemon');
+  }
+  const info = JSON.parse(result.stdout);
+  if (!(info.MemTotal > 0) || !(info.NCPU > 0)) {
+    throw new Error('Docker daemon did not report MemTotal/NCPU');
+  }
+  return { memory: info.MemTotal, cpus: info.NCPU };
+}
+
+function resolveResourceOptions(options, runner, random = Math.random) {
+  const specs = {};
+  for (const field of ['memory', 'memorySwap', 'cpus']) {
+    if (options[field] !== null && options[field] !== undefined) {
+      specs[field] = parseResourceSpec(options[field], field === 'cpus');
+    }
+  }
+  const capacity = Object.values(specs).some((s) => s.percent)
+    ? readDaemonCapacity(runner)
+    : null;
+  const resolved = {};
+  const limits = normalizeResourceLimits(options.resourceLimits);
+  for (const [field, spec] of Object.entries(specs)) {
+    const amount = spec.percent
+      ? (capacity[field === 'cpus' ? 'cpus' : 'memory'] *
+          (spec.min + (spec.max - spec.min) * random())) /
+        100
+      : spec.amount;
+    resolved[field] =
+      field === 'cpus' ? Math.floor(amount * 1e9) / 1e9 : Math.floor(amount);
+    if (resolved[field] < (field === 'cpus' ? 0.01 : 6 * 1024 * 1024)) {
+      throw new Error(`Resolved ${field} is below Docker's minimum`);
+    }
+  }
+  if (
+    resolved.memory !== null &&
+    resolved.memory !== undefined &&
+    (resolved.memorySwap === null || resolved.memorySwap === undefined)
+  ) {
+    resolved.memorySwap = resolved.memory;
+  }
+  const existingMemory = limits
+    .find((flag) => flag.startsWith('--memory='))
+    ?.slice(9);
+  const memory =
+    resolved.memory ??
+    (existingMemory ? parseResourceSpec(existingMemory).amount : null);
+  if (
+    resolved.memorySwap !== null &&
+    resolved.memorySwap !== undefined &&
+    (memory === null || memory === undefined || resolved.memorySwap < memory)
+  ) {
+    throw new Error('memory-swap must be at least memory');
+  }
+  for (const [field, amount] of Object.entries(resolved)) {
+    const flag = field === 'memorySwap' ? '--memory-swap' : `--${field}`;
+    for (let i = limits.length - 1; i >= 0; i--) {
+      if (
+        limits[i].startsWith(`${flag}=`) ||
+        (field === 'cpus' && /^--cpu-(quota|period)=/.test(limits[i]))
+      ) {
+        limits.splice(i, 1);
+      }
+    }
+    limits.push(`${flag}=${amount}`);
+  }
+  return {
+    resourceLimits: limits,
+    resolvedLimits: resolved,
+    resourceLimitSpecs: Object.fromEntries(
+      Object.keys(specs).map((field) => [field, options[field]])
+    ),
+  };
+}
+
+function limitsLogLine(options) {
+  const values = options.resolvedLimits || {};
+  return Object.keys(values).length
+    ? `Limits: ${Object.entries(values)
+        .map(
+          ([field, value]) =>
+            `${field}=${value}${options.resourceLimitSpecs?.[field] ? ` (${options.resourceLimitSpecs[field]} -> ${value})` : ''}`
+        )
+        .join(' ')}`
+    : null;
+}
+
+module.exports = {
+  parse,
+  validate,
+  parseResourceSpec,
+  readDaemonCapacity,
+  resolveResourceOptions,
+  limitsLogLine,
+};

package/src/lib/docker-runtime-args.js

@@ -97,6 +97,12 @@ function buildDockerRuntimeStatusLines(options = {}) {
   if (limitsLine) {
     lines.push(limitsLine);
   }
+  const limitsSummary = require('./docker-resource-options').limitsLogLine(
+    options
+  );
+  if (limitsSummary) {
+    lines.push(limitsSummary);
+  }
   const onKillResume = getOnKillResume(options);
   if (onKillResume) {
     const what = options.recoveryCommand || 'the original command';
@@ -131,6 +137,15 @@ function buildDockerRuntimeMetadata(options = {}) {
         ? options.networkAliases
         : null,
     resourceLimits: resourceLimits.length > 0 ? resourceLimits : null,
+    resolvedLimits: options.resolvedLimits || null,
+    resourceLimitSpecs: options.resourceLimitSpecs || null,
+    onKillResumeMemory: options.onKillResumeMemory || null,
+    ...(options.cpuPenaltyConfig
+      ? {
+          cpuPenaltyConfig: options.cpuPenaltyConfig,
+          baseResourceLimits: resourceLimits,
+        }
+      : {}),
     onKillResume: getOnKillResume(options),
     recoveryCommand: options.recoveryCommand || null,
     onKillResumeDelay: getOnKillResume(options)