start-command 0.35.2 → 0.35.4

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # start-command
 
+## 0.35.4
+
+### Patch Changes
+
+- 6544236: Scope explicit resume status, memory evidence and log reads to execution attempts; preserve earlier evidence in attempt history and report launch, watcher, output and terminal lifecycle separately.
+
+## 0.35.3
+
+### Patch Changes
+
+- ec0a9cf: Report OOM kill scope as unknown when only cgroup memory counters are available. Allocation events and killed processes have different units, so their comparison cannot establish container, parent or host scope. Preserve raw counters and the exit-code guard for earlier child OOM kills.
+
 ## 0.35.2
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.35.2",
+  "version": "0.35.4",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/lib/cgroup-memory.js

@@ -5,14 +5,16 @@
  * it says that *some* process in the container was OOM-killed at some point,
  * but not how many, not whether the container hit its own `--memory` limit or
  * the whole host ran out, and not how close the execution came to its limit.
- * The kernel keeps all three per cgroup
+ * The kernel provides raw counters per cgroup
  * (https://docs.kernel.org/admin-guide/cgroup-v2.html#memory-interface-files):
- *   memory.events `oom`       the cgroup hit its own limit
- *   memory.events `oom_kill`  processes killed here by *any* OOM killer, so
- *                             `oom_kill > oom` points to a host-wide (or
- *                             parent cgroup) OOM
+ *   memory.events `oom`       allocation events reaching the memory limit
+ *   memory.events `oom_kill`  processes killed here by *any* OOM killer
  *   memory.peak / memory.max  peak usage (Linux 5.19+) and limit (`max` when
  *                             unlimited)
+ * These counters have different units and are hierarchical. A group OOM can
+ * kill several processes for one allocation event; earlier allocation events
+ * can also coexist with a later host OOM. Their comparison cannot establish
+ * container, parent or host scope (issue #185).
  *
  * The cgroup is removed when the container stops, so the values cannot be read
  * afterwards. The detached completion watcher therefore samples them while the
@@ -122,9 +124,8 @@ function buildCgroupMemoryLogSnippet(quotedLogPath) {
   // A function, so `$1`.. of the watcher script itself stay untouched.
   return (
     `__start_command_cgroup_log() { __scm_note=''; ` +
-    `if [ "$4" -gt 0 ] 2>/dev/null; then if [ "$4" -gt "$3" ] 2>/dev/null; then ` +
-    `__scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.HOST_OR_PARENT]})'; ` +
-    `else __scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.CONTAINER_LIMIT]})'; fi; fi; ` +
+    `if [ "$4" -gt 0 ] 2>/dev/null; then ` +
+    `__scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.UNKNOWN]})'; fi; ` +
     `printf 'Memory:     memory.max=%s memory.peak=%s oom=%s oom_kill=%s%s\\n' ` +
     `"$1" "$2" "$3" "$4" "$__scm_note"; }; ` +
     `if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath}; fi`
@@ -151,14 +152,17 @@ function formatCgroupMemoryLogLine(text) {
 
 /** Where the OOM killer that killed processes in the container came from. */
 const OOM_SCOPE = {
+  UNKNOWN: 'unknown',
+  // Retained scope names; raw counters alone cannot establish either one.
   CONTAINER_LIMIT: 'container-limit',
   HOST_OR_PARENT: 'host-or-parent',
 };
 
 const OOM_SCOPE_NOTES = {
+  [OOM_SCOPE.UNKNOWN]: 'OOM kill scope unknown',
   [OOM_SCOPE.CONTAINER_LIMIT]: 'the container hit its own memory limit',
   [OOM_SCOPE.HOST_OR_PARENT]:
-    'oom_kill > oom: a host-wide or parent cgroup OOM killed processes here',
+    'a host-wide or parent cgroup OOM killed processes here',
 };
 
 function parseCounter(text) {
@@ -216,16 +220,17 @@ function normalizeCgroupMemory(value) {
 
 /**
  * @param {?object} memory - Counters
- * @returns {?string} One of OOM_SCOPE, or null when nothing was OOM-killed
+ * Raw counters do not attribute the OOM killer's scope. Separately attributed
+ * kernel/cgroup evidence would be needed to establish container, parent or
+ * host scope. An unknown oomEvents count remains unknown, never zero.
+ * @returns {?string} OOM_SCOPE.UNKNOWN for observed kills, otherwise null
  */
 function describeCgroupOomScope(memory) {
   const counters = normalizeCgroupMemory(memory);
   if (!counters || !(counters.oomKills > 0)) {
     return null;
   }
-  return counters.oomKills > (counters.oomEvents || 0)
-    ? OOM_SCOPE.HOST_OR_PARENT
-    : OOM_SCOPE.CONTAINER_LIMIT;
+  return OOM_SCOPE.UNKNOWN;
 }
 
 function formatBytes(bytes) {

package/src/lib/detached-finalize.js

@@ -85,6 +85,17 @@ function finalizeDetachedExecution(options = {}) {
   if (!record) {
     return { updated: false, reason: 'record-not-found' };
   }
+  const started = normalizeDockerTimestamp(options.startedAt);
+  const attemptNumber = options.attemptNumber ?? null;
+  if (
+    record.attempt &&
+    ((attemptNumber !== null &&
+      Number(attemptNumber) !== record.attempt.number) ||
+      (started && Date.parse(started) < Date.parse(record.attempt.startedAt)) ||
+      (attemptNumber === null && !started))
+  ) {
+    return { updated: false, reason: 'stale-attempt' };
+  }
   if (record.status === 'executed' && record.endTime) {
     // Already finalized (e.g. a `--status` query got there first, or the
     // watcher ran twice after a resume). Nothing to correct.
@@ -146,11 +157,24 @@ function finalizeDetachedExecution(options = {}) {
     record.options = { ...(record.options || {}), containerError };
   }
 
+  const {
+    appendLifecycle,
+    readAttemptActivity,
+  } = require('./execution-attempt');
+  readAttemptActivity(record);
   try {
     store.save(record);
   } catch (err) {
     return { updated: false, reason: `save-failed: ${err.message}`, record };
   }
+  appendLifecycle(record, 'terminal', {
+    status: record.status,
+    exitCode: record.exitCode,
+    endTime: record.endTime,
+    exitReason: record.exitReason || null,
+    oomKilled: record.oomKilled ?? null,
+    cgroupMemory: record.cgroupMemory || null,
+  });
   return { updated: true, record };
 }
 
@@ -163,12 +187,10 @@ function finalizeDetachedExecution(options = {}) {
 function resolveReason(record) {
   try {
     const { resolveExitReason } = require('./exit-reason');
-    const {
-      FATAL_MARKER_TAIL_BYTES,
-      readLogTail,
-    } = require('./isolation-log-utils');
+    const { FATAL_MARKER_TAIL_BYTES } = require('./isolation-log-utils');
+    const { readAttemptLogTail } = require('./execution-attempt');
     const logTail = record.logPath
-      ? readLogTail(record.logPath, FATAL_MARKER_TAIL_BYTES)
+      ? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
       : null;
     return resolveExitReason({
       exitCode: record.exitCode,
@@ -251,13 +273,13 @@ function reconcileFinalizedRecord(store, record) {
  * @param {string} executionId - UUID of the record to finalize
  * @returns {string} POSIX shell fragment
  */
-function buildDetachedFinalizeSnippet(executionId) {
+function buildDetachedFinalizeSnippet(executionId, attemptNumber = null) {
   const v = SHELL_VARS;
   return (
     `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
     `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
     `"$${v.started}" "$${v.finished}" "$${v.error}" "$${v.running}" ` +
-    `"$${CGROUP_SHELL_VARS.sample}" >/dev/null 2>&1 || true`
+    `"$${CGROUP_SHELL_VARS.sample}"${attemptNumber === null ? '' : ` ${shellQuote(attemptNumber)}`} >/dev/null 2>&1 || true`
   );
 }
 
@@ -279,6 +301,7 @@ function main(argv) {
     containerError,
     running,
     cgroupMemory,
+    attemptNumber,
   ] = argv;
   if (!executionId || process.env.START_DISABLE_TRACKING === 'true') {
     return;
@@ -300,6 +323,7 @@ function main(argv) {
       containerError,
       running,
       cgroupMemory,
+      attemptNumber: attemptNumber || null,
     });
   } catch {
     // Deliberately silent: see the exit-0 contract above.

package/src/lib/detached-output.js

@@ -0,0 +1,61 @@
+/** Stream Docker output without buffering lines; observe only timestamped output. */
+const fs = require('fs');
+const { StringDecoder } = require('string_decoder');
+const { activityPath } = require('./execution-attempt');
+
+class OutputObserver {
+  constructor(logPath, number, since) {
+    this.logPath = logPath;
+    this.activity = activityPath({ logPath, attempt: { number } });
+    this.since = Date.parse(since);
+    this.prefix = '';
+    this.atLineStart = true;
+    this.latest = null;
+    this.decoder = new StringDecoder('utf8');
+  }
+
+  write(chunk) {
+    fs.appendFileSync(this.logPath, chunk);
+    let latest = null;
+    for (const character of this.decoder.write(chunk)) {
+      if (character === '\n') {
+        this.atLineStart = true;
+        this.prefix = '';
+      } else if (this.atLineStart) {
+        this.prefix += character;
+        if (character === ' ') {
+          const text = this.prefix.trim();
+          const time = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(
+            text
+          )
+            ? Date.parse(text)
+            : NaN;
+          if (
+            Number.isFinite(time) &&
+            time >= this.since &&
+            (this.latest === null || time > this.latest)
+          ) {
+            this.latest = time;
+            latest = new Date(time).toISOString();
+          }
+          this.atLineStart = false;
+        } else if (this.prefix.length > 80) {
+          this.atLineStart = false;
+        }
+      }
+    }
+    if (latest) {
+      const temporary = `${this.activity}.${process.pid}.tmp`;
+      fs.writeFileSync(temporary, latest);
+      fs.renameSync(temporary, this.activity);
+    }
+  }
+}
+
+if (require.main === module) {
+  const [logPath, number, since] = process.argv.slice(2);
+  const observer = new OutputObserver(logPath, Number(number), since);
+  process.stdin.on('data', (chunk) => observer.write(chunk));
+}
+
+module.exports = { OutputObserver };

package/src/lib/docker-cleanup.js

@@ -1,4 +1,5 @@
 const { spawn, spawnSync } = require('child_process');
+const path = require('path');
 const {
   appendLogFile,
   createShellLogFooterSnippet,
@@ -381,7 +382,16 @@ function buildDetachedDockerCompletionScript(
     const since = watcherOptions.since
       ? ` --since ${shellQuote(watcherOptions.since)}`
       : '';
-    parts.push(`docker logs -f${since} ${quotedName} >> ${quotedLogPath} 2>&1`);
+    if (watcherOptions.attemptNumber) {
+      const capture = shellQuote(path.join(__dirname, 'detached-output.js'));
+      parts.push(
+        `docker logs -f --timestamps${since} ${quotedName} 2>&1 | ${shellQuote(process.execPath)} ${capture} ${quotedLogPath} ${watcherOptions.attemptNumber} ${shellQuote(watcherOptions.since)}`
+      );
+    } else {
+      parts.push(
+        `docker logs -f${since} ${quotedName} >> ${quotedLogPath} 2>&1`
+      );
+    }
     parts.push(buildDockerWaitForExitSnippet(containerName, quotedLogPath));
     parts.push(buildCgroupSamplerStopSnippet());
     parts.push(buildDockerStateSnippet(containerName));
@@ -420,7 +430,9 @@ function buildDetachedDockerCompletionScript(
 
   if (executionId) {
     // Last, so the record is only marked terminal once the log is complete.
-    exited.push(buildDetachedFinalizeSnippet(executionId));
+    exited.push(
+      buildDetachedFinalizeSnippet(executionId, watcherOptions.attemptNumber)
+    );
   }
 
   const stillRunning = logPath
@@ -474,7 +486,13 @@ function startDetachedDockerCompletionWatcher(
       stdio: 'ignore',
     }
   );
+  watcher.on('error', (error) => {
+    if (process.env.START_DEBUG === '1' || process.env.START_DEBUG === 'true') {
+      console.error(`[DEBUG] completion watcher failed: ${error.message}`);
+    }
+  });
   watcher.unref();
+  return Boolean(watcher.pid);
 }
 
 function spawnAttachedDocker(dockerArgs, logPath) {

package/src/lib/execution-attempt.js

@@ -0,0 +1,153 @@
+/** Attempt-scoped evidence for logical executions sharing a UUID and log. */
+const fs = require('fs');
+const { appendLogFile, readLogTail } = require('./isolation-log-utils');
+
+const TERMINAL_FIELDS = [
+  'endTimeSource',
+  'observedAt',
+  'staleDetectedAt',
+  'containerStartedAt',
+  'exitReason',
+  'oomKilled',
+  'memoryExhausted',
+  'memoryExhaustedReason',
+  'cgroupMemory',
+];
+
+function activityPath(record) {
+  return `${record.logPath}.attempt-${record.attempt.number}.activity`;
+}
+
+function readAttemptActivity(record) {
+  if (!record.attempt || !record.logPath) {
+    return record;
+  }
+  let fd;
+  try {
+    fd = fs.openSync(activityPath(record), 'r');
+    const buffer = Buffer.alloc(128);
+    const length = fs.readSync(fd, buffer, 0, buffer.length, 0);
+    const at = buffer.subarray(0, length).toString().trim();
+    if (
+      Number.isFinite(Date.parse(at)) &&
+      Date.parse(at) >= Date.parse(record.attempt.startedAt)
+    ) {
+      record.attempt = { ...record.attempt, lastOutputAt: at };
+    }
+  } catch {
+    // Quiet, missing, or unreadable output is not evidence of task progress.
+  } finally {
+    if (fd !== undefined) {
+      fs.closeSync(fd);
+    }
+  }
+  return record;
+}
+
+function createAttempt(record, plan) {
+  let logOffset = null;
+  if (record.logPath) {
+    try {
+      logOffset = fs.statSync(record.logPath).size;
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        logOffset = 0;
+      }
+    }
+  }
+  return {
+    number:
+      (record.attempt?.number ||
+        (Number(record.options?.resumeCount) || 0) +
+          (Number(record.options?.recoveryAttempts) || 0) +
+          1) + 1,
+    startedAt: new Date().toISOString(),
+    logOffset,
+    lastOutputAt: null,
+    launchAcceptedAt: null,
+    watcherAttachedAt: null,
+    mode: plan.mode,
+    previousSessionName: record.options?.sessionName || null,
+    sessionName: plan.newSessionName || plan.sessionName,
+  };
+}
+
+function archiveAttempt(record) {
+  readAttemptActivity(record);
+  const snapshot = {
+    ...(record.attempt || {
+      number:
+        (Number(record.options?.resumeCount) || 0) +
+        (Number(record.options?.recoveryAttempts) || 0) +
+        1,
+      startedAt: record.options?.resumedAt || record.startTime,
+      logOffset: null,
+    }),
+    command: record.command,
+    sessionName: record.options?.sessionName || null,
+    containerId: record.options?.containerId || null,
+    containerError: record.options?.containerError || null,
+    status: record.status,
+    exitCode: record.exitCode,
+    endTime: record.endTime,
+  };
+  for (const field of TERMINAL_FIELDS) {
+    if (record[field] !== undefined) {
+      snapshot[field] = record[field];
+    }
+    record[field] = undefined;
+  }
+  record.attemptHistory = [...(record.attemptHistory || []), snapshot];
+  delete record.options.containerError;
+}
+
+function appendLifecycle(record, event, details = {}) {
+  if (!record.logPath || !record.attempt) {
+    return;
+  }
+  appendLogFile(
+    record.logPath,
+    `\n[Start Command Lifecycle] ${JSON.stringify({
+      event,
+      at: new Date().toISOString(),
+      uuid: record.uuid,
+      attemptNumber: record.attempt.number,
+      ...record.attempt,
+      ...details,
+    })}\n`
+  );
+}
+
+function patchAttempt(store, record, fields) {
+  if (typeof store.patchAttempt === 'function') {
+    return store.patchAttempt(record.uuid, record.attempt.number, fields);
+  }
+  // Compatibility for consumers injecting a minimal in-memory store.
+  const current = store.get(record.uuid) || record;
+  if (current.attempt?.number !== record.attempt.number) {
+    return null;
+  }
+  current.attempt = { ...current.attempt, ...fields };
+  store.save(current);
+  return current;
+}
+
+function readAttemptLogTail(record, bytes) {
+  if (
+    !record.logPath ||
+    (record.attempt && record.attempt.logOffset === null)
+  ) {
+    return null;
+  }
+  return readLogTail(record.logPath, bytes, record.attempt?.logOffset || 0);
+}
+
+module.exports = {
+  activityPath,
+  appendLifecycle,
+  archiveAttempt,
+  createAttempt,
+  patchAttempt,
+  readAttemptActivity,
+  readAttemptLogTail,
+};

package/src/lib/execution-recovery.js

@@ -318,8 +318,34 @@ function recoverKilledExecution(params = {}) {
   }
 
   const since = now().toISOString();
+  const {
+    appendLifecycle,
+    archiveAttempt,
+    createAttempt,
+    patchAttempt,
+  } = require('./execution-attempt');
+  const nextAttempt = record.attempt
+    ? createAttempt(record, {
+        mode: 'automatic-recovery',
+        sessionName: containerName,
+      })
+    : null;
+  if (nextAttempt) {
+    nextAttempt.startedAt = since;
+    appendLifecycle(
+      { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
+      'resume-started'
+    );
+  }
   const started = runner(getDockerCommand(), ['start', containerName]);
   if (!started.success) {
+    if (nextAttempt) {
+      appendLifecycle(
+        { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
+        'launch-failed',
+        { error: failureDetail(started) }
+      );
+    }
     return giveUp(`docker start failed: ${failureDetail(started)}`);
   }
 
@@ -327,6 +353,19 @@ function recoverKilledExecution(params = {}) {
   const delayed = delayRange ? `, delayMs=${delayMs}` : '';
   // The killed run's own counters: the resumed run starts a fresh cgroup.
   const memory = parseCgroupMemorySample(params.cgroupMemory);
+  if (nextAttempt) {
+    Object.assign(record, {
+      status: 'executed',
+      exitCode: described.code,
+      endTime: params.finishedAt || null,
+      containerStartedAt: params.startedAt || undefined,
+      oomKilled: params.oomKilled === true || params.oomKilled === 'true',
+      cgroupMemory: memory || undefined,
+    });
+    archiveAttempt(record);
+    nextAttempt.launchAcceptedAt = new Date().toISOString();
+    record.attempt = nextAttempt;
+  }
   const counted = memory
     ? ['oomEvents', 'oomKills']
         .filter((field) => memory[field] !== null)
@@ -350,6 +389,12 @@ function recoverKilledExecution(params = {}) {
   record.exitReason = undefined;
   record.oomKilled = undefined;
   record.cgroupMemory = undefined;
+  record.memoryExhausted = undefined;
+  record.memoryExhaustedReason = undefined;
+  record.endTimeSource = undefined;
+  record.observedAt = undefined;
+  record.staleDetectedAt = undefined;
+  record.containerStartedAt = undefined;
   try {
     store.save(record);
   } catch {
@@ -359,13 +404,28 @@ function recoverKilledExecution(params = {}) {
 
   const startWatcher =
     params.startWatcher || startDetachedDockerCompletionWatcher;
+  if (nextAttempt) {
+    appendLifecycle(record, 'launch-accepted');
+  }
   startWatcher(
     containerName,
     getDockerContainerCleanupPolicy(opts),
     record.logPath || null,
     record.uuid,
-    { since, recoverOnKill: true }
+    {
+      since,
+      recoverOnKill: true,
+      ...(nextAttempt ? { attemptNumber: nextAttempt.number } : {}),
+    }
   );
+  if (nextAttempt) {
+    const current = patchAttempt(store, record, {
+      watcherAttachedAt: new Date().toISOString(),
+    });
+    if (current) {
+      appendLifecycle(current, 'watcher-attached');
+    }
+  }
   return {
     recovered: true,
     reason: 'resumed',

package/src/lib/execution-resume-all.js

@@ -131,7 +131,18 @@ function resumeAllExecutions(store, deps = {}) {
         sessionName,
         getDockerContainerCleanupPolicy(record.options || {}),
         record.logPath || null,
-        record.uuid || null
+        record.uuid || null,
+        record.attempt
+          ? {
+              since: record.attempt.startedAt,
+              attemptNumber: record.attempt.number,
+              recoverOnKill:
+                Boolean(record.options?.onKillResume) &&
+                ['docker-start', 'automatic-recovery', 'relaunch'].includes(
+                  record.attempt.mode
+                ),
+            }
+          : {}
       );
       executions.push(
         describeRecord(