start-command 0.35.3 → 0.35.4

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # start-command
 
+## 0.35.4
+
+### Patch Changes
+
+- 6544236: Scope explicit resume status, memory evidence and log reads to execution attempts; preserve earlier evidence in attempt history and report launch, watcher, output and terminal lifecycle separately.
+
 ## 0.35.3
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.35.3",
+  "version": "0.35.4",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/lib/detached-finalize.js

@@ -85,6 +85,17 @@ function finalizeDetachedExecution(options = {}) {
   if (!record) {
     return { updated: false, reason: 'record-not-found' };
   }
+  const started = normalizeDockerTimestamp(options.startedAt);
+  const attemptNumber = options.attemptNumber ?? null;
+  if (
+    record.attempt &&
+    ((attemptNumber !== null &&
+      Number(attemptNumber) !== record.attempt.number) ||
+      (started && Date.parse(started) < Date.parse(record.attempt.startedAt)) ||
+      (attemptNumber === null && !started))
+  ) {
+    return { updated: false, reason: 'stale-attempt' };
+  }
   if (record.status === 'executed' && record.endTime) {
     // Already finalized (e.g. a `--status` query got there first, or the
     // watcher ran twice after a resume). Nothing to correct.
@@ -146,11 +157,24 @@ function finalizeDetachedExecution(options = {}) {
     record.options = { ...(record.options || {}), containerError };
   }
 
+  const {
+    appendLifecycle,
+    readAttemptActivity,
+  } = require('./execution-attempt');
+  readAttemptActivity(record);
   try {
     store.save(record);
   } catch (err) {
     return { updated: false, reason: `save-failed: ${err.message}`, record };
   }
+  appendLifecycle(record, 'terminal', {
+    status: record.status,
+    exitCode: record.exitCode,
+    endTime: record.endTime,
+    exitReason: record.exitReason || null,
+    oomKilled: record.oomKilled ?? null,
+    cgroupMemory: record.cgroupMemory || null,
+  });
   return { updated: true, record };
 }
 
@@ -163,12 +187,10 @@ function finalizeDetachedExecution(options = {}) {
 function resolveReason(record) {
   try {
     const { resolveExitReason } = require('./exit-reason');
-    const {
-      FATAL_MARKER_TAIL_BYTES,
-      readLogTail,
-    } = require('./isolation-log-utils');
+    const { FATAL_MARKER_TAIL_BYTES } = require('./isolation-log-utils');
+    const { readAttemptLogTail } = require('./execution-attempt');
     const logTail = record.logPath
-      ? readLogTail(record.logPath, FATAL_MARKER_TAIL_BYTES)
+      ? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
       : null;
     return resolveExitReason({
       exitCode: record.exitCode,
@@ -251,13 +273,13 @@ function reconcileFinalizedRecord(store, record) {
  * @param {string} executionId - UUID of the record to finalize
  * @returns {string} POSIX shell fragment
  */
-function buildDetachedFinalizeSnippet(executionId) {
+function buildDetachedFinalizeSnippet(executionId, attemptNumber = null) {
   const v = SHELL_VARS;
   return (
     `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
     `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
     `"$${v.started}" "$${v.finished}" "$${v.error}" "$${v.running}" ` +
-    `"$${CGROUP_SHELL_VARS.sample}" >/dev/null 2>&1 || true`
+    `"$${CGROUP_SHELL_VARS.sample}"${attemptNumber === null ? '' : ` ${shellQuote(attemptNumber)}`} >/dev/null 2>&1 || true`
   );
 }
 
@@ -279,6 +301,7 @@ function main(argv) {
     containerError,
     running,
     cgroupMemory,
+    attemptNumber,
   ] = argv;
   if (!executionId || process.env.START_DISABLE_TRACKING === 'true') {
     return;
@@ -300,6 +323,7 @@ function main(argv) {
       containerError,
       running,
       cgroupMemory,
+      attemptNumber: attemptNumber || null,
     });
   } catch {
     // Deliberately silent: see the exit-0 contract above.

package/src/lib/detached-output.js

@@ -0,0 +1,61 @@
+/** Stream Docker output without buffering lines; observe only timestamped output. */
+const fs = require('fs');
+const { StringDecoder } = require('string_decoder');
+const { activityPath } = require('./execution-attempt');
+
+class OutputObserver {
+  constructor(logPath, number, since) {
+    this.logPath = logPath;
+    this.activity = activityPath({ logPath, attempt: { number } });
+    this.since = Date.parse(since);
+    this.prefix = '';
+    this.atLineStart = true;
+    this.latest = null;
+    this.decoder = new StringDecoder('utf8');
+  }
+
+  write(chunk) {
+    fs.appendFileSync(this.logPath, chunk);
+    let latest = null;
+    for (const character of this.decoder.write(chunk)) {
+      if (character === '\n') {
+        this.atLineStart = true;
+        this.prefix = '';
+      } else if (this.atLineStart) {
+        this.prefix += character;
+        if (character === ' ') {
+          const text = this.prefix.trim();
+          const time = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(
+            text
+          )
+            ? Date.parse(text)
+            : NaN;
+          if (
+            Number.isFinite(time) &&
+            time >= this.since &&
+            (this.latest === null || time > this.latest)
+          ) {
+            this.latest = time;
+            latest = new Date(time).toISOString();
+          }
+          this.atLineStart = false;
+        } else if (this.prefix.length > 80) {
+          this.atLineStart = false;
+        }
+      }
+    }
+    if (latest) {
+      const temporary = `${this.activity}.${process.pid}.tmp`;
+      fs.writeFileSync(temporary, latest);
+      fs.renameSync(temporary, this.activity);
+    }
+  }
+}
+
+if (require.main === module) {
+  const [logPath, number, since] = process.argv.slice(2);
+  const observer = new OutputObserver(logPath, Number(number), since);
+  process.stdin.on('data', (chunk) => observer.write(chunk));
+}
+
+module.exports = { OutputObserver };

package/src/lib/docker-cleanup.js

@@ -1,4 +1,5 @@
 const { spawn, spawnSync } = require('child_process');
+const path = require('path');
 const {
   appendLogFile,
   createShellLogFooterSnippet,
@@ -381,7 +382,16 @@ function buildDetachedDockerCompletionScript(
     const since = watcherOptions.since
       ? ` --since ${shellQuote(watcherOptions.since)}`
       : '';
-    parts.push(`docker logs -f${since} ${quotedName} >> ${quotedLogPath} 2>&1`);
+    if (watcherOptions.attemptNumber) {
+      const capture = shellQuote(path.join(__dirname, 'detached-output.js'));
+      parts.push(
+        `docker logs -f --timestamps${since} ${quotedName} 2>&1 | ${shellQuote(process.execPath)} ${capture} ${quotedLogPath} ${watcherOptions.attemptNumber} ${shellQuote(watcherOptions.since)}`
+      );
+    } else {
+      parts.push(
+        `docker logs -f${since} ${quotedName} >> ${quotedLogPath} 2>&1`
+      );
+    }
     parts.push(buildDockerWaitForExitSnippet(containerName, quotedLogPath));
     parts.push(buildCgroupSamplerStopSnippet());
     parts.push(buildDockerStateSnippet(containerName));
@@ -420,7 +430,9 @@ function buildDetachedDockerCompletionScript(
 
   if (executionId) {
     // Last, so the record is only marked terminal once the log is complete.
-    exited.push(buildDetachedFinalizeSnippet(executionId));
+    exited.push(
+      buildDetachedFinalizeSnippet(executionId, watcherOptions.attemptNumber)
+    );
   }
 
   const stillRunning = logPath
@@ -474,7 +486,13 @@ function startDetachedDockerCompletionWatcher(
       stdio: 'ignore',
     }
   );
+  watcher.on('error', (error) => {
+    if (process.env.START_DEBUG === '1' || process.env.START_DEBUG === 'true') {
+      console.error(`[DEBUG] completion watcher failed: ${error.message}`);
+    }
+  });
   watcher.unref();
+  return Boolean(watcher.pid);
 }
 
 function spawnAttachedDocker(dockerArgs, logPath) {

package/src/lib/execution-attempt.js

@@ -0,0 +1,153 @@
+/** Attempt-scoped evidence for logical executions sharing a UUID and log. */
+const fs = require('fs');
+const { appendLogFile, readLogTail } = require('./isolation-log-utils');
+
+const TERMINAL_FIELDS = [
+  'endTimeSource',
+  'observedAt',
+  'staleDetectedAt',
+  'containerStartedAt',
+  'exitReason',
+  'oomKilled',
+  'memoryExhausted',
+  'memoryExhaustedReason',
+  'cgroupMemory',
+];
+
+function activityPath(record) {
+  return `${record.logPath}.attempt-${record.attempt.number}.activity`;
+}
+
+function readAttemptActivity(record) {
+  if (!record.attempt || !record.logPath) {
+    return record;
+  }
+  let fd;
+  try {
+    fd = fs.openSync(activityPath(record), 'r');
+    const buffer = Buffer.alloc(128);
+    const length = fs.readSync(fd, buffer, 0, buffer.length, 0);
+    const at = buffer.subarray(0, length).toString().trim();
+    if (
+      Number.isFinite(Date.parse(at)) &&
+      Date.parse(at) >= Date.parse(record.attempt.startedAt)
+    ) {
+      record.attempt = { ...record.attempt, lastOutputAt: at };
+    }
+  } catch {
+    // Quiet, missing, or unreadable output is not evidence of task progress.
+  } finally {
+    if (fd !== undefined) {
+      fs.closeSync(fd);
+    }
+  }
+  return record;
+}
+
+function createAttempt(record, plan) {
+  let logOffset = null;
+  if (record.logPath) {
+    try {
+      logOffset = fs.statSync(record.logPath).size;
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        logOffset = 0;
+      }
+    }
+  }
+  return {
+    number:
+      (record.attempt?.number ||
+        (Number(record.options?.resumeCount) || 0) +
+          (Number(record.options?.recoveryAttempts) || 0) +
+          1) + 1,
+    startedAt: new Date().toISOString(),
+    logOffset,
+    lastOutputAt: null,
+    launchAcceptedAt: null,
+    watcherAttachedAt: null,
+    mode: plan.mode,
+    previousSessionName: record.options?.sessionName || null,
+    sessionName: plan.newSessionName || plan.sessionName,
+  };
+}
+
+function archiveAttempt(record) {
+  readAttemptActivity(record);
+  const snapshot = {
+    ...(record.attempt || {
+      number:
+        (Number(record.options?.resumeCount) || 0) +
+        (Number(record.options?.recoveryAttempts) || 0) +
+        1,
+      startedAt: record.options?.resumedAt || record.startTime,
+      logOffset: null,
+    }),
+    command: record.command,
+    sessionName: record.options?.sessionName || null,
+    containerId: record.options?.containerId || null,
+    containerError: record.options?.containerError || null,
+    status: record.status,
+    exitCode: record.exitCode,
+    endTime: record.endTime,
+  };
+  for (const field of TERMINAL_FIELDS) {
+    if (record[field] !== undefined) {
+      snapshot[field] = record[field];
+    }
+    record[field] = undefined;
+  }
+  record.attemptHistory = [...(record.attemptHistory || []), snapshot];
+  delete record.options.containerError;
+}
+
+function appendLifecycle(record, event, details = {}) {
+  if (!record.logPath || !record.attempt) {
+    return;
+  }
+  appendLogFile(
+    record.logPath,
+    `\n[Start Command Lifecycle] ${JSON.stringify({
+      event,
+      at: new Date().toISOString(),
+      uuid: record.uuid,
+      attemptNumber: record.attempt.number,
+      ...record.attempt,
+      ...details,
+    })}\n`
+  );
+}
+
+function patchAttempt(store, record, fields) {
+  if (typeof store.patchAttempt === 'function') {
+    return store.patchAttempt(record.uuid, record.attempt.number, fields);
+  }
+  // Compatibility for consumers injecting a minimal in-memory store.
+  const current = store.get(record.uuid) || record;
+  if (current.attempt?.number !== record.attempt.number) {
+    return null;
+  }
+  current.attempt = { ...current.attempt, ...fields };
+  store.save(current);
+  return current;
+}
+
+function readAttemptLogTail(record, bytes) {
+  if (
+    !record.logPath ||
+    (record.attempt && record.attempt.logOffset === null)
+  ) {
+    return null;
+  }
+  return readLogTail(record.logPath, bytes, record.attempt?.logOffset || 0);
+}
+
+module.exports = {
+  activityPath,
+  appendLifecycle,
+  archiveAttempt,
+  createAttempt,
+  patchAttempt,
+  readAttemptActivity,
+  readAttemptLogTail,
+};

package/src/lib/execution-recovery.js

@@ -318,8 +318,34 @@ function recoverKilledExecution(params = {}) {
   }
 
   const since = now().toISOString();
+  const {
+    appendLifecycle,
+    archiveAttempt,
+    createAttempt,
+    patchAttempt,
+  } = require('./execution-attempt');
+  const nextAttempt = record.attempt
+    ? createAttempt(record, {
+        mode: 'automatic-recovery',
+        sessionName: containerName,
+      })
+    : null;
+  if (nextAttempt) {
+    nextAttempt.startedAt = since;
+    appendLifecycle(
+      { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
+      'resume-started'
+    );
+  }
   const started = runner(getDockerCommand(), ['start', containerName]);
   if (!started.success) {
+    if (nextAttempt) {
+      appendLifecycle(
+        { uuid: record.uuid, logPath: record.logPath, attempt: nextAttempt },
+        'launch-failed',
+        { error: failureDetail(started) }
+      );
+    }
     return giveUp(`docker start failed: ${failureDetail(started)}`);
   }
 
@@ -327,6 +353,19 @@ function recoverKilledExecution(params = {}) {
   const delayed = delayRange ? `, delayMs=${delayMs}` : '';
   // The killed run's own counters: the resumed run starts a fresh cgroup.
   const memory = parseCgroupMemorySample(params.cgroupMemory);
+  if (nextAttempt) {
+    Object.assign(record, {
+      status: 'executed',
+      exitCode: described.code,
+      endTime: params.finishedAt || null,
+      containerStartedAt: params.startedAt || undefined,
+      oomKilled: params.oomKilled === true || params.oomKilled === 'true',
+      cgroupMemory: memory || undefined,
+    });
+    archiveAttempt(record);
+    nextAttempt.launchAcceptedAt = new Date().toISOString();
+    record.attempt = nextAttempt;
+  }
   const counted = memory
     ? ['oomEvents', 'oomKills']
         .filter((field) => memory[field] !== null)
@@ -350,6 +389,12 @@ function recoverKilledExecution(params = {}) {
   record.exitReason = undefined;
   record.oomKilled = undefined;
   record.cgroupMemory = undefined;
+  record.memoryExhausted = undefined;
+  record.memoryExhaustedReason = undefined;
+  record.endTimeSource = undefined;
+  record.observedAt = undefined;
+  record.staleDetectedAt = undefined;
+  record.containerStartedAt = undefined;
   try {
     store.save(record);
   } catch {
@@ -359,13 +404,28 @@ function recoverKilledExecution(params = {}) {
 
   const startWatcher =
     params.startWatcher || startDetachedDockerCompletionWatcher;
+  if (nextAttempt) {
+    appendLifecycle(record, 'launch-accepted');
+  }
   startWatcher(
     containerName,
     getDockerContainerCleanupPolicy(opts),
     record.logPath || null,
     record.uuid,
-    { since, recoverOnKill: true }
+    {
+      since,
+      recoverOnKill: true,
+      ...(nextAttempt ? { attemptNumber: nextAttempt.number } : {}),
+    }
   );
+  if (nextAttempt) {
+    const current = patchAttempt(store, record, {
+      watcherAttachedAt: new Date().toISOString(),
+    });
+    if (current) {
+      appendLifecycle(current, 'watcher-attached');
+    }
+  }
   return {
     recovered: true,
     reason: 'resumed',

package/src/lib/execution-resume-all.js

@@ -131,7 +131,18 @@ function resumeAllExecutions(store, deps = {}) {
         sessionName,
         getDockerContainerCleanupPolicy(record.options || {}),
         record.logPath || null,
-        record.uuid || null
+        record.uuid || null,
+        record.attempt
+          ? {
+              since: record.attempt.startedAt,
+              attemptNumber: record.attempt.number,
+              recoverOnKill:
+                Boolean(record.options?.onKillResume) &&
+                ['docker-start', 'automatic-recovery', 'relaunch'].includes(
+                  record.attempt.mode
+                ),
+            }
+          : {}
       );
       executions.push(
         describeRecord(

package/src/lib/execution-resume.js

@@ -31,6 +31,12 @@ const {
   readDockerResourceLimits,
 } = require('./docker-resource-limits');
 const { appendLogFile } = require('./isolation-log-utils');
+const {
+  appendLifecycle,
+  archiveAttempt,
+  createAttempt,
+  patchAttempt,
+} = require('./execution-attempt');
 const { runCommand } = require('./execution-control');
 const { SessionState, probeSession } = require('./session-probe');
 const { getDockerNetworks } = require('./docker-network-lifecycle');
@@ -105,6 +111,7 @@ function buildLaunchOptions(record) {
     resourceLimits: normalizeResourceLimits(opts.resourceLimits),
     // Append to the same log so one logical session keeps one gap-free record.
     logPath: record.logPath || null,
+    deferCompletionWatcher: true,
   };
 }
 
@@ -339,10 +346,17 @@ function formatResumeResult(result, outputFormat) {
  * @param {?string} containerId - New container id, when one was created
  * @returns {object} The updated record
  */
-function applyResumeToRecord(record, plan, containerId) {
+function applyResumeToRecord(
+  record,
+  plan,
+  containerId,
+  attempt = createAttempt(record, plan)
+) {
+  archiveAttempt(record);
+  record.attempt = attempt;
   const options = { ...(record.options || {}) };
   options.resumeCount = plan.attempt;
-  options.resumedAt = new Date().toISOString();
+  options.resumedAt = attempt.startedAt;
   // A resume is a new deliberate start: launch-time recovery applies again.
   delete options.stopRequestedAt;
   if (plan.resourceLimits && plan.resourceLimits.length > 0) {
@@ -423,6 +437,13 @@ async function resumeExecution(store, identifier, deps = {}) {
   }
 
   let containerId = null;
+  const attempt = createAttempt(record, plan);
+  const lifecycleRecord = {
+    uuid: record.uuid,
+    logPath: record.logPath,
+    attempt,
+  };
+  appendLifecycle(lifecycleRecord, 'resume-started');
 
   if (plan.mode === ResumeMode.RELAUNCH) {
     const runIsolated = deps.runIsolated || require('./isolation').runIsolated;
@@ -432,6 +453,9 @@ async function resumeExecution(store, identifier, deps = {}) {
       plan.launchOptions
     );
     if (!launchResult || !launchResult.success) {
+      appendLifecycle(lifecycleRecord, 'launch-failed', {
+        error: launchResult?.message || 'unknown error',
+      });
       return {
         success: false,
         error: `Failed to relaunch ${plan.backend} session "${plan.sessionName}": ${
@@ -448,6 +472,7 @@ async function resumeExecution(store, identifier, deps = {}) {
           (result.stderr || '').trim() ||
           result.error ||
           `exit code ${result.status}`;
+        appendLifecycle(lifecycleRecord, 'launch-failed', { error: detail });
         return {
           success: false,
           error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${detail}`,
@@ -460,28 +485,53 @@ async function resumeExecution(store, identifier, deps = {}) {
     if (limitsLine && record.logPath) {
       appendLogFile(record.logPath, `${limitsLine}\n`);
     }
-
-    // The completion watcher died with the previous run (or with the
-    // supervisor), so a new one must follow the resumed container.
-    const startWatcher =
-      deps.startWatcher || startDetachedDockerCompletionWatcher;
-    startWatcher(
-      activeSessionName(plan),
-      getDockerContainerCleanupPolicy(record.options || {}),
-      record.logPath || null,
-      record.uuid || null,
-      // `docker start` re-runs the launch-time selector, so the session keeps
-      // its kill recovery (issue #176); a snapshot container has no selector.
-      {
-        recoverOnKill:
-          plan.mode === ResumeMode.DOCKER_START && Boolean(opts.onKillResume),
-      }
-    );
   }
 
   const previousSessionName = plan.newSessionName ? plan.sessionName : null;
-  const updated = applyResumeToRecord(record, plan, containerId);
+  attempt.launchAcceptedAt = new Date().toISOString();
+  const updated = applyResumeToRecord(record, plan, containerId, attempt);
   store.save(updated);
+  appendLifecycle(updated, 'launch-accepted');
+
+  if (plan.backend === 'docker') {
+    try {
+      const startWatcher =
+        deps.startWatcher || startDetachedDockerCompletionWatcher;
+      const watcher = startWatcher(
+        activeSessionName(plan),
+        getDockerContainerCleanupPolicy(updated.options),
+        updated.logPath || null,
+        updated.uuid || null,
+        {
+          since: attempt.startedAt,
+          attemptNumber: attempt.number,
+          recoverOnKill:
+            plan.mode !== ResumeMode.DOCKER_SNAPSHOT &&
+            Boolean(opts.onKillResume),
+        }
+      );
+      if (watcher === false) {
+        throw new Error('Watcher process could not be started');
+      }
+      const current = patchAttempt(store, updated, {
+        watcherAttachedAt: new Date().toISOString(),
+      });
+      if (current) {
+        appendLifecycle(current, 'watcher-attached');
+      }
+    } catch (error) {
+      const current = patchAttempt(store, updated, {
+        watcherError: error.message,
+      });
+      appendLifecycle(current || updated, 'watcher-attachment-failed', {
+        error: error.message,
+      });
+      return {
+        success: false,
+        error: `Launch accepted, but completion watcher attachment failed: ${error.message}`,
+      };
+    }
+  }
 
   return {
     success: true,

package/src/lib/execution-store.js

@@ -75,6 +75,8 @@ class ExecutionRecord {
     this.logPath = options.logPath || '';
     this.startTime = options.startTime || new Date().toISOString();
     this.endTime = options.endTime || null;
+    this.attempt = options.attempt;
+    this.attemptHistory = options.attemptHistory;
     // Provenance of `endTime` (issue #170.2). `--status` used to fabricate a
     // finish time with `new Date()` whenever the record had none, which made an
     // observation indistinguishable from a real finish time. One of
@@ -152,6 +154,12 @@ class ExecutionRecord {
       startTime: this.startTime,
       endTime: this.endTime,
     };
+    if (this.attempt) {
+      obj.attempt = this.attempt;
+    }
+    if (this.attemptHistory) {
+      obj.attemptHistory = this.attemptHistory;
+    }
     if (this.endTimeSource !== undefined && this.endTimeSource !== null) {
       obj.endTimeSource = this.endTimeSource;
     }
@@ -557,6 +565,29 @@ class ExecutionStore {
     }
   }
 
+  // Merge watcher metadata under the database lock without replacing terminal state.
+  patchAttempt(uuid, number, fields) {
+    const lock = new LockManager(this.lockFilePath);
+    if (!lock.acquire()) {
+      throw new Error('Failed to acquire lock for attempt metadata');
+    }
+    try {
+      const records = this.readLinoRecords();
+      const current = records.find((record) => record.uuid === uuid);
+      if (current?.attempt?.number !== number) {
+        return null;
+      }
+      current.attempt = { ...current.attempt, ...fields };
+      this.writeLinoRecords(records);
+      if (this.useLinks) {
+        this.writeLinksRecord(current);
+      }
+      return current;
+    } finally {
+      lock.release();
+    }
+  }
+
   /**
    * Get an execution record by UUID or session name
    * First tries exact UUID match, then the current session name, then any