start-command 0.33.0 → 0.34.0

package/src/lib/status-formatter.js

@@ -14,11 +14,17 @@ const {
 } = require('./output-blocks');
 const { collectProcessIds } = require('./execution-control');
 const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
-const { resolveExitReason, resolveMemoryExhaustion } = require('./exit-reason');
+const {
+  describeExitCode,
+  resolveExitReason,
+  resolveMemoryExhaustion,
+} = require('./exit-reason');
 const {
   FATAL_MARKER_TAIL_BYTES,
   readLogTail,
 } = require('./isolation-log-utils');
+const { normalizeDockerTimestamp } = require('./docker-post-mortem');
+const { END_TIME_SOURCE } = require('./detached-finalize');
 
 /**
  * Inspect the live state of a detached docker container by name.
@@ -44,7 +50,8 @@ function inspectDockerState(sessionName) {
     [
       'inspect',
       '-f',
-      '{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}}',
+      '{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ' +
+        '{{.State.StartedAt}} {{.State.FinishedAt}}',
       sessionName,
     ],
     getDockerSpawnOptions({
@@ -56,13 +63,18 @@ function inspectDockerState(sessionName) {
   if (result.error || result.status !== 0 || !result.stdout) {
     return null;
   }
-  const [runningRaw, exitRaw, oomKilledRaw] = result.stdout.trim().split(/\s+/);
+  // Parsed positionally and defensively: an older docker (or a stubbed one)
+  // may answer with fewer fields than the template asks for.
+  const [runningRaw, exitRaw, oomKilledRaw, startedRaw, finishedRaw] =
+    result.stdout.trim().split(/\s+/);
   const exitCode = Number.parseInt(exitRaw, 10);
   return {
     running: runningRaw === 'true',
     exitCode: Number.isFinite(exitCode) ? exitCode : null,
     oomKilled:
       oomKilledRaw === 'true' ? true : oomKilledRaw === 'false' ? false : null,
+    startedAt: normalizeDockerTimestamp(startedRaw),
+    finishedAt: normalizeDockerTimestamp(finishedRaw),
   };
 }
 
@@ -193,7 +205,50 @@ function isDetachedSessionAlive(record) {
  * `start` appends itself (issue #150).
  */
 const LOG_FOOTER_PATTERN =
-  /^={10,}[ \t]*\r?\n^Finished:[^\r\n]*\r?\n^Exit Code:[ \t]*(-?\d+)[ \t]*(?![^\r\n])/gm;
+  /^={10,}[ \t]*\r?\n^Finished:[ \t]*([^\r\n]*)\r?\n^Exit Code:[ \t]*(-?\d+)[ \t]*(?![^\r\n])/gm;
+
+/**
+ * Parse the `Finished:` line of the footer into an ISO timestamp.
+ *
+ * `start` writes it as `YYYY-MM-DD HH:MM:SS[.mmm]` in UTC (both from JS and
+ * from the shell snippet). Anything else — a truncated log, a test fixture, a
+ * future format — yields null so the caller falls back instead of inventing a
+ * timestamp out of an unparseable string.
+ *
+ * @param {string|null|undefined} value - Raw text after `Finished:`
+ * @returns {string|null} ISO timestamp, or null
+ */
+function parseFooterTimestamp(value) {
+  if (typeof value !== 'string') {
+    return null;
+  }
+  const trimmed = value.trim();
+  if (!/^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(\.\d+)?Z?$/.test(trimmed)) {
+    return null;
+  }
+  const parsed = Date.parse(`${trimmed.replace(' ', 'T').replace(/Z$/, '')}Z`);
+  return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
+}
+
+/**
+ * Read the terminal footer written at the very end of a log.
+ * @param {string|null} tail - Tail of the execution log
+ * @returns {{exitCode: number|null, finishedAt: string|null}} Footer facts
+ */
+function parseFooterFromTail(tail) {
+  if (tail === null || tail === undefined) {
+    return { exitCode: null, finishedAt: null };
+  }
+  const matches = [...tail.matchAll(LOG_FOOTER_PATTERN)];
+  if (matches.length === 0) {
+    return { exitCode: null, finishedAt: null };
+  }
+  const last = matches[matches.length - 1];
+  return {
+    exitCode: parseInt(last[2], 10),
+    finishedAt: parseFooterTimestamp(last[1]),
+  };
+}
 
 /**
  * Read the terminal exit code from the anchored footer at the end of a log.
@@ -207,11 +262,7 @@ function parseExitCodeFromTail(tail) {
   if (tail === null || tail === undefined) {
     return null;
   }
-  const matches = [...tail.matchAll(LOG_FOOTER_PATTERN)];
-  if (matches.length === 0) {
-    return null;
-  }
-  return parseInt(matches[matches.length - 1][1], 10);
+  return parseFooterFromTail(tail).exitCode;
 }
 
 function readExitCodeFromLog(logPath) {
@@ -230,8 +281,43 @@ function readExitCodeFromLog(logPath) {
  * @param {string|null} logTail - Tail of the execution log (already read once)
  * @returns {Object} Possibly updated execution record
  */
+/**
+ * Decide the moment a finished execution ended, and say where that moment came
+ * from (issue #170.2).
+ *
+ * Precedence: the container's own `State.FinishedAt`, then the `Finished:` line
+ * of the footer `start` wrote itself, and only then the time the end was
+ * observed — which is explicitly marked as such, so a consumer can tell a real
+ * finish time from a query-time observation.
+ *
+ * @param {Object} enriched - Record copy being enriched (mutated)
+ * @param {Object|null} dockerState - Inspected docker state, when available
+ * @param {string|null} footerFinishedAt - Footer timestamp, when parseable
+ * @returns {void}
+ */
+function applyEndTime(enriched, dockerState, footerFinishedAt) {
+  if (enriched.endTime) {
+    return;
+  }
+  const dockerFinishedAt = dockerState ? dockerState.finishedAt : null;
+  if (dockerFinishedAt) {
+    enriched.endTime = new Date(dockerFinishedAt).toISOString();
+    enriched.endTimeSource = END_TIME_SOURCE.DOCKER_FINISHED_AT;
+    return;
+  }
+  if (footerFinishedAt) {
+    enriched.endTime = footerFinishedAt;
+    enriched.endTimeSource = END_TIME_SOURCE.LOG_FOOTER;
+    return;
+  }
+  enriched.endTime = new Date().toISOString();
+  enriched.endTimeSource = END_TIME_SOURCE.OBSERVED_AT;
+  enriched.observedAt = enriched.endTime;
+}
+
 function resolveDetachedStatus(record, logTail) {
-  const footerExit = parseExitCodeFromTail(logTail);
+  const footer = parseFooterFromTail(logTail);
+  const footerExit = footer.exitCode;
   const dockerState = isDetachedDockerRecord(record)
     ? readDockerState(record)
     : null;
@@ -268,9 +354,7 @@ function resolveDetachedStatus(record, logTail) {
         const enriched = cloneRecord();
         enriched.status = 'executed';
         enriched.exitCode = footerExit;
-        if (!enriched.endTime) {
-          enriched.endTime = new Date().toISOString();
-        }
+        applyEndTime(enriched, dockerState, footer.finishedAt);
         return enriched;
       }
       if (oomKilled === true) {
@@ -282,9 +366,7 @@ function resolveDetachedStatus(record, logTail) {
         const enriched = cloneRecord();
         enriched.status = 'executed';
         enriched.exitCode = 137;
-        if (!enriched.endTime) {
-          enriched.endTime = new Date().toISOString();
-        }
+        applyEndTime(enriched, dockerState, footer.finishedAt);
         return enriched;
       }
     }
@@ -324,9 +406,7 @@ function resolveDetachedStatus(record, logTail) {
         footerExit ??
         (oomKilled === true ? 137 : -1);
     }
-    if (!enriched.endTime) {
-      enriched.endTime = new Date().toISOString();
-    }
+    applyEndTime(enriched, dockerState, footer.finishedAt);
   }
 
   return enriched;
@@ -562,7 +642,7 @@ function formatRecordAsText(record) {
     `UUID:              ${obj.uuid}`,
     `Status:            ${obj.status}`,
     `Command:           ${obj.command}`,
-    `Exit Code:         ${obj.exitCode !== null ? obj.exitCode : 'N/A'}`,
+    `Exit Code:         ${obj.exitCode !== null ? describeExitCode(obj.exitCode).text : 'N/A'}`,
     ...(obj.oomKilled !== undefined
       ? [`OOM Killed:        ${obj.oomKilled}`]
       : []),
@@ -584,10 +664,19 @@ function formatRecordAsText(record) {
   if (obj.currentTime) {
     lines.push(`Current Time:      ${obj.currentTime}`);
   }
-  lines.push(
-    `End Time:          ${obj.endTime || 'N/A'}`,
-    `Log Path:          ${obj.logPath}`
-  );
+  lines.push(`End Time:          ${obj.endTime || 'N/A'}`);
+  if (obj.endTimeSource) {
+    // Where `End Time` came from: a real finish time, or merely when `start`
+    // noticed the execution was over (issue #170.2).
+    lines.push(`End Time Source:   ${obj.endTimeSource}`);
+  }
+  if (obj.containerStartedAt) {
+    lines.push(`Container Started: ${obj.containerStartedAt}`);
+  }
+  if (obj.staleDetectedAt) {
+    lines.push(`Stale Detected At: ${obj.staleDetectedAt}`);
+  }
+  lines.push(`Log Path:          ${obj.logPath}`);
 
   // Format options as nested list instead of JSON
   const optionEntries = Object.entries(obj.options || {}).filter(
@@ -778,6 +867,8 @@ module.exports = {
   listExecutions,
   isDetachedSessionAlive,
   enrichDetachedStatus,
+  parseFooterFromTail,
+  parseFooterTimestamp,
   readExitCodeFromLog,
   readLogTail,
   attachExitReason,

package/test/execution-store.js

@@ -573,11 +573,14 @@ describe('ExecutionStore cleanupStale', () => {
     const result = store.cleanupStale({ dryRun: false });
     expect(result.cleaned).toBe(1);
 
-    // Verify it's now "executed" with exit code -1
+    // Verify it's now "executed" with exit code -1. The finish time stays
+    // unknown — cleanup records *when the loss was detected* instead of
+    // fabricating a finish time it cannot know (issue #170.3).
     retrieved = store.get(record.uuid);
     expect(retrieved.status).toBe(ExecutionStatus.EXECUTED);
     expect(retrieved.exitCode).toBe(-1);
-    expect(retrieved.endTime).toBeTruthy();
+    expect(retrieved.endTime).toBeNull();
+    expect(retrieved.staleDetectedAt).toBeTruthy();
   });
 
   it('should handle custom max age', () => {

package/test/regression-170.js

@@ -0,0 +1,321 @@
+/**
+ * Regression tests for issue #170: detached Docker executions never persist a
+ * terminal state, and `--status` fabricates `endTime` with `new Date()`.
+ *
+ * The three defects covered here:
+ *   170.1 the detached completion watcher must write the terminal state back
+ *         into the store instead of leaving the record `executing` forever;
+ *   170.2 `--status` must derive `endTime` from a real clock
+ *         (`State.FinishedAt` -> log footer `Finished:`) and mark the
+ *         provenance via `endTimeSource` when it falls back to observation time;
+ *   170.3 `cleanupStale()` must record `staleDetectedAt`, not a fabricated
+ *         finish time.
+ */
+
+const { describe, it, expect, afterEach } = require('bun:test');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const {
+  ExecutionStore,
+  ExecutionRecord,
+  ExecutionStatus,
+} = require('../src/lib/execution-store');
+const { resolveDetachedStatus } = require('../src/lib/status-formatter');
+const {
+  buildDetachedDockerCompletionScript,
+} = require('../src/lib/docker-cleanup');
+const { finalizeDetachedExecution } = require('../src/lib/detached-finalize');
+
+const tempDirs = [];
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function createExecutable(filePath, content) {
+  fs.writeFileSync(filePath, content, 'utf8');
+  fs.chmodSync(filePath, 0o755);
+}
+
+/**
+ * A fake `docker` that answers `inspect` with a fixed state line, whatever the
+ * `-f` template asks for. The template is echoed verbatim by the caller, so the
+ * test controls exactly how many fields the formatter has to cope with.
+ *
+ * Windows gets a `.cmd` batch stub: `spawnSync()` there cannot execute an
+ * extensionless `#!/bin/sh` script, and `--status` resolves the docker facts
+ * with a plain `spawnSync()` on every platform.
+ */
+function withFakeDockerInspect(stateLine, fn) {
+  const fakeBin = makeTempDir('fake-docker-170-');
+  const isWindows = process.platform === 'win32';
+  const dockerPath = path.join(fakeBin, isWindows ? 'docker.cmd' : 'docker');
+  createExecutable(
+    dockerPath,
+    isWindows
+      ? [
+          '@echo off',
+          'if not "%1"=="inspect" exit /b 1',
+          `echo ${stateLine}`,
+          'exit /b 0',
+          '',
+        ].join('\r\n')
+      : [
+          '#!/bin/sh',
+          '[ "$1" = "inspect" ] || exit 1',
+          `echo "${stateLine}"`,
+          '',
+        ].join('\n')
+  );
+  const originalPath = process.env.PATH;
+  const originalDockerBin = process.env.START_DOCKER_BIN;
+  process.env.PATH = `${fakeBin}${path.delimiter}${originalPath || ''}`;
+  process.env.START_DOCKER_BIN = dockerPath;
+  try {
+    return fn();
+  } finally {
+    process.env.PATH = originalPath;
+    if (originalDockerBin === undefined) {
+      delete process.env.START_DOCKER_BIN;
+    } else {
+      process.env.START_DOCKER_BIN = originalDockerBin;
+    }
+  }
+}
+
+function withFakeDockerMissingContainer(fn) {
+  const fakeBin = makeTempDir('fake-docker-gone-170-');
+  const dockerPath = path.join(fakeBin, 'docker');
+  createExecutable(dockerPath, ['#!/bin/sh', 'exit 1', ''].join('\n'));
+  const originalPath = process.env.PATH;
+  const originalDockerBin = process.env.START_DOCKER_BIN;
+  process.env.PATH = `${fakeBin}${path.delimiter}${originalPath || ''}`;
+  process.env.START_DOCKER_BIN = dockerPath;
+  try {
+    return fn();
+  } finally {
+    process.env.PATH = originalPath;
+    if (originalDockerBin === undefined) {
+      delete process.env.START_DOCKER_BIN;
+    } else {
+      process.env.START_DOCKER_BIN = originalDockerBin;
+    }
+  }
+}
+
+function detachedDockerRecord(overrides = {}) {
+  return new ExecutionRecord({
+    pid: null,
+    status: ExecutionStatus.EXECUTING,
+    exitCode: null,
+    command: 'sleep 100',
+    startTime: '2026-09-15T22:21:40.000Z',
+    endTime: null,
+    options: {
+      isolated: 'docker',
+      isolationMode: 'detached',
+      sessionName: 'start-command-170',
+    },
+    ...overrides,
+  });
+}
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    const dir = tempDirs.pop();
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+describe('issue #170.2: endTime is never fabricated silently', () => {
+  it('derives endTime from docker State.FinishedAt for a stopped container', () => {
+    const record = detachedDockerRecord();
+    const enriched = withFakeDockerInspect(
+      'false 137 false 2026-09-15T22:21:40.942007645Z 2026-09-15T22:21:46.740817278Z',
+      () => resolveDetachedStatus(record, null)
+    );
+
+    expect(enriched.status).toBe('executed');
+    expect(enriched.exitCode).toBe(137);
+    expect(enriched.endTimeSource).toBe('docker-finished-at');
+    expect(new Date(enriched.endTime).toISOString()).toBe(
+      '2026-09-15T22:21:46.740Z'
+    );
+  });
+
+  it('falls back to the log footer Finished: line when docker is gone', () => {
+    const record = detachedDockerRecord();
+    const tail = [
+      'output line',
+      '='.repeat(50),
+      'Finished: 2026-09-15 22:21:46.740',
+      'Exit Code: 137',
+      '',
+    ].join('\n');
+
+    const enriched = withFakeDockerMissingContainer(() =>
+      resolveDetachedStatus(record, tail)
+    );
+
+    expect(enriched.status).toBe('executed');
+    expect(enriched.exitCode).toBe(137);
+    expect(enriched.endTimeSource).toBe('log-footer');
+    expect(new Date(enriched.endTime).toISOString()).toBe(
+      '2026-09-15T22:21:46.740Z'
+    );
+  });
+
+  it('marks the observation-time fallback instead of pretending it is a finish time', () => {
+    const record = detachedDockerRecord();
+    const tail = [
+      '='.repeat(50),
+      'Finished: not-a-timestamp',
+      'Exit Code: 3',
+      '',
+    ].join('\n');
+
+    const enriched = withFakeDockerMissingContainer(() =>
+      resolveDetachedStatus(record, tail)
+    );
+
+    expect(enriched.status).toBe('executed');
+    expect(enriched.exitCode).toBe(3);
+    expect(enriched.endTimeSource).toBe('observed-at');
+    expect(enriched.observedAt).toBe(enriched.endTime);
+  });
+
+  it('keeps an already recorded endTime and marks it as persisted', () => {
+    const record = detachedDockerRecord({
+      status: ExecutionStatus.EXECUTED,
+      exitCode: 0,
+      endTime: '2026-09-15T22:21:46.740Z',
+      endTimeSource: 'docker-finished-at',
+    });
+
+    const enriched = withFakeDockerInspect('false 0 false', () =>
+      resolveDetachedStatus(record, null)
+    );
+
+    expect(enriched.endTime).toBe('2026-09-15T22:21:46.740Z');
+    expect(enriched.endTimeSource).toBe('docker-finished-at');
+  });
+});
+
+describe('issue #170.3: cleanupStale records the detection time', () => {
+  it('sets staleDetectedAt and leaves endTime unknown', () => {
+    const appFolder = makeTempDir('stale-170-');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+
+    const record = new ExecutionRecord({
+      pid: 999999,
+      status: ExecutionStatus.EXECUTING,
+      command: 'sleep 1000',
+      startTime: new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString(),
+    });
+    store.save(record);
+
+    const result = store.cleanupStale({ maxAge: 1000 });
+    expect(result.cleaned).toBe(1);
+
+    const cleaned = store.get(record.uuid);
+    expect(cleaned.status).toBe(ExecutionStatus.EXECUTED);
+    expect(cleaned.exitCode).toBe(-1);
+    expect(cleaned.endTime).toBeNull();
+    expect(typeof cleaned.staleDetectedAt).toBe('string');
+    expect(Number.isNaN(Date.parse(cleaned.staleDetectedAt))).toBe(false);
+  });
+});
+
+describe('issue #170.1: the detached watcher persists the terminal state', () => {
+  it('invokes the finalizer with the inspected docker facts', () => {
+    const script = buildDetachedDockerCompletionScript(
+      'start-command-170',
+      'default',
+      '/tmp/start-command-170.log',
+      'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+    );
+
+    expect(script).toContain('detached-finalize');
+    expect(script).toContain('aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee');
+    expect(script).toContain('$__start_command_exit');
+    expect(script).toContain('$__start_command_finished');
+  });
+
+  it('stays backward compatible when no execution id is supplied', () => {
+    const script = buildDetachedDockerCompletionScript(
+      'start-command-170',
+      'default',
+      '/tmp/start-command-170.log'
+    );
+    expect(script).not.toContain('detached-finalize');
+  });
+
+  it('writes status, exitCode, endTime and provenance into the store', () => {
+    const appFolder = makeTempDir('finalize-170-');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const record = detachedDockerRecord();
+    store.save(record);
+
+    const outcome = finalizeDetachedExecution({
+      store,
+      executionId: record.uuid,
+      exitCode: 137,
+      oomKilled: false,
+      startedAt: '2026-09-15T22:21:40.942007645Z',
+      finishedAt: '2026-09-15T22:21:46.740817278Z',
+      containerError: '',
+    });
+
+    expect(outcome.updated).toBe(true);
+
+    const stored = store.get(record.uuid);
+    expect(stored.status).toBe(ExecutionStatus.EXECUTED);
+    expect(stored.exitCode).toBe(137);
+    expect(stored.oomKilled).toBe(false);
+    expect(stored.endTimeSource).toBe('docker-finished-at');
+    expect(new Date(stored.endTime).toISOString()).toBe(
+      '2026-09-15T22:21:46.740Z'
+    );
+    expect(stored.exitReason).toContain('SIGKILL');
+  });
+
+  it('falls back to observation time when docker reports no finish time', () => {
+    const appFolder = makeTempDir('finalize-170-zero-');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const record = detachedDockerRecord();
+    store.save(record);
+
+    finalizeDetachedExecution({
+      store,
+      executionId: record.uuid,
+      exitCode: 0,
+      oomKilled: false,
+      startedAt: 'unknown',
+      finishedAt: '0001-01-01T00:00:00Z',
+      containerError: '',
+    });
+
+    const stored = store.get(record.uuid);
+    expect(stored.status).toBe(ExecutionStatus.EXECUTED);
+    expect(stored.exitCode).toBe(0);
+    expect(stored.endTimeSource).toBe('observed-at');
+    expect(stored.observedAt).toBe(stored.endTime);
+    expect(new Date(stored.endTime).getUTCFullYear()).toBeGreaterThan(2000);
+  });
+
+  it('never fails when the record is gone', () => {
+    const appFolder = makeTempDir('finalize-170-missing-');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const outcome = finalizeDetachedExecution({
+      store,
+      executionId: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
+      exitCode: 1,
+    });
+    expect(outcome.updated).toBe(false);
+    expect(outcome.reason).toBe('record-not-found');
+  });
+});