start-command 0.33.0 → 0.34.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # start-command
 
+## 0.34.0
+
+### Minor Changes
+
+- 22c1158: Persist the terminal state of a detached Docker execution and record the container post-mortem. The completion watcher now writes `status`, `exitCode`, `oomKilled` and a real `endTime` back into the store instead of leaving the record `executing` forever, so `--status` stops calling the clock at query time and reporting a different "finish" on every call. `endTime` carries its provenance in `endTimeSource` (`docker-finished-at`, `log-footer`, `observed-at`); a stale record keeps `endTime` empty and records `staleDetectedAt` instead of a fabricated finish time. The single `docker inspect` now also collects `StartedAt`, `FinishedAt` and `State.Error`, and both the detached and the attached path write a post-mortem block for a kept container and a one-line note for a removed one. Signal decoding (`128+n`) is shared by the log and `--status`, so `137` reads as `137 (SIGKILL - 128+9)` everywhere.
+
 ## 0.33.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.33.0",
+  "version": "0.34.0",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/bin/cli.js

@@ -33,6 +33,7 @@ const {
 } = require('../lib/user-manager');
 const { handleFailure } = require('../lib/failure-handler');
 const { ExecutionStore, ExecutionRecord } = require('../lib/execution-store');
+const { reconcileFinalizedRecord } = require('../lib/detached-finalize');
 const { dispatchQueryCommand, hasQueryMode } = require('../lib/query-commands');
 const { printVersion } = require('../lib/version');
 const { createStartBlock, createFinishBlock } = require('../lib/output-blocks');
@@ -469,6 +470,9 @@ async function runWithIsolation(
       ...buildDockerRuntimeMetadata(options),
       shell: options.shell,
       logPath: logFilePath,
+      // Lets the detached completion watcher write the terminal state back
+      // into this record once the container is gone (issue #170.1).
+      executionId: executionRecord ? executionRecord.uuid : null,
     });
   } else if (createdUser) {
     // Run directly as the created user (no isolation environment)
@@ -504,7 +508,11 @@ async function runWithIsolation(
       executionRecord.complete(exitCode);
     }
     try {
-      store.save(executionRecord);
+      store.save(
+        mode === 'detached'
+          ? reconcileFinalizedRecord(store, executionRecord)
+          : executionRecord
+      );
     } catch (err) {
       if (config.verbose) {
         console.error(

package/src/lib/detached-finalize.js

@@ -0,0 +1,281 @@
+/**
+ * Detached completion finalizer (issue #170)
+ *
+ * A detached Docker session outlives the CLI process that started it: the
+ * foreground `start` invocation returns as soon as the container is up, so
+ * nobody is left to write the terminal state back into the execution store.
+ * The record therefore stayed `executing` forever, with `exitCode: null` and
+ * `endTime: null`, and every `--status` query had to re-derive the outcome from
+ * scratch — fabricating `endTime` with `new Date()` in the process.
+ *
+ * The detached completion watcher (see `docker-cleanup.js`) now re-invokes this
+ * module once the container is gone, handing over the facts it already read out
+ * of a single `docker inspect`. This module is deliberately tiny and
+ * dependency-light: it is executed as a short-lived child process on a host
+ * whose CLI invocation is long gone, so it must never throw and never block.
+ */
+
+const path = require('path');
+
+const {
+  SHELL_VARS,
+  describeExitCode,
+  normalizeDockerTimestamp,
+} = require('./docker-post-mortem');
+const { shellQuote } = require('./isolation-log-utils');
+
+/** Provenance markers for `endTime` (issue #170.2). */
+const END_TIME_SOURCE = {
+  /** `docker inspect .State.FinishedAt` — the container's own clock. */
+  DOCKER_FINISHED_AT: 'docker-finished-at',
+  /** The anchored `Finished:` line `start` itself wrote into the log. */
+  LOG_FOOTER: 'log-footer',
+  /** No real finish time exists: this is when the end was *observed*. */
+  OBSERVED_AT: 'observed-at',
+};
+
+/**
+ * Persist the terminal state of a detached execution.
+ *
+ * Purely additive with respect to the observation-vs-verdict discipline of
+ * issues #148/#151/#162: `oomKilled` and `exitReason` are recorded as facts
+ * next to the exit code, never as a substitute for it.
+ *
+ * @param {Object} options - Finalization input
+ * @param {Object} options.store - An `ExecutionStore`
+ * @param {string} options.executionId - UUID (or session name) of the record
+ * @param {number|string|null} [options.exitCode] - `.State.ExitCode`
+ * @param {boolean|string|null} [options.oomKilled] - `.State.OOMKilled`
+ * @param {string|null} [options.startedAt] - `.State.StartedAt`
+ * @param {string|null} [options.finishedAt] - `.State.FinishedAt`
+ * @param {string|null} [options.containerError] - `.State.Error`
+ * @returns {{updated: boolean, reason?: string, record?: Object}} Outcome
+ */
+function finalizeDetachedExecution(options = {}) {
+  const { store, executionId } = options;
+  if (!store || !executionId) {
+    return { updated: false, reason: 'missing-arguments' };
+  }
+
+  let record;
+  try {
+    record = store.get(executionId);
+  } catch {
+    return { updated: false, reason: 'store-unreadable' };
+  }
+  if (!record) {
+    return { updated: false, reason: 'record-not-found' };
+  }
+  if (record.status === 'executed' && record.endTime) {
+    // Already finalized (e.g. a `--status` query got there first, or the
+    // watcher ran twice after a resume). Nothing to correct.
+    return { updated: false, reason: 'already-final', record };
+  }
+
+  const described = describeExitCode(options.exitCode);
+  record.status = 'executed';
+  if (described.code !== null) {
+    record.exitCode = described.code;
+  } else if (record.exitCode === null || record.exitCode === undefined) {
+    record.exitCode = -1;
+  }
+
+  const finishedAt = normalizeDockerTimestamp(options.finishedAt);
+  if (finishedAt) {
+    record.endTime = new Date(finishedAt).toISOString();
+    record.endTimeSource = END_TIME_SOURCE.DOCKER_FINISHED_AT;
+  } else {
+    // Docker has no finish time for this container (it never started, or it
+    // was removed before we could look). Record *when we noticed*, and say so,
+    // rather than passing observation time off as a finish time (issue #170.2).
+    record.endTime = new Date().toISOString();
+    record.endTimeSource = END_TIME_SOURCE.OBSERVED_AT;
+    record.observedAt = record.endTime;
+  }
+
+  const startedAt = normalizeDockerTimestamp(options.startedAt);
+  if (startedAt) {
+    record.containerStartedAt = startedAt;
+  }
+
+  const oomKilled = normalizeBoolean(options.oomKilled);
+  if (oomKilled !== null) {
+    record.oomKilled = oomKilled;
+  }
+
+  const exitReason = resolveReason(record);
+  if (exitReason) {
+    record.exitReason = exitReason;
+  }
+
+  const containerError = normalizeContainerError(options.containerError);
+  if (containerError) {
+    record.options = { ...(record.options || {}), containerError };
+  }
+
+  try {
+    store.save(record);
+  } catch (err) {
+    return { updated: false, reason: `save-failed: ${err.message}`, record };
+  }
+  return { updated: true, record };
+}
+
+/**
+ * Derive the `exitReason` hint from the same evidence `--status` would use.
+ * Kept in a helper so a failure to read the log never aborts finalization.
+ * @param {Object} record - Execution record being finalized
+ * @returns {string|null} Reason hint, or null
+ */
+function resolveReason(record) {
+  try {
+    const { resolveExitReason } = require('./exit-reason');
+    const {
+      FATAL_MARKER_TAIL_BYTES,
+      readLogTail,
+    } = require('./isolation-log-utils');
+    const logTail = record.logPath
+      ? readLogTail(record.logPath, FATAL_MARKER_TAIL_BYTES)
+      : null;
+    return resolveExitReason({
+      exitCode: record.exitCode,
+      logTail,
+      oomKilled: record.oomKilled,
+    });
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * @param {boolean|string|null|undefined} value - Raw shell-provided flag
+ * @returns {boolean|null} Parsed flag, or null when it carries no information
+ */
+function normalizeBoolean(value) {
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  if (value === 'true') {
+    return true;
+  }
+  if (value === 'false') {
+    return false;
+  }
+  return null;
+}
+
+/**
+ * @param {string|null|undefined} value - Raw `.State.Error`
+ * @returns {string|null} Error text, or null when docker reported none
+ */
+function normalizeContainerError(value) {
+  if (typeof value !== 'string') {
+    return null;
+  }
+  const trimmed = value.trim();
+  if (trimmed === '' || trimmed === '<no value>' || trimmed === '(none)') {
+    return null;
+  }
+  return trimmed;
+}
+
+/**
+ * Reconcile an in-memory record with one the detached watcher already
+ * finalized.
+ *
+ * A container that exits almost immediately can be finalized by the watcher
+ * before the foreground `start` invocation writes its own last update. Saving
+ * the stale in-memory copy would resurrect `status: executing` and throw the
+ * terminal facts away, so the stored record wins and only the fields the
+ * foreground process learned (its `options`, e.g. `containerId`) are merged in.
+ *
+ * @param {Object} store - An `ExecutionStore`
+ * @param {Object} record - The in-memory record about to be saved
+ * @returns {Object} The record to save
+ */
+function reconcileFinalizedRecord(store, record) {
+  try {
+    const stored = store.get(record.uuid);
+    if (!stored || stored.status !== 'executed' || !stored.endTime) {
+      return record;
+    }
+    stored.options = { ...(stored.options || {}), ...(record.options || {}) };
+    return stored;
+  } catch {
+    return record;
+  }
+}
+
+/**
+ * Shell fragment that hands the inspected docker facts to this module.
+ *
+ * Runs with the same runtime that started the session, inherits
+ * `START_APP_FOLDER`/`START_DISABLE_TRACKING` from the detached watcher, and is
+ * suffixed with `|| true` so a finalization failure can never abort the
+ * watcher's remaining cleanup work.
+ *
+ * @param {string} executionId - UUID of the record to finalize
+ * @returns {string} POSIX shell fragment
+ */
+function buildDetachedFinalizeSnippet(executionId) {
+  const v = SHELL_VARS;
+  return (
+    `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
+    `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
+    `"$${v.started}" "$${v.finished}" "$${v.error}" ` +
+    `>/dev/null 2>&1 || true`
+  );
+}
+
+/**
+ * Entry point used by the detached watcher:
+ *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error>
+ * Always exits 0 — a bookkeeping failure must never turn into a visible error
+ * in a log the user is reading for the command's own output.
+ * @param {string[]} argv - Positional arguments
+ * @returns {void}
+ */
+function main(argv) {
+  const [
+    executionId,
+    exitCode,
+    oomKilled,
+    startedAt,
+    finishedAt,
+    containerError,
+  ] = argv;
+  if (!executionId || process.env.START_DISABLE_TRACKING === 'true') {
+    return;
+  }
+  try {
+    const { ExecutionStore } = require('./execution-store');
+    const store = new ExecutionStore(
+      process.env.START_APP_FOLDER
+        ? { appFolder: path.resolve(process.env.START_APP_FOLDER) }
+        : {}
+    );
+    finalizeDetachedExecution({
+      store,
+      executionId,
+      exitCode,
+      oomKilled,
+      startedAt,
+      finishedAt,
+      containerError,
+    });
+  } catch {
+    // Deliberately silent: see the exit-0 contract above.
+  }
+}
+
+if (require.main === module) {
+  main(process.argv.slice(2));
+}
+
+module.exports = {
+  END_TIME_SOURCE,
+  buildDetachedFinalizeSnippet,
+  finalizeDetachedExecution,
+  main,
+  reconcileFinalizedRecord,
+};

package/src/lib/docker-cleanup.js

@@ -7,6 +7,16 @@ const {
   shellQuote,
 } = require('./isolation-log-utils');
 const { resolveMemoryExhaustion } = require('./exit-reason');
+const {
+  buildDockerPostMortemSnippet,
+  buildDockerRemovalNoteSnippet,
+  buildDockerStateSnippet,
+  DOCKER_STATE_INSPECT_FORMAT,
+  formatContainerPostMortem,
+  formatContainerRemovalNote,
+  normalizeDockerTimestamp,
+} = require('./docker-post-mortem');
+const { buildDetachedFinalizeSnippet } = require('./detached-finalize');
 
 const DOCKER_CONTAINER_CLEANUP_POLICY = {
   DEFAULT: 'default',
@@ -91,27 +101,85 @@ function appendDockerContainerCleanupPolicyMessage(
   return `${message}\nContainer will be removed after command completes.`;
 }
 
-function readDockerContainerOomKilled(containerName) {
-  const result = spawnSync(
-    getDockerCommand(),
-    ['inspect', '-f', '{{.State.OOMKilled}}', containerName],
-    getDockerSpawnOptions({
-      encoding: 'utf8',
-      env: process.env,
-      stdio: ['pipe', 'pipe', 'pipe'],
-    })
-  );
-  if (result.error || result.status !== 0) {
+/**
+ * Read a finished container's post-mortem facts in a single `docker inspect`.
+ *
+ * The attached path only ever inspected `State.OOMKilled`, which left its
+ * "Container kept for investigation" message unable to say *why* the container
+ * died — the same gap the detached watcher had (issue #171.1). Reading the
+ * facts once, before any `docker rm`, also makes them available to the removal
+ * path, where the container is about to stop existing (issue #171.3).
+ *
+ * @param {string} containerName - Container to inspect
+ * @returns {object|null} Facts accepted by the `docker-post-mortem` formatters,
+ *   or null when the container could not be inspected at all
+ */
+function readDockerContainerState(containerName) {
+  const inspect = (format) => {
+    const result = spawnSync(
+      getDockerCommand(),
+      ['inspect', '-f', format, containerName],
+      getDockerSpawnOptions({
+        encoding: 'utf8',
+        env: process.env,
+        stdio: ['pipe', 'pipe', 'pipe'],
+      })
+    );
+    if (result.error || result.status !== 0) {
+      return null;
+    }
+    return String(result.stdout || '').trim();
+  };
+  const state = inspect(DOCKER_STATE_INSPECT_FORMAT);
+  if (state === null) {
     return null;
   }
-  const value = String(result.stdout || '').trim();
-  if (value === 'true') {
-    return true;
+  const [exitCode, oomKilled, startedAt, finishedAt] = state.split(/\s+/);
+  // `State.Error` is free-form text that may contain spaces, so it cannot ride
+  // along in the whitespace-separated template above.
+  const error = inspect('{{.State.Error}}');
+  const parsedExit = Number.parseInt(exitCode, 10);
+  return {
+    containerName,
+    exitCode: Number.isFinite(parsedExit) ? parsedExit : null,
+    oomKilled:
+      oomKilled === 'true' ? true : oomKilled === 'false' ? false : null,
+    startedAt: normalizeDockerTimestamp(startedAt),
+    finishedAt: normalizeDockerTimestamp(finishedAt),
+    error: error || null,
+  };
+}
+
+/**
+ * Write the post-mortem for an attached run into its log and return the same
+ * text for the console message.
+ *
+ * Kept and removed containers both get their facts recorded — a kept container
+ * gets the full block, a removed one the single line — so the log of an
+ * attached run carries exactly what the detached watcher writes (issues #171.2
+ * and #171.3).
+ *
+ * @param {object} params - Container name, inspected state, log path, and
+ *   whether the container was removed
+ * @returns {string} Message lines to append, or '' when nothing is known
+ */
+function recordAttachedDockerPostMortem({
+  containerName,
+  state,
+  logPath = null,
+  removed = false,
+}) {
+  if (!state) {
+    return '';
   }
-  if (value === 'false') {
-    return false;
+  const facts = { ...state, containerName };
+  const text = removed
+    ? formatContainerRemovalNote(facts)
+    : formatContainerPostMortem(facts);
+  if (logPath) {
+    appendLogFile(logPath, `\n${text}`);
   }
-  return null;
+  return `\n${text.trimEnd()}`;
 }
 
 function readDockerContainerStatus(containerName) {
@@ -231,37 +299,68 @@ function buildSuccessfulNonOomCondition() {
   );
 }
 
-function buildDetachedDockerCompletionScript(containerName, policy, logPath) {
+/**
+ * Build the POSIX shell run by the detached completion watcher.
+ *
+ * The watcher is the only observer left once `start --detached` returns, so it
+ * carries three responsibilities that used to be missing or deferred:
+ *   1. it reads the container's post-mortem facts out of a single
+ *      `docker inspect` (issue #171.1);
+ *   2. it writes those facts into the log on every path — kept *and* removed —
+ *      so an operator reading the log never has to reconstruct them from a
+ *      container that no longer exists (issues #171.2, #171.3);
+ *   3. it hands the same facts to the finalizer so the execution record stops
+ *      being `executing` forever (issue #170.1).
+ *
+ * @param {string} containerName - Docker container name
+ * @param {string} policy - One of DOCKER_CONTAINER_CLEANUP_POLICY
+ * @param {string|null} logPath - Log file to append to, or null
+ * @param {string|null} [executionId] - Execution UUID to finalize, or null
+ * @returns {string} The shell script
+ */
+function buildDetachedDockerCompletionScript(
+  containerName,
+  policy,
+  logPath,
+  executionId = null
+) {
   const quotedName = shellQuote(containerName);
   const parts = [];
 
   if (logPath) {
     const quotedLogPath = shellQuote(logPath);
-    parts.push(`docker logs -f ${quotedName} >> ${quotedLogPath} 2>&1`);
-    parts.push(
-      `__start_command_state=$(docker inspect -f '{{.State.ExitCode}} {{.State.OOMKilled}}' ${quotedName} 2>/dev/null || printf '%s' '-1 false')`
+    const removalNote = buildDockerRemovalNoteSnippet(
+      containerName,
+      quotedLogPath
+    );
+    const postMortem = buildDockerPostMortemSnippet(
+      containerName,
+      quotedLogPath
     );
-    parts.push('__start_command_exit=${__start_command_state%% *}');
-    parts.push('__start_command_oom=${__start_command_state##* }');
+    const remove = `docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true; ${removalNote}`;
+    const keep = `${postMortem}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
+
+    parts.push(`docker logs -f ${quotedName} >> ${quotedLogPath} 2>&1`);
+    parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
-      parts.push(`docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true`);
+      parts.push(remove);
     } else if (
       policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT ||
       policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
     ) {
-      const successCondition = buildSuccessfulNonOomCondition();
       parts.push(
-        `if ${successCondition}; then docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true; else ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}; fi`
+        `if ${buildSuccessfulNonOomCondition()}; then ${remove}; else ${keep}; fi`
       );
+    } else {
+      // KEEP: the container is never removed, so the log used to end with the
+      // raw command output and nothing else. The post-mortem is exactly the
+      // information an operator needs before deciding what to do with it.
+      parts.push(keep);
     }
     parts.push(`${createShellLogFooterSnippet()} >> ${quotedLogPath}`);
   } else {
     parts.push(`docker wait ${quotedName} >/dev/null 2>&1`);
-    parts.push(
-      `__start_command_state=$(docker inspect -f '{{.State.ExitCode}} {{.State.OOMKilled}}' ${quotedName} 2>/dev/null || printf '%s' '-1 false')`
-    );
-    parts.push('__start_command_exit=${__start_command_state%% *}');
-    parts.push('__start_command_oom=${__start_command_state##* }');
+    parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
       parts.push(`docker rm -f ${quotedName} >/dev/null 2>&1 || true`);
     } else if (
@@ -274,13 +373,39 @@ function buildDetachedDockerCompletionScript(containerName, policy, logPath) {
     }
   }
 
+  if (executionId) {
+    // Last, so the record is only marked terminal once the log is complete.
+    parts.push(buildDetachedFinalizeSnippet(executionId));
+  }
+
   return parts.join('; ');
 }
 
-function startDetachedDockerCompletionWatcher(containerName, policy, logPath) {
+/**
+ * Start the detached completion watcher.
+ * @param {string} containerName - Docker container name
+ * @param {string} policy - One of DOCKER_CONTAINER_CLEANUP_POLICY
+ * @param {string|null} logPath - Log file to append to, or null
+ * @param {string|null} [executionId] - Execution UUID to finalize, or null
+ * @returns {void}
+ */
+function startDetachedDockerCompletionWatcher(
+  containerName,
+  policy,
+  logPath,
+  executionId = null
+) {
   const watcher = spawn(
     'sh',
-    ['-c', buildDetachedDockerCompletionScript(containerName, policy, logPath)],
+    [
+      '-c',
+      buildDetachedDockerCompletionScript(
+        containerName,
+        policy,
+        logPath,
+        executionId
+      ),
+    ],
     {
       detached: true,
       stdio: 'ignore',
@@ -327,8 +452,9 @@ module.exports = {
   shouldCleanupDockerContainer,
   getDockerContainerCleanupInstructions,
   appendDockerContainerCleanupPolicyMessage,
-  readDockerContainerOomKilled,
+  readDockerContainerState,
   readDockerContainerStatus,
+  recordAttachedDockerPostMortem,
   removeDockerContainer,
   buildDetachedDockerCompletionScript,
   startDetachedDockerCompletionWatcher,