start-command 0.40.0 → 0.42.0

package/CHANGELOG.md

@@ -1,5 +1,27 @@
 # start-command
 
+## 0.42.0
+
+### Minor Changes
+
+- 5ebb7ec: `--use-formal-ai`: keep the cause of Formal AI server errors, offer structured file tools, and report capability gaps as incomplete (#257, #258, #259).
+
+  - **Server errors (#257):** an HTTP error from the Formal AI runtime now keeps its sanitized, bounded cause: the error type (100 chars), the message (300), a body preview (500), the body length and the number of attempts. The cause appears in the final message (`Formal AI returned HTTP 500: formal_ai_error: Aborted(...) (after 1 retry)`) and under `outcome.diagnostics` in the evidence. Authorization values, credentials, control characters and the environment are never recorded. There is still no fallback, the exit status is still 70, and 4xx responses are not retried.
+  - **File tools (#258):** next to `bash`, Formal AI is offered a structured `read` tool, plus `edit` (one literal replacement) and `write` (replace a file it has read) in `START_FORMAL_AI_EFFECTS=workspace-write` mode. The tools follow the workspace path policy: no paths outside the working directory, and a change only to a file read in the same task that has not changed since. They keep comments, whitespace, line endings, a byte order mark, the file mode and all unrelated content. Each change is checked by reading the file back, and the evidence records its hashes and effects. Files over `START_FORMAL_AI_MAX_FILE_BYTES` (1 MiB by default, at most 16 MiB) are refused, and interpreters are still not offered.
+  - **Capability gaps (#258):** an answer that matches one of Formal AI's known capability-gap records (`workspace-edit-capability-gap/v1` or `workspace-change-verification-gap/v1`, with `complete` and `verified` both `false`) is now reported as `capability-gap` with exit 65. Its remaining obligations are kept. Plain answers to questions still exit 0.
+
+## 0.41.0
+
+### Minor Changes
+
+- 65ce70c: Add the experimental `--use-formal-ai` mode: `$ --use-formal-ai -- <request>` sends a natural-language request to a local Formal AI runtime and runs the tool calls it chooses (#255).
+
+  - **Opt-in only:** without the flag nothing changes. The request after `--` is never read by a shell or the alias table. Without a qualified runtime (Formal AI 0.355.27 or newer, from `START_FORMAL_AI_ROOT` or an explicit `START_FORMAL_AI_URL`), start exits 69 and runs nothing. There is no fallback to aliases, a shell or an LLM.
+  - **Tools:** a small read-only set of programs, run without a shell. It includes `ls`, `cat`, `head`, `tail`, `wc`, `find`, `grep`, `sort`, `diff`, `stat`, `tree`, `du` and read-only `git` (see docs/formal-ai.md). Changes (`mkdir`, `touch`, `rmdir`, `rm`, `cp`, `mv`) need `START_FORMAL_AI_EFFECTS=workspace-write`; a change may only name targets observed before it and is verified by readback. Shell syntax, other programs and paths outside the working directory are refused (exit 77).
+  - **Bounds:** steps, retries, total time, per-tool time, output bytes and request length (`START_FORMAL_AI_*`). Exhausted budgets exit 124 and unsupported requests exit 65. A cancelled run ends with the signal status.
+  - **Evidence:** the log records the runtime identity, every response, tool decision, exit status and file-system effect. The log stays local: this mode never uploads logs or creates issues. Tool output appears in the evidence only as a short preview, and the environment is not recorded.
+  - **Dependencies:** `lino-objects-codec` 0.9.4 and `eslint` 10.14.0; CI runs on Node.js 26.
+
 ## 0.40.0
 
 ### Minor Changes

package/eslint.config.mjs

@@ -59,6 +59,36 @@ export default [
       'spaced-comment': ['error', 'always', { markers: ['/'] }],
     },
   },
+  {
+    // CommonJS test fixtures and helpers (.cjs): the test runner collects
+    // every .js/.mjs file under test/, so non-test modules there use .cjs.
+    files: ['**/*.cjs'],
+    plugins: {
+      prettier: prettierPlugin,
+    },
+    languageOptions: {
+      ecmaVersion: 'latest',
+      sourceType: 'commonjs',
+      globals: {
+        require: 'readonly',
+        module: 'readonly',
+        exports: 'readonly',
+        __dirname: 'readonly',
+        __filename: 'readonly',
+        console: 'readonly',
+        process: 'readonly',
+        Buffer: 'readonly',
+      },
+    },
+    rules: {
+      'prettier/prettier': 'error',
+      eqeqeq: ['error', 'always'],
+      curly: ['error', 'all'],
+      'no-var': 'error',
+      'prefer-const': 'error',
+      'prefer-template': 'error',
+    },
+  },
   {
     // ES module files (.mjs)
     files: ['**/*.mjs'],
@@ -112,6 +142,7 @@ export default [
     files: [
       'test/**/*.js',
       'test/**/*.mjs',
+      'test/**/*.cjs',
       'tests/**/*.js',
       'experiments/**/*.js',
     ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.40.0",
+  "version": "0.42.0",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {
@@ -49,7 +49,7 @@
   "devDependencies": {
     "@changesets/cli": "^3.0.3",
     "@eslint/js": "^10.0.1",
-    "eslint": "^10.13.0",
+    "eslint": "^10.14.0",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-prettier": "^5.5.7",
     "husky": "^9.1.7",
@@ -70,7 +70,7 @@
   "dependencies": {
     "@link-foundation/sensitive-data-sanitizer": "^0.12.2",
     "command-stream": "^2.6.0",
-    "lino-objects-codec": "^0.9.3"
+    "lino-objects-codec": "^0.9.5"
   },
   "overrides": {
     "brace-expansion": "^5.0.12",

package/src/bin/cli.js

@@ -223,7 +223,9 @@ if (!isQueryMode && (!parsedCommand || parsedCommand.trim() === '')) {
 let command = parsedCommand;
 let substitutionResult = null;
 
-if (!config.disableSubstitutions && parsedCommand && parsedCommand.trim()) {
+// A --use-formal-ai request is only ever read by Formal AI (issue #255).
+const useFormalAi = wrapperOptions.useFormalAi;
+if (!config.disableSubstitutions && !useFormalAi && parsedCommand?.trim()) {
   substitutionResult = processCommand(parsedCommand, {
     customLinoPath: config.substitutionsPath,
     verbose: config.verbose,
@@ -270,6 +272,14 @@ let launch = null;
     console.error(`Error: ${error.message}`);
     process.exit(1);
   }
+  if (useFormalAi) {
+    const { requestText, runFormalAiCli } = require('../lib/formal-ai-cli');
+    const request = requestText(parsedArgs.rawCommand);
+    const store = getExecutionStore();
+    process.exit(
+      await runFormalAiCli({ request, sessionId, launch, store, config })
+    );
+  }
 
   if (wrapperOptions.isolated === 'docker') {
     const {

package/src/lib/args-parser.js

@@ -27,6 +27,7 @@
  * --keep-container-on-fail         Remove successful docker containers, keep failed or OOM-killed ones
  * --shell <shell>                  Shell to use in isolation environments: auto, bash, zsh, sh (default: auto)
  * --use-command-stream             Use command-stream library for command execution (experimental)
+ * --use-formal-ai                  Perform a natural-language request through Formal AI (experimental)
  * --verbose                        Enable verbose/debug output (sets START_VERBOSE=1)
  * --status <uuid>                  Show status of a previous command execution by UUID
  * --list [--running]               List all (or only running) tracked command executions
@@ -55,6 +56,7 @@ const { parseSequence, isSequence } = require('./sequence-parser');
 const { buildCommandString } = require('./shell-utils');
 const {
   VALID_OUTPUT_FORMATS,
+  activeQueryModes,
   createQueryOptionDefaults,
   parseQueryOption,
   validateQueryOptions,
@@ -220,6 +222,7 @@ function parseArgs(args) {
     keepContainerOnFail: false, // Keep docker container filesystem when command fails or OOM-kills
     shell: 'auto', // Shell to use in isolation environments: auto, bash, zsh, sh
     useCommandStream: false, // Use command-stream library for command execution
+    useFormalAi: false, // Natural-language request performed through Formal AI (issue #255)
     ...createQueryOptionDefaults(),
     ...snapshotResumeOptions.createSnapshotOptionDefaults(),
     ...launchIdentityOptions.createLaunchIdentityOptionDefaults(),
@@ -557,6 +560,11 @@ function parseOption(args, index, options) {
     return 1;
   }
 
+  if (arg === '--use-formal-ai') {
+    options.useFormalAi = true;
+    return 1;
+  }
+
   // --verbose (enable verbose/debug output, sets START_VERBOSE env var)
   if (arg === '--verbose') {
     process.env.START_VERBOSE = '1';
@@ -849,6 +857,7 @@ function validateOptions(options) {
   snapshotResumeOptions.validate(options);
   validateQueryOptions(options);
   launchIdentityOptions.validate(options);
+  validateFormalAi(options);
 
   // Validate shell option
   if (options.shell !== null && options.shell !== undefined) {
@@ -869,6 +878,30 @@ function validateOptions(options) {
   }
 }
 
+/**
+ * `--use-formal-ai` runs its tool calls directly in the current directory;
+ * other execution modes would change where and how they run (issue #255).
+ * @param {object} options - Parsed options
+ * @throws {Error} When combined with an unsupported option
+ */
+function validateFormalAi(options) {
+  if (!options.useFormalAi) {
+    return;
+  }
+  const conflicts = [
+    ...(options.isolated !== null ? ['--isolated'] : []),
+    ...(options.user ? ['--isolated-user'] : []),
+    ...(options.detached ? ['--detached'] : []),
+    ...(options.useCommandStream ? ['--use-command-stream'] : []),
+    ...activeQueryModes(options),
+  ];
+  if (conflicts.length) {
+    throw new Error(
+      `--use-formal-ai cannot be combined with ${conflicts.join(', ')} (experimental mode, see docs/formal-ai.md)`
+    );
+  }
+}
+
 /**
  * Generate a unique session name
  * @param {string} [prefix='start'] - Prefix for the session name

package/src/lib/formal-ai-cli.js

@@ -0,0 +1,237 @@
+/**
+ * CLI side of the experimental `--use-formal-ai` mode (issue #255): resolves
+ * the bounds and the runtime, tracks the execution like a direct run (start
+ * and finish blocks, log, execution record, START_* environment) and writes
+ * the evidence of the run into its log.
+ *
+ * The log stays local: this mode never creates issues or uploads logs.
+ */
+
+const process = require('process');
+const { EXIT, resolveBounds } = require('./formal-ai-policy');
+const runtimeLib = require('./formal-ai-runtime');
+const { createClient } = require('./formal-ai-client');
+const { runFormalAiTask } = require('./formal-ai');
+const { ExecutionRecord } = require('./execution-store');
+const { createStartBlock, createFinishBlock } = require('./output-blocks');
+const { appendLogFile } = require('./isolation-log-utils');
+const launchIdentity = require('./launch-identity');
+
+const EVIDENCE_SCHEMA = 'start-formal-ai-evidence/1';
+const PREFIX = '[formal-ai]';
+const { AbortController } = globalThis;
+
+function timestamp() {
+  return new Date().toISOString().replace('T', ' ').substring(0, 23);
+}
+
+/**
+ * Text of the request: the argv after `--`, joined by single spaces. It is
+ * only ever sent to Formal AI, never to a shell.
+ * @param {string[]} rawCommand - Parsed command argv
+ * @returns {string}
+ */
+function requestText(rawCommand) {
+  return rawCommand.join(' ').trim();
+}
+
+/**
+ * Run `--use-formal-ai`.
+ * @param {object} options
+ * @param {string} options.request - Request text
+ * @param {string} options.sessionId - Execution UUID
+ * @param {object} options.launch - prepareLaunch() result
+ * @param {object|null} options.store - ExecutionStore, or null
+ * @param {object} [options.config] - CLI configuration
+ * @param {object} [options.env] - Environment (process.env)
+ * @param {string} [options.cwd] - Working directory (process.cwd())
+ * @param {object} [options.out] - stdout stream
+ * @param {object} [options.err] - stderr stream
+ * @returns {Promise<number>} Exit code
+ */
+async function runFormalAiCli(options) {
+  const {
+    request,
+    sessionId,
+    launch,
+    store,
+    config = {},
+    env = process.env,
+    cwd = process.cwd(),
+    out = process.stdout,
+    err = process.stderr,
+  } = options;
+  const note = (text) =>
+    err.write(`${text.replace(/^/gm, `${PREFIX} `).replace(/ +$/gm, '')}\n`);
+  let bounds;
+  try {
+    bounds = resolveBounds(env);
+  } catch (error) {
+    note(`Error: ${error.message}`);
+    return error.exitCode || EXIT.CONFIG;
+  }
+  if (!request) {
+    note('Error: --use-formal-ai needs a request after --');
+    return EXIT.USAGE;
+  }
+  if (request.length > bounds.maxRequestChars) {
+    note(
+      `Error: the request has ${request.length} characters; START_FORMAL_AI_MAX_REQUEST_CHARS is ${bounds.maxRequestChars}`
+    );
+    return EXIT.USAGE;
+  }
+  const deadline = Date.now() + bounds.timeoutMs;
+  let runtime;
+  try {
+    runtime = await runtimeLib.startRuntime({ env, bounds });
+  } catch (error) {
+    note(`Error: ${error.message}`);
+    note(
+      'No command was run: --use-formal-ai never falls back to aliases, a shell or an LLM.'
+    );
+    return error.exitCode || EXIT.UNAVAILABLE;
+  }
+  note(`runtime: ${runtimeLib.describeIdentity(runtime.identity)}`);
+  note(`effects: ${bounds.effects}`);
+
+  const logPath = launch.logPath;
+  const startTime = timestamp();
+  const startMs = Date.now();
+  const record = store
+    ? new ExecutionRecord({
+        uuid: sessionId,
+        command: request,
+        logPath,
+        shell: 'formal-ai',
+        workingDirectory: cwd,
+        options: {
+          formalAi: {
+            request,
+            effects: bounds.effects,
+            runtime: runtime.identity,
+          },
+        },
+      })
+    : null;
+  if (record) {
+    launchIdentity.applyLaunch(record, launch);
+    launchIdentity.claimDirectLaunch(store, record, launch, config);
+  }
+  const header = [
+    '=== Start Command Log ===',
+    `Timestamp: ${startTime}`,
+    `Session ID: ${sessionId}`,
+    'Mode: --use-formal-ai (experimental)',
+    `Request: ${request}`,
+    `Formal AI: ${runtimeLib.describeIdentity(runtime.identity)}`,
+    `Effects: ${bounds.effects}`,
+    `Platform: ${process.platform}`,
+    `Working Directory: ${cwd}`,
+    '='.repeat(50),
+    '',
+    '',
+  ].join('\n');
+  launchIdentity.writeLaunchLog(logPath, header, launch, record);
+  const logLine = (text) => appendLogFile(logPath, text);
+
+  out.write(
+    `${createStartBlock({ sessionId, timestamp: startTime, command: `--use-formal-ai -- ${request}` })}\n\n`
+  );
+
+  const evidence = {
+    schema: EVIDENCE_SCHEMA,
+    sessionId,
+    runtime: runtime.identity,
+  };
+  const controller = new AbortController();
+  let finalized = false;
+  const finalize = (exitCode) => {
+    if (finalized) {
+      return;
+    }
+    finalized = true;
+    runtime.stop();
+    if (!evidence.outcome) {
+      evidence.outcome = {
+        status: 'cancelled',
+        exitCode,
+        message: 'start exited before the task finished',
+        remainingObligations: [
+          'the task was interrupted; later steps did not run',
+        ],
+      };
+    }
+    evidence.durationMs = Date.now() - startMs;
+    const endTime = timestamp();
+    logLine(
+      `\n=== Formal AI Evidence ===\n${JSON.stringify(evidence, null, 2)}\n` +
+        `\n${'='.repeat(50)}\nFinished: ${endTime}\nExit Code: ${exitCode}\n`
+    );
+    if (record) {
+      record.options.formalAi.outcome = evidence.outcome.status;
+      record.complete(exitCode);
+      try {
+        store.save(record);
+      } catch (error) {
+        if (config.verbose) {
+          note(
+            `[Tracking] Warning: Could not save execution record: ${error.message}`
+          );
+        }
+      }
+    }
+    return endTime;
+  };
+  // Signals end start through process.exit() in the CLI; finish the
+  // evidence, stop the tool's process group and the runtime synchronously.
+  const onExit = (code) => {
+    if (evidence.activeKill) {
+      evidence.activeKill('SIGKILL');
+    }
+    finalize(code);
+  };
+  process.on('exit', onExit);
+  try {
+    const client = createClient({ baseUrl: runtime.baseUrl, bounds, deadline });
+    await runFormalAiTask({
+      request,
+      cwd,
+      env: launchIdentity.directLaunchEnv(sessionId, launch),
+      bounds,
+      client,
+      deadline,
+      signal: controller.signal,
+      evidence,
+      io: {
+        stdout: out,
+        stderr: err,
+        note: (text) => {
+          note(text);
+          logLine(`${PREFIX} ${text}\n`);
+        },
+      },
+    });
+    evidence.formalAiRequests = client.requests;
+  } catch (error) {
+    evidence.outcome = {
+      status: 'internal-error',
+      exitCode: 1,
+      message: error.message,
+      remainingObligations: ['start failed; the task state is unknown'],
+    };
+  } finally {
+    process.removeListener('exit', onExit);
+  }
+  const { outcome } = evidence;
+  note(`${outcome.status}: ${outcome.message}`);
+  for (const obligation of outcome.remainingObligations) {
+    note(`remaining: ${obligation}`);
+  }
+  const endTime = finalize(outcome.exitCode);
+  out.write(
+    `\n${createFinishBlock({ sessionId, timestamp: endTime, exitCode: outcome.exitCode, logPath, durationMs: evidence.durationMs })}\n`
+  );
+  return outcome.exitCode;
+}
+
+module.exports = { EVIDENCE_SCHEMA, requestText, runFormalAiCli };