start-command 0.39.2 → 0.41.0

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # start-command
 
+## 0.41.0
+
+### Minor Changes
+
+- 65ce70c: Add the experimental `--use-formal-ai` mode: `$ --use-formal-ai -- <request>` sends a natural-language request to a local Formal AI runtime and runs the tool calls it chooses (#255).
+
+  - **Opt-in only:** without the flag nothing changes. The request after `--` is never read by a shell or the alias table. Without a qualified runtime (Formal AI 0.355.27 or newer, from `START_FORMAL_AI_ROOT` or an explicit `START_FORMAL_AI_URL`), start exits 69 and runs nothing. There is no fallback to aliases, a shell or an LLM.
+  - **Tools:** a small read-only set of programs, run without a shell. It includes `ls`, `cat`, `head`, `tail`, `wc`, `find`, `grep`, `sort`, `diff`, `stat`, `tree`, `du` and read-only `git` (see docs/formal-ai.md). Changes (`mkdir`, `touch`, `rmdir`, `rm`, `cp`, `mv`) need `START_FORMAL_AI_EFFECTS=workspace-write`; a change may only name targets observed before it and is verified by readback. Shell syntax, other programs and paths outside the working directory are refused (exit 77).
+  - **Bounds:** steps, retries, total time, per-tool time, output bytes and request length (`START_FORMAL_AI_*`). Exhausted budgets exit 124 and unsupported requests exit 65. A cancelled run ends with the signal status.
+  - **Evidence:** the log records the runtime identity, every response, tool decision, exit status and file-system effect. The log stays local: this mode never uploads logs or creates issues. Tool output appears in the evidence only as a short preview, and the environment is not recorded.
+  - **Dependencies:** `lino-objects-codec` 0.9.4 and `eslint` 10.14.0; CI runs on Node.js 26.
+
+## 0.40.0
+
+### Minor Changes
+
+- 47a9dd8: Upgrade every dependency to its latest release, and let CI reject pull requests that fall behind (#253).
+
+  - **Upgrades:** `@link-foundation/sensitive-data-sanitizer` 0.12.2, `command-stream` 2.6.0, `lino-objects-codec` 0.9.3, the lint tools, the GitHub Actions, the runners (`ubuntu-26.04`, `macos-26`) and the .NET 10 SDK that clink 4.x needs.
+  - **Store format:** the execution store is now written in lino-objects-codec 0.9's readable format. Stores written by older releases are still read, but older releases of start cannot read a store written by this one.
+  - **Links database:** the store had stayed empty, because clink 4.x rejected every query it sent. It now writes one record-scoped link per property and verifies the writes. The existing `recordToLinksNotation()` helper remains available with the corrected format.
+  - **`--use-command-stream`:** command output is appended to the log again. command-stream 2.x returns it as snapshots.
+  - **Sanitizer:** start's own `Authorization` masking pass is gone, because the sanitizer now masks only the scheme and token itself. A NUL byte in a log no longer blocks the upload (`--allow-binary`).
+  - **Dependency fixes:** preserve Authorization placeholders, bracketed tokens and following control bytes; avoid quadratic scanning of dense credentials. Load command-stream's lightweight `core` entry for command execution.
+  - **CI:** `scripts/check-dependency-freshness.mjs` fails a pull request when any npm package, crate, action, CI tool, runtime, runner or translator pin is not its latest release. The one exception is a version that links an open blocking issue.
+
 ## 0.39.2
 
 ### Patch Changes

package/eslint.config.mjs

@@ -59,6 +59,36 @@ export default [
       'spaced-comment': ['error', 'always', { markers: ['/'] }],
     },
   },
+  {
+    // CommonJS test fixtures and helpers (.cjs): the test runner collects
+    // every .js/.mjs file under test/, so non-test modules there use .cjs.
+    files: ['**/*.cjs'],
+    plugins: {
+      prettier: prettierPlugin,
+    },
+    languageOptions: {
+      ecmaVersion: 'latest',
+      sourceType: 'commonjs',
+      globals: {
+        require: 'readonly',
+        module: 'readonly',
+        exports: 'readonly',
+        __dirname: 'readonly',
+        __filename: 'readonly',
+        console: 'readonly',
+        process: 'readonly',
+        Buffer: 'readonly',
+      },
+    },
+    rules: {
+      'prettier/prettier': 'error',
+      eqeqeq: ['error', 'always'],
+      curly: ['error', 'all'],
+      'no-var': 'error',
+      'prefer-const': 'error',
+      'prefer-template': 'error',
+    },
+  },
   {
     // ES module files (.mjs)
     files: ['**/*.mjs'],
@@ -80,6 +110,7 @@ export default [
         // Web-standard globals Node exposes; used by the repository-level
         // scripts/ directory, which the root config lints (issue #168).
         AbortController: 'readonly',
+        AbortSignal: 'readonly',
         URL: 'readonly',
         setTimeout: 'readonly',
         clearTimeout: 'readonly',
@@ -111,6 +142,7 @@ export default [
     files: [
       'test/**/*.js',
       'test/**/*.mjs',
+      'test/**/*.cjs',
       'tests/**/*.js',
       'experiments/**/*.js',
     ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.39.2",
+  "version": "0.41.0",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {
@@ -49,12 +49,12 @@
   "devDependencies": {
     "@changesets/cli": "^3.0.3",
     "@eslint/js": "^10.0.1",
-    "eslint": "^10.3.0",
+    "eslint": "^10.14.0",
     "eslint-config-prettier": "^10.1.8",
-    "eslint-plugin-prettier": "^5.5.5",
+    "eslint-plugin-prettier": "^5.5.7",
     "husky": "^9.1.7",
-    "lint-staged": "^17.0.4",
-    "prettier": "^3.8.3"
+    "lint-staged": "^17.6.0",
+    "prettier": "^3.9.10"
   },
   "lint-staged": {
     "*.{js,mjs,cjs}": [
@@ -68,13 +68,12 @@
     ]
   },
   "dependencies": {
-    "@link-foundation/sensitive-data-sanitizer": "^0.12.0",
-    "command-stream": "^0.9.4",
-    "lino-objects-codec": "^0.4.0"
+    "@link-foundation/sensitive-data-sanitizer": "^0.12.2",
+    "command-stream": "^2.6.0",
+    "lino-objects-codec": "^0.9.4"
   },
   "overrides": {
     "brace-expansion": "^5.0.12",
-    "flatted": "^3.4.2",
-    "js-yaml": "^4.3.2"
+    "flatted": "^3.4.4"
   }
 }

package/src/bin/cli.js

@@ -223,7 +223,9 @@ if (!isQueryMode && (!parsedCommand || parsedCommand.trim() === '')) {
 let command = parsedCommand;
 let substitutionResult = null;
 
-if (!config.disableSubstitutions && parsedCommand && parsedCommand.trim()) {
+// A --use-formal-ai request is only ever read by Formal AI (issue #255).
+const useFormalAi = wrapperOptions.useFormalAi;
+if (!config.disableSubstitutions && !useFormalAi && parsedCommand?.trim()) {
   substitutionResult = processCommand(parsedCommand, {
     customLinoPath: config.substitutionsPath,
     verbose: config.verbose,
@@ -270,6 +272,14 @@ let launch = null;
     console.error(`Error: ${error.message}`);
     process.exit(1);
   }
+  if (useFormalAi) {
+    const { requestText, runFormalAiCli } = require('../lib/formal-ai-cli');
+    const request = requestText(parsedArgs.rawCommand);
+    const store = getExecutionStore();
+    process.exit(
+      await runFormalAiCli({ request, sessionId, launch, store, config })
+    );
+  }
 
   if (wrapperOptions.isolated === 'docker') {
     const {
@@ -814,7 +824,7 @@ async function runDirectWithCommandStream(
   sessionId
 ) {
   // Lazy load command-stream
-  const { getCommandStream } = require('../lib/command-stream');
+  const { getCommandStream, outputText } = require('../lib/command-stream');
   const { $, raw } = await getCommandStream();
 
   // Get the command name (first word of the actual command to execute)
@@ -921,11 +931,13 @@ async function runDirectWithCommandStream(
     }
 
     // Collect output for log
-    if (result.stdout) {
-      appendLogFile(logFilePath, result.stdout);
+    const stdout = outputText(result.stdout);
+    const stderr = outputText(result.stderr);
+    if (stdout) {
+      appendLogFile(logFilePath, stdout);
     }
-    if (result.stderr) {
-      appendLogFile(logFilePath, result.stderr);
+    if (stderr) {
+      appendLogFile(logFilePath, stderr);
     }
   } catch (err) {
     exitCode = err.code || 1;

package/src/lib/args-parser.js

@@ -27,6 +27,7 @@
  * --keep-container-on-fail         Remove successful docker containers, keep failed or OOM-killed ones
  * --shell <shell>                  Shell to use in isolation environments: auto, bash, zsh, sh (default: auto)
  * --use-command-stream             Use command-stream library for command execution (experimental)
+ * --use-formal-ai                  Perform a natural-language request through Formal AI (experimental)
  * --verbose                        Enable verbose/debug output (sets START_VERBOSE=1)
  * --status <uuid>                  Show status of a previous command execution by UUID
  * --list [--running]               List all (or only running) tracked command executions
@@ -55,6 +56,7 @@ const { parseSequence, isSequence } = require('./sequence-parser');
 const { buildCommandString } = require('./shell-utils');
 const {
   VALID_OUTPUT_FORMATS,
+  activeQueryModes,
   createQueryOptionDefaults,
   parseQueryOption,
   validateQueryOptions,
@@ -220,6 +222,7 @@ function parseArgs(args) {
     keepContainerOnFail: false, // Keep docker container filesystem when command fails or OOM-kills
     shell: 'auto', // Shell to use in isolation environments: auto, bash, zsh, sh
     useCommandStream: false, // Use command-stream library for command execution
+    useFormalAi: false, // Natural-language request performed through Formal AI (issue #255)
     ...createQueryOptionDefaults(),
     ...snapshotResumeOptions.createSnapshotOptionDefaults(),
     ...launchIdentityOptions.createLaunchIdentityOptionDefaults(),
@@ -557,6 +560,11 @@ function parseOption(args, index, options) {
     return 1;
   }
 
+  if (arg === '--use-formal-ai') {
+    options.useFormalAi = true;
+    return 1;
+  }
+
   // --verbose (enable verbose/debug output, sets START_VERBOSE env var)
   if (arg === '--verbose') {
     process.env.START_VERBOSE = '1';
@@ -849,6 +857,7 @@ function validateOptions(options) {
   snapshotResumeOptions.validate(options);
   validateQueryOptions(options);
   launchIdentityOptions.validate(options);
+  validateFormalAi(options);
 
   // Validate shell option
   if (options.shell !== null && options.shell !== undefined) {
@@ -869,6 +878,30 @@ function validateOptions(options) {
   }
 }
 
+/**
+ * `--use-formal-ai` runs its tool calls directly in the current directory;
+ * other execution modes would change where and how they run (issue #255).
+ * @param {object} options - Parsed options
+ * @throws {Error} When combined with an unsupported option
+ */
+function validateFormalAi(options) {
+  if (!options.useFormalAi) {
+    return;
+  }
+  const conflicts = [
+    ...(options.isolated !== null ? ['--isolated'] : []),
+    ...(options.user ? ['--isolated-user'] : []),
+    ...(options.detached ? ['--detached'] : []),
+    ...(options.useCommandStream ? ['--use-command-stream'] : []),
+    ...activeQueryModes(options),
+  ];
+  if (conflicts.length) {
+    throw new Error(
+      `--use-formal-ai cannot be combined with ${conflicts.join(', ')} (experimental mode, see docs/formal-ai.md)`
+    );
+  }
+}
+
 /**
  * Generate a unique session name
  * @param {string} [prefix='start'] - Prefix for the session name

package/src/lib/command-stream.js

@@ -26,11 +26,24 @@ let commandStream = null;
  */
 async function getCommandStream() {
   if (!commandStream) {
-    commandStream = await import('command-stream');
+    // Execution needs no compatibility layers, terminal renderers or REPL.
+    commandStream = await import('command-stream/core');
   }
   return commandStream;
 }
 
+/**
+ * Text of a completed command's stdout or stderr. Since command-stream 2.x
+ * these are readable stream snapshots rather than strings, and an empty one
+ * is still truthy (#253).
+ *
+ * @param {unknown} output - result.stdout or result.stderr
+ * @returns {string}
+ */
+function outputText(output) {
+  return output === undefined || output === null ? '' : String(output);
+}
+
 /**
  * Execute a shell command synchronously and return the result
  * Uses command-stream's $ function with .sync() for blocking execution.
@@ -54,8 +67,8 @@ async function execCommand(command, options = {}) {
     const result = $cmd`${command}`.sync();
 
     return {
-      stdout: (result.stdout || '').trim(),
-      stderr: (result.stderr || '').trim(),
+      stdout: outputText(result.stdout).trim(),
+      stderr: outputText(result.stderr).trim(),
       code: resolveChildExitCode(result.code),
     };
   } catch (err) {
@@ -91,8 +104,8 @@ async function execCommandAsync(command, options = {}) {
     const result = await $cmd`${command}`;
 
     return {
-      stdout: (result.stdout || '').trim(),
-      stderr: (result.stderr || '').trim(),
+      stdout: outputText(result.stdout).trim(),
+      stderr: outputText(result.stderr).trim(),
       code: resolveChildExitCode(result.code),
     };
   } catch (err) {
@@ -250,6 +263,7 @@ async function runWithHandlers(command, handlers = {}, options = {}) {
 
 module.exports = {
   getCommandStream,
+  outputText,
   execCommand,
   execCommandAsync,
   commandExists,

package/src/lib/execution-chain.js

@@ -21,7 +21,7 @@ const { executionAliases, hasUserNames } = require('./execution-identity');
 function buildChain(records, record) {
   const seen = new Set([record.uuid]);
   const chain = [record];
-  for (let cur = record; cur.continues; ) {
+  for (let cur = record; cur.continues;) {
     const prev = records.find((r) => r.uuid === cur.continues);
     if (!prev || seen.has(prev.uuid)) {
       break;
@@ -30,7 +30,7 @@ function buildChain(records, record) {
     chain.unshift(prev);
     cur = prev;
   }
-  for (let cur = record; ; ) {
+  for (let cur = record; ;) {
     const next = records
       .filter((r) => r.continues === cur.uuid && !seen.has(r.uuid))
       .sort((a, b) => Date.parse(b.startTime) - Date.parse(a.startTime))[0];

package/src/lib/execution-store.js

@@ -245,6 +245,16 @@ class ExecutionRecord {
   }
 }
 
+/**
+ * Quote text as a clink reference: `"..."` with inner quotes doubled.
+ * clink 4.x rejects backslash escapes inside references.
+ * @param {string} text
+ * @returns {string}
+ */
+function clinkReference(text) {
+  return `"${text.replaceAll('"', '""')}"`;
+}
+
 /**
  * Check if clink is installed
  * @returns {boolean}
@@ -343,70 +353,69 @@ class ExecutionStore {
   }
 
   /**
-   * Convert execution record to clink links notation format
-   * Uses string aliases for readable IDs
+   * Convert a record's properties to clink links notation.
+   *
+   * The record is one link `(uuid: ExecutionRecord uuid)` plus one link per
+   * property, `(uuid.key: uuid "key=value")`. Property links have the record
+   * as their source, so two records that share a value (both `executing`) are
+   * still distinct links: doublets are deduplicated by (source, target), and a
+   * shared `(status executing)` link would carry only one record's name. The
+   * `key=` prefix keeps the target from being a bare number, which clink 4.x
+   * reads as a link address even when quoted (issue #253).
    * @param {ExecutionRecord} record
    * @returns {string}
    */
   recordToLinksNotation(record) {
-    // Using clink's string alias feature for readable identifiers
-    // Format: (uuid: uuid-value) (pid: pid-value) etc.
     const obj = record.toObject();
-    const parts = [];
+    const links = [];
 
     for (const [key, value] of Object.entries(obj)) {
       if (value !== null && value !== undefined) {
-        // Escape value properly for links notation
-        const escapedValue =
+        const text =
           typeof value === 'object' ? JSON.stringify(value) : String(value);
-        parts.push(`(${record.uuid}.${key}: ${key} "${escapedValue}")`);
+        links.push(
+          `(${record.uuid}.${key}: ${record.uuid} ${clinkReference(`${key}=${text}`)})`
+        );
       }
     }
 
-    return parts.join(' ');
+    return links.join(' ');
+  }
+
+  /** Build the create query using the same property notation as the public helper. */
+  buildClinkCreateQuery(record) {
+    return `() ((${record.uuid}: ExecutionRecord ${record.uuid}) ${this.recordToLinksNotation(record)})`;
   }
 
   /**
-   * Build clink query for creating/updating a record
-   * @param {ExecutionRecord} record
+   * Build clink query deleting every property link of a record
+   * @param {string} uuid
    * @returns {string}
    */
-  buildClinkCreateQuery(record) {
-    const obj = record.toObject();
-    const links = [];
-
-    // Create main record link
-    links.push(`(${record.uuid}: ExecutionRecord ${record.uuid})`);
-
-    // Create property links
-    for (const [key, value] of Object.entries(obj)) {
-      if (value !== null && value !== undefined) {
-        const escapedValue =
-          typeof value === 'object' ? JSON.stringify(value) : String(value);
-        // Using format: (uuid.property: property "value")
-        links.push(`(${record.uuid}.${key}: ${key} "${escapedValue}")`);
-      }
-    }
-
-    // Format: () ((links)) - creates new links
-    return `() ((${links.join(') (')}))`;
+  buildClinkDeletePropertiesQuery(uuid) {
+    return `(($id: ${uuid} $value)) ()`;
   }
 
   /**
    * Execute clink command
    * @param {string} query
+   * @param {string[]} [extraArgs] - flags appended after `--db <path>`
    * @returns {{success: boolean, output: string}}
    */
-  execClink(query) {
+  execClink(query, extraArgs = []) {
     try {
       // The query embeds record values verbatim; a shell string would let a
       // single quote in any of them break out of the argument
       // (CodeQL js/shell-command-constructed-from-input, issue #168).
-      const result = execFileSync('clink', [query, '--db', this.linksDbPath], {
-        encoding: 'utf8',
-        timeout: 10000,
-        stdio: ['pipe', 'pipe', 'pipe'],
-      });
+      const result = execFileSync(
+        'clink',
+        [query, '--db', this.linksDbPath, ...extraArgs],
+        {
+          encoding: 'utf8',
+          timeout: 10000,
+          stdio: ['pipe', 'pipe', 'pipe'],
+        }
+      );
       return { success: true, output: result };
     } catch (err) {
       this.log(`Clink error: ${err.message}`);
@@ -417,15 +426,22 @@ class ExecutionStore {
   /**
    * Write a record to the links database using clink
    * @param {ExecutionRecord} record
+   * @param {{isNew?: boolean}} [options] - isNew skips dropping old links
    * @returns {boolean}
    */
-  writeLinksRecord(record) {
+  writeLinksRecord(record, { isNew = false } = {}) {
     if (!this.useLinks) {
       return false;
     }
 
-    const query = this.buildClinkCreateQuery(record);
-    const result = this.execClink(query);
+    // Replace, not merge: drop the previous property links first. clink
+    // reports an error when the record has no links yet, which is harmless.
+    if (!isNew) {
+      this.execClink(this.buildClinkDeletePropertiesQuery(record.uuid));
+    }
+    const result = this.execClink(this.buildClinkCreateQuery(record), [
+      '--auto-create-missing-references',
+    ]);
 
     if (result.success) {
       this.log(`Wrote record ${record.uuid} to links database`);
@@ -444,9 +460,8 @@ class ExecutionStore {
       return false;
     }
 
-    // Delete all links with this UUID prefix
-    const query = `(($id: ${uuid} $any)) ()`;
-    const result = this.execClink(query);
+    this.execClink(this.buildClinkDeletePropertiesQuery(uuid));
+    const result = this.execClink(`((${uuid}: ExecutionRecord ${uuid})) ()`);
     return result.success;
   }
 
@@ -492,7 +507,7 @@ class ExecutionStore {
 
       // Also write to links database if available
       if (this.useLinks) {
-        this.writeLinksRecord(record);
+        this.writeLinksRecord(record, { isNew: existingIndex < 0 });
       }
 
       return true;
@@ -810,8 +825,10 @@ class ExecutionStore {
 
     // Query links database for all ExecutionRecord links
     try {
+      // Without --changes clink prints nothing for a read-only match
       const queryResult = this.execClink(
-        `((($id: ExecutionRecord $uuid)) (($id: ExecutionRecord $uuid)))`
+        `((($id: ExecutionRecord $uuid)) (($id: ExecutionRecord $uuid)))`,
+        ['--changes']
       );
 
       if (queryResult.success) {
@@ -928,6 +945,7 @@ module.exports = {
   ExecutionStatus,
   LockManager,
   isClinkInstalled,
+  clinkReference,
   DEFAULT_APP_FOLDER,
   LINO_DB_FILE,
   LINKS_DB_FILE,