start-command 0.40.0 → 0.42.0

package/test/fixtures/formal-ai/replay.cjs

@@ -0,0 +1,194 @@
+/**
+ * Replies of the offline Formal AI stand-in (issue #255), shared by the fake
+ * runtime process (fake-server.cjs) and in-process tests.
+ *
+ * A request gets its recorded response number k from receipts.json, where k
+ * is the number of assistant turns already in the conversation. Unknown
+ * requests get the runtime's "did not understand" reply. SYNTHETIC scenarios
+ * (requests starting with `synthetic:`) cover states a real runtime cannot be
+ * made to produce on demand. A number reply is an HTTP error status, and a
+ * `{status, text}` reply an HTTP error with that exact body.
+ *
+ * Receipts of issue #258 (receipts-258.json) also record the tool names start
+ * offered; they answer only a request offering exactly those tools, so the
+ * released Bash-only contract and the structured file tools replay apart.
+ */
+
+const fs = require('fs');
+const http = require('http');
+const path = require('path');
+
+const load = (name) =>
+  JSON.parse(fs.readFileSync(path.join(__dirname, name), 'utf8'));
+const { receipts } = load('receipts.json');
+const edits = load('receipts-258.json');
+
+/**
+ * The HTTP 500 body a released start discarded (issue #257), captured from a
+ * Formal AI 0.355.28 runtime whose local policy forbids its Rust grammar WASM.
+ */
+const CAPTURED_500 =
+  '{"error":{"message":"Aborted(Error: LocalProductWasmForbidden). Build with -sASSERTIONS for more info.","type":"formal_ai_error"}}';
+
+function toolCall(command, id = `call_${Math.random().toString(16).slice(2)}`) {
+  return {
+    role: 'assistant',
+    content: 'Let me run the requested command to get that for you.',
+    tool_calls: [
+      {
+        id,
+        type: 'function',
+        function: { name: 'bash', arguments: JSON.stringify({ command }) },
+      },
+    ],
+  };
+}
+
+const answer = (content) => ({ role: 'assistant', content });
+
+/**
+ * SYNTHETIC scenarios: request -> (assistantTurns, messages, body) ->
+ * message | status | {status, text}.
+ * A constant list matched by string equality: request text only selects among
+ * these entries and is never used as a property key, so `constructor`,
+ * `toString` and similar names reach the "did not understand" reply.
+ */
+const SYNTHETIC = Object.freeze(
+  Object.entries({
+    // Never stops asking for more reads: exhausts the step budget.
+    'synthetic: loop forever': () => toolCall('pwd'),
+    // A tool start must not run.
+    'synthetic: call an unknown tool': () => ({
+      role: 'assistant',
+      content: '',
+      tool_calls: [
+        {
+          id: 'call_x',
+          type: 'function',
+          function: { name: 'python', arguments: '{"code":"1"}' },
+        },
+      ],
+    }),
+    // A long-running read (a fake `cat` from the test PATH sleeps).
+    'synthetic: read slowly': (turns) =>
+      turns === 0 ? toolCall('cat slow.txt') : answer('done'),
+    // A tool that does not exist on PATH.
+    'synthetic: run a missing tool': (turns) =>
+      turns === 0 ? toolCall('tree') : answer('The `tree` command failed.'),
+    // Writes outside the observed workspace.
+    'synthetic: overwrite without looking': () => toolCall('touch README.md'),
+    'synthetic: write outside': () => toolCall('touch ../escape.txt'),
+    // A server error on every request.
+    'synthetic: server error': () => 500,
+    // The captured runtime failure of issue #257 on every request.
+    'synthetic: server error (captured #257)': () => ({
+      status: 500,
+      text: CAPTURED_500,
+    }),
+    // A 503 whose body echoes credentials and control characters.
+    'synthetic: server error with secrets': () => ({
+      status: 503,
+      text: JSON.stringify({
+        error: {
+          type: 'upstream_error',
+          message: `proxy refused Authorization: Bearer sk-test-0123456789\u0007 api_key=sk-live-abcdef`,
+        },
+      }),
+    }),
+    // A write that claims success but leaves no trace (fake `mkdir` on PATH).
+    'synthetic: unverified mkdir': (turns) =>
+      turns === 0 ? toolCall('test ! -e out') : toolCall('mkdir out'),
+  })
+);
+
+function reply(body) {
+  const messages = Array.isArray(body.messages) ? body.messages : [];
+  const request = (messages.find((m) => m.role === 'user') || {}).content || '';
+  const turns = messages.filter((m) => m.role === 'assistant').length;
+  const scenario = SYNTHETIC.find(([name]) => name === request);
+  if (scenario) {
+    const [, respond] = scenario;
+    return respond(turns, messages, body);
+  }
+  const offered = (Array.isArray(body.tools) ? body.tools : [])
+    .map((tool) => tool && tool.function && tool.function.name)
+    .join(',');
+  const edit = edits.receipts.find(
+    (r) => r.request === request && r.offered.join(',') === offered
+  );
+  if (edit && edit.responses[turns]) {
+    return edit.responses[turns].message;
+  }
+  const receipt = receipts.find((r) => r.request === request);
+  if (receipt && receipt.responses[turns]) {
+    return receipt.responses[turns].message;
+  }
+  return {
+    role: 'assistant',
+    content: "I didn't understand you. Could you rephrase?",
+    thinking_steps: [{ step: 'dispatch_handler', detail: 'unknown' }],
+  };
+}
+
+/**
+ * HTTP server speaking the agent-mode subset start uses.
+ * @param {string} [log] - File that receives every request body as a JSON line
+ * @returns {import('http').Server}
+ */
+function createFakeServer(log) {
+  return http.createServer((req, res) => {
+    let text = '';
+    req.on('data', (chunk) => {
+      text += chunk;
+    });
+    req.on('end', () => {
+      const send = (status, payload) => {
+        res.writeHead(status, { 'content-type': 'application/json' });
+        res.end(JSON.stringify(payload));
+      };
+      if (req.method === 'GET' && req.url.endsWith('/models')) {
+        send(200, { object: 'list', data: [{ id: 'formal-ai' }] });
+        return;
+      }
+      if (req.method !== 'POST' || !req.url.endsWith('/chat/completions')) {
+        send(404, { error: 'not found' });
+        return;
+      }
+      const body = JSON.parse(text);
+      if (log) {
+        fs.appendFileSync(log, `${JSON.stringify(body)}\n`);
+      }
+      const message = reply(body);
+      if (typeof message === 'number') {
+        send(message, { error: 'synthetic failure' });
+        return;
+      }
+      if (typeof message.status === 'number') {
+        res.writeHead(message.status, { 'content-type': 'application/json' });
+        res.end(message.text);
+        return;
+      }
+      send(200, {
+        id: 'chatcmpl_fake',
+        object: 'chat.completion',
+        model: 'formal-ai',
+        choices: [
+          {
+            index: 0,
+            message,
+            finish_reason: message.tool_calls ? 'tool_calls' : 'stop',
+          },
+        ],
+      });
+    });
+  });
+}
+
+module.exports = {
+  CAPTURED_500,
+  SYNTHETIC,
+  createFakeServer,
+  edits,
+  receipts,
+  reply,
+};

package/test/regression-255-cli.js

@@ -0,0 +1,296 @@
+/**
+ * Issue #255: `$ --use-formal-ai -- <request>` end to end through the CLI,
+ * with an offline Formal AI checkout (test/fixtures/formal-ai) started exactly
+ * like a real one. Covers parsing of `--`, option-like text and shell
+ * metacharacters, the no-fallback guarantee, logs, evidence and cancellation.
+ */
+const { describe, it, expect, afterAll } = require('bun:test');
+const { spawn, spawnSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const { parseArgs } = require('../src/lib/args-parser');
+const { requestText, EVIDENCE_SCHEMA } = require('../src/lib/formal-ai-cli');
+const {
+  CLI,
+  cleanup,
+  cliEnv,
+  fakeBin,
+  fakeCheckout,
+  readEvidence,
+  readLog,
+  receiptFixture,
+  sentRequests,
+  snapshot,
+  tempDir,
+} = require('./fixtures/formal-ai/helpers.cjs');
+
+const posix = process.platform !== 'win32';
+const dirs = [];
+const checkout = fakeCheckout();
+const oldCheckout = fakeCheckout({ version: '0.355.26' });
+dirs.push(checkout, oldCheckout);
+
+afterAll(() => cleanup(...dirs));
+
+/** Run the CLI in a fresh fixture with an isolated app folder. */
+function cli(args, { env = {}, root = tempDir('cli'), cwd } = {}) {
+  dirs.push(root);
+  const dir = cwd || receiptFixture();
+  dirs.push(dir);
+  const before = snapshot(dir);
+  const result = spawnSync(process.execPath, [CLI, ...args], {
+    cwd: dir,
+    env: cliEnv(root, env),
+    encoding: 'utf8',
+    timeout: 60000,
+  });
+  return {
+    ...result,
+    root,
+    cwd: dir,
+    before,
+    after: snapshot(dir),
+    requests: sentRequests(root),
+  };
+}
+
+const withRuntime = { START_FORMAL_AI_ROOT: checkout };
+
+describe('--use-formal-ai parsing (#255)', () => {
+  it('keeps everything after -- as request text', () => {
+    const cases = [
+      [
+        ['print', 'list', 'of', 'files', 'in', 'current', 'directory'],
+        'print list of files in current directory',
+      ],
+      [['--help'], '--help'],
+      [['--isolated', 'docker', 'ls'], '--isolated docker ls'],
+      [['ls;', 'rm', '-rf', '~'], 'ls; rm -rf ~'],
+      [
+        ['ls; rm -rf ~ && echo $(id) | sh > x'],
+        'ls; rm -rf ~ && echo $(id) | sh > x',
+      ],
+      [['--', 'what is', '--version?'], '-- what is --version?'],
+    ];
+    for (const [words, text] of cases) {
+      const result = parseArgs(['--use-formal-ai', '--', ...words]);
+      expect(result.wrapperOptions.useFormalAi).toBe(true);
+      expect(result.rawCommand).toEqual(words);
+      expect(requestText(result.rawCommand)).toBe(text);
+      // Option-like request text never becomes a start option.
+      expect(result.wrapperOptions.isolated).toBeFalsy();
+      expect(result.wrapperOptions.help).toBeFalsy();
+    }
+  });
+
+  it('is off unless requested', () => {
+    expect(parseArgs(['--', 'ls']).wrapperOptions.useFormalAi).toBe(false);
+    expect(
+      parseArgs(['ls', '--use-formal-ai']).wrapperOptions.useFormalAi
+    ).toBe(false);
+  });
+
+  it('rejects combinations the experimental mode does not support', () => {
+    for (const args of [
+      ['--use-formal-ai', '--isolated', 'docker', '--', 'ls'],
+      ['--use-formal-ai', '--detached', '--isolated', 'screen', '--', 'ls'],
+      ['--use-formal-ai', '--use-command-stream', '--', 'ls'],
+    ]) {
+      expect(() => parseArgs(args)).toThrow(
+        /--use-formal-ai cannot be combined with/
+      );
+    }
+  });
+});
+
+describe.skipIf(!posix)('--use-formal-ai never falls back (#255)', () => {
+  it('runs nothing without a qualified runtime', () => {
+    const cases = [
+      [{}, 'needs a Formal AI runtime and has no fallback'],
+      [{ START_FORMAL_AI_ROOT: oldCheckout }, 'needs 0.355.27 or newer'],
+      [
+        { ...withRuntime, START_FORMAL_AI_EXPECTED_REVISION: 'abc123' },
+        'not START_FORMAL_AI_EXPECTED_REVISION=abc123',
+      ],
+      [{ START_FORMAL_AI_URL: 'http://127.0.0.1:9' }, 'cannot be qualified'],
+    ];
+    for (const [env, message] of cases) {
+      // "list files in docs" is also a built-in alias for `ls docs`.
+      const result = cli(
+        ['--use-formal-ai', '--', 'list', 'files', 'in', 'docs'],
+        {
+          env,
+        }
+      );
+      expect(result.status).toBe(69);
+      expect(result.stderr).toContain(message);
+      expect(result.stderr).toContain(
+        'No command was run: --use-formal-ai never falls back to aliases, a shell or an LLM.'
+      );
+      expect(result.stdout).not.toContain('guide.md');
+      expect(result.after).toEqual(result.before);
+      expect(fs.existsSync(path.join(result.root, 'logs'))).toBe(false);
+    }
+  });
+
+  it('rejects an empty, overlong or misconfigured request before starting', () => {
+    let result = cli(['--use-formal-ai', '--', 'list', 'files'], {
+      env: { ...withRuntime, START_FORMAL_AI_MAX_REQUEST_CHARS: '5' },
+    });
+    expect(result.status).toBe(64);
+    expect(result.stderr).toContain('START_FORMAL_AI_MAX_REQUEST_CHARS is 5');
+    result = cli(['--use-formal-ai', '--', 'ls'], {
+      env: { ...withRuntime, START_FORMAL_AI_MAX_STEPS: 'many' },
+    });
+    expect(result.status).toBe(78);
+    expect(result.requests).toEqual([]);
+  });
+});
+
+describe.skipIf(!posix)('--use-formal-ai end to end (#255)', () => {
+  it('lists the working directory, leaves it unchanged and keeps evidence', () => {
+    const result = cli(
+      [
+        '--use-formal-ai',
+        '--',
+        'print',
+        'list',
+        'of',
+        'files',
+        'in',
+        'current',
+        'directory',
+      ],
+      { env: withRuntime }
+    );
+    expect(result.status).toBe(0);
+    for (const name of fs.readdirSync(result.cwd)) {
+      expect(result.stdout).toContain(name);
+    }
+    expect(result.stdout).toContain(
+      '--use-formal-ai -- print list of files in current directory'
+    );
+    expect(result.stderr).toContain('[formal-ai] runtime: formal-ai 0.355.27');
+    expect(result.stderr).toContain('[formal-ai] $ ls');
+    expect(result.stderr).toContain(
+      '[formal-ai] completed: finished after 1 tool call(s); no file changed'
+    );
+    expect(result.after).toEqual(result.before);
+
+    // The request went to Formal AI verbatim; read-only mode offers bash and
+    // the structured read tool, never the change tools (#258).
+    expect(result.requests[0].messages).toEqual([
+      { role: 'user', content: 'print list of files in current directory' },
+    ]);
+    expect(result.requests[0].tools.map((t) => t.function.name)).toEqual([
+      'bash',
+      'read',
+    ]);
+
+    const log = readLog(result.root);
+    expect(log).toContain('Mode: --use-formal-ai (experimental)');
+    expect(log).toContain('Request: print list of files in current directory');
+    expect(log).toContain('Effects: read-only');
+    expect(log).toMatch(/Exit Code: 0\n$/);
+    const evidence = readEvidence(log);
+    expect(evidence).toMatchObject({
+      schema: EVIDENCE_SCHEMA,
+      request: 'print list of files in current directory',
+      effectsMode: 'read-only',
+      runtime: {
+        source: 'checkout',
+        version: '0.355.27',
+        entry: 'js/server/main.mjs',
+        memory: 'START_FORMAL_AI_MEMORY_PATH',
+      },
+      effects: { added: [], removed: [], changed: [], complete: true },
+      outcome: { status: 'completed', exitCode: 0, remainingObligations: [] },
+      formalAiRequests: 2,
+    });
+    expect(evidence.steps.map((s) => s.kind)).toEqual([
+      'response',
+      'tool',
+      'response',
+    ]);
+    expect(evidence.steps[1]).toMatchObject({
+      command: 'ls',
+      argv: ['ls'],
+      decision: 'allowed',
+      effect: 'read',
+      exitCode: 0,
+    });
+
+    const db = fs.readFileSync(
+      path.join(result.root, 'app', 'executions.lino'),
+      'utf8'
+    );
+    expect(db).toContain('print list of files in current directory');
+    expect(db).toContain('formal-ai');
+  });
+
+  it('never applies aliases or a shell to the request', () => {
+    const home = tempDir('home');
+    dirs.push(home);
+    fs.writeFileSync(path.join(home, 'canary'), 'keep');
+    let result = cli(['--use-formal-ai', '--', 'list', 'files', 'in', 'docs'], {
+      env: withRuntime,
+    });
+    expect(result.requests[0].messages[0].content).toBe('list files in docs');
+    expect(result.status).toBe(0);
+
+    result = cli(['--use-formal-ai', '--', 'ls; rm -rf ~'], {
+      env: { ...withRuntime, HOME: home },
+    });
+    expect(result.requests[0].messages[0].content).toBe('ls; rm -rf ~');
+    expect(result.status).toBe(77);
+    expect(result.stderr).toContain('needs shell syntax start does not run');
+    expect(fs.readFileSync(path.join(home, 'canary'), 'utf8')).toBe('keep');
+    expect(result.after).toEqual(result.before);
+  });
+
+  it('reports unsupported requests explicitly', () => {
+    const result = cli(['--use-formal-ai', '--', 'book me a flight to Paris'], {
+      env: withRuntime,
+    });
+    expect(result.status).toBe(65);
+    expect(result.stderr).toContain(
+      'unsupported: Formal AI did not understand the request; no tool ran'
+    );
+    expect(result.after).toEqual(result.before);
+  });
+
+  it('ends with the signal status and evidence when cancelled', async () => {
+    const root = tempDir('cancel');
+    const cwd = receiptFixture();
+    const bin = fakeBin({ cat: 'exec sleep 30' });
+    dirs.push(root, cwd, bin);
+    const env = cliEnv(root, {
+      ...withRuntime,
+      PATH: `${bin}:${process.env.PATH}`,
+    });
+    const child = spawn(
+      process.execPath,
+      [CLI, '--use-formal-ai', '--', 'synthetic: read slowly'],
+      { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }
+    );
+    let stderr = '';
+    const status = await new Promise((resolve) => {
+      const timer = setTimeout(() => child.kill('SIGKILL'), 20000);
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk;
+        if (stderr.includes('$ cat slow.txt') && !child.signalled) {
+          child.signalled = true;
+          setTimeout(() => child.kill('SIGTERM'), 200);
+        }
+      });
+      child.on('close', (code) => {
+        clearTimeout(timer);
+        resolve(code);
+      });
+    });
+    expect(status).toBe(143);
+    const evidence = readEvidence(readLog(root));
+    expect(evidence.outcome.status).toBe('cancelled');
+    expect(evidence.outcome.exitCode).toBe(143);
+  });
+});

package/test/regression-255-e2e.js

@@ -0,0 +1,167 @@
+/**
+ * Issue #255: opt-in end-to-end test of `--use-formal-ai` against a REAL
+ * Formal AI checkout. Skipped unless both are set:
+ *
+ *   START_FORMAL_AI_E2E=1 START_FORMAL_AI_ROOT=/path/to/formal-ai \
+ *     bun test ./test/regression-255-e2e.js
+ *
+ * It runs the issue's demo in an isolated directory whose names contain
+ * spaces and Unicode, compares the listing with an independent readdir and
+ * proves the directory is unchanged. Issue #258 adds an isolated
+ * read/edit/read task in workspace-write mode, the explicit capability gap of
+ * the same request in read-only mode, and a plain question answered without
+ * tools. Use START_FORMAL_AI_NODE to pick the Node.js that runs the Formal AI
+ * server.
+ */
+const { describe, it, expect, afterAll } = require('bun:test');
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const {
+  CLI,
+  cleanup,
+  cliEnv,
+  populate,
+  readEvidence,
+  readLog,
+  snapshot,
+  tempDir,
+} = require('./fixtures/formal-ai/helpers.cjs');
+
+const enabled =
+  process.env.START_FORMAL_AI_E2E === '1' &&
+  Boolean(process.env.START_FORMAL_AI_ROOT) &&
+  process.platform !== 'win32';
+const dirs = [];
+afterAll(() => cleanup(...dirs));
+
+function run(request, files, extra = {}) {
+  const root = tempDir('e2e');
+  const cwd = populate(tempDir('e2e-fixture'), files);
+  dirs.push(root, cwd);
+  const before = snapshot(cwd);
+  const env = cliEnv(root, {
+    START_FORMAL_AI_ROOT: process.env.START_FORMAL_AI_ROOT,
+    START_FORMAL_AI_NODE: process.env.START_FORMAL_AI_NODE || 'node',
+    ...extra,
+  });
+  delete env.START_FORMAL_AI_MEMORY_PATH;
+  const result = spawnSync(
+    process.execPath,
+    [CLI, '--use-formal-ai', '--', ...request.split(' ')],
+    { cwd, env, encoding: 'utf8', timeout: 120000 }
+  );
+  return { ...result, root, cwd, before, after: snapshot(cwd) };
+}
+
+describe.skipIf(!enabled)('--use-formal-ai with a real runtime (#255)', () => {
+  it(
+    'prints the list of files in the current directory',
+    () => {
+      const result = run('print list of files in current directory', {
+        'a b.txt': 'one\n',
+        'ünï cødé.md': '# two\n',
+        'sub dir': null,
+        plain: '',
+      });
+      expect(result.status).toBe(0);
+      const listed = result.stdout
+        .split('\n')
+        .filter((line) => line && fs.existsSync(path.join(result.cwd, line)));
+      expect(listed.sort()).toEqual(fs.readdirSync(result.cwd).sort());
+      expect(result.after).toEqual(result.before);
+      const evidence = readEvidence(readLog(result.root));
+      expect(evidence.runtime.source).toBe('checkout');
+      expect(evidence.outcome.status).toBe('completed');
+      expect(evidence.effects).toMatchObject({
+        added: [],
+        removed: [],
+        changed: [],
+      });
+    },
+    { timeout: 150000 }
+  );
+
+  it(
+    'fails explicitly on an unsupported request',
+    () => {
+      const result = run('book me a flight to Paris', { 'a.txt': 'a' });
+      expect(result.status).toBe(65);
+      expect(result.after).toEqual(result.before);
+    },
+    { timeout: 150000 }
+  );
+});
+
+const SNIPPET =
+  '// keep this comment\nconst label = "before";\n\n\texport const shown = label;  \n/* "before" stays here */\n';
+const EDIT =
+  'In snippet.mjs, replace «const label = "before";» with «const label = "after";». Preserve all other content.';
+
+describe.skipIf(!enabled)(
+  '--use-formal-ai edits with a real runtime (#258)',
+  () => {
+    it(
+      'reads, edits and reads back one file in workspace-write mode',
+      () => {
+        const result = run(
+          EDIT,
+          { 'snippet.mjs': SNIPPET, 'other.txt': 'unrelated\n' },
+          { START_FORMAL_AI_EFFECTS: 'workspace-write' }
+        );
+        expect(result.status).toBe(0);
+        expect(
+          fs.readFileSync(path.join(result.cwd, 'snippet.mjs'), 'utf8')
+        ).toBe(
+          SNIPPET.replace('const label = "before";', 'const label = "after";')
+        );
+        expect(
+          fs.readFileSync(path.join(result.cwd, 'other.txt'), 'utf8')
+        ).toBe('unrelated\n');
+        const evidence = readEvidence(readLog(result.root));
+        expect(
+          evidence.steps.filter((s) => s.kind === 'tool').map((s) => s.command)
+        ).toEqual([
+          'read snippet.mjs',
+          'edit snippet.mjs',
+          'sha256sum -- snippet.mjs',
+        ]);
+        expect(evidence.effects).toMatchObject({
+          added: [],
+          removed: [],
+          changed: ['snippet.mjs'],
+        });
+      },
+      { timeout: 150000 }
+    );
+
+    it(
+      'exits nonzero with the capability gap in read-only mode',
+      () => {
+        const result = run(EDIT, { 'snippet.mjs': SNIPPET });
+        expect(result.status).toBe(65);
+        expect(result.after).toEqual(result.before);
+        const evidence = readEvidence(readLog(result.root));
+        expect(evidence.outcome).toMatchObject({
+          status: 'capability-gap',
+          gap: {
+            schema: 'workspace-edit-capability-gap/v1',
+            target: 'snippet.mjs',
+          },
+        });
+      },
+      { timeout: 150000 }
+    );
+
+    it(
+      'answers a plain question with exit 0',
+      () => {
+        const result = run('What is 2 + 2?', { 'a.txt': 'a' });
+        expect(result.status).toBe(0);
+        expect(result.stderr).toContain('2 + 2 = 4');
+        expect(result.after).toEqual(result.before);
+      },
+      { timeout: 150000 }
+    );
+  }
+);