start-command 0.41.0 → 0.43.0

package/test/regression-255-cli.js

@@ -177,12 +177,14 @@ describe.skipIf(!posix)('--use-formal-ai end to end (#255)', () => {
     );
     expect(result.after).toEqual(result.before);
 
-    // The request went to Formal AI verbatim, with only the bash tool.
+    // The request went to Formal AI verbatim; read-only mode offers bash and
+    // the structured read tool, never the change tools (#258).
     expect(result.requests[0].messages).toEqual([
       { role: 'user', content: 'print list of files in current directory' },
     ]);
     expect(result.requests[0].tools.map((t) => t.function.name)).toEqual([
       'bash',
+      'read',
     ]);
 
     const log = readLog(result.root);

package/test/regression-255-e2e.js

@@ -7,8 +7,11 @@
  *
  * It runs the issue's demo in an isolated directory whose names contain
  * spaces and Unicode, compares the listing with an independent readdir and
- * proves the directory is unchanged. Use START_FORMAL_AI_NODE to pick the
- * Node.js that runs the Formal AI server.
+ * proves the directory is unchanged. Issue #258 adds an isolated
+ * read/edit/read task in workspace-write mode, the explicit capability gap of
+ * the same request in read-only mode, and a plain question answered without
+ * tools. Use START_FORMAL_AI_NODE to pick the Node.js that runs the Formal AI
+ * server.
  */
 const { describe, it, expect, afterAll } = require('bun:test');
 const { spawnSync } = require('child_process');
@@ -32,7 +35,7 @@ const enabled =
 const dirs = [];
 afterAll(() => cleanup(...dirs));
 
-function run(request, files) {
+function run(request, files, extra = {}) {
   const root = tempDir('e2e');
   const cwd = populate(tempDir('e2e-fixture'), files);
   dirs.push(root, cwd);
@@ -40,6 +43,7 @@ function run(request, files) {
   const env = cliEnv(root, {
     START_FORMAL_AI_ROOT: process.env.START_FORMAL_AI_ROOT,
     START_FORMAL_AI_NODE: process.env.START_FORMAL_AI_NODE || 'node',
+    ...extra,
   });
   delete env.START_FORMAL_AI_MEMORY_PATH;
   const result = spawnSync(
@@ -88,3 +92,76 @@ describe.skipIf(!enabled)('--use-formal-ai with a real runtime (#255)', () => {
     { timeout: 150000 }
   );
 });
+
+const SNIPPET =
+  '// keep this comment\nconst label = "before";\n\n\texport const shown = label;  \n/* "before" stays here */\n';
+const EDIT =
+  'In snippet.mjs, replace «const label = "before";» with «const label = "after";». Preserve all other content.';
+
+describe.skipIf(!enabled)(
+  '--use-formal-ai edits with a real runtime (#258)',
+  () => {
+    it(
+      'reads, edits and reads back one file in workspace-write mode',
+      () => {
+        const result = run(
+          EDIT,
+          { 'snippet.mjs': SNIPPET, 'other.txt': 'unrelated\n' },
+          { START_FORMAL_AI_EFFECTS: 'workspace-write' }
+        );
+        expect(result.status).toBe(0);
+        expect(
+          fs.readFileSync(path.join(result.cwd, 'snippet.mjs'), 'utf8')
+        ).toBe(
+          SNIPPET.replace('const label = "before";', 'const label = "after";')
+        );
+        expect(
+          fs.readFileSync(path.join(result.cwd, 'other.txt'), 'utf8')
+        ).toBe('unrelated\n');
+        const evidence = readEvidence(readLog(result.root));
+        expect(
+          evidence.steps.filter((s) => s.kind === 'tool').map((s) => s.command)
+        ).toEqual([
+          'read snippet.mjs',
+          'edit snippet.mjs',
+          'sha256sum -- snippet.mjs',
+        ]);
+        expect(evidence.effects).toMatchObject({
+          added: [],
+          removed: [],
+          changed: ['snippet.mjs'],
+        });
+      },
+      { timeout: 150000 }
+    );
+
+    it(
+      'exits nonzero with the capability gap in read-only mode',
+      () => {
+        const result = run(EDIT, { 'snippet.mjs': SNIPPET });
+        expect(result.status).toBe(65);
+        expect(result.after).toEqual(result.before);
+        const evidence = readEvidence(readLog(result.root));
+        expect(evidence.outcome).toMatchObject({
+          status: 'capability-gap',
+          gap: {
+            schema: 'workspace-edit-capability-gap/v1',
+            target: 'snippet.mjs',
+          },
+        });
+      },
+      { timeout: 150000 }
+    );
+
+    it(
+      'answers a plain question with exit 0',
+      () => {
+        const result = run('What is 2 + 2?', { 'a.txt': 'a' });
+        expect(result.status).toBe(0);
+        expect(result.stderr).toContain('2 + 2 = 4');
+        expect(result.after).toEqual(result.before);
+      },
+      { timeout: 150000 }
+    );
+  }
+);

package/test/regression-255-loop.js

@@ -339,8 +339,13 @@ describe('--use-formal-ai refusals and unsupported requests (#255)', () => {
     expect(result.outcome.status).toBe('protocol-error');
     expect(result.outcome.exitCode).toBe(EXIT.PROTOCOL);
     expect(result.outcome.message).toBe(
-      'Formal AI returned HTTP 500 (after 2 retries)'
+      'Formal AI returned HTTP 500: synthetic failure (after 2 retries)'
     );
+    expect(result.outcome.diagnostics).toMatchObject({
+      httpStatus: 500,
+      errorMessage: 'synthetic failure',
+      attempts: 3,
+    });
     expect(result.requests).toBe(3);
   });
 });

package/test/regression-255-policy.js

@@ -340,7 +340,7 @@ describe('--use-formal-ai protocol (#255)', () => {
     ).toEqual({
       content: 'x',
       finishReason: 'tool_calls',
-      toolCalls: [{ id: 'c1', name: 'bash', command: 'ls' }],
+      toolCalls: [{ id: 'c1', name: 'bash', tool: 'bash', command: 'ls' }],
       handler: null,
     });
     expect(

package/test/regression-257.js

@@ -0,0 +1,265 @@
+/**
+ * Issue #257: an HTTP 5xx reply of the Formal AI runtime keeps its bounded,
+ * sanitized cause (error type, message and body preview) and the attempt
+ * count in the final error and in the evidence, also after bounded retries,
+ * without a fallback and without exposing credentials or the environment.
+ *
+ * The offline stand-in replays the exact 500 body a released start discarded:
+ * {"error":{"message":"Aborted(Error: LocalProductWasmForbidden). ...",
+ * "type":"formal_ai_error"}}.
+ */
+const { describe, it, expect, beforeAll, afterAll } = require('bun:test');
+const { spawnSync } = require('child_process');
+const http = require('http');
+const { resolveBounds, EXIT } = require('../src/lib/formal-ai-policy');
+const {
+  ProtocolError,
+  createClient,
+  httpErrorDetails,
+  sanitizeDiagnostic,
+} = require('../src/lib/formal-ai-client');
+const { runFormalAiTask } = require('../src/lib/formal-ai');
+const {
+  CAPTURED_500,
+  createFakeServer,
+} = require('./fixtures/formal-ai/replay.cjs');
+const {
+  CLI,
+  cleanup,
+  cliEnv,
+  fakeCheckout,
+  readEvidence,
+  readLog,
+  receiptFixture,
+  sentRequests,
+  snapshot,
+  tempDir,
+} = require('./fixtures/formal-ai/helpers.cjs');
+
+const CAUSE =
+  'Aborted(Error: LocalProductWasmForbidden). Build with -sASSERTIONS for more info.';
+
+let server;
+let baseUrl;
+const dirs = [];
+
+beforeAll(async () => {
+  server = createFakeServer();
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+  baseUrl = `http://127.0.0.1:${server.address().port}`;
+});
+
+afterAll(() => {
+  server.close();
+  cleanup(...dirs);
+});
+
+const ask = (request) => [{ role: 'user', content: request }];
+
+/** One client request against the stand-in; resolves to the thrown error. */
+async function failure(request, { maxRetries = 0, token = null } = {}) {
+  const bounds = { ...resolveBounds({}), maxRetries };
+  const client = createClient({
+    baseUrl,
+    bounds,
+    deadline: Date.now() + bounds.timeoutMs,
+    token,
+  });
+  try {
+    await client.complete(ask(request));
+  } catch (error) {
+    return { error, requests: client.requests };
+  }
+  throw new Error('expected the request to fail');
+}
+
+describe('5xx diagnostics of the Formal AI client (#257)', () => {
+  it('keeps the captured cause when no retry is allowed', async () => {
+    const { error, requests } = await failure(
+      'synthetic: server error (captured #257)'
+    );
+    expect(error).toBeInstanceOf(ProtocolError);
+    expect(error.exitCode).toBe(EXIT.PROTOCOL);
+    expect(error.message).toBe(
+      `Formal AI returned HTTP 500: formal_ai_error: ${CAUSE}`
+    );
+    expect(error.details).toEqual({
+      httpStatus: 500,
+      errorType: 'formal_ai_error',
+      errorMessage: CAUSE,
+      bodyPreview: CAPTURED_500,
+      bodyChars: CAPTURED_500.length,
+      attempts: 1,
+    });
+    expect(requests).toBe(1);
+  });
+
+  it('keeps the cause and the attempt count after bounded retries', async () => {
+    const { error, requests } = await failure(
+      'synthetic: server error (captured #257)',
+      { maxRetries: 1 }
+    );
+    expect(error.exitCode).toBe(EXIT.PROTOCOL);
+    expect(error.message).toBe(
+      `Formal AI returned HTTP 500: formal_ai_error: ${CAUSE} (after 1 retry)`
+    );
+    expect(error.details).toMatchObject({
+      httpStatus: 500,
+      errorType: 'formal_ai_error',
+      errorMessage: CAUSE,
+      attempts: 2,
+    });
+    expect(requests).toBe(2);
+  });
+
+  it('never exposes the token, authorization values or control characters', async () => {
+    const token = 'sk-test-0123456789';
+    const { error } = await failure('synthetic: server error with secrets', {
+      maxRetries: 1,
+      token,
+    });
+    const text = JSON.stringify({ message: error.message, ...error.details });
+    expect(text).not.toContain(token);
+    expect(text).not.toContain('sk-live-abcdef');
+    expect(text).not.toContain('\u0007');
+    expect(error.details).toMatchObject({
+      httpStatus: 503,
+      errorType: 'upstream_error',
+      attempts: 2,
+    });
+    expect(error.message).toContain('HTTP 503: upstream_error: proxy refused');
+    expect(error.message).toContain('[REDACTED]');
+  });
+
+  it('bounds and sanitizes any body', () => {
+    const long = `${'x'.repeat(2000)}`;
+    const details = httpErrorDetails(502, long, []);
+    expect(details.bodyChars).toBe(2000);
+    expect(details.bodyPreview.length).toBeLessThan(600);
+    expect(details.bodyPreview).toContain('[1500 more chars]');
+    expect(details.errorMessage).toBeNull();
+    expect(httpErrorDetails(500, '', [])).toMatchObject({
+      bodyPreview: '',
+      bodyChars: 0,
+      errorType: null,
+      errorMessage: null,
+    });
+    expect(httpErrorDetails(500, '{"error":"plain text"}', [])).toMatchObject({
+      errorMessage: 'plain text',
+    });
+    expect(
+      sanitizeDiagnostic(
+        'Authorization: Basic dXNlcjpwYXNz\nproxy-authorization: Bearer abc password=hunter2 secret-value',
+        500,
+        ['secret-value']
+      )
+    ).not.toMatch(/dXNlcjpwYXNz|abc|hunter2|secret-value|\n/);
+  });
+
+  it('keeps 4xx bodies and does not retry them', async () => {
+    const error4xx = http.createServer((req, res) => {
+      req.resume();
+      req.on('end', () => {
+        res.writeHead(400, { 'content-type': 'application/json' });
+        res.end('{"error":{"message":"bad tools","type":"invalid_request"}}');
+      });
+    });
+    await new Promise((resolve) => error4xx.listen(0, '127.0.0.1', resolve));
+    try {
+      const bounds = { ...resolveBounds({}), maxRetries: 2 };
+      const client = createClient({
+        baseUrl: `http://127.0.0.1:${error4xx.address().port}`,
+        bounds,
+        deadline: Date.now() + bounds.timeoutMs,
+      });
+      const error = await client.complete(ask('x')).catch((e) => e);
+      expect(error.message).toBe(
+        'Formal AI returned HTTP 400: invalid_request: bad tools'
+      );
+      expect(error.details).toMatchObject({ httpStatus: 400, attempts: 1 });
+      expect(client.requests).toBe(1);
+    } finally {
+      error4xx.close();
+    }
+  });
+
+  it('records the cause in the task evidence and changes nothing', async () => {
+    const cwd = receiptFixture();
+    dirs.push(cwd);
+    const before = snapshot(cwd);
+    const bounds = {
+      ...resolveBounds({}),
+      maxRetries: 1,
+      effects: 'workspace-write',
+    };
+    const client = createClient({
+      baseUrl,
+      bounds,
+      deadline: Date.now() + bounds.timeoutMs,
+    });
+    const evidence = await runFormalAiTask({
+      request: 'synthetic: server error (captured #257)',
+      cwd,
+      env: { PATH: process.env.PATH },
+      bounds,
+      client,
+    });
+    expect(evidence.outcome).toMatchObject({
+      status: 'protocol-error',
+      exitCode: EXIT.PROTOCOL,
+      message: `Formal AI returned HTTP 500: formal_ai_error: ${CAUSE} (after 1 retry)`,
+      diagnostics: {
+        httpStatus: 500,
+        errorType: 'formal_ai_error',
+        errorMessage: CAUSE,
+        attempts: 2,
+      },
+    });
+    expect(evidence.steps.filter((s) => s.kind === 'tool')).toEqual([]);
+    expect(snapshot(cwd)).toEqual(before);
+  });
+});
+
+describe('5xx diagnostics through the CLI (#257)', () => {
+  it('exits 70 with the cause in stderr, log and evidence', () => {
+    const checkout = fakeCheckout();
+    const root = tempDir('cli-257');
+    const cwd = receiptFixture();
+    dirs.push(checkout, root, cwd);
+    const before = snapshot(cwd);
+    const result = spawnSync(
+      process.execPath,
+      [CLI, '--use-formal-ai', '--', 'synthetic: server error (captured #257)'],
+      {
+        cwd,
+        env: cliEnv(root, {
+          START_FORMAL_AI_ROOT: checkout,
+          START_FORMAL_AI_MAX_RETRIES: '0',
+        }),
+        encoding: 'utf8',
+        timeout: 60000,
+      }
+    );
+    expect(result.status).toBe(EXIT.PROTOCOL);
+    expect(result.stderr).toContain(CAUSE);
+    expect(sentRequests(root)).toHaveLength(1);
+    expect(snapshot(cwd)).toEqual(before);
+    const log = readLog(root);
+    const evidence = readEvidence(log);
+    expect(evidence.outcome).toMatchObject({
+      status: 'protocol-error',
+      exitCode: EXIT.PROTOCOL,
+      message: `Formal AI returned HTTP 500: formal_ai_error: ${CAUSE}`,
+      diagnostics: {
+        httpStatus: 500,
+        errorType: 'formal_ai_error',
+        errorMessage: CAUSE,
+        bodyPreview: CAPTURED_500,
+        attempts: 1,
+      },
+    });
+    expect(evidence.formalAiRequests).toBe(1);
+    // The evidence never carries the environment.
+    expect(JSON.stringify(evidence)).not.toContain(process.env.PATH);
+  });
+});