start-command 0.41.0 → 0.42.0

package/test/regression-255-policy.js

@@ -340,7 +340,7 @@ describe('--use-formal-ai protocol (#255)', () => {
     ).toEqual({
       content: 'x',
       finishReason: 'tool_calls',
-      toolCalls: [{ id: 'c1', name: 'bash', command: 'ls' }],
+      toolCalls: [{ id: 'c1', name: 'bash', tool: 'bash', command: 'ls' }],
       handler: null,
     });
     expect(

package/test/regression-257.js

@@ -0,0 +1,265 @@
+/**
+ * Issue #257: an HTTP 5xx reply of the Formal AI runtime keeps its bounded,
+ * sanitized cause (error type, message and body preview) and the attempt
+ * count in the final error and in the evidence, also after bounded retries,
+ * without a fallback and without exposing credentials or the environment.
+ *
+ * The offline stand-in replays the exact 500 body a released start discarded:
+ * {"error":{"message":"Aborted(Error: LocalProductWasmForbidden). ...",
+ * "type":"formal_ai_error"}}.
+ */
+const { describe, it, expect, beforeAll, afterAll } = require('bun:test');
+const { spawnSync } = require('child_process');
+const http = require('http');
+const { resolveBounds, EXIT } = require('../src/lib/formal-ai-policy');
+const {
+  ProtocolError,
+  createClient,
+  httpErrorDetails,
+  sanitizeDiagnostic,
+} = require('../src/lib/formal-ai-client');
+const { runFormalAiTask } = require('../src/lib/formal-ai');
+const {
+  CAPTURED_500,
+  createFakeServer,
+} = require('./fixtures/formal-ai/replay.cjs');
+const {
+  CLI,
+  cleanup,
+  cliEnv,
+  fakeCheckout,
+  readEvidence,
+  readLog,
+  receiptFixture,
+  sentRequests,
+  snapshot,
+  tempDir,
+} = require('./fixtures/formal-ai/helpers.cjs');
+
+const CAUSE =
+  'Aborted(Error: LocalProductWasmForbidden). Build with -sASSERTIONS for more info.';
+
+let server;
+let baseUrl;
+const dirs = [];
+
+beforeAll(async () => {
+  server = createFakeServer();
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+  baseUrl = `http://127.0.0.1:${server.address().port}`;
+});
+
+afterAll(() => {
+  server.close();
+  cleanup(...dirs);
+});
+
+const ask = (request) => [{ role: 'user', content: request }];
+
+/** One client request against the stand-in; resolves to the thrown error. */
+async function failure(request, { maxRetries = 0, token = null } = {}) {
+  const bounds = { ...resolveBounds({}), maxRetries };
+  const client = createClient({
+    baseUrl,
+    bounds,
+    deadline: Date.now() + bounds.timeoutMs,
+    token,
+  });
+  try {
+    await client.complete(ask(request));
+  } catch (error) {
+    return { error, requests: client.requests };
+  }
+  throw new Error('expected the request to fail');
+}
+
+describe('5xx diagnostics of the Formal AI client (#257)', () => {
+  it('keeps the captured cause when no retry is allowed', async () => {
+    const { error, requests } = await failure(
+      'synthetic: server error (captured #257)'
+    );
+    expect(error).toBeInstanceOf(ProtocolError);
+    expect(error.exitCode).toBe(EXIT.PROTOCOL);
+    expect(error.message).toBe(
+      `Formal AI returned HTTP 500: formal_ai_error: ${CAUSE}`
+    );
+    expect(error.details).toEqual({
+      httpStatus: 500,
+      errorType: 'formal_ai_error',
+      errorMessage: CAUSE,
+      bodyPreview: CAPTURED_500,
+      bodyChars: CAPTURED_500.length,
+      attempts: 1,
+    });
+    expect(requests).toBe(1);
+  });
+
+  it('keeps the cause and the attempt count after bounded retries', async () => {
+    const { error, requests } = await failure(
+      'synthetic: server error (captured #257)',
+      { maxRetries: 1 }
+    );
+    expect(error.exitCode).toBe(EXIT.PROTOCOL);
+    expect(error.message).toBe(
+      `Formal AI returned HTTP 500: formal_ai_error: ${CAUSE} (after 1 retry)`
+    );
+    expect(error.details).toMatchObject({
+      httpStatus: 500,
+      errorType: 'formal_ai_error',
+      errorMessage: CAUSE,
+      attempts: 2,
+    });
+    expect(requests).toBe(2);
+  });
+
+  it('never exposes the token, authorization values or control characters', async () => {
+    const token = 'sk-test-0123456789';
+    const { error } = await failure('synthetic: server error with secrets', {
+      maxRetries: 1,
+      token,
+    });
+    const text = JSON.stringify({ message: error.message, ...error.details });
+    expect(text).not.toContain(token);
+    expect(text).not.toContain('sk-live-abcdef');
+    expect(text).not.toContain('\u0007');
+    expect(error.details).toMatchObject({
+      httpStatus: 503,
+      errorType: 'upstream_error',
+      attempts: 2,
+    });
+    expect(error.message).toContain('HTTP 503: upstream_error: proxy refused');
+    expect(error.message).toContain('[REDACTED]');
+  });
+
+  it('bounds and sanitizes any body', () => {
+    const long = `${'x'.repeat(2000)}`;
+    const details = httpErrorDetails(502, long, []);
+    expect(details.bodyChars).toBe(2000);
+    expect(details.bodyPreview.length).toBeLessThan(600);
+    expect(details.bodyPreview).toContain('[1500 more chars]');
+    expect(details.errorMessage).toBeNull();
+    expect(httpErrorDetails(500, '', [])).toMatchObject({
+      bodyPreview: '',
+      bodyChars: 0,
+      errorType: null,
+      errorMessage: null,
+    });
+    expect(httpErrorDetails(500, '{"error":"plain text"}', [])).toMatchObject({
+      errorMessage: 'plain text',
+    });
+    expect(
+      sanitizeDiagnostic(
+        'Authorization: Basic dXNlcjpwYXNz\nproxy-authorization: Bearer abc password=hunter2 secret-value',
+        500,
+        ['secret-value']
+      )
+    ).not.toMatch(/dXNlcjpwYXNz|abc|hunter2|secret-value|\n/);
+  });
+
+  it('keeps 4xx bodies and does not retry them', async () => {
+    const error4xx = http.createServer((req, res) => {
+      req.resume();
+      req.on('end', () => {
+        res.writeHead(400, { 'content-type': 'application/json' });
+        res.end('{"error":{"message":"bad tools","type":"invalid_request"}}');
+      });
+    });
+    await new Promise((resolve) => error4xx.listen(0, '127.0.0.1', resolve));
+    try {
+      const bounds = { ...resolveBounds({}), maxRetries: 2 };
+      const client = createClient({
+        baseUrl: `http://127.0.0.1:${error4xx.address().port}`,
+        bounds,
+        deadline: Date.now() + bounds.timeoutMs,
+      });
+      const error = await client.complete(ask('x')).catch((e) => e);
+      expect(error.message).toBe(
+        'Formal AI returned HTTP 400: invalid_request: bad tools'
+      );
+      expect(error.details).toMatchObject({ httpStatus: 400, attempts: 1 });
+      expect(client.requests).toBe(1);
+    } finally {
+      error4xx.close();
+    }
+  });
+
+  it('records the cause in the task evidence and changes nothing', async () => {
+    const cwd = receiptFixture();
+    dirs.push(cwd);
+    const before = snapshot(cwd);
+    const bounds = {
+      ...resolveBounds({}),
+      maxRetries: 1,
+      effects: 'workspace-write',
+    };
+    const client = createClient({
+      baseUrl,
+      bounds,
+      deadline: Date.now() + bounds.timeoutMs,
+    });
+    const evidence = await runFormalAiTask({
+      request: 'synthetic: server error (captured #257)',
+      cwd,
+      env: { PATH: process.env.PATH },
+      bounds,
+      client,
+    });
+    expect(evidence.outcome).toMatchObject({
+      status: 'protocol-error',
+      exitCode: EXIT.PROTOCOL,
+      message: `Formal AI returned HTTP 500: formal_ai_error: ${CAUSE} (after 1 retry)`,
+      diagnostics: {
+        httpStatus: 500,
+        errorType: 'formal_ai_error',
+        errorMessage: CAUSE,
+        attempts: 2,
+      },
+    });
+    expect(evidence.steps.filter((s) => s.kind === 'tool')).toEqual([]);
+    expect(snapshot(cwd)).toEqual(before);
+  });
+});
+
+describe('5xx diagnostics through the CLI (#257)', () => {
+  it('exits 70 with the cause in stderr, log and evidence', () => {
+    const checkout = fakeCheckout();
+    const root = tempDir('cli-257');
+    const cwd = receiptFixture();
+    dirs.push(checkout, root, cwd);
+    const before = snapshot(cwd);
+    const result = spawnSync(
+      process.execPath,
+      [CLI, '--use-formal-ai', '--', 'synthetic: server error (captured #257)'],
+      {
+        cwd,
+        env: cliEnv(root, {
+          START_FORMAL_AI_ROOT: checkout,
+          START_FORMAL_AI_MAX_RETRIES: '0',
+        }),
+        encoding: 'utf8',
+        timeout: 60000,
+      }
+    );
+    expect(result.status).toBe(EXIT.PROTOCOL);
+    expect(result.stderr).toContain(CAUSE);
+    expect(sentRequests(root)).toHaveLength(1);
+    expect(snapshot(cwd)).toEqual(before);
+    const log = readLog(root);
+    const evidence = readEvidence(log);
+    expect(evidence.outcome).toMatchObject({
+      status: 'protocol-error',
+      exitCode: EXIT.PROTOCOL,
+      message: `Formal AI returned HTTP 500: formal_ai_error: ${CAUSE}`,
+      diagnostics: {
+        httpStatus: 500,
+        errorType: 'formal_ai_error',
+        errorMessage: CAUSE,
+        bodyPreview: CAPTURED_500,
+        attempts: 1,
+      },
+    });
+    expect(evidence.formalAiRequests).toBe(1);
+    // The evidence never carries the environment.
+    expect(JSON.stringify(evidence)).not.toContain(process.env.PATH);
+  });
+});

package/test/regression-258-files.js

@@ -0,0 +1,430 @@
+/**
+ * Issue #258: the structured read/edit/write tools of `--use-formal-ai`
+ * (src/lib/formal-ai-files.js) and their loop integration, offline.
+ */
+const { describe, it, expect, afterAll } = require('bun:test');
+const fs = require('fs');
+const path = require('path');
+const {
+  BOUND_DEFAULTS,
+  resolveBounds,
+  EXIT,
+} = require('../src/lib/formal-ai-policy');
+const files = require('../src/lib/formal-ai-files');
+const { runFormalAiTask } = require('../src/lib/formal-ai');
+const {
+  cleanup,
+  populate,
+  sink,
+  snapshot,
+  tempDir,
+} = require('./fixtures/formal-ai/helpers.cjs');
+
+const posix = process.platform !== 'win32';
+const dirs = [];
+afterAll(() => cleanup(...dirs));
+
+const TEXT =
+  '// keep this comment\r\nconst a = 1;\t \n\n  const b = 2; // b\nconst a2 = 1;\n';
+
+function workspace(extra = {}) {
+  const dir = populate(tempDir('files'), {
+    'a.js': TEXT,
+    'sub/b.txt': 'one two one\n',
+    ...extra,
+  });
+  dirs.push(dir);
+  return dir;
+}
+
+function context(cwd, effects = 'workspace-write') {
+  return {
+    cwd,
+    effects,
+    observed: new Set(),
+    observedDirs: new Set(),
+    contents: new Map(),
+  };
+}
+
+const bounds = { maxFileBytes: BOUND_DEFAULTS.maxFileBytes.value };
+
+/** Normalize, decide and run one call like the loop does. */
+function call(tool, args, ctx, limits = bounds) {
+  const normalized = files.normalizeFileArgs(tool, args);
+  if (normalized.problem) {
+    return { problem: normalized.problem };
+  }
+  const decision = files.evaluateFileCall(tool, normalized.args, ctx);
+  if (!decision.allowed) {
+    return { refused: decision.reason };
+  }
+  return files.runFileCall(tool, normalized.args, decision, ctx, limits);
+}
+
+describe('file tool arguments (#258)', () => {
+  it('normalizes aliases and rejects malformed arguments', () => {
+    expect(
+      files.normalizeFileArgs('edit', {
+        filePath: 'a.js',
+        oldString: 'x',
+        newString: 'y',
+      })
+    ).toEqual({
+      args: { path: 'a.js', oldString: 'x', newString: 'y', replaceAll: false },
+    });
+    for (const [tool, args] of [
+      ['read', null],
+      ['read', []],
+      ['read', {}],
+      ['read', { path: 1 }],
+      ['read', { path: ' ' }],
+      ['read', { path: 'a\0b' }],
+      ['read', { path: 'a', file_path: 'b' }],
+      ['edit', { path: 'a', old_string: 'x' }],
+      ['edit', { path: 'a', old_string: 'x', new_string: 'y', replace_all: 1 }],
+      ['write', { path: 'a' }],
+    ]) {
+      expect(files.normalizeFileArgs(tool, args).problem).toBeString();
+    }
+  });
+});
+
+describe('file tool policy (#258)', () => {
+  it('refuses changes in read-only mode, outside paths and unobserved targets', () => {
+    const cwd = workspace();
+    const outside = tempDir('outside');
+    dirs.push(outside);
+    fs.writeFileSync(path.join(outside, 'secret.txt'), 'secret');
+    fs.symlinkSync(path.join(outside, 'secret.txt'), path.join(cwd, 'link'));
+    const ro = context(cwd, 'read-only');
+    expect(
+      call('edit', { path: 'a.js', old_string: 'a', new_string: 'b' }, ro)
+    ).toMatchObject({ refused: expect.stringContaining('read-only') });
+    expect(call('write', { path: 'a.js', content: '' }, ro).refused).toContain(
+      'START_FORMAL_AI_EFFECTS=workspace-write'
+    );
+    const rw = context(cwd);
+    for (const target of [
+      '../x',
+      '/etc/passwd',
+      '~/.ssh/id_rsa',
+      'link',
+      path.join(outside, 'secret.txt'),
+    ]) {
+      expect(call('read', { path: target }, rw).refused).toContain(
+        'outside the working directory'
+      );
+    }
+    expect(call('read', { path: '.' }, rw).refused).toContain('not a file');
+    expect(
+      call('edit', { path: 'a.js', old_string: 'a', new_string: 'b' }, rw)
+        .refused
+    ).toBe('target "a.js" was not read before the change');
+    expect(call('write', { path: 'new.txt', content: 'x' }, rw).refused).toBe(
+      'target "new.txt" was not observed before the change'
+    );
+    rw.observedDirs.add('.');
+    // A new file in a listed directory may be created, an existing one not.
+    expect(call('write', { path: 'a.js', content: 'x' }, rw).refused).toBe(
+      'target "a.js" was not observed before the change'
+    );
+    expect(
+      call('write', { path: 'new.txt', content: 'x\n' }, rw)
+    ).toMatchObject({ exitCode: 0, verification: { verified: true } });
+    expect(fs.readFileSync(path.join(cwd, 'new.txt'), 'utf8')).toBe('x\n');
+    expect(fs.readFileSync(path.join(outside, 'secret.txt'), 'utf8')).toBe(
+      'secret'
+    );
+  });
+});
+
+describe('read (#258)', () => {
+  it('returns the exact text and records the observation', () => {
+    const cwd = workspace();
+    const ctx = context(cwd, 'read-only');
+    const receipt = call('read', { path: './a.js' }, ctx);
+    expect(receipt).toMatchObject({
+      exitCode: 0,
+      result: TEXT,
+      bytes: Buffer.byteLength(TEXT),
+    });
+    expect(ctx.observed.has('a.js')).toBe(true);
+    expect(ctx.contents.get('a.js')).toBe(receipt.sha256);
+  });
+
+  it('fails for missing, directory, oversized and non-text files', () => {
+    const cwd = workspace({ 'bin.dat': '', 'big.txt': 'x'.repeat(64) });
+    fs.writeFileSync(path.join(cwd, 'bin.dat'), Buffer.from([0xff, 0xfe, 0]));
+    fs.writeFileSync(path.join(cwd, 'nul.txt'), 'a\0b');
+    const ctx = context(cwd, 'read-only');
+    const error = (name, limits) => call('read', { path: name }, ctx, limits);
+    expect(error('missing.txt')).toMatchObject({
+      exitCode: 1,
+      error: 'missing.txt: no such file',
+    });
+    expect(JSON.parse(error('missing.txt').result)).toEqual({
+      ok: false,
+      is_error: true,
+      error: 'missing.txt: no such file',
+    });
+    expect(error('sub').error).toBe('sub: is a directory, not a file');
+    expect(error('bin.dat').error).toBe('bin.dat: is not UTF-8 text');
+    expect(error('nul.txt').error).toBe('nul.txt: is not UTF-8 text');
+    expect(error('big.txt', { maxFileBytes: 10 }).error).toBe(
+      'big.txt: has 64 bytes; START_FORMAL_AI_MAX_FILE_BYTES is 10'
+    );
+    expect(ctx.observed.size).toBe(0);
+  });
+});
+
+describe('edit (#258)', () => {
+  it('replaces one literal occurrence, preserving every other byte and the mode', () => {
+    const cwd = workspace();
+    const file = path.join(cwd, 'a.js');
+    if (posix) {
+      fs.chmodSync(file, 0o640);
+    }
+    const ctx = context(cwd);
+    call('read', { path: 'a.js' }, ctx);
+    const receipt = call(
+      'edit',
+      {
+        path: 'a.js',
+        old_string: 'const b = 2;',
+        new_string: 'const b = $&3;',
+      },
+      ctx
+    );
+    expect(receipt).toMatchObject({
+      exitCode: 0,
+      result: '',
+      replacements: 1,
+      verification: { verified: true },
+    });
+    expect(fs.readFileSync(file, 'utf8')).toBe(
+      TEXT.replace('const b = 2;', () => 'const b = $&3;')
+    );
+    if (posix) {
+      expect(fs.statSync(file).mode & 0o777).toBe(0o640);
+    }
+    // No temporary file is left behind.
+    expect(fs.readdirSync(cwd).sort()).toEqual(['a.js', 'sub']);
+  });
+
+  it('keeps a byte order mark', () => {
+    const cwd = workspace({ 'bom.txt': '\ufefffirst\nsecond\n' });
+    const ctx = context(cwd);
+    expect(call('read', { path: 'bom.txt' }, ctx).result).toBe(
+      '\ufefffirst\nsecond\n'
+    );
+    const receipt = call(
+      'edit',
+      { path: 'bom.txt', old_string: 'second', new_string: '2nd' },
+      ctx
+    );
+    expect(receipt.exitCode).toBe(0);
+    expect(fs.readFileSync(path.join(cwd, 'bom.txt'))).toEqual(
+      Buffer.from('\ufefffirst\n2nd\n', 'utf8')
+    );
+  });
+
+  it('refuses ambiguous, missing, empty and no-op replacements', () => {
+    const cwd = workspace();
+    const ctx = context(cwd);
+    call('read', { path: 'sub/b.txt' }, ctx);
+    const edit = (args) => call('edit', { path: 'sub/b.txt', ...args }, ctx);
+    expect(edit({ old_string: 'one', new_string: '1' }).error).toBe(
+      'sub/b.txt: old_string occurs 2 times; give more surrounding text or set replace_all; nothing changed'
+    );
+    expect(edit({ old_string: 'three', new_string: '3' }).error).toContain(
+      'not found'
+    );
+    expect(edit({ old_string: '', new_string: '3' }).error).toContain('empty');
+    expect(edit({ old_string: 'two', new_string: 'two' }).error).toContain(
+      'the same'
+    );
+    expect(fs.readFileSync(path.join(cwd, 'sub/b.txt'), 'utf8')).toBe(
+      'one two one\n'
+    );
+    expect(
+      edit({ old_string: 'one', new_string: '1', replace_all: true })
+    ).toMatchObject({ exitCode: 0, replacements: 2 });
+    expect(fs.readFileSync(path.join(cwd, 'sub/b.txt'), 'utf8')).toBe(
+      '1 two 1\n'
+    );
+  });
+
+  it('does not overwrite a file changed since it was read', () => {
+    const cwd = workspace();
+    const ctx = context(cwd);
+    call('read', { path: 'a.js' }, ctx);
+    fs.writeFileSync(path.join(cwd, 'a.js'), 'someone else\n');
+    expect(
+      call('edit', { path: 'a.js', old_string: 'else', new_string: 'x' }, ctx)
+        .error
+    ).toContain('changed since start read it');
+    expect(call('write', { path: 'a.js', content: 'x' }, ctx).error).toContain(
+      'changed since start read it'
+    );
+    expect(fs.readFileSync(path.join(cwd, 'a.js'), 'utf8')).toBe(
+      'someone else\n'
+    );
+  });
+
+  it('keeps the file size bound for the new text', () => {
+    const cwd = workspace();
+    const ctx = context(cwd);
+    const limits = { maxFileBytes: Buffer.byteLength(TEXT) };
+    call('read', { path: 'a.js' }, ctx, limits);
+    expect(
+      call(
+        'edit',
+        {
+          path: 'a.js',
+          old_string: 'const a = 1;',
+          new_string: 'const a = 10;',
+        },
+        ctx,
+        limits
+      ).error
+    ).toContain('START_FORMAL_AI_MAX_FILE_BYTES');
+    expect(fs.readFileSync(path.join(cwd, 'a.js'), 'utf8')).toBe(TEXT);
+  });
+});
+
+describe('write (#258)', () => {
+  it('replaces a read file and refuses a missing directory', () => {
+    const cwd = workspace();
+    const ctx = context(cwd);
+    call('read', { path: 'a.js' }, ctx);
+    expect(
+      call('write', { path: 'a.js', content: 'new\n' }, ctx)
+    ).toMatchObject({
+      exitCode: 0,
+      bytes: 4,
+      verification: { verified: true },
+    });
+    expect(fs.readFileSync(path.join(cwd, 'a.js'), 'utf8')).toBe('new\n');
+    ctx.observedDirs.add('gone');
+    expect(call('write', { path: 'gone/x.txt', content: 'x' }, ctx).error).toBe(
+      'gone/x.txt: its directory does not exist'
+    );
+  });
+});
+
+describe('START_FORMAL_AI_MAX_FILE_BYTES (#258)', () => {
+  it('defaults to 1 MiB and is bounded', () => {
+    expect(resolveBounds({}).maxFileBytes).toBe(1024 * 1024);
+    expect(
+      resolveBounds({ START_FORMAL_AI_MAX_FILE_BYTES: '4096' }).maxFileBytes
+    ).toBe(4096);
+    for (const value of ['0', '-1', 'abc', String(64 * 1024 * 1024)]) {
+      expect(() =>
+        resolveBounds({ START_FORMAL_AI_MAX_FILE_BYTES: value })
+      ).toThrow();
+    }
+  });
+});
+
+describe('file tools in the task loop (#258)', () => {
+  /** A scripted client: each turn returns the next message. */
+  function scripted(turns) {
+    let index = 0;
+    return {
+      complete: () => {
+        const message = turns[index++];
+        return Promise.resolve({
+          content: message.content || '',
+          finishReason: message.toolCalls ? 'tool_calls' : 'stop',
+          handler: null,
+          toolCalls: message.toolCalls || [],
+          raw: {},
+        });
+      },
+    };
+  }
+  const fileCall = (tool, args) => ({
+    id: `c_${tool}`,
+    name: tool,
+    tool,
+    args: files.normalizeFileArgs(tool, args).args,
+    arguments: JSON.stringify(args),
+    command: `${tool} ${args.path}`,
+  });
+
+  async function loop(turns, effects = 'workspace-write') {
+    const cwd = workspace();
+    const before = snapshot(cwd);
+    const evidence = await runFormalAiTask({
+      request: 'scripted',
+      cwd,
+      env: { PATH: process.env.PATH },
+      bounds: { ...resolveBounds({}), effects },
+      client: scripted(turns),
+      io: { stdout: sink(), stderr: sink(), note: () => {} },
+    });
+    return { cwd, before, after: snapshot(cwd), evidence };
+  }
+
+  it('stops with 77 before an unobserved edit runs', async () => {
+    const result = await loop([
+      {
+        toolCalls: [
+          fileCall('edit', { path: 'a.js', old_string: 'a', new_string: 'b' }),
+        ],
+      },
+    ]);
+    expect(result.evidence.outcome).toMatchObject({
+      status: 'refused',
+      exitCode: EXIT.REFUSED,
+      remainingObligations: [
+        'the refused call did not run; nothing after it ran either',
+      ],
+    });
+    expect(result.after).toEqual(result.before);
+  });
+
+  it('refuses edits in read-only mode even after a read', async () => {
+    const result = await loop(
+      [
+        { toolCalls: [fileCall('read', { path: 'a.js' })] },
+        {
+          toolCalls: [
+            fileCall('edit', {
+              path: 'a.js',
+              old_string: 'a',
+              new_string: 'b',
+            }),
+          ],
+        },
+      ],
+      'read-only'
+    );
+    expect(result.evidence.outcome.exitCode).toBe(EXIT.REFUSED);
+    expect(result.evidence.outcome.message).toContain('read-only');
+    expect(result.after).toEqual(result.before);
+  });
+
+  it('reports a failed edit with its real status and no change', async () => {
+    const result = await loop([
+      { toolCalls: [fileCall('read', { path: 'a.js' })] },
+      {
+        toolCalls: [
+          fileCall('edit', {
+            path: 'a.js',
+            old_string: 'zzz',
+            new_string: 'b',
+          }),
+        ],
+      },
+      { content: 'The edit failed.' },
+    ]);
+    expect(result.evidence.outcome).toMatchObject({
+      status: 'completed',
+      exitCode: 1,
+      remainingObligations: ['the last tool call failed'],
+    });
+    expect(result.after).toEqual(result.before);
+  });
+});