tabwright 1.0.0 → 2.0.0

package/src/replay-eval.ts

@@ -1,1039 +0,0 @@
-import fs from 'node:fs'
-import http from 'node:http'
-import path from 'node:path'
-import util from 'node:util'
-import { chromium, type Browser, type Page } from '@xmorse/playwright-core'
-import pc from 'picocolors'
-import { prepareCapabilityRun, runCapabilityWithExecutor, type CapabilityExecutor } from './capability-runner.js'
-import { createReplayAiIndexFromRecording } from './replay-ai-index.js'
-import { compileReplayWorkflow } from './replay-workflow-compiler.js'
-import type { ExecuteResult } from './executor.js'
-import type { RrwebEvent } from './protocol.js'
-import { getTabwrightUserDataDir } from './product-paths.js'
-
-type ReplayEvalExpectedStatus = 'completed' | 'needs_ai' | 'compile_failed'
-
-type ReplayEvalCase = {
-  id: string
-  title: string
-  description: string
-  editLabel: string
-  addLabel: string
-  submitLabel: string
-  okLabel: string
-  demonstratedValue: string
-  runValue: string
-  initialValues?: string[]
-  alreadyEditing?: boolean
-  draftAction?: 'restart' | 'continue'
-  brokenAdd?: boolean
-  unsupported?: boolean
-  addAnnotation?: boolean
-  deleteAnnotation?: boolean
-  expectedStatus: ReplayEvalExpectedStatus
-  expectedReasonIncludes?: string
-  expectedAnnotationCount?: number
-}
-
-export type ReplayEvalCaseResult = {
-  id: string
-  title: string
-  status: 'passed' | 'failed'
-  durationMs: number
-  replayId: string
-  phases: Array<'recording' | 'index' | 'compile' | 'run' | 'verify'>
-  index?: {
-    actionCount: number
-    fieldCount: number
-    annotationCount: number
-  }
-  compile?: {
-    actionKind: string
-    confidence: string
-    demonstratedValue?: string
-  }
-  run?: {
-    status?: string
-    value?: string
-    reason?: string
-    completed?: number
-    total?: number
-  }
-  error?: string
-}
-
-export type ReplayEvalReport = {
-  generatedAt: string
-  durationMs: number
-  passed: number
-  failed: number
-  total: number
-  results: ReplayEvalCaseResult[]
-  artifactsDir: string
-}
-
-export type RunReplayEvalOptions = {
-  caseId?: string
-  reportPath?: string
-  keepArtifacts?: boolean
-  headed?: boolean
-}
-
-type ExampleCaseState = {
-  values: string[]
-}
-
-type ExampleServer = {
-  baseUrl: string
-  close: () => Promise<void>
-}
-
-type AsyncFunctionConstructor = new (...args: string[]) => (...args: unknown[]) => Promise<unknown>
-
-const REPLAY_EVAL_CASES: ReplayEvalCase[] = [
-  {
-    id: 'zh-list-append',
-    title: 'Chinese admin list append',
-    description: '编辑 -> Add entry -> 填值 -> 提交 -> OK',
-    editLabel: '编辑',
-    addLabel: 'Add entry',
-    submitLabel: '提交',
-    okLabel: 'OK',
-    demonstratedValue: 'test2',
-    runValue: 'test3',
-    expectedStatus: 'completed',
-  },
-  {
-    id: 'en-list-append',
-    title: 'English list append',
-    description: 'Edit -> Add entry -> fill -> Submit -> OK',
-    editLabel: 'Edit',
-    addLabel: 'Add entry',
-    submitLabel: 'Submit',
-    okLabel: 'OK',
-    demonstratedValue: 'alpha',
-    runValue: 'beta',
-    expectedStatus: 'completed',
-  },
-  {
-    id: 'cn-add-confirm',
-    title: 'Chinese add and confirm labels',
-    description: '新增/确定 button labels should still compile and run.',
-    editLabel: '编辑',
-    addLabel: '新增',
-    submitLabel: '提交',
-    okLabel: '确定',
-    demonstratedValue: '中文值1',
-    runValue: '中文值2',
-    expectedStatus: 'completed',
-  },
-  {
-    id: 'already-editing',
-    title: 'Already editing page',
-    description: 'Generated script should continue from an already-editing page.',
-    editLabel: '编辑',
-    addLabel: 'Add entry',
-    submitLabel: '提交',
-    okLabel: 'OK',
-    demonstratedValue: 'inline-demo',
-    runValue: 'inline-run',
-    alreadyEditing: true,
-    expectedStatus: 'completed',
-  },
-  {
-    id: 'draft-restart',
-    title: 'Draft dialog restart',
-    description: 'Replay clicked 重新编辑, so generated script should restart the draft.',
-    editLabel: '编辑',
-    addLabel: 'Add entry',
-    submitLabel: '提交',
-    okLabel: 'OK',
-    demonstratedValue: 'restart-demo',
-    runValue: 'restart-run',
-    draftAction: 'restart',
-    expectedStatus: 'completed',
-  },
-  {
-    id: 'draft-continue',
-    title: 'Draft dialog continue',
-    description: 'Replay clicked 继续编辑, so generated script should continue the draft.',
-    editLabel: '编辑',
-    addLabel: 'Add entry',
-    submitLabel: '提交',
-    okLabel: 'OK',
-    demonstratedValue: 'continue-demo',
-    runValue: 'continue-run',
-    draftAction: 'continue',
-    expectedStatus: 'completed',
-  },
-  {
-    id: 'duplicate-short-circuit',
-    title: 'Duplicate value short-circuit',
-    description: 'If the target value is already visible, script should return completed without editing.',
-    editLabel: '编辑',
-    addLabel: 'Add entry',
-    submitLabel: '提交',
-    okLabel: 'OK',
-    demonstratedValue: 'existing-demo',
-    runValue: 'existing-demo',
-    initialValues: ['existing-demo'],
-    expectedStatus: 'completed',
-  },
-  {
-    id: 'drift-add-fails',
-    title: 'Page drift handoff',
-    description: 'The add button no longer creates an input, so script should return needs_ai.',
-    editLabel: '编辑',
-    addLabel: 'Add entry',
-    submitLabel: '提交',
-    okLabel: 'OK',
-    demonstratedValue: 'drift-demo',
-    runValue: 'drift-run',
-    brokenAdd: true,
-    expectedStatus: 'needs_ai',
-    expectedReasonIncludes: 'Add entry did not create a new list input',
-  },
-  {
-    id: 'annotation-delete-index',
-    title: 'Deleted annotation index',
-    description: 'A deleted annotation should not reach AI index output.',
-    editLabel: '编辑',
-    addLabel: 'Add entry',
-    submitLabel: '提交',
-    okLabel: 'OK',
-    demonstratedValue: 'annotation-demo',
-    runValue: 'annotation-run',
-    addAnnotation: true,
-    deleteAnnotation: true,
-    expectedStatus: 'completed',
-    expectedAnnotationCount: 0,
-  },
-  {
-    id: 'unsupported-click-only',
-    title: 'Unsupported click-only replay',
-    description: 'Compiler should fail clearly instead of generating a fake workflow.',
-    editLabel: '编辑',
-    addLabel: 'Add entry',
-    submitLabel: '提交',
-    okLabel: 'OK',
-    demonstratedValue: '',
-    runValue: '',
-    unsupported: true,
-    expectedStatus: 'compile_failed',
-  },
-]
-
-function ensureDir(dir: string): void {
-  fs.mkdirSync(dir, { recursive: true })
-}
-
-function createLocalTempDir(prefix: string): string {
-  const root = path.join(process.cwd(), 'tmp')
-  ensureDir(root)
-  return fs.mkdtempSync(path.join(root, prefix))
-}
-
-function writeJson(filePath: string, value: unknown): void {
-  ensureDir(path.dirname(filePath))
-  fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`)
-}
-
-function normalizeError(error: unknown): string {
-  return error instanceof Error ? error.message : String(error)
-}
-
-function textNode(id: number, textContent: string): Record<string, unknown> {
-  return { id, type: 3, textContent }
-}
-
-function elementNode(options: {
-  id: number
-  tagName: string
-  attributes?: Record<string, string>
-  childNodes?: Array<Record<string, unknown>>
-}): Record<string, unknown> {
-  return {
-    id: options.id,
-    type: 2,
-    tagName: options.tagName,
-    attributes: options.attributes || {},
-    childNodes: options.childNodes || [],
-  }
-}
-
-function clickEvent(options: { timestamp: number; id: number }): RrwebEvent {
-  return {
-    type: 3,
-    timestamp: options.timestamp,
-    data: {
-      source: 2,
-      type: 2,
-      id: options.id,
-      x: 10,
-      y: 10,
-      pointerType: 0,
-    },
-  }
-}
-
-function inputEvent(options: { timestamp: number; id: number; text: string }): RrwebEvent {
-  return {
-    type: 3,
-    timestamp: options.timestamp,
-    data: {
-      source: 5,
-      id: options.id,
-      text: options.text,
-      isChecked: false,
-    },
-  }
-}
-
-function annotationEvent(options: { timestamp: number; id: string; text: string }): RrwebEvent {
-  return {
-    type: 5,
-    timestamp: options.timestamp,
-    data: {
-      tag: 'playwriter.annotation',
-      payload: {
-        schemaVersion: 1,
-        id: options.id,
-        text: options.text,
-        timestamp: options.timestamp,
-        target: {
-          tagName: 'textarea',
-          label: 'List value',
-          selectorHints: ['textarea.designer-input'],
-          rect: { x: 10, y: 20, width: 240, height: 32 },
-        },
-      },
-    },
-  }
-}
-
-function annotationDeleteEvent(options: { timestamp: number; id: string }): RrwebEvent {
-  return {
-    type: 5,
-    timestamp: options.timestamp,
-    data: {
-      tag: 'playwriter.annotation.delete',
-      payload: {
-        schemaVersion: 1,
-        id: options.id,
-        timestamp: options.timestamp,
-      },
-    },
-  }
-}
-
-function createReplayEvents(testCase: ReplayEvalCase): RrwebEvent[] {
-  const nodes = [
-    elementNode({ id: 10, tagName: 'button', childNodes: [textNode(11, testCase.editLabel)] }),
-    elementNode({
-      id: 12,
-      tagName: 'button',
-      attributes: { class: 'designer-btn designer-btn-dangerous' },
-      childNodes: [textNode(13, testCase.draftAction === 'restart' ? '重新编辑' : 'Restart')],
-    }),
-    elementNode({
-      id: 14,
-      tagName: 'button',
-      attributes: { class: 'designer-btn designer-btn-primary' },
-      childNodes: [textNode(15, testCase.draftAction === 'continue' ? '继续编辑' : 'Continue')],
-    }),
-    elementNode({
-      id: 20,
-      tagName: 'button',
-      attributes: { class: 'designer-formily-array-base-addition' },
-      childNodes: [textNode(21, testCase.addLabel)],
-    }),
-    elementNode({ id: 30, tagName: 'textarea', attributes: { class: 'designer-input' } }),
-    elementNode({ id: 40, tagName: 'button', childNodes: [textNode(41, testCase.submitLabel)] }),
-    elementNode({
-      id: 50,
-      tagName: 'button',
-      attributes: { class: 'designer-btn designer-btn-primary' },
-      childNodes: [textNode(51, testCase.okLabel)],
-    }),
-  ]
-  const events: RrwebEvent[] = [
-    {
-      type: 2,
-      timestamp: 1000,
-      data: {
-        node: {
-          id: 1,
-          type: 0,
-          childNodes: [
-            {
-              id: 2,
-              type: 1,
-              childNodes: [
-                elementNode({
-                  id: 3,
-                  tagName: 'body',
-                  childNodes: nodes,
-                }),
-              ],
-            },
-          ],
-        },
-      },
-    },
-  ]
-  if (testCase.unsupported) {
-    events.push(clickEvent({ timestamp: 1100, id: 10 }))
-    return events
-  }
-  if (!testCase.alreadyEditing) {
-    events.push(clickEvent({ timestamp: 1100, id: 10 }))
-  }
-  if (testCase.draftAction === 'restart') {
-    events.push(clickEvent({ timestamp: 1200, id: 12 }))
-  }
-  if (testCase.draftAction === 'continue') {
-    events.push(clickEvent({ timestamp: 1200, id: 14 }))
-  }
-  events.push(clickEvent({ timestamp: 1300, id: 20 }))
-  events.push(inputEvent({ timestamp: 1400, id: 30, text: testCase.demonstratedValue }))
-  if (testCase.addAnnotation) {
-    events.push(annotationEvent({ timestamp: 1450, id: `${testCase.id}-annotation`, text: '重点填写这个列表项' }))
-  }
-  if (testCase.deleteAnnotation) {
-    events.push(annotationDeleteEvent({ timestamp: 1460, id: `${testCase.id}-annotation` }))
-  }
-  events.push(clickEvent({ timestamp: 1500, id: 40 }))
-  events.push(clickEvent({ timestamp: 1600, id: 50 }))
-  return events
-}
-
-function htmlEscape(value: string): string {
-  return value
-    .replace(/&/g, '&amp;')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&#39;')
-}
-
-function buildExampleHtml(testCase: ReplayEvalCase, state: ExampleCaseState): string {
-  const config = {
-    id: testCase.id,
-    labels: {
-      edit: testCase.editLabel,
-      add: testCase.addLabel,
-      submit: testCase.submitLabel,
-      ok: testCase.okLabel,
-      restart: testCase.draftAction === 'restart' ? '重新编辑' : 'Restart',
-      continue: testCase.draftAction === 'continue' ? '继续编辑' : 'Continue',
-    },
-    initialValues: state.values,
-    alreadyEditing: Boolean(testCase.alreadyEditing),
-    draftAction: testCase.draftAction || null,
-    brokenAdd: Boolean(testCase.brokenAdd),
-  }
-  return `<!doctype html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <title>${htmlEscape(testCase.title)}</title>
-    <style>
-      body { font-family: system-ui, sans-serif; padding: 24px; color: #172033; }
-      button { margin: 6px; padding: 7px 12px; border: 1px solid #cbd5e1; border-radius: 6px; background: white; cursor: pointer; }
-      textarea { display: block; width: 320px; min-height: 36px; margin: 8px 0; border: 1px solid #cbd5e1; border-radius: 6px; padding: 7px; }
-      .designer-modal { position: fixed; inset: auto 24px 24px auto; width: 360px; padding: 16px; border: 1px solid #cbd5e1; border-radius: 8px; background: white; box-shadow: 0 18px 48px rgba(15,23,42,.18); }
-      .readonly-value { padding: 4px 0; }
-    </style>
-  </head>
-  <body>
-    <h1>${htmlEscape(testCase.title)}</h1>
-    <div id="app"></div>
-    <script>
-      const config = ${JSON.stringify(config)};
-      const app = document.getElementById('app');
-      const state = {
-        values: [...config.initialValues],
-        editing: config.alreadyEditing,
-        draftShown: false,
-      };
-      function renderReadOnly() {
-        app.innerHTML = '<button id="edit">' + config.labels.edit + '</button><div id="values"></div>';
-        const values = document.getElementById('values');
-        state.values.forEach((value) => {
-          const item = document.createElement('div');
-          item.className = 'readonly-value';
-          item.textContent = value;
-          values.appendChild(item);
-        });
-        document.getElementById('edit').addEventListener('click', () => {
-          if (config.draftAction && !state.draftShown) {
-            state.draftShown = true;
-            renderDraftDialog();
-            return;
-          }
-          state.editing = true;
-          render();
-        });
-      }
-      function renderDraftDialog() {
-        renderReadOnly();
-        const modal = document.createElement('div');
-        modal.className = 'designer-modal';
-        modal.innerHTML = '<p>继续编辑上次修改 / Continue previous edit draft</p><button class="designer-btn designer-btn-dangerous" id="restart">' + config.labels.restart + '</button><button class="designer-btn designer-btn-primary" id="continue">' + config.labels.continue + '</button>';
-        document.body.appendChild(modal);
-        document.getElementById('restart').addEventListener('click', () => {
-          modal.remove();
-          state.values = [];
-          state.editing = true;
-          render();
-        });
-        document.getElementById('continue').addEventListener('click', () => {
-          modal.remove();
-          state.editing = true;
-          render();
-        });
-      }
-      function renderEditor() {
-        app.innerHTML = '<div id="editor"></div><button class="designer-formily-array-base-addition" id="add">' + config.labels.add + '</button><button id="submit">' + config.labels.submit + '</button>';
-        const editor = document.getElementById('editor');
-        state.values.forEach((value, index) => {
-          const input = document.createElement('textarea');
-          input.className = 'designer-input';
-          input.value = value;
-          input.addEventListener('input', () => {
-            state.values[index] = input.value;
-          });
-          editor.appendChild(input);
-        });
-        document.getElementById('add').addEventListener('click', () => {
-          if (!config.brokenAdd) {
-            state.values.push('');
-          }
-          render();
-        });
-        document.getElementById('submit').addEventListener('click', () => {
-          renderConfirm();
-        });
-      }
-      function renderConfirm() {
-        const modal = document.createElement('div');
-        modal.className = 'designer-modal';
-        modal.innerHTML = '<p>Confirm values: ' + state.values.join(', ') + '</p><button class="designer-btn designer-btn-primary" id="ok">' + config.labels.ok + '</button>';
-        document.body.appendChild(modal);
-        document.getElementById('ok').addEventListener('click', async () => {
-          await fetch('/enhancedConfigs/update', {
-            method: 'POST',
-            headers: { 'content-type': 'application/json' },
-            body: JSON.stringify({ caseId: config.id, values: state.values }),
-          });
-          await fetch('/enhancedConfigs/publish', { method: 'POST' });
-          await fetch('/enhancedConfigs/detail?id=' + encodeURIComponent(config.id));
-          modal.remove();
-          state.editing = false;
-          render();
-        });
-      }
-      function render() {
-        if (state.editing) {
-          renderEditor();
-          return;
-        }
-        renderReadOnly();
-      }
-      render();
-    </script>
-  </body>
-</html>`
-}
-
-function requestBody(request: http.IncomingMessage): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const chunks: Buffer[] = []
-    request.on('data', (chunk: Buffer) => {
-      chunks.push(chunk)
-    })
-    request.on('end', () => {
-      resolve(Buffer.concat(chunks).toString('utf-8'))
-    })
-    request.on('error', (error) => {
-      reject(error)
-    })
-  })
-}
-
-async function startExampleServer(cases: ReplayEvalCase[]): Promise<ExampleServer> {
-  const casesById = new Map(
-    cases.map((testCase) => {
-      return [testCase.id, testCase] as const
-    }),
-  )
-  const states: Map<string, ExampleCaseState> = new Map(
-    cases.map((testCase) => {
-      const state: ExampleCaseState = { values: [...(testCase.initialValues || [])] }
-      return [testCase.id, state]
-    }),
-  )
-  const server = http.createServer((request, response) => {
-    const host = request.headers.host || 'localhost'
-    const url = new URL(request.url || '/', `http://${host}`)
-    const sendJson = (statusCode: number, value: unknown): void => {
-      response.writeHead(statusCode, { 'content-type': 'application/json' })
-      response.end(JSON.stringify(value))
-    }
-    if (request.method === 'GET' && url.pathname.startsWith('/case/')) {
-      const caseId = decodeURIComponent(url.pathname.slice('/case/'.length))
-      const testCase = casesById.get(caseId)
-      const state = states.get(caseId)
-      if (!testCase || !state) {
-        response.writeHead(404)
-        response.end('Not found')
-        return
-      }
-      response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' })
-      response.end(buildExampleHtml(testCase, state))
-      return
-    }
-    if (request.method === 'POST' && url.pathname === '/enhancedConfigs/update') {
-      requestBody(request)
-        .then((body) => {
-          const parsed = JSON.parse(body || '{}') as { caseId?: unknown; values?: unknown }
-          const caseId = typeof parsed.caseId === 'string' ? parsed.caseId : ''
-          const state = states.get(caseId)
-          if (state && Array.isArray(parsed.values)) {
-            state.values = parsed.values.map((value) => {
-              return String(value)
-            })
-          }
-          sendJson(200, { code: 0, data: true })
-        })
-        .catch((error: unknown) => {
-          sendJson(500, { code: 1, message: normalizeError(error) })
-        })
-      return
-    }
-    if (request.method === 'POST' && url.pathname === '/enhancedConfigs/publish') {
-      sendJson(200, { code: 0, data: true })
-      return
-    }
-    if (request.method === 'GET' && url.pathname === '/enhancedConfigs/detail') {
-      const caseId = url.searchParams.get('id') || ''
-      const state = states.get(caseId)
-      sendJson(200, {
-        code: 0,
-        data: {
-          config: {
-            value: JSON.stringify(state?.values || []),
-          },
-        },
-      })
-      return
-    }
-    response.writeHead(404)
-    response.end('Not found')
-  })
-
-  await new Promise<void>((resolve, reject) => {
-    server.on('error', reject)
-    server.listen(0, '127.0.0.1', () => {
-      resolve()
-    })
-  })
-  const address = server.address()
-  if (!address || typeof address === 'string') {
-    throw new Error('Replay eval server did not expose a TCP port')
-  }
-  return {
-    baseUrl: `http://127.0.0.1:${address.port}`,
-    close: async () => {
-      await new Promise<void>((resolve, reject) => {
-        server.close((error) => {
-          if (error) {
-            reject(error)
-            return
-          }
-          resolve()
-        })
-      })
-    },
-  }
-}
-
-function getChromeExecutable(): string | undefined {
-  const candidates = [
-    process.env.TABWRIGHT_TEST_CHROME || process.env.PLAYWRITER_TEST_CHROME,
-    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
-    '/Applications/Chromium.app/Contents/MacOS/Chromium',
-    '/usr/bin/google-chrome',
-    '/usr/bin/google-chrome-stable',
-    '/usr/bin/chromium',
-    '/usr/bin/chromium-browser',
-    'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
-    'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
-  ].filter((candidate): candidate is string => {
-    return Boolean(candidate)
-  })
-  return candidates.find((candidate) => {
-    return fs.existsSync(candidate)
-  })
-}
-
-async function launchEvalBrowser(options: { headed?: boolean }): Promise<Browser> {
-  const executablePath = getChromeExecutable()
-  return await chromium.launch({
-    ...(executablePath ? { executablePath } : { channel: 'chromium' }),
-    headless: !options.headed,
-  })
-}
-
-function formatOutput(value: unknown): string {
-  return `[return value] ${util.inspect(value, {
-    depth: 5,
-    colors: false,
-    maxArrayLength: 80,
-    maxStringLength: 2000,
-    breakLength: 100,
-  })}`
-}
-
-class PageEvalExecutor implements CapabilityExecutor {
-  constructor(private page: Page) {}
-
-  async execute(code: string, timeout = 15000): Promise<ExecuteResult> {
-    const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor as AsyncFunctionConstructor
-    const runner = new AsyncFunction(
-      'page',
-      'snapshot',
-      'console',
-      'setTimeout',
-      'clearTimeout',
-      'URL',
-      'URLSearchParams',
-      'fetch',
-      'Buffer',
-      code,
-    )
-    const snapshot = async (): Promise<string> => {
-      return await this.page.locator('body').innerText({ timeout: 2000 }).catch((error: unknown) => {
-        return `snapshot failed: ${normalizeError(error)}`
-      })
-    }
-    try {
-      const structuredResult = await Promise.race([
-        runner(this.page, snapshot, console, setTimeout, clearTimeout, URL, URLSearchParams, fetch, Buffer),
-        new Promise((_, reject) => {
-          setTimeout(() => {
-            reject(new Error(`Replay eval execution timed out after ${timeout}ms`))
-          }, timeout)
-        }),
-      ])
-      return {
-        text: formatOutput(structuredResult),
-        images: [],
-        screenshots: [],
-        isError: false,
-        structuredResult: JSON.parse(JSON.stringify(structuredResult)),
-      }
-    } catch (error: unknown) {
-      return {
-        text: normalizeError(error),
-        images: [],
-        screenshots: [],
-        isError: true,
-      }
-    }
-  }
-}
-
-function writeRecording(options: {
-  home: string
-  replayId: string
-  url: string
-  events: RrwebEvent[]
-}): void {
-  const recordingsDir = path.join(getTabwrightUserDataDir({ homeDir: options.home }), 'rrweb-recordings')
-  const recordingPath = path.join(recordingsDir, `${options.replayId}.json`)
-  writeJson(recordingPath, options.events)
-  writeJson(path.join(recordingsDir, 'index.json'), [
-    {
-      id: options.replayId,
-      path: recordingPath,
-      startedAt: 1000,
-      savedAt: 2000,
-      duration: 1000,
-      size: fs.statSync(recordingPath).size,
-      eventCount: options.events.length,
-      tabId: 1,
-      url: options.url,
-    },
-  ])
-}
-
-function getOutputStatus(output: unknown): string | undefined {
-  if (!output || typeof output !== 'object') {
-    return undefined
-  }
-  const candidate = output as { status?: unknown }
-  return typeof candidate.status === 'string' ? candidate.status : undefined
-}
-
-function outputField(output: unknown, key: string): string | undefined {
-  if (!output || typeof output !== 'object') {
-    return undefined
-  }
-  const value = (output as Record<string, unknown>)[key]
-  return typeof value === 'string' ? value : undefined
-}
-
-function outputNumberField(output: unknown, key: string): number | undefined {
-  if (!output || typeof output !== 'object') {
-    return undefined
-  }
-  const value = (output as Record<string, unknown>)[key]
-  return typeof value === 'number' ? value : undefined
-}
-
-function assertCaseResult(options: { testCase: ReplayEvalCase; output: unknown; annotationCount: number }): void {
-  if (options.testCase.expectedAnnotationCount !== undefined && options.annotationCount !== options.testCase.expectedAnnotationCount) {
-    throw new Error(`Expected ${options.testCase.expectedAnnotationCount} annotations, got ${options.annotationCount}`)
-  }
-  const status = getOutputStatus(options.output)
-  if (status !== options.testCase.expectedStatus) {
-    throw new Error(`Expected run status ${options.testCase.expectedStatus}, got ${status || 'undefined'}`)
-  }
-  if (options.testCase.expectedReasonIncludes) {
-    const reason = outputField(options.output, 'reason') || ''
-    const needsAi = options.output && typeof options.output === 'object' ? (options.output as Record<string, unknown>).needsAi : undefined
-    const nestedReason = outputField(needsAi, 'reason') || ''
-    if (!reason.includes(options.testCase.expectedReasonIncludes) && !nestedReason.includes(options.testCase.expectedReasonIncludes)) {
-      throw new Error(`Expected reason to include "${options.testCase.expectedReasonIncludes}", got "${reason || nestedReason}"`)
-    }
-  }
-}
-
-async function runOneEvalCase(options: {
-  testCase: ReplayEvalCase
-  server: ExampleServer
-  browser: Browser
-  artifactsDir: string
-}): Promise<ReplayEvalCaseResult> {
-  const start = Date.now()
-  const phases: ReplayEvalCaseResult['phases'] = []
-  const replayId = `eval-${options.testCase.id}`
-  const home = path.join(options.artifactsDir, replayId, 'home')
-  const cwd = path.join(options.artifactsDir, replayId, 'cwd')
-  ensureDir(home)
-  ensureDir(cwd)
-  const previousHome = process.env.HOME
-  const url = `${options.server.baseUrl}/case/${encodeURIComponent(options.testCase.id)}?key=${encodeURIComponent(options.testCase.id)}`
-  try {
-    process.env.HOME = home
-    const events = createReplayEvents(options.testCase)
-    writeRecording({ home, replayId, url, events })
-    phases.push('recording')
-
-    const index = createReplayAiIndexFromRecording(replayId)
-    phases.push('index')
-
-    if (options.testCase.expectedStatus === 'compile_failed') {
-      try {
-        compileReplayWorkflow({ replayId, id: `${options.testCase.id}-capability`, cwd, overwrite: true })
-      } catch (error: unknown) {
-        phases.push('compile')
-        return {
-          id: options.testCase.id,
-          title: options.testCase.title,
-          status: 'passed',
-          durationMs: Date.now() - start,
-          replayId,
-          phases,
-          index: {
-            actionCount: index.actions.length,
-            fieldCount: index.fields.length,
-            annotationCount: index.annotations.length,
-          },
-          error: normalizeError(error),
-        }
-      }
-      throw new Error('Expected replay compiler to fail, but it produced a capability')
-    }
-
-    const compiled = compileReplayWorkflow({ replayId, id: `${options.testCase.id}-capability`, cwd, overwrite: true })
-    phases.push('compile')
-
-    const page = await options.browser.newPage()
-    try {
-      await page.goto(url)
-      const executor = new PageEvalExecutor(page)
-      const runResult = await runCapabilityWithExecutor({
-        executor,
-        id: `${options.testCase.id}-capability`,
-        input: { value: options.testCase.runValue },
-        cwd,
-        force: true,
-        confirmation: `${options.testCase.id}-capability`,
-        timeout: 20000,
-      })
-      phases.push('run')
-      assertCaseResult({
-        testCase: options.testCase,
-        output: runResult.output,
-        annotationCount: index.annotations.length,
-      })
-      phases.push('verify')
-      return {
-        id: options.testCase.id,
-        title: options.testCase.title,
-        status: 'passed',
-        durationMs: Date.now() - start,
-        replayId,
-        phases,
-        index: {
-          actionCount: index.actions.length,
-          fieldCount: index.fields.length,
-          annotationCount: index.annotations.length,
-        },
-        compile: {
-          actionKind: compiled.analysis.actionKind,
-          confidence: compiled.analysis.confidence,
-          demonstratedValue: compiled.analysis.demonstratedValue,
-        },
-        run: {
-          status: getOutputStatus(runResult.output),
-          value: outputField(runResult.output, 'value'),
-          reason: outputField(runResult.output, 'reason'),
-          completed: outputNumberField(runResult.output, 'completed'),
-          total: outputNumberField(runResult.output, 'total'),
-        },
-      }
-    } finally {
-      await page.close().catch(() => {})
-    }
-  } catch (error: unknown) {
-    return {
-      id: options.testCase.id,
-      title: options.testCase.title,
-      status: 'failed',
-      durationMs: Date.now() - start,
-      replayId,
-      phases,
-      error: normalizeError(error),
-    }
-  } finally {
-    if (previousHome === undefined) {
-      delete process.env.HOME
-    } else {
-      process.env.HOME = previousHome
-    }
-  }
-}
-
-function renderReportHtml(report: ReplayEvalReport): string {
-  const rows = report.results
-    .map((result) => {
-      const statusColor = result.status === 'passed' ? '#15803d' : '#b91c1c'
-      return `<tr>
-        <td>${htmlEscape(result.id)}</td>
-        <td>${htmlEscape(result.title)}</td>
-        <td style="color:${statusColor};font-weight:700">${htmlEscape(result.status)}</td>
-        <td>${result.durationMs}ms</td>
-        <td>${htmlEscape(result.phases.join(' -> '))}</td>
-        <td>${htmlEscape(result.run?.status || result.error || '-')}</td>
-      </tr>`
-    })
-    .join('\n')
-  return `<!doctype html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <title>Tabwright Replay Eval</title>
-    <style>
-      body { font-family: ui-sans-serif, system-ui, sans-serif; margin: 32px; color: #172033; }
-      table { border-collapse: collapse; width: 100%; }
-      th, td { border-bottom: 1px solid #e2e8f0; padding: 8px; text-align: left; vertical-align: top; }
-      th { background: #f8fafc; font-size: 12px; text-transform: uppercase; color: #64748b; }
-      .summary { margin-bottom: 18px; padding: 14px; border: 1px solid #e2e8f0; border-radius: 8px; background: #f8fafc; }
-    </style>
-  </head>
-  <body>
-    <h1>Tabwright Replay Eval</h1>
-    <div class="summary">
-      <strong>${report.passed}/${report.total} passed</strong>
-      <span>generated ${htmlEscape(report.generatedAt)}</span>
-      <span>duration ${report.durationMs}ms</span>
-      <span>artifacts ${htmlEscape(report.artifactsDir)}</span>
-    </div>
-    <table>
-      <thead>
-        <tr><th>Case</th><th>Title</th><th>Status</th><th>Duration</th><th>Phases</th><th>Run/Error</th></tr>
-      </thead>
-      <tbody>${rows}</tbody>
-    </table>
-  </body>
-</html>`
-}
-
-export async function runReplayEval(options: RunReplayEvalOptions = {}): Promise<ReplayEvalReport> {
-  const selectedCases = options.caseId
-    ? REPLAY_EVAL_CASES.filter((testCase) => {
-        return testCase.id === options.caseId
-      })
-    : REPLAY_EVAL_CASES
-  if (selectedCases.length === 0) {
-    throw new Error(`Replay eval case not found: ${options.caseId}`)
-  }
-  const start = Date.now()
-  const artifactsDir = createLocalTempDir('replay-eval-')
-  const server = await startExampleServer(selectedCases)
-  const browser = await launchEvalBrowser({ headed: options.headed })
-  try {
-    const results: ReplayEvalCaseResult[] = []
-    for (const testCase of selectedCases) {
-      results.push(
-        await runOneEvalCase({
-          testCase,
-          server,
-          browser,
-          artifactsDir,
-        }),
-      )
-    }
-    const passed = results.filter((result) => {
-      return result.status === 'passed'
-    }).length
-    const report: ReplayEvalReport = {
-      generatedAt: new Date().toISOString(),
-      durationMs: Date.now() - start,
-      passed,
-      failed: results.length - passed,
-      total: results.length,
-      results,
-      artifactsDir,
-    }
-    if (options.reportPath) {
-      ensureDir(path.dirname(options.reportPath))
-      fs.writeFileSync(options.reportPath, renderReportHtml(report))
-    }
-    if (!options.keepArtifacts) {
-      fs.rmSync(artifactsDir, { recursive: true, force: true })
-      return { ...report, artifactsDir: `${artifactsDir} (removed)` }
-    }
-    return report
-  } finally {
-    await browser.close().catch(() => {})
-    await server.close().catch(() => {})
-  }
-}
-
-export function formatReplayEvalReport(report: ReplayEvalReport): string {
-  const lines = [
-    `Replay eval: ${report.passed}/${report.total} passed in ${report.durationMs}ms`,
-    `Artifacts: ${report.artifactsDir}`,
-    '',
-    ...report.results.map((result) => {
-      const status = result.status === 'passed' ? pc.green('PASS') : pc.red('FAIL')
-      const detail = result.error || result.run?.status || ''
-      return `${status} ${result.id} (${result.durationMs}ms) ${detail}`
-    }),
-  ]
-  return lines.join('\n')
-}