tabwright 1.0.0

package/dist/replay-eval.js

@@ -0,0 +1,902 @@
+import fs from 'node:fs';
+import http from 'node:http';
+import path from 'node:path';
+import util from 'node:util';
+import { chromium } from '@xmorse/playwright-core';
+import pc from 'picocolors';
+import { runCapabilityWithExecutor } from './capability-runner.js';
+import { createReplayAiIndexFromRecording } from './replay-ai-index.js';
+import { compileReplayWorkflow } from './replay-workflow-compiler.js';
+import { getTabwrightUserDataDir } from './product-paths.js';
+const REPLAY_EVAL_CASES = [
+    {
+        id: 'zh-list-append',
+        title: 'Chinese admin list append',
+        description: '编辑 -> Add entry -> 填值 -> 提交 -> OK',
+        editLabel: '编辑',
+        addLabel: 'Add entry',
+        submitLabel: '提交',
+        okLabel: 'OK',
+        demonstratedValue: 'test2',
+        runValue: 'test3',
+        expectedStatus: 'completed',
+    },
+    {
+        id: 'en-list-append',
+        title: 'English list append',
+        description: 'Edit -> Add entry -> fill -> Submit -> OK',
+        editLabel: 'Edit',
+        addLabel: 'Add entry',
+        submitLabel: 'Submit',
+        okLabel: 'OK',
+        demonstratedValue: 'alpha',
+        runValue: 'beta',
+        expectedStatus: 'completed',
+    },
+    {
+        id: 'cn-add-confirm',
+        title: 'Chinese add and confirm labels',
+        description: '新增/确定 button labels should still compile and run.',
+        editLabel: '编辑',
+        addLabel: '新增',
+        submitLabel: '提交',
+        okLabel: '确定',
+        demonstratedValue: '中文值1',
+        runValue: '中文值2',
+        expectedStatus: 'completed',
+    },
+    {
+        id: 'already-editing',
+        title: 'Already editing page',
+        description: 'Generated script should continue from an already-editing page.',
+        editLabel: '编辑',
+        addLabel: 'Add entry',
+        submitLabel: '提交',
+        okLabel: 'OK',
+        demonstratedValue: 'inline-demo',
+        runValue: 'inline-run',
+        alreadyEditing: true,
+        expectedStatus: 'completed',
+    },
+    {
+        id: 'draft-restart',
+        title: 'Draft dialog restart',
+        description: 'Replay clicked 重新编辑, so generated script should restart the draft.',
+        editLabel: '编辑',
+        addLabel: 'Add entry',
+        submitLabel: '提交',
+        okLabel: 'OK',
+        demonstratedValue: 'restart-demo',
+        runValue: 'restart-run',
+        draftAction: 'restart',
+        expectedStatus: 'completed',
+    },
+    {
+        id: 'draft-continue',
+        title: 'Draft dialog continue',
+        description: 'Replay clicked 继续编辑, so generated script should continue the draft.',
+        editLabel: '编辑',
+        addLabel: 'Add entry',
+        submitLabel: '提交',
+        okLabel: 'OK',
+        demonstratedValue: 'continue-demo',
+        runValue: 'continue-run',
+        draftAction: 'continue',
+        expectedStatus: 'completed',
+    },
+    {
+        id: 'duplicate-short-circuit',
+        title: 'Duplicate value short-circuit',
+        description: 'If the target value is already visible, script should return completed without editing.',
+        editLabel: '编辑',
+        addLabel: 'Add entry',
+        submitLabel: '提交',
+        okLabel: 'OK',
+        demonstratedValue: 'existing-demo',
+        runValue: 'existing-demo',
+        initialValues: ['existing-demo'],
+        expectedStatus: 'completed',
+    },
+    {
+        id: 'drift-add-fails',
+        title: 'Page drift handoff',
+        description: 'The add button no longer creates an input, so script should return needs_ai.',
+        editLabel: '编辑',
+        addLabel: 'Add entry',
+        submitLabel: '提交',
+        okLabel: 'OK',
+        demonstratedValue: 'drift-demo',
+        runValue: 'drift-run',
+        brokenAdd: true,
+        expectedStatus: 'needs_ai',
+        expectedReasonIncludes: 'Add entry did not create a new list input',
+    },
+    {
+        id: 'annotation-delete-index',
+        title: 'Deleted annotation index',
+        description: 'A deleted annotation should not reach AI index output.',
+        editLabel: '编辑',
+        addLabel: 'Add entry',
+        submitLabel: '提交',
+        okLabel: 'OK',
+        demonstratedValue: 'annotation-demo',
+        runValue: 'annotation-run',
+        addAnnotation: true,
+        deleteAnnotation: true,
+        expectedStatus: 'completed',
+        expectedAnnotationCount: 0,
+    },
+    {
+        id: 'unsupported-click-only',
+        title: 'Unsupported click-only replay',
+        description: 'Compiler should fail clearly instead of generating a fake workflow.',
+        editLabel: '编辑',
+        addLabel: 'Add entry',
+        submitLabel: '提交',
+        okLabel: 'OK',
+        demonstratedValue: '',
+        runValue: '',
+        unsupported: true,
+        expectedStatus: 'compile_failed',
+    },
+];
+function ensureDir(dir) {
+    fs.mkdirSync(dir, { recursive: true });
+}
+function createLocalTempDir(prefix) {
+    const root = path.join(process.cwd(), 'tmp');
+    ensureDir(root);
+    return fs.mkdtempSync(path.join(root, prefix));
+}
+function writeJson(filePath, value) {
+    ensureDir(path.dirname(filePath));
+    fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`);
+}
+function normalizeError(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function textNode(id, textContent) {
+    return { id, type: 3, textContent };
+}
+function elementNode(options) {
+    return {
+        id: options.id,
+        type: 2,
+        tagName: options.tagName,
+        attributes: options.attributes || {},
+        childNodes: options.childNodes || [],
+    };
+}
+function clickEvent(options) {
+    return {
+        type: 3,
+        timestamp: options.timestamp,
+        data: {
+            source: 2,
+            type: 2,
+            id: options.id,
+            x: 10,
+            y: 10,
+            pointerType: 0,
+        },
+    };
+}
+function inputEvent(options) {
+    return {
+        type: 3,
+        timestamp: options.timestamp,
+        data: {
+            source: 5,
+            id: options.id,
+            text: options.text,
+            isChecked: false,
+        },
+    };
+}
+function annotationEvent(options) {
+    return {
+        type: 5,
+        timestamp: options.timestamp,
+        data: {
+            tag: 'playwriter.annotation',
+            payload: {
+                schemaVersion: 1,
+                id: options.id,
+                text: options.text,
+                timestamp: options.timestamp,
+                target: {
+                    tagName: 'textarea',
+                    label: 'List value',
+                    selectorHints: ['textarea.designer-input'],
+                    rect: { x: 10, y: 20, width: 240, height: 32 },
+                },
+            },
+        },
+    };
+}
+function annotationDeleteEvent(options) {
+    return {
+        type: 5,
+        timestamp: options.timestamp,
+        data: {
+            tag: 'playwriter.annotation.delete',
+            payload: {
+                schemaVersion: 1,
+                id: options.id,
+                timestamp: options.timestamp,
+            },
+        },
+    };
+}
+function createReplayEvents(testCase) {
+    const nodes = [
+        elementNode({ id: 10, tagName: 'button', childNodes: [textNode(11, testCase.editLabel)] }),
+        elementNode({
+            id: 12,
+            tagName: 'button',
+            attributes: { class: 'designer-btn designer-btn-dangerous' },
+            childNodes: [textNode(13, testCase.draftAction === 'restart' ? '重新编辑' : 'Restart')],
+        }),
+        elementNode({
+            id: 14,
+            tagName: 'button',
+            attributes: { class: 'designer-btn designer-btn-primary' },
+            childNodes: [textNode(15, testCase.draftAction === 'continue' ? '继续编辑' : 'Continue')],
+        }),
+        elementNode({
+            id: 20,
+            tagName: 'button',
+            attributes: { class: 'designer-formily-array-base-addition' },
+            childNodes: [textNode(21, testCase.addLabel)],
+        }),
+        elementNode({ id: 30, tagName: 'textarea', attributes: { class: 'designer-input' } }),
+        elementNode({ id: 40, tagName: 'button', childNodes: [textNode(41, testCase.submitLabel)] }),
+        elementNode({
+            id: 50,
+            tagName: 'button',
+            attributes: { class: 'designer-btn designer-btn-primary' },
+            childNodes: [textNode(51, testCase.okLabel)],
+        }),
+    ];
+    const events = [
+        {
+            type: 2,
+            timestamp: 1000,
+            data: {
+                node: {
+                    id: 1,
+                    type: 0,
+                    childNodes: [
+                        {
+                            id: 2,
+                            type: 1,
+                            childNodes: [
+                                elementNode({
+                                    id: 3,
+                                    tagName: 'body',
+                                    childNodes: nodes,
+                                }),
+                            ],
+                        },
+                    ],
+                },
+            },
+        },
+    ];
+    if (testCase.unsupported) {
+        events.push(clickEvent({ timestamp: 1100, id: 10 }));
+        return events;
+    }
+    if (!testCase.alreadyEditing) {
+        events.push(clickEvent({ timestamp: 1100, id: 10 }));
+    }
+    if (testCase.draftAction === 'restart') {
+        events.push(clickEvent({ timestamp: 1200, id: 12 }));
+    }
+    if (testCase.draftAction === 'continue') {
+        events.push(clickEvent({ timestamp: 1200, id: 14 }));
+    }
+    events.push(clickEvent({ timestamp: 1300, id: 20 }));
+    events.push(inputEvent({ timestamp: 1400, id: 30, text: testCase.demonstratedValue }));
+    if (testCase.addAnnotation) {
+        events.push(annotationEvent({ timestamp: 1450, id: `${testCase.id}-annotation`, text: '重点填写这个列表项' }));
+    }
+    if (testCase.deleteAnnotation) {
+        events.push(annotationDeleteEvent({ timestamp: 1460, id: `${testCase.id}-annotation` }));
+    }
+    events.push(clickEvent({ timestamp: 1500, id: 40 }));
+    events.push(clickEvent({ timestamp: 1600, id: 50 }));
+    return events;
+}
+function htmlEscape(value) {
+    return value
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&#39;');
+}
+function buildExampleHtml(testCase, state) {
+    const config = {
+        id: testCase.id,
+        labels: {
+            edit: testCase.editLabel,
+            add: testCase.addLabel,
+            submit: testCase.submitLabel,
+            ok: testCase.okLabel,
+            restart: testCase.draftAction === 'restart' ? '重新编辑' : 'Restart',
+            continue: testCase.draftAction === 'continue' ? '继续编辑' : 'Continue',
+        },
+        initialValues: state.values,
+        alreadyEditing: Boolean(testCase.alreadyEditing),
+        draftAction: testCase.draftAction || null,
+        brokenAdd: Boolean(testCase.brokenAdd),
+    };
+    return `<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>${htmlEscape(testCase.title)}</title>
+    <style>
+      body { font-family: system-ui, sans-serif; padding: 24px; color: #172033; }
+      button { margin: 6px; padding: 7px 12px; border: 1px solid #cbd5e1; border-radius: 6px; background: white; cursor: pointer; }
+      textarea { display: block; width: 320px; min-height: 36px; margin: 8px 0; border: 1px solid #cbd5e1; border-radius: 6px; padding: 7px; }
+      .designer-modal { position: fixed; inset: auto 24px 24px auto; width: 360px; padding: 16px; border: 1px solid #cbd5e1; border-radius: 8px; background: white; box-shadow: 0 18px 48px rgba(15,23,42,.18); }
+      .readonly-value { padding: 4px 0; }
+    </style>
+  </head>
+  <body>
+    <h1>${htmlEscape(testCase.title)}</h1>
+    <div id="app"></div>
+    <script>
+      const config = ${JSON.stringify(config)};
+      const app = document.getElementById('app');
+      const state = {
+        values: [...config.initialValues],
+        editing: config.alreadyEditing,
+        draftShown: false,
+      };
+      function renderReadOnly() {
+        app.innerHTML = '<button id="edit">' + config.labels.edit + '</button><div id="values"></div>';
+        const values = document.getElementById('values');
+        state.values.forEach((value) => {
+          const item = document.createElement('div');
+          item.className = 'readonly-value';
+          item.textContent = value;
+          values.appendChild(item);
+        });
+        document.getElementById('edit').addEventListener('click', () => {
+          if (config.draftAction && !state.draftShown) {
+            state.draftShown = true;
+            renderDraftDialog();
+            return;
+          }
+          state.editing = true;
+          render();
+        });
+      }
+      function renderDraftDialog() {
+        renderReadOnly();
+        const modal = document.createElement('div');
+        modal.className = 'designer-modal';
+        modal.innerHTML = '<p>继续编辑上次修改 / Continue previous edit draft</p><button class="designer-btn designer-btn-dangerous" id="restart">' + config.labels.restart + '</button><button class="designer-btn designer-btn-primary" id="continue">' + config.labels.continue + '</button>';
+        document.body.appendChild(modal);
+        document.getElementById('restart').addEventListener('click', () => {
+          modal.remove();
+          state.values = [];
+          state.editing = true;
+          render();
+        });
+        document.getElementById('continue').addEventListener('click', () => {
+          modal.remove();
+          state.editing = true;
+          render();
+        });
+      }
+      function renderEditor() {
+        app.innerHTML = '<div id="editor"></div><button class="designer-formily-array-base-addition" id="add">' + config.labels.add + '</button><button id="submit">' + config.labels.submit + '</button>';
+        const editor = document.getElementById('editor');
+        state.values.forEach((value, index) => {
+          const input = document.createElement('textarea');
+          input.className = 'designer-input';
+          input.value = value;
+          input.addEventListener('input', () => {
+            state.values[index] = input.value;
+          });
+          editor.appendChild(input);
+        });
+        document.getElementById('add').addEventListener('click', () => {
+          if (!config.brokenAdd) {
+            state.values.push('');
+          }
+          render();
+        });
+        document.getElementById('submit').addEventListener('click', () => {
+          renderConfirm();
+        });
+      }
+      function renderConfirm() {
+        const modal = document.createElement('div');
+        modal.className = 'designer-modal';
+        modal.innerHTML = '<p>Confirm values: ' + state.values.join(', ') + '</p><button class="designer-btn designer-btn-primary" id="ok">' + config.labels.ok + '</button>';
+        document.body.appendChild(modal);
+        document.getElementById('ok').addEventListener('click', async () => {
+          await fetch('/enhancedConfigs/update', {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ caseId: config.id, values: state.values }),
+          });
+          await fetch('/enhancedConfigs/publish', { method: 'POST' });
+          await fetch('/enhancedConfigs/detail?id=' + encodeURIComponent(config.id));
+          modal.remove();
+          state.editing = false;
+          render();
+        });
+      }
+      function render() {
+        if (state.editing) {
+          renderEditor();
+          return;
+        }
+        renderReadOnly();
+      }
+      render();
+    </script>
+  </body>
+</html>`;
+}
+function requestBody(request) {
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        request.on('data', (chunk) => {
+            chunks.push(chunk);
+        });
+        request.on('end', () => {
+            resolve(Buffer.concat(chunks).toString('utf-8'));
+        });
+        request.on('error', (error) => {
+            reject(error);
+        });
+    });
+}
+async function startExampleServer(cases) {
+    const casesById = new Map(cases.map((testCase) => {
+        return [testCase.id, testCase];
+    }));
+    const states = new Map(cases.map((testCase) => {
+        const state = { values: [...(testCase.initialValues || [])] };
+        return [testCase.id, state];
+    }));
+    const server = http.createServer((request, response) => {
+        const host = request.headers.host || 'localhost';
+        const url = new URL(request.url || '/', `http://${host}`);
+        const sendJson = (statusCode, value) => {
+            response.writeHead(statusCode, { 'content-type': 'application/json' });
+            response.end(JSON.stringify(value));
+        };
+        if (request.method === 'GET' && url.pathname.startsWith('/case/')) {
+            const caseId = decodeURIComponent(url.pathname.slice('/case/'.length));
+            const testCase = casesById.get(caseId);
+            const state = states.get(caseId);
+            if (!testCase || !state) {
+                response.writeHead(404);
+                response.end('Not found');
+                return;
+            }
+            response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
+            response.end(buildExampleHtml(testCase, state));
+            return;
+        }
+        if (request.method === 'POST' && url.pathname === '/enhancedConfigs/update') {
+            requestBody(request)
+                .then((body) => {
+                const parsed = JSON.parse(body || '{}');
+                const caseId = typeof parsed.caseId === 'string' ? parsed.caseId : '';
+                const state = states.get(caseId);
+                if (state && Array.isArray(parsed.values)) {
+                    state.values = parsed.values.map((value) => {
+                        return String(value);
+                    });
+                }
+                sendJson(200, { code: 0, data: true });
+            })
+                .catch((error) => {
+                sendJson(500, { code: 1, message: normalizeError(error) });
+            });
+            return;
+        }
+        if (request.method === 'POST' && url.pathname === '/enhancedConfigs/publish') {
+            sendJson(200, { code: 0, data: true });
+            return;
+        }
+        if (request.method === 'GET' && url.pathname === '/enhancedConfigs/detail') {
+            const caseId = url.searchParams.get('id') || '';
+            const state = states.get(caseId);
+            sendJson(200, {
+                code: 0,
+                data: {
+                    config: {
+                        value: JSON.stringify(state?.values || []),
+                    },
+                },
+            });
+            return;
+        }
+        response.writeHead(404);
+        response.end('Not found');
+    });
+    await new Promise((resolve, reject) => {
+        server.on('error', reject);
+        server.listen(0, '127.0.0.1', () => {
+            resolve();
+        });
+    });
+    const address = server.address();
+    if (!address || typeof address === 'string') {
+        throw new Error('Replay eval server did not expose a TCP port');
+    }
+    return {
+        baseUrl: `http://127.0.0.1:${address.port}`,
+        close: async () => {
+            await new Promise((resolve, reject) => {
+                server.close((error) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve();
+                });
+            });
+        },
+    };
+}
+function getChromeExecutable() {
+    const candidates = [
+        process.env.TABWRIGHT_TEST_CHROME || process.env.PLAYWRITER_TEST_CHROME,
+        '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+        '/Applications/Chromium.app/Contents/MacOS/Chromium',
+        '/usr/bin/google-chrome',
+        '/usr/bin/google-chrome-stable',
+        '/usr/bin/chromium',
+        '/usr/bin/chromium-browser',
+        'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
+        'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
+    ].filter((candidate) => {
+        return Boolean(candidate);
+    });
+    return candidates.find((candidate) => {
+        return fs.existsSync(candidate);
+    });
+}
+async function launchEvalBrowser(options) {
+    const executablePath = getChromeExecutable();
+    return await chromium.launch({
+        ...(executablePath ? { executablePath } : { channel: 'chromium' }),
+        headless: !options.headed,
+    });
+}
+function formatOutput(value) {
+    return `[return value] ${util.inspect(value, {
+        depth: 5,
+        colors: false,
+        maxArrayLength: 80,
+        maxStringLength: 2000,
+        breakLength: 100,
+    })}`;
+}
+class PageEvalExecutor {
+    page;
+    constructor(page) {
+        this.page = page;
+    }
+    async execute(code, timeout = 15000) {
+        const AsyncFunction = Object.getPrototypeOf(async function () { }).constructor;
+        const runner = new AsyncFunction('page', 'snapshot', 'console', 'setTimeout', 'clearTimeout', 'URL', 'URLSearchParams', 'fetch', 'Buffer', code);
+        const snapshot = async () => {
+            return await this.page.locator('body').innerText({ timeout: 2000 }).catch((error) => {
+                return `snapshot failed: ${normalizeError(error)}`;
+            });
+        };
+        try {
+            const structuredResult = await Promise.race([
+                runner(this.page, snapshot, console, setTimeout, clearTimeout, URL, URLSearchParams, fetch, Buffer),
+                new Promise((_, reject) => {
+                    setTimeout(() => {
+                        reject(new Error(`Replay eval execution timed out after ${timeout}ms`));
+                    }, timeout);
+                }),
+            ]);
+            return {
+                text: formatOutput(structuredResult),
+                images: [],
+                screenshots: [],
+                isError: false,
+                structuredResult: JSON.parse(JSON.stringify(structuredResult)),
+            };
+        }
+        catch (error) {
+            return {
+                text: normalizeError(error),
+                images: [],
+                screenshots: [],
+                isError: true,
+            };
+        }
+    }
+}
+function writeRecording(options) {
+    const recordingsDir = path.join(getTabwrightUserDataDir({ homeDir: options.home }), 'rrweb-recordings');
+    const recordingPath = path.join(recordingsDir, `${options.replayId}.json`);
+    writeJson(recordingPath, options.events);
+    writeJson(path.join(recordingsDir, 'index.json'), [
+        {
+            id: options.replayId,
+            path: recordingPath,
+            startedAt: 1000,
+            savedAt: 2000,
+            duration: 1000,
+            size: fs.statSync(recordingPath).size,
+            eventCount: options.events.length,
+            tabId: 1,
+            url: options.url,
+        },
+    ]);
+}
+function getOutputStatus(output) {
+    if (!output || typeof output !== 'object') {
+        return undefined;
+    }
+    const candidate = output;
+    return typeof candidate.status === 'string' ? candidate.status : undefined;
+}
+function outputField(output, key) {
+    if (!output || typeof output !== 'object') {
+        return undefined;
+    }
+    const value = output[key];
+    return typeof value === 'string' ? value : undefined;
+}
+function outputNumberField(output, key) {
+    if (!output || typeof output !== 'object') {
+        return undefined;
+    }
+    const value = output[key];
+    return typeof value === 'number' ? value : undefined;
+}
+function assertCaseResult(options) {
+    if (options.testCase.expectedAnnotationCount !== undefined && options.annotationCount !== options.testCase.expectedAnnotationCount) {
+        throw new Error(`Expected ${options.testCase.expectedAnnotationCount} annotations, got ${options.annotationCount}`);
+    }
+    const status = getOutputStatus(options.output);
+    if (status !== options.testCase.expectedStatus) {
+        throw new Error(`Expected run status ${options.testCase.expectedStatus}, got ${status || 'undefined'}`);
+    }
+    if (options.testCase.expectedReasonIncludes) {
+        const reason = outputField(options.output, 'reason') || '';
+        const needsAi = options.output && typeof options.output === 'object' ? options.output.needsAi : undefined;
+        const nestedReason = outputField(needsAi, 'reason') || '';
+        if (!reason.includes(options.testCase.expectedReasonIncludes) && !nestedReason.includes(options.testCase.expectedReasonIncludes)) {
+            throw new Error(`Expected reason to include "${options.testCase.expectedReasonIncludes}", got "${reason || nestedReason}"`);
+        }
+    }
+}
+async function runOneEvalCase(options) {
+    const start = Date.now();
+    const phases = [];
+    const replayId = `eval-${options.testCase.id}`;
+    const home = path.join(options.artifactsDir, replayId, 'home');
+    const cwd = path.join(options.artifactsDir, replayId, 'cwd');
+    ensureDir(home);
+    ensureDir(cwd);
+    const previousHome = process.env.HOME;
+    const url = `${options.server.baseUrl}/case/${encodeURIComponent(options.testCase.id)}?key=${encodeURIComponent(options.testCase.id)}`;
+    try {
+        process.env.HOME = home;
+        const events = createReplayEvents(options.testCase);
+        writeRecording({ home, replayId, url, events });
+        phases.push('recording');
+        const index = createReplayAiIndexFromRecording(replayId);
+        phases.push('index');
+        if (options.testCase.expectedStatus === 'compile_failed') {
+            try {
+                compileReplayWorkflow({ replayId, id: `${options.testCase.id}-capability`, cwd, overwrite: true });
+            }
+            catch (error) {
+                phases.push('compile');
+                return {
+                    id: options.testCase.id,
+                    title: options.testCase.title,
+                    status: 'passed',
+                    durationMs: Date.now() - start,
+                    replayId,
+                    phases,
+                    index: {
+                        actionCount: index.actions.length,
+                        fieldCount: index.fields.length,
+                        annotationCount: index.annotations.length,
+                    },
+                    error: normalizeError(error),
+                };
+            }
+            throw new Error('Expected replay compiler to fail, but it produced a capability');
+        }
+        const compiled = compileReplayWorkflow({ replayId, id: `${options.testCase.id}-capability`, cwd, overwrite: true });
+        phases.push('compile');
+        const page = await options.browser.newPage();
+        try {
+            await page.goto(url);
+            const executor = new PageEvalExecutor(page);
+            const runResult = await runCapabilityWithExecutor({
+                executor,
+                id: `${options.testCase.id}-capability`,
+                input: { value: options.testCase.runValue },
+                cwd,
+                force: true,
+                confirmation: `${options.testCase.id}-capability`,
+                timeout: 20000,
+            });
+            phases.push('run');
+            assertCaseResult({
+                testCase: options.testCase,
+                output: runResult.output,
+                annotationCount: index.annotations.length,
+            });
+            phases.push('verify');
+            return {
+                id: options.testCase.id,
+                title: options.testCase.title,
+                status: 'passed',
+                durationMs: Date.now() - start,
+                replayId,
+                phases,
+                index: {
+                    actionCount: index.actions.length,
+                    fieldCount: index.fields.length,
+                    annotationCount: index.annotations.length,
+                },
+                compile: {
+                    actionKind: compiled.analysis.actionKind,
+                    confidence: compiled.analysis.confidence,
+                    demonstratedValue: compiled.analysis.demonstratedValue,
+                },
+                run: {
+                    status: getOutputStatus(runResult.output),
+                    value: outputField(runResult.output, 'value'),
+                    reason: outputField(runResult.output, 'reason'),
+                    completed: outputNumberField(runResult.output, 'completed'),
+                    total: outputNumberField(runResult.output, 'total'),
+                },
+            };
+        }
+        finally {
+            await page.close().catch(() => { });
+        }
+    }
+    catch (error) {
+        return {
+            id: options.testCase.id,
+            title: options.testCase.title,
+            status: 'failed',
+            durationMs: Date.now() - start,
+            replayId,
+            phases,
+            error: normalizeError(error),
+        };
+    }
+    finally {
+        if (previousHome === undefined) {
+            delete process.env.HOME;
+        }
+        else {
+            process.env.HOME = previousHome;
+        }
+    }
+}
+function renderReportHtml(report) {
+    const rows = report.results
+        .map((result) => {
+        const statusColor = result.status === 'passed' ? '#15803d' : '#b91c1c';
+        return `<tr>
+        <td>${htmlEscape(result.id)}</td>
+        <td>${htmlEscape(result.title)}</td>
+        <td style="color:${statusColor};font-weight:700">${htmlEscape(result.status)}</td>
+        <td>${result.durationMs}ms</td>
+        <td>${htmlEscape(result.phases.join(' -> '))}</td>
+        <td>${htmlEscape(result.run?.status || result.error || '-')}</td>
+      </tr>`;
+    })
+        .join('\n');
+    return `<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Tabwright Replay Eval</title>
+    <style>
+      body { font-family: ui-sans-serif, system-ui, sans-serif; margin: 32px; color: #172033; }
+      table { border-collapse: collapse; width: 100%; }
+      th, td { border-bottom: 1px solid #e2e8f0; padding: 8px; text-align: left; vertical-align: top; }
+      th { background: #f8fafc; font-size: 12px; text-transform: uppercase; color: #64748b; }
+      .summary { margin-bottom: 18px; padding: 14px; border: 1px solid #e2e8f0; border-radius: 8px; background: #f8fafc; }
+    </style>
+  </head>
+  <body>
+    <h1>Tabwright Replay Eval</h1>
+    <div class="summary">
+      <strong>${report.passed}/${report.total} passed</strong>
+      <span>generated ${htmlEscape(report.generatedAt)}</span>
+      <span>duration ${report.durationMs}ms</span>
+      <span>artifacts ${htmlEscape(report.artifactsDir)}</span>
+    </div>
+    <table>
+      <thead>
+        <tr><th>Case</th><th>Title</th><th>Status</th><th>Duration</th><th>Phases</th><th>Run/Error</th></tr>
+      </thead>
+      <tbody>${rows}</tbody>
+    </table>
+  </body>
+</html>`;
+}
+export async function runReplayEval(options = {}) {
+    const selectedCases = options.caseId
+        ? REPLAY_EVAL_CASES.filter((testCase) => {
+            return testCase.id === options.caseId;
+        })
+        : REPLAY_EVAL_CASES;
+    if (selectedCases.length === 0) {
+        throw new Error(`Replay eval case not found: ${options.caseId}`);
+    }
+    const start = Date.now();
+    const artifactsDir = createLocalTempDir('replay-eval-');
+    const server = await startExampleServer(selectedCases);
+    const browser = await launchEvalBrowser({ headed: options.headed });
+    try {
+        const results = [];
+        for (const testCase of selectedCases) {
+            results.push(await runOneEvalCase({
+                testCase,
+                server,
+                browser,
+                artifactsDir,
+            }));
+        }
+        const passed = results.filter((result) => {
+            return result.status === 'passed';
+        }).length;
+        const report = {
+            generatedAt: new Date().toISOString(),
+            durationMs: Date.now() - start,
+            passed,
+            failed: results.length - passed,
+            total: results.length,
+            results,
+            artifactsDir,
+        };
+        if (options.reportPath) {
+            ensureDir(path.dirname(options.reportPath));
+            fs.writeFileSync(options.reportPath, renderReportHtml(report));
+        }
+        if (!options.keepArtifacts) {
+            fs.rmSync(artifactsDir, { recursive: true, force: true });
+            return { ...report, artifactsDir: `${artifactsDir} (removed)` };
+        }
+        return report;
+    }
+    finally {
+        await browser.close().catch(() => { });
+        await server.close().catch(() => { });
+    }
+}
+export function formatReplayEvalReport(report) {
+    const lines = [
+        `Replay eval: ${report.passed}/${report.total} passed in ${report.durationMs}ms`,
+        `Artifacts: ${report.artifactsDir}`,
+        '',
+        ...report.results.map((result) => {
+            const status = result.status === 'passed' ? pc.green('PASS') : pc.red('FAIL');
+            const detail = result.error || result.run?.status || '';
+            return `${status} ${result.id} (${result.durationMs}ms) ${detail}`;
+        }),
+    ];
+    return lines.join('\n');
+}
+//# sourceMappingURL=replay-eval.js.map