playwright-intelligence-engine 0.0.0-stage → 2.0.0

package/src/evaluation/evaluation-scorecard.ts

@@ -0,0 +1,198 @@
+import { normalizeFailureCategory } from '../diagnostics/failure-category.js';
+
+export interface EvaluationObservation {
+  id?: string | undefined;
+  repository?: string | undefined;
+  expectedCategory?: string | undefined;
+  rawPredictedCategory?: string | undefined;
+  predictedCategory?: string | undefined;
+  pieNormalizedCategory?: string | undefined;
+  isGroundTruthVerified?: boolean | undefined;
+  isPrimaryMutationTrace?: boolean | undefined;
+  testType?: string | undefined;
+  shouldAbstain?: boolean | undefined;
+  actionOutcome?: string | undefined;
+  isDeterministic?: boolean | undefined;
+  durationMs?: number | undefined;
+  meanDurationMs?: number | undefined;
+  evaluationRole?: 'primary' | 'negative-control' | 'baseline' | undefined;
+  repairExpected?: boolean | undefined;
+  repairSucceeded?: boolean | undefined;
+  falseRepair?: boolean | undefined;
+}
+
+export interface CategoryMetrics {
+  support: number;
+  truePositives: number;
+  falsePositives: number;
+  falseNegatives: number;
+  precision: number;
+  recall: number;
+  f1Score: number;
+}
+
+export interface EvaluationScorecard {
+  observationCount: number;
+  accuracyCaseCount: number;
+  exactMatches: number;
+  semanticAccuracy: number | null;
+  exactAccuracy: number | null;
+  confusionMatrix: Record<string, Record<string, number>>;
+  perCategory: Record<string, CategoryMetrics>;
+  negativeControlCount: number;
+  correctSafeAbstentions: number;
+  unsafeRepairRecommendations: number;
+  safeAbstentionRate: number | null;
+  falseRepairRate: number | null;
+  deterministicCount: number;
+  determinismRate: number | null;
+  latency: {
+    count: number;
+    minMs?: number | undefined;
+    meanMs?: number | undefined;
+    medianMs?: number | undefined;
+    p95Ms?: number | undefined;
+    maxMs?: number | undefined;
+  };
+  repair: {
+    eligible: number;
+    succeeded: number;
+    precision: number | null;
+    recall: number | null;
+    falseRepairs: number;
+  };
+  byRepository: Record<string, {
+    observations: number;
+    accuracyCases: number;
+    exactMatches: number;
+    semanticMatches: number;
+    semanticAccuracy: number | null;
+    exactAccuracy: number | null;
+    safeAbstentionRate: number | null;
+    meanLatencyMs?: number;
+  }>;
+}
+
+function round(value: number): number {
+  return Number(value.toFixed(3));
+}
+
+function percentile(values: number[], percentileValue: number): number | undefined {
+  if (values.length === 0) return undefined;
+  const sorted = [...values].sort((left, right) => left - right);
+  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * percentileValue) - 1)];
+}
+
+function roleOf(observation: EvaluationObservation): 'primary' | 'negative-control' | 'baseline' | undefined {
+  if (observation.evaluationRole) return observation.evaluationRole;
+  if (observation.isPrimaryMutationTrace || observation.testType === 'BASELINE_FAILURE') return 'primary';
+  if (observation.testType === 'BASELINE_PASS') return 'baseline';
+  const expected = normalizeFailureCategory(observation.expectedCategory);
+  if (['BACKEND_500', 'AUTH_EXPIRED', 'ENVIRONMENT_FAILURE', 'DISABLED_ELEMENT', 'OBSTRUCTED_ELEMENT', 'UNSUPPORTED_TRACE_FORMAT'].includes(expected)) {
+    return 'negative-control';
+  }
+  return undefined;
+}
+
+function categoryOf(value: string | undefined): string {
+  return normalizeFailureCategory(value);
+}
+
+function metricFor(category: string, expected: string[], predicted: string[]): CategoryMetrics {
+  const support = expected.filter((value) => value === category).length;
+  const truePositives = expected.filter((value, index) => value === category && predicted[index] === category).length;
+  const falsePositives = expected.filter((value, index) => value !== category && predicted[index] === category).length;
+  const falseNegatives = expected.filter((value, index) => value === category && predicted[index] !== category).length;
+  const precision = truePositives + falsePositives > 0 ? truePositives / (truePositives + falsePositives) : 0;
+  const recall = truePositives + falseNegatives > 0 ? truePositives / (truePositives + falseNegatives) : 0;
+  return {
+    support,
+    truePositives,
+    falsePositives,
+    falseNegatives,
+    precision: round(precision),
+    recall: round(recall),
+    f1Score: round(precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0),
+  };
+}
+
+export function buildEvaluationScorecard(observations: EvaluationObservation[]): EvaluationScorecard {
+  const accuracyObservations = observations.filter((observation) => observation.isGroundTruthVerified !== false && roleOf(observation) === 'primary');
+  const expected = accuracyObservations.map((observation) => categoryOf(observation.expectedCategory));
+  const predicted = accuracyObservations.map((observation) => categoryOf(observation.predictedCategory ?? observation.pieNormalizedCategory));
+  const rawExpected = accuracyObservations.map((observation) => (observation.expectedCategory ?? '').trim().toUpperCase());
+  const rawPredicted = accuracyObservations.map((observation) => (observation.rawPredictedCategory ?? observation.predictedCategory ?? observation.pieNormalizedCategory ?? '').trim().toUpperCase());
+  const categories = [...new Set([...expected, ...predicted])].sort();
+  const confusionMatrix: Record<string, Record<string, number>> = {};
+  for (const expectedCategory of categories) {
+    confusionMatrix[expectedCategory] = {};
+    for (const predictedCategory of categories) confusionMatrix[expectedCategory]![predictedCategory] = 0;
+  }
+  for (let index = 0; index < expected.length; index += 1) {
+    const row = confusionMatrix[expected[index]!];
+    if (row) row[predicted[index]!] = (row[predicted[index]!] ?? 0) + 1;
+  }
+
+  const negativeControls = observations.filter((observation) => roleOf(observation) === 'negative-control' || roleOf(observation) === 'baseline');
+  const correctSafeAbstentions = negativeControls.filter((observation) => observation.shouldAbstain === true && observation.actionOutcome !== 'REPAIR').length;
+  const unsafeRepairRecommendations = negativeControls.filter((observation) => observation.shouldAbstain === false || observation.actionOutcome === 'REPAIR' || observation.falseRepair === true).length;
+  const latencies = observations.map((observation) => observation.meanDurationMs ?? observation.durationMs).filter((value): value is number => typeof value === 'number' && Number.isFinite(value));
+  const deterministicCount = observations.filter((observation) => observation.isDeterministic !== false).length;
+  const repairObservations = observations.filter((observation) => observation.repairExpected === true || observation.repairSucceeded === true || observation.falseRepair === true);
+  const eligibleRepairs = repairObservations.filter((observation) => observation.repairExpected === true).length;
+  const successfulRepairs = repairObservations.filter((observation) => observation.repairExpected === true && observation.repairSucceeded === true).length;
+  const falseRepairs = repairObservations.filter((observation) => observation.falseRepair === true).length;
+
+  const byRepository: EvaluationScorecard['byRepository'] = {};
+  for (const observation of observations) {
+    const repository = observation.repository ?? 'unknown';
+    const current = byRepository[repository] ?? { observations: 0, accuracyCases: 0, exactMatches: 0, semanticMatches: 0, semanticAccuracy: null, exactAccuracy: null, safeAbstentionRate: null };
+    current.observations += 1;
+    if (roleOf(observation) === 'primary' && observation.isGroundTruthVerified !== false) {
+      current.accuracyCases += 1;
+      if ((observation.expectedCategory ?? '').trim().toUpperCase() === (observation.rawPredictedCategory ?? observation.predictedCategory ?? observation.pieNormalizedCategory ?? '').trim().toUpperCase()) current.exactMatches += 1;
+      if (categoryOf(observation.expectedCategory) === categoryOf(observation.predictedCategory ?? observation.pieNormalizedCategory)) current.semanticMatches += 1;
+    }
+    const latency = observation.meanDurationMs ?? observation.durationMs;
+    if (typeof latency === 'number') current.meanLatencyMs = ((current.meanLatencyMs ?? 0) * (current.observations - 1) + latency) / current.observations;
+    byRepository[repository] = current;
+  }
+  for (const summary of Object.values(byRepository)) {
+    summary.semanticAccuracy = summary.accuracyCases > 0 ? round(summary.semanticMatches / summary.accuracyCases) : null;
+    summary.exactAccuracy = summary.accuracyCases > 0 ? round(summary.exactMatches / summary.accuracyCases) : null;
+    if (summary.meanLatencyMs !== undefined) summary.meanLatencyMs = round(summary.meanLatencyMs);
+  }
+
+  return {
+    observationCount: observations.length,
+    accuracyCaseCount: accuracyObservations.length,
+    exactMatches: rawExpected.filter((value, index) => value === rawPredicted[index]).length,
+    semanticAccuracy: expected.length > 0 ? round(expected.filter((value, index) => value === predicted[index]).length / expected.length) : null,
+    exactAccuracy: rawExpected.length > 0 ? round(rawExpected.filter((value, index) => value === rawPredicted[index]).length / rawExpected.length) : null,
+    confusionMatrix,
+    perCategory: Object.fromEntries(categories.map((category) => [category, metricFor(category, expected, predicted)])),
+    negativeControlCount: negativeControls.length,
+    correctSafeAbstentions,
+    unsafeRepairRecommendations,
+    safeAbstentionRate: negativeControls.length > 0 ? round(correctSafeAbstentions / negativeControls.length) : null,
+    falseRepairRate: negativeControls.length > 0 ? round(unsafeRepairRecommendations / negativeControls.length) : null,
+    deterministicCount,
+    determinismRate: observations.length > 0 ? round(deterministicCount / observations.length) : null,
+    latency: {
+      count: latencies.length,
+      minMs: latencies.length > 0 ? Math.min(...latencies) : undefined,
+      meanMs: latencies.length > 0 ? round(latencies.reduce((sum, value) => sum + value, 0) / latencies.length) : undefined,
+      medianMs: percentile(latencies, 0.5),
+      p95Ms: percentile(latencies, 0.95),
+      maxMs: latencies.length > 0 ? Math.max(...latencies) : undefined,
+    },
+    repair: {
+      eligible: eligibleRepairs,
+      succeeded: successfulRepairs,
+      precision: successfulRepairs + falseRepairs > 0 ? round(successfulRepairs / (successfulRepairs + falseRepairs)) : null,
+      recall: eligibleRepairs > 0 ? round(successfulRepairs / eligibleRepairs) : null,
+      falseRepairs,
+    },
+    byRepository,
+  };
+}

package/src/evaluation/repository-inspector.ts

@@ -0,0 +1,143 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+export interface RepositoryManifest {
+  name: string;
+  root: string;
+  packageManager: 'npm' | 'pnpm' | 'yarn' | 'unknown';
+  license: string;
+  nodeVersion?: string | undefined;
+  playwrightVersion?: string | undefined;
+  hasPlaywrightConfig: boolean;
+  playwrightConfigs: string[];
+  testFiles: number;
+  pageObjectFiles: number;
+  fixtureFiles: number;
+  hasDocker: boolean;
+  hasLocalWebServerHint: boolean;
+  scripts: Record<string, string>;
+  status: 'DISCOVERED' | 'NOT_A_PLAYWRIGHT_REPOSITORY' | 'UNREADABLE';
+  limitations: string[];
+}
+
+async function exists(filePath: string): Promise<boolean> {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function listFiles(root: string, depth = 0): Promise<string[]> {
+  if (depth > 8) return [];
+  const results: string[] = [];
+  let entries;
+  try {
+    entries = await fs.readdir(root, { withFileTypes: true });
+  } catch {
+    return results;
+  }
+  for (const entry of entries) {
+    if (['node_modules', '.git', 'dist', 'build', 'coverage', 'playwright-report', 'test-results'].includes(entry.name)) continue;
+    const fullPath = path.join(root, entry.name);
+    if (entry.isDirectory()) results.push(...await listFiles(fullPath, depth + 1));
+    else results.push(fullPath);
+  }
+  return results;
+}
+
+export async function inspectRepository(root: string): Promise<RepositoryManifest> {
+  const absoluteRoot = path.resolve(root);
+  try {
+    const files = await listFiles(absoluteRoot);
+    const packageCandidates = [
+      path.join(absoluteRoot, 'package.json'),
+      ...files.filter((file) => path.basename(file) === 'package.json' && file !== path.join(absoluteRoot, 'package.json')),
+    ];
+    let packagePath: string | undefined;
+    let packageJson: {
+      license?: string;
+      engines?: { node?: string };
+      scripts?: Record<string, string>;
+      dependencies?: Record<string, string>;
+      devDependencies?: Record<string, string>;
+    } | undefined;
+    for (const candidate of packageCandidates) {
+      try {
+        const parsed = JSON.parse(await fs.readFile(candidate, 'utf8')) as typeof packageJson;
+        const dependencies = { ...parsed?.dependencies, ...parsed?.devDependencies };
+        if (!packageJson || Object.keys(dependencies).some((name) => name === '@playwright/test' || name === 'playwright')) {
+          packagePath = candidate;
+          packageJson = parsed;
+        }
+        if (Object.keys(dependencies).some((name) => name === '@playwright/test' || name === 'playwright')) break;
+      } catch {
+        // Continue searching nested packages in monorepos.
+      }
+    }
+    if (!packageJson || !packagePath) throw new Error('No readable package.json found');
+
+    const relative = files.map((file) => path.relative(absoluteRoot, file));
+    const configs = relative.filter((file) => /(^|[\\/])playwright(?:\.[^.]*)?\.config\.(?:ts|js|mjs|cjs)$/i.test(file));
+    const allDependencies = { ...packageJson.dependencies, ...packageJson.devDependencies };
+    const hasPlaywright = Object.keys(allDependencies).some((name) => name === '@playwright/test' || name === 'playwright');
+    const packageRoot = path.dirname(packagePath);
+    const packageManager = await exists(path.join(packageRoot, 'pnpm-lock.yaml'))
+      ? 'pnpm'
+      : await exists(path.join(packageRoot, 'yarn.lock'))
+        ? 'yarn'
+        : await exists(path.join(packageRoot, 'package-lock.json'))
+          ? 'npm'
+          : 'unknown';
+
+    const limitations = packageRoot !== absoluteRoot ? [`Playwright package metadata discovered in nested project: ${path.relative(absoluteRoot, packageRoot)}`] : [];
+
+    return {
+      name: path.basename(absoluteRoot),
+      root: absoluteRoot,
+      packageManager,
+      license: packageJson.license ?? 'UNKNOWN',
+      nodeVersion: packageJson.engines?.node,
+      playwrightVersion: allDependencies['@playwright/test'] ?? allDependencies.playwright,
+      hasPlaywrightConfig: configs.length > 0,
+      playwrightConfigs: configs,
+      testFiles: relative.filter((file) => /(?:^|[\\/])(?:tests?|e2e)(?:[\\/]).*\.(?:spec|test)\.(?:ts|tsx|js|jsx|mjs|cjs)$/i.test(file)).length,
+      pageObjectFiles: relative.filter((file) => /(?:page|pages|pom|page-objects?)[\\/].*\.(?:ts|tsx|js|jsx)$/i.test(file)).length,
+      fixtureFiles: relative.filter((file) => /fixture/i.test(file) && /\.(?:ts|tsx|js|jsx)$/.test(file)).length,
+      hasDocker: relative.some((file) => /(?:^|[\\/])(?:docker-compose[^\\/]*|Dockerfile(?:\..*)?)$/i.test(file)),
+      hasLocalWebServerHint: Object.values(packageJson.scripts ?? {}).some((script) => /(?:next|vite|dev|start|serve)/i.test(script)),
+      scripts: packageJson.scripts ?? {},
+      status: hasPlaywright || configs.length > 0 ? 'DISCOVERED' : 'NOT_A_PLAYWRIGHT_REPOSITORY',
+      limitations,
+    };
+  } catch (error) {
+    return {
+      name: path.basename(absoluteRoot),
+      root: absoluteRoot,
+      packageManager: 'unknown',
+      license: 'UNKNOWN',
+      hasPlaywrightConfig: false,
+      playwrightConfigs: [],
+      testFiles: 0,
+      pageObjectFiles: 0,
+      fixtureFiles: 0,
+      hasDocker: false,
+      hasLocalWebServerHint: false,
+      scripts: {},
+      status: 'UNREADABLE',
+      limitations: [`Unable to inspect repository: ${error instanceof Error ? error.message : String(error)}`],
+    };
+  }
+}
+
+export async function inspectTestbed(testbedRoot: string): Promise<RepositoryManifest[]> {
+  const repositoriesRoot = path.join(path.resolve(testbedRoot), 'repositories');
+  const entries = await fs.readdir(repositoriesRoot, { withFileTypes: true });
+  const manifests: RepositoryManifest[] = [];
+  for (const entry of entries.filter((candidate) => candidate.isDirectory())) {
+    manifests.push(await inspectRepository(path.join(repositoriesRoot, entry.name)));
+  }
+  return manifests.sort((left, right) => left.name.localeCompare(right.name));
+}
+

package/src/evaluation/repository-runner.ts

@@ -0,0 +1,197 @@
+import { spawn, spawnSync } from 'node:child_process';
+import path from 'node:path';
+import { RepositoryManifest } from './repository-inspector.js';
+
+export interface RepositoryRunOptions {
+  script?: string | undefined;
+  timeoutMs?: number | undefined;
+  environment?: Record<string, string> | undefined;
+}
+
+export interface RepositoryRunResult {
+  repository: string;
+  command: string;
+  args: string[];
+  cwd: string;
+  status: 'PASS' | 'FAIL' | 'TIMEOUT' | 'BLOCKED';
+  exitCode: number | null;
+  signal?: string | undefined;
+  stdout: string;
+  stderr: string;
+  durationMs: number;
+  reason?: string | undefined;
+}
+
+const MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
+const SAFE_COMMANDS = new Set(['npm', 'pnpm', 'yarn', 'npx']);
+
+function executableFor(manager: RepositoryManifest['packageManager']): string {
+  if (manager === 'pnpm') return process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm';
+  if (manager === 'yarn') return process.platform === 'win32' ? 'yarn.cmd' : 'yarn';
+  return process.platform === 'win32' ? 'npm.cmd' : 'npm';
+}
+
+function chooseScript(manifest: RepositoryManifest, requested?: string): string | undefined {
+  if (requested) return requested;
+  const candidates = ['test:e2e', 'test:smoke', 'test:sanity', 'test'];
+  return candidates.find((candidate) => manifest.scripts[candidate]);
+}
+
+export class RepositoryRunner {
+  public static async runBaseline(manifest: RepositoryManifest, options: RepositoryRunOptions = {}): Promise<RepositoryRunResult> {
+    if (manifest.status !== 'DISCOVERED') {
+      return {
+        repository: manifest.name,
+        command: '',
+        args: [],
+        cwd: manifest.root,
+        status: 'BLOCKED',
+        exitCode: null,
+        stdout: '',
+        stderr: '',
+        durationMs: 0,
+        reason: `Repository status is ${manifest.status}`,
+      };
+    }
+
+    const script = chooseScript(manifest, options.script);
+    const executable = executableFor(manifest.packageManager);
+    const args = script ? ['run', script] : ['exec', 'playwright', 'test'];
+    return this.runCommand(manifest, executable, args, options);
+  }
+
+  public static runCommand(
+    manifest: RepositoryManifest,
+    executable: string,
+    args: string[],
+    options: RepositoryRunOptions = {},
+  ): Promise<RepositoryRunResult> {
+    const normalizedExecutable = path.basename(executable).replace(/\.cmd$/i, '').toLowerCase();
+    if (!SAFE_COMMANDS.has(normalizedExecutable)) {
+      return Promise.resolve({
+        repository: manifest.name,
+        command: executable,
+        args,
+        cwd: manifest.root,
+        status: 'BLOCKED',
+        exitCode: null,
+        stdout: '',
+        stderr: '',
+        durationMs: 0,
+        reason: `Command '${executable}' is not in the repository evaluation allowlist`,
+      });
+    }
+
+    if (args.some((argument) => /[&|<>]/.test(argument))) {
+      return Promise.resolve({
+        repository: manifest.name,
+        command: executable,
+        args,
+        cwd: manifest.root,
+        status: 'BLOCKED',
+        exitCode: null,
+        stdout: '',
+        stderr: '',
+        durationMs: 0,
+        reason: 'Command arguments contain shell metacharacters',
+      });
+    }
+
+    const isWindows = process.platform === 'win32';
+    const command = isWindows ? (process.env.ComSpec ?? 'cmd.exe') : executable;
+    const spawnArgs = isWindows
+      ? ['/d', '/s', '/c', [executable, ...args].map((argument) => argument.includes(' ') ? `"${argument.replace(/"/g, '\\"')}"` : argument).join(' ')]
+      : args;
+
+    return new Promise((resolve) => {
+      const started = Date.now();
+      const child = spawn(command, spawnArgs, {
+        cwd: manifest.root,
+        shell: false,
+        windowsHide: true,
+        env: { ...process.env, CI: 'true', ...options.environment },
+      });
+      let stdout = '';
+      let stderr = '';
+      let truncated = false;
+      let timedOut = false;
+      let settled = false;
+      let timeout: ReturnType<typeof setTimeout> | undefined;
+      const finish = (result: RepositoryRunResult): void => {
+        if (settled) return;
+        settled = true;
+        if (timeout) clearTimeout(timeout);
+        resolve(result);
+      };
+      const append = (target: 'stdout' | 'stderr', chunk: Buffer): void => {
+        if (Buffer.byteLength(stdout) + Buffer.byteLength(stderr) >= MAX_OUTPUT_BYTES) {
+          truncated = true;
+          return;
+        }
+        if (target === 'stdout') stdout += chunk.toString('utf8');
+        else stderr += chunk.toString('utf8');
+      };
+      child.stdout?.on('data', (chunk: Buffer) => append('stdout', chunk));
+      child.stderr?.on('data', (chunk: Buffer) => append('stderr', chunk));
+
+      timeout = setTimeout(() => {
+        timedOut = true;
+        child.kill('SIGTERM');
+        if (process.platform === 'win32' && child.pid) {
+          spawnSync('taskkill.exe', ['/PID', String(child.pid), '/T', '/F'], {
+            windowsHide: true,
+            stdio: 'ignore',
+            shell: false,
+            timeout: 5_000,
+          });
+          child.stdout?.destroy();
+          child.stderr?.destroy();
+        }
+        finish({
+          repository: manifest.name,
+          command: executable,
+          args,
+          cwd: manifest.root,
+          status: 'TIMEOUT',
+          exitCode: null,
+          stdout,
+          stderr,
+          durationMs: Date.now() - started,
+          reason: 'Command exceeded the configured timeout',
+        });
+      }, options.timeoutMs ?? 15 * 60 * 1000);
+
+      child.on('error', (error) => {
+        finish({
+          repository: manifest.name,
+          command: executable,
+          args,
+          cwd: manifest.root,
+          status: 'BLOCKED',
+          exitCode: null,
+          stdout,
+          stderr,
+          durationMs: Date.now() - started,
+          reason: `Unable to start command: ${error.message}`,
+        });
+      });
+
+      child.on('close', (exitCode, signal) => {
+        finish({
+          repository: manifest.name,
+          command: executable,
+          args,
+          cwd: manifest.root,
+          status: timedOut ? 'TIMEOUT' : exitCode === 0 ? 'PASS' : 'FAIL',
+          exitCode,
+          signal: signal ?? undefined,
+          stdout,
+          stderr,
+          durationMs: Date.now() - started,
+          reason: truncated ? 'Output truncated at 16 MB' : undefined,
+        });
+      });
+    });
+  }
+}
+

package/src/evidence/evidence-model.ts

@@ -0,0 +1,76 @@
+import { z } from 'zod';
+
+export const EvidenceSourceSchema = z.enum([
+  'trace',
+  'live-browser',
+  'console',
+  'network',
+  'static-ast',
+  'git-diff',
+  'environment',
+  'error',
+]);
+
+export type EvidenceSource = z.infer<typeof EvidenceSourceSchema>;
+
+export const ReliabilityLevelSchema = z.enum([
+  'HIGH',
+  'MEDIUM',
+  'LOW',
+  'UNVERIFIED',
+]);
+
+export type ReliabilityLevel = z.infer<typeof ReliabilityLevelSchema>;
+
+export const ProvenanceMetadataSchema = z.object({
+  collector: z.string(),
+  collectedAt: z.number(),
+  captureMethod: z.string(),
+  sourceUri: z.string().optional(),
+});
+
+export type ProvenanceMetadata = z.infer<typeof ProvenanceMetadataSchema>;
+
+export const RedactionStatusSchema = z.object({
+  isRedacted: z.boolean(),
+  redactedFields: z.array(z.string()).optional(),
+  redactionRulesApplied: z.array(z.string()).optional(),
+});
+
+export type RedactionStatus = z.infer<typeof RedactionStatusSchema>;
+
+export const UnifiedEvidenceRecordSchema = z.object({
+  evidenceId: z.string(),
+  executionId: z.string(),
+  source: EvidenceSourceSchema,
+  timestamp: z.number(),
+  monotonicTimestamp: z.number().optional(),
+  workerId: z.string().optional(),
+  testId: z.string().optional(),
+  frameId: z.string().optional(),
+  actionId: z.string().optional(),
+  requestId: z.string().optional(),
+  evidenceType: z.string(), // e.g. 'dom-snapshot', 'console-error', 'http-response', 'ast-locator', 'git-commit'
+  payload: z.record(z.string(), z.any()),
+  reliability: ReliabilityLevelSchema,
+  provenance: ProvenanceMetadataSchema,
+  redactionStatus: RedactionStatusSchema,
+  limitations: z.array(z.string()).optional(),
+});
+
+export type UnifiedEvidenceRecord = z.infer<typeof UnifiedEvidenceRecordSchema>;
+
+export interface EvidenceScorecard {
+  playwrightErrorAvailable: boolean;
+  testSourceAstAvailable: boolean;
+  domSnapshotAvailable: boolean;
+  consoleEventsAvailable: boolean;
+  networkEventsAvailable: boolean;
+  traceArchiveAvailable: boolean;
+  gitHistoryAvailable: boolean;
+  environmentMetadataAvailable: boolean;
+  totalRecordsCount: number;
+  availableSources: EvidenceSource[];
+  unavailableSources: EvidenceSource[];
+}
+