playwright-intelligence-engine 0.0.0-stage → 2.0.0

package/src/evidence/evidence-normalizer.ts

@@ -0,0 +1,322 @@
+import {
+  UnifiedEvidenceRecord,
+  EvidenceScorecard,
+  EvidenceSource,
+} from './evidence-model.js';
+import {
+  DiagnosticContext,
+  NetworkEvent,
+  ConsoleEvent,
+  FailedAction,
+} from '../core/types/diagnostic-context.js';
+
+export class EvidenceNormalizer {
+  private static readonly ALL_SOURCES: EvidenceSource[] = [
+    'trace',
+    'live-browser',
+    'console',
+    'network',
+    'static-ast',
+    'git-diff',
+    'environment',
+    'error',
+  ];
+
+  /**
+   * Computes an EvidenceScorecard summarizing the presence and completeness of all evidence channels.
+   */
+  public static calculateScorecard(records: UnifiedEvidenceRecord[]): EvidenceScorecard {
+    const presentSources = new Set<EvidenceSource>(records.map((r) => r.source));
+
+    const playwrightErrorAvailable = records.some(
+      (r) => r.source === 'error' || r.evidenceType === 'playwright-error' || Boolean(r.payload?.errorMessage)
+    );
+    const testSourceAstAvailable = records.some(
+      (r) => r.source === 'static-ast' || r.evidenceType === 'ast-analysis'
+    );
+    const domSnapshotAvailable = records.some(
+      (r) => r.evidenceType === 'dom-snapshot' || r.evidenceType === 'accessibility-tree' || Boolean(r.payload?.html || r.payload?.snapshot)
+    );
+    const consoleEventsAvailable = records.some(
+      (r) => r.source === 'console' || r.evidenceType === 'console-entry' || r.evidenceType === 'console-event'
+    );
+    const networkEventsAvailable = records.some(
+      (r) => r.source === 'network' || r.evidenceType === 'http-response' || r.evidenceType === 'http-request'
+    );
+    const traceArchiveAvailable = records.some((r) => r.source === 'trace');
+    const gitHistoryAvailable = records.some((r) => r.source === 'git-diff');
+    const environmentMetadataAvailable = records.some((r) => r.source === 'environment');
+
+    const availableSources = Array.from(presentSources);
+    const unavailableSources = EvidenceNormalizer.ALL_SOURCES.filter((s) => !presentSources.has(s));
+
+    return {
+      playwrightErrorAvailable,
+      testSourceAstAvailable,
+      domSnapshotAvailable,
+      consoleEventsAvailable,
+      networkEventsAvailable,
+      traceArchiveAvailable,
+      gitHistoryAvailable,
+      environmentMetadataAvailable,
+      totalRecordsCount: records.length,
+      availableSources,
+      unavailableSources,
+    };
+  }
+
+  /**
+   * Normalizes a collection of multi-source UnifiedEvidenceRecords into a structured DiagnosticContext.
+   * This allows the failure classifier and repair pipeline to function without requiring a trace.zip file.
+   */
+  public static toDiagnosticContext(records: UnifiedEvidenceRecord[]): DiagnosticContext {
+    // 1. Extract FailedAction
+    let failedAction: FailedAction = {
+      callId: 'multi-source-0',
+      action: 'unknown',
+    };
+
+    const errorRecord = records.find(
+      (r) => r.source === 'error' || r.evidenceType === 'playwright-error'
+    );
+    if (errorRecord && errorRecord.payload) {
+      failedAction = {
+        callId: errorRecord.actionId || errorRecord.evidenceId,
+        action: errorRecord.payload.action || 'action',
+        selector: errorRecord.payload.selector,
+        sourceFile: errorRecord.payload.sourceFile,
+        line: errorRecord.payload.line,
+        column: errorRecord.payload.column,
+        errorMessage: errorRecord.payload.errorMessage,
+        errorStack: errorRecord.payload.errorStack,
+        durationMs: errorRecord.payload.durationMs,
+      };
+    } else {
+      // Fallback to any record with an errorMessage
+      const fallbackRecord = records.find((r) => r.payload?.errorMessage);
+      if (fallbackRecord) {
+        failedAction = {
+          callId: fallbackRecord.actionId || fallbackRecord.evidenceId,
+          action: fallbackRecord.payload.action || 'action',
+          selector: fallbackRecord.payload.selector,
+          errorMessage: fallbackRecord.payload.errorMessage,
+        };
+      }
+    }
+
+    // 2. Extract DOM Snapshots
+    const snapshots: { before?: string; action?: string; after?: string } = {};
+    const domRecords = records.filter(
+      (r) => r.evidenceType === 'dom-snapshot' || r.evidenceType === 'accessibility-tree'
+    );
+    for (const domRec of domRecords) {
+      const html = domRec.payload.html || domRec.payload.snapshot || domRec.payload.content;
+      if (html) {
+        const phase = domRec.payload.phase || 'action';
+        if (phase === 'before') snapshots.before = html;
+        else if (phase === 'after') snapshots.after = html;
+        else snapshots.action = html;
+      }
+    }
+
+    // 3. Extract Network Events
+    const networkEvents: NetworkEvent[] = [];
+    const networkRecords = records.filter(
+      (r) => r.source === 'network' || r.evidenceType === 'http-response' || r.evidenceType === 'http-request'
+    );
+    for (const netRec of networkRecords) {
+      if (netRec.payload?.url) {
+        networkEvents.push({
+          url: netRec.payload.url,
+          method: netRec.payload.method || 'GET',
+          status: typeof netRec.payload.status === 'number' ? netRec.payload.status : 0,
+          statusText: netRec.payload.statusText,
+          headers: netRec.payload.headers,
+          timestamp: netRec.timestamp,
+          durationMs: netRec.payload.durationMs,
+          errorText: netRec.payload.errorText,
+          requestId: netRec.requestId || netRec.payload.requestId,
+          frameId: netRec.frameId || netRec.payload.frameId,
+          resourceType: netRec.payload.resourceType,
+          initiator: netRec.payload.initiator,
+        });
+      }
+    }
+
+    // 4. Extract Console Events
+    const consoleEvents: ConsoleEvent[] = [];
+    const consoleRecords = records.filter(
+      (r) => r.source === 'console' || r.evidenceType === 'console-entry' || r.evidenceType === 'console-event'
+    );
+    for (const conRec of consoleRecords) {
+      if (conRec.payload?.text) {
+        consoleEvents.push({
+          type: conRec.payload.type || 'log',
+          text: conRec.payload.text,
+          timestamp: conRec.timestamp,
+          location: conRec.payload.location,
+        });
+      }
+    }
+
+    // 5. Extract Environment Metadata
+    const envRecord = records.find((r) => r.source === 'environment');
+    const playwrightVersion = envRecord?.payload?.playwrightVersion || '1.40.0';
+    const traceFormatVersion = envRecord?.payload?.traceFormatVersion || 0;
+
+    const hasSnapshots = Object.keys(snapshots).length > 0;
+    const hasNetwork = networkEvents.length > 0;
+    const hasSource = records.some((r) => r.source === 'static-ast');
+
+    return {
+      playwrightVersion,
+      traceFormatVersion,
+      failedAction,
+      snapshots,
+      networkEvents,
+      consoleEvents,
+      evidenceCompleteness: {
+        domAvailable: hasSnapshots,
+        networkAvailable: hasNetwork,
+        sourceAvailable: hasSource,
+      },
+    };
+  }
+
+  /**
+   * Converts a DiagnosticContext into UnifiedEvidenceRecords.
+   */
+  public static fromDiagnosticContext(
+    context: DiagnosticContext,
+    executionId = `exec-${Date.now()}`
+  ): UnifiedEvidenceRecord[] {
+    const records: UnifiedEvidenceRecord[] = [];
+    const now = Date.now();
+
+    // 1. Error / Failed action
+    records.push({
+      evidenceId: `err-${context.failedAction.callId || '0'}`,
+      executionId,
+      source: 'error',
+      timestamp: now,
+      actionId: context.failedAction.callId,
+      evidenceType: 'playwright-error',
+      payload: {
+        action: context.failedAction.action,
+        selector: context.failedAction.selector,
+        errorMessage: context.failedAction.errorMessage,
+        errorStack: context.failedAction.errorStack,
+        sourceFile: context.failedAction.sourceFile,
+        line: context.failedAction.line,
+        column: context.failedAction.column,
+        durationMs: context.failedAction.durationMs,
+      },
+      reliability: 'HIGH',
+      provenance: {
+        collector: 'DiagnosticContextBridge',
+        collectedAt: now,
+        captureMethod: 'context-export',
+      },
+      redactionStatus: { isRedacted: false },
+    });
+
+    // 2. DOM Snapshots
+    for (const [phase, html] of Object.entries(context.snapshots)) {
+      if (html) {
+        records.push({
+          evidenceId: `dom-${phase}-${Date.now()}`,
+          executionId,
+          source: 'trace',
+          timestamp: now,
+          evidenceType: 'dom-snapshot',
+          payload: { phase, html },
+          reliability: 'HIGH',
+          provenance: {
+            collector: 'DiagnosticContextBridge',
+            collectedAt: now,
+            captureMethod: 'snapshot-extraction',
+          },
+          redactionStatus: { isRedacted: false },
+        });
+      }
+    }
+
+    // 3. Network Events
+    let netIdx = 0;
+    for (const net of context.networkEvents) {
+      records.push({
+        evidenceId: `net-${net.requestId || netIdx++}`,
+        executionId,
+        source: 'network',
+        timestamp: net.timestamp || now,
+        requestId: net.requestId,
+        frameId: net.frameId,
+        evidenceType: 'http-response',
+        payload: {
+          url: net.url,
+          method: net.method,
+          status: net.status,
+          statusText: net.statusText,
+          headers: net.headers,
+          durationMs: net.durationMs,
+          errorText: net.errorText,
+          resourceType: net.resourceType,
+          initiator: net.initiator,
+        },
+        reliability: 'HIGH',
+        provenance: {
+          collector: 'DiagnosticContextBridge',
+          collectedAt: now,
+          captureMethod: 'network-event',
+        },
+        redactionStatus: { isRedacted: false },
+      });
+    }
+
+    // 4. Console Events
+    let conIdx = 0;
+    for (const con of context.consoleEvents) {
+      records.push({
+        evidenceId: `con-${conIdx++}`,
+        executionId,
+        source: 'console',
+        timestamp: con.timestamp || now,
+        evidenceType: 'console-entry',
+        payload: {
+          type: con.type,
+          text: con.text,
+          location: con.location,
+        },
+        reliability: 'MEDIUM',
+        provenance: {
+          collector: 'DiagnosticContextBridge',
+          collectedAt: now,
+          captureMethod: 'console-event',
+        },
+        redactionStatus: { isRedacted: false },
+      });
+    }
+
+    // 5. Environment
+    records.push({
+      evidenceId: `env-${Date.now()}`,
+      executionId,
+      source: 'environment',
+      timestamp: now,
+      evidenceType: 'environment-metadata',
+      payload: {
+        playwrightVersion: context.playwrightVersion,
+        traceFormatVersion: context.traceFormatVersion,
+      },
+      reliability: 'HIGH',
+      provenance: {
+        collector: 'DiagnosticContextBridge',
+        collectedAt: now,
+        captureMethod: 'environment-probe',
+      },
+      redactionStatus: { isRedacted: false },
+    });
+
+    return records;
+  }
+}

package/src/evidence/evidence-redactor.ts

@@ -0,0 +1,218 @@
+import { UnifiedEvidenceRecord, RedactionStatus } from './evidence-model.js';
+
+export class EvidenceRedactor {
+  private static readonly SENSITIVE_HEADER_KEYS = new Set([
+    'authorization',
+    'cookie',
+    'set-cookie',
+    'x-api-key',
+    'x-auth-token',
+    'proxy-authorization',
+    'proxy-authenticate',
+    'www-authenticate',
+    'session-token',
+    'access-token',
+  ]);
+
+  private static readonly SENSITIVE_FIELD_PATTERNS = [
+    /pass(word)?/i,
+    /secret/i,
+    /token/i,
+    /api[_-]?key/i,
+    /bearer/i,
+    /credit[_-]?card/i,
+    /cvv/i,
+    /ssn/i,
+    /social[_-]?security/i,
+    /private[_-]?key/i,
+  ];
+
+  private static readonly PATTERNS = [
+    {
+      name: 'BEARER_TOKEN',
+      regex: /Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi,
+      replacement: 'Bearer [REDACTED]',
+    },
+    {
+      name: 'JWT_TOKEN',
+      regex: /eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.?[A-Za-z0-9-_.+/=]*/g,
+      replacement: '[JWT_TOKEN_REDACTED]',
+    },
+    {
+      name: 'CREDIT_CARD',
+      // 13-19 digits with optional spaces or dashes
+      regex: /\b(?:\d[ -]*?){13,19}\b/g,
+      replacement: '[CARD_REDACTED]',
+    },
+    {
+      name: 'SSN',
+      regex: /\b\d{3}-\d{2}-\d{4}\b/g,
+      replacement: '[SSN_REDACTED]',
+    },
+    {
+      name: 'BASIC_AUTH_URL',
+      regex: /https?:\/\/[^:]+:[^@]+@/gi,
+      replacement: 'https://[USER:PASS_REDACTED]@',
+    },
+  ];
+
+  /**
+   * Redacts sensitive string patterns such as JWT tokens, credit cards, SSNs, and passwords.
+   */
+  public static redactText(text: string): { redactedText: string; rulesApplied: string[] } {
+    if (!text || typeof text !== 'string') {
+      return { redactedText: text, rulesApplied: [] };
+    }
+
+    let result = text;
+    const rulesApplied: string[] = [];
+
+    for (const rule of EvidenceRedactor.PATTERNS) {
+      if (rule.regex.test(result)) {
+        rulesApplied.push(rule.name);
+        result = result.replace(rule.regex, rule.replacement);
+      }
+      // Reset regex state if global
+      rule.regex.lastIndex = 0;
+    }
+
+    return { redactedText: result, rulesApplied };
+  }
+
+  /**
+   * Redacts sensitive HTTP header maps.
+   */
+  public static redactHeaders(headers?: Record<string, string>): {
+    redactedHeaders: Record<string, string>;
+    redactedFields: string[];
+  } {
+    if (!headers || typeof headers !== 'object') {
+      return { redactedHeaders: {}, redactedFields: [] };
+    }
+
+    const redactedHeaders: Record<string, string> = {};
+    const redactedFields: string[] = [];
+
+    for (const [key, value] of Object.entries(headers)) {
+      const lower = key.toLowerCase();
+      if (EvidenceRedactor.SENSITIVE_HEADER_KEYS.has(lower)) {
+        redactedHeaders[key] = '[REDACTED]';
+        redactedFields.push(key);
+      } else {
+        const { redactedText, rulesApplied } = EvidenceRedactor.redactText(String(value));
+        redactedHeaders[key] = redactedText;
+        if (rulesApplied.length > 0) {
+          redactedFields.push(key);
+        }
+      }
+    }
+
+    return { redactedHeaders, redactedFields };
+  }
+
+  /**
+   * Recursively redacts objects, arrays, and primitive payload fields.
+   */
+  public static redactPayload(payload: any): {
+    redactedPayload: any;
+    redactedFields: string[];
+    rulesApplied: string[];
+  } {
+    const redactedFields: string[] = [];
+    const rulesApplied: string[] = [];
+
+    function recurse(value: any, path: string): any {
+      if (value === null || value === undefined) {
+        return value;
+      }
+
+      if (typeof value === 'string') {
+        const { redactedText, rulesApplied: applied } = EvidenceRedactor.redactText(value);
+        if (applied.length > 0) {
+          redactedFields.push(path);
+          for (const rule of applied) {
+            if (!rulesApplied.includes(rule)) rulesApplied.push(rule);
+          }
+        }
+        return redactedText;
+      }
+
+      if (Array.isArray(value)) {
+        return value.map((item, idx) => recurse(item, `${path}[${idx}]`));
+      }
+
+      if (typeof value === 'object') {
+        const result: Record<string, any> = {};
+        for (const [k, v] of Object.entries(value)) {
+          const currentPath = path ? `${path}.${k}` : k;
+          const isSensitiveField = EvidenceRedactor.SENSITIVE_FIELD_PATTERNS.some((p) => p.test(k));
+
+          if (isSensitiveField) {
+            result[k] = '[REDACTED]';
+            redactedFields.push(currentPath);
+            if (!rulesApplied.includes('SENSITIVE_FIELD_KEY')) {
+              rulesApplied.push('SENSITIVE_FIELD_KEY');
+            }
+          } else {
+            result[k] = recurse(v, currentPath);
+          }
+        }
+        return result;
+      }
+
+      return value;
+    }
+
+    const redactedPayload = recurse(payload, '');
+    return { redactedPayload, redactedFields, rulesApplied };
+  }
+
+  /**
+   * Sanitizes DOM HTML strings, scrubbing passwords, secret inputs, tokens, and PII.
+   */
+  public static redactDomContent(html: string): string {
+    if (!html || typeof html !== 'string') return '';
+    let clean = html;
+    // 1. Password input values
+    clean = clean.replace(/(<input[^>]*type=["']password["'][^>]*value=["'])([^"']*)(["'])/gi, '$1[REDACTED]$3');
+    // 2. Hidden input tokens
+    clean = clean.replace(/(<input[^>]*name=["'](?:csrf|token|secret|auth|apikey|api_key)["'][^>]*value=["'])([^"']*)(["'])/gi, '$1[REDACTED]$3');
+    // 3. General pattern redaction (Bearer, JWT, CC, SSN)
+    clean = EvidenceRedactor.redactText(clean).redactedText;
+    return clean;
+  }
+
+  /**
+   * Enforces complete redaction across a UnifiedEvidenceRecord.
+   */
+  public static redactRecord(record: UnifiedEvidenceRecord): UnifiedEvidenceRecord {
+    let rawPayload = record.payload;
+    if (record.evidenceType === 'dom-snapshot' && rawPayload?.html) {
+      rawPayload = {
+        ...rawPayload,
+        html: EvidenceRedactor.redactDomContent(rawPayload.html),
+      };
+    }
+
+    const { redactedPayload, redactedFields, rulesApplied } = EvidenceRedactor.redactPayload(rawPayload);
+
+    const existingFields = record.redactionStatus.redactedFields ?? [];
+    const existingRules = record.redactionStatus.redactionRulesApplied ?? [];
+
+    const mergedFields = Array.from(new Set([...existingFields, ...redactedFields]));
+    const mergedRules = Array.from(new Set([...existingRules, ...rulesApplied]));
+
+    const redactionStatus: RedactionStatus = {
+      isRedacted: mergedFields.length > 0 || record.redactionStatus.isRedacted,
+      redactedFields: mergedFields,
+      redactionRulesApplied: mergedRules,
+    };
+
+    return {
+      ...record,
+      payload: redactedPayload,
+      redactionStatus,
+    };
+  }
+}
+

package/src/framework/config-adapter.ts

@@ -0,0 +1,131 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { Project, SyntaxKind, ObjectLiteralExpression } from 'ts-morph';
+
+export interface PlaywrightConfigSummary {
+  configPath: string;
+  testDir: string;
+  timeoutMs: number;
+  retries: number;
+  workers?: number | string;
+  traceSetting: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'unknown';
+  projects: Array<{ name: string; use?: Record<string, any> }>;
+}
+
+export class ConfigAdapter {
+  private static readonly CONFIG_NAMES = [
+    'playwright.config.ts',
+    'playwright.config.js',
+    'playwright.config.mjs',
+    'playwright.config.cjs',
+  ];
+
+  /**
+   * Finds and summarizes the Playwright config in a repository.
+   */
+  public static async findConfig(repoDir: string): Promise<PlaywrightConfigSummary | null> {
+    for (const name of ConfigAdapter.CONFIG_NAMES) {
+      const fullPath = path.resolve(repoDir, name);
+      try {
+        await fs.access(fullPath);
+        return ConfigAdapter.parseConfig(fullPath);
+      } catch {
+        continue;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Statically parses a playwright.config.ts file via ts-morph AST without evaluating untrusted code.
+   */
+  public static parseConfig(configPath: string): PlaywrightConfigSummary {
+    const project = new Project({ skipAddingFilesFromTsConfig: true });
+    const sourceFile = project.addSourceFileAtPath(configPath);
+
+    let testDir = './tests';
+    let timeoutMs = 30000;
+    let retries = 0;
+    let traceSetting: PlaywrightConfigSummary['traceSetting'] = 'unknown';
+    const projects: PlaywrightConfigSummary['projects'] = [];
+
+    // Look for defineProperty or export default defineConfig({...})
+    sourceFile.forEachDescendant((node) => {
+      if (node.isKind(SyntaxKind.ObjectLiteralExpression)) {
+        const obj = node as ObjectLiteralExpression;
+
+        const testDirProp = obj.getProperty('testDir');
+        if (testDirProp && 'getInitializer' in testDirProp) {
+          const init = (testDirProp as any).getInitializer();
+          if (init) testDir = init.getText().replace(/['"]/g, '');
+        }
+
+        const timeoutProp = obj.getProperty('timeout');
+        if (timeoutProp && 'getInitializer' in timeoutProp) {
+          const init = (timeoutProp as any).getInitializer();
+          if (init && !isNaN(Number(init.getText()))) {
+            timeoutMs = Number(init.getText());
+          }
+        }
+
+        const retriesProp = obj.getProperty('retries');
+        if (retriesProp && 'getInitializer' in retriesProp) {
+          const init = (retriesProp as any).getInitializer();
+          if (init && !isNaN(Number(init.getText()))) {
+            retries = Number(init.getText());
+          }
+        }
+
+        // Check use.trace
+        const useProp = obj.getProperty('use');
+        if (useProp && 'getInitializer' in useProp) {
+          const useInit = (useProp as any).getInitializer();
+          if (useInit && useInit.isKind(SyntaxKind.ObjectLiteralExpression)) {
+            const traceProp = (useInit as ObjectLiteralExpression).getProperty('trace');
+            if (traceProp && 'getInitializer' in traceProp) {
+              const traceInit = (traceProp as any).getInitializer();
+              const val = traceInit?.getText().replace(/['"]/g, '');
+              if (['off', 'on', 'retain-on-failure', 'on-first-retry'].includes(val)) {
+                traceSetting = val as any;
+              }
+            }
+          }
+        }
+      }
+    });
+
+    return {
+      configPath,
+      testDir,
+      timeoutMs,
+      retries,
+      traceSetting,
+      projects,
+    };
+  }
+
+  /**
+   * Generates a temporary verification config string with retries disabled and tracing enabled.
+   */
+  public static generateVerificationOverride(baseConfigPath?: string): string {
+    const importStatement = baseConfigPath
+      ? `import baseConfig from './${path.basename(baseConfigPath).replace(/\.ts$/, '')}';\n`
+      : `import { defineConfig } from '@playwright/test';\n`;
+
+    return `${importStatement}
+// Auto-generated by Playwright Intelligence Engine for isolated worktree verification
+export default {
+  ...(typeof baseConfig !== 'undefined' ? baseConfig : {}),
+  retries: 0, // Deterministic: retries disabled to measure true flakiness
+  workers: 1, // Deterministic single-worker isolation
+  use: {
+    ...(typeof baseConfig !== 'undefined' && baseConfig.use ? baseConfig.use : {}),
+    trace: 'retain-on-failure',
+    video: 'off',
+    screenshot: 'only-on-failure',
+  },
+};
+`;
+  }
+}
+