playwright-intelligence-engine 0.0.0-stage → 2.0.0

package/src/diagnostics/diagnostic-contract.ts

@@ -0,0 +1,79 @@
+import { z } from 'zod';
+import { DiagnosticContext, FailureCategorySchema } from '../core/types/diagnostic-context.js';
+import { ClassificationResult } from './failure-classifier.js';
+import { normalizeTypedFailureCategory } from './failure-category.js';
+
+export const DiagnosticRecommendationSchema = z.enum([
+  'REPAIR_CANDIDATE',
+  'REPRODUCE_WITH_MORE_EVIDENCE',
+  'REPORT_APPLICATION_OR_ENVIRONMENT_FAILURE',
+  'ABSTAIN_FROM_REPAIR',
+]);
+
+export type DiagnosticRecommendation = z.infer<typeof DiagnosticRecommendationSchema>;
+
+export const DiagnosticDecisionSchema = z.object({
+  category: FailureCategorySchema,
+  confidence: z.number().min(0).max(1),
+  shouldAbstain: z.boolean(),
+  reason: z.string(),
+  recommendation: DiagnosticRecommendationSchema,
+  supportingEvidence: z.array(z.string()),
+  missingEvidence: z.array(z.string()),
+  provenance: z.object({
+    failedActionId: z.string(),
+    traceFormatVersion: z.number(),
+    playwrightVersion: z.string(),
+  }),
+});
+
+export type DiagnosticDecision = z.infer<typeof DiagnosticDecisionSchema>;
+
+function recommendationFor(classification: ClassificationResult): DiagnosticRecommendation {
+  if (classification.shouldAbstain) {
+    if (
+      classification.category === 'BACKEND_500' ||
+      classification.category === 'AUTH_EXPIRED' ||
+      classification.category === 'AUTH_SESSION_EXPIRED' ||
+      classification.category === 'ENVIRONMENT_FAILURE' ||
+      classification.category === 'ASSERTION_FAILURE' ||
+      classification.category === 'ASSERTION_FAILED'
+    ) {
+      return 'REPORT_APPLICATION_OR_ENVIRONMENT_FAILURE';
+    }
+    return 'ABSTAIN_FROM_REPAIR';
+  }
+
+  if (classification.category === 'INCOMPLETE_EVIDENCE') return 'REPRODUCE_WITH_MORE_EVIDENCE';
+  if (classification.category === 'LOCATOR_DRIFT') return 'REPAIR_CANDIDATE';
+  return 'ABSTAIN_FROM_REPAIR';
+}
+
+function missingEvidenceFor(context: DiagnosticContext): string[] {
+  const missing: string[] = [];
+  if (!context.evidenceCompleteness.domAvailable) missing.push('DOM or accessibility snapshot');
+  if (!context.evidenceCompleteness.networkAvailable) missing.push('network timeline');
+  if (!context.evidenceCompleteness.sourceAvailable) missing.push('test source and Page Object context');
+  return missing;
+}
+
+export function createDiagnosticDecision(
+  context: DiagnosticContext,
+  classification: ClassificationResult,
+): DiagnosticDecision {
+  return DiagnosticDecisionSchema.parse({
+    category: normalizeTypedFailureCategory(classification.category),
+    confidence: classification.confidence,
+    shouldAbstain: classification.shouldAbstain,
+    reason: classification.reason,
+    recommendation: recommendationFor(classification),
+    supportingEvidence: classification.supportingEvidence,
+    missingEvidence: missingEvidenceFor(context),
+    provenance: {
+      failedActionId: context.failedAction.callId,
+      traceFormatVersion: context.traceFormatVersion,
+      playwrightVersion: context.playwrightVersion,
+    },
+  });
+}
+

package/src/diagnostics/diagnostic-report.ts

@@ -0,0 +1,204 @@
+import { CorrelatedEvidence } from './evidence-correlator.js';
+import { EvidenceScorecard } from '../evidence/evidence-model.js';
+import { CandidateExtractor } from '../selector/candidate-extractor.js';
+import { RepairCandidateRanker } from '../selector/repair-candidate-ranker.js';
+import { AbstentionPolicy } from '../selector/abstention-policy.js';
+import { ProjectIntelligenceSnapshot } from '../intelligence/project-intelligence.js';
+import { RankedRepairCandidate } from '../selector/repair-candidate-ranker.js';
+
+export class DiagnosticReportGenerator {
+  public static selectRepairCandidate(
+    evidence: CorrelatedEvidence,
+    intelligence?: ProjectIntelligenceSnapshot,
+  ): RankedRepairCandidate | undefined {
+    const { context, classification } = evidence;
+    if (classification.shouldAbstain) return undefined;
+    const domRoot = (context as any).domRoot;
+    if (!domRoot) return undefined;
+    try {
+      const candidates = CandidateExtractor.extractCandidates(domRoot, context.failedAction);
+      if (candidates.length === 0) return undefined;
+      const ranked = RepairCandidateRanker.rank(
+        candidates,
+        context.failedAction.selector,
+        context.failedAction.action,
+        intelligence,
+        domRoot,
+        context.failedAction.selector
+      );
+      const decision = AbstentionPolicy.evaluate(ranked);
+      if (!decision.shouldAbstain && decision.selectedCandidate) {
+        return decision.selectedCandidate as RankedRepairCandidate;
+      }
+    } catch {
+      return undefined;
+    }
+    return undefined;
+  }
+
+  public static extractProposedLocator(
+    evidence: CorrelatedEvidence,
+    intelligence?: ProjectIntelligenceSnapshot,
+  ): string | undefined {
+    return DiagnosticReportGenerator.selectRepairCandidate(evidence, intelligence)?.locatorExpression;
+  }
+
+  public static generateTerminalReport(
+    evidence: CorrelatedEvidence,
+    scorecard?: EvidenceScorecard,
+    intelligence?: ProjectIntelligenceSnapshot,
+  ): string {
+    const { context, classification } = evidence;
+    const { failedAction } = context;
+
+    const divider = '─'.repeat(72);
+    const lines: string[] = [];
+
+    lines.push('');
+    lines.push('╔══════════════════════════════════════════════════════════════════════╗');
+    lines.push('║             PLAYWRIGHT INTELLIGENCE ENGINE: DIAGNOSTIC REPORT        ║');
+    lines.push('╚══════════════════════════════════════════════════════════════════════╝');
+    lines.push('');
+
+    // Summary Header
+    lines.push(`Root Cause Classification : ${classification.category}`);
+    lines.push(`Confidence Level          : ${(classification.confidence * 100).toFixed(1)}%`);
+    lines.push(`Failure Fingerprint       : ${evidence.failureFingerprint.key}`);
+    lines.push(`Action Outcome            : ${classification.shouldAbstain ? '⛔ ABSTAIN FROM CODE REPAIR' : '🔍 REPAIR CANDIDATE AVAILABLE'}`);
+    lines.push(divider);
+
+    // Failed Action Context
+    lines.push('Failed Action Details:');
+    lines.push(`  • Call ID     : ${failedAction.callId}`);
+    lines.push(`  • Method      : ${failedAction.action}`);
+    if (failedAction.selector) {
+      lines.push(`  • Selector    : ${failedAction.selector}`);
+    }
+    if (failedAction.sourceFile) {
+      lines.push(`  • Source File : ${failedAction.sourceFile}:${failedAction.line ?? 0}:${failedAction.column ?? 0}`);
+    }
+    if (failedAction.durationMs) {
+      lines.push(`  • Duration    : ${failedAction.durationMs}ms`);
+    }
+    if (failedAction.errorMessage) {
+      lines.push(`  • Error       : ${failedAction.errorMessage}`);
+    }
+    lines.push(divider);
+
+    // Evidence Scorecard / Completeness
+    if (scorecard) {
+      lines.push('Multi-Source Evidence Scorecard:');
+      lines.push(`  • Playwright Error Msg   : ${scorecard.playwrightErrorAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • Test Source AST        : ${scorecard.testSourceAstAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • DOM Snapshot           : ${scorecard.domSnapshotAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • Network Events         : ${scorecard.networkEventsAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • Console Events         : ${scorecard.consoleEventsAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • Trace Archive          : ${scorecard.traceArchiveAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • Git Working Tree Diff  : ${scorecard.gitHistoryAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • Environment Metadata   : ${scorecard.environmentMetadataAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • Total Ingested Records : ${scorecard.totalRecordsCount}`);
+    } else {
+      const comp = context.evidenceCompleteness;
+      lines.push('Evidence Provenance:');
+      lines.push(`  • DOM Snapshot Available : ${comp.domAvailable ? '✅ YES' : '❌ NO'}`);
+      lines.push(`  • Network Logs Ingested  : ${comp.networkAvailable ? `✅ YES (${context.networkEvents.length} events)` : '❌ NONE'}`);
+      lines.push(`  • Source Code Mapped     : ${comp.sourceAvailable ? '✅ YES' : '❌ NO'}`);
+    }
+    lines.push(divider);
+
+    // Causal Evidence Graph summary
+    if (classification.causalGraph) {
+      const g = classification.causalGraph;
+      lines.push('Causal Evidence Graph:');
+      lines.push(`  • Graph Size   : ${g.nodes.length} nodes, ${g.edges?.length ?? 0} causal edges`);
+      if (g.hasCausallyRelevantServerError && g.offendingNetworkEvent) {
+        lines.push(`  • Offending Request: HTTP ${g.offendingNetworkEvent.status} on ${g.offendingNetworkEvent.method} ${g.offendingNetworkEvent.url}`);
+      }
+      lines.push(divider);
+    }
+
+    // Suspicious Network & Console Context
+    if (evidence.suspiciousNetworkEvents.length > 0) {
+      lines.push('Correlated Network Anomalies:');
+      for (const net of evidence.suspiciousNetworkEvents.slice(0, 5)) {
+        lines.push(`  ⚠️  HTTP ${net.status} [${net.method}] ${net.url}`);
+      }
+      lines.push(divider);
+    }
+
+    if (evidence.suspiciousConsoleEvents.length > 0) {
+      lines.push('Correlated Console Errors:');
+      for (const con of evidence.suspiciousConsoleEvents.slice(0, 3)) {
+        lines.push(`  ⚠️  ${con.text}`);
+      }
+      lines.push(divider);
+    }
+
+    // Abstention or Next Action
+    if (classification.shouldAbstain) {
+      lines.push('Abstention Rationale:');
+      lines.push(`  ${classification.abstentionReason ?? classification.reason}`);
+      lines.push('  Engine guardrail enforced: zero unsafe AST mutations attempted.');
+    } else {
+      lines.push('Diagnostic Conclusion:');
+      lines.push(`  ${classification.reason}`);
+      lines.push('  Ready for candidate extraction and isolated worktree verification.');
+
+      const proposed = DiagnosticReportGenerator.extractProposedLocator(evidence, intelligence);
+      if (proposed) {
+        lines.push(`  Proposed Replacement Locator: \`${proposed}\``);
+      }
+    }
+
+    lines.push(divider);
+    lines.push('');
+
+    return lines.join('\n');
+  }
+
+  public static generateMarkdownReport(
+    evidence: CorrelatedEvidence,
+    scorecard?: EvidenceScorecard
+  ): string {
+    const { context, classification } = evidence;
+    const { failedAction } = context;
+
+    const md: string[] = [];
+    md.push('# Playwright Intelligence Engine — Diagnostic Report');
+    md.push('');
+    md.push(`**Classification:** \`${classification.category}\` (Confidence: ${(classification.confidence * 100).toFixed(1)}%)  `);
+    md.push(`**Failure fingerprint:** \`${evidence.failureFingerprint.key}\`  `);
+    md.push(`**Status:** ${classification.shouldAbstain ? '⛔ **Abstained from repair**' : '🔍 **Repair candidate eligible**'}  `);
+    md.push(`**Reason:** ${classification.reason}  `);
+    md.push('');
+
+    md.push('## Failed Action');
+    md.push(`- **Action:** \`${failedAction.action}\``);
+    if (failedAction.selector) md.push(`- **Selector:** \`${failedAction.selector}\``);
+    if (failedAction.sourceFile) md.push(`- **Location:** \`${failedAction.sourceFile}:${failedAction.line ?? 0}\``);
+    if (failedAction.errorMessage) md.push(`- **Error:** \`${failedAction.errorMessage}\``);
+    md.push('');
+
+    if (scorecard) {
+      md.push('## Evidence Scorecard');
+      md.push('| Evidence Channel | Available |');
+      md.push('|---|---|');
+      md.push(`| Playwright Error Message | ${scorecard.playwrightErrorAvailable ? '✅' : '❌'} |`);
+      md.push(`| Test Source Code (AST) | ${scorecard.testSourceAstAvailable ? '✅' : '❌'} |`);
+      md.push(`| Browser DOM Snapshot | ${scorecard.domSnapshotAvailable ? '✅' : '❌'} |`);
+      md.push(`| Network Interception | ${scorecard.networkEventsAvailable ? '✅' : '❌'} |`);
+      md.push(`| Browser Console | ${scorecard.consoleEventsAvailable ? '✅' : '❌'} |`);
+      md.push(`| Playwright Trace | ${scorecard.traceArchiveAvailable ? '✅' : '❌'} |`);
+      md.push(`| Git History / Diff | ${scorecard.gitHistoryAvailable ? '✅' : '❌'} |`);
+      md.push(`| Environment Metadata | ${scorecard.environmentMetadataAvailable ? '✅' : '❌'} |`);
+      md.push('');
+    }
+
+    if (classification.shouldAbstain) {
+      md.push('## Abstention Guardrail Enforced');
+      md.push(`> [!IMPORTANT]\n> ${classification.abstentionReason ?? classification.reason}`);
+    }
+
+    return md.join('\n');
+  }
+}

package/src/diagnostics/evidence-correlator.ts

@@ -0,0 +1,117 @@
+import {
+  DiagnosticContext,
+  NetworkEvent,
+  ConsoleEvent,
+} from '../core/types/diagnostic-context.js';
+import { FailureClassifier, ClassificationResult } from './failure-classifier.js';
+import { FailureFingerprint, FailureObservation, fingerprintFailure } from '../intelligence/failure-clustering.js';
+
+export interface CorrelatedEvidence {
+  context: DiagnosticContext;
+  classification: ClassificationResult;
+  failureObservation: FailureObservation;
+  failureFingerprint: FailureFingerprint;
+  suspiciousNetworkEvents: NetworkEvent[];
+  suspiciousConsoleEvents: ConsoleEvent[];
+  temporalWindowMs: number;
+}
+
+export class EvidenceCorrelator {
+  /**
+   * Correlates network and console anomalies within an action-anchored temporal window
+   * leading up to the failure (AUDIT-12).
+   * Filters events FIRST before classification so the classifier evaluates only
+   * causally relevant, frame-matched events.
+   */
+  public static correlate(
+    context: DiagnosticContext,
+    windowMs = 5000
+  ): CorrelatedEvidence {
+    // 1. Anchor reference time strictly to failedAction.timestamp if available
+    let refTime: number | undefined = context.failedAction.timestamp;
+
+    if (refTime === undefined) {
+      // Fallback: collect timestamps from network and console events
+      const allTimestamps: number[] = [];
+      for (const net of context.networkEvents) {
+        if (typeof net.timestamp === 'number') allTimestamps.push(net.timestamp);
+      }
+      for (const con of context.consoleEvents) {
+        if (typeof con.timestamp === 'number') allTimestamps.push(con.timestamp);
+      }
+      refTime = allTimestamps.length > 0 ? Math.max(...allTimestamps) : undefined;
+    }
+
+    const windowStart = refTime !== undefined ? refTime - windowMs : undefined;
+    const windowEnd = refTime !== undefined ? refTime + 1000 : undefined;
+
+    // 2. Filter network events that occurred within the action-anchored temporal window with error status codes (4xx, 5xx)
+    const suspiciousNetworkEvents = context.networkEvents.filter((event) => {
+      const isError = event.status >= 400 || Boolean(event.errorText);
+      if (!isError) return false;
+
+      if (windowStart !== undefined && windowEnd !== undefined && typeof event.timestamp === 'number') {
+        const sameScale =
+          (refTime! > 1e11 && event.timestamp > 1e11) ||
+          (refTime! < 1e11 && event.timestamp < 1e11);
+
+        if (sameScale) {
+          return event.timestamp >= windowStart && event.timestamp <= windowEnd;
+        }
+      }
+      return true;
+    });
+
+    // 3. If frameId is specified, sort matching frame events to the front
+    if (context.failedAction.frameId) {
+      const targetFrame = context.failedAction.frameId;
+      suspiciousNetworkEvents.sort((a, b) => {
+        const aMatch = a.frameId === targetFrame ? 1 : 0;
+        const bMatch = b.frameId === targetFrame ? 1 : 0;
+        return bMatch - aMatch;
+      });
+    }
+
+    // 4. Filter console errors within the action-anchored temporal window
+    const suspiciousConsoleEvents = context.consoleEvents.filter((event) => {
+      const isConsoleError = event.type === 'error' || event.text.toLowerCase().includes('uncaught');
+      if (!isConsoleError) return false;
+
+      if (windowStart !== undefined && windowEnd !== undefined && typeof event.timestamp === 'number') {
+        return event.timestamp >= windowStart && event.timestamp <= windowEnd;
+      }
+      return true;
+    });
+
+    // 5. Pipe filtered, frame-matched events directly to the FailureClassifier
+    const windowedContext: DiagnosticContext = {
+      ...context,
+      networkEvents: suspiciousNetworkEvents,
+      consoleEvents: suspiciousConsoleEvents,
+    };
+
+    const classification = FailureClassifier.classify(windowedContext);
+    const failureObservation: FailureObservation = {
+      id: context.failedAction.callId,
+      category: classification.category,
+      action: context.failedAction.action,
+      selector: context.failedAction.selector,
+      sourceFile: context.failedAction.sourceFile,
+      errorMessage: context.failedAction.errorMessage,
+      errorStack: context.failedAction.errorStack,
+      networkStatuses: suspiciousNetworkEvents.map((event) => event.status),
+      timestamp: context.failedAction.timestamp,
+      durationMs: context.failedAction.durationMs,
+    };
+
+    return {
+      context,
+      classification,
+      failureObservation,
+      failureFingerprint: fingerprintFailure(failureObservation),
+      suspiciousNetworkEvents,
+      suspiciousConsoleEvents,
+      temporalWindowMs: windowMs,
+    };
+  }
+}

package/src/diagnostics/external-failure-evidence.ts

@@ -0,0 +1,72 @@
+import { DiagnosticContext, FailedAction } from '../core/types/diagnostic-context.js';
+import { EvidenceRedactor } from '../evidence/evidence-redactor.js';
+
+export interface ExternalFailureEvidence {
+  errorMessage: string;
+  errorStack?: string | undefined;
+  action?: string | undefined;
+  selector?: string | undefined;
+  sourceFile?: string | undefined;
+  line?: number | undefined;
+  column?: number | undefined;
+}
+
+function extractSelector(text: string): string | undefined {
+  const match = text.match(/waiting for (?:locator|getByRole|getByText|getByTestId|getByLabel|getByPlaceholder)\((['"`])([\s\S]*?)\1\)/i)
+    ?? text.match(/locator\((['"`])([\s\S]*?)\1\)/i);
+  return match?.[2];
+}
+
+function extractAction(text: string): string | undefined {
+  const match = text.match(/\b(page|locator|frameLocator)\.(click|fill|press|check|uncheck|hover|selectOption|goto|waitFor|dblclick)\b/i);
+  return match ? `${match[1]}.${match[2]}` : undefined;
+}
+
+function extractLocation(text: string): Pick<ExternalFailureEvidence, 'sourceFile' | 'line' | 'column'> {
+  const lines = text.split(/\r?\n/);
+  for (const line of lines) {
+    const match = line.match(/(?:at\s+.*?\s+)?((?:[A-Za-z]:[\\/]|\/)?[^()\r\n]+?\.(?:ts|tsx|js|jsx):([0-9]+):([0-9]+))/i);
+    if (match?.[1] && match[2] && match[3]) {
+      return { sourceFile: match[1].trim(), line: Number(match[2]), column: Number(match[3]) };
+    }
+  }
+  return {};
+}
+
+/** Parses a Playwright error-context.md file without treating it as trusted source code. */
+export function parseExternalFailureEvidence(rawText: string): ExternalFailureEvidence {
+  const raw = rawText.trim();
+  const { redactedText: errorMessage } = EvidenceRedactor.redactText(raw);
+  const location = extractLocation(raw);
+  return {
+    errorMessage: errorMessage.slice(0, 20000),
+    errorStack: errorMessage.slice(0, 20000),
+    action: extractAction(raw),
+    selector: extractSelector(raw),
+    ...location,
+  };
+}
+
+/**
+ * Supplements a trace context only when the trace did not contain a usable
+ * failed action. Existing trace evidence remains authoritative.
+ */
+export function supplementDiagnosticContext(
+  context: DiagnosticContext,
+  external: ExternalFailureEvidence,
+): DiagnosticContext {
+  const isFallback = context.failedAction.callId === 'unknown' || context.failedAction.errorMessage === 'No explicit action failure recorded in trace';
+  if (!isFallback) return context;
+
+  const failedAction: FailedAction = {
+    ...context.failedAction,
+    ...(external.action ? { action: external.action } : {}),
+    ...(external.selector ? { selector: external.selector } : {}),
+    ...(external.sourceFile ? { sourceFile: external.sourceFile } : {}),
+    ...(external.line !== undefined ? { line: external.line } : {}),
+    ...(external.column !== undefined ? { column: external.column } : {}),
+    errorMessage: external.errorMessage,
+    ...(external.errorStack ? { errorStack: external.errorStack } : {}),
+  };
+  return { ...context, failedAction };
+}

package/src/diagnostics/failure-category.ts

@@ -0,0 +1,40 @@
+import { FailureCategory } from '../core/types/diagnostic-context.js';
+
+/**
+ * Converts historical, external, and compatibility labels into the
+ * canonical categories used by current diagnostics and reporting.
+ * Unknown labels are preserved in normalized uppercase form so external
+ * evaluation data remains auditable instead of being silently discarded.
+ */
+const CATEGORY_ALIASES: Record<string, string> = {
+  AUTH_SESSION_EXPIRED: 'AUTH_EXPIRED',
+  AUTHENTICATION_FAILURE: 'AUTH_EXPIRED',
+  AUTHENTICATION_EXPIRED: 'AUTH_EXPIRED',
+  AUTHENTICATION_ERROR: 'AUTH_EXPIRED',
+  ASSERTION_FAILED: 'ASSERTION_FAILURE',
+  ASSERTION_MISMATCH: 'ASSERTION_FAILURE',
+  ASSERTION_DISCREPANCY: 'ASSERTION_FAILURE',
+  BACKEND_ERROR: 'BACKEND_500',
+  BACKEND_SERVICE_OUTAGE: 'BACKEND_500',
+  BACKEND_OUTAGE: 'BACKEND_500',
+  AMBIGUOUS_LOCATOR: 'AMBIGUOUS_MATCH',
+  AMBIGUOUS_CONTROL: 'AMBIGUOUS_MATCH',
+  DISABLED_CONTROL: 'DISABLED_ELEMENT',
+  DYNAMIC_CONTENT_SLOW: 'HYDRATION_OR_TIMING',
+  HYDRATION_DELAY: 'HYDRATION_OR_TIMING',
+  NETWORK_ERROR: 'NETWORK_OFFLINE',
+  UNRELATED_NETWORK_ERROR: 'NETWORK_OFFLINE',
+  TIMEOUT: 'TIMING_RACE',
+  UNKNOWN: 'UNKNOWN_ERROR',
+};
+
+export function normalizeFailureCategory(category: string | undefined): string {
+  const normalized = (category ?? '').trim().toUpperCase().replace(/[\s-]+/g, '_');
+  if (!normalized) return 'UNKNOWN_ERROR';
+  return CATEGORY_ALIASES[normalized] ?? normalized;
+}
+
+export function normalizeTypedFailureCategory(category: FailureCategory): FailureCategory {
+  return normalizeFailureCategory(category) as FailureCategory;
+}
+