quality-intelligence-engine 2.2.3 → 2.2.4

package/src/pipeline/failure-grouping.pipeline.ts

@@ -0,0 +1,67 @@
+// src/pipeline/failure-grouping.pipeline.ts
+
+import { FailureAnalysisResult } from "../types/failure.types";
+import { saveFailureHistory, loadFailureHistory } from "../history/failure-history.store";
+import { analyzeFailureTrends } from "../history/failure-trend.analyzer";
+
+export interface FailureGroupSummary {
+  dominantCategory: string;
+  groupedCounts: Record<string, number>;
+  confidence: number;
+  flakyCount: number;
+  trend?: ReturnType<typeof analyzeFailureTrends>;
+}
+
+export function groupFailures(
+  results: FailureAnalysisResult[]
+): FailureGroupSummary {
+  const groupedCounts: Record<string, number> = {};
+  let flakyCount = 0;
+
+  for (const r of results) {
+    const key = r.classification ?? "UNKNOWN";
+    groupedCounts[key] = (groupedCounts[key] || 0) + 1;
+
+    if (key === "FLAKY") {
+      flakyCount++;
+    }
+  }
+
+  let dominantCategory = "UNKNOWN";
+  let dominantCount = 0;
+
+  for (const [category, count] of Object.entries(groupedCounts)) {
+    if (count > dominantCount) {
+      dominantCategory = category;
+      dominantCount = count;
+    }
+  }
+
+  const total = results.length || 1;
+  let confidence = dominantCount / total;
+
+  // 🔻 FLAKY PENALTY (KEY LOGIC)
+  if (flakyCount > 0) {
+    const penalty = Math.min(0.3, flakyCount / total);
+    confidence = Math.max(0, confidence - penalty);
+  }
+
+  const summary: FailureGroupSummary = {
+    dominantCategory,
+    groupedCounts,
+    confidence,
+    flakyCount
+  };
+
+  saveFailureHistory({
+    runId: Date.now().toString(),
+    timestamp: new Date().toISOString(),
+    dominantCategory,
+    groupedCounts,
+    confidence
+  });
+
+  summary.trend = analyzeFailureTrends(loadFailureHistory());
+
+  return summary;
+}

package/src/playwright/qi.reporter.ts

@@ -0,0 +1,26 @@
+import type {
+  Reporter,
+  TestCase,
+  TestResult
+} from "@playwright/test/reporter";
+
+import { trackFailure } from "../intelligence/failure-fingerprinting/failure.tracker";
+
+export default class QIReporter implements Reporter {
+  onTestEnd(test: TestCase, result: TestResult) {
+    if (result.status !== "failed") return;
+
+    const errorText =
+      result.error?.message ||
+      result.error?.stack ||
+      "unknown error";
+
+    trackFailure(
+      {
+        error: errorText,
+        testName: test.title
+      },
+      process.env.QI_RUN_ID ?? "local-run"
+    );
+  }
+}

package/src/reporter.ts

@@ -0,0 +1,88 @@
+import { AnalysisResult, PassRisk } from './types';
+import { writeAnalysisMarkdown } from './markdownWriter';
+import { RunContext } from './runManager';
+
+/**
+ * FAILURES: Print summary + write markdown reports
+ */
+export function printSummaryAndWriteReports(
+  grouped: Map<string, AnalysisResult[]>,
+  run: RunContext,
+  config: any
+) {
+  let printed = 0;
+
+  grouped.forEach((items, key) => {
+    const sample = items[0];
+
+    // Respect confidence threshold
+    if (
+      sample.confidence <
+      config.engine.confidenceThresholds.fail
+    ) {
+      return;
+    }
+
+    if (printed === 0) {
+      console.log(
+        `❌ Failures Detected (${grouped.size} unique issues)\n`
+      );
+    }
+
+    printed++;
+
+    console.log(
+      `${printed}) ${sample.category} (${items.length} occurrences)`
+    );
+    console.log(
+      `   Location : ${sample.location.file}:${sample.location.line}`
+    );
+    console.log(
+      `   API      : ${sample.failedApi.endpoint} → ${sample.failedApi.status}`
+    );
+    console.log(
+      `   Cause    : ${sample.rootCause}`
+    );
+    console.log(
+      `   Details  : output/${run.runDir.split('/').pop()}/${key}/analysis.md\n`
+    );
+
+    writeAnalysisMarkdown(
+      run.runDir,
+      key,
+      sample,
+      items.length
+    );
+  });
+
+  if (printed === 0) {
+    console.log('✅ No failures exceeded confidence threshold\n');
+  }
+}
+
+/**
+ * PASSES: Print non-blocking risk warnings
+ */
+export function printPassRisks(
+  risks: PassRisk[],
+  _run: RunContext
+) {
+  if (risks.length === 0) return;
+
+  console.log(`⚠️ Pass Risks Detected (${risks.length})\n`);
+
+  risks.forEach((risk, i) => {
+    console.log(
+      `${i + 1}) ${risk.test} → ${risk.riskType}`
+    );
+    console.log(
+      `   Location : ${risk.location.file}`
+    );
+    console.log(
+      `   Reason   : ${risk.reason}`
+    );
+    console.log(
+      `   Fix      : ${risk.fix}\n`
+    );
+  });
+}

package/src/reporters/qi-reporter.js

@@ -0,0 +1,34 @@
+const fs = require('fs');
+const path = require('path');
+
+class QiReporter {
+  onTestEnd(test, result) {
+    if (result.status !== 'failed') return;
+
+    const message = result.error && result.error.message;
+    if (!message) return;
+
+    const isNumericMismatch =
+      /Expected:\s*\d+/i.test(message) &&
+      /Received:\s*\d+/i.test(message);
+
+    if (!isNumericMismatch) return;
+
+    // ✅ Playwright GUARANTEED directory
+    const outputDir = result.outputDir;
+
+    fs.writeFileSync(
+      path.join(outputDir, 'failure-hint.json'),
+      JSON.stringify(
+        {
+          type: 'API_BUG',
+          reason: 'Numeric data mismatch inferred from UI assertion'
+        },
+        null,
+        2
+      )
+    );
+  }
+}
+
+module.exports = QiReporter;

package/src/rules.ts

@@ -0,0 +1,35 @@
+import { AnalysisResult } from './types';
+
+export function generateDummyResult(): AnalysisResult {
+  return {
+    category: 'BACKEND_API_FAILURE',
+    confidence: 0.93,
+    location: {
+      file: 'AuthController.java',
+      line: 87
+    },
+    failedApi: {
+      method: 'GET',
+      endpoint: '/users/{id}',
+      status: 404,
+      response: 'User not found'
+    },
+    rootCause:
+      'The requested user ID does not exist in the database.',
+    retryGuidance:
+      'Automated CI retry is not advised (high confidence application failure).',
+    reproductionSteps: [
+      'Start backend service',
+      'Run: curl GET /users/{id}',
+      'Observe: 404 Not Found'
+    ],
+    expected:
+      'User details are returned successfully.',
+    actual:
+      'API returns 404 and request fails in AuthController.java:87',
+    suggestedActions: [
+      'Validate user existence before API call',
+      'Add graceful handling for missing user records'
+    ]
+  };
+}

package/src/runManager.ts

@@ -0,0 +1,21 @@
+import fs from 'fs';
+import path from 'path';
+
+export interface RunContext {
+  runId: string;
+  runDir: string;
+}
+
+export function createRunContext(): RunContext {
+  const now = new Date();
+  const runId = now
+    .toISOString()
+    .replace(/[:.]/g, '-')
+    .replace('T', '_')
+    .replace('Z', '');
+
+  const runDir = path.join(process.cwd(), 'output', `run-${runId}`);
+  fs.mkdirSync(runDir, { recursive: true });
+
+  return { runId, runDir };
+}

package/src/runner.ts

@@ -0,0 +1,12 @@
+// src/runner.ts
+
+import { runFailureAnalysisFromPlaywright } from "./pipeline/failure-analysis.pipeline";
+import { groupFailures } from "./pipeline/failure-grouping.pipeline";
+import { enrichWithIntelligence } from "./intelligence/intelligence.pipeline";
+
+export async function runQualityIntelligence(input: string) {
+  const analysisResults = await runFailureAnalysisFromPlaywright(input);
+  const summary = groupFailures(analysisResults);
+  const enriched = await enrichWithIntelligence(summary);
+  return enriched;
+}

package/src/runtime/networkCollector.ts

@@ -0,0 +1,36 @@
+import { Page } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+
+export function attachNetworkCollector(page: Page, testId: string) {
+  const failures: any[] = [];
+
+  page.on('response', response => {
+    if (response.status() >= 400) {
+      failures.push({
+        method: response.request().method(),
+        url: new URL(response.url()).pathname,
+        status: response.status()
+      });
+    }
+  });
+
+  page.on('requestfailed', request => {
+    failures.push({
+      method: request.method(),
+      url: new URL(request.url()).pathname,
+      error: request.failure()?.errorText
+    });
+  });
+
+  process.on('exit', () => {
+    if (failures.length === 0) return;
+
+    const dir = path.join('test-results', testId);
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(
+      path.join(dir, 'network.json'),
+      JSON.stringify(failures, null, 2)
+    );
+  });
+}

package/src/stackParser.ts

@@ -0,0 +1,22 @@
+export interface ParsedLocation {
+  file: string;
+  line: number;
+}
+
+export function parseStackTrace(stack: string[]): ParsedLocation {
+  if (!stack || stack.length === 0) {
+    return { file: 'Unknown', line: -1 };
+  }
+
+  // Supports formats like: Class.method(File.java:87)
+  const match = stack[0].match(/\((.*):(\d+)\)/);
+
+  if (!match) {
+    return { file: 'Unknown', line: -1 };
+  }
+
+  return {
+    file: match[1],
+    line: Number(match[2])
+  };
+}

package/src/test-run.ts

@@ -0,0 +1,59 @@
+// src/test-run.ts
+
+import { runQualityIntelligence } from "./runner";
+
+function line(char = "=", count = 60) {
+  console.log(char.repeat(count));
+}
+
+async function main() {
+  const playwrightResultsDir = "playwright-results";
+
+  const result = await runQualityIntelligence(playwrightResultsDir);
+
+  line();
+  console.log("QUALITY INTELLIGENCE – FINAL OUTPUT");
+  line();
+
+  console.log(`Primary Failure   : ${result.dominantCategory}`);
+  console.log(`Confidence Score  : ${result.confidence}`);
+  console.log(`Flaky Failures    : ${result.flakyCount}`);
+  console.log(`Trend Signal      : ${result.trend?.signal ?? "N/A"}`);
+
+  line("-");
+  console.log("Failure Breakdown:");
+  for (const [k, v] of Object.entries(result.groupedCounts)) {
+    console.log(`  - ${k}: ${v}`);
+  }
+
+  line("-");
+  console.log("Confidence Diagnosis:");
+  for (const lineText of result.intelligence.confidence.explanation) {
+    console.log(`  • ${lineText}`);
+  }
+
+  line("-");
+  console.log("AI Explanation:");
+  console.log(`  Summary: ${result.intelligence.explanation.summary}`);
+
+  if (result.intelligence.explanation.reasoning?.length) {
+    console.log("  Reasoning:");
+    for (const r of result.intelligence.explanation.reasoning) {
+      console.log(`    - ${r}`);
+    }
+  }
+
+  if (result.intelligence.explanation.recommendedAction?.length) {
+    console.log("  Recommended Actions:");
+    for (const a of result.intelligence.explanation.recommendedAction) {
+      console.log(`    - ${a}`);
+    }
+  }
+
+  line();
+}
+
+main().catch(err => {
+  console.error("QUALITY INTELLIGENCE FAILED");
+  console.error(err);
+});

package/src/types/analysis-result.ts

@@ -0,0 +1,16 @@
+export type FailureType = 'TEST_BUG' | 'UI_BUG' | 'API_BUG'
+
+export interface AnalysisFailure {
+  testName: string
+  failureType: FailureType
+  confidence: number
+
+  // Location
+  file?: string
+  line?: number
+
+  // Intelligence sections
+  rootCause?: string[]
+  evidence?: string[]
+  diagnosis?: string[]
+}

package/src/types/failure.types.ts

@@ -0,0 +1,28 @@
+// src/types/failure.types.ts
+
+/**
+ * Shared failure contract
+ * Used by:
+ * - failure-analysis pipeline
+ * - grouping / history
+ * - CLI reporter
+ *
+ * Fields are OPTIONAL where produced later in the pipeline.
+ */
+export interface FailureAnalysisResult {
+  /** Core classification */
+  classification: string;
+
+  /** Optional human message */
+  message?: string;
+
+  /** CLI / reporting fields */
+  testName?: string;
+  failureType?: string;
+  confidence?: number;
+
+  /** Diagnosis-style outputs */
+  rootCause?: string[];
+  evidence?: string[];
+  diagnosis?: string[];
+}

package/src/types/playwright-failure.ts

@@ -0,0 +1,10 @@
+export type PlaywrightFailure = {
+  testName: string
+  errorMessage: string
+  stack?: string
+
+  filePath: string
+  lineNumber: number
+
+  retryCount: number
+}

package/src/types.ts

@@ -0,0 +1,102 @@
+// =====================
+// RAW INPUT TYPES
+// =====================
+
+export interface RawApiCall {
+  method: string;
+  endpoint: string;
+  status: number;
+  response?: string;
+}
+
+export interface RawFailureInput {
+  errorType: 'API_ERROR' | 'TEST_ERROR' | 'ENVIRONMENT_ERROR';
+  api?: RawApiCall;
+  message: string;
+  stackTrace: string;
+}
+
+export interface RawPassInput {
+  testName: string;
+  duration: number;
+  retries?: number;
+  apiCalls?: RawApiCall[];
+}
+
+// =====================
+// FAILURE TYPES
+// =====================
+
+export type FailureCategory =
+  | 'BACKEND_API_FAILURE'
+  | 'TEST_FAILURE'
+  | 'FLAKY_TEST'
+  | 'ENVIRONMENT_ISSUE';
+
+export interface FailureLocation {
+  file: string;
+  line: number;
+}
+
+export interface FailedApiCall {
+  method: string;
+  endpoint: string;
+  status: number;
+  response?: string;
+}
+
+export interface AnalysisResult {
+  category: FailureCategory;
+  confidence: number;
+  location: FailureLocation;
+  failedApi: FailedApiCall;
+  rootCause: string;
+  retryGuidance: string;
+  reproductionSteps: string[];
+  expected: string;
+  actual: string;
+  suggestedActions: string[];
+}
+
+// =====================
+// PASS RISK TYPES
+// =====================
+
+export type PassRiskType =
+  | 'FLAKY_PASS'
+  | 'API_WARNING'
+  | 'PERFORMANCE_DRIFT';
+
+export interface PassRisk {
+  test: string;
+  riskType: PassRiskType;
+  location: {
+    file: string;
+    line: number;
+  };
+  reason: string;
+  fix: string;
+}
+
+// =====================
+// CONFIGURATION TYPES
+// =====================
+
+export interface QIConfig {
+  engine: {
+    mode: 'standard' | 'strict' | 'lenient';
+    confidenceThresholds: {
+      fail: number;      // 0-1
+      passRisk: number;  // 0-1
+    };
+  };
+  output: {
+    writeMarkdown: boolean;
+    maxHistoryRuns: number;
+  };
+  passAnalysis: {
+    enable: boolean;
+    reportFlakyPass: boolean;
+    reportApiWarnings: boolean;
+  };
+}

package/src/utils/artifact.locator.ts

@@ -0,0 +1,42 @@
+import fs from 'fs'
+import path from 'path'
+
+type Artifacts = {
+  trace?: string
+  screenshots: string[]
+}
+
+export function locatePlaywrightArtifacts(): Artifacts {
+  const resultsDir = path.resolve('test-results')
+  const screenshots: string[] = []
+  let trace: string | undefined
+
+  if (!fs.existsSync(resultsDir)) {
+    return { screenshots, trace }
+  }
+
+  const runDirs = fs.readdirSync(resultsDir)
+
+  for (const dir of runDirs) {
+    const fullDir = path.join(resultsDir, dir)
+    if (!fs.statSync(fullDir).isDirectory()) continue
+
+    const files = fs.readdirSync(fullDir)
+
+    for (const file of files) {
+      const fullPath = path.join(fullDir, file)
+
+      // 🎥 Trace ZIP (take the latest retry if exists)
+      if (file === 'trace.zip') {
+        trace = fullPath
+      }
+
+      // 📸 Screenshots
+      if (file.endsWith('.png')) {
+        screenshots.push(fullPath)
+      }
+    }
+  }
+
+  return { trace, screenshots }
+}