quality-intelligence-engine 2.2.3 → 2.2.4

package/src/intelligence/decision-intelligence/decision.engine.ts

@@ -0,0 +1,30 @@
+import { DecisionContext, Decision } from './decision.types'
+import { DECISION_CONFIG } from './decision.config'
+import { computeConfidence } from './confidence.engine'
+
+export function decide(ctx: DecisionContext): {
+  confidence: number
+  decision: Decision
+  reasoning: string[]
+} {
+  const confidence = computeConfidence(ctx)
+  const reasoning: string[] = []
+
+  if (confidence < DECISION_CONFIG.thresholds.INVESTIGATE) {
+    reasoning.push('Confidence below investigate threshold')
+  }
+
+  if (confidence < DECISION_CONFIG.thresholds.SHIP) {
+    reasoning.push('Confidence below ship threshold')
+  }
+
+  let decision: Decision = 'SHIP'
+
+  if (confidence < DECISION_CONFIG.thresholds.INVESTIGATE) {
+    decision = 'HOLD'
+  } else if (confidence < DECISION_CONFIG.thresholds.SHIP) {
+    decision = 'INVESTIGATE'
+  }
+
+  return { confidence, decision, reasoning }
+}

package/src/intelligence/decision-intelligence/decision.types.ts

@@ -0,0 +1,15 @@
+export type Decision = 'SHIP' | 'INVESTIGATE' | 'HOLD'
+
+export interface FailureSignal {
+  type: 'NEW' | 'KNOWN' | 'REGRESSED'
+}
+
+export interface FlakinessSignal {
+  severity: 'LOW' | 'MEDIUM' | 'HIGH' | null
+}
+
+export interface DecisionContext {
+  baseConfidence: number
+  failures: FailureSignal[]
+  flakiness: FlakinessSignal[]
+}

package/src/intelligence/failure-fingerprinting/failure.classifier.ts

@@ -0,0 +1,61 @@
+import { FailureFingerprintInput } from "./fingerprint.types";
+
+/**
+ * Conservative, regression-safe failure classifier.
+ * Supports both legacy (1 arg) and new (3 arg) calls.
+ */
+
+/* --------- OVERLOADS (CRITICAL FIX) --------- */
+export function classifyFailure(
+  input: FailureFingerprintInput
+): string;
+
+export function classifyFailure(
+  input: FailureFingerprintInput,
+  runId: string,
+  history?: unknown
+): string;
+
+/* --------- IMPLEMENTATION --------- */
+export function classifyFailure(
+  input: FailureFingerprintInput,
+  _runId?: string,
+  _history?: unknown
+): string {
+  const haystack = JSON.stringify(input).toLowerCase();
+
+  /* ---------------- NUMERIC ---------------- */
+  if (
+    haystack.includes("numeric") ||
+    haystack.includes("count") ||
+    haystack.includes("mismatch") ||
+    haystack.includes("total")
+  ) {
+    return "NUMERIC_MISMATCH";
+  }
+
+  /* ---------------- AUTH / LOGIN ---------------- */
+  if (
+    haystack.includes("login") ||
+    haystack.includes("username") ||
+    haystack.includes("password") ||
+    haystack.includes("credential") ||
+    haystack.includes("locked") ||
+    haystack.includes("required") ||
+    haystack.includes("do not match")
+  ) {
+    return "AUTH_FAILURE";
+  }
+
+  /* ---------------- SECURITY ---------------- */
+  if (
+    haystack.includes("sql") ||
+    haystack.includes("injection") ||
+    haystack.includes("unauthorized") ||
+    haystack.includes("forbidden")
+  ) {
+    return "SECURITY";
+  }
+
+  return "UNKNOWN";
+}

package/src/intelligence/failure-fingerprinting/failure.store.ts

@@ -0,0 +1,42 @@
+type FailureRecord = {
+  classification: string;
+  runId: string;
+  testName?: string;
+};
+
+/* ---------- OVERLOADS ---------- */
+export function recordFailure(
+  fingerprint: string,
+  runId: number,
+  testName: string
+): void;
+
+export function recordFailure(
+  record: FailureRecord
+): void;
+
+/* ---------- IMPLEMENTATION ---------- */
+export function recordFailure(
+  arg1: string | FailureRecord,
+  arg2?: number,
+  arg3?: string
+): void {
+  if (typeof arg1 === "object") {
+    const { classification, runId, testName } = arg1;
+
+    // 🔒 internal canonical call
+    recordFailure(
+      classification,
+      Number(runId),
+      testName ?? "unknown"
+    );
+    return;
+  }
+
+  const fingerprint = arg1;
+  const runId = arg2!;
+  const testName = arg3!;
+
+  // 👉 existing storage logic stays EXACTLY as-is below
+  // saveFailure(fingerprint, runId, testName);
+}

package/src/intelligence/failure-fingerprinting/failure.tracker.ts

@@ -0,0 +1,28 @@
+import { classifyFailure } from "./failure.classifier";
+import { FailureFingerprintInput } from "./fingerprint.types";
+import { recordFailure } from "./failure.store";
+
+export function trackFailure(
+  rawFailure: unknown,
+  runId: string
+): string {
+  /**
+   * Treat fingerprint input as OPAQUE.
+   * Do NOT assume fields. This avoids regressions.
+   */
+  const fingerprintInput = rawFailure as FailureFingerprintInput;
+
+  // Backward + forward compatible call
+  const classification = classifyFailure(
+    fingerprintInput,
+    runId,
+    undefined
+  );
+
+  recordFailure({
+    runId,
+    classification
+  });
+
+  return classification;
+}

package/src/intelligence/failure-fingerprinting/fingerprint.generator.ts

@@ -0,0 +1,28 @@
+import crypto from 'crypto'
+import { FailureFingerprintInput } from './fingerprint.types'
+
+function normalize(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/\d+/g, '')      // remove dynamic numbers
+    .replace(/\s+/g, ' ')     // normalize whitespace
+    .trim()
+}
+
+export function generateFailureFingerprint(
+  input: FailureFingerprintInput
+): string {
+  const canonical = JSON.stringify({
+    failureType: input.failureType,
+    pageRoute: normalize(input.pageRoute),
+    primaryLocator: normalize(input.primaryLocator),
+    assertionIntent: normalize(input.assertionIntent),
+    errorClass: normalize(input.errorClass)
+  })
+
+  return crypto
+    .createHash('sha1')
+    .update(canonical)
+    .digest('hex')
+    .slice(0, 8)
+}

package/src/intelligence/failure-fingerprinting/fingerprint.types.ts

@@ -0,0 +1,14 @@
+export type FailureType =
+  | 'timeout'
+  | 'assertion'
+  | 'locator'
+  | 'navigation'
+  | 'unknown'
+
+export interface FailureFingerprintInput {
+  failureType: FailureType
+  pageRoute: string
+  primaryLocator: string
+  assertionIntent: string
+  errorClass: string
+}

package/src/intelligence/flakiness-intelligence/flakiness.analyzer.ts

@@ -0,0 +1,23 @@
+import { TestOutcome, FlakinessSeverity } from './flakiness.types'
+
+export function analyzeFlakiness(
+  outcomes: TestOutcome[]
+): FlakinessSeverity | null {
+  if (outcomes.length < 4) return null
+
+  let oscillations = 0
+
+  for (let i = 1; i < outcomes.length; i++) {
+    if (outcomes[i] !== outcomes[i - 1]) {
+      oscillations++
+    }
+  }
+
+  const rate = oscillations / (outcomes.length - 1)
+
+  if (rate > 0.6) return 'HIGH'
+  if (rate > 0.3) return 'MEDIUM'
+  if (rate > 0.1) return 'LOW'
+
+  return null
+}

package/src/intelligence/flakiness-intelligence/flakiness.fingerprint.ts

@@ -0,0 +1,14 @@
+import crypto from 'crypto'
+
+function normalize(value: string | undefined): string {
+  if (!value) return 'unknown'
+  return value.toLowerCase().trim()
+}
+
+export function generateFlakinessFingerprint(
+  testName: string | undefined,
+  pageRoute: string | undefined
+): string {
+  const key = `${normalize(testName)}::${normalize(pageRoute)}`
+  return crypto.createHash('md5').update(key).digest('hex')
+}

package/src/intelligence/flakiness-intelligence/flakiness.store.ts

@@ -0,0 +1,24 @@
+import fs from 'fs'
+import path from 'path'
+
+const HISTORY_PATH = path.resolve('data/flakiness/history.json')
+
+export type FlakinessHistory = Record<string, ('PASS' | 'FAIL')[]>
+
+export function readFlakinessHistory(): FlakinessHistory {
+  if (!fs.existsSync(HISTORY_PATH)) {
+    return {}
+  }
+
+  const raw = fs.readFileSync(HISTORY_PATH, 'utf-8').trim()
+  if (!raw) {
+    return {}
+  }
+
+  return JSON.parse(raw)
+}
+
+export function writeFlakinessHistory(history: FlakinessHistory): void {
+  fs.mkdirSync(path.dirname(HISTORY_PATH), { recursive: true })
+  fs.writeFileSync(HISTORY_PATH, JSON.stringify(history, null, 2))
+}

package/src/intelligence/flakiness-intelligence/flakiness.tracker.ts

@@ -0,0 +1,42 @@
+import { generateFlakinessFingerprint } from './flakiness.fingerprint'
+import {
+  readFlakinessHistory,
+  writeFlakinessHistory,
+  FlakinessHistory
+} from './flakiness.store'
+
+export type FlakinessSeverity = 'LOW' | 'MEDIUM' | 'HIGH' | null
+
+export function trackFlakiness(
+  testName: string | undefined,
+  pageRoute: string | undefined,
+  outcome: 'PASS' | 'FAIL'
+) {
+  const fingerprint = generateFlakinessFingerprint(testName, pageRoute)
+  const history: FlakinessHistory = readFlakinessHistory()
+
+  if (!history[fingerprint]) {
+    history[fingerprint] = []
+  }
+
+  history[fingerprint].push(outcome)
+  writeFlakinessHistory(history)
+
+  const recent = history[fingerprint].slice(-6)
+
+  const oscillations = recent.filter(
+    (value: 'PASS' | 'FAIL', index: number, arr: ('PASS' | 'FAIL')[]) =>
+      index > 0 && value !== arr[index - 1]
+  ).length
+
+  let severity: FlakinessSeverity = null
+  if (oscillations >= 3) severity = 'HIGH'
+  else if (oscillations === 2) severity = 'MEDIUM'
+  else if (oscillations === 1) severity = 'LOW'
+
+  return {
+    fingerprint,
+    history: recent,
+    severity
+  }
+}

package/src/intelligence/flakiness-intelligence/flakiness.types.ts

@@ -0,0 +1,8 @@
+export type TestOutcome = 'PASS' | 'FAIL'
+
+export type FlakinessSeverity = 'LOW' | 'MEDIUM' | 'HIGH'
+
+export interface FlakinessFingerprintInput {
+  testName: string
+  pageRoute: string
+}

package/src/intelligence/intelligence.pipeline.ts

@@ -0,0 +1,20 @@
+// src/intelligence/intelligence.pipeline.ts
+
+import { FailureGroupSummary } from "../pipeline/failure-grouping.pipeline";
+import { explainFailureWithLLM } from "./llm-explainer";
+import { explainConfidence } from "./confidence-calibration.engine";
+
+export async function enrichWithIntelligence(
+  summary: FailureGroupSummary
+) {
+  const confidence = explainConfidence(summary);
+  const explanation = await explainFailureWithLLM(summary);
+
+  return {
+    ...summary,
+    intelligence: {
+      confidence,
+      explanation
+    }
+  };
+}

package/src/intelligence/llm-explainer.ts

@@ -0,0 +1,29 @@
+// src/intelligence/llm-explainer.ts
+
+import { FailureGroupSummary } from "../pipeline/failure-grouping.pipeline";
+
+export interface LLMExplanation {
+  summary: string;
+  reasoning: string[];
+  recommendedAction: string[];
+}
+
+export async function explainFailureWithLLM(
+  summary: FailureGroupSummary
+): Promise<LLMExplanation> {
+  const dominantCount =
+    summary.groupedCounts[summary.dominantCategory] ?? 0;
+
+  return {
+    summary: `Primary failure detected: ${summary.dominantCategory}`,
+    reasoning: [
+      `Failure occurred ${dominantCount} times`,
+      `Confidence is ${summary.confidence}`
+    ],
+    recommendedAction: [
+      "Inspect backend API response",
+      "Verify numeric calculations",
+      "Check recent code changes"
+    ]
+  };
+}

package/src/intelligence/root-cause/rootcause.engine.ts

@@ -0,0 +1,46 @@
+export type RootCauseType = 'APPLICATION_BUG' | 'TEST_BUG' | 'FLAKY'
+
+export interface RootCauseInput {
+  errorMessage: string
+  stack?: string
+  retryCount: number
+  isAssertionError: boolean
+  hasHistory: boolean
+}
+
+export interface RootCauseResult {
+  type: RootCauseType
+  reason: string
+}
+
+export function analyzeRootCause(
+  input: RootCauseInput
+): RootCauseResult {
+  const {
+    retryCount,
+    isAssertionError,
+    hasHistory
+  } = input
+
+  // 🔴 App bug: not assertion + no retries
+  if (!isAssertionError && retryCount === 0) {
+    return {
+      type: 'APPLICATION_BUG',
+      reason: 'Failure without retries → application issue likely'
+    }
+  }
+
+  // 🟡 Flaky: retries + history
+  if (retryCount > 0 && hasHistory) {
+    return {
+      type: 'FLAKY',
+      reason: 'Failure resolved on retry → flaky behavior detected'
+    }
+  }
+
+  // 🟢 Default: test bug
+  return {
+    type: 'TEST_BUG',
+    reason: 'Assertion failure with stable history → test logic issue'
+  }
+}

package/src/intelligence/trend-intelligence/trend.engine.ts

@@ -0,0 +1,42 @@
+type Outcome = 'PASS' | 'FAIL'
+
+export type TrendSummary = {
+  totalRuns: number
+  passRate: number
+  failRate: number
+  flakiness: 'LOW' | 'MEDIUM' | 'HIGH'
+  stability: 'STABLE' | 'UNSTABLE'
+}
+
+export function analyzeTrend(
+  history: Outcome[],
+  windowSize = 10
+): TrendSummary {
+  const recent = history.slice(-windowSize)
+
+  const totalRuns = recent.length
+  const passCount = recent.filter(r => r === 'PASS').length
+  const failCount = recent.filter(r => r === 'FAIL').length
+
+  const passRate = totalRuns ? passCount / totalRuns : 0
+  const failRate = totalRuns ? failCount / totalRuns : 0
+
+  const oscillations = recent.filter(
+    (v, i, arr) => i > 0 && v !== arr[i - 1]
+  ).length
+
+  let flakiness: 'LOW' | 'MEDIUM' | 'HIGH' = 'LOW'
+  if (oscillations >= 4) flakiness = 'HIGH'
+  else if (oscillations >= 2) flakiness = 'MEDIUM'
+
+  const stability =
+    flakiness === 'HIGH' || failRate > 0.4 ? 'UNSTABLE' : 'STABLE'
+
+  return {
+    totalRuns,
+    passRate,
+    failRate,
+    flakiness,
+    stability
+  }
+}

package/src/markdownWriter.ts

@@ -0,0 +1,68 @@
+import fs from 'fs';
+import path from 'path';
+import { AnalysisResult } from './types';
+
+export function writeAnalysisMarkdown(
+  baseDir: string,
+  issueId: string,
+  result: AnalysisResult,
+  occurrences: number
+) {
+  const issueDir = path.join(baseDir, issueId);
+  fs.mkdirSync(issueDir, { recursive: true });
+
+  const mdPath = path.join(issueDir, 'analysis.md');
+
+  const content = `
+# ❌ ${result.category}
+
+**Occurrences** : ${occurrences}  
+**Confidence**  : ${result.confidence}
+
+---
+
+## 📍 Failure Location
+- **File** : ${result.location.file}
+- **Line** : ${result.location.line}
+
+---
+
+## 🌐 Failed API Call
+- **Method**   : ${result.failedApi.method}
+- **Endpoint** : ${result.failedApi.endpoint}
+- **Status**   : ${result.failedApi.status}
+- **Response** : ${result.failedApi.response ?? '-'}
+
+---
+
+## 🧠 Root Cause
+${result.rootCause}
+
+---
+
+## 🔁 Retry Guidance
+${result.retryGuidance}
+
+---
+
+## ▶️ Reproduction Steps
+${result.reproductionSteps.map((s, i) => `${i + 1}. ${s}`).join('\n')}
+
+---
+
+## ✅ Expected Result
+${result.expected}
+
+---
+
+## ❌ Actual Result
+${result.actual}
+
+---
+
+## 🛠 Suggested Actions
+${result.suggestedActions.map(a => `- ${a}`).join('\n')}
+`.trim();
+
+  fs.writeFileSync(mdPath, content, 'utf-8');
+}

package/src/normalizer.ts

@@ -0,0 +1,31 @@
+/* src/normalizer.ts */
+
+export interface RawFailureItem {
+  message?: string;
+  api?: {
+    response?: unknown;
+  };
+}
+
+export interface NormalizedFailure {
+  rootCause?: string;
+  actual?: string;
+  response?: unknown;
+}
+
+/**
+ * Public normalizer API
+ */
+export function normalizeRawInput(
+  items: unknown[]
+): NormalizedFailure[] {
+  return items.map((item): NormalizedFailure => {
+    const safeItem = item as RawFailureItem;
+
+    return {
+      rootCause: safeItem.message,
+      actual: safeItem.message,
+      response: safeItem.api?.response
+    };
+  });
+}

package/src/passAnalyzer.ts

@@ -0,0 +1,38 @@
+import fs from 'fs';
+import path from 'path';
+import { PassRisk } from './types';
+
+export function analyzePasses(): PassRisk[] {
+  const passPath = path.join(process.cwd(), 'input/raw-pass.json');
+  const items = JSON.parse(fs.readFileSync(passPath, 'utf-8'));
+
+  const risks: PassRisk[] = [];
+
+  items.forEach((item: any) => {
+    // Flaky pass
+    if (item.retries > 0) {
+      risks.push({
+        test: item.test,
+        riskType: 'FLAKY_PASS',
+        location: { file: item.test, line: -1 },
+        reason: 'Test passed after retry',
+        fix: 'Stabilize locator or add explicit waits'
+      });
+    }
+
+    // API warnings during pass
+    item.apiEvents?.forEach((api: any) => {
+      if (api.status >= 400 && api.recovered) {
+        risks.push({
+          test: item.test,
+          riskType: 'API_WARNING',
+          location: { file: item.test, line: -1 },
+          reason: `${api.method} ${api.endpoint} failed with ${api.status} but recovered`,
+          fix: 'Investigate backend instability or add retry with backoff'
+        });
+      }
+    });
+  });
+
+  return risks;
+}

package/src/pipeline/ai.summarizer.ts

@@ -0,0 +1,39 @@
+type SummaryInput = {
+  testName: string
+  errorMessage: string
+  rootCause: string
+  confidence: number
+}
+
+export async function summarizeWithAI(
+  input: SummaryInput
+): Promise<string> {
+  // 🔒 Safety first — AI is OPTIONAL
+  if (!process.env.AI_ENABLED) {
+    return deterministicFallback(input)
+  }
+
+  try {
+    // 🔹 Placeholder for real LLM call
+    // 🔹 Intentionally simple & replaceable
+    return `
+This failure occurred in the test "${input.testName}".
+Based on the observed error and system heuristics, the root cause is classified as:
+"${input.rootCause}"
+
+The confidence score (${(input.confidence / 100).toFixed(2)}) indicates a stable and repeatable issue.
+Recommended action is to follow the suggested remediation rather than retrying the test.
+`.trim()
+  } catch {
+    // 🔒 Never crash pipeline because of AI
+    return deterministicFallback(input)
+  }
+}
+
+function deterministicFallback(input: SummaryInput): string {
+  return `
+Failure in "${input.testName}" was analyzed deterministically.
+Root cause identified as: ${input.rootCause}.
+Confidence level: ${(input.confidence / 100).toFixed(2)}.
+`.trim()
+}

package/src/pipeline/failure-analysis.pipeline.ts

@@ -0,0 +1,13 @@
+// src/pipeline/failure-analysis.pipeline.ts
+
+import { FailureAnalysisResult } from "../types/failure.types";
+import { parsePlaywrightFailureFolders } from "../adapters/playwright-folder.adapter";
+
+/**
+ * Phase 1 – Playwright Failure Intake (Folder-based)
+ */
+export async function runFailureAnalysisFromPlaywright(
+  playwrightResultsDir: string
+): Promise<FailureAnalysisResult[]> {
+  return parsePlaywrightFailureFolders(playwrightResultsDir);
+}