quality-intelligence-engine 2.2.0

package/README.md

@@ -0,0 +1,123 @@
+# Quality Intelligence Engine
+
+**Deterministic Test Failure Classification & Analysis for Playwright**
+
+A rule-based intelligent system that automatically classifies test failures, provides root cause analysis, and delivers actionable insights—without relying on AI black boxes.
+
+---
+
+## 🎯 Overview
+
+The Quality Intelligence Engine analyzes Playwright test failures and classifies them into:
+
+- **API_BUG**: Backend/API failures (authentication errors, data mismatches, server errors)
+- **UI_BUG**: Frontend/UI issues (visibility problems, rendering failures, element issues)
+
+Each classification comes with:
+- ✅ **Confidence score** (0-100%) - How certain we are about the classification
+- ✅ **Root cause analysis** - What went wrong and why
+- ✅ **Evidence** - Signals that led to the classification
+- ✅ **Diagnosis** - Step-by-step breakdown of the failure
+- ✅ **Suggested actions** - Concrete next steps to fix the issue
+
+---
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+npm install
+```
+
+### Run Analysis
+
+```bash
+# Run Playwright tests
+npx playwright test
+
+# Run Quality Intelligence Engine
+npm start
+```
+
+The engine will:
+1. Read test results from `playwright-report/report.json`
+2. Classify failures using the taxonomy
+3. Generate detailed analysis reports
+4. Output results to `output/` directory
+
+---
+
+## 📁 Project Structure
+
+```
+quality-intelligence-engine/
+├── src/
+│   ├── intelligence/         # Classification algorithms
+│   ├── pipeline/             # Data processing
+│   ├── utils/                # Utilities
+│   ├── types.ts              # Type definitions
+│   ├── normalizer.ts         # Input normalization
+│   ├── configLoader.ts       # Configuration
+│   └── reporter.ts           # Output generation
+├── tests/                    # Example test cases
+├── config/                   # Configuration files
+└── playwright.config.ts      # Playwright configuration
+```
+
+---
+
+## ⚙️ Configuration
+
+Edit `config/agent.config.json`:
+
+```json
+{
+  "engine": {
+    "mode": "standard",
+    "confidenceThresholds": {
+      "fail": 0.85,
+      "passRisk": 0.6
+    }
+  }
+}
+```
+
+---
+
+## 🏷️ Taxonomy System
+
+Tests are tagged with their expected failure type:
+
+```typescript
+test('Data | NUMERIC_MISMATCH | inventory count wrong', async ({ page }) => {
+  // Test code...
+});
+```
+
+---
+
+## 📊 Confidence Scoring
+
+Deterministic, explainable confidence calculations:
+
+- **Base Confidence**: 0.6 - 0.95
+- **Adjustments**: +0.05 - +0.1 based on signals
+- **Maximum**: 0.95 (never 100% certain)
+
+---
+
+## 🔧 Development
+
+```bash
+# Run tests
+npx playwright test
+
+# Type check
+npx tsc --noEmit
+```
+
+---
+
+**Version**: 1.0.0 (Refactored)  
+**Last Updated**: 2026-01-29

package/dist/bin/qi.js

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = __importDefault(require("path"));
+/**
+ * CLI entry point for Quality Intelligence
+ * Compiled and executed as CommonJS
+ */
+async function main() {
+    const args = process.argv.slice(2);
+    const playwrightResultsDir = args.find(a => !a.startsWith("--")) ?? "playwright-results";
+    const showIssues = args.includes("--issue") || args.includes("--issues");
+    // Resolve project root (works after npm install)
+    const projectRoot = path_1.default.resolve(__dirname, "..");
+    // IMPORTANT: call compiled JS, never TS
+    const finalRunPath = path_1.default.join(projectRoot, "src", "final-run.js");
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { runFinal } = require(finalRunPath);
+    runFinal(playwrightResultsDir, showIssues);
+}
+main().catch(err => {
+    console.error("QI failed:", err);
+    process.exit(1);
+});

package/dist/src/adapters/playwright-folder.adapter.js

@@ -0,0 +1,68 @@
+"use strict";
+// src/adapters/playwright-folder.adapter.ts
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parsePlaywrightFailureFolders = parsePlaywrightFailureFolders;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+/**
+ * Improved Playwright failure folder parser
+ * Focus: reduce UNKNOWN classifications
+ */
+function parsePlaywrightFailureFolders(resultsDir) {
+    const fullDir = path_1.default.resolve(resultsDir);
+    if (!fs_1.default.existsSync(fullDir)) {
+        throw new Error(`Playwright results directory not found: ${fullDir}`);
+    }
+    const entries = fs_1.default.readdirSync(fullDir, { withFileTypes: true });
+    const failures = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory())
+            continue;
+        const name = entry.name;
+        if (!name.includes("FAIL"))
+            continue;
+        const lower = name.toLowerCase();
+        let classification = "UNKNOWN";
+        // 🔢 Numeric / data mismatches
+        if (lower.includes("numeric-mismatch") ||
+            lower.includes("count") ||
+            lower.includes("total") ||
+            lower.includes("sum") ||
+            lower.includes("price")) {
+            classification = "NUMERIC_MISMATCH";
+        }
+        // 🔐 Auth / permission issues
+        else if (lower.includes("unauthorized") ||
+            lower.includes("forbidden") ||
+            lower.includes("permission")) {
+            classification = "AUTH_BUG";
+        }
+        // ⏱ Timeouts / waits
+        else if (lower.includes("timeout") ||
+            lower.includes("timed-out") ||
+            lower.includes("wait")) {
+            classification = "TIMEOUT";
+        }
+        // 🧭 UI / locator issues
+        else if (lower.includes("locator") ||
+            lower.includes("not-visible") ||
+            lower.includes("hidden") ||
+            lower.includes("detached")) {
+            classification = "UI_STATE";
+        }
+        // 🔁 Retry / flaky signal
+        else if (lower.includes("retry")) {
+            classification = "FLAKY";
+        }
+        failures.push({
+            testName: name,
+            classification,
+            failureType: classification,
+            message: `Derived from Playwright folder name: ${name}`
+        });
+    }
+    return failures;
+}

package/dist/src/adapters/playwright-json.adapter.js

@@ -0,0 +1,46 @@
+"use strict";
+// src/adapters/playwright-json.adapter.ts
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parsePlaywrightJsonReport = parsePlaywrightJsonReport;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+/**
+ * Converts Playwright JSON report → FailureAnalysisResult[]
+ */
+function parsePlaywrightJsonReport(reportPath) {
+    const fullPath = path_1.default.resolve(reportPath);
+    const raw = fs_1.default.readFileSync(fullPath, "utf-8");
+    const report = JSON.parse(raw);
+    const failures = [];
+    for (const suite of report.suites ?? []) {
+        for (const spec of suite.specs ?? []) {
+            for (const test of spec.tests ?? []) {
+                for (const result of test.results ?? []) {
+                    if (result.status === "failed") {
+                        const error = result.error?.message ?? "";
+                        let classification = "UNKNOWN";
+                        if (error.includes("toHaveCount")) {
+                            classification = "NUMERIC_MISMATCH";
+                        }
+                        else if (error.includes("timeout")) {
+                            classification = "TIMEOUT";
+                        }
+                        else if (error.includes("locator")) {
+                            classification = "LOCATOR_CHANGED";
+                        }
+                        failures.push({
+                            testName: test.title,
+                            classification,
+                            failureType: classification,
+                            message: error
+                        });
+                    }
+                }
+            }
+        }
+    }
+    return failures;
+}

package/dist/src/adapters/playwright.adapter.js

@@ -0,0 +1,36 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readPlaywrightFailures = readPlaywrightFailures;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+function readPlaywrightFailures(reportDir) {
+    const reportFile = path_1.default.join(reportDir, 'report.json');
+    if (!fs_1.default.existsSync(reportFile)) {
+        throw new Error(`Playwright JSON report not found.\nExpected at: ${reportFile}`);
+    }
+    const raw = JSON.parse(fs_1.default.readFileSync(reportFile, 'utf-8'));
+    const failures = [];
+    for (const suite of raw.suites ?? []) {
+        for (const spec of suite.specs ?? []) {
+            for (const test of spec.tests ?? []) {
+                const failedResults = test.results?.filter((r) => r.status === 'failed');
+                if (!failedResults || failedResults.length === 0)
+                    continue;
+                const firstFailure = failedResults[0];
+                const error = firstFailure.error;
+                failures.push({
+                    testName: spec.title,
+                    errorMessage: error?.message ?? 'Unknown error',
+                    stack: error?.stack,
+                    filePath: error?.location?.file ?? spec.file,
+                    lineNumber: error?.location?.line ?? spec.line ?? 0,
+                    retryCount: failedResults.length - 1
+                });
+            }
+        }
+    }
+    return failures;
+}

package/dist/src/cli/qi-test.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runQiTest = runQiTest;
+const child_process_1 = require("child_process");
+function runQiTest(args) {
+    const playwrightArgs = [
+        'playwright',
+        'test',
+        '--config',
+        'playwright.config.ts',
+        ...args
+    ];
+    const child = (0, child_process_1.spawn)('npx', playwrightArgs, {
+        stdio: 'inherit',
+        shell: true
+    });
+    child.on('exit', code => {
+        process.exit(code ?? 0);
+    });
+}

package/dist/src/cli/qi.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+"use strict";
+// src/cli/qi.ts
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runQiCli = runQiCli;
+const runner_1 = require("../runner");
+/**
+ * Core CLI runner (reusable)
+ */
+async function runQiCli(input) {
+    const results = await (0, runner_1.runQualityIntelligence)(input);
+    if (!Array.isArray(results) || results.length === 0) {
+        console.log("No failures detected.");
+        return;
+    }
+    for (const f of results) {
+        const testName = f.testName ?? "UNKNOWN_TEST";
+        const isExpected = testName.includes("EXPECTED");
+        console.log("=================================");
+        console.log(`Test Name     : ${testName}${isExpected ? " (EXPECTED)" : ""}`);
+        console.log(`Failure Type  : ${f.failureType ?? f.classification}`);
+        console.log(`Confidence    : ${f.confidence ?? "N/A"}\n`);
+        if (f.rootCause?.length) {
+            console.log("Root Cause:");
+            for (const line of f.rootCause) {
+                console.log(` - ${line}`);
+            }
+            console.log();
+        }
+        if (f.evidence?.length) {
+            console.log("Evidence:");
+            for (const line of f.evidence) {
+                console.log(` - ${line}`);
+            }
+            console.log();
+        }
+        if (f.diagnosis?.length) {
+            console.log("Diagnosis:");
+            for (const line of f.diagnosis) {
+                console.log(` - ${line}`);
+            }
+            console.log();
+        }
+    }
+}
+/**
+ * CLI ENTRY POINT
+ * Allows: npx qi <path>
+ */
+async function main() {
+    const input = process.argv[2] ?? "playwright-results";
+    await runQiCli(input);
+}
+if (process.argv[1]?.includes("qi")) {
+    main().catch(err => {
+        console.error("QI CLI failed:", err);
+        process.exit(1);
+    });
+}

package/dist/src/cli/ui/divider.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.divider = divider;
+function divider(title, width = 60) {
+    const line = '━'.repeat(width);
+    if (!title) {
+        return `\n${line}\n`;
+    }
+    return `\n${line}\n${title}\n${line}\n`;
+}

package/dist/src/cli/ui/failureLogger.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logFailure = logFailure;
+function logFailure(input) {
+    const confidencePercent = Math.round(input.confidence * 100);
+    console.log(`
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+❌ Automation Test Failure
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+Test Name     : ${input.testName}
+Failure Type  : ${input.failureType}
+Severity      : ${input.severity}
+Priority      : ${input.priority}
+Confidence    : ${confidencePercent}%
+
+Failure Location:
+- Test File   : ${input.file}
+- Line Number : ${input.line}
+
+Root Cause:
+- ${input.rootCause}
+`);
+}

package/dist/src/configLoader.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadConfig = loadConfig;
+const file_utils_1 = require("./utils/file-utils");
+/**
+ * Validate configuration object structure
+ */
+function validateConfig(config) {
+    if (typeof config !== 'object' || config === null) {
+        return false;
+    }
+    const cfg = config;
+    // Validate engine section
+    if (typeof cfg.engine !== 'object' ||
+        cfg.engine === null) {
+        return false;
+    }
+    const engine = cfg.engine;
+    if (typeof engine.mode !== 'string' ||
+        typeof engine.confidenceThresholds !== 'object') {
+        return false;
+    }
+    return true;
+}
+/**
+ * Load and validate configuration
+ */
+function loadConfig() {
+    try {
+        const configPath = (0, file_utils_1.resolvePath)('config', 'agent.config.json');
+        const config = (0, file_utils_1.readJsonFile)(configPath);
+        if (!validateConfig(config)) {
+            throw new Error('Invalid configuration structure');
+        }
+        return config;
+    }
+    catch (error) {
+        console.error('❌ Failed to load configuration:', error);
+        console.error('\n💡 Using default configuration');
+        // Return safe defaults
+        return {
+            engine: {
+                mode: 'standard',
+                confidenceThresholds: {
+                    fail: 0.85,
+                    passRisk: 0.6
+                }
+            },
+            output: {
+                writeMarkdown: true,
+                maxHistoryRuns: 10
+            },
+            passAnalysis: {
+                enable: true,
+                reportFlakyPass: true,
+                reportApiWarnings: true
+            }
+        };
+    }
+}

package/dist/src/console/issue-view.js

@@ -0,0 +1,122 @@
+"use strict";
+// src/console/issue-view.ts
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractIssues = extractIssues;
+exports.filterIssues = filterIssues;
+exports.printIssues = printIssues;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+/* ---------------- Issue Core ---------------- */
+function extractIssues(groupedCounts, confidence) {
+    return Object.entries(groupedCounts).map(([type, count]) => ({
+        type,
+        occurrences: count,
+        confidence
+    }));
+}
+/* ---------------- Decision Logic ---------------- */
+function computeIssueDecision(issue) {
+    switch (issue.type) {
+        case "FLAKY":
+            return {
+                autoHeal: true,
+                reason: "Retry-based flaky behavior detected; stabilization is safe"
+            };
+        case "NUMERIC_MISMATCH":
+            return {
+                autoHeal: false,
+                reason: "Numeric mismatch likely backend or data-related"
+            };
+        default:
+            return {
+                autoHeal: false,
+                reason: "Insufficient signal to safely auto-heal"
+            };
+    }
+}
+/* ---------------- Owner Suggestion ---------------- */
+function computeIssueOwner(issue) {
+    switch (issue.type) {
+        case "NUMERIC_MISMATCH":
+            return {
+                team: "Backend",
+                reason: "Mismatch usually originates from API data or backend calculations"
+            };
+        case "FLAKY":
+            return {
+                team: "QA",
+                reason: "Flaky behavior typically caused by timing, waits, or test stability"
+            };
+        case "UNKNOWN":
+        default:
+            return {
+                team: "Frontend",
+                reason: "Failure source unclear; frontend investigation needed first"
+            };
+    }
+}
+/* ---------------- Evidence Picker ---------------- */
+function pickIssueEvidence(failureFolder) {
+    if (!fs_1.default.existsSync(failureFolder))
+        return {};
+    const files = fs_1.default.readdirSync(failureFolder);
+    const screenshot = files.find(f => f.startsWith("test-failed") && f.endsWith(".png"));
+    const trace = files.includes("trace.zip") ? "trace.zip" : undefined;
+    return {
+        screenshot: screenshot
+            ? path_1.default.join(failureFolder, screenshot)
+            : undefined,
+        trace: trace
+            ? path_1.default.join(failureFolder, trace)
+            : undefined
+    };
+}
+/* ---------------- Filtering ---------------- */
+function filterIssues(issues, filter) {
+    let result = [...issues];
+    if (filter.keyword) {
+        const k = filter.keyword.toLowerCase();
+        result = result.filter(i => i.type.toLowerCase().includes(k));
+    }
+    result.sort((a, b) => b.occurrences * b.confidence -
+        a.occurrences * a.confidence);
+    if (filter.topN !== undefined) {
+        result = result.slice(0, filter.topN);
+    }
+    return result;
+}
+/* ---------------- Console Output ---------------- */
+function printIssues(issues, failureFolder) {
+    if (issues.length === 0) {
+        console.log("No matching issues found.");
+        return;
+    }
+    const evidence = pickIssueEvidence(failureFolder);
+    issues.forEach((issue, index) => {
+        const decision = computeIssueDecision(issue);
+        const owner = computeIssueOwner(issue);
+        console.log(`================ ISSUE #${index + 1} =================`);
+        console.log(`Type        : ${issue.type}`);
+        console.log(`Occurrences : ${issue.occurrences}`);
+        console.log(`Confidence  : ${issue.confidence}`);
+        console.log("Decision:");
+        console.log(`  - Auto-heal: ${decision.autoHeal ? "YES" : "NO"}`);
+        console.log(`  - Reason   : ${decision.reason}`);
+        console.log("Owner:");
+        console.log(`  - Team     : ${owner.team}`);
+        console.log(`  - Why      : ${owner.reason}`);
+        if (evidence.screenshot || evidence.trace) {
+            console.log("\nEvidence:");
+            if (evidence.screenshot) {
+                console.log(`  Screenshot : ${evidence.screenshot}`);
+            }
+            if (evidence.trace) {
+                console.log(`  Trace      : npx playwright show-trace ${evidence.trace}`);
+            }
+        }
+        console.log("======================================================");
+    });
+}

package/dist/src/failure-analysis/failure-analyzer.js

@@ -0,0 +1,38 @@
+"use strict";
+// src/failure-analysis/failure-analyzer.ts
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FailureAnalyzer = void 0;
+/**
+ * FailureAnalyzer
+ * ----------------
+ * Phase 1 core analyzer.
+ * This class is intentionally simple and stable.
+ * Do NOT add history, trends, or intelligence here.
+ */
+class FailureAnalyzer {
+    analyze(input) {
+        if (!input || !Array.isArray(input)) {
+            return [];
+        }
+        return input.map((item) => {
+            // Minimal safe classification logic
+            if (item?.type === "NUMERIC_MISMATCH") {
+                return {
+                    classification: "NUMERIC_MISMATCH",
+                    message: item.message
+                };
+            }
+            if (item?.type === "API_BUG") {
+                return {
+                    classification: "API_BUG",
+                    message: item.message
+                };
+            }
+            return {
+                classification: "UNKNOWN",
+                message: item?.message
+            };
+        });
+    }
+}
+exports.FailureAnalyzer = FailureAnalyzer;