qa-intelligence 1.0.0

package/README.md

@@ -0,0 +1,184 @@
+# qa-intelligence
+
+CI intelligence engine for Playwright test pipelines.
+
+- AI-powered failure analysis
+- PR failure diff (new, flaky, still failing, fixed)
+- Flaky test detection (retry-aware)
+- PR blocking on new non-flaky failures
+- Recurrence tracking ("3rd time since...")
+
+Use it **without the framework template** — install the package into any existing Playwright project.
+
+---
+
+## Install
+
+```bash
+npm install qa-intelligence @playwright/test
+```
+
+---
+
+## Quick setup (existing project)
+
+### 1. Environment (`.env`)
+
+```env
+BASE_URL=https://your-app.example.com
+HEADLESS=true
+PW_WORKERS=2
+PW_RETRIES=1
+```
+
+### 2. `playwright.config.ts`
+
+```ts
+import { defineConfig } from "@playwright/test";
+import { env } from "qa-intelligence/config/env";
+
+export default defineConfig({
+  testDir: "./tests",
+  retries: env.PW_RETRIES,
+  workers: env.PW_WORKERS,
+  globalSetup: require.resolve("qa-intelligence/playwright/globalSetup"),
+  globalTeardown: require.resolve("qa-intelligence/playwright/globalTeardown"),
+  use: {
+    baseURL: env.BASE_URL,
+    headless: env.HEADLESS,
+    trace: "on-first-retry",
+    screenshot: "only-on-failure",
+    video: "retain-on-failure",
+  },
+});
+```
+
+### 3. Write tests
+
+Always import `test` from the package — **not** directly from Playwright:
+
+```ts
+import { test, expect } from "qa-intelligence/playwright";
+
+test("user can login", async ({ page }) => {
+  await page.goto("/");
+  await expect(page).toHaveTitle(/My App/);
+});
+```
+
+Optional helpers:
+
+```ts
+import { step } from "qa-intelligence/playwright/steps";
+import { BasePage } from "qa-intelligence/playwright/basePage";
+```
+
+### 4. What you provide
+
+| You write | Package provides |
+|-----------|------------------|
+| `tests/` | Test hooks, artifact capture |
+| `.env` | Env validation |
+| `playwright.config.ts` | `globalSetup`, `globalTeardown`, AI teardown |
+| `.github/workflows/ci.yml` | `qa-intelligence-diff`, `qa-intelligence-history`, `qa-intelligence-comment` |
+| Page objects (optional) | `BasePage`, `step()` |
+
+---
+
+## CI setup (GitHub Actions)
+
+### Option A: Copy the full workflow (recommended)
+
+Copy `.github/workflows/ci.yml` from the [framework template](https://github.com/ardithaqi/qa-intelligence-framework/blob/master/.github/workflows/ci.yml) into your repo.
+
+It includes everything wired together:
+
+- Docker test execution
+- Artifact upload on every run
+- Baseline download from `main` on pull requests
+- `qa-intelligence-diff`, `qa-intelligence-history`, `qa-intelligence-comment`
+- PR blocking on new non-flaky failures
+
+Then update:
+
+- `BASE_URL` in the `docker run` step
+- GitHub secrets (see below)
+
+### Option B: Add to your existing workflow
+
+If you already run Playwright in CI, add these steps **after** tests run and `artifacts/` are uploaded:
+
+```bash
+npm install qa-intelligence
+npx qa-intelligence-diff --baseline baseline-artifacts --current artifacts
+npx qa-intelligence-history
+npx qa-intelligence-comment \
+  --diff failure-diff.json \
+  --repo owner/repo \
+  --pr 123 \
+  --token $GITHUB_TOKEN
+```
+
+You must also handle on your own:
+
+- Uploading `artifacts/` after each run
+- Downloading `baseline-artifacts/` from the target branch on PRs
+
+See the [framework CI workflow](https://github.com/ardithaqi/qa-intelligence-framework/blob/master/.github/workflows/ci.yml) for reference.
+
+### Required GitHub secrets
+
+- `OPENAI_API_KEY` — enables AI failure analysis in teardown
+- `TEST_USERNAME` / `TEST_PASSWORD` — if your tests need credentials
+
+Set `AI_ANALYSIS=true` in CI when running tests inside Docker.
+
+---
+
+## PR behavior
+
+| Failure type | Blocks PR? | Shown in comment |
+|--------------|------------|------------------|
+| New (not flaky) | Yes | New Issues |
+| Flaky | No | Flaky |
+| Still failing from base branch | No | Still Failing |
+| Fixed since base branch | No | Fixed Issues |
+
+---
+
+## CLI tools
+
+| Command | Purpose |
+|---------|---------|
+| `qa-intelligence-diff` | Compare baseline vs current failures |
+| `qa-intelligence-history` | Add recurrence tracking |
+| `qa-intelligence-comment` | Post/update PR summary comment |
+
+---
+
+## Package exports
+
+| Import path | What it gives you |
+|-------------|-------------------|
+| `qa-intelligence/playwright` | `test`, `expect`, `env` |
+| `qa-intelligence/playwright/globalSetup` | Artifact run setup |
+| `qa-intelligence/playwright/globalTeardown` | AI failure analysis |
+| `qa-intelligence/playwright/basePage` | Base page object |
+| `qa-intelligence/playwright/steps` | `step()` helper |
+| `qa-intelligence/config/env` | Validated env config |
+
+---
+
+## Alternative: use the full template
+
+If you prefer a ready-made project with examples, Docker, and CI pre-wired:
+
+**[qa-intelligence-framework](https://github.com/ardithaqi/qa-intelligence-framework)** — click "Use this template".
+
+The template uses this package under the hood.
+
+---
+
+## License
+
+MIT

package/dist/ai/failureAnalyzer.js

@@ -0,0 +1,149 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.analyzeFailureFile = analyzeFailureFile;
+exports.analyzeLatestFailure = analyzeLatestFailure;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const openai_1 = __importDefault(require("openai"));
+function getOpenAIClient() {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        console.warn("OPENAI_API_KEY is not set. Skipping AI failure analysis.");
+        return null;
+    }
+    return new openai_1.default({ apiKey });
+}
+function findLatestMetaFile(root = "artifacts") {
+    if (!fs_1.default.existsSync(root))
+        return null;
+    let latestPath = null;
+    let latestMtime = -1;
+    function walk(dir) {
+        for (const entry of fs_1.default.readdirSync(dir, { withFileTypes: true })) {
+            const fullPath = path_1.default.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                walk(fullPath);
+                continue;
+            }
+            if (entry.name !== "meta.json")
+                continue;
+            const stats = fs_1.default.statSync(fullPath);
+            if (stats.mtimeMs > latestMtime) {
+                latestMtime = stats.mtimeMs;
+                latestPath = fullPath;
+            }
+        }
+    }
+    walk(root);
+    return latestPath;
+}
+async function analyzeFailureFile(jsonFilePath) {
+    const openai = getOpenAIClient();
+    if (!openai)
+        return null;
+    const htmlPath = jsonFilePath.replace(".json", ".html");
+    const meta = JSON.parse(fs_1.default.readFileSync(jsonFilePath, "utf-8"));
+    const html = fs_1.default.existsSync(htmlPath)
+        ? fs_1.default.readFileSync(htmlPath, "utf-8")
+        : "";
+    const prompt = `
+        You are a senior QA automation engineer analyzing a failed Playwright test.
+        
+        STRICT RULES:
+        - Do NOT speculate about deployments, databases, or external systems.
+        - Do NOT guess about business logic changes.
+        - Base reasoning ONLY on stack trace, metadata, and DOM snapshot.
+        - If the original assertion is visible in the stack trace, extract it exactly as written.
+        - Remove absolute user directories such as /Users/... and return only project-relative path starting from tests/.
+        - If the stack trace includes the original source line (e.g. expect(x).toBe(y)), extract that exact line instead of Playwright's formatted error message.
+        - Never output expect(received).toBe(expected) if a concrete assertion exists.
+        - Prefer the real test assertion over Playwright’s generic expect(received).toBe(expected).
+        - failure_type MUST be exactly one of:
+          assertion_mismatch
+          selector_not_found
+          timeout
+          navigation_failure
+          environment_error
+          unknown
+        - Severity MUST be exactly one of: low, medium, high.
+        - Severity must follow this logic:
+          low for forced/debug assertions.
+          medium for deterministic assertion mismatches.
+          high only for navigation_failure, timeout, or environment_error.
+        - confidence must reflect how strongly the evidence supports the classification.
+        - For direct numeric assertion mismatches (e.g. expect(5).toBe(6)), confidence MUST be >= 90.
+        - expected and received must be numbers if numeric.
+        
+        Failure metadata:
+        ${JSON.stringify(meta, null, 2)}
+        
+        Primary error:
+        ${meta.errorMessage}
+        
+        DOM snapshot (first 4000 chars):
+        ${html.substring(0, 4000)}
+        
+        Your response must contain TWO sections.
+        
+        SECTION 1: HUMAN ANALYSIS
+        
+        Format EXACTLY like this:
+        
+        File: <relative path only>
+        Line: <number>
+        Assertion: <exact assertion from test file>
+        Expected: <value>
+        Received: <value>
+        
+        Root cause:
+        Line 1: <what the assertion compares>
+        Line 2: <what was actually observed>
+        Line 3: <classification sentence>
+        
+        No extra commentary.
+        No speculation.
+        No extra paragraphs.
+        
+        SECTION 2: STRUCTURED_JSON
+        
+        Return STRICTLY raw JSON only.
+        Do NOT wrap in markdown.
+        Do NOT add text before or after JSON.
+        
+        {
+          "file": "relative/path/to/file",
+          "line": 0,
+          "failure_type": "assertion_mismatch | selector_not_found | timeout | navigation_failure | environment_error | unknown",
+          "expected": 0,
+          "received": 0,
+          "is_flaky_suspected": false,
+          "severity": "low | medium | high",
+          "confidence": 0
+        }
+        `;
+    const response = await openai.chat.completions.create({
+        model: "gpt-4o-mini",
+        messages: [{ role: "user", content: prompt }],
+    });
+    const content = response.choices[0]?.message?.content ?? null;
+    const usage = response.usage;
+    if (usage) {
+        console.log("Token usage:", usage);
+        // Rough estimate for gpt-4o-mini (adjust if pricing changes)
+        const estimatedCost = (usage.total_tokens / 1000) * 0.00015;
+        console.log(`Estimated cost (approx): $${estimatedCost.toFixed(6)}`);
+    }
+    return content;
+}
+async function analyzeLatestFailure() {
+    const latestMetaFile = findLatestMetaFile();
+    if (!latestMetaFile) {
+        console.log("No failure metadata found in artifacts.");
+        return null;
+    }
+    console.log(`Analyzing latest failure: ${latestMetaFile}`);
+    return analyzeFailureFile(latestMetaFile);
+}

package/dist/ai/runAnalyzer.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+require("dotenv/config");
+const failureAnalyzer_1 = require("./failureAnalyzer");
+(0, failureAnalyzer_1.analyzeLatestFailure)().catch(console.error);

package/dist/cli/diff.js

@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const computeDiff_1 = require("../lib/computeDiff");
+function getArg(name) {
+    const idx = process.argv.indexOf(`--${name}`);
+    if (idx === -1)
+        return undefined;
+    return process.argv[idx + 1];
+}
+function getBoolArg(name, defaultVal) {
+    const v = getArg(name);
+    if (!v)
+        return defaultVal;
+    return v === "true" || v === "1" || v === "yes";
+}
+async function main() {
+    const baselineDir = getArg("baseline") ?? "baseline-artifacts";
+    const currentDir = getArg("current") ?? "artifacts";
+    const outFile = getArg("out") ?? "failure-diff.json";
+    const failOnBlocking = getBoolArg("fail-on-blocking", true);
+    const diff = (0, computeDiff_1.computeDiff)(baselineDir, currentDir);
+    fs_1.default.writeFileSync(outFile, JSON.stringify(diff, null, 2));
+    console.log(`Wrote diff to ${outFile}`);
+    console.log(`New failures: ${diff.newFailures.length}`);
+    console.log(`Blocking failures: ${diff.blockingFailures.length}`);
+    console.log(`Still failing: ${diff.unchangedFailures.length}`);
+    console.log(`Fixed failures: ${diff.fixedFailures.length}`);
+    if (failOnBlocking && diff.blockingFailures.length > 0) {
+        console.error(`Failing job because ${diff.blockingFailures.length} new non-flaky failure(s) were detected.`);
+        process.exit(1);
+    }
+}
+main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+});

package/dist/cli/history.js

@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * Enriches failure-diff with recurrence: first_seen, occurrence_count.
+ * Persists run history in a cache file for the next run.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const HISTORY_PATH = ".cache/failure-history.json";
+const MAX_RUNS = 20;
+function failureKey(f) {
+    return `${f.file}:${f.line}:${f.failure_type}`;
+}
+function loadHistory() {
+    if (!fs_1.default.existsSync(HISTORY_PATH))
+        return { runs: [] };
+    try {
+        const data = JSON.parse(fs_1.default.readFileSync(HISTORY_PATH, "utf8"));
+        return Array.isArray(data.runs) ? { runs: data.runs } : { runs: [] };
+    }
+    catch {
+        return { runs: [] };
+    }
+}
+function enrichFailures(failures, history, currentKeys) {
+    const runsIncludingThis = [...history.runs];
+    const thisRun = {
+        sha: process.env.GITHUB_SHA ?? "",
+        timestamp: new Date().toISOString(),
+        failureKeys: [...currentKeys],
+    };
+    runsIncludingThis.push(thisRun);
+    return failures.map((f) => {
+        const key = failureKey(f);
+        const enriched = { ...f };
+        const runsWithThis = runsIncludingThis.filter((r) => r.failureKeys.includes(key));
+        if (runsWithThis.length > 0) {
+            enriched.occurrence_count = runsWithThis.length;
+            enriched.first_seen = runsWithThis[0].timestamp;
+        }
+        return enriched;
+    });
+}
+function main() {
+    const diffPath = "failure-diff.json";
+    if (!fs_1.default.existsSync(diffPath)) {
+        console.log("No failure-diff.json, skipping history update.");
+        return;
+    }
+    const diff = JSON.parse(fs_1.default.readFileSync(diffPath, "utf8"));
+    const { newFailures, unchangedFailures, fixedFailures } = diff;
+    const currentKeys = new Set();
+    for (const f of [...newFailures, ...unchangedFailures]) {
+        currentKeys.add(failureKey(f));
+    }
+    const history = loadHistory();
+    const enrichedNew = enrichFailures(newFailures, history, currentKeys);
+    const enrichedUnchanged = enrichFailures(unchangedFailures, history, currentKeys);
+    const enrichedFixed = enrichFailures(fixedFailures, history, currentKeys);
+    const updatedHistory = {
+        runs: [...history.runs, { sha: process.env.GITHUB_SHA ?? "", timestamp: new Date().toISOString(), failureKeys: [...currentKeys] }].slice(-MAX_RUNS),
+    };
+    fs_1.default.mkdirSync(path_1.default.dirname(HISTORY_PATH), { recursive: true });
+    fs_1.default.writeFileSync(HISTORY_PATH, JSON.stringify(updatedHistory, null, 2));
+    const result = {
+        newFailures: enrichedNew,
+        unchangedFailures: enrichedUnchanged,
+        fixedFailures: enrichedFixed,
+    };
+    fs_1.default.writeFileSync(diffPath, JSON.stringify(result, null, 2));
+    console.log("Updated failure-diff with recurrence; history runs:", updatedHistory.runs.length);
+}
+main();

package/dist/cli/index.js

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const child_process_1 = require("child_process");
+function getArg(name) {
+    const idx = process.argv.indexOf(`--${name}`);
+    if (idx === -1)
+        return undefined;
+    return process.argv[idx + 1];
+}
+async function main() {
+    const baseline = getArg("baseline") ?? "baseline-artifacts";
+    const current = getArg("current") ?? "artifacts";
+    const repo = getArg("repo");
+    const pr = getArg("pr");
+    const token = getArg("token");
+    console.log("Running failure diff...");
+    (0, child_process_1.execSync)(`node ${__dirname}/diff.js --baseline ${baseline} --current ${current} --out failure-diff.json`, {
+        stdio: "inherit"
+    });
+    if (repo && pr && token) {
+        console.log("Posting PR comment...");
+        (0, child_process_1.execSync)(`node ${__dirname}/postComment.js --diff failure-diff.json --repo ${repo} --pr ${pr} --token ${token}`, {
+            stdio: "inherit"
+        });
+    }
+}
+main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+});

package/dist/cli/postComment.js

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const format_1 = require("../lib/format");
+const rest_1 = require("@octokit/rest");
+function getArg(name) {
+    const idx = process.argv.indexOf(`--${name}`);
+    if (idx === -1)
+        return undefined;
+    return process.argv[idx + 1];
+}
+async function main() {
+    const diffPath = getArg("diff");
+    const repoArg = getArg("repo");
+    const prArg = getArg("pr");
+    const token = getArg("token");
+    if (!diffPath || !repoArg || !prArg || !token) {
+        console.error("Usage: qa-intelligence-comment --diff <file> --repo <owner/repo> --pr <number> --token <token>");
+        process.exit(1);
+    }
+    const [owner, repo] = repoArg.split("/");
+    const issue_number = Number(prArg);
+    const raw = fs_1.default.readFileSync(diffPath, "utf8");
+    const diff = JSON.parse(raw);
+    if (!(0, format_1.hasFailureChanges)(diff)) {
+        console.log("No failure changes. Skipping comment.");
+        return;
+    }
+    const body = (0, format_1.formatDiffComment)(diff);
+    const octokit = new rest_1.Octokit({ auth: token });
+    // Find existing comment to update
+    const { data: comments } = await octokit.issues.listComments({
+        owner,
+        repo,
+        issue_number,
+        per_page: 100,
+    });
+    const existing = comments.find((c) => c.user?.type === "Bot" &&
+        typeof c.body === "string" &&
+        c.body.startsWith("## AI Failure Diff Summary"));
+    if (existing) {
+        await octokit.issues.updateComment({
+            owner,
+            repo,
+            comment_id: existing.id,
+            body,
+        });
+        console.log("Updated existing AI diff comment.");
+    }
+    else {
+        await octokit.issues.createComment({
+            owner,
+            repo,
+            issue_number,
+            body,
+        });
+        console.log("Created new AI diff comment.");
+    }
+}
+main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+});

package/dist/config/env.js

@@ -0,0 +1,64 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.env = void 0;
+const dotenv = __importStar(require("dotenv"));
+const zod_1 = require("zod");
+dotenv.config();
+const EnvSchema = zod_1.z.object({
+    ENV: zod_1.z.enum(["dev", "staging", "prod"]).default("dev"),
+    BASE_URL: zod_1.z.string().url(),
+    HEADLESS: zod_1.z
+        .string()
+        .transform((v) => v === "true")
+        .default(true),
+    PW_WORKERS: zod_1.z
+        .string()
+        .transform((v) => Number(v))
+        .pipe(zod_1.z.number().int().positive())
+        .default(2),
+    PW_RETRIES: zod_1.z
+        .string()
+        .transform((v) => Number(v))
+        .pipe(zod_1.z.number().int().nonnegative())
+        .default(1),
+});
+const parsed = EnvSchema.safeParse(process.env);
+if (!parsed.success) {
+    console.error("Invalid environment variables:");
+    console.error(parsed.error.flatten().fieldErrors);
+    process.exit(1);
+}
+exports.env = parsed.data;

package/dist/lib/computeDiff.js

@@ -0,0 +1,81 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeDiff = computeDiff;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+function extractTrailingJson(content) {
+    const trimmed = content.trimEnd();
+    for (let i = trimmed.lastIndexOf("{"); i >= 0; i = trimmed.lastIndexOf("{", i - 1)) {
+        try {
+            return JSON.parse(trimmed.slice(i));
+        }
+        catch { }
+    }
+    return null;
+}
+function collectFailures(root) {
+    const failures = [];
+    if (!fs_1.default.existsSync(root))
+        return failures;
+    function walk(dir) {
+        for (const file of fs_1.default.readdirSync(dir)) {
+            const full = path_1.default.join(dir, file);
+            if (fs_1.default.statSync(full).isDirectory()) {
+                walk(full);
+            }
+            else if (file === "ai.txt") {
+                const content = fs_1.default.readFileSync(full, "utf8");
+                const data = extractTrailingJson(content);
+                if (!data)
+                    continue;
+                failures.push({
+                    file: data.file,
+                    line: data.line,
+                    failure_type: data.failure_type,
+                    severity: data.severity,
+                    confidence: data.confidence,
+                    is_flaky_suspected: data.is_flaky_suspected ?? false
+                });
+            }
+        }
+    }
+    walk(root);
+    return failures;
+}
+function toMap(arr) {
+    const map = new Map();
+    for (const item of arr) {
+        const key = `${item.file}:${item.line}:${item.failure_type}`;
+        map.set(key, item);
+    }
+    return map;
+}
+function computeDiff(baselineDir, currentDir) {
+    const baseline = collectFailures(baselineDir);
+    const current = collectFailures(currentDir);
+    const baselineMap = toMap(baseline);
+    const currentMap = toMap(current);
+    const newFailures = [];
+    const unchangedFailures = [];
+    const fixedFailures = [];
+    for (const [key, value] of currentMap.entries()) {
+        if (!baselineMap.has(key))
+            newFailures.push(value);
+        else
+            unchangedFailures.push(value);
+    }
+    for (const [key, value] of baselineMap.entries()) {
+        if (!currentMap.has(key))
+            fixedFailures.push(value);
+    }
+    const blockingFailures = newFailures.filter((f) => !f.is_flaky_suspected);
+    return {
+        newFailures,
+        unchangedFailures,
+        fixedFailures,
+        blockingFailures
+    };
+}

package/dist/lib/failureReader.js

@@ -0,0 +1,107 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.failureKey = failureKey;
+exports.readFailures = readFailures;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+function isDirectory(p) {
+    return fs_1.default.existsSync(p) && fs_1.default.statSync(p).isDirectory();
+}
+function walkForAiTxt(dir, results = []) {
+    for (const entry of fs_1.default.readdirSync(dir)) {
+        const full = path_1.default.join(dir, entry);
+        const stat = fs_1.default.statSync(full);
+        if (stat.isDirectory()) {
+            walkForAiTxt(full, results);
+        }
+        else if (entry === "ai.txt" || entry === "failure-meta.json") {
+            results.push(full);
+        }
+    }
+    return results;
+}
+/**
+ * ai.txt contains human section + raw JSON.
+ * We extract JSON by finding the first "{" and parsing the rest.
+ */
+function parseAiTxt(filePath) {
+    if (filePath.endsWith("failure-meta.json")) {
+        try {
+            const raw = JSON.parse(fs_1.default.readFileSync(filePath, "utf8"));
+            if (!raw || !Array.isArray(raw.failures) || raw.failures.length === 0) {
+                return null;
+            }
+            const parsed = raw.failures[0];
+            return {
+                file: parsed.file,
+                line: parsed.line,
+                failure_type: parsed.failure_type,
+                expected: parsed.expected,
+                received: parsed.received,
+                is_flaky_suspected: Boolean(parsed.is_flaky_suspected),
+                severity: parsed.severity,
+                confidence: parsed.confidence,
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    const content = fs_1.default.readFileSync(filePath, "utf8");
+    const jsonStart = content.indexOf("{");
+    if (jsonStart === -1)
+        return null;
+    const jsonRaw = content.substring(jsonStart).trim();
+    try {
+        const parsed = JSON.parse(jsonRaw);
+        // Minimal validation (avoid hard crash in CI)
+        if (!parsed ||
+            typeof parsed.file !== "string" ||
+            (typeof parsed.line !== "number" && typeof parsed.line !== "string") ||
+            typeof parsed.failure_type !== "string" ||
+            typeof parsed.severity !== "string" ||
+            typeof parsed.confidence !== "number") {
+            return null;
+        }
+        const lineNum = typeof parsed.line === "string" ? Number(parsed.line) : parsed.line;
+        const failure = {
+            file: parsed.file,
+            line: Number.isFinite(lineNum) ? lineNum : 0,
+            failure_type: parsed.failure_type,
+            expected: parsed.expected,
+            received: parsed.received,
+            is_flaky_suspected: Boolean(parsed.is_flaky_suspected),
+            severity: parsed.severity,
+            confidence: parsed.confidence,
+        };
+        return failure;
+    }
+    catch {
+        return null;
+    }
+}
+function failureKey(f) {
+    return `${f.file}:${f.line}:${f.failure_type}`;
+}
+function readFailures(options) {
+    const root = options.rootDir;
+    if (!isDirectory(root)) {
+        return { failures: [], byKey: new Map(), sourceFiles: [] };
+    }
+    const aiFiles = walkForAiTxt(root);
+    const failures = [];
+    const byKey = new Map();
+    for (const fp of aiFiles) {
+        const parsed = parseAiTxt(fp);
+        if (!parsed)
+            continue;
+        const key = failureKey(parsed);
+        failures.push(parsed);
+        // If duplicates, keep the latest encountered (fine for now)
+        byKey.set(key, parsed);
+    }
+    return { failures, byKey, sourceFiles: aiFiles };
+}

package/dist/lib/format.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatDiffComment = formatDiffComment;
+exports.hasFailureChanges = hasFailureChanges;
+function recurrenceSuffix(f) {
+    if (f.occurrence_count == null)
+        return "";
+    const n = f.occurrence_count;
+    const date = f.first_seen
+        ? new Date(f.first_seen).toISOString().slice(0, 10)
+        : "";
+    if (n === 1)
+        return " (first time)";
+    const ord = n % 10 === 1 && n % 100 !== 11
+        ? "st"
+        : n % 10 === 2 && n % 100 !== 12
+            ? "nd"
+            : n % 10 === 3 && n % 100 !== 13
+                ? "rd"
+                : "th";
+    return date ? ` (${n}${ord} time since ${date})` : ` (${n}×)`;
+}
+function formatSection(title, list, includeSeverity = true) {
+    if (list.length === 0)
+        return "";
+    let section = `### ${title} (${list.length})\n\n`;
+    for (const item of list) {
+        const severityPart = includeSeverity
+            ? ` | severity: ${item.severity}`
+            : "";
+        const recur = recurrenceSuffix(item);
+        section += `• ${item.file}:${item.line} | ${item.failure_type}${severityPart} | confidence: ${item.confidence}${recur}\n`;
+    }
+    return section + "\n";
+}
+function formatDiffComment(diff) {
+    const unchangedFailures = diff.unchangedFailures ?? [];
+    const newFailures = diff.newFailures ?? [];
+    const fixedFailures = diff.fixedFailures ?? [];
+    const flaky = newFailures.filter((f) => f.is_flaky_suspected);
+    const realNewFailures = newFailures.filter((f) => !f.is_flaky_suspected);
+    const commit = process.env.GITHUB_SHA?.slice(0, 7);
+    let body = "## AI Failure Diff Summary\n\n";
+    if (commit)
+        body += `Commit: ${commit}\n`;
+    body += "\n";
+    if (unchangedFailures.length > 0) {
+        body +=
+            "⚠️ **Existing issues** from the base branch are still failing on this branch. This PR was not blocked by them. Fix these on this branch or on the base branch to get to green.\n\n";
+    }
+    body += formatSection("New Issues", realNewFailures);
+    body += formatSection("Flaky", flaky, false);
+    body += formatSection("Still Failing", unchangedFailures);
+    body += formatSection("Fixed Issues", fixedFailures);
+    return body;
+}
+function hasFailureChanges(diff) {
+    return ((diff.newFailures?.length ?? 0) > 0 ||
+        (diff.unchangedFailures?.length ?? 0) > 0 ||
+        (diff.fixedFailures?.length ?? 0) > 0);
+}

package/dist/lib/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/playwright/basePage.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BasePage = void 0;
+const baseTest_1 = require("./baseTest");
+const logger_1 = require("../reporting/logger");
+class BasePage {
+    constructor(page) {
+        this.page = page;
+    }
+    async navigate(path) {
+        const url = path.startsWith("http")
+            ? path
+            : `${baseTest_1.env.BASE_URL}${path}`;
+        logger_1.logger.info(`Navigating to: ${url}`);
+        await this.page.goto(url, { waitUntil: "networkidle" });
+    }
+}
+exports.BasePage = BasePage;

package/dist/playwright/baseTest.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.env = exports.expect = exports.test = void 0;
+const test_1 = require("@playwright/test");
+Object.defineProperty(exports, "expect", { enumerable: true, get: function () { return test_1.expect; } });
+const env_1 = require("../config/env");
+Object.defineProperty(exports, "env", { enumerable: true, get: function () { return env_1.env; } });
+require("./testHooks");
+exports.test = test_1.test;

package/dist/playwright/globalSetup.js

@@ -0,0 +1,14 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = globalSetup;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+async function globalSetup() {
+    const runId = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`;
+    const runDir = path_1.default.join("artifacts", runId);
+    fs_1.default.mkdirSync(runDir, { recursive: true });
+    fs_1.default.writeFileSync(path_1.default.join("artifacts", ".current-run"), runDir);
+}

package/dist/playwright/globalTeardown.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = globalTeardown;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const failureAnalyzer_1 = require("../ai/failureAnalyzer");
+function findMetaFiles(dir, results = []) {
+    const entries = fs_1.default.readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path_1.default.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            findMetaFiles(fullPath, results);
+        }
+        else if (entry.name === "meta.json") {
+            results.push(fullPath);
+        }
+    }
+    return results;
+}
+async function globalTeardown() {
+    if (process.env.AI_ANALYSIS !== "true")
+        return;
+    const runDir = fs_1.default.readFileSync(path_1.default.join("artifacts", ".current-run"), "utf-8");
+    if (!runDir || !fs_1.default.existsSync(runDir))
+        return;
+    const metaFiles = findMetaFiles(runDir);
+    if (metaFiles.length === 0)
+        return;
+    for (const metaPath of metaFiles) {
+        console.log(`Analyzing: ${metaPath}`);
+        try {
+            const analysis = await (0, failureAnalyzer_1.analyzeFailureFile)(metaPath);
+            if (!analysis)
+                continue;
+            const outputFile = metaPath.replace("meta.json", "ai.txt");
+            fs_1.default.writeFileSync(outputFile, analysis);
+            console.log(`Saved AI analysis: ${path_1.default.basename(outputFile)}\n`);
+        }
+        catch (error) {
+            console.error(`AI analysis failed for ${metaPath}:`, error);
+        }
+    }
+}

package/dist/playwright/steps.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.step = step;
+const test_1 = require("@playwright/test");
+async function step(name, fn) {
+    return await test_1.test.step(name, fn);
+}

package/dist/playwright/testHooks.js

@@ -0,0 +1,53 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.test = void 0;
+const test_1 = require("@playwright/test");
+const fs_1 = __importDefault(require("fs"));
+const logger_1 = require("../reporting/logger");
+const path_1 = __importDefault(require("path"));
+exports.test = test_1.test;
+exports.test.afterEach(async ({ page }, testInfo) => {
+    const logFile = (0, logger_1.getLogFilePath)();
+    if (fs_1.default.existsSync(logFile)) {
+        await testInfo.attach("run.log", {
+            path: logFile,
+            contentType: "text/plain",
+        });
+    }
+    const isRealFailure = testInfo.status !== testInfo.expectedStatus;
+    const isFlaky = testInfo.retry > 0 &&
+        testInfo.status === testInfo.expectedStatus;
+    if (isRealFailure || isFlaky) {
+        const screenshot = await page.screenshot({ fullPage: true });
+        await testInfo.attach("failure-screenshot", {
+            body: screenshot,
+            contentType: "image/png",
+        });
+        const runDir = fs_1.default.readFileSync(path_1.default.join("artifacts", ".current-run"), "utf-8");
+        const specName = path_1.default.basename(testInfo.file);
+        const safeTitle = testInfo.title.replace(/[^\w\d]/g, "_");
+        const attemptDir = `attempt-${testInfo.retry}`;
+        const testDir = path_1.default.join(runDir, specName, safeTitle, attemptDir);
+        fs_1.default.mkdirSync(testDir, { recursive: true });
+        const html = await page.content();
+        fs_1.default.writeFileSync(path_1.default.join(testDir, "dom.html"), html);
+        const severity = isFlaky ? "low" : "medium";
+        const meta = {
+            title: testInfo.title,
+            status: testInfo.status,
+            expectedStatus: testInfo.expectedStatus,
+            errorMessage: testInfo.error?.message,
+            stack: testInfo.error?.stack,
+            url: page.url(),
+            project: testInfo.project.name,
+            duration: testInfo.duration,
+            retry: testInfo.retry,
+            is_flaky_suspected: isFlaky,
+            severity: severity
+        };
+        fs_1.default.writeFileSync(path_1.default.join(testDir, "meta.json"), JSON.stringify(meta, null, 2));
+    }
+});

package/dist/reporting/logger.js

@@ -0,0 +1,27 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logger = void 0;
+exports.getLogFilePath = getLogFilePath;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const logsDir = path_1.default.resolve(process.cwd(), "artifacts/logs");
+if (!fs_1.default.existsSync(logsDir))
+    fs_1.default.mkdirSync(logsDir, { recursive: true });
+const logFile = path_1.default.join(logsDir, `run-${new Date().toISOString().replace(/[:.]/g, "-")}.log`);
+function write(level, message) {
+    const line = `${new Date().toISOString()} [${level}] ${message}\n`;
+    fs_1.default.appendFileSync(logFile, line);
+    process.stdout.write(line);
+}
+exports.logger = {
+    info: (msg) => write("INFO", msg),
+    warn: (msg) => write("WARN", msg),
+    error: (msg) => write("ERROR", msg),
+    debug: (msg) => write("DEBUG", msg),
+};
+function getLogFilePath() {
+    return logFile;
+}

package/package.json

@@ -0,0 +1,66 @@
+{
+    "name": "qa-intelligence",
+    "version": "1.0.0",
+    "engines": {
+        "node": ">=18"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "description": "CI intelligence engine for test pipelines. Detects regressions, flaky tests, and failure lifecycle in pull requests.",
+    "keywords": [
+        "ci",
+        "qa",
+        "test-automation",
+        "playwright",
+        "regression-detection",
+        "flaky-tests",
+        "qa-intelligence"
+    ],
+    "license": "MIT",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/ardithaqi/qa-intelligence.git"
+    },
+    "type": "commonjs",
+    "main": "dist/index.js",
+    "exports": {
+        "./playwright": "./dist/playwright/baseTest.js",
+        "./playwright/globalSetup": "./dist/playwright/globalSetup.js",
+        "./playwright/globalTeardown": "./dist/playwright/globalTeardown.js",
+        "./playwright/basePage": "./dist/playwright/basePage.js",
+        "./playwright/steps": "./dist/playwright/steps.js",
+        "./config/env": "./dist/config/env.js"
+    },
+    "peerDependencies": {
+        "@playwright/test": ">=1.40.0"
+    },
+    "bin": {
+        "qa-intelligence": "dist/cli/index.js",
+        "qa-intelligence-diff": "dist/cli/diff.js",
+        "qa-intelligence-history": "dist/cli/history.js",
+        "qa-intelligence-comment": "dist/cli/postComment.js"
+    },
+    "scripts": {
+        "build": "tsc -p tsconfig.json",
+        "prepare": "npm run build",
+        "dev:diff": "ts-node src/cli/diff.ts --baseline baseline-artifacts --current artifacts --out failure-diff.json",
+        "dev:comment": "ts-node src/cli/postComment.ts --diff failure-diff.json --repo owner/repo --pr 1 --token TOKEN"
+    },
+    "dependencies": {
+        "@octokit/rest": "^22.0.1",
+        "dotenv": "^17.3.1",
+        "openai": "^6.29.0",
+        "zod": "^4.3.6"
+    },
+    "devDependencies": {
+        "@playwright/test": "^1.58.2",
+        "@types/node": "^22.0.0",
+        "ts-node": "^10.9.2",
+        "tsx": "^4.21.0",
+        "typescript": "^5.9.3"
+    },
+    "files": [
+        "dist"
+    ]
+}