quality-intelligence-engine 2.2.2 → 2.2.4

package/src/utils/confidence-constants.ts

@@ -0,0 +1,111 @@
+/**
+ * Confidence Calculation Constants
+ * 
+ * These constants define the base confidence levels and adjustments
+ * used throughout the quality intelligence engine.
+ */
+
+/**
+ * Base confidence levels for different error types
+ */
+export const BASE_CONFIDENCE = {
+  /** Strong backend failure signal (5xx errors) */
+  BACKEND_SERVER_ERROR: 0.95,
+  
+  /** Backend API failure (any API error) */
+  BACKEND_API_ERROR: 0.9,
+  
+  /** Test or automation failure */
+  TEST_ERROR: 0.75,
+  
+  /** Unknown or weak signal */
+  UNKNOWN: 0.6,
+} as const;
+
+/**
+ * Confidence adjustments based on test context and signals
+ */
+export const CONFIDENCE_ADJUSTMENTS = {
+  /** UI successfully rendered and reached assertion */
+  UI_RENDERED: 0.05,
+  
+  /** Test involves authentication or login flow */
+  AUTH_FLOW_DETECTED: 0.05,
+  
+  /** Explicit authentication failure detected */
+  EXPLICIT_AUTH_FAILURE: 0.1,
+  
+  /** Test involves inventory or data operations */
+  DATA_OPERATION_DETECTED: 0.05,
+  
+  /** Numeric mismatch detected in assertions */
+  NUMERIC_MISMATCH: 0.05,
+} as const;
+
+/**
+ * Maximum and minimum confidence bounds
+ */
+export const CONFIDENCE_BOUNDS = {
+  /** Maximum confidence score (never 100% certain) */
+  MAX: 0.95,
+  
+  /** Minimum confidence score for reporting */
+  MIN: 0.5,
+} as const;
+
+/**
+ * Confidence thresholds for decision making
+ */
+export const CONFIDENCE_THRESHOLDS = {
+  /** Threshold for blocking CI/CD (high confidence failure) */
+  FAIL: 0.85,
+  
+  /** Threshold for reporting pass risks */
+  PASS_RISK: 0.6,
+  
+  /** Threshold for investigating (medium confidence) */
+  INVESTIGATE: 0.7,
+  
+  /** Threshold for shipping (high confidence in safety) */
+  SHIP: 0.85,
+} as const;
+
+/**
+ * Taxonomy rule base confidence levels
+ */
+export const TAXONOMY_BASE_CONFIDENCE = {
+  /** Numeric data mismatch */
+  NUMERIC_MISMATCH: 0.6,
+  
+  /** Authentication failure */
+  AUTH_FAILURE: 0.7,
+  
+  /** UI visibility issue */
+  VISIBILITY: 0.7,
+  
+  /** Network timeout */
+  TIMEOUT: 0.65,
+  
+  /** Element not found */
+  ELEMENT_NOT_FOUND: 0.7,
+  
+  /** Default fallback */
+  DEFAULT: 0.7,
+} as const;
+
+/**
+ * Helper function to clamp confidence within bounds
+ */
+export function clampConfidence(value: number): number {
+  return Math.max(
+    CONFIDENCE_BOUNDS.MIN,
+    Math.min(CONFIDENCE_BOUNDS.MAX, value)
+  );
+}
+
+/**
+ * Helper function to format confidence as percentage string
+ */
+export function formatConfidence(value: number): string {
+  return `${(value * 100).toFixed(1)}%`;
+}

package/src/utils/file-utils.ts

@@ -0,0 +1,146 @@
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Custom error class for file operations
+ */
+export class FileOperationError extends Error {
+  constructor(
+    message: string,
+    public readonly filePath: string,
+    public readonly operation: 'read' | 'write' | 'parse',
+    public readonly originalError?: Error
+  ) {
+    super(message);
+    this.name = 'FileOperationError';
+  }
+}
+
+/**
+ * Safely read and parse a JSON file with proper error handling
+ * @param filePath - Absolute or relative path to the JSON file
+ * @returns Parsed JSON object
+ * @throws FileOperationError if file cannot be read or parsed
+ */
+export function readJsonFile<T>(filePath: string): T {
+  try {
+    // Check if file exists
+    if (!fs.existsSync(filePath)) {
+      throw new FileOperationError(
+        `File not found: ${filePath}`,
+        filePath,
+        'read'
+      );
+    }
+
+    // Read file content
+    let content: string;
+    try {
+      content = fs.readFileSync(filePath, 'utf-8');
+    } catch (error) {
+      throw new FileOperationError(
+        `Failed to read file: ${filePath}`,
+        filePath,
+        'read',
+        error as Error
+      );
+    }
+
+    // Parse JSON
+    try {
+      return JSON.parse(content) as T;
+    } catch (error) {
+      throw new FileOperationError(
+        `Invalid JSON in file: ${filePath}`,
+        filePath,
+        'parse',
+        error as Error
+      );
+    }
+  } catch (error) {
+    if (error instanceof FileOperationError) {
+      throw error;
+    }
+    throw new FileOperationError(
+      `Unexpected error reading ${filePath}: ${(error as Error).message}`,
+      filePath,
+      'read',
+      error as Error
+    );
+  }
+}
+
+/**
+ * Safely write JSON to a file with proper error handling
+ * @param filePath - Absolute or relative path to the JSON file
+ * @param data - Data to write (will be JSON.stringified)
+ * @param pretty - Whether to pretty-print the JSON (default: true)
+ */
+export function writeJsonFile<T>(
+  filePath: string,
+  data: T,
+  pretty: boolean = true
+): void {
+  try {
+    // Ensure directory exists
+    const dir = path.dirname(filePath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    // Stringify data
+    const content = pretty
+      ? JSON.stringify(data, null, 2)
+      : JSON.stringify(data);
+
+    // Write to file
+    try {
+      fs.writeFileSync(filePath, content, 'utf-8');
+    } catch (error) {
+      throw new FileOperationError(
+        `Failed to write file: ${filePath}`,
+        filePath,
+        'write',
+        error as Error
+      );
+    }
+  } catch (error) {
+    if (error instanceof FileOperationError) {
+      throw error;
+    }
+    throw new FileOperationError(
+      `Unexpected error writing ${filePath}: ${(error as Error).message}`,
+      filePath,
+      'write',
+      error as Error
+    );
+  }
+}
+
+/**
+ * Check if a file exists
+ * @param filePath - Path to check
+ * @returns true if file exists
+ */
+export function fileExists(filePath: string): boolean {
+  return fs.existsSync(filePath);
+}
+
+/**
+ * Ensure a directory exists, creating it if necessary
+ * @param dirPath - Directory path to ensure
+ */
+export function ensureDirectory(dirPath: string): void {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+/**
+ * Resolve a path relative to the project root
+ * @param relativePath - Path relative to project root
+ * @returns Absolute path
+ */
+export function resolvePath(...segments: string[]): string {
+  return path.join(process.cwd(), ...segments);
+}

package/src/v2/llm/llm-advisor.ts

@@ -0,0 +1,22 @@
+// src/v2/llm/llm-advisor.ts
+
+export interface LLMAdvice {
+  summary: string;
+  fixRecommendation: string[];
+}
+
+/**
+ * Stub for ChatGPT / Azure / Local LLM
+ */
+export async function getLLMAdvice(
+  failureSummary: string
+): Promise<LLMAdvice> {
+  return {
+    summary: `AI analysis of failure: ${failureSummary}`,
+    fixRecommendation: [
+      "Check backend API response",
+      "Validate UI rendering timing",
+      "Review recent code changes"
+    ]
+  };
+}

package/src/v2/pipeline/v2-intelligence.pipeline.ts

@@ -0,0 +1,21 @@
+// src/v2/pipeline/v2-intelligence.pipeline.ts
+
+import { analyzeTrace } from "../trace-intelligence/trace-analyzer";
+import { attemptSelfHeal } from "../self-healing/self-healer";
+import { getLLMAdvice } from "../llm/llm-advisor";
+
+export async function runV2Intelligence(
+  failureFolder: string,
+  failureType: string,
+  errorMessage: string
+) {
+  const trace = analyzeTrace(failureFolder);
+  const healing = attemptSelfHeal(failureType, errorMessage);
+  const llm = await getLLMAdvice(failureType);
+
+  return {
+    trace,
+    healing,
+    llm
+  };
+}

package/src/v2/self-healing/self-healer.ts

@@ -0,0 +1,60 @@
+// src/v2/self-healing/self-healer.ts
+
+export interface HealingSuggestion {
+  reason: string;
+  suggestedFix: string;
+  confidence: number;
+}
+
+/**
+ * Self-Healing Decision Engine (V2)
+ * Decides WHAT can be auto-fixed and WHAT should not
+ */
+export function attemptSelfHeal(
+  failureType: string,
+  errorMessage: string
+): HealingSuggestion | null {
+
+  const msg = errorMessage.toLowerCase();
+
+  // 🔁 Flaky tests
+  if (failureType === "FLAKY") {
+    return {
+      reason: "Intermittent retry-based failure detected",
+      suggestedFix: "Add proper waits or retry assertions",
+      confidence: 0.6
+    };
+  }
+
+  // 🧭 UI / Locator issues
+  if (failureType === "UI_STATE" || failureType === "LOCATOR_CHANGED") {
+    return {
+      reason: "UI element state or locator changed",
+      suggestedFix: "Re-locate element using role/text-based selectors",
+      confidence: 0.7
+    };
+  }
+
+  // 🔢 Numeric mismatch — CONDITIONAL healing
+  if (failureType === "NUMERIC_MISMATCH") {
+
+    // UI-level numeric problems (healable)
+    if (
+      msg.includes("count") ||
+      msg.includes("items") ||
+      msg.includes("length") ||
+      msg.includes("round")
+    ) {
+      return {
+        reason: "UI count or rounding issue detected",
+        suggestedFix: "Update assertion logic or normalize UI values",
+        confidence: 0.55
+      };
+    }
+
+    // Backend/API numeric issues (DO NOT HEAL)
+    return null;
+  }
+
+  return null;
+}

package/src/v2/trace-intelligence/trace-analyzer.ts

@@ -0,0 +1,42 @@
+// src/v2/trace-intelligence/trace-analyzer.ts
+
+import fs from "fs";
+import path from "path";
+
+export interface TraceInsight {
+  page: string;
+  failureStep: string;
+  screenshots: string[];
+  traceFile?: string;
+  note?: string;
+}
+
+export function analyzeTrace(failureFolder: string): TraceInsight {
+  const fullPath = path.resolve(failureFolder);
+
+  if (!fs.existsSync(fullPath)) {
+    return {
+      page: "UNKNOWN",
+      failureStep: "Trace folder not found",
+      screenshots: [],
+      note: `Missing folder: ${failureFolder}`
+    };
+  }
+
+  const files = fs.readdirSync(fullPath);
+
+  const screenshots = files
+    .filter(f => f.endsWith(".png") || f.endsWith(".jpg"))
+    .map(f => path.join(fullPath, f));
+
+  const traceFile = files.includes("trace.zip")
+    ? path.join(fullPath, "trace.zip")
+    : undefined;
+
+  return {
+    page: "UNKNOWN_PAGE",
+    failureStep: "Action before failure",
+    screenshots,
+    traceFile
+  };
+}

package/src/v2-test-run.ts

@@ -0,0 +1,74 @@
+// src/v2-test-run.ts
+
+import fs from "fs";
+import path from "path";
+import { runV2Intelligence } from "./v2/pipeline/v2-intelligence.pipeline";
+
+function pickOneFailureFolder(): string | null {
+  const base = "playwright-results";
+
+  if (!fs.existsSync(base)) return null;
+
+  const entries = fs.readdirSync(base, { withFileTypes: true });
+  const folder = entries.find(e => e.isDirectory());
+
+  return folder ? path.join(base, folder.name) : null;
+}
+
+async function run() {
+  const failureFolder = pickOneFailureFolder();
+
+  if (!failureFolder) {
+    console.error("No Playwright failure folders found.");
+    return;
+  }
+
+  const result = await runV2Intelligence(
+    failureFolder,
+    "NUMERIC_MISMATCH",
+    "Expected 10, received 6"
+  );
+
+  console.log("==================================================");
+  console.log("V2 TRACE INTELLIGENCE OUTPUT");
+  console.log("==================================================");
+
+  console.log(`Failure Folder : ${failureFolder}`);
+  console.log(`Failure Type   : NUMERIC_MISMATCH`);
+
+  console.log("\nEvidence:");
+  if (result.trace.screenshots.length > 0) {
+    console.log("  Screenshots:");
+    result.trace.screenshots.forEach(s =>
+      console.log(`    - ${s}`)
+    );
+  } else {
+    console.log("  Screenshots: none");
+  }
+
+  if (result.trace.traceFile) {
+    console.log(`  Trace File  : ${result.trace.traceFile}`);
+    console.log(`  Open with   : npx playwright show-trace ${result.trace.traceFile}`);
+  } else {
+    console.log("  Trace File  : none");
+  }
+
+  console.log("\nSelf-Healing:");
+  if (result.healing) {
+    console.log(`  Reason       : ${result.healing.reason}`);
+    console.log(`  SuggestedFix : ${result.healing.suggestedFix}`);
+    console.log(`  Confidence   : ${result.healing.confidence}`);
+  } else {
+    console.log("  Not safe to auto-heal");
+  }
+
+  console.log("\nAI Advice:");
+  console.log(`  Summary: ${result.llm.summary}`);
+  result.llm.fixRecommendation.forEach(r =>
+    console.log(`    - ${r}`)
+  );
+
+  console.log("==================================================");
+}
+
+run();

package/tests/fixtures.ts

@@ -0,0 +1,40 @@
+import { test as base, expect } from '@playwright/test';
+
+type FailureHint = {
+  type: 'API_BUG' | 'UI_BUG';
+  reason: string;
+};
+
+export const test = base.extend({
+  page: async ({ page }, use, testInfo) => {
+    try {
+      await use(page);
+    } catch (error: any) {
+      const message = error?.message ?? '';
+
+      let hint: FailureHint | null = null;
+
+      // ✅ Numeric assertion mismatch → API_BUG
+      if (
+        /Expected:\s*\d+/i.test(message) &&
+        /Received:\s*\d+/i.test(message)
+      ) {
+        hint = {
+          type: 'API_BUG',
+          reason: 'Numeric data mismatch inferred from UI assertion'
+        };
+      }
+
+      if (hint) {
+        await testInfo.attach('failure-hint', {
+          body: JSON.stringify(hint, null, 2),
+          contentType: 'application/json'
+        });
+      }
+
+      throw error;
+    }
+  }
+});
+
+export { expect };

package/tests/saucedemo-login-validation.spec.ts

@@ -0,0 +1,74 @@
+// tests/saucedemo-login-validation.spec.ts
+import { test, expect } from '@playwright/test';
+
+const URL = 'https://www.saucedemo.com/';
+const USERNAME = '#user-name';
+const PASSWORD = '#password';
+const LOGIN_BTN = '#login-button';
+const ERROR = '[data-test="error"]';
+
+/**
+ * 1️⃣ VALID LOGIN
+ */
+test('Login | VALID | standard_user should login successfully', async ({ page }) => {
+  await page.goto(URL);
+
+  await page.fill(USERNAME, 'standard_user');
+  await page.fill(PASSWORD, 'secret_sauce');
+  await page.click(LOGIN_BTN);
+
+  await expect(page).toHaveURL(/inventory.html/);
+});
+
+/**
+ * 2️⃣ INVALID USERNAME
+ */
+test('Login | INVALID | wrong username should fail', async ({ page }) => {
+  await page.goto(URL);
+
+  await page.fill(USERNAME, 'invalid_user');
+  await page.fill(PASSWORD, 'secret_sauce');
+  await page.click(LOGIN_BTN);
+
+  await expect(page.locator(ERROR)).toBeVisible();
+  await expect(page.locator(ERROR)).toContainText(/do not match/i);
+});
+
+/**
+ * 3️⃣ LOCKED OUT USER
+ */
+test('Login | LOCKED | locked_out_user should be blocked', async ({ page }) => {
+  await page.goto(URL);
+
+  await page.fill(USERNAME, 'locked_out_user');
+  await page.fill(PASSWORD, 'secret_sauce');
+  await page.click(LOGIN_BTN);
+
+  await expect(page.locator(ERROR)).toBeVisible();
+  await expect(page.locator(ERROR)).toContainText(/locked out/i);
+});
+
+/**
+ * 4️⃣ EMPTY CREDENTIALS
+ */
+test('Login | EMPTY | username and password required', async ({ page }) => {
+  await page.goto(URL);
+
+  await page.click(LOGIN_BTN);
+
+  await expect(page.locator(ERROR)).toBeVisible();
+  await expect(page.locator(ERROR)).toContainText(/Username is required/i);
+});
+
+/**
+ * 5️⃣ SQL-LIKE INPUT (SECURITY SANITY)
+ */
+test('Login | SECURITY | SQL-like input should be rejected', async ({ page }) => {
+  await page.goto(URL);
+
+  await page.fill(USERNAME, `' OR 1=1 --`);
+  await page.fill(PASSWORD, `' OR 1=1 --`);
+  await page.click(LOGIN_BTN);
+
+  await expect(page.locator(ERROR)).toBeVisible();
+});

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": ".",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "target": "ES2022",
+    "esModuleInterop": true,
+    "strict": true,
+    "skipLibCheck": true
+  },
+  "include": [
+    "src/**/*",
+    "bin/**/*"
+  ]
+}

package/tsconfig.playwright.json

@@ -0,0 +1,13 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "types": ["node", "@playwright/test"],
+    "noImplicitAny": true,
+    "strict": true
+  },
+  "include": [
+    "tests/**/*.ts",
+    "tests/**/*.tsx",
+    "src/**/*.ts"
+  ]
+}