qai-cli 3.0.0

package/scripts/visual-regression.cjs

@@ -0,0 +1,339 @@
+/**
+ * Visual Regression Testing Utility
+ * Compares screenshots between runs to detect visual changes
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { PNG } = require('pngjs');
+const pixelmatch = require('pixelmatch');
+
+/**
+ * Compare two images and generate a diff
+ *
+ * @param {string} baselinePath - Path to baseline image
+ * @param {string} currentPath - Path to current image
+ * @param {string} diffPath - Path to save diff image
+ * @param {Object} options - Comparison options
+ * @returns {Promise<{match: boolean, diffPixels: number, diffPercent: number, dimensions: Object}>}
+ */
+async function compareImages(baselinePath, currentPath, diffPath, options = {}) {
+  const { threshold = 0.1, includeAA = false } = options;
+
+  // Read images
+  const baseline = PNG.sync.read(fs.readFileSync(baselinePath));
+  const current = PNG.sync.read(fs.readFileSync(currentPath));
+
+  // Check dimensions match
+  if (baseline.width !== current.width || baseline.height !== current.height) {
+    return {
+      match: false,
+      error: 'dimension_mismatch',
+      baseline: { width: baseline.width, height: baseline.height },
+      current: { width: current.width, height: current.height },
+      diffPixels: -1,
+      diffPercent: 100,
+    };
+  }
+
+  const { width, height } = baseline;
+  const diff = new PNG({ width, height });
+
+  // Compare pixels
+  const diffPixels = pixelmatch(baseline.data, current.data, diff.data, width, height, {
+    threshold,
+    includeAA,
+    alpha: 0.1,
+    diffColor: [255, 0, 0], // Red for differences
+    diffColorAlt: [0, 255, 0], // Green for anti-aliasing
+  });
+
+  // Calculate percentage
+  const totalPixels = width * height;
+  const diffPercent = (diffPixels / totalPixels) * 100;
+
+  // Save diff image
+  if (diffPath) {
+    fs.mkdirSync(path.dirname(diffPath), { recursive: true });
+    fs.writeFileSync(diffPath, PNG.sync.write(diff));
+  }
+
+  return {
+    match: diffPixels === 0,
+    diffPixels,
+    diffPercent: parseFloat(diffPercent.toFixed(2)),
+    dimensions: { width, height },
+  };
+}
+
+/**
+ * Compare all screenshots in two directories
+ *
+ * @param {string} baselineDir - Directory with baseline screenshots
+ * @param {string} currentDir - Directory with current screenshots
+ * @param {string} diffDir - Directory to save diff images
+ * @param {Object} options - Comparison options
+ * @returns {Promise<{summary: Object, results: Array}>}
+ */
+async function compareDirectories(baselineDir, currentDir, diffDir, options = {}) {
+  const { threshold = 0.1, failThreshold = 0.5 } = options;
+
+  const results = [];
+  let passed = 0;
+  let failed = 0;
+  let missing = 0;
+  let newImages = 0;
+
+  // Get all PNG files from both directories
+  const baselineFiles = fs.existsSync(baselineDir)
+    ? fs.readdirSync(baselineDir).filter((f) => f.endsWith('.png'))
+    : [];
+  const currentFiles = fs.existsSync(currentDir)
+    ? fs.readdirSync(currentDir).filter((f) => f.endsWith('.png'))
+    : [];
+
+  const allFiles = new Set([...baselineFiles, ...currentFiles]);
+
+  for (const file of allFiles) {
+    const baselinePath = path.join(baselineDir, file);
+    const currentPath = path.join(currentDir, file);
+    const diffPath = path.join(diffDir, `diff-${file}`);
+
+    const result = { file };
+
+    if (!fs.existsSync(baselinePath)) {
+      // New screenshot (no baseline)
+      result.status = 'new';
+      result.message = 'New screenshot - no baseline to compare';
+      newImages++;
+    } else if (!fs.existsSync(currentPath)) {
+      // Missing screenshot (was in baseline)
+      result.status = 'missing';
+      result.message = 'Screenshot missing from current run';
+      missing++;
+    } else {
+      // Compare images
+      const comparison = await compareImages(baselinePath, currentPath, diffPath, { threshold });
+
+      if (comparison.error === 'dimension_mismatch') {
+        result.status = 'failed';
+        result.message = `Dimension mismatch: baseline ${comparison.baseline.width}x${comparison.baseline.height} vs current ${comparison.current.width}x${comparison.current.height}`;
+        result.diffPath = null;
+        failed++;
+      } else if (comparison.diffPercent > failThreshold) {
+        result.status = 'failed';
+        result.message = `${comparison.diffPercent}% pixels differ (threshold: ${failThreshold}%)`;
+        result.diffPixels = comparison.diffPixels;
+        result.diffPercent = comparison.diffPercent;
+        result.diffPath = diffPath;
+        failed++;
+      } else if (comparison.diffPixels > 0) {
+        result.status = 'warning';
+        result.message = `Minor differences: ${comparison.diffPercent}% pixels differ`;
+        result.diffPixels = comparison.diffPixels;
+        result.diffPercent = comparison.diffPercent;
+        result.diffPath = diffPath;
+        passed++;
+      } else {
+        result.status = 'passed';
+        result.message = 'Images match exactly';
+        result.diffPercent = 0;
+        passed++;
+      }
+    }
+
+    results.push(result);
+  }
+
+  return {
+    summary: {
+      total: allFiles.size,
+      passed,
+      failed,
+      missing,
+      new: newImages,
+      passRate: allFiles.size > 0 ? parseFloat(((passed / allFiles.size) * 100).toFixed(1)) : 100,
+    },
+    results,
+  };
+}
+
+/**
+ * Generate markdown report for visual regression results
+ *
+ * @param {Object} comparison - Result from compareDirectories
+ * @returns {string} Markdown formatted report
+ */
+function generateReport(comparison) {
+  const { summary, results } = comparison;
+
+  let report = `## Visual Regression Report\n\n`;
+
+  // Summary
+  report += `### Summary\n`;
+  report += `- **Total Screenshots**: ${summary.total}\n`;
+  report += `- **Passed**: ${summary.passed} ✅\n`;
+  report += `- **Failed**: ${summary.failed} ❌\n`;
+  report += `- **New**: ${summary.new} 🆕\n`;
+  report += `- **Missing**: ${summary.missing} ⚠️\n`;
+  report += `- **Pass Rate**: ${summary.passRate}%\n\n`;
+
+  // Failed comparisons
+  const failed = results.filter((r) => r.status === 'failed');
+  if (failed.length > 0) {
+    report += `### Failed Comparisons ❌\n\n`;
+    failed.forEach((r) => {
+      report += `#### ${r.file}\n`;
+      report += `- **Status**: Failed\n`;
+      report += `- **Reason**: ${r.message}\n`;
+      if (r.diffPath) {
+        report += `- **Diff Image**: ${path.basename(r.diffPath)}\n`;
+      }
+      report += `\n`;
+    });
+  }
+
+  // Warnings (minor differences)
+  const warnings = results.filter((r) => r.status === 'warning');
+  if (warnings.length > 0) {
+    report += `### Minor Differences ⚠️\n\n`;
+    warnings.forEach((r) => {
+      report += `- **${r.file}**: ${r.diffPercent}% difference\n`;
+    });
+    report += `\n`;
+  }
+
+  // New screenshots
+  const newScreenshots = results.filter((r) => r.status === 'new');
+  if (newScreenshots.length > 0) {
+    report += `### New Screenshots 🆕\n\n`;
+    report += `These screenshots have no baseline yet:\n`;
+    newScreenshots.forEach((r) => {
+      report += `- ${r.file}\n`;
+    });
+    report += `\n`;
+  }
+
+  // Missing screenshots
+  const missingScreenshots = results.filter((r) => r.status === 'missing');
+  if (missingScreenshots.length > 0) {
+    report += `### Missing Screenshots ⚠️\n\n`;
+    report += `These screenshots existed in baseline but not in current run:\n`;
+    missingScreenshots.forEach((r) => {
+      report += `- ${r.file}\n`;
+    });
+    report += `\n`;
+  }
+
+  return report;
+}
+
+/**
+ * Update baseline screenshots from current run
+ *
+ * @param {string} currentDir - Directory with current screenshots
+ * @param {string} baselineDir - Directory to save as baseline
+ * @param {Object} options - Options
+ * @returns {Object} Update summary
+ */
+function updateBaseline(currentDir, baselineDir, options = {}) {
+  const { overwrite = true } = options;
+
+  fs.mkdirSync(baselineDir, { recursive: true });
+
+  const currentFiles = fs.readdirSync(currentDir).filter((f) => f.endsWith('.png'));
+  let copied = 0;
+  let skipped = 0;
+
+  currentFiles.forEach((file) => {
+    const src = path.join(currentDir, file);
+    const dest = path.join(baselineDir, file);
+
+    if (!overwrite && fs.existsSync(dest)) {
+      skipped++;
+    } else {
+      fs.copyFileSync(src, dest);
+      copied++;
+    }
+  });
+
+  return {
+    copied,
+    skipped,
+    total: currentFiles.length,
+  };
+}
+
+/**
+ * CLI interface
+ */
+async function main() {
+  const args = process.argv.slice(2);
+  const command = args[0];
+
+  if (command === 'compare') {
+    const baselineDir = args[1] || './screenshots/baseline';
+    const currentDir = args[2] || './screenshots';
+    const diffDir = args[3] || './screenshots/diff';
+
+    console.log(`Comparing screenshots...`);
+    console.log(`  Baseline: ${baselineDir}`);
+    console.log(`  Current: ${currentDir}`);
+    console.log(`  Diff output: ${diffDir}`);
+
+    const result = await compareDirectories(baselineDir, currentDir, diffDir);
+    const report = generateReport(result);
+
+    console.log('\n' + report);
+
+    // Save report
+    fs.writeFileSync('visual-regression-report.md', report);
+    console.log('Report saved to visual-regression-report.md');
+
+    // Exit with error if any failed
+    if (result.summary.failed > 0) {
+      process.exit(1);
+    }
+  } else if (command === 'update-baseline') {
+    const currentDir = args[1] || './screenshots';
+    const baselineDir = args[2] || './screenshots/baseline';
+
+    console.log(`Updating baseline screenshots...`);
+    console.log(`  Source: ${currentDir}`);
+    console.log(`  Baseline: ${baselineDir}`);
+
+    const result = updateBaseline(currentDir, baselineDir);
+    console.log(`\nBaseline updated:`);
+    console.log(`  Copied: ${result.copied}`);
+    console.log(`  Skipped: ${result.skipped}`);
+    console.log(`  Total: ${result.total}`);
+  } else {
+    console.log(`
+Visual Regression Testing Utility
+
+Usage:
+  node visual-regression.js compare [baselineDir] [currentDir] [diffDir]
+    Compare screenshots and generate diff images
+
+  node visual-regression.js update-baseline [currentDir] [baselineDir]
+    Update baseline screenshots from current run
+
+Examples:
+  node visual-regression.js compare ./baseline ./screenshots ./diff
+  node visual-regression.js update-baseline ./screenshots ./baseline
+    `);
+  }
+}
+
+// Export functions for programmatic use
+module.exports = {
+  compareImages,
+  compareDirectories,
+  generateReport,
+  updateBaseline,
+};
+
+// Run CLI if executed directly
+if (require.main === module) {
+  main().catch(console.error);
+}

package/src/analyze.js

@@ -0,0 +1,365 @@
+/**
+ * qaie - Library Export
+ *
+ * Use this in your Playwright tests to add AI-powered QA analysis.
+ *
+ * @example
+ * import { test, expect } from '@playwright/test';
+ * import { analyzeWithAI } from 'qaie';
+ *
+ * test('AI QA: homepage', async ({ page }) => {
+ *   await page.goto('/');
+ *   const report = await analyzeWithAI(page);
+ *
+ *   // Attach screenshots to test report
+ *   for (const screenshot of report.screenshots) {
+ *     await test.info().attach(screenshot.name, {
+ *       body: screenshot.buffer,
+ *       contentType: 'image/png'
+ *     });
+ *   }
+ *
+ *   expect(report.criticalBugs).toHaveLength(0);
+ * });
+ */
+
+const { getProvider, createProvider } = require('./providers');
+
+/**
+ * Viewport configurations
+ */
+const VIEWPORT_CONFIGS = {
+  mobile: { width: 375, height: 667, name: 'mobile' },
+  tablet: { width: 768, height: 1024, name: 'tablet' },
+  desktop: { width: 1920, height: 1080, name: 'desktop' },
+};
+
+/**
+ * Analyze a page with AI
+ *
+ * @param {import('playwright').Page} page - Playwright page object
+ * @param {Object} options - Analysis options
+ * @param {string[]} [options.viewports=['desktop', 'mobile']] - Viewports to test
+ * @param {string} [options.focus='all'] - Focus area (all, accessibility, performance, forms, visual)
+ * @param {string} [options.provider] - LLM provider (anthropic, openai, gemini, ollama)
+ * @param {string} [options.apiKey] - API key (uses env var if not provided)
+ * @returns {Promise<AnalysisReport>} Analysis report with bugs, screenshots, and recommendations
+ */
+async function analyzeWithAI(page, options = {}) {
+  const {
+    viewports = ['desktop', 'mobile'],
+    focus = 'all',
+    provider: providerName,
+    apiKey,
+  } = options;
+
+  const startTime = Date.now();
+
+  // Get or create provider
+  let provider;
+  if (providerName && apiKey) {
+    provider = createProvider(providerName, apiKey);
+  } else {
+    provider = getProvider();
+  }
+
+  // Capture page data
+  const captureData = await capturePageData(page, viewports);
+
+  // Analyze with AI
+  const analysis = await provider.analyze(captureData, { focus });
+
+  // Build report
+  const report = {
+    url: page.url(),
+    title: await page.title(),
+    timestamp: new Date().toISOString(),
+    duration: `${((Date.now() - startTime) / 1000).toFixed(1)}s`,
+    score: analysis.score,
+    summary: analysis.summary,
+    bugs: analysis.bugs || [],
+    criticalBugs: (analysis.bugs || []).filter(
+      (b) => b.severity === 'critical' || b.severity === 'high',
+    ),
+    recommendations: analysis.recommendations || [],
+    consoleErrors: captureData.consoleErrors,
+    networkErrors: captureData.networkErrors,
+    screenshots: captureData.screenshots,
+    viewports,
+    focus,
+  };
+
+  return report;
+}
+
+/**
+ * Capture page data including screenshots, console errors, and network errors
+ *
+ * @param {import('playwright').Page} page - Playwright page object
+ * @param {string[]} viewports - Viewports to capture
+ * @returns {Promise<CaptureData>}
+ */
+async function capturePageData(page, viewports) {
+  const consoleErrors = [];
+  const networkErrors = [];
+  const screenshots = [];
+
+  // Set up console listener
+  const consoleHandler = (msg) => {
+    if (msg.type() === 'error') {
+      consoleErrors.push(msg.text());
+    }
+  };
+  page.on('console', consoleHandler);
+
+  // Set up network error listener
+  const requestFailedHandler = (request) => {
+    networkErrors.push({
+      url: request.url(),
+      method: request.method(),
+      failure: request.failure()?.errorText || 'Unknown error',
+    });
+  };
+  page.on('requestfailed', requestFailedHandler);
+
+  // Set up response listener for HTTP errors
+  const responseHandler = (response) => {
+    if (response.status() >= 400) {
+      networkErrors.push({
+        url: response.url(),
+        method: response.request().method(),
+        status: response.status(),
+      });
+    }
+  };
+  page.on('response', responseHandler);
+
+  // Capture ARIA snapshot for accessibility analysis
+  let ariaSnapshot = null;
+  try {
+    ariaSnapshot = await page.accessibility.snapshot();
+    if (ariaSnapshot) {
+      ariaSnapshot = JSON.stringify(ariaSnapshot, null, 2).slice(0, 8000);
+    }
+  } catch {
+    // accessibility.snapshot() may not be available in all contexts
+  }
+
+  // Capture DOM summary
+  let domSummary = null;
+  try {
+    /* eslint-disable no-undef */
+    domSummary = await page.evaluate(() => {
+      const headings = [...document.querySelectorAll('h1,h2,h3,h4,h5,h6')].map(
+        (h) => `${h.tagName}: ${h.textContent.trim().slice(0, 80)}`,
+      );
+      const links = document.querySelectorAll('a[href]').length;
+      const buttons = document.querySelectorAll('button').length;
+      const inputs = document.querySelectorAll('input,textarea,select').length;
+      const images = [...document.querySelectorAll('img')].map((img) => ({
+        src: img.src.slice(0, 100),
+        alt: img.alt || '(no alt)',
+      }));
+      const noAlt = images.filter((i) => i.alt === '(no alt)').length;
+
+      return [
+        'Headings: ' + (headings.join(' | ') || 'None'),
+        'Links: ' + links + ', Buttons: ' + buttons + ', Inputs: ' + inputs,
+        'Images: ' + images.length + ' total, ' + noAlt + ' missing alt text',
+      ].join('\n');
+    });
+    /* eslint-enable no-undef */
+  } catch {
+    // DOM evaluation may fail in some contexts
+  }
+
+  // Store original viewport
+  const originalViewport = page.viewportSize();
+
+  // Capture screenshots at each viewport
+  for (const viewportName of viewports) {
+    const config = VIEWPORT_CONFIGS[viewportName.toLowerCase()];
+    if (!config) {
+      console.warn(`Unknown viewport: ${viewportName}, skipping`);
+      continue;
+    }
+
+    // Set viewport
+    await page.setViewportSize({ width: config.width, height: config.height });
+
+    // Wait for any layout shifts
+    await page.waitForTimeout(500);
+
+    // Capture screenshot
+    const buffer = await page.screenshot({ fullPage: true });
+
+    screenshots.push({
+      name: `${config.name}-${config.width}x${config.height}`,
+      viewport: config.name,
+      width: config.width,
+      height: config.height,
+      buffer,
+      base64: buffer.toString('base64'),
+    });
+  }
+
+  // Restore original viewport
+  if (originalViewport) {
+    await page.setViewportSize(originalViewport);
+  }
+
+  // Clean up listeners
+  page.off('console', consoleHandler);
+  page.off('requestfailed', requestFailedHandler);
+  page.off('response', responseHandler);
+
+  return {
+    pageUrl: page.url(),
+    pageTitle: await page.title(),
+    timestamp: new Date().toISOString(),
+    consoleErrors,
+    networkErrors,
+    screenshots,
+    ariaSnapshot,
+    domSummary,
+  };
+}
+
+/**
+ * Create a Playwright test helper that runs AI analysis
+ * Use this to create reusable test fixtures
+ *
+ * @param {Object} defaultOptions - Default options for all analyses
+ * @returns {Function} Configured analyzeWithAI function
+ *
+ * @example
+ * // In your playwright fixtures
+ * import { createAnalyzer } from 'qaie';
+ *
+ * const analyzeWithAI = createAnalyzer({
+ *   viewports: ['desktop', 'mobile', 'tablet'],
+ *   focus: 'accessibility',
+ * });
+ *
+ * // In your tests
+ * test('homepage', async ({ page }) => {
+ *   await page.goto('/');
+ *   const report = await analyzeWithAI(page);
+ * });
+ */
+function createAnalyzer(defaultOptions = {}) {
+  return (page, options = {}) => {
+    return analyzeWithAI(page, { ...defaultOptions, ...options });
+  };
+}
+
+/**
+ * Attach all screenshots from a report to the Playwright test
+ * Convenience helper for test files
+ *
+ * @param {Object} testInfo - Playwright test.info() object
+ * @param {AnalysisReport} report - The AI analysis report
+ *
+ * @example
+ * test('AI QA', async ({ page }, testInfo) => {
+ *   await page.goto('/');
+ *   const report = await analyzeWithAI(page);
+ *   await attachScreenshots(testInfo, report);
+ *   expect(report.criticalBugs).toHaveLength(0);
+ * });
+ */
+async function attachScreenshots(testInfo, report) {
+  for (const screenshot of report.screenshots) {
+    await testInfo.attach(screenshot.name, {
+      body: screenshot.buffer,
+      contentType: 'image/png',
+    });
+  }
+}
+
+/**
+ * Attach bug details as a test attachment
+ *
+ * @param {Object} testInfo - Playwright test.info() object
+ * @param {AnalysisReport} report - The AI analysis report
+ */
+async function attachBugReport(testInfo, report) {
+  const bugReport = {
+    score: report.score,
+    summary: report.summary,
+    bugs: report.bugs,
+    recommendations: report.recommendations,
+    consoleErrors: report.consoleErrors,
+    networkErrors: report.networkErrors,
+  };
+
+  await testInfo.attach('qai-report', {
+    body: JSON.stringify(bugReport, null, 2),
+    contentType: 'application/json',
+  });
+}
+
+module.exports = {
+  analyzeWithAI,
+  createAnalyzer,
+  attachScreenshots,
+  attachBugReport,
+  capturePageData,
+  VIEWPORT_CONFIGS,
+};
+
+/**
+ * @typedef {Object} AnalysisReport
+ * @property {string} url - Page URL
+ * @property {string} title - Page title
+ * @property {string} timestamp - ISO timestamp
+ * @property {string} duration - Analysis duration
+ * @property {number|null} score - QA score (0-100)
+ * @property {string} summary - AI-generated summary
+ * @property {Bug[]} bugs - All bugs found
+ * @property {Bug[]} criticalBugs - Only critical/high severity bugs
+ * @property {string[]} recommendations - AI recommendations
+ * @property {string[]} consoleErrors - Console errors captured
+ * @property {NetworkError[]} networkErrors - Network errors captured
+ * @property {Screenshot[]} screenshots - Screenshots taken
+ * @property {string[]} viewports - Viewports tested
+ * @property {string} focus - Focus area used
+ */
+
+/**
+ * @typedef {Object} Bug
+ * @property {string} title - Bug title
+ * @property {string} description - Bug description
+ * @property {'critical'|'high'|'medium'|'low'} severity - Bug severity
+ * @property {string} category - Bug category
+ * @property {string} [viewport] - Viewport where bug was found
+ * @property {string} [recommendation] - How to fix
+ */
+
+/**
+ * @typedef {Object} Screenshot
+ * @property {string} name - Screenshot name
+ * @property {string} viewport - Viewport name
+ * @property {number} width - Viewport width
+ * @property {number} height - Viewport height
+ * @property {Buffer} buffer - Screenshot buffer
+ * @property {string} base64 - Base64 encoded screenshot
+ */
+
+/**
+ * @typedef {Object} NetworkError
+ * @property {string} url - Request URL
+ * @property {string} method - HTTP method
+ * @property {number} [status] - HTTP status code
+ * @property {string} [failure] - Failure reason
+ */
+
+/**
+ * @typedef {Object} CaptureData
+ * @property {string} pageUrl - Page URL
+ * @property {string} pageTitle - Page title
+ * @property {string} timestamp - ISO timestamp
+ * @property {string[]} consoleErrors - Console errors
+ * @property {NetworkError[]} networkErrors - Network errors
+ * @property {Screenshot[]} screenshots - Screenshots
+ */