qai-cli 3.0.0

package/.github/workflows/visual-regression.yml

@@ -0,0 +1,192 @@
+name: Visual Regression Testing
+
+on:
+  workflow_dispatch:
+    inputs:
+      url:
+        description: 'URL to test'
+        required: true
+        type: string
+      update_baseline:
+        description: 'Update baseline screenshots'
+        required: false
+        type: boolean
+        default: false
+      baseline_run_id:
+        description: 'Run ID to use as baseline (leave empty for latest)'
+        required: false
+        type: string
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+  actions: read
+
+env:
+  NODE_VERSION: '20'
+  SCREENSHOTS_DIR: './screenshots'
+  BASELINE_DIR: './screenshots/baseline'
+  DIFF_DIR: './screenshots/diff'
+
+jobs:
+  visual-regression:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Install Playwright Browsers
+        run: npx playwright install chromium --with-deps
+
+      - name: Download Baseline Screenshots
+        if: ${{ !inputs.update_baseline }}
+        uses: dawidd6/action-download-artifact@v3
+        with:
+          name: visual-baseline
+          path: ${{ env.BASELINE_DIR }}
+          workflow: visual-regression.yml
+          run_id: ${{ inputs.baseline_run_id }}
+          if_no_artifact_found: warn
+        continue-on-error: true
+
+      - name: Create Directories
+        run: |
+          mkdir -p ${{ env.SCREENSHOTS_DIR }}
+          mkdir -p ${{ env.BASELINE_DIR }}
+          mkdir -p ${{ env.DIFF_DIR }}
+
+      - name: Capture Screenshots
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          echo "Capturing screenshots for visual regression testing..."
+          echo "URL: ${{ inputs.url }}"
+
+          npx @anthropic-ai/claude-code \
+            --print \
+            --mcp-config .claude/mcp-config.json \
+            -- \
+            'Navigate to ${{ inputs.url }} and capture screenshots. Save to ./screenshots/: desktop-full.png at 1920x1080, tablet-full.png at 768x1024, mobile-full.png at 375x667, desktop-fold.png, navigation.png, footer.png. Wait for page load before each screenshot.'
+
+      - name: Run Visual Regression Comparison
+        if: ${{ !inputs.update_baseline }}
+        id: comparison
+        run: |
+          echo "Running visual regression comparison..."
+
+          if [ -d "${{ env.BASELINE_DIR }}" ] && [ "$(ls -A ${{ env.BASELINE_DIR }} 2>/dev/null)" ]; then
+            node scripts/visual-regression.cjs compare \
+              "${{ env.BASELINE_DIR }}" \
+              "${{ env.SCREENSHOTS_DIR }}" \
+              "${{ env.DIFF_DIR }}"
+
+            # Capture exit code
+            EXIT_CODE=$?
+
+            # Read summary for output
+            if [ -f "visual-regression-report.md" ]; then
+              echo "report_exists=true" >> $GITHUB_OUTPUT
+            fi
+
+            exit $EXIT_CODE
+          else
+            echo "No baseline found - this will become the new baseline"
+            echo "no_baseline=true" >> $GITHUB_OUTPUT
+          fi
+        continue-on-error: true
+
+      - name: Update Baseline
+        if: ${{ inputs.update_baseline || steps.comparison.outputs.no_baseline == 'true' }}
+        run: |
+          echo "Updating baseline screenshots..."
+          node scripts/visual-regression.cjs update-baseline \
+            "${{ env.SCREENSHOTS_DIR }}" \
+            "${{ env.BASELINE_DIR }}"
+
+      - name: Upload Current Screenshots
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-screenshots-${{ github.run_number }}
+          path: ${{ env.SCREENSHOTS_DIR }}
+          retention-days: 30
+
+      - name: Upload Diff Images
+        if: ${{ !inputs.update_baseline && steps.comparison.outputs.report_exists == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-diff-${{ github.run_number }}
+          path: ${{ env.DIFF_DIR }}
+          retention-days: 30
+          if-no-files-found: ignore
+
+      - name: Upload Baseline (for future comparisons)
+        if: ${{ inputs.update_baseline || steps.comparison.outputs.no_baseline == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-baseline
+          path: ${{ env.BASELINE_DIR }}
+          retention-days: 90
+
+      - name: Upload Regression Report
+        if: ${{ !inputs.update_baseline }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-regression-report-${{ github.run_number }}
+          path: visual-regression-report.md
+          retention-days: 30
+          if-no-files-found: ignore
+
+      - name: Comment on PR (if applicable)
+        if: github.event_name == 'pull_request' && !inputs.update_baseline
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+
+            let report = '## 📸 Visual Regression Report\n\n';
+
+            if (fs.existsSync('visual-regression-report.md')) {
+              report += fs.readFileSync('visual-regression-report.md', 'utf8');
+            } else {
+              report += 'No visual regression report generated.\n';
+            }
+
+            report += `\n\n### 📎 Artifacts\n`;
+            report += `- [Screenshots](../actions/runs/${{ github.run_id }})\n`;
+            report += `- [Diff Images](../actions/runs/${{ github.run_id }})\n`;
+
+            report += '\n---\n*Visual regression testing by qai*';
+
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: report
+            });
+
+      - name: Summary
+        run: |
+          echo "## Visual Regression Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          if [ -f "visual-regression-report.md" ]; then
+            cat visual-regression-report.md >> $GITHUB_STEP_SUMMARY
+          elif [ "${{ inputs.update_baseline }}" == "true" ]; then
+            echo "✅ Baseline screenshots updated successfully" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "📸 Screenshots captured - no baseline to compare against" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "Run this workflow again with 'Update baseline screenshots' checked to create a baseline." >> $GITHUB_STEP_SUMMARY
+          fi

package/.prettierrc.json

@@ -0,0 +1,10 @@
+{
+  "semi": true,
+  "trailingComma": "all",
+  "singleQuote": true,
+  "printWidth": 100,
+  "tabWidth": 2,
+  "useTabs": false,
+  "arrowParens": "always",
+  "endOfLine": "lf"
+}

package/README.md

@@ -0,0 +1,111 @@
+# qai
+
+AI-powered QA engineer for your terminal. Scan websites, review PRs, generate tests.
+
+[![npm version](https://img.shields.io/npm/v/qai-cli)](https://www.npmjs.com/package/qai-cli)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+[![Lint](https://github.com/tyler-james-bridges/qaie/actions/workflows/lint.yml/badge.svg)](https://github.com/tyler-james-bridges/qaie/actions/workflows/lint.yml)
+
+## Install
+
+```bash
+npm install -g qai-cli
+```
+
+## Commands
+
+### `qai scan` — Visual QA Analysis
+
+Capture screenshots, detect console/network errors, and get AI-powered bug reports.
+
+```bash
+# Scan a URL
+qai scan https://mysite.com
+
+# Multiple viewports
+VIEWPORTS=desktop,mobile,tablet qai scan https://mysite.com
+
+# Focus on accessibility
+FOCUS=accessibility qai scan https://mysite.com
+```
+
+### `qai review` — PR Code Review _(Coming Soon)_
+
+Deep code review with full codebase context. Not just the diff — traces through dependencies, callers, and related tests.
+
+```bash
+# Review a PR
+qai review 42
+
+# Review current branch against main
+qai review --base main
+```
+
+### `qai generate` — Test Generation _(Coming Soon)_
+
+Auto-generate Playwright E2E tests from URLs or unit tests from source files.
+
+```bash
+# Generate E2E tests by crawling a site
+qai generate https://mysite.com
+
+# Generate unit tests from source
+qai generate src/billing.ts
+```
+
+## Playwright Integration
+
+Use qai inside your existing Playwright test suite:
+
+```typescript
+import { test, expect } from '@playwright/test';
+import { analyzeWithAI, attachScreenshots } from 'qai-cli';
+
+test('homepage has no critical issues', async ({ page }, testInfo) => {
+  await page.goto('/');
+
+  const report = await analyzeWithAI(page, {
+    viewports: ['desktop', 'mobile'],
+    focus: 'all',
+  });
+
+  await attachScreenshots(testInfo, report);
+  expect(report.criticalBugs).toHaveLength(0);
+});
+```
+
+## GitHub Action
+
+```yaml
+- name: QAI Scan
+  uses: tyler-james-bridges/qaie@main
+  with:
+    url: ${{ env.PREVIEW_URL }}
+    viewports: desktop,mobile
+  env:
+    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+```
+
+## AI Providers
+
+Works with any major LLM. Set one env var:
+
+| Provider  | Env Var             | Default Model   |
+| --------- | ------------------- | --------------- |
+| Anthropic | `ANTHROPIC_API_KEY` | claude-sonnet-4 |
+| OpenAI    | `OPENAI_API_KEY`    | gpt-4o          |
+| Google    | `GEMINI_API_KEY`    | gemini-pro      |
+| Ollama    | `OLLAMA_HOST`       | llama3          |
+
+## Features
+
+- **Multi-viewport** — Desktop, tablet, mobile screenshots
+- **Console errors** — JavaScript errors and warnings
+- **Network errors** — Failed APIs, slow requests, 4xx/5xx
+- **Visual regression** — Pixel-level comparison with baselines
+- **Structured reports** — JSON + Markdown output
+- **CI/CD ready** — GitHub Action + exit codes for pipelines
+
+## License
+
+MIT

package/action.yml

@@ -0,0 +1,149 @@
+name: 'qai'
+description: 'AI-powered QA engineer. Code review, testing, and bug detection. Supports multiple LLM providers.'
+author: 'tyler-james-bridges'
+
+branding:
+  icon: 'check-circle'
+  color: 'green'
+
+inputs:
+  url:
+    description: 'URL to test'
+    required: true
+
+  # Provider selection (auto-detected if not specified)
+  provider:
+    description: 'LLM provider: anthropic, openai, codex, gemini, ollama'
+    required: false
+
+  # Provider-specific API keys (pass whichever you have)
+  anthropic_api_key:
+    description: 'Anthropic API key (for Claude)'
+    required: false
+  openai_api_key:
+    description: 'OpenAI API key (for GPT-4)'
+    required: false
+  codex_api_key:
+    description: 'Codex API key'
+    required: false
+  gemini_api_key:
+    description: 'Google Gemini API key'
+    required: false
+
+  # Generic key (use with provider input)
+  api_key:
+    description: 'Generic API key (use with provider input)'
+    required: false
+
+  # Ollama settings (for local models)
+  ollama_base_url:
+    description: 'Ollama base URL (default: http://localhost:11434)'
+    required: false
+    default: 'http://localhost:11434'
+  ollama_model:
+    description: 'Ollama model to use (default: llava)'
+    required: false
+    default: 'llava'
+
+  # Test configuration
+  viewports:
+    description: 'Viewports to test (comma-separated): desktop,mobile,tablet'
+    required: false
+    default: 'desktop,mobile'
+  focus:
+    description: 'Focus area: all, accessibility, performance, forms, visual'
+    required: false
+    default: 'all'
+  timeout:
+    description: 'Timeout in seconds'
+    required: false
+    default: '300'
+
+  # Output options
+  fail_on_bugs:
+    description: 'Fail workflow if critical/high severity bugs found'
+    required: false
+    default: 'false'
+  output_format:
+    description: 'Report format: markdown, json, sarif'
+    required: false
+    default: 'markdown'
+
+outputs:
+  report:
+    description: 'Path to the QA report file'
+  screenshots:
+    description: 'Path to screenshots directory'
+  bugs_found:
+    description: 'Number of bugs found'
+  critical_bugs:
+    description: 'Number of critical/high severity bugs'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        cd ${{ github.action_path }}
+        npm ci
+
+    - name: Install Playwright
+      shell: bash
+      run: npx playwright install chromium --with-deps
+
+    - name: Run QA Tests
+      id: qa-test
+      shell: bash
+      env:
+        INPUT_URL: ${{ inputs.url }}
+        INPUT_PROVIDER: ${{ inputs.provider }}
+        INPUT_ANTHROPIC_API_KEY: ${{ inputs.anthropic_api_key }}
+        INPUT_OPENAI_API_KEY: ${{ inputs.openai_api_key }}
+        INPUT_CODEX_API_KEY: ${{ inputs.codex_api_key }}
+        INPUT_GEMINI_API_KEY: ${{ inputs.gemini_api_key }}
+        INPUT_API_KEY: ${{ inputs.api_key }}
+        INPUT_OLLAMA_BASE_URL: ${{ inputs.ollama_base_url }}
+        INPUT_OLLAMA_MODEL: ${{ inputs.ollama_model }}
+        INPUT_VIEWPORTS: ${{ inputs.viewports }}
+        INPUT_FOCUS: ${{ inputs.focus }}
+        INPUT_TIMEOUT: ${{ inputs.timeout }}
+        INPUT_OUTPUT_FORMAT: ${{ inputs.output_format }}
+      run: |
+        cd ${{ github.action_path }}
+        node src/index.js
+
+    - name: Upload Screenshots
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: qa-screenshots-${{ github.run_number }}
+        path: screenshots/
+        retention-days: 14
+        if-no-files-found: ignore
+
+    - name: Upload Report
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: qa-report-${{ github.run_number }}
+        path: qa-report.*
+        retention-days: 14
+        if-no-files-found: ignore
+
+    - name: Check for critical bugs
+      if: ${{ inputs.fail_on_bugs == 'true' }}
+      shell: bash
+      run: |
+        if [ -f "qa-report.json" ]; then
+          CRITICAL=$(cat qa-report.json | jq '.bugs | map(select(.severity == "critical" or .severity == "high")) | length')
+          if [ "$CRITICAL" -gt 0 ]; then
+            echo "::error::Found $CRITICAL critical/high severity bugs"
+            exit 1
+          fi
+        fi

package/docs/BUGS.md

@@ -0,0 +1,43 @@
+# Intentional Bugs in Demo Site
+
+This site contains intentional bugs for testing qai.
+
+| Bug                                    | Category      | Severity | Location                 |
+| -------------------------------------- | ------------- | -------- | ------------------------ |
+| Broken "Resources" link                | Functional    | High     | index.html:14            |
+| Missing alt text on hero image         | Accessibility | Medium   | index.html:19            |
+| Hero image 404                         | Network       | Medium   | index.html:19            |
+| Mobile nav doesn't work                | Functional    | High     | style.css:186, app.js:56 |
+| Horizontal scroll on mobile            | Visual        | Medium   | style.css:79             |
+| Missing focus styles                   | Accessibility | Medium   | style.css:197            |
+| Form has no validation                 | Functional    | Medium   | index.html:75-82         |
+| Email input wrong type                 | Accessibility | Low      | index.html:79            |
+| Console error: undefined siteConfig    | JavaScript    | Medium   | app.js:7                 |
+| Console error: undefined trackPurchase | JavaScript    | Medium   | app.js:51                |
+| Console error: undefined analytics     | JavaScript    | Low      | app.js:71                |
+| No required fields on form             | Functional    | Medium   | index.html:75-82         |
+| Disabled button with no explanation    | UX            | Low      | index.html:66            |
+| Scroll handler not throttled           | Performance   | Low      | app.js:62                |
+| Generic form error message             | UX            | Low      | app.js:41                |
+
+## Expected QA Findings
+
+A thorough QA test should find:
+
+- **3+** console errors
+- **2+** accessibility issues
+- **2+** mobile/responsive issues
+- **1** broken link
+- **1** 404 resource
+
+## Testing This Site
+
+```bash
+# Via GitHub Actions
+Go to Actions → qai → Run workflow
+URL: https://tyler-james-bridges.github.io/qaie/
+
+# Via Visual Regression
+Go to Actions → Visual Regression Testing → Run workflow
+URL: https://tyler-james-bridges.github.io/qaie/
+```

package/docs/app.js

@@ -0,0 +1,101 @@
+// qai Demo Site - JavaScript
+// This file contains intentional bugs for QA testing
+
+document.addEventListener('DOMContentLoaded', function () {
+  // BUG: Referencing undefined variable causes console error
+  console.log('Demo site loaded');
+  console.log('Config:', siteConfig); // ReferenceError: siteConfig is not defined
+
+  // CTA Button click handler
+  const ctaBtn = document.getElementById('cta-btn');
+  if (ctaBtn) {
+    ctaBtn.addEventListener('click', function () {
+      // BUG: Alert is not accessible and annoying
+      alert('Thanks for clicking! This is a demo site.');
+    });
+  }
+
+  // Form submission handler
+  const form = document.getElementById('contact-form');
+  const statusEl = document.getElementById('form-status');
+
+  if (form) {
+    form.addEventListener('submit', function (e) {
+      e.preventDefault();
+
+      const name = document.getElementById('name').value;
+      const email = document.getElementById('email').value;
+      const message = document.getElementById('message').value;
+
+      // BUG: No client-side validation before "sending"
+      // BUG: Race condition - status shows before "processing" completes
+      statusEl.textContent = 'Sending...';
+      statusEl.className = 'form-status';
+
+      // Simulate API call
+      setTimeout(function () {
+        // BUG: Always succeeds, never validates email format
+        if (name && email && message) {
+          statusEl.textContent = 'Message sent successfully!';
+          statusEl.className = 'form-status success';
+          form.reset();
+        } else {
+          // BUG: Generic error message doesn't tell user what's missing
+          statusEl.textContent = 'Error: Please fill out the form.';
+          statusEl.className = 'form-status error';
+        }
+      }, 1000);
+    });
+  }
+
+  // Buy button handlers
+  const buyButtons = document.querySelectorAll('.buy-btn');
+  buyButtons.forEach(function (btn) {
+    btn.addEventListener('click', function () {
+      if (!btn.disabled) {
+        // BUG: Uses deprecated API
+        const plan = btn.parentElement.querySelector('h3').innerText;
+        console.log('Selected plan:', plan);
+
+        // BUG: Another console error - undefined function
+        trackPurchase(plan); // ReferenceError: trackPurchase is not defined
+      }
+    });
+  });
+
+  // Mobile nav toggle
+  const navToggle = document.querySelector('.nav-toggle');
+  const navLinks = document.querySelector('.nav-links');
+
+  if (navToggle) {
+    navToggle.addEventListener('click', function () {
+      // BUG: Toggle doesn't actually work - CSS hides nav-links on mobile with no toggle class
+      navLinks.classList.toggle('active');
+      console.log('Nav toggled, but CSS does not support this');
+    });
+  }
+
+  // BUG: Scroll event listener without throttling (performance issue)
+  window.addEventListener('scroll', function () {
+    const scrollPos = window.scrollY;
+    // Doing unnecessary work on every scroll event
+    document.querySelectorAll('.feature-card').forEach(function (card, index) {
+      card.style.opacity = Math.min(1, scrollPos / 500 + 0.5);
+    });
+  });
+
+  // BUG: Set timeout that logs after page might be navigated away
+  setTimeout(function () {
+    console.log('Delayed analytics ping');
+    // BUG: Trying to call undefined analytics
+    analytics.track('page_view_delayed'); // ReferenceError
+  }, 5000);
+});
+
+// BUG: Global error handler that swallows errors (bad practice)
+window.onerror = function (msg, url, lineNo, columnNo, error) {
+  console.log('Error caught:', msg);
+  // BUG: Returns true which prevents error from being logged to console properly
+  // return true; // Uncomment to hide all errors (worse bug)
+  return false;
+};