qai-cli 3.0.0

package/.github/workflows/playwright-qa.yml

@@ -0,0 +1,223 @@
+name: Playwright + AI QA
+
+on:
+  workflow_call:
+    inputs:
+      working_directory:
+        description: 'Directory containing playwright.config (default: repo root)'
+        required: false
+        type: string
+        default: '.'
+      test_command:
+        description: 'Command to run tests (default: npx playwright test)'
+        required: false
+        type: string
+        default: 'npx playwright test'
+      preview_url:
+        description: 'Preview URL to test against (optional - uses baseURL from config if not provided)'
+        required: false
+        type: string
+      node_version:
+        description: 'Node.js version'
+        required: false
+        type: string
+        default: '20'
+    secrets:
+      ANTHROPIC_API_KEY:
+        required: false
+      OPENAI_API_KEY:
+        required: false
+
+jobs:
+  playwright-qa:
+    name: Playwright + AI QA
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ inputs.node_version }}
+          cache: 'npm'
+          cache-dependency-path: ${{ inputs.working_directory }}/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+        working-directory: ${{ inputs.working_directory }}
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps chromium
+        working-directory: ${{ inputs.working_directory }}
+
+      - name: Run Playwright tests
+        id: playwright
+        run: |
+          # Run tests with HTML reporter, continue even if tests fail
+          ${{ inputs.test_command }} --reporter=html,json || true
+
+          # Check if report was generated
+          if [ -d "playwright-report" ]; then
+            echo "report_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "report_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+          # Parse test results if JSON exists
+          if [ -f "test-results.json" ]; then
+            total=$(jq '.stats.expected + .stats.unexpected + .stats.flaky + .stats.skipped' test-results.json 2>/dev/null || echo "0")
+            passed=$(jq '.stats.expected' test-results.json 2>/dev/null || echo "0")
+            failed=$(jq '.stats.unexpected' test-results.json 2>/dev/null || echo "0")
+            flaky=$(jq '.stats.flaky' test-results.json 2>/dev/null || echo "0")
+            skipped=$(jq '.stats.skipped' test-results.json 2>/dev/null || echo "0")
+
+            echo "total=$total" >> $GITHUB_OUTPUT
+            echo "passed=$passed" >> $GITHUB_OUTPUT
+            echo "failed=$failed" >> $GITHUB_OUTPUT
+            echo "flaky=$flaky" >> $GITHUB_OUTPUT
+            echo "skipped=$skipped" >> $GITHUB_OUTPUT
+          fi
+        working-directory: ${{ inputs.working_directory }}
+        env:
+          PLAYWRIGHT_BASE_URL: ${{ inputs.preview_url }}
+
+      - name: Checkout qaie
+        uses: actions/checkout@v4
+        with:
+          repository: tyler-james-bridges/qaie
+          ref: main
+          path: _qaie
+
+      - name: Setup qaie
+        run: |
+          cd _qaie
+          npm ci
+          npx playwright install chromium
+
+      - name: Run AI QA Analysis
+        id: qai
+        if: ${{ inputs.preview_url != '' }}
+        run: |
+          cd _qaie
+          node src/index.js
+        env:
+          URL: ${{ inputs.preview_url }}
+          VIEWPORTS: desktop,mobile
+          FOCUS: all
+          OUTPUT_FORMAT: markdown
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        continue-on-error: true
+
+      - name: Upload Playwright Report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: ${{ inputs.working_directory }}/playwright-report/
+          retention-days: 14
+
+      - name: Upload AI QA Screenshots
+        uses: actions/upload-artifact@v4
+        if: always() && inputs.preview_url != ''
+        with:
+          name: qai-screenshots
+          path: _qaie/screenshots/
+          retention-days: 14
+
+      - name: Post Results to PR
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+
+            // Build comment
+            let comment = '## 🎭 Playwright + AI QA Report\n\n';
+
+            // Playwright results
+            const total = '${{ steps.playwright.outputs.total }}' || '0';
+            const passed = '${{ steps.playwright.outputs.passed }}' || '0';
+            const failed = '${{ steps.playwright.outputs.failed }}' || '0';
+            const flaky = '${{ steps.playwright.outputs.flaky }}' || '0';
+            const skipped = '${{ steps.playwright.outputs.skipped }}' || '0';
+
+            comment += '### 🧪 Playwright Test Results\n\n';
+
+            if (total !== '0') {
+              const passRate = ((parseInt(passed) / parseInt(total)) * 100).toFixed(1);
+              const statusEmoji = failed === '0' ? '✅' : '❌';
+
+              comment += `| Status | Count |\n`;
+              comment += `|--------|-------|\n`;
+              comment += `| ${statusEmoji} Passed | ${passed} |\n`;
+              if (failed !== '0') comment += `| ❌ Failed | ${failed} |\n`;
+              if (flaky !== '0') comment += `| ⚠️ Flaky | ${flaky} |\n`;
+              if (skipped !== '0') comment += `| ⏭️ Skipped | ${skipped} |\n`;
+              comment += `| **Total** | **${total}** |\n\n`;
+              comment += `**Pass Rate:** ${passRate}%\n\n`;
+            } else {
+              comment += '_No test results found_\n\n';
+            }
+
+            // AI QA results
+            const previewUrl = '${{ inputs.preview_url }}';
+            if (previewUrl) {
+              comment += '### 🤖 AI QA Analysis\n\n';
+
+              const aiReportPath = '_qaie/qa-report.md';
+              if (fs.existsSync(aiReportPath)) {
+                let aiReport = fs.readFileSync(aiReportPath, 'utf8');
+                // Remove the header since we have our own
+                aiReport = aiReport.replace(/^# QA Report\n+/, '');
+                comment += aiReport + '\n\n';
+              } else {
+                comment += '_AI QA analysis did not complete_\n\n';
+              }
+            }
+
+            // Artifacts link
+            comment += '---\n';
+            comment += `📊 **[View Full Playwright Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})**\n\n`;
+            comment += '_Download the `playwright-report` artifact and run `npx playwright show-report` to view interactive results_\n';
+
+            // Find or update existing comment
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const botComment = comments.find(c =>
+              c.user.type === 'Bot' &&
+              c.body.includes('Playwright + AI QA Report')
+            );
+
+            if (botComment) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: comment
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: comment
+              });
+            }
+
+      - name: Fail if tests failed
+        if: steps.playwright.outputs.failed != '0' && steps.playwright.outputs.failed != ''
+        run: |
+          echo "❌ ${{ steps.playwright.outputs.failed }} test(s) failed"
+          exit 1

package/.github/workflows/qa-engineer.yml

@@ -0,0 +1,309 @@
+name: qai
+
+# This workflow is designed to be copied to your repository.
+# It can run manually OR on pull requests.
+#
+# For PR testing, you need a preview deployment. Options:
+# 1. Vercel/Netlify: Set PREVIEW_URL env var in your deployment workflow
+# 2. workflow_call: Call this workflow from another workflow with the URL
+# 3. Manual: Uncomment pull_request trigger and set PREVIEW_URL secret/var
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+    inputs:
+      url:
+        description: 'URL to test (required for manual runs)'
+        required: false
+        type: string
+      focus:
+        description: 'Focus area: accessibility, performance, forms, mobile, all'
+        required: false
+        type: string
+        default: 'all'
+      fail_on_bugs:
+        description: 'Fail workflow if critical/high bugs found'
+        required: false
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      url:
+        description: 'URL to test'
+        required: true
+        type: string
+      focus:
+        description: 'Focus area'
+        required: false
+        type: string
+        default: 'all'
+      fail_on_bugs:
+        description: 'Fail on critical/high bugs'
+        required: false
+        type: boolean
+        default: true
+    secrets:
+      ANTHROPIC_API_KEY:
+        required: true
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+env:
+  NODE_VERSION: '20'
+  PLAYWRIGHT_VERSION: '1.48.0'
+
+jobs:
+  qa-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 12
+
+    steps:
+      - name: Determine Test URL
+        id: get-url
+        env:
+          INPUT_URL: ${{ inputs.url }}
+          PREVIEW_URL: ${{ vars.PREVIEW_URL || secrets.PREVIEW_URL || '' }}
+        run: |
+          # Priority: explicit input > PREVIEW_URL var/secret
+          if [ -n "$INPUT_URL" ]; then
+            echo "test_url=$INPUT_URL" >> $GITHUB_OUTPUT
+            echo "✓ Using provided URL: $INPUT_URL"
+          elif [ -n "$PREVIEW_URL" ]; then
+            echo "test_url=$PREVIEW_URL" >> $GITHUB_OUTPUT
+            echo "✓ Using PREVIEW_URL: $PREVIEW_URL"
+          else
+            echo "❌ Error: No URL provided"
+            echo ""
+            echo "For manual runs: provide a URL in the workflow inputs"
+            echo "For PR runs: set PREVIEW_URL as a repository variable or secret"
+            echo "For Vercel/Netlify: chain this workflow after your deploy workflow"
+            exit 1
+          fi
+
+      - name: Validate API Key
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          if [ -z "$ANTHROPIC_API_KEY" ]; then
+            echo "❌ Error: ANTHROPIC_API_KEY secret is not configured"
+            echo "Please add your Anthropic API key to repository secrets."
+            exit 1
+          fi
+          echo "✓ API key configured"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Cache Playwright Browsers
+        id: playwright-cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ env.PLAYWRIGHT_VERSION }}
+          restore-keys: |
+            playwright-${{ runner.os }}-
+
+      - name: Install Dependencies
+        run: npm ci
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 'true'
+
+      - name: Install Playwright Browsers
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+        run: npx playwright install chromium --with-deps
+
+      - name: Install Playwright Deps (cached)
+        if: steps.playwright-cache.outputs.cache-hit == 'true'
+        run: npx playwright install-deps chromium
+
+      - name: Check URL is reachable
+        id: url-check
+        run: |
+          echo "Checking if URL is reachable..."
+          TEST_URL="${{ steps.get-url.outputs.test_url }}"
+
+          # Try to reach the URL
+          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$TEST_URL" || echo "000")
+
+          if [ "$HTTP_CODE" = "000" ]; then
+            echo "⚠️ Warning: Could not reach $TEST_URL (connection failed)"
+            echo "reachable=false" >> $GITHUB_OUTPUT
+          elif [ "$HTTP_CODE" -ge 400 ]; then
+            echo "⚠️ Warning: $TEST_URL returned HTTP $HTTP_CODE"
+            echo "reachable=false" >> $GITHUB_OUTPUT
+          else
+            echo "✓ URL is reachable (HTTP $HTTP_CODE)"
+            echo "reachable=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create Screenshots Directory
+        run: mkdir -p screenshots
+
+      - name: Run qai
+        id: qa-run
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TEST_URL: ${{ steps.get-url.outputs.test_url }}
+          TEST_FOCUS: ${{ inputs.focus || 'all' }}
+        run: |
+          echo "::group::QA Testing Configuration"
+          echo "URL: $TEST_URL"
+          echo "Focus Area: $TEST_FOCUS"
+          echo "::endgroup::"
+
+          echo "::group::Starting qai"
+          START_TIME=$(date +%s)
+
+          # 5 minute timeout, max 15 turns - be aggressive about speed
+          timeout 300 npx @anthropic-ai/claude-code --print \
+            --mcp-config .claude/mcp-config.json \
+            --max-turns 15 \
+            "SPEED IS CRITICAL. You have 3 minutes max.
+
+             Test $TEST_URL:
+             1. Navigate to URL
+             2. Screenshot desktop view -> ./screenshots/desktop.png
+             3. Screenshot mobile (375px) -> ./screenshots/mobile.png
+             4. Check for console errors
+             5. Click 2-3 buttons/links to verify they work
+             6. Write ./qa-report.md: list any bugs found, or 'No issues found'
+
+             DO NOT: test multiple viewports, do exhaustive testing, write long reports.
+             STOP after basic checks. Speed > thoroughness." || {
+            EXIT_CODE=$?
+            if [ $EXIT_CODE -eq 124 ]; then
+              echo "⚠️ QA testing timed out after 5 minutes"
+            else
+              echo "⚠️ QA testing encountered an error (exit code: $EXIT_CODE)"
+            fi
+          }
+
+          END_TIME=$(date +%s)
+          DURATION=$((END_TIME - START_TIME))
+          echo "✓ QA testing completed in ${DURATION}s"
+          echo "::endgroup::"
+          echo "qa_duration_seconds=$DURATION" >> $GITHUB_OUTPUT
+
+      - name: Upload Screenshots
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: qa-screenshots-${{ github.run_number }}
+          path: screenshots/
+          retention-days: 14
+          if-no-files-found: ignore
+
+      - name: Upload QA Report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: qa-report-${{ github.run_number }}
+          path: qa-report.md
+          retention-days: 14
+          if-no-files-found: ignore
+
+      - name: Summary
+        if: always()
+        run: |
+          echo "## qai Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**URL Tested:** ${{ steps.get-url.outputs.test_url }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Focus Area:** ${{ inputs.focus || 'all' }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          if [ -f "qa-report.md" ]; then
+            echo "### Report" >> $GITHUB_STEP_SUMMARY
+            cat qa-report.md >> $GITHUB_STEP_SUMMARY
+          else
+            echo "⚠️ No QA report generated. Check the logs for details." >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Comment on PR
+        if: github.event_name == 'pull_request' && always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+
+            let body = '## 🤖 qai Report\n\n';
+            body += `**URL Tested:** ${{ steps.get-url.outputs.test_url }}\n`;
+            body += `**Focus Area:** ${{ inputs.focus || 'all' }}\n\n`;
+
+            if (fs.existsSync('qa-report.md')) {
+              const report = fs.readFileSync('qa-report.md', 'utf8');
+              body += report;
+            } else {
+              body += '⚠️ No QA report generated. Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.\n';
+            }
+
+            body += `\n\n### 📎 Artifacts\n`;
+            body += `- [Screenshots](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\n`;
+            body += `- [Full Report](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\n`;
+            body += '\n---\n*Automated QA testing by [qai](https://github.com/tyler-james-bridges/qaie)*';
+
+            // Find existing comment to update (avoid spam)
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const botComment = comments.find(comment =>
+              comment.user.type === 'Bot' &&
+              comment.body.includes('qai Report')
+            );
+
+            if (botComment) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: body
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: body
+              });
+            }
+
+      - name: Check for critical bugs
+        if: ${{ inputs.fail_on_bugs }}
+        run: |
+          if [ ! -f "qa-report.md" ]; then
+            echo "No QA report found - skipping bug check"
+            exit 0
+          fi
+
+          echo "Checking for critical/high severity bugs..."
+
+          # Case-insensitive search for critical or high severity indicators
+          CRITICAL_COUNT=$(grep -i -c -E '(critical|severity:\s*critical)' qa-report.md || echo "0")
+          HIGH_COUNT=$(grep -i -c -E '(high|severity:\s*high)' qa-report.md || echo "0")
+
+          echo "Critical bugs: $CRITICAL_COUNT"
+          echo "High severity bugs: $HIGH_COUNT"
+
+          if [ "$CRITICAL_COUNT" -gt 0 ]; then
+            echo "::error::Found $CRITICAL_COUNT critical bug(s) - failing workflow"
+            exit 1
+          fi
+
+          if [ "$HIGH_COUNT" -gt 0 ]; then
+            echo "::warning::Found $HIGH_COUNT high severity bug(s) - failing workflow"
+            exit 1
+          fi
+
+          echo "✓ No critical/high severity bugs found"