playwright-ai-insights 1.0.0

package/.github/workflows/publish.yml

@@ -0,0 +1,39 @@
+name: Publish NPM Package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+          registry-url: 'https://registry.npmjs.org'
+      
+      - name: Install dependencies
+        run: |
+          cd playwright-ai-insights
+          npm install
+      
+      - name: Build
+        run: |
+          cd playwright-ai-insights
+          npm run build
+      
+      - name: Publish to NPM
+        run: |
+          cd playwright-ai-insights
+          npm publish --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      
+      - name: Create GitHub Release Assets
+        uses: actions/upload-artifact@v3
+        with:
+          name: build-artifacts
+          path: playwright-ai-insights/dist/

package/.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: Test NPM Package
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - 'playwright-ai-insights/**'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'playwright-ai-insights/**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      
+      - name: Install dependencies
+        run: |
+          cd playwright-ai-insights
+          npm install
+      
+      - name: Typecheck
+        run: |
+          cd playwright-ai-insights
+          npm run typecheck
+      
+      - name: Build
+        run: |
+          cd playwright-ai-insights
+          npm run build
+      
+      - name: Lint
+        run: |
+          cd playwright-ai-insights
+          npm run lint || true

package/EXAMPLES.md

@@ -0,0 +1,324 @@
+# Usage Examples
+
+## Integration with Existing Playwright Projects
+
+### Example 1: Add AI Analysis to Test Hooks
+
+```javascript
+// tests/fixtures/withAI.js
+import { test as base } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+import PlaywrightAI, { analyzeTestFailures } from '@playwright-ai/insights';
+
+// Initialize once
+PlaywrightAI.initialize({
+  openaiApiKey: process.env.OPENAI_API_KEY,
+  debug: process.env.DEBUG === 'true',
+});
+
+const failuresDir = './test-results/failures';
+
+export const test = base.extend({
+  analyzeFail: async ({ page }, use) => {
+    const failures = [];
+
+    await use(async (testName, errorMessage, captureDOM = true) => {
+      let domSnapshot = '';
+      if (captureDOM) {
+        try {
+          domSnapshot = await page.content();
+        } catch (e) {
+          console.warn('Failed to capture DOM');
+        }
+      }
+
+      failures.push({
+        testName,
+        error: errorMessage,
+        domSnapshot,
+      });
+    });
+
+    // Analyze failures after test
+    if (failures.length > 0) {
+      const report = await analyzeTestFailures(failures);
+      
+      // Save report
+      fs.mkdirSync(failuresDir, { recursive: true });
+      fs.writeFileSync(
+        path.join(failuresDir, `${failures[0].testName}.json`),
+        JSON.stringify(report, null, 2)
+      );
+    }
+  },
+});
+
+export { expect } from '@playwright/test';
+```
+
+Usage in test:
+
+```javascript
+import { test, expect } from './fixtures/withAI';
+
+test('login with valid credentials', async ({ page, analyzeFail }) => {
+  await page.goto('https://example.com/login');
+  
+  try {
+    await expect(page.locator('#login-btn')).toBeVisible({ timeout: 5000 });
+  } catch (error) {
+    await analyzeFail('Login button should be visible', error.message);
+    throw error;
+  }
+});
+```
+
+### Example 2: Standalone Analysis Script
+
+```javascript
+// scripts/analyze-failures.js
+import fs from 'fs';
+import PlaywrightAI, { analyzeTestFailures } from '@playwright-ai/insights';
+
+async function main() {
+  PlaywrightAI.initialize({
+    openaiApiKey: process.env.OPENAI_API_KEY,
+  });
+
+  // Read failures from playwright-report
+  const failures = [];
+  const reportDir = './playwright-report/data';
+
+  for (const file of fs.readdirSync(reportDir)) {
+    if (file.endsWith('.md')) {
+      const content = fs.readFileSync(`${reportDir}/${file}`, 'utf-8');
+      
+      failures.push({
+        testName: file.replace('.md', ''),
+        error: content.split('\n')[0],
+        domSnapshot: content,
+      });
+    }
+  }
+
+  console.log(`Analyzing ${failures.length} failures...`);
+  const report = await analyzeTestFailures(failures);
+
+  // Save HTML report
+  const html = generateHTML(report);
+  fs.writeFileSync('./ai-insights-report.html', html);
+  console.log('✅ Report saved to ./ai-insights-report.html');
+}
+
+function generateHTML(report) {
+  return `
+    <html>
+      <head>
+        <title>AI Insights Report</title>
+        <style>
+          body { font-family: sans-serif; margin: 20px; }
+          h1 { color: #333; }
+          .summary { background: #f5f5f5; padding: 15px; border-radius: 5px; }
+          .failure { margin: 20px 0; border-left: 4px solid #007bff; padding: 10px; }
+          .suggestion { background: #e7f3ff; padding: 10px; margin: 10px 0; border-radius: 3px; }
+        </style>
+      </head>
+      <body>
+        <h1>AI Insights Report</h1>
+        <div class="summary">
+          <p><strong>Total Failures:</strong> ${report.summary.totalFailures}</p>
+          <p><strong>Locator Issues:</strong> ${report.summary.locatorIssues}</p>
+          <p><strong>Average Confidence:</strong> ${report.summary.averageConfidence}%</p>
+        </div>
+        <div>
+          ${report.failures.map(f => `
+            <div class="failure">
+              <h3>${f.testName}</h3>
+              <p>${f.rootCause}</p>
+              ${report.suggestions
+                .filter(s => s.testName === f.testName)
+                .map(s => `
+                  <div class="suggestion">
+                    <strong>Suggested Locator:</strong> <code>${s.suggestedLocator}</code>
+                    <p>${s.reason}</p>
+                  </div>
+                `).join('')}
+            </div>
+          `).join('')}
+        </div>
+      </body>
+    </html>
+  `;
+}
+
+main().catch(e => {
+  console.error('Error:', e);
+  process.exit(1);
+});
+```
+
+### Example 3: Custom Report Generator
+
+```javascript
+// scripts/generate-insights.js
+import PlaywrightAI, { 
+  analyzeTestFailures,
+  utils 
+} from '@playwright-ai/insights';
+import fs from 'fs';
+
+async function generateInsights() {
+  PlaywrightAI.initialize({
+    openaiApiKey: process.env.OPENAI_API_KEY,
+    debug: true,
+  });
+
+  // Read test history
+  const history = JSON.parse(
+    fs.readFileSync('./reports/history.json', 'utf-8')
+  );
+
+  // Identify flaky tests
+  const flakyTests = history.filter(test => {
+    const { isFlaky, confidence } = PlaywrightAI.checkTestFlakiness(test.runs);
+    return isFlaky && confidence > 0.6;
+  });
+
+  console.log(`Found ${flakyTests.length} flaky tests`);
+
+  // Analyze recent failures
+  const recentFailures = history
+    .filter(t => t.runs[t.runs.length - 1]?.status === 'FAILED')
+    .slice(0, 10);
+
+  const report = await analyzeTestFailures(
+    recentFailures.map(t => ({
+      testName: t.name,
+      error: t.runs[t.runs.length - 1].error,
+      domSnapshot: t.lastDomSnapshot,
+    }))
+  );
+
+  // Generate summary
+  const summary = {
+    timestamp: new Date().toISOString(),
+    flakyTestsCount: flakyTests.length,
+    failuresAnalyzed: report.failures.length,
+    averageConfidence: report.summary.averageConfidence,
+    topSuggestions: report.suggestions.slice(0, 5),
+  };
+
+  fs.writeFileSync(
+    './reports/ai-summary.json',
+    JSON.stringify(summary, null, 2)
+  );
+
+  console.log('✅ Summary written to ./reports/ai-summary.json');
+}
+
+generateInsights().catch(e => {
+  console.error('Error:', e);
+  process.exit(1);
+});
+```
+
+## Integration Patterns
+
+### Pattern 1: Playwright Test Plugin
+
+```javascript
+// plugins/ai-plugin.js
+export function defineConfig(config) {
+  return {
+    ...config,
+    use: {
+      ...config.use,
+      // Custom plugin setup
+    },
+  };
+}
+```
+
+### Pattern 2: GitHub Actions Workflow
+
+```yaml
+# .github/workflows/test-and-analyze.yml
+name: Test with AI Analysis
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      
+      - run: npm install
+      - run: npm test || true
+      
+      - name: Analyze failures with AI
+        if: failure()
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          npx playwright-ai analyze \
+            --results=./test-results/failures.json \
+            --output=./ai-insights.json
+      
+      - name: Upload AI report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ai-insights
+          path: ./ai-insights.json
+```
+
+### Pattern 3: Docker Integration
+
+```dockerfile
+FROM mcr.microsoft.com/playwright:v1.40.0-jammy
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm ci && npm install @playwright-ai/insights
+
+COPY . .
+
+ENV OPENAI_API_KEY=${OPENAI_API_KEY}
+
+RUN npm test
+
+RUN npx playwright-ai analyze \
+    --results=./test-results/failures.json \
+    --output=./ai-insights.json
+```
+
+## Cost Optimization
+
+```javascript
+// Batch multiple analyses for efficiency
+const batchSize = 10;
+for (let i = 0; i < failures.length; i += batchSize) {
+  const batch = failures.slice(i, i + batchSize);
+  const report = await analyzeTestFailures(batch);
+  // Process batch results
+}
+
+// Use caching to avoid re-analyzing
+const cache = new Map();
+async function getAnalysis(failure) {
+  const key = JSON.stringify(failure);
+  if (cache.has(key)) return cache.get(key);
+  
+  const result = await analyzeTestFailures([failure]);
+  cache.set(key, result);
+  return result;
+}
+```
+
+See README.md for more examples and API documentation.

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Playwright AI Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,289 @@
+# @playwright-ai/insights
+
+AI-powered Playwright test failure analysis and self-healing with automatic locator suggestions.
+
+## Features
+
+✨ **Core Capabilities**
+- 🤖 **AI Failure Analysis** - Analyzes test failures with OpenAI API
+- 🔧 **Self-Healing Locators** - AI-suggested improved selectors
+- 🟣 **Flakiness Detection** - Identify and quarantine flaky tests
+- 📊 **Smart Reports** - HTML reports with AI insights and statistics
+- 🎯 **Minimal Setup** - Just add API key, works with any Playwright framework
+
+## Installation
+
+```bash
+npm install @playwright-ai/insights
+```
+
+Requires Node.js 18+ and an OpenAI API key.
+
+## Quick Start
+
+### 1. Initialize Configuration
+
+```javascript
+import PlaywrightAI from '@playwright-ai/insights';
+
+PlaywrightAI.initialize({
+  openaiApiKey: process.env.OPENAI_API_KEY,
+  model: 'gpt-4o-mini',
+  enableFlakyDetection: true,
+});
+```
+
+### 2. Analyze Test Failures
+
+```javascript
+import { analyzeTestFailures } from '@playwright-ai/insights';
+
+const failures = [
+  {
+    testName: 'Login form should accept valid credentials',
+    error: 'Timeout waiting for element with id "login-button"',
+    domSnapshot: '<html>...</html>',
+    screenshotPath: '/path/to/screenshot.png',
+  },
+];
+
+const report = await analyzeTestFailures(failures);
+
+console.log(report.summary);
+// {
+//   totalFailures: 1,
+//   locatorIssues: 1,
+//   timeoutIssues: 0,
+//   assertionIssues: 0,
+//   averageConfidence: 82
+// }
+
+console.log(report.suggestions);
+// [
+//   {
+//     testName: 'Login form...',
+//     failedLocatorGuess: '#login-button',
+//     suggestedLocator: '[data-testid="submitButton"]',
+//     confidence: 82,
+//     reason: 'data-testid selectors are more stable than IDs...'
+//   }
+// ]
+```
+
+## Usage Examples
+
+### With Playwright Test Hooks
+
+Automatically collect and analyze failures:
+
+```javascript
+// tests/hooks.js
+import { test as base } from '@playwright/test';
+import { analyzeTestFailures } from '@playwright-ai/insights';
+import PlaywrightAI from '@playwright-ai/insights';
+
+PlaywrightAI.initialize({
+  openaiApiKey: process.env.OPENAI_API_KEY,
+});
+
+export const test = base.extend({
+  captureFailure: async ({}, use) => {
+    const failures = [];
+    
+    await use((data) => failures.push(data));
+    
+    if (failures.length > 0) {
+      const report = await analyzeTestFailures(failures);
+      console.log('AI Report:', report);
+    }
+  },
+});
+```
+
+### Directly Analyze a Failure
+
+```javascript
+import { getFailureInsight } from '@playwright-ai/insights';
+
+const failure = {
+  testName: 'My Test',
+  error: 'Element not found: .sidebar-nav > li',
+  domSnapshot: pageHTML,
+};
+
+const insight = await getFailureInsight(failure);
+console.log(insight.rootCause);
+console.log(insight.suggestions);
+```
+
+### Detect Flaky Tests
+
+```javascript
+import { checkTestFlakiness } from '@playwright-ai/insights';
+
+const history = [
+  { status: 'PASSED', timestamp: '2024-01-01' },
+  { status: 'FAILED', timestamp: '2024-01-02', failureType: '⏱ Timeout' },
+  { status: 'PASSED', timestamp: '2024-01-03' },
+];
+
+const { isFlaky, confidence } = checkTestFlakiness(history);
+console.log(`Flaky: ${isFlaky}, Confidence: ${confidence}`);
+```
+
+### Get Locator Suggestions
+
+```javascript
+import { suggestLocatorFixes } from '@playwright-ai/insights/agents';
+
+const suggestions = await suggestLocatorFixes([
+  {
+    testName: 'Search test',
+    error: 'Locator "#search_box" not found',
+    dom: pageHTML,
+  },
+]);
+
+suggestions.forEach((s) => {
+  console.log(`${s.testName}:`);
+  console.log(`  Failed: ${s.failedLocatorGuess}`);
+  console.log(`  Suggested: ${s.suggestedLocator}`);
+  console.log(`  Confidence: ${s.confidence}%`);
+});
+```
+
+## CLI Tool
+
+Analyze test failures from the command line:
+
+```bash
+# Set your API key
+export OPENAI_API_KEY=sk-...
+
+# Analyze failures from JSON file
+npx playwright-ai analyze --results=./test-failures.json
+
+# Save report to custom location
+npx playwright-ai analyze --results=failures.json --output=insights.json
+
+# Enable debug logging
+npx playwright-ai analyze --results=failures.json --debug
+```
+
+## API Reference
+
+### `initialize(config: Partial<PlaywrightAIConfig>)`
+
+Initialize the library with configuration.
+
+**Options:**
+- `openaiApiKey` (required): Your OpenAI API key
+- `model`: 'gpt-4o' | 'gpt-4o-mini' (default) | 'gpt-4-turbo'
+- `enableFlakyDetection`: boolean (default: true)
+- `maxDomSnapshotChars`: number (default: 6000)
+- `temperature`: number (default: 0.2)
+- `debug`: boolean (default: false)
+
+### `analyzeTestFailures(failures: TestFailure[]): Promise<AnalysisReport>`
+
+Analyze multiple test failures and generate a comprehensive report.
+
+**Returns:**
+```typescript
+{
+  timestamp: string;
+  failures: AIAnalysisResult[];
+  suggestions: LocatorSuggestion[];
+  summary: {
+    totalFailures: number;
+    locatorIssues: number;
+    timeoutIssues: number;
+    assertionIssues: number;
+    averageConfidence: number;
+  };
+}
+```
+
+### `getFailureInsight(failure: TestFailure): Promise<AIAnalysisResult>`
+
+Get AI analysis for a single test failure.
+
+### `checkTestFlakiness(testHistory): { isFlaky: boolean; confidence: number }`
+
+Check if a test is flaky based on its run history.
+
+### `utils`
+
+Access to utility functions:
+- `classifyFailure(error: string): FailureType`
+- `isLocatorRelated(error: string): boolean`
+- `isFlaky(history): boolean`
+- `calculateFlakyConfidence(history): number`
+- `logger`, `setDebugMode(enabled: boolean)`, `setConfig()`
+
+## Types
+
+```typescript
+interface TestFailure {
+  testName: string;
+  error: string;
+  errorType?: string;
+  stack?: string;
+  domSnapshot?: string;
+  screenshot?: string;
+}
+
+interface AIAnalysisResult {
+  testName: string;
+  rootCause: string;
+  suggestions: string[];
+  confidence: number;
+  failureType: FailureType;
+}
+
+interface LocatorSuggestion {
+  failedLocatorGuess: string;
+  suggestedLocator: string;
+  confidence: number;
+  reason: string;
+  testName?: string;
+}
+```
+
+## Environment Variables
+
+- `OPENAI_API_KEY` - Your OpenAI API key (required)
+- `DEBUG` - Enable debug logging: `DEBUG=true`
+
+## Pricing & Costs
+
+Uses OpenAI's **gpt-4o-mini** model by default (most cost-effective):
+- ~$0.015 per 1M input tokens
+- ~$0.06 per 1M output tokens
+
+Typical cost per failure analysis: **$0.001 - $0.003**
+
+## Best Practices
+
+1. **Batch Analysis**: Send multiple failures at once for efficiency
+2. **Capture DOM**: Include DOM snapshots for better locator suggestions
+3. **Set Debug Mode**: Use `debug: true` during development
+4. **Cache Results**: Store analysis results to avoid re-analyzing
+5. **Use data-testid**: Tests using data-testid attributes get better suggestions
+
+## Limitations
+
+- DOM snapshots capped at 6000 characters
+- OpenAI API rate limits apply
+- Requires internet connectivity
+- Works best with descriptive error messages
+
+## License
+
+MIT
+
+## Support
+
+- GitHub: https://github.com/playwright-ai/insights
+- Issues: https://github.com/playwright-ai/insights/issues
+- Docs: https://playwright-ai.dev