testeiya 0.2.1 → 0.3.9

package/skills/playwright-best-practices-skill/infrastructure-ci-cd/reporting.md

@@ -1,424 +0,0 @@
-# Test Reports & Artifacts
-
-## Table of Contents
-
-1. [CLI Commands](#cli-commands)
-2. [Reporter Configuration](#reporter-configuration)
-3. [Custom Reporter](#custom-reporter)
-4. [Trace Configuration](#trace-configuration)
-5. [Screenshot & Video Settings](#screenshot--video-settings)
-6. [Artifact Directory Structure](#artifact-directory-structure)
-7. [CI Artifact Upload](#ci-artifact-upload)
-8. [Decision Guide](#decision-guide)
-9. [Anti-Patterns](#anti-patterns)
-10. [Troubleshooting](#troubleshooting)
-
-> **When to use**: Configuring test output for debugging, CI dashboards, and team visibility.
-
-## CLI Commands
-
-```bash
-# Display last HTML report
-npx playwright show-report
-
-# Specify reporter
-npx playwright test --reporter=html
-npx playwright test --reporter=dot           # minimal CI output
-npx playwright test --reporter=line          # one line per test
-npx playwright test --reporter=json          # machine-readable
-npx playwright test --reporter=junit         # CI integration
-
-# Combine reporters
-npx playwright test --reporter=dot,html
-
-# Merge sharded reports
-npx playwright merge-reports --reporter=html ./blob-report
-```
-
-## Reporter Configuration
-
-### Environment-Based Setup
-
-```typescript
-// playwright.config.ts
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  reporter: process.env.CI
-    ? [
-        ['dot'],
-        ['html', { open: 'never' }],
-        ['junit', { outputFile: 'results/junit.xml' }],
-        ['github'],
-      ]
-    : [
-        ['list'],
-        ['html', { open: 'on-failure' }],
-      ],
-});
-```
-
-### Reporter Types
-
-| Reporter | Output | Use Case |
-|---|---|---|
-| `list` | One line per test | Local development |
-| `line` | Single updating line | Local, less verbose |
-| `dot` | `.` pass, `F` fail | CI logs |
-| `html` | Interactive HTML page | Post-run analysis |
-| `json` | Machine-readable JSON | Custom tooling |
-| `junit` | JUnit XML | CI platforms |
-| `github` | PR annotations | GitHub Actions |
-| `blob` | Binary archive | Shard merging |
-
-### JSON Output to File
-
-```typescript
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  reporter: [
-    ['json', { outputFile: 'results/output.json' }],
-  ],
-});
-```
-
-### JUnit Customization
-
-```typescript
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  reporter: [
-    ['junit', {
-      outputFile: 'results/junit.xml',
-      stripANSIControlSequences: true,
-      includeProjectInTestName: true,
-    }],
-  ],
-});
-```
-
-## Custom Reporter
-
-Build custom reporters for Slack notifications, database logging, or dashboards.
-
-```typescript
-// reporters/notification-reporter.ts
-import type {
-  FullResult,
-  Reporter,
-  TestCase,
-  TestResult,
-} from '@playwright/test/reporter';
-
-class NotificationReporter implements Reporter {
-  private passed = 0;
-  private failed = 0;
-  private skipped = 0;
-  private failures: string[] = [];
-
-  onTestEnd(test: TestCase, result: TestResult) {
-    switch (result.status) {
-      case 'passed':
-        this.passed++;
-        break;
-      case 'failed':
-      case 'timedOut':
-        this.failed++;
-        this.failures.push(`${test.title}: ${result.error?.message?.split('\n')[0]}`);
-        break;
-      case 'skipped':
-        this.skipped++;
-        break;
-    }
-  }
-
-  async onEnd(result: FullResult) {
-    const total = this.passed + this.failed + this.skipped;
-    const status = this.failed > 0 ? 'FAILED' : 'PASSED';
-    const message = [
-      `Tests ${status}`,
-      `Passed: ${this.passed} | Failed: ${this.failed} | Skipped: ${this.skipped}`,
-      `Duration: ${(result.duration / 1000).toFixed(1)}s`,
-    ];
-
-    if (this.failures.length > 0) {
-      message.push('', 'Failures:');
-      this.failures.slice(0, 5).forEach((f) => message.push(`  - ${f}`));
-      if (this.failures.length > 5) {
-        message.push(`  ...and ${this.failures.length - 5} more`);
-      }
-    }
-
-    const webhookUrl = process.env.NOTIFICATION_WEBHOOK;
-    if (webhookUrl) {
-      const controller = new AbortController();
-      const timeout = setTimeout(() => controller.abort(), 5000);
-      try {
-        await fetch(webhookUrl, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ text: message.join('\n') }),
-          signal: controller.signal,
-        });
-      } catch (error) {
-        // Intentionally swallow notifier failures to avoid blocking test completion
-        console.warn('Webhook notification failed:', error.message);
-      } finally {
-        clearTimeout(timeout);
-      }
-    }
-  }
-}
-
-export default NotificationReporter;
-```
-
-**Register custom reporter:**
-
-```typescript
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  reporter: [
-    ['dot'],
-    ['html', { open: 'never' }],
-    ['./reporters/notification-reporter.ts'],
-  ],
-});
-```
-
-## Trace Configuration
-
-Traces capture actions, network requests, DOM snapshots, and console logs.
-
-```typescript
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  retries: process.env.CI ? 2 : 0,
-  use: {
-    trace: 'on-first-retry',
-  },
-});
-```
-
-### Trace Options
-
-| Value | Behavior | Overhead |
-|---|---|---|
-| `'off'` | Never records | None |
-| `'on'` | Every test | High |
-| `'on-first-retry'` | On first retry after failure | Minimal |
-| `'retain-on-failure'` | Records all, keeps failures | Medium |
-| `'retain-on-first-failure'` | Records all, keeps first failure | Medium |
-
-### Viewing Traces
-
-```bash
-# Local trace viewer
-npx playwright show-trace results/my-test/trace.zip
-
-# From HTML report (click Traces tab)
-npx playwright show-report
-
-# Online viewer: https://trace.playwright.dev
-```
-
-## Screenshot & Video Settings
-
-```typescript
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  use: {
-    screenshot: 'only-on-failure',
-    video: 'retain-on-failure',
-  },
-});
-```
-
-### Video with Custom Size
-
-```typescript
-use: {
-  video: {
-    mode: 'retain-on-failure',
-    size: { width: 1280, height: 720 },
-  },
-},
-```
-
-### Screenshot Options
-
-| Value | Captures | Disk Cost |
-|---|---|---|
-| `'off'` | Never | None |
-| `'on'` | Every test | High |
-| `'only-on-failure'` | Failed tests | Low |
-
-### Video Options
-
-| Value | Records | Keeps | Disk Cost |
-|---|---|---|---|
-| `'off'` | Never | — | None |
-| `'on'` | Every test | All | Very high |
-| `'on-first-retry'` | On retry | Retried | Low |
-| `'retain-on-failure'` | Every test | Failed | Medium |
-
-## Artifact Directory Structure
-
-```text
-test-results/
-├── checkout-test-chromium/
-│   ├── trace.zip
-│   ├── test-failed-1.png
-│   └── video.webm
-├── login-test-firefox/
-│   ├── trace.zip
-│   └── test-failed-1.png
-└── junit.xml
-
-playwright-report/
-├── index.html
-└── data/
-
-blob-report/
-└── report-1.zip
-```
-
-## CI Artifact Upload
-
-### GitHub Actions
-
-```yaml
-- uses: actions/upload-artifact@v4
-  if: ${{ !cancelled() }}
-  with:
-    name: playwright-report
-    path: playwright-report/
-    retention-days: 14
-
-- uses: actions/upload-artifact@v4
-  if: failure()
-  with:
-    name: test-traces
-    path: |
-      test-results/**/trace.zip
-      test-results/**/*.png
-      test-results/**/*.webm
-    retention-days: 7
-```
-
-## Decision Guide
-
-| Scenario | Reporter Configuration |
-|---|---|
-| Local development | `[['list'], ['html', { open: 'on-failure' }]]` |
-| GitHub Actions | `[['dot'], ['html'], ['github']]` |
-| GitLab CI | `[['dot'], ['html'], ['junit']]` |
-| Azure DevOps / Jenkins | `[['dot'], ['html'], ['junit']]` |
-| Sharded CI | `[['blob'], ['github']]` |
-| Custom dashboard | `[['json', { outputFile: '...' }]]` + custom reporter |
-
-| Artifact | When to Collect | Retention | Upload Condition |
-|---|---|---|---|
-| HTML report | Always | 14 days | `if: ${{ !cancelled() }}` |
-| Traces | On failure | 7 days | `if: failure()` |
-| Screenshots | On failure | 7 days | `if: failure()` |
-| Videos | On failure | 7 days | `if: failure()` |
-| JUnit XML | Always | 14 days | `if: ${{ !cancelled() }}` |
-| Blob report | Always (sharded) | 1 day | `if: ${{ !cancelled() }}` |
-
-## Anti-Patterns
-
-| Anti-Pattern | Problem | Solution |
-|---|---|---|
-| No reporter configured | Default `list` only; no persistent report | Configure `html` + CI reporter |
-| `trace: 'on'` in CI | Massive artifacts, slow uploads | Use `trace: 'on-first-retry'` |
-| `video: 'on'` in CI | Enormous storage, slower tests | Use `video: 'retain-on-failure'` |
-| Upload artifacts only on failure | No report when tests pass | Upload with `if: ${{ !cancelled() }}` |
-| No retention limits | CI storage fills quickly | Set `retention-days: 7-14` |
-| Only `dot` reporter | Cannot drill into failures | Pair `dot` with `html` |
-| JUnit to stdout | Interferes with console output | Write to file |
-| Blocking `onEnd` in custom reporter | Slow HTTP calls delay pipeline | Use `Promise.race` with timeout |
-
-## Troubleshooting
-
-### Empty HTML Report
-
-Check reporter config. HTML report defaults to `playwright-report/`:
-
-```typescript
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  reporter: [['html', { outputFolder: 'playwright-report', open: 'never' }]],
-});
-```
-
-### Traces Too Large
-
-Switch from `trace: 'on'` to `'on-first-retry'` with retries enabled:
-
-```typescript
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  retries: process.env.CI ? 2 : 0,
-  use: {
-    trace: 'on-first-retry',
-  },
-});
-```
-
-### JUnit XML Not Recognized
-
-Ensure path matches CI configuration:
-
-```typescript
-reporter: [['junit', { outputFile: 'results/junit.xml' }]],
-```
-
-```yaml
-# GitHub Actions
-- uses: dorny/test-reporter@latest
-  with:
-    path: results/junit.xml
-    reporter: java-junit
-
-# Azure DevOps
-- task: PublishTestResults@latest
-  inputs:
-    testResultsFiles: 'results/junit.xml'
-
-# Jenkins
-junit 'results/junit.xml'
-```
-
-### Empty Merged Report
-
-Use `blob` reporter for sharded runs (not `html`):
-
-```typescript
-import { defineConfig } from '@playwright/test';
-
-export default defineConfig({
-  reporter: process.env.CI
-    ? [['blob'], ['dot']]
-    : [['html', { open: 'on-failure' }]],
-});
-```
-
-### Missing Screenshots in Report
-
-Enable screenshots and keep both directories:
-
-```typescript
-use: {
-  screenshot: 'only-on-failure',
-},
-```
-
-The HTML report embeds screenshots from `test-results/`. Deleting that directory removes screenshots from the report.