qa360 2.1.7 → 2.2.0

package/core/src/reporting/index.ts

@@ -0,0 +1,16 @@
+/**
+ * QA360 Reporting Module
+ *
+ * Generates test reports in various formats (HTML, JSON, JUnit)
+ */
+
+export {
+  HTMLReporter,
+  generateHTMLReport,
+  type ReportData,
+  type TestReport,
+  type StepReport,
+  type ReportScreenshotArtifact as ScreenshotArtifact,
+  type ReportVideoArtifact as VideoArtifact,
+  type ReportTraceArtifact as TraceArtifact,
+} from './html-reporter.js';

package/core/src/retry/README.md

@@ -0,0 +1,192 @@
+# F8 Smart Retry Module
+
+**Vision 2.0 - Phase 2**
+
+The final piece of the complete flakiness system: **Detection → Scoring → Quarantine → Smart Retry**
+
+## Overview
+
+The F8 Smart Retry Module provides intelligent, adaptive retry strategies for test execution. It integrates with the flakiness scoring system to make data-driven retry decisions, reducing wasted time on hopeless retries while maximizing recovery from transient failures.
+
+## Features
+
+- **6 Retry Strategies**: NONE, FIXED, LINEAR, EXPONENTIAL, ADAPTIVE, INTELLIGENT
+- **Flakiness-Aware**: Adjusts retry behavior based on historical test flakiness scores
+- **Pattern-Based**: Detects failure patterns (timing, race conditions, selectors) and adapts
+- **Non-Retriable Detection**: Identifies errors that should never be retried
+- **Jitter Support**: Configurable jitter to avoid thundering herd problem
+- **Vault Integration**: Persists retry history to SQLite for analytics
+
+## Installation
+
+```bash
+import {
+  SmartRetryEngine,
+  RetryStrategy,
+  createSmartRetryEngine
+} from 'qa360/core/retry';
+```
+
+## Quick Start
+
+```typescript
+// Create a retry engine with adaptive strategy
+const engine = createSmartRetryEngine({
+  maxRetries: 3,
+  initialDelayMs: 1000,
+  strategy: RetryStrategy.ADAPTIVE,
+  jitterFactor: 0.1
+});
+
+// Execute a test with retry
+const result = await engine.executeWithRetry('test-1', async () => {
+  // Your test code here
+  await runMyTest();
+});
+
+console.log(result);
+// {
+//   testId: 'test-1',
+//   success: true,
+//   totalAttempts: 2,
+//   successfulAttempt: 2,
+//   attempts: [...],
+//   totalDurationMs: 1500,
+//   strategy: 'adaptive',
+//   stopReason: 'success'
+// }
+```
+
+## Retry Strategies
+
+### NONE
+No retry, fail immediately. Use for tests that should never retry.
+
+### FIXED
+Fixed delay between retries. Good for simple transient failures.
+
+### LINEAR
+Delay increases linearly with each retry. Good for rate-limited APIs.
+
+### EXPONENTIAL
+Delay doubles with each retry. Good for network issues.
+
+### ADAPTIVE
+Adjusts strategy based on flakiness score:
+- Score ≥ 90: No retry (Legendary/Solid)
+- Score ≥ 75: Fixed delay (Good)
+- Score ≥ 50: Linear backoff (Shaky)
+- Score < 50: Exponential backoff (Unstable)
+
+### INTELLIGENT
+Pattern-based retry with intelligent delays:
+- Timing issues: Exponential backoff starting at 2s
+- External dependency: Linear with longer delays
+- Selector issues: Fixed delay (resolves quickly)
+- Race conditions: Exponential with jitter
+
+## Flakiness-Aware Retry
+
+```typescript
+import { FlakinessAwareRetryEngine } from 'qa360/core/retry';
+
+const engine = new FlakinessAwareRetryEngine();
+
+// Load historical flakiness data
+engine.loadFlakinessData([
+  {
+    testId: 'test-1',
+    score: 85,  // Good - occasionally flaky
+    category: 'good',
+    totalRuns: 10,
+    successfulRuns: 8
+  }
+]);
+
+// Get retry config for a specific test
+const config = engine.getRetryConfigForTest('test-1', baseConfig);
+// → maxRetries: 2, strategy: FIXED
+
+// Get comprehensive retry recommendation
+const rec = engine.getRetryRecommendation('test-1');
+// → {
+//   shouldRetry: true,
+//   strategy: RetryStrategy.FIXED,
+//   maxRetries: 2,
+//   initialDelayMs: 1000,
+//   confidence: 0.75,
+//   reason: 'Test is occasionally flaky'
+// }
+```
+
+## Configuration
+
+```typescript
+interface RetryConfig {
+  maxRetries: number;              // Maximum retry attempts (default: 3)
+  initialDelayMs: number;          // Initial delay in ms (default: 1000)
+  maxDelayMs: number;              // Maximum delay in ms (default: 30000)
+  attemptTimeoutMs: number;        // Timeout per attempt (default: 30000)
+  overallTimeoutMs?: number;       // Overall timeout (default: 120000)
+  strategy: RetryStrategy;         // Retry strategy (default: ADAPTIVE)
+  backoffMultiplier: number;       // Backoff multiplier (default: 2)
+  jitterFactor: number;            // Jitter 0-1 (default: 0.1)
+  retryOnTimeout: boolean;         // Retry on timeout? (default: false)
+  retryOnAssertionFailure: boolean;// Retry assertions? (default: false)
+  nonRetriableErrors: string[];    // Errors to never retry
+}
+```
+
+## Non-Retriable Errors
+
+By default, these errors are never retried:
+- `OutOfMemoryError`
+- `StackOverflowError`
+- `SecurityError`
+- `PermissionDenied`
+- `InvalidArgument`
+- `ValidationError`
+- `SyntaxError`
+
+You can customize this list:
+
+```typescript
+const engine = createSmartRetryEngine({
+  nonRetriableErrors: [
+    'MyCustomError',
+    'DatabaseConnectionError'
+  ]
+});
+```
+
+## Testing
+
+The module includes comprehensive tests:
+
+```bash
+# Run all retry tests
+pnpm test retry
+
+# Run specific test files
+pnpm test retry-engine
+pnpm test flakiness-integration
+```
+
+**Test Coverage:**
+- `retry-engine.test.ts`: 30 tests
+- `flakiness-integration.test.ts`: 18 tests
+- **Total: 48 tests**
+
+## Files
+
+| File | Description |
+|------|-------------|
+| `types.ts` | Type definitions and enums |
+| `retry-engine.ts` | Core SmartRetryEngine class |
+| `flakiness-integration.ts` | FlakinessAwareRetryEngine |
+| `vault.ts` | SQLite persistence for retry history |
+| `index.ts` | Public API exports |
+
+## License
+
+MIT

package/core/src/retry/__tests__/flakiness-integration.test.ts

@@ -0,0 +1,475 @@
+/**
+ * Flakiness Integration Tests
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  FlakinessAwareRetryEngine,
+  createFlakinessAwareRetryEngine
+} from '../flakiness-integration.js';
+import { RetryStrategy } from '../types.js';
+import type { FlakinessResult } from '../flakiness/index.js';
+import type { TestResult } from '../flakiness/index.js';
+
+describe('FlakinessAwareRetryEngine', () => {
+  let engine: FlakinessAwareRetryEngine;
+
+  beforeEach(() => {
+    engine = createFlakinessAwareRetryEngine();
+  });
+
+  describe('loading flakiness data', () => {
+    it('should load flakiness results', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'test1',
+          testName: 'Test 1',
+          filePath: '/test1.ts',
+          gate: 'api',
+          score: 80,
+          category: 'good' as any,
+          totalRuns: 10,
+          successfulRuns: 8,
+          avgDurationMs: 100
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      const history = engine.getFlakinessHistory();
+      expect(history.size).toBe(1);
+      expect(history.get('test1')?.score).toBe(80);
+    });
+
+    it('should handle empty results', () => {
+      engine.loadFlakinessData([]);
+
+      const history = engine.getFlakinessHistory();
+      expect(history.size).toBe(0);
+    });
+  });
+
+  describe('adding test results', () => {
+    it('should add test result to history', () => {
+      const result: TestResult = {
+        testId: 'test1',
+        testName: 'Test 1',
+        filePath: '/test1.ts',
+        gate: 'api',
+        success: true,
+        durationMs: 100,
+        timestamp: Date.now(),
+        environment: 'local'
+      };
+
+      engine.addTestResult('test1', result);
+
+      // Should be stored internally
+      expect(engine.getFlakinessHistory().size).toBe(0); // No flakiness data yet
+    });
+
+    it('should accumulate results for same test', () => {
+      const result: TestResult = {
+        testId: 'test1',
+        testName: 'Test 1',
+        filePath: '/test1.ts',
+        gate: 'api',
+        success: true,
+        durationMs: 100,
+        timestamp: Date.now(),
+        environment: 'local'
+      };
+
+      engine.addTestResult('test1', result);
+      engine.addTestResult('test1', { ...result, success: false });
+
+      // Results accumulate
+      const history = engine.getFlakinessHistory();
+      expect(history.size).toBe(0);
+    });
+  });
+
+  describe('getRetryConfigForTest', () => {
+    const baseConfig = {
+      maxRetries: 3,
+      initialDelayMs: 1000,
+      attemptTimeoutMs: 30000,
+      overallTimeoutMs: 120000,
+      strategy: RetryStrategy.ADAPTIVE,
+      backoffMultiplier: 2,
+      jitterFactor: 0.1,
+      retryOnTimeout: true,
+      retryOnAssertionFailure: false,
+      nonRetriableErrors: [],
+      useFlakinessScore: true,
+      minHistoricalRuns: 3,
+      boostRetriesForFlaky: true,
+      reduceRetriesForStable: true
+    };
+
+    it('should return base config when no flakiness data', () => {
+      const config = engine.getRetryConfigForTest('unknown', baseConfig as any);
+
+      expect(config.maxRetries).toBe(3);
+      expect(config.strategy).toBe(RetryStrategy.ADAPTIVE);
+    });
+
+    it('should reduce retries for stable tests', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'stable-test',
+          testName: 'Stable Test',
+          filePath: '/stable.ts',
+          gate: 'api',
+          score: 95,
+          category: 'solid' as any,
+          totalRuns: 10,
+          successfulRuns: 9,
+          avgDurationMs: 100
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      // Add test history
+      for (let i = 0; i < 5; i++) {
+        engine.addTestResult('stable-test', {
+          testId: 'stable-test',
+          testName: 'Stable Test',
+          filePath: '/stable.ts',
+          gate: 'api',
+          success: true,
+          durationMs: 100,
+          timestamp: Date.now(),
+          environment: 'local'
+        });
+      }
+
+      const config = engine.getRetryConfigForTest('stable-test', baseConfig as any);
+
+      expect(config.maxRetries).toBeLessThan(baseConfig.maxRetries);
+      expect(config.strategy).toBe(RetryStrategy.FIXED);
+    });
+
+    it('should boost retries for flaky tests', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'flaky-test',
+          testName: 'Flaky Test',
+          filePath: '/flaky.ts',
+          gate: 'api',
+          score: 40,
+          category: 'unstable' as any,
+          totalRuns: 10,
+          successfulRuns: 4,
+          avgDurationMs: 100
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      for (let i = 0; i < 5; i++) {
+        engine.addTestResult('flaky-test', {
+          testId: 'flaky-test',
+          testName: 'Flaky Test',
+          filePath: '/flaky.ts',
+          gate: 'api',
+          success: i < 2,
+          durationMs: 100,
+          timestamp: Date.now(),
+          environment: 'local'
+        });
+      }
+
+      const config = engine.getRetryConfigForTest('flaky-test', baseConfig as any);
+
+      expect(config.maxRetries).toBeGreaterThan(baseConfig.maxRetries);
+      expect(config.strategy).toBe(RetryStrategy.INTELLIGENT);
+    });
+
+    it('should adjust based on pattern type', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'timing-test',
+          testName: 'Timing Test',
+          filePath: '/timing.ts',
+          gate: 'ui',
+          score: 60,
+          category: 'shaky' as any,
+          totalRuns: 10,
+          successfulRuns: 6,
+          avgDurationMs: 200,
+          patternType: 'timing'
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      for (let i = 0; i < 5; i++) {
+        engine.addTestResult('timing-test', {
+          testId: 'timing-test',
+          testName: 'Timing Test',
+          filePath: '/timing.ts',
+          gate: 'ui',
+          success: true,
+          durationMs: 200,
+          timestamp: Date.now(),
+          environment: 'local'
+        });
+      }
+
+      const config = engine.getRetryConfigForTest('timing-test', baseConfig as any);
+
+      expect(config.initialDelayMs).toBeGreaterThanOrEqual(2000);
+      expect(config.retryOnTimeout).toBe(true);
+    });
+
+    it('should use exponential for external dependency pattern', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'external-test',
+          testName: 'External Test',
+          filePath: '/external.ts',
+          gate: 'api',
+          score: 50,
+          category: 'shaky' as any,
+          totalRuns: 10,
+          successfulRuns: 5,
+          avgDurationMs: 150,
+          patternType: 'external_dependency'
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      for (let i = 0; i < 5; i++) {
+        engine.addTestResult('external-test', {
+          testId: 'external-test',
+          testName: 'External Test',
+          filePath: '/external.ts',
+          gate: 'api',
+          success: true,
+          durationMs: 150,
+          timestamp: Date.now(),
+          environment: 'local'
+        });
+      }
+
+      const config = engine.getRetryConfigForTest('external-test', baseConfig as any);
+
+      expect(config.strategy).toBe(RetryStrategy.EXPONENTIAL);
+    });
+
+    it('should use fixed for selector issues', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'selector-test',
+          testName: 'Selector Test',
+          filePath: '/selector.ts',
+          gate: 'ui',
+          score: 50,
+          category: 'shaky' as any,
+          totalRuns: 10,
+          successfulRuns: 5,
+          avgDurationMs: 100,
+          patternType: 'selector_issue'
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      for (let i = 0; i < 5; i++) {
+        engine.addTestResult('selector-test', {
+          testId: 'selector-test',
+          testName: 'Selector Test',
+          filePath: '/selector.ts',
+          gate: 'ui',
+          success: true,
+          durationMs: 100,
+          timestamp: Date.now(),
+          environment: 'local'
+        });
+      }
+
+      const config = engine.getRetryConfigForTest('selector-test', baseConfig as any);
+
+      expect(config.strategy).toBe(RetryStrategy.FIXED);
+    });
+  });
+
+  describe('getRetryRecommendation', () => {
+    it('should return default recommendation without data', () => {
+      const rec = engine.getRetryRecommendation('unknown');
+
+      expect(rec.shouldRetry).toBe(true);
+      expect(rec.strategy).toBe(RetryStrategy.ADAPTIVE);
+      expect(rec.confidence).toBe(0.5);
+      expect(rec.reason).toContain('Insufficient historical data');
+    });
+
+    it('should return no-retry for stable tests', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'stable-test',
+          testName: 'Stable Test',
+          filePath: '/stable.ts',
+          gate: 'api',
+          score: 95,
+          category: 'solid' as any,
+          totalRuns: 10,
+          successfulRuns: 9,
+          avgDurationMs: 100
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      for (let i = 0; i < 5; i++) {
+        engine.addTestResult('stable-test', {
+          testId: 'stable-test',
+          testName: 'Stable Test',
+          filePath: '/stable.ts',
+          gate: 'api',
+          success: true,
+          durationMs: 100,
+          timestamp: Date.now(),
+          environment: 'local'
+        });
+      }
+
+      const rec = engine.getRetryRecommendation('stable-test');
+
+      expect(rec.shouldRetry).toBe(false);
+      expect(rec.strategy).toBe(RetryStrategy.NONE);
+      expect(rec.confidence).toBe(0.9);
+    });
+
+    it('should return aggressive retry for unstable tests', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'unstable-test',
+          testName: 'Unstable Test',
+          filePath: '/unstable.ts',
+          gate: 'api',
+          score: 20,
+          category: 'unstable' as any,
+          totalRuns: 10,
+          successfulRuns: 2,
+          avgDurationMs: 100
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      for (let i = 0; i < 5; i++) {
+        engine.addTestResult('unstable-test', {
+          testId: 'unstable-test',
+          testName: 'Unstable Test',
+          filePath: '/unstable.ts',
+          gate: 'api',
+          success: false,
+          durationMs: 100,
+          timestamp: Date.now(),
+          environment: 'local'
+        });
+      }
+
+      const rec = engine.getRetryRecommendation('unstable-test');
+
+      expect(rec.shouldRetry).toBe(true);
+      expect(rec.strategy).toBe(RetryStrategy.INTELLIGENT);
+      expect(rec.maxRetries).toBe(5);
+      expect(rec.reason).toContain('aggressive');
+    });
+  });
+
+  describe('shouldSkipTest', () => {
+    it('should not skip test without data', () => {
+      expect(engine.shouldSkipTest('unknown')).toBe(false);
+    });
+
+    it('should skip very flaky tests with many runs', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'very-flaky',
+          testName: 'Very Flaky',
+          filePath: '/flaky.ts',
+          gate: 'api',
+          score: 20,
+          category: 'unstable' as any,
+          totalRuns: 15,
+          successfulRuns: 3,
+          avgDurationMs: 100
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      expect(engine.shouldSkipTest('very-flaky')).toBe(true);
+    });
+
+    it('should not skip tests with good scores', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'good-test',
+          testName: 'Good Test',
+          filePath: '/good.ts',
+          gate: 'api',
+          score: 80,
+          category: 'good' as any,
+          totalRuns: 15,
+          successfulRuns: 12,
+          avgDurationMs: 100
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+
+      expect(engine.shouldSkipTest('good-test')).toBe(false);
+    });
+  });
+
+  describe('clearHistory', () => {
+    it('should clear all history', () => {
+      const results: FlakinessResult[] = [
+        {
+          testId: 'test1',
+          testName: 'Test 1',
+          filePath: '/test1.ts',
+          gate: 'api',
+          score: 80,
+          category: 'good' as any,
+          totalRuns: 10,
+          successfulRuns: 8,
+          avgDurationMs: 100
+        }
+      ];
+
+      engine.loadFlakinessData(results);
+      engine.addTestResult('test1', {
+        testId: 'test1',
+        testName: 'Test 1',
+        filePath: '/test1.ts',
+        gate: 'api',
+        success: true,
+        durationMs: 100,
+        timestamp: Date.now(),
+        environment: 'local'
+      });
+
+      expect(engine.getFlakinessHistory().size).toBe(1);
+
+      engine.clearHistory();
+
+      expect(engine.getFlakinessHistory().size).toBe(0);
+    });
+  });
+});
+
+describe('createFlakinessAwareRetryEngine', () => {
+  it('should create FlakinessAwareRetryEngine instance', () => {
+    const engine = createFlakinessAwareRetryEngine();
+
+    expect(engine).toBeInstanceOf(FlakinessAwareRetryEngine);
+  });
+});