qa360 2.1.7 → 2.2.0

package/core/src/retry/__tests__/retry-engine.test.ts

@@ -0,0 +1,424 @@
+/**
+ * Smart Retry Engine Tests
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import {
+  SmartRetryEngine,
+  createSmartRetryEngine
+} from '../retry-engine.js';
+import { RetryStrategy } from '../types.js';
+import type { RetryConfig } from '../types.js';
+
+describe('SmartRetryEngine', () => {
+  let engine: SmartRetryEngine;
+
+  beforeEach(() => {
+    engine = createSmartRetryEngine();
+  });
+
+  describe('construction', () => {
+    it('should create engine with default config', () => {
+      const config = engine.getConfig();
+      expect(config.maxRetries).toBe(3);
+      expect(config.initialDelayMs).toBe(1000);
+      expect(config.strategy).toBe(RetryStrategy.ADAPTIVE);
+    });
+
+    it('should create engine with custom config', () => {
+      const customEngine = createSmartRetryEngine({
+        maxRetries: 5,
+        initialDelayMs: 2000,
+        strategy: RetryStrategy.EXPONENTIAL
+      });
+
+      const config = customEngine.getConfig();
+      expect(config.maxRetries).toBe(5);
+      expect(config.initialDelayMs).toBe(2000);
+      expect(config.strategy).toBe(RetryStrategy.EXPONENTIAL);
+    });
+  });
+
+  describe('executeWithRetry - immediate success', () => {
+    it('should succeed on first attempt', async () => {
+      const executor = vi.fn().mockResolvedValue('success');
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(true);
+      expect(result.totalAttempts).toBe(1);
+      expect(result.successfulAttempt).toBe(1);
+      expect(result.attempts.length).toBe(1);
+      expect(result.attempts[0].success).toBe(true);
+      expect(executor).toHaveBeenCalledTimes(1);
+    });
+
+    it('should not delay on first attempt', async () => {
+      const executor = vi.fn().mockResolvedValue('success');
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.attempts[0].delayMs).toBe(0);
+    });
+  });
+
+  describe('executeWithRetry - retry on failure', () => {
+    it('should retry on failure', async () => {
+      let attempts = 0;
+      const executor = vi.fn().mockImplementation(() => {
+        attempts++;
+        if (attempts < 2) throw new Error('Temporary failure');
+        return 'success';
+      });
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(true);
+      expect(result.totalAttempts).toBe(2);
+      expect(result.successfulAttempt).toBe(2);
+      expect(executor).toHaveBeenCalledTimes(2);
+    }, 10000);
+
+    it('should respect max retries', async () => {
+      const executor = vi.fn().mockRejectedValue(new Error('Always fails'));
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(false);
+      expect(result.totalAttempts).toBe(4); // 1 initial + 3 retries
+      expect(result.stopReason).toBe('max_retries');
+      expect(executor).toHaveBeenCalledTimes(4);
+    }, 20000);
+
+    it('should stop on successful retry', async () => {
+      let attempts = 0;
+      const executor = vi.fn().mockImplementation(() => {
+        attempts++;
+        if (attempts < 3) throw new Error('Fails first two times');
+        return 'success';
+      });
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(true);
+      expect(result.totalAttempts).toBe(3);
+      expect(result.stopReason).toBe('success');
+    }, 10000);
+  });
+
+  describe('retry strategies', () => {
+    it('should use fixed delay strategy', async () => {
+      const fixedEngine = createSmartRetryEngine({
+        maxRetries: 2,
+        initialDelayMs: 100,
+        strategy: RetryStrategy.FIXED,
+        jitterFactor: 0
+      });
+
+      const executor = vi.fn()
+        .mockRejectedValueOnce(new Error('Fail 1'))
+        .mockRejectedValueOnce(new Error('Fail 2'))
+        .mockResolvedValue('success');
+
+      await fixedEngine.executeWithRetry('test-1', executor);
+
+      expect(executor).toHaveBeenCalledTimes(3);
+    }, 10000);
+
+    it('should use exponential backoff', async () => {
+      const expEngine = createSmartRetryEngine({
+        maxRetries: 3,
+        initialDelayMs: 50,
+        strategy: RetryStrategy.EXPONENTIAL,
+        backoffMultiplier: 2,
+        jitterFactor: 0
+      });
+
+      const executor = vi.fn()
+        .mockRejectedValue(new Error('Fail'));
+
+      await expEngine.executeWithRetry('test-1', executor);
+
+      expect(executor).toHaveBeenCalledTimes(4);
+    }, 10000);
+
+    it('should use linear backoff', async () => {
+      const linearEngine = createSmartRetryEngine({
+        maxRetries: 3,
+        initialDelayMs: 50,
+        strategy: RetryStrategy.LINEAR,
+        backoffMultiplier: 2,
+        jitterFactor: 0
+      });
+
+      const executor = vi.fn()
+        .mockRejectedValue(new Error('Fail'));
+
+      await linearEngine.executeWithRetry('test-1', executor);
+
+      expect(executor).toHaveBeenCalledTimes(4);
+    }, 10000);
+  });
+
+  describe('non-retriable errors', () => {
+    it('should not retry on OutOfMemoryError', async () => {
+      const executor = vi.fn().mockRejectedValue(new Error('OutOfMemoryError'));
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(false);
+      expect(result.totalAttempts).toBe(1);
+      expect(result.stopReason).toBe('unrecoverable');
+      expect(executor).toHaveBeenCalledTimes(1);
+    }, 15000);
+
+    it('should not retry on PermissionDenied', async () => {
+      const executor = vi.fn().mockRejectedValue(new Error('PermissionDenied'));
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(false);
+      expect(result.totalAttempts).toBe(1);
+      expect(result.stopReason).toBe('unrecoverable');
+    }, 15000);
+
+    it('should not retry on SyntaxError', async () => {
+      const executor = vi.fn().mockRejectedValue(new SyntaxError('Bad syntax'));
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(false);
+      expect(result.totalAttempts).toBe(1);
+      expect(result.stopReason).toBe('unrecoverable');
+    }, 15000);
+  });
+
+  describe('assertion failures', () => {
+    it('should not retry assertion failures by default', async () => {
+      const executor = vi.fn().mockRejectedValue(new Error('AssertionError: Expected true'));
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(false);
+      expect(result.totalAttempts).toBe(1);
+      expect(result.stopReason).toBe('unrecoverable');
+    }, 15000);
+
+    it('should retry assertion failures when enabled', async () => {
+      const retryEngine = createSmartRetryEngine({
+        maxRetries: 2,
+        retryOnAssertionFailure: true
+      });
+
+      let attempts = 0;
+      const executor = vi.fn().mockImplementation(() => {
+        attempts++;
+        if (attempts === 1) throw new Error('AssertionError: Expected true');
+        return 'success';
+      });
+
+      const result = await retryEngine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(true);
+      expect(result.totalAttempts).toBe(2);
+    });
+  });
+
+  describe('timeout handling', () => {
+    it('should not retry timeout by default', async () => {
+      const executor = vi.fn().mockRejectedValue(new Error('TimeoutError'));
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(false);
+      expect(result.totalAttempts).toBe(1);
+      expect(result.stopReason).toBe('unrecoverable');
+    }, 15000);
+
+    it('should retry timeout when enabled', async () => {
+      const retryEngine = createSmartRetryEngine({
+        maxRetries: 2,
+        retryOnTimeout: true
+      });
+
+      let attempts = 0;
+      const executor = vi.fn().mockImplementation(() => {
+        attempts++;
+        if (attempts === 1) throw new Error('TimeoutError');
+        return 'success';
+      });
+
+      const result = await retryEngine.executeWithRetry('test-1', executor);
+
+      expect(result.success).toBe(true);
+      expect(result.totalAttempts).toBe(2);
+    });
+
+    it('should stop on overall timeout', async () => {
+      const retryEngine = createSmartRetryEngine({
+        maxRetries: 10,
+        overallTimeoutMs: 100,
+        initialDelayMs: 50
+      });
+
+      const executor = vi.fn().mockRejectedValue(new Error('Fail'));
+
+      const result = await retryEngine.executeWithRetry('test-1', executor);
+
+      // Should stop before max retries due to overall timeout
+      expect(result.totalAttempts).toBeLessThan(11);
+      expect(result.stopReason).toBe('timeout');
+    });
+  });
+
+  describe('retry recommendations', () => {
+    it('should not retry for high flakiness score', () => {
+      const rec = engine.getRetryRecommendation({
+        flakinessScore: 95
+      });
+
+      expect(rec.shouldRetry).toBe(false);
+      expect(rec.strategy).toBe(RetryStrategy.NONE);
+      expect(rec.reason).toContain('highly reliable');
+    });
+
+    it('should recommend fixed retry for medium flakiness', () => {
+      const rec = engine.getRetryRecommendation({
+        flakinessScore: 80
+      });
+
+      expect(rec.shouldRetry).toBe(true);
+      expect(rec.strategy).toBe(RetryStrategy.FIXED);
+      expect(rec.maxRetries).toBe(2);
+    });
+
+    it('should recommend adaptive retry for low flakiness', () => {
+      const rec = engine.getRetryRecommendation({
+        flakinessScore: 60
+      });
+
+      expect(rec.shouldRetry).toBe(true);
+      expect(rec.strategy).toBe(RetryStrategy.ADAPTIVE);
+      expect(rec.maxRetries).toBe(3);
+    });
+
+    it('should recommend intelligent retry for very low flakiness', () => {
+      const rec = engine.getRetryRecommendation({
+        flakinessScore: 30
+      });
+
+      expect(rec.shouldRetry).toBe(true);
+      expect(rec.strategy).toBe(RetryStrategy.INTELLIGENT);
+      expect(rec.maxRetries).toBe(5);
+      expect(rec.reason).toContain('aggressive');
+    });
+
+    it('should not retry for non-retriable errors', () => {
+      const rec = engine.getRetryRecommendation({
+        flakinessScore: 50,
+        errorType: 'OutOfMemoryError'
+      });
+
+      expect(rec.shouldRetry).toBe(false);
+      expect(rec.reason).toContain('not retriable');
+    });
+  });
+
+  describe('jitter', () => {
+    it('should apply jitter to delays', async () => {
+      const jitterEngine = createSmartRetryEngine({
+        maxRetries: 2,
+        initialDelayMs: 100,
+        jitterFactor: 0.5
+      });
+
+      const delays: number[] = [];
+      const executor = vi.fn().mockImplementation(async () => {
+        if (executor.mock.calls.length > 1) {
+          delays.push(jitterEngine.getConfig().initialDelayMs);
+        }
+        throw new Error('Fail');
+      });
+
+      await jitterEngine.executeWithRetry('test-1', executor);
+
+      // Delays should vary due to jitter
+      expect(executor).toHaveBeenCalledTimes(3);
+    });
+  });
+
+  describe('config updates', () => {
+    it('should update config', () => {
+      engine.updateConfig({ maxRetries: 5 });
+
+      const config = engine.getConfig();
+      expect(config.maxRetries).toBe(5);
+    });
+
+    it('should preserve other config values when updating', () => {
+      const originalDelay = engine.getConfig().initialDelayMs;
+      engine.updateConfig({ maxRetries: 5 });
+
+      const config = engine.getConfig();
+      expect(config.initialDelayMs).toBe(originalDelay);
+    });
+  });
+
+  describe('attempt tracking', () => {
+    it('should record all attempt details', async () => {
+      const executor = vi.fn()
+        .mockRejectedValueOnce(new Error('Fail 1'))
+        .mockRejectedValueOnce(new Error('Fail 2'))
+        .mockResolvedValue('success');
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.attempts.length).toBe(3);
+
+      // First attempt
+      expect(result.attempts[0].attemptNumber).toBe(1);
+      expect(result.attempts[0].success).toBe(false);
+      expect(result.attempts[0].delayMs).toBe(0);
+      expect(result.attempts[0].errorMessage).toBe('Fail 1');
+
+      // Second attempt
+      expect(result.attempts[1].attemptNumber).toBe(2);
+      expect(result.attempts[1].success).toBe(false);
+      expect(result.attempts[1].delayMs).toBeGreaterThan(0);
+      expect(result.attempts[1].errorMessage).toBe('Fail 2');
+
+      // Third attempt
+      expect(result.attempts[2].attemptNumber).toBe(3);
+      expect(result.attempts[2].success).toBe(true);
+    }, 15000);
+
+    it('should calculate total duration correctly', async () => {
+      const executor = vi.fn().mockImplementation(async () => {
+        await new Promise(resolve => setTimeout(resolve, 10));
+        if (executor.mock.calls.length < 2) throw new Error('Fail');
+        return 'success';
+      });
+
+      const result = await engine.executeWithRetry('test-1', executor);
+
+      expect(result.totalDurationMs).toBeGreaterThanOrEqual(20); // 2 attempts * 10ms
+      expect(result.success).toBe(true);
+    }, 15000);
+  });
+});
+
+describe('createSmartRetryEngine', () => {
+  it('should create SmartRetryEngine instance', () => {
+    const engine = createSmartRetryEngine();
+
+    expect(engine).toBeInstanceOf(SmartRetryEngine);
+  });
+
+  it('should pass config to engine', () => {
+    const engine = createSmartRetryEngine({
+      maxRetries: 10
+    });
+
+    expect(engine.getConfig().maxRetries).toBe(10);
+  });
+});

package/core/src/retry/flakiness-integration.ts

@@ -0,0 +1,267 @@
+/**
+ * Flakiness Integration for Smart Retry
+ *
+ * Integrates Smart Retry with the Flakiness Detection Engine
+ * to make intelligent retry decisions based on historical data.
+ */
+
+import type { FlakinessResult, TestResult as FlakinessTestResult } from '../flakiness/index.js';
+import type {
+  RetryConfig,
+  RetryRecommendation
+} from './types.js';
+import { RetryStrategy } from './types.js';
+
+/**
+ * Enhanced retry config with flakiness awareness
+ */
+export interface FlakinessAwareRetryConfig extends RetryConfig {
+  /** Use flakiness score to determine retry strategy */
+  useFlakinessScore?: boolean;
+  /** Minimum historical runs before using flakiness data */
+  minHistoricalRuns?: number;
+  /** Boost retry attempts for known flaky tests */
+  boostRetriesForFlaky?: boolean;
+  /** Reduce retries for known stable tests */
+  reduceRetriesForStable?: boolean;
+}
+
+/**
+ * Flakiness-aware retry engine
+ */
+export class FlakinessAwareRetryEngine {
+  private flakinessHistory: Map<string, FlakinessResult> = new Map();
+  private testHistory: Map<string, FlakinessTestResult[]> = new Map();
+
+  /**
+   * Load flakiness data
+   */
+  loadFlakinessData(results: FlakinessResult[]): void {
+    for (const result of results) {
+      this.flakinessHistory.set(result.testId, result);
+    }
+  }
+
+  /**
+   * Add test result to history
+   */
+  addTestResult(testId: string, result: FlakinessTestResult): void {
+    if (!this.testHistory.has(testId)) {
+      this.testHistory.set(testId, []);
+    }
+    this.testHistory.get(testId)!.push(result);
+  }
+
+  /**
+   * Get retry config for a specific test based on flakiness
+   */
+  getRetryConfigForTest(
+    testId: string,
+    baseConfig: FlakinessAwareRetryConfig
+  ): RetryConfig {
+    const flakiness = this.flakinessHistory.get(testId);
+    const recentResults = this.testHistory.get(testId);
+
+    // If no flakiness data, return base config
+    const minRuns = baseConfig.minHistoricalRuns ?? 0;
+    if (!flakiness || !recentResults || recentResults.length < minRuns) {
+      return baseConfig;
+    }
+
+    const config = { ...baseConfig };
+
+    // Adjust based on flakiness score
+    if (baseConfig.useFlakinessScore) {
+      const score = flakiness.score;
+
+      if (score >= 90) {
+        // Legendary/Solid - reduce retries
+        if (baseConfig.reduceRetriesForStable) {
+          config.maxRetries = Math.max(0, baseConfig.maxRetries - 1);
+          config.strategy = RetryStrategy.FIXED;
+        }
+      } else if (score >= 75) {
+        // Good - keep base config
+        config.strategy = RetryStrategy.FIXED;
+      } else if (score >= 50) {
+        // Shaky - increase retries slightly
+        if (baseConfig.boostRetriesForFlaky) {
+          config.maxRetries = baseConfig.maxRetries + 1;
+          config.strategy = RetryStrategy.ADAPTIVE;
+        }
+      } else {
+        // Unstable - aggressive retry
+        if (baseConfig.boostRetriesForFlaky) {
+          config.maxRetries = baseConfig.maxRetries + 2;
+          config.strategy = RetryStrategy.INTELLIGENT;
+        }
+      }
+    }
+
+    // Adjust based on pattern type
+    if (flakiness.patternType) {
+      switch (flakiness.patternType) {
+        case 'timing':
+          config.initialDelayMs = Math.max(config.initialDelayMs, 2000);
+          config.retryOnTimeout = true;
+          break;
+        case 'external_dependency':
+          config.initialDelayMs = Math.max(config.initialDelayMs, 3000);
+          config.strategy = RetryStrategy.EXPONENTIAL;
+          break;
+        case 'race_condition':
+          config.strategy = RetryStrategy.EXPONENTIAL;
+          config.jitterFactor = Math.max(config.jitterFactor, 0.2);
+          break;
+        case 'selector_issue':
+          config.strategy = RetryStrategy.FIXED;
+          config.maxRetries = Math.min(config.maxRetries, 3);
+          break;
+      }
+    }
+
+    // Adjust based on recent streak
+    if (flakiness.currentStreak && flakiness.currentStreak > 3) {
+      if (flakiness.streakType === 'fail') {
+        // Long fail streak - boost retries
+        config.maxRetries = config.maxRetries + 1;
+      } else {
+        // Long pass streak - reduce retries
+        config.maxRetries = Math.max(1, config.maxRetries - 1);
+      }
+    }
+
+    return config;
+  }
+
+  /**
+   * Get comprehensive retry recommendation
+   */
+  getRetryRecommendation(testId: string): RetryRecommendation {
+    const flakiness = this.flakinessHistory.get(testId);
+    const recentResults = this.testHistory.get(testId);
+
+    if (!flakiness || !recentResults || recentResults.length < 3) {
+      return {
+        shouldRetry: true,
+        strategy: RetryStrategy.ADAPTIVE,
+        maxRetries: 3,
+        initialDelayMs: 1000,
+        confidence: 0.5,
+        reason: 'Insufficient historical data, using default adaptive retry'
+      };
+    }
+
+    // Analyze recent failures
+    const recentFailures = recentResults.slice(-10).filter(r => !r.success);
+    const failureRate = recentFailures.length / Math.min(recentResults.length, 10);
+
+    // Build recommendation
+    let strategy: RetryStrategy;
+    let maxRetries: number;
+    let initialDelayMs: number;
+    let confidence: number;
+    const reasons: string[] = [];
+
+    // Base recommendation on flakiness score
+    if (flakiness.score >= 90) {
+      strategy = RetryStrategy.NONE;
+      maxRetries = 0;
+      initialDelayMs = 0;
+      confidence = 0.9;
+      reasons.push('Test is highly reliable');
+    } else if (flakiness.score >= 75) {
+      strategy = RetryStrategy.FIXED;
+      maxRetries = 2;
+      initialDelayMs = 1000;
+      confidence = 0.75;
+      reasons.push('Test is occasionally flaky');
+    } else if (flakiness.score >= 50) {
+      strategy = RetryStrategy.ADAPTIVE;
+      maxRetries = 3;
+      initialDelayMs = 1000;
+      confidence = 0.85;
+      reasons.push('Test is often flaky');
+    } else {
+      strategy = RetryStrategy.INTELLIGENT;
+      maxRetries = 5;
+      initialDelayMs = 2000;
+      confidence = 0.9;
+      reasons.push('Test is very flaky, aggressive retry recommended');
+    }
+
+    // Adjust based on pattern
+    if (flakiness.patternType) {
+      reasons.push(`Pattern: ${flakiness.patternType}`);
+
+      switch (flakiness.patternType) {
+        case 'timing':
+          initialDelayMs = Math.max(initialDelayMs, 2000);
+          strategy = flakiness.score < 75 ? RetryStrategy.EXPONENTIAL : strategy;
+          break;
+        case 'external_dependency':
+          initialDelayMs = Math.max(initialDelayMs, 3000);
+          break;
+        case 'selector_issue':
+          strategy = RetryStrategy.FIXED;
+          maxRetries = Math.min(maxRetries, 3);
+          break;
+      }
+    }
+
+    // Adjust based on recent failure rate
+    if (failureRate > 0.5) {
+      maxRetries = Math.min(maxRetries + 1, 5);
+      reasons.push(`High recent failure rate (${Math.round(failureRate * 100)}%)`);
+    } else if (failureRate < 0.1 && flakiness.score >= 75) {
+      // For highly reliable tests (score >= 90), keep maxRetries at 0
+      if (flakiness.score < 90) {
+        maxRetries = Math.max(maxRetries - 1, 1);
+      }
+      reasons.push('Low recent failure rate');
+    }
+
+    return {
+      shouldRetry: maxRetries > 0,
+      strategy,
+      maxRetries,
+      initialDelayMs,
+      confidence,
+      reason: reasons.join('; '),
+      pattern: flakiness.patternType
+    };
+  }
+
+  /**
+   * Check if test should be skipped (too flaky)
+   */
+  shouldSkipTest(testId: string): boolean {
+    const flakiness = this.flakinessHistory.get(testId);
+    if (!flakiness) return false;
+
+    // Skip if score is very low and has many runs
+    return flakiness.score < 30 && flakiness.totalRuns >= 10;
+  }
+
+  /**
+   * Get flakiness history
+   */
+  getFlakinessHistory(): Map<string, FlakinessResult> {
+    return new Map(this.flakinessHistory);
+  }
+
+  /**
+   * Clear all history
+   */
+  clearHistory(): void {
+    this.flakinessHistory.clear();
+    this.testHistory.clear();
+  }
+}
+
+/**
+ * Create a flakiness-aware retry engine
+ */
+export function createFlakinessAwareRetryEngine(): FlakinessAwareRetryEngine {
+  return new FlakinessAwareRetryEngine();
+}