qa360 2.2.1 → 2.2.13

package/core/src/retry/__tests__/retry-engine.test.ts

@@ -1,424 +0,0 @@
-/**
- * Smart Retry Engine Tests
- */
-
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import {
-  SmartRetryEngine,
-  createSmartRetryEngine
-} from '../retry-engine.js';
-import { RetryStrategy } from '../types.js';
-import type { RetryConfig } from '../types.js';
-
-describe('SmartRetryEngine', () => {
-  let engine: SmartRetryEngine;
-
-  beforeEach(() => {
-    engine = createSmartRetryEngine();
-  });
-
-  describe('construction', () => {
-    it('should create engine with default config', () => {
-      const config = engine.getConfig();
-      expect(config.maxRetries).toBe(3);
-      expect(config.initialDelayMs).toBe(1000);
-      expect(config.strategy).toBe(RetryStrategy.ADAPTIVE);
-    });
-
-    it('should create engine with custom config', () => {
-      const customEngine = createSmartRetryEngine({
-        maxRetries: 5,
-        initialDelayMs: 2000,
-        strategy: RetryStrategy.EXPONENTIAL
-      });
-
-      const config = customEngine.getConfig();
-      expect(config.maxRetries).toBe(5);
-      expect(config.initialDelayMs).toBe(2000);
-      expect(config.strategy).toBe(RetryStrategy.EXPONENTIAL);
-    });
-  });
-
-  describe('executeWithRetry - immediate success', () => {
-    it('should succeed on first attempt', async () => {
-      const executor = vi.fn().mockResolvedValue('success');
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(true);
-      expect(result.totalAttempts).toBe(1);
-      expect(result.successfulAttempt).toBe(1);
-      expect(result.attempts.length).toBe(1);
-      expect(result.attempts[0].success).toBe(true);
-      expect(executor).toHaveBeenCalledTimes(1);
-    });
-
-    it('should not delay on first attempt', async () => {
-      const executor = vi.fn().mockResolvedValue('success');
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.attempts[0].delayMs).toBe(0);
-    });
-  });
-
-  describe('executeWithRetry - retry on failure', () => {
-    it('should retry on failure', async () => {
-      let attempts = 0;
-      const executor = vi.fn().mockImplementation(() => {
-        attempts++;
-        if (attempts < 2) throw new Error('Temporary failure');
-        return 'success';
-      });
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(true);
-      expect(result.totalAttempts).toBe(2);
-      expect(result.successfulAttempt).toBe(2);
-      expect(executor).toHaveBeenCalledTimes(2);
-    }, 10000);
-
-    it('should respect max retries', async () => {
-      const executor = vi.fn().mockRejectedValue(new Error('Always fails'));
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(false);
-      expect(result.totalAttempts).toBe(4); // 1 initial + 3 retries
-      expect(result.stopReason).toBe('max_retries');
-      expect(executor).toHaveBeenCalledTimes(4);
-    }, 20000);
-
-    it('should stop on successful retry', async () => {
-      let attempts = 0;
-      const executor = vi.fn().mockImplementation(() => {
-        attempts++;
-        if (attempts < 3) throw new Error('Fails first two times');
-        return 'success';
-      });
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(true);
-      expect(result.totalAttempts).toBe(3);
-      expect(result.stopReason).toBe('success');
-    }, 10000);
-  });
-
-  describe('retry strategies', () => {
-    it('should use fixed delay strategy', async () => {
-      const fixedEngine = createSmartRetryEngine({
-        maxRetries: 2,
-        initialDelayMs: 100,
-        strategy: RetryStrategy.FIXED,
-        jitterFactor: 0
-      });
-
-      const executor = vi.fn()
-        .mockRejectedValueOnce(new Error('Fail 1'))
-        .mockRejectedValueOnce(new Error('Fail 2'))
-        .mockResolvedValue('success');
-
-      await fixedEngine.executeWithRetry('test-1', executor);
-
-      expect(executor).toHaveBeenCalledTimes(3);
-    }, 10000);
-
-    it('should use exponential backoff', async () => {
-      const expEngine = createSmartRetryEngine({
-        maxRetries: 3,
-        initialDelayMs: 50,
-        strategy: RetryStrategy.EXPONENTIAL,
-        backoffMultiplier: 2,
-        jitterFactor: 0
-      });
-
-      const executor = vi.fn()
-        .mockRejectedValue(new Error('Fail'));
-
-      await expEngine.executeWithRetry('test-1', executor);
-
-      expect(executor).toHaveBeenCalledTimes(4);
-    }, 10000);
-
-    it('should use linear backoff', async () => {
-      const linearEngine = createSmartRetryEngine({
-        maxRetries: 3,
-        initialDelayMs: 50,
-        strategy: RetryStrategy.LINEAR,
-        backoffMultiplier: 2,
-        jitterFactor: 0
-      });
-
-      const executor = vi.fn()
-        .mockRejectedValue(new Error('Fail'));
-
-      await linearEngine.executeWithRetry('test-1', executor);
-
-      expect(executor).toHaveBeenCalledTimes(4);
-    }, 10000);
-  });
-
-  describe('non-retriable errors', () => {
-    it('should not retry on OutOfMemoryError', async () => {
-      const executor = vi.fn().mockRejectedValue(new Error('OutOfMemoryError'));
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(false);
-      expect(result.totalAttempts).toBe(1);
-      expect(result.stopReason).toBe('unrecoverable');
-      expect(executor).toHaveBeenCalledTimes(1);
-    }, 15000);
-
-    it('should not retry on PermissionDenied', async () => {
-      const executor = vi.fn().mockRejectedValue(new Error('PermissionDenied'));
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(false);
-      expect(result.totalAttempts).toBe(1);
-      expect(result.stopReason).toBe('unrecoverable');
-    }, 15000);
-
-    it('should not retry on SyntaxError', async () => {
-      const executor = vi.fn().mockRejectedValue(new SyntaxError('Bad syntax'));
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(false);
-      expect(result.totalAttempts).toBe(1);
-      expect(result.stopReason).toBe('unrecoverable');
-    }, 15000);
-  });
-
-  describe('assertion failures', () => {
-    it('should not retry assertion failures by default', async () => {
-      const executor = vi.fn().mockRejectedValue(new Error('AssertionError: Expected true'));
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(false);
-      expect(result.totalAttempts).toBe(1);
-      expect(result.stopReason).toBe('unrecoverable');
-    }, 15000);
-
-    it('should retry assertion failures when enabled', async () => {
-      const retryEngine = createSmartRetryEngine({
-        maxRetries: 2,
-        retryOnAssertionFailure: true
-      });
-
-      let attempts = 0;
-      const executor = vi.fn().mockImplementation(() => {
-        attempts++;
-        if (attempts === 1) throw new Error('AssertionError: Expected true');
-        return 'success';
-      });
-
-      const result = await retryEngine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(true);
-      expect(result.totalAttempts).toBe(2);
-    });
-  });
-
-  describe('timeout handling', () => {
-    it('should not retry timeout by default', async () => {
-      const executor = vi.fn().mockRejectedValue(new Error('TimeoutError'));
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(false);
-      expect(result.totalAttempts).toBe(1);
-      expect(result.stopReason).toBe('unrecoverable');
-    }, 15000);
-
-    it('should retry timeout when enabled', async () => {
-      const retryEngine = createSmartRetryEngine({
-        maxRetries: 2,
-        retryOnTimeout: true
-      });
-
-      let attempts = 0;
-      const executor = vi.fn().mockImplementation(() => {
-        attempts++;
-        if (attempts === 1) throw new Error('TimeoutError');
-        return 'success';
-      });
-
-      const result = await retryEngine.executeWithRetry('test-1', executor);
-
-      expect(result.success).toBe(true);
-      expect(result.totalAttempts).toBe(2);
-    });
-
-    it('should stop on overall timeout', async () => {
-      const retryEngine = createSmartRetryEngine({
-        maxRetries: 10,
-        overallTimeoutMs: 100,
-        initialDelayMs: 50
-      });
-
-      const executor = vi.fn().mockRejectedValue(new Error('Fail'));
-
-      const result = await retryEngine.executeWithRetry('test-1', executor);
-
-      // Should stop before max retries due to overall timeout
-      expect(result.totalAttempts).toBeLessThan(11);
-      expect(result.stopReason).toBe('timeout');
-    });
-  });
-
-  describe('retry recommendations', () => {
-    it('should not retry for high flakiness score', () => {
-      const rec = engine.getRetryRecommendation({
-        flakinessScore: 95
-      });
-
-      expect(rec.shouldRetry).toBe(false);
-      expect(rec.strategy).toBe(RetryStrategy.NONE);
-      expect(rec.reason).toContain('highly reliable');
-    });
-
-    it('should recommend fixed retry for medium flakiness', () => {
-      const rec = engine.getRetryRecommendation({
-        flakinessScore: 80
-      });
-
-      expect(rec.shouldRetry).toBe(true);
-      expect(rec.strategy).toBe(RetryStrategy.FIXED);
-      expect(rec.maxRetries).toBe(2);
-    });
-
-    it('should recommend adaptive retry for low flakiness', () => {
-      const rec = engine.getRetryRecommendation({
-        flakinessScore: 60
-      });
-
-      expect(rec.shouldRetry).toBe(true);
-      expect(rec.strategy).toBe(RetryStrategy.ADAPTIVE);
-      expect(rec.maxRetries).toBe(3);
-    });
-
-    it('should recommend intelligent retry for very low flakiness', () => {
-      const rec = engine.getRetryRecommendation({
-        flakinessScore: 30
-      });
-
-      expect(rec.shouldRetry).toBe(true);
-      expect(rec.strategy).toBe(RetryStrategy.INTELLIGENT);
-      expect(rec.maxRetries).toBe(5);
-      expect(rec.reason).toContain('aggressive');
-    });
-
-    it('should not retry for non-retriable errors', () => {
-      const rec = engine.getRetryRecommendation({
-        flakinessScore: 50,
-        errorType: 'OutOfMemoryError'
-      });
-
-      expect(rec.shouldRetry).toBe(false);
-      expect(rec.reason).toContain('not retriable');
-    });
-  });
-
-  describe('jitter', () => {
-    it('should apply jitter to delays', async () => {
-      const jitterEngine = createSmartRetryEngine({
-        maxRetries: 2,
-        initialDelayMs: 100,
-        jitterFactor: 0.5
-      });
-
-      const delays: number[] = [];
-      const executor = vi.fn().mockImplementation(async () => {
-        if (executor.mock.calls.length > 1) {
-          delays.push(jitterEngine.getConfig().initialDelayMs);
-        }
-        throw new Error('Fail');
-      });
-
-      await jitterEngine.executeWithRetry('test-1', executor);
-
-      // Delays should vary due to jitter
-      expect(executor).toHaveBeenCalledTimes(3);
-    });
-  });
-
-  describe('config updates', () => {
-    it('should update config', () => {
-      engine.updateConfig({ maxRetries: 5 });
-
-      const config = engine.getConfig();
-      expect(config.maxRetries).toBe(5);
-    });
-
-    it('should preserve other config values when updating', () => {
-      const originalDelay = engine.getConfig().initialDelayMs;
-      engine.updateConfig({ maxRetries: 5 });
-
-      const config = engine.getConfig();
-      expect(config.initialDelayMs).toBe(originalDelay);
-    });
-  });
-
-  describe('attempt tracking', () => {
-    it('should record all attempt details', async () => {
-      const executor = vi.fn()
-        .mockRejectedValueOnce(new Error('Fail 1'))
-        .mockRejectedValueOnce(new Error('Fail 2'))
-        .mockResolvedValue('success');
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.attempts.length).toBe(3);
-
-      // First attempt
-      expect(result.attempts[0].attemptNumber).toBe(1);
-      expect(result.attempts[0].success).toBe(false);
-      expect(result.attempts[0].delayMs).toBe(0);
-      expect(result.attempts[0].errorMessage).toBe('Fail 1');
-
-      // Second attempt
-      expect(result.attempts[1].attemptNumber).toBe(2);
-      expect(result.attempts[1].success).toBe(false);
-      expect(result.attempts[1].delayMs).toBeGreaterThan(0);
-      expect(result.attempts[1].errorMessage).toBe('Fail 2');
-
-      // Third attempt
-      expect(result.attempts[2].attemptNumber).toBe(3);
-      expect(result.attempts[2].success).toBe(true);
-    }, 15000);
-
-    it('should calculate total duration correctly', async () => {
-      const executor = vi.fn().mockImplementation(async () => {
-        await new Promise(resolve => setTimeout(resolve, 10));
-        if (executor.mock.calls.length < 2) throw new Error('Fail');
-        return 'success';
-      });
-
-      const result = await engine.executeWithRetry('test-1', executor);
-
-      expect(result.totalDurationMs).toBeGreaterThanOrEqual(20); // 2 attempts * 10ms
-      expect(result.success).toBe(true);
-    }, 15000);
-  });
-});
-
-describe('createSmartRetryEngine', () => {
-  it('should create SmartRetryEngine instance', () => {
-    const engine = createSmartRetryEngine();
-
-    expect(engine).toBeInstanceOf(SmartRetryEngine);
-  });
-
-  it('should pass config to engine', () => {
-    const engine = createSmartRetryEngine({
-      maxRetries: 10
-    });
-
-    expect(engine.getConfig().maxRetries).toBe(10);
-  });
-});

package/core/src/retry/flakiness-integration.ts

@@ -1,267 +0,0 @@
-/**
- * Flakiness Integration for Smart Retry
- *
- * Integrates Smart Retry with the Flakiness Detection Engine
- * to make intelligent retry decisions based on historical data.
- */
-
-import type { FlakinessResult, TestResult as FlakinessTestResult } from '../flakiness/index.js';
-import type {
-  RetryConfig,
-  RetryRecommendation
-} from './types.js';
-import { RetryStrategy } from './types.js';
-
-/**
- * Enhanced retry config with flakiness awareness
- */
-export interface FlakinessAwareRetryConfig extends RetryConfig {
-  /** Use flakiness score to determine retry strategy */
-  useFlakinessScore?: boolean;
-  /** Minimum historical runs before using flakiness data */
-  minHistoricalRuns?: number;
-  /** Boost retry attempts for known flaky tests */
-  boostRetriesForFlaky?: boolean;
-  /** Reduce retries for known stable tests */
-  reduceRetriesForStable?: boolean;
-}
-
-/**
- * Flakiness-aware retry engine
- */
-export class FlakinessAwareRetryEngine {
-  private flakinessHistory: Map<string, FlakinessResult> = new Map();
-  private testHistory: Map<string, FlakinessTestResult[]> = new Map();
-
-  /**
-   * Load flakiness data
-   */
-  loadFlakinessData(results: FlakinessResult[]): void {
-    for (const result of results) {
-      this.flakinessHistory.set(result.testId, result);
-    }
-  }
-
-  /**
-   * Add test result to history
-   */
-  addTestResult(testId: string, result: FlakinessTestResult): void {
-    if (!this.testHistory.has(testId)) {
-      this.testHistory.set(testId, []);
-    }
-    this.testHistory.get(testId)!.push(result);
-  }
-
-  /**
-   * Get retry config for a specific test based on flakiness
-   */
-  getRetryConfigForTest(
-    testId: string,
-    baseConfig: FlakinessAwareRetryConfig
-  ): RetryConfig {
-    const flakiness = this.flakinessHistory.get(testId);
-    const recentResults = this.testHistory.get(testId);
-
-    // If no flakiness data, return base config
-    const minRuns = baseConfig.minHistoricalRuns ?? 0;
-    if (!flakiness || !recentResults || recentResults.length < minRuns) {
-      return baseConfig;
-    }
-
-    const config = { ...baseConfig };
-
-    // Adjust based on flakiness score
-    if (baseConfig.useFlakinessScore) {
-      const score = flakiness.score;
-
-      if (score >= 90) {
-        // Legendary/Solid - reduce retries
-        if (baseConfig.reduceRetriesForStable) {
-          config.maxRetries = Math.max(0, baseConfig.maxRetries - 1);
-          config.strategy = RetryStrategy.FIXED;
-        }
-      } else if (score >= 75) {
-        // Good - keep base config
-        config.strategy = RetryStrategy.FIXED;
-      } else if (score >= 50) {
-        // Shaky - increase retries slightly
-        if (baseConfig.boostRetriesForFlaky) {
-          config.maxRetries = baseConfig.maxRetries + 1;
-          config.strategy = RetryStrategy.ADAPTIVE;
-        }
-      } else {
-        // Unstable - aggressive retry
-        if (baseConfig.boostRetriesForFlaky) {
-          config.maxRetries = baseConfig.maxRetries + 2;
-          config.strategy = RetryStrategy.INTELLIGENT;
-        }
-      }
-    }
-
-    // Adjust based on pattern type
-    if (flakiness.patternType) {
-      switch (flakiness.patternType) {
-        case 'timing':
-          config.initialDelayMs = Math.max(config.initialDelayMs, 2000);
-          config.retryOnTimeout = true;
-          break;
-        case 'external_dependency':
-          config.initialDelayMs = Math.max(config.initialDelayMs, 3000);
-          config.strategy = RetryStrategy.EXPONENTIAL;
-          break;
-        case 'race_condition':
-          config.strategy = RetryStrategy.EXPONENTIAL;
-          config.jitterFactor = Math.max(config.jitterFactor, 0.2);
-          break;
-        case 'selector_issue':
-          config.strategy = RetryStrategy.FIXED;
-          config.maxRetries = Math.min(config.maxRetries, 3);
-          break;
-      }
-    }
-
-    // Adjust based on recent streak
-    if (flakiness.currentStreak && flakiness.currentStreak > 3) {
-      if (flakiness.streakType === 'fail') {
-        // Long fail streak - boost retries
-        config.maxRetries = config.maxRetries + 1;
-      } else {
-        // Long pass streak - reduce retries
-        config.maxRetries = Math.max(1, config.maxRetries - 1);
-      }
-    }
-
-    return config;
-  }
-
-  /**
-   * Get comprehensive retry recommendation
-   */
-  getRetryRecommendation(testId: string): RetryRecommendation {
-    const flakiness = this.flakinessHistory.get(testId);
-    const recentResults = this.testHistory.get(testId);
-
-    if (!flakiness || !recentResults || recentResults.length < 3) {
-      return {
-        shouldRetry: true,
-        strategy: RetryStrategy.ADAPTIVE,
-        maxRetries: 3,
-        initialDelayMs: 1000,
-        confidence: 0.5,
-        reason: 'Insufficient historical data, using default adaptive retry'
-      };
-    }
-
-    // Analyze recent failures
-    const recentFailures = recentResults.slice(-10).filter(r => !r.success);
-    const failureRate = recentFailures.length / Math.min(recentResults.length, 10);
-
-    // Build recommendation
-    let strategy: RetryStrategy;
-    let maxRetries: number;
-    let initialDelayMs: number;
-    let confidence: number;
-    const reasons: string[] = [];
-
-    // Base recommendation on flakiness score
-    if (flakiness.score >= 90) {
-      strategy = RetryStrategy.NONE;
-      maxRetries = 0;
-      initialDelayMs = 0;
-      confidence = 0.9;
-      reasons.push('Test is highly reliable');
-    } else if (flakiness.score >= 75) {
-      strategy = RetryStrategy.FIXED;
-      maxRetries = 2;
-      initialDelayMs = 1000;
-      confidence = 0.75;
-      reasons.push('Test is occasionally flaky');
-    } else if (flakiness.score >= 50) {
-      strategy = RetryStrategy.ADAPTIVE;
-      maxRetries = 3;
-      initialDelayMs = 1000;
-      confidence = 0.85;
-      reasons.push('Test is often flaky');
-    } else {
-      strategy = RetryStrategy.INTELLIGENT;
-      maxRetries = 5;
-      initialDelayMs = 2000;
-      confidence = 0.9;
-      reasons.push('Test is very flaky, aggressive retry recommended');
-    }
-
-    // Adjust based on pattern
-    if (flakiness.patternType) {
-      reasons.push(`Pattern: ${flakiness.patternType}`);
-
-      switch (flakiness.patternType) {
-        case 'timing':
-          initialDelayMs = Math.max(initialDelayMs, 2000);
-          strategy = flakiness.score < 75 ? RetryStrategy.EXPONENTIAL : strategy;
-          break;
-        case 'external_dependency':
-          initialDelayMs = Math.max(initialDelayMs, 3000);
-          break;
-        case 'selector_issue':
-          strategy = RetryStrategy.FIXED;
-          maxRetries = Math.min(maxRetries, 3);
-          break;
-      }
-    }
-
-    // Adjust based on recent failure rate
-    if (failureRate > 0.5) {
-      maxRetries = Math.min(maxRetries + 1, 5);
-      reasons.push(`High recent failure rate (${Math.round(failureRate * 100)}%)`);
-    } else if (failureRate < 0.1 && flakiness.score >= 75) {
-      // For highly reliable tests (score >= 90), keep maxRetries at 0
-      if (flakiness.score < 90) {
-        maxRetries = Math.max(maxRetries - 1, 1);
-      }
-      reasons.push('Low recent failure rate');
-    }
-
-    return {
-      shouldRetry: maxRetries > 0,
-      strategy,
-      maxRetries,
-      initialDelayMs,
-      confidence,
-      reason: reasons.join('; '),
-      pattern: flakiness.patternType
-    };
-  }
-
-  /**
-   * Check if test should be skipped (too flaky)
-   */
-  shouldSkipTest(testId: string): boolean {
-    const flakiness = this.flakinessHistory.get(testId);
-    if (!flakiness) return false;
-
-    // Skip if score is very low and has many runs
-    return flakiness.score < 30 && flakiness.totalRuns >= 10;
-  }
-
-  /**
-   * Get flakiness history
-   */
-  getFlakinessHistory(): Map<string, FlakinessResult> {
-    return new Map(this.flakinessHistory);
-  }
-
-  /**
-   * Clear all history
-   */
-  clearHistory(): void {
-    this.flakinessHistory.clear();
-    this.testHistory.clear();
-  }
-}
-
-/**
- * Create a flakiness-aware retry engine
- */
-export function createFlakinessAwareRetryEngine(): FlakinessAwareRetryEngine {
-  return new FlakinessAwareRetryEngine();
-}