qa360 2.1.7 → 2.2.0

package/core/src/retry/index.ts

@@ -0,0 +1,48 @@
+/**
+ * QA360 Smart Retry Module (F8)
+ *
+ * Intelligent retry strategies based on flakiness scoring and pattern detection.
+ * Completes the flakiness system: Detection → Scoring → Quarantine → Smart Retry
+ *
+ * @module retry
+ */
+
+// Types
+export {
+  RetryStrategy,
+  DEFAULT_RETRY_CONFIG
+} from './types.js';
+export type {
+  RetryAttempt,
+  RetryResult,
+  RetryConfig,
+  RetryRecommendation,
+  RetryHistoryRecord,
+  RetryStatistics
+} from './types.js';
+
+// Core engine
+export {
+  SmartRetryEngine,
+  createSmartRetryEngine,
+  type TestExecutor
+} from './retry-engine.js';
+
+// Flakiness integration
+export {
+  FlakinessAwareRetryEngine,
+  createFlakinessAwareRetryEngine,
+  type FlakinessAwareRetryConfig
+} from './flakiness-integration.js';
+
+// Vault integration
+export {
+  RetryVault
+} from './vault.js';
+
+// Re-export flakiness types for convenience
+export type {
+  FlakinessResult,
+  FlakinessCategory,
+  TestResult as FlakinessTestResult
+} from '../flakiness/index.js';

package/core/src/retry/retry-engine.ts

@@ -0,0 +1,368 @@
+/**
+ * Smart Retry Engine
+ *
+ * Core retry logic with multiple strategies and adaptive behavior.
+ */
+
+import type {
+  RetryConfig,
+  RetryResult,
+  RetryAttempt,
+  RetryRecommendation
+} from './types.js';
+import {
+  DEFAULT_RETRY_CONFIG,
+  RetryStrategy
+} from './types.js';
+
+/**
+ * Test execution function type
+ */
+export type TestExecutor<T = unknown> = () => Promise<T>;
+
+/**
+ * Smart Retry Engine
+ */
+export class SmartRetryEngine {
+  private config: RetryConfig;
+
+  constructor(config: Partial<RetryConfig> = {}) {
+    this.config = { ...DEFAULT_RETRY_CONFIG, ...config };
+  }
+
+  /**
+   * Execute a test with retry logic
+   */
+  async executeWithRetry(
+    testId: string,
+    executor: TestExecutor,
+    context?: {
+      testName?: string;
+      gate?: string;
+      flakinessScore?: number;
+      patternType?: string;
+    }
+  ): Promise<RetryResult> {
+    const startTime = Date.now();
+    const attempts: RetryAttempt[] = [];
+    let lastError: Error | undefined;
+    let stopReason: RetryResult['stopReason'] = 'max_retries';
+
+    // Adjust strategy based on flakiness if adaptive
+    const effectiveStrategy = this.getEffectiveStrategy(
+      this.config.strategy,
+      context?.flakinessScore,
+      context?.patternType
+    );
+
+    for (let attempt = 1; attempt <= this.config.maxRetries + 1; attempt++) {
+      // Calculate delay before this attempt
+      const delayMs = attempt === 1 ? 0 : this.calculateDelay(attempt, effectiveStrategy);
+
+      // Apply delay if not first attempt
+      if (delayMs > 0) {
+        await this.delay(delayMs);
+      }
+
+      const attemptStartTime = Date.now();
+      let attemptSuccess = false;
+      let errorType: string | undefined;
+      let errorMessage: string | undefined;
+      let shouldStop = false;
+
+      try {
+        await executor();
+        attemptSuccess = true;
+        stopReason = 'success';
+      } catch (error) {
+        const err = error as Error;
+        errorType = err.constructor?.name || 'Error';
+        errorMessage = err.message;
+
+        // Check if error is non-retriable
+        if (this.isNonRetriableError(errorType, errorMessage)) {
+          stopReason = 'unrecoverable';
+          lastError = err;
+          shouldStop = true;
+        }
+
+        // Check if we should retry on assertion failure
+        if (!shouldStop && !this.config.retryOnAssertionFailure && this.isAssertionError(errorType, errorMessage)) {
+          stopReason = 'unrecoverable';
+          lastError = err;
+          shouldStop = true;
+        }
+
+        // Check if we should retry on timeout
+        if (!shouldStop && !this.config.retryOnTimeout && this.isTimeoutError(errorType, errorMessage)) {
+          stopReason = 'unrecoverable';
+          lastError = err;
+          shouldStop = true;
+        }
+
+        if (!shouldStop) {
+          lastError = err;
+        }
+      }
+
+      const attemptDuration = Date.now() - attemptStartTime;
+
+      // Always record the attempt
+      attempts.push({
+        attemptNumber: attempt,
+        success: attemptSuccess,
+        durationMs: attemptDuration,
+        errorType,
+        errorMessage,
+        delayMs,
+        timestamp: attemptStartTime
+      });
+
+      if (attemptSuccess || shouldStop) {
+        break;
+      }
+
+      // Check overall timeout
+      if (this.config.overallTimeoutMs) {
+        const elapsed = Date.now() - startTime;
+        if (elapsed + this.config.maxDelayMs > this.config.overallTimeoutMs) {
+          stopReason = 'timeout';
+          break;
+        }
+      }
+    }
+
+    const totalDuration = Date.now() - startTime;
+    const success = attempts.some(a => a.success);
+    const successfulAttempt = attempts.find(a => a.success)?.attemptNumber;
+
+    return {
+      testId,
+      success,
+      totalAttempts: attempts.length,
+      successfulAttempt,
+      attempts,
+      totalDurationMs: totalDuration,
+      strategy: effectiveStrategy,
+      stopReason
+    };
+  }
+
+  /**
+   * Get retry recommendation based on flakiness and pattern
+   */
+  getRetryRecommendation(params: {
+    flakinessScore?: number;
+    patternType?: string;
+    errorType?: string;
+    errorCount?: number;
+    recentAttempts?: number;
+  }): RetryRecommendation {
+    const { flakinessScore = 100, patternType, errorType, errorCount = 1, recentAttempts = 0 } = params;
+
+    // Non-retriable errors should not be retried
+    if (errorType && this.isNonRetriableError(errorType)) {
+      return {
+        shouldRetry: false,
+        strategy: RetryStrategy.NONE,
+        maxRetries: 0,
+        initialDelayMs: 0,
+        confidence: 0.95,
+        reason: `Error type "${errorType}" is not retriable`
+      };
+    }
+
+    // High flakiness score (reliable tests) -> fewer retries
+    if (flakinessScore >= 90) {
+      return {
+        shouldRetry: false,
+        strategy: RetryStrategy.NONE,
+        maxRetries: 0,
+        initialDelayMs: 0,
+        confidence: 0.9,
+        reason: 'Test is highly reliable (Legendary/Solid), retry not recommended'
+      };
+    }
+
+    // Medium flakiness -> conservative retry
+    if (flakinessScore >= 75) {
+      return {
+        shouldRetry: true,
+        strategy: RetryStrategy.FIXED,
+        maxRetries: 2,
+        initialDelayMs: 1000,
+        confidence: 0.75,
+        reason: 'Test is occasionally flaky (Good), conservative retry recommended'
+      };
+    }
+
+    // Low flakiness -> adaptive retry
+    if (flakinessScore >= 50) {
+      return {
+        shouldRetry: true,
+        strategy: RetryStrategy.ADAPTIVE,
+        maxRetries: 3,
+        initialDelayMs: 1000,
+        confidence: 0.85,
+        reason: 'Test is often flaky (Shaky), adaptive retry recommended',
+        pattern: patternType
+      };
+    }
+
+    // Very low flakiness -> intelligent retry
+    return {
+      shouldRetry: true,
+      strategy: RetryStrategy.INTELLIGENT,
+      maxRetries: 5,
+      initialDelayMs: 2000,
+      confidence: 0.9,
+      reason: 'Test is very flaky (Unstable), aggressive retry with pattern analysis',
+      pattern: patternType
+    };
+  }
+
+  /**
+   * Calculate delay before retry attempt
+   */
+  private calculateDelay(attemptNumber: number, strategy: RetryStrategy): number {
+    const baseDelay = this.config.initialDelayMs;
+
+    switch (strategy) {
+      case RetryStrategy.FIXED:
+        return this.applyJitter(baseDelay);
+
+      case RetryStrategy.LINEAR:
+        const linearDelay = baseDelay * (attemptNumber - 1) * this.config.backoffMultiplier;
+        return this.applyJitter(Math.min(linearDelay, this.config.maxDelayMs));
+
+      case RetryStrategy.EXPONENTIAL:
+        const expDelay = baseDelay * Math.pow(this.config.backoffMultiplier, attemptNumber - 1);
+        return this.applyJitter(Math.min(expDelay, this.config.maxDelayMs));
+
+      case RetryStrategy.ADAPTIVE:
+        // Start with exponential, but reduce if attempts are high
+        const adaptiveDelay = baseDelay * Math.pow(this.config.backoffMultiplier, Math.min(attemptNumber - 1, 3));
+        return this.applyJitter(Math.min(adaptiveDelay, this.config.maxDelayMs));
+
+      case RetryStrategy.INTELLIGENT:
+        // Use pattern-based delay calculation
+        return this.getIntelligentDelay(attemptNumber);
+
+      case RetryStrategy.NONE:
+      default:
+        return 0;
+    }
+  }
+
+  /**
+   * Get effective strategy based on context
+   */
+  private getEffectiveStrategy(
+    configuredStrategy: RetryStrategy,
+    flakinessScore?: number,
+    patternType?: string
+  ): RetryStrategy {
+    // If explicitly set to none, respect it
+    if (configuredStrategy === RetryStrategy.NONE) {
+      return RetryStrategy.NONE;
+    }
+
+    // Adaptive strategy adjusts based on flakiness
+    if (configuredStrategy === RetryStrategy.ADAPTIVE && flakinessScore !== undefined) {
+      if (flakinessScore >= 90) return RetryStrategy.NONE;
+      if (flakinessScore >= 75) return RetryStrategy.FIXED;
+      if (flakinessScore >= 50) return RetryStrategy.LINEAR;
+      return RetryStrategy.EXPONENTIAL;
+    }
+
+    // Intelligent strategy considers patterns
+    if (configuredStrategy === RetryStrategy.INTELLIGENT && patternType) {
+      // Timing issues -> exponential backoff
+      if (patternType.includes('timing')) return RetryStrategy.EXPONENTIAL;
+      // External dependency -> linear with longer delays
+      if (patternType.includes('external')) return RetryStrategy.LINEAR;
+      // Selector issues -> fixed delay (usually resolves quickly)
+      if (patternType.includes('selector')) return RetryStrategy.FIXED;
+      // Race condition -> exponential with jitter
+      if (patternType.includes('race')) return RetryStrategy.EXPONENTIAL;
+    }
+
+    return configuredStrategy;
+  }
+
+  /**
+   * Calculate intelligent delay based on common patterns
+   */
+  private getIntelligentDelay(attemptNumber: number): number {
+    // Timing issues: exponential backoff starting at 2s
+    if (attemptNumber === 1) return 2000;
+    if (attemptNumber === 2) return 3000;
+    if (attemptNumber === 3) return 5000;
+    return 8000;
+  }
+
+  /**
+   * Apply jitter to delay to avoid thundering herd
+   */
+  private applyJitter(delayMs: number): number {
+    if (this.config.jitterFactor === 0) return delayMs;
+    const jitter = delayMs * this.config.jitterFactor * (Math.random() * 2 - 1);
+    return Math.max(0, Math.round(delayMs + jitter));
+  }
+
+  /**
+   * Check if error type is non-retriable
+   */
+  private isNonRetriableError(errorType: string, errorMessage?: string): boolean {
+    return this.config.nonRetriableErrors.some(
+      nonRetriable => errorType.includes(nonRetriable) ||
+                     (errorMessage && errorMessage.includes(nonRetriable))
+    );
+  }
+
+  /**
+   * Check if error is an assertion failure
+   */
+  private isAssertionError(errorType: string, errorMessage?: string): boolean {
+    return errorType.toLowerCase().includes('assertion') ||
+           errorType === 'AssertionError' ||
+           errorType === 'ExpectFailError' ||
+           Boolean(errorMessage && errorMessage.toLowerCase().includes('assertion'));
+  }
+
+  /**
+   * Check if error is a timeout
+   */
+  private isTimeoutError(errorType: string, errorMessage?: string): boolean {
+    return errorType.toLowerCase().includes('timeout') ||
+           Boolean(errorMessage && errorMessage.toLowerCase().includes('timeout'));
+  }
+
+  /**
+   * Delay for specified milliseconds
+   */
+  private delay(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Update retry configuration
+   */
+  updateConfig(config: Partial<RetryConfig>): void {
+    this.config = { ...this.config, ...config };
+  }
+
+  /**
+   * Get current configuration
+   */
+  getConfig(): RetryConfig {
+    return { ...this.config };
+  }
+}
+
+/**
+ * Create a smart retry engine
+ */
+export function createSmartRetryEngine(config?: Partial<RetryConfig>): SmartRetryEngine {
+  return new SmartRetryEngine(config);
+}

package/core/src/retry/types.ts

@@ -0,0 +1,208 @@
+/**
+ * QA360 Smart Retry Module (F8)
+ *
+ * Intelligent retry strategies based on flakiness scoring and pattern detection.
+ * Completes the flakiness system: Detection → Scoring → Quarantine → Smart Retry
+ *
+ * @module retry
+ */
+
+/**
+ * Retry strategy types
+ */
+export enum RetryStrategy {
+  /** No retry - fail immediately */
+  NONE = 'none',
+  /** Fixed delay between retries */
+  FIXED = 'fixed',
+  /** Linear backoff (delay increases linearly) */
+  LINEAR = 'linear',
+  /** Exponential backoff (delay doubles each retry) */
+  EXPONENTIAL = 'exponential',
+  /** Adaptive retry based on flakiness score */
+  ADAPTIVE = 'adaptive',
+  /** Intelligent retry based on pattern detection */
+  INTELLIGENT = 'intelligent'
+}
+
+/**
+ * Retry attempt result
+ */
+export interface RetryAttempt {
+  /** Attempt number (1-indexed) */
+  attemptNumber: number;
+  /** Whether the attempt succeeded */
+  success: boolean;
+  /** Duration in milliseconds */
+  durationMs: number;
+  /** Error type if failed */
+  errorType?: string;
+  /** Error message if failed */
+  errorMessage?: string;
+  /** Delay before this attempt in ms */
+  delayMs: number;
+  /** Timestamp of attempt */
+  timestamp: number;
+}
+
+/**
+ * Retry result
+ */
+export interface RetryResult {
+  /** Test identifier */
+  testId: string;
+  /** Whether any retry attempt succeeded */
+  success: boolean;
+  /** Total number of attempts */
+  totalAttempts: number;
+  /** Successful attempt number (0 if none) */
+  successfulAttempt?: number;
+  /** All retry attempts */
+  attempts: RetryAttempt[];
+  /** Total duration including delays */
+  totalDurationMs: number;
+  /** Strategy used */
+  strategy: RetryStrategy;
+  /** Reason for stopping retries */
+  stopReason: 'success' | 'max_retries' | 'timeout' | 'unrecoverable';
+}
+
+/**
+ * Retry configuration
+ */
+export interface RetryConfig {
+  /** Maximum number of retry attempts */
+  maxRetries: number;
+  /** Initial delay in milliseconds */
+  initialDelayMs: number;
+  /** Maximum delay in milliseconds */
+  maxDelayMs: number;
+  /** Timeout for each attempt */
+  attemptTimeoutMs: number;
+  /** Overall timeout including retries */
+  overallTimeoutMs?: number;
+  /** Retry strategy */
+  strategy: RetryStrategy;
+  /** Multiplier for exponential/linear backoff */
+  backoffMultiplier: number;
+  /** Jitter for backoff (0-1) to avoid thundering herd */
+  jitterFactor: number;
+  /** Whether to retry on timeout */
+  retryOnTimeout: boolean;
+  /** Whether to retry on assertion failures */
+  retryOnAssertionFailure: boolean;
+  /** Error types that should not be retried */
+  nonRetriableErrors: string[];
+  /** Adaptive retry: minimum flakiness score to enable retries */
+  adaptiveMinFlakinessScore?: number;
+  /** Intelligent retry: enable pattern-based retry */
+  enablePatternBasedRetry?: boolean;
+}
+
+/**
+ * Default retry configuration
+ */
+export const DEFAULT_RETRY_CONFIG: Required<Omit<RetryConfig, 'nonRetriableErrors' | 'adaptiveMinFlakinessScore' | 'enablePatternBasedRetry'>> & {
+  nonRetriableErrors: string[];
+  adaptiveMinFlakinessScore?: number;
+  enablePatternBasedRetry?: boolean;
+} = {
+  maxRetries: 3,
+  initialDelayMs: 1000,
+  maxDelayMs: 30000,
+  attemptTimeoutMs: 30000,
+  overallTimeoutMs: 120000,
+  strategy: RetryStrategy.ADAPTIVE,
+  backoffMultiplier: 2,
+  jitterFactor: 0.1,
+  retryOnTimeout: false,
+  retryOnAssertionFailure: false,
+  nonRetriableErrors: [
+    'OutOfMemoryError',
+    'StackOverflowError',
+    'SecurityError',
+    'PermissionDenied',
+    'InvalidArgument',
+    'ValidationError',
+    'SyntaxError'
+  ],
+  adaptiveMinFlakinessScore: 75,
+  enablePatternBasedRetry: true
+};
+
+/**
+ * Flakiness-based retry recommendation
+ */
+export interface RetryRecommendation {
+  /** Whether retry is recommended */
+  shouldRetry: boolean;
+  /** Suggested retry strategy */
+  strategy: RetryStrategy;
+  /** Suggested max retries */
+  maxRetries: number;
+  /** Suggested initial delay */
+  initialDelayMs: number;
+  /** Confidence in recommendation (0-1) */
+  confidence: number;
+  /** Reason for recommendation */
+  reason: string;
+  /** Specific pattern detected that informs retry */
+  pattern?: string;
+}
+
+/**
+ * Retry history record for vault storage
+ */
+export interface RetryHistoryRecord {
+  id: number;
+  run_id: string;
+  test_id: string;
+  test_name: string;
+  gate: string;
+  strategy: RetryStrategy;
+  max_retries: number;
+  actual_attempts: number;
+  success: boolean;
+  successful_attempt: number | null;
+  total_duration_ms: number;
+  stop_reason: string;
+  error_type: string | null;
+  error_message: string | null;
+  flakiness_score: number | null;
+  pattern_type: string | null;
+  timestamp: number;
+}
+
+/**
+ * Retry statistics
+ */
+export interface RetryStatistics {
+  /** Total retries attempted */
+  totalRetries: number;
+  /** Total tests that were retried */
+  testsRetried: number;
+  /** Tests that passed after retry */
+  recoveredTests: number;
+  /** Tests that failed all retries */
+  failedTests: number;
+  /** Recovery rate (percentage) */
+  recoveryRate: number;
+  /** Average attempts per retried test */
+  avgAttemptsPerTest: number;
+  /** Time spent retrying (ms) */
+  totalRetryTimeMs: number;
+  /** Breakdown by strategy */
+  byStrategy: Record<RetryStrategy, {
+    attempts: number;
+    successes: number;
+    avgDurationMs: number;
+  }>;
+  /** Breakdown by error type */
+  byErrorType: Record<string, number>;
+  /** Breakdown by flakiness score */
+  byFlakinessScore: {
+    low: number;    // 0-49
+    medium: number; // 50-74
+    high: number;   // 75-100
+  };
+}