qa360 2.2.1 → 2.2.7

package/cli/dist/core/retry/retry-engine.js

@@ -0,0 +1,296 @@
+/**
+ * Smart Retry Engine
+ *
+ * Core retry logic with multiple strategies and adaptive behavior.
+ */
+import { DEFAULT_RETRY_CONFIG, RetryStrategy } from './types.js';
+/**
+ * Smart Retry Engine
+ */
+export class SmartRetryEngine {
+    config;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_RETRY_CONFIG, ...config };
+    }
+    /**
+     * Execute a test with retry logic
+     */
+    async executeWithRetry(testId, executor, context) {
+        const startTime = Date.now();
+        const attempts = [];
+        let lastError;
+        let stopReason = 'max_retries';
+        // Adjust strategy based on flakiness if adaptive
+        const effectiveStrategy = this.getEffectiveStrategy(this.config.strategy, context?.flakinessScore, context?.patternType);
+        for (let attempt = 1; attempt <= this.config.maxRetries + 1; attempt++) {
+            // Calculate delay before this attempt
+            const delayMs = attempt === 1 ? 0 : this.calculateDelay(attempt, effectiveStrategy);
+            // Apply delay if not first attempt
+            if (delayMs > 0) {
+                await this.delay(delayMs);
+            }
+            const attemptStartTime = Date.now();
+            let attemptSuccess = false;
+            let errorType;
+            let errorMessage;
+            let shouldStop = false;
+            try {
+                await executor();
+                attemptSuccess = true;
+                stopReason = 'success';
+            }
+            catch (error) {
+                const err = error;
+                errorType = err.constructor?.name || 'Error';
+                errorMessage = err.message;
+                // Check if error is non-retriable
+                if (this.isNonRetriableError(errorType, errorMessage)) {
+                    stopReason = 'unrecoverable';
+                    lastError = err;
+                    shouldStop = true;
+                }
+                // Check if we should retry on assertion failure
+                if (!shouldStop && !this.config.retryOnAssertionFailure && this.isAssertionError(errorType, errorMessage)) {
+                    stopReason = 'unrecoverable';
+                    lastError = err;
+                    shouldStop = true;
+                }
+                // Check if we should retry on timeout
+                if (!shouldStop && !this.config.retryOnTimeout && this.isTimeoutError(errorType, errorMessage)) {
+                    stopReason = 'unrecoverable';
+                    lastError = err;
+                    shouldStop = true;
+                }
+                if (!shouldStop) {
+                    lastError = err;
+                }
+            }
+            const attemptDuration = Date.now() - attemptStartTime;
+            // Always record the attempt
+            attempts.push({
+                attemptNumber: attempt,
+                success: attemptSuccess,
+                durationMs: attemptDuration,
+                errorType,
+                errorMessage,
+                delayMs,
+                timestamp: attemptStartTime
+            });
+            if (attemptSuccess || shouldStop) {
+                break;
+            }
+            // Check overall timeout
+            if (this.config.overallTimeoutMs) {
+                const elapsed = Date.now() - startTime;
+                if (elapsed + this.config.maxDelayMs > this.config.overallTimeoutMs) {
+                    stopReason = 'timeout';
+                    break;
+                }
+            }
+        }
+        const totalDuration = Date.now() - startTime;
+        const success = attempts.some(a => a.success);
+        const successfulAttempt = attempts.find(a => a.success)?.attemptNumber;
+        return {
+            testId,
+            success,
+            totalAttempts: attempts.length,
+            successfulAttempt,
+            attempts,
+            totalDurationMs: totalDuration,
+            strategy: effectiveStrategy,
+            stopReason
+        };
+    }
+    /**
+     * Get retry recommendation based on flakiness and pattern
+     */
+    getRetryRecommendation(params) {
+        const { flakinessScore = 100, patternType, errorType, errorCount = 1, recentAttempts = 0 } = params;
+        // Non-retriable errors should not be retried
+        if (errorType && this.isNonRetriableError(errorType)) {
+            return {
+                shouldRetry: false,
+                strategy: RetryStrategy.NONE,
+                maxRetries: 0,
+                initialDelayMs: 0,
+                confidence: 0.95,
+                reason: `Error type "${errorType}" is not retriable`
+            };
+        }
+        // High flakiness score (reliable tests) -> fewer retries
+        if (flakinessScore >= 90) {
+            return {
+                shouldRetry: false,
+                strategy: RetryStrategy.NONE,
+                maxRetries: 0,
+                initialDelayMs: 0,
+                confidence: 0.9,
+                reason: 'Test is highly reliable (Legendary/Solid), retry not recommended'
+            };
+        }
+        // Medium flakiness -> conservative retry
+        if (flakinessScore >= 75) {
+            return {
+                shouldRetry: true,
+                strategy: RetryStrategy.FIXED,
+                maxRetries: 2,
+                initialDelayMs: 1000,
+                confidence: 0.75,
+                reason: 'Test is occasionally flaky (Good), conservative retry recommended'
+            };
+        }
+        // Low flakiness -> adaptive retry
+        if (flakinessScore >= 50) {
+            return {
+                shouldRetry: true,
+                strategy: RetryStrategy.ADAPTIVE,
+                maxRetries: 3,
+                initialDelayMs: 1000,
+                confidence: 0.85,
+                reason: 'Test is often flaky (Shaky), adaptive retry recommended',
+                pattern: patternType
+            };
+        }
+        // Very low flakiness -> intelligent retry
+        return {
+            shouldRetry: true,
+            strategy: RetryStrategy.INTELLIGENT,
+            maxRetries: 5,
+            initialDelayMs: 2000,
+            confidence: 0.9,
+            reason: 'Test is very flaky (Unstable), aggressive retry with pattern analysis',
+            pattern: patternType
+        };
+    }
+    /**
+     * Calculate delay before retry attempt
+     */
+    calculateDelay(attemptNumber, strategy) {
+        const baseDelay = this.config.initialDelayMs;
+        switch (strategy) {
+            case RetryStrategy.FIXED:
+                return this.applyJitter(baseDelay);
+            case RetryStrategy.LINEAR:
+                const linearDelay = baseDelay * (attemptNumber - 1) * this.config.backoffMultiplier;
+                return this.applyJitter(Math.min(linearDelay, this.config.maxDelayMs));
+            case RetryStrategy.EXPONENTIAL:
+                const expDelay = baseDelay * Math.pow(this.config.backoffMultiplier, attemptNumber - 1);
+                return this.applyJitter(Math.min(expDelay, this.config.maxDelayMs));
+            case RetryStrategy.ADAPTIVE:
+                // Start with exponential, but reduce if attempts are high
+                const adaptiveDelay = baseDelay * Math.pow(this.config.backoffMultiplier, Math.min(attemptNumber - 1, 3));
+                return this.applyJitter(Math.min(adaptiveDelay, this.config.maxDelayMs));
+            case RetryStrategy.INTELLIGENT:
+                // Use pattern-based delay calculation
+                return this.getIntelligentDelay(attemptNumber);
+            case RetryStrategy.NONE:
+            default:
+                return 0;
+        }
+    }
+    /**
+     * Get effective strategy based on context
+     */
+    getEffectiveStrategy(configuredStrategy, flakinessScore, patternType) {
+        // If explicitly set to none, respect it
+        if (configuredStrategy === RetryStrategy.NONE) {
+            return RetryStrategy.NONE;
+        }
+        // Adaptive strategy adjusts based on flakiness
+        if (configuredStrategy === RetryStrategy.ADAPTIVE && flakinessScore !== undefined) {
+            if (flakinessScore >= 90)
+                return RetryStrategy.NONE;
+            if (flakinessScore >= 75)
+                return RetryStrategy.FIXED;
+            if (flakinessScore >= 50)
+                return RetryStrategy.LINEAR;
+            return RetryStrategy.EXPONENTIAL;
+        }
+        // Intelligent strategy considers patterns
+        if (configuredStrategy === RetryStrategy.INTELLIGENT && patternType) {
+            // Timing issues -> exponential backoff
+            if (patternType.includes('timing'))
+                return RetryStrategy.EXPONENTIAL;
+            // External dependency -> linear with longer delays
+            if (patternType.includes('external'))
+                return RetryStrategy.LINEAR;
+            // Selector issues -> fixed delay (usually resolves quickly)
+            if (patternType.includes('selector'))
+                return RetryStrategy.FIXED;
+            // Race condition -> exponential with jitter
+            if (patternType.includes('race'))
+                return RetryStrategy.EXPONENTIAL;
+        }
+        return configuredStrategy;
+    }
+    /**
+     * Calculate intelligent delay based on common patterns
+     */
+    getIntelligentDelay(attemptNumber) {
+        // Timing issues: exponential backoff starting at 2s
+        if (attemptNumber === 1)
+            return 2000;
+        if (attemptNumber === 2)
+            return 3000;
+        if (attemptNumber === 3)
+            return 5000;
+        return 8000;
+    }
+    /**
+     * Apply jitter to delay to avoid thundering herd
+     */
+    applyJitter(delayMs) {
+        if (this.config.jitterFactor === 0)
+            return delayMs;
+        const jitter = delayMs * this.config.jitterFactor * (Math.random() * 2 - 1);
+        return Math.max(0, Math.round(delayMs + jitter));
+    }
+    /**
+     * Check if error type is non-retriable
+     */
+    isNonRetriableError(errorType, errorMessage) {
+        return this.config.nonRetriableErrors.some(nonRetriable => errorType.includes(nonRetriable) ||
+            (errorMessage && errorMessage.includes(nonRetriable)));
+    }
+    /**
+     * Check if error is an assertion failure
+     */
+    isAssertionError(errorType, errorMessage) {
+        return errorType.toLowerCase().includes('assertion') ||
+            errorType === 'AssertionError' ||
+            errorType === 'ExpectFailError' ||
+            Boolean(errorMessage && errorMessage.toLowerCase().includes('assertion'));
+    }
+    /**
+     * Check if error is a timeout
+     */
+    isTimeoutError(errorType, errorMessage) {
+        return errorType.toLowerCase().includes('timeout') ||
+            Boolean(errorMessage && errorMessage.toLowerCase().includes('timeout'));
+    }
+    /**
+     * Delay for specified milliseconds
+     */
+    delay(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+    /**
+     * Update retry configuration
+     */
+    updateConfig(config) {
+        this.config = { ...this.config, ...config };
+    }
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+}
+/**
+ * Create a smart retry engine
+ */
+export function createSmartRetryEngine(config) {
+    return new SmartRetryEngine(config);
+}

package/cli/dist/core/retry/types.d.ts

@@ -0,0 +1,178 @@
+/**
+ * QA360 Smart Retry Module (F8)
+ *
+ * Intelligent retry strategies based on flakiness scoring and pattern detection.
+ * Completes the flakiness system: Detection → Scoring → Quarantine → Smart Retry
+ *
+ * @module retry
+ */
+/**
+ * Retry strategy types
+ */
+export declare enum RetryStrategy {
+    /** No retry - fail immediately */
+    NONE = "none",
+    /** Fixed delay between retries */
+    FIXED = "fixed",
+    /** Linear backoff (delay increases linearly) */
+    LINEAR = "linear",
+    /** Exponential backoff (delay doubles each retry) */
+    EXPONENTIAL = "exponential",
+    /** Adaptive retry based on flakiness score */
+    ADAPTIVE = "adaptive",
+    /** Intelligent retry based on pattern detection */
+    INTELLIGENT = "intelligent"
+}
+/**
+ * Retry attempt result
+ */
+export interface RetryAttempt {
+    /** Attempt number (1-indexed) */
+    attemptNumber: number;
+    /** Whether the attempt succeeded */
+    success: boolean;
+    /** Duration in milliseconds */
+    durationMs: number;
+    /** Error type if failed */
+    errorType?: string;
+    /** Error message if failed */
+    errorMessage?: string;
+    /** Delay before this attempt in ms */
+    delayMs: number;
+    /** Timestamp of attempt */
+    timestamp: number;
+}
+/**
+ * Retry result
+ */
+export interface RetryResult {
+    /** Test identifier */
+    testId: string;
+    /** Whether any retry attempt succeeded */
+    success: boolean;
+    /** Total number of attempts */
+    totalAttempts: number;
+    /** Successful attempt number (0 if none) */
+    successfulAttempt?: number;
+    /** All retry attempts */
+    attempts: RetryAttempt[];
+    /** Total duration including delays */
+    totalDurationMs: number;
+    /** Strategy used */
+    strategy: RetryStrategy;
+    /** Reason for stopping retries */
+    stopReason: 'success' | 'max_retries' | 'timeout' | 'unrecoverable';
+}
+/**
+ * Retry configuration
+ */
+export interface RetryConfig {
+    /** Maximum number of retry attempts */
+    maxRetries: number;
+    /** Initial delay in milliseconds */
+    initialDelayMs: number;
+    /** Maximum delay in milliseconds */
+    maxDelayMs: number;
+    /** Timeout for each attempt */
+    attemptTimeoutMs: number;
+    /** Overall timeout including retries */
+    overallTimeoutMs?: number;
+    /** Retry strategy */
+    strategy: RetryStrategy;
+    /** Multiplier for exponential/linear backoff */
+    backoffMultiplier: number;
+    /** Jitter for backoff (0-1) to avoid thundering herd */
+    jitterFactor: number;
+    /** Whether to retry on timeout */
+    retryOnTimeout: boolean;
+    /** Whether to retry on assertion failures */
+    retryOnAssertionFailure: boolean;
+    /** Error types that should not be retried */
+    nonRetriableErrors: string[];
+    /** Adaptive retry: minimum flakiness score to enable retries */
+    adaptiveMinFlakinessScore?: number;
+    /** Intelligent retry: enable pattern-based retry */
+    enablePatternBasedRetry?: boolean;
+}
+/**
+ * Default retry configuration
+ */
+export declare const DEFAULT_RETRY_CONFIG: Required<Omit<RetryConfig, 'nonRetriableErrors' | 'adaptiveMinFlakinessScore' | 'enablePatternBasedRetry'>> & {
+    nonRetriableErrors: string[];
+    adaptiveMinFlakinessScore?: number;
+    enablePatternBasedRetry?: boolean;
+};
+/**
+ * Flakiness-based retry recommendation
+ */
+export interface RetryRecommendation {
+    /** Whether retry is recommended */
+    shouldRetry: boolean;
+    /** Suggested retry strategy */
+    strategy: RetryStrategy;
+    /** Suggested max retries */
+    maxRetries: number;
+    /** Suggested initial delay */
+    initialDelayMs: number;
+    /** Confidence in recommendation (0-1) */
+    confidence: number;
+    /** Reason for recommendation */
+    reason: string;
+    /** Specific pattern detected that informs retry */
+    pattern?: string;
+}
+/**
+ * Retry history record for vault storage
+ */
+export interface RetryHistoryRecord {
+    id: number;
+    run_id: string;
+    test_id: string;
+    test_name: string;
+    gate: string;
+    strategy: RetryStrategy;
+    max_retries: number;
+    actual_attempts: number;
+    success: boolean;
+    successful_attempt: number | null;
+    total_duration_ms: number;
+    stop_reason: string;
+    error_type: string | null;
+    error_message: string | null;
+    flakiness_score: number | null;
+    pattern_type: string | null;
+    timestamp: number;
+}
+/**
+ * Retry statistics
+ */
+export interface RetryStatistics {
+    /** Total retries attempted */
+    totalRetries: number;
+    /** Total tests that were retried */
+    testsRetried: number;
+    /** Tests that passed after retry */
+    recoveredTests: number;
+    /** Tests that failed all retries */
+    failedTests: number;
+    /** Recovery rate (percentage) */
+    recoveryRate: number;
+    /** Average attempts per retried test */
+    avgAttemptsPerTest: number;
+    /** Time spent retrying (ms) */
+    totalRetryTimeMs: number;
+    /** Breakdown by strategy */
+    byStrategy: Record<RetryStrategy, {
+        attempts: number;
+        successes: number;
+        avgDurationMs: number;
+    }>;
+    /** Breakdown by error type */
+    byErrorType: Record<string, number>;
+    /** Breakdown by flakiness score */
+    byFlakinessScore: {
+        low: number;
+        medium: number;
+        high: number;
+    };
+}

package/cli/dist/core/retry/types.js

@@ -0,0 +1,52 @@
+/**
+ * QA360 Smart Retry Module (F8)
+ *
+ * Intelligent retry strategies based on flakiness scoring and pattern detection.
+ * Completes the flakiness system: Detection → Scoring → Quarantine → Smart Retry
+ *
+ * @module retry
+ */
+/**
+ * Retry strategy types
+ */
+export var RetryStrategy;
+(function (RetryStrategy) {
+    /** No retry - fail immediately */
+    RetryStrategy["NONE"] = "none";
+    /** Fixed delay between retries */
+    RetryStrategy["FIXED"] = "fixed";
+    /** Linear backoff (delay increases linearly) */
+    RetryStrategy["LINEAR"] = "linear";
+    /** Exponential backoff (delay doubles each retry) */
+    RetryStrategy["EXPONENTIAL"] = "exponential";
+    /** Adaptive retry based on flakiness score */
+    RetryStrategy["ADAPTIVE"] = "adaptive";
+    /** Intelligent retry based on pattern detection */
+    RetryStrategy["INTELLIGENT"] = "intelligent";
+})(RetryStrategy || (RetryStrategy = {}));
+/**
+ * Default retry configuration
+ */
+export const DEFAULT_RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelayMs: 1000,
+    maxDelayMs: 30000,
+    attemptTimeoutMs: 30000,
+    overallTimeoutMs: 120000,
+    strategy: RetryStrategy.ADAPTIVE,
+    backoffMultiplier: 2,
+    jitterFactor: 0.1,
+    retryOnTimeout: false,
+    retryOnAssertionFailure: false,
+    nonRetriableErrors: [
+        'OutOfMemoryError',
+        'StackOverflowError',
+        'SecurityError',
+        'PermissionDenied',
+        'InvalidArgument',
+        'ValidationError',
+        'SyntaxError'
+    ],
+    adaptiveMinFlakinessScore: 75,
+    enablePatternBasedRetry: true
+};

package/cli/dist/core/retry/vault.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Retry Vault Integration
+ *
+ * Stores retry history and statistics in the Evidence Vault.
+ */
+import type { Database } from 'sqlite3';
+import type { RetryResult, RetryHistoryRecord, RetryStatistics } from './types.js';
+import { RetryStrategy } from './types.js';
+/**
+ * Retry Vault class
+ */
+export declare class RetryVault {
+    private db;
+    private dbRun;
+    private dbAll;
+    private dbGet;
+    constructor(db: Database);
+    /**
+     * Initialize retry tables
+     */
+    initialize(): Promise<void>;
+    /**
+     * Store retry result
+     */
+    storeRetryResult(runId: string, result: RetryResult, context?: {
+        testName?: string;
+        gate?: string;
+        flakinessScore?: number;
+        patternType?: string;
+    }): Promise<number>;
+    /**
+     * Get retry history for a test
+     */
+    getRetryHistory(testId: string, limit?: number): Promise<RetryHistoryRecord[]>;
+    /**
+     * Get retry history for a run
+     */
+    getRetryHistoryForRun(runId: string): Promise<RetryHistoryRecord[]>;
+    /**
+     * Get retry statistics
+     */
+    getStatistics(days?: number): Promise<RetryStatistics>;
+    /**
+     * Get strategy performance
+     */
+    getStrategyPerformance(days?: number): Promise<Array<{
+        strategy: RetryStrategy;
+        date: string;
+        attempts: number;
+        successes: number;
+        successRate: number;
+        avgDurationMs: number;
+    }>>;
+    /**
+     * Get most retried tests
+     */
+    getMostRetriedTests(limit?: number, days?: number): Promise<Array<{
+        testId: string;
+        testName: string;
+        gate: string;
+        retryCount: number;
+        recoveryRate: number;
+        avgAttempts: number;
+    }>>;
+    /**
+     * Update daily statistics
+     */
+    private updateStatistics;
+    /**
+     * Map database rows to RetryHistoryRecord objects
+     */
+    private mapRowsToRecords;
+    /**
+     * Delete old retry history
+     */
+    deleteOldHistory(olderThanMs: number): Promise<number>;
+}