qa360 2.2.1 → 2.2.13

package/cli/dist/core/retry/vault.js

@@ -0,0 +1,304 @@
+/**
+ * Retry Vault Integration
+ *
+ * Stores retry history and statistics in the Evidence Vault.
+ */
+import { promisify } from 'util';
+import { RetryStrategy } from './types.js';
+/**
+ * Retry Vault class
+ */
+export class RetryVault {
+    db;
+    dbRun;
+    dbAll;
+    dbGet;
+    constructor(db) {
+        this.db = db;
+        this.dbRun = promisify(db.run.bind(db));
+        this.dbAll = promisify(db.all.bind(db));
+        this.dbGet = promisify(db.get.bind(db));
+    }
+    /**
+     * Initialize retry tables
+     */
+    async initialize() {
+        const schema = `
+      -- Retry history table
+      CREATE TABLE IF NOT EXISTS retry_history (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+          test_id TEXT NOT NULL,
+          test_name TEXT NOT NULL,
+          gate TEXT NOT NULL,
+          strategy TEXT NOT NULL,
+          max_retries INTEGER NOT NULL,
+          actual_attempts INTEGER NOT NULL,
+          success BOOLEAN NOT NULL,
+          successful_attempt INTEGER,
+          total_duration_ms INTEGER NOT NULL,
+          stop_reason TEXT NOT NULL,
+          error_type TEXT,
+          error_message TEXT,
+          flakiness_score REAL,
+          pattern_type TEXT,
+          timestamp INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_retry_run_id ON retry_history(run_id);
+      CREATE INDEX IF NOT EXISTS idx_retry_test_id ON retry_history(test_id);
+      CREATE INDEX IF NOT EXISTS idx_retry_strategy ON retry_history(strategy);
+      CREATE INDEX IF NOT EXISTS idx_retry_success ON retry_history(success);
+      CREATE INDEX IF NOT EXISTS idx_retry_timestamp ON retry_history(timestamp);
+
+      -- Retry statistics summary table
+      CREATE TABLE IF NOT EXISTS retry_stats (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          date TEXT NOT NULL,
+          total_retries INTEGER NOT NULL DEFAULT 0,
+          tests_retried INTEGER NOT NULL DEFAULT 0,
+          recovered_tests INTEGER NOT NULL DEFAULT 0,
+          failed_tests INTEGER NOT NULL DEFAULT 0,
+          recovery_rate REAL NOT NULL DEFAULT 0,
+          avg_attempts_per_test REAL NOT NULL DEFAULT 0,
+          total_retry_time_ms INTEGER NOT NULL DEFAULT 0,
+          UNIQUE(date)
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_retry_stats_date ON retry_stats(date);
+
+      -- Strategy performance table
+      CREATE TABLE IF NOT EXISTS retry_strategy_performance (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          strategy TEXT NOT NULL,
+          date TEXT NOT NULL,
+          attempts INTEGER NOT NULL DEFAULT 0,
+          successes INTEGER NOT NULL DEFAULT 0,
+          avg_duration_ms REAL NOT NULL DEFAULT 0,
+          UNIQUE(strategy, date)
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_retry_strategy_perf_date ON retry_strategy_performance(date);
+    `;
+        const statements = schema
+            .split(';')
+            .map(s => s.trim())
+            .filter(s => s.length > 0);
+        for (const statement of statements) {
+            await this.dbRun(statement, []);
+        }
+    }
+    /**
+     * Store retry result
+     */
+    async storeRetryResult(runId, result, context) {
+        const lastAttempt = result.attempts[result.attempts.length - 1];
+        const errorType = lastAttempt?.errorType;
+        const errorMessage = lastAttempt?.errorMessage;
+        const sql = `
+      INSERT INTO retry_history (
+        run_id, test_id, test_name, gate, strategy, max_retries,
+        actual_attempts, success, successful_attempt, total_duration_ms,
+        stop_reason, error_type, error_message, flakiness_score, pattern_type, timestamp
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `;
+        const result2 = await this.dbRun(sql, [
+            runId,
+            result.testId,
+            context?.testName || result.testId,
+            context?.gate || 'unknown',
+            result.strategy,
+            result.totalAttempts - 1, // max_retries = attempts - 1 (first attempt is not a retry)
+            result.totalAttempts,
+            result.success ? 1 : 0,
+            result.successfulAttempt || null,
+            result.totalDurationMs,
+            result.stopReason,
+            errorType || null,
+            errorMessage || null,
+            context?.flakinessScore || null,
+            context?.patternType || null,
+            Date.now()
+        ]);
+        // Update statistics
+        await this.updateStatistics(result);
+        return result2.lastID;
+    }
+    /**
+     * Get retry history for a test
+     */
+    async getRetryHistory(testId, limit = 50) {
+        const rows = await this.dbAll(`SELECT * FROM retry_history WHERE test_id = ? ORDER BY timestamp DESC LIMIT ?`, [testId, limit]);
+        return this.mapRowsToRecords(rows);
+    }
+    /**
+     * Get retry history for a run
+     */
+    async getRetryHistoryForRun(runId) {
+        const rows = await this.dbAll(`SELECT * FROM retry_history WHERE run_id = ? ORDER BY timestamp DESC`, [runId]);
+        return this.mapRowsToRecords(rows);
+    }
+    /**
+     * Get retry statistics
+     */
+    async getStatistics(days = 30) {
+        const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+        const total = await this.dbGet(`SELECT COUNT(*) as count, SUM(actual_attempts) as attempts, SUM(total_duration_ms) as duration
+       FROM retry_history WHERE timestamp >= ?`, [cutoff]);
+        const testsRetried = await this.dbGet(`SELECT COUNT(DISTINCT test_id) as count FROM retry_history WHERE timestamp >= ?`, [cutoff]);
+        const recovered = await this.dbGet(`SELECT COUNT(*) as count FROM retry_history WHERE success = 1 AND timestamp >= ?`, [cutoff]);
+        const failed = await this.dbGet(`SELECT COUNT(*) as count FROM retry_history WHERE success = 0 AND timestamp >= ?`, [cutoff]);
+        const byStrategy = await this.dbAll(`SELECT strategy, COUNT(*) as attempts, SUM(success) as successes, AVG(total_duration_ms) as avg_duration
+       FROM retry_history WHERE timestamp >= ?
+       GROUP BY strategy`, [cutoff]);
+        const byErrorType = await this.dbAll(`SELECT error_type, COUNT(*) as count FROM retry_history
+       WHERE timestamp >= ? AND error_type IS NOT NULL
+       GROUP BY error_type
+       ORDER BY count DESC`, [cutoff]);
+        const byFlakiness = await this.dbGet(`SELECT
+         SUM(CASE WHEN flakiness_score < 50 THEN 1 ELSE 0 END) as low,
+         SUM(CASE WHEN flakiness_score >= 50 AND flakiness_score < 75 THEN 1 ELSE 0 END) as medium,
+         SUM(CASE WHEN flakiness_score >= 75 THEN 1 ELSE 0 END) as high
+       FROM retry_history WHERE timestamp >= ?`, [cutoff]);
+        const totalRetries = total?.attempts || 0;
+        const totalRecovered = recovered?.count || 0;
+        const totalFailed = failed?.count || 0;
+        const totalTests = testsRetried?.count || 0;
+        const byStrategyMap = {
+            [RetryStrategy.NONE]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+            [RetryStrategy.FIXED]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+            [RetryStrategy.LINEAR]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+            [RetryStrategy.EXPONENTIAL]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+            [RetryStrategy.ADAPTIVE]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+            [RetryStrategy.INTELLIGENT]: { attempts: 0, successes: 0, avgDurationMs: 0 }
+        };
+        for (const row of byStrategy) {
+            byStrategyMap[row.strategy] = {
+                attempts: row.attempts,
+                successes: row.successes,
+                avgDurationMs: Math.round(row.avg_duration)
+            };
+        }
+        const byErrorTypeMap = {};
+        for (const row of byErrorType) {
+            byErrorTypeMap[row.error_type] = row.count;
+        }
+        return {
+            totalRetries,
+            testsRetried: totalTests,
+            recoveredTests: totalRecovered,
+            failedTests: totalFailed,
+            recoveryRate: totalRetries > 0 ? Math.round((totalRecovered / (totalRecovered + totalFailed)) * 100) : 0,
+            avgAttemptsPerTest: totalTests > 0 ? Math.round(totalRetries / totalTests) : 0,
+            totalRetryTimeMs: total?.duration || 0,
+            byStrategy: byStrategyMap,
+            byErrorType: byErrorTypeMap,
+            byFlakinessScore: {
+                low: byFlakiness?.low || 0,
+                medium: byFlakiness?.medium || 0,
+                high: byFlakiness?.high || 0
+            }
+        };
+    }
+    /**
+     * Get strategy performance
+     */
+    async getStrategyPerformance(days = 30) {
+        const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+        const rows = await this.dbAll(`SELECT strategy, date, attempts, successes, avg_duration_ms
+       FROM retry_strategy_performance
+       WHERE date >= datetime(?, 'unixepoch')
+       ORDER BY date DESC, strategy`, [Math.floor(cutoff / 1000)]);
+        return rows.map((r) => ({
+            strategy: r.strategy,
+            date: r.date,
+            attempts: r.attempts,
+            successes: r.successes,
+            successRate: r.attempts > 0 ? Math.round((r.successes / r.attempts) * 100) : 0,
+            avgDurationMs: Math.round(r.avg_duration_ms)
+        }));
+    }
+    /**
+     * Get most retried tests
+     */
+    async getMostRetriedTests(limit = 10, days = 30) {
+        const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+        const rows = await this.dbAll(`SELECT
+         test_id,
+         test_name,
+         gate,
+         COUNT(*) as retry_count,
+         SUM(success) as recovered,
+         AVG(actual_attempts) as avg_attempts
+       FROM retry_history
+       WHERE timestamp >= ?
+       GROUP BY test_id, test_name, gate
+       ORDER BY retry_count DESC
+       LIMIT ?`, [cutoff, limit]);
+        return rows.map((r) => ({
+            testId: r.test_id,
+            testName: r.test_name,
+            gate: r.gate,
+            retryCount: r.retry_count,
+            recoveryRate: r.retry_count > 0 ? Math.round((r.recovered / r.retry_count) * 100) : 0,
+            avgAttempts: Math.round(r.avg_attempts)
+        }));
+    }
+    /**
+     * Update daily statistics
+     */
+    async updateStatistics(result) {
+        const today = new Date().toISOString().split('T')[0];
+        // Update main stats
+        await this.dbRun(`
+      INSERT INTO retry_stats (date, total_retries, tests_retried, recovered_tests, failed_tests)
+      VALUES (?, 1, 1, ?, ?)
+      ON CONFLICT(date) DO UPDATE SET
+        total_retries = total_retries + 1,
+        tests_retried = tests_retried + 1,
+        recovered_tests = recovered_tests + ?,
+        failed_tests = failed_tests + ?
+    `, [today, result.success ? 1 : 0, result.success ? 0 : 1, result.success ? 1 : 0, result.success ? 0 : 1]);
+        // Update strategy performance
+        await this.dbRun(`
+      INSERT INTO retry_strategy_performance (strategy, date, attempts, successes, avg_duration_ms)
+      VALUES (?, ?, 1, ?, ?)
+      ON CONFLICT(strategy, date) DO UPDATE SET
+        attempts = attempts + 1,
+        successes = successes + ?,
+        avg_duration_ms = (avg_duration_ms * (attempts - 1) + ?) / attempts
+    `, [result.strategy, today, result.success ? 1 : 0, result.totalDurationMs, result.success ? 1 : 0, result.totalDurationMs]);
+    }
+    /**
+     * Map database rows to RetryHistoryRecord objects
+     */
+    mapRowsToRecords(rows) {
+        return rows.map((r) => ({
+            id: r.id,
+            run_id: r.run_id,
+            test_id: r.test_id,
+            test_name: r.test_name,
+            gate: r.gate,
+            strategy: r.strategy,
+            max_retries: r.max_retries,
+            actual_attempts: r.actual_attempts,
+            success: r.success === 1,
+            successful_attempt: r.successful_attempt,
+            total_duration_ms: r.total_duration_ms,
+            stop_reason: r.stop_reason,
+            error_type: r.error_type,
+            error_message: r.error_message,
+            flakiness_score: r.flakiness_score,
+            pattern_type: r.pattern_type,
+            timestamp: r.timestamp
+        }));
+    }
+    /**
+     * Delete old retry history
+     */
+    async deleteOldHistory(olderThanMs) {
+        const cutoff = Date.now() - olderThanMs;
+        const result = await this.dbRun(`DELETE FROM retry_history WHERE timestamp < ?`, [cutoff]);
+        return result.changes || 0;
+    }
+}

package/cli/dist/core/runner/e2e-helpers.d.ts

@@ -0,0 +1,102 @@
+/**
+ * E2E Test Helpers
+ *
+ * Utility functions for E2E testing including health checks,
+ * site availability verification, and graceful test skipping.
+ */
+/**
+ * Health check result for a target site
+ */
+export interface HealthCheckResult {
+    url: string;
+    available: boolean;
+    status?: number;
+    latency?: number;
+    error?: string;
+}
+/**
+ * Health check options
+ */
+export interface HealthCheckOptions {
+    timeout?: number;
+    method?: 'GET' | 'HEAD' | 'OPTIONS';
+    expectedStatus?: number[];
+}
+/**
+ * Perform a health check on a URL
+ *
+ * @param url - URL to check
+ * @param options - Health check options
+ * @returns Health check result
+ */
+export declare function healthCheck(url: string, options?: HealthCheckOptions): Promise<HealthCheckResult>;
+/**
+ * Check if a site is healthy before running E2E tests
+ *
+ * @param url - URL to check
+ * @param options - Health check options
+ * @returns true if site is healthy, false otherwise
+ */
+export declare function isSiteHealthy(url: string, options?: HealthCheckOptions): Promise<boolean>;
+/**
+ * Perform health checks on multiple URLs
+ *
+ * @param urls - URLs to check
+ * @param options - Health check options
+ * @returns Array of health check results
+ */
+export declare function healthCheckMany(urls: string[], options?: HealthCheckOptions): Promise<HealthCheckResult[]>;
+/**
+ * Skip test if site is unhealthy with a clear message
+ *
+ * Usage in beforeEach:
+ *   await skipIfUnhealthy('https://example.com');
+ *
+ * @param url - URL to check
+ * @param options - Health check options
+ */
+export declare function skipIfUnhealthy(url: string, options?: HealthCheckOptions): Promise<void>;
+/**
+ * Health check configuration for known E2E targets
+ */
+export declare const E2E_TARGETS: {
+    readonly b2bshop: {
+        readonly url: "https://b2bshop.lovable.app";
+        readonly timeout: 10000;
+        readonly description: "B2BShop E-commerce platform";
+    };
+    readonly reqres: {
+        readonly url: "https://reqres.in/api";
+        readonly timeout: 5000;
+        readonly description: "Reqres.in fake REST API";
+    };
+};
+/**
+ * Get health check config for a known target
+ *
+ * @param target - Target name
+ * @returns Health check options for the target
+ */
+export declare function getTargetConfig(target: keyof typeof E2E_TARGETS): HealthCheckOptions & {
+    url: string;
+    description: string;
+};
+/**
+ * Perform health check on a known E2E target
+ *
+ * @param target - Target name
+ * @returns Health check result
+ */
+export declare function healthCheckE2ETarget(target: keyof typeof E2E_TARGETS): Promise<HealthCheckResult>;
+/**
+ * Skip test if known E2E target is unhealthy
+ *
+ * @param target - Target name
+ */
+export declare function skipIfE2ETargetUnhealthy(target: keyof typeof E2E_TARGETS): Promise<void>;
+/**
+ * Log health status of all known E2E targets
+ *
+ * Useful for debugging or CI output
+ */
+export declare function logE2ETargetsStatus(): Promise<void>;

package/cli/dist/core/runner/e2e-helpers.js

@@ -0,0 +1,153 @@
+/**
+ * E2E Test Helpers
+ *
+ * Utility functions for E2E testing including health checks,
+ * site availability verification, and graceful test skipping.
+ */
+/**
+ * Perform a health check on a URL
+ *
+ * @param url - URL to check
+ * @param options - Health check options
+ * @returns Health check result
+ */
+export async function healthCheck(url, options = {}) {
+    const { timeout = 5000, method = 'HEAD', expectedStatus = [200, 301, 302, 304] } = options;
+    const startTime = Date.now();
+    try {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), timeout);
+        const response = await fetch(url, {
+            method,
+            signal: controller.signal,
+            // Don't follow redirects for health check
+            redirect: 'manual'
+        });
+        clearTimeout(timeoutId);
+        const latency = Date.now() - startTime;
+        const isAvailable = expectedStatus.includes(response.status);
+        return {
+            url,
+            available: isAvailable,
+            status: response.status,
+            latency
+        };
+    }
+    catch (error) {
+        const latency = Date.now() - startTime;
+        return {
+            url,
+            available: false,
+            latency,
+            error: error instanceof Error ? error.message : String(error)
+        };
+    }
+}
+/**
+ * Check if a site is healthy before running E2E tests
+ *
+ * @param url - URL to check
+ * @param options - Health check options
+ * @returns true if site is healthy, false otherwise
+ */
+export async function isSiteHealthy(url, options) {
+    const result = await healthCheck(url, options);
+    return result.available;
+}
+/**
+ * Perform health checks on multiple URLs
+ *
+ * @param urls - URLs to check
+ * @param options - Health check options
+ * @returns Array of health check results
+ */
+export async function healthCheckMany(urls, options) {
+    return Promise.all(urls.map(url => healthCheck(url, options)));
+}
+/**
+ * Skip test if site is unhealthy with a clear message
+ *
+ * Usage in beforeEach:
+ *   await skipIfUnhealthy('https://example.com');
+ *
+ * @param url - URL to check
+ * @param options - Health check options
+ */
+export async function skipIfUnhealthy(url, options) {
+    const result = await healthCheck(url, options);
+    if (!result.available) {
+        const message = `Skipping E2E tests: ${url} is not available` +
+            (result.status ? ` (status: ${result.status})` : '') +
+            (result.error ? ` - ${result.error}` : '');
+        console.warn(`\n⚠️  ${message}`);
+        // Throw a special error that test frameworks should handle as skip
+        const skipError = new Error(message);
+        skipError.skip = true;
+        throw skipError;
+    }
+}
+/**
+ * Health check configuration for known E2E targets
+ */
+export const E2E_TARGETS = {
+    b2bshop: {
+        url: 'https://b2bshop.lovable.app',
+        timeout: 10000,
+        description: 'B2BShop E-commerce platform'
+    },
+    reqres: {
+        url: 'https://reqres.in/api',
+        timeout: 5000,
+        description: 'Reqres.in fake REST API'
+    }
+};
+/**
+ * Get health check config for a known target
+ *
+ * @param target - Target name
+ * @returns Health check options for the target
+ */
+export function getTargetConfig(target) {
+    const config = E2E_TARGETS[target];
+    return {
+        url: config.url,
+        timeout: config.timeout,
+        description: config.description
+    };
+}
+/**
+ * Perform health check on a known E2E target
+ *
+ * @param target - Target name
+ * @returns Health check result
+ */
+export async function healthCheckE2ETarget(target) {
+    const config = E2E_TARGETS[target];
+    return healthCheck(config.url, { timeout: config.timeout });
+}
+/**
+ * Skip test if known E2E target is unhealthy
+ *
+ * @param target - Target name
+ */
+export async function skipIfE2ETargetUnhealthy(target) {
+    const config = E2E_TARGETS[target];
+    await skipIfUnhealthy(config.url, { timeout: config.timeout });
+}
+/**
+ * Log health status of all known E2E targets
+ *
+ * Useful for debugging or CI output
+ */
+export async function logE2ETargetsStatus() {
+    console.log('\n🔍 E2E Targets Health Check:');
+    console.log('━'.repeat(50));
+    for (const [name, config] of Object.entries(E2E_TARGETS)) {
+        const result = await healthCheck(config.url, { timeout: config.timeout });
+        const status = result.available ? '✅' : '❌';
+        const latency = result.latency ? `${result.latency}ms` : 'N/A';
+        const statusText = result.status ? `(${result.status})` : '';
+        console.log(`${status} ${name.padEnd(10)} ${config.url.padEnd(35)} ${latency.padStart(8)} ${statusText}`);
+    }
+    console.log('━'.repeat(50));
+}