qa360 2.2.1 → 2.2.13

package/cli/dist/core/flakiness/index.js

@@ -0,0 +1,384 @@
+/**
+ * QA360 Flakiness Detection Engine
+ *
+ * Detects and scores test flakiness using consecutive runs and pattern analysis.
+ * This is the core differentiator of QA360 - measuring test reliability.
+ *
+ * Scoring Categories:
+ * - Legendary (100): Never flaky - 100% reliable
+ * - Solid (90-99): Rarely flaky
+ * - Good (75-89): Occasionally flaky
+ * - Shaky (50-74): Often flaky - attention needed
+ * - Unstable (0-49): Very flaky - quarantine recommended
+ *
+ * @module flakiness
+ */
+import { createHash } from 'crypto';
+/**
+ * Flakiness score category
+ */
+export var FlakinessCategory;
+(function (FlakinessCategory) {
+    FlakinessCategory["LEGENDARY"] = "legendary";
+    FlakinessCategory["SOLID"] = "solid";
+    FlakinessCategory["GOOD"] = "good";
+    FlakinessCategory["SHAKY"] = "shaky";
+    FlakinessCategory["UNSTABLE"] = "unstable"; // 0-49
+})(FlakinessCategory || (FlakinessCategory = {}));
+/**
+ * Flakiness category metadata
+ */
+export const FLAKINESS_CATEGORIES = {
+    [FlakinessCategory.LEGENDARY]: {
+        min: 100,
+        max: 100,
+        label: 'LEGENDARY',
+        emoji: '🟢',
+        color: '#10B981',
+        description: 'Jamais flaky - 100% fiable',
+        action: 'None - keep doing what you\'re doing'
+    },
+    [FlakinessCategory.SOLID]: {
+        min: 90,
+        max: 99,
+        label: 'SOLID',
+        emoji: '🟢',
+        color: '#34D399',
+        description: 'Rarement flaky',
+        action: 'Monitor - occasional hiccups'
+    },
+    [FlakinessCategory.GOOD]: {
+        min: 75,
+        max: 89,
+        label: 'GOOD',
+        emoji: '🟡',
+        color: '#FBBF24',
+        description: 'Occasionnellement flaky',
+        action: 'Watch - may need attention'
+    },
+    [FlakinessCategory.SHAKY]: {
+        min: 50,
+        max: 74,
+        label: 'SHAKY',
+        emoji: '🟠',
+        color: '#F97316',
+        description: 'Souvent flaky - attention requise',
+        action: 'Investigate - fix recommended'
+    },
+    [FlakinessCategory.UNSTABLE]: {
+        min: 0,
+        max: 49,
+        label: 'UNSTABLE',
+        emoji: '🔴',
+        color: '#EF4444',
+        description: 'Très flaky - quarantaine recommandée',
+        action: 'Quarantine - unblock the team'
+    }
+};
+/**
+ * Default flakiness detection options
+ */
+export const DEFAULT_FLAKINESS_OPTIONS = {
+    minRuns: 3,
+    consecutiveRuns: 3,
+    timeWindowDays: 30,
+    enablePatternDetection: true
+};
+/**
+ * Flakiness pattern types
+ */
+export var FlakinessPattern;
+(function (FlakinessPattern) {
+    FlakinessPattern["TIMING"] = "timing";
+    FlakinessPattern["RACE_CONDITION"] = "race_condition";
+    FlakinessPattern["EXTERNAL_DEPENDENCY"] = "external_dependency";
+    FlakinessPattern["ENVIRONMENT_SPECIFIC"] = "environment_specific";
+    FlakinessPattern["SELECTOR_ISSUE"] = "selector_issue";
+    FlakinessPattern["UNKNOWN"] = "unknown";
+})(FlakinessPattern || (FlakinessPattern = {}));
+/**
+ * Calculate flakiness score from test results
+ * @param results Array of test results
+ * @returns Flakiness score (0-100)
+ */
+export function calculateFlakinessScore(results) {
+    if (results.length === 0) {
+        return 100; // No failures means 100% by default
+    }
+    const successCount = results.filter(r => r.success).length;
+    const score = Math.round((successCount / results.length) * 100);
+    return Math.max(0, Math.min(100, score));
+}
+/**
+ * Get flakiness category from score
+ * @param score Flakiness score (0-100)
+ * @returns Flakiness category
+ */
+export function getFlakinessCategory(score) {
+    if (score === 100)
+        return FlakinessCategory.LEGENDARY;
+    if (score >= 90)
+        return FlakinessCategory.SOLID;
+    if (score >= 75)
+        return FlakinessCategory.GOOD;
+    if (score >= 50)
+        return FlakinessCategory.SHAKY;
+    return FlakinessCategory.UNSTABLE;
+}
+/**
+ * Generate a unique test ID
+ * @param testName Test name
+ * @param filePath File path
+ * @returns Unique test ID
+ */
+export function generateTestId(testName, filePath) {
+    const hash = createHash('sha256')
+        .update(`${filePath}:${testName}`)
+        .digest('hex')
+        .substring(0, 16);
+    return hash;
+}
+/**
+ * Analyze test results for flakiness patterns
+ * @param results Array of test results for the same test
+ * @returns Pattern detection result or undefined
+ */
+export function detectFlakinessPattern(results) {
+    if (results.length < 3) {
+        return undefined;
+    }
+    const failures = results.filter(r => !r.success);
+    if (failures.length === 0) {
+        return undefined; // No failures to analyze
+    }
+    // Collect error messages as lowercase strings
+    const errorMessages = failures.map(f => (f.errorMessage || '').toLowerCase());
+    // Helper to check if any error contains a keyword
+    const anyErrorContains = (keywords) => {
+        return errorMessages.some(err => keywords.some(kw => err.includes(kw)));
+    };
+    // Check for timing issues
+    if (anyErrorContains(['timeout', 'timed out', 'waiting for', 'element not found', 'waiting for selector'])) {
+        return {
+            patternType: FlakinessPattern.TIMING,
+            description: 'Timing issues - test is too fast for the application',
+            suggestedFix: 'Increase timeout or add explicit wait (waitForSelector, waitForLoadState)',
+            confidence: 0.85,
+            autoFixPossible: true
+        };
+    }
+    // Check for selector issues
+    if (anyErrorContains(['selector', 'detached', 'stale element', 'element not found'])) {
+        return {
+            patternType: FlakinessPattern.SELECTOR_ISSUE,
+            description: 'Selector issues - element not consistently found',
+            suggestedFix: 'Use data-testid selectors instead of CSS selectors for better reliability',
+            confidence: 0.80,
+            autoFixPossible: true
+        };
+    }
+    // Check for external dependency issues
+    if (anyErrorContains(['econnrefused', 'connection refused', 'connection reset', 'network error', 'network unreachable', 'api timeout'])) {
+        return {
+            patternType: FlakinessPattern.EXTERNAL_DEPENDENCY,
+            description: 'External dependency issues - network or service unavailable',
+            suggestedFix: 'Add retry logic with exponential backoff or mock the external dependency',
+            confidence: 0.90,
+            autoFixPossible: true
+        };
+    }
+    // Check for race conditions
+    if (anyErrorContains(['race condition', 'state changed', 'unexpected state', 'assertion failed'])) {
+        return {
+            patternType: FlakinessPattern.RACE_CONDITION,
+            description: 'Possible race condition - state changes unexpectedly',
+            suggestedFix: 'Add proper synchronization/locks or use explicit waits',
+            confidence: 0.70,
+            autoFixPossible: false
+        };
+    }
+    // Check for environment-specific issues
+    const envs = new Set(results.map(r => r.environment));
+    if (envs.size > 1) {
+        const failEnvs = new Set(failures.map(r => r.environment));
+        const passEnvs = new Set(results.filter(r => r.success).map(r => r.environment));
+        // Check if failures are concentrated in one environment
+        for (const env of failEnvs) {
+            if (!passEnvs.has(env)) {
+                return {
+                    patternType: FlakinessPattern.ENVIRONMENT_SPECIFIC,
+                    description: `Environment-specific issue - fails on ${env} only`,
+                    suggestedFix: 'Check environment-specific configuration (OS, timezone, locale)',
+                    confidence: 0.75,
+                    autoFixPossible: false
+                };
+            }
+        }
+    }
+    return {
+        patternType: FlakinessPattern.UNKNOWN,
+        description: 'Flaky behavior detected but pattern unclear',
+        suggestedFix: 'Manual investigation recommended - check logs for specific failure patterns',
+        confidence: 0.50,
+        autoFixPossible: false
+    };
+}
+/**
+ * Calculate consecutive streak (passes or failures)
+ * @param results Array of test results sorted by timestamp
+ * @returns Current streak info
+ */
+export function calculateStreak(results) {
+    if (results.length === 0) {
+        return { currentStreak: 0, streakType: 'pass' };
+    }
+    // Sort by timestamp (newest first)
+    const sorted = [...results].sort((a, b) => b.timestamp - a.timestamp);
+    let streak = 0;
+    let streakType = sorted[0].success ? 'pass' : 'fail';
+    for (const result of sorted) {
+        if (result.success === (streakType === 'pass')) {
+            streak++;
+        }
+        else {
+            break;
+        }
+    }
+    return { currentStreak: streak, streakType };
+}
+/**
+ * Format flakiness score for display
+ * @param score Flakiness score (0-100)
+ * @returns Formatted string with emoji
+ */
+export function formatFlakinessScore(score) {
+    const category = getFlakinessCategory(score);
+    const meta = FLAKINESS_CATEGORIES[category];
+    return `${meta.emoji} ${category.toUpperCase()} (${score}%)`;
+}
+/**
+ * Flakiness Detection Engine
+ *
+ * Main class for detecting and scoring test flakiness
+ */
+export class FlakinessDetector {
+    options;
+    constructor(options = {}) {
+        this.options = { ...DEFAULT_FLAKINESS_OPTIONS, ...options };
+    }
+    /**
+     * Analyze a single test's flakiness from its run history
+     * @param testId Test identifier
+     * @param results Array of test results (must be from the same test)
+     * @returns Flakiness result
+     */
+    analyzeTest(testId, results) {
+        // Filter by time window
+        const timeWindow = this.options.timeWindowDays * 24 * 60 * 60 * 1000;
+        const now = Date.now();
+        const cutoff = now - timeWindow;
+        const filteredResults = results.filter(r => r.timestamp >= cutoff);
+        // Need minimum runs to calculate score
+        if (filteredResults.length < this.options.minRuns) {
+            return {
+                testId,
+                testName: results[0]?.testName || 'Unknown',
+                filePath: results[0]?.filePath || 'Unknown',
+                gate: results[0]?.gate || 'Unknown',
+                score: 100, // Default to 100 if not enough data
+                category: FlakinessCategory.LEGENDARY,
+                totalRuns: filteredResults.length,
+                successfulRuns: filteredResults.filter(r => r.success).length,
+                avgDurationMs: this.averageDuration(filteredResults)
+            };
+        }
+        // Calculate score
+        const score = calculateFlakinessScore(filteredResults);
+        const category = getFlakinessCategory(score);
+        // Calculate streak
+        const { currentStreak, streakType } = calculateStreak(filteredResults);
+        // Detect patterns if enabled
+        let patternType;
+        let suggestedFix;
+        let confidence;
+        if (this.options.enablePatternDetection) {
+            const pattern = detectFlakinessPattern(filteredResults);
+            if (pattern) {
+                patternType = pattern.patternType;
+                suggestedFix = pattern.suggestedFix;
+                confidence = pattern.confidence;
+            }
+        }
+        return {
+            testId,
+            testName: results[0].testName,
+            filePath: results[0].filePath,
+            gate: results[0].gate,
+            score,
+            category,
+            totalRuns: filteredResults.length,
+            successfulRuns: filteredResults.filter(r => r.success).length,
+            avgDurationMs: this.averageDuration(filteredResults),
+            patternType,
+            suggestedFix,
+            confidence,
+            firstSeen: Math.min(...filteredResults.map(r => r.timestamp)),
+            lastSeen: Math.max(...filteredResults.map(r => r.timestamp)),
+            currentStreak,
+            streakType
+        };
+    }
+    /**
+     * Analyze multiple tests and return flakiness results
+     * @param testResults Map of testId to array of results
+     * @returns Array of flakiness results
+     */
+    analyzeAll(testResults) {
+        const flakinessResults = [];
+        for (const [testId, testRuns] of testResults.entries()) {
+            if (testRuns.length > 0) {
+                flakinessResults.push(this.analyzeTest(testId, testRuns));
+            }
+        }
+        // Sort by score (lowest first) - most flaky first
+        flakinessResults.sort((a, b) => a.score - b.score);
+        return flakinessResults;
+    }
+    /**
+     * Check if a test should be quarantined based on flakiness
+     * @param result Flakiness result
+     * @returns Whether to quarantine
+     */
+    shouldQuarantine(result) {
+        // Quarantine if:
+        // 1. Score is below 50 (Unstable)
+        // 2. Score is below 75 (Shaky) AND has more than 5 runs
+        return (result.category === FlakinessCategory.UNSTABLE ||
+            (result.category === FlakinessCategory.SHAKY && result.totalRuns >= 5));
+    }
+    /**
+     * Get quarantine recommendation message
+     * @param result Flakiness result
+     * @returns Recommendation message
+     */
+    getQuarantineRecommendation(result) {
+        if (!this.shouldQuarantine(result)) {
+            return `Test is ${FLAKINESS_CATEGORIES[result.category].emoji} ${result.category} - no quarantine needed`;
+        }
+        return [
+            `❌ Test "${result.testName}" is ${FLAKINESS_CATEGORIES[result.category].emoji} ${FLAKINESS_CATEGORIES[result.category].label} (${result.score}% flaky)`,
+            `   Passed ${result.successfulRuns}/${result.totalRuns} times`,
+            result.suggestedFix ? `   💡 Suggested: ${result.suggestedFix}` : '',
+            `   📋 Action: ${FLAKINESS_CATEGORIES[result.category].action}`
+        ].filter(Boolean).join('\n');
+    }
+    /**
+     * Calculate average duration
+     */
+    averageDuration(results) {
+        if (results.length === 0)
+            return 0;
+        const total = results.reduce((sum, r) => sum + r.durationMs, 0);
+        return Math.round(total / results.length);
+    }
+}

package/cli/dist/core/generation/code-formatter.d.ts

@@ -0,0 +1,111 @@
+/**
+ * QA360 Code Formatter
+ *
+ * Formats and validates generated test code.
+ * Supports TypeScript, JavaScript, Go, and Python.
+ */
+/**
+ * Formatter configuration
+ */
+export interface FormatterConfig {
+    /**
+     * Indentation style
+     */
+    indentStyle?: 'spaces' | 'tabs';
+    /**
+     * Indentation size
+     */
+    indentSize?: number;
+    /**
+     * Line width
+     */
+    lineWidth?: number;
+    /**
+     * Semicolons (for JS/TS)
+     */
+    semicolons?: boolean;
+    /**
+     * Quotes style
+     */
+    quotes?: 'single' | 'double';
+    /**
+     * Trailing commas
+     */
+    trailingCommas?: boolean;
+    /**
+     * Sort imports
+     */
+    sortImports?: boolean;
+}
+/**
+ * Formatting result
+ */
+export interface FormatResult {
+    formatted: string;
+    errors: string[];
+    warnings: string[];
+}
+/**
+ * Code Formatter class
+ */
+export declare class CodeFormatter {
+    private readonly config;
+    private readonly defaultConfig;
+    constructor(config?: FormatterConfig);
+    /**
+     * Format code based on language
+     */
+    format(code: string, language: string): string;
+    /**
+     * Format TypeScript code
+     */
+    formatTypeScript(code: string): string;
+    /**
+     * Format JavaScript code
+     */
+    formatJavaScript(code: string): string;
+    /**
+     * Format Go code
+     */
+    formatGo(code: string): string;
+    /**
+     * Format Python code
+     */
+    formatPython(code: string): string;
+    /**
+     * Generic formatting
+     */
+    formatGeneric(code: string): string;
+    /**
+     * Validate code syntax
+     */
+    validate(code: string, language: string): FormatResult;
+    /**
+     * Sort imports
+     */
+    private sortImports;
+    /**
+     * Sort import lines
+     */
+    private sortImportLines;
+    /**
+     * Normalize quotes
+     */
+    private normalizeQuotes;
+    /**
+     * Normalize braces spacing
+     */
+    private normalizeBraces;
+    /**
+     * Fix common formatting issues
+     */
+    private fixCommonIssues;
+    /**
+     * Normalize Python indentation
+     */
+    private normalizePythonIndentation;
+}
+/**
+ * Create a formatter with default config
+ */
+export declare function createFormatter(config?: FormatterConfig): CodeFormatter;