qa360 1.4.5 → 2.0.1

package/dist/core/discoverer/index.js

@@ -0,0 +1,250 @@
+/**
+ * QA360 Test Discoverer
+ *
+ * Automatically discovers and categorizes test files in a project.
+ * Supports Playwright, K6, Cypress, and other test frameworks.
+ */
+import { stat } from 'fs/promises';
+import { relative } from 'path';
+import { glob } from 'glob';
+/**
+ * Test Discoverer
+ *
+ * Scans a project directory for test files and categorizes them
+ * by framework and type.
+ */
+export class TestDiscoverer {
+    rootDir;
+    defaultPatterns;
+    // Default test patterns
+    DEFAULT_PATTERNS = [
+        'tests/**/*',
+        'test/**/*',
+        '**/*.spec.ts',
+        '**/*.spec.js',
+        '**/*.test.ts',
+        '**/*.test.js',
+        '**/__tests__/**/*',
+    ];
+    // Default exclusions
+    DEFAULT_EXCLUDES = [
+        '**/node_modules/**',
+        '**/dist/**',
+        '**/build/**',
+        '**/.next/**',
+        '**/.nuxt/**',
+        '**/coverage/**',
+        '**/*.d.ts',
+    ];
+    constructor(options = {}) {
+        this.rootDir = options.rootDir || process.cwd();
+        this.defaultPatterns = options.patterns || this.DEFAULT_PATTERNS;
+    }
+    /**
+     * Discover all test files in the project
+     */
+    async discover() {
+        const files = await this.findTestFiles();
+        const categorized = {
+            playwright: [],
+            k6: [],
+            cypress: [],
+            jest: [],
+            vitest: [],
+            other: [],
+        };
+        for (const file of files) {
+            const category = this.categorizeFile(file);
+            categorized[category].push(file);
+        }
+        return categorized;
+    }
+    /**
+     * Find all test files using glob patterns
+     */
+    async findTestFiles() {
+        const files = await glob(this.defaultPatterns, {
+            cwd: this.rootDir,
+            absolute: true,
+            ignore: this.DEFAULT_EXCLUDES,
+        });
+        const results = [];
+        for (const filePath of files) {
+            const statResult = await stat(filePath).catch(() => null);
+            if (!statResult?.isFile())
+                continue;
+            results.push({
+                path: filePath,
+                type: this.detectType(filePath),
+                framework: this.detectFramework(filePath),
+                relativePath: relative(this.rootDir, filePath),
+            });
+        }
+        return results;
+    }
+    /**
+     * Categorize a file into a test category
+     */
+    categorizeFile(file) {
+        switch (file.framework) {
+            case 'playwright':
+                return 'playwright';
+            case 'k6':
+                return 'k6';
+            case 'cypress':
+                return 'cypress';
+            case 'jest':
+                return 'jest';
+            case 'vitest':
+                return 'vitest';
+            default:
+                return 'other';
+        }
+    }
+    /**
+     * Detect the test framework from file path and content
+     */
+    detectFramework(filePath) {
+        const path = filePath.toLowerCase();
+        // Check by path patterns first
+        if (path.includes('playwright') || path.endsWith('.spec.ts') || path.endsWith('.spec.js')) {
+            return 'playwright';
+        }
+        if (path.includes('k6') || path.includes('perf') || path.includes('load')) {
+            return 'k6';
+        }
+        if (path.includes('cypress') || path.includes('.cy.')) {
+            return 'cypress';
+        }
+        if (path.includes('vitest')) {
+            return 'vitest';
+        }
+        if (path.includes('__tests__') || path.includes('.test.')) {
+            return 'jest';
+        }
+        return 'unknown';
+    }
+    /**
+     * Detect the type of test (api, ui, perf, etc.)
+     */
+    detectType(filePath) {
+        const path = filePath.toLowerCase();
+        const name = filePath.toLowerCase();
+        // Detect by directory/file name
+        if (name.includes('api') || name.includes('endpoint')) {
+            return 'api';
+        }
+        if (name.includes('ui') || name.includes('e2e') || name.includes('ui-e2e')) {
+            return 'e2e';
+        }
+        if (name.includes('perf') || name.includes('performance') || name.includes('load')) {
+            return 'perf';
+        }
+        if (name.includes('security') || name.includes('sast')) {
+            return 'sast';
+        }
+        if (name.includes('a11y') || name.includes('accessibility')) {
+            return 'a11y';
+        }
+        if (name.includes('unit')) {
+            return 'unit';
+        }
+        if (name.includes('integration')) {
+            return 'integration';
+        }
+        // Default for framework-specific conventions
+        const framework = this.detectFramework(filePath);
+        switch (framework) {
+            case 'k6':
+                return 'perf';
+            case 'cypress':
+                return 'e2e';
+            default:
+                return 'unit';
+        }
+    }
+    /**
+     * Route a test file to the appropriate QA360 adapter
+     */
+    routeToAdapter(test) {
+        switch (test.framework) {
+            case 'playwright':
+                if (test.type === 'api') {
+                    return { adapter: 'playwright-api' };
+                }
+                return { adapter: 'playwright-ui' };
+            case 'k6':
+                return { adapter: 'k6-perf', config: { testType: test.type } };
+            case 'cypress':
+                return { adapter: 'cypress-ui' };
+            default:
+                return { adapter: 'generic', config: { testPath: test.path } };
+        }
+    }
+    /**
+     * Discover tests for a specific gate
+     */
+    async discoverForGate(gate) {
+        const allTests = await this.discover();
+        switch (gate) {
+            case 'api_smoke':
+                return [
+                    ...allTests.playwright.filter(t => t.type === 'api'),
+                    ...allTests.jest.filter(t => t.type === 'api'),
+                    ...allTests.vitest.filter(t => t.type === 'api'),
+                ];
+            case 'ui':
+            case 'e2e':
+                return [
+                    ...allTests.playwright.filter(t => t.type === 'e2e' || t.type === 'ui'),
+                    ...allTests.cypress.filter(t => t.type === 'e2e'),
+                ];
+            case 'perf':
+                return allTests.k6;
+            case 'security':
+            case 'sast':
+                return [
+                    ...allTests.playwright.filter(t => t.path.includes('security')),
+                    ...allTests.other.filter(t => t.type === 'sast'),
+                ];
+            default:
+                return [];
+        }
+    }
+    /**
+     * Get summary statistics of discovered tests
+     */
+    async getSummary() {
+        const tests = await this.discover();
+        return {
+            total: this.countAll(tests),
+            byFramework: {
+                playwright: tests.playwright.length,
+                k6: tests.k6.length,
+                cypress: tests.cypress.length,
+                jest: tests.jest.length,
+                vitest: tests.vitest.length,
+                other: tests.other.length,
+            },
+            byType: this.countByType(tests),
+        };
+    }
+    countAll(tests) {
+        return Object.values(tests).reduce((sum, arr) => sum + arr.length, 0);
+    }
+    countByType(tests) {
+        const counts = {};
+        for (const category of Object.values(tests)) {
+            for (const test of category) {
+                counts[test.type] = (counts[test.type] || 0) + 1;
+            }
+        }
+        return counts;
+    }
+}
+/**
+ * Create a test discoverer
+ */
+export function createTestDiscoverer(options) {
+    return new TestDiscoverer(options);
+}

package/dist/core/flakiness/index.d.ts

@@ -0,0 +1,228 @@
+/**
+ * QA360 Flakiness Detection Engine
+ *
+ * Detects and scores test flakiness using consecutive runs and pattern analysis.
+ * This is the core differentiator of QA360 - measuring test reliability.
+ *
+ * Scoring Categories:
+ * - Legendary (100): Never flaky - 100% reliable
+ * - Solid (90-99): Rarely flaky
+ * - Good (75-89): Occasionally flaky
+ * - Shaky (50-74): Often flaky - attention needed
+ * - Unstable (0-49): Very flaky - quarantine recommended
+ *
+ * @module flakiness
+ */
+/**
+ * Flakiness score category
+ */
+export declare enum FlakinessCategory {
+    LEGENDARY = "legendary",// 100
+    SOLID = "solid",// 90-99
+    GOOD = "good",// 75-89
+    SHAKY = "shaky",// 50-74
+    UNSTABLE = "unstable"
+}
+/**
+ * Flakiness category metadata
+ */
+export declare const FLAKINESS_CATEGORIES: Record<FlakinessCategory, {
+    min: number;
+    max: number;
+    label: string;
+    emoji: string;
+    color: string;
+    description: string;
+    action: string;
+}>;
+/**
+ * Test result from a single run
+ */
+export interface TestResult {
+    /** Test identifier (unique per test) */
+    testId: string;
+    /** Test name/title */
+    testName: string;
+    /** File path */
+    filePath: string;
+    /** Gate name */
+    gate: string;
+    /** Whether the test passed */
+    success: boolean;
+    /** Duration in milliseconds */
+    durationMs: number;
+    /** Error type if failed */
+    errorType?: string;
+    /** Error message if failed */
+    errorMessage?: string;
+    /** Timestamp of run */
+    timestamp: number;
+    /** Environment (local, ci, staging) */
+    environment: string;
+    /** Git branch */
+    branch?: string;
+    /** Commit hash */
+    commitHash?: string;
+    /** CI run ID */
+    ciRunId?: string;
+    /** Tags (team, repo, etc.) */
+    tags?: Record<string, string>;
+}
+/**
+ * Flakiness detection result for a single test
+ */
+export interface FlakinessResult {
+    /** Test identifier */
+    testId: string;
+    /** Test name */
+    testName: string;
+    /** File path */
+    filePath: string;
+    /** Gate name */
+    gate: string;
+    /** Flakiness score (0-100) */
+    score: number;
+    /** Flakiness category */
+    category: FlakinessCategory;
+    /** Total number of runs analyzed */
+    totalRuns: number;
+    /** Number of successful runs */
+    successfulRuns: number;
+    /** Average duration in ms */
+    avgDurationMs: number;
+    /** Detected pattern type (if any) */
+    patternType?: string;
+    /** Suggested fix */
+    suggestedFix?: string;
+    /** Confidence in pattern detection (0-1) */
+    confidence?: number;
+    /** First seen timestamp */
+    firstSeen?: number;
+    /** Last seen timestamp */
+    lastSeen?: number;
+    /** Current streak (consecutive passes or failures) */
+    currentStreak?: number;
+    /** Streak type */
+    streakType?: 'pass' | 'fail';
+}
+/**
+ * Flakiness detection options
+ */
+export interface FlakinessDetectionOptions {
+    /** Minimum runs before calculating score */
+    minRuns?: number;
+    /** Number of consecutive runs for detection */
+    consecutiveRuns?: number;
+    /** Time window in days (only consider runs within this window) */
+    timeWindowDays?: number;
+    /** Enable pattern detection */
+    enablePatternDetection?: boolean;
+}
+/**
+ * Default flakiness detection options
+ */
+export declare const DEFAULT_FLAKINESS_OPTIONS: Required<FlakinessDetectionOptions>;
+/**
+ * Flakiness pattern types
+ */
+export declare enum FlakinessPattern {
+    TIMING = "timing",
+    RACE_CONDITION = "race_condition",
+    EXTERNAL_DEPENDENCY = "external_dependency",
+    ENVIRONMENT_SPECIFIC = "environment_specific",
+    SELECTOR_ISSUE = "selector_issue",
+    UNKNOWN = "unknown"
+}
+/**
+ * Pattern detection result
+ */
+export interface PatternDetection {
+    /** Pattern type */
+    patternType: FlakinessPattern;
+    /** Pattern description */
+    description: string;
+    /** Suggested fix */
+    suggestedFix: string;
+    /** Confidence (0-1) */
+    confidence: number;
+    /** Whether auto-fix is possible */
+    autoFixPossible: boolean;
+}
+/**
+ * Calculate flakiness score from test results
+ * @param results Array of test results
+ * @returns Flakiness score (0-100)
+ */
+export declare function calculateFlakinessScore(results: TestResult[]): number;
+/**
+ * Get flakiness category from score
+ * @param score Flakiness score (0-100)
+ * @returns Flakiness category
+ */
+export declare function getFlakinessCategory(score: number): FlakinessCategory;
+/**
+ * Generate a unique test ID
+ * @param testName Test name
+ * @param filePath File path
+ * @returns Unique test ID
+ */
+export declare function generateTestId(testName: string, filePath: string): string;
+/**
+ * Analyze test results for flakiness patterns
+ * @param results Array of test results for the same test
+ * @returns Pattern detection result or undefined
+ */
+export declare function detectFlakinessPattern(results: TestResult[]): PatternDetection | undefined;
+/**
+ * Calculate consecutive streak (passes or failures)
+ * @param results Array of test results sorted by timestamp
+ * @returns Current streak info
+ */
+export declare function calculateStreak(results: TestResult[]): {
+    currentStreak: number;
+    streakType: 'pass' | 'fail';
+};
+/**
+ * Format flakiness score for display
+ * @param score Flakiness score (0-100)
+ * @returns Formatted string with emoji
+ */
+export declare function formatFlakinessScore(score: number): string;
+/**
+ * Flakiness Detection Engine
+ *
+ * Main class for detecting and scoring test flakiness
+ */
+export declare class FlakinessDetector {
+    private options;
+    constructor(options?: FlakinessDetectionOptions);
+    /**
+     * Analyze a single test's flakiness from its run history
+     * @param testId Test identifier
+     * @param results Array of test results (must be from the same test)
+     * @returns Flakiness result
+     */
+    analyzeTest(testId: string, results: TestResult[]): FlakinessResult;
+    /**
+     * Analyze multiple tests and return flakiness results
+     * @param testResults Map of testId to array of results
+     * @returns Array of flakiness results
+     */
+    analyzeAll(testResults: Map<string, TestResult[]>): FlakinessResult[];
+    /**
+     * Check if a test should be quarantined based on flakiness
+     * @param result Flakiness result
+     * @returns Whether to quarantine
+     */
+    shouldQuarantine(result: FlakinessResult): boolean;
+    /**
+     * Get quarantine recommendation message
+     * @param result Flakiness result
+     * @returns Recommendation message
+     */
+    getQuarantineRecommendation(result: FlakinessResult): string;
+    /**
+     * Calculate average duration
+     */
+    private averageDuration;
+}