qa360 2.2.1 → 2.2.13

package/cli/dist/core/regression/detector.js

@@ -0,0 +1,497 @@
+/**
+ * Regression Detector
+ *
+ * Detects regressions using statistical methods.
+ * Supports Z-score, T-test, Mann-Whitney U, and percentile-based detection.
+ */
+/**
+ * Regression Detector class
+ */
+export class RegressionDetector {
+    config;
+    baselines = new Map();
+    constructor(config) {
+        this.config = {
+            ...this.createDefaultConfig(),
+            ...config
+        };
+    }
+    /**
+     * Detect regressions by comparing current values to baseline
+     */
+    detectRegressions(currentData, currentRunId, gate, context) {
+        const regressions = [];
+        for (const [metricName, currentValue] of Object.entries(currentData)) {
+            if (this.shouldIgnore(metricName)) {
+                continue;
+            }
+            const baseline = this.baselines.get(metricName);
+            if (!baseline || baseline.sampleSize < this.config.minDataPoints) {
+                continue;
+            }
+            const regression = this.detectMetricRegression(metricName, currentValue, baseline, currentRunId, gate, context);
+            if (regression) {
+                regressions.push(regression);
+            }
+        }
+        return regressions;
+    }
+    /**
+     * Detect regression for a single metric
+     */
+    detectMetricRegression(metricName, currentValue, baseline, runId, gate, context) {
+        const threshold = this.getThresholdForMetric(metricName);
+        if (!threshold) {
+            return null;
+        }
+        // Calculate statistical significance
+        const test = this.performStatisticalTest(currentValue, baseline);
+        // Calculate change
+        const absoluteChange = currentValue - baseline.mean;
+        const percentChange = baseline.mean !== 0
+            ? (absoluteChange / baseline.mean) * 100
+            : 0;
+        // Determine direction
+        const direction = this.getDIRECTION(metricName, absoluteChange, threshold);
+        // Check if regression detected
+        const isRegression = this.isRegression(currentValue, baseline, threshold, test, direction);
+        if (!isRegression) {
+            return null;
+        }
+        // Determine severity
+        const severity = this.calculateSeverity(absoluteChange, percentChange, threshold, test.pValue);
+        // Generate suggestions
+        const suggestions = this.generateSuggestions(metricName, direction, severity, currentValue, baseline.mean);
+        return {
+            id: `regression_${Date.now()}_${metricName.replace(/\W/g, '_')}`,
+            runId,
+            baselineRunId: baseline.runIds[baseline.runIds.length - 1],
+            type: this.getTypeForMetric(metricName),
+            severity,
+            status: 'detected',
+            metricName,
+            baselineValue: baseline.mean,
+            currentValue,
+            absoluteChange,
+            percentChange,
+            significance: test.pValue,
+            confidence: 1 - test.pValue,
+            direction,
+            affectedComponent: this.extractComponent(metricName, context),
+            gate,
+            context,
+            suggestions,
+            timestamp: Date.now()
+        };
+    }
+    /**
+     * Perform statistical test
+     */
+    performStatisticalTest(currentValue, baseline) {
+        let test;
+        switch (this.config.method) {
+            case 'zscore':
+                test = this.zScoreTest(currentValue, baseline);
+                break;
+            case 'ttest':
+                test = this.tTest(currentValue, baseline);
+                break;
+            case 'percentile':
+                test = this.percentileTest(currentValue, baseline);
+                break;
+            case 'mannwhitney':
+            default:
+                test = this.zScoreTest(currentValue, baseline); // Default to z-score for single value
+        }
+        return test;
+    }
+    /**
+     * Z-score test
+     */
+    zScoreTest(value, baseline) {
+        if (baseline.stdDev === 0) {
+            // When baseline has no variance, any change is highly significant
+            if (value !== baseline.mean) {
+                return {
+                    test: 'zscore',
+                    pValue: 0.0001, // Very small p-value for high significance
+                    significant: true,
+                    statistic: value > baseline.mean ? Infinity : -Infinity
+                };
+            }
+            return {
+                test: 'zscore',
+                pValue: 1,
+                significant: false,
+                statistic: 0
+            };
+        }
+        const zScore = (value - baseline.mean) / baseline.stdDev;
+        // Two-tailed p-value
+        const pValue = 2 * (1 - this.normalCDF(Math.abs(zScore)));
+        return {
+            test: 'zscore',
+            pValue,
+            significant: pValue < 0.05,
+            statistic: zScore,
+            confidenceInterval: {
+                lower: baseline.mean - 1.96 * baseline.stdDev,
+                upper: baseline.mean + 1.96 * baseline.stdDev
+            }
+        };
+    }
+    /**
+     * T-test (simplified for single value vs baseline)
+     */
+    tTest(value, baseline) {
+        // For single value, use z-score approximation with n-1 degrees of freedom
+        const df = baseline.sampleSize - 1;
+        const tScore = (value - baseline.mean) / (baseline.stdDev / Math.sqrt(baseline.sampleSize));
+        // Approximate p-value (simplified)
+        const pValue = 2 * (1 - this.studentTCDF(Math.abs(tScore), df));
+        return {
+            test: 'ttest',
+            pValue,
+            significant: pValue < 0.05,
+            statistic: tScore
+        };
+    }
+    /**
+     * Percentile test
+     */
+    percentileTest(value, baseline) {
+        const percentile = this.calculatePercentile(value, baseline);
+        // P-value based on percentile (extreme values have low p-value)
+        const pValue = percentile > 95
+            ? (100 - percentile) / 100
+            : percentile < 5
+                ? percentile / 100
+                : 0.5;
+        return {
+            test: 'percentile',
+            pValue,
+            significant: pValue < 0.05,
+            statistic: percentile
+        };
+    }
+    /**
+     * Calculate percentile for a value in baseline
+     */
+    calculatePercentile(value, baseline) {
+        if (value < baseline.min)
+            return 0;
+        if (value > baseline.max)
+            return 100;
+        // Estimate percentile using mean and stdDev (assuming normal distribution)
+        const zScore = (value - baseline.mean) / baseline.stdDev;
+        return this.normalCDF(zScore) * 100;
+    }
+    /**
+     * Normal CDF (cumulative distribution function)
+     */
+    normalCDF(x) {
+        // Approximation of normal CDF
+        const a1 = 0.254829592;
+        const a2 = -0.284496736;
+        const a3 = 1.421413741;
+        const a4 = -1.453152027;
+        const a5 = 1.061405429;
+        const p = 0.3275911;
+        const sign = x < 0 ? -1 : 1;
+        x = Math.abs(x) / Math.sqrt(2);
+        const t = 1.0 / (1.0 + p * x);
+        const y = 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
+        return 0.5 * (1.0 + sign * y);
+    }
+    /**
+     * Student's t CDF approximation
+     */
+    studentTCDF(t, df) {
+        // Simplified approximation for large df
+        if (df > 30) {
+            return this.normalCDF(t);
+        }
+        // For small df, use approximation
+        return this.normalCDF(t * 0.9); // Conservative approximation
+    }
+    /**
+     * Check if change constitutes a regression
+     */
+    isRegression(currentValue, baseline, threshold, test, direction) {
+        // Must be statistically significant
+        if (!test.significant && test.pValue > threshold.significanceLevel) {
+            return false;
+        }
+        // Must exceed minimum delta
+        const absChange = Math.abs(currentValue - baseline.mean);
+        if (absChange < threshold.minDelta) {
+            return false;
+        }
+        // Must be in the "worse" direction
+        if (direction !== 'worse') {
+            return false;
+        }
+        // For percentage-based thresholds
+        const percentChange = baseline.mean !== 0
+            ? (absChange / baseline.mean) * 100
+            : 0;
+        return percentChange > threshold.maxRegression;
+    }
+    /**
+     * Get direction of change (worse/better/neutral)
+     */
+    getDIRECTION(metricName, change, threshold) {
+        // For 'increase' direction metrics (like latency): higher value is worse
+        // For 'decrease' direction metrics (like coverage): lower value is worse
+        const isWorse = (threshold.direction === 'increase' && change > 0) ||
+            (threshold.direction === 'decrease' && change < 0);
+        const isBetter = (threshold.direction === 'increase' && change < 0) ||
+            (threshold.direction === 'decrease' && change > 0);
+        if (isWorse) {
+            return 'worse';
+        }
+        if (isBetter) {
+            return 'better';
+        }
+        return 'neutral';
+    }
+    /**
+     * Calculate regression severity
+     */
+    calculateSeverity(absoluteChange, percentChange, threshold, pValue) {
+        // Critical: extreme regression or very high significance
+        if (percentChange > threshold.maxRegression * 5 || pValue < 0.001) {
+            return 'critical';
+        }
+        // Major: severe regression
+        if (percentChange > threshold.maxRegression * 3 || pValue < 0.01) {
+            return 'major';
+        }
+        // Moderate: notable regression
+        if (percentChange > threshold.maxRegression * 2 || pValue < 0.05) {
+            return 'moderate';
+        }
+        // Minor: slight regression
+        return 'minor';
+    }
+    /**
+     * Generate suggestions for regression
+     */
+    generateSuggestions(metricName, direction, severity, currentValue, baselineValue) {
+        const suggestions = [];
+        if (direction === 'worse') {
+            suggestions.push(`Investigate recent changes to ${metricName}`);
+            suggestions.push(`Compare with baseline: ${baselineValue.toFixed(2)} → ${currentValue.toFixed(2)}`);
+            if (severity === 'critical' || severity === 'major') {
+                suggestions.push('Consider rolling back recent changes');
+                suggestions.push('Review test environment for external factors');
+            }
+            if (metricName.includes('duration') || metricName.includes('latency')) {
+                suggestions.push('Check for infinite loops or blocking operations');
+                suggestions.push('Profile the code to identify bottlenecks');
+            }
+            if (metricName.includes('memory') || metricName.includes('heap')) {
+                suggestions.push('Check for memory leaks');
+                suggestions.push('Review allocation patterns');
+            }
+        }
+        return suggestions;
+    }
+    /**
+     * Update baseline with new data
+     */
+    updateBaseline(metricName, data) {
+        if (data.length === 0)
+            return;
+        const values = data.map(d => d.value);
+        const sorted = [...values].sort((a, b) => a - b);
+        const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
+        const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length;
+        const stdDev = Math.sqrt(variance);
+        this.baselines.set(metricName, {
+            metric: metricName,
+            mean,
+            stdDev,
+            min: sorted[0],
+            max: sorted[sorted.length - 1],
+            percentiles: {
+                p50: sorted[Math.floor(sorted.length * 0.5)],
+                p90: sorted[Math.floor(sorted.length * 0.9)],
+                p95: sorted[Math.floor(sorted.length * 0.95)],
+                p99: sorted[Math.floor(sorted.length * 0.99)]
+            },
+            sampleSize: values.length,
+            runIds: data.map(d => d.runId),
+            timestamp: Date.now()
+        });
+    }
+    /**
+     * Get baseline for a metric
+     */
+    getBaseline(metricName) {
+        return this.baselines.get(metricName);
+    }
+    /**
+     * Get all baselines
+     */
+    getAllBaselines() {
+        return Array.from(this.baselines.values());
+    }
+    /**
+     * Get threshold for a metric
+     */
+    getThresholdForMetric(metricName) {
+        // Check all threshold categories
+        for (const thresholds of Object.values(this.config.thresholds)) {
+            const match = thresholds.find(t => this.metricMatchesPattern(metricName, t.metric));
+            if (match)
+                return match;
+        }
+        return undefined;
+    }
+    /**
+     * Check if metric matches pattern
+     */
+    metricMatchesPattern(metricName, pattern) {
+        if (pattern === '*')
+            return true;
+        if (pattern.endsWith('*')) {
+            const prefix = pattern.slice(0, -1);
+            return metricName.startsWith(prefix);
+        }
+        return metricName === pattern;
+    }
+    /**
+     * Get regression type for metric
+     */
+    getTypeForMetric(metricName) {
+        if (metricName.includes('duration') || metricName.includes('latency') || metricName.includes('time')) {
+            return 'performance';
+        }
+        if (metricName.includes('coverage')) {
+            return 'coverage';
+        }
+        if (metricName.includes('flaky') || metricName.includes('stability')) {
+            return 'reliability';
+        }
+        if (metricName.includes('vulnerability') || metricName.includes('security')) {
+            return 'security';
+        }
+        if (metricName.includes('api') || metricName.includes('endpoint')) {
+            return 'api_breaking';
+        }
+        return 'quality';
+    }
+    /**
+     * Extract component name from metric and context
+     */
+    extractComponent(metricName, context) {
+        if (context?.file)
+            return String(context.file);
+        if (context?.test)
+            return String(context.test);
+        if (context?.endpoint)
+            return String(context.endpoint);
+        // Extract from metric name
+        const parts = metricName.split(/[._/]/);
+        return parts[0] || metricName;
+    }
+    /**
+     * Check if metric should be ignored
+     */
+    shouldIgnore(metricName) {
+        return this.config.ignorePatterns.some(pattern => this.metricMatchesPattern(metricName, pattern));
+    }
+    /**
+     * Create default configuration
+     */
+    createDefaultConfig() {
+        return {
+            enabled: true,
+            autoDetect: true,
+            baselineWindow: 30,
+            minDataPoints: 5,
+            method: 'zscore',
+            thresholds: {
+                performance: [
+                    {
+                        metric: '*_duration',
+                        maxRegression: 20, // 20% slower
+                        minDelta: 100, // 100ms minimum change
+                        significanceLevel: 0.05,
+                        direction: 'increase',
+                        severity: 'moderate'
+                    },
+                    {
+                        metric: '*_latency',
+                        maxRegression: 15,
+                        minDelta: 50,
+                        significanceLevel: 0.05,
+                        direction: 'increase',
+                        severity: 'moderate'
+                    }
+                ],
+                quality: [
+                    {
+                        metric: 'pass_rate',
+                        maxRegression: 5, // 5% decrease
+                        minDelta: 1,
+                        significanceLevel: 0.05,
+                        direction: 'decrease',
+                        severity: 'major'
+                    },
+                    {
+                        metric: 'failure_count',
+                        maxRegression: 10, // 10% increase
+                        minDelta: 1,
+                        significanceLevel: 0.05,
+                        direction: 'increase',
+                        severity: 'moderate'
+                    }
+                ],
+                coverage: [
+                    {
+                        metric: 'line_coverage',
+                        maxRegression: 2, // 2% decrease
+                        minDelta: 0.5,
+                        significanceLevel: 0.1,
+                        direction: 'decrease',
+                        severity: 'minor'
+                    }
+                ],
+                reliability: [
+                    {
+                        metric: 'flakiness_rate',
+                        maxRegression: 5, // 5% increase
+                        minDelta: 1,
+                        significanceLevel: 0.05,
+                        direction: 'increase',
+                        severity: 'major'
+                    }
+                ],
+                security: [
+                    {
+                        metric: 'vulnerability_count',
+                        maxRegression: 0, // Any increase is critical
+                        minDelta: 1,
+                        significanceLevel: 0.01,
+                        direction: 'increase',
+                        severity: 'critical'
+                    }
+                ]
+            },
+            alerts: {
+                enabled: true,
+                onNewRegression: true,
+                onSeverity: ['major', 'critical']
+            },
+            ignorePatterns: ['*_timestamp', '*_version', 'build_*']
+        };
+    }
+}
+/**
+ * Create a regression detector
+ */
+export function createRegressionDetector(config) {
+    return new RegressionDetector(config);
+}

package/cli/dist/core/regression/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * QA360 Regression Detection Module
+ *
+ * Exports all regression detection functionality.
+ */
+export type { RegressionType, RegressionSeverity, RegressionStatus, RegressionDetection, RegressionThreshold, RegressionConfig, TimeSeriesPoint, StatisticalTest, RegressionTrend, RegressionReport, Baseline } from './types.js';
+export { RegressionDetector, createRegressionDetector } from './detector.js';
+export { RegressionTrendAnalyzer, createRegressionTrendAnalyzer, type TrendAnalysis, type ChangePoint } from './trend-analyzer.js';
+export { RegressionVault } from './vault.js';

package/cli/dist/core/regression/index.js

@@ -0,0 +1,11 @@
+/**
+ * QA360 Regression Detection Module
+ *
+ * Exports all regression detection functionality.
+ */
+// Detector
+export { RegressionDetector, createRegressionDetector } from './detector.js';
+// Trend Analyzer
+export { RegressionTrendAnalyzer, createRegressionTrendAnalyzer } from './trend-analyzer.js';
+// Vault integration
+export { RegressionVault } from './vault.js';

package/cli/dist/core/regression/trend-analyzer.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Regression Trend Analyzer
+ *
+ * Analyzes metric trends to predict potential regressions.
+ * Uses linear regression and change point detection.
+ */
+import type { TimeSeriesPoint, RegressionTrend, RegressionDetection } from './types.js';
+/**
+ * Trend analysis result
+ */
+export interface TrendAnalysis {
+    /** Slope of trend */
+    slope: number;
+    /** Intercept */
+    intercept: number;
+    /** Correlation coefficient (R²) */
+    rSquared: number;
+    /** Is trend significant? */
+    significant: boolean;
+    /** Direction */
+    direction: 'increasing' | 'decreasing' | 'stable';
+    /** Projected value at future timestamp */
+    project(ms: number): number;
+}
+/**
+ * Change point detection result
+ */
+export interface ChangePoint {
+    /** Index of change point */
+    index: number;
+    /** Timestamp of change */
+    timestamp: number;
+    /** Mean before change */
+    beforeMean: number;
+    /** Mean after change */
+    afterMean: number;
+    /** Magnitude of change */
+    magnitude: number;
+    /** Statistical significance */
+    pValue: number;
+}
+/**
+ * Regression Trend Analyzer class
+ */
+export declare class RegressionTrendAnalyzer {
+    /**
+     * Analyze trend for a metric
+     */
+    analyzeTrend(data: TimeSeriesPoint[], windowMs?: number): RegressionTrend | null;
+    /**
+     * Perform linear regression on time series
+     */
+    linearRegression(data: TimeSeriesPoint[]): TrendAnalysis;
+    /**
+     * Detect change points in time series
+     */
+    detectChangePoints(data: TimeSeriesPoint[], minSegmentSize?: number): ChangePoint[];
+    /**
+     * Estimate p-value for change point
+     */
+    private estimatePValue;
+    /**
+     * Project trend forward
+     */
+    private projectTrend;
+    /**
+     * Get trend direction for reporting
+     */
+    private getTrendDirection;
+    /**
+     * Calculate time until threshold breach
+     */
+    calculateTimeToThreshold(data: TimeSeriesPoint[], threshold: number, isIncreasingBad: boolean): number | null;
+    /**
+     * Compare two periods for regression
+     */
+    comparePeriods(baseline: TimeSeriesPoint[], current: TimeSeriesPoint[], metric: string): RegressionDetection | null;
+    /**
+     * Infer regression type from metric name
+     */
+    private inferType;
+    /**
+     * Check if change direction indicates regression
+     */
+    private isRegressionDirection;
+    /**
+     * Normal CDF approximation
+     */
+    private normalCDF;
+    /**
+     * Get trend description
+     */
+    getTrendDescription(trend: RegressionTrend): string;
+    /**
+     * Format time to threshold
+     */
+    formatTimeToThreshold(ms: number): string;
+}
+/**
+ * Create a regression trend analyzer
+ */
+export declare function createRegressionTrendAnalyzer(): RegressionTrendAnalyzer;