qa360 2.1.7 → 2.2.0

package/core/src/regression/detector.ts

@@ -0,0 +1,629 @@
+/**
+ * Regression Detector
+ *
+ * Detects regressions using statistical methods.
+ * Supports Z-score, T-test, Mann-Whitney U, and percentile-based detection.
+ */
+
+import type {
+  RegressionDetection,
+  RegressionType,
+  RegressionSeverity,
+  RegressionConfig,
+  RegressionThreshold,
+  TimeSeriesPoint,
+  Baseline,
+  StatisticalTest
+} from './types.js';
+
+/**
+ * Regression Detector class
+ */
+export class RegressionDetector {
+  private config: RegressionConfig;
+  private baselines: Map<string, Baseline> = new Map();
+
+  constructor(config?: Partial<RegressionConfig>) {
+    this.config = {
+      ...this.createDefaultConfig(),
+      ...config
+    };
+  }
+
+  /**
+   * Detect regressions by comparing current values to baseline
+   */
+  detectRegressions(
+    currentData: Record<string, number>,
+    currentRunId: string,
+    gate: string,
+    context?: Record<string, unknown>
+  ): RegressionDetection[] {
+    const regressions: RegressionDetection[] = [];
+
+    for (const [metricName, currentValue] of Object.entries(currentData)) {
+      if (this.shouldIgnore(metricName)) {
+        continue;
+      }
+
+      const baseline = this.baselines.get(metricName);
+      if (!baseline || baseline.sampleSize < this.config.minDataPoints) {
+        continue;
+      }
+
+      const regression = this.detectMetricRegression(
+        metricName,
+        currentValue,
+        baseline,
+        currentRunId,
+        gate,
+        context
+      );
+
+      if (regression) {
+        regressions.push(regression);
+      }
+    }
+
+    return regressions;
+  }
+
+  /**
+   * Detect regression for a single metric
+   */
+  private detectMetricRegression(
+    metricName: string,
+    currentValue: number,
+    baseline: Baseline,
+    runId: string,
+    gate: string,
+    context?: Record<string, unknown>
+  ): RegressionDetection | null {
+    const threshold = this.getThresholdForMetric(metricName);
+    if (!threshold) {
+      return null;
+    }
+
+    // Calculate statistical significance
+    const test = this.performStatisticalTest(currentValue, baseline);
+
+    // Calculate change
+    const absoluteChange = currentValue - baseline.mean;
+    const percentChange = baseline.mean !== 0
+      ? (absoluteChange / baseline.mean) * 100
+      : 0;
+
+    // Determine direction
+    const direction = this.getDIRECTION(metricName, absoluteChange, threshold);
+
+    // Check if regression detected
+    const isRegression = this.isRegression(
+      currentValue,
+      baseline,
+      threshold,
+      test,
+      direction
+    );
+
+    if (!isRegression) {
+      return null;
+    }
+
+    // Determine severity
+    const severity = this.calculateSeverity(
+      absoluteChange,
+      percentChange,
+      threshold,
+      test.pValue
+    );
+
+    // Generate suggestions
+    const suggestions = this.generateSuggestions(
+      metricName,
+      direction,
+      severity,
+      currentValue,
+      baseline.mean
+    );
+
+    return {
+      id: `regression_${Date.now()}_${metricName.replace(/\W/g, '_')}`,
+      runId,
+      baselineRunId: baseline.runIds[baseline.runIds.length - 1],
+      type: this.getTypeForMetric(metricName),
+      severity,
+      status: 'detected',
+      metricName,
+      baselineValue: baseline.mean,
+      currentValue,
+      absoluteChange,
+      percentChange,
+      significance: test.pValue,
+      confidence: 1 - test.pValue,
+      direction,
+      affectedComponent: this.extractComponent(metricName, context),
+      gate,
+      context,
+      suggestions,
+      timestamp: Date.now()
+    };
+  }
+
+  /**
+   * Perform statistical test
+   */
+  private performStatisticalTest(
+    currentValue: number,
+    baseline: Baseline
+  ): StatisticalTest {
+    let test: StatisticalTest;
+
+    switch (this.config.method) {
+      case 'zscore':
+        test = this.zScoreTest(currentValue, baseline);
+        break;
+      case 'ttest':
+        test = this.tTest(currentValue, baseline);
+        break;
+      case 'percentile':
+        test = this.percentileTest(currentValue, baseline);
+        break;
+      case 'mannwhitney':
+      default:
+        test = this.zScoreTest(currentValue, baseline); // Default to z-score for single value
+    }
+
+    return test;
+  }
+
+  /**
+   * Z-score test
+   */
+  private zScoreTest(value: number, baseline: Baseline): StatisticalTest {
+    if (baseline.stdDev === 0) {
+      // When baseline has no variance, any change is highly significant
+      if (value !== baseline.mean) {
+        return {
+          test: 'zscore',
+          pValue: 0.0001, // Very small p-value for high significance
+          significant: true,
+          statistic: value > baseline.mean ? Infinity : -Infinity
+        };
+      }
+      return {
+        test: 'zscore',
+        pValue: 1,
+        significant: false,
+        statistic: 0
+      };
+    }
+
+    const zScore = (value - baseline.mean) / baseline.stdDev;
+    // Two-tailed p-value
+    const pValue = 2 * (1 - this.normalCDF(Math.abs(zScore)));
+
+    return {
+      test: 'zscore',
+      pValue,
+      significant: pValue < 0.05,
+      statistic: zScore,
+      confidenceInterval: {
+        lower: baseline.mean - 1.96 * baseline.stdDev,
+        upper: baseline.mean + 1.96 * baseline.stdDev
+      }
+    };
+  }
+
+  /**
+   * T-test (simplified for single value vs baseline)
+   */
+  private tTest(value: number, baseline: Baseline): StatisticalTest {
+    // For single value, use z-score approximation with n-1 degrees of freedom
+    const df = baseline.sampleSize - 1;
+    const tScore = (value - baseline.mean) / (baseline.stdDev / Math.sqrt(baseline.sampleSize));
+
+    // Approximate p-value (simplified)
+    const pValue = 2 * (1 - this.studentTCDF(Math.abs(tScore), df));
+
+    return {
+      test: 'ttest',
+      pValue,
+      significant: pValue < 0.05,
+      statistic: tScore
+    };
+  }
+
+  /**
+   * Percentile test
+   */
+  private percentileTest(value: number, baseline: Baseline): StatisticalTest {
+    const percentile = this.calculatePercentile(value, baseline);
+
+    // P-value based on percentile (extreme values have low p-value)
+    const pValue = percentile > 95
+      ? (100 - percentile) / 100
+      : percentile < 5
+        ? percentile / 100
+        : 0.5;
+
+    return {
+      test: 'percentile',
+      pValue,
+      significant: pValue < 0.05,
+      statistic: percentile
+    };
+  }
+
+  /**
+   * Calculate percentile for a value in baseline
+   */
+  private calculatePercentile(value: number, baseline: Baseline): number {
+    if (value < baseline.min) return 0;
+    if (value > baseline.max) return 100;
+
+    // Estimate percentile using mean and stdDev (assuming normal distribution)
+    const zScore = (value - baseline.mean) / baseline.stdDev;
+    return this.normalCDF(zScore) * 100;
+  }
+
+  /**
+   * Normal CDF (cumulative distribution function)
+   */
+  private normalCDF(x: number): number {
+    // Approximation of normal CDF
+    const a1 = 0.254829592;
+    const a2 = -0.284496736;
+    const a3 = 1.421413741;
+    const a4 = -1.453152027;
+    const a5 = 1.061405429;
+    const p = 0.3275911;
+
+    const sign = x < 0 ? -1 : 1;
+    x = Math.abs(x) / Math.sqrt(2);
+
+    const t = 1.0 / (1.0 + p * x);
+    const y = 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
+
+    return 0.5 * (1.0 + sign * y);
+  }
+
+  /**
+   * Student's t CDF approximation
+   */
+  private studentTCDF(t: number, df: number): number {
+    // Simplified approximation for large df
+    if (df > 30) {
+      return this.normalCDF(t);
+    }
+    // For small df, use approximation
+    return this.normalCDF(t * 0.9); // Conservative approximation
+  }
+
+  /**
+   * Check if change constitutes a regression
+   */
+  private isRegression(
+    currentValue: number,
+    baseline: Baseline,
+    threshold: RegressionThreshold,
+    test: StatisticalTest,
+    direction: 'worse' | 'better' | 'neutral'
+  ): boolean {
+    // Must be statistically significant
+    if (!test.significant && test.pValue > threshold.significanceLevel) {
+      return false;
+    }
+
+    // Must exceed minimum delta
+    const absChange = Math.abs(currentValue - baseline.mean);
+    if (absChange < threshold.minDelta) {
+      return false;
+    }
+
+    // Must be in the "worse" direction
+    if (direction !== 'worse') {
+      return false;
+    }
+
+    // For percentage-based thresholds
+    const percentChange = baseline.mean !== 0
+      ? (absChange / baseline.mean) * 100
+      : 0;
+
+    return percentChange > threshold.maxRegression;
+  }
+
+  /**
+   * Get direction of change (worse/better/neutral)
+   */
+  private getDIRECTION(
+    metricName: string,
+    change: number,
+    threshold: RegressionThreshold
+  ): 'worse' | 'better' | 'neutral' {
+    // For 'increase' direction metrics (like latency): higher value is worse
+    // For 'decrease' direction metrics (like coverage): lower value is worse
+    const isWorse = (threshold.direction === 'increase' && change > 0) ||
+                    (threshold.direction === 'decrease' && change < 0);
+    const isBetter = (threshold.direction === 'increase' && change < 0) ||
+                     (threshold.direction === 'decrease' && change > 0);
+
+    if (isWorse) {
+      return 'worse';
+    }
+    if (isBetter) {
+      return 'better';
+    }
+    return 'neutral';
+  }
+
+  /**
+   * Calculate regression severity
+   */
+  private calculateSeverity(
+    absoluteChange: number,
+    percentChange: number,
+    threshold: RegressionThreshold,
+    pValue: number
+  ): RegressionSeverity {
+    // Critical: extreme regression or very high significance
+    if (percentChange > threshold.maxRegression * 5 || pValue < 0.001) {
+      return 'critical';
+    }
+
+    // Major: severe regression
+    if (percentChange > threshold.maxRegression * 3 || pValue < 0.01) {
+      return 'major';
+    }
+
+    // Moderate: notable regression
+    if (percentChange > threshold.maxRegression * 2 || pValue < 0.05) {
+      return 'moderate';
+    }
+
+    // Minor: slight regression
+    return 'minor';
+  }
+
+  /**
+   * Generate suggestions for regression
+   */
+  private generateSuggestions(
+    metricName: string,
+    direction: string,
+    severity: RegressionSeverity,
+    currentValue: number,
+    baselineValue: number
+  ): string[] {
+    const suggestions: string[] = [];
+
+    if (direction === 'worse') {
+      suggestions.push(`Investigate recent changes to ${metricName}`);
+      suggestions.push(`Compare with baseline: ${baselineValue.toFixed(2)} → ${currentValue.toFixed(2)}`);
+
+      if (severity === 'critical' || severity === 'major') {
+        suggestions.push('Consider rolling back recent changes');
+        suggestions.push('Review test environment for external factors');
+      }
+
+      if (metricName.includes('duration') || metricName.includes('latency')) {
+        suggestions.push('Check for infinite loops or blocking operations');
+        suggestions.push('Profile the code to identify bottlenecks');
+      }
+
+      if (metricName.includes('memory') || metricName.includes('heap')) {
+        suggestions.push('Check for memory leaks');
+        suggestions.push('Review allocation patterns');
+      }
+    }
+
+    return suggestions;
+  }
+
+  /**
+   * Update baseline with new data
+   */
+  updateBaseline(metricName: string, data: TimeSeriesPoint[]): void {
+    if (data.length === 0) return;
+
+    const values = data.map(d => d.value);
+    const sorted = [...values].sort((a, b) => a - b);
+
+    const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
+    const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length;
+    const stdDev = Math.sqrt(variance);
+
+    this.baselines.set(metricName, {
+      metric: metricName,
+      mean,
+      stdDev,
+      min: sorted[0],
+      max: sorted[sorted.length - 1],
+      percentiles: {
+        p50: sorted[Math.floor(sorted.length * 0.5)],
+        p90: sorted[Math.floor(sorted.length * 0.9)],
+        p95: sorted[Math.floor(sorted.length * 0.95)],
+        p99: sorted[Math.floor(sorted.length * 0.99)]
+      },
+      sampleSize: values.length,
+      runIds: data.map(d => d.runId),
+      timestamp: Date.now()
+    });
+  }
+
+  /**
+   * Get baseline for a metric
+   */
+  getBaseline(metricName: string): Baseline | undefined {
+    return this.baselines.get(metricName);
+  }
+
+  /**
+   * Get all baselines
+   */
+  getAllBaselines(): Baseline[] {
+    return Array.from(this.baselines.values());
+  }
+
+  /**
+   * Get threshold for a metric
+   */
+  private getThresholdForMetric(metricName: string): RegressionThreshold | undefined {
+    // Check all threshold categories
+    for (const thresholds of Object.values(this.config.thresholds)) {
+      const match = thresholds.find(t => this.metricMatchesPattern(metricName, t.metric));
+      if (match) return match;
+    }
+    return undefined;
+  }
+
+  /**
+   * Check if metric matches pattern
+   */
+  private metricMatchesPattern(metricName: string, pattern: string): boolean {
+    if (pattern === '*') return true;
+    if (pattern.endsWith('*')) {
+      const prefix = pattern.slice(0, -1);
+      return metricName.startsWith(prefix);
+    }
+    return metricName === pattern;
+  }
+
+  /**
+   * Get regression type for metric
+   */
+  private getTypeForMetric(metricName: string): RegressionType {
+    if (metricName.includes('duration') || metricName.includes('latency') || metricName.includes('time')) {
+      return 'performance';
+    }
+    if (metricName.includes('coverage')) {
+      return 'coverage';
+    }
+    if (metricName.includes('flaky') || metricName.includes('stability')) {
+      return 'reliability';
+    }
+    if (metricName.includes('vulnerability') || metricName.includes('security')) {
+      return 'security';
+    }
+    if (metricName.includes('api') || metricName.includes('endpoint')) {
+      return 'api_breaking';
+    }
+    return 'quality';
+  }
+
+  /**
+   * Extract component name from metric and context
+   */
+  private extractComponent(metricName: string, context?: Record<string, unknown>): string {
+    if (context?.file) return String(context.file);
+    if (context?.test) return String(context.test);
+    if (context?.endpoint) return String(context.endpoint);
+
+    // Extract from metric name
+    const parts = metricName.split(/[._/]/);
+    return parts[0] || metricName;
+  }
+
+  /**
+   * Check if metric should be ignored
+   */
+  private shouldIgnore(metricName: string): boolean {
+    return this.config.ignorePatterns.some(pattern =>
+      this.metricMatchesPattern(metricName, pattern)
+    );
+  }
+
+  /**
+   * Create default configuration
+   */
+  private createDefaultConfig(): RegressionConfig {
+    return {
+      enabled: true,
+      autoDetect: true,
+      baselineWindow: 30,
+      minDataPoints: 5,
+      method: 'zscore',
+      thresholds: {
+        performance: [
+          {
+            metric: '*_duration',
+            maxRegression: 20, // 20% slower
+            minDelta: 100, // 100ms minimum change
+            significanceLevel: 0.05,
+            direction: 'increase',
+            severity: 'moderate'
+          },
+          {
+            metric: '*_latency',
+            maxRegression: 15,
+            minDelta: 50,
+            significanceLevel: 0.05,
+            direction: 'increase',
+            severity: 'moderate'
+          }
+        ],
+        quality: [
+          {
+            metric: 'pass_rate',
+            maxRegression: 5, // 5% decrease
+            minDelta: 1,
+            significanceLevel: 0.05,
+            direction: 'decrease',
+            severity: 'major'
+          },
+          {
+            metric: 'failure_count',
+            maxRegression: 10, // 10% increase
+            minDelta: 1,
+            significanceLevel: 0.05,
+            direction: 'increase',
+            severity: 'moderate'
+          }
+        ],
+        coverage: [
+          {
+            metric: 'line_coverage',
+            maxRegression: 2, // 2% decrease
+            minDelta: 0.5,
+            significanceLevel: 0.1,
+            direction: 'decrease',
+            severity: 'minor'
+          }
+        ],
+        reliability: [
+          {
+            metric: 'flakiness_rate',
+            maxRegression: 5, // 5% increase
+            minDelta: 1,
+            significanceLevel: 0.05,
+            direction: 'increase',
+            severity: 'major'
+          }
+        ],
+        security: [
+          {
+            metric: 'vulnerability_count',
+            maxRegression: 0, // Any increase is critical
+            minDelta: 1,
+            significanceLevel: 0.01,
+            direction: 'increase',
+            severity: 'critical'
+          }
+        ]
+      },
+      alerts: {
+        enabled: true,
+        onNewRegression: true,
+        onSeverity: ['major', 'critical']
+      },
+      ignorePatterns: ['*_timestamp', '*_version', 'build_*']
+    };
+  }
+}
+
+/**
+ * Create a regression detector
+ */
+export function createRegressionDetector(config?: Partial<RegressionConfig>): RegressionDetector {
+  return new RegressionDetector(config);
+}

package/core/src/regression/index.ts

@@ -0,0 +1,34 @@
+/**
+ * QA360 Regression Detection Module
+ *
+ * Exports all regression detection functionality.
+ */
+
+// Types
+export type {
+  RegressionType,
+  RegressionSeverity,
+  RegressionStatus,
+  RegressionDetection,
+  RegressionThreshold,
+  RegressionConfig,
+  TimeSeriesPoint,
+  StatisticalTest,
+  RegressionTrend,
+  RegressionReport,
+  Baseline
+} from './types.js';
+
+// Detector
+export { RegressionDetector, createRegressionDetector } from './detector.js';
+
+// Trend Analyzer
+export {
+  RegressionTrendAnalyzer,
+  createRegressionTrendAnalyzer,
+  type TrendAnalysis,
+  type ChangePoint
+} from './trend-analyzer.js';
+
+// Vault integration
+export { RegressionVault } from './vault.js';