qa360 2.2.1 → 2.2.13

package/core/src/regression/detector.ts

@@ -1,629 +0,0 @@
-/**
- * Regression Detector
- *
- * Detects regressions using statistical methods.
- * Supports Z-score, T-test, Mann-Whitney U, and percentile-based detection.
- */
-
-import type {
-  RegressionDetection,
-  RegressionType,
-  RegressionSeverity,
-  RegressionConfig,
-  RegressionThreshold,
-  TimeSeriesPoint,
-  Baseline,
-  StatisticalTest
-} from './types.js';
-
-/**
- * Regression Detector class
- */
-export class RegressionDetector {
-  private config: RegressionConfig;
-  private baselines: Map<string, Baseline> = new Map();
-
-  constructor(config?: Partial<RegressionConfig>) {
-    this.config = {
-      ...this.createDefaultConfig(),
-      ...config
-    };
-  }
-
-  /**
-   * Detect regressions by comparing current values to baseline
-   */
-  detectRegressions(
-    currentData: Record<string, number>,
-    currentRunId: string,
-    gate: string,
-    context?: Record<string, unknown>
-  ): RegressionDetection[] {
-    const regressions: RegressionDetection[] = [];
-
-    for (const [metricName, currentValue] of Object.entries(currentData)) {
-      if (this.shouldIgnore(metricName)) {
-        continue;
-      }
-
-      const baseline = this.baselines.get(metricName);
-      if (!baseline || baseline.sampleSize < this.config.minDataPoints) {
-        continue;
-      }
-
-      const regression = this.detectMetricRegression(
-        metricName,
-        currentValue,
-        baseline,
-        currentRunId,
-        gate,
-        context
-      );
-
-      if (regression) {
-        regressions.push(regression);
-      }
-    }
-
-    return regressions;
-  }
-
-  /**
-   * Detect regression for a single metric
-   */
-  private detectMetricRegression(
-    metricName: string,
-    currentValue: number,
-    baseline: Baseline,
-    runId: string,
-    gate: string,
-    context?: Record<string, unknown>
-  ): RegressionDetection | null {
-    const threshold = this.getThresholdForMetric(metricName);
-    if (!threshold) {
-      return null;
-    }
-
-    // Calculate statistical significance
-    const test = this.performStatisticalTest(currentValue, baseline);
-
-    // Calculate change
-    const absoluteChange = currentValue - baseline.mean;
-    const percentChange = baseline.mean !== 0
-      ? (absoluteChange / baseline.mean) * 100
-      : 0;
-
-    // Determine direction
-    const direction = this.getDIRECTION(metricName, absoluteChange, threshold);
-
-    // Check if regression detected
-    const isRegression = this.isRegression(
-      currentValue,
-      baseline,
-      threshold,
-      test,
-      direction
-    );
-
-    if (!isRegression) {
-      return null;
-    }
-
-    // Determine severity
-    const severity = this.calculateSeverity(
-      absoluteChange,
-      percentChange,
-      threshold,
-      test.pValue
-    );
-
-    // Generate suggestions
-    const suggestions = this.generateSuggestions(
-      metricName,
-      direction,
-      severity,
-      currentValue,
-      baseline.mean
-    );
-
-    return {
-      id: `regression_${Date.now()}_${metricName.replace(/\W/g, '_')}`,
-      runId,
-      baselineRunId: baseline.runIds[baseline.runIds.length - 1],
-      type: this.getTypeForMetric(metricName),
-      severity,
-      status: 'detected',
-      metricName,
-      baselineValue: baseline.mean,
-      currentValue,
-      absoluteChange,
-      percentChange,
-      significance: test.pValue,
-      confidence: 1 - test.pValue,
-      direction,
-      affectedComponent: this.extractComponent(metricName, context),
-      gate,
-      context,
-      suggestions,
-      timestamp: Date.now()
-    };
-  }
-
-  /**
-   * Perform statistical test
-   */
-  private performStatisticalTest(
-    currentValue: number,
-    baseline: Baseline
-  ): StatisticalTest {
-    let test: StatisticalTest;
-
-    switch (this.config.method) {
-      case 'zscore':
-        test = this.zScoreTest(currentValue, baseline);
-        break;
-      case 'ttest':
-        test = this.tTest(currentValue, baseline);
-        break;
-      case 'percentile':
-        test = this.percentileTest(currentValue, baseline);
-        break;
-      case 'mannwhitney':
-      default:
-        test = this.zScoreTest(currentValue, baseline); // Default to z-score for single value
-    }
-
-    return test;
-  }
-
-  /**
-   * Z-score test
-   */
-  private zScoreTest(value: number, baseline: Baseline): StatisticalTest {
-    if (baseline.stdDev === 0) {
-      // When baseline has no variance, any change is highly significant
-      if (value !== baseline.mean) {
-        return {
-          test: 'zscore',
-          pValue: 0.0001, // Very small p-value for high significance
-          significant: true,
-          statistic: value > baseline.mean ? Infinity : -Infinity
-        };
-      }
-      return {
-        test: 'zscore',
-        pValue: 1,
-        significant: false,
-        statistic: 0
-      };
-    }
-
-    const zScore = (value - baseline.mean) / baseline.stdDev;
-    // Two-tailed p-value
-    const pValue = 2 * (1 - this.normalCDF(Math.abs(zScore)));
-
-    return {
-      test: 'zscore',
-      pValue,
-      significant: pValue < 0.05,
-      statistic: zScore,
-      confidenceInterval: {
-        lower: baseline.mean - 1.96 * baseline.stdDev,
-        upper: baseline.mean + 1.96 * baseline.stdDev
-      }
-    };
-  }
-
-  /**
-   * T-test (simplified for single value vs baseline)
-   */
-  private tTest(value: number, baseline: Baseline): StatisticalTest {
-    // For single value, use z-score approximation with n-1 degrees of freedom
-    const df = baseline.sampleSize - 1;
-    const tScore = (value - baseline.mean) / (baseline.stdDev / Math.sqrt(baseline.sampleSize));
-
-    // Approximate p-value (simplified)
-    const pValue = 2 * (1 - this.studentTCDF(Math.abs(tScore), df));
-
-    return {
-      test: 'ttest',
-      pValue,
-      significant: pValue < 0.05,
-      statistic: tScore
-    };
-  }
-
-  /**
-   * Percentile test
-   */
-  private percentileTest(value: number, baseline: Baseline): StatisticalTest {
-    const percentile = this.calculatePercentile(value, baseline);
-
-    // P-value based on percentile (extreme values have low p-value)
-    const pValue = percentile > 95
-      ? (100 - percentile) / 100
-      : percentile < 5
-        ? percentile / 100
-        : 0.5;
-
-    return {
-      test: 'percentile',
-      pValue,
-      significant: pValue < 0.05,
-      statistic: percentile
-    };
-  }
-
-  /**
-   * Calculate percentile for a value in baseline
-   */
-  private calculatePercentile(value: number, baseline: Baseline): number {
-    if (value < baseline.min) return 0;
-    if (value > baseline.max) return 100;
-
-    // Estimate percentile using mean and stdDev (assuming normal distribution)
-    const zScore = (value - baseline.mean) / baseline.stdDev;
-    return this.normalCDF(zScore) * 100;
-  }
-
-  /**
-   * Normal CDF (cumulative distribution function)
-   */
-  private normalCDF(x: number): number {
-    // Approximation of normal CDF
-    const a1 = 0.254829592;
-    const a2 = -0.284496736;
-    const a3 = 1.421413741;
-    const a4 = -1.453152027;
-    const a5 = 1.061405429;
-    const p = 0.3275911;
-
-    const sign = x < 0 ? -1 : 1;
-    x = Math.abs(x) / Math.sqrt(2);
-
-    const t = 1.0 / (1.0 + p * x);
-    const y = 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
-
-    return 0.5 * (1.0 + sign * y);
-  }
-
-  /**
-   * Student's t CDF approximation
-   */
-  private studentTCDF(t: number, df: number): number {
-    // Simplified approximation for large df
-    if (df > 30) {
-      return this.normalCDF(t);
-    }
-    // For small df, use approximation
-    return this.normalCDF(t * 0.9); // Conservative approximation
-  }
-
-  /**
-   * Check if change constitutes a regression
-   */
-  private isRegression(
-    currentValue: number,
-    baseline: Baseline,
-    threshold: RegressionThreshold,
-    test: StatisticalTest,
-    direction: 'worse' | 'better' | 'neutral'
-  ): boolean {
-    // Must be statistically significant
-    if (!test.significant && test.pValue > threshold.significanceLevel) {
-      return false;
-    }
-
-    // Must exceed minimum delta
-    const absChange = Math.abs(currentValue - baseline.mean);
-    if (absChange < threshold.minDelta) {
-      return false;
-    }
-
-    // Must be in the "worse" direction
-    if (direction !== 'worse') {
-      return false;
-    }
-
-    // For percentage-based thresholds
-    const percentChange = baseline.mean !== 0
-      ? (absChange / baseline.mean) * 100
-      : 0;
-
-    return percentChange > threshold.maxRegression;
-  }
-
-  /**
-   * Get direction of change (worse/better/neutral)
-   */
-  private getDIRECTION(
-    metricName: string,
-    change: number,
-    threshold: RegressionThreshold
-  ): 'worse' | 'better' | 'neutral' {
-    // For 'increase' direction metrics (like latency): higher value is worse
-    // For 'decrease' direction metrics (like coverage): lower value is worse
-    const isWorse = (threshold.direction === 'increase' && change > 0) ||
-                    (threshold.direction === 'decrease' && change < 0);
-    const isBetter = (threshold.direction === 'increase' && change < 0) ||
-                     (threshold.direction === 'decrease' && change > 0);
-
-    if (isWorse) {
-      return 'worse';
-    }
-    if (isBetter) {
-      return 'better';
-    }
-    return 'neutral';
-  }
-
-  /**
-   * Calculate regression severity
-   */
-  private calculateSeverity(
-    absoluteChange: number,
-    percentChange: number,
-    threshold: RegressionThreshold,
-    pValue: number
-  ): RegressionSeverity {
-    // Critical: extreme regression or very high significance
-    if (percentChange > threshold.maxRegression * 5 || pValue < 0.001) {
-      return 'critical';
-    }
-
-    // Major: severe regression
-    if (percentChange > threshold.maxRegression * 3 || pValue < 0.01) {
-      return 'major';
-    }
-
-    // Moderate: notable regression
-    if (percentChange > threshold.maxRegression * 2 || pValue < 0.05) {
-      return 'moderate';
-    }
-
-    // Minor: slight regression
-    return 'minor';
-  }
-
-  /**
-   * Generate suggestions for regression
-   */
-  private generateSuggestions(
-    metricName: string,
-    direction: string,
-    severity: RegressionSeverity,
-    currentValue: number,
-    baselineValue: number
-  ): string[] {
-    const suggestions: string[] = [];
-
-    if (direction === 'worse') {
-      suggestions.push(`Investigate recent changes to ${metricName}`);
-      suggestions.push(`Compare with baseline: ${baselineValue.toFixed(2)} → ${currentValue.toFixed(2)}`);
-
-      if (severity === 'critical' || severity === 'major') {
-        suggestions.push('Consider rolling back recent changes');
-        suggestions.push('Review test environment for external factors');
-      }
-
-      if (metricName.includes('duration') || metricName.includes('latency')) {
-        suggestions.push('Check for infinite loops or blocking operations');
-        suggestions.push('Profile the code to identify bottlenecks');
-      }
-
-      if (metricName.includes('memory') || metricName.includes('heap')) {
-        suggestions.push('Check for memory leaks');
-        suggestions.push('Review allocation patterns');
-      }
-    }
-
-    return suggestions;
-  }
-
-  /**
-   * Update baseline with new data
-   */
-  updateBaseline(metricName: string, data: TimeSeriesPoint[]): void {
-    if (data.length === 0) return;
-
-    const values = data.map(d => d.value);
-    const sorted = [...values].sort((a, b) => a - b);
-
-    const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
-    const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length;
-    const stdDev = Math.sqrt(variance);
-
-    this.baselines.set(metricName, {
-      metric: metricName,
-      mean,
-      stdDev,
-      min: sorted[0],
-      max: sorted[sorted.length - 1],
-      percentiles: {
-        p50: sorted[Math.floor(sorted.length * 0.5)],
-        p90: sorted[Math.floor(sorted.length * 0.9)],
-        p95: sorted[Math.floor(sorted.length * 0.95)],
-        p99: sorted[Math.floor(sorted.length * 0.99)]
-      },
-      sampleSize: values.length,
-      runIds: data.map(d => d.runId),
-      timestamp: Date.now()
-    });
-  }
-
-  /**
-   * Get baseline for a metric
-   */
-  getBaseline(metricName: string): Baseline | undefined {
-    return this.baselines.get(metricName);
-  }
-
-  /**
-   * Get all baselines
-   */
-  getAllBaselines(): Baseline[] {
-    return Array.from(this.baselines.values());
-  }
-
-  /**
-   * Get threshold for a metric
-   */
-  private getThresholdForMetric(metricName: string): RegressionThreshold | undefined {
-    // Check all threshold categories
-    for (const thresholds of Object.values(this.config.thresholds)) {
-      const match = thresholds.find(t => this.metricMatchesPattern(metricName, t.metric));
-      if (match) return match;
-    }
-    return undefined;
-  }
-
-  /**
-   * Check if metric matches pattern
-   */
-  private metricMatchesPattern(metricName: string, pattern: string): boolean {
-    if (pattern === '*') return true;
-    if (pattern.endsWith('*')) {
-      const prefix = pattern.slice(0, -1);
-      return metricName.startsWith(prefix);
-    }
-    return metricName === pattern;
-  }
-
-  /**
-   * Get regression type for metric
-   */
-  private getTypeForMetric(metricName: string): RegressionType {
-    if (metricName.includes('duration') || metricName.includes('latency') || metricName.includes('time')) {
-      return 'performance';
-    }
-    if (metricName.includes('coverage')) {
-      return 'coverage';
-    }
-    if (metricName.includes('flaky') || metricName.includes('stability')) {
-      return 'reliability';
-    }
-    if (metricName.includes('vulnerability') || metricName.includes('security')) {
-      return 'security';
-    }
-    if (metricName.includes('api') || metricName.includes('endpoint')) {
-      return 'api_breaking';
-    }
-    return 'quality';
-  }
-
-  /**
-   * Extract component name from metric and context
-   */
-  private extractComponent(metricName: string, context?: Record<string, unknown>): string {
-    if (context?.file) return String(context.file);
-    if (context?.test) return String(context.test);
-    if (context?.endpoint) return String(context.endpoint);
-
-    // Extract from metric name
-    const parts = metricName.split(/[._/]/);
-    return parts[0] || metricName;
-  }
-
-  /**
-   * Check if metric should be ignored
-   */
-  private shouldIgnore(metricName: string): boolean {
-    return this.config.ignorePatterns.some(pattern =>
-      this.metricMatchesPattern(metricName, pattern)
-    );
-  }
-
-  /**
-   * Create default configuration
-   */
-  private createDefaultConfig(): RegressionConfig {
-    return {
-      enabled: true,
-      autoDetect: true,
-      baselineWindow: 30,
-      minDataPoints: 5,
-      method: 'zscore',
-      thresholds: {
-        performance: [
-          {
-            metric: '*_duration',
-            maxRegression: 20, // 20% slower
-            minDelta: 100, // 100ms minimum change
-            significanceLevel: 0.05,
-            direction: 'increase',
-            severity: 'moderate'
-          },
-          {
-            metric: '*_latency',
-            maxRegression: 15,
-            minDelta: 50,
-            significanceLevel: 0.05,
-            direction: 'increase',
-            severity: 'moderate'
-          }
-        ],
-        quality: [
-          {
-            metric: 'pass_rate',
-            maxRegression: 5, // 5% decrease
-            minDelta: 1,
-            significanceLevel: 0.05,
-            direction: 'decrease',
-            severity: 'major'
-          },
-          {
-            metric: 'failure_count',
-            maxRegression: 10, // 10% increase
-            minDelta: 1,
-            significanceLevel: 0.05,
-            direction: 'increase',
-            severity: 'moderate'
-          }
-        ],
-        coverage: [
-          {
-            metric: 'line_coverage',
-            maxRegression: 2, // 2% decrease
-            minDelta: 0.5,
-            significanceLevel: 0.1,
-            direction: 'decrease',
-            severity: 'minor'
-          }
-        ],
-        reliability: [
-          {
-            metric: 'flakiness_rate',
-            maxRegression: 5, // 5% increase
-            minDelta: 1,
-            significanceLevel: 0.05,
-            direction: 'increase',
-            severity: 'major'
-          }
-        ],
-        security: [
-          {
-            metric: 'vulnerability_count',
-            maxRegression: 0, // Any increase is critical
-            minDelta: 1,
-            significanceLevel: 0.01,
-            direction: 'increase',
-            severity: 'critical'
-          }
-        ]
-      },
-      alerts: {
-        enabled: true,
-        onNewRegression: true,
-        onSeverity: ['major', 'critical']
-      },
-      ignorePatterns: ['*_timestamp', '*_version', 'build_*']
-    };
-  }
-}
-
-/**
- * Create a regression detector
- */
-export function createRegressionDetector(config?: Partial<RegressionConfig>): RegressionDetector {
-  return new RegressionDetector(config);
-}

package/core/src/regression/index.ts

@@ -1,34 +0,0 @@
-/**
- * QA360 Regression Detection Module
- *
- * Exports all regression detection functionality.
- */
-
-// Types
-export type {
-  RegressionType,
-  RegressionSeverity,
-  RegressionStatus,
-  RegressionDetection,
-  RegressionThreshold,
-  RegressionConfig,
-  TimeSeriesPoint,
-  StatisticalTest,
-  RegressionTrend,
-  RegressionReport,
-  Baseline
-} from './types.js';
-
-// Detector
-export { RegressionDetector, createRegressionDetector } from './detector.js';
-
-// Trend Analyzer
-export {
-  RegressionTrendAnalyzer,
-  createRegressionTrendAnalyzer,
-  type TrendAnalysis,
-  type ChangePoint
-} from './trend-analyzer.js';
-
-// Vault integration
-export { RegressionVault } from './vault.js';