qa360 2.1.7 → 2.2.0

package/core/src/regression/__tests__/detector.test.ts

@@ -0,0 +1,396 @@
+/**
+ * Regression Detector Tests
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest';
+import { RegressionDetector, createRegressionDetector } from '../detector.js';
+import type { TimeSeriesPoint, RegressionConfig } from '../types.js';
+
+describe('RegressionDetector', () => {
+  let detector: RegressionDetector;
+
+  beforeEach(() => {
+    detector = createRegressionDetector();
+  });
+
+  describe('baseline management', () => {
+    it('should create baseline from time series data', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 1000, value: 100, runId: 'run1' },
+        { timestamp: 2000, value: 105, runId: 'run2' },
+        { timestamp: 3000, value: 95, runId: 'run3' },
+        { timestamp: 4000, value: 102, runId: 'run4' },
+        { timestamp: 5000, value: 98, runId: 'run5' }
+      ];
+
+      detector.updateBaseline('test_metric', data);
+
+      const baseline = detector.getBaseline('test_metric');
+
+      expect(baseline).toBeDefined();
+      expect(baseline!.mean).toBeCloseTo(100, 0);
+      expect(baseline!.sampleSize).toBe(5);
+      expect(baseline!.min).toBe(95);
+      expect(baseline!.max).toBe(105);
+    });
+
+    it('should calculate correct percentiles', () => {
+      const data: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 100; i++) {
+        data.push({
+          timestamp: i * 1000,
+          value: i,
+          runId: `run${i}`
+        });
+      }
+
+      detector.updateBaseline('metric', data);
+      const baseline = detector.getBaseline('metric');
+
+      // Using Math.floor for percentiles:
+      // p50 = sorted[ Math.floor(100 * 0.5) ] = sorted[50] = 50
+      // p90 = sorted[ Math.floor(100 * 0.9) ] = sorted[90] = 90
+      // p95 = sorted[ Math.floor(100 * 0.95) ] = sorted[95] = 95
+      expect(baseline!.percentiles.p50).toBe(50);
+      expect(baseline!.percentiles.p90).toBe(90);
+      expect(baseline!.percentiles.p95).toBe(95);
+    });
+
+    it('should handle empty data', () => {
+      detector.updateBaseline('empty_metric', []);
+      const baseline = detector.getBaseline('empty_metric');
+
+      expect(baseline).toBeUndefined();
+    });
+  });
+
+  describe('regression detection', () => {
+    it('should detect performance regression when value increases significantly', () => {
+      // Create detector with custom thresholds for testing
+      const testConfig: Partial<RegressionConfig> = {
+        thresholds: {
+          performance: [{
+            metric: 'api_latency_ms',
+            maxRegression: 10, // 10% threshold
+            minDelta: 5,
+            significanceLevel: 0.1,
+            direction: 'increase',
+            severity: 'moderate'
+          }],
+          quality: [],
+          coverage: [],
+          reliability: [],
+          security: []
+        },
+        ignorePatterns: []
+      };
+      const testDetector = createRegressionDetector(testConfig);
+
+      // Create baseline with stable latency around 100ms
+      const baselineData: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 20; i++) {
+        baselineData.push({
+          timestamp: i * 60000,
+          value: 100 + Math.random() * 10, // 90-110 range
+          runId: `run${i}`
+        });
+      }
+
+      testDetector.updateBaseline('api_latency_ms', baselineData);
+
+      // Current value significantly higher (50% increase)
+      const currentMetrics = {
+        api_latency_ms: 150 // 50% increase from baseline ~100
+      };
+
+      const regressions = testDetector.detectRegressions(
+        currentMetrics,
+        'run_current',
+        'api_smoke'
+      );
+
+      // Should detect regression due to 50% increase
+      expect(regressions.length).toBeGreaterThan(0);
+      const regression = regressions.find(r => r.metricName === 'api_latency_ms');
+      expect(regression).toBeDefined();
+      expect(regression!.direction).toBe('worse');
+      expect(regression!.type).toBe('performance');
+    });
+
+    it('should not flag minor fluctuations as regressions', () => {
+      // Create detector with threshold that won't trigger for small changes
+      const testConfig: Partial<RegressionConfig> = {
+        thresholds: {
+          performance: [{
+            metric: 'metric',
+            maxRegression: 20, // 20% threshold - won't trigger for 2% change
+            minDelta: 10, // Minimum 10 units
+            significanceLevel: 0.05,
+            direction: 'increase',
+            severity: 'minor'
+          }],
+          quality: [],
+          coverage: [],
+          reliability: [],
+          security: []
+        },
+        ignorePatterns: [],
+        minDataPoints: 5
+      };
+      const testDetector = createRegressionDetector(testConfig);
+
+      const baselineData: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 20; i++) {
+        baselineData.push({
+          timestamp: i * 60000,
+          value: 100 + Math.random() * 5,
+          runId: `run${i}`
+        });
+      }
+
+      testDetector.updateBaseline('metric', baselineData);
+
+      // Small change within normal variation
+      const currentMetrics = { metric: 102 };
+      const regressions = testDetector.detectRegressions(currentMetrics, 'run_current', 'test');
+
+      expect(regressions.length).toBe(0);
+    });
+
+    it('should detect coverage decrease regression', () => {
+      // Create detector with coverage threshold
+      const testConfig: Partial<RegressionConfig> = {
+        thresholds: {
+          performance: [],
+          quality: [],
+          coverage: [{
+            metric: 'line_coverage',
+            maxRegression: 5, // 5% threshold
+            minDelta: 1,
+            significanceLevel: 0.1,
+            direction: 'decrease',
+            severity: 'minor'
+          }],
+          reliability: [],
+          security: []
+        },
+        ignorePatterns: [],
+        minDataPoints: 5
+      };
+      const testDetector = createRegressionDetector(testConfig);
+
+      const baselineData: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 20; i++) {
+        baselineData.push({
+          timestamp: i * 60000,
+          value: 85 + Math.random() * 2, // 85-87% coverage
+          runId: `run${i}`
+        });
+      }
+
+      testDetector.updateBaseline('line_coverage', baselineData);
+
+      // Coverage dropped to 75%
+      const currentMetrics = { line_coverage: 75 };
+      const regressions = testDetector.detectRegressions(currentMetrics, 'run_current', 'ui');
+
+      expect(regressions.length).toBeGreaterThan(0);
+      const regression = regressions.find(r => r.metricName === 'line_coverage');
+      expect(regression!.type).toBe('coverage');
+    });
+
+    it('should ignore metrics matching ignore patterns', () => {
+      const config: Partial<RegressionConfig> = {
+        ignorePatterns: ['*_timestamp', 'build_*']
+      };
+      const customDetector = createRegressionDetector(config);
+
+      const baselineData: TimeSeriesPoint[] = [
+        { timestamp: 1000, value: 100, runId: 'run1' },
+        { timestamp: 2000, value: 105, runId: 'run2' }
+      ];
+
+      customDetector.updateBaseline('build_timestamp', baselineData);
+
+      const currentMetrics = {
+        build_timestamp: 1000000, // Large change but should be ignored
+        other_metric: 200
+      };
+
+      const regressions = customDetector.detectRegressions(currentMetrics, 'run_current', 'test');
+
+      // Should not detect regression for ignored metric
+      expect(regressions.find(r => r.metricName === 'build_timestamp')).toBeUndefined();
+    });
+  });
+
+  describe('statistical tests', () => {
+    it('should calculate z-score correctly', () => {
+      const baselineData: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 100; i++) {
+        baselineData.push({
+          timestamp: i * 1000,
+          value: 100, // Constant for zero std dev
+          runId: `run${i}`
+        });
+      }
+
+      detector.updateBaseline('constant_metric', baselineData);
+      const baseline = detector.getBaseline('constant_metric');
+
+      // With zero std dev, z-score should be undefined/handled
+      expect(baseline!.stdDev).toBe(0);
+    });
+
+    it('should detect regression with statistical significance', () => {
+      // Create detector with appropriate threshold
+      const testConfig: Partial<RegressionConfig> = {
+        thresholds: {
+          performance: [{
+            metric: 'duration_ms',
+            maxRegression: 10, // 10% threshold
+            minDelta: 10, // 10ms minimum
+            significanceLevel: 0.1,
+            direction: 'increase',
+            severity: 'moderate'
+          }],
+          quality: [],
+          coverage: [],
+          reliability: [],
+          security: []
+        },
+        ignorePatterns: [],
+        minDataPoints: 5
+      };
+      const testDetector = createRegressionDetector(testConfig);
+
+      // Baseline: mean=100, low variance
+      const baselineData: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 30; i++) {
+        baselineData.push({
+          timestamp: i * 60000,
+          value: 100 + (Math.random() - 0.5) * 2, // 99-101 range
+          runId: `run${i}`
+        });
+      }
+
+      testDetector.updateBaseline('duration_ms', baselineData);
+
+      // Current: 30% increase (well outside normal variation)
+      const currentMetrics = { duration_ms: 130 };
+      const regressions = testDetector.detectRegressions(currentMetrics, 'run_current', 'perf');
+
+      expect(regressions.length).toBeGreaterThan(0);
+
+      const regression = regressions[0];
+      expect(regression.confidence).toBeGreaterThan(0.95); // High confidence
+    });
+  });
+
+  describe('severity calculation', () => {
+    it('should classify critical regressions correctly', () => {
+      // Create detector with security threshold
+      const testConfig: Partial<RegressionConfig> = {
+        thresholds: {
+          performance: [],
+          quality: [],
+          coverage: [],
+          reliability: [],
+          security: [{
+            metric: 'security_vulnerabilities',
+            maxRegression: 0, // Any increase is critical
+            minDelta: 1,
+            significanceLevel: 0.1,
+            direction: 'increase',
+            severity: 'critical'
+          }]
+        },
+        ignorePatterns: [],
+        minDataPoints: 5
+      };
+      const testDetector = createRegressionDetector(testConfig);
+
+      const baselineData: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 20; i++) {
+        baselineData.push({
+          timestamp: i * 60000,
+          value: 100,
+          runId: `run${i}`
+        });
+      }
+
+      testDetector.updateBaseline('security_vulnerabilities', baselineData);
+
+      // Any increase in vulnerabilities should be detected
+      const currentMetrics = { security_vulnerabilities: 105 };
+      const regressions = testDetector.detectRegressions(currentMetrics, 'run_current', 'sast');
+
+      const regression = regressions.find(r => r.metricName === 'security_vulnerabilities');
+      expect(regression).toBeDefined();
+      expect(regression!.type).toBe('security');
+    });
+  });
+
+  describe('suggestions generation', () => {
+    it('should generate relevant suggestions for performance regressions', () => {
+      // Create detector with latency threshold
+      const testConfig: Partial<RegressionConfig> = {
+        thresholds: {
+          performance: [{
+            metric: 'api_latency_p95',
+            maxRegression: 10, // 10% threshold
+            minDelta: 10,
+            significanceLevel: 0.1,
+            direction: 'increase',
+            severity: 'moderate'
+          }],
+          quality: [],
+          coverage: [],
+          reliability: [],
+          security: []
+        },
+        ignorePatterns: [],
+        minDataPoints: 5
+      };
+      const testDetector = createRegressionDetector(testConfig);
+
+      const baselineData: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 10; i++) {
+        baselineData.push({
+          timestamp: i * 60000,
+          value: 100,
+          runId: `run${i}`
+        });
+      }
+
+      testDetector.updateBaseline('api_latency_p95', baselineData);
+
+      const currentMetrics = { api_latency_p95: 150 };
+      const regressions = testDetector.detectRegressions(currentMetrics, 'run_current', 'api');
+
+      expect(regressions.length).toBeGreaterThan(0);
+      const regression = regressions[0];
+      expect(regression.suggestions.length).toBeGreaterThan(0);
+      expect(regression.suggestions.some(s => s.includes('latency') || s.includes('bottleneck'))).toBe(true);
+    });
+  });
+});
+
+describe('createRegressionDetector', () => {
+  it('should create detector with default config', () => {
+    const detector = createRegressionDetector();
+
+    expect(detector).toBeInstanceOf(RegressionDetector);
+  });
+
+  it('should create detector with custom config', () => {
+    const config: Partial<RegressionConfig> = {
+      minDataPoints: 10,
+      method: 'ttest'
+    };
+
+    const detector = createRegressionDetector(config);
+
+    expect(detector).toBeInstanceOf(RegressionDetector);
+  });
+});

package/core/src/regression/__tests__/trend-analyzer.test.ts

@@ -0,0 +1,300 @@
+/**
+ * Regression Trend Analyzer Tests
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  RegressionTrendAnalyzer,
+  createRegressionTrendAnalyzer
+} from '../trend-analyzer.js';
+import type { TimeSeriesPoint } from '../types.js';
+
+describe('RegressionTrendAnalyzer', () => {
+  let analyzer: RegressionTrendAnalyzer;
+
+  beforeEach(() => {
+    analyzer = createRegressionTrendAnalyzer();
+  });
+
+  describe('linear regression', () => {
+    it('should calculate correct slope for increasing trend', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 100, runId: 'run1' },
+        { timestamp: 1000, value: 110, runId: 'run2' },
+        { timestamp: 2000, value: 120, runId: 'run3' },
+        { timestamp: 3000, value: 130, runId: 'run4' },
+        { timestamp: 4000, value: 140, runId: 'run5' }
+      ];
+
+      const result = analyzer.linearRegression(data);
+
+      expect(result.direction).toBe('increasing');
+      // Slope is 10 units per second (values increase by 10 every 1000ms)
+      expect(result.slope).toBeCloseTo(10, 1);
+      expect(result.intercept).toBeCloseTo(100, 5);
+      expect(result.rSquared).toBeGreaterThan(0.95); // Strong linear relationship
+    });
+
+    it('should calculate correct slope for decreasing trend', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 140, runId: 'run1' },
+        { timestamp: 1000, value: 130, runId: 'run2' },
+        { timestamp: 2000, value: 120, runId: 'run3' },
+        { timestamp: 3000, value: 110, runId: 'run4' },
+        { timestamp: 4000, value: 100, runId: 'run5' }
+      ];
+
+      const result = analyzer.linearRegression(data);
+
+      expect(result.direction).toBe('decreasing');
+      // Slope is -10 units per second (values decrease by 10 every 1000ms)
+      expect(result.slope).toBeCloseTo(-10, 1);
+    });
+
+    it('should detect stable trend', () => {
+      const data: TimeSeriesPoint[] = [];
+      for (let i = 0; i < 10; i++) {
+        data.push({
+          timestamp: i * 1000,
+          value: 100 + (Math.random() - 0.5) * 2, // Small noise
+          runId: `run${i}`
+        });
+      }
+
+      const result = analyzer.linearRegression(data);
+
+      // With only 2 units of variation over 9 seconds, slope is very small
+      expect(Math.abs(result.slope)).toBeLessThan(1);
+      // Direction might be stable, increasing, or decreasing depending on randomness
+      // Just check slope is close to 0
+    });
+
+    it('should handle single data point', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 100, runId: 'run1' }
+      ];
+
+      const result = analyzer.linearRegression(data);
+
+      expect(result.direction).toBe('stable');
+      expect(result.slope).toBe(0);
+    });
+  });
+
+  describe('trend analysis', () => {
+    it('should analyze trend correctly', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 100, runId: 'run1' },
+        { timestamp: 1000, value: 102, runId: 'run2' },
+        { timestamp: 2000, value: 104, runId: 'run3' },
+        { timestamp: 3000, value: 106, runId: 'run4' },
+        { timestamp: 4000, value: 108, runId: 'run5' }
+      ];
+
+      const trend = analyzer.analyzeTrend(data);
+
+      expect(trend).toBeDefined();
+      expect(trend!.direction).toBe('degrading'); // Increasing for latency metrics is bad
+      expect(trend!.dataPoints).toBe(5);
+      expect(trend!.strength).toBeGreaterThan(0.9); // Strong correlation
+    });
+
+    it('should return null for insufficient data', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 100, runId: 'run1' }
+      ];
+
+      const trend = analyzer.analyzeTrend(data);
+
+      expect(trend).toBeNull();
+    });
+  });
+
+  describe('change point detection', () => {
+    it('should detect change point in data', () => {
+      const data: TimeSeriesPoint[] = [];
+
+      // First half: values around 100
+      for (let i = 0; i < 10; i++) {
+        data.push({
+          timestamp: i * 1000,
+          value: 100 + Math.random() * 5,
+          runId: `run${i}`
+        });
+      }
+
+      // Second half: values around 150 (clear change)
+      for (let i = 10; i < 20; i++) {
+        data.push({
+          timestamp: i * 1000,
+          value: 150 + Math.random() * 5,
+          runId: `run${i}`
+        });
+      }
+
+      const changePoints = analyzer.detectChangePoints(data, 5);
+
+      expect(changePoints.length).toBeGreaterThan(0);
+
+      const cp = changePoints[0];
+      expect(cp!.index).toBeCloseTo(10, 2);
+      expect(cp!.magnitude).toBeGreaterThan(40); // ~50 unit change
+      expect(cp!.afterMean).toBeGreaterThan(cp!.beforeMean);
+    });
+
+    it('should return empty array for too small dataset', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 100, runId: 'run1' },
+        { timestamp: 1000, value: 105, runId: 'run2' }
+      ];
+
+      const changePoints = analyzer.detectChangePoints(data, 5);
+
+      expect(changePoints).toEqual([]);
+    });
+  });
+
+  describe('period comparison', () => {
+    it('should detect regression when comparing periods', () => {
+      const baseline: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 100, runId: 'run1' },
+        { timestamp: 1000, value: 102, runId: 'run2' },
+        { timestamp: 2000, value: 98, runId: 'run3' },
+        { timestamp: 3000, value: 101, runId: 'run4' },
+        { timestamp: 4000, value: 99, runId: 'run5' }
+      ];
+
+      const current: TimeSeriesPoint[] = [
+        { timestamp: 5000, value: 120, runId: 'run6' },
+        { timestamp: 6000, value: 125, runId: 'run7' },
+        { timestamp: 7000, value: 118, runId: 'run8' },
+        { timestamp: 8000, value: 122, runId: 'run9' },
+        { timestamp: 9000, value: 123, runId: 'run10' }
+      ];
+
+      const regression = analyzer.comparePeriods(baseline, current, 'api_latency');
+
+      expect(regression).toBeDefined();
+      expect(regression!.direction).toBe('worse');
+      expect(regression!.percentChange).toBeGreaterThan(15); // ~20% increase
+      expect(regression!.type).toBe('performance');
+    });
+
+    it('should detect improvement when comparing periods', () => {
+      const baseline: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 120, runId: 'run1' },
+        { timestamp: 1000, value: 125, runId: 'run2' },
+        { timestamp: 2000, value: 118, runId: 'run3' }
+      ];
+
+      const current: TimeSeriesPoint[] = [
+        { timestamp: 3000, value: 100, runId: 'run4' },
+        { timestamp: 4000, value: 98, runId: 'run5' },
+        { timestamp: 5000, value: 102, runId: 'run6' }
+      ];
+
+      const regression = analyzer.comparePeriods(baseline, current, 'api_latency');
+
+      // For performance, lower is better, so this should NOT be a regression
+      expect(regression).toBeNull();
+    });
+  });
+
+  describe('time to threshold calculation', () => {
+    it('should calculate time to threshold for degrading metric', () => {
+      const data: TimeSeriesPoint[] = [];
+      // Linearly increasing with a steeper slope for significance
+      // Using hourly timestamps to get a larger slope
+      for (let i = 0; i < 10; i++) {
+        data.push({
+          timestamp: i * 3600000, // Hourly in ms
+          value: 100 + i * 5, // 5 units per hour increase
+          runId: `run${i}`
+        });
+      }
+
+      const timeToThreshold = analyzer.calculateTimeToThreshold(
+        data,
+        150, // Threshold
+        true  // Increasing is bad
+      );
+
+      expect(timeToThreshold).toBeDefined();
+      expect(typeof timeToThreshold).toBe('number');
+      expect(timeToThreshold!).toBeGreaterThan(0);
+
+      // Last value is 100 + 9*5 = 145, need 5 more to reach 150
+      // With slope of ~5 units per hour, need ~1 hour
+      expect(timeToThreshold!).toBeGreaterThan(0);
+      expect(timeToThreshold!).toBeLessThan(5 * 3600000); // Less than 5 hours
+    });
+
+    it('should return null if not moving toward threshold', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 100, runId: 'run1' },
+        { timestamp: 1000, value: 99, runId: 'run2' },
+        { timestamp: 2000, value: 98, runId: 'run3' }
+      ];
+
+      const timeToThreshold = analyzer.calculateTimeToThreshold(
+        data,
+        150, // Threshold above current values
+        true  // Increasing is bad
+      );
+
+      // Trend is decreasing, not moving toward threshold
+      expect(timeToThreshold).toBeNull();
+    });
+
+    it('should return 0 if already at threshold', () => {
+      const data: TimeSeriesPoint[] = [
+        { timestamp: 0, value: 150, runId: 'run1' },
+        { timestamp: 1000, value: 155, runId: 'run2' },
+        { timestamp: 2000, value: 160, runId: 'run3' }
+      ];
+
+      const timeToThreshold = analyzer.calculateTimeToThreshold(
+        data,
+        150,
+        true  // Increasing is bad, and we're already at threshold
+      );
+
+      // Current value is 160, which is past threshold of 150
+      // So diff = 150 - 160 = -10 <= 0, should return 0
+      expect(timeToThreshold).toBe(0);
+    });
+  });
+
+  describe('utility functions', () => {
+    it('should format time to threshold correctly', () => {
+      expect(analyzer.formatTimeToThreshold(-1000)).toBe('Already breached');
+      expect(analyzer.formatTimeToThreshold(0)).toBe('At threshold');
+      expect(analyzer.formatTimeToThreshold(3600000)).toBe('1 hour');
+      expect(analyzer.formatTimeToThreshold(86400000)).toBe('1 day');
+      expect(analyzer.formatTimeToThreshold(172800000)).toBe('2 days');
+    });
+
+    it('should describe trends correctly', () => {
+      const trend = {
+        metric: 'test_metric',
+        direction: 'degrading' as const,
+        strength: 0.85,
+        dataPoints: 10,
+        regressions: 0
+      };
+
+      const description = analyzer.getTrendDescription(trend);
+
+      expect(description).toContain('Degrading');
+      expect(description).toContain('85%');
+    });
+  });
+});
+
+describe('createRegressionTrendAnalyzer', () => {
+  it('should create trend analyzer', () => {
+    const analyzer = createRegressionTrendAnalyzer();
+
+    expect(analyzer).toBeInstanceOf(RegressionTrendAnalyzer);
+  });
+});