qa360 2.1.7 → 2.2.0

package/cli/src/commands/regression.ts

@@ -0,0 +1,400 @@
+/**
+ * QA360 Regression Command
+ *
+ * CLI command for regression detection and analysis.
+ */
+
+import { Command } from 'commander';
+import {
+  RegressionDetector,
+  createRegressionDetector,
+  RegressionTrendAnalyzer,
+  createRegressionTrendAnalyzer,
+  type TimeSeriesPoint,
+  type RegressionDetection,
+  type RegressionSeverity
+} from 'qa360-core';
+
+export const regressionCommand = new Command('regression');
+
+regressionCommand.description('Regression detection and analysis');
+
+// Detect regressions
+regressionCommand
+  .command('detect')
+  .description('Detect regressions by comparing current metrics to baseline')
+  .option('--baseline <file>', 'Baseline metrics JSON file')
+  .option('--current <file>', 'Current metrics JSON file')
+  .option('--run-id <id>', 'Current run ID')
+  .option('--gate <name>', 'Gate name')
+  .option('--json', 'Output as JSON')
+  .action(async (options) => {
+    const detector = createRegressionDetector();
+
+    // Demo: Create baseline from simulated data
+    const baselineData: TimeSeriesPoint[] = [];
+    for (let i = 0; i < 30; i++) {
+      baselineData.push({
+        timestamp: Date.now() - (30 - i) * 3600000,
+        value: 100 + Math.random() * 10,
+        runId: `run_${i}`
+      });
+    }
+
+    detector.updateBaseline('api_latency_p95', baselineData);
+    detector.updateBaseline('pass_rate', baselineData.map(d => ({
+      ...d,
+      value: 95 + Math.random() * 3
+    })));
+    detector.updateBaseline('line_coverage', baselineData.map(d => ({
+      ...d,
+      value: 80 + Math.random() * 2
+    })));
+
+    // Simulate current metrics with one regression
+    const currentMetrics = {
+      api_latency_p95: 145, // 45% increase - regression!
+      pass_rate: 94,
+      line_coverage: 79.5
+    };
+
+    const regressions = detector.detectRegressions(
+      currentMetrics,
+      options.runId || 'current',
+      options.gate || 'api_smoke'
+    );
+
+    if (options.json) {
+      console.log(JSON.stringify(regressions, null, 2));
+      return;
+    }
+
+    console.log('\n🔍 Regression Detection Results\n');
+    console.log(`Run ID: ${options.runId || 'current'}`);
+    console.log(`Gate: ${options.gate || 'api_smoke'}\n`);
+
+    if (regressions.length === 0) {
+      console.log('✅ No regressions detected!\n');
+      return;
+    }
+
+    console.log(`Detected ${regressions.length} regression(s):\n`);
+
+    for (const reg of regressions) {
+      const icon = getSeverityIcon(reg.severity);
+      const arrow = reg.direction === 'worse' ? '↑' : '↓';
+      const changeStr = reg.percentChange > 0 ? '+' : '';
+
+      console.log(`  ${icon} ${reg.metricName}`);
+      console.log(`      Type: ${reg.type}`);
+      console.log(`      Severity: ${reg.severity.toUpperCase()}`);
+      console.log(`      Baseline: ${reg.baselineValue.toFixed(2)} → Current: ${reg.currentValue.toFixed(2)}`);
+      console.log(`      Change: ${arrow} ${changeStr}${reg.percentChange.toFixed(1)}% (${reg.absoluteChange.toFixed(2)})`);
+      console.log(`      Confidence: ${(reg.confidence * 100).toFixed(1)}%`);
+      console.log(`      Component: ${reg.affectedComponent}`);
+      console.log(`      Suggestions:`);
+      for (const suggestion of reg.suggestions) {
+        console.log(`        - ${suggestion}`);
+      }
+      console.log('');
+    }
+  });
+
+// Analyze trends
+regressionCommand
+  .command('trend')
+  .description('Analyze metric trends for potential future regressions')
+  .option('--metric <name>', 'Metric name to analyze')
+  .option('--points <n>', 'Number of data points to use', '30')
+  .option('--json', 'Output as JSON')
+  .action(async (options) => {
+    const analyzer = createRegressionTrendAnalyzer();
+    const metricName = options.metric || 'api_latency_p95';
+
+    // Generate demo trend data
+    const data: TimeSeriesPoint[] = [];
+    for (let i = 0; i < parseInt(options.points); i++) {
+      // Slight upward trend (degrading)
+      const trend = i * 0.5;
+      const noise = (Math.random() - 0.5) * 5;
+      data.push({
+        timestamp: Date.now() - (parseInt(options.points) - i) * 3600000,
+        value: 100 + trend + noise,
+        runId: `run_${i}`,
+        metadata: { metric: metricName }
+      });
+    }
+
+    const trend = analyzer.analyzeTrend(data);
+
+    if (options.json) {
+      console.log(JSON.stringify(trend, null, 2));
+      return;
+    }
+
+    console.log('\n📈 Trend Analysis\n');
+    console.log(`Metric: ${metricName}`);
+    console.log(`Data Points: ${data.length}\n`);
+
+    if (!trend) {
+      console.log('⚠️  Insufficient data for trend analysis\n');
+      return;
+    }
+
+    const directionIcon = trend.direction === 'degrading' ? '📉' : trend.direction === 'improving' ? '📈' : '➡️';
+    const strengthPercent = Math.round(trend.strength * 100);
+
+    console.log(`  ${directionIcon} Direction: ${trend.direction.toUpperCase()}`);
+    console.log(`  Strength: ${strengthPercent}% confidence`);
+
+    if (trend.projected !== undefined) {
+      console.log(`  Projected (7 days): ${trend.projected.toFixed(2)}`);
+    }
+
+    console.log('');
+  });
+
+// List recent regressions
+regressionCommand
+  .command('list')
+  .description('List recent regressions')
+  .option('--severity <level>', 'Filter by severity')
+  .option('--type <type>', 'Filter by type')
+  .option('--limit <n>', 'Max results', '20')
+  .option('--json', 'Output as JSON')
+  .action(async (options) => {
+    // Demo: Generate some regressions
+    const regressions: RegressionDetection[] = [
+      {
+        id: 'reg_1',
+        runId: 'run_123',
+        baselineRunId: 'run_100',
+        type: 'performance',
+        severity: 'major',
+        status: 'detected',
+        metricName: 'api_latency_p95',
+        baselineValue: 100,
+        currentValue: 145,
+        absoluteChange: 45,
+        percentChange: 45,
+        significance: 0.001,
+        confidence: 0.999,
+        direction: 'worse',
+        affectedComponent: '/api/users',
+        gate: 'api_smoke',
+        suggestions: ['Investigate recent API changes', 'Profile the endpoint'],
+        timestamp: Date.now() - 3600000
+      },
+      {
+        id: 'reg_2',
+        runId: 'run_123',
+        baselineRunId: 'run_100',
+        type: 'coverage',
+        severity: 'minor',
+        status: 'detected',
+        metricName: 'line_coverage',
+        baselineValue: 82,
+        currentValue: 79.5,
+        absoluteChange: -2.5,
+        percentChange: -3.05,
+        significance: 0.03,
+        confidence: 0.97,
+        direction: 'worse',
+        affectedComponent: 'src/components/',
+        gate: 'ui',
+        suggestions: ['Review deleted test cases'],
+        timestamp: Date.now() - 3600000
+      },
+      {
+        id: 'reg_3',
+        runId: 'run_122',
+        baselineRunId: 'run_99',
+        type: 'quality',
+        severity: 'critical',
+        status: 'resolved',
+        metricName: 'pass_rate',
+        baselineValue: 98,
+        currentValue: 85,
+        absoluteChange: -13,
+        percentChange: -13.27,
+        significance: 0.0001,
+        confidence: 0.9999,
+        direction: 'worse',
+        affectedComponent: 'test/auth',
+        gate: 'unit',
+        suggestions: ['Fix authentication tests'],
+        timestamp: Date.now() - 86400000
+      }
+    ];
+
+    // Filter
+    let filtered = regressions;
+    if (options.severity) {
+      filtered = filtered.filter(r => r.severity === options.severity);
+    }
+    if (options.type) {
+      filtered = filtered.filter(r => r.type === options.type);
+    }
+    filtered = filtered.slice(0, parseInt(options.limit));
+
+    if (options.json) {
+      console.log(JSON.stringify(filtered, null, 2));
+      return;
+    }
+
+    console.log('\n🔍 Recent Regressions\n');
+
+    if (filtered.length === 0) {
+      console.log('No regressions found.\n');
+      return;
+    }
+
+    for (const reg of filtered) {
+      const icon = getSeverityIcon(reg.severity);
+      const statusIcon = reg.status === 'resolved' ? '✅' : reg.status === 'investigating' ? '🔬' : '❗';
+
+      console.log(`  ${icon} ${statusIcon} ${reg.metricName}`);
+      console.log(`      Severity: ${reg.severity.toUpperCase()} | Status: ${reg.status}`);
+      console.log(`      Type: ${reg.type} | Gate: ${reg.gate}`);
+      console.log(`      ${new Date(reg.timestamp).toISOString()}`);
+      console.log('');
+    }
+  });
+
+// Compare two runs
+regressionCommand
+  .command('compare')
+  .description('Compare two runs for regressions')
+  .option('--baseline <runId>', 'Baseline run ID')
+  .option('--current <runId>', 'Current run ID')
+  .option('--metric <name>', 'Metric to compare')
+  .option('--json', 'Output as JSON')
+  .action(async (options) => {
+    const analyzer = createRegressionTrendAnalyzer();
+    const metricName = options.metric || 'api_latency';
+
+    // Generate demo data
+    const baseline: TimeSeriesPoint[] = [];
+    const current: TimeSeriesPoint[] = [];
+
+    for (let i = 0; i < 10; i++) {
+      baseline.push({
+        timestamp: Date.now() - (20 - i) * 3600000,
+        value: 100 + Math.random() * 5,
+        runId: options.baseline || `baseline_${i}`
+      });
+    }
+
+    for (let i = 0; i < 10; i++) {
+      current.push({
+        timestamp: Date.now() - (10 - i) * 3600000,
+        value: 120 + Math.random() * 5, // 20% higher
+        runId: options.current || `current_${i}`
+      });
+    }
+
+    const regression = analyzer.comparePeriods(baseline, current, metricName);
+
+    if (options.json) {
+      console.log(JSON.stringify(regression, null, 2));
+      return;
+    }
+
+    console.log('\n🔍 Regression Comparison\n');
+    console.log(`Metric: ${metricName}`);
+    console.log(`Baseline: ${options.baseline || 'baseline_*'}`);
+    console.log(`Current: ${options.current || 'current_*'}\n`);
+
+    if (!regression) {
+      console.log('✅ No regression detected!\n');
+      return;
+    }
+
+    const icon = getSeverityIcon(regression.severity);
+    console.log(`  ${icon} Regression Detected`);
+    console.log(`      Type: ${regression.type}`);
+    console.log(`      Severity: ${regression.severity.toUpperCase()}`);
+    console.log(`      Baseline: ${regression.baselineValue.toFixed(2)} → Current: ${regression.currentValue.toFixed(2)}`);
+    console.log(`      Change: ${regression.percentChange > 0 ? '+' : ''}${regression.percentChange.toFixed(1)}%`);
+    console.log(`      Confidence: ${(regression.confidence * 100).toFixed(1)}%\n`);
+  });
+
+// Show regression statistics
+regressionCommand
+  .command('stats')
+  .description('Show regression statistics')
+  .option('--json', 'Output as JSON')
+  .action(async (_options) => {
+    // Demo statistics
+    const stats = {
+      total: 15,
+      bySeverity: {
+        critical: 2,
+        major: 3,
+        moderate: 5,
+        minor: 3,
+        info: 2
+      },
+      byType: {
+        performance: 5,
+        quality: 4,
+        coverage: 2,
+        reliability: 2,
+        security: 1,
+        api_breaking: 1
+      },
+      byStatus: {
+        detected: 5,
+        investigating: 3,
+        confirmed: 2,
+        resolved: 4,
+        ignored: 1
+      },
+      recent: 3 // Last 7 days
+    };
+
+    console.log('\n📊 Regression Statistics\n');
+    console.log(`Total Regressions: ${stats.total}`);
+    console.log(`Recent (7 days): ${stats.recent}\n`);
+
+    console.log('By Severity:');
+    for (const [severity, count] of Object.entries(stats.bySeverity)) {
+      if (count > 0) {
+        const icon = getSeverityIcon(severity as RegressionSeverity);
+        console.log(`  ${icon} ${severity.toUpperCase()}: ${count}`);
+      }
+    }
+
+    console.log('\nBy Type:');
+    for (const [type, count] of Object.entries(stats.byType)) {
+      if (count > 0) {
+        console.log(`  • ${type}: ${count}`);
+      }
+    }
+
+    console.log('\nBy Status:');
+    for (const [status, count] of Object.entries(stats.byStatus)) {
+      if (count > 0) {
+        console.log(`  • ${status}: ${count}`);
+      }
+    }
+
+    console.log('');
+  });
+
+function getSeverityIcon(severity: RegressionSeverity): string {
+  switch (severity) {
+    case 'critical':
+      return '🔴';
+    case 'major':
+      return '🟠';
+    case 'moderate':
+      return '🟡';
+    case 'minor':
+      return '🟢';
+    default:
+      return '⚪';
+  }
+}
+
+export default regressionCommand;