qa360 2.1.7 → 2.2.0

package/core/src/regression/types.ts

@@ -0,0 +1,295 @@
+/**
+ * QA360 Regression Detection Module
+ *
+ * Types for detecting performance and quality regressions.
+ * Uses statistical analysis to identify significant changes in metrics.
+ */
+
+/**
+ * Regression types
+ */
+export type RegressionType =
+  | 'performance'      // Performance degradation
+  | 'quality'          // Quality/test failure increase
+  | 'coverage'         // Coverage decrease
+  | 'reliability'      // Reliability/flakiness increase
+  | 'security'         // New security vulnerabilities
+  | 'api_breaking'     // Breaking API changes
+  | 'custom';          // Custom regression type
+
+/**
+ * Regression severity
+ */
+export type RegressionSeverity = 'info' | 'minor' | 'moderate' | 'major' | 'critical';
+
+/**
+ * Regression status
+ */
+export type RegressionStatus = 'detected' | 'investigating' | 'confirmed' | 'false_positive' | 'resolved' | 'ignored';
+
+/**
+ * Regression detection result
+ */
+export interface RegressionDetection {
+  /** Unique identifier */
+  id: string;
+
+  /** Run ID where regression was detected */
+  runId: string;
+
+  /** Baseline run ID for comparison */
+  baselineRunId: string;
+
+  /** Regression type */
+  type: RegressionType;
+
+  /** Severity level */
+  severity: RegressionSeverity;
+
+  /** Status */
+  status: RegressionStatus;
+
+  /** Metric name that regressed */
+  metricName: string;
+
+  /** Baseline value */
+  baselineValue: number;
+
+  /** Current value */
+  currentValue: number;
+
+  /** Absolute change */
+  absoluteChange: number;
+
+  /** Percentage change */
+  percentChange: number;
+
+  /** Statistical significance (p-value) */
+  significance: number;
+
+  /** Confidence (0-1) */
+  confidence: number;
+
+  /** Direction (worse = degradation, better = improvement) */
+  direction: 'worse' | 'better' | 'neutral';
+
+  /** Affected component/test */
+  affectedComponent: string;
+
+  /** Gate where detected */
+  gate: string;
+
+  /** Context information */
+  context?: {
+    file?: string;
+    line?: number;
+    test?: string;
+    endpoint?: string;
+    [key: string]: unknown;
+  };
+
+  /** Suggested actions */
+  suggestions: string[];
+
+  /** Related commits */
+  commits?: string[];
+
+  /** Detection timestamp */
+  timestamp: number;
+
+  /** Notes */
+  notes?: string;
+}
+
+/**
+ * Regression threshold configuration
+ */
+export interface RegressionThreshold {
+  /** Metric name */
+  metric: string;
+
+  /** Maximum allowed regression (percentage) */
+  maxRegression: number;
+
+  /** Minimum change to trigger alert (absolute) */
+  minDelta: number;
+
+  /** Statistical significance threshold (p-value) */
+  significanceLevel: number;
+
+  /** Direction to monitor */
+  direction: 'increase' | 'decrease' | 'both';
+
+  /** Regression severity if threshold exceeded */
+  severity: RegressionSeverity;
+}
+
+/**
+ * Regression configuration
+ */
+export interface RegressionConfig {
+  /** Enable regression detection */
+  enabled: boolean;
+
+  /** Automatic detection on each run */
+  autoDetect: boolean;
+
+  /** Baseline window (number of runs to average) */
+  baselineWindow: number;
+
+  /** Minimum data points for statistical analysis */
+  minDataPoints: number;
+
+  /** Statistical method */
+  method: 'zscore' | 'ttest' | 'mannwhitney' | 'percentile';
+
+  /** Thresholds by metric type */
+  thresholds: {
+    performance: RegressionThreshold[];
+    quality: RegressionThreshold[];
+    coverage: RegressionThreshold[];
+    reliability: RegressionThreshold[];
+    security: RegressionThreshold[];
+  };
+
+  /** Alert configuration */
+  alerts: {
+    enabled: boolean;
+    onNewRegression: boolean;
+    onSeverity: RegressionSeverity[];
+  };
+
+  /** Ignore patterns (metrics to ignore) */
+  ignorePatterns: string[];
+}
+
+/**
+ * Time series data point
+ */
+export interface TimeSeriesPoint {
+  timestamp: number;
+  value: number;
+  runId: string;
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Statistical test result
+ */
+export interface StatisticalTest {
+  /** Test name */
+  test: string;
+
+  /** P-value */
+  pValue: number;
+
+  /** Is significant? */
+  significant: boolean;
+
+  /** Test statistic */
+  statistic: number;
+
+  /** Confidence interval */
+  confidenceInterval?: {
+    lower: number;
+    upper: number;
+  };
+}
+
+/**
+ * Regression trend analysis
+ */
+export interface RegressionTrend {
+  /** Metric name */
+  metric: string;
+
+  /** Current trend direction */
+  direction: 'improving' | 'degrading' | 'stable';
+
+  /** Trend strength (0-1) */
+  strength: number;
+
+  /** Projected value (if trend continues) */
+  projected?: number;
+
+  /** Time to threshold (if degrading) */
+  timeToThreshold?: number;
+
+  /** Data points analyzed */
+  dataPoints: number;
+
+  /** Regression detections in trend */
+  regressions: number;
+}
+
+/**
+ * Regression report
+ */
+export interface RegressionReport {
+  /** Report ID */
+  id: string;
+
+  /** Report timestamp */
+  timestamp: number;
+
+  /** Comparison period */
+  period: {
+    start: number;
+    end: number;
+    baselineStart: number;
+    baselineEnd: number;
+  };
+
+  /** Detected regressions */
+  regressions: RegressionDetection[];
+
+  /** Summary statistics */
+  summary: {
+    total: number;
+    bySeverity: Record<RegressionSeverity, number>;
+    byType: Record<RegressionType, number>;
+    new: number;
+    resolved: number;
+    ongoing: number;
+  };
+
+  /** Overall health status */
+  healthStatus: 'healthy' | 'warning' | 'critical';
+}
+
+/**
+ * Baseline data
+ */
+export interface Baseline {
+  /** Metric name */
+  metric: string;
+
+  /** Average value */
+  mean: number;
+
+  /** Standard deviation */
+  stdDev: number;
+
+  /** Minimum value */
+  min: number;
+
+  /** Maximum value */
+  max: number;
+
+  /** Percentiles */
+  percentiles: {
+    p50: number;
+    p90: number;
+    p95: number;
+    p99: number;
+  };
+
+  /** Sample size */
+  sampleSize: number;
+
+  /** Calculated from run IDs */
+  runIds: string[];
+
+  /** Baseline timestamp */
+  timestamp: number;
+}

package/core/src/regression/vault.ts

@@ -0,0 +1,419 @@
+/**
+ * Regression Vault Integration
+ *
+ * Integrates regression data storage and retrieval with the Evidence Vault.
+ */
+
+import type { Database } from 'sqlite3';
+import type { RegressionDetection, RegressionReport, RegressionTrend } from './types.js';
+import { promisify } from 'util';
+
+/**
+ * Regression record for storage
+ */
+interface RegressionRecord {
+  id: string;
+  run_id: string;
+  baseline_run_id: string;
+  type: string;
+  severity: string;
+  status: string;
+  metric_name: string;
+  baseline_value: number;
+  current_value: number;
+  absolute_change: number;
+  percent_change: number;
+  significance: number;
+  confidence: number;
+  direction: string;
+  affected_component: string;
+  gate: string;
+  context_json?: string;
+  suggestions_json: string;
+  commits_json?: string;
+  timestamp: number;
+  notes?: string;
+}
+
+/**
+ * Regression Vault class
+ */
+export class RegressionVault {
+  private db: Database;
+  private dbRun: (sql: string, params: any[]) => Promise<any>;
+  private dbAll: (sql: string, params: any[]) => Promise<any[]>;
+  private dbGet: (sql: string, params: any[]) => Promise<any>;
+
+  constructor(db: Database) {
+    this.db = db;
+    this.dbRun = promisify(db.run.bind(db)) as any;
+    this.dbAll = promisify(db.all.bind(db)) as any;
+    this.dbGet = promisify(db.get.bind(db)) as any;
+  }
+
+  /**
+   * Initialize regression tables
+   */
+  async initialize(): Promise<void> {
+    const schema = `
+      -- Regression detections table
+      CREATE TABLE IF NOT EXISTS regression_detections (
+          id TEXT PRIMARY KEY,
+          run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+          baseline_run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE SET NULL,
+          type TEXT NOT NULL,
+          severity TEXT NOT NULL,
+          status TEXT NOT NULL DEFAULT 'detected',
+          metric_name TEXT NOT NULL,
+          baseline_value REAL NOT NULL,
+          current_value REAL NOT NULL,
+          absolute_change REAL NOT NULL,
+          percent_change REAL NOT NULL,
+          significance REAL NOT NULL,
+          confidence REAL NOT NULL,
+          direction TEXT NOT NULL,
+          affected_component TEXT,
+          gate TEXT NOT NULL,
+          context_json TEXT,
+          suggestions_json TEXT NOT NULL,
+          commits_json TEXT,
+          timestamp INTEGER NOT NULL,
+          notes TEXT
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_regression_run_id ON regression_detections(run_id);
+      CREATE INDEX IF NOT EXISTS idx_regression_baseline_run_id ON regression_detections(baseline_run_id);
+      CREATE INDEX IF NOT EXISTS idx_regression_type ON regression_detections(type);
+      CREATE INDEX IF NOT EXISTS idx_regression_severity ON regression_detections(severity);
+      CREATE INDEX IF NOT EXISTS idx_regression_status ON regression_detections(status);
+      CREATE INDEX IF NOT EXISTS idx_regression_metric ON regression_detections(metric_name);
+      CREATE INDEX IF NOT EXISTS idx_regression_timestamp ON regression_detections(timestamp);
+
+      -- Regression snapshots (aggregate metrics over time)
+      CREATE TABLE IF NOT EXISTS regression_snapshots (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+          metric_name TEXT NOT NULL,
+          value REAL NOT NULL,
+          baseline_value REAL,
+          change_percent REAL,
+          is_regression INTEGER DEFAULT 0,
+          timestamp INTEGER NOT NULL,
+          UNIQUE(run_id, metric_name)
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_regression_snapshots_run_id ON regression_snapshots(run_id);
+      CREATE INDEX IF NOT EXISTS idx_regression_snapshots_metric ON regression_snapshots(metric_name);
+
+      -- Regression history for trend analysis
+      CREATE TABLE IF NOT EXISTS regression_history (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          metric_name TEXT NOT NULL,
+          run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+          value REAL NOT NULL,
+          mean REAL NOT NULL,
+          std_dev REAL NOT NULL,
+          sample_size INTEGER NOT NULL,
+          window_start INTEGER NOT NULL,
+          window_end INTEGER NOT NULL,
+          timestamp INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_regression_history_metric ON regression_history(metric_name);
+      CREATE INDEX IF NOT EXISTS idx_regression_history_timestamp ON regression_history(timestamp);
+    `;
+
+    const statements = schema
+      .split(';')
+      .map(s => s.trim())
+      .filter(s => s.length > 0);
+
+    for (const statement of statements) {
+      await this.dbRun(statement, []);
+    }
+  }
+
+  /**
+   * Store a regression detection
+   */
+  async storeRegression(regression: RegressionDetection): Promise<void> {
+    await this.dbRun(
+      `INSERT INTO regression_detections (
+        id, run_id, baseline_run_id, type, severity, status,
+        metric_name, baseline_value, current_value, absolute_change,
+        percent_change, significance, confidence, direction,
+        affected_component, gate, context_json, suggestions_json,
+        commits_json, timestamp, notes
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        regression.id,
+        regression.runId,
+        regression.baselineRunId,
+        regression.type,
+        regression.severity,
+        regression.status,
+        regression.metricName,
+        regression.baselineValue,
+        regression.currentValue,
+        regression.absoluteChange,
+        regression.percentChange,
+        regression.significance,
+        regression.confidence,
+        regression.direction,
+        regression.affectedComponent,
+        regression.gate,
+        regression.context ? JSON.stringify(regression.context) : null,
+        JSON.stringify(regression.suggestions),
+        regression.commits ? JSON.stringify(regression.commits) : null,
+        regression.timestamp,
+        regression.notes || null
+      ]
+    );
+  }
+
+  /**
+   * Store multiple regressions
+   */
+  async storeRegressions(regressions: RegressionDetection[]): Promise<void> {
+    for (const regression of regressions) {
+      await this.storeRegression(regression);
+    }
+  }
+
+  /**
+   * Get regressions by run ID
+   */
+  async getRegressionsByRun(runId: string): Promise<RegressionDetection[]> {
+    const rows = await this.dbAll(
+      `SELECT * FROM regression_detections WHERE run_id = ? ORDER BY severity DESC, percent_change DESC`,
+      [runId]
+    );
+
+    return this.mapRowsToDetections(rows);
+  }
+
+  /**
+   * Get regressions by type
+   */
+  async getRegressionsByType(type: string, limit = 100): Promise<RegressionDetection[]> {
+    const rows = await this.dbAll(
+      `SELECT * FROM regression_detections WHERE type = ? ORDER BY timestamp DESC LIMIT ?`,
+      [type, limit]
+    );
+
+    return this.mapRowsToDetections(rows);
+  }
+
+  /**
+   * Get regressions by severity
+   */
+  async getRegressionsBySeverity(severity: string, limit = 100): Promise<RegressionDetection[]> {
+    const rows = await this.dbAll(
+      `SELECT * FROM regression_detections WHERE severity = ? ORDER BY timestamp DESC LIMIT ?`,
+      [severity, limit]
+    );
+
+    return this.mapRowsToDetections(rows);
+  }
+
+  /**
+   * Get regressions by status
+   */
+  async getRegressionsByStatus(status: string, limit = 100): Promise<RegressionDetection[]> {
+    const rows = await this.dbAll(
+      `SELECT * FROM regression_detections WHERE status = ? ORDER BY timestamp DESC LIMIT ?`,
+      [status, limit]
+    );
+
+    return this.mapRowsToDetections(rows);
+  }
+
+  /**
+   * Get recent regressions
+   */
+  async getRecentRegressions(limit = 50): Promise<RegressionDetection[]> {
+    const rows = await this.dbAll(
+      `SELECT * FROM regression_detections ORDER BY timestamp DESC LIMIT ?`,
+      [limit]
+    );
+
+    return this.mapRowsToDetections(rows);
+  }
+
+  /**
+   * Get regressions for a metric
+   */
+  async getRegressionsByMetric(metricName: string, limit = 50): Promise<RegressionDetection[]> {
+    const rows = await this.dbAll(
+      `SELECT * FROM regression_detections WHERE metric_name = ? ORDER BY timestamp DESC LIMIT ?`,
+      [metricName, limit]
+    );
+
+    return this.mapRowsToDetections(rows);
+  }
+
+  /**
+   * Update regression status
+   */
+  async updateRegressionStatus(id: string, status: RegressionDetection['status'], notes?: string): Promise<void> {
+    let sql = `UPDATE regression_detections SET status = ?`;
+    const params: any[] = [status];
+
+    if (notes) {
+      sql += `, notes = ?`;
+      params.push(notes);
+    }
+
+    sql += ` WHERE id = ?`;
+    params.push(id);
+
+    await this.dbRun(sql, params);
+  }
+
+  /**
+   * Store regression snapshot
+   */
+  async storeSnapshot(
+    runId: string,
+    metricName: string,
+    value: number,
+    baselineValue?: number
+  ): Promise<void> {
+    const changePercent = baselineValue !== undefined
+      ? ((value - baselineValue) / baselineValue) * 100
+      : null;
+
+    await this.dbRun(
+      `INSERT OR REPLACE INTO regression_snapshots (
+        run_id, metric_name, value, baseline_value, change_percent, timestamp
+      ) VALUES (?, ?, ?, ?, ?, ?)`,
+      [runId, metricName, value, baselineValue, changePercent, Date.now()]
+    );
+  }
+
+  /**
+   * Get metric history for trend analysis
+   */
+  async getMetricHistory(
+    metricName: string,
+    limit = 100
+  ): Promise<Array<{
+    runId: string;
+    value: number;
+    timestamp: number;
+  }>> {
+    const rows = await this.dbAll(
+      `SELECT run_id, value, timestamp FROM regression_snapshots
+       WHERE metric_name = ?
+       ORDER BY timestamp DESC
+       LIMIT ?`,
+      [metricName, limit]
+    );
+
+    return rows.map((r: any) => ({
+      runId: r.run_id,
+      value: r.value,
+      timestamp: r.timestamp
+    }));
+  }
+
+  /**
+   * Get regression statistics
+   */
+  async getStatistics(): Promise<{
+    total: number;
+    bySeverity: Record<string, number>;
+    byType: Record<string, number>;
+    byStatus: Record<string, number>;
+    recent: number; // Last 7 days
+  }> {
+    const total = await this.dbGet(`SELECT COUNT(*) as count FROM regression_detections`, []);
+
+    const bySeverity = await this.dbAll(
+      `SELECT severity, COUNT(*) as count FROM regression_detections GROUP BY severity`,
+      []
+    );
+
+    const byType = await this.dbAll(
+      `SELECT type, COUNT(*) as count FROM regression_detections GROUP BY type`,
+      []
+    );
+
+    const byStatus = await this.dbAll(
+      `SELECT status, COUNT(*) as count FROM regression_detections GROUP BY status`,
+      []
+    );
+
+    const weekAgo = Date.now() - 7 * 24 * 60 * 60 * 1000;
+    const recent = await this.dbGet(
+      `SELECT COUNT(*) as count FROM regression_detections WHERE timestamp >= ?`,
+      [weekAgo]
+    );
+
+    const severityMap: Record<string, number> = {};
+    for (const row of bySeverity) {
+      severityMap[row.severity] = row.count;
+    }
+
+    const typeMap: Record<string, number> = {};
+    for (const row of byType) {
+      typeMap[row.type] = row.count;
+    }
+
+    const statusMap: Record<string, number> = {};
+    for (const row of byStatus) {
+      statusMap[row.status] = row.count;
+    }
+
+    return {
+      total: total?.count || 0,
+      bySeverity: severityMap,
+      byType: typeMap,
+      byStatus: statusMap,
+      recent: recent?.count || 0
+    };
+  }
+
+  /**
+   * Delete old regressions
+   */
+  async deleteOldRegressions(olderThanMs: number): Promise<number> {
+    const cutoff = Date.now() - olderThanMs;
+
+    const result = await this.dbRun(
+      `DELETE FROM regression_detections WHERE timestamp < ?`,
+      [cutoff]
+    );
+
+    return result.changes || 0;
+  }
+
+  /**
+   * Map database rows to RegressionDetection objects
+   */
+  private mapRowsToDetections(rows: any[]): RegressionDetection[] {
+    return rows.map((r: RegressionRecord) => ({
+      id: r.id,
+      runId: r.run_id,
+      baselineRunId: r.baseline_run_id,
+      type: r.type as any,
+      severity: r.severity as any,
+      status: r.status as any,
+      metricName: r.metric_name,
+      baselineValue: r.baseline_value,
+      currentValue: r.current_value,
+      absoluteChange: r.absolute_change,
+      percentChange: r.percent_change,
+      significance: r.significance,
+      confidence: r.confidence,
+      direction: r.direction as any,
+      affectedComponent: r.affected_component,
+      gate: r.gate,
+      context: r.context_json ? JSON.parse(r.context_json) : undefined,
+      suggestions: JSON.parse(r.suggestions_json),
+      commits: r.commits_json ? JSON.parse(r.commits_json) : undefined,
+      timestamp: r.timestamp,
+      notes: r.notes
+    }));
+  }
+}