qa360 2.2.1 → 2.2.13

package/cli/dist/core/regression/trend-analyzer.js

@@ -0,0 +1,345 @@
+/**
+ * Regression Trend Analyzer
+ *
+ * Analyzes metric trends to predict potential regressions.
+ * Uses linear regression and change point detection.
+ */
+/**
+ * Regression Trend Analyzer class
+ */
+export class RegressionTrendAnalyzer {
+    /**
+     * Analyze trend for a metric
+     */
+    analyzeTrend(data, windowMs) {
+        if (data.length < 3) {
+            return null;
+        }
+        const analysis = this.linearRegression(data);
+        const direction = this.getTrendDirection(analysis.slope);
+        const strength = Math.abs(analysis.rSquared);
+        // Project future value
+        const projected = this.projectTrend(data, analysis);
+        return {
+            metric: data[0].metadata?.metric || 'unknown',
+            direction,
+            strength,
+            projected,
+            dataPoints: data.length,
+            regressions: 0 // Will be populated by detector
+        };
+    }
+    /**
+     * Perform linear regression on time series
+     */
+    linearRegression(data) {
+        const n = data.length;
+        if (n < 2) {
+            return {
+                slope: 0,
+                intercept: data[0]?.value || 0,
+                rSquared: 0,
+                significant: false,
+                direction: 'stable',
+                project: () => 0
+            };
+        }
+        // Normalize timestamps to start from 0
+        const baseTime = data[0].timestamp;
+        let sumX = 0;
+        let sumY = 0;
+        let sumXY = 0;
+        let sumXX = 0;
+        let sumYY = 0;
+        for (const point of data) {
+            const x = (point.timestamp - baseTime) / 1000; // Convert to seconds
+            const y = point.value;
+            sumX += x;
+            sumY += y;
+            sumXY += x * y;
+            sumXX += x * x;
+            sumYY += y * y;
+        }
+        const slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX);
+        const intercept = (sumY - slope * sumX) / n;
+        // Calculate R²
+        const meanY = sumY / n;
+        let ssTotal = 0;
+        let ssResidual = 0;
+        for (const point of data) {
+            const x = (point.timestamp - baseTime) / 1000;
+            const y = point.value;
+            const predicted = slope * x + intercept;
+            ssTotal += Math.pow(y - meanY, 2);
+            ssResidual += Math.pow(y - predicted, 2);
+        }
+        const rSquared = ssTotal > 0 ? 1 - ssResidual / ssTotal : 0;
+        // Determine significance (simplified)
+        const significant = rSquared > 0.7 && Math.abs(slope) > 0.001;
+        const direction = Math.abs(slope) < 0.001 ? 'stable'
+            : slope > 0 ? 'increasing'
+                : 'decreasing';
+        return {
+            slope,
+            intercept,
+            rSquared,
+            significant,
+            direction,
+            project: (ms) => {
+                const x = ms / 1000;
+                return slope * x + intercept;
+            }
+        };
+    }
+    /**
+     * Detect change points in time series
+     */
+    detectChangePoints(data, minSegmentSize = 5) {
+        if (data.length < minSegmentSize * 2) {
+            return [];
+        }
+        const changePoints = [];
+        const values = data.map(d => d.value);
+        // Use simplified CUSUM-like approach
+        let maxChange = 0;
+        let maxIndex = 0;
+        for (let i = minSegmentSize; i < values.length - minSegmentSize; i++) {
+            const before = values.slice(0, i);
+            const after = values.slice(i);
+            const beforeMean = before.reduce((a, b) => a + b, 0) / before.length;
+            const afterMean = after.reduce((a, b) => a + b, 0) / after.length;
+            const magnitude = Math.abs(afterMean - beforeMean);
+            if (magnitude > maxChange) {
+                maxChange = magnitude;
+                maxIndex = i;
+            }
+        }
+        if (maxChange > 0) {
+            const splitIdx = maxIndex;
+            const before = values.slice(0, splitIdx);
+            const after = values.slice(splitIdx);
+            changePoints.push({
+                index: splitIdx,
+                timestamp: data[splitIdx].timestamp,
+                beforeMean: before.reduce((a, b) => a + b, 0) / before.length,
+                afterMean: after.reduce((a, b) => a + b, 0) / after.length,
+                magnitude: maxChange,
+                pValue: this.estimatePValue(maxChange, before.length, after.length)
+            });
+        }
+        return changePoints;
+    }
+    /**
+     * Estimate p-value for change point
+     */
+    estimatePValue(magnitude, n1, n2) {
+        // Simplified estimation using t-distribution approximation
+        const pooledStd = Math.sqrt((magnitude * magnitude) / (1 / n1 + 1 / n2));
+        const tStat = magnitude / (pooledStd || 1);
+        // Rough p-value approximation
+        return Math.max(0.001, 1 - Math.min(1, tStat / 5));
+    }
+    /**
+     * Project trend forward
+     */
+    projectTrend(data, analysis) {
+        if (!analysis.significant) {
+            return undefined;
+        }
+        // Project 7 days forward
+        const weekMs = 7 * 24 * 60 * 60 * 1000;
+        return analysis.project(weekMs);
+    }
+    /**
+     * Get trend direction for reporting
+     */
+    getTrendDirection(slope) {
+        if (Math.abs(slope) < 0.001) {
+            return 'stable';
+        }
+        // Note: For some metrics (like latency), increasing is bad
+        // For others (like coverage), increasing is good
+        // This is simplified - should consider metric type
+        return slope > 0 ? 'degrading' : 'improving';
+    }
+    /**
+     * Calculate time until threshold breach
+     */
+    calculateTimeToThreshold(data, threshold, isIncreasingBad) {
+        const analysis = this.linearRegression(data);
+        if (!analysis.significant || Math.abs(analysis.slope) < 0.00001) {
+            return null;
+        }
+        const currentValue = data[data.length - 1].value;
+        // Determine if we're moving toward threshold
+        const movingTowardBad = isIncreasingBad
+            ? analysis.slope > 0
+            : analysis.slope < 0;
+        if (!movingTowardBad) {
+            return null; // Not moving toward threshold
+        }
+        // Calculate time to reach threshold
+        // slope is in units per second, so we need to convert
+        const diff = isIncreasingBad
+            ? threshold - currentValue
+            : currentValue - threshold;
+        if (diff <= 0) {
+            return 0; // Already at/beyond threshold
+        }
+        // Convert: slope (units/sec) to time needed (ms)
+        // time_ms = (diff / slope) * 1000
+        const timeToThreshold = (diff / Math.abs(analysis.slope)) * 1000;
+        return Math.max(0, timeToThreshold);
+    }
+    /**
+     * Compare two periods for regression
+     */
+    comparePeriods(baseline, current, metric) {
+        if (baseline.length === 0 || current.length === 0) {
+            return null;
+        }
+        const baselineMean = baseline.reduce((sum, p) => sum + p.value, 0) / baseline.length;
+        const currentMean = current.reduce((sum, p) => sum + p.value, 0) / current.length;
+        const absoluteChange = currentMean - baselineMean;
+        const percentChange = baselineMean !== 0
+            ? (absoluteChange / baselineMean) * 100
+            : 0;
+        // Determine if regression based on metric type
+        const type = this.inferType(metric);
+        const direction = this.isRegressionDirection(type, absoluteChange);
+        if (!direction) {
+            return null;
+        }
+        // Calculate significance using t-test
+        const baselineVariance = baseline.reduce((sum, p) => sum + Math.pow(p.value - baselineMean, 2), 0) / baseline.length;
+        const currentVariance = current.reduce((sum, p) => sum + Math.pow(p.value - currentMean, 2), 0) / current.length;
+        const pooledStd = Math.sqrt(((baseline.length - 1) * baselineVariance + (current.length - 1) * currentVariance) /
+            (baseline.length + current.length - 2));
+        const tStat = absoluteChange / (pooledStd * Math.sqrt(1 / baseline.length + 1 / current.length) || 1);
+        const pValue = 2 * (1 - this.normalCDF(Math.abs(tStat)));
+        // Severity based on percent change and significance
+        let severity;
+        if (Math.abs(percentChange) > 50 && pValue < 0.01) {
+            severity = 'critical';
+        }
+        else if (Math.abs(percentChange) > 20 && pValue < 0.05) {
+            severity = 'major';
+        }
+        else if (Math.abs(percentChange) > 10) {
+            severity = 'moderate';
+        }
+        else {
+            severity = 'minor';
+        }
+        return {
+            id: `regression_${Date.now()}_${metric.replace(/\W/g, '_')}`,
+            runId: current[current.length - 1].runId,
+            baselineRunId: baseline[baseline.length - 1].runId,
+            type,
+            severity,
+            status: 'detected',
+            metricName: metric,
+            baselineValue: baselineMean,
+            currentValue: currentMean,
+            absoluteChange,
+            percentChange,
+            significance: pValue,
+            confidence: 1 - pValue,
+            direction: absoluteChange > 0 ? 'worse' : 'better',
+            affectedComponent: metric,
+            gate: 'unknown',
+            suggestions: [
+                `Review changes between ${new Date(baseline[0].timestamp).toISOString()} and ${new Date(current[0].timestamp).toISOString()}`,
+                `${percentChange > 0 ? 'Increased' : 'Decreased'} by ${Math.abs(percentChange).toFixed(1)}%`
+            ],
+            timestamp: Date.now()
+        };
+    }
+    /**
+     * Infer regression type from metric name
+     */
+    inferType(metric) {
+        if (metric.includes('duration') || metric.includes('latency') || metric.includes('time')) {
+            return 'performance';
+        }
+        if (metric.includes('coverage')) {
+            return 'coverage';
+        }
+        if (metric.includes('flaky') || metric.includes('stability')) {
+            return 'reliability';
+        }
+        if (metric.includes('security') || metric.includes('vulnerability')) {
+            return 'security';
+        }
+        return 'quality';
+    }
+    /**
+     * Check if change direction indicates regression
+     */
+    isRegressionDirection(type, change) {
+        // For performance, reliability: increase is bad
+        if (type === 'performance' || type === 'reliability') {
+            return change > 0;
+        }
+        // For coverage: decrease is bad
+        if (type === 'coverage') {
+            return change < 0;
+        }
+        // Default: increase is bad
+        return change > 0;
+    }
+    /**
+     * Normal CDF approximation
+     */
+    normalCDF(x) {
+        const a1 = 0.254829592;
+        const a2 = -0.284496736;
+        const a3 = 1.421413741;
+        const a4 = -1.453152027;
+        const a5 = 1.061405429;
+        const p = 0.3275911;
+        const sign = x < 0 ? -1 : 1;
+        x = Math.abs(x) / Math.sqrt(2);
+        const t = 1.0 / (1.0 + p * x);
+        const y = 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
+        return 0.5 * (1.0 + sign * y);
+    }
+    /**
+     * Get trend description
+     */
+    getTrendDescription(trend) {
+        const percentStrength = Math.round(trend.strength * 100);
+        switch (trend.direction) {
+            case 'degrading':
+                return `Degrading (${percentStrength}% confidence)`;
+            case 'improving':
+                return `Improving (${percentStrength}% confidence)`;
+            default:
+                return `Stable (${percentStrength}% confidence)`;
+        }
+    }
+    /**
+     * Format time to threshold
+     */
+    formatTimeToThreshold(ms) {
+        if (ms < 0)
+            return 'Already breached';
+        if (ms === 0)
+            return 'At threshold';
+        const days = Math.floor(ms / (24 * 60 * 60 * 1000));
+        const hours = Math.floor((ms % (24 * 60 * 60 * 1000)) / (60 * 60 * 1000));
+        if (days > 0) {
+            return `${days} day${days > 1 ? 's' : ''}`;
+        }
+        if (hours > 0) {
+            return `${hours} hour${hours > 1 ? 's' : ''}`;
+        }
+        return '< 1 hour';
+    }
+}
+/**
+ * Create a regression trend analyzer
+ */
+export function createRegressionTrendAnalyzer() {
+    return new RegressionTrendAnalyzer();
+}

package/cli/dist/core/regression/types.d.ts

@@ -0,0 +1,222 @@
+/**
+ * QA360 Regression Detection Module
+ *
+ * Types for detecting performance and quality regressions.
+ * Uses statistical analysis to identify significant changes in metrics.
+ */
+/**
+ * Regression types
+ */
+export type RegressionType = 'performance' | 'quality' | 'coverage' | 'reliability' | 'security' | 'api_breaking' | 'custom';
+/**
+ * Regression severity
+ */
+export type RegressionSeverity = 'info' | 'minor' | 'moderate' | 'major' | 'critical';
+/**
+ * Regression status
+ */
+export type RegressionStatus = 'detected' | 'investigating' | 'confirmed' | 'false_positive' | 'resolved' | 'ignored';
+/**
+ * Regression detection result
+ */
+export interface RegressionDetection {
+    /** Unique identifier */
+    id: string;
+    /** Run ID where regression was detected */
+    runId: string;
+    /** Baseline run ID for comparison */
+    baselineRunId: string;
+    /** Regression type */
+    type: RegressionType;
+    /** Severity level */
+    severity: RegressionSeverity;
+    /** Status */
+    status: RegressionStatus;
+    /** Metric name that regressed */
+    metricName: string;
+    /** Baseline value */
+    baselineValue: number;
+    /** Current value */
+    currentValue: number;
+    /** Absolute change */
+    absoluteChange: number;
+    /** Percentage change */
+    percentChange: number;
+    /** Statistical significance (p-value) */
+    significance: number;
+    /** Confidence (0-1) */
+    confidence: number;
+    /** Direction (worse = degradation, better = improvement) */
+    direction: 'worse' | 'better' | 'neutral';
+    /** Affected component/test */
+    affectedComponent: string;
+    /** Gate where detected */
+    gate: string;
+    /** Context information */
+    context?: {
+        file?: string;
+        line?: number;
+        test?: string;
+        endpoint?: string;
+        [key: string]: unknown;
+    };
+    /** Suggested actions */
+    suggestions: string[];
+    /** Related commits */
+    commits?: string[];
+    /** Detection timestamp */
+    timestamp: number;
+    /** Notes */
+    notes?: string;
+}
+/**
+ * Regression threshold configuration
+ */
+export interface RegressionThreshold {
+    /** Metric name */
+    metric: string;
+    /** Maximum allowed regression (percentage) */
+    maxRegression: number;
+    /** Minimum change to trigger alert (absolute) */
+    minDelta: number;
+    /** Statistical significance threshold (p-value) */
+    significanceLevel: number;
+    /** Direction to monitor */
+    direction: 'increase' | 'decrease' | 'both';
+    /** Regression severity if threshold exceeded */
+    severity: RegressionSeverity;
+}
+/**
+ * Regression configuration
+ */
+export interface RegressionConfig {
+    /** Enable regression detection */
+    enabled: boolean;
+    /** Automatic detection on each run */
+    autoDetect: boolean;
+    /** Baseline window (number of runs to average) */
+    baselineWindow: number;
+    /** Minimum data points for statistical analysis */
+    minDataPoints: number;
+    /** Statistical method */
+    method: 'zscore' | 'ttest' | 'mannwhitney' | 'percentile';
+    /** Thresholds by metric type */
+    thresholds: {
+        performance: RegressionThreshold[];
+        quality: RegressionThreshold[];
+        coverage: RegressionThreshold[];
+        reliability: RegressionThreshold[];
+        security: RegressionThreshold[];
+    };
+    /** Alert configuration */
+    alerts: {
+        enabled: boolean;
+        onNewRegression: boolean;
+        onSeverity: RegressionSeverity[];
+    };
+    /** Ignore patterns (metrics to ignore) */
+    ignorePatterns: string[];
+}
+/**
+ * Time series data point
+ */
+export interface TimeSeriesPoint {
+    timestamp: number;
+    value: number;
+    runId: string;
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Statistical test result
+ */
+export interface StatisticalTest {
+    /** Test name */
+    test: string;
+    /** P-value */
+    pValue: number;
+    /** Is significant? */
+    significant: boolean;
+    /** Test statistic */
+    statistic: number;
+    /** Confidence interval */
+    confidenceInterval?: {
+        lower: number;
+        upper: number;
+    };
+}
+/**
+ * Regression trend analysis
+ */
+export interface RegressionTrend {
+    /** Metric name */
+    metric: string;
+    /** Current trend direction */
+    direction: 'improving' | 'degrading' | 'stable';
+    /** Trend strength (0-1) */
+    strength: number;
+    /** Projected value (if trend continues) */
+    projected?: number;
+    /** Time to threshold (if degrading) */
+    timeToThreshold?: number;
+    /** Data points analyzed */
+    dataPoints: number;
+    /** Regression detections in trend */
+    regressions: number;
+}
+/**
+ * Regression report
+ */
+export interface RegressionReport {
+    /** Report ID */
+    id: string;
+    /** Report timestamp */
+    timestamp: number;
+    /** Comparison period */
+    period: {
+        start: number;
+        end: number;
+        baselineStart: number;
+        baselineEnd: number;
+    };
+    /** Detected regressions */
+    regressions: RegressionDetection[];
+    /** Summary statistics */
+    summary: {
+        total: number;
+        bySeverity: Record<RegressionSeverity, number>;
+        byType: Record<RegressionType, number>;
+        new: number;
+        resolved: number;
+        ongoing: number;
+    };
+    /** Overall health status */
+    healthStatus: 'healthy' | 'warning' | 'critical';
+}
+/**
+ * Baseline data
+ */
+export interface Baseline {
+    /** Metric name */
+    metric: string;
+    /** Average value */
+    mean: number;
+    /** Standard deviation */
+    stdDev: number;
+    /** Minimum value */
+    min: number;
+    /** Maximum value */
+    max: number;
+    /** Percentiles */
+    percentiles: {
+        p50: number;
+        p90: number;
+        p95: number;
+        p99: number;
+    };
+    /** Sample size */
+    sampleSize: number;
+    /** Calculated from run IDs */
+    runIds: string[];
+    /** Baseline timestamp */
+    timestamp: number;
+}

package/cli/dist/core/regression/types.js

@@ -0,0 +1,7 @@
+/**
+ * QA360 Regression Detection Module
+ *
+ * Types for detecting performance and quality regressions.
+ * Uses statistical analysis to identify significant changes in metrics.
+ */
+export {};

package/cli/dist/core/regression/vault.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Regression Vault Integration
+ *
+ * Integrates regression data storage and retrieval with the Evidence Vault.
+ */
+import type { Database } from 'sqlite3';
+import type { RegressionDetection } from './types.js';
+/**
+ * Regression Vault class
+ */
+export declare class RegressionVault {
+    private db;
+    private dbRun;
+    private dbAll;
+    private dbGet;
+    constructor(db: Database);
+    /**
+     * Initialize regression tables
+     */
+    initialize(): Promise<void>;
+    /**
+     * Store a regression detection
+     */
+    storeRegression(regression: RegressionDetection): Promise<void>;
+    /**
+     * Store multiple regressions
+     */
+    storeRegressions(regressions: RegressionDetection[]): Promise<void>;
+    /**
+     * Get regressions by run ID
+     */
+    getRegressionsByRun(runId: string): Promise<RegressionDetection[]>;
+    /**
+     * Get regressions by type
+     */
+    getRegressionsByType(type: string, limit?: number): Promise<RegressionDetection[]>;
+    /**
+     * Get regressions by severity
+     */
+    getRegressionsBySeverity(severity: string, limit?: number): Promise<RegressionDetection[]>;
+    /**
+     * Get regressions by status
+     */
+    getRegressionsByStatus(status: string, limit?: number): Promise<RegressionDetection[]>;
+    /**
+     * Get recent regressions
+     */
+    getRecentRegressions(limit?: number): Promise<RegressionDetection[]>;
+    /**
+     * Get regressions for a metric
+     */
+    getRegressionsByMetric(metricName: string, limit?: number): Promise<RegressionDetection[]>;
+    /**
+     * Update regression status
+     */
+    updateRegressionStatus(id: string, status: RegressionDetection['status'], notes?: string): Promise<void>;
+    /**
+     * Store regression snapshot
+     */
+    storeSnapshot(runId: string, metricName: string, value: number, baselineValue?: number): Promise<void>;
+    /**
+     * Get metric history for trend analysis
+     */
+    getMetricHistory(metricName: string, limit?: number): Promise<Array<{
+        runId: string;
+        value: number;
+        timestamp: number;
+    }>>;
+    /**
+     * Get regression statistics
+     */
+    getStatistics(): Promise<{
+        total: number;
+        bySeverity: Record<string, number>;
+        byType: Record<string, number>;
+        byStatus: Record<string, number>;
+        recent: number;
+    }>;
+    /**
+     * Delete old regressions
+     */
+    deleteOldRegressions(olderThanMs: number): Promise<number>;
+    /**
+     * Map database rows to RegressionDetection objects
+     */
+    private mapRowsToDetections;
+}