qa360 2.1.7 → 2.2.0

package/core/src/retry/vault.ts

@@ -0,0 +1,413 @@
+/**
+ * Retry Vault Integration
+ *
+ * Stores retry history and statistics in the Evidence Vault.
+ */
+
+import type { Database } from 'sqlite3';
+import { promisify } from 'util';
+import type {
+  RetryResult,
+  RetryHistoryRecord,
+  RetryStatistics
+} from './types.js';
+import { RetryStrategy } from './types.js';
+
+/**
+ * Retry Vault class
+ */
+export class RetryVault {
+  private db: Database;
+  private dbRun: (sql: string, params: any[]) => Promise<any>;
+  private dbAll: (sql: string, params: any[]) => Promise<any[]>;
+  private dbGet: (sql: string, params: any[]) => Promise<any>;
+
+  constructor(db: Database) {
+    this.db = db;
+    this.dbRun = promisify(db.run.bind(db)) as any;
+    this.dbAll = promisify(db.all.bind(db)) as any;
+    this.dbGet = promisify(db.get.bind(db)) as any;
+  }
+
+  /**
+   * Initialize retry tables
+   */
+  async initialize(): Promise<void> {
+    const schema = `
+      -- Retry history table
+      CREATE TABLE IF NOT EXISTS retry_history (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+          test_id TEXT NOT NULL,
+          test_name TEXT NOT NULL,
+          gate TEXT NOT NULL,
+          strategy TEXT NOT NULL,
+          max_retries INTEGER NOT NULL,
+          actual_attempts INTEGER NOT NULL,
+          success BOOLEAN NOT NULL,
+          successful_attempt INTEGER,
+          total_duration_ms INTEGER NOT NULL,
+          stop_reason TEXT NOT NULL,
+          error_type TEXT,
+          error_message TEXT,
+          flakiness_score REAL,
+          pattern_type TEXT,
+          timestamp INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_retry_run_id ON retry_history(run_id);
+      CREATE INDEX IF NOT EXISTS idx_retry_test_id ON retry_history(test_id);
+      CREATE INDEX IF NOT EXISTS idx_retry_strategy ON retry_history(strategy);
+      CREATE INDEX IF NOT EXISTS idx_retry_success ON retry_history(success);
+      CREATE INDEX IF NOT EXISTS idx_retry_timestamp ON retry_history(timestamp);
+
+      -- Retry statistics summary table
+      CREATE TABLE IF NOT EXISTS retry_stats (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          date TEXT NOT NULL,
+          total_retries INTEGER NOT NULL DEFAULT 0,
+          tests_retried INTEGER NOT NULL DEFAULT 0,
+          recovered_tests INTEGER NOT NULL DEFAULT 0,
+          failed_tests INTEGER NOT NULL DEFAULT 0,
+          recovery_rate REAL NOT NULL DEFAULT 0,
+          avg_attempts_per_test REAL NOT NULL DEFAULT 0,
+          total_retry_time_ms INTEGER NOT NULL DEFAULT 0,
+          UNIQUE(date)
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_retry_stats_date ON retry_stats(date);
+
+      -- Strategy performance table
+      CREATE TABLE IF NOT EXISTS retry_strategy_performance (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          strategy TEXT NOT NULL,
+          date TEXT NOT NULL,
+          attempts INTEGER NOT NULL DEFAULT 0,
+          successes INTEGER NOT NULL DEFAULT 0,
+          avg_duration_ms REAL NOT NULL DEFAULT 0,
+          UNIQUE(strategy, date)
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_retry_strategy_perf_date ON retry_strategy_performance(date);
+    `;
+
+    const statements = schema
+      .split(';')
+      .map(s => s.trim())
+      .filter(s => s.length > 0);
+
+    for (const statement of statements) {
+      await this.dbRun(statement, []);
+    }
+  }
+
+  /**
+   * Store retry result
+   */
+  async storeRetryResult(
+    runId: string,
+    result: RetryResult,
+    context?: {
+      testName?: string;
+      gate?: string;
+      flakinessScore?: number;
+      patternType?: string;
+    }
+  ): Promise<number> {
+    const lastAttempt = result.attempts[result.attempts.length - 1];
+    const errorType = lastAttempt?.errorType;
+    const errorMessage = lastAttempt?.errorMessage;
+
+    const sql = `
+      INSERT INTO retry_history (
+        run_id, test_id, test_name, gate, strategy, max_retries,
+        actual_attempts, success, successful_attempt, total_duration_ms,
+        stop_reason, error_type, error_message, flakiness_score, pattern_type, timestamp
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `;
+
+    const result2 = await this.dbRun(sql, [
+      runId,
+      result.testId,
+      context?.testName || result.testId,
+      context?.gate || 'unknown',
+      result.strategy,
+      result.totalAttempts - 1, // max_retries = attempts - 1 (first attempt is not a retry)
+      result.totalAttempts,
+      result.success ? 1 : 0,
+      result.successfulAttempt || null,
+      result.totalDurationMs,
+      result.stopReason,
+      errorType || null,
+      errorMessage || null,
+      context?.flakinessScore || null,
+      context?.patternType || null,
+      Date.now()
+    ]);
+
+    // Update statistics
+    await this.updateStatistics(result);
+
+    return result2.lastID;
+  }
+
+  /**
+   * Get retry history for a test
+   */
+  async getRetryHistory(
+    testId: string,
+    limit = 50
+  ): Promise<RetryHistoryRecord[]> {
+    const rows = await this.dbAll(
+      `SELECT * FROM retry_history WHERE test_id = ? ORDER BY timestamp DESC LIMIT ?`,
+      [testId, limit]
+    );
+
+    return this.mapRowsToRecords(rows);
+  }
+
+  /**
+   * Get retry history for a run
+   */
+  async getRetryHistoryForRun(runId: string): Promise<RetryHistoryRecord[]> {
+    const rows = await this.dbAll(
+      `SELECT * FROM retry_history WHERE run_id = ? ORDER BY timestamp DESC`,
+      [runId]
+    );
+
+    return this.mapRowsToRecords(rows);
+  }
+
+  /**
+   * Get retry statistics
+   */
+  async getStatistics(days = 30): Promise<RetryStatistics> {
+    const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+
+    const total = await this.dbGet(
+      `SELECT COUNT(*) as count, SUM(actual_attempts) as attempts, SUM(total_duration_ms) as duration
+       FROM retry_history WHERE timestamp >= ?`,
+      [cutoff]
+    );
+
+    const testsRetried = await this.dbGet(
+      `SELECT COUNT(DISTINCT test_id) as count FROM retry_history WHERE timestamp >= ?`,
+      [cutoff]
+    );
+
+    const recovered = await this.dbGet(
+      `SELECT COUNT(*) as count FROM retry_history WHERE success = 1 AND timestamp >= ?`,
+      [cutoff]
+    );
+
+    const failed = await this.dbGet(
+      `SELECT COUNT(*) as count FROM retry_history WHERE success = 0 AND timestamp >= ?`,
+      [cutoff]
+    );
+
+    const byStrategy = await this.dbAll(
+      `SELECT strategy, COUNT(*) as attempts, SUM(success) as successes, AVG(total_duration_ms) as avg_duration
+       FROM retry_history WHERE timestamp >= ?
+       GROUP BY strategy`,
+      [cutoff]
+    );
+
+    const byErrorType = await this.dbAll(
+      `SELECT error_type, COUNT(*) as count FROM retry_history
+       WHERE timestamp >= ? AND error_type IS NOT NULL
+       GROUP BY error_type
+       ORDER BY count DESC`,
+      [cutoff]
+    );
+
+    const byFlakiness = await this.dbGet(
+      `SELECT
+         SUM(CASE WHEN flakiness_score < 50 THEN 1 ELSE 0 END) as low,
+         SUM(CASE WHEN flakiness_score >= 50 AND flakiness_score < 75 THEN 1 ELSE 0 END) as medium,
+         SUM(CASE WHEN flakiness_score >= 75 THEN 1 ELSE 0 END) as high
+       FROM retry_history WHERE timestamp >= ?`,
+      [cutoff]
+    );
+
+    const totalRetries = total?.attempts || 0;
+    const totalRecovered = recovered?.count || 0;
+    const totalFailed = failed?.count || 0;
+    const totalTests = testsRetried?.count || 0;
+
+    const byStrategyMap: Record<RetryStrategy, { attempts: number; successes: number; avgDurationMs: number }> = {
+      [RetryStrategy.NONE]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+      [RetryStrategy.FIXED]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+      [RetryStrategy.LINEAR]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+      [RetryStrategy.EXPONENTIAL]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+      [RetryStrategy.ADAPTIVE]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+      [RetryStrategy.INTELLIGENT]: { attempts: 0, successes: 0, avgDurationMs: 0 }
+    };
+
+    for (const row of byStrategy) {
+      byStrategyMap[row.strategy as RetryStrategy] = {
+        attempts: row.attempts,
+        successes: row.successes,
+        avgDurationMs: Math.round(row.avg_duration)
+      };
+    }
+
+    const byErrorTypeMap: Record<string, number> = {};
+    for (const row of byErrorType) {
+      byErrorTypeMap[row.error_type] = row.count;
+    }
+
+    return {
+      totalRetries,
+      testsRetried: totalTests,
+      recoveredTests: totalRecovered,
+      failedTests: totalFailed,
+      recoveryRate: totalRetries > 0 ? Math.round((totalRecovered / (totalRecovered + totalFailed)) * 100) : 0,
+      avgAttemptsPerTest: totalTests > 0 ? Math.round(totalRetries / totalTests) : 0,
+      totalRetryTimeMs: total?.duration || 0,
+      byStrategy: byStrategyMap,
+      byErrorType: byErrorTypeMap,
+      byFlakinessScore: {
+        low: byFlakiness?.low || 0,
+        medium: byFlakiness?.medium || 0,
+        high: byFlakiness?.high || 0
+      }
+    };
+  }
+
+  /**
+   * Get strategy performance
+   */
+  async getStrategyPerformance(days = 30): Promise<Array<{
+    strategy: RetryStrategy;
+    date: string;
+    attempts: number;
+    successes: number;
+    successRate: number;
+    avgDurationMs: number;
+  }>> {
+    const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+
+    const rows = await this.dbAll(
+      `SELECT strategy, date, attempts, successes, avg_duration_ms
+       FROM retry_strategy_performance
+       WHERE date >= datetime(?, 'unixepoch')
+       ORDER BY date DESC, strategy`,
+      [Math.floor(cutoff / 1000)]
+    );
+
+    return rows.map((r: any) => ({
+      strategy: r.strategy as RetryStrategy,
+      date: r.date,
+      attempts: r.attempts,
+      successes: r.successes,
+      successRate: r.attempts > 0 ? Math.round((r.successes / r.attempts) * 100) : 0,
+      avgDurationMs: Math.round(r.avg_duration_ms)
+    }));
+  }
+
+  /**
+   * Get most retried tests
+   */
+  async getMostRetriedTests(limit = 10, days = 30): Promise<Array<{
+    testId: string;
+    testName: string;
+    gate: string;
+    retryCount: number;
+    recoveryRate: number;
+    avgAttempts: number;
+  }>> {
+    const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+
+    const rows = await this.dbAll(
+      `SELECT
+         test_id,
+         test_name,
+         gate,
+         COUNT(*) as retry_count,
+         SUM(success) as recovered,
+         AVG(actual_attempts) as avg_attempts
+       FROM retry_history
+       WHERE timestamp >= ?
+       GROUP BY test_id, test_name, gate
+       ORDER BY retry_count DESC
+       LIMIT ?`,
+      [cutoff, limit]
+    );
+
+    return rows.map((r: any) => ({
+      testId: r.test_id,
+      testName: r.test_name,
+      gate: r.gate,
+      retryCount: r.retry_count,
+      recoveryRate: r.retry_count > 0 ? Math.round((r.recovered / r.retry_count) * 100) : 0,
+      avgAttempts: Math.round(r.avg_attempts)
+    }));
+  }
+
+  /**
+   * Update daily statistics
+   */
+  private async updateStatistics(result: RetryResult): Promise<void> {
+    const today = new Date().toISOString().split('T')[0];
+
+    // Update main stats
+    await this.dbRun(`
+      INSERT INTO retry_stats (date, total_retries, tests_retried, recovered_tests, failed_tests)
+      VALUES (?, 1, 1, ?, ?)
+      ON CONFLICT(date) DO UPDATE SET
+        total_retries = total_retries + 1,
+        tests_retried = tests_retried + 1,
+        recovered_tests = recovered_tests + ?,
+        failed_tests = failed_tests + ?
+    `, [today, result.success ? 1 : 0, result.success ? 0 : 1, result.success ? 1 : 0, result.success ? 0 : 1]);
+
+    // Update strategy performance
+    await this.dbRun(`
+      INSERT INTO retry_strategy_performance (strategy, date, attempts, successes, avg_duration_ms)
+      VALUES (?, ?, 1, ?, ?)
+      ON CONFLICT(strategy, date) DO UPDATE SET
+        attempts = attempts + 1,
+        successes = successes + ?,
+        avg_duration_ms = (avg_duration_ms * (attempts - 1) + ?) / attempts
+    `, [result.strategy, today, result.success ? 1 : 0, result.totalDurationMs, result.success ? 1 : 0, result.totalDurationMs]);
+  }
+
+  /**
+   * Map database rows to RetryHistoryRecord objects
+   */
+  private mapRowsToRecords(rows: any[]): RetryHistoryRecord[] {
+    return rows.map((r: any) => ({
+      id: r.id,
+      run_id: r.run_id,
+      test_id: r.test_id,
+      test_name: r.test_name,
+      gate: r.gate,
+      strategy: r.strategy,
+      max_retries: r.max_retries,
+      actual_attempts: r.actual_attempts,
+      success: r.success === 1,
+      successful_attempt: r.successful_attempt,
+      total_duration_ms: r.total_duration_ms,
+      stop_reason: r.stop_reason,
+      error_type: r.error_type,
+      error_message: r.error_message,
+      flakiness_score: r.flakiness_score,
+      pattern_type: r.pattern_type,
+      timestamp: r.timestamp
+    }));
+  }
+
+  /**
+   * Delete old retry history
+   */
+  async deleteOldHistory(olderThanMs: number): Promise<number> {
+    const cutoff = Date.now() - olderThanMs;
+
+    const result = await this.dbRun(
+      `DELETE FROM retry_history WHERE timestamp < ?`,
+      [cutoff]
+    );
+
+    return result.changes || 0;
+  }
+}