qa360 2.1.7 → 2.2.0

package/core/src/flakiness/index.ts

@@ -0,0 +1,536 @@
+/**
+ * QA360 Flakiness Detection Engine
+ *
+ * Detects and scores test flakiness using consecutive runs and pattern analysis.
+ * This is the core differentiator of QA360 - measuring test reliability.
+ *
+ * Scoring Categories:
+ * - Legendary (100): Never flaky - 100% reliable
+ * - Solid (90-99): Rarely flaky
+ * - Good (75-89): Occasionally flaky
+ * - Shaky (50-74): Often flaky - attention needed
+ * - Unstable (0-49): Very flaky - quarantine recommended
+ *
+ * @module flakiness
+ */
+
+import { createHash } from 'crypto';
+
+/**
+ * Flakiness score category
+ */
+export enum FlakinessCategory {
+  LEGENDARY = 'legendary',  // 100
+  SOLID = 'solid',          // 90-99
+  GOOD = 'good',            // 75-89
+  SHAKY = 'shaky',          // 50-74
+  UNSTABLE = 'unstable'     // 0-49
+}
+
+/**
+ * Flakiness category metadata
+ */
+export const FLAKINESS_CATEGORIES: Record<FlakinessCategory, {
+  min: number;
+  max: number;
+  label: string;
+  emoji: string;
+  color: string;
+  description: string;
+  action: string;
+}> = {
+  [FlakinessCategory.LEGENDARY]: {
+    min: 100,
+    max: 100,
+    label: 'LEGENDARY',
+    emoji: '🟢',
+    color: '#10B981',
+    description: 'Jamais flaky - 100% fiable',
+    action: 'None - keep doing what you\'re doing'
+  },
+  [FlakinessCategory.SOLID]: {
+    min: 90,
+    max: 99,
+    label: 'SOLID',
+    emoji: '🟢',
+    color: '#34D399',
+    description: 'Rarement flaky',
+    action: 'Monitor - occasional hiccups'
+  },
+  [FlakinessCategory.GOOD]: {
+    min: 75,
+    max: 89,
+    label: 'GOOD',
+    emoji: '🟡',
+    color: '#FBBF24',
+    description: 'Occasionnellement flaky',
+    action: 'Watch - may need attention'
+  },
+  [FlakinessCategory.SHAKY]: {
+    min: 50,
+    max: 74,
+    label: 'SHAKY',
+    emoji: '🟠',
+    color: '#F97316',
+    description: 'Souvent flaky - attention requise',
+    action: 'Investigate - fix recommended'
+  },
+  [FlakinessCategory.UNSTABLE]: {
+    min: 0,
+    max: 49,
+    label: 'UNSTABLE',
+    emoji: '🔴',
+    color: '#EF4444',
+    description: 'Très flaky - quarantaine recommandée',
+    action: 'Quarantine - unblock the team'
+  }
+};
+
+/**
+ * Test result from a single run
+ */
+export interface TestResult {
+  /** Test identifier (unique per test) */
+  testId: string;
+  /** Test name/title */
+  testName: string;
+  /** File path */
+  filePath: string;
+  /** Gate name */
+  gate: string;
+  /** Whether the test passed */
+  success: boolean;
+  /** Duration in milliseconds */
+  durationMs: number;
+  /** Error type if failed */
+  errorType?: string;
+  /** Error message if failed */
+  errorMessage?: string;
+  /** Timestamp of run */
+  timestamp: number;
+  /** Environment (local, ci, staging) */
+  environment: string;
+  /** Git branch */
+  branch?: string;
+  /** Commit hash */
+  commitHash?: string;
+  /** CI run ID */
+  ciRunId?: string;
+  /** Tags (team, repo, etc.) */
+  tags?: Record<string, string>;
+}
+
+/**
+ * Flakiness detection result for a single test
+ */
+export interface FlakinessResult {
+  /** Test identifier */
+  testId: string;
+  /** Test name */
+  testName: string;
+  /** File path */
+  filePath: string;
+  /** Gate name */
+  gate: string;
+  /** Flakiness score (0-100) */
+  score: number;
+  /** Flakiness category */
+  category: FlakinessCategory;
+  /** Total number of runs analyzed */
+  totalRuns: number;
+  /** Number of successful runs */
+  successfulRuns: number;
+  /** Average duration in ms */
+  avgDurationMs: number;
+  /** Detected pattern type (if any) */
+  patternType?: string;
+  /** Suggested fix */
+  suggestedFix?: string;
+  /** Confidence in pattern detection (0-1) */
+  confidence?: number;
+  /** First seen timestamp */
+  firstSeen?: number;
+  /** Last seen timestamp */
+  lastSeen?: number;
+  /** Current streak (consecutive passes or failures) */
+  currentStreak?: number;
+  /** Streak type */
+  streakType?: 'pass' | 'fail';
+}
+
+/**
+ * Flakiness detection options
+ */
+export interface FlakinessDetectionOptions {
+  /** Minimum runs before calculating score */
+  minRuns?: number;
+  /** Number of consecutive runs for detection */
+  consecutiveRuns?: number;
+  /** Time window in days (only consider runs within this window) */
+  timeWindowDays?: number;
+  /** Enable pattern detection */
+  enablePatternDetection?: boolean;
+}
+
+/**
+ * Default flakiness detection options
+ */
+export const DEFAULT_FLAKINESS_OPTIONS: Required<FlakinessDetectionOptions> = {
+  minRuns: 3,
+  consecutiveRuns: 3,
+  timeWindowDays: 30,
+  enablePatternDetection: true
+};
+
+/**
+ * Flakiness pattern types
+ */
+export enum FlakinessPattern {
+  TIMING = 'timing',
+  RACE_CONDITION = 'race_condition',
+  EXTERNAL_DEPENDENCY = 'external_dependency',
+  ENVIRONMENT_SPECIFIC = 'environment_specific',
+  SELECTOR_ISSUE = 'selector_issue',
+  UNKNOWN = 'unknown'
+}
+
+/**
+ * Pattern detection result
+ */
+export interface PatternDetection {
+  /** Pattern type */
+  patternType: FlakinessPattern;
+  /** Pattern description */
+  description: string;
+  /** Suggested fix */
+  suggestedFix: string;
+  /** Confidence (0-1) */
+  confidence: number;
+  /** Whether auto-fix is possible */
+  autoFixPossible: boolean;
+}
+
+/**
+ * Calculate flakiness score from test results
+ * @param results Array of test results
+ * @returns Flakiness score (0-100)
+ */
+export function calculateFlakinessScore(results: TestResult[]): number {
+  if (results.length === 0) {
+    return 100; // No failures means 100% by default
+  }
+
+  const successCount = results.filter(r => r.success).length;
+  const score = Math.round((successCount / results.length) * 100);
+
+  return Math.max(0, Math.min(100, score));
+}
+
+/**
+ * Get flakiness category from score
+ * @param score Flakiness score (0-100)
+ * @returns Flakiness category
+ */
+export function getFlakinessCategory(score: number): FlakinessCategory {
+  if (score === 100) return FlakinessCategory.LEGENDARY;
+  if (score >= 90) return FlakinessCategory.SOLID;
+  if (score >= 75) return FlakinessCategory.GOOD;
+  if (score >= 50) return FlakinessCategory.SHAKY;
+  return FlakinessCategory.UNSTABLE;
+}
+
+/**
+ * Generate a unique test ID
+ * @param testName Test name
+ * @param filePath File path
+ * @returns Unique test ID
+ */
+export function generateTestId(testName: string, filePath: string): string {
+  const hash = createHash('sha256')
+    .update(`${filePath}:${testName}`)
+    .digest('hex')
+    .substring(0, 16);
+  return hash;
+}
+
+/**
+ * Analyze test results for flakiness patterns
+ * @param results Array of test results for the same test
+ * @returns Pattern detection result or undefined
+ */
+export function detectFlakinessPattern(results: TestResult[]): PatternDetection | undefined {
+  if (results.length < 3) {
+    return undefined;
+  }
+
+  const failures = results.filter(r => !r.success);
+  if (failures.length === 0) {
+    return undefined; // No failures to analyze
+  }
+
+  // Collect error messages as lowercase strings
+  const errorMessages = failures.map(f => (f.errorMessage || '').toLowerCase());
+
+  // Helper to check if any error contains a keyword
+  const anyErrorContains = (keywords: string[]): boolean => {
+    return errorMessages.some(err => keywords.some(kw => err.includes(kw)));
+  };
+
+  // Check for timing issues
+  if (anyErrorContains(['timeout', 'timed out', 'waiting for', 'element not found', 'waiting for selector'])) {
+    return {
+      patternType: FlakinessPattern.TIMING,
+      description: 'Timing issues - test is too fast for the application',
+      suggestedFix: 'Increase timeout or add explicit wait (waitForSelector, waitForLoadState)',
+      confidence: 0.85,
+      autoFixPossible: true
+    };
+  }
+
+  // Check for selector issues
+  if (anyErrorContains(['selector', 'detached', 'stale element', 'element not found'])) {
+    return {
+      patternType: FlakinessPattern.SELECTOR_ISSUE,
+      description: 'Selector issues - element not consistently found',
+      suggestedFix: 'Use data-testid selectors instead of CSS selectors for better reliability',
+      confidence: 0.80,
+      autoFixPossible: true
+    };
+  }
+
+  // Check for external dependency issues
+  if (anyErrorContains(['econnrefused', 'connection refused', 'connection reset', 'network error', 'network unreachable', 'api timeout'])) {
+    return {
+      patternType: FlakinessPattern.EXTERNAL_DEPENDENCY,
+      description: 'External dependency issues - network or service unavailable',
+      suggestedFix: 'Add retry logic with exponential backoff or mock the external dependency',
+      confidence: 0.90,
+      autoFixPossible: true
+    };
+  }
+
+  // Check for race conditions
+  if (anyErrorContains(['race condition', 'state changed', 'unexpected state', 'assertion failed'])) {
+    return {
+      patternType: FlakinessPattern.RACE_CONDITION,
+      description: 'Possible race condition - state changes unexpectedly',
+      suggestedFix: 'Add proper synchronization/locks or use explicit waits',
+      confidence: 0.70,
+      autoFixPossible: false
+    };
+  }
+
+  // Check for environment-specific issues
+  const envs = new Set(results.map(r => r.environment));
+  if (envs.size > 1) {
+    const failEnvs = new Set(failures.map(r => r.environment));
+    const passEnvs = new Set(results.filter(r => r.success).map(r => r.environment));
+
+    // Check if failures are concentrated in one environment
+    for (const env of failEnvs) {
+      if (!passEnvs.has(env)) {
+        return {
+          patternType: FlakinessPattern.ENVIRONMENT_SPECIFIC,
+          description: `Environment-specific issue - fails on ${env} only`,
+          suggestedFix: 'Check environment-specific configuration (OS, timezone, locale)',
+          confidence: 0.75,
+          autoFixPossible: false
+        };
+      }
+    }
+  }
+
+  return {
+    patternType: FlakinessPattern.UNKNOWN,
+    description: 'Flaky behavior detected but pattern unclear',
+    suggestedFix: 'Manual investigation recommended - check logs for specific failure patterns',
+    confidence: 0.50,
+    autoFixPossible: false
+  };
+}
+
+/**
+ * Calculate consecutive streak (passes or failures)
+ * @param results Array of test results sorted by timestamp
+ * @returns Current streak info
+ */
+export function calculateStreak(results: TestResult[]): {
+  currentStreak: number;
+  streakType: 'pass' | 'fail';
+} {
+  if (results.length === 0) {
+    return { currentStreak: 0, streakType: 'pass' };
+  }
+
+  // Sort by timestamp (newest first)
+  const sorted = [...results].sort((a, b) => b.timestamp - a.timestamp);
+
+  let streak = 0;
+  let streakType: 'pass' | 'fail' = sorted[0].success ? 'pass' : 'fail';
+
+  for (const result of sorted) {
+    if (result.success === (streakType === 'pass')) {
+      streak++;
+    } else {
+      break;
+    }
+  }
+
+  return { currentStreak: streak, streakType };
+}
+
+/**
+ * Format flakiness score for display
+ * @param score Flakiness score (0-100)
+ * @returns Formatted string with emoji
+ */
+export function formatFlakinessScore(score: number): string {
+  const category = getFlakinessCategory(score);
+  const meta = FLAKINESS_CATEGORIES[category];
+  return `${meta.emoji} ${category.toUpperCase()} (${score}%)`;
+}
+
+/**
+ * Flakiness Detection Engine
+ *
+ * Main class for detecting and scoring test flakiness
+ */
+export class FlakinessDetector {
+  private options: Required<FlakinessDetectionOptions>;
+
+  constructor(options: FlakinessDetectionOptions = {}) {
+    this.options = { ...DEFAULT_FLAKINESS_OPTIONS, ...options };
+  }
+
+  /**
+   * Analyze a single test's flakiness from its run history
+   * @param testId Test identifier
+   * @param results Array of test results (must be from the same test)
+   * @returns Flakiness result
+   */
+  analyzeTest(testId: string, results: TestResult[]): FlakinessResult {
+    // Filter by time window
+    const timeWindow = this.options.timeWindowDays * 24 * 60 * 60 * 1000;
+    const now = Date.now();
+    const cutoff = now - timeWindow;
+
+    const filteredResults = results.filter(r => r.timestamp >= cutoff);
+
+    // Need minimum runs to calculate score
+    if (filteredResults.length < this.options.minRuns) {
+      return {
+        testId,
+        testName: results[0]?.testName || 'Unknown',
+        filePath: results[0]?.filePath || 'Unknown',
+        gate: results[0]?.gate || 'Unknown',
+        score: 100, // Default to 100 if not enough data
+        category: FlakinessCategory.LEGENDARY,
+        totalRuns: filteredResults.length,
+        successfulRuns: filteredResults.filter(r => r.success).length,
+        avgDurationMs: this.averageDuration(filteredResults)
+      };
+    }
+
+    // Calculate score
+    const score = calculateFlakinessScore(filteredResults);
+    const category = getFlakinessCategory(score);
+
+    // Calculate streak
+    const { currentStreak, streakType } = calculateStreak(filteredResults);
+
+    // Detect patterns if enabled
+    let patternType: string | undefined;
+    let suggestedFix: string | undefined;
+    let confidence: number | undefined;
+
+    if (this.options.enablePatternDetection) {
+      const pattern = detectFlakinessPattern(filteredResults);
+      if (pattern) {
+        patternType = pattern.patternType;
+        suggestedFix = pattern.suggestedFix;
+        confidence = pattern.confidence;
+      }
+    }
+
+    return {
+      testId,
+      testName: results[0].testName,
+      filePath: results[0].filePath,
+      gate: results[0].gate,
+      score,
+      category,
+      totalRuns: filteredResults.length,
+      successfulRuns: filteredResults.filter(r => r.success).length,
+      avgDurationMs: this.averageDuration(filteredResults),
+      patternType,
+      suggestedFix,
+      confidence,
+      firstSeen: Math.min(...filteredResults.map(r => r.timestamp)),
+      lastSeen: Math.max(...filteredResults.map(r => r.timestamp)),
+      currentStreak,
+      streakType
+    };
+  }
+
+  /**
+   * Analyze multiple tests and return flakiness results
+   * @param testResults Map of testId to array of results
+   * @returns Array of flakiness results
+   */
+  analyzeAll(testResults: Map<string, TestResult[]>): FlakinessResult[] {
+    const flakinessResults: FlakinessResult[] = [];
+
+    for (const [testId, testRuns] of testResults.entries()) {
+      if (testRuns.length > 0) {
+        flakinessResults.push(this.analyzeTest(testId, testRuns));
+      }
+    }
+
+    // Sort by score (lowest first) - most flaky first
+    flakinessResults.sort((a, b) => a.score - b.score);
+
+    return flakinessResults;
+  }
+
+  /**
+   * Check if a test should be quarantined based on flakiness
+   * @param result Flakiness result
+   * @returns Whether to quarantine
+   */
+  shouldQuarantine(result: FlakinessResult): boolean {
+    // Quarantine if:
+    // 1. Score is below 50 (Unstable)
+    // 2. Score is below 75 (Shaky) AND has more than 5 runs
+    return (
+      result.category === FlakinessCategory.UNSTABLE ||
+      (result.category === FlakinessCategory.SHAKY && result.totalRuns >= 5)
+    );
+  }
+
+  /**
+   * Get quarantine recommendation message
+   * @param result Flakiness result
+   * @returns Recommendation message
+   */
+  getQuarantineRecommendation(result: FlakinessResult): string {
+    if (!this.shouldQuarantine(result)) {
+      return `Test is ${FLAKINESS_CATEGORIES[result.category].emoji} ${result.category} - no quarantine needed`;
+    }
+
+    return [
+      `❌ Test "${result.testName}" is ${FLAKINESS_CATEGORIES[result.category].emoji} ${FLAKINESS_CATEGORIES[result.category].label} (${result.score}% flaky)`,
+      `   Passed ${result.successfulRuns}/${result.totalRuns} times`,
+      result.suggestedFix ? `   💡 Suggested: ${result.suggestedFix}` : '',
+      `   📋 Action: ${FLAKINESS_CATEGORIES[result.category].action}`
+    ].filter(Boolean).join('\n');
+  }
+
+  /**
+   * Calculate average duration
+   */
+  private averageDuration(results: TestResult[]): number {
+    if (results.length === 0) return 0;
+    const total = results.reduce((sum, r) => sum + r.durationMs, 0);
+    return Math.round(total / results.length);
+  }
+}

package/core/src/generation/__tests__/code-formatter.test.ts

@@ -0,0 +1,170 @@
+/**
+ * Code Formatter Unit Tests
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest';
+import { CodeFormatter } from '../code-formatter.js';
+
+describe('CodeFormatter', () => {
+  let formatter: CodeFormatter;
+
+  beforeEach(() => {
+    formatter = new CodeFormatter();
+  });
+
+  describe('formatTypeScript', () => {
+    it('should format TypeScript code', () => {
+      const code = `import{foo}from'./foo';const x=foo();`;
+      const result = formatter.formatTypeScript(code);
+
+      expect(result).toContain('import');
+      expect(result).toContain('from');
+    });
+
+    it('should normalize quotes to single', () => {
+      const code = 'const x = "hello";';
+      const result = formatter.formatTypeScript(code);
+
+      // Should have single quotes
+      expect(result).toContain("'");
+    });
+
+    it('should remove trailing whitespace', () => {
+      const code = `const x = 1;   \nconst y = 2;`;
+      const result = formatter.formatTypeScript(code);
+
+      expect(result).toContain('const x = 1;\nconst y = 2;');
+    });
+  });
+
+  describe('formatJavaScript', () => {
+    it('should format JavaScript code', () => {
+      const code = `const x=1;`;
+      const result = formatter.formatJavaScript(code);
+
+      expect(result).toBeTruthy();
+    });
+  });
+
+  describe('formatGo', () => {
+    it('should normalize Go indentation to tabs', () => {
+      const code = `func main() {\n    fmt.Println("hello")\n}`;
+      const result = formatter.formatGo(code);
+
+      // Go uses tabs
+      expect(result).toContain('\tfmt.Println');
+    });
+
+    it('should ensure file ends with newline', () => {
+      const code = 'package main';
+      const result = formatter.formatGo(code);
+
+      // Check for trailing newline
+      expect(result.slice(-1)).toBe('\n');
+    });
+  });
+
+  describe('formatPython', () => {
+    it('should normalize Python indentation to 4 spaces', () => {
+      const code = `def test():\n  print('hello')`;
+      const result = formatter.formatPython(code);
+
+      expect(result).toContain('    print');
+    });
+
+    it('should ensure file ends with newline', () => {
+      const code = 'def test(): pass';
+      const result = formatter.formatPython(code);
+
+      expect(result.slice(-1)).toBe('\n');
+    });
+  });
+
+  describe('validate', () => {
+    it('should detect unbalanced braces', () => {
+      const code = 'function test() { return true;';
+      const result = formatter.validate(code, 'typescript');
+
+      expect(result.errors.length).toBeGreaterThan(0);
+      expect(result.errors[0]).toContain('braces');
+    });
+
+    it('should detect unbalanced parentheses', () => {
+      const code = 'const x = (1 + 2;';
+      const result = formatter.validate(code, 'typescript');
+
+      expect(result.errors.length).toBeGreaterThan(0);
+      expect(result.errors[0]).toContain('parentheses');
+    });
+
+    it('should detect empty code', () => {
+      const result = formatter.validate('', 'typescript');
+
+      expect(result.errors).toContain('Code is empty');
+    });
+
+    it('should warn about console.log', () => {
+      const code = `
+        console.log('test');
+        console.log('test2');
+        console.log('test3');
+      `;
+      const result = formatter.validate(code, 'typescript');
+
+      expect(result.warnings.length).toBeGreaterThan(0);
+      expect(result.warnings[0]).toContain('console.log');
+    });
+
+    it('should warn about any type', () => {
+      const code = 'const x: any = 1;';
+      const result = formatter.validate(code, 'typescript');
+
+      expect(result.warnings.length).toBeGreaterThan(0);
+      expect(result.warnings[0]).toContain('any');
+    });
+
+    it('should return no errors for valid code', () => {
+      const code = `
+        import { test } from 'vitest';
+
+        test('example', () => {
+          expect(1 + 1).toBe(2);
+        });
+      `;
+      const result = formatter.validate(code, 'typescript');
+
+      expect(result.errors).toHaveLength(0);
+    });
+  });
+
+  describe('configuration', () => {
+    it('should use custom indent size', () => {
+      const formatter = new CodeFormatter({ indentSize: 4 });
+      const code = 'const x = 1;';
+      const result = formatter.format(code, 'typescript');
+
+      expect(result).toBeTruthy();
+    });
+
+    it('should use double quotes when configured', () => {
+      const formatter = new CodeFormatter({ quotes: 'double' });
+      const code = "const x = 'hello';";
+      const result = formatter.formatTypeScript(code);
+
+      // With double quotes preference, should have more double quotes
+      expect(result).toContain('"');
+    });
+
+    it('should respect sortImports option', () => {
+      const noSortFormatter = new CodeFormatter({ sortImports: false });
+      const code = `
+        import { c } from './c';
+        import { a } from './a';
+      `;
+      const result = noSortFormatter.formatTypeScript(code);
+
+      // Should not reorder when sort is disabled
+      expect(result).toBeTruthy();
+    });
+  });
+});