qa360 2.1.7 → 2.2.0

package/core/src/flakiness/__tests__/flakiness.test.ts

@@ -0,0 +1,554 @@
+/**
+ * Flakiness Detection Engine Tests
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  calculateFlakinessScore,
+  getFlakinessCategory,
+  generateTestId,
+  detectFlakinessPattern,
+  calculateStreak,
+  formatFlakinessScore,
+  FlakinessDetector,
+  FlakinessCategory,
+  FlakinessPattern,
+  type TestResult,
+  type FlakinessResult,
+  DEFAULT_FLAKINESS_OPTIONS
+} from '../index.js';
+
+describe('FlakinessDetection', () => {
+  describe('calculateFlakinessScore', () => {
+    it('should return 100 for empty results', () => {
+      const score = calculateFlakinessScore([]);
+      expect(score).toBe(100);
+    });
+
+    it('should calculate correct percentage for all passes', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: true }),
+        createTestResult({ success: true }),
+        createTestResult({ success: true })
+      ];
+      const score = calculateFlakinessScore(results);
+      expect(score).toBe(100);
+    });
+
+    it('should calculate correct percentage for mixed results', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: true }),
+        createTestResult({ success: false }),
+        createTestResult({ success: true })
+      ];
+      const score = calculateFlakinessScore(results);
+      expect(score).toBe(67); // 2/3 = 66.67% rounded
+    });
+
+    it('should return 0 for all failures', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: false }),
+        createTestResult({ success: false }),
+        createTestResult({ success: false })
+      ];
+      const score = calculateFlakinessScore(results);
+      expect(score).toBe(0);
+    });
+
+    it('should clamp scores between 0 and 100', () => {
+      // Edge case testing
+      const singlePass: TestResult[] = [createTestResult({ success: true })];
+      expect(calculateFlakinessScore(singlePass)).toBe(100);
+
+      const singleFail: TestResult[] = [createTestResult({ success: false })];
+      expect(calculateFlakinessScore(singleFail)).toBe(0);
+    });
+  });
+
+  describe('getFlakinessCategory', () => {
+    it('should return LEGENDARY for score 100', () => {
+      expect(getFlakinessCategory(100)).toBe(FlakinessCategory.LEGENDARY);
+    });
+
+    it('should return SOLID for scores 90-99', () => {
+      expect(getFlakinessCategory(90)).toBe(FlakinessCategory.SOLID);
+      expect(getFlakinessCategory(95)).toBe(FlakinessCategory.SOLID);
+      expect(getFlakinessCategory(99)).toBe(FlakinessCategory.SOLID);
+    });
+
+    it('should return GOOD for scores 75-89', () => {
+      expect(getFlakinessCategory(75)).toBe(FlakinessCategory.GOOD);
+      expect(getFlakinessCategory(80)).toBe(FlakinessCategory.GOOD);
+      expect(getFlakinessCategory(89)).toBe(FlakinessCategory.GOOD);
+    });
+
+    it('should return SHAKY for scores 50-74', () => {
+      expect(getFlakinessCategory(50)).toBe(FlakinessCategory.SHAKY);
+      expect(getFlakinessCategory(60)).toBe(FlakinessCategory.SHAKY);
+      expect(getFlakinessCategory(74)).toBe(FlakinessCategory.SHAKY);
+    });
+
+    it('should return UNSTABLE for scores 0-49', () => {
+      expect(getFlakinessCategory(0)).toBe(FlakinessCategory.UNSTABLE);
+      expect(getFlakinessCategory(25)).toBe(FlakinessCategory.UNSTABLE);
+      expect(getFlakinessCategory(49)).toBe(FlakinessCategory.UNSTABLE);
+    });
+  });
+
+  describe('generateTestId', () => {
+    it('should generate consistent IDs for same test', () => {
+      const id1 = generateTestId('myTest', '/path/to/test.ts');
+      const id2 = generateTestId('myTest', '/path/to/test.ts');
+      expect(id1).toBe(id2);
+    });
+
+    it('should generate different IDs for different tests', () => {
+      const id1 = generateTestId('testA', '/path/to/test.ts');
+      const id2 = generateTestId('testB', '/path/to/test.ts');
+      expect(id1).not.toBe(id2);
+    });
+
+    it('should generate different IDs for same test in different files', () => {
+      const id1 = generateTestId('myTest', '/path/to/testA.ts');
+      const id2 = generateTestId('myTest', '/path/to/testB.ts');
+      expect(id1).not.toBe(id2);
+    });
+
+    it('should generate 16-character hex strings', () => {
+      const id = generateTestId('test', 'file.ts');
+      expect(id).toMatch(/^[a-f0-9]{16}$/);
+    });
+  });
+
+  describe('detectFlakinessPattern', () => {
+    it('should return undefined for insufficient results', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: false }),
+        createTestResult({ success: false })
+      ];
+      const pattern = detectFlakinessPattern(results);
+      expect(pattern).toBeUndefined();
+    });
+
+    it('should detect timing issues from error messages', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: false, errorMessage: 'timeout exceeded' }),
+        createTestResult({ success: true }),
+        createTestResult({ success: false, errorMessage: 'timed out waiting for selector' })
+      ];
+      const pattern = detectFlakinessPattern(results);
+      expect(pattern?.patternType).toBe(FlakinessPattern.TIMING);
+      expect(pattern?.description).toContain('Timing');
+      expect(pattern?.autoFixPossible).toBe(true);
+      expect(pattern?.confidence).toBeGreaterThan(0.8);
+    });
+
+    it('should detect selector issues from error messages', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: false, errorMessage: 'selector not found' }),
+        createTestResult({ success: true }),
+        createTestResult({ success: false, errorMessage: 'element detached from DOM' })
+      ];
+      const pattern = detectFlakinessPattern(results);
+      expect(pattern?.patternType).toBe(FlakinessPattern.SELECTOR_ISSUE);
+      expect(pattern?.suggestedFix).toContain('data-testid');
+      expect(pattern?.autoFixPossible).toBe(true);
+    });
+
+    it('should detect external dependency issues', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: false, errorMessage: 'ECONNREFUSED' }),
+        createTestResult({ success: true }),
+        createTestResult({ success: false, errorMessage: 'connection refused' })
+      ];
+      const pattern = detectFlakinessPattern(results);
+      expect(pattern?.patternType).toBe(FlakinessPattern.EXTERNAL_DEPENDENCY);
+      expect(pattern?.suggestedFix).toContain('retry');
+      expect(pattern?.confidence).toBeGreaterThan(0.85);
+    });
+
+    it('should detect race conditions', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: false, errorMessage: 'race condition detected' }),
+        createTestResult({ success: true }),
+        createTestResult({ success: false, errorMessage: 'state changed unexpectedly' })
+      ];
+      const pattern = detectFlakinessPattern(results);
+      expect(pattern?.patternType).toBe(FlakinessPattern.RACE_CONDITION);
+      expect(pattern?.autoFixPossible).toBe(false);
+      expect(pattern?.description).toContain('race condition');
+    });
+
+    it('should detect environment-specific issues', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: true, environment: 'local' }),
+        createTestResult({ success: false, environment: 'ci' }),
+        createTestResult({ success: true, environment: 'local' }),
+        createTestResult({ success: false, environment: 'ci' })
+      ];
+      const pattern = detectFlakinessPattern(results);
+      expect(pattern?.patternType).toBe(FlakinessPattern.ENVIRONMENT_SPECIFIC);
+      expect(pattern?.description).toContain('ci');
+      expect(pattern?.autoFixPossible).toBe(false);
+    });
+
+    it('should return unknown pattern for unclear issues', () => {
+      const results: TestResult[] = [
+        createTestResult({ success: false, errorMessage: 'unknown error' }),
+        createTestResult({ success: true }),
+        createTestResult({ success: false, errorMessage: 'something broke' })
+      ];
+      const pattern = detectFlakinessPattern(results);
+      expect(pattern?.patternType).toBe(FlakinessPattern.UNKNOWN);
+      expect(pattern?.confidence).toBeLessThanOrEqual(0.6);
+      expect(pattern?.autoFixPossible).toBe(false);
+    });
+  });
+
+  describe('calculateStreak', () => {
+    it('should return empty streak for empty results', () => {
+      const streak = calculateStreak([]);
+      expect(streak).toEqual({ currentStreak: 0, streakType: 'pass' });
+    });
+
+    it('should calculate consecutive pass streak', () => {
+      const now = Date.now();
+      const results: TestResult[] = [
+        createTestResult({ success: true, timestamp: now }),
+        createTestResult({ success: true, timestamp: now - 1000 }),
+        createTestResult({ success: false, timestamp: now - 2000 })
+      ];
+      const streak = calculateStreak(results);
+      expect(streak.streakType).toBe('pass');
+      expect(streak.currentStreak).toBe(2);
+    });
+
+    it('should calculate consecutive fail streak', () => {
+      const now = Date.now();
+      const results: TestResult[] = [
+        createTestResult({ success: false, timestamp: now }),
+        createTestResult({ success: false, timestamp: now - 1000 }),
+        createTestResult({ success: true, timestamp: now - 2000 })
+      ];
+      const streak = calculateStreak(results);
+      expect(streak.streakType).toBe('fail');
+      expect(streak.currentStreak).toBe(2);
+    });
+
+    it('should handle all passes', () => {
+      const now = Date.now();
+      const results: TestResult[] = [
+        createTestResult({ success: true, timestamp: now }),
+        createTestResult({ success: true, timestamp: now - 1000 }),
+        createTestResult({ success: true, timestamp: now - 2000 })
+      ];
+      const streak = calculateStreak(results);
+      expect(streak.streakType).toBe('pass');
+      expect(streak.currentStreak).toBe(3);
+    });
+  });
+
+  describe('formatFlakinessScore', () => {
+    it('should format legendary score', () => {
+      const formatted = formatFlakinessScore(100);
+      expect(formatted).toContain('LEGENDARY');
+      expect(formatted).toContain('100%');
+      expect(formatted).toContain('🟢');
+    });
+
+    it('should format solid score', () => {
+      const formatted = formatFlakinessScore(95);
+      expect(formatted).toContain('SOLID');
+      expect(formatted).toContain('95%');
+    });
+
+    it('should format unstable score', () => {
+      const formatted = formatFlakinessScore(30);
+      expect(formatted).toContain('UNSTABLE');
+      expect(formatted).toContain('30%');
+      expect(formatted).toContain('🔴');
+    });
+  });
+
+  describe('FlakinessDetector', () => {
+    it('should use default options', () => {
+      const detector = new FlakinessDetector();
+      expect(detector).toBeDefined();
+    });
+
+    it('should analyze test with minimum runs', () => {
+      const detector = new FlakinessDetector({ minRuns: 2 });
+      const results: TestResult[] = [
+        createTestResult({
+          testName: 'myTest',
+          filePath: '/test.ts',
+          gate: 'api_smoke',
+          success: true
+        }),
+        createTestResult({
+          testName: 'myTest',
+          filePath: '/test.ts',
+          gate: 'api_smoke',
+          success: false
+        })
+      ];
+
+      const analysis = detector.analyzeTest('test-id', results);
+      expect(analysis.testId).toBe('test-id');
+      expect(analysis.score).toBe(50);
+      expect(analysis.category).toBe(FlakinessCategory.SHAKY);
+    });
+
+    it('should return legendary for insufficient data', () => {
+      const detector = new FlakinessDetector({ minRuns: 5 });
+      const results: TestResult[] = [
+        createTestResult({
+          testName: 'myTest',
+          filePath: '/test.ts',
+          gate: 'api_smoke',
+          success: true
+        })
+      ];
+
+      const analysis = detector.analyzeTest('test-id', results);
+      expect(analysis.category).toBe(FlakinessCategory.LEGENDARY);
+      expect(analysis.score).toBe(100);
+    });
+
+    it('should analyze all tests and sort by score', () => {
+      const detector = new FlakinessDetector({ minRuns: 2 });
+      const testResults = new Map<string, TestResult[]>();
+
+      testResults.set('test1', [
+        createTestResult({ testName: 'Test 1', success: true, gate: 'api', filePath: '/test1.ts' }),
+        createTestResult({ testName: 'Test 1', success: true, gate: 'api', filePath: '/test1.ts' })
+      ]);
+
+      testResults.set('test2', [
+        createTestResult({ testName: 'Test 2', success: true, gate: 'ui', filePath: '/test2.ts' }),
+        createTestResult({ testName: 'Test 2', success: false, gate: 'ui', filePath: '/test2.ts' })
+      ]);
+
+      const results = detector.analyzeAll(testResults);
+      expect(results.length).toBeGreaterThanOrEqual(1);
+      // Shaky (50%) should come first
+      expect(results[0].score).toBe(50);
+      expect(results[0].testId).toBeDefined();
+    });
+
+    it('should recommend quarantine for unstable tests', () => {
+      const detector = new FlakinessDetector({ minRuns: 2 });
+      const result: FlakinessResult = {
+        testId: 'test1',
+        testName: 'Unstable Test',
+        filePath: '/test.ts',
+        gate: 'api',
+        score: 30,
+        category: FlakinessCategory.UNSTABLE,
+        totalRuns: 5,
+        successfulRuns: 1,
+        avgDurationMs: 100
+      };
+
+      expect(detector.shouldQuarantine(result)).toBe(true);
+    });
+
+    it('should recommend quarantine for shaky tests with 5+ runs', () => {
+      const detector = new FlakinessDetector({ minRuns: 2 });
+      const result: FlakinessResult = {
+        testId: 'test1',
+        testName: 'Shaky Test',
+        filePath: '/test.ts',
+        gate: 'api',
+        score: 60,
+        category: FlakinessCategory.SHAKY,
+        totalRuns: 5,
+        successfulRuns: 3,
+        avgDurationMs: 100
+      };
+
+      expect(detector.shouldQuarantine(result)).toBe(true);
+    });
+
+    it('should not quarantine shaky tests with less than 5 runs', () => {
+      const detector = new FlakinessDetector({ minRuns: 2 });
+      const result: FlakinessResult = {
+        testId: 'test1',
+        testName: 'Shaky Test',
+        filePath: '/test.ts',
+        gate: 'api',
+        score: 60,
+        category: FlakinessCategory.SHAKY,
+        totalRuns: 3,
+        successfulRuns: 2,
+        avgDurationMs: 100
+      };
+
+      expect(detector.shouldQuarantine(result)).toBe(false);
+    });
+
+    it('should not quarantine good or better tests', () => {
+      const detector = new FlakinessDetector({ minRuns: 2 });
+
+      const goodResult: FlakinessResult = {
+        testId: 'test1',
+        testName: 'Good Test',
+        filePath: '/test.ts',
+        gate: 'api',
+        score: 80,
+        category: FlakinessCategory.GOOD,
+        totalRuns: 5,
+        successfulRuns: 4,
+        avgDurationMs: 100
+      };
+
+      expect(detector.shouldQuarantine(goodResult)).toBe(false);
+    });
+
+    it('should generate quarantine recommendation message', () => {
+      const detector = new FlakinessDetector({ minRuns: 2 });
+      const result: FlakinessResult = {
+        testId: 'test1',
+        testName: 'Bad Test',
+        filePath: '/test.ts',
+        gate: 'api',
+        score: 20,
+        category: FlakinessCategory.UNSTABLE,
+        totalRuns: 10,
+        successfulRuns: 2,
+        avgDurationMs: 150,
+        suggestedFix: 'Add retry logic'
+      };
+
+      const message = detector.getQuarantineRecommendation(result);
+      expect(message).toContain('Bad Test');
+      expect(message).toContain('UNSTABLE');
+      expect(message).toContain('20%');
+      expect(message).toContain('2/10');
+      expect(message).toContain('retry logic');
+    });
+
+    it('should include pattern detection when enabled', () => {
+      const detector = new FlakinessDetector({
+        minRuns: 2,
+        enablePatternDetection: true
+      });
+
+      const results: TestResult[] = [
+        createTestResult({
+          testName: 'Timeout Test',
+          gate: 'ui',
+          filePath: '/test.ts',
+          success: false,
+          errorMessage: 'timeout exceeded'
+        }),
+        createTestResult({
+          testName: 'Timeout Test',
+          gate: 'ui',
+          filePath: '/test.ts',
+          success: true
+        }),
+        createTestResult({
+          testName: 'Timeout Test',
+          gate: 'ui',
+          filePath: '/test.ts',
+          success: false,
+          errorMessage: 'timed out'
+        })
+      ];
+
+      const analysis = detector.analyzeTest('test-id', results);
+      // Pattern detection needs at least 3 results with failures
+      expect(analysis.patternType).toBe('timing');
+      expect(analysis.suggestedFix).toBeDefined();
+      expect(analysis.confidence).toBeDefined();
+    });
+
+    it('should skip pattern detection when disabled', () => {
+      const detector = new FlakinessDetector({
+        minRuns: 2,
+        enablePatternDetection: false
+      });
+
+      const results: TestResult[] = [
+        createTestResult({
+          testName: 'Timeout Test',
+          gate: 'ui',
+          success: false,
+          errorMessage: 'timeout exceeded'
+        }),
+        createTestResult({
+          testName: 'Timeout Test',
+          gate: 'ui',
+          success: true
+        })
+      ];
+
+      const analysis = detector.analyzeTest('test-id', results);
+      expect(analysis.patternType).toBeUndefined();
+      expect(analysis.suggestedFix).toBeUndefined();
+      expect(analysis.confidence).toBeUndefined();
+    });
+
+    it('should filter results by time window', () => {
+      const detector = new FlakinessDetector({
+        minRuns: 1,
+        timeWindowDays: 1
+      });
+
+      const now = Date.now();
+      const dayInMs = 24 * 60 * 60 * 1000;
+
+      const results: TestResult[] = [
+        createTestResult({
+          testName: 'Recent Test',
+          gate: 'api',
+          success: true,
+          timestamp: now
+        }),
+        createTestResult({
+          testName: 'Recent Test',
+          gate: 'api',
+          success: true,
+          timestamp: now - dayInMs / 2
+        }),
+        // Old result should be filtered
+        createTestResult({
+          testName: 'Recent Test',
+          gate: 'api',
+          success: false,
+          timestamp: now - dayInMs * 2
+        })
+      ];
+
+      const analysis = detector.analyzeTest('test-id', results);
+      // Only 2 results within time window
+      expect(analysis.totalRuns).toBe(2);
+      expect(analysis.score).toBe(100);
+    });
+  });
+
+  describe('DEFAULT_FLAKINESS_OPTIONS', () => {
+    it('should have sensible defaults', () => {
+      expect(DEFAULT_FLAKINESS_OPTIONS.minRuns).toBe(3);
+      expect(DEFAULT_FLAKINESS_OPTIONS.consecutiveRuns).toBe(3);
+      expect(DEFAULT_FLAKINESS_OPTIONS.timeWindowDays).toBe(30);
+      expect(DEFAULT_FLAKINESS_OPTIONS.enablePatternDetection).toBe(true);
+    });
+  });
+});
+
+// Helper function to create test results
+function createTestResult(overrides: Partial<TestResult> = {}): TestResult {
+  return {
+    testId: 'test-id',
+    testName: 'test',
+    filePath: '/test.ts',
+    gate: 'api_smoke',
+    success: true,
+    durationMs: 100,
+    timestamp: Date.now(),
+    environment: 'local',
+    ...overrides
+  };
+}