qa360 2.2.1 → 2.2.13

package/core/src/flakiness/__tests__/flakiness.test.ts

@@ -1,554 +0,0 @@
-/**
- * Flakiness Detection Engine Tests
- */
-
-import { describe, it, expect } from 'vitest';
-import {
-  calculateFlakinessScore,
-  getFlakinessCategory,
-  generateTestId,
-  detectFlakinessPattern,
-  calculateStreak,
-  formatFlakinessScore,
-  FlakinessDetector,
-  FlakinessCategory,
-  FlakinessPattern,
-  type TestResult,
-  type FlakinessResult,
-  DEFAULT_FLAKINESS_OPTIONS
-} from '../index.js';
-
-describe('FlakinessDetection', () => {
-  describe('calculateFlakinessScore', () => {
-    it('should return 100 for empty results', () => {
-      const score = calculateFlakinessScore([]);
-      expect(score).toBe(100);
-    });
-
-    it('should calculate correct percentage for all passes', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: true }),
-        createTestResult({ success: true }),
-        createTestResult({ success: true })
-      ];
-      const score = calculateFlakinessScore(results);
-      expect(score).toBe(100);
-    });
-
-    it('should calculate correct percentage for mixed results', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: true }),
-        createTestResult({ success: false }),
-        createTestResult({ success: true })
-      ];
-      const score = calculateFlakinessScore(results);
-      expect(score).toBe(67); // 2/3 = 66.67% rounded
-    });
-
-    it('should return 0 for all failures', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: false }),
-        createTestResult({ success: false }),
-        createTestResult({ success: false })
-      ];
-      const score = calculateFlakinessScore(results);
-      expect(score).toBe(0);
-    });
-
-    it('should clamp scores between 0 and 100', () => {
-      // Edge case testing
-      const singlePass: TestResult[] = [createTestResult({ success: true })];
-      expect(calculateFlakinessScore(singlePass)).toBe(100);
-
-      const singleFail: TestResult[] = [createTestResult({ success: false })];
-      expect(calculateFlakinessScore(singleFail)).toBe(0);
-    });
-  });
-
-  describe('getFlakinessCategory', () => {
-    it('should return LEGENDARY for score 100', () => {
-      expect(getFlakinessCategory(100)).toBe(FlakinessCategory.LEGENDARY);
-    });
-
-    it('should return SOLID for scores 90-99', () => {
-      expect(getFlakinessCategory(90)).toBe(FlakinessCategory.SOLID);
-      expect(getFlakinessCategory(95)).toBe(FlakinessCategory.SOLID);
-      expect(getFlakinessCategory(99)).toBe(FlakinessCategory.SOLID);
-    });
-
-    it('should return GOOD for scores 75-89', () => {
-      expect(getFlakinessCategory(75)).toBe(FlakinessCategory.GOOD);
-      expect(getFlakinessCategory(80)).toBe(FlakinessCategory.GOOD);
-      expect(getFlakinessCategory(89)).toBe(FlakinessCategory.GOOD);
-    });
-
-    it('should return SHAKY for scores 50-74', () => {
-      expect(getFlakinessCategory(50)).toBe(FlakinessCategory.SHAKY);
-      expect(getFlakinessCategory(60)).toBe(FlakinessCategory.SHAKY);
-      expect(getFlakinessCategory(74)).toBe(FlakinessCategory.SHAKY);
-    });
-
-    it('should return UNSTABLE for scores 0-49', () => {
-      expect(getFlakinessCategory(0)).toBe(FlakinessCategory.UNSTABLE);
-      expect(getFlakinessCategory(25)).toBe(FlakinessCategory.UNSTABLE);
-      expect(getFlakinessCategory(49)).toBe(FlakinessCategory.UNSTABLE);
-    });
-  });
-
-  describe('generateTestId', () => {
-    it('should generate consistent IDs for same test', () => {
-      const id1 = generateTestId('myTest', '/path/to/test.ts');
-      const id2 = generateTestId('myTest', '/path/to/test.ts');
-      expect(id1).toBe(id2);
-    });
-
-    it('should generate different IDs for different tests', () => {
-      const id1 = generateTestId('testA', '/path/to/test.ts');
-      const id2 = generateTestId('testB', '/path/to/test.ts');
-      expect(id1).not.toBe(id2);
-    });
-
-    it('should generate different IDs for same test in different files', () => {
-      const id1 = generateTestId('myTest', '/path/to/testA.ts');
-      const id2 = generateTestId('myTest', '/path/to/testB.ts');
-      expect(id1).not.toBe(id2);
-    });
-
-    it('should generate 16-character hex strings', () => {
-      const id = generateTestId('test', 'file.ts');
-      expect(id).toMatch(/^[a-f0-9]{16}$/);
-    });
-  });
-
-  describe('detectFlakinessPattern', () => {
-    it('should return undefined for insufficient results', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: false }),
-        createTestResult({ success: false })
-      ];
-      const pattern = detectFlakinessPattern(results);
-      expect(pattern).toBeUndefined();
-    });
-
-    it('should detect timing issues from error messages', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: false, errorMessage: 'timeout exceeded' }),
-        createTestResult({ success: true }),
-        createTestResult({ success: false, errorMessage: 'timed out waiting for selector' })
-      ];
-      const pattern = detectFlakinessPattern(results);
-      expect(pattern?.patternType).toBe(FlakinessPattern.TIMING);
-      expect(pattern?.description).toContain('Timing');
-      expect(pattern?.autoFixPossible).toBe(true);
-      expect(pattern?.confidence).toBeGreaterThan(0.8);
-    });
-
-    it('should detect selector issues from error messages', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: false, errorMessage: 'selector not found' }),
-        createTestResult({ success: true }),
-        createTestResult({ success: false, errorMessage: 'element detached from DOM' })
-      ];
-      const pattern = detectFlakinessPattern(results);
-      expect(pattern?.patternType).toBe(FlakinessPattern.SELECTOR_ISSUE);
-      expect(pattern?.suggestedFix).toContain('data-testid');
-      expect(pattern?.autoFixPossible).toBe(true);
-    });
-
-    it('should detect external dependency issues', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: false, errorMessage: 'ECONNREFUSED' }),
-        createTestResult({ success: true }),
-        createTestResult({ success: false, errorMessage: 'connection refused' })
-      ];
-      const pattern = detectFlakinessPattern(results);
-      expect(pattern?.patternType).toBe(FlakinessPattern.EXTERNAL_DEPENDENCY);
-      expect(pattern?.suggestedFix).toContain('retry');
-      expect(pattern?.confidence).toBeGreaterThan(0.85);
-    });
-
-    it('should detect race conditions', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: false, errorMessage: 'race condition detected' }),
-        createTestResult({ success: true }),
-        createTestResult({ success: false, errorMessage: 'state changed unexpectedly' })
-      ];
-      const pattern = detectFlakinessPattern(results);
-      expect(pattern?.patternType).toBe(FlakinessPattern.RACE_CONDITION);
-      expect(pattern?.autoFixPossible).toBe(false);
-      expect(pattern?.description).toContain('race condition');
-    });
-
-    it('should detect environment-specific issues', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: true, environment: 'local' }),
-        createTestResult({ success: false, environment: 'ci' }),
-        createTestResult({ success: true, environment: 'local' }),
-        createTestResult({ success: false, environment: 'ci' })
-      ];
-      const pattern = detectFlakinessPattern(results);
-      expect(pattern?.patternType).toBe(FlakinessPattern.ENVIRONMENT_SPECIFIC);
-      expect(pattern?.description).toContain('ci');
-      expect(pattern?.autoFixPossible).toBe(false);
-    });
-
-    it('should return unknown pattern for unclear issues', () => {
-      const results: TestResult[] = [
-        createTestResult({ success: false, errorMessage: 'unknown error' }),
-        createTestResult({ success: true }),
-        createTestResult({ success: false, errorMessage: 'something broke' })
-      ];
-      const pattern = detectFlakinessPattern(results);
-      expect(pattern?.patternType).toBe(FlakinessPattern.UNKNOWN);
-      expect(pattern?.confidence).toBeLessThanOrEqual(0.6);
-      expect(pattern?.autoFixPossible).toBe(false);
-    });
-  });
-
-  describe('calculateStreak', () => {
-    it('should return empty streak for empty results', () => {
-      const streak = calculateStreak([]);
-      expect(streak).toEqual({ currentStreak: 0, streakType: 'pass' });
-    });
-
-    it('should calculate consecutive pass streak', () => {
-      const now = Date.now();
-      const results: TestResult[] = [
-        createTestResult({ success: true, timestamp: now }),
-        createTestResult({ success: true, timestamp: now - 1000 }),
-        createTestResult({ success: false, timestamp: now - 2000 })
-      ];
-      const streak = calculateStreak(results);
-      expect(streak.streakType).toBe('pass');
-      expect(streak.currentStreak).toBe(2);
-    });
-
-    it('should calculate consecutive fail streak', () => {
-      const now = Date.now();
-      const results: TestResult[] = [
-        createTestResult({ success: false, timestamp: now }),
-        createTestResult({ success: false, timestamp: now - 1000 }),
-        createTestResult({ success: true, timestamp: now - 2000 })
-      ];
-      const streak = calculateStreak(results);
-      expect(streak.streakType).toBe('fail');
-      expect(streak.currentStreak).toBe(2);
-    });
-
-    it('should handle all passes', () => {
-      const now = Date.now();
-      const results: TestResult[] = [
-        createTestResult({ success: true, timestamp: now }),
-        createTestResult({ success: true, timestamp: now - 1000 }),
-        createTestResult({ success: true, timestamp: now - 2000 })
-      ];
-      const streak = calculateStreak(results);
-      expect(streak.streakType).toBe('pass');
-      expect(streak.currentStreak).toBe(3);
-    });
-  });
-
-  describe('formatFlakinessScore', () => {
-    it('should format legendary score', () => {
-      const formatted = formatFlakinessScore(100);
-      expect(formatted).toContain('LEGENDARY');
-      expect(formatted).toContain('100%');
-      expect(formatted).toContain('🟢');
-    });
-
-    it('should format solid score', () => {
-      const formatted = formatFlakinessScore(95);
-      expect(formatted).toContain('SOLID');
-      expect(formatted).toContain('95%');
-    });
-
-    it('should format unstable score', () => {
-      const formatted = formatFlakinessScore(30);
-      expect(formatted).toContain('UNSTABLE');
-      expect(formatted).toContain('30%');
-      expect(formatted).toContain('🔴');
-    });
-  });
-
-  describe('FlakinessDetector', () => {
-    it('should use default options', () => {
-      const detector = new FlakinessDetector();
-      expect(detector).toBeDefined();
-    });
-
-    it('should analyze test with minimum runs', () => {
-      const detector = new FlakinessDetector({ minRuns: 2 });
-      const results: TestResult[] = [
-        createTestResult({
-          testName: 'myTest',
-          filePath: '/test.ts',
-          gate: 'api_smoke',
-          success: true
-        }),
-        createTestResult({
-          testName: 'myTest',
-          filePath: '/test.ts',
-          gate: 'api_smoke',
-          success: false
-        })
-      ];
-
-      const analysis = detector.analyzeTest('test-id', results);
-      expect(analysis.testId).toBe('test-id');
-      expect(analysis.score).toBe(50);
-      expect(analysis.category).toBe(FlakinessCategory.SHAKY);
-    });
-
-    it('should return legendary for insufficient data', () => {
-      const detector = new FlakinessDetector({ minRuns: 5 });
-      const results: TestResult[] = [
-        createTestResult({
-          testName: 'myTest',
-          filePath: '/test.ts',
-          gate: 'api_smoke',
-          success: true
-        })
-      ];
-
-      const analysis = detector.analyzeTest('test-id', results);
-      expect(analysis.category).toBe(FlakinessCategory.LEGENDARY);
-      expect(analysis.score).toBe(100);
-    });
-
-    it('should analyze all tests and sort by score', () => {
-      const detector = new FlakinessDetector({ minRuns: 2 });
-      const testResults = new Map<string, TestResult[]>();
-
-      testResults.set('test1', [
-        createTestResult({ testName: 'Test 1', success: true, gate: 'api', filePath: '/test1.ts' }),
-        createTestResult({ testName: 'Test 1', success: true, gate: 'api', filePath: '/test1.ts' })
-      ]);
-
-      testResults.set('test2', [
-        createTestResult({ testName: 'Test 2', success: true, gate: 'ui', filePath: '/test2.ts' }),
-        createTestResult({ testName: 'Test 2', success: false, gate: 'ui', filePath: '/test2.ts' })
-      ]);
-
-      const results = detector.analyzeAll(testResults);
-      expect(results.length).toBeGreaterThanOrEqual(1);
-      // Shaky (50%) should come first
-      expect(results[0].score).toBe(50);
-      expect(results[0].testId).toBeDefined();
-    });
-
-    it('should recommend quarantine for unstable tests', () => {
-      const detector = new FlakinessDetector({ minRuns: 2 });
-      const result: FlakinessResult = {
-        testId: 'test1',
-        testName: 'Unstable Test',
-        filePath: '/test.ts',
-        gate: 'api',
-        score: 30,
-        category: FlakinessCategory.UNSTABLE,
-        totalRuns: 5,
-        successfulRuns: 1,
-        avgDurationMs: 100
-      };
-
-      expect(detector.shouldQuarantine(result)).toBe(true);
-    });
-
-    it('should recommend quarantine for shaky tests with 5+ runs', () => {
-      const detector = new FlakinessDetector({ minRuns: 2 });
-      const result: FlakinessResult = {
-        testId: 'test1',
-        testName: 'Shaky Test',
-        filePath: '/test.ts',
-        gate: 'api',
-        score: 60,
-        category: FlakinessCategory.SHAKY,
-        totalRuns: 5,
-        successfulRuns: 3,
-        avgDurationMs: 100
-      };
-
-      expect(detector.shouldQuarantine(result)).toBe(true);
-    });
-
-    it('should not quarantine shaky tests with less than 5 runs', () => {
-      const detector = new FlakinessDetector({ minRuns: 2 });
-      const result: FlakinessResult = {
-        testId: 'test1',
-        testName: 'Shaky Test',
-        filePath: '/test.ts',
-        gate: 'api',
-        score: 60,
-        category: FlakinessCategory.SHAKY,
-        totalRuns: 3,
-        successfulRuns: 2,
-        avgDurationMs: 100
-      };
-
-      expect(detector.shouldQuarantine(result)).toBe(false);
-    });
-
-    it('should not quarantine good or better tests', () => {
-      const detector = new FlakinessDetector({ minRuns: 2 });
-
-      const goodResult: FlakinessResult = {
-        testId: 'test1',
-        testName: 'Good Test',
-        filePath: '/test.ts',
-        gate: 'api',
-        score: 80,
-        category: FlakinessCategory.GOOD,
-        totalRuns: 5,
-        successfulRuns: 4,
-        avgDurationMs: 100
-      };
-
-      expect(detector.shouldQuarantine(goodResult)).toBe(false);
-    });
-
-    it('should generate quarantine recommendation message', () => {
-      const detector = new FlakinessDetector({ minRuns: 2 });
-      const result: FlakinessResult = {
-        testId: 'test1',
-        testName: 'Bad Test',
-        filePath: '/test.ts',
-        gate: 'api',
-        score: 20,
-        category: FlakinessCategory.UNSTABLE,
-        totalRuns: 10,
-        successfulRuns: 2,
-        avgDurationMs: 150,
-        suggestedFix: 'Add retry logic'
-      };
-
-      const message = detector.getQuarantineRecommendation(result);
-      expect(message).toContain('Bad Test');
-      expect(message).toContain('UNSTABLE');
-      expect(message).toContain('20%');
-      expect(message).toContain('2/10');
-      expect(message).toContain('retry logic');
-    });
-
-    it('should include pattern detection when enabled', () => {
-      const detector = new FlakinessDetector({
-        minRuns: 2,
-        enablePatternDetection: true
-      });
-
-      const results: TestResult[] = [
-        createTestResult({
-          testName: 'Timeout Test',
-          gate: 'ui',
-          filePath: '/test.ts',
-          success: false,
-          errorMessage: 'timeout exceeded'
-        }),
-        createTestResult({
-          testName: 'Timeout Test',
-          gate: 'ui',
-          filePath: '/test.ts',
-          success: true
-        }),
-        createTestResult({
-          testName: 'Timeout Test',
-          gate: 'ui',
-          filePath: '/test.ts',
-          success: false,
-          errorMessage: 'timed out'
-        })
-      ];
-
-      const analysis = detector.analyzeTest('test-id', results);
-      // Pattern detection needs at least 3 results with failures
-      expect(analysis.patternType).toBe('timing');
-      expect(analysis.suggestedFix).toBeDefined();
-      expect(analysis.confidence).toBeDefined();
-    });
-
-    it('should skip pattern detection when disabled', () => {
-      const detector = new FlakinessDetector({
-        minRuns: 2,
-        enablePatternDetection: false
-      });
-
-      const results: TestResult[] = [
-        createTestResult({
-          testName: 'Timeout Test',
-          gate: 'ui',
-          success: false,
-          errorMessage: 'timeout exceeded'
-        }),
-        createTestResult({
-          testName: 'Timeout Test',
-          gate: 'ui',
-          success: true
-        })
-      ];
-
-      const analysis = detector.analyzeTest('test-id', results);
-      expect(analysis.patternType).toBeUndefined();
-      expect(analysis.suggestedFix).toBeUndefined();
-      expect(analysis.confidence).toBeUndefined();
-    });
-
-    it('should filter results by time window', () => {
-      const detector = new FlakinessDetector({
-        minRuns: 1,
-        timeWindowDays: 1
-      });
-
-      const now = Date.now();
-      const dayInMs = 24 * 60 * 60 * 1000;
-
-      const results: TestResult[] = [
-        createTestResult({
-          testName: 'Recent Test',
-          gate: 'api',
-          success: true,
-          timestamp: now
-        }),
-        createTestResult({
-          testName: 'Recent Test',
-          gate: 'api',
-          success: true,
-          timestamp: now - dayInMs / 2
-        }),
-        // Old result should be filtered
-        createTestResult({
-          testName: 'Recent Test',
-          gate: 'api',
-          success: false,
-          timestamp: now - dayInMs * 2
-        })
-      ];
-
-      const analysis = detector.analyzeTest('test-id', results);
-      // Only 2 results within time window
-      expect(analysis.totalRuns).toBe(2);
-      expect(analysis.score).toBe(100);
-    });
-  });
-
-  describe('DEFAULT_FLAKINESS_OPTIONS', () => {
-    it('should have sensible defaults', () => {
-      expect(DEFAULT_FLAKINESS_OPTIONS.minRuns).toBe(3);
-      expect(DEFAULT_FLAKINESS_OPTIONS.consecutiveRuns).toBe(3);
-      expect(DEFAULT_FLAKINESS_OPTIONS.timeWindowDays).toBe(30);
-      expect(DEFAULT_FLAKINESS_OPTIONS.enablePatternDetection).toBe(true);
-    });
-  });
-});
-
-// Helper function to create test results
-function createTestResult(overrides: Partial<TestResult> = {}): TestResult {
-  return {
-    testId: 'test-id',
-    testName: 'test',
-    filePath: '/test.ts',
-    gate: 'api_smoke',
-    success: true,
-    durationMs: 100,
-    timestamp: Date.now(),
-    environment: 'local',
-    ...overrides
-  };
-}