qa360 2.1.2 → 2.1.4

package/core/src/vault/__tests__/flakiness-vault.test.ts

@@ -0,0 +1,562 @@
+/**
+ * Flakiness Vault Extension Tests
+ *
+ * Tests the flakiness-specific tables and methods in the Evidence Vault.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtemp, rm } from 'fs/promises';
+import { join } from 'path';
+import { EvidenceVault } from '../index.js';
+import { FlakinessCategory, FlakinessPattern } from '../../flakiness/index.js';
+
+describe('EvidenceVault - Flakiness Extension', () => {
+  let tempDir: string;
+  let vault: EvidenceVault;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(process.cwd(), '.tmp-test-'));
+    vault = await EvidenceVault.open(tempDir);
+  });
+
+  afterEach(async () => {
+    try {
+      await vault.close();
+      await rm(tempDir, { recursive: true, force: true });
+    } catch {
+      // Ignore cleanup errors
+    }
+  });
+
+  describe('Flakiness History', () => {
+    it('should record flakiness analysis results', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      const flakinessId = await vault.recordFlakiness(result.runId, {
+        test_id: 'test-123',
+        test_name: 'MyFlakyTest',
+        gate: 'ui',
+        file_path: '/tests/ui.spec.ts',
+        score: 45,
+        category: 'shaky',
+        total_runs: 5,
+        successful_runs: 3,
+        avg_duration_ms: 1200,
+        pattern_type: 'timing',
+        suggested_fix: 'Increase timeout',
+        confidence: 0.8,
+        first_seen: Date.now() - 100000,
+        last_seen: Date.now(),
+        current_streak: 2,
+        streak_type: 'pass'
+      });
+
+      expect(flakinessId).toBeGreaterThan(0);
+
+      const history = await vault.getFlakinessHistory('test-123');
+      expect(history).toHaveLength(1);
+      expect(history[0].test_name).toBe('MyFlakyTest');
+      expect(history[0].score).toBe(45);
+      expect(history[0].category).toBe('shaky');
+    });
+
+    it('should retrieve flakiness history for a run', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      await vault.recordFlakiness(result.runId, {
+        test_id: 'test-1',
+        test_name: 'Test1',
+        gate: 'ui',
+        file_path: '/test1.spec.ts',
+        score: 100,
+        category: 'legendary',
+        total_runs: 3,
+        successful_runs: 3,
+        avg_duration_ms: 500,
+        first_seen: Date.now(),
+        last_seen: Date.now()
+      });
+
+      await vault.recordFlakiness(result.runId, {
+        test_id: 'test-2',
+        test_name: 'Test2',
+        gate: 'api',
+        file_path: '/test2.spec.ts',
+        score: 30,
+        category: 'unstable',
+        total_runs: 5,
+        successful_runs: 2,
+        avg_duration_ms: 1000,
+        first_seen: Date.now(),
+        last_seen: Date.now()
+      });
+
+      const runFlakiness = await vault.getRunFlakiness(result.runId);
+      expect(runFlakiness).toHaveLength(2);
+      // Should be sorted by score ASC (most flaky first)
+      expect(runFlakiness[0].test_id).toBe('test-2');
+      expect(runFlakiness[0].score).toBe(30);
+      expect(runFlakiness[1].test_id).toBe('test-1');
+      expect(runFlakiness[1].score).toBe(100);
+    });
+
+    it('should limit history results', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      const testId = 'test-history';
+      for (let i = 0; i < 10; i++) {
+        await vault.recordFlakiness(result.runId, {
+          test_id: testId,
+          test_name: `Test${i}`,
+          gate: 'ui',
+          file_path: '/test.spec.ts',
+          score: 50 + i * 5,
+          category: 'shaky',
+          total_runs: 3,
+          successful_runs: 2,
+          avg_duration_ms: 500,
+          first_seen: Date.now(),
+          last_seen: Date.now()
+        });
+      }
+
+      const history = await vault.getFlakinessHistory(testId, 5);
+      expect(history.length).toBeLessThanOrEqual(5);
+    });
+
+    it('should track flakiness trends over time', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      const testId = 'test-trends';
+
+      // Simulate improving test over time (within 30 day window)
+      const now = Date.now();
+      // Add more points to ensure trend detection works
+      for (let i = 0; i < 5; i++) {
+        const score = 30 + i * 15; // 30, 45, 60, 75, 90
+        await vault.recordFlakiness(result.runId, {
+          test_id: testId,
+          test_name: 'TrendTest',
+          gate: 'ui',
+          file_path: '/test.spec.ts',
+          score,
+          category: score >= 75 ? 'good' : score >= 50 ? 'shaky' : 'unstable',
+          total_runs: 5,
+          successful_runs: Math.floor((score / 100) * 5),
+          avg_duration_ms: 500,
+          first_seen: now - (5 - i) * 12 * 60 * 60 * 1000, // 5 points spread over 5 days
+          last_seen: now - (5 - i) * 12 * 60 * 60 * 1000
+        });
+      }
+
+      const trends = await vault.getFlakinessTrends(testId, 30);
+      expect(trends.testId).toBe(testId);
+      expect(trends.currentScore).toBe(90);
+      expect(trends.averageScore).toBeGreaterThanOrEqual(30);
+      expect(trends.trend).toBe('improving');
+      expect(trends.dataPoints).toHaveLength(5);
+    });
+  });
+
+  describe('Flakiness Patterns', () => {
+    it('should record new flakiness patterns', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      const patternId = await vault.recordFlakinessPattern({
+        test_id: 'test-456',
+        test_name: 'TimeoutTest',
+        pattern_type: 'timing',
+        description: 'Test times out waiting for selector',
+        suggested_fix: 'Add explicit wait',
+        confidence: 0.9,
+        auto_fix_possible: true
+      });
+
+      expect(patternId).toBeGreaterThan(0);
+
+      const pattern = await vault.getFlakinessPattern('test-456', 'timing');
+      expect(pattern).toBeDefined();
+      expect(pattern!.test_name).toBe('TimeoutTest');
+      expect(pattern!.pattern_type).toBe('timing');
+      expect(pattern!.detection_count).toBe(1);
+    });
+
+    it('should update existing patterns on re-detection', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      const testId = 'test-pattern-update';
+
+      // First detection
+      await vault.recordFlakinessPattern({
+        test_id: testId,
+        test_name: 'PatternTest',
+        pattern_type: 'selector_issue',
+        description: 'Element not found',
+        suggested_fix: 'Use data-testid',
+        confidence: 0.8,
+        auto_fix_possible: true
+      });
+
+      // Second detection (with higher confidence)
+      await vault.recordFlakinessPattern({
+        test_id: testId,
+        test_name: 'PatternTest',
+        pattern_type: 'selector_issue',
+        description: 'Element not found again',
+        suggested_fix: 'Use data-testid',
+        confidence: 0.9,
+        auto_fix_possible: true
+      });
+
+      const pattern = await vault.getFlakinessPattern(testId, 'selector_issue');
+      expect(pattern!.detection_count).toBe(2);
+      // Note: confidence is NOT updated when pattern already exists, only detection_count
+      expect(pattern!.confidence).toBe(0.8);
+    });
+
+    it('should support multiple pattern types per test', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      const testId = 'test-multiple-patterns';
+
+      await vault.recordFlakinessPattern({
+        test_id: testId,
+        test_name: 'MultiPatternTest',
+        pattern_type: 'timing',
+        description: 'Timeout issues',
+        suggested_fix: 'Increase timeout',
+        confidence: 0.8,
+        auto_fix_possible: true
+      });
+
+      await vault.recordFlakinessPattern({
+        test_id: testId,
+        test_name: 'MultiPatternTest',
+        pattern_type: 'race_condition',
+        description: 'Race condition detected',
+        suggested_fix: 'Add synchronization',
+        confidence: 0.7,
+        auto_fix_possible: false
+      });
+
+      const timingPattern = await vault.getFlakinessPattern(testId, 'timing');
+      const racePattern = await vault.getFlakinessPattern(testId, 'race_condition');
+
+      expect(timingPattern).toBeDefined();
+      expect(racePattern).toBeDefined();
+      expect(timingPattern!.id).not.toBe(racePattern!.id);
+    });
+  });
+
+  describe('Flakiness Quarantine', () => {
+    it('should add tests to quarantine', async () => {
+      const quarantineId = await vault.addToQuarantine({
+        test_id: 'test-quarantine-1',
+        test_name: 'VeryFlakyTest',
+        gate: 'ui',
+        file_path: '/tests/flaky.spec.ts',
+        reason: 'Consistently fails - 20% reliability',
+        score: 20,
+        category: 'unstable',
+        quarantined_at: Date.now(),
+        quarantined_by: 'auto'
+      });
+
+      expect(quarantineId).toBeGreaterThan(0);
+
+      const quarantined = await vault.getQuarantinedTests();
+      expect(quarantined).toHaveLength(1);
+      expect(quarantined[0].test_name).toBe('VeryFlakyTest');
+      // resolved_at should be null (SQLite returns null for undefined)
+      expect(quarantined[0].resolved_at).toBeNull();
+    });
+
+    it('should not add same test twice', async () => {
+      const testId = 'test-duplicate-quarantine';
+
+      await vault.addToQuarantine({
+        test_id: testId,
+        test_name: 'DuplicateTest',
+        gate: 'ui',
+        file_path: '/test.spec.ts',
+        reason: 'Flaky',
+        score: 40,
+        category: 'shaky',
+        quarantined_at: Date.now(),
+        quarantined_by: 'auto'
+      });
+
+      const secondId = await vault.addToQuarantine({
+        test_id: testId,
+        test_name: 'DuplicateTest',
+        gate: 'ui',
+        file_path: '/test.spec.ts',
+        reason: 'Flaky again',
+        score: 35,
+        category: 'shaky',
+        quarantined_at: Date.now(),
+        quarantined_by: 'auto'
+      });
+
+      // Should return the existing quarantine ID
+      expect(secondId).toBeGreaterThan(0);
+
+      const quarantined = await vault.getQuarantinedTests();
+      expect(quarantined).toHaveLength(1);
+    });
+
+    it('should remove tests from quarantine', async () => {
+      const testId = 'test-remove-quarantine';
+
+      await vault.addToQuarantine({
+        test_id: testId,
+        test_name: 'RemoveTest',
+        gate: 'ui',
+        file_path: '/test.spec.ts',
+        reason: 'Was flaky',
+        score: 30,
+        category: 'unstable',
+        quarantined_at: Date.now(),
+        quarantined_by: 'auto'
+      });
+
+      await vault.removeFromQuarantine(testId, 'user-123', 'Test fixed by adding explicit waits');
+
+      const quarantine = await vault.getQuarantine(testId);
+      expect(quarantine!.resolved_at).toBeDefined();
+      expect(quarantine!.resolved_by).toBe('user-123');
+      expect(quarantine!.notes).toContain('fixed');
+
+      const activeQuarantine = await vault.getQuarantinedTests(false);
+      expect(activeQuarantine).toHaveLength(0);
+    });
+
+    it('should list resolved quarantines when requested', async () => {
+      await vault.addToQuarantine({
+        test_id: 'test-1',
+        test_name: 'Test1',
+        gate: 'ui',
+        file_path: '/test1.spec.ts',
+        reason: 'Flaky',
+        score: 30,
+        category: 'unstable',
+        quarantined_at: Date.now(),
+        quarantined_by: 'auto'
+      });
+
+      await vault.addToQuarantine({
+        test_id: 'test-2',
+        test_name: 'Test2',
+        gate: 'api',
+        file_path: '/test2.spec.ts',
+        reason: 'Flaky',
+        score: 40,
+        category: 'shaky',
+        quarantined_at: Date.now(),
+        quarantined_by: 'auto'
+      });
+
+      await vault.removeFromQuarantine('test-1', 'user-1', 'Fixed');
+
+      const allQuarantined = await vault.getQuarantinedTests(true);
+      expect(allQuarantined).toHaveLength(2);
+
+      const activeQuarantined = await vault.getQuarantinedTests(false);
+      expect(activeQuarantined).toHaveLength(1);
+      expect(activeQuarantined[0].test_id).toBe('test-2');
+    });
+  });
+
+  describe('Schema Migration', () => {
+    it('should create flakiness tables on init', async () => {
+      // The vault was already initialized in beforeEach
+      // Verify tables exist by inserting and querying
+
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      // Test flakiness_history table
+      await vault.recordFlakiness(result.runId, {
+        test_id: 'test-123',
+        test_name: 'SchemaTest',
+        gate: 'ui',
+        file_path: '/test.spec.ts',
+        score: 100,
+        category: 'legendary',
+        total_runs: 5,
+        successful_runs: 5,
+        avg_duration_ms: 500,
+        first_seen: Date.now(),
+        last_seen: Date.now()
+      });
+
+      // Test flakiness_patterns table
+      await vault.recordFlakinessPattern({
+        test_id: 'test-123',
+        test_name: 'SchemaTest',
+        pattern_type: 'timing',
+        description: 'No pattern',
+        suggested_fix: 'None',
+        confidence: 1.0,
+        auto_fix_possible: false
+      });
+
+      // Test flakiness_quarantine table
+      await vault.addToQuarantine({
+        test_id: 'test-quarantine',
+        test_name: 'QuarantineTest',
+        gate: 'ui',
+        file_path: '/test.spec.ts',
+        reason: 'Test',
+        score: 10,
+        category: 'unstable',
+        quarantined_at: Date.now(),
+        quarantined_by: 'test'
+      });
+
+      // Verify all data is accessible
+      const history = await vault.getFlakinessHistory('test-123');
+      expect(history).toHaveLength(1);
+
+      const pattern = await vault.getFlakinessPattern('test-123', 'timing');
+      expect(pattern).toBeDefined();
+
+      const quarantine = await vault.getQuarantine('test-quarantine');
+      expect(quarantine).toBeDefined();
+    });
+
+    it('should enforce category constraints', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      // Valid categories
+      const validCategories = ['legendary', 'solid', 'good', 'shaky', 'unstable'] as const;
+
+      for (const category of validCategories) {
+        await vault.recordFlakiness(result.runId, {
+          test_id: `test-${category}`,
+          test_name: `${category}Test`,
+          gate: 'ui',
+          file_path: '/test.spec.ts',
+          score: 50,
+          category,
+          total_runs: 5,
+          successful_runs: 3,
+          avg_duration_ms: 500,
+          first_seen: Date.now(),
+          last_seen: Date.now()
+        });
+      }
+
+      const allHistory = await vault.getFlakinessHistory('test-legendary');
+      expect(allHistory).toHaveLength(1);
+    });
+
+    it('should enforce pattern type constraints', async () => {
+      const validPatterns = [
+        'timing',
+        'race_condition',
+        'external_dependency',
+        'environment_specific',
+        'selector_issue',
+        'unknown'
+      ] as const;
+
+      for (const pattern of validPatterns) {
+        await vault.recordFlakinessPattern({
+          test_id: `test-${pattern}`,
+          test_name: `${pattern}Test`,
+          pattern_type: pattern,
+          description: `Test ${pattern}`,
+          suggested_fix: 'Fix it',
+          confidence: 0.5,
+          auto_fix_possible: false
+        });
+      }
+
+      const pattern = await vault.getFlakinessPattern('test-timing', 'timing');
+      expect(pattern).toBeDefined();
+      expect(pattern!.pattern_type).toBe('timing');
+    });
+  });
+
+  describe('Flakiness Stats Queries', () => {
+    it('should get vault stats including flakiness data', async () => {
+      const result = await vault.beginRun({
+        pack_path: 'test.yaml',
+        pack_hash: 'abc123'
+      });
+
+      // Add some flakiness data
+      await vault.recordFlakiness(result.runId, {
+        test_id: 'test-stats',
+        test_name: 'StatsTest',
+        gate: 'ui',
+        file_path: '/test.spec.ts',
+        score: 70,
+        category: 'good',
+        total_runs: 10,
+        successful_runs: 7,
+        avg_duration_ms: 800,
+        first_seen: Date.now(),
+        last_seen: Date.now()
+      });
+
+      await vault.recordFlakinessPattern({
+        test_id: 'test-stats',
+        test_name: 'StatsTest',
+        pattern_type: 'timing',
+        description: 'Timeout',
+        suggested_fix: 'Wait',
+        confidence: 0.8,
+        auto_fix_possible: true
+      });
+
+      await vault.addToQuarantine({
+        test_id: 'test-quarantine-stats',
+        test_name: 'QuarantineStatsTest',
+        gate: 'ui',
+        file_path: '/test.spec.ts',
+        reason: 'Flaky',
+        score: 25,
+        category: 'unstable',
+        quarantined_at: Date.now(),
+        quarantined_by: 'auto'
+      });
+
+      // Finish the run
+      await vault.finishRun(result.runId, {
+        status: 'passed',
+        trust_score: 70
+      });
+
+      const stats = await vault.getStats();
+      expect(stats.totalRuns).toBe(1);
+    });
+  });
+});