qa360 2.1.7 → 2.2.0

package/core/src/runner/__tests__/flakiness-integration.test.ts

@@ -0,0 +1,566 @@
+/**
+ * Flakiness Detection Integration Tests
+ *
+ * Tests the integration between Phase3Runner and FlakinessDetector
+ * to ensure consecutive runs and scoring work correctly.
+ */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { mkdtemp, rm } from 'fs/promises';
+import { join } from 'path';
+import { Phase3Runner } from '../phase3-runner.js';
+import { PackConfigV2 } from '../../types/pack-v2.js';
+import { FlakinessCategory } from '../../flakiness/index.js';
+import { EvidenceVault } from '../../vault/index.js';
+import { initializeKeys } from '../../proof/signer.js';
+
+describe('Flakiness Detection Integration', () => {
+  let tempDir: string;
+  let vaultDir: string;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(process.cwd(), '.tmp-test-'));
+    vaultDir = join(tempDir, '.qa360');
+    // Initialize keys for testing
+    await initializeKeys();
+  });
+
+  afterEach(async () => {
+    try {
+      await rm(tempDir, { recursive: true, force: true });
+    } catch {
+      // Ignore cleanup errors
+    }
+  });
+
+  describe('Phase3Runner with flakyDetect enabled', () => {
+    it('should include flakiness results when enabled', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Flakiness Test Pack',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200',
+                'GET /status/404 -> 404'
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 2 // Reduce runs for faster testing
+      });
+
+      const result = await runner.run();
+
+      expect(result).toBeDefined();
+      expect(result.flakiness).toBeDefined();
+      expect(Array.isArray(result.flakiness)).toBe(true);
+
+      // Should have flakiness results for each test
+      if (result.flakiness && result.flakiness.length > 0) {
+        const firstFlaky = result.flakiness[0];
+        expect(firstFlaky.testId).toBeDefined();
+        expect(firstFlaky.score).toBeGreaterThanOrEqual(0);
+        expect(firstFlaky.score).toBeLessThanOrEqual(100);
+        expect(firstFlaky.category).toBeDefined();
+        expect(firstFlaky.totalRuns).toBeGreaterThan(0);
+      }
+    }, 60000);
+
+    it('should not include flakiness results when disabled', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'No Flakiness Test Pack',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200'
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: false
+      });
+
+      const result = await runner.run();
+
+      expect(result.flakiness).toBeUndefined();
+    }, 60000);
+
+    it('should calculate correct scores based on consecutive runs', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Score Calculation Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200'  // Should always pass
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 3
+      });
+
+      const result = await runner.run();
+
+      expect(result.flakiness).toBeDefined();
+
+      // Find a stable test (should have 100% score)
+      const stableTests = result.flakiness?.filter(f => f.score === 100);
+      expect(stableTests?.length).toBeGreaterThan(0);
+
+      if (stableTests && stableTests.length > 0) {
+        const stableTest = stableTests[0];
+        expect(stableTest.category).toBe(FlakinessCategory.LEGENDARY);
+        expect(stableTest.successfulRuns).toBe(stableTest.totalRuns);
+      }
+    }, 90000);
+
+    it('should detect patterns from failed tests', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Pattern Detection Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200',
+                'GET /status/404 -> 404',
+                'GET /delay/5 -> 200'  // May timeout
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 2000  // Low timeout to trigger failures
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 2
+      });
+
+      const result = await runner.run();
+
+      expect(result.flakiness).toBeDefined();
+
+      // Check that unstable tests have pattern detection
+      const unstableTests = result.flakiness?.filter(f => f.score < 100);
+      if (unstableTests && unstableTests.length > 0) {
+        const unstable = unstableTests[0];
+        // Pattern detection may or may not be present depending on the error
+        if (unstable.patternType) {
+          expect(unstable.suggestedFix).toBeDefined();
+          expect(unstable.confidence).toBeGreaterThan(0);
+        }
+      }
+    }, 90000);
+
+    it('should save flakiness findings to vault', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Vault Storage Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200',
+                'GET /status/500 -> 500'  // Intentional failure
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 2
+      });
+
+      await runner.run();
+
+      // Check vault for flakiness findings
+      const vault = await EvidenceVault.open(vaultDir);
+      const runs = await vault.listRuns({ limit: 1 });
+
+      expect(runs.length).toBeGreaterThan(0);
+
+      const runId = runs[0].id;
+      const findings = await vault.getFindings(runId);
+
+      // Check that flaky tests are recorded as findings
+      const flakyFindings = findings.filter(f => f.rule === 'flaky-test-detected');
+
+      // We may have flaky findings if tests were unstable
+      if (flakyFindings.length > 0) {
+        expect(flakyFindings[0].severity).toMatch(/^(high|medium)$/);
+        expect(flakyFindings[0].message).toContain('reliability');
+      }
+    }, 60000);
+  });
+
+  describe('extractTestResults integration', () => {
+    it('should extract results from Playwright API adapter', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Extract Results Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200',
+                'GET /status/201 -> 201',
+                'POST /post -> 200'
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 2
+      });
+
+      const result = await runner.run();
+
+      expect(result.flakiness).toBeDefined();
+      expect(result.flakiness?.length).toBeGreaterThan(0);
+
+      // Each HTTP request should be tracked as a separate test
+      const testNames = result.flakiness?.map(f => f.testName);
+      expect(testNames).toContain('GET /status/200 -> 200');
+    }, 60000);
+
+    it('should handle mixed pass/fail scenarios', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Mixed Results Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200'  // Stable
+              ]
+            }
+          },
+          api_unstable: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://this-domain-does-not-exist-invalid-12345.com',
+              smoke: [
+                'GET /test -> 200'  // Will always fail (DNS error)
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 5000,
+          on_failure: 'continue'
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 2
+      });
+
+      const result = await runner.run();
+
+      expect(result.flakiness).toBeDefined();
+
+      // Find tests with different scores
+      const legendary = result.flakiness?.filter(f => f.category === FlakinessCategory.LEGENDARY);
+      const unstable = result.flakiness?.filter(f => f.category === FlakinessCategory.UNSTABLE);
+
+      // We should have at least one stable test
+      expect(legendary?.length).toBeGreaterThan(0);
+
+      // We should have at least one unstable test (DNS failures)
+      expect(unstable?.length).toBeGreaterThan(0);
+
+      // Verify the unstable test has a low score
+      if (unstable && unstable.length > 0) {
+        expect(unstable[0].score).toBeLessThan(50); // Failed tests = low reliability
+      }
+    }, 60000);
+  });
+
+  describe('Flakiness streak tracking', () => {
+    it('should track consecutive pass/fail streaks', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Streak Tracking Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200'
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 3
+      });
+
+      const result = await runner.run();
+
+      expect(result.flakiness).toBeDefined();
+      expect(result.flakiness?.length).toBeGreaterThan(0);
+
+      // Check that at least one test has streak info
+      // Streak info is present when minRuns is reached (which is 2 in Phase3Runner)
+      const testWithStreak = result.flakiness?.find(f => f.currentStreak !== undefined);
+
+      if (testWithStreak) {
+        expect(testWithStreak.currentStreak).toBeGreaterThan(0);
+        expect(testWithStreak.currentStreak).toBeLessThanOrEqual(3);
+        expect(testWithStreak.streakType).toMatch(/^(pass|fail)$/);
+      } else {
+        // If no streak info, it means minRuns wasn't reached, but we should still have flakiness data
+        expect(result.flakiness?.length).toBeGreaterThan(0);
+      }
+    }, 60000);
+  });
+
+  describe('Flakiness quarantine recommendations', () => {
+    it('should recommend quarantine for unstable tests', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Quarantine Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200',
+                'GET /status/500 -> 500',
+                'GET /status/503 -> 503'
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 3
+      });
+
+      const result = await runner.run();
+
+      expect(result.flakiness).toBeDefined();
+
+      // Find unstable or shaky tests
+      const needsQuarantine = result.flakiness?.filter(f =>
+        f.category === FlakinessCategory.UNSTABLE ||
+        f.category === FlakinessCategory.SHAKY
+      );
+
+      if (needsQuarantine && needsQuarantine.length > 0) {
+        // Verify these tests have proper metadata for quarantine
+        const test = needsQuarantine[0];
+        expect(test.score).toBeLessThan(75);
+        expect(test.totalRuns).toBeGreaterThanOrEqual(2);
+      }
+    }, 60000);
+  });
+
+  describe('Edge cases', () => {
+    it('should handle empty gate results gracefully', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Empty Gates Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: []  // No tests
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 2
+      });
+
+      const result = await runner.run();
+
+      // Should complete without error even with no tests
+      expect(result).toBeDefined();
+      // Flakiness may be undefined or empty
+      if (result.flakiness) {
+        expect(result.flakiness.length).toBe(0);
+      }
+    }, 60000);
+
+    it('should handle single run correctly', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Single Run Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://httpbin.org',
+              smoke: [
+                'GET /status/200 -> 200'
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 10000
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 1  // Only 1 run - should still work
+      });
+
+      const result = await runner.run();
+
+      expect(result.flakiness).toBeDefined();
+      // With only 1 run and minRuns=2 default, tests may default to LEGENDARY
+      expect(result.flakiness?.length).toBeGreaterThan(0);
+    }, 60000);
+
+    it('should handle adapter errors gracefully', async () => {
+      const pack: PackConfigV2 = {
+        version: 2,
+        name: 'Adapter Error Test',
+        gates: {
+          api_smoke: {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://this-domain-does-not-exist-12345.com',
+              smoke: [
+                'GET /test -> 200'
+              ]
+            }
+          }
+        },
+        execution: {
+          default_timeout: 5000,
+          on_failure: 'continue'
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: tempDir,
+        pack,
+        flakyDetect: true,
+        flakyRuns: 2
+      });
+
+      const result = await runner.run();
+
+      // Should complete without crashing
+      expect(result).toBeDefined();
+      expect(result.flakiness).toBeDefined();
+      // Tests should be marked as unstable due to connection errors
+      const unstable = result.flakiness?.filter(f => f.category === FlakinessCategory.UNSTABLE);
+      expect(unstable?.length).toBeGreaterThan(0);
+    }, 60000);
+  });
+});