qa360 2.1.7 → 2.2.0

package/core/src/runner/__tests__/phase3-e2e-b2bshop.test.ts

@@ -0,0 +1,218 @@
+/**
+ * QA360 Internal E2E Tests - B2BShop
+ *
+ * Real E2E tests using the B2BShop dataset to validate QA360 auth integration.
+ * These tests ensure QA360 can authenticate and run UI tests on real applications.
+ *
+ * Run with: ENABLE_E2E_TESTS=1 pnpm test:core -- e2e-b2bshop
+ */
+
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { mkdirSync, rmSync, existsSync } from 'fs';
+import { describe, it, expect, beforeEach, afterEach, beforeAll } from 'vitest';
+import { Phase3Runner } from '../phase3-runner.js';
+import type { PackConfigV2 } from '../../types/pack-v2.js';
+import { skipIfE2ETargetUnhealthy, healthCheckE2ETarget } from '../e2e-helpers.js';
+
+// Import B2BShop dataset (only in non-production environments)
+const getB2bShopPack = async (): Promise<PackConfigV2> => {
+  if (process.env.NODE_ENV === 'production') {
+    throw new Error('E2E tests with real credentials should not run in production');
+  }
+
+  // Try multiple possible paths for the dataset
+  const possiblePaths = [
+    '../../../dataset/b2bshop.js',
+    '../../../../dataset/b2bshop.js',
+    // When running from monorepo root with NODE_PATH set
+    'dataset/b2bshop'
+  ];
+
+  let lastError: Error | null = null;
+
+  for (const importPath of possiblePaths) {
+    try {
+      const module = await import(importPath);
+      return module.b2bshopPackV2 as PackConfigV2;
+    } catch (error) {
+      lastError = error as Error;
+    }
+  }
+
+  throw new Error(
+    `B2BShop dataset not found. Searched paths: ${possiblePaths.join(', ')}. ` +
+    `Make sure the dataset directory exists at the project root.`
+  );
+};
+
+describe('Phase3Runner - Internal E2E (B2BShop)', () => {
+  let testDir: string;
+
+  // Timeout for real network requests
+  const NETWORK_TIMEOUT = 60000; // 60s
+
+  // Skip these tests by default - enable with ENABLE_E2E_TESTS=1
+  const shouldRun = process.env.ENABLE_E2E_TESTS === '1';
+
+  // Health check before all B2BShop tests
+  beforeAll(async () => {
+    if (!shouldRun) return;
+
+    const health = await healthCheckE2ETarget('b2bshop');
+    if (!health.available) {
+      console.warn(`\n⚠️  B2BShop E2E tests skipped: site is ${health.error ? `unreachable (${health.error})` : `down (status: ${health.status})`}`);
+    }
+  });
+
+  let siteHealthy = true;
+
+  beforeEach(async () => {
+    if (!shouldRun) return;
+
+    // Check site health before each test
+    const health = await healthCheckE2ETarget('b2bshop');
+    siteHealthy = health.available;
+
+    if (!siteHealthy) {
+      return; // Skip test setup
+    }
+
+    testDir = join(tmpdir(), `qa360-e2e-b2bshop-${Date.now()}-${Math.random()}`);
+    mkdirSync(testDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    if (existsSync(testDir)) {
+      rmSync(testDir, { recursive: true, force: true });
+    }
+  });
+
+  describe.skipIf(!shouldRun)('UI Login Authentication', () => {
+    beforeEach(async () => {
+      if (!shouldRun || !siteHealthy) return;
+      await skipIfE2ETargetUnhealthy('b2bshop');
+    });
+
+    it('should authenticate with B2BShop using UI login', async () => {
+      const pack = await getB2bShopPack();
+
+      const runner = new Phase3Runner({
+        workingDir: testDir,
+        pack
+      });
+
+      const result = await runner.run();
+
+      // Verify execution completed
+      expect(result).toBeDefined();
+      expect(result.gates).toHaveLength(1);
+
+      // Check if gate executed
+      const gate = result.gates[0];
+      expect(gate.gate).toBe('ui_e2e');
+      expect(gate.duration).toBeGreaterThanOrEqual(0);
+    }, NETWORK_TIMEOUT);
+
+    it('should generate proof bundle after E2E test', async () => {
+      const pack = await getB2bShopPack();
+
+      const runner = new Phase3Runner({
+        workingDir: testDir,
+        pack
+      });
+
+      const result = await runner.run();
+
+      // Proof should be generated
+      expect(result.proofPath).toBeDefined();
+
+      // Check proof file exists
+      if (result.proofPath) {
+        const fs = await import('fs');
+        expect(fs.existsSync(result.proofPath)).toBe(true);
+      }
+    }, NETWORK_TIMEOUT);
+  });
+
+  describe.skipIf(!shouldRun)('Multi-page Navigation', () => {
+    it('should navigate through multiple B2BShop pages', async () => {
+      const basePack = await getB2bShopPack();
+
+      // Test with multiple pages
+      const multiPagePack: PackConfigV2 = {
+        ...basePack,
+        gates: {
+          ui_navigation: {
+            adapter: 'playwright-ui',
+            enabled: true,
+            auth: 'b2bshop-login',
+            config: {
+              baseUrl: 'https://b2bshop.lovable.app',
+              pages: [
+                'https://b2bshop.lovable.app/catalog',
+                'https://b2bshop.lovable.app/products',
+                'https://b2bshop.lovable.app/cart'
+              ]
+            }
+          }
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: testDir,
+        pack: multiPagePack
+      });
+
+      const result = await runner.run();
+
+      expect(result.gates).toHaveLength(1);
+      expect(result.gates[0].gate).toBe('ui_navigation');
+      expect(result.gates[0].duration).toBeGreaterThanOrEqual(0);
+    }, NETWORK_TIMEOUT);
+  });
+
+  describe.skipIf(!shouldRun)('Auth Credentials Caching', () => {
+    it('should cache auth credentials across multiple gates', async () => {
+      const basePack = await getB2bShopPack();
+
+      // Test with multiple gates using same auth
+      const multiGatePack: PackConfigV2 = {
+        ...basePack,
+        gates: {
+          catalog_page: {
+            adapter: 'playwright-ui',
+            enabled: true,
+            auth: 'b2bshop-login',
+            config: {
+              baseUrl: 'https://b2bshop.lovable.app',
+              pages: ['https://b2bshop.lovable.app/catalog']
+            }
+          },
+          products_page: {
+            adapter: 'playwright-ui',
+            enabled: true,
+            auth: 'b2bshop-login',
+            config: {
+              baseUrl: 'https://b2bshop.lovable.app',
+              pages: ['https://b2bshop.lovable.app/products']
+            }
+          }
+        }
+      };
+
+      const runner = new Phase3Runner({
+        workingDir: testDir,
+        pack: multiGatePack
+      });
+
+      const result = await runner.run();
+
+      expect(result.gates).toHaveLength(2);
+
+      // Both gates should use cached auth
+      expect(result.gates[0].gate).toBe('catalog_page');
+      expect(result.gates[1].gate).toBe('products_page');
+    }, NETWORK_TIMEOUT * 2);
+  });
+});

package/core/src/runner/__tests__/phase3-e2e-reqres.test.ts

@@ -0,0 +1,199 @@
+/**
+ * QA360 Internal E2E Tests - Reqres.in
+ *
+ * Real E2E tests using Reqres.in API to validate QA360 with bearer auth.
+ * Reqres.in is a stable, free fake API for testing.
+ *
+ * Run with: ENABLE_E2E_TESTS=1 pnpm test:core -- e2e-reqres
+ */
+
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { mkdirSync, rmSync, existsSync } from 'fs';
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { Phase3Runner } from '../phase3-runner.js';
+import type { PackConfigV2 } from '../../types/pack-v2.js';
+
+// Import Reqres.in dataset (only in non-production environments)
+const getReqresPack = async (): Promise<PackConfigV2> => {
+  if (process.env.NODE_ENV === 'production') {
+    throw new Error('E2E tests should not run in production');
+  }
+
+  const possiblePaths = [
+    '../../../dataset/reqres.js',
+    '../../../../dataset/reqres.js',
+    'dataset/reqres'
+  ];
+
+  let lastError: Error | null = null;
+
+  for (const importPath of possiblePaths) {
+    try {
+      const module = await import(importPath);
+      return module.reqresPackV2 as PackConfigV2;
+    } catch (error) {
+      lastError = error as Error;
+    }
+  }
+
+  throw new Error(
+    `Reqres dataset not found. Searched paths: ${possiblePaths.join(', ')}.`
+  );
+};
+
+describe('Phase3Runner - Internal E2E (Reqres.in)', () => {
+  let testDir: string;
+
+  const NETWORK_TIMEOUT = 60000;
+  const shouldRun = process.env.ENABLE_E2E_TESTS === '1';
+
+  beforeEach(() => {
+    if (!shouldRun) return;
+    testDir = join(tmpdir(), `qa360-e2e-reqres-${Date.now()}-${Math.random()}`);
+    mkdirSync(testDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    if (existsSync(testDir)) {
+      rmSync(testDir, { recursive: true, force: true });
+    }
+  });
+
+  describe.skipIf(!shouldRun)('API Smoke Tests', () => {
+    it('should run API smoke tests with bearer auth', async () => {
+      const pack = await getReqresPack();
+
+      const runner = new Phase3Runner({
+        workingDir: testDir,
+        pack
+      });
+
+      const result = await runner.run();
+
+      expect(result).toBeDefined();
+      expect(result.gates.length).toBeGreaterThan(0);
+      expect(result.gates[0].success).toBe(true);
+    }, NETWORK_TIMEOUT);
+  });
+
+  describe.skipIf(!shouldRun)('Auth Scenarios', () => {
+    it('should run without auth when not specified', async () => {
+      const noAuthPack: PackConfigV2 = {
+        version: 2,
+        name: 'reqres-no-auth',
+        gates: {
+          'public-api': {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://reqres.in/api',
+              smoke: ['GET /users -> 200', 'GET /users/2 -> 200']
+            }
+          }
+        }
+      };
+
+      const runner = new Phase3Runner({ workingDir: testDir, pack: noAuthPack });
+      const result = await runner.run();
+
+      expect(result.gates).toHaveLength(1);
+      expect(result.gates[0].success).toBe(true);
+    }, NETWORK_TIMEOUT);
+
+    it('should use bearer auth with token', async () => {
+      const pack = await getReqresPack();
+      const runner = new Phase3Runner({
+        workingDir: testDir,
+        pack: {
+          ...pack,
+          gates: { 'api-health': pack.gates['api-health'] }
+        }
+      });
+
+      const result = await runner.run();
+      expect(result.gates[0].success).toBe(true);
+    }, NETWORK_TIMEOUT);
+  });
+
+  describe.skipIf(!shouldRun)('Error Scenarios', () => {
+    it('should handle 404 responses correctly', async () => {
+      const errorPack: PackConfigV2 = {
+        version: 2,
+        name: 'reqres-errors',
+        gates: {
+          'error-handling': {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://reqres.in/api',
+              smoke: ['GET /users/999 -> 404']
+            }
+          }
+        }
+      };
+
+      const runner = new Phase3Runner({ workingDir: testDir, pack: errorPack });
+      const result = await runner.run();
+
+      expect(result.gates[0].success).toBe(true);
+    }, NETWORK_TIMEOUT);
+
+    it('should handle timeout with delayed response', async () => {
+      const timeoutPack: PackConfigV2 = {
+        version: 2,
+        name: 'reqres-timeout',
+        gates: {
+          'delayed-test': {
+            adapter: 'playwright-api',
+            enabled: true,
+            config: {
+              baseUrl: 'https://reqres.in/api',
+              smoke: ['GET /users?delay=2 -> 200'],
+              timeout: 5000
+            }
+          }
+        }
+      };
+
+      const runner = new Phase3Runner({ workingDir: testDir, pack: timeoutPack });
+      const result = await runner.run();
+
+      expect(result.gates[0].success).toBe(true);
+    }, NETWORK_TIMEOUT);
+  });
+
+  describe.skipIf(!shouldRun)('Multiple Gates', () => {
+    it('should run multiple gates sequentially', async () => {
+      const pack = await getReqresPack();
+      const runner = new Phase3Runner({ workingDir: testDir, pack });
+      const result = await runner.run();
+
+      expect(result.gates.length).toBeGreaterThan(1);
+      const passedGates = result.gates.filter(g => g.success).length;
+      expect(passedGates).toBeGreaterThan(0);
+    }, NETWORK_TIMEOUT * 2);
+  });
+
+  describe.skipIf(!shouldRun)('Proof Generation', () => {
+    it('should generate and store proof bundle', async () => {
+      const pack = await getReqresPack();
+      const runner = new Phase3Runner({
+        workingDir: testDir,
+        pack: {
+          ...pack,
+          gates: { 'api-health': pack.gates['api-health'] }
+        }
+      });
+
+      const result = await runner.run();
+
+      expect(result.proofPath).toBeDefined();
+
+      if (result.proofPath) {
+        const fs = await import('fs');
+        expect(fs.existsSync(result.proofPath)).toBe(true);
+      }
+    }, NETWORK_TIMEOUT);
+  });
+});