qa360 2.2.1 → 2.2.7

package/core/src/generation/__tests__/generation-e2e-b2bshop.test.ts

@@ -1,718 +0,0 @@
-/**
- * QA360 AI Generation E2E Tests - B2BShop
- *
- * Real E2E tests for AI-powered test generation using the B2BShop application.
- * Tests the complete workflow: URL discovery → test generation → pack creation → execution.
- *
- * Prerequisites:
- * - ENABLE_E2E_TESTS=1 environment variable
- * - B2BShop site must be accessible
- *
- * Run with: ENABLE_E2E_TESTS=1 pnpm test:core -- generation-e2e-b2bshop
- */
-
-import { tmpdir } from 'os';
-import { join } from 'path';
-import { mkdirSync, rmSync, existsSync, writeFileSync, readFileSync } from 'fs';
-import { describe, it, expect, beforeEach, afterEach, beforeAll } from 'vitest';
-import { Phase3Runner } from '../../runner/phase3-runner.js';
-import type { PackConfigV2 } from '../../types/pack-v2.js';
-import { skipIfE2ETargetUnhealthy, healthCheckE2ETarget } from '../../runner/e2e-helpers.js';
-import { PackGenerator } from '../pack-generator.js';
-import { CodeGenerator } from '../code-generator.js';
-import { MockProvider } from '../../ai/mock-provider.js';
-import type { UiTestSpec, ApiTestSpec } from '../types.js';
-
-// E2E test configuration
-const B2BSHOP_URL = 'https://b2bshop.lovable.app';
-const NETWORK_TIMEOUT = 120000; // 2 minutes for AI generation
-const shouldRun = process.env.ENABLE_E2E_TESTS === '1';
-
-describe('AI Generation E2E (B2BShop)', () => {
-  let testDir: string;
-  let siteHealthy = true;
-
-  beforeAll(async () => {
-    if (!shouldRun) return;
-
-    const health = await healthCheckE2ETarget('b2bshop');
-    if (!health.available) {
-      console.warn(`\n⚠️  B2BShop E2E tests skipped: site is ${health.error ? `unreachable (${health.error})` : `down (status: ${health.status})`}`);
-    }
-  });
-
-  beforeEach(async () => {
-    if (!shouldRun) return;
-
-    const health = await healthCheckE2ETarget('b2bshop');
-    siteHealthy = health.available;
-
-    if (!siteHealthy) return;
-
-    testDir = join(tmpdir(), `qa360-e2e-gen-b2bshop-${Date.now()}-${Math.random()}`);
-    mkdirSync(testDir, { recursive: true });
-  });
-
-  afterEach(() => {
-    if (existsSync(testDir)) {
-      rmSync(testDir, { recursive: true, force: true });
-    }
-  });
-
-  describe.skipIf(!shouldRun)('URL Discovery and Analysis', () => {
-    it('should discover B2BShop pages from URL', async () => {
-      // This test simulates discovering the site structure
-      // In real implementation, this would use a crawler
-
-      const discoveredPages = [
-        { name: 'Homepage', url: `${B2BSHOP_URL}/`, actions: [] },
-        { name: 'Catalog', url: `${B2BSHOP_URL}/catalog`, actions: [] },
-        { name: 'Products', url: `${B2BSHOP_URL}/products`, actions: [] },
-        { name: 'Cart', url: `${B2BSHOP_URL}/cart`, actions: [] },
-        { name: 'Login', url: `${B2BSHOP_URL}/login`, actions: [] }
-      ];
-
-      expect(discoveredPages).toHaveLength(5);
-      expect(discoveredPages[0].url).toBe(B2BSHOP_URL + '/');
-    });
-
-    it('should analyze B2BShop API endpoints', async () => {
-      // Simulated API discovery for B2BShop
-      const discoveredEndpoints = [
-        { method: 'GET' as const, path: '/api/health', expectations: { status: 200 } },
-        { method: 'GET' as const, path: '/api/products', expectations: { status: 200 } },
-        { method: 'GET' as const, path: '/api/products/:id', expectations: { status: 200 } },
-        { method: 'POST' as const, path: '/api/cart', expectations: { status: 201 } }
-      ];
-
-      expect(discoveredEndpoints).toHaveLength(4);
-    });
-  });
-
-  describe.skipIf(!shouldRun)('Spec Generation from B2BShop', () => {
-    it('should generate UI test spec from B2BShop URL', () => {
-      const uiSpec: UiTestSpec = {
-        type: 'ui',
-        name: 'B2BShop UI Tests',
-        baseUrl: B2BSHOP_URL,
-        viewport: { width: 1920, height: 1080 },
-        pages: [
-          {
-            name: 'Homepage',
-            url: `${B2BSHOP_URL}/`,
-            actions: [
-              { type: 'goto', selector: '/' },
-              { type: 'wait', timeout: 2000 },
-              { type: 'screenshot' }
-            ]
-          },
-          {
-            name: 'Catalog Page',
-            url: `${B2BSHOP_URL}/catalog`,
-            actions: [
-              { type: 'goto', selector: '/catalog' },
-              { type: 'wait', timeout: 2000 },
-              { type: 'assert', selector: '.product-list', expected: 'visible' }
-            ]
-          },
-          {
-            name: 'Products Page',
-            url: `${B2BSHOP_URL}/products`,
-            actions: [
-              { type: 'goto', selector: '/products' },
-              { type: 'wait', timeout: 2000 }
-            ]
-          }
-        ]
-      };
-
-      expect(uiSpec.pages).toHaveLength(3);
-      expect(uiSpec.pages[0].name).toBe('Homepage');
-      expect(uiSpec.baseUrl).toBe(B2BSHOP_URL);
-    });
-
-    it('should generate API test spec from B2BShop API', () => {
-      const apiSpec: ApiTestSpec = {
-        type: 'api',
-        name: 'B2BShop API Tests',
-        baseUrl: `${B2BSHOP_URL}/api`,
-        endpoints: [
-          {
-            method: 'GET',
-            path: '/health',
-            description: 'Health check endpoint',
-            expectations: { status: 200 }
-          },
-          {
-            method: 'GET',
-            path: '/products',
-            description: 'List all products',
-            expectations: { status: 200 }
-          },
-          {
-            method: 'POST',
-            path: '/cart/items',
-            description: 'Add item to cart',
-            expectations: { status: 201 }
-          }
-        ]
-      };
-
-      expect(apiSpec.endpoints).toHaveLength(3);
-      expect(apiSpec.baseUrl).toBe(`${B2BSHOP_URL}/api`);
-    });
-  });
-
-  describe.skipIf(!shouldRun)('Pack Generation for B2BShop', () => {
-    it('should generate B2BShop UI pack from spec', () => {
-      const uiSpec: UiTestSpec = {
-        type: 'ui',
-        name: 'B2BShop E2E Tests',
-        baseUrl: B2BSHOP_URL,
-        viewport: { width: 1920, height: 1080 },
-        pages: [
-          {
-            name: 'Homepage',
-            url: `${B2BSHOP_URL}/`,
-            actions: [
-              { type: 'goto', selector: '/' },
-              { type: 'wait', timeout: 2000 }
-            ]
-          },
-          {
-            name: 'Catalog',
-            url: `${B2BSHOP_URL}/catalog`,
-            actions: [
-              { type: 'goto', selector: '/catalog' },
-              { type: 'wait', timeout: 2000 }
-            ]
-          },
-          {
-            name: 'Products',
-            url: `${B2BSHOP_URL}/products`,
-            actions: [
-              { type: 'goto', selector: '/products' },
-              { type: 'wait', timeout: 2000 }
-            ]
-          },
-          {
-            name: 'Cart',
-            url: `${B2BSHOP_URL}/cart`,
-            actions: [
-              { type: 'goto', selector: '/cart' },
-              { type: 'wait', timeout: 2000 }
-            ]
-          }
-        ]
-      };
-
-      const packGenerator = new PackGenerator({
-        name: 'b2bshop-ui-tests',
-        timeout: 30000,
-        retries: 2
-      });
-
-      const result = packGenerator.generateFromUiSpec(uiSpec);
-
-      expect(result.success).toBe(true);
-      expect(result.yaml).toContain('version: 2');
-      expect(result.yaml).toContain('name: b2bshop-ui-tests');
-      expect(result.yaml).toContain('ui_smoke:');
-      expect(result.yaml).toContain('ui_full:');
-      expect(result.yaml).toContain(B2BSHOP_URL);
-    });
-
-    it('should generate B2BShop API pack from spec', () => {
-      const apiSpec: ApiTestSpec = {
-        type: 'api',
-        name: 'B2BShop API Tests',
-        baseUrl: `${B2BSHOP_URL}/api`,
-        endpoints: [
-          {
-            method: 'GET',
-            path: '/health',
-            description: 'Health check',
-            expectations: { status: 200 }
-          },
-          {
-            method: 'GET',
-            path: '/products',
-            description: 'List products',
-            expectations: { status: 200 }
-          }
-        ]
-      };
-
-      const packGenerator = new PackGenerator({
-        name: 'b2bshop-api-tests'
-      });
-
-      const result = packGenerator.generateFromApiSpec(apiSpec);
-
-      expect(result.success).toBe(true);
-      expect(result.yaml).toContain('b2bshop-api-tests');
-      expect(result.yaml).toContain('api_smoke:');
-      expect(result.yaml).toContain('/health');
-      expect(result.yaml).toContain('/products');
-    });
-
-    it('should save generated pack to file', () => {
-      const uiSpec: UiTestSpec = {
-        type: 'ui',
-        name: 'B2BShop Saved Pack',
-        baseUrl: B2BSHOP_URL,
-        pages: [
-          { name: 'Home', url: `${B2BSHOP_URL}/`, actions: [] }
-        ]
-      };
-
-      const packGenerator = new PackGenerator({ name: 'b2bshop-saved' });
-      const result = packGenerator.generateFromUiSpec(uiSpec);
-
-      // Save to file
-      const packPath = join(testDir, 'b2bshop-generated.yml');
-      writeFileSync(packPath, result.yaml, 'utf-8');
-
-      expect(existsSync(packPath)).toBe(true);
-
-      // Verify content
-      const content = readFileSync(packPath, 'utf-8');
-      expect(content).toContain('version: 2');
-      expect(content).toContain(B2BSHOP_URL);
-    });
-  });
-
-  describe.skipIf(!shouldRun)('AI-Powered Test Generation', () => {
-    it('should generate tests from B2BShop URL using Mock AI', async () => {
-      const mockProvider = new MockProvider({
-        model: 'mock-b2bshop-model',
-        responses: {
-          default: {
-            content: `import { test, expect } from '@playwright/test';
-
-test.describe('B2BShop Generated Tests', () => {
-  test('homepage loads correctly', async ({ page }) => {
-    await page.goto('${B2BSHOP_URL}/');
-    await expect(page).toHaveTitle(/B2BShop/);
-  });
-
-  test('can navigate to catalog', async ({ page }) => {
-    await page.goto('${B2BSHOP_URL}/');
-    await page.click('a[href="/catalog"]');
-    await expect(page).toHaveURL(/.*catalog/);
-  });
-
-  test('product search works', async ({ page }) => {
-    await page.goto('${B2BSHOP_URL}/');
-    await page.fill('input[placeholder*="search"]', 'test product');
-    await page.click('button[type="submit"]');
-    await expect(page.locator('.product-list')).toBeVisible();
-  });
-});`,
-            usage: { promptTokens: 150, completionTokens: 200, totalTokens: 350 },
-            model: 'mock-b2bshop-model',
-            finishReason: 'stop'
-          }
-        }
-      });
-
-      const codeGenerator = new CodeGenerator({
-        timeout: 10000,
-        temperature: 0.3
-      }) as any;
-      codeGenerator.provider = mockProvider;
-
-      const result = await codeGenerator.generateFromUrl(B2BSHOP_URL);
-
-      expect(result.success).toBe(true);
-      expect(result.tests).toHaveLength(1);
-      expect(result.tests[0].code).toContain('B2BShop');
-      expect(result.tests[0].code).toContain('homepage loads correctly');
-      expect(result.tests[0].code).toContain('can navigate to catalog');
-      expect(result.tests[0].framework).toBe('playwright');
-    });
-
-    it('should generate tests from B2BShop description', async () => {
-      const mockProvider = new MockProvider({
-        model: 'mock-description-model',
-        responses: {
-          default: {
-            content: `import { test, expect } from '@playwright/test';
-
-test.describe('B2BShop API Tests', () => {
-  test('GET /api/health returns 200', async ({ request }) => {
-    const response = await request.get('${B2BSHOP_URL}/api/health');
-    expect(response.status()).toBe(200);
-  });
-
-  test('GET /api/products returns array', async ({ request }) => {
-    const response = await request.get('${B2BSHOP_URL}/api/products');
-    expect(response.status()).toBe(200);
-    const body = await response.json();
-    expect(Array.isArray(body.products)).toBe(true);
-  });
-});`,
-            usage: { promptTokens: 100, completionTokens: 150, totalTokens: 250 },
-            model: 'mock-description-model',
-            finishReason: 'stop'
-          }
-        }
-      });
-
-      const codeGenerator = new CodeGenerator() as any;
-      codeGenerator.provider = mockProvider;
-
-      const description = `
-Generate API tests for B2BShop e-commerce platform:
-- Health check endpoint at /api/health
-- Products listing at /api/products (returns array of products)
-- Product details at /api/products/:id
-- Cart operations at /api/cart
-Include authentication and error handling.
-      `;
-
-      const result = await codeGenerator.generateFromDescription(description);
-
-      expect(result.success).toBe(true);
-      expect(result.tests[0].code).toContain('/api/health');
-      expect(result.tests[0].code).toContain('/api/products');
-    });
-  });
-
-  describe.skipIf(!shouldRun)('Full E2E Workflow', () => {
-    it('should complete full workflow: URL → Spec → Pack → Run', async () => {
-      // Step 1: Generate spec from URL (simulated)
-      const uiSpec: UiTestSpec = {
-        type: 'ui',
-        name: 'B2BShop E2E Workflow Test',
-        baseUrl: B2BSHOP_URL,
-        pages: [
-          {
-            name: 'Homepage',
-            url: `${B2BSHOP_URL}/`,
-            actions: [
-              { type: 'goto', selector: '/' },
-              { type: 'wait', timeout: 2000 },
-              { type: 'assert', selector: 'body', expected: 'visible' }
-            ]
-          }
-        ]
-      };
-
-      // Step 2: Generate pack from spec
-      const packGenerator = new PackGenerator({
-        name: 'b2bshop-e2e-workflow',
-        timeout: 30000,
-        retries: 1
-      });
-
-      const packResult = packGenerator.generateFromUiSpec(uiSpec);
-      expect(packResult.success).toBe(true);
-
-      // Step 3: Save pack to file
-      const packPath = join(testDir, 'b2bshop-workflow.yml');
-      writeFileSync(packPath, packResult.yaml, 'utf-8');
-
-      // Step 4: Load and validate pack
-      const yaml = readFileSync(packPath, 'utf-8');
-      expect(yaml).toContain('version: 2');
-      expect(yaml).toContain(B2BSHOP_URL);
-
-      // The pack is now ready to be executed
-      // In real scenario, we would run: qa360 run b2bshop-workflow.yml
-      expect(existsSync(packPath)).toBe(true);
-    }, NETWORK_TIMEOUT);
-
-    it('should generate comprehensive B2BShop test suite', async () => {
-      const packGenerator = new PackGenerator({
-        name: 'b2bshop-comprehensive',
-        timeout: 60000,
-        retries: 2
-      });
-
-      // Generate UI pack
-      const uiSpec: UiTestSpec = {
-        type: 'ui',
-        name: 'B2BShop Comprehensive UI',
-        baseUrl: B2BSHOP_URL,
-        viewport: { width: 1920, height: 1080 },
-        pages: [
-          { name: 'Homepage', url: `${B2BSHOP_URL}/`, actions: [] },
-          { name: 'Catalog', url: `${B2BSHOP_URL}/catalog`, actions: [] },
-          { name: 'Products', url: `${B2BSHOP_URL}/products`, actions: [] },
-          { name: 'Cart', url: `${B2BSHOP_URL}/cart`, actions: [] }
-        ]
-      };
-
-      const uiPack = packGenerator.generateFromUiSpec(uiSpec);
-      expect(uiPack.success).toBe(true);
-      expect(uiPack.yaml).toContain('ui_smoke:');
-      expect(uiPack.yaml).toContain('ui_full:');
-
-      // Generate API pack
-      const apiSpec: ApiTestSpec = {
-        type: 'api',
-        name: 'B2BShop Comprehensive API',
-        baseUrl: `${B2BSHOP_URL}/api`,
-        endpoints: [
-          { method: 'GET', path: '/health', expectations: { status: 200 } },
-          { method: 'GET', path: '/products', expectations: { status: 200 } },
-          { method: 'GET', path: '/products/:id', expectations: { status: 200 } }
-        ]
-      };
-
-      const apiPack = packGenerator.generateFromApiSpec(apiSpec);
-      expect(apiPack.success).toBe(true);
-      expect(apiPack.yaml).toContain('api_smoke:');
-
-      // Both packs should be valid
-      expect(uiPack.config.version).toBe(2);
-      expect(apiPack.config.version).toBe(2);
-    }, NETWORK_TIMEOUT);
-  });
-
-  describe.skipIf(!shouldRun)('Real Site Execution', () => {
-    beforeEach(async () => {
-      if (!shouldRun || !siteHealthy) return;
-      await skipIfE2ETargetUnhealthy('b2bshop');
-    });
-
-    it('should run generated B2BShop pack successfully', async () => {
-      // Create a minimal pack for B2BShop
-      const pack: PackConfigV2 = {
-        version: 2,
-        name: 'b2bshop-real-execution',
-        description: 'Generated B2BShop test pack',
-        gates: {
-          ui_smoke: {
-            adapter: 'playwright-ui',
-            enabled: true,
-            config: {
-              baseUrl: B2BSHOP_URL,
-              pages: [
-                `${B2BSHOP_URL}/`
-              ]
-            },
-            options: {
-              timeout: 30000,
-              retries: 2
-            },
-            budgets: {
-              a11y_min: 50
-            }
-          }
-        },
-        execution: {
-          parallel: false,
-          on_failure: 'continue'
-        }
-      };
-
-      const runner = new Phase3Runner({
-        workingDir: testDir,
-        pack
-      });
-
-      const result = await runner.run();
-
-      // Verify execution completed
-      expect(result).toBeDefined();
-      expect(result.gates).toHaveLength(1);
-
-      const gate = result.gates[0];
-      expect(gate.gate).toBe('ui_smoke');
-      expect(gate.status).toMatch(/^(passed|failed|skipped)$/);
-      expect(gate.duration).toBeGreaterThanOrEqual(0);
-
-      // Proof should be generated
-      if (result.proofPath) {
-        expect(existsSync(result.proofPath)).toBe(true);
-      }
-    }, NETWORK_TIMEOUT);
-
-    it('should handle multiple B2BShop pages in single run', async () => {
-      const pack: PackConfigV2 = {
-        version: 2,
-        name: 'b2bshop-multi-page',
-        gates: {
-          ui_pages: {
-            adapter: 'playwright-ui',
-            enabled: true,
-            config: {
-              baseUrl: B2BSHOP_URL,
-              pages: [
-                `${B2BSHOP_URL}/`,
-                `${B2BSHOP_URL}/catalog`,
-                `${B2BSHOP_URL}/products`
-              ]
-            },
-            options: {
-              timeout: 30000,
-              retries: 1
-            }
-          }
-        },
-        execution: {
-          parallel: false,
-          on_failure: 'continue'
-        }
-      };
-
-      const runner = new Phase3Runner({
-        workingDir: testDir,
-        pack
-      });
-
-      const result = await runner.run();
-
-      expect(result.gates).toHaveLength(1);
-      expect(result.gates[0].gate).toBe('ui_pages');
-    }, NETWORK_TIMEOUT * 2);
-  });
-
-  describe.skipIf(!shouldRun)('AI Generation Quality', () => {
-    it('should generate tests with proper assertions', async () => {
-      const mockProvider = new MockProvider({
-        model: 'quality-test',
-        responses: {
-          default: {
-            content: `import { test, expect } from '@playwright/test';
-
-test('B2BShop homepage has expected elements', async ({ page }) => {
-  await page.goto('${B2BSHOP_URL}/');
-
-  // Check for navigation
-  await expect(page.locator('nav')).toBeVisible();
-
-  // Check for main content
-  await expect(page.locator('main')).toBeVisible();
-
-  // Check accessibility
-  const heading = page.locator('h1');
-  await expect(heading).toBeVisible();
-});`,
-            usage: { promptTokens: 80, completionTokens: 120, totalTokens: 200 },
-            model: 'quality-test',
-            finishReason: 'stop'
-          }
-        }
-      });
-
-      const codeGenerator = new CodeGenerator() as any;
-      codeGenerator.provider = mockProvider;
-
-      const result = await codeGenerator.generateFromDescription(
-        'Generate UI tests for B2BShop homepage with proper assertions'
-      );
-
-      expect(result.success).toBe(true);
-      expect(result.tests[0].code).toContain('expect');
-      expect(result.tests[0].code).toContain('toBeVisible');
-    });
-
-    it('should include error handling in generated tests', async () => {
-      const mockProvider = new MockProvider({
-        model: 'error-handling-test',
-        responses: {
-          default: {
-            content: `import { test, expect } from '@playwright/test';
-
-test('API error handling', async ({ request }) => {
-  // Test 404 handling
-  const response = await request.get('${B2BSHOP_URL}/api/nonexistent');
-  expect(response.status()).toBe(404);
-
-  // Test error response format
-  const body = await response.json();
-  expect(body).toHaveProperty('error');
-});`,
-            usage: { promptTokens: 60, completionTokens: 100, totalTokens: 160 },
-            model: 'error-handling-test',
-            finishReason: 'stop'
-          }
-        }
-      });
-
-      const codeGenerator = new CodeGenerator() as any;
-      codeGenerator.provider = mockProvider;
-
-      const result = await codeGenerator.generateFromDescription(
-        'Generate API tests for B2BShop with error handling'
-      );
-
-      expect(result.success).toBe(true);
-      expect(result.tests[0].code).toContain('404');
-      expect(result.tests[0].code).toContain('error');
-    });
-  });
-});
-
-describe('B2BShop Test Data Generation', () => {
-  describe('Test Data Helpers', () => {
-    it('should provide realistic test data for B2BShop', () => {
-      const testData = {
-        products: [
-          { id: '1', name: 'Wireless Headphones', price: 79.99, category: 'Electronics' },
-          { id: '2', name: 'USB-C Cable', price: 12.99, category: 'Accessories' },
-          { id: '3', name: 'Laptop Stand', price: 45.00, category: 'Office' }
-        ],
-        users: [
-          { email: 'test@example.com', password: 'SecurePass123!', name: 'Test User' },
-          { email: 'admin@b2bshop.com', password: 'AdminPass123!', name: 'Admin User' }
-        ],
-        cart: {
-          items: [
-            { productId: '1', quantity: 2 },
-            { productId: '2', quantity: 1 }
-          ],
-          total: 172.97
-        }
-      };
-
-      expect(testData.products).toHaveLength(3);
-      expect(testData.users).toHaveLength(2);
-      expect(testData.cart.items).toHaveLength(2);
-    });
-
-    it('should generate test scenarios for B2BShop workflows', () => {
-      const workflows = [
-        {
-          name: 'Browse and Add to Cart',
-          steps: [
-            'Navigate to homepage',
-            'Browse catalog',
-            'Select product',
-            'Add to cart',
-            'Verify cart count'
-          ]
-        },
-        {
-          name: 'User Registration',
-          steps: [
-            'Navigate to registration',
-            'Fill user details',
-            'Submit form',
-            'Verify success message',
-            'Verify login capability'
-          ]
-        },
-        {
-          name: 'Checkout Process',
-          steps: [
-            'Add items to cart',
-            'Navigate to checkout',
-            'Enter shipping details',
-            'Select payment method',
-            'Complete order',
-            'Verify order confirmation'
-          ]
-        }
-      ];
-
-      expect(workflows).toHaveLength(3);
-      expect(workflows[0].steps).toHaveLength(5);
-      expect(workflows[1].steps).toHaveLength(5);
-      expect(workflows[2].steps).toHaveLength(6);
-    });
-  });
-});