qa360 2.1.7 → 2.2.0

package/core/src/generation/__tests__/generation-integration.test.ts

@@ -0,0 +1,655 @@
+/**
+ * Generation Integration Tests
+ *
+ * End-to-end tests for AI test generation workflow:
+ * 1. Generate test specification (from description, code, or URL)
+ * 2. Convert to QA360 pack YAML
+ * 3. Validate pack format
+ * 4. Verify pack is executable
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest';
+import { PackGenerator, type PackGeneratorOptions } from '../pack-generator.js';
+import { CodeGenerator } from '../code-generator.js';
+import type { ApiTestSpec, UiTestSpec, PerfTestSpec } from '../types.js';
+import { MockProvider } from '../../ai/mock-provider.js';
+
+// Mock provider for integration tests
+const createMockProvider = () => {
+  return new MockProvider({
+    model: 'mock-integration-model',
+    responses: {
+      default: {
+        content: `import { test, expect } from '@playwright/test';
+
+test.describe('Generated Tests', () => {
+  test('should pass', async ({ page }) => {
+    await page.goto('/');
+    await expect(page).toHaveTitle(/Home/);
+  });
+
+  test('should have working navigation', async ({ page }) => {
+    await page.goto('/');
+    await page.click('nav a');
+    await expect(page).toHaveURL(/./);
+  });
+});`,
+        usage: { promptTokens: 50, completionTokens: 100, totalTokens: 150 },
+        model: 'mock-integration-model',
+        finishReason: 'stop'
+      }
+    }
+  });
+};
+
+describe('Generation Integration Tests', () => {
+  describe('Spec to YAML Conversion', () => {
+    let packGenerator: PackGenerator;
+
+    beforeEach(() => {
+      packGenerator = new PackGenerator({
+        timeout: 30000,
+        retries: 2
+      });
+    });
+
+    describe('API Test Generation Workflow', () => {
+      it('should convert API spec to valid pack YAML', () => {
+        const apiSpec: ApiTestSpec = {
+          type: 'api',
+          name: 'User API Tests',
+          baseUrl: 'https://api.example.com',
+          endpoints: [
+            {
+              method: 'GET',
+              path: '/health',
+              description: 'Health check endpoint',
+              expectations: { status: 200 }
+            },
+            {
+              method: 'GET',
+              path: '/users',
+              description: 'List all users',
+              expectations: { status: 200 }
+            },
+            {
+              method: 'POST',
+              path: '/users',
+              description: 'Create new user',
+              expectations: { status: 201 }
+            }
+          ]
+        };
+
+        const result = packGenerator.generateFromApiSpec(apiSpec);
+
+        // Verify success
+        expect(result.success).toBe(true);
+        expect(result.yaml).toBeTruthy();
+        expect(result.config).toBeTruthy();
+
+        // Verify YAML structure
+        const yaml = result.yaml;
+        expect(yaml).toContain('version: 2');
+        expect(yaml).toContain('gates:');
+        expect(yaml).toContain('api_smoke:');
+        expect(yaml).toContain('api_full:');
+        expect(yaml).toContain('adapter: playwright-native-api');
+
+        // Verify config structure
+        expect(result.config.version).toBe(2);
+        expect(result.config.gates).toHaveProperty('api_smoke');
+        expect(result.config.gates).toHaveProperty('api_full');
+        expect(result.config.execution).toBeDefined();
+        expect(result.config.execution?.on_failure).toBe('continue');
+      });
+
+      it('should include auth in generated pack when specified', () => {
+        const apiSpec: ApiTestSpec = {
+          type: 'api',
+          name: 'Secure API',
+          baseUrl: 'https://api.example.com',
+          auth: {
+            type: 'bearer',
+            config: { token: '${ENV:API_TOKEN}' }
+          },
+          endpoints: [
+            { method: 'GET', path: '/secure', expectations: { status: 200 } }
+          ]
+        };
+
+        const result = packGenerator.generateFromApiSpec(apiSpec);
+
+        expect(result.success).toBe(true);
+        expect(result.yaml).toContain('auth:');
+        expect(result.yaml).toContain('profiles:');
+        expect(result.yaml).toContain('type: bearer');
+        expect(result.config.auth).toBeDefined();
+        expect(result.config.auth?.profiles).toBeDefined();
+      });
+
+      it('should handle complex API specs with multiple auth types', () => {
+        const specs: ApiTestSpec[] = [
+          {
+            type: 'api',
+            name: 'Basic Auth API',
+            baseUrl: 'https://api.example.com',
+            auth: { type: 'basic', config: { username: 'user', password: 'pass' } },
+            endpoints: [{ method: 'GET', path: '/data', expectations: { status: 200 } }]
+          },
+          {
+            type: 'api',
+            name: 'API Key API',
+            baseUrl: 'https://api.example.com',
+            auth: { type: 'api_key', config: { header: 'X-API-Key', key: 'secret' } },
+            endpoints: [{ method: 'GET', path: '/data', expectations: { status: 200 } }]
+          },
+          {
+            type: 'api',
+            name: 'JWT Auth API',
+            baseUrl: 'https://api.example.com',
+            auth: { type: 'jwt', config: { tokenEndpoint: '/oauth/token' } },
+            endpoints: [{ method: 'GET', path: '/data', expectations: { status: 200 } }]
+          }
+        ];
+
+        for (const spec of specs) {
+          const result = packGenerator.generateFromApiSpec(spec);
+          expect(result.success).toBe(true);
+          expect(result.yaml).toContain('auth:');
+        }
+      });
+    });
+
+    describe('UI Test Generation Workflow', () => {
+      it('should convert UI spec to valid pack YAML', () => {
+        const uiSpec: UiTestSpec = {
+          type: 'ui',
+          name: 'E-commerce UI Tests',
+          baseUrl: 'https://shop.example.com',
+          viewport: { width: 1920, height: 1080 },
+          pages: [
+            {
+              name: 'Homepage',
+              url: 'https://shop.example.com',
+              actions: [
+                { type: 'goto', selector: '/' },
+                { type: 'click', selector: 'button[class="cta"]' },
+                { type: 'wait', timeout: 1000 }
+              ]
+            },
+            {
+              name: 'Product Page',
+              url: 'https://shop.example.com/product/1',
+              actions: [
+                { type: 'goto', selector: '/product/1' },
+                { type: 'click', selector: 'button[id="add-to-cart"]' }
+              ]
+            },
+            {
+              name: 'Cart',
+              url: 'https://shop.example.com/cart',
+              actions: [
+                { type: 'goto', selector: '/cart' }
+              ]
+            },
+            {
+              name: 'Checkout',
+              url: 'https://shop.example.com/checkout',
+              actions: [
+                { type: 'goto', selector: '/checkout' }
+              ]
+            }
+          ]
+        };
+
+        const result = packGenerator.generateFromUiSpec(uiSpec);
+
+        expect(result.success).toBe(true);
+        expect(result.yaml).toContain('version: 2');
+        expect(result.yaml).toContain('ui_smoke:');
+        expect(result.yaml).toContain('ui_full:');
+        expect(result.yaml).toContain('adapter: playwright-ui');
+        expect(result.yaml).toContain('width: 1920');
+        expect(result.yaml).toContain('height: 1080');
+        expect(result.yaml).toContain('a11y_min:');
+      });
+
+      it('should split UI tests into smoke and full suites', () => {
+        const uiSpec: UiTestSpec = {
+          type: 'ui',
+          name: 'Large Site Tests',
+          baseUrl: 'https://example.com',
+          pages: Array.from({ length: 10 }, (_, i) => ({
+            name: `Page ${i + 1}`,
+            url: `https://example.com/page/${i + 1}`,
+            actions: [{ type: 'goto', selector: `/page/${i + 1}` }]
+          }))
+        };
+
+        const result = packGenerator.generateFromUiSpec(uiSpec);
+
+        expect(result.success).toBe(true);
+        expect(result.yaml).toContain('ui_smoke:');
+        expect(result.yaml).toContain('ui_full:');
+
+        // Smoke should have first 3 pages, full should have all
+        expect(result.config.gates.ui_smoke?.config).toBeDefined();
+        expect(result.config.gates.ui_full?.config).toBeDefined();
+      });
+    });
+
+    describe('Performance Test Generation Workflow', () => {
+      it('should convert performance spec to valid pack YAML', () => {
+        const perfSpec: PerfTestSpec = {
+          type: 'perf',
+          name: 'API Load Tests',
+          baseUrl: 'https://api.example.com',
+          scenarios: [
+            {
+              name: 'Browse Load',
+              weight: 70,
+              requests: [
+                { method: 'GET', path: '/products' },
+                { method: 'GET', path: '/products/{id}' }
+              ]
+            },
+            {
+              name: 'Checkout Load',
+              weight: 20,
+              requests: [
+                { method: 'POST', path: '/cart' },
+                { method: 'POST', path: '/orders' }
+              ]
+            },
+            {
+              name: 'Admin Load',
+              weight: 10,
+              requests: [
+                { method: 'GET', path: '/admin/dashboard' },
+                { method: 'GET', path: '/admin/users' }
+              ]
+            }
+          ],
+          options: {
+            duration: '5m',
+            vus: 50,
+            maxVus: 200,
+            thresholds: [
+              { metric: 'http_req_duration', condition: 'p(95)<500' },
+              { metric: 'http_req_duration', condition: 'p(99)<1000' },
+              { metric: 'http_req_failed', condition: 'rate<0.05' }
+            ]
+          }
+        };
+
+        const result = packGenerator.generateFromPerfSpec(perfSpec);
+
+        expect(result.success).toBe(true);
+        expect(result.yaml).toContain('version: 2');
+        expect(result.yaml).toContain('perf:');
+        expect(result.yaml).toContain('adapter: k6-perf');
+        expect(result.yaml).toContain('scenarios:');
+        expect(result.yaml).toContain('Browse Load');
+        expect(result.yaml).toContain('Checkout Load');
+        expect(result.yaml).toContain('Admin Load');
+        expect(result.yaml).toContain('duration: 5m');
+        expect(result.yaml).toContain('vus: 50');
+        expect(result.yaml).toContain('maxVus: 200');
+        expect(result.yaml).toContain('budgets:');
+        expect(result.yaml).toContain('p95_ms: 500');
+        expect(result.yaml).toContain('p99_ms: 1000');
+      });
+
+      it('should extract threshold values correctly', () => {
+        const perfSpec: PerfTestSpec = {
+          type: 'perf',
+          name: 'Threshold Tests',
+          baseUrl: 'https://api.example.com',
+          scenarios: [
+            {
+              name: 'Simple',
+              requests: [{ method: 'GET', path: '/test' }]
+            }
+          ],
+          options: {
+            thresholds: [
+              { metric: 'http_req_duration', condition: 'p(95)<250' },
+              { metric: 'http_req_duration', condition: 'p(99)<750' },
+              { metric: 'http_req_failed', condition: 'rate<0.01' }
+            ]
+          }
+        };
+
+        const result = packGenerator.generateFromPerfSpec(perfSpec);
+
+        expect(result.success).toBe(true);
+        expect(result.yaml).toContain('p95_ms: 250');
+        expect(result.yaml).toContain('p99_ms: 750');
+      });
+    });
+
+    describe('Custom Generator Options', () => {
+      it('should apply custom timeout and retries', () => {
+        const options: PackGeneratorOptions = {
+          name: 'custom-pack',
+          timeout: 60000,
+          retries: 5,
+          onFailure: 'stop'
+        };
+
+        const generator = new PackGenerator(options);
+        const apiSpec: ApiTestSpec = {
+          type: 'api',
+          name: 'API Tests',
+          baseUrl: 'https://api.example.com',
+          endpoints: [{ method: 'GET', path: '/health' }]
+        };
+
+        const result = generator.generateFromApiSpec(apiSpec);
+
+        expect(result.success).toBe(true);
+        expect(result.yaml).toContain('timeout: 60000');
+        expect(result.yaml).toContain('retries: 5');
+        expect(result.yaml).toContain('on_failure: stop');
+      });
+    });
+
+    describe('Error Handling', () => {
+      it('should return error for unsupported test type', () => {
+        const result = packGenerator.generateFromSpec({
+          type: 'unit',
+          name: 'Unit Tests'
+        } as any);
+
+        expect(result.success).toBe(false);
+        expect(result.errors).toBeDefined();
+        expect(result.errors?.[0]).toContain('not supported');
+      });
+    });
+  });
+
+  describe('YAML Output Validation', () => {
+    let packGenerator: PackGenerator;
+
+    beforeEach(() => {
+      packGenerator = new PackGenerator();
+    });
+
+    it('should generate valid YAML structure', () => {
+      const apiSpec: ApiTestSpec = {
+        type: 'api',
+        name: 'Test API',
+        baseUrl: 'https://api.example.com',
+        endpoints: [
+          { method: 'GET', path: '/test' }
+        ]
+      };
+
+      const result = packGenerator.generateFromApiSpec(apiSpec);
+      const lines = result.yaml.split('\n');
+
+      // Verify YAML structure
+      expect(lines[0]).toBe('version: 2');
+      expect(lines[1]).toMatch(/^name: /);
+
+      // Check indentation is consistent
+      const gateLines = lines.filter(l => l.includes('adapter:'));
+      expect(gateLines.length).toBeGreaterThan(0);
+      gateLines.forEach(line => {
+        expect(line.startsWith('    ')).toBe(true); // 4 spaces for adapter
+      });
+    });
+
+    it('should handle nested objects correctly', () => {
+      const apiSpec: ApiTestSpec = {
+        type: 'api',
+        name: 'Complex API',
+        baseUrl: 'https://api.example.com',
+        auth: {
+          type: 'oauth2',
+          config: {
+            tokenUrl: 'https://auth.example.com/token',
+            clientId: 'test-client',
+            scopes: ['read', 'write']
+          }
+        },
+        endpoints: [
+          { method: 'GET', path: '/data' }
+        ]
+      };
+
+      const result = packGenerator.generateFromApiSpec(apiSpec);
+
+      expect(result.yaml).toContain('auth:');
+      expect(result.yaml).toContain('profiles:');
+      expect(result.yaml).toContain('tokenUrl:');
+      expect(result.yaml).toContain('clientId:');
+      expect(result.yaml).toContain('scopes:');
+    });
+
+    it('should handle arrays in YAML', () => {
+      const perfSpec: PerfTestSpec = {
+        type: 'perf',
+        name: 'Array Test',
+        baseUrl: 'https://api.example.com',
+        scenarios: [
+          {
+            name: 'Scenario 1',
+            requests: [
+              { method: 'GET', path: '/path1' },
+              { method: 'GET', path: '/path2' }
+            ]
+          }
+        ]
+      };
+
+      const result = packGenerator.generateFromPerfSpec(perfSpec);
+
+      expect(result.yaml).toContain('-');
+      expect(result.yaml).toContain('method: GET');
+    });
+  });
+
+  describe('Pack Executability', () => {
+    it('should generate pack that can be validated', () => {
+      const packGenerator = new PackGenerator({ timeout: 30000 });
+      const apiSpec: ApiTestSpec = {
+        type: 'api',
+        name: 'Executable API Tests',
+        baseUrl: 'https://api.example.com',
+        endpoints: [
+          { method: 'GET', path: '/health', expectations: { status: 200 } },
+          { method: 'GET', path: '/users', expectations: { status: 200 } }
+        ]
+      };
+
+      const result = packGenerator.generateFromApiSpec(apiSpec);
+
+      // Verify the pack has all required fields
+      expect(result.config.version).toBe(2);
+      expect(result.config.name).toBeDefined();
+      expect(result.config.gates).toBeDefined();
+      expect(Object.keys(result.config.gates).length).toBeGreaterThan(0);
+      expect(result.config.execution).toBeDefined();
+      expect(result.config.execution?.on_failure).toBeDefined();
+
+      // Verify each gate has required fields
+      for (const [gateName, gate] of Object.entries(result.config.gates)) {
+        expect(gate.adapter).toBeDefined();
+        expect(gate.config).toBeDefined();
+      }
+    });
+
+    it('should generate pack with sensible defaults', () => {
+      const packGenerator = new PackGenerator(); // Use all defaults
+      const uiSpec: UiTestSpec = {
+        type: 'ui',
+        name: 'Default UI Tests',
+        baseUrl: 'https://example.com',
+        pages: [
+          { name: 'Home', url: 'https://example.com', actions: [] }
+        ]
+      };
+
+      const result = packGenerator.generateFromUiSpec(uiSpec);
+
+      expect(result.config.name).toBeDefined();
+      expect(result.config.execution?.on_failure).toBe('continue');
+    });
+  });
+
+  describe('Convenience Functions', () => {
+    it('should work with generatePackFromApiSpec', async () => {
+      const { generatePackFromApiSpec } = await import('../pack-generator.js');
+
+      const spec: ApiTestSpec = {
+        type: 'api',
+        name: 'Convenience API',
+        baseUrl: 'https://api.example.com',
+        endpoints: [{ method: 'GET', path: '/test' }]
+      };
+
+      const result = generatePackFromApiSpec(spec, { name: 'convenience-pack' });
+
+      expect(result.success).toBe(true);
+      expect(result.config.name).toBe('convenience-pack');
+    });
+
+    it('should work with generatePackFromUiSpec with custom name', async () => {
+      const { generatePackFromUiSpec } = await import('../pack-generator.js');
+
+      const spec: UiTestSpec = {
+        type: 'ui',
+        name: 'Convenience UI',
+        baseUrl: 'https://example.com',
+        pages: [{ name: 'Home', url: 'https://example.com', actions: [] }]
+      };
+
+      const result = generatePackFromUiSpec(spec, { name: 'my-ui-pack' });
+
+      expect(result.success).toBe(true);
+      expect(result.config.name).toBe('my-ui-pack');
+    });
+
+    it('should work with generatePackFromPerfSpec', async () => {
+      const { generatePackFromPerfSpec } = await import('../pack-generator.js');
+
+      const spec: PerfTestSpec = {
+        type: 'perf',
+        name: 'Convenience Perf',
+        baseUrl: 'https://api.example.com',
+        scenarios: [{ name: 'Load', requests: [{ method: 'GET', path: '/test' }] }]
+      };
+
+      const result = generatePackFromPerfSpec(spec);
+
+      expect(result.success).toBe(true);
+    });
+  });
+});
+
+describe('Generation E2E Workflow', () => {
+  describe('Description to Pack YAML', () => {
+    it('should generate pack from natural language description', async () => {
+      const mockProvider = createMockProvider();
+
+      const codeGenerator = new CodeGenerator({
+        timeout: 5000,
+        temperature: 0.3
+      }) as any;
+      codeGenerator.provider = mockProvider;
+
+      const packGenerator = new PackGenerator({
+        name: 'description-generated-pack'
+      });
+
+      // Step 1: Generate test code from description
+      const description = 'Create API smoke tests for a user management REST API';
+      const codeResult = await codeGenerator.generateFromDescription(description);
+
+      expect(codeResult.success).toBe(true);
+      expect(codeResult.tests).toHaveLength(1);
+      expect(codeResult.tests[0].code).toBeTruthy();
+
+      // Step 2: Create API spec from the generated test info
+      const apiSpec: ApiTestSpec = {
+        type: 'api',
+        name: 'User Management API',
+        baseUrl: 'https://api.example.com',
+        endpoints: [
+          { method: 'GET', path: '/health', expectations: { status: 200 } },
+          { method: 'GET', path: '/users', expectations: { status: 200 } },
+          { method: 'POST', path: '/users', expectations: { status: 201 } }
+        ]
+      };
+
+      // Step 3: Generate pack YAML from spec
+      const packResult = packGenerator.generateFromApiSpec(apiSpec);
+
+      expect(packResult.success).toBe(true);
+      expect(packResult.yaml).toContain('version: 2');
+      expect(packResult.yaml).toContain('gates:');
+
+      // Verify the full workflow produced valid outputs
+      expect(codeResult.metadata).toBeDefined();
+      expect(packResult.config).toBeDefined();
+    });
+  });
+
+  describe('Batch Generation', () => {
+    it('should generate multiple packs from different sources', async () => {
+      const mockProvider = createMockProvider();
+      const codeGenerator = new CodeGenerator() as any;
+      codeGenerator.provider = mockProvider;
+
+      const packGenerator = new PackGenerator();
+
+      const sources = [
+        {
+          name: 'Health Check Pack',
+          spec: {
+            type: 'api' as const,
+            name: 'Health API',
+            baseUrl: 'https://api.example.com',
+            endpoints: [{ method: 'GET', path: '/health' }]
+          }
+        },
+        {
+          name: 'UI Pack',
+          spec: {
+            type: 'ui' as const,
+            name: 'Main UI',
+            baseUrl: 'https://example.com',
+            pages: [{ name: 'Home', url: 'https://example.com', actions: [] }]
+          }
+        },
+        {
+          name: 'Performance Pack',
+          spec: {
+            type: 'perf' as const,
+            name: 'Load Test',
+            baseUrl: 'https://api.example.com',
+            scenarios: [{ name: 'Load', requests: [{ method: 'GET', path: '/test' }] }]
+          }
+        }
+      ];
+
+      const results = sources.map(({ spec }) => {
+        if (spec.type === 'api') {
+          return packGenerator.generateFromApiSpec(spec);
+        } else if (spec.type === 'ui') {
+          return packGenerator.generateFromUiSpec(spec);
+        } else {
+          return packGenerator.generateFromPerfSpec(spec);
+        }
+      });
+
+      expect(results).toHaveLength(3);
+      expect(results.every(r => r.success)).toBe(true);
+      expect(results[0].yaml).toContain('api_smoke:');
+      expect(results[1].yaml).toContain('ui_smoke:');
+      expect(results[2].yaml).toContain('perf:');
+    });
+  });
+});