qa360 2.1.7 → 2.2.0

package/core/src/generation/__tests__/code-generator.test.ts

@@ -0,0 +1,586 @@
+/**
+ * Tests for Code Generator
+ *
+ * Unit tests for AI-powered test code generation using mocked LLM providers.
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { CodeGenerator, type CodeGeneratorOptions, type GenerationResult } from '../code-generator.js';
+import type { LLMProvider } from '../../ai/index.js';
+
+// Mock LLM Provider for testing
+class MockLLMProvider implements LLMProvider {
+  name = 'mock-test-provider';
+  models = ['mock-model', 'mock-large'];
+
+  private shouldFail = false;
+  private delay = 0;
+  private fixedResponse?: string;
+
+  setFixedResponse(response: string): void {
+    this.fixedResponse = response;
+  }
+
+  setShouldFail(shouldFail: boolean): void {
+    this.shouldFail = shouldFail;
+  }
+
+  setDelay(ms: number): void {
+    this.delay = ms;
+  }
+
+  async generate(request: {
+    prompt: string;
+    systemPrompt?: string;
+    temperature?: number;
+    maxTokens?: number;
+    jsonMode?: boolean;
+  }) {
+    if (this.delay > 0) {
+      await new Promise(resolve => setTimeout(resolve, this.delay));
+    }
+
+    if (this.shouldFail) {
+      throw new Error('Mock LLM provider failed');
+    }
+
+    const response = this.fixedResponse || this.getDefaultResponse(request);
+    return {
+      content: response,
+      usage: {
+        promptTokens: 100,
+        completionTokens: 200,
+        totalTokens: 300
+      },
+      model: 'mock-model',
+      finishReason: 'stop' as const
+    };
+  }
+
+  private getDefaultResponse(request: { prompt: string }): string {
+    // Generate relevant mock code based on the prompt
+    const prompt = request.prompt.toLowerCase();
+
+    if (prompt.includes('api') || prompt.includes('endpoint')) {
+      return this.getApiTestCode();
+    }
+
+    if (prompt.includes('ui') || prompt.includes('page')) {
+      return this.getUiTestCode();
+    }
+
+    if (prompt.includes('performance') || prompt.includes('load')) {
+      return this.getPerfTestCode();
+    }
+
+    return this.getGenericTestCode();
+  }
+
+  private getApiTestCode(): string {
+    return `import { test, expect } from '@playwright/test';
+
+test.describe('API Smoke Tests', () => {
+  test('GET /health should return 200', async ({ request }) => {
+    const response = await request.get('/health');
+    expect(response.status()).toBe(200);
+  });
+
+  test('GET /users should return user list', async ({ request }) => {
+    const response = await request.get('/users');
+    expect(response.status()).toBe(200);
+    const body = await response.json();
+    expect(Array.isArray(body)).toBe(true);
+  });
+
+  test('POST /users should create user', async ({ request }) => {
+    const userData = {
+      name: 'Test User',
+      email: 'test@example.com'
+    };
+    const response = await request.post('/users', { data: userData });
+    expect(response.status()).toBe(201);
+    const body = await response.json();
+    expect(body).toHaveProperty('id');
+  });
+});`;
+  }
+
+  private getUiTestCode(): string {
+    return `import { test, expect } from '@playwright/test';
+
+test.describe('UI Tests', () => {
+  test('homepage should load', async ({ page }) => {
+    await page.goto('/');
+    await expect(page).toHaveTitle(/Home/);
+  });
+
+  test('navigation should work', async ({ page }) => {
+    await page.goto('/');
+    await page.click('nav a[href="/products"]');
+    await expect(page).toHaveURL(/.*products/);
+  });
+
+  test('search functionality', async ({ page }) => {
+    await page.goto('/');
+    await page.fill('input[name="search"]', 'test product');
+    await page.click('button[type="submit"]');
+    await expect(page.locator('.product-list')).toBeVisible();
+  });
+});`;
+  }
+
+  private getPerfTestCode(): string {
+    return `import { check } from 'k6';
+import http from 'k6/http';
+
+export let options = {
+  vus: 10,
+  duration: '30s',
+  thresholds: {
+    http_req_duration: ['p(95)<500', 'p(99)<1000'],
+    http_req_failed: ['rate<0.01']
+  }
+};
+
+export default function () {
+  let response = http.get('https://api.example.com/health');
+  check(response, {
+    'status is 200': (r) => r.status === 200,
+    'response time < 500ms': (r) => r.timings.duration < 500
+  });
+}`;
+  }
+
+  private getGenericTestCode(): string {
+    return `import { test, expect } from '@playwright/test';
+
+test('generic test', async ({ page }) => {
+  await page.goto('/');
+  await expect(page).toBeVisible();
+});`;
+  }
+
+  countTokens(text: string): number {
+    return Math.ceil(text.length / 4);
+  }
+
+  async isAvailable(): Promise<boolean> {
+    return true;
+  }
+}
+
+describe('CodeGenerator', () => {
+  let mockProvider: MockLLMProvider;
+  let generator: CodeGenerator;
+
+  beforeEach(() => {
+    mockProvider = new MockLLMProvider();
+    // Create generator with mocked provider
+    generator = new CodeGenerator({
+      timeout: 5000,
+      temperature: 0.3,
+      maxTokens: 2000
+    }) as any;
+
+    // Replace the provider with our mock
+    (generator as any).provider = mockProvider;
+  });
+
+  describe('Constructor', () => {
+    it('should create generator with default options', () => {
+      const gen = new CodeGenerator();
+      expect(gen).toBeInstanceOf(CodeGenerator);
+    });
+
+    it('should validate timeout bounds', () => {
+      expect(() => new CodeGenerator({ timeout: 50 })).toThrow('timeout must be between 100ms');
+      expect(() => new CodeGenerator({ timeout: 700000 })).toThrow('timeout must be between');
+    });
+
+    it('should validate temperature bounds', () => {
+      expect(() => new CodeGenerator({ temperature: -0.1 })).toThrow('temperature must be between 0 and 1');
+      expect(() => new CodeGenerator({ temperature: 1.5 })).toThrow('temperature must be between 0 and 1');
+    });
+
+    it('should accept valid options', () => {
+      const gen = new CodeGenerator({
+        timeout: 10000,
+        temperature: 0.7,
+        maxTokens: 4000,
+        formatCode: false
+      });
+      expect(gen).toBeInstanceOf(CodeGenerator);
+    });
+  });
+
+  describe('generateFromDescription', () => {
+    it('should generate tests from description', async () => {
+      const result = await generator.generateFromDescription('Create API smoke tests for a REST API');
+
+      expect(result.success).toBe(true);
+      expect(result.tests).toHaveLength(1);
+      expect(result.tests[0].code).toContain('test.describe');
+      expect(result.tests[0].language).toBe('typescript');
+      expect(result.metadata.model).toBe('mock-model');
+    });
+
+    it('should include metadata in result', async () => {
+      const result = await generator.generateFromDescription('Generate UI tests');
+
+      expect(result.metadata).toBeDefined();
+      expect(result.metadata.duration).toBeGreaterThanOrEqual(0);
+      expect(result.metadata.model).toBe('mock-model');
+      expect(result.metadata.tokensUsed).toBe(300);
+    });
+
+    it('should handle generation errors gracefully', async () => {
+      mockProvider.setShouldFail(true);
+
+      const result = await generator.generateFromDescription('This should fail');
+
+      expect(result.success).toBe(false);
+      expect(result.tests).toHaveLength(0);
+      expect(result.errors).toBeDefined();
+      expect(result.errors?.length).toBeGreaterThan(0);
+    });
+
+    it('should handle timeout', async () => {
+      // Set a delay longer than timeout
+      mockProvider.setDelay(6000);
+
+      const fastGenerator = new CodeGenerator({ timeout: 100 }) as any;
+      (fastGenerator as any).provider = mockProvider;
+
+      const result = await fastGenerator.generateFromDescription('This will timeout');
+
+      expect(result.success).toBe(false);
+      expect(result.errors?.[0]).toContain('TIMEOUT');
+    });
+  });
+
+  describe('generateFromSpec', () => {
+    it('should generate tests from API spec', async () => {
+      const spec = {
+        type: 'api' as const,
+        name: 'User API Tests',
+        baseUrl: 'https://api.example.com',
+        endpoints: [
+          { method: 'GET' as const, path: '/users' },
+          { method: 'POST' as const, path: '/users' }
+        ]
+      };
+
+      const result = await generator.generateFromSpec(spec);
+
+      expect(result.success).toBe(true);
+      expect(result.tests).toHaveLength(1);
+      expect(result.tests[0].code).toContain('API');
+    });
+
+    it('should generate tests from UI spec', async () => {
+      const spec = {
+        type: 'ui' as const,
+        name: 'UI Tests',
+        baseUrl: 'https://example.com',
+        pages: [
+          {
+            name: 'Homepage',
+            url: 'https://example.com',
+            actions: [
+              { type: 'goto' as const, selector: '/' },
+              { type: 'click' as const, selector: 'button' }
+            ]
+          }
+        ]
+      };
+
+      const result = await generator.generateFromSpec(spec);
+
+      expect(result.success).toBe(true);
+      expect(result.tests).toHaveLength(1);
+      expect(result.tests[0].framework).toBe('playwright');
+    });
+
+    it('should generate tests from performance spec', async () => {
+      const spec = {
+        type: 'perf' as const,
+        name: 'Load Tests',
+        baseUrl: 'https://api.example.com',
+        scenarios: [
+          {
+            name: 'Read Load',
+            requests: [
+              { method: 'GET' as const, path: '/api/data' }
+            ]
+          }
+        ]
+      };
+
+      const result = await generator.generateFromSpec(spec);
+
+      expect(result.success).toBe(true);
+      expect(result.tests).toHaveLength(1);
+      // Note: framework is determined by context, defaults to playwright for specs
+      expect(result.tests[0].framework).toBeTruthy();
+    });
+  });
+
+  describe('generateFromOpenAPI', () => {
+    it('should generate tests from OpenAPI spec path', async () => {
+      // Since we're using a mock, this will succeed regardless of file existence
+      // In real scenario, this would parse the OpenAPI file
+      const result = await generator.generateFromOpenAPI('/path/to/openapi.yaml');
+
+      expect(result.success).toBe(true);
+    });
+  });
+
+  describe('generateFromUrl', () => {
+    it('should generate tests from URL', async () => {
+      const result = await generator.generateFromUrl('https://example.com');
+
+      expect(result.success).toBe(true);
+      expect(result.tests[0].code).toContain('test');
+    });
+  });
+
+  describe('generateFromCode', () => {
+    it('should generate tests from code snippet', async () => {
+      const code = `
+export class UserService {
+  async getUser(id: string) {
+    return database.findUser(id);
+  }
+
+  async createUser(data: any) {
+    return database.insertUser(data);
+  }
+}
+      `;
+
+      const result = await generator.generateFromCode(code, 'typescript');
+
+      expect(result.success).toBe(true);
+      expect(result.tests[0].language).toBe('typescript');
+    });
+  });
+
+  describe('listModels', () => {
+    it('should return available models', async () => {
+      const models = await generator.listModels();
+
+      expect(models).toEqual(['mock-model', 'mock-large']);
+    });
+  });
+
+  describe('isAvailable', () => {
+    it('should check provider availability', async () => {
+      const available = await generator.isAvailable();
+
+      expect(available).toBe(true);
+    });
+
+    it('should return false when provider is unavailable', async () => {
+      mockProvider.setShouldFail(true);
+
+      // Mock the isAvailable method
+      vi.spyOn(mockProvider, 'isAvailable').mockResolvedValue(false);
+
+      const available = await generator.isAvailable();
+
+      expect(available).toBe(false);
+    });
+  });
+
+  describe('Code extraction', () => {
+    it('should extract code from markdown code blocks', async () => {
+      mockProvider.setFixedResponse(`
+Here is the generated test code:
+
+\`\`\`typescript
+import { test } from '@playwright/test';
+
+test('example', async ({ page }) => {
+  await page.goto('/');
+});
+\`\`\`
+
+Additional text here.
+      `);
+
+      const result = await generator.generateFromDescription('Test with markdown');
+
+      expect(result.success).toBe(true);
+      expect(result.tests[0].code).toContain('import');
+      expect(result.tests[0].code).toContain('test(');
+    });
+
+    it('should handle multiple code blocks', async () => {
+      mockProvider.setFixedResponse(`
+\`\`\`typescript
+test('first', () => {});
+\`\`\`
+
+\`\`\`typescript
+test('second', () => {});
+\`\`\`
+      `);
+
+      const result = await generator.generateFromDescription('Multiple tests');
+
+      expect(result.success).toBe(true);
+      // Should parse both blocks
+      expect(result.tests[0].code).toContain('first');
+    });
+
+    it('should handle plain code without markdown', async () => {
+      mockProvider.setFixedResponse(`
+import { test } from '@playwright/test';
+
+test('plain code', async ({ page }) => {
+  await page.goto('/');
+});
+      `);
+
+      const result = await generator.generateFromDescription('Plain code');
+
+      expect(result.success).toBe(true);
+      expect(result.tests[0].code).toContain('import');
+      expect(result.tests[0].code).toContain('test(');
+    });
+  });
+
+  describe('File path generation', () => {
+    it('should generate file path with .ts extension', async () => {
+      const result = await generator.generateFromDescription('Create API tests for users endpoint');
+
+      expect(result.tests[0].filePath).toContain('.ts');
+      expect(result.tests[0].filePath).toContain('tests');
+    });
+
+    it('should include test name in file path', async () => {
+      mockProvider.setFixedResponse(MockLLMProvider.prototype.getUiTestCode());
+
+      const result = await generator.generateFromDescription('Create UI tests for homepage');
+
+      // File path should be based on the test name
+      expect(result.tests[0].filePath).toBeTruthy();
+      expect(result.tests[0].filePath.length).toBeGreaterThan(0);
+    });
+
+    it('should generate performance test paths', async () => {
+      mockProvider.setFixedResponse(MockLLMProvider.prototype.getPerfTestCode());
+
+      const result = await generator.generateFromDescription('Create load tests');
+
+      // Note: framework detection is based on defaults, not code content
+      // The generator defaults to 'playwright' for descriptions
+      expect(result.tests[0].filePath).toBeTruthy();
+    });
+  });
+
+  describe('Test summary', () => {
+    it('should calculate test summary correctly', async () => {
+      const result = await generator.generateFromDescription('Generate multiple tests');
+
+      expect(result.tests[0].summary).toBeDefined();
+      expect(result.tests[0].summary.lines).toBeGreaterThan(0);
+      expect(result.tests[0].summary.testCount).toBeGreaterThan(0);
+    });
+  });
+
+  describe('Custom output directory', () => {
+    it('should use custom output directory when specified', async () => {
+      const customGen = new CodeGenerator({ outputDir: '/custom/output' }) as any;
+      (customGen as any).provider = mockProvider;
+
+      const result = await customGen.generateFromDescription('Test output dir');
+
+      expect(result.tests[0].filePath).toContain('/custom/output');
+    });
+  });
+
+  describe('Prompt options', () => {
+    it('should pass prompt options to builder', async () => {
+      const gen = new CodeGenerator({
+        promptOptions: {
+          includeComments: false,
+          includeDetailedAssertions: true,
+          includeRetryLogic: true,
+          codeStyle: 'concise'
+        }
+      }) as any;
+
+      (gen as any).provider = mockProvider;
+
+      const result = await gen.generateFromDescription('Test with options');
+
+      expect(result.success).toBe(true);
+    });
+  });
+});
+
+describe('CodeGenerator Integration', () => {
+  let mockProvider: MockLLMProvider;
+
+  beforeEach(() => {
+    mockProvider = new MockLLMProvider();
+  });
+
+  describe('Full generation workflow', () => {
+    it('should complete full workflow from description to executable test', async () => {
+      const generator = new CodeGenerator({
+        formatCode: true,
+        outputDir: './tests/generated'
+      }) as any;
+
+      (generator as any).provider = mockProvider;
+
+      const description = `
+Create comprehensive API tests for a user management API with the following endpoints:
+- GET /users - List all users
+- GET /users/:id - Get user by ID
+- POST /users - Create new user
+- PUT /users/:id - Update user
+- DELETE /users/:id - Delete user
+
+Include authentication, error handling, and data validation.
+      `;
+
+      const result = await generator.generateFromDescription(description);
+
+      // Verify result structure
+      expect(result.success).toBe(true);
+      expect(result.tests).toHaveLength(1);
+      expect(result.tests[0].name).toBeTruthy();
+      expect(result.tests[0].code).toBeTruthy();
+      expect(result.tests[0].language).toBe('typescript');
+      expect(result.tests[0].framework).toBe('playwright');
+      expect(result.tests[0].filePath).toBeTruthy();
+
+      // Verify generated code quality
+      const code = result.tests[0].code;
+      expect(code).toContain('test');
+      expect(code).toContain('expect');
+      expect(result.tests[0].summary.testCount).toBeGreaterThan(0);
+    });
+
+    it('should handle batch generation from multiple sources', async () => {
+      const generator = new CodeGenerator() as any;
+      (generator as any).provider = mockProvider;
+
+      const sources = [
+        { type: 'description' as const, description: 'API health check tests' },
+        { type: 'description' as const, description: 'UI navigation tests' },
+        { type: 'description' as const, description: 'Load tests' }
+      ];
+
+      const results = await Promise.all(
+        sources.map(s => generator.generateFromSource(s, {}))
+      );
+
+      expect(results).toHaveLength(3);
+      expect(results.every(r => r.success)).toBe(true);
+    });
+  });
+});