qa360 2.1.2 → 2.1.4

package/core/src/generation/prompt-builder.ts

@@ -0,0 +1,772 @@
+/**
+ * QA360 Prompt Builder
+ *
+ * Constructs effective prompts for LLM test generation.
+ * Uses system prompts, few-shot examples, and structured templates.
+ */
+
+import type { TestSpec, ApiTestSpec, UiTestSpec, PerfTestSpec, UnitTestSpec, GenerationSource } from './types.js';
+
+/**
+ * Prompt builder options
+ */
+export interface PromptBuilderOptions {
+  /**
+   * Include code comments in generated tests
+   */
+  includeComments?: boolean;
+
+  /**
+   * Include detailed assertions
+   */
+  includeDetailedAssertions?: boolean;
+
+  /**
+   * Include retry logic for flaky tests
+   */
+  includeRetryLogic?: boolean;
+
+  /**
+   * Target code style
+   */
+  codeStyle?: 'clean' | 'verbose' | 'concise';
+
+  /**
+   * Few-shot examples to include
+   */
+  examples?: string[];
+
+  /**
+   * Custom system prompt
+   */
+  systemPrompt?: string;
+}
+
+/**
+ * Built prompt result
+ */
+export interface BuiltPrompt {
+  system: string;
+  user: string;
+  estimatedTokens: number;
+}
+
+/**
+ * Prompt Builder class
+ */
+export class PromptBuilder {
+  private readonly options: Required<PromptBuilderOptions>;
+
+  private readonly defaultOptions: Required<PromptBuilderOptions> = {
+    includeComments: true,
+    includeDetailedAssertions: true,
+    includeRetryLogic: true,
+    codeStyle: 'clean',
+    examples: [],
+    systemPrompt: '',
+  };
+
+  constructor(options: PromptBuilderOptions = {}) {
+    this.options = { ...this.defaultOptions, ...options };
+  }
+
+  /**
+   * Build prompt from generation source
+   */
+  async buildFromSource(source: GenerationSource): Promise<BuiltPrompt> {
+    switch (source.type) {
+      case 'openapi':
+        return this.buildFromOpenAPI(source.urlOrPath);
+      case 'har':
+        return this.buildFromHAR(source.filePath);
+      case 'url':
+        return this.buildFromUrl(source.url);
+      case 'spec':
+        return this.buildFromSpec(source.spec);
+      case 'description':
+        return this.buildFromDescription(source.description);
+      case 'code':
+        return this.buildFromCode(source.code, source.language);
+      default:
+        throw new Error(`Unsupported source type: ${(source as any).type}`);
+    }
+  }
+
+  /**
+   * Build prompt from OpenAPI/Swagger spec
+   */
+  buildFromOpenAPI(specPath: string): BuiltPrompt {
+    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
+    const user = `You are a test generation expert. Generate comprehensive tests from the following OpenAPI/Swagger specification.
+
+**OpenAPI Spec Location**: ${specPath}
+
+**Requirements**:
+1. Generate tests for all major endpoints
+2. Include happy path tests for each operation
+3. Include error cases (400, 401, 404, 500)
+4. Use TypeScript with Playwright
+5. Include proper assertions for status codes and response schemas
+6. Add authentication if the spec requires it
+
+**Output Format**:
+Return valid TypeScript code that can be used with Playwright testing framework.
+Include clear comments describing each test.
+
+${this.getCodeStyleInstructions()}
+
+${this.getFewShotExample('api')}`;
+
+    return {
+      system,
+      user,
+      estimatedTokens: this.countTokens(system + user),
+    };
+  }
+
+  /**
+   * Build prompt from HAR file
+   */
+  buildFromHAR(filePath: string): BuiltPrompt {
+    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
+    const user = `You are a test generation expert. Convert the following HAR (HTTP Archive) file into Playwright API tests.
+
+**HAR File**: ${filePath}
+
+**Requirements**:
+1. Create a test for each unique request in the HAR
+2. Extract request parameters, headers, and body
+3. Add assertions based on the response status and structure
+4. Group related tests logically
+5. Use TypeScript with Playwright
+
+**Output Format**:
+Return valid TypeScript code with Playwright API tests.
+Include setup/teardown for any required state.
+
+${this.getCodeStyleInstructions()}
+
+${this.getFewShotExample('api')}`;
+
+    return {
+      system,
+      user,
+      estimatedTokens: this.countTokens(system + user),
+    };
+  }
+
+  /**
+   * Build prompt from URL
+   */
+  buildFromUrl(url: string): BuiltPrompt {
+    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
+    const user = `You are a test generation expert. Generate comprehensive API smoke tests for the following URL.
+
+**Target URL**: ${url}
+
+**Requirements**:
+1. Analyze the URL structure to infer API endpoints
+2. Generate tests for common REST patterns (list, get, create, update, delete)
+3. Include health check if applicable
+4. Add authentication placeholders
+5. Use TypeScript with Playwright
+6. Include proper error handling
+
+**Assumptions**:
+- Assume RESTful API conventions
+- Use reasonable defaults for any missing information
+- Add TODO comments where assumptions are made
+
+${this.getCodeStyleInstructions()}
+
+${this.getFewShotExample('api')}`;
+
+    return {
+      system,
+      user,
+      estimatedTokens: this.countTokens(system + user),
+    };
+  }
+
+  /**
+   * Build prompt from test spec
+   */
+  buildFromSpec(spec: TestSpec): BuiltPrompt {
+    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
+
+    let specPrompt = '';
+    let exampleType = 'api';
+
+    switch (spec.type) {
+      case 'api':
+        specPrompt = this.buildApiSpecPrompt(spec as ApiTestSpec);
+        exampleType = 'api';
+        break;
+      case 'ui':
+        specPrompt = this.buildUiSpecPrompt(spec as UiTestSpec);
+        exampleType = 'ui';
+        break;
+      case 'perf':
+        specPrompt = this.buildPerfSpecPrompt(spec as PerfTestSpec);
+        exampleType = 'perf';
+        break;
+      case 'unit':
+        specPrompt = this.buildUnitSpecPrompt(spec as UnitTestSpec);
+        exampleType = 'unit';
+        break;
+    }
+
+    const user = `You are a test generation expert. Generate tests based on the following specification.
+
+**Test Specification**:
+${specPrompt}
+
+**Requirements**:
+1. Follow the specification exactly
+2. Include all specified endpoints/actions/scenarios
+3. Add appropriate assertions
+4. Handle edge cases
+5. Use clean, maintainable code
+
+${this.getCodeStyleInstructions()}
+
+${this.getFewShotExample(exampleType)}`;
+
+    return {
+      system,
+      user,
+      estimatedTokens: this.countTokens(system + user),
+    };
+  }
+
+  /**
+   * Build prompt from description
+   */
+  buildFromDescription(description: string): BuiltPrompt {
+    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
+    const user = `You are a test generation expert. Generate tests based on the following description.
+
+**Description**:
+${description}
+
+**Requirements**:
+1. Infer the test requirements from the description
+2. Generate comprehensive, production-ready tests
+3. Make reasonable assumptions and document them
+4. Include proper assertions and error handling
+
+${this.getCodeStyleInstructions()}
+
+${this.getFewShotExample('api')}`;
+
+    return {
+      system,
+      user,
+      estimatedTokens: this.countTokens(system + user),
+    };
+  }
+
+  /**
+   * Build prompt from existing code
+   */
+  buildFromCode(code: string, language: string): BuiltPrompt {
+    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
+    const user = `You are a test generation expert. Generate comprehensive tests for the following code.
+
+**Code Language**: ${language}
+**Code to Test**:
+\`\`\`
+${code}
+\`\`\`
+
+**Requirements**:
+1. Test all exported functions/classes
+2. Include edge cases and error conditions
+3. Test valid and invalid inputs
+4. Mock external dependencies
+5. Aim for high coverage
+6. Use ${language === 'typescript' ? 'Vitest' : language === 'python' ? 'pytest' : 'appropriate testing framework'}
+
+${this.getCodeStyleInstructions()}
+
+${this.getFewShotExample('unit')}`;
+
+    return {
+      system,
+      user,
+      estimatedTokens: this.countTokens(system + user),
+    };
+  }
+
+  /**
+   * Build API spec prompt
+   */
+  private buildApiSpecPrompt(spec: ApiTestSpec): string {
+    const parts = [
+      `**Type**: API Tests`,
+      `**Base URL**: ${spec.baseUrl}`,
+      `**Test Name**: ${spec.name}`,
+    ];
+
+    if (spec.description) {
+      parts.push(`**Description**: ${spec.description}`);
+    }
+
+    parts.push('\n**Endpoints**:');
+    for (const endpoint of spec.endpoints) {
+      parts.push(`- ${endpoint.method} ${endpoint.path}`);
+      if (endpoint.description) {
+        parts.push(`  Description: ${endpoint.description}`);
+      }
+      if (endpoint.expectations && endpoint.expectations.length > 0) {
+        parts.push(`  Expectations:`);
+        for (const exp of endpoint.expectations) {
+          parts.push(`    - Status: ${exp.status}, Response Time: ${exp.responseTime || 'N/A'}`);
+        }
+      }
+    }
+
+    if (spec.auth) {
+      parts.push(`\n**Authentication**: ${spec.auth.type}`);
+    }
+
+    return parts.join('\n');
+  }
+
+  /**
+   * Build UI spec prompt
+   */
+  private buildUiSpecPrompt(spec: UiTestSpec): string {
+    const parts = [
+      `**Type**: UI/E2E Tests`,
+      `**Base URL**: ${spec.baseUrl}`,
+      `**Test Name**: ${spec.name}`,
+    ];
+
+    if (spec.description) {
+      parts.push(`**Description**: ${spec.description}`);
+    }
+
+    if (spec.viewport) {
+      parts.push(`**Viewport**: ${spec.viewport.width}x${spec.viewport.height}`);
+    }
+
+    parts.push('\n**Pages**:');
+    for (const page of spec.pages) {
+      parts.push(`- ${page.name}: ${page.url}`);
+      parts.push('  Actions:');
+      for (const action of page.actions) {
+        parts.push(`    - ${action.type} ${action.selector || ''} ${action.value || ''}`);
+      }
+      if (page.assertions && page.assertions.length > 0) {
+        parts.push('  Assertions:');
+        for (const assertion of page.assertions) {
+          parts.push(`    - ${assertion.type} ${assertion.expected}`);
+        }
+      }
+    }
+
+    return parts.join('\n');
+  }
+
+  /**
+   * Build performance spec prompt
+   */
+  private buildPerfSpecPrompt(spec: PerfTestSpec): string {
+    const parts = [
+      `**Type**: Performance Tests`,
+      `**Base URL**: ${spec.baseUrl}`,
+      `**Test Name**: ${spec.name}`,
+    ];
+
+    if (spec.options) {
+      parts.push(`**Duration**: ${spec.options.duration || '30s'}`);
+      parts.push(`**Rate**: ${spec.options.rate || 'auto'} req/s`);
+      parts.push(`**VUs**: ${spec.options.vus || 10}`);
+    }
+
+    parts.push('\n**Scenarios**:');
+    for (const scenario of spec.scenarios) {
+      parts.push(`- ${scenario.name} (weight: ${scenario.weight || 1})`);
+      for (const req of scenario.requests) {
+        parts.push(`  - ${req.method} ${req.path}`);
+      }
+    }
+
+    if (spec.options?.thresholds) {
+      parts.push('\n**Thresholds**:');
+      for (const threshold of spec.options.thresholds) {
+        parts.push(`  - ${threshold.metric}: ${threshold.condition}`);
+      }
+    }
+
+    return parts.join('\n');
+  }
+
+  /**
+   * Build unit test spec prompt
+   */
+  private buildUnitSpecPrompt(spec: UnitTestSpec): string {
+    const parts = [
+      `**Type**: Unit Tests`,
+      `**Test Name**: ${spec.name}`,
+      `**Language**: ${spec.source.language}`,
+    ];
+
+    if (spec.source.exports) {
+      parts.push(`**Functions/Classes to Test**: ${spec.source.exports.join(', ')}`);
+    }
+
+    parts.push(`**Framework**: ${spec.framework || 'vitest'}`);
+
+    return parts.join('\n');
+  }
+
+  /**
+   * Get default system prompt
+   */
+  private getDefaultSystemPrompt(): string {
+    return `You are an expert test generation engineer. Your role is to create high-quality, maintainable test code.
+
+**Principles**:
+1. **Clarity**: Tests should be self-documenting with clear names and structure
+2. **Reliability**: Use proper waits, assertions, and error handling
+3. **Maintainability**: Follow DRY principles, use helpers when appropriate
+4. **Coverage**: Test happy paths AND edge cases
+5. **Best Practices**: Follow testing framework conventions and idioms
+
+**Output Requirements**:
+- Return ONLY the test code, no explanations outside the code
+- Use descriptive test names that explain what is being tested
+- Include assertions for all critical conditions
+- Handle errors appropriately
+- Add comments for complex logic`;
+  }
+
+  /**
+   * Get code style instructions
+   */
+  private getCodeStyleInstructions(): string {
+    const style = this.options.codeStyle;
+    const instructions = [];
+
+    instructions.push('**Code Style**:\n');
+
+    switch (style) {
+      case 'clean':
+        instructions.push('- Use clear, descriptive names');
+        instructions.push('- Group related tests with describe blocks');
+        instructions.push('- Extract common setup into beforeEach/beforeAll');
+        break;
+      case 'verbose':
+        instructions.push('- Include detailed comments for each step');
+        instructions.push('- Add explanatory comments for assertions');
+        instructions.push('- Document setup and teardown');
+        break;
+      case 'concise':
+        instructions.push('- Keep tests brief and to the point');
+        instructions.push('- Use minimal comments (only for complex logic)');
+        instructions.push('- Leverage framework features for conciseness');
+        break;
+    }
+
+    if (this.options.includeComments) {
+      instructions.push('- Include helpful comments explaining test intent');
+    } else {
+      instructions.push('- Minimize comments (let code be self-documenting)');
+    }
+
+    if (this.options.includeDetailedAssertions) {
+      instructions.push('- Include detailed assertions for all response properties');
+    }
+
+    if (this.options.includeRetryLogic) {
+      instructions.push('- Add retry logic for potentially flaky network operations');
+    }
+
+    return instructions.join('\n');
+  }
+
+  /**
+   * Get few-shot example
+   */
+  private getFewShotExample(type: string): string {
+    if (this.options.examples.length > 0) {
+      return '**Example**:\n' + this.options.examples.join('\n\n');
+    }
+
+    // Built-in examples
+    const examples: Record<string, string> = {
+      api: this.getApiExample(),
+      ui: this.getUiExample(),
+      perf: this.getPerfExample(),
+      unit: this.getUnitExample(),
+    };
+
+    return examples[type] || '';
+  }
+
+  /**
+   * Get API test example
+   */
+  private getApiExample(): string {
+    return `**Example: API Test with Playwright**
+
+\`\`\`typescript
+import { test, expect } from '@playwright/test';
+
+test.describe('Users API', () => {
+  const baseUrl = 'https://api.example.com';
+  let authToken: string;
+
+  test.beforeAll(async () => {
+    // Setup: Get auth token
+    const response = await fetch(\`\${baseUrl}/auth/login\`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ username: 'test', password: 'test' }),
+    });
+    const data = await response.json();
+    authToken = data.token;
+  });
+
+  test('GET /users - should return list of users', async () => {
+    const response = await fetch(\`\${baseUrl}/users\`, {
+      headers: { 'Authorization': \`Bearer \${authToken}\` },
+    });
+
+    expect(response.status).toBe(200);
+    const data = await response.json();
+    expect(Array.isArray(data.users)).toBe(true);
+    expect(data.users.length).toBeGreaterThan(0);
+  });
+
+  test('GET /users/:id - should return specific user', async () => {
+    const response = await fetch(\`\${baseUrl}/users/1\`, {
+      headers: { 'Authorization': \`Bearer \${authToken}\` },
+    });
+
+    expect(response.status).toBe(200);
+    const user = await response.json();
+    expect(user.id).toBe(1);
+    expect(user).toHaveProperty('name');
+    expect(user).toHaveProperty('email');
+  });
+
+  test('POST /users - should create new user', async () => {
+    const newUser = {
+      name: 'Test User',
+      email: 'test@example.com',
+    };
+
+    const response = await fetch(\`\${baseUrl}/users\`, {
+      method: 'POST',
+      headers: {
+        'Authorization': \`Bearer \${authToken}\`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(newUser),
+    });
+
+    expect(response.status).toBe(201);
+    const created = await response.json();
+    expect(created.id).toBeDefined();
+    expect(created.name).toBe(newUser.name);
+  });
+
+  test('GET /users/:id - should return 404 for non-existent user', async () => {
+    const response = await fetch(\`\${baseUrl}/users/999999\`, {
+      headers: { 'Authorization': \`Bearer \${authToken}\` },
+    });
+
+    expect(response.status).toBe(404);
+  });
+});
+\`\`\``;
+  }
+
+  /**
+   * Get UI test example
+   */
+  private getUiExample(): string {
+    return `**Example: UI Test with Playwright**
+
+\`\`\`typescript
+import { test, expect } from '@playwright/test';
+
+test.describe('Login Page', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.goto('https://example.com/login');
+  });
+
+  test('should show login form', async ({ page }) => {
+    await expect(page.locator('input[name="email"]')).toBeVisible();
+    await expect(page.locator('input[name="password"]')).toBeVisible();
+    await expect(page.locator('button[type="submit"]')).toBeVisible();
+  });
+
+  test('should login with valid credentials', async ({ page }) => {
+    await page.fill('input[name="email"]', 'test@example.com');
+    await page.fill('input[name="password"]', 'password123');
+    await page.click('button[type="submit"]');
+
+    // Wait for navigation
+    await page.waitForURL('**/dashboard');
+    await expect(page).toHaveURL(/.*dashboard/);
+  });
+
+  test('should show error with invalid credentials', async ({ page }) => {
+    await page.fill('input[name="email"]', 'invalid@example.com');
+    await page.fill('input[name="password"]', 'wrongpassword');
+    await page.click('button[type="submit"]');
+
+    await expect(page.locator('.error-message')).toBeVisible();
+    await expect(page.locator('.error-message')).toContainText('Invalid credentials');
+  });
+
+  test('should require email field', async ({ page }) => {
+    await page.fill('input[name="password"]', 'password123');
+    await page.click('button[type="submit"]');
+
+    await expect(page.locator('input[name="email"]')).toHaveAttribute('required', '');
+  });
+});
+\`\`\``;
+  }
+
+  /**
+   * Get performance test example
+   */
+  private getPerfExample(): string {
+    return `**Example: Performance Test with K6**
+
+\`\`\`javascript
+import http from 'k6/http';
+import { check, sleep } from 'k6';
+import { Rate } from 'k6/metrics';
+
+const errorRate = new Rate('errors');
+
+export const options = {
+  stages: [
+    { duration: '30s', target: 20 },   // Ramp up
+    { duration: '1m', target: 20 },     // Stay at 20
+    { duration: '20s', target: 0 },     // Ramp down
+  ],
+  thresholds: {
+    http_req_duration: ['p(95)<500', 'p(99)<1000'],
+    http_req_failed: ['rate<0.01'],
+    errors: ['rate<0.05'],
+  },
+};
+
+const BASE_URL = 'https://api.example.com';
+
+export default function () {
+  // List users
+  const listRes = http.get(\`\${BASE_URL}/users\`);
+  check(listRes, {
+    'list status is 200': (r) => r.status === 200,
+    'list has users': (r) => JSON.parse(r.body).users.length > 0,
+  }) || errorRate.add(1);
+
+  sleep(1);
+
+  // Get specific user
+  const getRes = http.get(\`\${BASE_URL}/users/1\`);
+  check(getRes, {
+    'get status is 200': (r) => r.status === 200,
+    'user has name': (r) => JSON.parse(r.body).name !== undefined,
+  }) || errorRate.add(1);
+
+  sleep(Math.random() * 2 + 1); // Random think time 1-3s
+}
+\`\`\``;
+  }
+
+  /**
+   * Get unit test example
+   */
+  private getUnitExample(): string {
+    return `**Example: Unit Test with Vitest**
+
+\`\`\`typescript
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+describe('UserService', () => {
+  let userService: UserService;
+  let mockRepository: any;
+
+  beforeEach(() => {
+    mockRepository = {
+      findById: vi.fn(),
+      create: vi.fn(),
+      update: vi.fn(),
+      delete: vi.fn(),
+    };
+    userService = new UserService(mockRepository);
+  });
+
+  describe('findById', () => {
+    it('should return user when found', async () => {
+      const expectedUser = { id: 1, name: 'Test User' };
+      mockRepository.findById.mockResolvedValue(expectedUser);
+
+      const result = await userService.findById(1);
+
+      expect(result).toEqual(expectedUser);
+      expect(mockRepository.findById).toHaveBeenCalledWith(1);
+    });
+
+    it('should return null when not found', async () => {
+      mockRepository.findById.mockResolvedValue(null);
+
+      const result = await userService.findById(999);
+
+      expect(result).toBeNull();
+    });
+
+    it('should throw on repository error', async () => {
+      mockRepository.findById.mockRejectedValue(new Error('Database error'));
+
+      await expect(userService.findById(1)).rejects.toThrow('Database error');
+    });
+  });
+
+  describe('create', () => {
+    it('should create user with valid data', async () => {
+      const userData = { name: 'New User', email: 'new@example.com' };
+      const createdUser = { id: 1, ...userData };
+      mockRepository.create.mockResolvedValue(createdUser);
+
+      const result = await userService.create(userData);
+
+      expect(result).toEqual(createdUser);
+      expect(mockRepository.create).toHaveBeenCalledWith(userData);
+    });
+
+    it('should throw validation error for invalid email', async () => {
+      const userData = { name: 'User', email: 'invalid-email' };
+
+      await expect(userService.create(userData)).rejects.toThrow('Invalid email');
+    });
+  });
+});
+\`\`\``;
+  }
+
+  /**
+   * Count approximate tokens
+   */
+  private countTokens(text: string): number {
+    // Rough approximation: ~4 chars per token
+    return Math.ceil(text.length / 4);
+  }
+}
+
+/**
+ * Create a prompt builder with default options
+ */
+export function createPromptBuilder(options?: PromptBuilderOptions): PromptBuilder {
+  return new PromptBuilder(options);
+}