qa360 2.2.1 → 2.2.13

package/core/src/generation/prompt-builder.ts

@@ -1,772 +0,0 @@
-/**
- * QA360 Prompt Builder
- *
- * Constructs effective prompts for LLM test generation.
- * Uses system prompts, few-shot examples, and structured templates.
- */
-
-import type { TestSpec, ApiTestSpec, UiTestSpec, PerfTestSpec, UnitTestSpec, GenerationSource } from './types.js';
-
-/**
- * Prompt builder options
- */
-export interface PromptBuilderOptions {
-  /**
-   * Include code comments in generated tests
-   */
-  includeComments?: boolean;
-
-  /**
-   * Include detailed assertions
-   */
-  includeDetailedAssertions?: boolean;
-
-  /**
-   * Include retry logic for flaky tests
-   */
-  includeRetryLogic?: boolean;
-
-  /**
-   * Target code style
-   */
-  codeStyle?: 'clean' | 'verbose' | 'concise';
-
-  /**
-   * Few-shot examples to include
-   */
-  examples?: string[];
-
-  /**
-   * Custom system prompt
-   */
-  systemPrompt?: string;
-}
-
-/**
- * Built prompt result
- */
-export interface BuiltPrompt {
-  system: string;
-  user: string;
-  estimatedTokens: number;
-}
-
-/**
- * Prompt Builder class
- */
-export class PromptBuilder {
-  private readonly options: Required<PromptBuilderOptions>;
-
-  private readonly defaultOptions: Required<PromptBuilderOptions> = {
-    includeComments: true,
-    includeDetailedAssertions: true,
-    includeRetryLogic: true,
-    codeStyle: 'clean',
-    examples: [],
-    systemPrompt: '',
-  };
-
-  constructor(options: PromptBuilderOptions = {}) {
-    this.options = { ...this.defaultOptions, ...options };
-  }
-
-  /**
-   * Build prompt from generation source
-   */
-  async buildFromSource(source: GenerationSource): Promise<BuiltPrompt> {
-    switch (source.type) {
-      case 'openapi':
-        return this.buildFromOpenAPI(source.urlOrPath);
-      case 'har':
-        return this.buildFromHAR(source.filePath);
-      case 'url':
-        return this.buildFromUrl(source.url);
-      case 'spec':
-        return this.buildFromSpec(source.spec);
-      case 'description':
-        return this.buildFromDescription(source.description);
-      case 'code':
-        return this.buildFromCode(source.code, source.language);
-      default:
-        throw new Error(`Unsupported source type: ${(source as any).type}`);
-    }
-  }
-
-  /**
-   * Build prompt from OpenAPI/Swagger spec
-   */
-  buildFromOpenAPI(specPath: string): BuiltPrompt {
-    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
-    const user = `You are a test generation expert. Generate comprehensive tests from the following OpenAPI/Swagger specification.
-
-**OpenAPI Spec Location**: ${specPath}
-
-**Requirements**:
-1. Generate tests for all major endpoints
-2. Include happy path tests for each operation
-3. Include error cases (400, 401, 404, 500)
-4. Use TypeScript with Playwright
-5. Include proper assertions for status codes and response schemas
-6. Add authentication if the spec requires it
-
-**Output Format**:
-Return valid TypeScript code that can be used with Playwright testing framework.
-Include clear comments describing each test.
-
-${this.getCodeStyleInstructions()}
-
-${this.getFewShotExample('api')}`;
-
-    return {
-      system,
-      user,
-      estimatedTokens: this.countTokens(system + user),
-    };
-  }
-
-  /**
-   * Build prompt from HAR file
-   */
-  buildFromHAR(filePath: string): BuiltPrompt {
-    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
-    const user = `You are a test generation expert. Convert the following HAR (HTTP Archive) file into Playwright API tests.
-
-**HAR File**: ${filePath}
-
-**Requirements**:
-1. Create a test for each unique request in the HAR
-2. Extract request parameters, headers, and body
-3. Add assertions based on the response status and structure
-4. Group related tests logically
-5. Use TypeScript with Playwright
-
-**Output Format**:
-Return valid TypeScript code with Playwright API tests.
-Include setup/teardown for any required state.
-
-${this.getCodeStyleInstructions()}
-
-${this.getFewShotExample('api')}`;
-
-    return {
-      system,
-      user,
-      estimatedTokens: this.countTokens(system + user),
-    };
-  }
-
-  /**
-   * Build prompt from URL
-   */
-  buildFromUrl(url: string): BuiltPrompt {
-    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
-    const user = `You are a test generation expert. Generate comprehensive API smoke tests for the following URL.
-
-**Target URL**: ${url}
-
-**Requirements**:
-1. Analyze the URL structure to infer API endpoints
-2. Generate tests for common REST patterns (list, get, create, update, delete)
-3. Include health check if applicable
-4. Add authentication placeholders
-5. Use TypeScript with Playwright
-6. Include proper error handling
-
-**Assumptions**:
-- Assume RESTful API conventions
-- Use reasonable defaults for any missing information
-- Add TODO comments where assumptions are made
-
-${this.getCodeStyleInstructions()}
-
-${this.getFewShotExample('api')}`;
-
-    return {
-      system,
-      user,
-      estimatedTokens: this.countTokens(system + user),
-    };
-  }
-
-  /**
-   * Build prompt from test spec
-   */
-  buildFromSpec(spec: TestSpec): BuiltPrompt {
-    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
-
-    let specPrompt = '';
-    let exampleType = 'api';
-
-    switch (spec.type) {
-      case 'api':
-        specPrompt = this.buildApiSpecPrompt(spec as ApiTestSpec);
-        exampleType = 'api';
-        break;
-      case 'ui':
-        specPrompt = this.buildUiSpecPrompt(spec as UiTestSpec);
-        exampleType = 'ui';
-        break;
-      case 'perf':
-        specPrompt = this.buildPerfSpecPrompt(spec as PerfTestSpec);
-        exampleType = 'perf';
-        break;
-      case 'unit':
-        specPrompt = this.buildUnitSpecPrompt(spec as UnitTestSpec);
-        exampleType = 'unit';
-        break;
-    }
-
-    const user = `You are a test generation expert. Generate tests based on the following specification.
-
-**Test Specification**:
-${specPrompt}
-
-**Requirements**:
-1. Follow the specification exactly
-2. Include all specified endpoints/actions/scenarios
-3. Add appropriate assertions
-4. Handle edge cases
-5. Use clean, maintainable code
-
-${this.getCodeStyleInstructions()}
-
-${this.getFewShotExample(exampleType)}`;
-
-    return {
-      system,
-      user,
-      estimatedTokens: this.countTokens(system + user),
-    };
-  }
-
-  /**
-   * Build prompt from description
-   */
-  buildFromDescription(description: string): BuiltPrompt {
-    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
-    const user = `You are a test generation expert. Generate tests based on the following description.
-
-**Description**:
-${description}
-
-**Requirements**:
-1. Infer the test requirements from the description
-2. Generate comprehensive, production-ready tests
-3. Make reasonable assumptions and document them
-4. Include proper assertions and error handling
-
-${this.getCodeStyleInstructions()}
-
-${this.getFewShotExample('api')}`;
-
-    return {
-      system,
-      user,
-      estimatedTokens: this.countTokens(system + user),
-    };
-  }
-
-  /**
-   * Build prompt from existing code
-   */
-  buildFromCode(code: string, language: string): BuiltPrompt {
-    const system = this.options.systemPrompt || this.getDefaultSystemPrompt();
-    const user = `You are a test generation expert. Generate comprehensive tests for the following code.
-
-**Code Language**: ${language}
-**Code to Test**:
-\`\`\`
-${code}
-\`\`\`
-
-**Requirements**:
-1. Test all exported functions/classes
-2. Include edge cases and error conditions
-3. Test valid and invalid inputs
-4. Mock external dependencies
-5. Aim for high coverage
-6. Use ${language === 'typescript' ? 'Vitest' : language === 'python' ? 'pytest' : 'appropriate testing framework'}
-
-${this.getCodeStyleInstructions()}
-
-${this.getFewShotExample('unit')}`;
-
-    return {
-      system,
-      user,
-      estimatedTokens: this.countTokens(system + user),
-    };
-  }
-
-  /**
-   * Build API spec prompt
-   */
-  private buildApiSpecPrompt(spec: ApiTestSpec): string {
-    const parts = [
-      `**Type**: API Tests`,
-      `**Base URL**: ${spec.baseUrl}`,
-      `**Test Name**: ${spec.name}`,
-    ];
-
-    if (spec.description) {
-      parts.push(`**Description**: ${spec.description}`);
-    }
-
-    parts.push('\n**Endpoints**:');
-    for (const endpoint of spec.endpoints) {
-      parts.push(`- ${endpoint.method} ${endpoint.path}`);
-      if (endpoint.description) {
-        parts.push(`  Description: ${endpoint.description}`);
-      }
-      if (endpoint.expectations && endpoint.expectations.length > 0) {
-        parts.push(`  Expectations:`);
-        for (const exp of endpoint.expectations) {
-          parts.push(`    - Status: ${exp.status}, Response Time: ${exp.responseTime || 'N/A'}`);
-        }
-      }
-    }
-
-    if (spec.auth) {
-      parts.push(`\n**Authentication**: ${spec.auth.type}`);
-    }
-
-    return parts.join('\n');
-  }
-
-  /**
-   * Build UI spec prompt
-   */
-  private buildUiSpecPrompt(spec: UiTestSpec): string {
-    const parts = [
-      `**Type**: UI/E2E Tests`,
-      `**Base URL**: ${spec.baseUrl}`,
-      `**Test Name**: ${spec.name}`,
-    ];
-
-    if (spec.description) {
-      parts.push(`**Description**: ${spec.description}`);
-    }
-
-    if (spec.viewport) {
-      parts.push(`**Viewport**: ${spec.viewport.width}x${spec.viewport.height}`);
-    }
-
-    parts.push('\n**Pages**:');
-    for (const page of spec.pages) {
-      parts.push(`- ${page.name}: ${page.url}`);
-      parts.push('  Actions:');
-      for (const action of page.actions) {
-        parts.push(`    - ${action.type} ${action.selector || ''} ${action.value || ''}`);
-      }
-      if (page.assertions && page.assertions.length > 0) {
-        parts.push('  Assertions:');
-        for (const assertion of page.assertions) {
-          parts.push(`    - ${assertion.type} ${assertion.expected}`);
-        }
-      }
-    }
-
-    return parts.join('\n');
-  }
-
-  /**
-   * Build performance spec prompt
-   */
-  private buildPerfSpecPrompt(spec: PerfTestSpec): string {
-    const parts = [
-      `**Type**: Performance Tests`,
-      `**Base URL**: ${spec.baseUrl}`,
-      `**Test Name**: ${spec.name}`,
-    ];
-
-    if (spec.options) {
-      parts.push(`**Duration**: ${spec.options.duration || '30s'}`);
-      parts.push(`**Rate**: ${spec.options.rate || 'auto'} req/s`);
-      parts.push(`**VUs**: ${spec.options.vus || 10}`);
-    }
-
-    parts.push('\n**Scenarios**:');
-    for (const scenario of spec.scenarios) {
-      parts.push(`- ${scenario.name} (weight: ${scenario.weight || 1})`);
-      for (const req of scenario.requests) {
-        parts.push(`  - ${req.method} ${req.path}`);
-      }
-    }
-
-    if (spec.options?.thresholds) {
-      parts.push('\n**Thresholds**:');
-      for (const threshold of spec.options.thresholds) {
-        parts.push(`  - ${threshold.metric}: ${threshold.condition}`);
-      }
-    }
-
-    return parts.join('\n');
-  }
-
-  /**
-   * Build unit test spec prompt
-   */
-  private buildUnitSpecPrompt(spec: UnitTestSpec): string {
-    const parts = [
-      `**Type**: Unit Tests`,
-      `**Test Name**: ${spec.name}`,
-      `**Language**: ${spec.source.language}`,
-    ];
-
-    if (spec.source.exports) {
-      parts.push(`**Functions/Classes to Test**: ${spec.source.exports.join(', ')}`);
-    }
-
-    parts.push(`**Framework**: ${spec.framework || 'vitest'}`);
-
-    return parts.join('\n');
-  }
-
-  /**
-   * Get default system prompt
-   */
-  private getDefaultSystemPrompt(): string {
-    return `You are an expert test generation engineer. Your role is to create high-quality, maintainable test code.
-
-**Principles**:
-1. **Clarity**: Tests should be self-documenting with clear names and structure
-2. **Reliability**: Use proper waits, assertions, and error handling
-3. **Maintainability**: Follow DRY principles, use helpers when appropriate
-4. **Coverage**: Test happy paths AND edge cases
-5. **Best Practices**: Follow testing framework conventions and idioms
-
-**Output Requirements**:
-- Return ONLY the test code, no explanations outside the code
-- Use descriptive test names that explain what is being tested
-- Include assertions for all critical conditions
-- Handle errors appropriately
-- Add comments for complex logic`;
-  }
-
-  /**
-   * Get code style instructions
-   */
-  private getCodeStyleInstructions(): string {
-    const style = this.options.codeStyle;
-    const instructions = [];
-
-    instructions.push('**Code Style**:\n');
-
-    switch (style) {
-      case 'clean':
-        instructions.push('- Use clear, descriptive names');
-        instructions.push('- Group related tests with describe blocks');
-        instructions.push('- Extract common setup into beforeEach/beforeAll');
-        break;
-      case 'verbose':
-        instructions.push('- Include detailed comments for each step');
-        instructions.push('- Add explanatory comments for assertions');
-        instructions.push('- Document setup and teardown');
-        break;
-      case 'concise':
-        instructions.push('- Keep tests brief and to the point');
-        instructions.push('- Use minimal comments (only for complex logic)');
-        instructions.push('- Leverage framework features for conciseness');
-        break;
-    }
-
-    if (this.options.includeComments) {
-      instructions.push('- Include helpful comments explaining test intent');
-    } else {
-      instructions.push('- Minimize comments (let code be self-documenting)');
-    }
-
-    if (this.options.includeDetailedAssertions) {
-      instructions.push('- Include detailed assertions for all response properties');
-    }
-
-    if (this.options.includeRetryLogic) {
-      instructions.push('- Add retry logic for potentially flaky network operations');
-    }
-
-    return instructions.join('\n');
-  }
-
-  /**
-   * Get few-shot example
-   */
-  private getFewShotExample(type: string): string {
-    if (this.options.examples.length > 0) {
-      return '**Example**:\n' + this.options.examples.join('\n\n');
-    }
-
-    // Built-in examples
-    const examples: Record<string, string> = {
-      api: this.getApiExample(),
-      ui: this.getUiExample(),
-      perf: this.getPerfExample(),
-      unit: this.getUnitExample(),
-    };
-
-    return examples[type] || '';
-  }
-
-  /**
-   * Get API test example
-   */
-  private getApiExample(): string {
-    return `**Example: API Test with Playwright**
-
-\`\`\`typescript
-import { test, expect } from '@playwright/test';
-
-test.describe('Users API', () => {
-  const baseUrl = 'https://api.example.com';
-  let authToken: string;
-
-  test.beforeAll(async () => {
-    // Setup: Get auth token
-    const response = await fetch(\`\${baseUrl}/auth/login\`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ username: 'test', password: 'test' }),
-    });
-    const data = await response.json();
-    authToken = data.token;
-  });
-
-  test('GET /users - should return list of users', async () => {
-    const response = await fetch(\`\${baseUrl}/users\`, {
-      headers: { 'Authorization': \`Bearer \${authToken}\` },
-    });
-
-    expect(response.status).toBe(200);
-    const data = await response.json();
-    expect(Array.isArray(data.users)).toBe(true);
-    expect(data.users.length).toBeGreaterThan(0);
-  });
-
-  test('GET /users/:id - should return specific user', async () => {
-    const response = await fetch(\`\${baseUrl}/users/1\`, {
-      headers: { 'Authorization': \`Bearer \${authToken}\` },
-    });
-
-    expect(response.status).toBe(200);
-    const user = await response.json();
-    expect(user.id).toBe(1);
-    expect(user).toHaveProperty('name');
-    expect(user).toHaveProperty('email');
-  });
-
-  test('POST /users - should create new user', async () => {
-    const newUser = {
-      name: 'Test User',
-      email: 'test@example.com',
-    };
-
-    const response = await fetch(\`\${baseUrl}/users\`, {
-      method: 'POST',
-      headers: {
-        'Authorization': \`Bearer \${authToken}\`,
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify(newUser),
-    });
-
-    expect(response.status).toBe(201);
-    const created = await response.json();
-    expect(created.id).toBeDefined();
-    expect(created.name).toBe(newUser.name);
-  });
-
-  test('GET /users/:id - should return 404 for non-existent user', async () => {
-    const response = await fetch(\`\${baseUrl}/users/999999\`, {
-      headers: { 'Authorization': \`Bearer \${authToken}\` },
-    });
-
-    expect(response.status).toBe(404);
-  });
-});
-\`\`\``;
-  }
-
-  /**
-   * Get UI test example
-   */
-  private getUiExample(): string {
-    return `**Example: UI Test with Playwright**
-
-\`\`\`typescript
-import { test, expect } from '@playwright/test';
-
-test.describe('Login Page', () => {
-  test.beforeEach(async ({ page }) => {
-    await page.goto('https://example.com/login');
-  });
-
-  test('should show login form', async ({ page }) => {
-    await expect(page.locator('input[name="email"]')).toBeVisible();
-    await expect(page.locator('input[name="password"]')).toBeVisible();
-    await expect(page.locator('button[type="submit"]')).toBeVisible();
-  });
-
-  test('should login with valid credentials', async ({ page }) => {
-    await page.fill('input[name="email"]', 'test@example.com');
-    await page.fill('input[name="password"]', 'password123');
-    await page.click('button[type="submit"]');
-
-    // Wait for navigation
-    await page.waitForURL('**/dashboard');
-    await expect(page).toHaveURL(/.*dashboard/);
-  });
-
-  test('should show error with invalid credentials', async ({ page }) => {
-    await page.fill('input[name="email"]', 'invalid@example.com');
-    await page.fill('input[name="password"]', 'wrongpassword');
-    await page.click('button[type="submit"]');
-
-    await expect(page.locator('.error-message')).toBeVisible();
-    await expect(page.locator('.error-message')).toContainText('Invalid credentials');
-  });
-
-  test('should require email field', async ({ page }) => {
-    await page.fill('input[name="password"]', 'password123');
-    await page.click('button[type="submit"]');
-
-    await expect(page.locator('input[name="email"]')).toHaveAttribute('required', '');
-  });
-});
-\`\`\``;
-  }
-
-  /**
-   * Get performance test example
-   */
-  private getPerfExample(): string {
-    return `**Example: Performance Test with K6**
-
-\`\`\`javascript
-import http from 'k6/http';
-import { check, sleep } from 'k6';
-import { Rate } from 'k6/metrics';
-
-const errorRate = new Rate('errors');
-
-export const options = {
-  stages: [
-    { duration: '30s', target: 20 },   // Ramp up
-    { duration: '1m', target: 20 },     // Stay at 20
-    { duration: '20s', target: 0 },     // Ramp down
-  ],
-  thresholds: {
-    http_req_duration: ['p(95)<500', 'p(99)<1000'],
-    http_req_failed: ['rate<0.01'],
-    errors: ['rate<0.05'],
-  },
-};
-
-const BASE_URL = 'https://api.example.com';
-
-export default function () {
-  // List users
-  const listRes = http.get(\`\${BASE_URL}/users\`);
-  check(listRes, {
-    'list status is 200': (r) => r.status === 200,
-    'list has users': (r) => JSON.parse(r.body).users.length > 0,
-  }) || errorRate.add(1);
-
-  sleep(1);
-
-  // Get specific user
-  const getRes = http.get(\`\${BASE_URL}/users/1\`);
-  check(getRes, {
-    'get status is 200': (r) => r.status === 200,
-    'user has name': (r) => JSON.parse(r.body).name !== undefined,
-  }) || errorRate.add(1);
-
-  sleep(Math.random() * 2 + 1); // Random think time 1-3s
-}
-\`\`\``;
-  }
-
-  /**
-   * Get unit test example
-   */
-  private getUnitExample(): string {
-    return `**Example: Unit Test with Vitest**
-
-\`\`\`typescript
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-
-describe('UserService', () => {
-  let userService: UserService;
-  let mockRepository: any;
-
-  beforeEach(() => {
-    mockRepository = {
-      findById: vi.fn(),
-      create: vi.fn(),
-      update: vi.fn(),
-      delete: vi.fn(),
-    };
-    userService = new UserService(mockRepository);
-  });
-
-  describe('findById', () => {
-    it('should return user when found', async () => {
-      const expectedUser = { id: 1, name: 'Test User' };
-      mockRepository.findById.mockResolvedValue(expectedUser);
-
-      const result = await userService.findById(1);
-
-      expect(result).toEqual(expectedUser);
-      expect(mockRepository.findById).toHaveBeenCalledWith(1);
-    });
-
-    it('should return null when not found', async () => {
-      mockRepository.findById.mockResolvedValue(null);
-
-      const result = await userService.findById(999);
-
-      expect(result).toBeNull();
-    });
-
-    it('should throw on repository error', async () => {
-      mockRepository.findById.mockRejectedValue(new Error('Database error'));
-
-      await expect(userService.findById(1)).rejects.toThrow('Database error');
-    });
-  });
-
-  describe('create', () => {
-    it('should create user with valid data', async () => {
-      const userData = { name: 'New User', email: 'new@example.com' };
-      const createdUser = { id: 1, ...userData };
-      mockRepository.create.mockResolvedValue(createdUser);
-
-      const result = await userService.create(userData);
-
-      expect(result).toEqual(createdUser);
-      expect(mockRepository.create).toHaveBeenCalledWith(userData);
-    });
-
-    it('should throw validation error for invalid email', async () => {
-      const userData = { name: 'User', email: 'invalid-email' };
-
-      await expect(userService.create(userData)).rejects.toThrow('Invalid email');
-    });
-  });
-});
-\`\`\``;
-  }
-
-  /**
-   * Count approximate tokens
-   */
-  private countTokens(text: string): number {
-    // Rough approximation: ~4 chars per token
-    return Math.ceil(text.length / 4);
-  }
-}
-
-/**
- * Create a prompt builder with default options
- */
-export function createPromptBuilder(options?: PromptBuilderOptions): PromptBuilder {
-  return new PromptBuilder(options);
-}