qa360 2.1.7 → 2.2.0

package/core/src/generation/__tests__/real-provider-integration.test.ts

@@ -0,0 +1,414 @@
+/**
+ * Real Provider Integration Tests
+ *
+ * Tests for actual LLM provider integration (Ollama, DeepSeek).
+ * These tests are skipped by default and require the actual providers to be available.
+ *
+ * Run with:
+ * - ENABLE_REAL_PROVIDER_TESTS=1 pnpm test:core -- real-provider
+ * - Or with specific provider: TEST_PROVIDER=ollama pnpm test:core -- real-provider
+ */
+
+import { describe, it, expect, beforeAll } from 'vitest';
+import { CodeGenerator } from '../code-generator.js';
+import { OllamaProvider } from '../../ai/ollama-provider.js';
+import { DeepSeekProvider } from '../../ai/deepseek-provider.js';
+import type { GenerationResult } from '../code-generator.js';
+
+// Test configuration
+const shouldRun = process.env.ENABLE_REAL_PROVIDER_TESTS === '1';
+const testProvider = process.env.TEST_PROVIDER || 'ollama';
+const testTimeout = 120000; // 2 minutes for real LLM calls
+
+// Skip tests if not enabled
+const describeOrSkip = shouldRun ? describe : describe.skip;
+
+describeOrSkip('Real Provider Integration Tests', () => {
+  let codeGenerator: CodeGenerator;
+  let providerAvailable = false;
+
+  beforeAll(async () => {
+    if (!shouldRun) return;
+
+    // Check if provider is available
+    let provider;
+    if (testProvider === 'ollama') {
+      provider = new OllamaProvider({ baseUrl: process.env.OLLAMA_BASE_URL });
+    } else if (testProvider === 'deepseek') {
+      provider = new DeepSeekProvider({
+        apiKey: process.env.DEEPSEEK_API_KEY || 'test'
+      });
+    } else {
+      return;
+    }
+
+    try {
+      providerAvailable = await provider.isAvailable();
+      if (!providerAvailable) {
+        console.warn(`\n⚠️  ${testProvider} provider is not available. Skipping real provider tests.`);
+        console.warn(`   To run these tests, ensure ${testProvider} is running and configured.`);
+      }
+    } catch (error) {
+      console.warn(`\n⚠️  Failed to check ${testProvider} availability:`, error);
+      providerAvailable = false;
+    }
+
+    // Create generator if provider is available
+    if (providerAvailable) {
+      codeGenerator = new CodeGenerator({
+        timeout: 60000,
+        temperature: 0.3,
+        maxTokens: 2000
+      }) as any;
+
+      // Replace with real provider
+      (codeGenerator as any).provider = provider;
+    }
+  });
+
+  describe.skipIf(!providerAvailable)('Code Generation with Real LLM', () => {
+    it('should generate tests from description', async () => {
+      const result = await codeGenerator.generateFromDescription(
+        'Generate a simple API smoke test for a health check endpoint'
+      );
+
+      expect(result.success).toBe(true);
+      expect(result.tests).toHaveLength(1);
+      expect(result.tests[0].code).toBeTruthy();
+      expect(result.tests[0].code.length).toBeGreaterThan(50);
+
+      console.log(`\n📝 Generated ${result.tests[0].code.length} characters of code`);
+      console.log(`🧠 Model: ${result.metadata.model}`);
+      console.log(`🔢 Tokens: ${result.metadata.tokensUsed}`);
+    }, testTimeout);
+
+    it('should generate tests from code snippet', async () => {
+      const code = `
+export class Calculator {
+  add(a: number, b: number): number {
+    return a + b;
+  }
+
+  subtract(a: number, b: number): number {
+    return a - b;
+  }
+
+  multiply(a: number, b: number): number {
+    return a * b;
+  }
+}
+      `;
+
+      const result = await codeGenerator.generateFromCode(code, 'typescript');
+
+      expect(result.success).toBe(true);
+      expect(result.tests[0].code).toBeTruthy();
+      // Generated code should test the Calculator class
+      expect(result.tests[0].code.toLowerCase()).toContain('calculator');
+    }, testTimeout);
+
+    it('should include error handling in generated tests', async () => {
+      const result = await codeGenerator.generateFromDescription(
+        'Generate API tests with error handling for a user service'
+      );
+
+      expect(result.success).toBe(true);
+      // Check for error-related patterns
+      const code = result.tests[0].code.toLowerCase();
+      const hasErrorHandling =
+        code.includes('error') ||
+        code.includes('catch') ||
+        code.includes('throw') ||
+        code.includes('400') ||
+        code.includes('500');
+
+      expect(hasErrorHandling).toBe(true);
+    }, testTimeout);
+  });
+
+  describe.skipIf(!providerAvailable)('Model Information', () => {
+    it('should list available models', async () => {
+      const models = await codeGenerator.listModels();
+
+      expect(Array.isArray(models)).toBe(true);
+      expect(models.length).toBeGreaterThan(0);
+
+      console.log(`\n
+    }, testTimeout);
+
+    it('should check provider availability', async () => {
+      const available = await codeGenerator.isAvailable();
+
+      expect(available).toBe(true);
+    });
+  });
+
+  describe.skipIf(!providerAvailable)('Generation Quality', () => {
+    it('should generate syntactically valid test code', async () => {
+      const result = await codeGenerator.generateFromDescription(
+        'Generate a test using @playwright/test framework for a login page'
+      );
+
+      expect(result.success).toBe(true);
+      const code = result.tests[0].code;
+
+      // Check for Playwright patterns
+      expect(code).toMatch(/test\s*\(|describe\s*\(/);
+      expect(code).toMatch(/expect|assert/);
+
+      console.log(`\n✅ Generated valid Playwright test`);
+    }, testTimeout);
+
+    it('should generate tests with proper assertions', async () => {
+      const result = await codeGenerator.generateFromDescription(
+        'Generate API tests for a products endpoint with proper assertions'
+      );
+
+      expect(result.success).toBe(true);
+      const code = result.tests[0].code.toLowerCase();
+
+      // Should have assertions
+      expect(code).toMatch(/expect|assert|to|equal/);
+    }, testTimeout);
+
+    it('should generate multiple test cases when appropriate', async () => {
+      const result = await codeGenerator.generateFromDescription(
+        'Generate comprehensive tests for a shopping cart API including add, remove, and checkout operations'
+      );
+
+      expect(result.success).toBe(true);
+      const code = result.tests[0].code.toLowerCase();
+
+      // Should have multiple tests
+      const testCount = (code.match(/test\s*\(|it\s*\(/g) || []).length;
+      expect(testCount).toBeGreaterThanOrEqual(2);
+
+      console.log(`\n📊 Generated ${testCount} test cases`);
+    }, testTimeout);
+  });
+
+  describe.skipIf(!providerAvailable)('Timeout Handling', () => {
+    it('should handle generation timeout gracefully', async () => {
+      // Create generator with very short timeout
+      const shortTimeoutGenerator = new CodeGenerator({
+        timeout: 100 // 100ms - should timeout
+      }) as any;
+
+      (shortTimeoutGenerator as any).provider = (codeGenerator as any).provider;
+
+      const result = await shortTimeoutGenerator.generateFromDescription(
+        'Generate a very long and complex test suite'
+      );
+
+      // Either success or timeout error
+      expect(result).toBeDefined();
+
+      if (!result.success) {
+        expect(result.errors?.[0]).toContain('TIMEOUT');
+        console.log(`\n⏱️  Correctly timed out after 100ms`);
+      }
+    });
+  });
+
+  describe.skipIf(!providerAvailable)('Temperature Effects', () => {
+    it('should generate more creative tests with higher temperature', async () => {
+      const creativeGenerator = new CodeGenerator({
+        temperature: 0.9,
+        timeout: 30000
+      }) as any;
+
+      (creativeGenerator as any).provider = (codeGenerator as any).provider;
+
+      const result = await creativeGenerator.generateFromDescription(
+        'Generate creative test scenarios for a social media API'
+      );
+
+      expect(result.success).toBe(true);
+      expect(result.tests[0].code.length).toBeGreaterThan(0);
+    }, testTimeout);
+  });
+
+  describe.skipIf(!providerAvailable)('Different Languages', () => {
+    it('should generate Python tests', async () => {
+      const code = `
+class UserService:
+    def get_user(self, user_id: str) -> User:
+        return self.db.find(user_id)
+
+    def create_user(self, data: dict) -> User:
+        return self.db.insert(data)
+      `;
+
+      const result = await codeGenerator.generateFromCode(code, 'python', {
+        language: 'python',
+        framework: 'pytest'
+      });
+
+      expect(result.success).toBe(true);
+      // Should include Python-specific patterns
+      expect(result.tests[0].code.toLowerCase()).toMatch(/def |class |pytest/);
+    }, testTimeout);
+
+    it('should generate Go tests', async () => {
+      const code = `
+package main
+
+func Add(a int, b int) int {
+    return a + b
+}
+
+func Multiply(a int, b int) int {
+    return a * b
+}
+      `;
+
+      const result = await codeGenerator.generateFromCode(code, 'go', {
+        language: 'go',
+        framework: 'go test'
+      });
+
+      expect(result.success).toBe(true);
+      // Should include Go-specific patterns
+      expect(result.tests[0].code).toMatch(/func |t\.Run|testing/);
+    }, testTimeout);
+  });
+});
+
+describe.skipIf(!shouldRun)('Provider Comparison Tests', () => {
+  it('should compare Ollama and DeepSeek responses', async () => {
+    const description = 'Generate a simple unit test for a function that adds two numbers';
+
+    const results = [];
+
+    // Test Ollama if available
+    const ollamaProvider = new OllamaProvider();
+    if (await ollamaProvider.isAvailable()) {
+      const ollamaGenerator = new CodeGenerator() as any;
+      ollamaGenerator.provider = ollamaProvider;
+
+      const ollamaResult = await ollamaGenerator.generateFromDescription(description);
+      results.push({ provider: 'ollama', result: ollamaResult });
+      console.log(`\n✅ Ollama: ${ollamaResult.tests[0].code.length} chars`);
+    }
+
+    // Test DeepSeek if API key is available
+    if (process.env.DEEPSEEK_API_KEY) {
+      const deepseekProvider = new DeepSeekProvider({
+        apiKey: process.env.DEEPSEEK_API_KEY
+      });
+
+      if (await deepseekProvider.isAvailable()) {
+        const deepseekGenerator = new CodeGenerator() as any;
+        deepseekGenerator.provider = deepseekProvider;
+
+        const deepseekResult = await deepseekGenerator.generateFromDescription(description);
+        results.push({ provider: 'deepseek', result: deepseekResult });
+        console.log(`✅ DeepSeek: ${deepseekResult.tests[0].code.length} chars`);
+      }
+    }
+
+    // At least one provider should work
+    expect(results.length).toBeGreaterThan(0);
+
+    // All should succeed
+    for (const { provider, result } of results) {
+      expect(result.success).toBe(true);
+      expect(result.tests[0].code.length).toBeGreaterThan(0);
+    }
+  }, 120000);
+});
+
+describe('Real Provider Test Helpers', () => {
+  describe('Provider Detection', () => {
+    it('should detect Ollama availability', async () => {
+      const provider = new OllamaProvider();
+      const available = await provider.isAvailable();
+
+      console.log(`\n${available ? '✅' : '❌'} Ollama available: ${available}`);
+      expect(typeof available).toBe('boolean');
+    });
+
+    it('should detect DeepSeek availability', async () => {
+      if (!process.env.DEEPSEEK_API_KEY) {
+        console.log('\n⚠️  DEEPSEEK_API_KEY not set, skipping DeepSeek check');
+        return;
+      }
+
+      const provider = new DeepSeekProvider({
+        apiKey: process.env.DEEPSEEK_API_KEY
+      });
+      const available = await provider.isAvailable();
+
+      console.log(`\n${available ? '✅' : '❌'} DeepSeek available: ${available}`);
+      expect(typeof available).toBe('boolean');
+    });
+  });
+
+  describe('Model Listing', () => {
+    it('should list Ollama models', async () => {
+      const provider = new OllamaProvider();
+      const available = await provider.isAvailable();
+
+      if (!available) {
+        console.log('\n⚠️  Ollama not available, skipping model listing');
+        return;
+      }
+
+      const models = await provider.listModels();
+
+      console.log(`\n📦 Ollama models: ${models.map(m => m.name).join(', ')}`);
+      expect(Array.isArray(models)).toBe(true);
+    }, 10000);
+  });
+});
+
+describe('Real Provider Smoke Tests', () => {
+  describe('Quick smoke test', () => {
+    it('should run quick smoke test with any available provider', async () => {
+      let providerAvailable = false;
+      let testResult: GenerationResult | null = null;
+
+      // Try Ollama first
+      const ollamaProvider = new OllamaProvider();
+      if (await ollamaProvider.isAvailable()) {
+        providerAvailable = true;
+
+        const generator = new CodeGenerator({
+          timeout: 30000
+        }) as any;
+        generator.provider = ollamaProvider;
+
+        testResult = await generator.generateFromDescription('Generate a simple test');
+        console.log('\n✅ Ollama smoke test passed');
+      }
+
+      // If Ollama not available, try DeepSeek
+      if (!providerAvailable && process.env.DEEPSEEK_API_KEY) {
+        const deepseekProvider = new DeepSeekProvider({
+          apiKey: process.env.DEEPSEEK_API_KEY
+        });
+
+        if (await deepseekProvider.isAvailable()) {
+          providerAvailable = true;
+
+          const generator = new CodeGenerator({
+            timeout: 30000
+          }) as any;
+          generator.provider = deepseekProvider;
+
+          testResult = await generator.generateFromDescription('Generate a simple test');
+          console.log('\n✅ DeepSeek smoke test passed');
+        }
+      }
+
+      if (!providerAvailable) {
+        console.log('\n⚠️  No real provider available for smoke test');
+        return;
+      }
+
+      expect(testResult).toBeDefined();
+      expect(testResult?.success).toBe(true);
+      expect(testResult?.tests[0].code.length).toBeGreaterThan(0);
+    }, 60000);
+  });
+});