qa360 2.1.7 → 2.2.0

package/core/src/ai/anthropic-provider.ts

@@ -0,0 +1,262 @@
+/**
+ * Anthropic LLM Provider
+ *
+ * Provider for Anthropic API (https://api.anthropic.com).
+ * Supports Claude 3.5 Sonnet, Claude 3 Opus, and Claude 3 Haiku.
+ *
+ * Environment variables:
+ * - ANTHROPIC_API_KEY: Required API key
+ * - ANTHROPIC_BASE_URL: Optional base URL (default: https://api.anthropic.com)
+ * - ANTHROPIC_MODEL: Model to use (default: claude-3-5-sonnet-20241022)
+ */
+
+import type { LLMProvider, GenerationRequest, GenerationResponse } from './index.js';
+
+export interface AnthropicConfig {
+  apiKey?: string;
+  baseURL?: string;
+  model?: string;
+  timeout?: number;
+}
+
+export class AnthropicProvider implements LLMProvider {
+  name = 'anthropic';
+
+  // Recommended models for code generation
+  models = [
+    'claude-3-5-sonnet-20241022',  // Most capable for code
+    'claude-3-5-sonnet-20240620',  // Previous Sonnet version
+    'claude-3-opus-20240229',      // Most capable overall
+    'claude-3-haiku-20240307',     // Fastest, cheapest
+  ];
+
+  private readonly apiKey: string;
+  private readonly baseURL: string;
+  private readonly defaultModel: string;
+  private readonly timeout: number;
+  private readonly version = '2023-06-01'; // API version header
+
+  constructor(config: AnthropicConfig = {}) {
+    this.apiKey = config.apiKey || process.env.ANTHROPIC_API_KEY || '';
+    this.baseURL = config.baseURL || process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com';
+    this.defaultModel = config.model || process.env.ANTHROPIC_MODEL || 'claude-3-5-sonnet-20241022';
+    this.timeout = config.timeout || 60000; // 1 minute default
+  }
+
+  async isAvailable(): Promise<boolean> {
+    // Check if API key is present
+    if (!this.apiKey) {
+      return false;
+    }
+
+    // Try a minimal API call
+    try {
+      // Use timeout with AbortController for compatibility
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), 5000);
+
+      const response = await fetch(`${this.baseURL}/v1/messages`, {
+        method: 'POST',
+        headers: {
+          'x-api-key': this.apiKey,
+          'anthropic-version': this.version,
+          'content-type': 'application/json',
+        },
+        signal: controller.signal,
+        body: JSON.stringify({
+          model: this.defaultModel,
+          max_tokens: 1,
+          messages: [{ role: 'user', content: 'Hi' }]
+        })
+      });
+
+      clearTimeout(timeoutId);
+      // Anthropic may return 400 for malformed request but still be available
+      // We consider it available if we get any response (not a network error)
+      return response.status !== 401 && response.status !== 403;
+    } catch {
+      return false;
+    }
+  }
+
+  async generate(request: GenerationRequest): Promise<GenerationResponse> {
+    if (!this.apiKey) {
+      throw new AnthropicError('ANTHROPIC_API_KEY is required', {
+        suggestion: 'Set ANTHROPIC_API_KEY environment variable or pass apiKey in config'
+      });
+    }
+
+    const messages = this.buildMessages(request);
+
+    // Use timeout with AbortController for compatibility
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseURL}/v1/messages`, {
+        method: 'POST',
+        headers: {
+          'x-api-key': this.apiKey,
+          'anthropic-version': this.version,
+          'content-type': 'application/json',
+        },
+        signal: controller.signal,
+        body: JSON.stringify({
+          model: this.defaultModel,
+          messages,
+          system: request.systemPrompt,
+          temperature: request.temperature ?? 0.7,
+          max_tokens: request.maxTokens ?? 4096,
+        })
+      });
+
+      if (!response.ok) {
+        const error = await response.text().catch(() => 'Unknown error');
+        throw new AnthropicError(`Anthropic request failed: ${response.status} ${error}`, {
+          status: response.status,
+          response: error
+        });
+      }
+
+      const data = await response.json();
+
+      if (data.error) {
+        throw new AnthropicError(`Anthropic API error: ${data.error.message}`, {
+          type: data.error.type
+        });
+      }
+
+      const content = this.extractContent(data.content);
+
+      return {
+        content,
+        usage: {
+          promptTokens: data.usage?.input_tokens || 0,
+          completionTokens: data.usage?.output_tokens || 0,
+          totalTokens: (data.usage?.input_tokens || 0) + (data.usage?.output_tokens || 0),
+        },
+        model: data.model,
+        finishReason: data.stop_reason === 'end_turn' ? 'stop' : 'length'
+      };
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  }
+
+  async *stream(request: GenerationRequest): AsyncIterable<string> {
+    if (!this.apiKey) {
+      throw new AnthropicError('ANTHROPIC_API_KEY is required');
+    }
+
+    const messages = this.buildMessages(request);
+
+    const response = await fetch(`${this.baseURL}/v1/messages`, {
+      method: 'POST',
+      headers: {
+        'x-api-key': this.apiKey,
+        'anthropic-version': this.version,
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({
+        model: this.defaultModel,
+        messages,
+        system: request.systemPrompt,
+        temperature: request.temperature ?? 0.7,
+        max_tokens: request.maxTokens ?? 4096,
+        stream: true,
+      })
+    });
+
+    if (!response.ok) {
+      throw new AnthropicError(`Anthropic stream failed: ${response.status}`);
+    }
+
+    const reader = response.body?.getReader();
+    if (!reader) {
+      throw new AnthropicError('No response body');
+    }
+
+    const decoder = new TextDecoder();
+    let buffer = '';
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      buffer += decoder.decode(value, { stream: true });
+
+      const lines = buffer.split('\n');
+      buffer = lines.pop() || '';
+
+      for (const line of lines) {
+        if (line.startsWith('data: ')) {
+          const data = line.slice(6);
+          if (data === '[DONE]') {
+            return;
+          }
+          try {
+            const parsed = JSON.parse(data);
+            if (parsed.type === 'content_block_delta') {
+              const content = parsed.delta?.text;
+              if (content) {
+                yield content;
+              }
+            }
+          } catch {
+            // Ignore invalid JSON
+          }
+        }
+      }
+    }
+  }
+
+  countTokens(text: string): number {
+    // Approximate token count (roughly 4 chars per token for English)
+    return Math.ceil(text.length / 4);
+  }
+
+  private buildMessages(request: GenerationRequest): Array<{ role: string; content: string }> {
+    return [{ role: 'user', content: request.prompt }];
+  }
+
+  private extractContent(content: unknown): string {
+    // Anthropic returns content as an array of blocks
+    if (Array.isArray(content)) {
+      const textBlocks = content
+        .filter((block): block is { type: string; text: string } => block?.type === 'text')
+        .map(block => block.text)
+        .join('');
+      return textBlocks;
+    }
+    return String(content ?? '');
+  }
+}
+
+/**
+ * Anthropic-specific error
+ */
+export class AnthropicError extends Error {
+  code = 'ANTHROPIC_ERROR';
+  details?: Record<string, unknown>;
+
+  constructor(message: string, details?: Record<string, unknown>) {
+    super(message);
+    this.name = 'AnthropicError';
+    this.details = details;
+  }
+}
+
+/**
+ * Create Anthropic provider with default settings
+ */
+export function createAnthropicProvider(config?: AnthropicConfig): AnthropicProvider {
+  return new AnthropicProvider(config);
+}
+
+/**
+ * Check if Anthropic is available and configured
+ */
+export async function checkAnthropicSetup(): Promise<boolean> {
+  const provider = new AnthropicProvider();
+  return provider.isAvailable();
+}

package/core/src/ai/deepseek-provider.ts

@@ -0,0 +1,315 @@
+/**
+ * DeepSeek LLM Provider
+ *
+ * Provider for DeepSeek API (https://api.deepseek.com).
+ * DeepSeek offers GPT-4 level performance at 95% lower cost.
+ *
+ * Environment variables:
+ * - DEEPSEEK_API_KEY: Required API key
+ * - DEEPSEEK_BASE_URL: Optional base URL (default: https://api.deepseek.com/v1)
+ * - DEEPSEEK_MODEL: Model to use (default: deepseek-chat)
+ *
+ * Pricing (as of 2025):
+ * - deepseek-chat: $0.14/M input tokens, $0.28/M output tokens
+ * - deepseek-coder: $0.14/M input tokens, $0.28/M output tokens
+ *
+ * Compared to GPT-4:
+ * - GPT-4: ~$30/M input, ~$60/M output
+ * - Savings: ~99.5% cheaper
+ */
+
+import type { LLMProvider, GenerationRequest, GenerationResponse } from './index.js';
+
+export interface DeepSeekConfig {
+  apiKey?: string;
+  baseURL?: string;
+  model?: string;
+  timeout?: number;
+}
+
+/**
+ * DeepSeek provider for cost-effective AI inference
+ *
+ * DeepSeek V3 matches GPT-4 performance at a fraction of the cost.
+ * Perfect for test generation, code analysis, and quality assurance tasks.
+ */
+export class DeepSeekProvider implements LLMProvider {
+  name = 'deepseek';
+
+  // Available DeepSeek models
+  models = [
+    'deepseek-chat',      // General purpose, GPT-4 level
+    'deepseek-coder',     // Specialized for code tasks
+    'deepseek-reasoner',  // Enhanced reasoning capabilities
+  ];
+
+  private readonly apiKey: string;
+  private readonly baseURL: string;
+  private readonly defaultModel: string;
+  private readonly timeout: number;
+
+  constructor(config: DeepSeekConfig = {}) {
+    this.apiKey = config.apiKey || process.env.DEEPSEEK_API_KEY || '';
+    this.baseURL = config.baseURL || process.env.DEEPSEEK_BASE_URL || 'https://api.deepseek.com';
+    this.defaultModel = config.model || process.env.DEEPSEEK_MODEL || 'deepseek-chat';
+    this.timeout = config.timeout || 60000; // 1 minute default
+  }
+
+  async isAvailable(): Promise<boolean> {
+    // Check if API key is present
+    if (!this.apiKey) {
+      return false;
+    }
+
+    // Try a minimal API call
+    try {
+      // Use timeout with AbortController for compatibility
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), 5000);
+
+      const response = await fetch(`${this.baseURL}/v1/models`, {
+        method: 'GET',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+        },
+        signal: controller.signal,
+      });
+
+      clearTimeout(timeoutId);
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  async generate(request: GenerationRequest): Promise<GenerationResponse> {
+    if (!this.apiKey) {
+      throw new DeepSeekError('DEEPSEEK_API_KEY is required', {
+        suggestion: 'Set DEEPSEEK_API_KEY environment variable or pass apiKey in config',
+        getApiKey: 'Get your API key at https://platform.deepseek.com/api_keys'
+      });
+    }
+
+    const messages = this.buildMessages(request);
+
+    // Use timeout with AbortController for compatibility
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+    try {
+      const response = await fetch(`${this.baseURL}/v1/chat/completions`, {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        signal: controller.signal,
+        body: JSON.stringify({
+          model: this.defaultModel,
+          messages,
+          temperature: request.temperature ?? 0.7,
+          max_tokens: request.maxTokens ?? 4096,
+        })
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text().catch(() => 'Unknown error');
+        throw new DeepSeekError(`DeepSeek request failed: ${response.status} ${errorText}`, {
+          status: response.status,
+          response: errorText
+        });
+      }
+
+      const data = await response.json();
+
+      const choice = data.choices?.[0];
+      if (!choice) {
+        throw new DeepSeekError('No choices returned from DeepSeek');
+      }
+
+      return {
+        content: choice.message.content,
+        usage: {
+          promptTokens: data.usage?.prompt_tokens || 0,
+          completionTokens: data.usage?.completion_tokens || 0,
+          totalTokens: data.usage?.total_tokens || 0,
+        },
+        model: data.model,
+        finishReason: choice.finish_reason === 'stop' ? 'stop' : 'length'
+      };
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  }
+
+  async *stream(request: GenerationRequest): AsyncIterable<string> {
+    if (!this.apiKey) {
+      throw new DeepSeekError('DEEPSEEK_API_KEY is required');
+    }
+
+    const messages = this.buildMessages(request);
+
+    const response = await fetch(`${this.baseURL}/v1/chat/completions`, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${this.apiKey}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        model: this.defaultModel,
+        messages,
+        temperature: request.temperature ?? 0.7,
+        max_tokens: request.maxTokens ?? 4096,
+        stream: true,
+      })
+    });
+
+    if (!response.ok) {
+      throw new DeepSeekError(`DeepSeek stream failed: ${response.status}`);
+    }
+
+    const reader = response.body?.getReader();
+    if (!reader) {
+      throw new DeepSeekError('No response body');
+    }
+
+    const decoder = new TextDecoder();
+    let buffer = '';
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      buffer += decoder.decode(value, { stream: true });
+
+      const lines = buffer.split('\n');
+      buffer = lines.pop() || '';
+
+      for (const line of lines) {
+        if (line.startsWith('data: ')) {
+          const data = line.slice(6);
+          if (data === '[DONE]') {
+            return;
+          }
+          try {
+            const parsed = JSON.parse(data);
+            const content = parsed.choices?.[0]?.delta?.content;
+            if (content) {
+              yield content;
+            }
+          } catch {
+            // Ignore invalid JSON
+          }
+        }
+      }
+    }
+  }
+
+  countTokens(text: string): number {
+    // DeepSeek uses the same tokenizer as GPT-4
+    // Approximate token count (roughly 4 chars per token)
+    return Math.ceil(text.length / 4);
+  }
+
+  private buildMessages(request: GenerationRequest): Array<{ role: string; content: string }> {
+    const messages: Array<{ role: string; content: string }> = [];
+
+    if (request.systemPrompt) {
+      messages.push({ role: 'system', content: request.systemPrompt });
+    }
+
+    messages.push({ role: 'user', content: request.prompt });
+
+    return messages;
+  }
+
+  /**
+   * Get cost estimate for a generation
+   * DeepSeek pricing: $0.14/M input, $0.28/M output
+   */
+  estimateCost(inputTokens: number, outputTokens: number): number {
+    const inputCost = (inputTokens / 1_000_000) * 0.14;
+    const outputCost = (outputTokens / 1_000_000) * 0.28;
+    return inputCost + outputCost;
+  }
+
+  /**
+   * Compare cost to GPT-4
+   * GPT-4: ~$30/M input, ~$60/M output
+   */
+  compareGPT4(inputTokens: number, outputTokens: number): {
+    deepseekCost: number;
+    gpt4Cost: number;
+    savings: number;
+    savingsPercent: number;
+  } {
+    const deepseekCost = this.estimateCost(inputTokens, outputTokens);
+    const gpt4Cost = (inputTokens / 1_000_000) * 30 + (outputTokens / 1_000_000) * 60;
+    const savings = gpt4Cost - deepseekCost;
+    const savingsPercent = (savings / gpt4Cost) * 100;
+
+    return {
+      deepseekCost,
+      gpt4Cost,
+      savings,
+      savingsPercent
+    };
+  }
+}
+
+/**
+ * DeepSeek-specific error
+ */
+export class DeepSeekError extends Error {
+  code = 'DEEPSEEK_ERROR';
+  details?: Record<string, unknown>;
+
+  constructor(message: string, details?: Record<string, unknown>) {
+    super(message);
+    this.name = 'DeepSeekError';
+    this.details = details;
+  }
+}
+
+/**
+ * Create DeepSeek provider with default settings
+ */
+export function createDeepSeekProvider(config?: DeepSeekConfig): DeepSeekProvider {
+  return new DeepSeekProvider(config);
+}
+
+/**
+ * Check if DeepSeek is available and configured
+ */
+export async function checkDeepSeekSetup(): Promise<boolean> {
+  const provider = new DeepSeekProvider();
+  return provider.isAvailable();
+}
+
+/**
+ * Get DeepSeek API key setup instructions
+ */
+export function getDeepSeekSetupInstructions(): string {
+  return `
+DeepSeek API Setup:
+
+1. Get your API key:
+   https://platform.deepseek.com/api_keys
+
+2. Set environment variable:
+   export DEEPSEEK_API_KEY=your_api_key_here
+
+3. Or configure in QA360:
+   qa360 secrets add DEEPSEEK_API_KEY
+
+Available models:
+- deepseek-chat    (General purpose, GPT-4 level)
+- deepseek-coder   (Code specialized)
+- deepseek-reasoner (Enhanced reasoning)
+
+Pricing (as of 2025):
+- Input:  $0.14 per million tokens
+- Output: $0.28 per million tokens
+- ~99.5% cheaper than GPT-4
+`;
+}

package/core/src/ai/index.ts

@@ -0,0 +1,87 @@
+/**
+ * QA360 AI Module - LLM Providers
+ *
+ * Multi-provider AI integration for test generation and analysis.
+ * Supports DeepSeek (best value), Ollama (local), OpenAI, Anthropic, and Mock providers.
+ *
+ * Default behavior: Automatically selects the first available provider
+ * using the factory pattern. DeepSeek is preferred for best value/cost ratio.
+ */
+
+// Re-export all providers
+export { OllamaProvider, createOllamaProvider, checkOllamaSetup, type OllamaConfig } from './ollama-provider.js';
+export { OpenAIProvider, createOpenAIProvider, checkOpenAISetup, type OpenAIConfig, OpenAIError } from './openai-provider.js';
+export { AnthropicProvider, createAnthropicProvider, checkAnthropicSetup, type AnthropicConfig, AnthropicError } from './anthropic-provider.js';
+export { DeepSeekProvider, createDeepSeekProvider, checkDeepSeekSetup, getDeepSeekSetupInstructions, type DeepSeekConfig, DeepSeekError } from './deepseek-provider.js';
+export { MockProvider, createMockProvider, createStandardMockProvider, MOCK_RESPONSES, type MockProviderConfig } from './mock-provider.js';
+
+// Factory exports
+export {
+  FallbackProvider,
+  createLLMProvider,
+  createBest,
+  createResilientLLMProvider,
+  getProviderStatus,
+  getProviderInfo,
+  detectBestProvider,
+  hasAnyProvider,
+  type ProviderFactoryConfig,
+  type ProviderType,
+  type ProviderSelection,
+  type ProviderInfo,
+  PROVIDER_CHAIN,
+} from './provider-factory.js';
+
+/**
+ * Base interface for LLM providers
+ */
+export interface LLMProvider {
+  name: string;
+  models: string[];
+
+  generate(request: GenerationRequest): Promise<GenerationResponse>;
+
+  /**
+   * Streamed generation (optional)
+   */
+  stream?(request: GenerationRequest): AsyncIterable<string>;
+
+  /**
+   * Count tokens in text (approximate)
+   */
+  countTokens(text: string): number;
+
+  /**
+   * Check if provider is available
+   */
+  isAvailable(): Promise<boolean>;
+}
+
+/**
+ * Request for LLM generation
+ */
+export interface GenerationRequest {
+  prompt: string;
+  systemPrompt?: string;
+  maxTokens?: number;
+  temperature?: number;
+  jsonMode?: boolean;  // Request JSON output
+}
+
+/**
+ * Response from LLM
+ */
+export interface GenerationResponse {
+  content: string;
+  usage: {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+  };
+  model: string;
+  finishReason: 'stop' | 'length' | 'content_filter';
+}
+
+// Re-export Ollama-specific types for backward compatibility
+export type { ModelInfo, PullProgress } from './ollama-provider.js';
+export { OllamaError } from './ollama-provider.js';

package/core/src/ai/llm-client.ts

@@ -0,0 +1,52 @@
+/**
+ * LLM Client Interface
+ *
+ * Abstract interface for LLM providers (Ollama, OpenAI, Anthropic, etc.).
+ * Allows switching between providers without changing application code.
+ */
+
+export interface LLMMessage {
+  role: 'system' | 'user' | 'assistant';
+  content: string;
+}
+
+export interface LLMResponse {
+  content: string;
+  model: string;
+  finish_reason?: 'stop' | 'length' | 'content_filter';
+  usage?: {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+  };
+}
+
+export interface LLMClientConfig {
+  apiKey?: string;
+  baseURL?: string;
+  model?: string;
+  timeout?: number;
+  maxRetries?: number;
+}
+
+export interface ILLMClient {
+  /**
+   * Provider name (ollama, openai, anthropic, mock)
+   */
+  readonly provider: string;
+
+  /**
+   * Check if the provider is available and configured
+   */
+  available(): Promise<boolean>;
+
+  /**
+   * Generate a completion from a prompt
+   */
+  complete(prompt: string, options?: LLMClientConfig): Promise<LLMResponse>;
+
+  /**
+   * Generate a completion from a chat (messages)
+   */
+  chat(messages: LLMMessage[], options?: LLMClientConfig): Promise<LLMResponse>;
+}