qa360 2.1.7 → 2.2.0

package/core/src/ai/provider-factory.ts

@@ -0,0 +1,408 @@
+/**
+ * LLM Provider Factory
+ *
+ * Factory pattern for creating LLM providers with automatic fallback.
+ * Tries providers in order: Ollama → OpenAI → Anthropic → Mock.
+ *
+ * Usage:
+ *   const provider = await createLLMProvider();
+ *   // Returns first available provider
+ */
+
+import type { LLMProvider, GenerationRequest, GenerationResponse } from './index.js';
+import { OllamaProvider, createOllamaProvider } from './index.js';
+import { OpenAIProvider, createOpenAIProvider } from './openai-provider.js';
+import { AnthropicProvider, createAnthropicProvider } from './anthropic-provider.js';
+import { DeepSeekProvider, createDeepSeekProvider } from './deepseek-provider.js';
+import { MockProvider, createStandardMockProvider } from './mock-provider.js';
+
+/**
+ * Provider priority order for fallback
+ * DeepSeek (best value) → Ollama (local, free) → OpenAI → Anthropic → Mock (always available)
+ */
+export const PROVIDER_CHAIN = [
+  'deepseek',
+  'ollama',
+  'openai',
+  'anthropic',
+  'mock'
+] as const;
+
+export type ProviderType = typeof PROVIDER_CHAIN[number];
+
+/**
+ * Provider factory configuration
+ */
+export interface ProviderFactoryConfig {
+  /**
+   * Preferred provider (will be tried first)
+   */
+  preferred?: ProviderType;
+
+  /**
+   * Ordered list of providers to try (overrides default chain)
+   */
+  chain?: ProviderType[];
+
+  /**
+   * Skip specific providers
+   */
+  skip?: ProviderType[];
+
+  /**
+   * Ollama configuration
+   */
+  ollama?: {
+    baseUrl?: string;
+    model?: string;
+    timeout?: number;
+  };
+
+  /**
+   * OpenAI configuration
+   */
+  openai?: {
+    apiKey?: string;
+    baseURL?: string;
+    model?: string;
+    timeout?: number;
+  };
+
+  /**
+   * Anthropic configuration
+   */
+  anthropic?: {
+    apiKey?: string;
+    baseURL?: string;
+    model?: string;
+    timeout?: number;
+  };
+
+  /**
+   * DeepSeek configuration
+   */
+  deepseek?: {
+    apiKey?: string;
+    baseURL?: string;
+    model?: string;
+    timeout?: number;
+  };
+
+  /**
+   * Mock configuration
+   */
+  mock?: {
+    delay?: number;
+    responses?: Record<string, string>;
+  };
+}
+
+/**
+ * Provider selection result
+ */
+export interface ProviderSelection {
+  provider: LLMProvider;
+  type: ProviderType;
+  reason: string;
+  skipped: ProviderType[];
+}
+
+/**
+ * Fallback provider that wraps the selected provider
+ */
+export class FallbackProvider implements LLMProvider {
+  name = 'fallback';
+  models: string[] = [];
+
+  private readonly provider: LLMProvider;
+  private readonly providerType: ProviderType;
+
+  constructor(provider: LLMProvider, providerType: ProviderType) {
+    this.provider = provider;
+    this.providerType = providerType;
+    this.name = `fallback-${providerType}`;
+    this.models = provider.models;
+  }
+
+  async isAvailable(): Promise<boolean> {
+    return this.provider.isAvailable();
+  }
+
+  async generate(request: GenerationRequest): Promise<GenerationResponse> {
+    return this.provider.generate(request);
+  }
+
+  async *stream(request: GenerationRequest): AsyncIterable<string> {
+    if (this.provider.stream) {
+      yield* this.provider.stream(request);
+    } else {
+      // Fallback to generate if stream not supported
+      const response = await this.provider.generate(request);
+      yield response.content;
+    }
+  }
+
+  countTokens(text: string): number {
+    return this.provider.countTokens(text);
+  }
+
+  /**
+   * Get the underlying provider type
+   */
+  getProviderType(): ProviderType {
+    return this.providerType;
+  }
+}
+
+/**
+ * Check availability of a provider without creating it
+ */
+async function checkProviderAvailability(
+  type: ProviderType,
+  config: ProviderFactoryConfig
+): Promise<boolean> {
+  try {
+    switch (type) {
+      case 'deepseek':
+        return await createDeepSeekProvider(config.deepseek).isAvailable();
+      case 'ollama':
+        return await createOllamaProvider(config.ollama).isAvailable();
+      case 'openai':
+        return await createOpenAIProvider(config.openai).isAvailable();
+      case 'anthropic':
+        return await createAnthropicProvider(config.anthropic).isAvailable();
+      case 'mock':
+        return true; // Mock is always available
+      default:
+        return false;
+    }
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Create provider instance by type
+ */
+function createProviderByType(
+  type: ProviderType,
+  config: ProviderFactoryConfig
+): LLMProvider {
+  switch (type) {
+    case 'deepseek':
+      return createDeepSeekProvider(config.deepseek);
+    case 'ollama':
+      return createOllamaProvider(config.ollama);
+    case 'openai':
+      return createOpenAIProvider(config.openai);
+    case 'anthropic':
+      return createAnthropicProvider(config.anthropic);
+    case 'mock':
+      return createStandardMockProvider();
+    default:
+      throw new Error(`Unknown provider type: ${type}`);
+  }
+}
+
+/**
+ * Create LLM provider with automatic fallback
+ *
+ * Tries providers in order until one is available.
+ * Logs which provider was selected and why.
+ */
+export async function createLLMProvider(
+  config: ProviderFactoryConfig = {}
+): Promise<FallbackProvider> {
+  const chain = config.chain || PROVIDER_CHAIN;
+  const skip = config.skip || [];
+
+  // Build the actual chain to try (respect skip list and preferred order)
+  let tryChain: ProviderType[] = [];
+
+  if (config.preferred && !skip.includes(config.preferred)) {
+    // Try preferred first, then the rest
+    tryChain = [config.preferred];
+    tryChain.push(...chain.filter(p => p !== config.preferred && !skip.includes(p)));
+  } else {
+    tryChain = chain.filter(p => !skip.includes(p));
+  }
+
+  // Ensure mock is always available as last resort
+  if (!tryChain.includes('mock')) {
+    tryChain.push('mock');
+  }
+
+  const skipped: ProviderType[] = [];
+
+  // Try each provider in order
+  for (const type of tryChain) {
+    if (skip.includes(type)) {
+      skipped.push(type);
+      continue;
+    }
+
+    const isAvailable = await checkProviderAvailability(type, config);
+
+    if (isAvailable) {
+      const provider = createProviderByType(type, config);
+
+      let reason = `Selected ${type}`;
+      if (type === 'mock' && tryChain.length > 1) {
+        reason = 'No other providers available, using mock';
+      } else if (skipped.length > 0) {
+        reason = `${type} (skipped: ${skipped.join(', ')})`;
+      }
+
+      return new FallbackProvider(provider, type);
+    }
+
+    skipped.push(type);
+  }
+
+  // Should never reach here since mock is always available
+  throw new Error('No LLM provider available (even mock failed)');
+}
+
+/**
+ * Get provider availability status for all providers
+ */
+export async function getProviderStatus(
+  config: ProviderFactoryConfig = {}
+): Promise<Record<ProviderType, boolean>> {
+  const status: Record<string, boolean> = {};
+
+  for (const type of PROVIDER_CHAIN) {
+    status[type] = await checkProviderAvailability(type, config);
+  }
+
+  return status as Record<ProviderType, boolean>;
+}
+
+/**
+ * Auto-detect best available provider
+ * Priority: DeepSeek (best value) → Ollama (free local) → OpenAI → Anthropic → Mock
+ */
+export async function detectBestProvider(
+  config: ProviderFactoryConfig = {}
+): Promise<ProviderType> {
+  const status = await getProviderStatus(config);
+
+  // Priority: DeepSeek > Ollama > OpenAI > Anthropic > Mock
+  if (status.deepseek) return 'deepseek';
+  if (status.ollama) return 'ollama';
+  if (status.openai) return 'openai';
+  if (status.anthropic) return 'anthropic';
+  return 'mock';
+}
+
+/**
+ * Create the best available provider automatically
+ * This is the recommended way to get a provider for most use cases.
+ *
+ * Selection criteria:
+ * 1. DeepSeek - Best value, GPT-4 level at 95% lower cost
+ * 2. Ollama - Free, local, privacy-preserving
+ * 3. OpenAI - Reliable, widely-used
+ * 4. Anthropic - High quality, Claude models
+ * 5. Mock - Fallback for testing
+ *
+ * @example
+ * const provider = await createBest();
+ * const response = await provider.generate({
+ *   prompt: 'Generate tests for login form'
+ * });
+ */
+export async function createBest(
+  config: ProviderFactoryConfig = {}
+): Promise<FallbackProvider> {
+  const providerType = await detectBestProvider(config);
+  const provider = createProviderByType(providerType, config);
+  return new FallbackProvider(provider, providerType);
+}
+
+/**
+ * Get provider information for display
+ */
+export interface ProviderInfo {
+  type: ProviderType;
+  name: string;
+  available: boolean;
+  description: string;
+  costLevel: 'free' | 'low' | 'medium' | 'high';
+  capabilities: string[];
+}
+
+export async function getProviderInfo(
+  config: ProviderFactoryConfig = {}
+): Promise<ProviderInfo[]> {
+  const status = await getProviderStatus(config);
+
+  return [
+    {
+      type: 'deepseek',
+      name: 'DeepSeek',
+      available: status.deepseek,
+      description: 'GPT-4 level performance at 95% lower cost',
+      costLevel: 'low',
+      capabilities: ['code-generation', 'test-generation', 'analysis', 'chat']
+    },
+    {
+      type: 'ollama',
+      name: 'Ollama',
+      available: status.ollama,
+      description: 'Local LLM inference, 100% free',
+      costLevel: 'free',
+      capabilities: ['code-generation', 'test-generation', 'offline', 'privacy']
+    },
+    {
+      type: 'openai',
+      name: 'OpenAI',
+      available: status.openai,
+      description: 'GPT-4, GPT-4 Turbo, reliable API',
+      costLevel: 'high',
+      capabilities: ['code-generation', 'test-generation', 'analysis', 'chat']
+    },
+    {
+      type: 'anthropic',
+      name: 'Anthropic',
+      available: status.anthropic,
+      description: 'Claude models, high quality outputs',
+      costLevel: 'high',
+      capabilities: ['code-generation', 'test-generation', 'analysis', 'chat']
+    },
+    {
+      type: 'mock',
+      name: 'Mock',
+      available: true,
+      description: 'Mock provider for testing',
+      costLevel: 'free',
+      capabilities: ['testing', 'mocking']
+    }
+  ];
+}
+
+/**
+ * Create provider with automatic retry on failure
+ *
+ * If the selected provider fails, automatically falls back
+ * to the next available provider in the chain.
+ */
+export async function createResilientLLMProvider(
+  config: ProviderFactoryConfig = {}
+): Promise<FallbackProvider> {
+  const provider = await createLLMProvider(config);
+
+  // Wrap with retry logic could be added here
+  // For now, return the standard fallback provider
+  return provider;
+}
+
+/**
+ * Quick check if any LLM provider is available
+ */
+export async function hasAnyProvider(
+  config: ProviderFactoryConfig = {}
+): Promise<boolean> {
+  const status = await getProviderStatus(config);
+  return Object.values(status).some(v => v);
+}

package/core/src/artifacts/README.md

@@ -0,0 +1,78 @@
+# QA360 Artifacts Module
+
+Manages screenshots, videos, traces, and other test artifacts with content-addressable storage (CAS) integration for deduplication.
+
+## Features
+
+- **Automatic Screenshots**: Before/after step screenshots, error screenshots
+- **Video Recording**: Optional video recording for test sessions
+- **Trace Capture**: Playwright trace files for debugging
+- **CAS Integration**: Content-addressable storage with SHA-256 hashing
+- **Metadata Tracking**: Full metadata for all artifacts
+
+## Usage
+
+```typescript
+import { UIArtifactsManager } from 'qa360-core';
+
+// Create artifacts manager
+const artifacts = new UIArtifactsManager(
+  '.qa360/artifacts/ui',  // Output directory
+  '.qa360/runs/cas'       // CAS directory
+);
+
+// Start test session
+artifacts.startTest('my-test');
+
+// Take screenshot
+const screenshot = await artifacts.takeScreenshot(page, {}, {
+  testId: 'my-test',
+  type: 'screenshot',
+  tags: ['initial'],
+});
+
+// Take before/after screenshots
+await artifacts.takeBeforeScreenshot(page, 'click-button', 0);
+// ... perform action ...
+await artifacts.takeAfterScreenshot(page, 'click-button', 0, true);
+
+// Take error screenshot
+await artifacts.takeErrorScreenshot(page, error, 'step-name');
+
+// End test session
+artifacts.endTest();
+
+// Get artifacts summary
+const summary = artifacts.generateSummary();
+console.log(`Screenshots: ${summary.screenshots}`);
+console.log(`CAS savings: ${summary.casDedupSavings} bytes`);
+```
+
+## Configuration
+
+```typescript
+interface ScreenshotOptions {
+  fullPage?: boolean;        // Full page screenshot (default: false)
+  quality?: number;          // JPEG/WebP quality (default: 90)
+  type?: 'png' | 'jpeg' | 'webp';  // Format (default: 'png')
+  animations?: 'disabled' | 'allow';  // CSS animations (default: 'disabled')
+}
+```
+
+## Artifacts
+
+Each artifact includes:
+- `casPath`: Path in content-addressable storage
+- `localPath`: Direct path for immediate access
+- `hash`: SHA-256 hash for deduplication
+- `size`: File size in bytes
+- `metadata`: Test ID, timestamp, status, tags
+
+## CAS Deduplication
+
+Identical artifacts are stored only once. The hash determines the storage path:
+```
+.cas/art/{hash[0:2]}/{hash[2:4]}/{hash[4:]}
+```
+
+This provides significant space savings for repeated screenshots.

package/core/src/artifacts/index.ts

@@ -0,0 +1,16 @@
+/**
+ * QA360 Artifacts Module
+ *
+ * Manages screenshots, videos, traces, and other test artifacts
+ */
+
+export {
+  UIArtifactsManager,
+  createUIArtifactsManager,
+  type ScreenshotOptions,
+  type ArtifactMetadata,
+  type StoredArtifact,
+  type VideoArtifact,
+  type ScreenshotArtifact,
+  type TraceArtifact,
+} from './ui-artifacts.js';