qa360 2.1.7 → 2.2.0

package/cli/src/commands/monitor.ts

@@ -0,0 +1,270 @@
+/**
+ * CLI Commands for TUI and Dashboard monitoring
+ * Phase 8: Real-time test execution visualization
+ */
+
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { TUIMonitor, monitorExecution, type TUIEvent } from 'qa360-core';
+import { DashboardServer, type DashboardEvent, EvidenceVault } from 'qa360-core';
+import { Phase3Runner } from 'qa360-core';
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { loadPackConfig } from '../utils/config.js';
+
+/**
+ * Monitor command - Run tests with TUI
+ */
+export const monitorCommand = async (pack: string | undefined, options: {
+  mode?: string;
+  retries?: string;
+  timeout?: string;
+  docker?: boolean;
+}): Promise<void> => {
+  const packPath = pack || findDefaultPack();
+  if (!packPath) {
+    console.error(chalk.red('Error: No pack file found. Specify a pack file or run qa360 init.'));
+    process.exit(1);
+  }
+
+  // Load pack config
+  const packConfig = await loadPackConfig(packPath);
+  const workingDir = process.cwd();
+
+  // Get gate names
+  const gateNames = getGateNames(packConfig);
+
+  // Create TUI monitor
+  const monitor = new TUIMonitor({
+    refreshRate: 500,
+    colors: true,
+    compact: false
+  });
+
+  monitor.start(gateNames.length);
+
+  try {
+    // Create runner with event emission
+    const runner = new Phase3Runner({
+      workingDir,
+      pack: packConfig
+    });
+
+    // Emit events to monitor
+    monitor.emit({
+      type: 'phase_start',
+      timestamp: Date.now(),
+      data: { phase: 'Initializing' }
+    });
+
+    // Run tests (simplified - in real implementation, Phase3Runner would emit events)
+    const result = await runner.run();
+
+    // Update monitor with results
+    for (const gate of result.gates) {
+      monitor.emit({
+        type: 'gate_start',
+        timestamp: Date.now(),
+        data: { gate: gate.gate }
+      });
+
+      monitor.emit({
+        type: 'gate_end',
+        timestamp: Date.now(),
+        data: {
+          gate: gate.gate,
+          success: gate.success,
+          results: {
+            total: gate.results?.summary?.total || 0,
+            passed: gate.results?.summary?.passed || 0,
+            failed: gate.results?.summary?.failed || 0
+          }
+        }
+      });
+    }
+
+    monitor.complete();
+  } catch (error) {
+    monitor.emit({
+      type: 'error',
+      timestamp: Date.now(),
+      data: { error: error instanceof Error ? error.message : String(error) }
+    });
+  }
+};
+
+/**
+ * Dashboard command - Run tests with web dashboard
+ */
+export const dashboardCommand = async (pack: string | undefined, options: {
+  port?: string;
+  host?: string;
+  autoOpen?: boolean;
+}): Promise<void> => {
+  const packPath = pack || findDefaultPack();
+  if (!packPath) {
+    console.error(chalk.red('Error: No pack file found. Specify a pack file or run qa360 init.'));
+    process.exit(1);
+  }
+
+  const port = options.port ? parseInt(options.port) : 8080;
+  const host = options.host || '127.0.0.1';
+  const workingDir = process.cwd();
+
+  // Load pack config
+  const packConfig = await loadPackConfig(packPath);
+  const gateNames = getGateNames(packConfig);
+
+  // Open vault for flakiness data
+  const vaultPath = join(workingDir, '.qa360');
+  const vault = await EvidenceVault.open(vaultPath);
+
+  // Create dashboard server
+  const dashboard = new DashboardServer({
+    port,
+    host,
+    autoOpen: options.autoOpen ?? true,
+    refreshRate: 1000
+  });
+
+  // Connect vault to dashboard for flakiness data
+  dashboard.setVault(vault);
+
+  await dashboard.start();
+  dashboard.setGates(gateNames);
+
+  console.log(chalk.green('✓ Dashboard started'));
+  console.log(chalk.gray(`   Open http://${host}:${port} in your browser`));
+
+  // Run tests with dashboard updates
+  try {
+    const runner = new Phase3Runner({
+      workingDir,
+      pack: packConfig
+    });
+
+    dashboard.emit({
+      type: 'start',
+      timestamp: Date.now(),
+      data: { phase: 'Running tests' }
+    });
+
+    const result = await runner.run();
+
+    // Set run ID for flakiness tracking
+    if (result.runId) {
+      dashboard.setRunId(result.runId);
+    }
+
+    // Update dashboard with gate results
+    for (const gate of result.gates) {
+      dashboard.emit({
+        type: 'gate_start',
+        timestamp: Date.now(),
+        data: { gate: gate.gate }
+      });
+
+      dashboard.emit({
+        type: 'gate_end',
+        timestamp: Date.now(),
+        data: {
+          gate: gate.gate,
+          success: gate.success,
+          duration: gate.duration,
+          results: {
+            total: gate.results?.summary?.total || 0,
+            passed: gate.results?.summary?.passed || 0,
+            failed: gate.results?.summary?.failed || 0
+          }
+        }
+      });
+    }
+
+    await dashboard.emit({
+      type: 'complete',
+      timestamp: Date.now(),
+      data: {}
+    });
+
+    console.log(chalk.green('✓ Tests completed. Dashboard will remain open.'));
+    console.log(chalk.gray('Press Ctrl+C to exit.'));
+
+  } catch (error) {
+    await dashboard.emit({
+      type: 'error',
+      timestamp: Date.now(),
+      data: { error: error instanceof Error ? error.message : String(error) }
+    });
+  }
+
+  // Keep dashboard open until user exits
+  // Vault stays open for flakiness data access
+  try {
+    await keepAlive();
+  } finally {
+    // Close vault when user exits
+    await vault.close();
+    await dashboard.stop();
+  }
+};
+
+/**
+ * Create monitor and dashboard commands
+ */
+export function createMonitorCommands(): Command {
+  const cmd = new Command('monitor');
+
+  cmd
+    .description('Run tests with terminal UI monitoring')
+    .argument('[pack]', 'Path to pack file')
+    .option('--mode <mode>', 'Execution mode: soft or strict', 'soft')
+    .option('--retries <number>', 'Number of retries for failed tests', '2')
+    .option('--timeout <ms>', 'Global timeout in milliseconds', '30000')
+    .option('--docker', 'Use Docker fallback for missing tools')
+    .action(monitorCommand);
+
+  return cmd;
+}
+
+export function createDashboardCommand(): Command {
+  const cmd = new Command('dashboard');
+
+  cmd
+    .description('Run tests with web dashboard')
+    .argument('[pack]', 'Path to pack file')
+    .option('--port <number>', 'Dashboard port', '8080')
+    .option('--host <address>', 'Dashboard host', '127.0.0.1')
+    .option('--no-auto-open', 'Do not auto-open browser')
+    .action(dashboardCommand);
+
+  return cmd;
+}
+
+// Helper functions
+
+function findDefaultPack(): string | undefined {
+  const candidates = ['qa360.yml', 'qa360.yaml', 'pack.yml', 'pack.yaml', '.qa360/pack.yml'];
+  for (const candidate of candidates) {
+    if (existsSync(join(process.cwd(), candidate))) {
+      return candidate;
+    }
+  }
+  return undefined;
+}
+
+function getGateNames(packConfig: any): string[] {
+  if (packConfig.version === 2 && packConfig.gates) {
+    // v2: gates is an object
+    return Object.keys(packConfig.gates);
+  } else if (packConfig.gates && Array.isArray(packConfig.gates)) {
+    // v1: gates is an array
+    return packConfig.gates;
+  }
+  return [];
+}
+
+async function keepAlive(): Promise<void> {
+  return new Promise(() => {
+    // Keep process alive
+  });
+}

package/cli/src/commands/ollama.ts

@@ -0,0 +1,337 @@
+/**
+ * QA360 Ollama Command
+ *
+ * Manage Ollama integration for AI-powered test generation.
+ * This command handles connection testing, model management, and configuration.
+ */
+
+import { Command } from 'commander';
+import chalk from 'chalk';
+import ora from 'ora';
+import { checkOllamaSetup, OllamaProvider, OllamaError } from '../core/index.js';
+
+/**
+ * Test Ollama connection
+ */
+export async function ollamaTestCommand(options: { model?: string } = {}): Promise<void> {
+  const spinner = ora('Testing Ollama connection...').start();
+  const model = options.model || 'deepseek-coder';
+
+  try {
+    const check = await checkOllamaSetup(model);
+
+    if (!check.available) {
+      spinner.fail('Ollama is not available');
+      console.log(chalk.red('\n❌ Ollama is not running or not installed'));
+      console.log(chalk.yellow('\n📦 To install Ollama:'));
+      console.log(chalk.gray('  brew install ollama     # macOS'));
+      console.log(chalk.gray('  # Or visit: https://ollama.com'));
+      console.log(chalk.yellow('\n🚀 To start Ollama:'));
+      console.log(chalk.gray('  ollama serve'));
+      return;
+    }
+
+    spinner.succeed('Ollama is running!');
+
+    console.log(chalk.green('\n✅ Ollama is available\n'));
+
+    if (check.models.length === 0) {
+      console.log(chalk.yellow('⚠️  No models found'));
+      console.log(chalk.yellow('\n📥 Pull DeepSeek Coder (recommended for code generation):'));
+      console.log(chalk.gray('  ollama pull deepseek-coder'));
+      return;
+    }
+
+    console.log(chalk.bold(`Found ${check.models.length} model(s):\n`));
+
+    for (const modelName of check.models) {
+      const isInstalled = modelName.includes(model);
+      const icon = isInstalled ? '✅' : '  ';
+      const color = isInstalled ? chalk.green : chalk.gray;
+      console.log(color(`  ${icon} ${modelName}`));
+    }
+
+    if (!check.modelInstalled) {
+      console.log(chalk.yellow(`\n⚠️  Model "${model}" is not installed`));
+      console.log(chalk.yellow(`\n📥 To install: ollama pull ${model}`));
+    } else {
+      console.log(chalk.green(`\n✅ Model "${model}" is ready!`));
+    }
+
+  } catch (error) {
+    spinner.fail('Connection test failed');
+    if (error instanceof OllamaError) {
+      console.error(chalk.red(error.message));
+    } else {
+      console.error(chalk.red('Unknown error:', error));
+    }
+  }
+}
+
+/**
+ * List available models
+ */
+export async function ollamaListCommand(): Promise<void> {
+  const spinner = ora('Connecting to Ollama...').start();
+
+  try {
+    const provider = new OllamaProvider();
+    const models = await provider.listModels();
+
+    spinner.succeed(`Found ${models.length} model(s)\n`);
+
+    console.log(chalk.bold('Available Models:\n'));
+
+    for (const model of models) {
+      const sizeGB = (model.size / 1024 / 1024 / 1024).toFixed(1);
+      console.log(chalk.cyan(`  ${model.name}`));
+      console.log(chalk.gray(`    Size: ${sizeGB} GB`));
+      console.log(chalk.gray(`    Modified: ${new Date(model.modifiedAt).toISOString()}\n`));
+    }
+
+    console.log(chalk.yellow('💡 Recommended for code generation:'));
+    console.log(chalk.gray('  • deepseek-coder (6.7GB) - Best for code'));
+    console.log(chalk.gray('  • codellama (34GB) - Also good'));
+    console.log(chalk.gray('  • llama3.2 (3-4GB) - General purpose\n'));
+
+    console.log(chalk.yellow('📥 To install a model:'));
+    console.log(chalk.gray('  ollama pull <model-name>\n'));
+
+  } catch (error) {
+    spinner.fail('Failed to list models');
+    if (error instanceof OllamaError) {
+      console.error(chalk.red(error.message));
+    } else {
+      console.error(chalk.red('Unknown error:', error));
+    }
+  }
+}
+
+/**
+ * Pull a model
+ */
+export async function ollamaPullCommand(model: string): Promise<void> {
+  console.log(chalk.cyan(`📥 Pulling model: ${model}\n`));
+  console.log(chalk.yellow('This may take a while depending on your connection...\n'));
+  console.log(chalk.gray('Alternatively, run this command in another terminal:\n'));
+  console.log(chalk.bold(`  ollama pull ${model}\n`));
+
+  try {
+    const provider = new OllamaProvider();
+    const models = await provider.listModels();
+
+    if (models.some(m => m.name === model)) {
+      console.log(chalk.green(`✅ Model "${model}" is already installed\n`));
+      return;
+    }
+
+    console.log(chalk.yellow('💡 Use the command above in a separate terminal to download the model.'));
+    console.log(chalk.gray('Large models (several GB) can take 10-30 minutes depending on your internet.\n'));
+
+  } catch (error) {
+    if (error instanceof OllamaError) {
+      console.error(chalk.red(error.message));
+    } else {
+      console.error(chalk.red('Unknown error:', error));
+    }
+  }
+}
+
+/**
+ * Generate tests using Ollama
+ */
+export async function ollamaGenerateCommand(
+  prompt: string,
+  options: { model?: string; type?: string; json?: boolean } = {}
+): Promise<void> {
+  const spinner = ora('Connecting to Ollama...').start();
+  const model = options.model || 'deepseek-coder';
+  const type = options.type || 'api';
+
+  try {
+    const provider = new OllamaProvider({ model });
+
+    // Check availability
+    const available = await provider.isAvailable();
+    if (!available) {
+      spinner.fail('Ollama not available');
+      console.log(chalk.red('\n❌ Ollama is not running'));
+      console.log(chalk.yellow('\nStart Ollama with: ollama serve'));
+      return;
+    }
+
+    spinner.succeed('Connected to Ollama');
+
+    // Build system prompt based on type
+    const systemPrompt = buildSystemPrompt(type);
+
+    const genSpinner = ora(`Generating ${type} tests...`).start();
+
+    const response = await provider.generate({
+      systemPrompt,
+      prompt,
+      maxTokens: 4096,
+      temperature: 0.7,
+    });
+
+    genSpinner.succeed('Tests generated!\n');
+
+    if (options.json) {
+      console.log(JSON.stringify({ content: response.content, usage: response.usage }, null, 2));
+    } else {
+      console.log(chalk.bold('Generated Tests:\n'));
+      console.log(chalk.gray('─'.repeat(60)));
+      console.log(response.content);
+      console.log(chalk.gray('─'.repeat(60)));
+      console.log(chalk.gray(`\n📊 Tokens: ${response.usage.totalTokens} (${response.usage.promptTokens} prompt + ${response.usage.completionTokens} completion)`));
+    }
+
+  } catch (error) {
+    spinner.fail('Generation failed');
+    if (error instanceof OllamaError) {
+      console.error(chalk.red(error.message));
+    } else {
+      console.error(chalk.red('Unknown error:', error));
+    }
+  }
+}
+
+/**
+ * Show Ollama configuration
+ */
+export async function ollamaConfigCommand(options: { setUrl?: string; setModel?: string } = {}): Promise<void> {
+  // TODO: Implement config file persistence
+  console.log(chalk.cyan('Ollama Configuration:\n'));
+  console.log(chalk.gray('  Base URL: http://localhost:11434 (default)'));
+  console.log(chalk.gray('  Model: deepseek-coder (default)\n'));
+  console.log(chalk.yellow('💡 Configuration will be persisted in .qa360/config.yml'));
+}
+
+/**
+ * Build system prompt for test generation
+ */
+function buildSystemPrompt(type: string): string {
+  const basePrompt = `You are QA360, an expert QA test generator.
+Generate production-ready, well-structured test code following best practices.
+
+Rules:
+- Generate COMPLETE, working code (no "...", no comments saying "to be implemented")
+- Include proper imports and setup
+- Handle errors gracefully
+- Use modern async/await patterns
+- Add helpful comments for complex logic
+- Return ONLY the code, no explanations before or after`;
+
+  const typePrompts: Record<string, string> = {
+    api: `${basePrompt}
+
+For API tests using Playwright:
+- Use @playwright/test
+- Use test() and expect() from Playwright
+- Test REST API endpoints (GET, POST, PUT, DELETE)
+- Include proper assertions for status codes, headers, and response body
+- Handle both success and error cases`,
+
+    ui: `${basePrompt}
+
+For UI/E2E tests using Playwright:
+- Use @playwright/test
+- Use page.goto(), page.click(), page.fill(), etc.
+- Include explicit waits where needed (waitForLoadState, waitForSelector)
+- Add assertions for page content
+- Use data-testid attributes when possible`,
+
+    perf: `${basePrompt}
+
+For performance tests using K6:
+- Use k6/http module
+- Define thresholds for response times and error rates
+- Include realistic user scenarios
+- Use options for VUs and duration`,
+
+    a11y: `${basePrompt}
+
+For accessibility tests:
+- Use @axe-core/playwright
+- Test WCAG compliance
+- Check for common a11y issues`,
+  };
+
+  return typePrompts[type] || typePrompts.api;
+}
+
+/**
+ * Create Ollama commands
+ */
+export function createOllamaCommands(): Command {
+  const ollamaCmd = new Command('ollama')
+    .description('Manage Ollama AI integration for test generation');
+
+  ollamaCmd
+    .command('test')
+    .description('Test Ollama connection and availability')
+    .option('-m, --model <model>', 'Model to test', 'deepseek-coder')
+    .action(async (options) => {
+      try {
+        await ollamaTestCommand(options);
+      } catch (error) {
+        console.error(chalk.red('Error testing Ollama:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  ollamaCmd
+    .command('list')
+    .description('List available Ollama models')
+    .action(async () => {
+      try {
+        await ollamaListCommand();
+      } catch (error) {
+        console.error(chalk.red('Error listing models:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  ollamaCmd
+    .command('pull <model>')
+    .description('Pull a model from Ollama registry')
+    .action(async (model) => {
+      try {
+        await ollamaPullCommand(model);
+      } catch (error) {
+        console.error(chalk.red('Error pulling model:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  ollamaCmd
+    .command('generate <prompt>')
+    .description('Generate tests using Ollama (preview)')
+    .option('-m, --model <model>', 'Model to use', 'deepseek-coder')
+    .option('-t, --type <type>', 'Test type (api, ui, perf)', 'api')
+    .option('--json', 'Output as JSON')
+    .action(async (prompt, options) => {
+      try {
+        await ollamaGenerateCommand(prompt, options);
+      } catch (error) {
+        console.error(chalk.red('Error generating tests:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  ollamaCmd
+    .command('config')
+    .description('Show or set Ollama configuration')
+    .option('--set-url <url>', 'Set Ollama base URL')
+    .option('--set-model <model>', 'Set default model')
+    .action(async (options) => {
+      try {
+        await ollamaConfigCommand(options);
+      } catch (error) {
+        console.error(chalk.red('Error showing config:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  return ollamaCmd;
+}