qa360 2.2.1 → 2.2.13

package/cli/src/commands/flakiness.ts

@@ -1,560 +0,0 @@
-/**
- * Flakiness CLI Commands
- *
- * Provides commands for managing test flakiness detection and quarantine.
- */
-
-import { Command } from 'commander';
-import chalk from 'chalk';
-import Table from 'cli-table3';
-import { join } from 'path';
-import { existsSync } from 'fs';
-import {
-  EvidenceVault,
-  FlakinessCategory,
-  FLAKINESS_CATEGORIES,
-  FlakinessRecord
-} from 'qa360-core';
-
-/**
- * Get the vault directory path
- */
-function getVaultPath(workingDir: string = process.cwd()): string {
-  return join(workingDir, '.qa360');
-}
-
-/**
- * Open the vault or exit with error
- */
-async function openVault(workingDir?: string): Promise<EvidenceVault> {
-  const vaultPath = getVaultPath(workingDir);
-  const vaultDbPath = join(vaultPath, 'vault.db');
-
-  if (!existsSync(vaultDbPath)) {
-    console.error(chalk.red(`❌ Vault not found at ${vaultDbPath}`));
-    console.error(chalk.yellow('💡 Run "qa360 run" first to create a vault'));
-    process.exit(1);
-  }
-
-  return await EvidenceVault.open(vaultPath);
-}
-
-/**
- * Format flakiness score with emoji
- */
-function formatScore(score: number, category: string): string {
-  const meta = FLAKINESS_CATEGORIES[category as FlakinessCategory];
-  const color = score >= 75 ? 'green' : score >= 50 ? 'yellow' : 'red';
-  return chalk[color](`${meta.emoji} ${score}%`);
-}
-
-/**
- * Format category with color
- */
-function formatCategory(category: string): string {
-  const meta = FLAKINESS_CATEGORIES[category as FlakinessCategory];
-  const color = category === 'legendary' || category === 'solid' ? 'green' :
-                category === 'good' ? 'yellow' :
-                category === 'shaky' ? 'yellow' : 'red';
-  return chalk[color](meta.label);
-}
-
-/**
- * List flakiness history for a test
- */
-export async function flakinessHistoryAction(testId: string, options: {
-  limit?: number;
-  json?: boolean;
-  workingDir?: string;
-}) {
-  const vault = await openVault(options.workingDir);
-  const history = await vault.getFlakinessHistory(testId, options.limit || 50);
-
-  if (options.json) {
-    console.log(JSON.stringify(history, null, 2));
-    await vault.close();
-    return;
-  }
-
-  if (history.length === 0) {
-    console.log(chalk.yellow(`\n⚠️  No flakiness history found for test: ${testId}`));
-    await vault.close();
-    return;
-  }
-
-  console.log(chalk.blue(`\n📊 Flakiness History: ${history[0].test_name}`));
-  console.log(chalk.gray(`Test ID: ${testId}`));
-  console.log(chalk.gray(`File: ${history[0].file_path}\n`));
-
-  const table = new Table({
-    head: [chalk.gray('Score'), chalk.gray('Category'), chalk.gray('Runs'), chalk.gray('Duration'), chalk.gray('Date')],
-    colWidths: [12, 15, 10, 12, 20],
-    style: { head: [], border: ['gray'] }
-  });
-
-  for (const record of history) {
-    const date = new Date(record.created_at).toLocaleDateString();
-    table.push([
-      formatScore(record.score, record.category),
-      formatCategory(record.category),
-      `${record.successful_runs}/${record.total_runs}`,
-      `${record.avg_duration_ms}ms`,
-      date
-    ]);
-  }
-
-  console.log(table.toString());
-  await vault.close();
-}
-
-/**
- * Show flakiness trends for a test
- */
-export async function flakinessTrendsAction(testId: string, options: {
-  days?: number;
-  json?: boolean;
-  workingDir?: string;
-}) {
-  const vault = await openVault(options.workingDir);
-  const trends = await vault.getFlakinessTrends(testId, options.days || 30);
-
-  if (options.json) {
-    console.log(JSON.stringify(trends, null, 2));
-    await vault.close();
-    return;
-  }
-
-  // Get test name from history
-  const history = await vault.getFlakinessHistory(testId, 1);
-  const testName = history[0]?.test_name || testId;
-
-  console.log(chalk.blue(`\n📈 Flakiness Trends: ${testName}`));
-  console.log(chalk.gray(`Test ID: ${testId}`));
-  console.log(chalk.gray(`Time Window: Last ${options.days || 30} days\n`));
-
-  const trendEmoji = trends.trend === 'improving' ? '📈' : trends.trend === 'degrading' ? '📉' : '➡️';
-  const trendColor = trends.trend === 'improving' ? 'green' : trends.trend === 'degrading' ? 'red' : 'gray';
-  const trendText = trends.trend === 'improving' ? 'Improving' : trends.trend === 'degrading' ? 'Degrading' : 'Stable';
-
-  const currentCategory = trends.currentScore >= 75 ? 'good' : trends.currentScore >= 50 ? 'shaky' : 'unstable';
-
-  console.log(`${chalk.gray('Current Score:')} ${formatScore(trends.currentScore, currentCategory)}`);
-  console.log(`${chalk.gray('Average Score:')} ${formatScore(Math.round(trends.averageScore), currentCategory)}`);
-  console.log(`${chalk.gray('Trend:')} ${chalk[trendColor](`${trendEmoji} ${trendText}`)}`);
-  console.log(`${chalk.gray('Data Points:')} ${trends.dataPoints.length}\n`);
-
-  await vault.close();
-}
-
-/**
- * List all flaky tests from recent runs
- */
-export async function flakinessListAction(options: {
-  runId?: string;
-  threshold?: number;
-  json?: boolean;
-  workingDir?: string;
-}) {
-  const vault = await openVault(options.workingDir);
-
-  let flakinessRecords: FlakinessRecord[] = [];
-
-  if (options.runId) {
-    flakinessRecords = await vault.getRunFlakiness(options.runId);
-  } else {
-    // Get recent runs and their flakiness
-    const runs = await vault.listRuns({ limit: 10 });
-    for (const run of runs) {
-      const runFlakiness = await vault.getRunFlakiness(run.id);
-      flakinessRecords.push(...runFlakiness);
-    }
-  }
-
-  // Filter by threshold if specified
-  if (options.threshold !== undefined) {
-    flakinessRecords = flakinessRecords.filter(r => r.score < options.threshold!);
-  }
-
-  // Sort by score (most flaky first)
-  flakinessRecords.sort((a, b) => a.score - b.score);
-
-  if (options.json) {
-    console.log(JSON.stringify(flakinessRecords, null, 2));
-    await vault.close();
-    return;
-  }
-
-  if (flakinessRecords.length === 0) {
-    console.log(chalk.yellow('\n✅ No flaky tests found!'));
-    await vault.close();
-    return;
-  }
-
-  console.log(chalk.blue(`\n🎲 Flakiness Report (${flakinessRecords.length} tests)\n`));
-
-  const table = new Table({
-    head: [chalk.gray('Test'), chalk.gray('Gate'), chalk.gray('Score'), chalk.gray('Category'), chalk.gray('Runs'), chalk.gray('Pattern')],
-    colWidths: [30, 12, 10, 15, 8, 20],
-    wordWrap: true,
-    style: { head: [], border: ['gray'] }
-  });
-
-  for (const record of flakinessRecords) {
-    table.push([
-      record.test_name,
-      record.gate,
-      formatScore(record.score, record.category),
-      formatCategory(record.category),
-      `${record.successful_runs}/${record.total_runs}`,
-      record.pattern_type || '-'
-    ]);
-  }
-
-  console.log(table.toString());
-  await vault.close();
-}
-
-/**
- * List quarantined tests
- */
-export async function flakinessQuarantineListAction(options: {
-  all?: boolean;
-  json?: boolean;
-  workingDir?: string;
-}) {
-  const vault = await openVault(options.workingDir);
-  const quarantined = await vault.getQuarantinedTests(options.all !== false);
-
-  if (options.json) {
-    console.log(JSON.stringify(quarantined, null, 2));
-    await vault.close();
-    return;
-  }
-
-  if (quarantined.length === 0) {
-    console.log(chalk.yellow('\n✅ No tests in quarantine!'));
-    await vault.close();
-    return;
-  }
-
-  console.log(chalk.blue(`\n🚫 Quarantined Tests (${quarantined.length} total)\n`));
-
-  const table = new Table({
-    head: [chalk.gray('Test'), chalk.gray('Gate'), chalk.gray('Category'), chalk.gray('Score'), chalk.gray('Reason'), chalk.gray('Status')],
-    colWidths: [25, 10, 12, 8, 30, 12],
-    wordWrap: true,
-    style: { head: [], border: ['gray'] }
-  });
-
-  for (const test of quarantined) {
-    const status = test.resolved_at ?
-      chalk.green('✅ Resolved') :
-      chalk.red('🚫 Active');
-
-    table.push([
-      test.test_name,
-      test.gate,
-      formatCategory(test.category),
-      `${test.score}%`,
-      test.reason,
-      status
-    ]);
-  }
-
-  console.log(table.toString());
-  await vault.close();
-}
-
-/**
- * Add a test to quarantine
- */
-export async function flakinessQuarantineAddAction(testId: string, options: {
-  reason?: string;
-  score?: number;
-  category?: string;
-  workingDir?: string;
-}) {
-  const vault = await openVault(options.workingDir);
-
-  // Check if already in quarantine
-  const existing = await vault.getQuarantine(testId);
-  const alreadyQuarantined = existing && !existing.resolved_at;
-
-  if (alreadyQuarantined) {
-    await vault.close();
-    console.log(chalk.yellow(`\n⚠️  Test "${existing.test_name}" is already in quarantine\n`));
-    return;
-  }
-
-  // Try to get test info from history
-  const history = await vault.getFlakinessHistory(testId, 1);
-
-  const testName = history[0]?.test_name || testId;
-  const gate = history[0]?.gate || 'unknown';
-  const filePath = history[0]?.file_path || 'unknown';
-  const score = options.score || history[0]?.score || 0;
-  const category = (options.category || history[0]?.category || 'unstable') as any;
-  const reason = options.reason || `Manually quarantined by user`;
-
-  const quarantineId = await vault.addToQuarantine({
-    test_id: testId,
-    test_name: testName,
-    gate,
-    file_path: filePath,
-    reason,
-    score,
-    category,
-    quarantined_at: Date.now(),
-    quarantined_by: 'user'
-  });
-
-  console.log(chalk.green(`\n✅ Test "${testName}" added to quarantine (ID: ${quarantineId})`));
-  console.log(chalk.gray(`   Reason: ${reason}`));
-  console.log(chalk.gray(`   Category: ${formatCategory(category)} (${score}%)\n`));
-
-  await vault.close();
-}
-
-/**
- * Remove a test from quarantine
- */
-export async function flakinessQuarantineRemoveAction(testId: string, options: {
-  notes?: string;
-  workingDir?: string;
-}) {
-  const vault = await openVault(options.workingDir);
-
-  const quarantine = await vault.getQuarantine(testId);
-
-  if (!quarantine) {
-    console.log(chalk.yellow(`\n⚠️  Test "${testId}" is not in quarantine\n`));
-    await vault.close();
-    return;
-  }
-
-  if (quarantine.resolved_at) {
-    console.log(chalk.yellow(`\n⚠️  Test "${testId}" was already resolved on ${new Date(quarantine.resolved_at).toLocaleString()}\n`));
-    await vault.close();
-    return;
-  }
-
-  const notes = options.notes || 'Resolved by user';
-  await vault.removeFromQuarantine(testId, 'user', notes);
-
-  console.log(chalk.green(`\n✅ Test "${quarantine.test_name}" removed from quarantine`));
-  console.log(chalk.gray(`   Notes: ${notes}\n`));
-
-  await vault.close();
-}
-
-/**
- * Show patterns for a test
- */
-export async function flakinessPatternsAction(testId: string, options: {
-  json?: boolean;
-  workingDir?: string;
-}) {
-  const vault = await openVault(options.workingDir);
-
-  // Get all pattern types and check each
-  const patternTypes = ['timing', 'race_condition', 'external_dependency', 'environment_specific', 'selector_issue', 'unknown'];
-  const patterns: any[] = [];
-
-  for (const type of patternTypes) {
-    const pattern = await vault.getFlakinessPattern(testId, type);
-    if (pattern) {
-      patterns.push(pattern);
-    }
-  }
-
-  if (options.json) {
-    console.log(JSON.stringify(patterns, null, 2));
-    await vault.close();
-    return;
-  }
-
-  if (patterns.length === 0) {
-    console.log(chalk.yellow(`\n⚠️  No patterns detected for test: ${testId}\n`));
-    await vault.close();
-    return;
-  }
-
-  console.log(chalk.blue(`\n🔍 Detected Patterns: ${patterns[0].test_name || testId}\n`));
-
-  const table = new Table({
-    head: [chalk.gray('Pattern'), chalk.gray('Description'), chalk.gray('Confidence'), chalk.gray('Fix'), chalk.gray('Detections')],
-    colWidths: [18, 30, 12, 30, 10],
-    wordWrap: true,
-    style: { head: [], border: ['gray'] }
-  });
-
-  for (const pattern of patterns) {
-    table.push([
-      chalk.cyan(pattern.pattern_type),
-      pattern.description,
-      `${Math.round(pattern.confidence * 100)}%`,
-      pattern.suggested_fix,
-      pattern.detection_count.toString()
-    ]);
-  }
-
-  console.log(table.toString());
-  await vault.close();
-}
-
-/**
- * Analyze flakiness for a specific run
- */
-export async function flakinessAnalyzeAction(runId: string, options: {
-  json?: boolean;
-  workingDir?: string;
-}) {
-  const vault = await openVault(options.workingDir);
-
-  const flakiness = await vault.getRunFlakiness(runId);
-
-  if (options.json) {
-    console.log(JSON.stringify(flakiness, null, 2));
-    await vault.close();
-    return;
-  }
-
-  if (flakiness.length === 0) {
-    console.log(chalk.yellow(`\n⚠️  No flakiness data found for run: ${runId}\n`));
-    await vault.close();
-    return;
-  }
-
-  // Calculate stats
-  const legendary = flakiness.filter((f: FlakinessRecord) => f.category === 'legendary').length;
-  const solid = flakiness.filter((f: FlakinessRecord) => f.category === 'solid').length;
-  const good = flakiness.filter((f: FlakinessRecord) => f.category === 'good').length;
-  const shaky = flakiness.filter((f: FlakinessRecord) => f.category === 'shaky').length;
-  const unstable = flakiness.filter((f: FlakinessRecord) => f.category === 'unstable').length;
-
-  const avgScore = Math.round(flakiness.reduce((sum: number, f: FlakinessRecord) => sum + f.score, 0) / flakiness.length);
-
-  console.log(chalk.blue(`\n📊 Flakiness Analysis for Run: ${runId}\n`));
-  console.log(chalk.gray(`Total Tests: ${flakiness.length}`));
-  console.log(chalk.gray(`Average Score: ${formatScore(avgScore, avgScore >= 75 ? 'good' : avgScore >= 50 ? 'shaky' : 'unstable')}\n`));
-
-  console.log(chalk.gray('Distribution:'));
-  console.log(`  ${chalk.green('🟢')} Legendary: ${legendary}`);
-  console.log(`  ${chalk.green('🟢')} Solid: ${solid}`);
-  console.log(`  ${chalk.yellow('🟡')} Good: ${good}`);
-  console.log(`  ${chalk.yellow('🟠')} Shaky: ${shaky}`);
-  console.log(`  ${chalk.red('🔴')} Unstable: ${unstable}\n`);
-
-  // Show tests needing attention
-  const needsAttention = flakiness.filter((f: FlakinessRecord) => f.score < 75);
-  if (needsAttention.length > 0) {
-    console.log(chalk.yellow(`⚠️  ${needsAttention.length} test(s) need attention:\n`));
-
-    const table = new Table({
-      head: [chalk.gray('Test'), chalk.gray('Score'), chalk.gray('Pattern'), chalk.gray('Suggested Fix')],
-      colWidths: [25, 10, 15, 40],
-      wordWrap: true,
-      style: { head: [], border: ['gray'] }
-    });
-
-    for (const test of needsAttention.slice(0, 10)) {
-      table.push([
-        test.test_name,
-        formatScore(test.score, test.category),
-        test.pattern_type || '-',
-        test.suggested_fix || '-'
-      ]);
-    }
-
-    console.log(table.toString());
-
-    if (needsAttention.length > 10) {
-      console.log(chalk.gray(`\n... and ${needsAttention.length - 10} more`));
-    }
-  }
-
-  await vault.close();
-}
-
-/**
- * Create all flakiness commands
- */
-export function createFlakinessCommands(): Command {
-  const flakinessCommand = new Command('flakiness')
-    .description('Test flakiness detection and quarantine management');
-
-  // History command
-  flakinessCommand
-    .command('history <testId>')
-    .description('Show flakiness history for a test')
-    .option('-l, --limit <number>', 'Number of records to show', '50')
-    .option('--json', 'Output as JSON')
-    .option('-w, --working-dir <path>', 'Working directory (default: current directory)')
-    .action(flakinessHistoryAction);
-
-  // Trends command
-  flakinessCommand
-    .command('trends <testId>')
-    .description('Show flakiness trends over time')
-    .option('-d, --days <number>', 'Time window in days', '30')
-    .option('--json', 'Output as JSON')
-    .option('-w, --working-dir <path>', 'Working directory')
-    .action(flakinessTrendsAction);
-
-  // List command
-  flakinessCommand
-    .command('list')
-    .description('List all flaky tests from recent runs')
-    .option('-r, --run-id <id>', 'Specific run ID')
-    .option('-t, --threshold <number>', 'Only show tests below this score')
-    .option('--json', 'Output as JSON')
-    .option('-w, --working-dir <path>', 'Working directory')
-    .action(flakinessListAction);
-
-  // Analyze command
-  flakinessCommand
-    .command('analyze <runId>')
-    .description('Analyze flakiness for a specific run')
-    .option('--json', 'Output as JSON')
-    .option('-w, --working-dir <path>', 'Working directory')
-    .action(flakinessAnalyzeAction);
-
-  // Patterns command
-  flakinessCommand
-    .command('patterns <testId>')
-    .description('Show detected patterns for a test')
-    .option('--json', 'Output as JSON')
-    .option('-w, --working-dir <path>', 'Working directory')
-    .action(flakinessPatternsAction);
-
-  // Quarantine subcommand
-  const quarantineCommand = flakinessCommand
-    .command('quarantine')
-    .description('Manage quarantined tests');
-
-  quarantineCommand
-    .command('list')
-    .description('List quarantined tests')
-    .option('-a, --all', 'Include resolved quarantines', false)
-    .option('--json', 'Output as JSON')
-    .option('-w, --working-dir <path>', 'Working directory')
-    .action(flakinessQuarantineListAction);
-
-  quarantineCommand
-    .command('add <testId>')
-    .description('Add a test to quarantine')
-    .option('-r, --reason <text>', 'Reason for quarantine')
-    .option('-s, --score <number>', 'Flakiness score (0-100)')
-    .option('-c, --category <category>', 'Category: shaky, unstable')
-    .option('-w, --working-dir <path>', 'Working directory')
-    .action(flakinessQuarantineAddAction);
-
-  quarantineCommand
-    .command('remove <testId>')
-    .description('Remove a test from quarantine')
-    .option('-n, --notes <text>', 'Resolution notes')
-    .option('-w, --working-dir <path>', 'Working directory')
-    .action(flakinessQuarantineRemoveAction);
-
-  return flakinessCommand;
-}