qa360 2.1.6 → 2.2.0

package/cli/src/commands/flakiness.ts

@@ -0,0 +1,560 @@
+/**
+ * Flakiness CLI Commands
+ *
+ * Provides commands for managing test flakiness detection and quarantine.
+ */
+
+import { Command } from 'commander';
+import chalk from 'chalk';
+import Table from 'cli-table3';
+import { join } from 'path';
+import { existsSync } from 'fs';
+import {
+  EvidenceVault,
+  FlakinessCategory,
+  FLAKINESS_CATEGORIES,
+  FlakinessRecord
+} from 'qa360-core';
+
+/**
+ * Get the vault directory path
+ */
+function getVaultPath(workingDir: string = process.cwd()): string {
+  return join(workingDir, '.qa360');
+}
+
+/**
+ * Open the vault or exit with error
+ */
+async function openVault(workingDir?: string): Promise<EvidenceVault> {
+  const vaultPath = getVaultPath(workingDir);
+  const vaultDbPath = join(vaultPath, 'vault.db');
+
+  if (!existsSync(vaultDbPath)) {
+    console.error(chalk.red(`❌ Vault not found at ${vaultDbPath}`));
+    console.error(chalk.yellow('💡 Run "qa360 run" first to create a vault'));
+    process.exit(1);
+  }
+
+  return await EvidenceVault.open(vaultPath);
+}
+
+/**
+ * Format flakiness score with emoji
+ */
+function formatScore(score: number, category: string): string {
+  const meta = FLAKINESS_CATEGORIES[category as FlakinessCategory];
+  const color = score >= 75 ? 'green' : score >= 50 ? 'yellow' : 'red';
+  return chalk[color](`${meta.emoji} ${score}%`);
+}
+
+/**
+ * Format category with color
+ */
+function formatCategory(category: string): string {
+  const meta = FLAKINESS_CATEGORIES[category as FlakinessCategory];
+  const color = category === 'legendary' || category === 'solid' ? 'green' :
+                category === 'good' ? 'yellow' :
+                category === 'shaky' ? 'yellow' : 'red';
+  return chalk[color](meta.label);
+}
+
+/**
+ * List flakiness history for a test
+ */
+export async function flakinessHistoryAction(testId: string, options: {
+  limit?: number;
+  json?: boolean;
+  workingDir?: string;
+}) {
+  const vault = await openVault(options.workingDir);
+  const history = await vault.getFlakinessHistory(testId, options.limit || 50);
+
+  if (options.json) {
+    console.log(JSON.stringify(history, null, 2));
+    await vault.close();
+    return;
+  }
+
+  if (history.length === 0) {
+    console.log(chalk.yellow(`\n⚠️  No flakiness history found for test: ${testId}`));
+    await vault.close();
+    return;
+  }
+
+  console.log(chalk.blue(`\n📊 Flakiness History: ${history[0].test_name}`));
+  console.log(chalk.gray(`Test ID: ${testId}`));
+  console.log(chalk.gray(`File: ${history[0].file_path}\n`));
+
+  const table = new Table({
+    head: [chalk.gray('Score'), chalk.gray('Category'), chalk.gray('Runs'), chalk.gray('Duration'), chalk.gray('Date')],
+    colWidths: [12, 15, 10, 12, 20],
+    style: { head: [], border: ['gray'] }
+  });
+
+  for (const record of history) {
+    const date = new Date(record.created_at).toLocaleDateString();
+    table.push([
+      formatScore(record.score, record.category),
+      formatCategory(record.category),
+      `${record.successful_runs}/${record.total_runs}`,
+      `${record.avg_duration_ms}ms`,
+      date
+    ]);
+  }
+
+  console.log(table.toString());
+  await vault.close();
+}
+
+/**
+ * Show flakiness trends for a test
+ */
+export async function flakinessTrendsAction(testId: string, options: {
+  days?: number;
+  json?: boolean;
+  workingDir?: string;
+}) {
+  const vault = await openVault(options.workingDir);
+  const trends = await vault.getFlakinessTrends(testId, options.days || 30);
+
+  if (options.json) {
+    console.log(JSON.stringify(trends, null, 2));
+    await vault.close();
+    return;
+  }
+
+  // Get test name from history
+  const history = await vault.getFlakinessHistory(testId, 1);
+  const testName = history[0]?.test_name || testId;
+
+  console.log(chalk.blue(`\n📈 Flakiness Trends: ${testName}`));
+  console.log(chalk.gray(`Test ID: ${testId}`));
+  console.log(chalk.gray(`Time Window: Last ${options.days || 30} days\n`));
+
+  const trendEmoji = trends.trend === 'improving' ? '📈' : trends.trend === 'degrading' ? '📉' : '➡️';
+  const trendColor = trends.trend === 'improving' ? 'green' : trends.trend === 'degrading' ? 'red' : 'gray';
+  const trendText = trends.trend === 'improving' ? 'Improving' : trends.trend === 'degrading' ? 'Degrading' : 'Stable';
+
+  const currentCategory = trends.currentScore >= 75 ? 'good' : trends.currentScore >= 50 ? 'shaky' : 'unstable';
+
+  console.log(`${chalk.gray('Current Score:')} ${formatScore(trends.currentScore, currentCategory)}`);
+  console.log(`${chalk.gray('Average Score:')} ${formatScore(Math.round(trends.averageScore), currentCategory)}`);
+  console.log(`${chalk.gray('Trend:')} ${chalk[trendColor](`${trendEmoji} ${trendText}`)}`);
+  console.log(`${chalk.gray('Data Points:')} ${trends.dataPoints.length}\n`);
+
+  await vault.close();
+}
+
+/**
+ * List all flaky tests from recent runs
+ */
+export async function flakinessListAction(options: {
+  runId?: string;
+  threshold?: number;
+  json?: boolean;
+  workingDir?: string;
+}) {
+  const vault = await openVault(options.workingDir);
+
+  let flakinessRecords: FlakinessRecord[] = [];
+
+  if (options.runId) {
+    flakinessRecords = await vault.getRunFlakiness(options.runId);
+  } else {
+    // Get recent runs and their flakiness
+    const runs = await vault.listRuns({ limit: 10 });
+    for (const run of runs) {
+      const runFlakiness = await vault.getRunFlakiness(run.id);
+      flakinessRecords.push(...runFlakiness);
+    }
+  }
+
+  // Filter by threshold if specified
+  if (options.threshold !== undefined) {
+    flakinessRecords = flakinessRecords.filter(r => r.score < options.threshold!);
+  }
+
+  // Sort by score (most flaky first)
+  flakinessRecords.sort((a, b) => a.score - b.score);
+
+  if (options.json) {
+    console.log(JSON.stringify(flakinessRecords, null, 2));
+    await vault.close();
+    return;
+  }
+
+  if (flakinessRecords.length === 0) {
+    console.log(chalk.yellow('\n✅ No flaky tests found!'));
+    await vault.close();
+    return;
+  }
+
+  console.log(chalk.blue(`\n🎲 Flakiness Report (${flakinessRecords.length} tests)\n`));
+
+  const table = new Table({
+    head: [chalk.gray('Test'), chalk.gray('Gate'), chalk.gray('Score'), chalk.gray('Category'), chalk.gray('Runs'), chalk.gray('Pattern')],
+    colWidths: [30, 12, 10, 15, 8, 20],
+    wordWrap: true,
+    style: { head: [], border: ['gray'] }
+  });
+
+  for (const record of flakinessRecords) {
+    table.push([
+      record.test_name,
+      record.gate,
+      formatScore(record.score, record.category),
+      formatCategory(record.category),
+      `${record.successful_runs}/${record.total_runs}`,
+      record.pattern_type || '-'
+    ]);
+  }
+
+  console.log(table.toString());
+  await vault.close();
+}
+
+/**
+ * List quarantined tests
+ */
+export async function flakinessQuarantineListAction(options: {
+  all?: boolean;
+  json?: boolean;
+  workingDir?: string;
+}) {
+  const vault = await openVault(options.workingDir);
+  const quarantined = await vault.getQuarantinedTests(options.all !== false);
+
+  if (options.json) {
+    console.log(JSON.stringify(quarantined, null, 2));
+    await vault.close();
+    return;
+  }
+
+  if (quarantined.length === 0) {
+    console.log(chalk.yellow('\n✅ No tests in quarantine!'));
+    await vault.close();
+    return;
+  }
+
+  console.log(chalk.blue(`\n🚫 Quarantined Tests (${quarantined.length} total)\n`));
+
+  const table = new Table({
+    head: [chalk.gray('Test'), chalk.gray('Gate'), chalk.gray('Category'), chalk.gray('Score'), chalk.gray('Reason'), chalk.gray('Status')],
+    colWidths: [25, 10, 12, 8, 30, 12],
+    wordWrap: true,
+    style: { head: [], border: ['gray'] }
+  });
+
+  for (const test of quarantined) {
+    const status = test.resolved_at ?
+      chalk.green('✅ Resolved') :
+      chalk.red('🚫 Active');
+
+    table.push([
+      test.test_name,
+      test.gate,
+      formatCategory(test.category),
+      `${test.score}%`,
+      test.reason,
+      status
+    ]);
+  }
+
+  console.log(table.toString());
+  await vault.close();
+}
+
+/**
+ * Add a test to quarantine
+ */
+export async function flakinessQuarantineAddAction(testId: string, options: {
+  reason?: string;
+  score?: number;
+  category?: string;
+  workingDir?: string;
+}) {
+  const vault = await openVault(options.workingDir);
+
+  // Check if already in quarantine
+  const existing = await vault.getQuarantine(testId);
+  const alreadyQuarantined = existing && !existing.resolved_at;
+
+  if (alreadyQuarantined) {
+    await vault.close();
+    console.log(chalk.yellow(`\n⚠️  Test "${existing.test_name}" is already in quarantine\n`));
+    return;
+  }
+
+  // Try to get test info from history
+  const history = await vault.getFlakinessHistory(testId, 1);
+
+  const testName = history[0]?.test_name || testId;
+  const gate = history[0]?.gate || 'unknown';
+  const filePath = history[0]?.file_path || 'unknown';
+  const score = options.score || history[0]?.score || 0;
+  const category = (options.category || history[0]?.category || 'unstable') as any;
+  const reason = options.reason || `Manually quarantined by user`;
+
+  const quarantineId = await vault.addToQuarantine({
+    test_id: testId,
+    test_name: testName,
+    gate,
+    file_path: filePath,
+    reason,
+    score,
+    category,
+    quarantined_at: Date.now(),
+    quarantined_by: 'user'
+  });
+
+  console.log(chalk.green(`\n✅ Test "${testName}" added to quarantine (ID: ${quarantineId})`));
+  console.log(chalk.gray(`   Reason: ${reason}`));
+  console.log(chalk.gray(`   Category: ${formatCategory(category)} (${score}%)\n`));
+
+  await vault.close();
+}
+
+/**
+ * Remove a test from quarantine
+ */
+export async function flakinessQuarantineRemoveAction(testId: string, options: {
+  notes?: string;
+  workingDir?: string;
+}) {
+  const vault = await openVault(options.workingDir);
+
+  const quarantine = await vault.getQuarantine(testId);
+
+  if (!quarantine) {
+    console.log(chalk.yellow(`\n⚠️  Test "${testId}" is not in quarantine\n`));
+    await vault.close();
+    return;
+  }
+
+  if (quarantine.resolved_at) {
+    console.log(chalk.yellow(`\n⚠️  Test "${testId}" was already resolved on ${new Date(quarantine.resolved_at).toLocaleString()}\n`));
+    await vault.close();
+    return;
+  }
+
+  const notes = options.notes || 'Resolved by user';
+  await vault.removeFromQuarantine(testId, 'user', notes);
+
+  console.log(chalk.green(`\n✅ Test "${quarantine.test_name}" removed from quarantine`));
+  console.log(chalk.gray(`   Notes: ${notes}\n`));
+
+  await vault.close();
+}
+
+/**
+ * Show patterns for a test
+ */
+export async function flakinessPatternsAction(testId: string, options: {
+  json?: boolean;
+  workingDir?: string;
+}) {
+  const vault = await openVault(options.workingDir);
+
+  // Get all pattern types and check each
+  const patternTypes = ['timing', 'race_condition', 'external_dependency', 'environment_specific', 'selector_issue', 'unknown'];
+  const patterns: any[] = [];
+
+  for (const type of patternTypes) {
+    const pattern = await vault.getFlakinessPattern(testId, type);
+    if (pattern) {
+      patterns.push(pattern);
+    }
+  }
+
+  if (options.json) {
+    console.log(JSON.stringify(patterns, null, 2));
+    await vault.close();
+    return;
+  }
+
+  if (patterns.length === 0) {
+    console.log(chalk.yellow(`\n⚠️  No patterns detected for test: ${testId}\n`));
+    await vault.close();
+    return;
+  }
+
+  console.log(chalk.blue(`\n🔍 Detected Patterns: ${patterns[0].test_name || testId}\n`));
+
+  const table = new Table({
+    head: [chalk.gray('Pattern'), chalk.gray('Description'), chalk.gray('Confidence'), chalk.gray('Fix'), chalk.gray('Detections')],
+    colWidths: [18, 30, 12, 30, 10],
+    wordWrap: true,
+    style: { head: [], border: ['gray'] }
+  });
+
+  for (const pattern of patterns) {
+    table.push([
+      chalk.cyan(pattern.pattern_type),
+      pattern.description,
+      `${Math.round(pattern.confidence * 100)}%`,
+      pattern.suggested_fix,
+      pattern.detection_count.toString()
+    ]);
+  }
+
+  console.log(table.toString());
+  await vault.close();
+}
+
+/**
+ * Analyze flakiness for a specific run
+ */
+export async function flakinessAnalyzeAction(runId: string, options: {
+  json?: boolean;
+  workingDir?: string;
+}) {
+  const vault = await openVault(options.workingDir);
+
+  const flakiness = await vault.getRunFlakiness(runId);
+
+  if (options.json) {
+    console.log(JSON.stringify(flakiness, null, 2));
+    await vault.close();
+    return;
+  }
+
+  if (flakiness.length === 0) {
+    console.log(chalk.yellow(`\n⚠️  No flakiness data found for run: ${runId}\n`));
+    await vault.close();
+    return;
+  }
+
+  // Calculate stats
+  const legendary = flakiness.filter((f: FlakinessRecord) => f.category === 'legendary').length;
+  const solid = flakiness.filter((f: FlakinessRecord) => f.category === 'solid').length;
+  const good = flakiness.filter((f: FlakinessRecord) => f.category === 'good').length;
+  const shaky = flakiness.filter((f: FlakinessRecord) => f.category === 'shaky').length;
+  const unstable = flakiness.filter((f: FlakinessRecord) => f.category === 'unstable').length;
+
+  const avgScore = Math.round(flakiness.reduce((sum: number, f: FlakinessRecord) => sum + f.score, 0) / flakiness.length);
+
+  console.log(chalk.blue(`\n📊 Flakiness Analysis for Run: ${runId}\n`));
+  console.log(chalk.gray(`Total Tests: ${flakiness.length}`));
+  console.log(chalk.gray(`Average Score: ${formatScore(avgScore, avgScore >= 75 ? 'good' : avgScore >= 50 ? 'shaky' : 'unstable')}\n`));
+
+  console.log(chalk.gray('Distribution:'));
+  console.log(`  ${chalk.green('🟢')} Legendary: ${legendary}`);
+  console.log(`  ${chalk.green('🟢')} Solid: ${solid}`);
+  console.log(`  ${chalk.yellow('🟡')} Good: ${good}`);
+  console.log(`  ${chalk.yellow('🟠')} Shaky: ${shaky}`);
+  console.log(`  ${chalk.red('🔴')} Unstable: ${unstable}\n`);
+
+  // Show tests needing attention
+  const needsAttention = flakiness.filter((f: FlakinessRecord) => f.score < 75);
+  if (needsAttention.length > 0) {
+    console.log(chalk.yellow(`⚠️  ${needsAttention.length} test(s) need attention:\n`));
+
+    const table = new Table({
+      head: [chalk.gray('Test'), chalk.gray('Score'), chalk.gray('Pattern'), chalk.gray('Suggested Fix')],
+      colWidths: [25, 10, 15, 40],
+      wordWrap: true,
+      style: { head: [], border: ['gray'] }
+    });
+
+    for (const test of needsAttention.slice(0, 10)) {
+      table.push([
+        test.test_name,
+        formatScore(test.score, test.category),
+        test.pattern_type || '-',
+        test.suggested_fix || '-'
+      ]);
+    }
+
+    console.log(table.toString());
+
+    if (needsAttention.length > 10) {
+      console.log(chalk.gray(`\n... and ${needsAttention.length - 10} more`));
+    }
+  }
+
+  await vault.close();
+}
+
+/**
+ * Create all flakiness commands
+ */
+export function createFlakinessCommands(): Command {
+  const flakinessCommand = new Command('flakiness')
+    .description('Test flakiness detection and quarantine management');
+
+  // History command
+  flakinessCommand
+    .command('history <testId>')
+    .description('Show flakiness history for a test')
+    .option('-l, --limit <number>', 'Number of records to show', '50')
+    .option('--json', 'Output as JSON')
+    .option('-w, --working-dir <path>', 'Working directory (default: current directory)')
+    .action(flakinessHistoryAction);
+
+  // Trends command
+  flakinessCommand
+    .command('trends <testId>')
+    .description('Show flakiness trends over time')
+    .option('-d, --days <number>', 'Time window in days', '30')
+    .option('--json', 'Output as JSON')
+    .option('-w, --working-dir <path>', 'Working directory')
+    .action(flakinessTrendsAction);
+
+  // List command
+  flakinessCommand
+    .command('list')
+    .description('List all flaky tests from recent runs')
+    .option('-r, --run-id <id>', 'Specific run ID')
+    .option('-t, --threshold <number>', 'Only show tests below this score')
+    .option('--json', 'Output as JSON')
+    .option('-w, --working-dir <path>', 'Working directory')
+    .action(flakinessListAction);
+
+  // Analyze command
+  flakinessCommand
+    .command('analyze <runId>')
+    .description('Analyze flakiness for a specific run')
+    .option('--json', 'Output as JSON')
+    .option('-w, --working-dir <path>', 'Working directory')
+    .action(flakinessAnalyzeAction);
+
+  // Patterns command
+  flakinessCommand
+    .command('patterns <testId>')
+    .description('Show detected patterns for a test')
+    .option('--json', 'Output as JSON')
+    .option('-w, --working-dir <path>', 'Working directory')
+    .action(flakinessPatternsAction);
+
+  // Quarantine subcommand
+  const quarantineCommand = flakinessCommand
+    .command('quarantine')
+    .description('Manage quarantined tests');
+
+  quarantineCommand
+    .command('list')
+    .description('List quarantined tests')
+    .option('-a, --all', 'Include resolved quarantines', false)
+    .option('--json', 'Output as JSON')
+    .option('-w, --working-dir <path>', 'Working directory')
+    .action(flakinessQuarantineListAction);
+
+  quarantineCommand
+    .command('add <testId>')
+    .description('Add a test to quarantine')
+    .option('-r, --reason <text>', 'Reason for quarantine')
+    .option('-s, --score <number>', 'Flakiness score (0-100)')
+    .option('-c, --category <category>', 'Category: shaky, unstable')
+    .option('-w, --working-dir <path>', 'Working directory')
+    .action(flakinessQuarantineAddAction);
+
+  quarantineCommand
+    .command('remove <testId>')
+    .description('Remove a test from quarantine')
+    .option('-n, --notes <text>', 'Resolution notes')
+    .option('-w, --working-dir <path>', 'Working directory')
+    .action(flakinessQuarantineRemoveAction);
+
+  return flakinessCommand;
+}