qa360 2.1.7 → 2.2.0

package/cli/src/commands/retry.ts

@@ -0,0 +1,380 @@
+/**
+ * QA360 Retry Commands (F8 Smart Retry Module)
+ *
+ * CLI commands for managing and viewing retry statistics and configuration.
+ */
+
+import { Command } from 'commander';
+import chalk from 'chalk';
+import Table from 'cli-table3';
+import {
+  createSmartRetryEngine,
+  RetryStrategy,
+  type RetryConfig,
+  type RetryStatistics
+} from 'qa360-core';
+
+/**
+ * Display retry statistics in a formatted table
+ */
+function displayStats(stats: RetryStatistics): void {
+  const table = new Table({
+    head: [chalk.bold('Metric'), chalk.bold('Value')],
+    style: { head: ['cyan', 'bold'] }
+  });
+
+  table.push(
+    ['Total Retries', stats.totalRetries.toString()],
+    ['Tests Retried', stats.testsRetried.toString()],
+    ['Recovered Tests', stats.recoveredTests.toString()],
+    ['Failed Tests', stats.failedTests.toString()],
+    ['Recovery Rate', `${stats.recoveryRate.toFixed(1)}%`],
+    ['Avg Attempts/Test', stats.avgAttemptsPerTest.toFixed(1)],
+    ['Total Retry Time', `${stats.totalRetryTimeMs}ms`]
+  );
+
+  console.log('\n' + chalk.bold.blue('📊 Retry Statistics'));
+  console.log(table.toString() + '\n');
+}
+
+/**
+ * Display retry strategy comparison
+ */
+function displayStrategyComparison(byStrategy: RetryStatistics['byStrategy']): void {
+  const table = new Table({
+    head: [chalk.bold('Strategy'), chalk.bold('Attempts'), chalk.bold('Successes'), chalk.bold('Avg Duration')],
+    style: { head: ['cyan', 'bold'] }
+  });
+
+  for (const [strategy, stats] of Object.entries(byStrategy)) {
+    table.push([
+      strategy,
+      stats.attempts.toString(),
+      stats.successes.toString(),
+      `${stats.avgDurationMs}ms`,
+    ]);
+  }
+
+  console.log(chalk.bold.blue('\n🎯 Performance by Strategy'));
+  console.log(table.toString() + '\n');
+}
+
+/**
+ * Display retry configuration
+ */
+function displayConfig(config: RetryConfig): void {
+  const table = new Table({
+    head: [chalk.bold('Setting'), chalk.bold('Value')],
+    style: { head: ['cyan', 'bold'] }
+  });
+
+  table.push(
+    ['Max Retries', config.maxRetries.toString()],
+    ['Initial Delay', `${config.initialDelayMs}ms`],
+    ['Max Delay', `${config.maxDelayMs}ms`],
+    ['Attempt Timeout', `${config.attemptTimeoutMs}ms`],
+    ['Overall Timeout', `${config.overallTimeoutMs}ms`],
+    ['Strategy', config.strategy],
+    ['Backoff Multiplier', config.backoffMultiplier.toString()],
+    ['Jitter Factor', config.jitterFactor.toString()],
+    ['Retry on Timeout', config.retryOnTimeout ? 'Yes' : 'No'],
+    ['Retry on Assertion', config.retryOnAssertionFailure ? 'Yes' : 'No']
+  );
+
+  console.log('\n' + chalk.bold.blue('⚙️  Retry Configuration'));
+  console.log(table.toString() + '\n');
+}
+
+/**
+ * Display flakiness-based recommendations
+ */
+function displayRecommendations(): void {
+  const engine = createSmartRetryEngine();
+
+  const recommendations = [
+    { score: 95, category: 'Legendary/Solid' },
+    { score: 80, category: 'Good' },
+    { score: 60, category: 'Shaky' },
+    { score: 30, category: 'Unstable' },
+  ];
+
+  const table = new Table({
+    head: [chalk.bold('Flakiness Score'), chalk.bold('Category'), chalk.bold('Strategy'), chalk.bold('Max Retries'), chalk.bold('Initial Delay')],
+    style: { head: ['cyan', 'bold'] }
+  });
+
+  for (const { score, category } of recommendations) {
+    const rec = engine.getRetryRecommendation({ flakinessScore: score });
+    table.push([
+      `${score}%`,
+      category,
+      rec.strategy,
+      rec.maxRetries.toString(),
+      `${rec.initialDelayMs}ms`,
+    ]);
+  }
+
+  console.log(chalk.bold.blue('\n🎯 Flakiness-Based Recommendations'));
+  console.log(table.toString() + '\n');
+}
+
+/**
+ * Retry history command
+ */
+export async function retryHistoryCommand(options: {
+  testId?: string;
+  limit?: number;
+  json?: boolean;
+}): Promise<void> {
+  if (options.json) {
+    console.log(JSON.stringify({
+      message: 'Retry history feature - querying vault for retry records',
+      testId: options.testId,
+      limit: options.limit,
+    }, null, 2));
+    return;
+  }
+
+  console.log(chalk.blue('\n📜 Retry History'));
+  console.log(chalk.gray('─'.repeat(60)));
+
+  if (options.testId) {
+    console.log(chalk.yellow(`Test ID: ${options.testId}`));
+  }
+
+  console.log(chalk.yellow('\nNote: Full history querying will be implemented with vault integration.'));
+  console.log(chalk.gray('Use: qa360 history list --type retry to see retry-related history.\n'));
+}
+
+/**
+ * Retry stats command
+ */
+export async function retryStatsCommand(options: {
+  strategy?: string;
+  json?: boolean;
+}): Promise<void> {
+  const mockStats: RetryStatistics = {
+    totalRetries: 1234,
+    testsRetried: 156,
+    recoveredTests: 142,
+    failedTests: 14,
+    recoveryRate: 91.0,
+    avgAttemptsPerTest: 2.3,
+    totalRetryTimeMs: 45678,
+    byStrategy: {
+      [RetryStrategy.NONE]: { attempts: 0, successes: 0, avgDurationMs: 0 },
+      [RetryStrategy.FIXED]: { attempts: 234, successes: 210, avgDurationMs: 1200 },
+      [RetryStrategy.LINEAR]: { attempts: 123, successes: 115, avgDurationMs: 1800 },
+      [RetryStrategy.EXPONENTIAL]: { attempts: 456, successes: 420, avgDurationMs: 2500 },
+      [RetryStrategy.ADAPTIVE]: { attempts: 345, successes: 320, avgDurationMs: 2000 },
+      [RetryStrategy.INTELLIGENT]: { attempts: 76, successes: 68, avgDurationMs: 3200 },
+    },
+    byErrorType: {
+      'TimeoutError': 234,
+      'NetworkError': 345,
+      'AssertionError': 123,
+      'ValidationError': 89,
+      'Other': 443,
+    },
+    byFlakinessScore: {
+      low: 234,
+      medium: 567,
+      high: 433,
+    },
+  };
+
+  if (options.strategy) {
+    const strategyStats = mockStats.byStrategy[options.strategy as RetryStrategy];
+    if (!strategyStats) {
+      console.log(chalk.red(`\nUnknown strategy: ${options.strategy}`));
+      console.log(chalk.gray('Available strategies: none, fixed, linear, exponential, adaptive, intelligent\n'));
+      return;
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify(strategyStats, null, 2));
+      return;
+    }
+
+    const table = new Table({
+      head: [chalk.bold('Metric'), chalk.bold('Value')],
+      style: { head: ['cyan', 'bold'] }
+    });
+
+    table.push(
+      ['Strategy', options.strategy],
+      ['Total Attempts', strategyStats.attempts.toString()],
+      ['Successes', strategyStats.successes.toString()],
+      ['Success Rate', `${((strategyStats.successes / strategyStats.attempts) * 100).toFixed(1)}%`],
+      ['Avg Duration', `${strategyStats.avgDurationMs}ms`]
+    );
+
+    console.log('\n' + chalk.bold.blue(`📊 ${options.strategy} Strategy Statistics`));
+    console.log(table.toString() + '\n');
+  } else {
+    if (options.json) {
+      console.log(JSON.stringify(mockStats, null, 2));
+      return;
+    }
+
+    displayStats(mockStats);
+    displayStrategyComparison(mockStats.byStrategy);
+  }
+}
+
+/**
+ * Retry configure command
+ */
+export async function retryConfigureCommand(options: {
+  strategy?: string;
+  maxRetries?: number;
+  delay?: number;
+  jitter?: number;
+  show?: boolean;
+}): Promise<void> {
+  const engine = createSmartRetryEngine();
+
+  if (options.show) {
+    displayConfig(engine.getConfig());
+    return;
+  }
+
+  const newConfig: Partial<RetryConfig> = {};
+
+  if (options.strategy) {
+    const validStrategies = ['none', 'fixed', 'linear', 'exponential', 'adaptive', 'intelligent'];
+    if (!validStrategies.includes(options.strategy.toLowerCase())) {
+      console.log(chalk.red(`\nInvalid strategy: ${options.strategy}`));
+      console.log(chalk.gray('Available strategies: none, fixed, linear, exponential, adaptive, intelligent\n'));
+      return;
+    }
+    newConfig.strategy = options.strategy.toUpperCase() as RetryStrategy;
+  }
+
+  if (options.maxRetries !== undefined) {
+    newConfig.maxRetries = options.maxRetries;
+  }
+
+  if (options.delay !== undefined) {
+    newConfig.initialDelayMs = options.delay;
+  }
+
+  if (options.jitter !== undefined) {
+    newConfig.jitterFactor = options.jitter;
+  }
+
+  if (Object.keys(newConfig).length === 0) {
+    console.log(chalk.yellow('\nNo configuration changes specified.\n'));
+    console.log(chalk.gray('Use: qa360 retry configure --show to view current configuration\n'));
+    return;
+  }
+
+  engine.updateConfig(newConfig);
+
+  console.log(chalk.green('\nRetry configuration updated:\n'));
+  displayConfig(engine.getConfig());
+
+  console.log(chalk.yellow('Note: Configuration changes apply to the current session only.'));
+  console.log(chalk.gray('To persist changes, add them to your qa360.yml config file.\n'));
+}
+
+/**
+ * Retry test command
+ */
+export async function retryTestCommand(options: {
+  testId?: string;
+  strategy?: string;
+  dryRun?: boolean;
+}): Promise<void> {
+  const engine = createSmartRetryEngine();
+
+  if (!options.testId) {
+    console.log(chalk.yellow('\nPlease specify a test ID with --test-id <id>\n'));
+    return;
+  }
+
+  const rec = engine.getRetryRecommendation({
+    flakinessScore: 70,
+  });
+
+  console.log(chalk.bold.blue(`\nRetry Recommendation for: ${options.testId}`));
+  console.log(chalk.gray('─'.repeat(60)));
+
+  const table = new Table({
+    head: [chalk.bold('Setting'), chalk.bold('Value')],
+    style: { head: ['cyan', 'bold'] }
+  });
+
+  table.push(
+    ['Should Retry', rec.shouldRetry ? chalk.green('Yes') : chalk.red('No')],
+    ['Strategy', rec.strategy],
+    ['Max Retries', rec.maxRetries.toString()],
+    ['Initial Delay', `${rec.initialDelayMs}ms`],
+    ['Confidence', `${(rec.confidence * 100).toFixed(0)}%`],
+    ['Reason', rec.reason]
+  );
+
+  console.log(table.toString() + '\n');
+
+  if (options.dryRun) {
+    console.log(chalk.yellow('Dry run mode - no tests executed.\n'));
+  }
+}
+
+/**
+ * Create retry commands
+ */
+export function createRetryCommands(): Command {
+  const retryCommand = new Command('retry')
+    .description('Smart Retry management and statistics (F8 Module)');
+
+  retryCommand
+    .command('history')
+    .description('View retry history from vault')
+    .option('--test-id <id>', 'Filter by specific test ID')
+    .option('--limit <n>', 'Limit number of entries', '10')
+    .option('--json', 'Output as JSON')
+    .action(async (options) => {
+      await retryHistoryCommand(options);
+    });
+
+  retryCommand
+    .command('stats')
+    .description('Display retry statistics')
+    .option('--strategy <name>', 'Filter by strategy (none, fixed, linear, exponential, adaptive, intelligent)')
+    .option('--json', 'Output as JSON')
+    .action(async (options) => {
+      await retryStatsCommand(options);
+    });
+
+  retryCommand
+    .command('configure')
+    .description('Configure retry settings')
+    .option('--show', 'Show current configuration')
+    .option('--strategy <name>', 'Set retry strategy')
+    .option('--max-retries <n>', 'Set maximum retry attempts')
+    .option('--delay <ms>', 'Set initial delay in milliseconds')
+    .option('--jitter <factor>', 'Set jitter factor (0-1)')
+    .action(async (options) => {
+      await retryConfigureCommand(options);
+    });
+
+  retryCommand
+    .command('test')
+    .description('Get retry recommendation for a test')
+    .option('--test-id <id>', 'Test ID to check')
+    .option('--strategy <name>', 'Override strategy')
+    .option('--dry-run', 'Show recommendation without executing')
+    .action(async (options) => {
+      await retryTestCommand(options);
+    });
+
+  retryCommand
+    .command('recommendations')
+    .description('Show flakiness-based retry recommendations')
+    .action(async () => {
+      displayRecommendations();
+    });
+
+  return retryCommand;
+}

package/cli/src/commands/run.ts

@@ -0,0 +1,218 @@
+/**
+ * QA360 Run Command - Execute quality gates
+ *
+ * Usage:
+ *   qa360 run [pack.yaml]
+ *   qa360 run --output ./results
+ *   qa360 run --verbose
+ */
+
+import { existsSync, readFileSync } from 'fs';
+import { join, resolve } from 'path';
+import chalk from 'chalk';
+import { load } from 'js-yaml';
+import { Phase3Runner, PackLoaderV2, type Phase3RunResult, type GateResult, type PackConfigV1, type PackConfigV2 } from 'qa360-core';
+
+/**
+ * CLI options for run command
+ */
+export interface RunOptions {
+  output?: string;
+  verbose?: boolean;
+  dryRun?: boolean;
+  strict?: boolean;
+  headed?: boolean;
+}
+
+/**
+ * Unified pack type - can be v1 or v2, Phase3Runner handles both
+ */
+export type PackConfig = PackConfigV1 | PackConfigV2;
+
+/**
+ * Format gates for display (handles both v1 array and v2 object formats)
+ */
+function formatGates(pack: PackConfig): string {
+  if (Array.isArray(pack.gates)) {
+    // v1 format: gates: ["api_smoke", "ui"]
+    return pack.gates.join(', ');
+  } else {
+    // v2 format: gates: { api_health: { type: api }, ui_e2e: { type: ui } }
+    const gateNames = Object.keys(pack.gates || {});
+    return gateNames.join(', ') || 'none';
+  }
+}
+
+/**
+ * Load and validate pack configuration (supports both v1 and v2)
+ * Uses PackLoaderV2 which automatically migrates v1 to v2
+ */
+export async function loadPack(packPath: string): Promise<PackConfig> {
+  if (!existsSync(packPath)) {
+    throw new Error(`Pack file not found: ${packPath}`);
+  }
+
+  // Use PackLoaderV2 which handles both v1 and v2 formats
+  const loader = new PackLoaderV2();
+  const result = await loader.load(packPath, {
+    validate: true,
+    migrate: true  // Auto-migrate v1 to v2
+  });
+
+  if (!result.success) {
+    throw new Error(`Failed to load pack: ${result.error?.message || 'Unknown error'}`);
+  }
+
+  // Handle validation errors
+  if (result.validationErrors && result.validationErrors.length > 0) {
+    const errors = result.validationErrors.map((e: any) => {
+      if (e.message) return `  • ${e.path ? e.path + ': ' : ''}${e.message}`;
+      return `  • ${JSON.stringify(e)}`;
+    }).join('\n');
+    throw new Error(`Invalid pack configuration:\n${errors}`);
+  }
+
+  // Show migration info if v1 was migrated
+  if (result.migrated) {
+    console.log(chalk.gray(`   📦 Migrated v1 → v2 format`));
+    if (result.changes && result.changes.length > 0) {
+      console.log(chalk.gray(`   Changes: ${result.changes.join(', ')}`));
+    }
+  }
+
+  // Show warnings if any
+  if (result.warnings && result.warnings.length > 0) {
+    console.log(chalk.yellow('\n⚠️  Pack warnings:'));
+    result.warnings.forEach((w: string) => console.log(chalk.yellow(`  • ${w}`)));
+  }
+
+  return result.pack as PackConfig;
+}
+
+/**
+ * Find pack file (search order: argument, .qa360/pack.yml, pack.yaml, pack.yml)
+ */
+export function findPackFile(packArg?: string): string {
+  if (packArg) {
+    return resolve(packArg);
+  }
+
+  const searchPaths = [
+    join(process.cwd(), '.qa360', 'pack.yml'),
+    join(process.cwd(), '.qa360', 'pack.yaml'),
+    join(process.cwd(), 'pack.yml'),
+    join(process.cwd(), 'pack.yaml'),
+    join(process.cwd(), 'qa360.yml'),
+    join(process.cwd(), 'qa360.yaml'),
+  ];
+
+  for (const path of searchPaths) {
+    if (existsSync(path)) {
+      return path;
+    }
+  }
+
+  throw new Error(
+    'No pack file found in current directory.\n\n' +
+    'To get started:\n' +
+    '  1. Copy an example: qa360 examples copy api-basic\n' +
+    '  2. Or create interactively: qa360 init\n' +
+    '  3. Or specify a path: qa360 run path/to/pack.yaml'
+  );
+}
+
+/**
+ * Display run results summary
+ */
+export function displayResults(result: Phase3RunResult): void {
+  console.log(chalk.bold('\n' + '═'.repeat(50)));
+  console.log(chalk.bold('📊 EXECUTION SUMMARY'));
+  console.log('═'.repeat(50));
+
+  console.log(`\n⏱️  Duration: ${result.duration}ms`);
+  console.log(`🎯 Trust Score: ${result.summary.trustScore}%`);
+  console.log(`📋 Gates: ${result.summary.passed}/${result.summary.total} passed`);
+
+  if (result.proofPath) {
+    console.log(`\n📄 Proof: ${result.proofPath}`);
+    console.log(chalk.gray('   Verify with: qa360 verify ' + result.proofPath));
+  }
+
+  if (result.success) {
+    console.log(chalk.green.bold('\n✅ All quality gates passed!'));
+  } else {
+    console.log(chalk.red.bold(`\n❌ ${result.summary.failed} gate(s) failed`));
+    
+    // Show failed gates
+    const failed = result.gates.filter((g: GateResult) => !g.success);
+    if (failed.length > 0) {
+      console.log(chalk.red('\nFailed gates:'));
+      failed.forEach((g: GateResult) => {
+        const errorMessage = g.error || 'Test failed - check logs above for details';
+        console.log(chalk.red(`  • ${g.gate}: ${errorMessage}`));
+      });
+    }
+  }
+
+  console.log('═'.repeat(50) + '\n');
+}
+
+/**
+ * Main run command
+ */
+export async function runCommand(
+  packArg?: string,
+  options: RunOptions = {}
+): Promise<void> {
+  try {
+    // Step 1: Find pack file
+    const packPath = findPackFile(packArg);
+    console.log(chalk.blue(`\n📦 Loading pack: ${packPath}`));
+
+    // Step 2: Load and validate pack
+    const pack = await loadPack(packPath);
+    console.log(chalk.green(`   ✅ Pack loaded: ${pack.name} v${pack.version}`));
+    console.log(chalk.gray(`   Gates: ${formatGates(pack)}`));
+
+    // Step 3: Dry run check
+    if (options.dryRun) {
+      console.log(chalk.yellow('\n🔍 Dry run mode - no tests executed'));
+      console.log(chalk.gray('   Remove --dry-run to execute tests'));
+      return;
+    }
+
+    // Step 4: Create and run Phase3Runner
+    const workingDir = process.cwd();
+    const outputDir = options.output 
+      ? resolve(options.output) 
+      : join(workingDir, '.qa360', 'runs');
+
+    const runner = new Phase3Runner({
+      workingDir,
+      pack,
+      outputDir,
+    });
+
+    const result = await runner.run();
+
+    // Step 5: Display results
+    displayResults(result);
+
+    // Step 6: Exit with appropriate code
+    process.exit(result.success ? 0 : 1);
+
+  } catch (error) {
+    console.error(chalk.red('\n❌ Execution failed:'));
+    console.error(chalk.red(`   ${error instanceof Error ? error.message : 'Unknown error'}`));
+    
+    if (options.verbose && error instanceof Error && error.stack) {
+      console.error(chalk.gray('\nStack trace:'));
+      console.error(chalk.gray(error.stack));
+    }
+
+    process.exit(1);
+  }
+}
+
+// Export for CLI integration
+export default runCommand;