qa360 1.4.5 → 2.0.1

package/dist/commands/flakiness.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Flakiness CLI Commands
+ *
+ * Provides commands for managing test flakiness detection and quarantine.
+ */
+import { Command } from 'commander';
+/**
+ * List flakiness history for a test
+ */
+export declare function flakinessHistoryAction(testId: string, options: {
+    limit?: number;
+    json?: boolean;
+    workingDir?: string;
+}): Promise<void>;
+/**
+ * Show flakiness trends for a test
+ */
+export declare function flakinessTrendsAction(testId: string, options: {
+    days?: number;
+    json?: boolean;
+    workingDir?: string;
+}): Promise<void>;
+/**
+ * List all flaky tests from recent runs
+ */
+export declare function flakinessListAction(options: {
+    runId?: string;
+    threshold?: number;
+    json?: boolean;
+    workingDir?: string;
+}): Promise<void>;
+/**
+ * List quarantined tests
+ */
+export declare function flakinessQuarantineListAction(options: {
+    all?: boolean;
+    json?: boolean;
+    workingDir?: string;
+}): Promise<void>;
+/**
+ * Add a test to quarantine
+ */
+export declare function flakinessQuarantineAddAction(testId: string, options: {
+    reason?: string;
+    score?: number;
+    category?: string;
+    workingDir?: string;
+}): Promise<void>;
+/**
+ * Remove a test from quarantine
+ */
+export declare function flakinessQuarantineRemoveAction(testId: string, options: {
+    notes?: string;
+    workingDir?: string;
+}): Promise<void>;
+/**
+ * Show patterns for a test
+ */
+export declare function flakinessPatternsAction(testId: string, options: {
+    json?: boolean;
+    workingDir?: string;
+}): Promise<void>;
+/**
+ * Analyze flakiness for a specific run
+ */
+export declare function flakinessAnalyzeAction(runId: string, options: {
+    json?: boolean;
+    workingDir?: string;
+}): Promise<void>;
+/**
+ * Create all flakiness commands
+ */
+export declare function createFlakinessCommands(): Command;

package/dist/commands/flakiness.js

@@ -0,0 +1,435 @@
+/**
+ * Flakiness CLI Commands
+ *
+ * Provides commands for managing test flakiness detection and quarantine.
+ */
+import { Command } from 'commander';
+import chalk from 'chalk';
+import Table from 'cli-table3';
+import { join } from 'path';
+import { existsSync } from 'fs';
+import { EvidenceVault, FLAKINESS_CATEGORIES } from '../core/index.js';
+/**
+ * Get the vault directory path
+ */
+function getVaultPath(workingDir = process.cwd()) {
+    return join(workingDir, '.qa360');
+}
+/**
+ * Open the vault or exit with error
+ */
+async function openVault(workingDir) {
+    const vaultPath = getVaultPath(workingDir);
+    const vaultDbPath = join(vaultPath, 'vault.db');
+    if (!existsSync(vaultDbPath)) {
+        console.error(chalk.red(`❌ Vault not found at ${vaultDbPath}`));
+        console.error(chalk.yellow('💡 Run "qa360 run" first to create a vault'));
+        process.exit(1);
+    }
+    return await EvidenceVault.open(vaultPath);
+}
+/**
+ * Format flakiness score with emoji
+ */
+function formatScore(score, category) {
+    const meta = FLAKINESS_CATEGORIES[category];
+    const color = score >= 75 ? 'green' : score >= 50 ? 'yellow' : 'red';
+    return chalk[color](`${meta.emoji} ${score}%`);
+}
+/**
+ * Format category with color
+ */
+function formatCategory(category) {
+    const meta = FLAKINESS_CATEGORIES[category];
+    const color = category === 'legendary' || category === 'solid' ? 'green' :
+        category === 'good' ? 'yellow' :
+            category === 'shaky' ? 'yellow' : 'red';
+    return chalk[color](meta.label);
+}
+/**
+ * List flakiness history for a test
+ */
+export async function flakinessHistoryAction(testId, options) {
+    const vault = await openVault(options.workingDir);
+    const history = await vault.getFlakinessHistory(testId, options.limit || 50);
+    if (options.json) {
+        console.log(JSON.stringify(history, null, 2));
+        await vault.close();
+        return;
+    }
+    if (history.length === 0) {
+        console.log(chalk.yellow(`\n⚠️  No flakiness history found for test: ${testId}`));
+        await vault.close();
+        return;
+    }
+    console.log(chalk.blue(`\n📊 Flakiness History: ${history[0].test_name}`));
+    console.log(chalk.gray(`Test ID: ${testId}`));
+    console.log(chalk.gray(`File: ${history[0].file_path}\n`));
+    const table = new Table({
+        head: [chalk.gray('Score'), chalk.gray('Category'), chalk.gray('Runs'), chalk.gray('Duration'), chalk.gray('Date')],
+        colWidths: [12, 15, 10, 12, 20],
+        style: { head: [], border: ['gray'] }
+    });
+    for (const record of history) {
+        const date = new Date(record.created_at).toLocaleDateString();
+        table.push([
+            formatScore(record.score, record.category),
+            formatCategory(record.category),
+            `${record.successful_runs}/${record.total_runs}`,
+            `${record.avg_duration_ms}ms`,
+            date
+        ]);
+    }
+    console.log(table.toString());
+    await vault.close();
+}
+/**
+ * Show flakiness trends for a test
+ */
+export async function flakinessTrendsAction(testId, options) {
+    const vault = await openVault(options.workingDir);
+    const trends = await vault.getFlakinessTrends(testId, options.days || 30);
+    if (options.json) {
+        console.log(JSON.stringify(trends, null, 2));
+        await vault.close();
+        return;
+    }
+    // Get test name from history
+    const history = await vault.getFlakinessHistory(testId, 1);
+    const testName = history[0]?.test_name || testId;
+    console.log(chalk.blue(`\n📈 Flakiness Trends: ${testName}`));
+    console.log(chalk.gray(`Test ID: ${testId}`));
+    console.log(chalk.gray(`Time Window: Last ${options.days || 30} days\n`));
+    const trendEmoji = trends.trend === 'improving' ? '📈' : trends.trend === 'degrading' ? '📉' : '➡️';
+    const trendColor = trends.trend === 'improving' ? 'green' : trends.trend === 'degrading' ? 'red' : 'gray';
+    const trendText = trends.trend === 'improving' ? 'Improving' : trends.trend === 'degrading' ? 'Degrading' : 'Stable';
+    const currentCategory = trends.currentScore >= 75 ? 'good' : trends.currentScore >= 50 ? 'shaky' : 'unstable';
+    console.log(`${chalk.gray('Current Score:')} ${formatScore(trends.currentScore, currentCategory)}`);
+    console.log(`${chalk.gray('Average Score:')} ${formatScore(Math.round(trends.averageScore), currentCategory)}`);
+    console.log(`${chalk.gray('Trend:')} ${chalk[trendColor](`${trendEmoji} ${trendText}`)}`);
+    console.log(`${chalk.gray('Data Points:')} ${trends.dataPoints.length}\n`);
+    await vault.close();
+}
+/**
+ * List all flaky tests from recent runs
+ */
+export async function flakinessListAction(options) {
+    const vault = await openVault(options.workingDir);
+    let flakinessRecords = [];
+    if (options.runId) {
+        flakinessRecords = await vault.getRunFlakiness(options.runId);
+    }
+    else {
+        // Get recent runs and their flakiness
+        const runs = await vault.listRuns({ limit: 10 });
+        for (const run of runs) {
+            const runFlakiness = await vault.getRunFlakiness(run.id);
+            flakinessRecords.push(...runFlakiness);
+        }
+    }
+    // Filter by threshold if specified
+    if (options.threshold !== undefined) {
+        flakinessRecords = flakinessRecords.filter(r => r.score < options.threshold);
+    }
+    // Sort by score (most flaky first)
+    flakinessRecords.sort((a, b) => a.score - b.score);
+    if (options.json) {
+        console.log(JSON.stringify(flakinessRecords, null, 2));
+        await vault.close();
+        return;
+    }
+    if (flakinessRecords.length === 0) {
+        console.log(chalk.yellow('\n✅ No flaky tests found!'));
+        await vault.close();
+        return;
+    }
+    console.log(chalk.blue(`\n🎲 Flakiness Report (${flakinessRecords.length} tests)\n`));
+    const table = new Table({
+        head: [chalk.gray('Test'), chalk.gray('Gate'), chalk.gray('Score'), chalk.gray('Category'), chalk.gray('Runs'), chalk.gray('Pattern')],
+        colWidths: [30, 12, 10, 15, 8, 20],
+        wordWrap: true,
+        style: { head: [], border: ['gray'] }
+    });
+    for (const record of flakinessRecords) {
+        table.push([
+            record.test_name,
+            record.gate,
+            formatScore(record.score, record.category),
+            formatCategory(record.category),
+            `${record.successful_runs}/${record.total_runs}`,
+            record.pattern_type || '-'
+        ]);
+    }
+    console.log(table.toString());
+    await vault.close();
+}
+/**
+ * List quarantined tests
+ */
+export async function flakinessQuarantineListAction(options) {
+    const vault = await openVault(options.workingDir);
+    const quarantined = await vault.getQuarantinedTests(options.all !== false);
+    if (options.json) {
+        console.log(JSON.stringify(quarantined, null, 2));
+        await vault.close();
+        return;
+    }
+    if (quarantined.length === 0) {
+        console.log(chalk.yellow('\n✅ No tests in quarantine!'));
+        await vault.close();
+        return;
+    }
+    console.log(chalk.blue(`\n🚫 Quarantined Tests (${quarantined.length} total)\n`));
+    const table = new Table({
+        head: [chalk.gray('Test'), chalk.gray('Gate'), chalk.gray('Category'), chalk.gray('Score'), chalk.gray('Reason'), chalk.gray('Status')],
+        colWidths: [25, 10, 12, 8, 30, 12],
+        wordWrap: true,
+        style: { head: [], border: ['gray'] }
+    });
+    for (const test of quarantined) {
+        const status = test.resolved_at ?
+            chalk.green('✅ Resolved') :
+            chalk.red('🚫 Active');
+        table.push([
+            test.test_name,
+            test.gate,
+            formatCategory(test.category),
+            `${test.score}%`,
+            test.reason,
+            status
+        ]);
+    }
+    console.log(table.toString());
+    await vault.close();
+}
+/**
+ * Add a test to quarantine
+ */
+export async function flakinessQuarantineAddAction(testId, options) {
+    const vault = await openVault(options.workingDir);
+    // Check if already in quarantine
+    const existing = await vault.getQuarantine(testId);
+    const alreadyQuarantined = existing && !existing.resolved_at;
+    if (alreadyQuarantined) {
+        await vault.close();
+        console.log(chalk.yellow(`\n⚠️  Test "${existing.test_name}" is already in quarantine\n`));
+        return;
+    }
+    // Try to get test info from history
+    const history = await vault.getFlakinessHistory(testId, 1);
+    const testName = history[0]?.test_name || testId;
+    const gate = history[0]?.gate || 'unknown';
+    const filePath = history[0]?.file_path || 'unknown';
+    const score = options.score || history[0]?.score || 0;
+    const category = (options.category || history[0]?.category || 'unstable');
+    const reason = options.reason || `Manually quarantined by user`;
+    const quarantineId = await vault.addToQuarantine({
+        test_id: testId,
+        test_name: testName,
+        gate,
+        file_path: filePath,
+        reason,
+        score,
+        category,
+        quarantined_at: Date.now(),
+        quarantined_by: 'user'
+    });
+    console.log(chalk.green(`\n✅ Test "${testName}" added to quarantine (ID: ${quarantineId})`));
+    console.log(chalk.gray(`   Reason: ${reason}`));
+    console.log(chalk.gray(`   Category: ${formatCategory(category)} (${score}%)\n`));
+    await vault.close();
+}
+/**
+ * Remove a test from quarantine
+ */
+export async function flakinessQuarantineRemoveAction(testId, options) {
+    const vault = await openVault(options.workingDir);
+    const quarantine = await vault.getQuarantine(testId);
+    if (!quarantine) {
+        console.log(chalk.yellow(`\n⚠️  Test "${testId}" is not in quarantine\n`));
+        await vault.close();
+        return;
+    }
+    if (quarantine.resolved_at) {
+        console.log(chalk.yellow(`\n⚠️  Test "${testId}" was already resolved on ${new Date(quarantine.resolved_at).toLocaleString()}\n`));
+        await vault.close();
+        return;
+    }
+    const notes = options.notes || 'Resolved by user';
+    await vault.removeFromQuarantine(testId, 'user', notes);
+    console.log(chalk.green(`\n✅ Test "${quarantine.test_name}" removed from quarantine`));
+    console.log(chalk.gray(`   Notes: ${notes}\n`));
+    await vault.close();
+}
+/**
+ * Show patterns for a test
+ */
+export async function flakinessPatternsAction(testId, options) {
+    const vault = await openVault(options.workingDir);
+    // Get all pattern types and check each
+    const patternTypes = ['timing', 'race_condition', 'external_dependency', 'environment_specific', 'selector_issue', 'unknown'];
+    const patterns = [];
+    for (const type of patternTypes) {
+        const pattern = await vault.getFlakinessPattern(testId, type);
+        if (pattern) {
+            patterns.push(pattern);
+        }
+    }
+    if (options.json) {
+        console.log(JSON.stringify(patterns, null, 2));
+        await vault.close();
+        return;
+    }
+    if (patterns.length === 0) {
+        console.log(chalk.yellow(`\n⚠️  No patterns detected for test: ${testId}\n`));
+        await vault.close();
+        return;
+    }
+    console.log(chalk.blue(`\n🔍 Detected Patterns: ${patterns[0].test_name || testId}\n`));
+    const table = new Table({
+        head: [chalk.gray('Pattern'), chalk.gray('Description'), chalk.gray('Confidence'), chalk.gray('Fix'), chalk.gray('Detections')],
+        colWidths: [18, 30, 12, 30, 10],
+        wordWrap: true,
+        style: { head: [], border: ['gray'] }
+    });
+    for (const pattern of patterns) {
+        table.push([
+            chalk.cyan(pattern.pattern_type),
+            pattern.description,
+            `${Math.round(pattern.confidence * 100)}%`,
+            pattern.suggested_fix,
+            pattern.detection_count.toString()
+        ]);
+    }
+    console.log(table.toString());
+    await vault.close();
+}
+/**
+ * Analyze flakiness for a specific run
+ */
+export async function flakinessAnalyzeAction(runId, options) {
+    const vault = await openVault(options.workingDir);
+    const flakiness = await vault.getRunFlakiness(runId);
+    if (options.json) {
+        console.log(JSON.stringify(flakiness, null, 2));
+        await vault.close();
+        return;
+    }
+    if (flakiness.length === 0) {
+        console.log(chalk.yellow(`\n⚠️  No flakiness data found for run: ${runId}\n`));
+        await vault.close();
+        return;
+    }
+    // Calculate stats
+    const legendary = flakiness.filter((f) => f.category === 'legendary').length;
+    const solid = flakiness.filter((f) => f.category === 'solid').length;
+    const good = flakiness.filter((f) => f.category === 'good').length;
+    const shaky = flakiness.filter((f) => f.category === 'shaky').length;
+    const unstable = flakiness.filter((f) => f.category === 'unstable').length;
+    const avgScore = Math.round(flakiness.reduce((sum, f) => sum + f.score, 0) / flakiness.length);
+    console.log(chalk.blue(`\n📊 Flakiness Analysis for Run: ${runId}\n`));
+    console.log(chalk.gray(`Total Tests: ${flakiness.length}`));
+    console.log(chalk.gray(`Average Score: ${formatScore(avgScore, avgScore >= 75 ? 'good' : avgScore >= 50 ? 'shaky' : 'unstable')}\n`));
+    console.log(chalk.gray('Distribution:'));
+    console.log(`  ${chalk.green('🟢')} Legendary: ${legendary}`);
+    console.log(`  ${chalk.green('🟢')} Solid: ${solid}`);
+    console.log(`  ${chalk.yellow('🟡')} Good: ${good}`);
+    console.log(`  ${chalk.yellow('🟠')} Shaky: ${shaky}`);
+    console.log(`  ${chalk.red('🔴')} Unstable: ${unstable}\n`);
+    // Show tests needing attention
+    const needsAttention = flakiness.filter((f) => f.score < 75);
+    if (needsAttention.length > 0) {
+        console.log(chalk.yellow(`⚠️  ${needsAttention.length} test(s) need attention:\n`));
+        const table = new Table({
+            head: [chalk.gray('Test'), chalk.gray('Score'), chalk.gray('Pattern'), chalk.gray('Suggested Fix')],
+            colWidths: [25, 10, 15, 40],
+            wordWrap: true,
+            style: { head: [], border: ['gray'] }
+        });
+        for (const test of needsAttention.slice(0, 10)) {
+            table.push([
+                test.test_name,
+                formatScore(test.score, test.category),
+                test.pattern_type || '-',
+                test.suggested_fix || '-'
+            ]);
+        }
+        console.log(table.toString());
+        if (needsAttention.length > 10) {
+            console.log(chalk.gray(`\n... and ${needsAttention.length - 10} more`));
+        }
+    }
+    await vault.close();
+}
+/**
+ * Create all flakiness commands
+ */
+export function createFlakinessCommands() {
+    const flakinessCommand = new Command('flakiness')
+        .description('Test flakiness detection and quarantine management');
+    // History command
+    flakinessCommand
+        .command('history <testId>')
+        .description('Show flakiness history for a test')
+        .option('-l, --limit <number>', 'Number of records to show', '50')
+        .option('--json', 'Output as JSON')
+        .option('-w, --working-dir <path>', 'Working directory (default: current directory)')
+        .action(flakinessHistoryAction);
+    // Trends command
+    flakinessCommand
+        .command('trends <testId>')
+        .description('Show flakiness trends over time')
+        .option('-d, --days <number>', 'Time window in days', '30')
+        .option('--json', 'Output as JSON')
+        .option('-w, --working-dir <path>', 'Working directory')
+        .action(flakinessTrendsAction);
+    // List command
+    flakinessCommand
+        .command('list')
+        .description('List all flaky tests from recent runs')
+        .option('-r, --run-id <id>', 'Specific run ID')
+        .option('-t, --threshold <number>', 'Only show tests below this score')
+        .option('--json', 'Output as JSON')
+        .option('-w, --working-dir <path>', 'Working directory')
+        .action(flakinessListAction);
+    // Analyze command
+    flakinessCommand
+        .command('analyze <runId>')
+        .description('Analyze flakiness for a specific run')
+        .option('--json', 'Output as JSON')
+        .option('-w, --working-dir <path>', 'Working directory')
+        .action(flakinessAnalyzeAction);
+    // Patterns command
+    flakinessCommand
+        .command('patterns <testId>')
+        .description('Show detected patterns for a test')
+        .option('--json', 'Output as JSON')
+        .option('-w, --working-dir <path>', 'Working directory')
+        .action(flakinessPatternsAction);
+    // Quarantine subcommand
+    const quarantineCommand = flakinessCommand
+        .command('quarantine')
+        .description('Manage quarantined tests');
+    quarantineCommand
+        .command('list')
+        .description('List quarantined tests')
+        .option('-a, --all', 'Include resolved quarantines', false)
+        .option('--json', 'Output as JSON')
+        .option('-w, --working-dir <path>', 'Working directory')
+        .action(flakinessQuarantineListAction);
+    quarantineCommand
+        .command('add <testId>')
+        .description('Add a test to quarantine')
+        .option('-r, --reason <text>', 'Reason for quarantine')
+        .option('-s, --score <number>', 'Flakiness score (0-100)')
+        .option('-c, --category <category>', 'Category: shaky, unstable')
+        .option('-w, --working-dir <path>', 'Working directory')
+        .action(flakinessQuarantineAddAction);
+    quarantineCommand
+        .command('remove <testId>')
+        .description('Remove a test from quarantine')
+        .option('-n, --notes <text>', 'Resolution notes')
+        .option('-w, --working-dir <path>', 'Working directory')
+        .action(flakinessQuarantineRemoveAction);
+    return flakinessCommand;
+}

package/dist/commands/generate.d.ts

@@ -0,0 +1,66 @@
+/**
+ * QA360 Generate Command
+ *
+ * AI-powered test generation using Ollama.
+ * Generates tests from OpenAPI specs, HAR files, URLs, or code.
+ *
+ * Phase 4 - AI Test Generation
+ */
+import { Command } from 'commander';
+/**
+ * Create generate commands group
+ */
+export declare function createGenerateCommands(): Command;
+/**
+ * Generate tests command handler
+ */
+export declare function generateCommand(source: string, options?: {
+    type?: 'openapi' | 'har' | 'url' | 'code';
+    output?: string;
+    framework?: 'playwright' | 'k6' | 'vitest' | 'jest';
+    optimize?: boolean;
+    model?: string;
+    format?: boolean;
+}): Promise<void>;
+/**
+ * Generate API tests from OpenAPI spec
+ */
+export declare function generateApiCommand(specPath: string, options?: {
+    output?: string;
+    model?: string;
+    optimize?: boolean;
+}): Promise<void>;
+/**
+ * Generate UI tests from URL
+ */
+export declare function generateUiCommand(url: string, options?: {
+    output?: string;
+    model?: string;
+    optimize?: boolean;
+}): Promise<void>;
+/**
+ * Generate performance tests
+ */
+export declare function generatePerfCommand(specPath: string, options?: {
+    output?: string;
+    model?: string;
+    duration?: string;
+    vus?: number;
+}): Promise<void>;
+/**
+ * Generate unit tests from code
+ */
+export declare function generateUnitCommand(filePath: string, options?: {
+    language?: string;
+    output?: string;
+    model?: string;
+    optimize?: boolean;
+}): Promise<void>;
+/**
+ * Build system prompt for test type
+ */
+export declare function buildSystemPrompt(type: string): string;
+/**
+ * Check if generation is available
+ */
+export declare function checkCommand(): Promise<void>;