qa360 2.2.1 → 2.2.7

package/cli/dist/commands/explain.js

@@ -0,0 +1,630 @@
+/**
+ * QA360 Explain Command
+ *
+ * Advanced error and failure explanations with:
+ * - Static error code database
+ * - AI-powered explanations
+ * - Vault-based historical context
+ * - Flakiness insights
+ *
+ * Usage:
+ *   qa360 explain QX001              # Static error code
+ *   qa360 explain --ai "my test failed"  # AI explanation
+ *   qa360 explain --run run-xxx      # Explain run results
+ *   qa360 explain --gate api_smoke   # Explain gate failure
+ */
+import chalk from 'chalk';
+import ora from 'ora';
+import { join } from 'path';
+import { existsSync } from 'fs';
+import { createBest, createLLMProvider } from '../core/index.js';
+import { EvidenceVault } from '../core/index.js';
+/**
+ * Static error code database
+ */
+const ERROR_CODES = {
+    'QX001': {
+        title: 'Node.js Version Incompatible',
+        cause: 'QA360 Core requires Node.js 18.0.0 or higher. You are running an older version.',
+        solution: 'Update Node.js: https://nodejs.org/ or use nvm/fnm: `nvm install 20`',
+        category: 'prerequisite'
+    },
+    'QX002': {
+        title: 'QA360 Directory Structure Missing',
+        cause: 'Required .qa360 directories not found in the current project.',
+        solution: 'Run "qa360 doctor --fix" to create missing directories automatically',
+        category: 'setup'
+    },
+    'QX003': {
+        title: 'Playwright Not Available',
+        cause: 'Playwright is not installed or not in PATH. Required for API and UI tests.',
+        solution: 'Install: "npx playwright install" or use --docker flag for containerized execution',
+        category: 'prerequisite'
+    },
+    'QX004': {
+        title: 'Puppeteer Not Available',
+        cause: 'Puppeteer is not installed (required for PDF report generation).',
+        solution: 'Install: "npm install puppeteer" or use --format json to skip PDF generation',
+        category: 'prerequisite'
+    },
+    'QX005': {
+        title: 'Network Connectivity Issues',
+        cause: 'Cannot reach external services (API, website). Check proxy/firewall settings.',
+        solution: 'Check network, configure proxy (export HTTP_PROXY), or test with local endpoints',
+        category: 'network'
+    },
+    'QX006': {
+        title: 'Secrets Configuration Missing',
+        cause: 'No secrets.enc file configured. Required for encrypted credential storage.',
+        solution: 'Run "qa360 doctor --fix" to create default secrets file',
+        category: 'setup'
+    },
+    'QX007': {
+        title: 'Cryptographic Keys Missing',
+        cause: 'Ed25519 key pair not found for signing proofs. Required for proof generation.',
+        solution: 'Run "qa360 doctor --fix" to generate new key pair automatically',
+        category: 'setup'
+    },
+    'QP100': {
+        title: 'Pack Validation Failed',
+        cause: 'The pack.yml file does not conform to the required schema or has invalid values.',
+        solution: 'Check pack.yml syntax with "qa360 pack validate". Ensure required fields (version, name, gates) are present.',
+        category: 'pack'
+    },
+    'QP200': {
+        title: 'Pack Version Not Supported',
+        cause: 'The pack version specified is not supported by this version of QA360.',
+        solution: 'Update QA360: "npm install -g qa360@latest" or use pack v1 format',
+        category: 'pack'
+    },
+    'QG200': {
+        title: 'Gate Execution Failed',
+        cause: 'A quality gate failed during execution. Check gate-specific error messages.',
+        solution: 'Review gate logs in .qa360/runs/ directory. Run with verbose mode for details.',
+        category: 'execution'
+    },
+    'QG300': {
+        title: 'Adapter Not Found',
+        cause: 'The required adapter for a gate is not available or not installed.',
+        solution: 'Ensure dependencies are installed. Run "qa360 doctor" to check prerequisites.',
+        category: 'execution'
+    },
+    'QV100': {
+        title: 'Proof Verification Failed',
+        cause: 'The proof signature is invalid or the proof has been tampered with.',
+        solution: 'Ensure the proof file has not been modified. Check the public key matches the signer.',
+        category: 'verification'
+    },
+    'QV200': {
+        title: 'Vault Corruption Detected',
+        cause: 'The evidence vault database (.qa360/vault.db) is corrupted or invalid.',
+        solution: 'Run "qa360 doctor --fix" to repair the vault. Restore from backup if available.',
+        category: 'vault'
+    },
+    'QT000': {
+        title: 'Test Timeout',
+        cause: 'A test exceeded the maximum allowed execution time.',
+        solution: 'Increase timeout in pack.yml (execution.timeout) or optimize the test',
+        category: 'execution'
+    },
+    'QT001': {
+        title: 'Flaky Test Detected',
+        cause: 'A test is showing inconsistent results across runs (flakiness score < 75).',
+        solution: 'Review "qa360 flakiness history" for patterns. Consider quarantining: "qa360 flakiness quarantine <testId>"',
+        category: 'flakiness'
+    },
+    'QD500': {
+        title: 'Dashboard Start Failed',
+        cause: 'The dashboard server cannot start (port in use or permission denied).',
+        solution: 'Use --port to specify a different port, or check if another instance is running.',
+        category: 'dashboard'
+    }
+};
+/**
+ * Gate-specific explanations
+ */
+const GATE_EXPLANATIONS = {
+    'api_smoke': {
+        title: 'API Smoke Tests',
+        commonFailures: [
+            'Endpoint not reachable (404, connection refused)',
+            'Unexpected response status (expected 200, got 500)',
+            'Response format mismatch',
+            'Authentication failure (401, 403)'
+        ],
+        tips: [
+            'Verify API is running: curl -I <apiUrl>',
+            'Check authentication tokens are valid',
+            'Review CORS settings for cross-origin requests',
+            'Ensure test environment matches expected configuration'
+        ]
+    },
+    'ui': {
+        title: 'UI/E2E Tests',
+        commonFailures: [
+            'Page load timeout',
+            'Element not found (selector changed)',
+            'JavaScript errors in browser console',
+            'Viewport/responsive issues'
+        ],
+        tips: [
+            'Use data-testid attributes for stable selectors',
+            'Add explicit waits for dynamic content',
+            'Check browser console for JavaScript errors',
+            'Test in headless mode matches real browser behavior'
+        ]
+    },
+    'a11y': {
+        title: 'Accessibility Tests',
+        commonFailures: [
+            'Missing alt text on images',
+            'Low color contrast ratio',
+            'Missing ARIA labels on interactive elements',
+            'Keyboard navigation issues'
+        ],
+        tips: [
+            'Use axe DevTools browser extension for debugging',
+            'Test with keyboard only (no mouse)',
+            'Run screen reader simulation to verify experience',
+            'Check contrast ratios with WCAG tools'
+        ]
+    },
+    'perf': {
+        title: 'Performance Tests',
+        commonFailures: [
+            'Response time exceeds budget',
+            'Error rate too high under load',
+            'Resource exhaustion (memory, CPU)',
+            'Timeouts during peak load'
+        ],
+        tips: [
+            'Start with low load and gradually increase',
+            'Monitor server resources during tests',
+            'Profile slow endpoints with performance tools',
+            'Consider caching and optimization strategies'
+        ]
+    },
+    'sast': {
+        title: 'Static Application Security Testing',
+        commonFailures: [
+            'Hardcoded secrets or passwords',
+            'SQL injection vulnerabilities',
+            'Unsafe function usage',
+            'Dependency vulnerabilities'
+        ],
+        tips: [
+            'Use environment variables for secrets',
+            'Parameterize all database queries',
+            'Keep dependencies updated',
+            'Review and understand each security finding'
+        ]
+    },
+    'dast': {
+        title: 'Dynamic Application Security Testing',
+        commonFailures: [
+            'Injection vulnerabilities (SQL, XSS)',
+            'Authentication/authorization bypass',
+            'Sensitive data exposure',
+            'Misconfigurations'
+        ],
+        tips: [
+            'Run against test environment only',
+            'DAST may trigger security alerts - inform your team',
+            'Review false positives carefully',
+            'Fix critical findings before production'
+        ]
+    },
+    'secrets': {
+        title: 'Secrets Scanning',
+        commonFailures: [
+            'API keys in source code',
+            'Passwords in config files',
+            'Certificates committed to repo',
+            'Tokens in logs'
+        ],
+        tips: [
+            'Use git-secrets or similar pre-commit hooks',
+            'Rotate any leaked credentials immediately',
+            'Use .env files with gitignore',
+            'Enable branch protection rules'
+        ]
+    },
+    'deps': {
+        title: 'Dependency Vulnerability Scanning',
+        commonFailures: [
+            'Known CVEs in dependencies',
+            'Outdated packages with security fixes',
+            'Transitive dependency issues',
+            'License violations'
+        ],
+        tips: [
+            'Run "npm audit fix" for automatic fixes',
+            'Review dependabot PRs promptly',
+            'Pin dependency versions where appropriate',
+            'Understand your dependency tree'
+        ]
+    }
+};
+/**
+ * Explain an error code
+ */
+async function explainErrorCode(code, json = false) {
+    const error = ERROR_CODES[code];
+    if (!error) {
+        console.log(chalk.red(`\n[X] Unknown error code: ${code}`));
+        console.log(chalk.blue('\nAvailable error codes:'));
+        const grouped = groupErrorCodes();
+        for (const [category, codes] of Object.entries(grouped)) {
+            console.log(chalk.yellow(`\n  ${category.toUpperCase()}:`));
+            console.log(chalk.gray(`    ${codes.join(', ')}`));
+        }
+        return;
+    }
+    if (json) {
+        console.log(JSON.stringify({ code, ...error }, null, 2));
+        return;
+    }
+    console.log(chalk.red(`\n[X] ${error.title}`));
+    console.log(chalk.gray(`   Code: ${code}`));
+    console.log(chalk.gray(`   Category: ${error.category}\n`));
+    console.log(chalk.yellow('Cause:'));
+    console.log(chalk.white(`   ${error.cause}\n`));
+    console.log(chalk.green('Solution:'));
+    console.log(chalk.white(`   ${error.solution}\n`));
+    // Show related commands
+    const relatedCommands = {
+        prerequisite: ['qa360 doctor', 'qa360 doctor --fix'],
+        setup: ['qa360 doctor --fix', 'qa360 secrets add'],
+        execution: ['qa360 run --verbose', 'qa360 history show <run-id>'],
+        pack: ['qa360 pack validate', 'qa360 init'],
+        vault: ['qa360 doctor --fix', 'qa360 history list'],
+        verification: ['qa360 verify --verbose'],
+        flakiness: ['qa360 flakiness history', 'qa360 flakiness quarantine'],
+        network: ['curl -I <url>', 'ping <host>'],
+        dashboard: ['qa360 dashboard --port 8081']
+    };
+    const commands = relatedCommands[error.category];
+    if (commands) {
+        console.log(chalk.cyan('Related commands:'));
+        commands.forEach(cmd => console.log(chalk.gray(`   ${cmd}`)));
+        console.log('');
+    }
+}
+/**
+ * Group error codes by category
+ */
+function groupErrorCodes() {
+    const grouped = {};
+    for (const [code, error] of Object.entries(ERROR_CODES)) {
+        if (!grouped[error.category]) {
+            grouped[error.category] = [];
+        }
+        grouped[error.category].push(code);
+    }
+    return grouped;
+}
+/**
+ * Explain gate failure with context
+ */
+async function explainGate(gateName, runId, json = false) {
+    const info = GATE_EXPLANATIONS[gateName];
+    if (!info) {
+        console.log(chalk.yellow(`\n[!] No specific information available for gate: ${gateName}`));
+        console.log(chalk.gray('Available gates: ' + Object.keys(GATE_EXPLANATIONS).join(', ')));
+        return;
+    }
+    if (json) {
+        console.log(JSON.stringify({ gate: gateName, ...info }, null, 2));
+        return;
+    }
+    console.log(chalk.bold(`\n${info.title} (${gateName})\n`));
+    console.log(chalk.yellow('Common failures:'));
+    info.commonFailures.forEach(f => console.log(chalk.gray(`  - ${f}`)));
+    console.log(chalk.cyan('\nTips:'));
+    info.tips.forEach(t => console.log(chalk.gray(`  - ${t}`)));
+    // Load from vault if runId provided
+    if (runId) {
+        await explainGateFromVault(gateName, runId);
+    }
+}
+/**
+ * Load and display gate-specific data from vault
+ */
+async function explainGateFromVault(gateName, runId) {
+    const vaultDir = join(process.cwd(), '.qa360');
+    if (!existsSync(vaultDir)) {
+        console.log(chalk.yellow('\n[!] Vault not found in current directory'));
+        return;
+    }
+    try {
+        const vault = new EvidenceVault({ baseDir: vaultDir });
+        // Get gate results from the run
+        const runInfo = await vault.getRun(runId);
+        if (!runInfo) {
+            console.log(chalk.yellow(`\n[!] Run ${runId} not found in vault`));
+            await vault.close();
+            return;
+        }
+        const gateResults = await vault.getGates(runId);
+        const gateResult = gateResults.find((g) => g.name === gateName);
+        if (gateResult) {
+            console.log(chalk.bold('\nRun Results:'));
+            console.log(chalk.gray(`  Status: ${gateResult.status}`));
+            console.log(chalk.gray(`  Duration: ${gateResult.duration_ms}ms`));
+            if (gateResult.metrics_json) {
+                const metrics = JSON.parse(gateResult.metrics_json);
+                if (metrics.total || metrics.passed || metrics.failed) {
+                    console.log(chalk.gray(`  Tests: ${metrics.passed || 0}/${metrics.total || 0} passed`));
+                }
+            }
+            // Get findings for this gate
+            const findings = await vault.getFindings(runId);
+            const gateFindings = findings.filter((f) => f.gate === gateName);
+            if (gateFindings.length > 0) {
+                console.log(chalk.yellow('\nFindings:'));
+                gateFindings.slice(0, 10).forEach((f) => {
+                    console.log(chalk.gray(`  - [${f.severity}] ${f.rule}: ${f.message}`));
+                });
+                if (gateFindings.length > 10) {
+                    console.log(chalk.gray(`  ... and ${gateFindings.length - 10} more`));
+                }
+            }
+        }
+        await vault.close();
+    }
+    catch (error) {
+        console.log(chalk.yellow(`\n[!] Could not load vault data: ${error.message}`));
+    }
+}
+/**
+ * AI-powered explanation
+ */
+async function explainWithAI(query, provider, runId, json = false) {
+    const spinner = ora('Connecting to AI provider...').start();
+    try {
+        // Get the AI provider
+        const aiProvider = provider
+            ? await createLLMProvider({ preferred: provider })
+            : await createBest();
+        if (!aiProvider) {
+            spinner.fail('No AI provider available');
+            console.log(chalk.yellow('\nSet up an AI provider:'));
+            console.log(chalk.gray('  qa360 ai list'));
+            console.log(chalk.gray('  export DEEPSEEK_API_KEY=your_key\n'));
+            return;
+        }
+        spinner.text = `Using ${aiProvider.getProviderType()}...`;
+        // Build context from vault if available
+        let context = '';
+        if (runId) {
+            const vaultDir = join(process.cwd(), '.qa360');
+            if (existsSync(vaultDir)) {
+                try {
+                    const vault = new EvidenceVault({ baseDir: vaultDir });
+                    const runInfo = await vault.getRun(runId);
+                    if (runInfo) {
+                        const duration = runInfo.ended_at && runInfo.started_at
+                            ? runInfo.ended_at - runInfo.started_at
+                            : undefined;
+                        context = `\n\nContext from run ${runId}:
+- Status: ${runInfo.status}
+- Trust Score: ${runInfo.trust_score ?? 'N/A'}
+- Pack: ${runInfo.pack_path || 'unknown'}
+- Duration: ${duration ? `${duration}ms` : 'N/A'}
+- Started: ${new Date(runInfo.started_at).toISOString()}`;
+                    }
+                    await vault.close();
+                }
+                catch {
+                    // Vault access failed, continue without context
+                }
+            }
+        }
+        // Build the explanation prompt
+        const systemPrompt = `You are QA360 Explain, an expert QA assistant.
+Explain test failures, errors, and quality issues in clear, actionable terms.
+
+Structure your response:
+1. **Summary**: What happened in one sentence
+2. **Root Cause**: Why it happened (technical explanation)
+3. **Impact**: What this means for the project
+4. **Solutions**: Specific steps to fix it (numbered list)
+5. **Prevention**: How to avoid this in the future
+
+Be concise but thorough. Use simple language but accurate technical terms.`;
+        const userPrompt = `Explain this QA issue:${context}\n\n${query}`;
+        const response = await aiProvider.generate({
+            prompt: userPrompt,
+            systemPrompt,
+            maxTokens: 2048,
+            temperature: 0.3
+        });
+        spinner.succeed(`Explanation using ${chalk.cyan(aiProvider.getProviderType())}\n`);
+        if (json) {
+            console.log(JSON.stringify({
+                provider: aiProvider.getProviderType(),
+                query,
+                explanation: response.content,
+                usage: response.usage
+            }, null, 2));
+        }
+        else {
+            console.log(chalk.bold('Explanation:\n'));
+            console.log(chalk.gray('-'.repeat(70)));
+            console.log(response.content);
+            console.log(chalk.gray('-'.repeat(70)));
+            console.log(chalk.gray(`\nTokens: ${response.usage.totalTokens}`));
+            console.log(chalk.gray(`  Prompt: ${response.usage.promptTokens}`));
+            console.log(chalk.gray(`  Completion: ${response.usage.completionTokens}\n`));
+        }
+    }
+    catch (error) {
+        spinner.fail('AI explanation failed');
+        console.log(chalk.red(`Error: ${error.message}`));
+        // Fallback to static explanation
+        console.log(chalk.yellow('\nYou can also try static explanations:'));
+        console.log(chalk.gray('  qa360 explain QX001    # List error codes'));
+        console.log(chalk.gray('  qa360 explain --gate api_smoke\n'));
+    }
+}
+/**
+ * Explain a run result
+ */
+async function explainRun(runId, options) {
+    const vaultDir = join(process.cwd(), '.qa360');
+    if (!existsSync(vaultDir)) {
+        console.log(chalk.red('\n[X] Vault not found in current directory'));
+        console.log(chalk.yellow('Run "qa360 run" first to generate test results.\n'));
+        return;
+    }
+    const spinner = ora('Loading run data...').start();
+    try {
+        const vault = new EvidenceVault({ baseDir: vaultDir });
+        const runInfo = await vault.getRun(runId);
+        if (!runInfo) {
+            spinner.fail(`Run ${runId} not found`);
+            console.log(chalk.yellow('\nAvailable runs:'));
+            const runs = await vault.listRuns({ limit: 20 });
+            runs.forEach((r) => console.log(chalk.gray(`  ${r.id} - ${r.status}`)));
+            await vault.close();
+            return;
+        }
+        spinner.succeed('Run loaded\n');
+        if (options.json) {
+            console.log(JSON.stringify(runInfo, null, 2));
+            await vault.close();
+            return;
+        }
+        // Calculate duration
+        const duration = runInfo.ended_at && runInfo.started_at
+            ? runInfo.ended_at - runInfo.started_at
+            : undefined;
+        // Display run summary
+        console.log(chalk.bold(`Run: ${runId}`));
+        console.log(chalk.gray(`   Status: ${runInfo.status}`));
+        console.log(chalk.gray(`   Trust Score: ${runInfo.trust_score ?? 'N/A'}/100`));
+        console.log(chalk.gray(`   Started: ${new Date(runInfo.started_at).toISOString()}`));
+        if (duration) {
+            console.log(chalk.gray(`   Duration: ${duration}ms\n`));
+        }
+        else {
+            console.log('');
+        }
+        // Get gate results
+        const gates = await vault.getGates(runId);
+        if (gates.length > 0) {
+            console.log(chalk.bold('Gates:'));
+            gates.forEach((g) => {
+                const icon = g.status === 'passed' ? '[OK]' : g.status === 'failed' ? '[FAIL]' : '[SKIP]';
+                const color = g.status === 'passed' ? chalk.green : g.status === 'failed' ? chalk.red : chalk.gray;
+                console.log(`  ${icon} ${color(g.name)} (${g.duration_ms || 'N/A'}ms)`);
+            });
+        }
+        // Get findings
+        const findings = await vault.getFindings(runId);
+        if (findings.length > 0) {
+            console.log(chalk.bold('\nFindings:'));
+            const bySeverity = {};
+            findings.forEach((f) => {
+                if (!bySeverity[f.severity])
+                    bySeverity[f.severity] = [];
+                bySeverity[f.severity].push(f);
+            });
+            ['critical', 'high', 'medium', 'low', 'info'].forEach(sev => {
+                if (bySeverity[sev]?.length) {
+                    console.log(chalk.red(`  ${sev.toUpperCase()}: ${bySeverity[sev].length}`));
+                    bySeverity[sev].slice(0, 5).forEach((f) => {
+                        const msg = f.message.length > 60 ? f.message.substring(0, 60) + '...' : f.message;
+                        console.log(chalk.gray(`    - ${f.rule}: ${msg}`));
+                    });
+                }
+            });
+        }
+        // Get flakiness data (may fail if vault doesn't have flakiness tables)
+        try {
+            const flakiness = await vault.getRunFlakiness(runId);
+            if (flakiness.length > 0) {
+                console.log(chalk.bold('\nFlakiness:'));
+                const unstable = flakiness.filter((f) => f.score < 75);
+                if (unstable.length > 0) {
+                    console.log(chalk.yellow(`  ${unstable.length} unstable tests detected`));
+                    unstable.forEach((f) => {
+                        console.log(chalk.gray(`    - ${f.test_name}: score ${f.score}`));
+                    });
+                }
+                else {
+                    console.log(chalk.green('  All tests are stable!'));
+                }
+            }
+        }
+        catch {
+            // Flakiness tables not available in this vault version
+        }
+        await vault.close();
+        // Offer AI explanation if requested
+        if (options.ai) {
+            console.log(chalk.cyan('\nGetting AI explanation...\n'));
+            await explainWithAI(`Explain this test run: ${runId} with status ${runInfo.status} and ${gates.length} gates`, options.provider, runId, options.json);
+        }
+    }
+    catch (error) {
+        spinner.fail('Failed to load run data');
+        console.log(chalk.red(`Error: ${error.message}`));
+    }
+}
+/**
+ * Main explain command
+ */
+export async function explainCommand(input, options = {}) {
+    // No input - show help
+    if (!input && !options.run && !options.gate) {
+        console.log(chalk.bold.cyan('\nQA360 Explain - Error and Failure Explanations\n'));
+        console.log(chalk.yellow('Usage:\n'));
+        console.log(chalk.gray('  qa360 explain QX001              # Explain error code'));
+        console.log(chalk.gray('  qa360 explain --ai "query"        # AI-powered explanation'));
+        console.log(chalk.gray('  qa360 explain --run run-xxx      # Explain test run'));
+        console.log(chalk.gray('  qa360 explain --gate api_smoke   # Explain gate failures\n'));
+        console.log(chalk.yellow('Available Error Codes:\n'));
+        const grouped = groupErrorCodes();
+        for (const [category, codes] of Object.entries(grouped)) {
+            console.log(chalk.cyan(`  ${category.toUpperCase()}:`));
+            codes.slice(0, 5).forEach(code => {
+                const error = ERROR_CODES[code];
+                console.log(chalk.gray(`    ${code} - ${error?.title}`));
+            });
+            if (codes.length > 5) {
+                console.log(chalk.gray(`    ... and ${codes.length - 5} more`));
+            }
+        }
+        console.log(chalk.yellow('\nAvailable Gates:\n'));
+        console.log(chalk.gray('  ' + Object.keys(GATE_EXPLANATIONS).join(', ') + '\n'));
+        return;
+    }
+    // Run mode
+    if (options.run) {
+        await explainRun(options.run, options);
+        return;
+    }
+    // Gate mode
+    if (options.gate) {
+        await explainGate(options.gate, options.run, options.json);
+        return;
+    }
+    // AI mode
+    if (options.ai) {
+        await explainWithAI(input || '', options.provider, options.run, options.json);
+        return;
+    }
+    // Error code mode (default)
+    if (input) {
+        // Check if it looks like an error code
+        if (/^Q[A-Z]\d{3}$/.test(input)) {
+            await explainErrorCode(input, options.json);
+        }
+        else {
+            // Treat as AI query
+            await explainWithAI(input, options.provider, options.run, options.json);
+        }
+    }
+}
+export default explainCommand;