qa360 2.1.7 → 2.2.0

package/dist/commands/ai.js

@@ -1,616 +0,0 @@
-/**
- * QA360 AI Command
- *
- * Manage AI provider integration for test generation.
- * Supports DeepSeek, Ollama, OpenAI, Anthropic with automatic selection.
- *
- * DeepSeek is recommended for best value (GPT-4 level at 95% lower cost).
- */
-import { Command } from 'commander';
-import chalk from 'chalk';
-import ora from 'ora';
-import Table from 'cli-table3';
-import { createBest, getProviderInfo, getProviderStatus, createLLMProvider, DeepSeekProvider, DeepSeekError, OllamaProvider, OllamaError, OpenAIProvider, OpenAIError, AnthropicProvider, AnthropicError } from '../core/index.js';
-/**
- * List all available AI providers
- */
-export async function aiListCommand() {
-    const spinner = ora('Checking AI providers...').start();
-    try {
-        const providers = await getProviderInfo();
-        spinner.succeed('AI Providers Status:\n');
-        // Create table for display
-        const table = new Table({
-            head: [chalk.bold('Provider'), chalk.bold('Status'), chalk.bold('Cost'), chalk.bold('Description')],
-            colWidths: [15, 10, 10, 40],
-            style: {
-                head: ['cyan', 'bold']
-            }
-        });
-        for (const provider of providers) {
-            const status = provider.available
-                ? chalk.green('✓ Available')
-                : chalk.gray('✗ Not set up');
-            // Get cost level display
-            let costLevel;
-            switch (provider.costLevel) {
-                case 'free':
-                    costLevel = chalk.green('FREE');
-                    break;
-                case 'low':
-                    costLevel = chalk.yellow('LOW');
-                    break;
-                case 'medium':
-                    costLevel = chalk.yellow('MED');
-                    break;
-                case 'high':
-                    costLevel = chalk.red('HIGH');
-                    break;
-                default:
-                    costLevel = provider.costLevel;
-            }
-            table.push([
-                chalk.bold(provider.name),
-                status,
-                costLevel,
-                provider.description
-            ]);
-        }
-        console.log(table.toString());
-        // Show setup instructions for unavailable providers
-        const unavailable = providers.filter((p) => !p.available && p.type !== 'mock');
-        if (unavailable.length > 0) {
-            console.log(chalk.yellow('\n[INFO] Setup available providers:\n'));
-            if (!providers.find((p) => p.type === 'deepseek')?.available) {
-                console.log(chalk.cyan('DeepSeek (Recommended - Best Value):'));
-                console.log(chalk.gray('  1. Get API key: https://platform.deepseek.com/api_keys'));
-                console.log(chalk.gray('  2. Run: qa360 secrets add DEEPSEEK_API_KEY\n'));
-            }
-            if (!providers.find((p) => p.type === 'ollama')?.available) {
-                console.log(chalk.cyan('Ollama (Free - Local):'));
-                console.log(chalk.gray('  1. Install: brew install ollama'));
-                console.log(chalk.gray('  2. Run: ollama serve'));
-                console.log(chalk.gray('  3. Pull model: ollama pull deepseek-coder\n'));
-            }
-            if (!providers.find((p) => p.type === 'openai')?.available) {
-                console.log(chalk.cyan('OpenAI:'));
-                console.log(chalk.gray('  1. Get API key: https://platform.openai.com/api-keys'));
-                console.log(chalk.gray('  2. Run: qa360 secrets add OPENAI_API_KEY\n'));
-            }
-            if (!providers.find((p) => p.type === 'anthropic')?.available) {
-                console.log(chalk.cyan('Anthropic:'));
-                console.log(chalk.gray('  1. Get API key: https://console.anthropic.com/'));
-                console.log(chalk.gray('  2. Run: qa360 secrets add ANTHROPIC_API_KEY\n'));
-            }
-        }
-        // Show best provider
-        const bestProvider = providers.find((p) => p.available && p.type !== 'mock');
-        if (bestProvider) {
-            console.log(chalk.green(`\n[STAR] Best available: ${chalk.bold(bestProvider.name)}`));
-            console.log(chalk.gray(`   ${bestProvider.description}`));
-        }
-        else {
-            console.log(chalk.red('\n[!] No AI providers available. Set up one above to use AI features.\n'));
-        }
-    }
-    catch (error) {
-        spinner.fail('Failed to check providers');
-        console.error(chalk.red('Error:'), error.message);
-    }
-}
-/**
- * Use a specific AI provider
- */
-export async function aiUseCommand(provider, options = {}) {
-    const spinner = ora(`Testing ${provider}...`).start();
-    try {
-        let available = false;
-        let error = '';
-        switch (provider.toLowerCase()) {
-            case 'deepseek': {
-                const p = new DeepSeekProvider();
-                available = await p.isAvailable();
-                if (!available)
-                    error = 'DEEPSEEK_API_KEY not set or invalid';
-                break;
-            }
-            case 'ollama': {
-                const p = new OllamaProvider();
-                available = await p.isAvailable();
-                if (!available)
-                    error = 'Ollama not running or not installed';
-                break;
-            }
-            case 'openai': {
-                const p = new OpenAIProvider();
-                available = await p.isAvailable();
-                if (!available)
-                    error = 'OPENAI_API_KEY not set or invalid';
-                break;
-            }
-            case 'anthropic': {
-                const p = new AnthropicProvider();
-                available = await p.isAvailable();
-                if (!available)
-                    error = 'ANTHROPIC_API_KEY not set or invalid';
-                break;
-            }
-            default:
-                spinner.fail(`Unknown provider: ${provider}`);
-                console.log(chalk.yellow('\nAvailable providers: deepseek, ollama, openai, anthropic'));
-                return;
-        }
-        if (available) {
-            spinner.succeed(`${provider} is available!`);
-            if (options.verify) {
-                await runVerificationTest(provider);
-            }
-            else {
-                console.log(chalk.green(`\n[OK] ${provider} is ready to use!\n`));
-                console.log(chalk.gray('Use --verify to run a test generation.\n'));
-            }
-        }
-        else {
-            spinner.fail(`${provider} is not available`);
-            console.log(chalk.red(`\n[X] ${error}\n`));
-            showSetupInstructions(provider);
-        }
-    }
-    catch (error) {
-        spinner.fail('Provider check failed');
-        console.error(chalk.red('Error:'), error.message);
-    }
-}
-/**
- * Benchmark all available providers
- */
-export async function aiBenchmarkCommand(options = {}) {
-    const spinner = ora('Checking providers...').start();
-    const testPrompt = options.prompt || 'Generate a simple API smoke test for https://api.example.com/health';
-    try {
-        const status = await getProviderStatus();
-        spinner.info('Starting benchmark...\n');
-        const results = [];
-        // Benchmark each provider
-        const providers = [
-            { key: 'deepseek', name: 'DeepSeek', factory: () => new DeepSeekProvider() },
-            { key: 'ollama', name: 'Ollama', factory: () => new OllamaProvider() },
-            { key: 'openai', name: 'OpenAI', factory: () => new OpenAIProvider() },
-            { key: 'anthropic', name: 'Anthropic', factory: () => new AnthropicProvider() },
-        ];
-        console.log(chalk.bold('Benchmark Results:\n'));
-        console.log(chalk.gray(`Prompt: "${testPrompt}"\n`));
-        const table = new Table({
-            head: [chalk.bold('Provider'), chalk.bold('Status'), chalk.bold('Time'), chalk.bold('Tokens')],
-            colWidths: [15, 15, 12, 15],
-            style: { head: ['cyan', 'bold'] }
-        });
-        for (const { key, name, factory } of providers) {
-            if (!status[key]) {
-                table.push([name, chalk.gray('Not available'), '-', '-']);
-                continue;
-            }
-            const pSpinner = ora(`Testing ${name}...`).start();
-            try {
-                const provider = factory();
-                const start = Date.now();
-                const response = await provider.generate({
-                    prompt: testPrompt,
-                    maxTokens: 500
-                });
-                const time = Date.now() - start;
-                pSpinner.succeed();
-                table.push([
-                    name,
-                    chalk.green('✓ Success'),
-                    chalk.cyan(`${time}ms`),
-                    chalk.gray(`${response.usage.totalTokens}`)
-                ]);
-            }
-            catch (error) {
-                pSpinner.fail();
-                table.push([
-                    name,
-                    chalk.red('✗ Failed'),
-                    '-',
-                    `-`
-                ]);
-            }
-        }
-        console.log(table.toString());
-        // Show cost comparison
-        console.log(chalk.yellow('\n[$] Cost Comparison (per 1M tokens):\n'));
-        console.log(chalk.gray('  DeepSeek:  $0.14 input,  $0.28 output  (~99.5% savings vs GPT-4)'));
-        console.log(chalk.gray('  Ollama:    FREE (local, privacy)'));
-        console.log(chalk.gray('  OpenAI:    ~$30 input,  ~$60 output'));
-        console.log(chalk.gray('  Anthropic: ~$30 input,  ~$60 output\n'));
-    }
-    catch (error) {
-        spinner.fail('Benchmark failed');
-        console.error(chalk.red('Error:'), error.message);
-    }
-}
-/**
- * Generate tests using the best available provider
- */
-export async function aiGenerateCommand(prompt, options = {}) {
-    let spinner;
-    try {
-        // Use pack generation if --pack flag is set
-        if (options.pack) {
-            await generatePackCommand(prompt, options);
-            return;
-        }
-        const systemPrompt = buildSystemPrompt(options.type || 'api');
-        const provider = options.provider
-            ? await createLLMProvider({ preferred: options.provider })
-            : await createBest();
-        const providerType = provider.getProviderType();
-        spinner = ora(`Using ${chalk.cyan(providerType)}...`).start();
-        const response = await provider.generate({
-            prompt,
-            systemPrompt,
-            maxTokens: 4096,
-            temperature: 0.7,
-        });
-        spinner.succeed(`Generated using ${chalk.cyan(providerType)}!\n`);
-        if (options.json) {
-            console.log(JSON.stringify({
-                provider: providerType,
-                content: response.content,
-                usage: response.usage,
-                model: response.model
-            }, null, 2));
-        }
-        else {
-            console.log(chalk.bold('Generated Content:\n'));
-            console.log(chalk.gray('─'.repeat(70)));
-            console.log(response.content);
-            console.log(chalk.gray('─'.repeat(70)));
-            console.log(chalk.gray(`\n[INFO] Tokens: ${response.usage.totalTokens} (${response.usage.promptTokens} in + ${response.usage.completionTokens} out)`));
-            // Show cost estimate for DeepSeek
-            if (providerType === 'deepseek') {
-                const deepSeek = new DeepSeekProvider();
-                const cost = deepSeek.estimateCost(response.usage.promptTokens, response.usage.completionTokens);
-                console.log(chalk.gray(`[$] Cost: $${cost.toFixed(6)} (vs ~$${(cost * 200).toFixed(4)} with GPT-4)`));
-            }
-        }
-    }
-    catch (error) {
-        spinner?.fail('Generation failed');
-        if (error instanceof DeepSeekError) {
-            console.error(chalk.red('\nDeepSeek Error:'), error.message);
-            if ('details' in error && error.details) {
-                console.error(chalk.gray(JSON.stringify(error.details, null, 2)));
-            }
-        }
-        else if (error instanceof OllamaError) {
-            console.error(chalk.red('\nOllama Error:'), error.message);
-        }
-        else if (error instanceof OpenAIError) {
-            console.error(chalk.red('\nOpenAI Error:'), error.message);
-        }
-        else if (error instanceof AnthropicError) {
-            console.error(chalk.red('\nAnthropic Error:'), error.message);
-        }
-        else {
-            const errMsg = error instanceof Error ? error.message : String(error);
-            console.error(chalk.red('Error:'), errMsg);
-        }
-    }
-}
-/**
- * Generate a QA360 pack YAML using AI
- */
-async function generatePackCommand(prompt, options = {}) {
-    const spinner = ora('Generating QA360 pack...').start();
-    try {
-        const systemPrompt = buildPackSystemPrompt();
-        const provider = options.provider
-            ? await createLLMProvider({ preferred: options.provider })
-            : await createBest();
-        const providerType = provider.getProviderType();
-        const enhancedPrompt = `Generate a QA360 v2 pack YAML for: ${prompt}
-
-Important: Return ONLY valid YAML starting with "version: 2". No code blocks, no explanations, just the YAML.`;
-        const response = await provider.generate({
-            prompt: enhancedPrompt,
-            systemPrompt,
-            maxTokens: 4096,
-            temperature: 0.3, // Lower temperature for more structured output
-        });
-        spinner.succeed(`Generated pack using ${chalk.cyan(providerType)}!\n`);
-        // Extract YAML from response (remove code blocks if present)
-        let yamlContent = response.content;
-        // Remove markdown code blocks
-        yamlContent = yamlContent.replace(/```yaml\n?/g, '').replace(/```\n?/g, '');
-        // Remove leading/trailing whitespace
-        yamlContent = yamlContent.trim();
-        // Determine output path
-        const outputPath = options.output || './qa360-pack.yml';
-        // Save the pack
-        const { writeFileSync } = await import('fs');
-        writeFileSync(outputPath, yamlContent, 'utf-8');
-        console.log(chalk.bold('Generated QA360 Pack:\n'));
-        console.log(chalk.gray('─'.repeat(70)));
-        console.log(yamlContent);
-        console.log(chalk.gray('─'.repeat(70)));
-        console.log(chalk.green(`\n✓ Pack saved to: ${outputPath}`));
-        console.log(chalk.gray(`\n[INFO] Tokens: ${response.usage.totalTokens} (${response.usage.promptTokens} in + ${response.usage.completionTokens} out)`));
-        console.log(chalk.gray(`[INFO] Provider: ${providerType} (${response.model})\n`));
-        // Show usage instructions
-        console.log(chalk.yellow('Next steps:'));
-        console.log(chalk.gray(`  1. Review the pack: cat ${outputPath}`));
-        console.log(chalk.gray(`  2. Run tests: qa360 run ${outputPath}`));
-        console.log(chalk.gray(`  3. Or validate: qa360 pack validate ${outputPath}\n`));
-    }
-    catch (error) {
-        spinner.fail('Pack generation failed');
-        console.error(chalk.red('Error:'), error instanceof Error ? error.message : String(error));
-    }
-}
-/**
- * Show AI configuration
- */
-export async function aiConfigCommand() {
-    console.log(chalk.cyan('QA360 AI Configuration:\n'));
-    const status = await getProviderStatus();
-    const configs = [
-        { provider: 'DeepSeek', envVar: 'DEEPSEEK_API_KEY', set: !!process.env.DEEPSEEK_API_KEY },
-        { provider: 'Ollama', envVar: 'OLLAMA_BASE_URL', set: !!process.env.OLLAMA_BASE_URL },
-        { provider: 'OpenAI', envVar: 'OPENAI_API_KEY', set: !!process.env.OPENAI_API_KEY },
-        { provider: 'Anthropic', envVar: 'ANTHROPIC_API_KEY', set: !!process.env.ANTHROPIC_API_KEY },
-    ];
-    const table = new Table({
-        head: [chalk.bold('Provider'), chalk.bold('Environment Variable'), chalk.bold('Status')],
-        colWidths: [15, 30, 15],
-        style: { head: ['cyan', 'bold'] }
-    });
-    for (const config of configs) {
-        const status = config.set
-            ? chalk.green('✓ Set')
-            : chalk.gray('✗ Not set');
-        table.push([config.provider, chalk.gray(config.envVar), status]);
-    }
-    console.log(table.toString());
-    console.log(chalk.yellow('\n[INFO] Set API keys using:'));
-    console.log(chalk.gray('  export VARIABLE=value'));
-    console.log(chalk.gray('  # OR'));
-    console.log(chalk.gray('  qa360 secrets add VARIABLE\n'));
-}
-/**
- * Run verification test for a provider
- */
-async function runVerificationTest(provider) {
-    const spinner = ora(`Running verification test with ${provider}...`).start();
-    try {
-        let testProvider;
-        switch (provider.toLowerCase()) {
-            case 'deepseek':
-                testProvider = new DeepSeekProvider();
-                break;
-            case 'ollama':
-                testProvider = new OllamaProvider();
-                break;
-            case 'openai':
-                testProvider = new OpenAIProvider();
-                break;
-            case 'anthropic':
-                testProvider = new AnthropicProvider();
-                break;
-            default:
-                throw new Error(`Unknown provider: ${provider}`);
-        }
-        const response = await testProvider.generate({
-            prompt: 'Say "OK" if you can read this.',
-            maxTokens: 10
-        });
-        spinner.succeed('Verification test passed!\n');
-        console.log(chalk.green('[OK] Provider is working correctly!'));
-        console.log(chalk.gray(`Response: ${response.content.trim()}\n`));
-    }
-    catch (error) {
-        spinner.fail('Verification test failed');
-        console.error(chalk.red('Error:'), error.message);
-    }
-}
-/**
- * Show setup instructions for a provider
- */
-function showSetupInstructions(provider) {
-    switch (provider.toLowerCase()) {
-        case 'deepseek':
-            console.log(chalk.yellow('[NOTE] To set up DeepSeek:\n'));
-            console.log(chalk.gray('  1. Get API key: https://platform.deepseek.com/api_keys'));
-            console.log(chalk.gray('  2. Run: export DEEPSEEK_API_KEY=your_key'));
-            console.log(chalk.gray('  3. Or: qa360 secrets add DEEPSEEK_API_KEY\n'));
-            break;
-        case 'ollama':
-            console.log(chalk.yellow('[NOTE] To set up Ollama:\n'));
-            console.log(chalk.gray('  1. Install: brew install ollama'));
-            console.log(chalk.gray('  2. Start: ollama serve'));
-            console.log(chalk.gray('  3. Pull: ollama pull deepseek-coder\n'));
-            break;
-        case 'openai':
-            console.log(chalk.yellow('[NOTE] To set up OpenAI:\n'));
-            console.log(chalk.gray('  1. Get API key: https://platform.openai.com/api-keys'));
-            console.log(chalk.gray('  2. Run: export OPENAI_API_KEY=your_key'));
-            console.log(chalk.gray('  3. Or: qa360 secrets add OPENAI_API_KEY\n'));
-            break;
-        case 'anthropic':
-            console.log(chalk.yellow('[NOTE] To set up Anthropic:\n'));
-            console.log(chalk.gray('  1. Get API key: https://console.anthropic.com/'));
-            console.log(chalk.gray('  2. Run: export ANTHROPIC_API_KEY=your_key'));
-            console.log(chalk.gray('  3. Or: qa360 secrets add ANTHROPIC_API_KEY\n'));
-            break;
-    }
-}
-/**
- * Build system prompt for test generation
- */
-function buildSystemPrompt(type) {
-    const basePrompt = `You are QA360, an expert QA test generator.
-Generate production-ready, well-structured test code following best practices.
-
-Rules:
-- Generate COMPLETE, working code (no "...", no comments saying "to be implemented")
-- Include proper imports and setup
-- Handle errors gracefully
-- Use modern async/await patterns
-- Add helpful comments for complex logic
-- Return ONLY the code, no explanations before or after`;
-    const typePrompts = {
-        api: `${basePrompt}
-
-For API tests using Playwright:
-- Use @playwright/test
-- Use test() and expect() from Playwright
-- Test REST API endpoints (GET, POST, PUT, DELETE)
-- Include proper assertions for status codes, headers, and response body
-- Handle both success and error cases`,
-        ui: `${basePrompt}
-
-For UI/E2E tests using Playwright:
-- Use @playwright/test
-- Use page.goto(), page.click(), page.fill(), etc.
-- Include explicit waits where needed (waitForLoadState, waitForSelector)
-- Add assertions for page content
-- Use data-testid attributes when possible`,
-        perf: `${basePrompt}
-
-For performance tests using K6:
-- Use k6/http module
-- Define thresholds for response times and error rates
-- Include realistic user scenarios
-- Use options for VUs and duration`,
-        a11y: `${basePrompt}
-
-For accessibility tests:
-- Use @axe-core/playwright
-- Test WCAG compliance
-- Check for common a11y issues`,
-    };
-    return typePrompts[type] || typePrompts.api;
-}
-/**
- * Build system prompt for QA360 pack YAML generation
- */
-function buildPackSystemPrompt() {
-    return `You are a QA360 pack YAML generator. Generate VALID QA360 v2 pack YAML files.
-
-QA360 Pack v2 Structure:
-\`\`\`yaml
-version: 2
-name: test-pack-name
-description: Test pack description
-gates:
-  gate-name:
-    adapter: playwright-api|playwright-ui|k6-perf
-    enabled: true
-    config:
-      baseUrl: "https://example.com"
-      # API adapter: use "smoke" array with "METHOD path -> status" format
-      smoke:
-        - "GET /api/health -> 200"
-        - "POST /api/users -> 201"
-      # UI adapter: use "scenario" array or "pages" array
-      scenario:
-        - action: navigate
-          url: /login
-        - action: fill
-          selector: "#email"
-          value: "test@example.com"
-        - action: click
-          selector: "button[type='submit']"
-    options:
-      timeout: 10000
-budgets:
-  a11y_min: 70
-  perf_p95_ms: 3000
-\`\`\`
-
-Rules:
-1. ALWAYS start with "version: 2"
-2. Use "adapter:" not "type:"
-3. For API tests: use adapter: playwright-api with smoke: array
-4. For UI tests: use adapter: playwright-ui with scenario: array or pages: array
-5. Smoke format: "METHOD path -> status" (e.g., "GET /users -> 200")
-6. Scenario actions: navigate, fill, click, waitForSelector, assertText, assertVisible
-7. Return ONLY the YAML, no markdown code blocks, no explanations
-8. Make sure YAML is valid and properly indented`;
-}
-/**
- * Create AI commands
- */
-export function createAICommands() {
-    const aiCmd = new Command('ai')
-        .description('Manage AI providers (DeepSeek, Ollama, OpenAI, Anthropic)');
-    aiCmd
-        .command('list')
-        .description('List all available AI providers and their status')
-        .action(async () => {
-        try {
-            await aiListCommand();
-        }
-        catch (error) {
-            console.error(chalk.red('Error listing providers:'), error.message);
-            process.exit(1);
-        }
-    });
-    aiCmd
-        .command('use <provider>')
-        .description('Test and use a specific AI provider')
-        .option('--verify', 'Run a verification test')
-        .action(async (provider, options) => {
-        try {
-            await aiUseCommand(provider, options);
-        }
-        catch (error) {
-            console.error(chalk.red('Error testing provider:'), error.message);
-            process.exit(1);
-        }
-    });
-    aiCmd
-        .command('benchmark')
-        .description('Benchmark all available AI providers')
-        .option('-p, --prompt <prompt>', 'Custom prompt for benchmark')
-        .action(async (options) => {
-        try {
-            await aiBenchmarkCommand(options);
-        }
-        catch (error) {
-            console.error(chalk.red('Error running benchmark:'), error.message);
-            process.exit(1);
-        }
-    });
-    aiCmd
-        .command('generate <prompt>')
-        .description('Generate tests using the best available AI provider')
-        .option('-P, --provider <provider>', 'Use specific provider (deepseek, ollama, openai, anthropic)')
-        .option('-t, --type <type>', 'Test type (api, ui, perf, a11y)', 'api')
-        .option('--json', 'Output as JSON')
-        .option('--pack', 'Generate a QA360 pack YAML (instead of test code)')
-        .option('-o, --output <path>', 'Output file path (for --pack: ./qa360-pack.yml)')
-        .action(async (prompt, options) => {
-        try {
-            await aiGenerateCommand(prompt, options);
-        }
-        catch (error) {
-            console.error(chalk.red('Error generating:'), error.message);
-            process.exit(1);
-        }
-    });
-    aiCmd
-        .command('config')
-        .description('Show AI configuration and environment variables')
-        .action(async () => {
-        try {
-            await aiConfigCommand();
-        }
-        catch (error) {
-            console.error(chalk.red('Error showing config:'), error.message);
-            process.exit(1);
-        }
-    });
-    return aiCmd;
-}