qa360 2.2.1 → 2.2.13

package/cli/dist/commands/ai.js

@@ -0,0 +1,616 @@
+/**
+ * QA360 AI Command
+ *
+ * Manage AI provider integration for test generation.
+ * Supports DeepSeek, Ollama, OpenAI, Anthropic with automatic selection.
+ *
+ * DeepSeek is recommended for best value (GPT-4 level at 95% lower cost).
+ */
+import { Command } from 'commander';
+import chalk from 'chalk';
+import ora from 'ora';
+import Table from 'cli-table3';
+import { createBest, getProviderInfo, getProviderStatus, createLLMProvider, DeepSeekProvider, DeepSeekError, OllamaProvider, OllamaError, OpenAIProvider, OpenAIError, AnthropicProvider, AnthropicError } from '../core/index.js';
+/**
+ * List all available AI providers
+ */
+export async function aiListCommand() {
+    const spinner = ora('Checking AI providers...').start();
+    try {
+        const providers = await getProviderInfo();
+        spinner.succeed('AI Providers Status:\n');
+        // Create table for display
+        const table = new Table({
+            head: [chalk.bold('Provider'), chalk.bold('Status'), chalk.bold('Cost'), chalk.bold('Description')],
+            colWidths: [15, 10, 10, 40],
+            style: {
+                head: ['cyan', 'bold']
+            }
+        });
+        for (const provider of providers) {
+            const status = provider.available
+                ? chalk.green('✓ Available')
+                : chalk.gray('✗ Not set up');
+            // Get cost level display
+            let costLevel;
+            switch (provider.costLevel) {
+                case 'free':
+                    costLevel = chalk.green('FREE');
+                    break;
+                case 'low':
+                    costLevel = chalk.yellow('LOW');
+                    break;
+                case 'medium':
+                    costLevel = chalk.yellow('MED');
+                    break;
+                case 'high':
+                    costLevel = chalk.red('HIGH');
+                    break;
+                default:
+                    costLevel = provider.costLevel;
+            }
+            table.push([
+                chalk.bold(provider.name),
+                status,
+                costLevel,
+                provider.description
+            ]);
+        }
+        console.log(table.toString());
+        // Show setup instructions for unavailable providers
+        const unavailable = providers.filter((p) => !p.available && p.type !== 'mock');
+        if (unavailable.length > 0) {
+            console.log(chalk.yellow('\n[INFO] Setup available providers:\n'));
+            if (!providers.find((p) => p.type === 'deepseek')?.available) {
+                console.log(chalk.cyan('DeepSeek (Recommended - Best Value):'));
+                console.log(chalk.gray('  1. Get API key: https://platform.deepseek.com/api_keys'));
+                console.log(chalk.gray('  2. Run: qa360 secrets add DEEPSEEK_API_KEY\n'));
+            }
+            if (!providers.find((p) => p.type === 'ollama')?.available) {
+                console.log(chalk.cyan('Ollama (Free - Local):'));
+                console.log(chalk.gray('  1. Install: brew install ollama'));
+                console.log(chalk.gray('  2. Run: ollama serve'));
+                console.log(chalk.gray('  3. Pull model: ollama pull deepseek-coder\n'));
+            }
+            if (!providers.find((p) => p.type === 'openai')?.available) {
+                console.log(chalk.cyan('OpenAI:'));
+                console.log(chalk.gray('  1. Get API key: https://platform.openai.com/api-keys'));
+                console.log(chalk.gray('  2. Run: qa360 secrets add OPENAI_API_KEY\n'));
+            }
+            if (!providers.find((p) => p.type === 'anthropic')?.available) {
+                console.log(chalk.cyan('Anthropic:'));
+                console.log(chalk.gray('  1. Get API key: https://console.anthropic.com/'));
+                console.log(chalk.gray('  2. Run: qa360 secrets add ANTHROPIC_API_KEY\n'));
+            }
+        }
+        // Show best provider
+        const bestProvider = providers.find((p) => p.available && p.type !== 'mock');
+        if (bestProvider) {
+            console.log(chalk.green(`\n[STAR] Best available: ${chalk.bold(bestProvider.name)}`));
+            console.log(chalk.gray(`   ${bestProvider.description}`));
+        }
+        else {
+            console.log(chalk.red('\n[!] No AI providers available. Set up one above to use AI features.\n'));
+        }
+    }
+    catch (error) {
+        spinner.fail('Failed to check providers');
+        console.error(chalk.red('Error:'), error.message);
+    }
+}
+/**
+ * Use a specific AI provider
+ */
+export async function aiUseCommand(provider, options = {}) {
+    const spinner = ora(`Testing ${provider}...`).start();
+    try {
+        let available = false;
+        let error = '';
+        switch (provider.toLowerCase()) {
+            case 'deepseek': {
+                const p = new DeepSeekProvider();
+                available = await p.isAvailable();
+                if (!available)
+                    error = 'DEEPSEEK_API_KEY not set or invalid';
+                break;
+            }
+            case 'ollama': {
+                const p = new OllamaProvider();
+                available = await p.isAvailable();
+                if (!available)
+                    error = 'Ollama not running or not installed';
+                break;
+            }
+            case 'openai': {
+                const p = new OpenAIProvider();
+                available = await p.isAvailable();
+                if (!available)
+                    error = 'OPENAI_API_KEY not set or invalid';
+                break;
+            }
+            case 'anthropic': {
+                const p = new AnthropicProvider();
+                available = await p.isAvailable();
+                if (!available)
+                    error = 'ANTHROPIC_API_KEY not set or invalid';
+                break;
+            }
+            default:
+                spinner.fail(`Unknown provider: ${provider}`);
+                console.log(chalk.yellow('\nAvailable providers: deepseek, ollama, openai, anthropic'));
+                return;
+        }
+        if (available) {
+            spinner.succeed(`${provider} is available!`);
+            if (options.verify) {
+                await runVerificationTest(provider);
+            }
+            else {
+                console.log(chalk.green(`\n[OK] ${provider} is ready to use!\n`));
+                console.log(chalk.gray('Use --verify to run a test generation.\n'));
+            }
+        }
+        else {
+            spinner.fail(`${provider} is not available`);
+            console.log(chalk.red(`\n[X] ${error}\n`));
+            showSetupInstructions(provider);
+        }
+    }
+    catch (error) {
+        spinner.fail('Provider check failed');
+        console.error(chalk.red('Error:'), error.message);
+    }
+}
+/**
+ * Benchmark all available providers
+ */
+export async function aiBenchmarkCommand(options = {}) {
+    const spinner = ora('Checking providers...').start();
+    const testPrompt = options.prompt || 'Generate a simple API smoke test for https://api.example.com/health';
+    try {
+        const status = await getProviderStatus();
+        spinner.info('Starting benchmark...\n');
+        const results = [];
+        // Benchmark each provider
+        const providers = [
+            { key: 'deepseek', name: 'DeepSeek', factory: () => new DeepSeekProvider() },
+            { key: 'ollama', name: 'Ollama', factory: () => new OllamaProvider() },
+            { key: 'openai', name: 'OpenAI', factory: () => new OpenAIProvider() },
+            { key: 'anthropic', name: 'Anthropic', factory: () => new AnthropicProvider() },
+        ];
+        console.log(chalk.bold('Benchmark Results:\n'));
+        console.log(chalk.gray(`Prompt: "${testPrompt}"\n`));
+        const table = new Table({
+            head: [chalk.bold('Provider'), chalk.bold('Status'), chalk.bold('Time'), chalk.bold('Tokens')],
+            colWidths: [15, 15, 12, 15],
+            style: { head: ['cyan', 'bold'] }
+        });
+        for (const { key, name, factory } of providers) {
+            if (!status[key]) {
+                table.push([name, chalk.gray('Not available'), '-', '-']);
+                continue;
+            }
+            const pSpinner = ora(`Testing ${name}...`).start();
+            try {
+                const provider = factory();
+                const start = Date.now();
+                const response = await provider.generate({
+                    prompt: testPrompt,
+                    maxTokens: 500
+                });
+                const time = Date.now() - start;
+                pSpinner.succeed();
+                table.push([
+                    name,
+                    chalk.green('✓ Success'),
+                    chalk.cyan(`${time}ms`),
+                    chalk.gray(`${response.usage.totalTokens}`)
+                ]);
+            }
+            catch (error) {
+                pSpinner.fail();
+                table.push([
+                    name,
+                    chalk.red('✗ Failed'),
+                    '-',
+                    `-`
+                ]);
+            }
+        }
+        console.log(table.toString());
+        // Show cost comparison
+        console.log(chalk.yellow('\n[$] Cost Comparison (per 1M tokens):\n'));
+        console.log(chalk.gray('  DeepSeek:  $0.14 input,  $0.28 output  (~99.5% savings vs GPT-4)'));
+        console.log(chalk.gray('  Ollama:    FREE (local, privacy)'));
+        console.log(chalk.gray('  OpenAI:    ~$30 input,  ~$60 output'));
+        console.log(chalk.gray('  Anthropic: ~$30 input,  ~$60 output\n'));
+    }
+    catch (error) {
+        spinner.fail('Benchmark failed');
+        console.error(chalk.red('Error:'), error.message);
+    }
+}
+/**
+ * Generate tests using the best available provider
+ */
+export async function aiGenerateCommand(prompt, options = {}) {
+    let spinner;
+    try {
+        // Use pack generation if --pack flag is set
+        if (options.pack) {
+            await generatePackCommand(prompt, options);
+            return;
+        }
+        const systemPrompt = buildSystemPrompt(options.type || 'api');
+        const provider = options.provider
+            ? await createLLMProvider({ preferred: options.provider })
+            : await createBest();
+        const providerType = provider.getProviderType();
+        spinner = ora(`Using ${chalk.cyan(providerType)}...`).start();
+        const response = await provider.generate({
+            prompt,
+            systemPrompt,
+            maxTokens: 4096,
+            temperature: 0.7,
+        });
+        spinner.succeed(`Generated using ${chalk.cyan(providerType)}!\n`);
+        if (options.json) {
+            console.log(JSON.stringify({
+                provider: providerType,
+                content: response.content,
+                usage: response.usage,
+                model: response.model
+            }, null, 2));
+        }
+        else {
+            console.log(chalk.bold('Generated Content:\n'));
+            console.log(chalk.gray('─'.repeat(70)));
+            console.log(response.content);
+            console.log(chalk.gray('─'.repeat(70)));
+            console.log(chalk.gray(`\n[INFO] Tokens: ${response.usage.totalTokens} (${response.usage.promptTokens} in + ${response.usage.completionTokens} out)`));
+            // Show cost estimate for DeepSeek
+            if (providerType === 'deepseek') {
+                const deepSeek = new DeepSeekProvider();
+                const cost = deepSeek.estimateCost(response.usage.promptTokens, response.usage.completionTokens);
+                console.log(chalk.gray(`[$] Cost: $${cost.toFixed(6)} (vs ~$${(cost * 200).toFixed(4)} with GPT-4)`));
+            }
+        }
+    }
+    catch (error) {
+        spinner?.fail('Generation failed');
+        if (error instanceof DeepSeekError) {
+            console.error(chalk.red('\nDeepSeek Error:'), error.message);
+            if ('details' in error && error.details) {
+                console.error(chalk.gray(JSON.stringify(error.details, null, 2)));
+            }
+        }
+        else if (error instanceof OllamaError) {
+            console.error(chalk.red('\nOllama Error:'), error.message);
+        }
+        else if (error instanceof OpenAIError) {
+            console.error(chalk.red('\nOpenAI Error:'), error.message);
+        }
+        else if (error instanceof AnthropicError) {
+            console.error(chalk.red('\nAnthropic Error:'), error.message);
+        }
+        else {
+            const errMsg = error instanceof Error ? error.message : String(error);
+            console.error(chalk.red('Error:'), errMsg);
+        }
+    }
+}
+/**
+ * Generate a QA360 pack YAML using AI
+ */
+async function generatePackCommand(prompt, options = {}) {
+    const spinner = ora('Generating QA360 pack...').start();
+    try {
+        const systemPrompt = buildPackSystemPrompt();
+        const provider = options.provider
+            ? await createLLMProvider({ preferred: options.provider })
+            : await createBest();
+        const providerType = provider.getProviderType();
+        const enhancedPrompt = `Generate a QA360 v2 pack YAML for: ${prompt}
+
+Important: Return ONLY valid YAML starting with "version: 2". No code blocks, no explanations, just the YAML.`;
+        const response = await provider.generate({
+            prompt: enhancedPrompt,
+            systemPrompt,
+            maxTokens: 4096,
+            temperature: 0.3, // Lower temperature for more structured output
+        });
+        spinner.succeed(`Generated pack using ${chalk.cyan(providerType)}!\n`);
+        // Extract YAML from response (remove code blocks if present)
+        let yamlContent = response.content;
+        // Remove markdown code blocks
+        yamlContent = yamlContent.replace(/```yaml\n?/g, '').replace(/```\n?/g, '');
+        // Remove leading/trailing whitespace
+        yamlContent = yamlContent.trim();
+        // Determine output path
+        const outputPath = options.output || './qa360-pack.yml';
+        // Save the pack
+        const { writeFileSync } = await import('fs');
+        writeFileSync(outputPath, yamlContent, 'utf-8');
+        console.log(chalk.bold('Generated QA360 Pack:\n'));
+        console.log(chalk.gray('─'.repeat(70)));
+        console.log(yamlContent);
+        console.log(chalk.gray('─'.repeat(70)));
+        console.log(chalk.green(`\n✓ Pack saved to: ${outputPath}`));
+        console.log(chalk.gray(`\n[INFO] Tokens: ${response.usage.totalTokens} (${response.usage.promptTokens} in + ${response.usage.completionTokens} out)`));
+        console.log(chalk.gray(`[INFO] Provider: ${providerType} (${response.model})\n`));
+        // Show usage instructions
+        console.log(chalk.yellow('Next steps:'));
+        console.log(chalk.gray(`  1. Review the pack: cat ${outputPath}`));
+        console.log(chalk.gray(`  2. Run tests: qa360 run ${outputPath}`));
+        console.log(chalk.gray(`  3. Or validate: qa360 pack validate ${outputPath}\n`));
+    }
+    catch (error) {
+        spinner.fail('Pack generation failed');
+        console.error(chalk.red('Error:'), error instanceof Error ? error.message : String(error));
+    }
+}
+/**
+ * Show AI configuration
+ */
+export async function aiConfigCommand() {
+    console.log(chalk.cyan('QA360 AI Configuration:\n'));
+    const status = await getProviderStatus();
+    const configs = [
+        { provider: 'DeepSeek', envVar: 'DEEPSEEK_API_KEY', set: !!process.env.DEEPSEEK_API_KEY },
+        { provider: 'Ollama', envVar: 'OLLAMA_BASE_URL', set: !!process.env.OLLAMA_BASE_URL },
+        { provider: 'OpenAI', envVar: 'OPENAI_API_KEY', set: !!process.env.OPENAI_API_KEY },
+        { provider: 'Anthropic', envVar: 'ANTHROPIC_API_KEY', set: !!process.env.ANTHROPIC_API_KEY },
+    ];
+    const table = new Table({
+        head: [chalk.bold('Provider'), chalk.bold('Environment Variable'), chalk.bold('Status')],
+        colWidths: [15, 30, 15],
+        style: { head: ['cyan', 'bold'] }
+    });
+    for (const config of configs) {
+        const status = config.set
+            ? chalk.green('✓ Set')
+            : chalk.gray('✗ Not set');
+        table.push([config.provider, chalk.gray(config.envVar), status]);
+    }
+    console.log(table.toString());
+    console.log(chalk.yellow('\n[INFO] Set API keys using:'));
+    console.log(chalk.gray('  export VARIABLE=value'));
+    console.log(chalk.gray('  # OR'));
+    console.log(chalk.gray('  qa360 secrets add VARIABLE\n'));
+}
+/**
+ * Run verification test for a provider
+ */
+async function runVerificationTest(provider) {
+    const spinner = ora(`Running verification test with ${provider}...`).start();
+    try {
+        let testProvider;
+        switch (provider.toLowerCase()) {
+            case 'deepseek':
+                testProvider = new DeepSeekProvider();
+                break;
+            case 'ollama':
+                testProvider = new OllamaProvider();
+                break;
+            case 'openai':
+                testProvider = new OpenAIProvider();
+                break;
+            case 'anthropic':
+                testProvider = new AnthropicProvider();
+                break;
+            default:
+                throw new Error(`Unknown provider: ${provider}`);
+        }
+        const response = await testProvider.generate({
+            prompt: 'Say "OK" if you can read this.',
+            maxTokens: 10
+        });
+        spinner.succeed('Verification test passed!\n');
+        console.log(chalk.green('[OK] Provider is working correctly!'));
+        console.log(chalk.gray(`Response: ${response.content.trim()}\n`));
+    }
+    catch (error) {
+        spinner.fail('Verification test failed');
+        console.error(chalk.red('Error:'), error.message);
+    }
+}
+/**
+ * Show setup instructions for a provider
+ */
+function showSetupInstructions(provider) {
+    switch (provider.toLowerCase()) {
+        case 'deepseek':
+            console.log(chalk.yellow('[NOTE] To set up DeepSeek:\n'));
+            console.log(chalk.gray('  1. Get API key: https://platform.deepseek.com/api_keys'));
+            console.log(chalk.gray('  2. Run: export DEEPSEEK_API_KEY=your_key'));
+            console.log(chalk.gray('  3. Or: qa360 secrets add DEEPSEEK_API_KEY\n'));
+            break;
+        case 'ollama':
+            console.log(chalk.yellow('[NOTE] To set up Ollama:\n'));
+            console.log(chalk.gray('  1. Install: brew install ollama'));
+            console.log(chalk.gray('  2. Start: ollama serve'));
+            console.log(chalk.gray('  3. Pull: ollama pull deepseek-coder\n'));
+            break;
+        case 'openai':
+            console.log(chalk.yellow('[NOTE] To set up OpenAI:\n'));
+            console.log(chalk.gray('  1. Get API key: https://platform.openai.com/api-keys'));
+            console.log(chalk.gray('  2. Run: export OPENAI_API_KEY=your_key'));
+            console.log(chalk.gray('  3. Or: qa360 secrets add OPENAI_API_KEY\n'));
+            break;
+        case 'anthropic':
+            console.log(chalk.yellow('[NOTE] To set up Anthropic:\n'));
+            console.log(chalk.gray('  1. Get API key: https://console.anthropic.com/'));
+            console.log(chalk.gray('  2. Run: export ANTHROPIC_API_KEY=your_key'));
+            console.log(chalk.gray('  3. Or: qa360 secrets add ANTHROPIC_API_KEY\n'));
+            break;
+    }
+}
+/**
+ * Build system prompt for test generation
+ */
+function buildSystemPrompt(type) {
+    const basePrompt = `You are QA360, an expert QA test generator.
+Generate production-ready, well-structured test code following best practices.
+
+Rules:
+- Generate COMPLETE, working code (no "...", no comments saying "to be implemented")
+- Include proper imports and setup
+- Handle errors gracefully
+- Use modern async/await patterns
+- Add helpful comments for complex logic
+- Return ONLY the code, no explanations before or after`;
+    const typePrompts = {
+        api: `${basePrompt}
+
+For API tests using Playwright:
+- Use @playwright/test
+- Use test() and expect() from Playwright
+- Test REST API endpoints (GET, POST, PUT, DELETE)
+- Include proper assertions for status codes, headers, and response body
+- Handle both success and error cases`,
+        ui: `${basePrompt}
+
+For UI/E2E tests using Playwright:
+- Use @playwright/test
+- Use page.goto(), page.click(), page.fill(), etc.
+- Include explicit waits where needed (waitForLoadState, waitForSelector)
+- Add assertions for page content
+- Use data-testid attributes when possible`,
+        perf: `${basePrompt}
+
+For performance tests using K6:
+- Use k6/http module
+- Define thresholds for response times and error rates
+- Include realistic user scenarios
+- Use options for VUs and duration`,
+        a11y: `${basePrompt}
+
+For accessibility tests:
+- Use @axe-core/playwright
+- Test WCAG compliance
+- Check for common a11y issues`,
+    };
+    return typePrompts[type] || typePrompts.api;
+}
+/**
+ * Build system prompt for QA360 pack YAML generation
+ */
+function buildPackSystemPrompt() {
+    return `You are a QA360 pack YAML generator. Generate VALID QA360 v2 pack YAML files.
+
+QA360 Pack v2 Structure:
+\`\`\`yaml
+version: 2
+name: test-pack-name
+description: Test pack description
+gates:
+  gate-name:
+    adapter: playwright-api|playwright-ui|k6-perf
+    enabled: true
+    config:
+      baseUrl: "https://example.com"
+      # API adapter: use "smoke" array with "METHOD path -> status" format
+      smoke:
+        - "GET /api/health -> 200"
+        - "POST /api/users -> 201"
+      # UI adapter: use "scenario" array or "pages" array
+      scenario:
+        - action: navigate
+          url: /login
+        - action: fill
+          selector: "#email"
+          value: "test@example.com"
+        - action: click
+          selector: "button[type='submit']"
+    options:
+      timeout: 10000
+budgets:
+  a11y_min: 70
+  perf_p95_ms: 3000
+\`\`\`
+
+Rules:
+1. ALWAYS start with "version: 2"
+2. Use "adapter:" not "type:"
+3. For API tests: use adapter: playwright-api with smoke: array
+4. For UI tests: use adapter: playwright-ui with scenario: array or pages: array
+5. Smoke format: "METHOD path -> status" (e.g., "GET /users -> 200")
+6. Scenario actions: navigate, fill, click, waitForSelector, assertText, assertVisible
+7. Return ONLY the YAML, no markdown code blocks, no explanations
+8. Make sure YAML is valid and properly indented`;
+}
+/**
+ * Create AI commands
+ */
+export function createAICommands() {
+    const aiCmd = new Command('ai')
+        .description('Manage AI providers (DeepSeek, Ollama, OpenAI, Anthropic)');
+    aiCmd
+        .command('list')
+        .description('List all available AI providers and their status')
+        .action(async () => {
+        try {
+            await aiListCommand();
+        }
+        catch (error) {
+            console.error(chalk.red('Error listing providers:'), error.message);
+            process.exit(1);
+        }
+    });
+    aiCmd
+        .command('use <provider>')
+        .description('Test and use a specific AI provider')
+        .option('--verify', 'Run a verification test')
+        .action(async (provider, options) => {
+        try {
+            await aiUseCommand(provider, options);
+        }
+        catch (error) {
+            console.error(chalk.red('Error testing provider:'), error.message);
+            process.exit(1);
+        }
+    });
+    aiCmd
+        .command('benchmark')
+        .description('Benchmark all available AI providers')
+        .option('-p, --prompt <prompt>', 'Custom prompt for benchmark')
+        .action(async (options) => {
+        try {
+            await aiBenchmarkCommand(options);
+        }
+        catch (error) {
+            console.error(chalk.red('Error running benchmark:'), error.message);
+            process.exit(1);
+        }
+    });
+    aiCmd
+        .command('generate <prompt>')
+        .description('Generate tests using the best available AI provider')
+        .option('-P, --provider <provider>', 'Use specific provider (deepseek, ollama, openai, anthropic)')
+        .option('-t, --type <type>', 'Test type (api, ui, perf, a11y)', 'api')
+        .option('--json', 'Output as JSON')
+        .option('--pack', 'Generate a QA360 pack YAML (instead of test code)')
+        .option('-o, --output <path>', 'Output file path (for --pack: ./qa360-pack.yml)')
+        .action(async (prompt, options) => {
+        try {
+            await aiGenerateCommand(prompt, options);
+        }
+        catch (error) {
+            console.error(chalk.red('Error generating:'), error.message);
+            process.exit(1);
+        }
+    });
+    aiCmd
+        .command('config')
+        .description('Show AI configuration and environment variables')
+        .action(async () => {
+        try {
+            await aiConfigCommand();
+        }
+        catch (error) {
+            console.error(chalk.red('Error showing config:'), error.message);
+            process.exit(1);
+        }
+    });
+    return aiCmd;
+}