qa360 2.1.2 → 2.1.4

package/scripts/stress-test.js

@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+
+// QA360 Cloud Stress Test Script
+// Tests performance with large Gherkin reports (1000+ steps)
+
+const crypto = require('crypto');
+const { performance } = require('perf_hooks');
+const { GherkinGenerator } = require('../lib/gherkin-generator');
+
+class StressTestRunner {
+  constructor() {
+    this.gherkinGenerator = new GherkinGenerator();
+    this.results = {
+      generation: [],
+      memory: [],
+      cpu: [],
+      errors: []
+    };
+  }
+
+  /**
+   * Generate large test dataset for stress testing
+   */
+  generateLargeTestDataset(pack, scenarioCount = 1000) {
+    const testResults = [];
+    const routes = [
+      'GET /pokemon/pikachu',
+      'GET /pokemon/charizard', 
+      'GET /pokemon/blastoise',
+      'GET /pokemon/venusaur',
+      'GET /type/electric',
+      'GET /type/fire',
+      'GET /type/water',
+      'GET /type/grass',
+      'GET /ability/static',
+      'GET /ability/blaze'
+    ];
+
+    for (let i = 0; i < scenarioCount; i++) {
+      const route = routes[i % routes.length];
+      const [method, path] = route.split(' ');
+      
+      testResults.push({
+        id: `test-${i}`,
+        title: `Pokemon API Test ${i + 1}`,
+        route: { method, path },
+        status: Math.random() > 0.1 ? 'passed' : 'failed',
+        duration: Math.floor(Math.random() * 500) + 50,
+        error: Math.random() > 0.9 ? `Simulated error ${i}` : undefined,
+        timestamp: new Date().toISOString()
+      });
+    }
+
+    return testResults;
+  }
+
+  /**
+   * Monitor memory usage
+   */
+  getMemoryUsage() {
+    const usage = process.memoryUsage();
+    return {
+      rss: Math.round(usage.rss / 1024 / 1024), // MB
+      heapUsed: Math.round(usage.heapUsed / 1024 / 1024), // MB
+      heapTotal: Math.round(usage.heapTotal / 1024 / 1024), // MB
+      external: Math.round(usage.external / 1024 / 1024) // MB
+    };
+  }
+
+  /**
+   * Monitor CPU usage (simplified)
+   */
+  getCPUUsage() {
+    const usage = process.cpuUsage();
+    return {
+      user: Math.round(usage.user / 1000), // ms
+      system: Math.round(usage.system / 1000) // ms
+    };
+  }
+
+  /**
+   * Run stress test with specified parameters
+   */
+  async runStressTest(pack, scenarioCount, iterations = 3) {
+    console.log(`\n🚀 Starting stress test for ${pack}`);
+    console.log(`📊 Scenarios: ${scenarioCount}, Iterations: ${iterations}\n`);
+
+    const testResults = this.generateLargeTestDataset(pack, scenarioCount);
+    
+    for (let i = 0; i < iterations; i++) {
+      console.log(`🔄 Iteration ${i + 1}/${iterations}`);
+      
+      const memoryBefore = this.getMemoryUsage();
+      const cpuBefore = this.getCPUUsage();
+      const startTime = performance.now();
+
+      try {
+        // Generate Gherkin report
+        const runId = crypto.randomUUID();
+        const gherkinReport = this.gherkinGenerator.generateReport(pack, testResults, runId);
+        
+        const endTime = performance.now();
+        const duration = endTime - startTime;
+        
+        const memoryAfter = this.getMemoryUsage();
+        const cpuAfter = this.getCPUUsage();
+
+        // Record results
+        this.results.generation.push({
+          iteration: i + 1,
+          duration: Math.round(duration),
+          scenarios: scenarioCount,
+          totalSteps: gherkinReport.summary.totalSteps,
+          reportSize: JSON.stringify(gherkinReport).length
+        });
+
+        this.results.memory.push({
+          iteration: i + 1,
+          before: memoryBefore,
+          after: memoryAfter,
+          delta: {
+            rss: memoryAfter.rss - memoryBefore.rss,
+            heapUsed: memoryAfter.heapUsed - memoryBefore.heapUsed
+          }
+        });
+
+        this.results.cpu.push({
+          iteration: i + 1,
+          before: cpuBefore,
+          after: cpuAfter,
+          delta: {
+            user: cpuAfter.user - cpuBefore.user,
+            system: cpuAfter.system - cpuBefore.system
+          }
+        });
+
+        console.log(`  ✅ Generated in ${Math.round(duration)}ms`);
+        console.log(`  📊 ${gherkinReport.summary.totalSteps} steps, ${Math.round(JSON.stringify(gherkinReport).length / 1024)}KB`);
+        console.log(`  💾 Memory: ${memoryAfter.heapUsed}MB (+${memoryAfter.heapUsed - memoryBefore.heapUsed}MB)`);
+
+        // Force garbage collection if available
+        if (global.gc) {
+          global.gc();
+        }
+
+        // Small delay between iterations
+        await new Promise(resolve => setTimeout(resolve, 100));
+
+      } catch (error) {
+        console.error(`  ❌ Error in iteration ${i + 1}:`, error.message);
+        this.results.errors.push({
+          iteration: i + 1,
+          error: error.message,
+          stack: error.stack
+        });
+      }
+    }
+  }
+
+  /**
+   * Analyze and display results
+   */
+  analyzeResults() {
+    console.log('\n📈 STRESS TEST RESULTS\n');
+
+    if (this.results.generation.length === 0) {
+      console.log('❌ No successful generations recorded');
+      return;
+    }
+
+    // Generation performance
+    const durations = this.results.generation.map(r => r.duration);
+    const avgDuration = durations.reduce((sum, d) => sum + d, 0) / durations.length;
+    const maxDuration = Math.max(...durations);
+    const minDuration = Math.min(...durations);
+
+    console.log('⚡ GENERATION PERFORMANCE:');
+    console.log(`  Average: ${Math.round(avgDuration)}ms`);
+    console.log(`  Min: ${minDuration}ms`);
+    console.log(`  Max: ${maxDuration}ms`);
+    console.log(`  Target: <1500ms ${avgDuration < 1500 ? '✅' : '❌'}`);
+
+    // Memory usage
+    const memoryDeltas = this.results.memory.map(r => r.delta.heapUsed);
+    const avgMemoryDelta = memoryDeltas.reduce((sum, d) => sum + d, 0) / memoryDeltas.length;
+    const maxMemoryUsed = Math.max(...this.results.memory.map(r => r.after.heapUsed));
+
+    console.log('\n💾 MEMORY USAGE:');
+    console.log(`  Peak usage: ${maxMemoryUsed}MB`);
+    console.log(`  Average delta: ${Math.round(avgMemoryDelta)}MB`);
+    console.log(`  Target: <300MB ${maxMemoryUsed < 300 ? '✅' : '❌'}`);
+
+    // Report sizes
+    const reportSizes = this.results.generation.map(r => r.reportSize);
+    const avgReportSize = reportSizes.reduce((sum, s) => sum + s, 0) / reportSizes.length;
+    const maxReportSize = Math.max(...reportSizes);
+
+    console.log('\n📄 REPORT SIZES:');
+    console.log(`  Average: ${Math.round(avgReportSize / 1024)}KB`);
+    console.log(`  Max: ${Math.round(maxReportSize / 1024)}KB`);
+    console.log(`  Target: <1.5MB ${maxReportSize < 1.5 * 1024 * 1024 ? '✅' : '❌'}`);
+
+    // Steps performance
+    const totalSteps = this.results.generation.map(r => r.totalSteps);
+    const avgSteps = totalSteps.reduce((sum, s) => sum + s, 0) / totalSteps.length;
+    const stepsPerMs = avgSteps / avgDuration;
+
+    console.log('\n🔢 STEPS PERFORMANCE:');
+    console.log(`  Average steps: ${Math.round(avgSteps)}`);
+    console.log(`  Steps/ms: ${stepsPerMs.toFixed(2)}`);
+
+    // Errors
+    if (this.results.errors.length > 0) {
+      console.log('\n❌ ERRORS:');
+      this.results.errors.forEach(error => {
+        console.log(`  Iteration ${error.iteration}: ${error.error}`);
+      });
+    }
+
+    // Overall assessment
+    console.log('\n🎯 OVERALL ASSESSMENT:');
+    const passed = avgDuration < 1500 && maxMemoryUsed < 300 && maxReportSize < 1.5 * 1024 * 1024;
+    console.log(`  Status: ${passed ? '✅ PASSED' : '❌ FAILED'}`);
+    
+    if (!passed) {
+      console.log('  Recommendations:');
+      if (avgDuration >= 1500) console.log('    - Optimize Gherkin generation algorithm');
+      if (maxMemoryUsed >= 300) console.log('    - Implement memory pooling or streaming');
+      if (maxReportSize >= 1.5 * 1024 * 1024) console.log('    - Add report compression or pagination');
+    }
+
+    return passed;
+  }
+
+  /**
+   * Export results to JSON
+   */
+  exportResults(filename = 'stress-test-results.json') {
+    const fs = require('fs');
+    const path = require('path');
+    
+    const exportData = {
+      timestamp: new Date().toISOString(),
+      summary: {
+        totalIterations: this.results.generation.length,
+        avgDuration: this.results.generation.reduce((sum, r) => sum + r.duration, 0) / this.results.generation.length,
+        maxMemory: Math.max(...this.results.memory.map(r => r.after.heapUsed)),
+        errorCount: this.results.errors.length
+      },
+      results: this.results
+    };
+
+    const filePath = path.join(__dirname, '..', filename);
+    fs.writeFileSync(filePath, JSON.stringify(exportData, null, 2));
+    console.log(`\n📁 Results exported to: ${filePath}`);
+  }
+}
+
+// CLI execution
+async function main() {
+  const args = process.argv.slice(2);
+  const pack = args[0] || 'realworld-api';
+  const scenarioCount = parseInt(args[1]) || 1000;
+  const iterations = parseInt(args[2]) || 3;
+
+  console.log('🧪 QA360 Cloud Stress Test');
+  console.log('==========================');
+
+  const stressTest = new StressTestRunner();
+  
+  try {
+    await stressTest.runStressTest(pack, scenarioCount, iterations);
+    const passed = stressTest.analyzeResults();
+    stressTest.exportResults();
+    
+    process.exit(passed ? 0 : 1);
+  } catch (error) {
+    console.error('\n💥 Stress test failed:', error);
+    process.exit(1);
+  }
+}
+
+// Run if called directly
+if (require.main === module) {
+  main().catch(console.error);
+}
+
+module.exports = { StressTestRunner };

package/scripts/validate-examples.mjs

@@ -0,0 +1,404 @@
+#!/usr/bin/env node
+/**
+ * QA360 Examples Validator
+ *
+ * Validates all example pack files to ensure they are syntactically correct
+ * and follow the Pack v2 schema.
+ *
+ * Usage:
+ *   node scripts/validate-examples.mjs
+ *   node scripts/validate-examples.mjs --check-exists
+ */
+
+import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
+import { resolve, join, dirname, relative } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const rootDir = resolve(__dirname, '..');
+
+// ANSI colors
+const colors = {
+  reset: '\x1b[0m',
+  red: '\x1b[31m',
+  green: '\x1b[32m',
+  yellow: '\x1b[33m',
+  blue: '\x1b[34m',
+  magenta: '\x1b[35m',
+  cyan: '\x1b[36m',
+  gray: '\x1b[90m'
+};
+
+function colorize(color, text) {
+  return `${colors[color]}${text}${colors.reset}`;
+}
+
+// Directories to skip (non-pack YAML files)
+const SKIP_DIRS = [
+  '.github',
+  '.gitlab-ci',
+  'ci',
+  'load-testing',  // K6/mcp files, not packs
+  'packs-conformance',  // Pack v1 format, validated separately
+];
+
+// Files that are definitely pack files (validate these)
+const KNOWN_PACK_FILES = [
+  'pack.yml',
+  'pack.yaml',
+];
+
+// Find all YAML files in a directory recursively
+function findYamlFiles(dir, results = []) {
+  if (!existsSync(dir)) return results;
+
+  const entries = readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+
+    // Skip certain directories
+    if (entry.isDirectory()) {
+      if (SKIP_DIRS.includes(entry.name) || entry.name.startsWith('.') || entry.name === 'node_modules') {
+        continue;
+      }
+      findYamlFiles(fullPath, results);
+    } else if (entry.isFile() && (entry.name.endsWith('.yml') || entry.name.endsWith('.yaml'))) {
+      // Include if it's a known pack file name or in a pack directory
+      const relativePath = relative(rootDir, fullPath);
+      const pathParts = relativePath.split('/');
+
+      // Include if it's named pack.yml/pack.yaml
+      if (KNOWN_PACK_FILES.includes(entry.name)) {
+        results.push(fullPath);
+      }
+      // Include if in a directory that contains "pack" in the name
+      else if (pathParts.some(p => p.includes('pack'))) {
+        results.push(fullPath);
+      }
+      // Include files in restful-api-dev, web-lite, httpbin, packs-business
+      else if (pathParts[0] === 'restful-api-dev' ||
+               pathParts[0] === 'web-lite' && entry.name === 'pack.yml' ||
+               pathParts[0] === 'httpbin' ||
+               pathParts[0] === 'packs-business') {
+        // Only include pack.yml in web-lite (services.yml is docker-compose)
+        if (pathParts[0] !== 'web-lite' || entry.name === 'pack.yml') {
+          results.push(fullPath);
+        }
+      }
+    }
+  }
+
+  return results;
+}
+
+// Basic YAML validation (check for common syntax errors)
+function validateYamlSyntax(content, filePath) {
+  const errors = [];
+  const warnings = [];
+  const lines = content.split('\n');
+
+  let inMultilineString = false;
+  let indentStack = [0];
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const lineNum = i + 1;
+    const trimmed = line.trimEnd();
+
+    // Skip empty lines and comments
+    if (!trimmed || trimmed.startsWith('#')) {
+      continue;
+    }
+
+    // Check for tabs (YAML doesn't allow tabs for indentation)
+    if (line.includes('\t')) {
+      errors.push({
+        line: lineNum,
+        message: 'Tab character found. Use spaces for indentation.',
+        code: 'SYNTAX_TAB'
+      });
+    }
+
+    // Check indentation consistency
+    if (trimmed && !line.startsWith(' ')) {
+      // Top level, reset stack
+      indentStack = [0];
+    } else if (trimmed) {
+      const indent = line.search(/\S/);
+
+      // Check for odd indentation
+      if (indent % 2 !== 0) {
+        warnings.push({
+          line: lineNum,
+          message: 'Odd indentation detected. Use 2-space increments.',
+          code: 'STYLE_INDENT'
+        });
+      }
+
+      // Check for too deep indentation
+      if (indent > 40) {
+        warnings.push({
+          line: lineNum,
+          message: 'Very deep indentation. Consider restructuring.',
+          code: 'STYLE_DEEP'
+        });
+      }
+    }
+
+    // Check for common YAML pitfalls
+    if (trimmed.startsWith('- ')) {
+      // List item
+      const valueIndent = line.indexOf('- ') + 2;
+      const afterDash = line.substring(valueIndent).trim();
+
+      // Check if the value after dash is a colon (nested mapping)
+      if (afterDash.endsWith(':')) {
+        // This is fine - it's a mapping in a list
+      }
+    }
+
+    // Check for trailing colon without space after
+    if (trimmed.endsWith(':') && !trimmed.endsWith(': ')) {
+      const nextLine = lines[i + 1];
+      if (nextLine && nextLine.trim() && !nextLine.trim().startsWith('#')) {
+        // Check if next line is indented more (nested block)
+        const nextIndent = nextLine.search(/\S/);
+        if (nextIndent <= line.search(/\S/)) {
+          warnings.push({
+            line: lineNum,
+            message: 'Mapping key without value. Use "key: value" format.',
+            code: 'SYNTAX_COLON'
+          });
+        }
+      }
+    }
+
+    // Check for unquoted special characters
+    if (trimmed.includes(':') && !trimmed.startsWith('-') && !trimmed.includes("'") && !trimmed.includes('"')) {
+      const colonIndex = trimmed.indexOf(':');
+      if (colonIndex > 0 && colonIndex < trimmed.length - 1) {
+        const afterColon = trimmed.substring(colonIndex + 1).trim();
+        if (afterColon && !afterColon.startsWith('#') && !afterColon.startsWith('|') && !afterColon.startsWith('>')) {
+          // This might be a time like "12:30" or URL like "http://", which is OK
+          const urlPattern = /^[a-z]+:\/\//;
+          if (!urlPattern.test(afterColon)) {
+            // Also check for HH:MM time format
+            const timePattern = /^\d{1,2}:\d{2}(:\d{2})?$/;
+            if (!timePattern.test(trimmed)) {
+              // Might be an issue, but could be valid YAML
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return { errors, warnings };
+}
+
+// Validate pack structure
+function validatePackStructure(content, filePath) {
+  const errors = [];
+  const warnings = [];
+  const lines = content.split('\n');
+
+  // Extract top-level keys
+  const topLevelKeys = new Set();
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed && !trimmed.startsWith('#')) {
+      const indent = line.search(/\S/);
+      if (indent === 0) {
+        const match = trimmed.match(/^([^:#]+):/);
+        if (match) {
+          topLevelKeys.add(match[1]);
+        }
+      }
+    }
+  }
+
+  // Check for required fields in pack files
+  if (filePath.includes('examples/') || filePath.includes('pack')) {
+    const requiredFields = ['version', 'name', 'gates'];
+    for (const field of requiredFields) {
+      if (!topLevelKeys.has(field)) {
+        errors.push({
+          line: 1,
+          message: `Missing required field: "${field}"`,
+          code: 'PACK_MISSING_FIELD',
+          suggestion: `Add ${field}: to your pack file`
+        });
+      }
+    }
+
+    // Check version field
+    if (topLevelKeys.has('version')) {
+      for (const line of lines) {
+        if (line.trim().startsWith('version:')) {
+          const version = line.trim().substring(8).trim().replace(/['"]/g, '');
+          // Accept simple version numbers (1, 2) or semantic versions (1.0.0, 2.1.3, etc.)
+          const validVersion = /^(1|2)(\.\d+)*$/.test(version);
+          if (!validVersion) {
+            warnings.push({
+              line: lines.indexOf(line) + 1,
+              message: `Unusual version value: ${version}. Expected 1.x or 2.x.`,
+              code: 'PACK_VERSION'
+            });
+          }
+          break;
+        }
+      }
+    }
+
+    // Check for known top-level keys (including extended keys)
+    const validKeys = ['version', 'name', 'description', 'gates', 'auth', 'targets', 'budgets', 'security', 'hooks', 'execution', 'metadata',
+                       // Extended keys for advanced packs
+                       'data', 'observability', 'environment', 'variables', 'templates'];
+    const invalidKeys = [...topLevelKeys].filter(k => !validKeys.includes(k));
+    if (invalidKeys.length > 0) {
+      warnings.push({
+        line: 1,
+        message: `Unrecognized top-level keys: ${invalidKeys.join(', ')}`,
+        code: 'PACK_UNKNOWN_KEYS'
+      });
+    }
+  }
+
+  return { errors, warnings };
+}
+
+// Validate a single YAML file
+function validateYamlFile(filePath) {
+  const relativePath = relative(rootDir, filePath);
+  const result = {
+    file: relativePath,
+    valid: true,
+    errors: [],
+    warnings: []
+  };
+
+  try {
+    const content = readFileSync(filePath, 'utf-8');
+
+    // Syntax validation
+    const syntaxResult = validateYamlSyntax(content, filePath);
+    result.errors.push(...syntaxResult.errors);
+    result.warnings.push(...syntaxResult.warnings);
+
+    // Structure validation
+    const structureResult = validatePackStructure(content, filePath);
+    result.errors.push(...structureResult.errors);
+    result.warnings.push(...structureResult.warnings);
+
+    // Check for empty file
+    if (!content.trim()) {
+      result.errors.push({
+        line: 1,
+        message: 'Empty file',
+        code: 'EMPTY_FILE'
+      });
+    }
+
+    result.valid = result.errors.length === 0;
+  } catch (error) {
+    result.valid = false;
+    result.errors.push({
+      line: 0,
+      message: `Failed to read file: ${error.message}`,
+      code: 'READ_ERROR'
+    });
+  }
+
+  return result;
+}
+
+// Main validation function
+async function validateExamples(options = {}) {
+  const examplesDir = resolve(rootDir, 'examples');
+
+  if (!existsSync(examplesDir)) {
+    console.error(colorize('red', `Error: Examples directory not found: ${examplesDir}`));
+    process.exit(1);
+  }
+
+  console.log(colorize('cyan', '🔍 QA360 Examples Validator'));
+  console.log(colorize('gray', '━'.repeat(50)));
+  console.log(`Scanning: ${examplesDir}\n`);
+
+  // Find all YAML files
+  const yamlFiles = findYamlFiles(examplesDir);
+
+  if (yamlFiles.length === 0) {
+    console.log(colorize('yellow', 'No YAML files found in examples directory.'));
+    return;
+  }
+
+  console.log(colorize('blue', `Found ${yamlFiles.length} YAML file(s) to validate\n`));
+
+  // Validate each file
+  const results = [];
+  let totalErrors = 0;
+  let totalWarnings = 0;
+  let validCount = 0;
+
+  for (const filePath of yamlFiles) {
+    const result = validateYamlFile(filePath);
+    results.push(result);
+
+    if (result.valid) {
+      validCount++;
+      console.log(colorize('green', `✓ ${result.file}`));
+    } else {
+      console.log(colorize('red', `✗ ${result.file}`));
+    }
+
+    // Print errors and warnings
+    if (result.errors.length > 0 || result.warnings.length > 0) {
+      for (const error of result.errors) {
+        console.log(colorize('red', `  [${error.code}] Line ${error.line}: ${error.message}`));
+        if (error.suggestion) {
+          console.log(colorize('gray', `    → ${error.suggestion}`));
+        }
+      }
+      for (const warning of result.warnings) {
+        console.log(colorize('yellow', `  [${warning.code}] Line ${warning.line}: ${warning.message}`));
+        if (warning.suggestion) {
+          console.log(colorize('gray', `    → ${warning.suggestion}`));
+        }
+      }
+    }
+
+    totalErrors += result.errors.length;
+    totalWarnings += result.warnings.length;
+  }
+
+  // Print summary
+  console.log('\n' + colorize('gray', '━'.repeat(50)));
+  console.log(colorize('cyan', 'Summary:'));
+  console.log(`  Total files: ${yamlFiles.length}`);
+  console.log(`  ${colorize('green', `Valid: ${validCount}`)}`);
+  console.log(`  ${colorize('red', `Invalid: ${yamlFiles.length - validCount}`)}`);
+  console.log(`  ${colorize('red', `Errors: ${totalErrors}`)}`);
+  console.log(`  ${colorize('yellow', `Warnings: ${totalWarnings}`)}`);
+
+  // Exit with error code if validation failed
+  if (totalErrors > 0) {
+    process.exit(1);
+  }
+}
+
+// Parse CLI arguments
+const args = process.argv.slice(2);
+const options = {};
+for (const arg of args) {
+  if (arg === '--check-exists') {
+    options.checkExists = true;
+  }
+}
+
+validateExamples(options).catch(error => {
+  console.error(colorize('red', `Error: ${error.message}`));
+  process.exit(1);
+});