qa360 2.1.7 → 2.2.0

package/scripts/ci/slo-assert.js

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+/**
+ * QA360 Phase 7 - SLO Assertion Script
+ * Validates Artillery load test results against SLO thresholds
+ * 
+ * Usage:
+ *   node scripts/ci/slo-assert.js <results.json> --p95-ms 150 --err-rate 0.01
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+// Parse CLI arguments
+const args = process.argv.slice(2);
+const resultsFile = args[0];
+const p95Threshold = parseFloat(args[args.indexOf('--p95-ms') + 1] || 150);
+const errorRateThreshold = parseFloat(args[args.indexOf('--err-rate') + 1] || 0.01);
+
+if (!resultsFile || !fs.existsSync(resultsFile)) {
+  console.error('❌ Error: Results file not found');
+  console.error(`Usage: node slo-assert.js <results.json> --p95-ms 150 --err-rate 0.01`);
+  process.exit(1);
+}
+
+// Load Artillery results
+const results = JSON.parse(fs.readFileSync(resultsFile, 'utf8'));
+
+// Extract metrics
+const summary = results.aggregate || {};
+const latency = summary.latency || {};
+const counters = summary.counters || {};
+
+const p95 = latency.p95 || 0;
+const p99 = latency.p99 || 0;
+const median = latency.median || 0;
+const totalRequests = counters['http.requests'] || counters['vusers.created'] || 0;
+const errors = counters['errors.total'] || counters['http.codes.500'] || 0;
+const errorRate = totalRequests > 0 ? errors / totalRequests : 0;
+
+// Calculate success rate
+const successRate = totalRequests > 0 ? ((totalRequests - errors) / totalRequests) * 100 : 0;
+
+// Display results
+console.log('\n🔍 SLO Validation Results\n');
+console.log('═══════════════════════════════════════════════════════');
+console.log(`📊 Latency Metrics:`);
+console.log(`   • P50 (Median): ${median.toFixed(2)}ms`);
+console.log(`   • P95: ${p95.toFixed(2)}ms (threshold: ${p95Threshold}ms)`);
+console.log(`   • P99: ${p99.toFixed(2)}ms`);
+console.log('');
+console.log(`📈 Request Metrics:`);
+console.log(`   • Total Requests: ${totalRequests}`);
+console.log(`   • Errors: ${errors}`);
+console.log(`   • Error Rate: ${(errorRate * 100).toFixed(2)}% (threshold: ${(errorRateThreshold * 100).toFixed(2)}%)`);
+console.log(`   • Success Rate: ${successRate.toFixed(2)}%`);
+console.log('═══════════════════════════════════════════════════════\n');
+
+// Validate SLO thresholds
+const sloResults = {
+  timestamp: new Date().toISOString(),
+  thresholds: {
+    p95_ms: p95Threshold,
+    error_rate: errorRateThreshold
+  },
+  actual: {
+    p95_ms: p95,
+    p99_ms: p99,
+    median_ms: median,
+    error_rate: errorRate,
+    success_rate: successRate / 100,
+    total_requests: totalRequests,
+    errors: errors
+  },
+  checks: {
+    p95_ok: p95 <= p95Threshold,
+    error_rate_ok: errorRate <= errorRateThreshold,
+    success_rate_ok: successRate >= 99.0
+  }
+};
+
+// Determine GO/NO-GO
+const allChecksPassed = Object.values(sloResults.checks).every(check => check === true);
+
+if (allChecksPassed) {
+  console.log('✅ GO - All SLO thresholds met\n');
+  sloResults.decision = 'GO';
+  sloResults.status = 'passed';
+} else {
+  console.log('❌ NO-GO - SLO thresholds violated\n');
+  sloResults.decision = 'NO-GO';
+  sloResults.status = 'failed';
+  
+  // List violations
+  console.log('Violations:');
+  if (!sloResults.checks.p95_ok) {
+    console.log(`  • P95 latency: ${p95.toFixed(2)}ms > ${p95Threshold}ms`);
+  }
+  if (!sloResults.checks.error_rate_ok) {
+    console.log(`  • Error rate: ${(errorRate * 100).toFixed(2)}% > ${(errorRateThreshold * 100).toFixed(2)}%`);
+  }
+  if (!sloResults.checks.success_rate_ok) {
+    console.log(`  • Success rate: ${successRate.toFixed(2)}% < 99.0%`);
+  }
+  console.log('');
+}
+
+// Generate performance summary
+const summaryPath = path.join(path.dirname(resultsFile), 'perf-summary.md');
+const summaryContent = `# QA360 Phase 7 - Performance Summary
+
+**Date**: ${new Date().toISOString()}  
+**Test**: Artillery Load Testing  
+**Decision**: ${sloResults.decision}
+
+## SLO Thresholds
+
+| Metric | Threshold | Actual | Status |
+|--------|-----------|--------|--------|
+| P95 Latency | ≤${p95Threshold}ms | ${p95.toFixed(2)}ms | ${sloResults.checks.p95_ok ? '✅' : '❌'} |
+| Error Rate | ≤${(errorRateThreshold * 100).toFixed(2)}% | ${(errorRate * 100).toFixed(2)}% | ${sloResults.checks.error_rate_ok ? '✅' : '❌'} |
+| Success Rate | ≥99.0% | ${successRate.toFixed(2)}% | ${sloResults.checks.success_rate_ok ? '✅' : '❌'} |
+
+## Latency Distribution
+
+- **P50 (Median)**: ${median.toFixed(2)}ms
+- **P95**: ${p95.toFixed(2)}ms
+- **P99**: ${p99.toFixed(2)}ms
+
+## Request Statistics
+
+- **Total Requests**: ${totalRequests}
+- **Successful**: ${totalRequests - errors}
+- **Errors**: ${errors}
+- **Error Rate**: ${(errorRate * 100).toFixed(2)}%
+
+## Decision
+
+${allChecksPassed ? 
+  '✅ **GO** - All SLO thresholds met. System performance validated.' :
+  '❌ **NO-GO** - SLO violations detected. Review performance issues before proceeding.'
+}
+
+---
+
+**Generated**: ${new Date().toISOString()}  
+**Source**: ${path.basename(resultsFile)}
+`;
+
+fs.writeFileSync(summaryPath, summaryContent);
+console.log(`📄 Performance summary: ${summaryPath}\n`);
+
+// Save detailed results
+const detailsPath = path.join(path.dirname(resultsFile), 'slo-results.json');
+fs.writeFileSync(detailsPath, JSON.stringify(sloResults, null, 2));
+console.log(`📊 Detailed results: ${detailsPath}\n`);
+
+// Exit with appropriate code
+process.exit(allChecksPassed ? 0 : 1);

package/scripts/ci/test-fault-tolerance.sh

@@ -0,0 +1,301 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# QA360 Fault Tolerance Test Suite
+# Tests: cancel, vault missing, invalid pack, SIGTERM handling
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+ARTIFACTS_DIR="$PROJECT_ROOT/artifacts/faults"
+TIMESTAMP=$(date -u +"%Y%m%d_%H%M%S")
+REPORT_FILE="$ARTIFACTS_DIR/faults-report-$TIMESTAMP.json"
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+mkdir -p "$ARTIFACTS_DIR"
+
+echo -e "${BLUE}QA360 Fault Tolerance Test Suite${NC}"
+echo "Timestamp: $TIMESTAMP"
+echo ""
+
+# Initialize report
+cat > "$REPORT_FILE" <<'EOF'
+{
+  "suite": "fault-tolerance",
+  "version": "1.0.0",
+  "tests": [],
+  "summary": {
+    "total": 0,
+    "passed": 0,
+    "failed": 0
+  }
+}
+EOF
+
+TOTAL_TESTS=0
+PASSED_TESTS=0
+FAILED_TESTS=0
+
+# Test 1: Cancel mid-run
+test_cancel_midrun() {
+  echo -e "${BLUE}Test 1: Cancel mid-run${NC}"
+  local status="fail"
+  local error=""
+  
+  # Start long-running pack in background
+  qa360 run --pack examples/packs-conformance/heavy-artifacts.yml > "$ARTIFACTS_DIR/cancel_run.log" 2>&1 &
+  local pid=$!
+  
+  sleep 2
+  
+  # Send SIGTERM
+  kill -TERM $pid 2>/dev/null || true
+  
+  # Wait for graceful shutdown
+  local waited=0
+  while kill -0 $pid 2>/dev/null && [ $waited -lt 10 ]; do
+    sleep 1
+    waited=$((waited + 1))
+  done
+  
+  if ! kill -0 $pid 2>/dev/null; then
+    # Check vault state
+    if grep -q "cancelled" "$ARTIFACTS_DIR/cancel_run.log" 2>/dev/null; then
+      status="pass"
+      echo -e "${GREEN}✓ Cancel handled gracefully${NC}"
+    else
+      error="Vault state not set to cancelled"
+      echo -e "${YELLOW}⚠ Vault state unclear${NC}"
+    fi
+  else
+    kill -9 $pid 2>/dev/null || true
+    error="Process did not terminate gracefully"
+    echo -e "${RED}✗ Process hung${NC}"
+  fi
+  
+  echo "{\"name\":\"cancel_midrun\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
+  
+  if [ "$status" = "pass" ]; then
+    PASSED_TESTS=$((PASSED_TESTS + 1))
+  else
+    FAILED_TESTS=$((FAILED_TESTS + 1))
+  fi
+  TOTAL_TESTS=$((TOTAL_TESTS + 1))
+  echo ""
+}
+
+# Test 2: Vault missing
+test_vault_missing() {
+  echo -e "${BLUE}Test 2: Vault missing${NC}"
+  local status="fail"
+  local error=""
+  
+  # Temporarily rename .qa360 directory
+  if [ -d "$HOME/.qa360" ]; then
+    mv "$HOME/.qa360" "$HOME/.qa360.backup" 2>/dev/null || true
+  fi
+  
+  # Try to run
+  if qa360 run --pack examples/packs-conformance/minimal.yml > "$ARTIFACTS_DIR/vault_missing.log" 2>&1; then
+    error="Should have failed with VAULT_MISSING"
+    echo -e "${RED}✗ No error raised${NC}"
+  else
+    if grep -q "VAULT_MISSING" "$ARTIFACTS_DIR/vault_missing.log" 2>/dev/null; then
+      status="pass"
+      echo -e "${GREEN}✓ VAULT_MISSING code returned${NC}"
+    else
+      error="Wrong error code"
+      echo -e "${YELLOW}⚠ Different error code${NC}"
+    fi
+  fi
+  
+  # Restore .qa360
+  if [ -d "$HOME/.qa360.backup" ]; then
+    mv "$HOME/.qa360.backup" "$HOME/.qa360" 2>/dev/null || true
+  fi
+  
+  echo "{\"name\":\"vault_missing\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
+  
+  if [ "$status" = "pass" ]; then
+    PASSED_TESTS=$((PASSED_TESTS + 1))
+  else
+    FAILED_TESTS=$((FAILED_TESTS + 1))
+  fi
+  TOTAL_TESTS=$((TOTAL_TESTS + 1))
+  echo ""
+}
+
+# Test 3: Invalid pack schema
+test_invalid_pack() {
+  echo -e "${BLUE}Test 3: Invalid pack schema${NC}"
+  local status="fail"
+  local error=""
+  
+  # Create invalid pack
+  cat > "$ARTIFACTS_DIR/invalid.yml" <<'INVALID'
+name: invalid-pack
+version: 1.0.0
+steps:
+  - name: missing-adapter
+    config:
+      url: "https://example.com"
+INVALID
+  
+  # Try to validate
+  if qa360 pack validate "$ARTIFACTS_DIR/invalid.yml" > "$ARTIFACTS_DIR/invalid_pack.log" 2>&1; then
+    error="Should have failed with INVALID_PACK_SCHEMA"
+    echo -e "${RED}✗ Invalid pack accepted${NC}"
+  else
+    if grep -q "INVALID_PACK_SCHEMA\|adapter.*required\|missing.*adapter" "$ARTIFACTS_DIR/invalid_pack.log" 2>/dev/null; then
+      status="pass"
+      echo -e "${GREEN}✓ INVALID_PACK_SCHEMA detected${NC}"
+    else
+      error="Wrong error code"
+      echo -e "${YELLOW}⚠ Different error${NC}"
+    fi
+  fi
+  
+  rm -f "$ARTIFACTS_DIR/invalid.yml"
+  
+  echo "{\"name\":\"invalid_pack_schema\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
+  
+  if [ "$status" = "pass" ]; then
+    PASSED_TESTS=$((PASSED_TESTS + 1))
+  else
+    FAILED_TESTS=$((FAILED_TESTS + 1))
+  fi
+  TOTAL_TESTS=$((TOTAL_TESTS + 1))
+  echo ""
+}
+
+# Test 4: SIGTERM graceful exit
+test_sigterm_graceful() {
+  echo -e "${BLUE}Test 4: SIGTERM graceful exit${NC}"
+  local status="fail"
+  local error=""
+  
+  # Start qa360 serve
+  qa360 serve --port 4061 > "$ARTIFACTS_DIR/sigterm.log" 2>&1 &
+  local pid=$!
+  
+  sleep 2
+  
+  # Send SIGTERM
+  kill -TERM $pid
+  
+  # Wait for exit
+  local exit_code=0
+  wait $pid || exit_code=$?
+  
+  if [ $exit_code -eq 0 ] || [ $exit_code -eq 143 ]; then
+    # Check cleanup
+    if ! lsof -i :4061 >/dev/null 2>&1; then
+      status="pass"
+      echo -e "${GREEN}✓ Graceful exit with cleanup${NC}"
+    else
+      error="Port still in use"
+      echo -e "${YELLOW}⚠ Cleanup incomplete${NC}"
+    fi
+  else
+    error="Non-zero exit code: $exit_code"
+    echo -e "${RED}✗ Exit code $exit_code${NC}"
+  fi
+  
+  echo "{\"name\":\"sigterm_graceful\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
+  
+  if [ "$status" = "pass" ]; then
+    PASSED_TESTS=$((PASSED_TESTS + 1))
+  else
+    FAILED_TESTS=$((FAILED_TESTS + 1))
+  fi
+  TOTAL_TESTS=$((TOTAL_TESTS + 1))
+  echo ""
+}
+
+# Test 5: Cleanup on error
+test_cleanup_on_error() {
+  echo -e "${BLUE}Test 5: Cleanup on error${NC}"
+  local status="fail"
+  local error=""
+  
+  # Run pack that will fail
+  qa360 run --pack examples/packs-conformance/timeouts.yml > "$ARTIFACTS_DIR/cleanup.log" 2>&1 || true
+  
+  # Check for orphan processes
+  local orphans=$(ps aux | grep -E "playwright|chromium|k6" | grep -v grep | wc -l)
+  
+  if [ "$orphans" -eq 0 ]; then
+    status="pass"
+    echo -e "${GREEN}✓ No orphan processes${NC}"
+  else
+    error="Found $orphans orphan processes"
+    echo -e "${YELLOW}⚠ $orphans orphans detected${NC}"
+  fi
+  
+  echo "{\"name\":\"cleanup_on_error\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
+  
+  if [ "$status" = "pass" ]; then
+    PASSED_TESTS=$((PASSED_TESTS + 1))
+  else
+    FAILED_TESTS=$((FAILED_TESTS + 1))
+  fi
+  TOTAL_TESTS=$((TOTAL_TESTS + 1))
+  echo ""
+}
+
+# Run all tests
+touch "$ARTIFACTS_DIR/test_results.tmp"
+
+test_cancel_midrun
+test_vault_missing
+test_invalid_pack
+test_sigterm_graceful
+test_cleanup_on_error
+
+# Build final report
+{
+  echo "{"
+  echo "  \"suite\": \"fault-tolerance\","
+  echo "  \"version\": \"1.0.0\","
+  echo "  \"timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\","
+  echo "  \"tests\": ["
+  
+  first=true
+  while IFS= read -r line; do
+    if [ "$first" = true ]; then
+      first=false
+    else
+      echo ","
+    fi
+    echo "    $line"
+  done < "$ARTIFACTS_DIR/test_results.tmp"
+  
+  echo ""
+  echo "  ],"
+  echo "  \"summary\": {"
+  echo "    \"total\": $TOTAL_TESTS,"
+  echo "    \"passed\": $PASSED_TESTS,"
+  echo "    \"failed\": $FAILED_TESTS"
+  echo "  }"
+  echo "}"
+} > "$REPORT_FILE"
+
+rm -f "$ARTIFACTS_DIR/test_results.tmp"
+
+# Summary
+echo "========================================="
+echo "Total: $TOTAL_TESTS | Passed: $PASSED_TESTS | Failed: $FAILED_TESTS"
+echo "Report: $REPORT_FILE"
+
+if [ $FAILED_TESTS -eq 0 ]; then
+  echo -e "${GREEN}✓ All fault tolerance tests passed${NC}"
+  exit 0
+else
+  echo -e "${RED}✗ $FAILED_TESTS test(s) failed${NC}"
+  exit 1
+fi

package/scripts/install-mcp.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# QA360 MCP CLI Installation Script
+# Safe for Windows, macOS, and Linux
+
+set -e
+
+echo "🚀 Installing QA360 MCP CLI..."
+
+# Detect OS
+OS="$(uname -s)"
+case "$OS" in
+  Linux*)   PLATFORM="linux";;
+  Darwin*)  PLATFORM="macos";;
+  MINGW*|MSYS*|CYGWIN*) PLATFORM="windows";;
+  *)        PLATFORM="unknown";;
+esac
+
+echo "📦 Platform detected: $PLATFORM"
+
+# Check Node.js
+if ! command -v node &> /dev/null; then
+  echo "❌ Node.js not found. Please install Node.js >= 18.0.0"
+  exit 1
+fi
+
+NODE_VERSION=$(node -v | cut -d'v' -f2 | cut -d'.' -f1)
+if [ "$NODE_VERSION" -lt 18 ]; then
+  echo "❌ Node.js version $NODE_VERSION is too old. Please upgrade to >= 18.0.0"
+  exit 1
+fi
+
+echo "✅ Node.js $(node -v) detected"
+
+# Check pnpm
+if ! command -v pnpm &> /dev/null; then
+  echo "📦 Installing pnpm..."
+  npm install -g pnpm@8.15.8
+fi
+
+echo "✅ pnpm $(pnpm -v) ready"
+
+# Install dependencies
+echo "📦 Installing dependencies..."
+pnpm install --frozen-lockfile
+
+# Build
+echo "🔨 Building QA360 MCP CLI..."
+pnpm --filter @qa360/mcp run build
+
+# Link globally (optional)
+read -p "🔗 Link globally? (y/N) " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+  cd packages/qa360-mcp-cli
+  npm link
+  echo "✅ qa360-mcp command available globally"
+else
+  echo "ℹ️  Run 'npm link' in packages/qa360-mcp-cli to link globally"
+fi
+
+echo "🎉 Installation complete!"
+echo ""
+echo "Usage:"
+echo "  qa360-mcp --version"
+echo "  qa360-mcp doctor"
+echo "  qa360-mcp init"

package/scripts/mcp-smoke.mjs

@@ -0,0 +1,27 @@
+import { spawn } from 'node:child_process';
+
+const srv = spawn('node', [
+  'packages/mcp-server-qa360/dist/index.js',
+  '--allow-run','--allow-secrets'
+], { stdio: ['pipe','pipe','inherit'] });
+
+const send = (id, method, params = {}) =>
+  srv.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
+
+let buf = "";
+srv.stdout.on('data', chunk => {
+  buf += chunk.toString('utf8');
+  for (const line of buf.split('\n')) {
+    const s = line.trim();
+    if (!s) continue;
+    try {
+      const msg = JSON.parse(s);
+      if (msg.id === 1) console.log('✅ initialize ok');
+      if (msg.id === 2) console.log('✅ tools/list ok →', (msg.result?.tools||[]).length, 'tools');
+    } catch { /* ignore non-JSON lines */ }
+  }
+});
+
+send(1, "initialize", { protocolVersion: "2024-11-05", capabilities: {} });
+setTimeout(() => send(2, "tools/list", {}), 200);
+setTimeout(() => srv.kill('SIGTERM'), 1200);

package/scripts/smoke.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+
+BASE="https://qa360-cloud-preview.vercel.app"
+QA360_CLOUD_API_KEY="176402b75ec5a56176fc99a1139f3bd16268cd4b1e71798d5dd55c76f33b69de"
+
+echo "🔍 Running QA360 Cloud smoke tests..."
+
+echo "✓ Testing /api/health"
+curl -sf "$BASE/api/health" | jq '.status=="ok"' > /dev/null
+
+echo "✓ Testing /api/metrics"
+curl -sf "$BASE/api/metrics" | jq '.totalRuns>=0' > /dev/null
+
+echo "✓ Testing /api/metrics cache headers"
+curl -sfI "$BASE/api/metrics" | grep -i "cache-control" > /dev/null
+
+echo "✓ Testing /metrics (Prometheus)"
+curl -sf "$BASE/metrics" | grep -E "^qa360_runs_total" > /dev/null
+
+echo "✓ Testing /api/pack execution"
+curl -sf -H "x-qa360-api-key: ${QA360_CLOUD_API_KEY}" \
+  -H "Content-Type: application/json" \
+  -d '{"pack":"httpbin"}' "$BASE/api/pack" | jq '.success==true' > /dev/null
+
+echo "SMOKE OK"