qa360 2.2.1 → 2.2.13

package/scripts/ci/slo-assert.js

@@ -1,158 +0,0 @@
-#!/usr/bin/env node
-/**
- * QA360 Phase 7 - SLO Assertion Script
- * Validates Artillery load test results against SLO thresholds
- * 
- * Usage:
- *   node scripts/ci/slo-assert.js <results.json> --p95-ms 150 --err-rate 0.01
- */
-
-const fs = require('fs');
-const path = require('path');
-
-// Parse CLI arguments
-const args = process.argv.slice(2);
-const resultsFile = args[0];
-const p95Threshold = parseFloat(args[args.indexOf('--p95-ms') + 1] || 150);
-const errorRateThreshold = parseFloat(args[args.indexOf('--err-rate') + 1] || 0.01);
-
-if (!resultsFile || !fs.existsSync(resultsFile)) {
-  console.error('❌ Error: Results file not found');
-  console.error(`Usage: node slo-assert.js <results.json> --p95-ms 150 --err-rate 0.01`);
-  process.exit(1);
-}
-
-// Load Artillery results
-const results = JSON.parse(fs.readFileSync(resultsFile, 'utf8'));
-
-// Extract metrics
-const summary = results.aggregate || {};
-const latency = summary.latency || {};
-const counters = summary.counters || {};
-
-const p95 = latency.p95 || 0;
-const p99 = latency.p99 || 0;
-const median = latency.median || 0;
-const totalRequests = counters['http.requests'] || counters['vusers.created'] || 0;
-const errors = counters['errors.total'] || counters['http.codes.500'] || 0;
-const errorRate = totalRequests > 0 ? errors / totalRequests : 0;
-
-// Calculate success rate
-const successRate = totalRequests > 0 ? ((totalRequests - errors) / totalRequests) * 100 : 0;
-
-// Display results
-console.log('\n🔍 SLO Validation Results\n');
-console.log('═══════════════════════════════════════════════════════');
-console.log(`📊 Latency Metrics:`);
-console.log(`   • P50 (Median): ${median.toFixed(2)}ms`);
-console.log(`   • P95: ${p95.toFixed(2)}ms (threshold: ${p95Threshold}ms)`);
-console.log(`   • P99: ${p99.toFixed(2)}ms`);
-console.log('');
-console.log(`📈 Request Metrics:`);
-console.log(`   • Total Requests: ${totalRequests}`);
-console.log(`   • Errors: ${errors}`);
-console.log(`   • Error Rate: ${(errorRate * 100).toFixed(2)}% (threshold: ${(errorRateThreshold * 100).toFixed(2)}%)`);
-console.log(`   • Success Rate: ${successRate.toFixed(2)}%`);
-console.log('═══════════════════════════════════════════════════════\n');
-
-// Validate SLO thresholds
-const sloResults = {
-  timestamp: new Date().toISOString(),
-  thresholds: {
-    p95_ms: p95Threshold,
-    error_rate: errorRateThreshold
-  },
-  actual: {
-    p95_ms: p95,
-    p99_ms: p99,
-    median_ms: median,
-    error_rate: errorRate,
-    success_rate: successRate / 100,
-    total_requests: totalRequests,
-    errors: errors
-  },
-  checks: {
-    p95_ok: p95 <= p95Threshold,
-    error_rate_ok: errorRate <= errorRateThreshold,
-    success_rate_ok: successRate >= 99.0
-  }
-};
-
-// Determine GO/NO-GO
-const allChecksPassed = Object.values(sloResults.checks).every(check => check === true);
-
-if (allChecksPassed) {
-  console.log('✅ GO - All SLO thresholds met\n');
-  sloResults.decision = 'GO';
-  sloResults.status = 'passed';
-} else {
-  console.log('❌ NO-GO - SLO thresholds violated\n');
-  sloResults.decision = 'NO-GO';
-  sloResults.status = 'failed';
-  
-  // List violations
-  console.log('Violations:');
-  if (!sloResults.checks.p95_ok) {
-    console.log(`  • P95 latency: ${p95.toFixed(2)}ms > ${p95Threshold}ms`);
-  }
-  if (!sloResults.checks.error_rate_ok) {
-    console.log(`  • Error rate: ${(errorRate * 100).toFixed(2)}% > ${(errorRateThreshold * 100).toFixed(2)}%`);
-  }
-  if (!sloResults.checks.success_rate_ok) {
-    console.log(`  • Success rate: ${successRate.toFixed(2)}% < 99.0%`);
-  }
-  console.log('');
-}
-
-// Generate performance summary
-const summaryPath = path.join(path.dirname(resultsFile), 'perf-summary.md');
-const summaryContent = `# QA360 Phase 7 - Performance Summary
-
-**Date**: ${new Date().toISOString()}  
-**Test**: Artillery Load Testing  
-**Decision**: ${sloResults.decision}
-
-## SLO Thresholds
-
-| Metric | Threshold | Actual | Status |
-|--------|-----------|--------|--------|
-| P95 Latency | ≤${p95Threshold}ms | ${p95.toFixed(2)}ms | ${sloResults.checks.p95_ok ? '✅' : '❌'} |
-| Error Rate | ≤${(errorRateThreshold * 100).toFixed(2)}% | ${(errorRate * 100).toFixed(2)}% | ${sloResults.checks.error_rate_ok ? '✅' : '❌'} |
-| Success Rate | ≥99.0% | ${successRate.toFixed(2)}% | ${sloResults.checks.success_rate_ok ? '✅' : '❌'} |
-
-## Latency Distribution
-
-- **P50 (Median)**: ${median.toFixed(2)}ms
-- **P95**: ${p95.toFixed(2)}ms
-- **P99**: ${p99.toFixed(2)}ms
-
-## Request Statistics
-
-- **Total Requests**: ${totalRequests}
-- **Successful**: ${totalRequests - errors}
-- **Errors**: ${errors}
-- **Error Rate**: ${(errorRate * 100).toFixed(2)}%
-
-## Decision
-
-${allChecksPassed ? 
-  '✅ **GO** - All SLO thresholds met. System performance validated.' :
-  '❌ **NO-GO** - SLO violations detected. Review performance issues before proceeding.'
-}
-
----
-
-**Generated**: ${new Date().toISOString()}  
-**Source**: ${path.basename(resultsFile)}
-`;
-
-fs.writeFileSync(summaryPath, summaryContent);
-console.log(`📄 Performance summary: ${summaryPath}\n`);
-
-// Save detailed results
-const detailsPath = path.join(path.dirname(resultsFile), 'slo-results.json');
-fs.writeFileSync(detailsPath, JSON.stringify(sloResults, null, 2));
-console.log(`📊 Detailed results: ${detailsPath}\n`);
-
-// Exit with appropriate code
-process.exit(allChecksPassed ? 0 : 1);

package/scripts/ci/test-fault-tolerance.sh

@@ -1,301 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# QA360 Fault Tolerance Test Suite
-# Tests: cancel, vault missing, invalid pack, SIGTERM handling
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-ARTIFACTS_DIR="$PROJECT_ROOT/artifacts/faults"
-TIMESTAMP=$(date -u +"%Y%m%d_%H%M%S")
-REPORT_FILE="$ARTIFACTS_DIR/faults-report-$TIMESTAMP.json"
-
-# Colors
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-BLUE='\033[0;34m'
-NC='\033[0m'
-
-mkdir -p "$ARTIFACTS_DIR"
-
-echo -e "${BLUE}QA360 Fault Tolerance Test Suite${NC}"
-echo "Timestamp: $TIMESTAMP"
-echo ""
-
-# Initialize report
-cat > "$REPORT_FILE" <<'EOF'
-{
-  "suite": "fault-tolerance",
-  "version": "1.0.0",
-  "tests": [],
-  "summary": {
-    "total": 0,
-    "passed": 0,
-    "failed": 0
-  }
-}
-EOF
-
-TOTAL_TESTS=0
-PASSED_TESTS=0
-FAILED_TESTS=0
-
-# Test 1: Cancel mid-run
-test_cancel_midrun() {
-  echo -e "${BLUE}Test 1: Cancel mid-run${NC}"
-  local status="fail"
-  local error=""
-  
-  # Start long-running pack in background
-  qa360 run --pack examples/packs-conformance/heavy-artifacts.yml > "$ARTIFACTS_DIR/cancel_run.log" 2>&1 &
-  local pid=$!
-  
-  sleep 2
-  
-  # Send SIGTERM
-  kill -TERM $pid 2>/dev/null || true
-  
-  # Wait for graceful shutdown
-  local waited=0
-  while kill -0 $pid 2>/dev/null && [ $waited -lt 10 ]; do
-    sleep 1
-    waited=$((waited + 1))
-  done
-  
-  if ! kill -0 $pid 2>/dev/null; then
-    # Check vault state
-    if grep -q "cancelled" "$ARTIFACTS_DIR/cancel_run.log" 2>/dev/null; then
-      status="pass"
-      echo -e "${GREEN}✓ Cancel handled gracefully${NC}"
-    else
-      error="Vault state not set to cancelled"
-      echo -e "${YELLOW}⚠ Vault state unclear${NC}"
-    fi
-  else
-    kill -9 $pid 2>/dev/null || true
-    error="Process did not terminate gracefully"
-    echo -e "${RED}✗ Process hung${NC}"
-  fi
-  
-  echo "{\"name\":\"cancel_midrun\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
-  
-  if [ "$status" = "pass" ]; then
-    PASSED_TESTS=$((PASSED_TESTS + 1))
-  else
-    FAILED_TESTS=$((FAILED_TESTS + 1))
-  fi
-  TOTAL_TESTS=$((TOTAL_TESTS + 1))
-  echo ""
-}
-
-# Test 2: Vault missing
-test_vault_missing() {
-  echo -e "${BLUE}Test 2: Vault missing${NC}"
-  local status="fail"
-  local error=""
-  
-  # Temporarily rename .qa360 directory
-  if [ -d "$HOME/.qa360" ]; then
-    mv "$HOME/.qa360" "$HOME/.qa360.backup" 2>/dev/null || true
-  fi
-  
-  # Try to run
-  if qa360 run --pack examples/packs-conformance/minimal.yml > "$ARTIFACTS_DIR/vault_missing.log" 2>&1; then
-    error="Should have failed with VAULT_MISSING"
-    echo -e "${RED}✗ No error raised${NC}"
-  else
-    if grep -q "VAULT_MISSING" "$ARTIFACTS_DIR/vault_missing.log" 2>/dev/null; then
-      status="pass"
-      echo -e "${GREEN}✓ VAULT_MISSING code returned${NC}"
-    else
-      error="Wrong error code"
-      echo -e "${YELLOW}⚠ Different error code${NC}"
-    fi
-  fi
-  
-  # Restore .qa360
-  if [ -d "$HOME/.qa360.backup" ]; then
-    mv "$HOME/.qa360.backup" "$HOME/.qa360" 2>/dev/null || true
-  fi
-  
-  echo "{\"name\":\"vault_missing\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
-  
-  if [ "$status" = "pass" ]; then
-    PASSED_TESTS=$((PASSED_TESTS + 1))
-  else
-    FAILED_TESTS=$((FAILED_TESTS + 1))
-  fi
-  TOTAL_TESTS=$((TOTAL_TESTS + 1))
-  echo ""
-}
-
-# Test 3: Invalid pack schema
-test_invalid_pack() {
-  echo -e "${BLUE}Test 3: Invalid pack schema${NC}"
-  local status="fail"
-  local error=""
-  
-  # Create invalid pack
-  cat > "$ARTIFACTS_DIR/invalid.yml" <<'INVALID'
-name: invalid-pack
-version: 1.0.0
-steps:
-  - name: missing-adapter
-    config:
-      url: "https://example.com"
-INVALID
-  
-  # Try to validate
-  if qa360 pack validate "$ARTIFACTS_DIR/invalid.yml" > "$ARTIFACTS_DIR/invalid_pack.log" 2>&1; then
-    error="Should have failed with INVALID_PACK_SCHEMA"
-    echo -e "${RED}✗ Invalid pack accepted${NC}"
-  else
-    if grep -q "INVALID_PACK_SCHEMA\|adapter.*required\|missing.*adapter" "$ARTIFACTS_DIR/invalid_pack.log" 2>/dev/null; then
-      status="pass"
-      echo -e "${GREEN}✓ INVALID_PACK_SCHEMA detected${NC}"
-    else
-      error="Wrong error code"
-      echo -e "${YELLOW}⚠ Different error${NC}"
-    fi
-  fi
-  
-  rm -f "$ARTIFACTS_DIR/invalid.yml"
-  
-  echo "{\"name\":\"invalid_pack_schema\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
-  
-  if [ "$status" = "pass" ]; then
-    PASSED_TESTS=$((PASSED_TESTS + 1))
-  else
-    FAILED_TESTS=$((FAILED_TESTS + 1))
-  fi
-  TOTAL_TESTS=$((TOTAL_TESTS + 1))
-  echo ""
-}
-
-# Test 4: SIGTERM graceful exit
-test_sigterm_graceful() {
-  echo -e "${BLUE}Test 4: SIGTERM graceful exit${NC}"
-  local status="fail"
-  local error=""
-  
-  # Start qa360 serve
-  qa360 serve --port 4061 > "$ARTIFACTS_DIR/sigterm.log" 2>&1 &
-  local pid=$!
-  
-  sleep 2
-  
-  # Send SIGTERM
-  kill -TERM $pid
-  
-  # Wait for exit
-  local exit_code=0
-  wait $pid || exit_code=$?
-  
-  if [ $exit_code -eq 0 ] || [ $exit_code -eq 143 ]; then
-    # Check cleanup
-    if ! lsof -i :4061 >/dev/null 2>&1; then
-      status="pass"
-      echo -e "${GREEN}✓ Graceful exit with cleanup${NC}"
-    else
-      error="Port still in use"
-      echo -e "${YELLOW}⚠ Cleanup incomplete${NC}"
-    fi
-  else
-    error="Non-zero exit code: $exit_code"
-    echo -e "${RED}✗ Exit code $exit_code${NC}"
-  fi
-  
-  echo "{\"name\":\"sigterm_graceful\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
-  
-  if [ "$status" = "pass" ]; then
-    PASSED_TESTS=$((PASSED_TESTS + 1))
-  else
-    FAILED_TESTS=$((FAILED_TESTS + 1))
-  fi
-  TOTAL_TESTS=$((TOTAL_TESTS + 1))
-  echo ""
-}
-
-# Test 5: Cleanup on error
-test_cleanup_on_error() {
-  echo -e "${BLUE}Test 5: Cleanup on error${NC}"
-  local status="fail"
-  local error=""
-  
-  # Run pack that will fail
-  qa360 run --pack examples/packs-conformance/timeouts.yml > "$ARTIFACTS_DIR/cleanup.log" 2>&1 || true
-  
-  # Check for orphan processes
-  local orphans=$(ps aux | grep -E "playwright|chromium|k6" | grep -v grep | wc -l)
-  
-  if [ "$orphans" -eq 0 ]; then
-    status="pass"
-    echo -e "${GREEN}✓ No orphan processes${NC}"
-  else
-    error="Found $orphans orphan processes"
-    echo -e "${YELLOW}⚠ $orphans orphans detected${NC}"
-  fi
-  
-  echo "{\"name\":\"cleanup_on_error\",\"status\":\"$status\",\"error\":\"$error\"}" >> "$ARTIFACTS_DIR/test_results.tmp"
-  
-  if [ "$status" = "pass" ]; then
-    PASSED_TESTS=$((PASSED_TESTS + 1))
-  else
-    FAILED_TESTS=$((FAILED_TESTS + 1))
-  fi
-  TOTAL_TESTS=$((TOTAL_TESTS + 1))
-  echo ""
-}
-
-# Run all tests
-touch "$ARTIFACTS_DIR/test_results.tmp"
-
-test_cancel_midrun
-test_vault_missing
-test_invalid_pack
-test_sigterm_graceful
-test_cleanup_on_error
-
-# Build final report
-{
-  echo "{"
-  echo "  \"suite\": \"fault-tolerance\","
-  echo "  \"version\": \"1.0.0\","
-  echo "  \"timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\","
-  echo "  \"tests\": ["
-  
-  first=true
-  while IFS= read -r line; do
-    if [ "$first" = true ]; then
-      first=false
-    else
-      echo ","
-    fi
-    echo "    $line"
-  done < "$ARTIFACTS_DIR/test_results.tmp"
-  
-  echo ""
-  echo "  ],"
-  echo "  \"summary\": {"
-  echo "    \"total\": $TOTAL_TESTS,"
-  echo "    \"passed\": $PASSED_TESTS,"
-  echo "    \"failed\": $FAILED_TESTS"
-  echo "  }"
-  echo "}"
-} > "$REPORT_FILE"
-
-rm -f "$ARTIFACTS_DIR/test_results.tmp"
-
-# Summary
-echo "========================================="
-echo "Total: $TOTAL_TESTS | Passed: $PASSED_TESTS | Failed: $FAILED_TESTS"
-echo "Report: $REPORT_FILE"
-
-if [ $FAILED_TESTS -eq 0 ]; then
-  echo -e "${GREEN}✓ All fault tolerance tests passed${NC}"
-  exit 0
-else
-  echo -e "${RED}✗ $FAILED_TESTS test(s) failed${NC}"
-  exit 1
-fi

package/scripts/install-mcp.sh

@@ -1,66 +0,0 @@
-#!/usr/bin/env bash
-# QA360 MCP CLI Installation Script
-# Safe for Windows, macOS, and Linux
-
-set -e
-
-echo "🚀 Installing QA360 MCP CLI..."
-
-# Detect OS
-OS="$(uname -s)"
-case "$OS" in
-  Linux*)   PLATFORM="linux";;
-  Darwin*)  PLATFORM="macos";;
-  MINGW*|MSYS*|CYGWIN*) PLATFORM="windows";;
-  *)        PLATFORM="unknown";;
-esac
-
-echo "📦 Platform detected: $PLATFORM"
-
-# Check Node.js
-if ! command -v node &> /dev/null; then
-  echo "❌ Node.js not found. Please install Node.js >= 18.0.0"
-  exit 1
-fi
-
-NODE_VERSION=$(node -v | cut -d'v' -f2 | cut -d'.' -f1)
-if [ "$NODE_VERSION" -lt 18 ]; then
-  echo "❌ Node.js version $NODE_VERSION is too old. Please upgrade to >= 18.0.0"
-  exit 1
-fi
-
-echo "✅ Node.js $(node -v) detected"
-
-# Check pnpm
-if ! command -v pnpm &> /dev/null; then
-  echo "📦 Installing pnpm..."
-  npm install -g pnpm@8.15.8
-fi
-
-echo "✅ pnpm $(pnpm -v) ready"
-
-# Install dependencies
-echo "📦 Installing dependencies..."
-pnpm install --frozen-lockfile
-
-# Build
-echo "🔨 Building QA360 MCP CLI..."
-pnpm --filter @qa360/mcp run build
-
-# Link globally (optional)
-read -p "🔗 Link globally? (y/N) " -n 1 -r
-echo
-if [[ $REPLY =~ ^[Yy]$ ]]; then
-  cd packages/qa360-mcp-cli
-  npm link
-  echo "✅ qa360-mcp command available globally"
-else
-  echo "ℹ️  Run 'npm link' in packages/qa360-mcp-cli to link globally"
-fi
-
-echo "🎉 Installation complete!"
-echo ""
-echo "Usage:"
-echo "  qa360-mcp --version"
-echo "  qa360-mcp doctor"
-echo "  qa360-mcp init"

package/scripts/mcp-smoke.mjs

@@ -1,27 +0,0 @@
-import { spawn } from 'node:child_process';
-
-const srv = spawn('node', [
-  'packages/mcp-server-qa360/dist/index.js',
-  '--allow-run','--allow-secrets'
-], { stdio: ['pipe','pipe','inherit'] });
-
-const send = (id, method, params = {}) =>
-  srv.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
-
-let buf = "";
-srv.stdout.on('data', chunk => {
-  buf += chunk.toString('utf8');
-  for (const line of buf.split('\n')) {
-    const s = line.trim();
-    if (!s) continue;
-    try {
-      const msg = JSON.parse(s);
-      if (msg.id === 1) console.log('✅ initialize ok');
-      if (msg.id === 2) console.log('✅ tools/list ok →', (msg.result?.tools||[]).length, 'tools');
-    } catch { /* ignore non-JSON lines */ }
-  }
-});
-
-send(1, "initialize", { protocolVersion: "2024-11-05", capabilities: {} });
-setTimeout(() => send(2, "tools/list", {}), 200);
-setTimeout(() => srv.kill('SIGTERM'), 1200);

package/scripts/smoke.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-BASE="https://qa360-cloud-preview.vercel.app"
-QA360_CLOUD_API_KEY="176402b75ec5a56176fc99a1139f3bd16268cd4b1e71798d5dd55c76f33b69de"
-
-echo "🔍 Running QA360 Cloud smoke tests..."
-
-echo "✓ Testing /api/health"
-curl -sf "$BASE/api/health" | jq '.status=="ok"' > /dev/null
-
-echo "✓ Testing /api/metrics"
-curl -sf "$BASE/api/metrics" | jq '.totalRuns>=0' > /dev/null
-
-echo "✓ Testing /api/metrics cache headers"
-curl -sfI "$BASE/api/metrics" | grep -i "cache-control" > /dev/null
-
-echo "✓ Testing /metrics (Prometheus)"
-curl -sf "$BASE/metrics" | grep -E "^qa360_runs_total" > /dev/null
-
-echo "✓ Testing /api/pack execution"
-curl -sf -H "x-qa360-api-key: ${QA360_CLOUD_API_KEY}" \
-  -H "Content-Type: application/json" \
-  -d '{"pack":"httpbin"}' "$BASE/api/pack" | jq '.success==true' > /dev/null
-
-echo "SMOKE OK"