qa360 2.1.7 → 2.2.0

package/examples/ci/gitlab-ci-serve.yml

@@ -0,0 +1,407 @@
+# QA360 GitLab CI avec Serve
+# Pipeline complet avec serveur d'observabilité et métriques
+
+stages:
+  - readiness
+  - test
+  - consolidate
+  - deploy
+
+variables:
+  QA360_VERSION: "latest"
+  NODE_VERSION: "18"
+  SERVE_PORT: "4060"
+  SERVE_HOST: "0.0.0.0"
+
+# Templates réutilisables
+.qa360_base: &qa360_base
+  image: node:${NODE_VERSION}-alpine
+  before_script:
+    - apk add --no-cache curl jq
+    - npm install -g @qa360/cli@${QA360_VERSION}
+    - qa360 --version
+
+.serve_setup: &serve_setup
+  - |
+    echo "🚀 Starting QA360 server..."
+    qa360 serve --port ${SERVE_PORT} --host ${SERVE_HOST} --verbose &
+    SERVER_PID=$!
+    echo $SERVER_PID > qa360-server.pid
+    
+    echo "⏳ Waiting for server to start..."
+    timeout 30 sh -c 'until curl -f http://localhost:${SERVE_PORT}/health; do sleep 1; done'
+    echo "✅ QA360 server ready"
+
+.serve_cleanup: &serve_cleanup
+  - |
+    if [ -f qa360-server.pid ]; then
+      echo "🛑 Stopping QA360 server..."
+      kill $(cat qa360-server.pid) 2>/dev/null || true
+      sleep 2
+      kill -9 $(cat qa360-server.pid) 2>/dev/null || true
+      rm qa360-server.pid
+      echo "✅ Server stopped"
+    fi
+
+# Stage 1: Readiness Check
+readiness_check:
+  <<: *qa360_base
+  stage: readiness
+  script:
+    - echo "🔍 QA360 Readiness Check"
+    - qa360 doctor --fix
+    - |
+      for i in $(seq 1 3); do
+        echo "Attempt $i/3..."
+        if qa360 serve --readiness-check; then
+          echo "✅ System ready"
+          exit 0
+        fi
+        echo "❌ Not ready, retrying in 10s..."
+        sleep 10
+      done
+      echo "❌ System not ready after 3 attempts"
+      qa360 doctor
+      exit 1
+  retry:
+    max: 2
+    when: script_failure
+  artifacts:
+    when: on_failure
+    paths:
+      - qa360-doctor-output.txt
+    expire_in: 1 day
+  only:
+    - merge_requests
+    - main
+    - develop
+
+# Stage 2: Tests parallèles avec observabilité
+.test_template: &test_template
+  <<: *qa360_base
+  stage: test
+  dependencies:
+    - readiness_check
+  before_script:
+    - !reference [.qa360_base, before_script]
+    - *serve_setup
+  script:
+    - echo "🧪 Running pack ${PACK_NAME}"
+    - |
+      # Generate run ID
+      RUN_ID="gl-${CI_PIPELINE_ID}-${PACK_NAME}-$(date +%s)"
+      echo "Run ID: $RUN_ID"
+      
+      # Get initial metrics
+      curl -s http://localhost:${SERVE_PORT}/metrics > metrics-before-${PACK_NAME}.txt
+      
+      # Run the pack
+      qa360 run packs/${PACK_NAME}.yml --run-id "$RUN_ID"
+      
+      # Get final metrics
+      curl -s http://localhost:${SERVE_PORT}/metrics > metrics-after-${PACK_NAME}.txt
+      
+      # Get diagnostics
+      curl -s http://localhost:${SERVE_PORT}/diag | jq . > diagnostics-${PACK_NAME}.json
+      
+      echo "📊 Test completed for ${PACK_NAME}"
+  after_script:
+    - *serve_cleanup
+  artifacts:
+    reports:
+      junit: qa360-${PACK_NAME}-results.xml
+    paths:
+      - qa360-${PACK_NAME}-report.html
+      - metrics-*-${PACK_NAME}.txt
+      - diagnostics-${PACK_NAME}.json
+      - screenshots/
+    expire_in: 1 week
+  coverage: '/Trust Score: (\d+)%/'
+
+api_tests:
+  <<: *test_template
+  variables:
+    PACK_NAME: "api-smoke"
+  parallel:
+    matrix:
+      - PACK_NAME: ["api-smoke", "api-integration", "api-security"]
+
+ui_tests:
+  <<: *test_template
+  variables:
+    PACK_NAME: "ui-critical"
+  parallel:
+    matrix:
+      - PACK_NAME: ["ui-critical", "ui-responsive", "ui-accessibility"]
+
+performance_tests:
+  <<: *test_template
+  variables:
+    PACK_NAME: "perf-baseline"
+  parallel:
+    matrix:
+      - PACK_NAME: ["perf-baseline", "perf-stress", "perf-endurance"]
+  only:
+    - main
+    - schedules
+
+# Test de cancel (fonctionnalité spéciale)
+cancel_test:
+  <<: *qa360_base
+  stage: test
+  dependencies:
+    - readiness_check
+  before_script:
+    - !reference [.qa360_base, before_script]
+    - *serve_setup
+  script:
+    - echo "🛑 Testing cancel functionality"
+    - |
+      # Start long-running test
+      qa360 run packs/long-running-test.yml --run-id "cancel-test" &
+      TEST_PID=$!
+      
+      sleep 5  # Let it start
+      
+      # Cancel via API
+      CANCEL_RESPONSE=$(curl -X POST http://localhost:${SERVE_PORT}/cancel/cancel-test)
+      echo "Cancel response: $CANCEL_RESPONSE"
+      
+      # Verify cancellation
+      if echo "$CANCEL_RESPONSE" | jq -r '.cancelled' | grep -q "true"; then
+        echo "✅ Cancel functionality works"
+      else
+        echo "❌ Cancel functionality failed"
+        exit 1
+      fi
+      
+      wait $TEST_PID || echo "✅ Test was cancelled successfully"
+  after_script:
+    - *serve_cleanup
+  artifacts:
+    paths:
+      - cancel-test-log.txt
+    expire_in: 1 day
+  only:
+    - merge_requests
+
+# Stage 3: Consolidation et quality gates
+consolidate_results:
+  <<: *qa360_base
+  stage: consolidate
+  dependencies:
+    - api_tests
+    - ui_tests
+    - performance_tests
+  before_script:
+    - !reference [.qa360_base, before_script]
+    - *serve_setup
+  script:
+    - echo "📊 Consolidating results and checking quality gates"
+    - |
+      # Generate consolidated report
+      qa360 report --consolidate --input . --output consolidated-report.html
+      
+      # Collect final metrics
+      curl -s http://localhost:${SERVE_PORT}/metrics > final-metrics.txt
+      
+      # Extract key metrics
+      TOTAL_RUNS=$(grep "qa360_runs_total" final-metrics.txt | awk '{print $2}' || echo "0")
+      TRUST_SCORE=$(grep "qa360_last_run_trust_score" final-metrics.txt | awk '{print $2}' | cut -d. -f1 || echo "0")
+      
+      echo "Total runs: $TOTAL_RUNS"
+      echo "Trust score: $TRUST_SCORE"
+      
+      # Quality gates
+      if [ "$TRUST_SCORE" -lt 70 ]; then
+        echo "❌ Quality gate failed: Trust score $TRUST_SCORE < 70"
+        exit 1
+      fi
+      
+      echo "✅ Quality gates passed"
+      
+      # Generate metrics summary
+      cat > metrics-summary.json << EOF
+      {
+        "pipeline_id": "${CI_PIPELINE_ID}",
+        "commit_sha": "${CI_COMMIT_SHA}",
+        "total_runs": $TOTAL_RUNS,
+        "trust_score": $TRUST_SCORE,
+        "timestamp": "$(date -Iseconds)"
+      }
+      EOF
+  after_script:
+    - *serve_cleanup
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: qa360-coverage.xml
+    paths:
+      - consolidated-report.html
+      - final-metrics.txt
+      - metrics-summary.json
+    expire_in: 30 days
+  coverage: '/Trust Score: (\d+)%/'
+
+# Monitoring continu (scheduled)
+monitoring_health:
+  <<: *qa360_base
+  stage: test
+  before_script:
+    - !reference [.qa360_base, before_script]
+    - *serve_setup
+  script:
+    - echo "🔍 Continuous monitoring check"
+    - |
+      # Extended health check
+      for i in $(seq 1 10); do
+        echo "Health check $i/10..."
+        HEALTH=$(curl -s http://localhost:${SERVE_PORT}/health)
+        STATUS=$(echo "$HEALTH" | jq -r '.status')
+        
+        if [ "$STATUS" != "ok" ]; then
+          echo "❌ Health check failed: $STATUS"
+          echo "$HEALTH" | jq .
+          exit 1
+        fi
+        
+        sleep 30
+      done
+      
+      echo "✅ All health checks passed"
+  after_script:
+    - *serve_cleanup
+  artifacts:
+    when: on_failure
+    paths:
+      - monitoring-failure.log
+    expire_in: 1 day
+  only:
+    - schedules
+
+# Stage 4: Déploiement des métriques
+deploy_metrics:
+  image: alpine:latest
+  stage: deploy
+  dependencies:
+    - consolidate_results
+  before_script:
+    - apk add --no-cache curl jq
+  script:
+    - echo "📤 Deploying metrics to monitoring system"
+    - |
+      if [ -f metrics-summary.json ]; then
+        # Send to Prometheus pushgateway
+        if [ -n "$PROMETHEUS_PUSHGATEWAY_URL" ]; then
+          TRUST_SCORE=$(jq -r '.trust_score' metrics-summary.json)
+          TOTAL_RUNS=$(jq -r '.total_runs' metrics-summary.json)
+          
+          cat << EOF | curl --data-binary @- "${PROMETHEUS_PUSHGATEWAY_URL}/metrics/job/qa360/instance/${CI_PIPELINE_ID}"
+      # TYPE qa360_pipeline_trust_score gauge
+      qa360_pipeline_trust_score{pipeline="${CI_PIPELINE_ID}",branch="${CI_COMMIT_REF_NAME}"} ${TRUST_SCORE}
+      # TYPE qa360_pipeline_runs_total counter
+      qa360_pipeline_runs_total{pipeline="${CI_PIPELINE_ID}",branch="${CI_COMMIT_REF_NAME}"} ${TOTAL_RUNS}
+      EOF
+          echo "✅ Metrics sent to Prometheus"
+        fi
+        
+        # Send to webhook
+        if [ -n "$METRICS_WEBHOOK_URL" ]; then
+          curl -X POST "$METRICS_WEBHOOK_URL" \
+            -H "Content-Type: application/json" \
+            -d @metrics-summary.json
+          echo "✅ Metrics sent to webhook"
+        fi
+      fi
+  only:
+    - main
+  when: on_success
+
+# Pages pour les rapports
+pages:
+  image: alpine:latest
+  stage: deploy
+  dependencies:
+    - consolidate_results
+  script:
+    - mkdir public
+    - cp consolidated-report.html public/index.html
+    - cp final-metrics.txt public/metrics.txt
+    - |
+      # Generate simple index
+      cat > public/metrics.html << 'EOF'
+      <!DOCTYPE html>
+      <html>
+      <head><title>QA360 Metrics</title></head>
+      <body>
+        <h1>QA360 Pipeline Metrics</h1>
+        <ul>
+          <li><a href="index.html">Consolidated Report</a></li>
+          <li><a href="metrics.txt">Raw Metrics</a></li>
+        </ul>
+      </body>
+      </html>
+      EOF
+  artifacts:
+    paths:
+      - public
+  only:
+    - main
+
+# Notifications
+notify_success:
+  image: alpine:latest
+  stage: .post
+  before_script:
+    - apk add --no-cache curl
+  script:
+    - echo "✅ QA360 pipeline completed successfully"
+    - |
+      if [ -n "$SLACK_WEBHOOK_URL" ]; then
+        curl -X POST "$SLACK_WEBHOOK_URL" \
+          -H "Content-Type: application/json" \
+          -d '{
+            "text": "✅ QA360 Pipeline Success",
+            "attachments": [{
+              "color": "good",
+              "fields": [
+                {"title": "Pipeline", "value": "'${CI_PIPELINE_ID}'", "short": true},
+                {"title": "Branch", "value": "'${CI_COMMIT_REF_NAME}'", "short": true},
+                {"title": "Commit", "value": "'${CI_COMMIT_SHORT_SHA}'", "short": true}
+              ]
+            }]
+          }'
+      fi
+  when: on_success
+  only:
+    - main
+
+notify_failure:
+  image: alpine:latest
+  stage: .post
+  before_script:
+    - apk add --no-cache curl
+  script:
+    - echo "❌ QA360 pipeline failed"
+    - |
+      if [ -n "$SLACK_WEBHOOK_URL" ]; then
+        curl -X POST "$SLACK_WEBHOOK_URL" \
+          -H "Content-Type: application/json" \
+          -d '{
+            "text": "❌ QA360 Pipeline Failed",
+            "attachments": [{
+              "color": "danger",
+              "fields": [
+                {"title": "Pipeline", "value": "'${CI_PIPELINE_ID}'", "short": true},
+                {"title": "Branch", "value": "'${CI_COMMIT_REF_NAME}'", "short": true},
+                {"title": "Commit", "value": "'${CI_COMMIT_SHORT_SHA}'", "short": true},
+                {"title": "Failed Job", "value": "'${CI_JOB_NAME}'", "short": true}
+              ]
+            }]
+          }'
+      fi
+  when: on_failure
+  only:
+    - main
+    - merge_requests

package/examples/datasets/README.md

@@ -0,0 +1,101 @@
+# QA360 E2E Test Datasets
+
+This directory contains real-world test datasets for E2E testing with QA360.
+
+## Structure
+
+```
+dataset/
+├── index.ts           # Registry and exports
+├── b2bshop.ts         # B2BShop Lovable test data
+├── README.md          # This file
+└── [domain].ts        # Add more domains as needed
+```
+
+## Adding a New Domain
+
+1. Create a new file `dataset/[domain].ts` (lowercase, no special chars)
+2. Export the dataset interface and data
+3. Add to the registry in `index.ts`
+
+### Template
+
+```typescript
+/**
+ * [Domain Name] - E2E Test Dataset
+ *
+ * Test data for https://example.com
+ */
+
+export interface [Domain]TestDataset {
+  baseUrl: string;
+  name: string;
+  auth: {
+    type: 'ui_login' | 'bearer' | 'api_key' | 'basic' | 'oauth2';
+    credentials: Record<string, unknown>;
+    selectors?: Record<string, string>;
+  };
+  pages: Array<{
+    name: string;
+    path: string;
+    expectedElements: string[];
+  }>;
+  flows: Array<{
+    name: string;
+    steps: Array<{
+      action: 'navigate' | 'click' | 'fill' | 'wait';
+      target?: string;
+      value?: string;
+      selector?: string;
+    }>;
+  }>;
+}
+
+export const [domain]Dataset: [Domain]TestDataset = {
+  // ... your test data
+};
+
+export const [domain]PackV2 = {
+  version: 2,
+  name: '[domain]-e2e',
+  auth: {
+    // ... auth profiles
+  },
+  gates: {
+    // ... gates
+  }
+};
+```
+
+## Available Datasets
+
+| Domain | URL | Auth Type | Description |
+|--------|-----|-----------|-------------|
+| b2bshop | https://b2bshop.lovable.app | ui_login | B2B E-commerce platform |
+| reqres | https://reqres.in/api | bearer | Fake REST API for testing |
+
+## Usage
+
+```typescript
+import { getDataset, getDatasetPack } from './dataset/index.js';
+
+// Get test dataset
+const dataset = await getDataset('b2bshop');
+
+// Get Pack v2 config
+const pack = await getDatasetPack('b2bshop');
+
+// Run with QA360
+const runner = new Phase3Runner({
+  workingDir: process.cwd(),
+  pack
+});
+await runner.run();
+```
+
+## Security
+
+**IMPORTANT**: These files contain real credentials. Never commit actual production credentials.
+- Use test accounts only
+- Rotate passwords regularly
+- Consider using environment variables for sensitive data

package/examples/datasets/b2bshop.ts

@@ -0,0 +1,155 @@
+/**
+ * B2BShop Lovable - E2E Test Dataset
+ *
+ * Test data for https://b2bshop.lovable.app
+ * Used for QA360 E2E authentication and navigation tests.
+ */
+
+export interface B2BShopTestDataset {
+  baseUrl: string;
+  name: string;
+  auth: {
+    type: 'ui_login';
+    credentials: {
+      email: string;
+      password: string;
+    };
+    selectors: {
+      emailInput: string;
+      passwordInput: string;
+      submitButton: string;
+    };
+  };
+  pages: {
+    name: string;
+    path: string;
+    expectedElements: string[];
+  }[];
+  flows: {
+    name: string;
+    steps: Array<{
+      action: 'navigate' | 'click' | 'fill' | 'wait';
+      target?: string;
+      value?: string;
+      selector?: string;
+    }>;
+  }[];
+}
+
+export const b2bshopDataset: B2BShopTestDataset = {
+  baseUrl: 'https://b2bshop.lovable.app',
+  name: 'B2BShop',
+
+  auth: {
+    type: 'ui_login',
+    credentials: {
+      email: 'buyer@acme.test',
+      password: 'Buyer123!'
+    },
+    selectors: {
+      emailInput: 'input[type="email"], input[name="email"], input[placeholder*="email" i]',
+      passwordInput: 'input[type="password"], input[name="password"], input[placeholder*="password" i]',
+      submitButton: 'button[type="submit"], button:has-text("Se connecter"), button:has-text("Login"), button:has-text("Sign in")'
+    }
+  },
+
+  pages: [
+    {
+      name: 'Home',
+      path: '/',
+      expectedElements: ['header', 'nav', 'main']
+    },
+    {
+      name: 'Catalog',
+      path: '/catalog',
+      expectedElements: ['.product-grid', '.product-card', '.filters']
+    },
+    {
+      name: 'Products',
+      path: '/products',
+      expectedElements: ['.product-list', '.product-item']
+    },
+    {
+      name: 'Cart',
+      path: '/cart',
+      expectedElements: ['.cart-items', '.cart-total', '.checkout-button']
+    },
+    {
+      name: 'Profile',
+      path: '/profile',
+      expectedElements: ['.user-info', '.order-history']
+    }
+  ],
+
+  flows: [
+    {
+      name: 'Login and Browse Catalog',
+      steps: [
+        { action: 'navigate', target: '/' },
+        { action: 'click', selector: 'a:has-text("Se connecter"), a:has-text("Login")' },
+        { action: 'fill', selector: 'input[type="email"]', value: 'buyer@acme.test' },
+        { action: 'fill', selector: 'input[type="password"]', value: 'Buyer123!' },
+        { action: 'click', selector: 'button[type="submit"]' },
+        { action: 'wait', target: '2000' },
+        { action: 'navigate', target: '/catalog' }
+      ]
+    },
+    {
+      name: 'Add Product to Cart',
+      steps: [
+        { action: 'navigate', target: '/' },
+        { action: 'click', selector: 'a:has-text("Catalogue"), a:has-text("Catalog")' },
+        { action: 'click', selector: '.product-card:first-child .add-to-cart' },
+        { action: 'wait', target: '1000' },
+        { action: 'navigate', target: '/cart' }
+      ]
+    },
+    {
+      name: 'View Profile',
+      steps: [
+        { action: 'navigate', target: '/' },
+        { action: 'click', selector: '.user-menu, a:has-text("Mon compte"), a:has-text("Profile")' },
+        { action: 'wait', target: '1000' }
+      ]
+    }
+  ]
+};
+
+// Pack v2 configuration for B2BShop E2E tests
+export const b2bshopPackV2 = {
+  version: 2,
+  name: 'b2bshop-e2e',
+  auth: {
+    ui: 'b2bshop-login',
+    profiles: {
+      'b2bshop-login': {
+        type: 'ui_login',
+        config: {
+          url: 'https://b2bshop.lovable.app/auth',
+          username: 'buyer@acme.test',
+          password: 'Buyer123!',
+          username_selector: '#email-signin',
+          password_selector: '#password-signin',
+          submit_selector: 'button[type="submit"]'
+        }
+      }
+    }
+  },
+  gates: {
+    ui_e2e: {
+      adapter: 'playwright-ui',
+      enabled: true,
+      auth: 'b2bshop-login',
+      config: {
+        baseUrl: 'https://b2bshop.lovable.app',
+        pages: [
+          'https://b2bshop.lovable.app/catalog',
+          'https://b2bshop.lovable.app/products',
+          'https://b2bshop.lovable.app/cart'
+        ]
+      }
+    }
+  }
+};
+
+export default b2bshopDataset;