qa360 2.1.2 → 2.1.4

package/examples/packs-conformance/full.yml

@@ -0,0 +1,152 @@
+name: full-conformance-pack
+version: 1.0.0
+description: Full pack with all gates and comprehensive testing
+
+secrets:
+  - name: API_KEY
+    required: true
+  - name: DB_PASSWORD
+    required: false
+
+hooks:
+  beforeAll: |
+    echo "Full pack initialization"
+    docker --version || echo "Docker not available"
+  afterAll: |
+    echo "Full pack cleanup"
+    docker ps -q | xargs -r docker stop || echo "No containers to stop"
+
+steps:
+  - name: api-comprehensive
+    adapter: playwright-api
+    config:
+      url: "https://httpbin.org/post"
+      method: POST
+      headers:
+        Content-Type: "application/json"
+        Authorization: "Bearer ${API_KEY}"
+      body: |
+        {
+          "test": "comprehensive",
+          "timestamp": "${TIMESTAMP}",
+          "pack": "full-conformance"
+        }
+      expect:
+        status: 200
+        body_contains: ["test", "comprehensive"]
+    budgets:
+      duration_ms: 3000
+      
+  - name: ui-comprehensive
+    adapter: playwright-ui
+    config:
+      url: "https://httpbin.org/forms/post"
+      actions:
+        - type: screenshot
+          name: "forms-initial"
+        - type: fill
+          selector: "input[name='custname']"
+          value: "QA360 Test User"
+        - type: fill
+          selector: "input[name='custtel']"
+          value: "1234567890"
+        - type: fill
+          selector: "input[name='custemail']"
+          value: "test@qa360.dev"
+        - type: select
+          selector: "select[name='size']"
+          value: "medium"
+        - type: screenshot
+          name: "forms-filled"
+        - type: click
+          selector: "input[type='submit']"
+        - type: wait
+          timeout: 2000
+        - type: screenshot
+          name: "forms-submitted"
+    budgets:
+      duration_ms: 15000
+      
+  - name: perf-comprehensive
+    adapter: k6-perf
+    config:
+      script: |
+        import http from 'k6/http';
+        import { check } from 'k6';
+        
+        export let options = {
+          stages: [
+            { duration: '30s', target: 5 },
+            { duration: '30s', target: 10 },
+            { duration: '30s', target: 5 },
+          ],
+        };
+        
+        export default function () {
+          let response = http.get('https://httpbin.org/delay/1');
+          check(response, {
+            'status is 200': (r) => r.status === 200,
+            'response time < 2000ms': (r) => r.timings.duration < 2000,
+          });
+        }
+    budgets:
+      p95_ms: 2500
+      p99_ms: 3000
+      error_rate: 0.05
+      
+  - name: sast-comprehensive
+    adapter: semgrep-sast
+    config:
+      rules: [
+        "javascript.lang.security.audit.xss",
+        "javascript.lang.security.audit.sqli",
+        "javascript.lang.security.audit.path-traversal",
+        "typescript.lang.security.audit.crypto.weak-crypto"
+      ]
+      paths: ["./core/src", "./cli/src", "./packages"]
+      exclude_paths: ["node_modules", "dist", "build"]
+    budgets:
+      high_findings: 2
+      critical_findings: 0
+      medium_findings: 10
+      
+  - name: deps-comprehensive
+    adapter: osv-deps
+    config:
+      manifest_paths: ["package.json", "core/package.json", "cli/package.json"]
+      severity_threshold: "MODERATE"
+    budgets:
+      high_findings: 3
+      critical_findings: 0
+      
+  - name: secrets-comprehensive
+    adapter: gitleaks-secrets
+    config:
+      paths: ["./"]
+      exclude_paths: ["node_modules", ".git", "artifacts"]
+      rules: ["api-key", "password", "token", "certificate"]
+    budgets:
+      findings: 0
+      
+  - name: dast-comprehensive
+    adapter: zap-dast
+    config:
+      target: "https://httpbin.org"
+      scan_type: "full"
+      auth:
+        type: "none"
+      spider_minutes: 2
+      scan_minutes: 5
+    budgets:
+      high_findings: 10
+      critical_findings: 1
+      
+  - name: a11y-comprehensive
+    adapter: playwright-a11y
+    config:
+      url: "https://httpbin.org/forms/post"
+      rules: ["wcag2a", "wcag2aa", "wcag21aa"]
+      include_selectors: ["form", "input", "button"]
+    budgets:
+      violations: 20
+      critical_violations: 0

package/examples/packs-conformance/heavy-artifacts.yml

@@ -0,0 +1,152 @@
+name: heavy-artifacts-conformance-pack
+version: 1.0.0
+description: Pack testing large artifact generation and storage
+
+hooks:
+  beforeAll: |
+    echo "Heavy artifacts pack - testing large output handling"
+    mkdir -p /tmp/qa360-heavy-test
+  afterAll: |
+    echo "Heavy artifacts cleanup"
+    rm -rf /tmp/qa360-heavy-test || true
+
+steps:
+  - name: api-large-response
+    adapter: playwright-api
+    config:
+      url: "https://httpbin.org/json"
+      method: GET
+      save_response: true
+      expect:
+        status: 200
+    budgets:
+      duration_ms: 5000
+      response_size_mb: 1
+      
+  - name: ui-heavy-screenshots
+    adapter: playwright-ui
+    config:
+      url: "https://httpbin.org/forms/post"
+      actions:
+        - type: screenshot
+          name: "heavy-initial"
+          full_page: true
+        - type: fill
+          selector: "input[name='custname']"
+          value: "QA360 Heavy Test User with Very Long Name That Exceeds Normal Limits"
+        - type: screenshot
+          name: "heavy-filled"
+          full_page: true
+        - type: fill
+          selector: "textarea[name='comments']"
+          value: |
+            This is a very long comment that simulates heavy text input.
+            Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+            Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+            Ut enim ad minim veniam, quis nostrud exercitation ullamco.
+            Laboris nisi ut aliquip ex ea commodo consequat.
+            Duis aute irure dolor in reprehenderit in voluptate velit esse.
+            Cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat.
+            Cupidatat non proident, sunt in culpa qui officia deserunt.
+            Mollit anim id est laborum. Sed ut perspiciatis unde omnis.
+            Iste natus error sit voluptatem accusantium doloremque laudantium.
+        - type: screenshot
+          name: "heavy-complete"
+          full_page: true
+        - type: wait
+          timeout: 2000
+        - type: screenshot
+          name: "heavy-final"
+          full_page: true
+    budgets:
+      duration_ms: 20000
+      screenshot_size_mb: 10
+      
+  - name: perf-heavy-load
+    adapter: k6-perf
+    config:
+      script: |
+        import http from 'k6/http';
+        import { check } from 'k6';
+        
+        export let options = {
+          stages: [
+            { duration: '60s', target: 20 },
+            { duration: '60s', target: 50 },
+            { duration: '60s', target: 20 },
+          ],
+        };
+        
+        export default function () {
+          // Generate large payload
+          let largePayload = {
+            data: 'x'.repeat(10000), // 10KB payload
+            timestamp: new Date().toISOString(),
+            iteration: __ITER,
+            vu: __VU
+          };
+          
+          let response = http.post('https://httpbin.org/post', JSON.stringify(largePayload), {
+            headers: { 'Content-Type': 'application/json' },
+          });
+          
+          check(response, {
+            'status is 200': (r) => r.status === 200,
+            'response time < 5000ms': (r) => r.timings.duration < 5000,
+          });
+        }
+    budgets:
+      p95_ms: 6000
+      p99_ms: 10000
+      error_rate: 0.1
+      
+  - name: sast-heavy-scan
+    adapter: semgrep-sast
+    config:
+      rules: [
+        "javascript.lang.security",
+        "typescript.lang.security",
+        "generic.secrets.security"
+      ]
+      paths: ["./core", "./cli", "./packages"]
+      exclude_paths: ["node_modules", "dist", "build", "artifacts"]
+      output_format: "json"
+      save_full_output: true
+    budgets:
+      duration_ms: 120000  # 2 minutes for heavy scan
+      high_findings: 50
+      critical_findings: 5
+      output_size_mb: 20
+      
+  - name: deps-heavy-analysis
+    adapter: osv-deps
+    config:
+      manifest_paths: [
+        "package.json",
+        "core/package.json", 
+        "cli/package.json",
+        "packages/mcp-server-qa360/package.json"
+      ]
+      severity_threshold: "LOW"
+      include_dev_deps: true
+      save_full_report: true
+    budgets:
+      duration_ms: 60000
+      high_findings: 100
+      critical_findings: 10
+      output_size_mb: 15
+      
+  - name: dast-heavy-scan
+    adapter: zap-dast
+    config:
+      target: "https://httpbin.org"
+      scan_type: "full"
+      spider_minutes: 5
+      scan_minutes: 10
+      save_full_report: true
+      include_response_bodies: true
+    budgets:
+      duration_ms: 900000  # 15 minutes
+      high_findings: 100
+      critical_findings: 20
+      output_size_mb: 50

package/examples/packs-conformance/minimal.yml

@@ -0,0 +1,71 @@
+name: minimal-conformance-pack
+version: 1.0.0
+description: Minimal pack for universal compatibility testing
+
+hooks:
+  beforeAll: |
+    echo "Minimal pack beforeAll hook"
+  afterAll: |
+    echo "Minimal pack afterAll hook"
+
+steps:
+  - name: api-minimal
+    adapter: playwright-api
+    config:
+      url: "https://httpbin.org/get"
+      method: GET
+      expect:
+        status: 200
+    budgets:
+      duration_ms: 5000
+    
+  - name: ui-minimal
+    adapter: playwright-ui
+    config:
+      url: "https://httpbin.org"
+      actions:
+        - type: screenshot
+          name: "httpbin-homepage"
+    budgets:
+      duration_ms: 10000
+
+  - name: perf-minimal
+    adapter: k6-perf
+    config:
+      script: |
+        import http from 'k6/http';
+        export default function () {
+          http.get('https://httpbin.org/get');
+        }
+      options:
+        duration: '10s'
+        vus: 1
+    budgets:
+      p95_ms: 1000
+      error_rate: 0.01
+
+  - name: sast-minimal
+    adapter: semgrep-sast
+    config:
+      rules: ["javascript.lang.security.audit.xss"]
+      paths: ["./"]
+    budgets:
+      high_findings: 0
+      critical_findings: 0
+
+  - name: dast-minimal
+    adapter: zap-dast
+    config:
+      target: "https://httpbin.org"
+      scan_type: "baseline"
+    budgets:
+      high_findings: 5
+      critical_findings: 0
+
+  - name: a11y-minimal
+    adapter: playwright-a11y
+    config:
+      url: "https://httpbin.org"
+      rules: ["wcag2a"]
+    budgets:
+      violations: 10

package/examples/packs-conformance/secrets-missing.yml

@@ -0,0 +1,97 @@
+name: secrets-missing-conformance-pack
+version: 1.0.0
+description: Pack testing secrets handling and missing credentials
+
+secrets:
+  - name: REQUIRED_API_KEY
+    required: true
+  - name: OPTIONAL_TOKEN
+    required: false
+  - name: DATABASE_PASSWORD
+    required: true
+  - name: ENCRYPTION_KEY
+    required: false
+
+hooks:
+  beforeAll: |
+    echo "Secrets pack - testing credential management"
+    echo "Required secrets: REQUIRED_API_KEY, DATABASE_PASSWORD"
+  afterAll: |
+    echo "Secrets pack cleanup - no credentials exposed"
+
+steps:
+  - name: api-with-required-secret
+    adapter: playwright-api
+    config:
+      url: "https://httpbin.org/bearer"
+      method: GET
+      headers:
+        Authorization: "Bearer ${REQUIRED_API_KEY}"
+      expect:
+        status: 200
+    budgets:
+      duration_ms: 3000
+      
+  - name: api-with-optional-secret
+    adapter: playwright-api
+    config:
+      url: "https://httpbin.org/headers"
+      method: GET
+      headers:
+        X-Optional-Token: "${OPTIONAL_TOKEN}"
+        X-Database-Pass: "${DATABASE_PASSWORD}"
+      expect:
+        status: 200
+    budgets:
+      duration_ms: 3000
+      
+  - name: ui-with-credentials
+    adapter: playwright-ui
+    config:
+      url: "https://httpbin.org/basic-auth/user/pass"
+      auth:
+        username: "user"
+        password: "${DATABASE_PASSWORD}"
+      actions:
+        - type: screenshot
+          name: "authenticated-page"
+    budgets:
+      duration_ms: 8000
+      
+  - name: perf-with-auth
+    adapter: k6-perf
+    config:
+      script: |
+        import http from 'k6/http';
+        export default function () {
+          let params = {
+            headers: {
+              'Authorization': 'Bearer ${REQUIRED_API_KEY}',
+              'X-Encryption-Key': '${ENCRYPTION_KEY}',
+            },
+          };
+          http.get('https://httpbin.org/headers', params);
+        }
+      options:
+        duration: '10s'
+        vus: 2
+    budgets:
+      p95_ms: 2000
+      error_rate: 0.1
+      
+  - name: sast-secrets-scan
+    adapter: semgrep-sast
+    config:
+      rules: ["generic.secrets.security.detected-private-key"]
+      paths: ["./examples/packs-conformance"]
+    budgets:
+      high_findings: 0
+      critical_findings: 0
+      
+  - name: secrets-detection
+    adapter: gitleaks-secrets
+    config:
+      paths: ["./examples/packs-conformance"]
+      rules: ["api-key", "password", "token"]
+    budgets:
+      findings: 0  # Should find no hardcoded secrets

package/examples/packs-conformance/timeouts.yml

@@ -0,0 +1,77 @@
+name: timeouts-conformance-pack
+version: 1.0.0
+description: Pack testing timeout handling and budget violations
+
+hooks:
+  beforeAll: |
+    echo "Timeouts pack - testing budget violations"
+  afterAll: |
+    echo "Timeouts pack cleanup"
+
+steps:
+  - name: api-timeout-soft
+    adapter: playwright-api
+    config:
+      url: "https://httpbin.org/delay/3"
+      method: GET
+      timeout: 5000
+    budgets:
+      duration_ms: 2000  # Soft budget violation
+      
+  - name: api-timeout-hard
+    adapter: playwright-api
+    config:
+      url: "https://httpbin.org/delay/10"
+      method: GET
+      timeout: 15000
+    budgets:
+      duration_ms: 5000  # Hard budget violation
+      
+  - name: ui-timeout-selector
+    adapter: playwright-ui
+    config:
+      url: "https://httpbin.org"
+      actions:
+        - type: click
+          selector: "#non-existent-element"
+          timeout: 3000
+        - type: screenshot
+          name: "timeout-test"
+    budgets:
+      duration_ms: 8000
+      
+  - name: perf-timeout-load
+    adapter: k6-perf
+    config:
+      script: |
+        import http from 'k6/http';
+        export let options = {
+          duration: '20s',
+          vus: 1,
+        };
+        export default function () {
+          http.get('https://httpbin.org/delay/5');
+        }
+    budgets:
+      p95_ms: 3000  # Will be exceeded
+      error_rate: 0.01
+      
+  - name: sast-timeout-large
+    adapter: semgrep-sast
+    config:
+      rules: ["javascript.lang.security"]
+      paths: ["./"]
+      timeout: 30
+    budgets:
+      duration_ms: 15000
+      high_findings: 100  # Generous budget
+      
+  - name: dast-timeout-scan
+    adapter: zap-dast
+    config:
+      target: "https://httpbin.org"
+      scan_type: "baseline"
+      timeout: 60
+    budgets:
+      duration_ms: 30000  # Tight budget for DAST
+      high_findings: 20

package/examples/proofs/e2e-playwright-proof.json

@@ -0,0 +1,75 @@
+{
+  "spec": "qa360.proof.v1",
+  "run": {
+    "id": "660e9511-f3ac-52e5-b827-557766551111",
+    "startedAt": "2025-10-26T14:20:00Z",
+    "finishedAt": "2025-10-26T14:22:15Z",
+    "environment": {
+      "os": "darwin",
+      "node": "20.19.0",
+      "arch": "arm64",
+      "ci": false
+    },
+    "packHash": "sha256-b2c3d4e5f6a78901234567890123456789012345678901234567890123456bcd",
+    "ciContext": {
+      "provider": null
+    }
+  },
+  "artifacts": [
+    {
+      "name": "ui-report.html",
+      "sha256": "sha256-2345678901bcdef2345678901bcdef2345678901bcdef2345678901bcdef234",
+      "size": 45678,
+      "path": ".qa360/artifacts/ui-report.html"
+    },
+    {
+      "name": "screenshot-homepage.png",
+      "sha256": "sha256-3456789012cdef3456789012cdef3456789012cdef3456789012cdef34567890",
+      "size": 234567,
+      "path": ".qa360/artifacts/screenshot-homepage.png"
+    },
+    {
+      "name": "screenshot-login.png",
+      "sha256": "sha256-456789013def456789013def456789013def456789013def456789013def4567",
+      "size": 198765,
+      "path": ".qa360/artifacts/screenshot-login.png"
+    }
+  ],
+  "results": {
+    "trustScore": 85,
+    "gates": [
+      {
+        "name": "ui",
+        "status": "pass",
+        "metrics": {
+          "buttons_tested": 12,
+          "forms_tested": 3,
+          "links_tested": 45,
+          "coverage": 0.78
+        }
+      },
+      {
+        "name": "a11y",
+        "status": "pass",
+        "metrics": {
+          "violations": 2,
+          "wcag_level": "AA",
+          "score": 92
+        }
+      }
+    ]
+  },
+  "signing": {
+    "algo": "ed25519",
+    "signerId": "local@qa360",
+    "timestamp": {
+      "type": "none",
+      "token": null
+    },
+    "identity": {
+      "type": "none",
+      "evidence": null
+    }
+  },
+  "signature": "VGhpcyBpcyBhbm90aGVyIGZha2Ugc2lnbmF0dXJlIGZvciBQbGF5d3JpZ2h0IGV4YW1wbGUuCg=="
+}

package/examples/proofs/httpbin-proof.json

@@ -0,0 +1,69 @@
+{
+  "spec": "qa360.proof.v1",
+  "run": {
+    "id": "550e8400-e29b-41d4-a716-446655440000",
+    "startedAt": "2025-10-26T12:34:56Z",
+    "finishedAt": "2025-10-26T12:35:42Z",
+    "environment": {
+      "os": "linux",
+      "node": "20.19.0",
+      "arch": "x64",
+      "ci": false
+    },
+    "packHash": "sha256-a1b2c3d4e5f6789012345678901234567890123456789012345678901234abcd",
+    "ciContext": {
+      "provider": null
+    }
+  },
+  "artifacts": [
+    {
+      "name": "api-report.html",
+      "sha256": "sha256-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
+      "size": 12345,
+      "path": ".qa360/artifacts/api-report.html"
+    },
+    {
+      "name": "api-report.json",
+      "sha256": "sha256-abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890",
+      "size": 5678,
+      "path": ".qa360/artifacts/api-report.json"
+    }
+  ],
+  "results": {
+    "trustScore": 92,
+    "gates": [
+      {
+        "name": "api_smoke",
+        "status": "pass",
+        "metrics": {
+          "total_requests": 10,
+          "success_rate": 1.0,
+          "p95_ms": 142,
+          "p99_ms": 280
+        }
+      },
+      {
+        "name": "perf",
+        "status": "pass",
+        "metrics": {
+          "p95_ms": 142,
+          "p99_ms": 280,
+          "throughput_rps": 100
+        }
+      }
+    ]
+  },
+  "signing": {
+    "algo": "ed25519",
+    "signerId": "local@qa360",
+    "timestamp": {
+      "type": "none",
+      "token": null
+    },
+    "identity": {
+      "type": "none",
+      "evidence": null
+    }
+  },
+  "signature": "SGVsbG8gV29ybGQhIFRoaXMgaXMgYSBmYWtlIHNpZ25hdHVyZSBmb3IgZGVtbyBwdXJwb3Nlcy4K=="
+}