ai-testing-swarm 0.1.14__tar.gz → 0.1.16__tar.gz

{ai_testing_swarm-0.1.14/src/ai_testing_swarm.egg-info → ai_testing_swarm-0.1.16}/PKG-INFO

@@ -1,13 +1,17 @@
 Metadata-Version: 2.4
 Name: ai-testing-swarm
-Version: 0.1.14
+Version: 0.1.16
 Summary: AI-powered testing swarm
 Author-email: Arif Shah <ashah7775@gmail.com>
 License: MIT
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.28
+Requires-Dist: PyYAML>=6.0
 Provides-Extra: openapi
 Requires-Dist: jsonschema>=4.0; extra == "openapi"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
 
 # AI Testing Swarm
 
@@ -68,7 +72,9 @@ A report is written under:
 - `./ai_swarm_reports/<METHOD>_<endpoint>/<METHOD>_<endpoint>_<timestamp>.<json|md|html>`
 
 Reports include:
-- per-test results
+- per-test results (including deterministic `risk_score` 0..100)
+- endpoint-level risk gate (`PASS`/`WARN`/`BLOCK`)
+- trend vs previous run for the same endpoint (risk delta + regressions)
 - summary counts by status code / failure type
 - optional AI summary (if enabled)
 
@@ -138,6 +144,28 @@ Then generates broad coverage across:
 
 ---
 
+## Auth matrix runner (multiple tokens/headers)
+
+To run the *same* request under multiple auth contexts (e.g., user/admin tokens), create `auth_matrix.yaml`:
+
+```yaml
+cases:
+  - name: user
+    headers:
+      Authorization: "Bearer USER_TOKEN"
+  - name: admin
+    headers:
+      Authorization: "Bearer ADMIN_TOKEN"
+```
+
+Run:
+
+```bash
+ai-test --input request.json --auth-matrix auth_matrix.yaml
+```
+
+Each auth case is written as a separate report using a `run_label` suffix (e.g. `__auth-user`).
+
 ## Safety mode (recommended for CI/demos)
 
 Mutation testing can be noisy and may accidentally stress a real environment.
@@ -189,6 +217,11 @@ Reports include:
 - `summary.counts_by_failure_type`
 - `summary.counts_by_status_code`
 - `summary.slow_tests` (based on SLA)
+- `meta.endpoint_risk_score` + `meta.gate_status`
+- `trend.*` (previous comparison if a prior report exists)
+
+A static dashboard index is generated at:
+- `./ai_swarm_reports/index.html` (latest JSON report per endpoint, sorted by regressions/risk)
 
 SLA threshold:
 - `AI_SWARM_SLA_MS` (default: `2000`)

ai_testing_swarm-0.1.14/PKG-INFO → ai_testing_swarm-0.1.16/README.md

@@ -1,14 +1,3 @@
-Metadata-Version: 2.4
-Name: ai-testing-swarm
-Version: 0.1.14
-Summary: AI-powered testing swarm
-Author-email: Arif Shah <ashah7775@gmail.com>
-License: MIT
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-Provides-Extra: openapi
-Requires-Dist: jsonschema>=4.0; extra == "openapi"
-
 # AI Testing Swarm
 
 AI Testing Swarm is a **super-advanced, mutation-driven API testing framework** (with optional OpenAPI + OpenAI augmentation) built on top of **pytest**.
@@ -68,7 +57,9 @@ A report is written under:
 - `./ai_swarm_reports/<METHOD>_<endpoint>/<METHOD>_<endpoint>_<timestamp>.<json|md|html>`
 
 Reports include:
-- per-test results
+- per-test results (including deterministic `risk_score` 0..100)
+- endpoint-level risk gate (`PASS`/`WARN`/`BLOCK`)
+- trend vs previous run for the same endpoint (risk delta + regressions)
 - summary counts by status code / failure type
 - optional AI summary (if enabled)
 
@@ -138,6 +129,28 @@ Then generates broad coverage across:
 
 ---
 
+## Auth matrix runner (multiple tokens/headers)
+
+To run the *same* request under multiple auth contexts (e.g., user/admin tokens), create `auth_matrix.yaml`:
+
+```yaml
+cases:
+  - name: user
+    headers:
+      Authorization: "Bearer USER_TOKEN"
+  - name: admin
+    headers:
+      Authorization: "Bearer ADMIN_TOKEN"
+```
+
+Run:
+
+```bash
+ai-test --input request.json --auth-matrix auth_matrix.yaml
+```
+
+Each auth case is written as a separate report using a `run_label` suffix (e.g. `__auth-user`).
+
 ## Safety mode (recommended for CI/demos)
 
 Mutation testing can be noisy and may accidentally stress a real environment.
@@ -189,6 +202,11 @@ Reports include:
 - `summary.counts_by_failure_type`
 - `summary.counts_by_status_code`
 - `summary.slow_tests` (based on SLA)
+- `meta.endpoint_risk_score` + `meta.gate_status`
+- `trend.*` (previous comparison if a prior report exists)
+
+A static dashboard index is generated at:
+- `./ai_swarm_reports/index.html` (latest JSON report per endpoint, sorted by regressions/risk)
 
 SLA threshold:
 - `AI_SWARM_SLA_MS` (default: `2000`)

{ai_testing_swarm-0.1.14 → ai_testing_swarm-0.1.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ai-testing-swarm"
-version = "0.1.14"
+version = "0.1.16"
 description = "AI-powered testing swarm"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -13,10 +13,18 @@ authors = [
 ]
 license = { text = "MIT" }
 
+dependencies = [
+  "requests>=2.28",
+  "PyYAML>=6.0",
+]
+
 [project.optional-dependencies]
 openapi = [
   "jsonschema>=4.0",
 ]
+dev = [
+  "pytest>=8.0",
+]
 
 [project.scripts]
 ai-test = "ai_testing_swarm.cli:main"

ai_testing_swarm-0.1.16/src/ai_testing_swarm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.16"

{ai_testing_swarm-0.1.14 → ai_testing_swarm-0.1.16}/src/ai_testing_swarm/cli.py

@@ -107,6 +107,29 @@ def main():
         help="Report format to write (default: json)",
     )
 
+    parser.add_argument(
+        "--auth-matrix",
+        default="",
+        help=(
+            "Optional path to auth_matrix.yaml/json to run the same endpoint under multiple auth headers. "
+            "Each case is reported separately via a run label suffix."
+        ),
+    )
+
+    # Batch1: risk gate thresholds (backward compatible defaults)
+    parser.add_argument(
+        "--gate-warn",
+        type=int,
+        default=30,
+        help="Gate WARN threshold for endpoint risk score (default: 30)",
+    )
+    parser.add_argument(
+        "--gate-block",
+        type=int,
+        default=80,
+        help="Gate BLOCK threshold for endpoint risk score (default: 80)",
+    )
+
     args = parser.parse_args()
 
     # ------------------------------------------------------------
@@ -127,24 +150,41 @@ def main():
         import os
         os.environ["AI_SWARM_PUBLIC_ONLY"] = "1"
 
-    decision, results = SwarmOrchestrator().run(request, report_format=args.report_format)
-
-    # ------------------------------------------------------------
-    # Console output
-    # ------------------------------------------------------------
-    print("\n=== RELEASE DECISION ===")
-    print(decision)
-
-    print("\n=== TEST RESULTS ===")
-    for r in results:
-        response = r.get("response", {})
-        status_code = response.get("status_code")
-
-        print(
-            f"{r.get('name'):25} "
-            f"{str(status_code):5} "
-            f"{r.get('reason')}"
+    orch = SwarmOrchestrator()
+
+    def _print_console(decision, results, *, label: str = ""):
+        if label:
+            print(f"\n=== AUTH CASE: {label} ===")
+        print("\n=== RELEASE DECISION ===")
+        print(decision)
+        print("\n=== TEST RESULTS ===")
+        for r in results:
+            response = r.get("response", {})
+            status_code = response.get("status_code")
+            print(f"{r.get('name'):25} {str(status_code):5} {r.get('reason')}")
+
+    if args.auth_matrix:
+        from ai_testing_swarm.core.auth_matrix import load_auth_matrix, merge_auth_headers
+
+        cases = load_auth_matrix(args.auth_matrix)
+        for c in cases:
+            req2 = merge_auth_headers(request, c)
+            decision, results = orch.run(
+                req2,
+                report_format=args.report_format,
+                gate_warn=args.gate_warn,
+                gate_block=args.gate_block,
+                run_label=f"auth-{c.name}",
+            )
+            _print_console(decision, results, label=c.name)
+    else:
+        decision, results = orch.run(
+            request,
+            report_format=args.report_format,
+            gate_warn=args.gate_warn,
+            gate_block=args.gate_block,
         )
+        _print_console(decision, results)
 
 
 if __name__ == "__main__":

ai_testing_swarm-0.1.16/src/ai_testing_swarm/core/auth_matrix.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+
+@dataclass(frozen=True)
+class AuthCase:
+    name: str
+    headers: dict[str, str]
+
+
+def _sanitize_case_name(name: str) -> str:
+    name = str(name or "").strip()
+    if not name:
+        return "case"
+    # Keep it filesystem-friendly.
+    out = []
+    for ch in name:
+        if ch.isalnum() or ch in ("-", "_", "."):
+            out.append(ch)
+        else:
+            out.append("-")
+    return "".join(out).strip("-") or "case"
+
+
+def load_auth_matrix(path: str | Path) -> list[AuthCase]:
+    """Load an auth matrix config (yaml/json).
+
+    Schema:
+      {
+        "cases": [
+          {"name": "user", "headers": {"Authorization": "Bearer ..."}},
+          {"name": "admin", "headers": {"Authorization": "Bearer ..."}}
+        ]
+      }
+
+    Notes:
+      - This is intentionally minimal and explicit.
+      - Headers are merged into the base request headers (case wins).
+    """
+
+    p = Path(path)
+    raw = p.read_text(encoding="utf-8")
+    if p.suffix.lower() in {".yaml", ".yml"}:
+        data = yaml.safe_load(raw) or {}
+    else:
+        data = json.loads(raw)
+
+    cases = data.get("cases") if isinstance(data, dict) else None
+    if not isinstance(cases, list) or not cases:
+        raise ValueError("auth matrix must contain a non-empty 'cases' list")
+
+    out: list[AuthCase] = []
+    for i, c in enumerate(cases):
+        if not isinstance(c, dict):
+            raise ValueError(f"auth case #{i} must be an object")
+        name = _sanitize_case_name(c.get("name") or f"case{i+1}")
+        headers = c.get("headers") or {}
+        if not isinstance(headers, dict):
+            raise ValueError(f"auth case '{name}' headers must be an object")
+        # stringify values (avoid accidental ints)
+        headers2 = {str(k): str(v) for k, v in headers.items() if v is not None}
+        out.append(AuthCase(name=name, headers=headers2))
+
+    # Ensure unique names
+    seen: set[str] = set()
+    uniq: list[AuthCase] = []
+    for c in out:
+        nm = c.name
+        if nm not in seen:
+            uniq.append(c)
+            seen.add(nm)
+        else:
+            j = 2
+            while f"{nm}-{j}" in seen:
+                j += 1
+            new = f"{nm}-{j}"
+            uniq.append(AuthCase(name=new, headers=c.headers))
+            seen.add(new)
+
+    return uniq
+
+
+def merge_auth_headers(request: dict, auth_case: AuthCase) -> dict:
+    req = dict(request)
+    base_headers = dict(req.get("headers") or {})
+    base_headers.update(auth_case.headers or {})
+    req["headers"] = base_headers
+    return req

ai_testing_swarm-0.1.16/src/ai_testing_swarm/core/risk.py

@@ -0,0 +1,161 @@
+"""Risk scoring for AI Testing Swarm.
+
+Batch1 additions:
+- Compute a numeric risk_score per test result (0..100)
+- Aggregate endpoint risk and drive gate thresholds (PASS/WARN/BLOCK)
+
+Design goals:
+- Backward compatible: consumers can ignore risk_score fields.
+- Deterministic and explainable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RiskThresholds:
+    """Thresholds for the endpoint gate.
+
+    Gate is computed from endpoint_risk_score (currently max test risk).
+
+    - PASS: score < warn
+    - WARN: warn <= score < block
+    - BLOCK: score >= block
+    """
+
+    warn: int = 30
+    block: int = 80
+
+
+# Keep aligned with orchestrator/release gate semantics
+EXPECTED_FAILURES: set[str] = {
+    "success",
+    "missing_param",
+    "invalid_param",
+    "security",
+    "method_not_allowed",
+    "not_found",
+    "content_negotiation",
+}
+
+RISKY_FAILURES: set[str] = {
+    "unknown",
+    "missing_param_accepted",
+    "null_param_accepted",
+    "invalid_param_accepted",
+    "headers_accepted",
+    "method_risk",
+}
+
+BLOCKING_FAILURES: set[str] = {
+    "auth_issue",
+    "infra",
+    "security_risk",
+    "server_error",
+}
+
+
+def clamp_int(x: int, lo: int, hi: int) -> int:
+    return lo if x < lo else hi if x > hi else x
+
+
+def compute_test_risk_score(result: dict, *, sla_ms: int | None = None) -> int:
+    """Compute a risk score for a single test result.
+
+    Inputs expected (best-effort):
+      - result['failure_type']
+      - result['status']
+      - result['mutation']['strategy'] (optional)
+      - result['response']['status_code']
+      - result['response']['elapsed_ms']
+      - result['response']['openapi_validation'] (list)
+
+    Returns: int in range 0..100.
+
+    Batch2: strategy-aware weighting.
+    The same failure_type can be more/less severe depending on the test strategy.
+    """
+
+    ft = str(result.get("failure_type") or "unknown")
+    status = str(result.get("status") or "")
+    mutation = result.get("mutation") or {}
+    strategy = str(mutation.get("strategy") or "").strip().lower()
+
+    resp = result.get("response") or {}
+    sc = resp.get("status_code")
+
+    # Base score from semantic classification.
+    if ft in EXPECTED_FAILURES:
+        base = 0
+    elif ft in RISKY_FAILURES:
+        base = 35
+    elif ft in BLOCKING_FAILURES:
+        base = 90
+    else:
+        # Unknown failure types are treated as high risk but not always a hard blocker.
+        base = 60
+
+    # Strategy-aware overrides (only when the strategy is known).
+    # These are designed to stay deterministic and explainable.
+    if strategy == "security" and ft == "security_risk":
+        base = max(base, 100)
+    if strategy in {"missing_param", "null_param", "invalid_param"} and ft.endswith("_accepted"):
+        # Validation bypass signals.
+        base = max(base, 80)
+    if strategy == "headers" and ft == "headers_accepted":
+        base = max(base, 55)
+    if strategy == "method_misuse" and ft == "method_risk":
+        base = max(base, 85)
+    if strategy == "auth" and ft == "auth_issue":
+        # Often indicates environment/config drift rather than product risk.
+        base = min(base, 70)
+
+    # Status-code adjustments (defense in depth)
+    if isinstance(sc, int):
+        if 500 <= sc:
+            base = max(base, 90)
+        elif sc in (401, 403):
+            base = max(base, 80)
+        elif 400 <= sc < 500:
+            # Client errors for negative tests are expected; keep base.
+            base = max(base, base)
+
+    # Explicit FAILED status implies something unexpected.
+    if status.upper() == "FAILED":
+        base = max(base, 70)
+
+    # OpenAPI validation issues are a small additive risk.
+    issues = resp.get("openapi_validation") or []
+    if isinstance(issues, list) and issues:
+        base += 10
+
+    # SLA breach is a small additive risk.
+    if sla_ms is not None:
+        elapsed = resp.get("elapsed_ms")
+        if isinstance(elapsed, int) and elapsed > sla_ms:
+            base += 5
+
+    return clamp_int(int(base), 0, 100)
+
+
+def compute_endpoint_risk_score(results: list[dict]) -> int:
+    """Aggregate endpoint risk score.
+
+    Current policy: endpoint risk is the max risk_score across tests.
+    (This makes gating stable and easy to interpret.)
+    """
+
+    scores = [r.get("risk_score") for r in (results or []) if isinstance(r.get("risk_score"), int)]
+    if not scores:
+        return 0
+    return int(max(scores))
+
+
+def gate_from_score(score: int, thresholds: RiskThresholds) -> str:
+    if score >= thresholds.block:
+        return "BLOCK"
+    if score >= thresholds.warn:
+        return "WARN"
+    return "PASS"

{ai_testing_swarm-0.1.14 → ai_testing_swarm-0.1.16}/src/ai_testing_swarm/orchestrator.py

@@ -5,6 +5,12 @@ from ai_testing_swarm.agents.learning_agent import LearningAgent
 from ai_testing_swarm.agents.release_gate_agent import ReleaseGateAgent
 from ai_testing_swarm.reporting.report_writer import write_report
 from ai_testing_swarm.core.safety import enforce_public_only
+from ai_testing_swarm.core.risk import (
+    RiskThresholds,
+    compute_test_risk_score,
+    compute_endpoint_risk_score,
+    gate_from_score,
+)
 
 import logging
 
@@ -40,8 +46,21 @@ class SwarmOrchestrator:
         self.learner = LearningAgent()
         self.release_gate = ReleaseGateAgent()
 
-    def run(self, request: dict, *, report_format: str = "json"):
-        """Runs the full AI testing swarm and returns (decision, results)."""
+    def run(
+        self,
+        request: dict,
+        *,
+        report_format: str = "json",
+        gate_warn: int = 30,
+        gate_block: int = 80,
+        run_label: str | None = None,
+    ):
+        """Runs the full AI testing swarm and returns (decision, results).
+
+        gate_warn/gate_block:
+          Thresholds for PASS/WARN/BLOCK gate based on endpoint risk.
+          (Kept optional for backward compatibility.)
+        """
 
         # Safety hook (currently no-op; kept for backward compatibility)
         enforce_public_only(request["url"])
@@ -78,6 +97,19 @@ class SwarmOrchestrator:
                     else "FAILED"
                 ),
             })
+
+            # Batch1: numeric risk score per test (0..100)
+            try:
+                from ai_testing_swarm.core.config import AI_SWARM_SLA_MS
+
+                execution_result["risk_score"] = compute_test_risk_score(
+                    execution_result,
+                    sla_ms=AI_SWARM_SLA_MS,
+                )
+            except Exception:
+                # Keep backwards compatibility: don't fail the run if scoring breaks.
+                execution_result["risk_score"] = 0
+
             # Optional learning step
             try:
                 self.learner.learn(test_name, classification)
@@ -97,14 +129,25 @@ class SwarmOrchestrator:
             results.append(results_by_name[t["name"]])
 
         # --------------------------------------------------------
-        # 3️⃣ RELEASE DECISION
+        # 3️⃣ RELEASE DECISION + RISK GATE
         # --------------------------------------------------------
         decision = self.release_gate.decide(results)
 
+        thresholds = RiskThresholds(warn=int(gate_warn), block=int(gate_block))
+        endpoint_risk_score = compute_endpoint_risk_score(results)
+        gate_status = gate_from_score(endpoint_risk_score, thresholds)
+
         # --------------------------------------------------------
-        # 4️⃣ WRITE JSON REPORT
+        # 4️⃣ WRITE REPORT
         # --------------------------------------------------------
-        meta = {"decision": decision}
+        meta = {
+            "decision": decision,
+            "gate_status": gate_status,
+            "gate_thresholds": {"warn": thresholds.warn, "block": thresholds.block},
+            "endpoint_risk_score": endpoint_risk_score,
+        }
+        if run_label:
+            meta["run_label"] = str(run_label)
 
         # Optional AI summary for humans (best-effort)
         try: