ai-testing-swarm 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

ai_testing_swarm/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.13"
+__version__ = "0.1.15"

ai_testing_swarm/agents/execution_agent.py

@@ -137,6 +137,27 @@ class ExecutionAgent:
                 except Exception:
                     body_snippet = resp.text
 
+                openapi_validation = []
+                try:
+                    ctx = request.get("_openapi") or {}
+                    if isinstance(ctx, dict) and ctx.get("spec") and ctx.get("path") and ctx.get("method"):
+                        # If method was mutated to something else, we can't reliably map to an OpenAPI op.
+                        if str(ctx.get("method")).upper() == str(method).upper():
+                            from ai_testing_swarm.core.openapi_validator import validate_openapi_response
+
+                            issues = validate_openapi_response(
+                                spec=ctx["spec"],
+                                path=ctx["path"],
+                                method=ctx["method"],
+                                status_code=resp.status_code,
+                                response_headers=dict(resp.headers),
+                                response_json=body_snippet if isinstance(body_snippet, (dict, list)) else None,
+                            )
+                            openapi_validation = [i.__dict__ for i in issues]
+                except Exception:
+                    # Best-effort only; never fail execution on validator issues.
+                    openapi_validation = []
+
                 return {
                     "name": test["name"],
                     "mutation": mutation,
@@ -151,7 +172,9 @@ class ExecutionAgent:
                         "status_code": resp.status_code,
                         "elapsed_ms": elapsed_ms,
                         "attempt": attempt,
+                        "headers": dict(resp.headers),
                         "body_snippet": body_snippet,
+                        "openapi_validation": openapi_validation,
                     },
                 }
 

ai_testing_swarm/cli.py

@@ -31,7 +31,7 @@ def normalize_request(payload: dict) -> dict:
         from ai_testing_swarm.core.openapi_loader import load_openapi, build_request_from_openapi
 
         spec = load_openapi(payload["openapi"])
-        return build_request_from_openapi(
+        req = build_request_from_openapi(
             spec,
             path=payload["path"],
             method=payload["method"],
@@ -40,6 +40,14 @@ def normalize_request(payload: dict) -> dict:
             query_params=payload.get("query_params") or {},
             body=payload.get("body"),
         )
+        # Attach OpenAPI context for optional response validation/reporting.
+        req["_openapi"] = {
+            "source": payload["openapi"],
+            "path": payload["path"],
+            "method": str(payload["method"]).upper(),
+            "spec": spec,
+        }
+        return req
 
     # Case 2: already normalized
     required_keys = {"method", "url"}
@@ -92,6 +100,27 @@ def main():
         help="Safety: allow only public test hosts (httpbin/postman-echo/reqres) for this run",
     )
 
+    parser.add_argument(
+        "--report-format",
+        default="json",
+        choices=["json", "md", "html"],
+        help="Report format to write (default: json)",
+    )
+
+    # Batch1: risk gate thresholds (backward compatible defaults)
+    parser.add_argument(
+        "--gate-warn",
+        type=int,
+        default=30,
+        help="Gate WARN threshold for endpoint risk score (default: 30)",
+    )
+    parser.add_argument(
+        "--gate-block",
+        type=int,
+        default=80,
+        help="Gate BLOCK threshold for endpoint risk score (default: 80)",
+    )
+
     args = parser.parse_args()
 
     # ------------------------------------------------------------
@@ -112,7 +141,12 @@ def main():
         import os
         os.environ["AI_SWARM_PUBLIC_ONLY"] = "1"
 
-    decision, results = SwarmOrchestrator().run(request)
+    decision, results = SwarmOrchestrator().run(
+        request,
+        report_format=args.report_format,
+        gate_warn=args.gate_warn,
+        gate_block=args.gate_block,
+    )
 
     # ------------------------------------------------------------
     # Console output

ai_testing_swarm/core/openapi_validator.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class OpenAPIValidationIssue:
+    type: str
+    message: str
+    details: dict | None = None
+
+
+def _matches_status_key(status_code: int, key: str) -> bool:
+    """OpenAPI response keys can be explicit ("200"), wildcard ("2XX"), or "default"."""
+    key = str(key).strip().upper()
+    if key == "DEFAULT":
+        return True
+    if len(key) == 3 and key.endswith("XX") and key[0].isdigit():
+        return int(key[0]) == int(status_code / 100)
+    return key == str(status_code)
+
+
+def _get_operation(spec: dict, *, path: str, method: str) -> dict | None:
+    paths = spec.get("paths") or {}
+    op = (paths.get(path) or {}).get(str(method).lower())
+    if isinstance(op, dict):
+        return op
+    return None
+
+
+def validate_openapi_response(
+    *,
+    spec: dict,
+    path: str,
+    method: str,
+    status_code: int | None,
+    response_headers: dict | None = None,
+    response_json=None,
+) -> list[OpenAPIValidationIssue]:
+    """Validate a response against an OpenAPI operation.
+
+    - Status code must be declared in operation.responses (supports 2XX and default)
+    - If response appears to be JSON and jsonschema is installed, validate body
+
+    Returns a list of issues (empty => OK or validation skipped).
+    """
+
+    issues: list[OpenAPIValidationIssue] = []
+
+    op = _get_operation(spec, path=path, method=method)
+    if not op:
+        return [
+            OpenAPIValidationIssue(
+                type="openapi_operation_missing",
+                message=f"Operation not found in spec: {str(method).upper()} {path}",
+            )
+        ]
+
+    responses = op.get("responses") or {}
+    if status_code is None:
+        # Network errors etc. Not an OpenAPI mismatch.
+        return issues
+
+    # --------------------
+    # Status validation
+    # --------------------
+    declared_keys = [str(k) for k in responses.keys()]
+    if declared_keys:
+        if not any(_matches_status_key(int(status_code), k) for k in declared_keys):
+            issues.append(
+                OpenAPIValidationIssue(
+                    type="openapi_status",
+                    message=f"Status {status_code} not declared in OpenAPI responses: {declared_keys}",
+                    details={"status_code": status_code, "declared": declared_keys},
+                )
+            )
+
+    # --------------------
+    # JSON schema validation (optional)
+    # --------------------
+    ct = ""
+    if response_headers and isinstance(response_headers, dict):
+        for k, v in response_headers.items():
+            if str(k).lower() == "content-type" and v:
+                ct = str(v).lower()
+                break
+
+    is_json = isinstance(response_json, (dict, list))
+    if not is_json:
+        # if body wasn't parsed as JSON, skip schema validation
+        return issues
+
+    if ct and "json" not in ct:
+        # Respect explicit non-JSON content-type
+        return issues
+
+    # Find best matching response schema: exact > 2XX > default
+    chosen_resp: dict | None = None
+    for k in (str(status_code), f"{int(status_code/100)}XX", "default"):
+        if k in responses:
+            chosen_resp = responses.get(k)
+            break
+
+    if not isinstance(chosen_resp, dict):
+        return issues
+
+    content = chosen_resp.get("content") or {}
+    schema = None
+    if isinstance(content, dict):
+        app_json = content.get("application/json") or content.get("application/*+json")
+        if isinstance(app_json, dict):
+            schema = app_json.get("schema")
+
+    if not schema:
+        return issues
+
+    try:
+        import jsonschema  # type: ignore
+
+        # Resolve in-document refs like #/components/schemas/X
+        resolver = jsonschema.RefResolver.from_schema(spec)  # type: ignore[attr-defined]
+        validator_cls = jsonschema.validators.validator_for(schema)  # type: ignore[attr-defined]
+        validator_cls.check_schema(schema)
+        validator = validator_cls(schema, resolver=resolver)
+        errors = sorted(validator.iter_errors(response_json), key=lambda e: list(e.path))
+        if errors:
+            # Keep just the first few errors for signal.
+            details = {
+                "error_count": len(errors),
+                "errors": [
+                    {
+                        "message": e.message,
+                        "path": list(e.path),
+                        "schema_path": list(e.schema_path),
+                    }
+                    for e in errors[:5]
+                ],
+            }
+            issues.append(
+                OpenAPIValidationIssue(
+                    type="openapi_schema",
+                    message="Response JSON does not match OpenAPI schema",
+                    details=details,
+                )
+            )
+    except ImportError:
+        # Optional dependency not installed: status validation still works.
+        return issues
+    except Exception as e:
+        issues.append(
+            OpenAPIValidationIssue(
+                type="openapi_schema_error",
+                message=f"OpenAPI schema validation failed: {e}",
+            )
+        )
+
+    return issues

ai_testing_swarm/core/risk.py

@@ -0,0 +1,139 @@
+"""Risk scoring for AI Testing Swarm.
+
+Batch1 additions:
+- Compute a numeric risk_score per test result (0..100)
+- Aggregate endpoint risk and drive gate thresholds (PASS/WARN/BLOCK)
+
+Design goals:
+- Backward compatible: consumers can ignore risk_score fields.
+- Deterministic and explainable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RiskThresholds:
+    """Thresholds for the endpoint gate.
+
+    Gate is computed from endpoint_risk_score (currently max test risk).
+
+    - PASS: score < warn
+    - WARN: warn <= score < block
+    - BLOCK: score >= block
+    """
+
+    warn: int = 30
+    block: int = 80
+
+
+# Keep aligned with orchestrator/release gate semantics
+EXPECTED_FAILURES: set[str] = {
+    "success",
+    "missing_param",
+    "invalid_param",
+    "security",
+    "method_not_allowed",
+    "not_found",
+    "content_negotiation",
+}
+
+RISKY_FAILURES: set[str] = {
+    "unknown",
+    "missing_param_accepted",
+    "null_param_accepted",
+    "invalid_param_accepted",
+    "headers_accepted",
+    "method_risk",
+}
+
+BLOCKING_FAILURES: set[str] = {
+    "auth_issue",
+    "infra",
+    "security_risk",
+    "server_error",
+}
+
+
+def clamp_int(x: int, lo: int, hi: int) -> int:
+    return lo if x < lo else hi if x > hi else x
+
+
+def compute_test_risk_score(result: dict, *, sla_ms: int | None = None) -> int:
+    """Compute a risk score for a single test result.
+
+    Inputs expected (best-effort):
+      - result['failure_type']
+      - result['status']
+      - result['response']['status_code']
+      - result['response']['elapsed_ms']
+      - result['response']['openapi_validation'] (list)
+
+    Returns: int in range 0..100.
+    """
+
+    ft = str(result.get("failure_type") or "unknown")
+    status = str(result.get("status") or "")
+    resp = result.get("response") or {}
+    sc = resp.get("status_code")
+
+    # Base score from semantic classification.
+    if ft in EXPECTED_FAILURES:
+        base = 0
+    elif ft in RISKY_FAILURES:
+        base = 35
+    elif ft in BLOCKING_FAILURES:
+        base = 90
+    else:
+        # Unknown failure types are treated as high risk but not always a hard blocker.
+        base = 60
+
+    # Status-code adjustments (defense in depth)
+    if isinstance(sc, int):
+        if 500 <= sc:
+            base = max(base, 90)
+        elif sc in (401, 403):
+            base = max(base, 80)
+        elif 400 <= sc < 500:
+            # Client errors for negative tests are expected; keep base.
+            base = max(base, base)
+
+    # Explicit FAILED status implies something unexpected.
+    if status.upper() == "FAILED":
+        base = max(base, 70)
+
+    # OpenAPI validation issues are a small additive risk.
+    issues = resp.get("openapi_validation") or []
+    if isinstance(issues, list) and issues:
+        base += 10
+
+    # SLA breach is a small additive risk.
+    if sla_ms is not None:
+        elapsed = resp.get("elapsed_ms")
+        if isinstance(elapsed, int) and elapsed > sla_ms:
+            base += 5
+
+    return clamp_int(int(base), 0, 100)
+
+
+def compute_endpoint_risk_score(results: list[dict]) -> int:
+    """Aggregate endpoint risk score.
+
+    Current policy: endpoint risk is the max risk_score across tests.
+    (This makes gating stable and easy to interpret.)
+    """
+
+    scores = [r.get("risk_score") for r in (results or []) if isinstance(r.get("risk_score"), int)]
+    if not scores:
+        return 0
+    return int(max(scores))
+
+
+def gate_from_score(score: int, thresholds: RiskThresholds) -> str:
+    if score >= thresholds.block:
+        return "BLOCK"
+    if score >= thresholds.warn:
+        return "WARN"
+    return "PASS"

ai_testing_swarm/orchestrator.py

@@ -5,6 +5,12 @@ from ai_testing_swarm.agents.learning_agent import LearningAgent
 from ai_testing_swarm.agents.release_gate_agent import ReleaseGateAgent
 from ai_testing_swarm.reporting.report_writer import write_report
 from ai_testing_swarm.core.safety import enforce_public_only
+from ai_testing_swarm.core.risk import (
+    RiskThresholds,
+    compute_test_risk_score,
+    compute_endpoint_risk_score,
+    gate_from_score,
+)
 
 import logging
 
@@ -40,8 +46,20 @@ class SwarmOrchestrator:
         self.learner = LearningAgent()
         self.release_gate = ReleaseGateAgent()
 
-    def run(self, request: dict):
-        """Runs the full AI testing swarm and returns (decision, results)."""
+    def run(
+        self,
+        request: dict,
+        *,
+        report_format: str = "json",
+        gate_warn: int = 30,
+        gate_block: int = 80,
+    ):
+        """Runs the full AI testing swarm and returns (decision, results).
+
+        gate_warn/gate_block:
+          Thresholds for PASS/WARN/BLOCK gate based on endpoint risk.
+          (Kept optional for backward compatibility.)
+        """
 
         # Safety hook (currently no-op; kept for backward compatibility)
         enforce_public_only(request["url"])
@@ -78,6 +96,19 @@ class SwarmOrchestrator:
                     else "FAILED"
                 ),
             })
+
+            # Batch1: numeric risk score per test (0..100)
+            try:
+                from ai_testing_swarm.core.config import AI_SWARM_SLA_MS
+
+                execution_result["risk_score"] = compute_test_risk_score(
+                    execution_result,
+                    sla_ms=AI_SWARM_SLA_MS,
+                )
+            except Exception:
+                # Keep backwards compatibility: don't fail the run if scoring breaks.
+                execution_result["risk_score"] = 0
+
             # Optional learning step
             try:
                 self.learner.learn(test_name, classification)
@@ -97,14 +128,23 @@ class SwarmOrchestrator:
             results.append(results_by_name[t["name"]])
 
         # --------------------------------------------------------
-        # 3️⃣ RELEASE DECISION
+        # 3️⃣ RELEASE DECISION + RISK GATE
         # --------------------------------------------------------
         decision = self.release_gate.decide(results)
 
+        thresholds = RiskThresholds(warn=int(gate_warn), block=int(gate_block))
+        endpoint_risk_score = compute_endpoint_risk_score(results)
+        gate_status = gate_from_score(endpoint_risk_score, thresholds)
+
         # --------------------------------------------------------
-        # 4️⃣ WRITE JSON REPORT
+        # 4️⃣ WRITE REPORT
         # --------------------------------------------------------
-        meta = {"decision": decision}
+        meta = {
+            "decision": decision,
+            "gate_status": gate_status,
+            "gate_thresholds": {"warn": thresholds.warn, "block": thresholds.block},
+            "endpoint_risk_score": endpoint_risk_score,
+        }
 
         # Optional AI summary for humans (best-effort)
         try:
@@ -122,8 +162,8 @@ class SwarmOrchestrator:
         except Exception as e:
             meta["ai_summary_error"] = str(e)
 
-        report_path = write_report(request, results, meta=meta)
-        logger.info("📄 Swarm JSON report written to: %s", report_path)
-        print(f"📄 Swarm JSON report written to: {report_path}")
+        report_path = write_report(request, results, meta=meta, report_format=report_format)
+        logger.info("📄 Swarm report written to: %s", report_path)
+        print(f"📄 Swarm report written to: {report_path}")
 
         return decision, results

ai_testing_swarm/reporting/dashboard.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+
+def _html_escape(s: object) -> str:
+    return (
+        str(s)
+        .replace("&", "&")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace('"', "&quot;")
+        .replace("'", "&#39;")
+    )
+
+
+@dataclass
+class EndpointRow:
+    endpoint_dir: str
+    endpoint: str
+    run_time: str
+    gate_status: str
+    endpoint_risk_score: int
+    decision: str
+    report_relpath: str
+    top_risks: list[dict]
+
+
+def _latest_json_report(endpoint_dir: Path) -> Path | None:
+    if not endpoint_dir.exists() or not endpoint_dir.is_dir():
+        return None
+    candidates = sorted(endpoint_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+    return candidates[0] if candidates else None
+
+
+def _load_report(p: Path) -> dict | None:
+    try:
+        return json.loads(p.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+
+
+def write_dashboard_index(reports_dir: Path) -> str:
+    """Create/overwrite a simple static index.html under reports_dir."""
+
+    reports_dir = Path(reports_dir)
+    reports_dir.mkdir(parents=True, exist_ok=True)
+
+    rows: list[EndpointRow] = []
+
+    for child in sorted([p for p in reports_dir.iterdir() if p.is_dir()]):
+        latest = _latest_json_report(child)
+        if not latest:
+            continue
+        rpt = _load_report(latest)
+        if not rpt:
+            continue
+
+        meta = rpt.get("meta") or {}
+        summary = rpt.get("summary") or {}
+
+        endpoint_risk = meta.get("endpoint_risk_score")
+        if endpoint_risk is None:
+            endpoint_risk = summary.get("endpoint_risk_score", 0)
+
+        row = EndpointRow(
+            endpoint_dir=child.name,
+            endpoint=str(rpt.get("endpoint") or child.name),
+            run_time=str(rpt.get("run_time") or ""),
+            gate_status=str(meta.get("gate_status") or ""),
+            endpoint_risk_score=int(endpoint_risk or 0),
+            decision=str(meta.get("decision") or ""),
+            report_relpath=str(child.name + "/" + latest.name),
+            top_risks=list(summary.get("top_risks") or []),
+        )
+        rows.append(row)
+
+    # Sort by risk (desc) then recent
+    rows.sort(key=lambda r: (r.endpoint_risk_score, r.run_time), reverse=True)
+
+    # Global top risks across endpoints
+    global_risks = []
+    for r in rows:
+        for item in (r.top_risks or [])[:3]:
+            global_risks.append(
+                {
+                    "endpoint": r.endpoint,
+                    "name": item.get("name"),
+                    "risk_score": item.get("risk_score"),
+                    "status": item.get("status"),
+                    "failure_type": item.get("failure_type"),
+                    "report": r.report_relpath,
+                }
+            )
+    global_risks.sort(key=lambda x: int(x.get("risk_score") or 0), reverse=True)
+    global_risks = global_risks[:15]
+
+    def badge(gate: str) -> str:
+        gate = (gate or "").upper()
+        cls = {"PASS": "pass", "WARN": "warn", "BLOCK": "block"}.get(gate, "")
+        return f"<span class='gate {cls}'>{_html_escape(gate)}</span>"
+
+    endpoint_rows_html = "".join(
+        "<tr>"
+        f"<td>{badge(r.gate_status)}</td>"
+        f"<td><code>{_html_escape(r.endpoint_risk_score)}</code></td>"
+        f"<td><a href='{_html_escape(r.report_relpath)}'>{_html_escape(r.endpoint)}</a></td>"
+        f"<td><code>{_html_escape(r.run_time)}</code></td>"
+        f"<td><code>{_html_escape(r.decision)}</code></td>"
+        "</tr>"
+        for r in rows
+    ) or "<tr><td colspan='5'>(no JSON reports found)</td></tr>"
+
+    top_risks_html = "".join(
+        "<tr>"
+        f"<td><code>{_html_escape(x.get('risk_score'))}</code></td>"
+        f"<td>{_html_escape(x.get('status'))}</td>"
+        f"<td><code>{_html_escape(x.get('failure_type'))}</code></td>"
+        f"<td>{_html_escape(x.get('name'))}</td>"
+        f"<td><a href='{_html_escape(x.get('report'))}'>{_html_escape(x.get('endpoint'))}</a></td>"
+        "</tr>"
+        for x in global_risks
+    ) or "<tr><td colspan='5'>(none)</td></tr>"
+
+    css = """
+    body{font-family:ui-sans-serif,system-ui,Segoe UI,Roboto,Arial; margin:20px;}
+    table{border-collapse:collapse; width:100%;}
+    th,td{border:1px solid #ddd; padding:8px; vertical-align:top}
+    th{background:#f6f7f9; text-align:left}
+    code{background:#f1f2f4; padding:1px 5px; border-radius:6px;}
+    .gate{display:inline-block; padding:2px 10px; border-radius:999px; border:1px solid #aaa; font-size:12px;}
+    .gate.pass{background:#e6ffed; border-color:#36b37e;}
+    .gate.warn{background:#fff7e6; border-color:#ffab00;}
+    .gate.block{background:#ffebe6; border-color:#ff5630;}
+    .muted{color:#555}
+    """
+
+    html = f"""<!doctype html>
+<html>
+<head>
+<meta charset='utf-8'/>
+<title>AI Testing Swarm — Dashboard</title>
+<style>{css}</style>
+</head>
+<body>
+<h1>AI Testing Swarm — Dashboard</h1>
+<p class='muted'>Static index generated from latest JSON report per endpoint.</p>
+
+<h2>Endpoints</h2>
+<table>
+  <thead>
+    <tr>
+      <th>Gate</th>
+      <th>Risk</th>
+      <th>Endpoint</th>
+      <th>Run time</th>
+      <th>Decision</th>
+    </tr>
+  </thead>
+  <tbody>
+    {endpoint_rows_html}
+  </tbody>
+</table>
+
+<h2>Top risks (across endpoints)</h2>
+<table>
+  <thead>
+    <tr>
+      <th>Risk</th>
+      <th>Status</th>
+      <th>Failure type</th>
+      <th>Test</th>
+      <th>Endpoint</th>
+    </tr>
+  </thead>
+  <tbody>
+    {top_risks_html}
+  </tbody>
+</table>
+
+</body>
+</html>"""
+
+    out = reports_dir / "index.html"
+    out.write_text(html, encoding="utf-8")
+    return str(out)

ai_testing_swarm/reporting/report_writer.py

@@ -58,6 +58,7 @@ def extract_endpoint_name(method: str, url: str) -> str:
 # 📝 REPORT WRITER
 # ============================================================
 from ai_testing_swarm.core.config import AI_SWARM_SLA_MS
+from ai_testing_swarm.reporting.dashboard import write_dashboard_index
 
 SENSITIVE_HEADER_KEYS = {
     "authorization",
@@ -92,7 +93,194 @@ def _redact_results(results: list) -> list:
     return redacted
 
 
-def write_report(request: dict, results: list, *, meta: dict | None = None) -> str:
+def _markdown_escape(s: str) -> str:
+    return str(s).replace("`", "\\`")
+
+
+def _render_markdown(report: dict) -> str:
+    lines: list[str] = []
+    lines.append(f"# AI Testing Swarm Report")
+    lines.append("")
+    lines.append(f"**Endpoint:** `{_markdown_escape(report.get('endpoint'))}`  ")
+    lines.append(f"**Run time:** `{_markdown_escape(report.get('run_time'))}`  ")
+    lines.append(f"**Total tests:** `{report.get('total_tests')}`")
+
+    meta = report.get("meta") or {}
+    if meta.get("gate_status"):
+        lines.append(f"**Gate:** `{_markdown_escape(meta.get('gate_status'))}`  ")
+    if meta.get("endpoint_risk_score") is not None:
+        lines.append(f"**Endpoint risk:** `{_markdown_escape(meta.get('endpoint_risk_score'))}`")
+
+    lines.append("")
+
+    summary = report.get("summary") or {}
+    counts_ft = summary.get("counts_by_failure_type") or {}
+    counts_sc = summary.get("counts_by_status_code") or {}
+
+    lines.append("## Summary")
+    lines.append("")
+    lines.append("### Counts by failure type")
+    for k, v in sorted(counts_ft.items(), key=lambda kv: (-kv[1], kv[0])):
+        lines.append(f"- **{_markdown_escape(k)}**: {v}")
+
+    lines.append("")
+    lines.append("### Counts by status code")
+    for k, v in sorted(counts_sc.items(), key=lambda kv: (-kv[1], kv[0])):
+        lines.append(f"- **{_markdown_escape(k)}**: {v}")
+
+    # Risky findings
+    results = report.get("results") or []
+    risky = [r for r in results if str(r.get("status")) == "RISK"]
+    failed = [r for r in results if str(r.get("status")) == "FAILED"]
+
+    lines.append("")
+    lines.append("## Top risky findings")
+    if not risky and not failed:
+        lines.append("- (none)")
+    else:
+        top = (risky + failed)[:10]
+        for r in top:
+            resp = r.get("response") or {}
+            sc = resp.get("status_code")
+            lines.append(
+                f"- **{_markdown_escape(r.get('status'))}** `{_markdown_escape(r.get('name'))}` "
+                f"(risk={_markdown_escape(r.get('risk_score'))}, status={sc}, "
+                f"failure_type={_markdown_escape(r.get('failure_type'))})"
+            )
+
+    return "\n".join(lines) + "\n"
+
+
+def _html_escape(s: str) -> str:
+    return (
+        str(s)
+        .replace("&", "&")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace('"', "&quot;")
+        .replace("'", "&#39;")
+    )
+
+
+def _render_html(report: dict) -> str:
+    endpoint = _html_escape(report.get("endpoint"))
+    run_time = _html_escape(report.get("run_time"))
+    total_tests = report.get("total_tests")
+    summary = report.get("summary") or {}
+    meta = report.get("meta") or {}
+
+    results = report.get("results") or []
+    risky = [r for r in results if str(r.get("status")) == "RISK"]
+    failed = [r for r in results if str(r.get("status")) == "FAILED"]
+    top_risky = (risky + failed)[:10]
+
+    def _kv_list(d: dict) -> str:
+        items = sorted((d or {}).items(), key=lambda kv: (-kv[1], kv[0]))
+        return "".join(f"<li><b>{_html_escape(k)}</b>: {v}</li>" for k, v in items) or "<li>(none)</li>"
+
+    def _pre(obj) -> str:
+        try:
+            txt = json.dumps(obj, indent=2, ensure_ascii=False)
+        except Exception:
+            txt = str(obj)
+        return f"<pre>{_html_escape(txt)}</pre>"
+
+    rows = []
+    for r in results:
+        resp = r.get("response") or {}
+        issues = resp.get("openapi_validation") or []
+        status = _html_escape(r.get("status"))
+        name = _html_escape(r.get("name"))
+        failure_type = _html_escape(r.get("failure_type"))
+        sc = _html_escape(resp.get("status_code"))
+        elapsed = _html_escape(resp.get("elapsed_ms"))
+        risk_score = _html_escape(r.get("risk_score"))
+        badge = {
+            "PASSED": "badge passed",
+            "FAILED": "badge failed",
+            "RISK": "badge risk",
+        }.get(r.get("status"), "badge")
+
+        rows.append(
+            "<details class='case'>"
+            f"<summary><span class='{badge}'>{status}</span> "
+            f"<b>{name}</b> — risk={risk_score}, status={sc}, elapsed_ms={elapsed}, failure_type={failure_type}"
+            f"{(' — openapi_issues=' + str(len(issues))) if issues else ''}"
+            "</summary>"
+            "<div class='grid'>"
+            f"<div><h4>Request</h4>{_pre(r.get('request'))}</div>"
+            f"<div><h4>Response</h4>{_pre(resp)}</div>"
+            f"<div><h4>Reasoning</h4>{_pre({k: r.get(k) for k in ('reason','confidence','failure_type','status','risk_score')})}</div>"
+            "</div>"
+            "</details>"
+        )
+
+    top_list = "".join(
+        f"<li><b>{_html_escape(r.get('status'))}</b> {_html_escape(r.get('name'))}"
+        f" (failure_type={_html_escape(r.get('failure_type'))})</li>"
+        for r in top_risky
+    ) or "<li>(none)</li>"
+
+    css = """
+    body{font-family:ui-sans-serif,system-ui,Segoe UI,Roboto,Arial; margin:20px;}
+    .meta{color:#444;margin-bottom:16px}
+    .grid{display:grid; grid-template-columns: 1fr 1fr 1fr; gap:12px; margin-top:10px}
+    pre{background:#0b1020;color:#e6e6e6;padding:10px;border-radius:8px;overflow:auto;max-height:360px}
+    details.case{border:1px solid #ddd; border-radius:10px; padding:10px; margin:10px 0}
+    summary{cursor:pointer}
+    .badge{display:inline-block; padding:2px 8px; border-radius:999px; font-size:12px; margin-right:8px; border:1px solid #aaa}
+    .badge.passed{background:#e6ffed;border-color:#36b37e}
+    .badge.failed{background:#ffebe6;border-color:#ff5630}
+    .badge.risk{background:#fff7e6;border-color:#ffab00}
+    """
+
+    return f"""<!doctype html>
+<html>
+<head>
+<meta charset='utf-8'/>
+<title>AI Testing Swarm Report</title>
+<style>{css}</style>
+</head>
+<body>
+<h1>AI Testing Swarm Report</h1>
+<div class='meta'>
+  <div><b>Endpoint:</b> <code>{endpoint}</code></div>
+  <div><b>Run time:</b> <code>{run_time}</code></div>
+  <div><b>Total tests:</b> <code>{total_tests}</code></div>
+  <div><b>Gate:</b> <code>{_html_escape(meta.get('gate_status'))}</code></div>
+  <div><b>Endpoint risk:</b> <code>{_html_escape(meta.get('endpoint_risk_score'))}</code></div>
+</div>
+
+<h2>Summary</h2>
+<div class='grid'>
+  <div><h3>Counts by failure type</h3><ul>{_kv_list(summary.get('counts_by_failure_type') or {})}</ul></div>
+  <div><h3>Counts by status code</h3><ul>{_kv_list(summary.get('counts_by_status_code') or {})}</ul></div>
+  <div><h3>Top risky findings</h3><ul>{top_list}</ul></div>
+</div>
+
+<h2>Results</h2>
+{''.join(rows)}
+
+</body>
+</html>"""
+
+
+def write_report(
+    request: dict,
+    results: list,
+    *,
+    meta: dict | None = None,
+    report_format: str = "json",
+    update_index: bool = True,
+) -> str:
+    """Write a swarm report.
+
+    report_format:
+      - json (default): full machine-readable report
+      - md: human-readable markdown summary
+      - html: single-file HTML report with collapsible sections
+    """
+
     REPORTS_DIR.mkdir(exist_ok=True)
 
     method = request.get("method", "UNKNOWN")
@@ -101,21 +289,18 @@ def write_report(request: dict, results: list, *, meta: dict | None = None) -> s
     endpoint_name = extract_endpoint_name(method, url)
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
 
-    # report_path = REPORTS_DIR / f"{timestamp}_{endpoint_name}.json"
-    # 🔥 PER-ENDPOINT FOLDER
     endpoint_dir = REPORTS_DIR / endpoint_name
     endpoint_dir.mkdir(parents=True, exist_ok=True)
 
-    # 🔥 FILE NAME FORMAT
-    report_path = endpoint_dir / f"{endpoint_name}_{timestamp}.json"
-
     safe_results = _redact_results(results)
 
-    # Summary (counts + performance)
     summary = {
         "counts_by_failure_type": {},
         "counts_by_status_code": {},
         "slow_tests": [],
+        # Batch1: risk aggregation (optional fields)
+        "endpoint_risk_score": 0,
+        "top_risks": [],
     }
 
     for r in safe_results:
@@ -130,6 +315,28 @@ def write_report(request: dict, results: list, *, meta: dict | None = None) -> s
         if isinstance(elapsed, int) and elapsed > AI_SWARM_SLA_MS:
             summary["slow_tests"].append({"name": r.get("name"), "elapsed_ms": elapsed})
 
+    # Batch1: endpoint risk summary (best-effort)
+    try:
+        scores = [r.get("risk_score") for r in safe_results if isinstance(r.get("risk_score"), int)]
+        summary["endpoint_risk_score"] = int(max(scores)) if scores else 0
+        top = sorted(
+            (
+                {
+                    "name": r.get("name"),
+                    "status": r.get("status"),
+                    "failure_type": r.get("failure_type"),
+                    "risk_score": r.get("risk_score"),
+                }
+                for r in safe_results
+                if isinstance(r.get("risk_score"), int)
+            ),
+            key=lambda x: int(x.get("risk_score") or 0),
+            reverse=True,
+        )
+        summary["top_risks"] = top[:10]
+    except Exception:
+        pass
+
     report = {
         "endpoint": f"{method} {url}",
         "run_time": timestamp,
@@ -139,9 +346,29 @@ def write_report(request: dict, results: list, *, meta: dict | None = None) -> s
         "results": safe_results,
     }
 
-    with open(report_path, "w") as f:
-        json.dump(report, f, indent=2)
-
-    print(f"📄 Swarm JSON report written to: {report_path}")
+    report_format = (report_format or "json").lower().strip()
+    if report_format not in {"json", "md", "html"}:
+        report_format = "json"
+
+    ext = {"json": "json", "md": "md", "html": "html"}[report_format]
+    report_path = endpoint_dir / f"{endpoint_name}_{timestamp}.{ext}"
+
+    if report_format == "json":
+        with open(report_path, "w", encoding="utf-8") as f:
+            json.dump(report, f, indent=2)
+    elif report_format == "md":
+        with open(report_path, "w", encoding="utf-8") as f:
+            f.write(_render_markdown(report))
+    else:
+        with open(report_path, "w", encoding="utf-8") as f:
+            f.write(_render_html(report))
+
+    # Batch1: update a simple static dashboard index.html
+    if update_index:
+        try:
+            write_dashboard_index(REPORTS_DIR)
+        except Exception:
+            # Dashboard is best-effort; never fail the run
+            pass
 
     return str(report_path)

{ai_testing_swarm-0.1.13.dist-info → ai_testing_swarm-0.1.15.dist-info}/METADATA

@@ -1,17 +1,21 @@
 Metadata-Version: 2.4
 Name: ai-testing-swarm
-Version: 0.1.13
+Version: 0.1.15
 Summary: AI-powered testing swarm
 Author-email: Arif Shah <ashah7775@gmail.com>
 License: MIT
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.28
+Requires-Dist: PyYAML>=6.0
+Provides-Extra: openapi
+Requires-Dist: jsonschema>=4.0; extra == "openapi"
 
 # AI Testing Swarm
 
 AI Testing Swarm is a **super-advanced, mutation-driven API testing framework** (with optional OpenAPI + OpenAI augmentation) built on top of **pytest**.
 
-It generates a large set of deterministic negative/edge/security test cases for an API request, executes them (optionally in parallel, with retries/throttling), and produces a JSON report with summaries.
+It generates a large set of deterministic negative/edge/security test cases for an API request, executes them (optionally in parallel, with retries/throttling), and produces a report (JSON/Markdown/HTML) with summaries.
 
 > Notes:
 > - UI testing is not the focus of the current releases.
@@ -25,6 +29,12 @@ It generates a large set of deterministic negative/edge/security test cases for
 pip install ai-testing-swarm
 ```
 
+Optional (OpenAPI JSON schema validation for responses):
+
+```bash
+pip install "ai-testing-swarm[openapi]"
+```
+
 CLI entrypoint:
 
 ```bash
@@ -49,9 +59,15 @@ Run:
 ai-test --input request.json
 ```
 
-A JSON report is written under:
+Choose a report format:
+
+```bash
+ai-test --input request.json --report-format html
+```
+
+A report is written under:
 
-- `./ai_swarm_reports/<METHOD>_<endpoint>/<METHOD>_<endpoint>_<timestamp>.json`
+- `./ai_swarm_reports/<METHOD>_<endpoint>/<METHOD>_<endpoint>_<timestamp>.<json|md|html>`
 
 Reports include:
 - per-test results
@@ -97,6 +113,8 @@ Reports include:
 - OpenAPI **JSON** works by default.
 - OpenAPI **YAML** requires `PyYAML` installed.
 - Base URL is read from `spec.servers[0].url`.
+- When using OpenAPI input, the swarm will also *optionally* validate response status codes against `operation.responses`.
+- If `jsonschema` is installed (via `ai-testing-swarm[openapi]`) and the response is JSON, response bodies are validated against the OpenAPI `application/json` schema.
   - Override with `AI_SWARM_OPENAPI_BASE_URL` if your spec doesn’t include servers.
 
 ---

{ai_testing_swarm-0.1.13.dist-info → ai_testing_swarm-0.1.15.dist-info}/RECORD

@@ -1,8 +1,8 @@
-ai_testing_swarm/__init__.py,sha256=khDKUuWafURKVs5EAZkpOMiUHI2-V7axlqrWLPUpuZo,23
-ai_testing_swarm/cli.py,sha256=o7JVbC5wNBFxAW9Ox8688JKA78RbY9eBdhUiWJvtgc0,4628
-ai_testing_swarm/orchestrator.py,sha256=WXOjMHwIEr054JkXbvIzmcU8yDYJ1qMP19kPmE7x0J4,4913
+ai_testing_swarm/__init__.py,sha256=qb0TalpSt1CbprnFyeLUKqgrqNtmnk9IoQQ7umAoXVY,23
+ai_testing_swarm/cli.py,sha256=IeCU0E1Ju8hfkHXT4ySI1KVSvKAlYK429Fa5atcAjdo,5626
+ai_testing_swarm/orchestrator.py,sha256=8uaUuyGy2lm3QzTNVl_8AeYtbyzrrxuykGQchRn1EEo,6295
 ai_testing_swarm/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ai_testing_swarm/agents/execution_agent.py,sha256=C4femMQLw8CPxxmaqinEcmweeDBAzkAOI3Df6CZOSIQ,6678
+ai_testing_swarm/agents/execution_agent.py,sha256=lkipsVOWe7agMt3kY9gDxjHfWujLtMh9DJCsAKhYzD8,8050
 ai_testing_swarm/agents/learning_agent.py,sha256=lHqikc8A8s2-Yln_qCm8hCxB9KTqVrTcNvFp20_QQU0,777
 ai_testing_swarm/agents/llm_reasoning_agent.py,sha256=nOA7Y8ixEh4sz67youSNPjFKipFSsfeup8XeSG4TvTg,8481
 ai_testing_swarm/agents/release_gate_agent.py,sha256=ontmkoWe86CmId2aLWVmzxUVw_-T-F5dCxt40o27oZ8,5888
@@ -15,11 +15,14 @@ ai_testing_swarm/core/config.py,sha256=loZcH45TlUseZMYZ9pkYpsK4AH1PaQ7PLv-AOZq_n
 ai_testing_swarm/core/curl_parser.py,sha256=0dPEhRGqn-4u00t-bp7kW9Ii7GSiO0wteB4kDhRKUBQ,1483
 ai_testing_swarm/core/openai_client.py,sha256=gxbrZZZUh_jBcXfxtsBei2t72_xEQrKZ0Z10HjS62Ss,5668
 ai_testing_swarm/core/openapi_loader.py,sha256=lZ1Y7lyyEL4Y90pc-naZQbCyNGndr4GmmUAvR71bpos,3639
+ai_testing_swarm/core/openapi_validator.py,sha256=8oZNSCBSf6O6CyHZj9MB8Ojaqw54DzJqSm6pbF57HuE,5153
+ai_testing_swarm/core/risk.py,sha256=9FtBWwq3_a1xZd5r-F78g9G6WaxA76xJBMe4LR90A_4,3646
 ai_testing_swarm/core/safety.py,sha256=MvOMr7pKFAn0z37pBcE8X8dOqPK-juxhypCZQ4YcriI,825
 ai_testing_swarm/reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ai_testing_swarm/reporting/report_writer.py,sha256=LvqKbhg3RUhEH_mWKgJQS0OmH6atdMvba9BQvqPcMZY,4227
-ai_testing_swarm-0.1.13.dist-info/METADATA,sha256=8c3_vsgmcfTOGUDIygejtMT1xwjS-XugAPo2B1pZETc,4919
-ai_testing_swarm-0.1.13.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-ai_testing_swarm-0.1.13.dist-info/entry_points.txt,sha256=vbW-IBcVcls5I-NA3xFUZxH4Ktevt7lA4w9P4Me0yXo,54
-ai_testing_swarm-0.1.13.dist-info/top_level.txt,sha256=OSqbej3vG04SKqgEcgzDTMn8QzpVsxwOzpSG7quhWJw,17
-ai_testing_swarm-0.1.13.dist-info/RECORD,,
+ai_testing_swarm/reporting/dashboard.py,sha256=W53SRD6ttT7C3VCcsB9nkXOGWaoRwzOp-cZaj1NcRx8,5682
+ai_testing_swarm/reporting/report_writer.py,sha256=wLndx2VPPldGxbCuMyCaWd3VesVRLBPGQ8n9jyyTctI,12554
+ai_testing_swarm-0.1.15.dist-info/METADATA,sha256=hAwNBIjwMxfpDBXHuomFLBHNM0csFl9ELwaXgAZtuyk,5562
+ai_testing_swarm-0.1.15.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ai_testing_swarm-0.1.15.dist-info/entry_points.txt,sha256=vbW-IBcVcls5I-NA3xFUZxH4Ktevt7lA4w9P4Me0yXo,54
+ai_testing_swarm-0.1.15.dist-info/top_level.txt,sha256=OSqbej3vG04SKqgEcgzDTMn8QzpVsxwOzpSG7quhWJw,17
+ai_testing_swarm-0.1.15.dist-info/RECORD,,