ai-testing-swarm 0.1.13__tar.gz → 0.1.15__tar.gz

{ai_testing_swarm-0.1.13/src/ai_testing_swarm.egg-info → ai_testing_swarm-0.1.15}/PKG-INFO

@@ -1,17 +1,21 @@
 Metadata-Version: 2.4
 Name: ai-testing-swarm
-Version: 0.1.13
+Version: 0.1.15
 Summary: AI-powered testing swarm
 Author-email: Arif Shah <ashah7775@gmail.com>
 License: MIT
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.28
+Requires-Dist: PyYAML>=6.0
+Provides-Extra: openapi
+Requires-Dist: jsonschema>=4.0; extra == "openapi"
 
 # AI Testing Swarm
 
 AI Testing Swarm is a **super-advanced, mutation-driven API testing framework** (with optional OpenAPI + OpenAI augmentation) built on top of **pytest**.
 
-It generates a large set of deterministic negative/edge/security test cases for an API request, executes them (optionally in parallel, with retries/throttling), and produces a JSON report with summaries.
+It generates a large set of deterministic negative/edge/security test cases for an API request, executes them (optionally in parallel, with retries/throttling), and produces a report (JSON/Markdown/HTML) with summaries.
 
 > Notes:
 > - UI testing is not the focus of the current releases.
@@ -25,6 +29,12 @@ It generates a large set of deterministic negative/edge/security test cases for
 pip install ai-testing-swarm
 ```
 
+Optional (OpenAPI JSON schema validation for responses):
+
+```bash
+pip install "ai-testing-swarm[openapi]"
+```
+
 CLI entrypoint:
 
 ```bash
@@ -49,9 +59,15 @@ Run:
 ai-test --input request.json
 ```
 
-A JSON report is written under:
+Choose a report format:
+
+```bash
+ai-test --input request.json --report-format html
+```
+
+A report is written under:
 
-- `./ai_swarm_reports/<METHOD>_<endpoint>/<METHOD>_<endpoint>_<timestamp>.json`
+- `./ai_swarm_reports/<METHOD>_<endpoint>/<METHOD>_<endpoint>_<timestamp>.<json|md|html>`
 
 Reports include:
 - per-test results
@@ -97,6 +113,8 @@ Reports include:
 - OpenAPI **JSON** works by default.
 - OpenAPI **YAML** requires `PyYAML` installed.
 - Base URL is read from `spec.servers[0].url`.
+- When using OpenAPI input, the swarm will also *optionally* validate response status codes against `operation.responses`.
+- If `jsonschema` is installed (via `ai-testing-swarm[openapi]`) and the response is JSON, response bodies are validated against the OpenAPI `application/json` schema.
   - Override with `AI_SWARM_OPENAPI_BASE_URL` if your spec doesn’t include servers.
 
 ---

ai_testing_swarm-0.1.13/PKG-INFO → ai_testing_swarm-0.1.15/README.md

@@ -1,17 +1,8 @@
-Metadata-Version: 2.4
-Name: ai-testing-swarm
-Version: 0.1.13
-Summary: AI-powered testing swarm
-Author-email: Arif Shah <ashah7775@gmail.com>
-License: MIT
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-
 # AI Testing Swarm
 
 AI Testing Swarm is a **super-advanced, mutation-driven API testing framework** (with optional OpenAPI + OpenAI augmentation) built on top of **pytest**.
 
-It generates a large set of deterministic negative/edge/security test cases for an API request, executes them (optionally in parallel, with retries/throttling), and produces a JSON report with summaries.
+It generates a large set of deterministic negative/edge/security test cases for an API request, executes them (optionally in parallel, with retries/throttling), and produces a report (JSON/Markdown/HTML) with summaries.
 
 > Notes:
 > - UI testing is not the focus of the current releases.
@@ -25,6 +16,12 @@ It generates a large set of deterministic negative/edge/security test cases for
 pip install ai-testing-swarm
 ```
 
+Optional (OpenAPI JSON schema validation for responses):
+
+```bash
+pip install "ai-testing-swarm[openapi]"
+```
+
 CLI entrypoint:
 
 ```bash
@@ -49,9 +46,15 @@ Run:
 ai-test --input request.json
 ```
 
-A JSON report is written under:
+Choose a report format:
+
+```bash
+ai-test --input request.json --report-format html
+```
+
+A report is written under:
 
-- `./ai_swarm_reports/<METHOD>_<endpoint>/<METHOD>_<endpoint>_<timestamp>.json`
+- `./ai_swarm_reports/<METHOD>_<endpoint>/<METHOD>_<endpoint>_<timestamp>.<json|md|html>`
 
 Reports include:
 - per-test results
@@ -97,6 +100,8 @@ Reports include:
 - OpenAPI **JSON** works by default.
 - OpenAPI **YAML** requires `PyYAML` installed.
 - Base URL is read from `spec.servers[0].url`.
+- When using OpenAPI input, the swarm will also *optionally* validate response status codes against `operation.responses`.
+- If `jsonschema` is installed (via `ai-testing-swarm[openapi]`) and the response is JSON, response bodies are validated against the OpenAPI `application/json` schema.
   - Override with `AI_SWARM_OPENAPI_BASE_URL` if your spec doesn’t include servers.
 
 ---

{ai_testing_swarm-0.1.13 → ai_testing_swarm-0.1.15}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ai-testing-swarm"
-version = "0.1.13"
+version = "0.1.15"
 description = "AI-powered testing swarm"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -13,6 +13,16 @@ authors = [
 ]
 license = { text = "MIT" }
 
+dependencies = [
+  "requests>=2.28",
+  "PyYAML>=6.0",
+]
+
+[project.optional-dependencies]
+openapi = [
+  "jsonschema>=4.0",
+]
+
 [project.scripts]
 ai-test = "ai_testing_swarm.cli:main"
 

ai_testing_swarm-0.1.15/src/ai_testing_swarm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.15"

{ai_testing_swarm-0.1.13 → ai_testing_swarm-0.1.15}/src/ai_testing_swarm/agents/execution_agent.py

@@ -137,6 +137,27 @@ class ExecutionAgent:
                 except Exception:
                     body_snippet = resp.text
 
+                openapi_validation = []
+                try:
+                    ctx = request.get("_openapi") or {}
+                    if isinstance(ctx, dict) and ctx.get("spec") and ctx.get("path") and ctx.get("method"):
+                        # If method was mutated to something else, we can't reliably map to an OpenAPI op.
+                        if str(ctx.get("method")).upper() == str(method).upper():
+                            from ai_testing_swarm.core.openapi_validator import validate_openapi_response
+
+                            issues = validate_openapi_response(
+                                spec=ctx["spec"],
+                                path=ctx["path"],
+                                method=ctx["method"],
+                                status_code=resp.status_code,
+                                response_headers=dict(resp.headers),
+                                response_json=body_snippet if isinstance(body_snippet, (dict, list)) else None,
+                            )
+                            openapi_validation = [i.__dict__ for i in issues]
+                except Exception:
+                    # Best-effort only; never fail execution on validator issues.
+                    openapi_validation = []
+
                 return {
                     "name": test["name"],
                     "mutation": mutation,
@@ -151,7 +172,9 @@ class ExecutionAgent:
                         "status_code": resp.status_code,
                         "elapsed_ms": elapsed_ms,
                         "attempt": attempt,
+                        "headers": dict(resp.headers),
                         "body_snippet": body_snippet,
+                        "openapi_validation": openapi_validation,
                     },
                 }
 

{ai_testing_swarm-0.1.13 → ai_testing_swarm-0.1.15}/src/ai_testing_swarm/cli.py

@@ -31,7 +31,7 @@ def normalize_request(payload: dict) -> dict:
         from ai_testing_swarm.core.openapi_loader import load_openapi, build_request_from_openapi
 
         spec = load_openapi(payload["openapi"])
-        return build_request_from_openapi(
+        req = build_request_from_openapi(
             spec,
             path=payload["path"],
             method=payload["method"],
@@ -40,6 +40,14 @@ def normalize_request(payload: dict) -> dict:
             query_params=payload.get("query_params") or {},
             body=payload.get("body"),
         )
+        # Attach OpenAPI context for optional response validation/reporting.
+        req["_openapi"] = {
+            "source": payload["openapi"],
+            "path": payload["path"],
+            "method": str(payload["method"]).upper(),
+            "spec": spec,
+        }
+        return req
 
     # Case 2: already normalized
     required_keys = {"method", "url"}
@@ -92,6 +100,27 @@ def main():
         help="Safety: allow only public test hosts (httpbin/postman-echo/reqres) for this run",
     )
 
+    parser.add_argument(
+        "--report-format",
+        default="json",
+        choices=["json", "md", "html"],
+        help="Report format to write (default: json)",
+    )
+
+    # Batch1: risk gate thresholds (backward compatible defaults)
+    parser.add_argument(
+        "--gate-warn",
+        type=int,
+        default=30,
+        help="Gate WARN threshold for endpoint risk score (default: 30)",
+    )
+    parser.add_argument(
+        "--gate-block",
+        type=int,
+        default=80,
+        help="Gate BLOCK threshold for endpoint risk score (default: 80)",
+    )
+
     args = parser.parse_args()
 
     # ------------------------------------------------------------
@@ -112,7 +141,12 @@ def main():
         import os
         os.environ["AI_SWARM_PUBLIC_ONLY"] = "1"
 
-    decision, results = SwarmOrchestrator().run(request)
+    decision, results = SwarmOrchestrator().run(
+        request,
+        report_format=args.report_format,
+        gate_warn=args.gate_warn,
+        gate_block=args.gate_block,
+    )
 
     # ------------------------------------------------------------
     # Console output

ai_testing_swarm-0.1.15/src/ai_testing_swarm/core/openapi_validator.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class OpenAPIValidationIssue:
+    type: str
+    message: str
+    details: dict | None = None
+
+
+def _matches_status_key(status_code: int, key: str) -> bool:
+    """OpenAPI response keys can be explicit ("200"), wildcard ("2XX"), or "default"."""
+    key = str(key).strip().upper()
+    if key == "DEFAULT":
+        return True
+    if len(key) == 3 and key.endswith("XX") and key[0].isdigit():
+        return int(key[0]) == int(status_code / 100)
+    return key == str(status_code)
+
+
+def _get_operation(spec: dict, *, path: str, method: str) -> dict | None:
+    paths = spec.get("paths") or {}
+    op = (paths.get(path) or {}).get(str(method).lower())
+    if isinstance(op, dict):
+        return op
+    return None
+
+
+def validate_openapi_response(
+    *,
+    spec: dict,
+    path: str,
+    method: str,
+    status_code: int | None,
+    response_headers: dict | None = None,
+    response_json=None,
+) -> list[OpenAPIValidationIssue]:
+    """Validate a response against an OpenAPI operation.
+
+    - Status code must be declared in operation.responses (supports 2XX and default)
+    - If response appears to be JSON and jsonschema is installed, validate body
+
+    Returns a list of issues (empty => OK or validation skipped).
+    """
+
+    issues: list[OpenAPIValidationIssue] = []
+
+    op = _get_operation(spec, path=path, method=method)
+    if not op:
+        return [
+            OpenAPIValidationIssue(
+                type="openapi_operation_missing",
+                message=f"Operation not found in spec: {str(method).upper()} {path}",
+            )
+        ]
+
+    responses = op.get("responses") or {}
+    if status_code is None:
+        # Network errors etc. Not an OpenAPI mismatch.
+        return issues
+
+    # --------------------
+    # Status validation
+    # --------------------
+    declared_keys = [str(k) for k in responses.keys()]
+    if declared_keys:
+        if not any(_matches_status_key(int(status_code), k) for k in declared_keys):
+            issues.append(
+                OpenAPIValidationIssue(
+                    type="openapi_status",
+                    message=f"Status {status_code} not declared in OpenAPI responses: {declared_keys}",
+                    details={"status_code": status_code, "declared": declared_keys},
+                )
+            )
+
+    # --------------------
+    # JSON schema validation (optional)
+    # --------------------
+    ct = ""
+    if response_headers and isinstance(response_headers, dict):
+        for k, v in response_headers.items():
+            if str(k).lower() == "content-type" and v:
+                ct = str(v).lower()
+                break
+
+    is_json = isinstance(response_json, (dict, list))
+    if not is_json:
+        # if body wasn't parsed as JSON, skip schema validation
+        return issues
+
+    if ct and "json" not in ct:
+        # Respect explicit non-JSON content-type
+        return issues
+
+    # Find best matching response schema: exact > 2XX > default
+    chosen_resp: dict | None = None
+    for k in (str(status_code), f"{int(status_code/100)}XX", "default"):
+        if k in responses:
+            chosen_resp = responses.get(k)
+            break
+
+    if not isinstance(chosen_resp, dict):
+        return issues
+
+    content = chosen_resp.get("content") or {}
+    schema = None
+    if isinstance(content, dict):
+        app_json = content.get("application/json") or content.get("application/*+json")
+        if isinstance(app_json, dict):
+            schema = app_json.get("schema")
+
+    if not schema:
+        return issues
+
+    try:
+        import jsonschema  # type: ignore
+
+        # Resolve in-document refs like #/components/schemas/X
+        resolver = jsonschema.RefResolver.from_schema(spec)  # type: ignore[attr-defined]
+        validator_cls = jsonschema.validators.validator_for(schema)  # type: ignore[attr-defined]
+        validator_cls.check_schema(schema)
+        validator = validator_cls(schema, resolver=resolver)
+        errors = sorted(validator.iter_errors(response_json), key=lambda e: list(e.path))
+        if errors:
+            # Keep just the first few errors for signal.
+            details = {
+                "error_count": len(errors),
+                "errors": [
+                    {
+                        "message": e.message,
+                        "path": list(e.path),
+                        "schema_path": list(e.schema_path),
+                    }
+                    for e in errors[:5]
+                ],
+            }
+            issues.append(
+                OpenAPIValidationIssue(
+                    type="openapi_schema",
+                    message="Response JSON does not match OpenAPI schema",
+                    details=details,
+                )
+            )
+    except ImportError:
+        # Optional dependency not installed: status validation still works.
+        return issues
+    except Exception as e:
+        issues.append(
+            OpenAPIValidationIssue(
+                type="openapi_schema_error",
+                message=f"OpenAPI schema validation failed: {e}",
+            )
+        )
+
+    return issues

ai_testing_swarm-0.1.15/src/ai_testing_swarm/core/risk.py

@@ -0,0 +1,139 @@
+"""Risk scoring for AI Testing Swarm.
+
+Batch1 additions:
+- Compute a numeric risk_score per test result (0..100)
+- Aggregate endpoint risk and drive gate thresholds (PASS/WARN/BLOCK)
+
+Design goals:
+- Backward compatible: consumers can ignore risk_score fields.
+- Deterministic and explainable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RiskThresholds:
+    """Thresholds for the endpoint gate.
+
+    Gate is computed from endpoint_risk_score (currently max test risk).
+
+    - PASS: score < warn
+    - WARN: warn <= score < block
+    - BLOCK: score >= block
+    """
+
+    warn: int = 30
+    block: int = 80
+
+
+# Keep aligned with orchestrator/release gate semantics
+EXPECTED_FAILURES: set[str] = {
+    "success",
+    "missing_param",
+    "invalid_param",
+    "security",
+    "method_not_allowed",
+    "not_found",
+    "content_negotiation",
+}
+
+RISKY_FAILURES: set[str] = {
+    "unknown",
+    "missing_param_accepted",
+    "null_param_accepted",
+    "invalid_param_accepted",
+    "headers_accepted",
+    "method_risk",
+}
+
+BLOCKING_FAILURES: set[str] = {
+    "auth_issue",
+    "infra",
+    "security_risk",
+    "server_error",
+}
+
+
+def clamp_int(x: int, lo: int, hi: int) -> int:
+    return lo if x < lo else hi if x > hi else x
+
+
+def compute_test_risk_score(result: dict, *, sla_ms: int | None = None) -> int:
+    """Compute a risk score for a single test result.
+
+    Inputs expected (best-effort):
+      - result['failure_type']
+      - result['status']
+      - result['response']['status_code']
+      - result['response']['elapsed_ms']
+      - result['response']['openapi_validation'] (list)
+
+    Returns: int in range 0..100.
+    """
+
+    ft = str(result.get("failure_type") or "unknown")
+    status = str(result.get("status") or "")
+    resp = result.get("response") or {}
+    sc = resp.get("status_code")
+
+    # Base score from semantic classification.
+    if ft in EXPECTED_FAILURES:
+        base = 0
+    elif ft in RISKY_FAILURES:
+        base = 35
+    elif ft in BLOCKING_FAILURES:
+        base = 90
+    else:
+        # Unknown failure types are treated as high risk but not always a hard blocker.
+        base = 60
+
+    # Status-code adjustments (defense in depth)
+    if isinstance(sc, int):
+        if 500 <= sc:
+            base = max(base, 90)
+        elif sc in (401, 403):
+            base = max(base, 80)
+        elif 400 <= sc < 500:
+            # Client errors for negative tests are expected; keep base.
+            base = max(base, base)
+
+    # Explicit FAILED status implies something unexpected.
+    if status.upper() == "FAILED":
+        base = max(base, 70)
+
+    # OpenAPI validation issues are a small additive risk.
+    issues = resp.get("openapi_validation") or []
+    if isinstance(issues, list) and issues:
+        base += 10
+
+    # SLA breach is a small additive risk.
+    if sla_ms is not None:
+        elapsed = resp.get("elapsed_ms")
+        if isinstance(elapsed, int) and elapsed > sla_ms:
+            base += 5
+
+    return clamp_int(int(base), 0, 100)
+
+
+def compute_endpoint_risk_score(results: list[dict]) -> int:
+    """Aggregate endpoint risk score.
+
+    Current policy: endpoint risk is the max risk_score across tests.
+    (This makes gating stable and easy to interpret.)
+    """
+
+    scores = [r.get("risk_score") for r in (results or []) if isinstance(r.get("risk_score"), int)]
+    if not scores:
+        return 0
+    return int(max(scores))
+
+
+def gate_from_score(score: int, thresholds: RiskThresholds) -> str:
+    if score >= thresholds.block:
+        return "BLOCK"
+    if score >= thresholds.warn:
+        return "WARN"
+    return "PASS"

{ai_testing_swarm-0.1.13 → ai_testing_swarm-0.1.15}/src/ai_testing_swarm/orchestrator.py

@@ -5,6 +5,12 @@ from ai_testing_swarm.agents.learning_agent import LearningAgent
 from ai_testing_swarm.agents.release_gate_agent import ReleaseGateAgent
 from ai_testing_swarm.reporting.report_writer import write_report
 from ai_testing_swarm.core.safety import enforce_public_only
+from ai_testing_swarm.core.risk import (
+    RiskThresholds,
+    compute_test_risk_score,
+    compute_endpoint_risk_score,
+    gate_from_score,
+)
 
 import logging
 
@@ -40,8 +46,20 @@ class SwarmOrchestrator:
         self.learner = LearningAgent()
         self.release_gate = ReleaseGateAgent()
 
-    def run(self, request: dict):
-        """Runs the full AI testing swarm and returns (decision, results)."""
+    def run(
+        self,
+        request: dict,
+        *,
+        report_format: str = "json",
+        gate_warn: int = 30,
+        gate_block: int = 80,
+    ):
+        """Runs the full AI testing swarm and returns (decision, results).
+
+        gate_warn/gate_block:
+          Thresholds for PASS/WARN/BLOCK gate based on endpoint risk.
+          (Kept optional for backward compatibility.)
+        """
 
         # Safety hook (currently no-op; kept for backward compatibility)
         enforce_public_only(request["url"])
@@ -78,6 +96,19 @@ class SwarmOrchestrator:
                     else "FAILED"
                 ),
             })
+
+            # Batch1: numeric risk score per test (0..100)
+            try:
+                from ai_testing_swarm.core.config import AI_SWARM_SLA_MS
+
+                execution_result["risk_score"] = compute_test_risk_score(
+                    execution_result,
+                    sla_ms=AI_SWARM_SLA_MS,
+                )
+            except Exception:
+                # Keep backwards compatibility: don't fail the run if scoring breaks.
+                execution_result["risk_score"] = 0
+
             # Optional learning step
             try:
                 self.learner.learn(test_name, classification)
@@ -97,14 +128,23 @@ class SwarmOrchestrator:
             results.append(results_by_name[t["name"]])
 
         # --------------------------------------------------------
-        # 3️⃣ RELEASE DECISION
+        # 3️⃣ RELEASE DECISION + RISK GATE
         # --------------------------------------------------------
         decision = self.release_gate.decide(results)
 
+        thresholds = RiskThresholds(warn=int(gate_warn), block=int(gate_block))
+        endpoint_risk_score = compute_endpoint_risk_score(results)
+        gate_status = gate_from_score(endpoint_risk_score, thresholds)
+
         # --------------------------------------------------------
-        # 4️⃣ WRITE JSON REPORT
+        # 4️⃣ WRITE REPORT
         # --------------------------------------------------------
-        meta = {"decision": decision}
+        meta = {
+            "decision": decision,
+            "gate_status": gate_status,
+            "gate_thresholds": {"warn": thresholds.warn, "block": thresholds.block},
+            "endpoint_risk_score": endpoint_risk_score,
+        }
 
         # Optional AI summary for humans (best-effort)
         try:
@@ -122,8 +162,8 @@ class SwarmOrchestrator:
         except Exception as e:
             meta["ai_summary_error"] = str(e)
 
-        report_path = write_report(request, results, meta=meta)
-        logger.info("📄 Swarm JSON report written to: %s", report_path)
-        print(f"📄 Swarm JSON report written to: {report_path}")
+        report_path = write_report(request, results, meta=meta, report_format=report_format)
+        logger.info("📄 Swarm report written to: %s", report_path)
+        print(f"📄 Swarm report written to: {report_path}")
 
         return decision, results