nat-engine 1__py3-none-any.whl

mannf/product/ingestors/traffic_ingestor.py

@@ -0,0 +1,679 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Production traffic ingestor plugin.
+
+Parses anonymized production traffic captures from HAR files and
+mitmproxy flow dumps (JSON or native format) and produces
+:class:`~mannf.product.ingestors.models.IngestedEndpoint` and
+:class:`~mannf.product.ingestors.models.IngestedTestCase` objects for
+seeding NAT's autonomous test loop with real traffic patterns.
+
+Supported formats
+-----------------
+* HAR files (``"log"``/``"entries"`` structure) — the same format as
+  :class:`~mannf.product.ingestors.har_ingestor.HARIngestor` but with
+  production-traffic-specific sampling and anonymization support.
+* mitmproxy JSON dumps (``mitmproxy --save-stream-file`` / ``mitmdump -w``
+  output converted to JSON via ``mitmproxy``) — a list of flow dicts with
+  ``"type": "http"``, ``"request"``, and ``"response"`` keys.
+* ``.mitmproxy`` / ``.pcap`` path extensions are accepted as format hints
+  (PCAP is treated as mitmproxy-compatible after conversion).
+
+Config keys
+-----------
+base_url_filter : str, optional
+    If set, only ingest traffic entries whose URL starts with this prefix.
+    Useful for isolating a single service in mixed prod captures.
+exclude_static : bool, default True
+    Skip static asset requests (.js, .css, images, fonts, etc.).
+include_response_data : bool, default False
+    Capture response bodies for regression testing.
+sample_rate : float, default 1.0
+    Fraction of entries to include (0.0–1.0).  Useful for very large
+    captures where full ingestion would be too slow.
+min_status : int, optional
+    Only include entries with HTTP status ≥ this value.
+max_status : int, optional
+    Only include entries with HTTP status ≤ this value.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from mannf.product.ingestors.base import IngestorPlugin
+from mannf.product.ingestors.models import (
+    IngestedEndpoint,
+    IngestedTestCase,
+    IngestorSource,
+    IngestResult,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_STATIC_EXTENSIONS: frozenset[str] = frozenset({
+    ".js", ".css", ".png", ".jpg", ".jpeg", ".gif", ".ico", ".svg",
+    ".woff", ".woff2", ".ttf", ".eot", ".map", ".webp", ".bmp", ".tiff",
+    ".pdf", ".zip", ".gz", ".tar", ".mp4", ".webm", ".mp3", ".wav",
+})
+
+_AUTH_HEADERS: frozenset[str] = frozenset({
+    "authorization",
+    "cookie",
+    "x-api-key",
+    "x-auth-token",
+    "x-access-token",
+    "x-csrf-token",
+})
+
+_SENSITIVE_HEADERS: frozenset[str] = frozenset({
+    "authorization",
+    "cookie",
+    "x-api-key",
+    "x-auth-token",
+    "x-access-token",
+    "x-csrf-token",
+    "proxy-authorization",
+    "set-cookie",
+})
+
+_RE_UUID = re.compile(
+    r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
+    re.IGNORECASE,
+)
+_RE_NUMERIC_ID = re.compile(r"^\d+$")
+
+
+# ---------------------------------------------------------------------------
+# Helper functions
+# ---------------------------------------------------------------------------
+
+
+def _normalise_path(path: str) -> str:
+    """Replace numeric IDs and UUIDs in *path* with ``{id}`` / ``{uuid}``."""
+    path = _RE_UUID.sub("{uuid}", path)
+    parts = path.split("/")
+    normalised = []
+    for part in parts:
+        if _RE_NUMERIC_ID.match(part):
+            normalised.append("{id}")
+        else:
+            normalised.append(part)
+    return "/".join(normalised)
+
+
+def _is_static_resource(path: str) -> bool:
+    """Return ``True`` if *path* looks like a static asset."""
+    suffix = Path(path).suffix.lower()
+    return suffix in _STATIC_EXTENSIONS
+
+
+def _endpoint_key(method: str, path: str) -> str:
+    return f"{method.upper()}:{path}"
+
+
+def _extract_auth_requirements(headers: list[dict]) -> list[str]:
+    found: list[str] = []
+    for h in headers:
+        name = str(h.get("name", "")).lower()
+        if name in _AUTH_HEADERS and name not in found:
+            found.append(name)
+    return found
+
+
+def _sanitise_headers(headers: list[dict]) -> list[dict]:
+    result: list[dict] = []
+    for h in headers:
+        name = str(h.get("name", ""))
+        if name.lower() in _SENSITIVE_HEADERS:
+            result.append({"name": name, "value": "[REDACTED]"})
+        else:
+            result.append(h)
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Parsers for different traffic dump formats
+# ---------------------------------------------------------------------------
+
+
+def _parse_har_entries(data: dict) -> list[dict[str, Any]]:
+    """Extract a flat list of normalised entry dicts from a HAR document."""
+    log = data.get("log")
+    if not isinstance(log, dict):
+        return []
+    entries = log.get("entries", []) or []
+    if not isinstance(entries, list):
+        return []
+    return entries
+
+
+def _parse_mitmproxy_entries(data: list) -> list[dict[str, Any]]:
+    """Convert a mitmproxy JSON flow list to HAR-compatible entry dicts.
+
+    mitmproxy JSON flows look like::
+
+        [
+          {
+            "type": "http",
+            "request": {
+              "method": "GET",
+              "url": "https://example.com/api",
+              "headers": [["Content-Type", "application/json"]],
+              "content": ""
+            },
+            "response": {
+              "status_code": 200,
+              "headers": [["Content-Type", "application/json"]],
+              "content": "{}"
+            }
+          },
+          ...
+        ]
+
+    Headers may be a list-of-pairs or a dict.
+    """
+    entries: list[dict[str, Any]] = []
+    for flow in data:
+        if not isinstance(flow, dict):
+            continue
+        if flow.get("type") not in ("http", None) and "request" not in flow:
+            continue
+        req = flow.get("request") or {}
+        resp = flow.get("response") or {}
+        if not isinstance(req, dict):
+            continue
+
+        url = req.get("url") or req.get("pretty_url") or ""
+        method = req.get("method", "GET") or "GET"
+
+        # Normalise headers: accept list-of-pairs or list-of-dicts
+        raw_req_headers = req.get("headers") or []
+        norm_req_headers: list[dict] = []
+        if isinstance(raw_req_headers, dict):
+            norm_req_headers = [{"name": k, "value": v} for k, v in raw_req_headers.items()]
+        elif isinstance(raw_req_headers, list):
+            for item in raw_req_headers:
+                if isinstance(item, (list, tuple)) and len(item) >= 2:
+                    norm_req_headers.append({"name": item[0], "value": item[1]})
+                elif isinstance(item, dict):
+                    norm_req_headers.append(item)
+
+        raw_resp_headers = resp.get("headers") or []
+        norm_resp_headers: list[dict] = []
+        if isinstance(raw_resp_headers, dict):
+            norm_resp_headers = [{"name": k, "value": v} for k, v in raw_resp_headers.items()]
+        elif isinstance(raw_resp_headers, list):
+            for item in raw_resp_headers:
+                if isinstance(item, (list, tuple)) and len(item) >= 2:
+                    norm_resp_headers.append({"name": item[0], "value": item[1]})
+                elif isinstance(item, dict):
+                    norm_resp_headers.append(item)
+
+        status_code = resp.get("status_code") or resp.get("status") or 0
+        try:
+            status_code = int(status_code)
+        except (TypeError, ValueError):
+            status_code = 0
+
+        resp_content_text = resp.get("content") or resp.get("text") or ""
+        if isinstance(resp_content_text, bytes):
+            resp_content_text = resp_content_text.decode("utf-8", errors="replace")
+
+        req_content = req.get("content") or req.get("text") or req.get("body") or ""
+        if isinstance(req_content, bytes):
+            req_content = req_content.decode("utf-8", errors="replace")
+
+        post_data: dict | None = None
+        if req_content:
+            content_type = ""
+            for h in norm_req_headers:
+                if h.get("name", "").lower() == "content-type":
+                    content_type = h.get("value", "")
+                    break
+            post_data = {"mimeType": content_type, "text": req_content}
+
+        entry: dict[str, Any] = {
+            "startedDateTime": flow.get("timestamp_start") or "",
+            "time": flow.get("timestamp_end", 0),
+            "request": {
+                "method": method.upper(),
+                "url": url,
+                "headers": norm_req_headers,
+                "queryString": [],
+                "postData": post_data,
+            },
+            "response": {
+                "status": status_code,
+                "headers": norm_resp_headers,
+                "content": {"text": resp_content_text, "mimeType": ""},
+            },
+        }
+        entries.append(entry)
+
+    return entries
+
+
+# ---------------------------------------------------------------------------
+# TrafficIngestor
+# ---------------------------------------------------------------------------
+
+
+class TrafficIngestor(IngestorPlugin):
+    """Ingestor plugin for anonymized production traffic captures.
+
+    Parses HAR files and mitmproxy JSON dumps containing real production
+    traffic and converts them into scenario seeds for NAT's autonomous
+    test loop.  This enables zero-cold-start testing: import production
+    traffic and immediately seed the autonomous loop with realistic flows.
+
+    Format detection
+    ----------------
+    * ``.mitmproxy`` extension → treated as mitmproxy JSON
+    * ``.pcap`` extension → treated as mitmproxy JSON (post-conversion)
+    * ``format="traffic"`` → auto-detect between HAR and mitmproxy based
+      on content structure
+    * HAR content (``{"log": ...}`` structure) → parsed as HAR
+    * JSON list of flow dicts → parsed as mitmproxy
+
+    Config keys
+    -----------
+    base_url_filter : str, optional
+        Only include entries whose URL starts with this prefix.
+    exclude_static : bool, default True
+        Skip static asset requests.
+    include_response_data : bool, default False
+        Capture response bodies.
+    sample_rate : float, default 1.0
+        Fraction of entries to include (0.0–1.0).
+    min_status : int, optional
+        Only include entries with HTTP status ≥ this value.
+    max_status : int, optional
+        Only include entries with HTTP status ≤ this value.
+    """
+
+    name = "traffic"
+    display_name = "Production Traffic"
+    description = (
+        "Ingest anonymized production traffic from HAR files or mitmproxy captures."
+    )
+    supported_extensions = [".mitmproxy", ".pcap"]
+    supported_formats = ["traffic", "mitmproxy", "pcap"]
+
+    # ------------------------------------------------------------------
+    # can_handle
+    # ------------------------------------------------------------------
+
+    def can_handle(self, source: IngestorSource) -> bool:
+        """Return ``True`` for traffic/mitmproxy sources."""
+        if source.format in self.supported_formats:
+            return True
+        if source.metadata.get("format_hint") in self.supported_formats:
+            return True
+        if source.path is not None:
+            ext = Path(source.path).suffix.lower()
+            if ext in self.supported_extensions:
+                return True
+        return False
+
+    # ------------------------------------------------------------------
+    # validate_config
+    # ------------------------------------------------------------------
+
+    def validate_config(self, config: dict) -> list[str]:
+        """Validate traffic ingestor configuration."""
+        errors: list[str] = []
+
+        if "min_status" in config:
+            try:
+                val = int(config["min_status"])
+                if not (100 <= val <= 599):
+                    errors.append("min_status must be between 100 and 599.")
+            except (TypeError, ValueError):
+                errors.append("min_status must be an integer.")
+
+        if "max_status" in config:
+            try:
+                val = int(config["max_status"])
+                if not (100 <= val <= 599):
+                    errors.append("max_status must be between 100 and 599.")
+            except (TypeError, ValueError):
+                errors.append("max_status must be an integer.")
+
+        if "min_status" in config and "max_status" in config:
+            try:
+                mn = int(config["min_status"])
+                mx = int(config["max_status"])
+                if mn > mx:
+                    errors.append("min_status must not be greater than max_status.")
+            except (TypeError, ValueError):
+                pass
+
+        if "base_url_filter" in config:
+            val = config["base_url_filter"]
+            if not isinstance(val, str) or not val.strip():
+                errors.append("base_url_filter must be a non-empty string.")
+
+        if "sample_rate" in config:
+            try:
+                val = float(config["sample_rate"])
+                if not (0.0 < val <= 1.0):
+                    errors.append("sample_rate must be between 0.0 (exclusive) and 1.0.")
+            except (TypeError, ValueError):
+                errors.append("sample_rate must be a number between 0.0 and 1.0.")
+
+        return errors
+
+    # ------------------------------------------------------------------
+    # ingest
+    # ------------------------------------------------------------------
+
+    async def ingest(self, source: IngestorSource, config: dict) -> IngestResult:
+        """Parse a production traffic capture and return a normalised :class:`IngestResult`.
+
+        Parameters
+        ----------
+        source:
+            The source to ingest.  Must provide either ``raw_content`` or a
+            valid ``path`` to a HAR or mitmproxy JSON file.
+        config:
+            Plugin-specific configuration.  See class docstring for keys.
+
+        Returns
+        -------
+        IngestResult
+            Populated result on success; ``success=False`` with ``errors`` on
+            failure.
+        """
+        # --- Read content ---
+        try:
+            content = self._read_source(source)
+        except (FileNotFoundError, ValueError) as exc:
+            return self._make_error_result(source, str(exc))
+
+        if not content or not content.strip():
+            return self._make_error_result(source, "Empty content — nothing to ingest.")
+
+        # --- Parse JSON ---
+        try:
+            data: Any = json.loads(content)
+        except (json.JSONDecodeError, ValueError) as exc:
+            return self._make_error_result(source, f"Failed to parse traffic JSON: {exc}")
+
+        # --- Determine format and extract entries ---
+        entries: list[dict[str, Any]] = []
+        detected_format = "unknown"
+
+        if isinstance(data, dict) and "log" in data:
+            # HAR format
+            entries = _parse_har_entries(data)
+            detected_format = "har"
+        elif isinstance(data, list):
+            # mitmproxy JSON list of flows
+            entries = _parse_mitmproxy_entries(data)
+            detected_format = "mitmproxy"
+        else:
+            return self._make_error_result(
+                source,
+                "Unrecognized traffic format. Expected HAR (object with 'log' key) "
+                "or mitmproxy JSON (array of flow objects).",
+            )
+
+        if not entries:
+            return self._make_error_result(
+                source,
+                f"No traffic entries found in {detected_format} capture.",
+            )
+
+        # --- Config options ---
+        base_url_filter: str = (config.get("base_url_filter") or "").strip()
+        exclude_static: bool = bool(config.get("exclude_static", True))
+        include_response_data: bool = bool(config.get("include_response_data", False))
+        min_status: int | None = None
+        max_status: int | None = None
+        if "min_status" in config:
+            try:
+                min_status = int(config["min_status"])
+            except (TypeError, ValueError):
+                pass
+        if "max_status" in config:
+            try:
+                max_status = int(config["max_status"])
+            except (TypeError, ValueError):
+                pass
+        sample_rate: float = 1.0
+        if "sample_rate" in config:
+            try:
+                sample_rate = float(config["sample_rate"])
+            except (TypeError, ValueError):
+                pass
+
+        # --- Apply sampling ---
+        if 0.0 < sample_rate < 1.0:
+            import math
+            keep_count = max(1, math.ceil(len(entries) * sample_rate))
+            # Systematic sampling to preserve temporal distribution
+            step = len(entries) / keep_count
+            entries = [entries[int(i * step)] for i in range(keep_count)]
+
+        warnings: list[str] = []
+        endpoint_map: dict[str, IngestedEndpoint] = {}
+        response_map: dict[str, list[tuple[int, str | None]]] = {}
+
+        skipped_static = 0
+        skipped_filter = 0
+        skipped_status = 0
+        total_entries = len(entries)
+
+        for entry in entries:
+            if not isinstance(entry, dict):
+                continue
+
+            request: dict[str, Any] = entry.get("request") or {}
+            response: dict[str, Any] = entry.get("response") or {}
+
+            if not isinstance(request, dict):
+                continue
+
+            raw_url: str = request.get("url", "") or ""
+            method: str = (request.get("method", "GET") or "GET").upper()
+
+            if not raw_url:
+                continue
+
+            try:
+                parsed = urlparse(raw_url)
+            except Exception:  # noqa: BLE001
+                warnings.append(f"Skipped entry with unparseable URL: {raw_url!r}")
+                continue
+
+            base_url_part = f"{parsed.scheme}://{parsed.netloc}" if parsed.netloc else ""
+            raw_path = parsed.path or "/"
+
+            if base_url_filter and not raw_url.startswith(base_url_filter):
+                skipped_filter += 1
+                continue
+
+            if exclude_static and _is_static_resource(raw_path):
+                skipped_static += 1
+                continue
+
+            response_status: int = 0
+            if isinstance(response, dict):
+                try:
+                    response_status = int(
+                        response.get("status") or response.get("status_code") or 0
+                    )
+                except (TypeError, ValueError):
+                    response_status = 0
+
+            if min_status is not None and response_status < min_status:
+                skipped_status += 1
+                continue
+            if max_status is not None and response_status > max_status:
+                skipped_status += 1
+                continue
+
+            normalised_path = _normalise_path(raw_path)
+            key = _endpoint_key(method, normalised_path)
+
+            raw_headers: list[dict] = request.get("headers", []) or []
+            auth_requirements = _extract_auth_requirements(raw_headers)
+
+            response_content_type = ""
+            response_body_text: str | None = None
+            if isinstance(response, dict):
+                resp_headers = response.get("headers", []) or []
+                for rh in resp_headers:
+                    if rh.get("name", "").lower() == "content-type":
+                        response_content_type = rh.get("value", "")
+                        break
+                if include_response_data:
+                    content_block: dict = response.get("content") or {}
+                    response_body_text = (
+                        content_block.get("text")
+                        or response.get("text")
+                        or None
+                    )
+
+            if key not in endpoint_map:
+                metadata: dict[str, Any] = {
+                    "base_url": base_url_part,
+                    "original_urls": [raw_url],
+                    "content_types_observed": [],
+                    "traffic_source": detected_format,
+                }
+                if response_content_type:
+                    metadata["content_types_observed"].append(response_content_type)
+
+                endpoint_map[key] = IngestedEndpoint(
+                    method=method,
+                    path=normalised_path,
+                    summary=f"{method} {normalised_path}",
+                    description=f"Production traffic: {method} {normalised_path}",
+                    request_body=request.get("postData"),
+                    responses={str(response_status): {"description": "Observed response"}}
+                    if response_status else {},
+                    auth_requirements=auth_requirements,
+                    tags=["traffic", detected_format, method.lower()],
+                    metadata=metadata,
+                )
+                response_map[key] = [(response_status, response_body_text)]
+            else:
+                existing = endpoint_map[key]
+                if raw_url not in existing.metadata.get("original_urls", []):
+                    existing.metadata.setdefault("original_urls", []).append(raw_url)
+                for scheme in auth_requirements:
+                    if scheme not in existing.auth_requirements:
+                        existing.auth_requirements.append(scheme)
+                if response_content_type:
+                    ct_list = existing.metadata.setdefault("content_types_observed", [])
+                    if response_content_type not in ct_list:
+                        ct_list.append(response_content_type)
+                if response_status and str(response_status) not in existing.responses:
+                    existing.responses[str(response_status)] = {
+                        "description": "Observed response"
+                    }
+                response_map[key].append((response_status, response_body_text))
+
+        if not endpoint_map and total_entries > 0:
+            filtered_count = skipped_static + skipped_filter + skipped_status
+            if filtered_count == total_entries:
+                warnings.append(
+                    "All entries were filtered out. Check base_url_filter, "
+                    "exclude_static, and min_status/max_status settings."
+                )
+            else:
+                warnings.append(
+                    "No valid endpoints could be extracted from the traffic capture."
+                )
+
+        if skipped_static:
+            warnings.append(
+                f"Skipped {skipped_static} static resource entries (exclude_static=True)."
+            )
+        if skipped_filter:
+            warnings.append(
+                f"Skipped {skipped_filter} entries that did not match "
+                f"base_url_filter={base_url_filter!r}."
+            )
+        if skipped_status:
+            warnings.append(
+                f"Skipped {skipped_status} entries outside the status code range."
+            )
+
+        endpoints = list(endpoint_map.values())
+        test_cases: list[IngestedTestCase] = []
+
+        for ep in endpoints:
+            ep_key = _endpoint_key(ep.method, ep.path)
+            recorded_responses = response_map.get(ep_key, [])
+            if recorded_responses:
+                statuses = [s for s, _ in recorded_responses if s]
+                recorded_status = max(set(statuses), key=statuses.count) if statuses else 200
+                resp_body = next(
+                    (b for _, b in recorded_responses if b is not None), None
+                )
+            else:
+                recorded_status = 200
+                resp_body = None
+
+            source_ref = f"{ep.method} {ep.path}"
+            steps: list[dict] = [
+                {"action": "send_request", "method": ep.method, "path": ep.path},
+                {"action": "assert_status", "expected": recorded_status},
+            ]
+            if include_response_data and resp_body:
+                steps.append({"action": "assert_response_body", "snapshot": True})
+
+            test_cases.append(IngestedTestCase(
+                name=f"[{ep.method}] {ep.path} — production replay",
+                description=(
+                    f"Replay of observed production {ep.method} request to {ep.path}. "
+                    f"Expected status: {recorded_status}."
+                ),
+                steps=steps,
+                expected_result=f"HTTP {recorded_status} response",
+                priority="medium",
+                tags=["traffic", detected_format, "replay", ep.method.lower()],
+                source_ref=source_ref,
+                metadata={
+                    "test_type": "traffic_replay",
+                    "traffic_source": detected_format,
+                    "recorded_status": recorded_status,
+                },
+            ))
+
+        stats: dict[str, Any] = {
+            "total_entries": total_entries,
+            "endpoints_count": len(endpoints),
+            "test_cases_count": len(test_cases),
+            "skipped_static": skipped_static,
+            "skipped_filter": skipped_filter,
+            "skipped_status": skipped_status,
+            "include_response_data": include_response_data,
+            "detected_format": detected_format,
+            "sample_rate": sample_rate,
+        }
+
+        return IngestResult(
+            success=True,
+            source=source,
+            endpoints=endpoints,
+            test_cases=test_cases,
+            errors=[],
+            warnings=warnings,
+            stats=stats,
+        )