nat-engine 1__py3-none-any.whl

mannf/product/orchestrator_models.py

@@ -0,0 +1,269 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Dataclass models for the orchestration pipeline.
+
+These models carry configuration and results through the
+ingest → transform → prioritize → scan → learn pipeline implemented
+by :class:`~mannf.product.orchestrator.IngestOrchestrator`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mannf.product.exporters.base import ExportSummary
+    from mannf.product.ingestors.models import IngestedEndpoint, IngestedTestCase
+    from mannf.product.security.models import SecurityReport
+
+
+@dataclass
+class OrchestrationConfig:
+    """Top-level configuration for a single orchestration run.
+
+    Attributes
+    ----------
+    ingestor_name:
+        Name of the ingestor plugin to use (e.g., ``"openapi"``, ``"postman"``).
+        If ``None``, auto-detection via :func:`~mannf.product.ingestors.loader.get_ingestor_for_source`
+        is attempted.
+    source_path:
+        File-system path to the source file to ingest.
+    source_url:
+        URL of the remote source to ingest.
+    source_format:
+        Format hint for the source (e.g., ``"openapi"``, ``"har"``).  Used
+        when constructing the :class:`~mannf.product.ingestors.models.IngestorSource`.
+    ingestor_config:
+        Arbitrary config dict passed verbatim to the ingestor's
+        :meth:`~mannf.product.ingestors.base.IngestorPlugin.ingest` method.
+    scan_config:
+        Scan-level config — must contain at least ``"base_url"`` for the
+        :class:`~mannf.product.security.scanner.SecurityScanner` to work.
+        May also contain ``"auth_token"``, ``"api_key"``, ``"timeout"``,
+        ``"severity_threshold"``, ``"belief_guided"``, ``"enabled_checks"``.
+    prioritizer_config:
+        Config hints for the belief-guided prioritizer.  Currently supports
+        ``"seed_belief_scores"`` (dict mapping endpoint keys to float scores)
+        and ``"risk_report"`` (list of legacy risk dicts).
+    exporter_config:
+        Config passed to the exporter plugin after scanning.  Must include
+        ``"exporter_name"`` if export is desired; ``None`` disables export.
+    feedback_config:
+        Config for the learning/feedback phase.  Supports
+        ``"save_path"`` (file path for persisting feedback JSON) and
+        ``"enabled"`` (bool, default ``True``).
+    llm_config:
+        Optional config for LLM-augmented edge-case generation.  When set,
+        the orchestrator will invoke the LLM to generate additional test cases
+        for high-priority endpoints before scanning.  Supported keys:
+
+        - ``"enabled"`` — bool (default ``True`` when this dict is non-empty)
+        - ``"provider"`` — ``"openai"`` or ``"anthropic"`` (default ``"openai"``)
+        - ``"model"`` — model identifier (e.g. ``"gpt-4o-mini"``)
+        - ``"api_key"`` — API key (falls back to env var)
+        - ``"priority_threshold"`` — minimum priority_score for a target to
+          receive LLM-generated test cases (default ``0.7``)
+        - ``"n_cases"`` — number of edge-case test cases to generate per
+          high-priority endpoint (default ``3``)
+        - Any other keys are passed verbatim to :class:`~mannf.product.llm.config.LLMConfig`.
+    """
+
+    ingestor_name: str | None = None
+    source_path: str | None = None
+    source_url: str | None = None
+    source_format: str | None = None
+    ingestor_config: dict = field(default_factory=dict)
+    scan_config: dict = field(default_factory=dict)
+    prioritizer_config: dict = field(default_factory=dict)
+    exporter_config: dict = field(default_factory=dict)
+    feedback_config: dict = field(default_factory=dict)
+    llm_config: dict = field(default_factory=dict)
+
+
+@dataclass
+class ScanTarget:
+    """A single scan target — one endpoint with associated test intents.
+
+    Attributes
+    ----------
+    endpoint:
+        The ingested endpoint to scan.
+    test_cases:
+        Test intents derived from the ingestion phase that are associated
+        with this endpoint (matched by path/tags).
+    priority_score:
+        Composite priority score in ``[0.0, 1.0]`` assigned by the
+        belief-guided prioritizer.  Higher means test first.
+    check_ids:
+        OWASP check IDs recommended for this endpoint by the prioritizer.
+    metadata:
+        Arbitrary additional data (e.g., source ingestor name, raw priority hint).
+    """
+
+    endpoint: IngestedEndpoint
+    test_cases: list[IngestedTestCase] = field(default_factory=list)
+    priority_score: float = 0.5
+    check_ids: list[str] = field(default_factory=list)
+    metadata: dict = field(default_factory=dict)
+
+    def endpoint_key(self) -> str:
+        """Return ``"METHOD /path"`` string for logging and lookup."""
+        return f"{self.endpoint.method.upper()} {self.endpoint.path}"
+
+
+@dataclass
+class ScanPlan:
+    """A prioritized, ordered list of endpoints to scan.
+
+    Attributes
+    ----------
+    targets:
+        Scan targets in descending priority order (highest first).
+    total_endpoints:
+        Number of distinct endpoints in the plan.
+    total_test_cases:
+        Total number of test intents across all targets.
+    ingest_stats:
+        Stats returned by the ingestor (passed through for reference).
+    metadata:
+        Arbitrary metadata (ingestor name, source path, timestamp, etc.).
+    """
+
+    targets: list[ScanTarget] = field(default_factory=list)
+    total_endpoints: int = 0
+    total_test_cases: int = 0
+    ingest_stats: dict = field(default_factory=dict)
+    metadata: dict = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        """Serialize to a plain dict suitable for JSON output."""
+        return {
+            "total_endpoints": self.total_endpoints,
+            "total_test_cases": self.total_test_cases,
+            "ingest_stats": self.ingest_stats,
+            "metadata": self.metadata,
+            "targets": [
+                {
+                    "endpoint": t.endpoint_key(),
+                    "priority_score": t.priority_score,
+                    "check_ids": t.check_ids,
+                    "test_cases": [tc.name for tc in t.test_cases],
+                    "metadata": t.metadata,
+                }
+                for t in self.targets
+            ],
+        }
+
+
+@dataclass
+class ScanFeedback:
+    """Learning data captured after a completed scan.
+
+    Attributes
+    ----------
+    scan_id:
+        UUID of the associated security scan.
+    total_targets:
+        Number of endpoints that were scanned.
+    findings_count:
+        Total number of security findings across all endpoints.
+    per_target:
+        Per-endpoint breakdown: list of dicts with keys
+        ``endpoint``, ``priority_score``, ``findings_count``,
+        ``matched_test_cases`` (names), ``outcome_matched`` (bool).
+    priority_accuracy:
+        Fraction of findings that came from the top-50 % of endpoints
+        by priority score.  Value in ``[0.0, 1.0]``; higher is better.
+    suggested_adjustments:
+        Endpoint keys that should receive a belief-score boost on the
+        next run (those that had unexpected findings or missed expected ones).
+    """
+
+    scan_id: str = ""
+    total_targets: int = 0
+    findings_count: int = 0
+    per_target: list[dict] = field(default_factory=list)
+    priority_accuracy: float = 0.0
+    suggested_adjustments: dict = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        """Serialize to a plain dict suitable for JSON output."""
+        return {
+            "scan_id": self.scan_id,
+            "total_targets": self.total_targets,
+            "findings_count": self.findings_count,
+            "per_target": self.per_target,
+            "priority_accuracy": self.priority_accuracy,
+            "suggested_adjustments": self.suggested_adjustments,
+        }
+
+
+@dataclass
+class OrchestrationResult:
+    """Full result of an orchestration run.
+
+    Attributes
+    ----------
+    config:
+        The configuration used for this run.
+    ingest_stats:
+        Raw stats returned by the ingestor plugin.
+    scan_plan:
+        The prioritized scan plan (populated after the prioritize phase).
+    scan_results:
+        The full :class:`~mannf.product.security.models.SecurityReport`,
+        or ``None`` in dry-run / plan-only mode.
+    feedback:
+        Learning data from the scan phase, or ``None`` in dry-run mode.
+    export_summaries:
+        One :class:`~mannf.product.exporters.base.ExportSummary` per exporter
+        that was invoked after scanning.  Empty when no exporter is configured
+        or when running in dry-run / plan-only mode.
+    errors:
+        Fatal errors encountered during the pipeline (ingestor failures, etc.).
+    warnings:
+        Non-fatal warnings emitted during the pipeline.
+    """
+
+    config: OrchestrationConfig = field(default_factory=OrchestrationConfig)
+    ingest_stats: dict = field(default_factory=dict)
+    scan_plan: ScanPlan = field(default_factory=ScanPlan)
+    scan_results: SecurityReport | None = None
+    feedback: ScanFeedback | None = None
+    export_summaries: list[ExportSummary] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    # LLM root cause suggestions for high-severity findings (Phase 10.3)
+    root_cause_suggestions: list[dict] = field(default_factory=list)
+
+    @property
+    def success(self) -> bool:
+        """True if there were no fatal errors in the pipeline."""
+        return len(self.errors) == 0
+
+    def to_dict(self) -> dict:
+        """Serialize to a plain dict suitable for JSON output."""
+        return {
+            "success": self.success,
+            "ingest_stats": self.ingest_stats,
+            "scan_plan": self.scan_plan.to_dict(),
+            "scan_results": self.scan_results.to_dict() if self.scan_results else None,
+            "feedback": self.feedback.to_dict() if self.feedback else None,
+            "root_cause_suggestions": self.root_cause_suggestions,
+            "export_summaries": [
+                {
+                    "total": s.total,
+                    "succeeded": s.succeeded,
+                    "failed": s.failed,
+                    "skipped": s.skipped,
+                }
+                for s in self.export_summaries
+            ],
+            "errors": self.errors,
+            "warnings": self.warnings,
+        }

mannf/product/regression/__init__.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Regression record/replay package for NAT.
+
+Provides :class:`~mannf.product.regression.recorder.RegressionRecorder`,
+:class:`~mannf.product.regression.replayer.RegressionReplayer`, and
+:class:`~mannf.product.regression.differ.RegressionDiffer` for recording API
+test runs as portable baselines and comparing future runs against them.
+"""
+
+from __future__ import annotations
+
+from mannf.product.regression.differ import RegressionDiffer
+from mannf.product.regression.masking import mask_sensitive
+from mannf.product.regression.models import (
+    RegressionBaseline,
+    RegressionDiff,
+    RequestRecord,
+    ResponseRecord,
+)
+from mannf.product.regression.recorder import RegressionRecorder
+from mannf.product.regression.replayer import RegressionReplayer
+
+__all__ = [
+    "RegressionRecorder",
+    "RegressionReplayer",
+    "RegressionDiffer",
+    "RegressionBaseline",
+    "RegressionDiff",
+    "RequestRecord",
+    "ResponseRecord",
+    "mask_sensitive",
+]

mannf/product/regression/differ.py

@@ -0,0 +1,172 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""RegressionDiffer — compares a baseline recording to a replay recording."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from mannf.product.regression.models import (
+    FieldChange,
+    RecordDiff,
+    RegressionBaseline,
+    RegressionDiff,
+    RegressionRecord,
+)
+
+# Latency regression threshold: flag if replay is ≥ this factor slower than baseline
+_LATENCY_REGRESSION_FACTOR = 2.0
+
+
+class RegressionDiffer:
+    """Compare a baseline :class:`~mannf.product.regression.models.RegressionBaseline` to a
+    replay baseline and produce a :class:`~mannf.product.regression.models.RegressionDiff`.
+
+    Parameters
+    ----------
+    latency_regression_factor:
+        Multiplier applied to the baseline latency.  If the replay latency
+        exceeds ``baseline_latency * factor``, a latency regression is reported.
+        Defaults to ``2.0`` (100 % slower = regression).
+    check_headers:
+        When ``True``, compare response headers between baseline and replay.
+        Defaults to ``False`` (headers are often volatile).
+    """
+
+    def __init__(
+        self,
+        latency_regression_factor: float = _LATENCY_REGRESSION_FACTOR,
+        check_headers: bool = False,
+    ) -> None:
+        self.latency_regression_factor = latency_regression_factor
+        self.check_headers = check_headers
+
+    def diff(
+        self,
+        baseline: RegressionBaseline,
+        replay: RegressionBaseline,
+    ) -> RegressionDiff:
+        """Compute the diff between *baseline* and *replay*.
+
+        Parameters
+        ----------
+        baseline:
+            The original recording.
+        replay:
+            The new recording produced by :class:`~mannf.product.regression.replayer.RegressionReplayer`.
+
+        Returns
+        -------
+        RegressionDiff
+            Structured diff report.
+        """
+        baseline_by_id: Dict[str, RegressionRecord] = {
+            r.test_case_id: r for r in baseline.records
+        }
+        replay_by_id: Dict[str, RegressionRecord] = {
+            r.test_case_id: r for r in replay.records
+        }
+
+        baseline_ids = set(baseline_by_id.keys())
+        replay_ids = set(replay_by_id.keys())
+
+        new_ids = replay_ids - baseline_ids
+        removed_ids = baseline_ids - replay_ids
+        common_ids = baseline_ids & replay_ids
+
+        changes: List[RecordDiff] = []
+
+        for tc_id in sorted(common_ids):
+            b_rec = baseline_by_id[tc_id]
+            r_rec = replay_by_id[tc_id]
+            diffs = self._compare_records(b_rec, r_rec)
+            if diffs:
+                changes.append(
+                    RecordDiff(
+                        test_case_id=tc_id,
+                        endpoint=f"{b_rec.request.method} {b_rec.request.url}",
+                        changes=diffs,
+                    )
+                )
+
+        total = len(baseline_ids) + len(new_ids)
+        changed = len(changes)
+        unchanged = len(common_ids) - changed
+
+        summary: Dict[str, int] = {
+            "total": total,
+            "unchanged": unchanged,
+            "changed": changed,
+            "new": len(new_ids),
+            "removed": len(removed_ids),
+        }
+
+        return RegressionDiff(summary=summary, changes=changes)
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _compare_records(
+        self,
+        baseline: RegressionRecord,
+        replay: RegressionRecord,
+    ) -> List[FieldChange]:
+        field_changes: List[FieldChange] = []
+
+        # Status code
+        if baseline.response.status_code != replay.response.status_code:
+            field_changes.append(
+                FieldChange(
+                    field="response.status_code",
+                    baseline=baseline.response.status_code,
+                    replay=replay.response.status_code,
+                )
+            )
+
+        # Response body
+        if baseline.response.body != replay.response.body:
+            field_changes.append(
+                FieldChange(
+                    field="response.body",
+                    baseline=baseline.response.body,
+                    replay=replay.response.body,
+                )
+            )
+
+        # Pass/fail
+        if baseline.passed != replay.passed:
+            field_changes.append(
+                FieldChange(
+                    field="passed",
+                    baseline=baseline.passed,
+                    replay=replay.passed,
+                )
+            )
+
+        # Latency regression
+        b_latency = baseline.response.latency_ms
+        r_latency = replay.response.latency_ms
+        if b_latency > 0 and r_latency > b_latency * self.latency_regression_factor:
+            field_changes.append(
+                FieldChange(
+                    field="response.latency_ms",
+                    baseline=b_latency,
+                    replay=r_latency,
+                )
+            )
+
+        # Response headers (optional)
+        if self.check_headers and baseline.response.headers != replay.response.headers:
+            field_changes.append(
+                FieldChange(
+                    field="response.headers",
+                    baseline=baseline.response.headers,
+                    replay=replay.response.headers,
+                )
+            )
+
+        return field_changes

mannf/product/regression/masking.py

@@ -0,0 +1,100 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Sensitive data masking for regression recordings.
+
+When ``--mask-sensitive`` is used, values in headers such as ``Authorization``,
+``X-API-Key``, cookie values, and OAuth tokens are replaced with
+``***MASKED***`` before being written to disk.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Optional
+
+_MASKED = "***MASKED***"
+
+# Header names that carry sensitive data (case-insensitive matching)
+_SENSITIVE_HEADERS = frozenset(
+    {
+        "authorization",
+        "proxy-authorization",
+        "x-api-key",
+        "x-auth-token",
+        "x-access-token",
+        "cookie",
+        "set-cookie",
+        "x-csrf-token",
+        "api-key",
+        "x-secret",
+        "x-token",
+    }
+)
+
+# Patterns in JSON body / query-param values that look like tokens
+_TOKEN_PATTERNS = [
+    re.compile(r"^Bearer\s+\S+", re.IGNORECASE),
+    re.compile(r"^Basic\s+\S+", re.IGNORECASE),
+]
+
+
+def _is_sensitive_header(name: str) -> bool:
+    return name.lower() in _SENSITIVE_HEADERS
+
+
+def mask_headers(headers: Dict[str, str]) -> Dict[str, str]:
+    """Return a copy of *headers* with sensitive values replaced by ``***MASKED***``."""
+    return {
+        k: (_MASKED if _is_sensitive_header(k) else v)
+        for k, v in headers.items()
+    }
+
+
+def _mask_value(value: Any) -> Any:
+    """Mask a single string value if it looks like a token."""
+    if not isinstance(value, str):
+        return value
+    for pattern in _TOKEN_PATTERNS:
+        if pattern.match(value):
+            return _MASKED
+    return value
+
+
+def _mask_dict_values(data: Any) -> Any:
+    """Recursively mask token-like string values inside *data*."""
+    if isinstance(data, dict):
+        return {k: _mask_dict_values(v) for k, v in data.items()}
+    if isinstance(data, list):
+        return [_mask_dict_values(item) for item in data]
+    return _mask_value(data)
+
+
+def mask_sensitive(
+    headers: Optional[Dict[str, str]] = None,
+    body: Any = None,
+    query_params: Optional[Dict[str, Any]] = None,
+) -> tuple:
+    """Apply all masking rules and return ``(masked_headers, masked_body, masked_params)``.
+
+    Parameters
+    ----------
+    headers:
+        HTTP headers dict (may be ``None``).
+    body:
+        Request/response body (any JSON-serialisable type).
+    query_params:
+        URL query parameters dict (may be ``None``).
+
+    Returns
+    -------
+    tuple
+        ``(masked_headers, masked_body, masked_query_params)`` with all
+        sensitive values replaced by ``***MASKED***``.
+    """
+    masked_hdrs = mask_headers(headers or {})
+    masked_body = _mask_dict_values(body)
+    masked_params = _mask_dict_values(query_params) if query_params else query_params
+    return masked_hdrs, masked_body, masked_params