nat-engine 1__py3-none-any.whl

mannf/core/prioritization/adaptive_controller.py

@@ -0,0 +1,509 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Adaptive Test Generation Controller.
+
+Extends the base ε-greedy :class:`~mannf.core.testing.adaptive_controller.AdaptiveController`
+with smarter test-generation strategies guided by BDI belief signals:
+
+* **Boundary value analysis** — focus on inputs at boundaries where the neural
+  network predicts highest fault likelihood.
+* **Mutation-based fuzzing** — mutate existing test cases more aggressively
+  for endpoints the agents believe are fault-prone.
+* **Sequence testing** — identify and test multi-step API flows (e.g.
+  create → read → update → delete).
+
+Integration with :class:`~mannf.core.prioritization.belief_prioritizer.BeliefPrioritizer`
+allocates test-generation effort proportionally to endpoint risk.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import logging
+import random
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+
+import numpy as np
+
+from mannf.core.agents.belief_state import BeliefState
+from mannf.core.neural import NeuralNetwork
+from mannf.core.prioritization.belief_prioritizer import BeliefPrioritizer
+from mannf.core.testing.models import TestCase, TestResult
+
+if TYPE_CHECKING:
+    from mannf.llm.base import LLMProvider
+
+logger = logging.getLogger(__name__)
+
+_FEATURE_DIM = 8
+
+
+class AdaptiveController:
+    """Adaptive test-generation controller with BDI-guided strategies.
+
+    Parameters
+    ----------
+    initial_epsilon:
+        Initial exploration probability (0–1).
+    min_epsilon:
+        Minimum exploration probability after decay.
+    epsilon_decay:
+        Multiplicative decay applied after each :meth:`update` call.
+    learning_rate:
+        Step size for the value-network update.
+    prioritizer:
+        Optional :class:`BeliefPrioritizer` to allocate generation effort.
+    mutation_intensity:
+        Base mutation intensity (0–1).  Higher values apply larger random
+        perturbations to mutated inputs.
+    """
+
+    # Common CRUD sequence patterns used by the sequence tester
+    _CRUD_SEQUENCE: List[str] = ["create", "read", "update", "delete"]
+    _CRUD_METHODS: Dict[str, str] = {
+        "create": "POST",
+        "read": "GET",
+        "update": "PUT",
+        "delete": "DELETE",
+    }
+
+    def __init__(
+        self,
+        initial_epsilon: float = 0.9,
+        min_epsilon: float = 0.05,
+        epsilon_decay: float = 0.995,
+        learning_rate: float = 0.005,
+        prioritizer: Optional[BeliefPrioritizer] = None,
+        mutation_intensity: float = 0.3,
+        llm_provider: Optional["LLMProvider"] = None,
+    ) -> None:
+        self.epsilon = initial_epsilon
+        self.min_epsilon = min_epsilon
+        self.epsilon_decay = epsilon_decay
+        self.mutation_intensity = mutation_intensity
+        self.prioritizer = prioritizer
+        self.llm_provider = llm_provider
+
+        self.value_net = NeuralNetwork(
+            layer_sizes=[_FEATURE_DIM, 16, 8, 1],
+            hidden_activation="relu",
+            learning_rate=learning_rate,
+        )
+
+        self._step_count = 0
+        self._cumulative_reward = 0.0
+
+        # Sequence memory: stores observed (endpoint, method) pairs to infer flows
+        self._sequence_memory: List[Tuple[str, str]] = []
+        self._max_sequence_memory = 200
+
+    # ------------------------------------------------------------------
+    # Selection (ε-greedy)
+    # ------------------------------------------------------------------
+
+    def select(self, candidates: List[TestCase]) -> Optional[TestCase]:
+        """Pick the next test case using ε-greedy selection.
+
+        If a *prioritizer* is set, candidates are pre-ordered by risk before
+        the greedy selection is applied.
+        """
+        if not candidates:
+            return None
+
+        ordered = candidates
+        if self.prioritizer is not None:
+            ordered = self.prioritizer.prioritize(candidates)
+
+        if np.random.random() < self.epsilon:
+            choice = ordered[int(np.random.randint(len(ordered)))]
+            logger.debug("AdaptiveController: exploring → %s", choice.id[:8])
+        else:
+            scores = [
+                float(self.value_net.predict(c.feature_vector(_FEATURE_DIM))[0])
+                for c in ordered
+            ]
+            best_idx = int(np.argmax(scores))
+            choice = ordered[best_idx]
+            logger.debug(
+                "AdaptiveController: exploiting → %s (score=%.4f)",
+                choice.id[:8],
+                scores[best_idx],
+            )
+        return choice
+
+    # ------------------------------------------------------------------
+    # Learning
+    # ------------------------------------------------------------------
+
+    def update(self, test_case: TestCase, result: TestResult) -> float:
+        """Update the value network with the observed reward signal.
+
+        Returns the MSE training loss for the current step.
+        """
+        reward = result.reward_signal()
+        self._cumulative_reward += reward
+
+        x = test_case.feature_vector(_FEATURE_DIM)
+        target = np.array([reward])
+        loss = self.value_net.train_step(x, target)
+
+        # Record for sequence inference
+        method = test_case.metadata.get("method", "GET")
+        self._record_sequence(test_case.target, method)
+
+        # Update prioritizer if available
+        if self.prioritizer is not None:
+            self.prioritizer.record_result(result, test_case.target)
+
+        self.epsilon = max(self.min_epsilon, self.epsilon * self.epsilon_decay)
+        self._step_count += 1
+
+        logger.debug(
+            "AdaptiveController update: reward=%.3f loss=%.6f ε=%.4f",
+            reward,
+            loss,
+            self.epsilon,
+        )
+        return loss
+
+    # ------------------------------------------------------------------
+    # Boundary value analysis
+    # ------------------------------------------------------------------
+
+    def generate_boundary_variants(
+        self,
+        base_case: TestCase,
+        belief_states: Optional[List[BeliefState]] = None,
+    ) -> List[TestCase]:
+        """Generate boundary-value variants of *base_case*.
+
+        For each numeric input parameter, creates test cases at:
+        * value = 0
+        * value = -1 (underflow)
+        * value = maximum observed + 1 (overflow)
+        * value = ``None`` / missing (null injection)
+
+        The intensity of boundary analysis is scaled by the endpoint's belief
+        signal — high-risk endpoints produce more variants.
+        """
+        risk_multiplier = self._endpoint_risk(base_case.target, belief_states)
+        # Number of variants scales with risk (1–4)
+        n_variants = max(1, round(risk_multiplier * 4))
+
+        variants: List[TestCase] = []
+        numeric_keys = [
+            k for k, v in base_case.inputs.items() if isinstance(v, (int, float))
+        ]
+
+        boundary_values: List[Any] = [0, -1, 2**31 - 1, None]
+
+        for key in numeric_keys:
+            original_val = base_case.inputs[key]
+            for bv in boundary_values[: n_variants]:
+                new_inputs = dict(base_case.inputs)
+                new_inputs[key] = bv
+                variant = TestCase(
+                    target=base_case.target,
+                    inputs=new_inputs,
+                    expected_behavior=(
+                        f"Boundary value {bv!r} for '{key}' — expect graceful handling"
+                    ),
+                    priority=min(1.0, base_case.priority + 0.1 * risk_multiplier),
+                    metadata={
+                        **base_case.metadata,
+                        "generation_strategy": "boundary_value_analysis",
+                        "boundary_key": key,
+                        "boundary_value": str(bv),
+                        "source_case_id": base_case.id,
+                    },
+                )
+                variants.append(variant)
+
+        if not variants:
+            # No numeric params: inject empty input as boundary
+            variant = TestCase(
+                target=base_case.target,
+                inputs={},
+                expected_behavior="Empty inputs — expect graceful handling",
+                priority=base_case.priority,
+                metadata={
+                    **base_case.metadata,
+                    "generation_strategy": "boundary_value_analysis",
+                    "boundary_key": "__empty__",
+                    "source_case_id": base_case.id,
+                },
+            )
+            variants.append(variant)
+
+        return variants
+
+    # ------------------------------------------------------------------
+    # Mutation-based fuzzing
+    # ------------------------------------------------------------------
+
+    def mutate(
+        self,
+        base_case: TestCase,
+        belief_states: Optional[List[BeliefState]] = None,
+        n_mutations: int = 3,
+    ) -> List[TestCase]:
+        """Generate mutated variants of *base_case*.
+
+        Mutation intensity scales with the endpoint's BDI risk signal:
+        high-risk endpoints receive more aggressive perturbations.
+
+        Mutation operations (applied randomly):
+        * Numeric values: add Gaussian noise scaled by mutation intensity.
+        * String values: random character substitution or length change.
+        * Boolean values: flip.
+        * Keys: randomly drop one key (missing-parameter test).
+        """
+        risk = self._endpoint_risk(base_case.target, belief_states)
+        intensity = self.mutation_intensity * (0.5 + risk)
+
+        mutations: List[TestCase] = []
+        for i in range(n_mutations):
+            mutated_inputs = self._apply_mutation(base_case.inputs, intensity)
+            mutated = TestCase(
+                target=base_case.target,
+                inputs=mutated_inputs,
+                expected_behavior=f"Mutation #{i + 1} of '{base_case.id[:8]}' — fault injection",
+                priority=min(1.0, base_case.priority + 0.05 * risk),
+                metadata={
+                    **base_case.metadata,
+                    "generation_strategy": "mutation_fuzzing",
+                    "mutation_index": i,
+                    "mutation_intensity": round(intensity, 3),
+                    "source_case_id": base_case.id,
+                },
+            )
+            mutations.append(mutated)
+        return mutations
+
+    async def mutate_with_llm(
+        self,
+        base_case: TestCase,
+        belief_states: Optional[List[BeliefState]] = None,
+        n_mutations: int = 3,
+    ) -> List[TestCase]:
+        """Generate mutated variants, adding one LLM-guided smart mutation when available.
+
+        The LLM mutation is appended to the random mutations produced by
+        :meth:`mutate` and tagged with ``generation_strategy = "llm_mutation"``.
+        Falls back to :meth:`mutate` if the LLM is unavailable or fails.
+        """
+        mutations = self.mutate(base_case, belief_states=belief_states, n_mutations=n_mutations)
+
+        if self.llm_provider is None or not self.llm_provider.is_available():
+            return mutations
+
+        from mannf.llm.prompts import MUTATION_PROMPT  # noqa: PLC0415
+
+        prompt = MUTATION_PROMPT.format(
+            n=1,
+            target=base_case.target,
+            inputs=json.dumps(base_case.inputs, indent=2),
+            expected_behavior=base_case.expected_behavior,
+        )
+        try:
+            raw = await self.llm_provider.generate(prompt)
+            raw_cases = self.llm_provider._parse_test_cases(raw)
+            for raw_case in raw_cases[:1]:
+                if not isinstance(raw_case, dict):
+                    continue
+                inputs = raw_case.get("inputs", {})
+                if not isinstance(inputs, dict):
+                    inputs = copy.deepcopy(base_case.inputs)
+                mutations.append(
+                    TestCase(
+                        target=base_case.target,
+                        inputs=inputs,
+                        expected_behavior=raw_case.get(
+                            "expected_behavior",
+                            f"LLM mutation of '{base_case.id[:8]}'",
+                        ),
+                        priority=min(1.0, base_case.priority + 0.1),
+                        metadata={
+                            **base_case.metadata,
+                            "generation_strategy": "llm_mutation",
+                            "generator": "llm",
+                            "source_case_id": base_case.id,
+                            "mutation_rationale": raw_case.get("mutation_rationale", ""),
+                        },
+                    )
+                )
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "AdaptiveController: LLM mutation failed for %s: %s",
+                base_case.target, exc,
+            )
+
+        return mutations
+
+    def _apply_mutation(self, inputs: Dict[str, Any], intensity: float) -> Dict[str, Any]:
+        mutated = copy.deepcopy(inputs)
+        if not mutated:
+            return mutated
+
+        keys = list(mutated.keys())
+        op = random.choice(["perturb", "drop", "flip", "null"])
+
+        if op == "drop" and len(keys) > 1:
+            drop_key = random.choice(keys)
+            del mutated[drop_key]
+        elif op == "null":
+            null_key = random.choice(keys)
+            mutated[null_key] = None
+        elif op == "flip":
+            bool_keys = [k for k, v in mutated.items() if isinstance(v, bool)]
+            if bool_keys:
+                flip_key = random.choice(bool_keys)
+                mutated[flip_key] = not mutated[flip_key]
+        else:  # perturb numerics / strings
+            for key in keys:
+                val = mutated[key]
+                if isinstance(val, (int, float)):
+                    noise = random.gauss(0, intensity * max(1.0, abs(val)))
+                    mutated[key] = type(val)(val + noise) if isinstance(val, float) else int(val + noise)
+                elif isinstance(val, str):
+                    if val and random.random() < intensity:
+                        idx = random.randrange(len(val))
+                        mutated[key] = val[:idx] + random.choice("abcdefXYZ!@#") + val[idx + 1:]
+
+        return mutated
+
+    def generate_sequence_tests(
+        self,
+        endpoints: List[str],
+        base_inputs: Optional[Dict[str, Dict[str, Any]]] = None,
+        belief_states: Optional[List[BeliefState]] = None,
+    ) -> List[List[TestCase]]:
+        """Generate multi-step API flow test sequences.
+
+        Infers CRUD sequences from the observed endpoint list and optionally
+        from the planner agent's sequence memory.  Returns a list of test-case
+        *sequences*, each representing a complete API flow.
+
+        Parameters
+        ----------
+        endpoints:
+            Known endpoint targets (e.g. ``["/users", "/users/{id}"]``).
+        base_inputs:
+            Mapping of endpoint → default inputs.  Defaults to empty dicts.
+        belief_states:
+            Optional belief states to prioritize high-risk sequences.
+
+        Returns
+        -------
+        list of list of TestCase
+            Each inner list is an ordered sequence of test cases forming a flow.
+        """
+        base_inputs = base_inputs or {}
+        sequences: List[List[TestCase]] = []
+
+        inferred = self._infer_sequences(endpoints)
+        for seq_endpoints in inferred:
+            seq = []
+            for i, (ep, method) in enumerate(seq_endpoints):
+                tc = TestCase(
+                    target=ep,
+                    inputs=dict(base_inputs.get(ep, {})),
+                    expected_behavior=f"Sequence step {i + 1}/{len(seq_endpoints)}: {method} {ep}",
+                    priority=self._endpoint_risk(ep, belief_states),
+                    metadata={
+                        "generation_strategy": "sequence_testing",
+                        "sequence_step": i,
+                        "sequence_length": len(seq_endpoints),
+                        "method": method,
+                    },
+                )
+                seq.append(tc)
+            if seq:
+                sequences.append(seq)
+
+        return sequences
+
+    def _infer_sequences(
+        self, endpoints: List[str]
+    ) -> List[List[Tuple[str, str]]]:
+        """Heuristically group endpoints into CRUD-like sequences."""
+        # Group endpoints by common resource root
+        # e.g. /users, /users/{id} → one group
+        groups: Dict[str, List[str]] = {}
+        for ep in endpoints:
+            root = ep.split("{")[0].rstrip("/")
+            if root not in groups:
+                groups[root] = []
+            groups[root].append(ep)
+
+        sequences = []
+        for root, eps in groups.items():
+            # Assign CRUD methods heuristically
+            seq: List[Tuple[str, str]] = []
+            has_id = any("{" in ep for ep in eps)
+            base_ep = min(eps, key=len)  # shortest = collection endpoint
+            detail_ep = max(eps, key=len) if has_id else base_ep
+
+            seq.append((base_ep, "POST"))   # create
+            seq.append((detail_ep, "GET"))  # read
+            if has_id:
+                seq.append((detail_ep, "PUT"))    # update
+                seq.append((detail_ep, "DELETE"))  # delete
+
+            if len(seq) >= 2:
+                sequences.append(seq)
+
+        # Also add any patterns observed in sequence memory
+        if len(self._sequence_memory) >= 3:
+            # Extract unique transitions
+            mem_seq: List[Tuple[str, str]] = []
+            seen: set = set()
+            for item in self._sequence_memory[-10:]:
+                if item not in seen:
+                    seen.add(item)
+                    mem_seq.append(item)
+            if len(mem_seq) >= 2:
+                sequences.append(mem_seq)
+
+        return sequences
+
+    def _record_sequence(self, endpoint: str, method: str) -> None:
+        self._sequence_memory.append((endpoint, method))
+        if len(self._sequence_memory) > self._max_sequence_memory:
+            self._sequence_memory.pop(0)
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _endpoint_risk(
+        self,
+        endpoint: str,
+        belief_states: Optional[List[BeliefState]],
+    ) -> float:
+        if not belief_states:
+            return 0.5
+        signals = [bs.fault_likelihood.get(endpoint, 0.5) for bs in belief_states]
+        return sum(signals) / len(signals)
+
+    # ------------------------------------------------------------------
+    # Diagnostics
+    # ------------------------------------------------------------------
+
+    @property
+    def step_count(self) -> int:
+        return self._step_count
+
+    @property
+    def average_reward(self) -> float:
+        if self._step_count == 0:
+            return 0.0
+        return self._cumulative_reward / self._step_count
+
+    def score(self, test_case: TestCase) -> float:
+        """Return the current predicted value for a test case."""
+        return float(self.value_net.predict(test_case.feature_vector(_FEATURE_DIM))[0])