ctrlcode 0.1.0__py3-none-any.whl

ctrlcode/fuzzing/context_fuzzer.py

@@ -0,0 +1,506 @@
+"""Context-aware fuzzer - generates input and environment test cases."""
+
+import json
+import logging
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from typing import Any, Literal, Optional
+
+import networkx as nx
+import numpy as np
+
+from ..analysis.bug_detector import BugPatternDetector
+from ..analysis.code_graphs import CodeGraphs
+from ..embeddings.embedder import CodeEmbedder
+from ..embeddings.vector_store import VectorStore
+from ..providers.base import Provider
+from ..storage.history_db import HistoryDB, StoredTest
+from .context import ContextDerivation
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class EnvironmentScenario:
+    """Environment simulation scenario."""
+
+    description: str
+    mock_setup: dict[str, Any]  # Dependency behaviors to simulate
+    expected_behavior: dict[str, Any]  # Derived oracle
+    invariants_checked: list[str]
+
+
+@dataclass
+class FuzzTestCase:
+    """Single fuzz test case (input and/or environment)."""
+
+    id: str
+    type: Literal["input", "environment", "combined"]
+    input: Optional[dict[str, Any]]  # Actual input (if type=input/combined)
+    environment: Optional[EnvironmentScenario]  # Environment sim (if type=environment/combined)
+    rationale: str
+    category: Literal["exploit", "explore", "cover", "stress", "environment"]
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary."""
+        return asdict(self)
+
+
+class ContextAwareFuzzer:
+    """
+    Generates test cases for context-aware differential fuzzing.
+
+    Unlike traditional input fuzzing, generates TWO kinds of test cases:
+    - Input fuzzing: diverse inputs to the function
+    - Environment fuzzing: simulated environmental conditions
+
+    Phase 3: Enhanced with graph-aware expansion, bug pattern detection,
+    and test case clustering/deduplication.
+    """
+
+    def __init__(
+        self,
+        provider: Provider,
+        history_db: Optional[HistoryDB] = None,
+        embedder: Optional[CodeEmbedder] = None,
+        bug_detector: Optional[BugPatternDetector] = None,
+        graph_expansion_depth: int = 2,
+        test_clustering_threshold: float = 0.8,
+    ):
+        """
+        Initialize context-aware fuzzer.
+
+        Args:
+            provider: LLM provider for test generation
+            history_db: Optional history database for bug patterns and test clustering
+            embedder: Optional code embedder for test clustering
+            bug_detector: Optional bug pattern detector
+            graph_expansion_depth: Max depth for graph traversal (default: 2)
+            test_clustering_threshold: Similarity threshold for test deduplication (default: 0.8)
+        """
+        self.provider = provider
+        self.history_db = history_db
+        self.embedder = embedder or CodeEmbedder()
+        self.bug_detector = bug_detector
+        self.graph_expansion_depth = graph_expansion_depth
+        self.test_clustering_threshold = test_clustering_threshold
+
+    async def generate_tests(
+        self,
+        spec: str,
+        code: str,
+        context: ContextDerivation,
+        previous_results: Optional[list[dict]] = None,
+        batch_size: int = 20,
+    ) -> list[FuzzTestCase]:
+        """
+        Generate batch of test cases with derived expected behaviors.
+
+        Args:
+            spec: Original specification
+            code: Generated code
+            context: Derived context from ContextDerivationEngine
+            previous_results: Previous test results (for guided fuzzing)
+            batch_size: Number of tests to generate
+
+        Returns:
+            List of FuzzTestCase instances
+
+        Distribution:
+        - 40% environment scenarios (dependency failures, timing, concurrency)
+        - 30% input edge cases
+        - 20% combined (tricky input + hostile environment)
+        - 10% invariant-focused (directly test each behavioral invariant)
+        """
+        # Phase 3: Check for similar bug patterns before generating tests
+        bug_warnings = ""
+        if self.bug_detector:
+            similar_bugs = self.bug_detector.check_patterns(code)
+            if similar_bugs:
+                bug_warnings = self._build_bug_warnings_section(similar_bugs)
+                logger.warning(f"Detected {len(similar_bugs)} similar bug patterns")
+
+        # Phase 3: Build graph expansion context for environment scenarios
+        graph_context = ""
+        if context.code_graphs:
+            graph_context = self._build_graph_context_section(context.code_graphs, context)
+
+        # Build system prompt (from DIFFFUZZTEST.md lines 232-294)
+        system_prompt = """You are a fuzz test generator for differential testing. Unlike traditional
+input fuzzing, you generate TWO kinds of test cases:
+
+**A) Input Fuzzing** — diverse inputs to the function itself
+**B) Environment Fuzzing** — simulated environmental conditions that test
+   how the code behaves in its derived system context
+
+You will be given:
+1. The original specification
+2. The generated code
+3. A Context Derivation Report (system placement, constraints, invariants,
+   edge cases, and implicit assumptions)
+4. Previous test results and analyses (if any)
+
+## For Input Fuzzing:
+Generate diverse inputs as before — edge cases, boundary values, adversarial
+inputs. Focus on the behavioral invariants from the context report.
+
+## For Environment Fuzzing:
+Generate SCENARIOS that simulate environmental conditions:
+- Dependency failures (API returns 500, database times out, etc.)
+- Resource pressure (slow responses, connection limits hit)
+- Concurrency scenarios (parallel calls, race conditions)
+- Configuration edge cases (missing env vars, wrong permissions)
+- Temporal edge cases (clock skew, timezone boundaries)
+
+Each environment scenario specifies:
+- What mocks/stubs to set up (dependency behavior)
+- What conditions to simulate (timing, ordering, resource state)
+- What the EXPECTED behavior is (derived from the invariants)
+
+Output format — JSON array:
+[
+  {
+    "id": "fuzz_001",
+    "type": "input | environment | combined",
+    "input": { ... },
+    "environment": {
+      "description": "API returns 429 on first 2 calls, then 200",
+      "mock_setup": {
+        "responses": [
+          {"status": 429, "headers": {"Retry-After": "1"}, "delay_ms": 0},
+          {"status": 429, "headers": {}, "delay_ms": 0},
+          {"status": 200, "body": {"ok": true}, "delay_ms": 50}
+        ]
+      },
+      "expected_behavior": {
+        "should_succeed": true,
+        "min_total_time_ms": 1000,
+        "max_retries": 2,
+        "invariants_checked": ["backoff_increases", "respects_retry_after"]
+      },
+      "invariants_checked": ["backoff_increases", "respects_retry_after"]
+    },
+    "rationale": "Tests basic retry-then-succeed with Retry-After header",
+    "category": "exploit | explore | cover | stress | environment"
+  }
+]
+
+CRITICAL: The `expected_behavior` field is your DERIVED ORACLE. Since there
+is no real system to compare against, the expected behavior IS the test
+oracle. Derive it strictly from the specification and the behavioral
+invariants in the context report."""
+
+        # Build user message
+        history_section = (
+            json.dumps(previous_results, indent=2)
+            if previous_results
+            else "First iteration, no previous results."
+        )
+
+        user_message = f"""## Specification
+{spec}
+
+## Generated Code
+```
+{code}
+```
+
+## Context Derivation Report
+{context.to_json()}
+
+{graph_context}
+
+{bug_warnings}
+
+## Previous Results
+{history_section}
+
+Generate a batch of {batch_size} test cases.
+Distribution:
+- 40% environment scenarios (dependency failures, timing, concurrency)
+- 30% input edge cases
+- 20% combined (tricky input + hostile environment)
+- 10% invariant-focused (directly test each behavioral invariant)"""
+
+        # Call LLM
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_message},
+        ]
+
+        response = await self.provider.generate(messages)
+        response_text = response.get("text", "").strip()
+
+        # Parse JSON response
+        try:
+            # Extract JSON from markdown code blocks if present
+            if "```json" in response_text:
+                start = response_text.find("```json") + 7
+                end = response_text.find("```", start)
+                response_text = response_text[start:end].strip()
+            elif "```" in response_text:
+                start = response_text.find("```") + 3
+                end = response_text.find("```", start)
+                response_text = response_text[start:end].strip()
+
+            data = json.loads(response_text)
+
+            # Convert to FuzzTestCase instances
+            test_cases = []
+            for item in data:
+                # Build environment scenario if present
+                env_scenario = None
+                if item.get("environment"):
+                    env_data = item["environment"]
+                    env_scenario = EnvironmentScenario(
+                        description=env_data["description"],
+                        mock_setup=env_data["mock_setup"],
+                        expected_behavior=env_data["expected_behavior"],
+                        invariants_checked=env_data.get("invariants_checked", []),
+                    )
+
+                test_case = FuzzTestCase(
+                    id=item["id"],
+                    type=item["type"],
+                    input=item.get("input"),
+                    environment=env_scenario,
+                    rationale=item["rationale"],
+                    category=item["category"],
+                )
+                test_cases.append(test_case)
+
+            return test_cases
+
+        except (json.JSONDecodeError, KeyError) as e:
+            raise ValueError(f"Failed to parse fuzzer response: {e}\nResponse: {response_text}")
+
+        # Phase 3: Cluster and deduplicate tests
+        if self.history_db:
+            test_cases = await self._cluster_and_deduplicate_tests(test_cases, context)
+
+        return test_cases
+
+    def _expand_graph_context(
+        self,
+        graphs: CodeGraphs,
+        start_functions: list[str],
+        max_depth: int = 2,
+    ) -> set[str]:
+        """Expand graph context using depth-limited BFS traversal.
+
+        Args:
+            graphs: Code relationship graphs
+            start_functions: Starting function nodes
+            max_depth: Maximum traversal depth (default: 2)
+
+        Returns:
+            Set of function names reachable within max_depth
+        """
+        if not graphs or not graphs.call_graph:
+            return set(start_functions)
+
+        reachable = set(start_functions)
+
+        for start_func in start_functions:
+            if start_func not in graphs.call_graph:
+                continue
+
+            # BFS traversal
+            visited = {start_func}
+            queue = [(start_func, 0)]  # (node, depth)
+
+            while queue:
+                current, depth = queue.pop(0)
+
+                if depth >= max_depth:
+                    continue
+
+                # Expand to callees (functions called by current)
+                for callee in graphs.get_callees(current):
+                    if callee not in visited:
+                        visited.add(callee)
+                        reachable.add(callee)
+                        queue.append((callee, depth + 1))
+
+                # Expand to callers (functions that call current)
+                for caller in graphs.get_callers(current):
+                    if caller not in visited:
+                        visited.add(caller)
+                        reachable.add(caller)
+                        queue.append((caller, depth + 1))
+
+        logger.debug(f"Graph expansion: {len(start_functions)} → {len(reachable)} functions")
+        return reachable
+
+    async def _cluster_and_deduplicate_tests(
+        self,
+        test_cases: list[FuzzTestCase],
+        context: ContextDerivation,
+    ) -> list[FuzzTestCase]:
+        """Cluster test cases and remove duplicates.
+
+        Args:
+            test_cases: Generated test cases
+            context: Context derivation with code graphs
+
+        Returns:
+            Deduplicated test cases (best test per cluster)
+        """
+        if len(test_cases) <= 1:
+            return test_cases
+
+        # Embed all test cases in batch for efficiency
+        test_strs = [json.dumps(test.to_dict(), sort_keys=True) for test in test_cases]
+        embeddings_array = self.embedder.embed_batch(test_strs)
+
+        # Cluster by similarity
+        clusters = self._cluster_by_similarity(
+            embeddings_array,
+            threshold=self.test_clustering_threshold,
+        )
+
+        # Keep best test per cluster
+        deduplicated = []
+        for cluster in clusters:
+            # Pick the test with most comprehensive rationale (as proxy for quality)
+            best_test = max(
+                (test_cases[i] for i in cluster),
+                key=lambda t: len(t.rationale),
+            )
+            deduplicated.append(best_test)
+
+        removed_count = len(test_cases) - len(deduplicated)
+        if removed_count > 0:
+            logger.info(f"Test clustering: removed {removed_count} duplicate tests")
+
+        return deduplicated
+
+    def _cluster_by_similarity(
+        self,
+        embeddings: np.ndarray,
+        threshold: float,
+    ) -> list[list[int]]:
+        """Cluster embeddings by cosine similarity.
+
+        Args:
+            embeddings: Array of embeddings (shape: [n, dim])
+            threshold: Similarity threshold (tests with similarity > threshold are clustered)
+
+        Returns:
+            List of clusters (each cluster is list of indices)
+        """
+        n = len(embeddings)
+        if n == 0:
+            return []
+
+        # Compute pairwise cosine similarities
+        # For normalized embeddings, cosine similarity = dot product
+        similarities = embeddings @ embeddings.T
+
+        # Build clusters using greedy approach
+        clusters = []
+        assigned = set()
+
+        for i in range(n):
+            if i in assigned:
+                continue
+
+            # Start new cluster with i
+            cluster = [i]
+            assigned.add(i)
+
+            # Add all similar unassigned items to this cluster
+            for j in range(i + 1, n):
+                if j in assigned:
+                    continue
+
+                if similarities[i, j] >= threshold:
+                    cluster.append(j)
+                    assigned.add(j)
+
+            clusters.append(cluster)
+
+        return clusters
+
+    def _build_bug_warnings_section(self, similar_bugs: list) -> str:
+        """Build bug warnings section for prompt.
+
+        Args:
+            similar_bugs: List of DetectedPattern instances
+
+        Returns:
+            Formatted bug warnings section
+        """
+        if not similar_bugs:
+            return ""
+
+        lines = ["## ⚠️  Similar Bug Patterns Detected"]
+        lines.append(
+            "The following bug patterns from historical fuzzing sessions are similar to this code:"
+        )
+        lines.append("")
+
+        for i, bug in enumerate(similar_bugs[:5], 1):  # Top 5 bugs
+            lines.append(
+                f"{i}. **{bug.description}** (Severity: {bug.severity}, Confidence: {bug.confidence})"
+            )
+            lines.append(f"   Similarity: {bug.similarity:.1%}")
+            lines.append("")
+
+        lines.append(
+            "**IMPORTANT**: Generate targeted test cases that specifically check for these bug patterns."
+        )
+        return "\n".join(lines)
+
+    def _build_graph_context_section(
+        self,
+        graphs: CodeGraphs,
+        context: ContextDerivation,
+    ) -> str:
+        """Build graph-aware context section for environment fuzzing.
+
+        Args:
+            graphs: Code relationship graphs
+            context: Context derivation
+
+        Returns:
+            Formatted graph context section
+        """
+        if not graphs or graphs.function_count == 0:
+            return ""
+
+        lines = ["## Graph-Aware Context for Environment Fuzzing"]
+
+        # Get all functions
+        functions = [
+            name for name, info in graphs.export_map.items() if info.symbol_type == "function"
+        ]
+
+        if functions:
+            lines.append(f"Defined functions ({len(functions)}): {', '.join(functions[:10])}")
+
+            # Expand graph context for environment scenarios
+            expanded_functions = self._expand_graph_context(
+                graphs, functions, max_depth=self.graph_expansion_depth
+            )
+
+            if len(expanded_functions) > len(functions):
+                additional = len(expanded_functions) - len(functions)
+                lines.append(
+                    f"Transitive dependencies ({additional} additional functions reachable via calls)"
+                )
+
+        # Integration contracts that might need mocking
+        if context.integration_contracts:
+            lines.append("")
+            lines.append("Functions that likely interact with external systems:")
+            for contract in context.integration_contracts[:5]:
+                lines.append(f"  - {contract.system}: {contract.contract}")
+
+        lines.append("")
+        lines.append(
+            "Use the call graph to generate comprehensive environment scenarios "
+            "that test transitive dependency failures."
+        )
+
+        return "\n".join(lines)