ctrlcode 0.1.0__py3-none-any.whl

ctrlcode/fuzzing/context.py

@@ -0,0 +1,665 @@
+"""Context derivation engine - derives behavioral oracle from code analysis."""
+
+import hashlib
+import json
+import logging
+import uuid
+from collections import OrderedDict
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Literal, Optional
+
+import numpy as np
+
+from ..analysis.code_graphs import CodeGraphBuilder, CodeGraphs
+from ..embeddings.embedder import CodeEmbedder
+from ..providers.base import Provider
+from ..storage.history_db import CodeRecord, HistoryDB, OracleRecord
+
+logger = logging.getLogger(__name__)
+
+
+class LRUCache:
+    """Simple LRU cache for oracle retrieval results."""
+
+    def __init__(self, max_size: int = 100):
+        """Initialize LRU cache.
+
+        Args:
+            max_size: Maximum number of entries to cache
+        """
+        self.cache: OrderedDict = OrderedDict()
+        self.max_size = max_size
+        self.hits = 0
+        self.misses = 0
+
+    def get(self, key: str) -> Optional[tuple]:
+        """Get value from cache.
+
+        Args:
+            key: Cache key
+
+        Returns:
+            Cached value or None
+        """
+        if key in self.cache:
+            # Move to end (most recently used)
+            self.cache.move_to_end(key)
+            self.hits += 1
+            return self.cache[key]
+        else:
+            self.misses += 1
+            return None
+
+    def put(self, key: str, value: tuple) -> None:
+        """Put value in cache.
+
+        Args:
+            key: Cache key
+            value: Value to cache
+        """
+        # If cache is disabled (max_size == 0), don't store anything
+        if self.max_size == 0:
+            return
+
+        if key in self.cache:
+            # Update existing entry
+            self.cache.move_to_end(key)
+        else:
+            # Add new entry
+            if len(self.cache) >= self.max_size:
+                # Remove oldest entry
+                self.cache.popitem(last=False)
+        self.cache[key] = value
+
+    def clear(self) -> None:
+        """Clear all cache entries."""
+        self.cache.clear()
+        self.hits = 0
+        self.misses = 0
+
+    @property
+    def hit_rate(self) -> float:
+        """Calculate cache hit rate."""
+        total = self.hits + self.misses
+        return self.hits / total if total > 0 else 0.0
+
+    @property
+    def size(self) -> int:
+        """Get current cache size."""
+        return len(self.cache)
+
+
+@dataclass
+class SystemPlacement:
+    """Derived system placement information."""
+
+    system_type: str  # "web service", "CLI tool", "library", etc.
+    layer: str  # "HTTP handler", "business logic", "data access", etc.
+    callers: str  # What likely calls this code
+    callees: str  # What this code calls
+
+
+@dataclass
+class IntegrationContract:
+    """Contract with external system."""
+
+    system: str  # "External API", "Database", "Message Queue"
+    contract: str  # Description of the contract
+    implicit_requirements: list[str]  # Implicit requirements
+
+
+@dataclass
+class ImplicitAssumption:
+    """Implicit assumption made by the code."""
+
+    assumption: str
+    risk: Literal["SAFE", "RISKY", "DANGEROUS"]
+    explanation: str
+
+
+@dataclass
+class ContextDerivation:
+    """Complete derived context for code under test."""
+
+    system_placement: SystemPlacement
+    environmental_constraints: dict[str, Any]
+    integration_contracts: list[IntegrationContract]
+    behavioral_invariants: list[str]
+    edge_case_surface: list[str]
+    implicit_assumptions: list[ImplicitAssumption]
+
+    # Phase 2 additions: graphs, embeddings, historical reuse
+    code_graphs: Optional[CodeGraphs] = None
+    function_invariants: dict[str, list[str]] = field(default_factory=dict)
+    embedding: Optional[np.ndarray] = None
+    retrieved_from: Optional[str] = None  # Session ID if oracle was reused
+
+    def to_json(self) -> str:
+        """Convert to JSON string (excludes graphs, embedding for serialization)."""
+        data = asdict(self)
+        # Remove non-serializable fields
+        data.pop("code_graphs", None)
+        data.pop("embedding", None)
+        return json.dumps(data, indent=2)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ContextDerivation":
+        """Create from dictionary."""
+        return cls(
+            system_placement=SystemPlacement(**data["system_placement"]),
+            environmental_constraints=data["environmental_constraints"],
+            integration_contracts=[
+                IntegrationContract(**c) for c in data["integration_contracts"]
+            ],
+            behavioral_invariants=data["behavioral_invariants"],
+            edge_case_surface=data["edge_case_surface"],
+            implicit_assumptions=[
+                ImplicitAssumption(**a) for a in data["implicit_assumptions"]
+            ],
+            function_invariants=data.get("function_invariants", {}),
+            retrieved_from=data.get("retrieved_from"),
+        )
+
+
+class ContextDerivationEngine:
+    """
+    Analyzes generated code to derive full operational context.
+
+    This replaces the "concrete system" in traditional differential fuzzing
+    with an inferred behavioral contract.
+
+    Phase 2: Supports historical oracle reuse via embeddings and graph analysis.
+    """
+
+    def __init__(
+        self,
+        provider: Provider,
+        history_db: Optional[HistoryDB] = None,
+        embedder: Optional[CodeEmbedder] = None,
+        graph_builder: Optional[CodeGraphBuilder] = None,
+        similarity_threshold: float = 0.85,
+        cache_size: int = 100,
+        vector_store_path: Optional[Path] = None,
+        auto_save_interval: int = 10,
+    ):
+        """
+        Initialize context derivation engine.
+
+        Args:
+            provider: LLM provider for analysis
+            history_db: Optional history database for oracle reuse
+            embedder: Optional code embedder (created if not provided)
+            graph_builder: Optional graph builder (created if not provided)
+            similarity_threshold: Minimum similarity for oracle reuse (default: 0.85)
+            cache_size: Max LRU cache entries for oracle retrieval (default: 100)
+            vector_store_path: Path to persistent vector store (default: ~/.ctrlcode/vector_store)
+            auto_save_interval: Save vector store after N additions (default: 10)
+        """
+        self.provider = provider
+        self.history_db = history_db
+        self.embedder = embedder or CodeEmbedder()
+        self.graph_builder = graph_builder or CodeGraphBuilder()
+        self.similarity_threshold = similarity_threshold
+        self.oracle_cache = LRUCache(max_size=cache_size)
+
+        # Initialize persistent vector store
+        if vector_store_path is None:
+            vector_store_path = Path.home() / ".ctrlcode" / "vector_store"
+
+        self.vector_store_path = Path(vector_store_path)
+        self.auto_save_interval = auto_save_interval
+        self._additions_since_save = 0
+
+        # Create or load vector store
+        from ..embeddings.vector_store import VectorStore
+
+        self.vector_store = VectorStore(dimension=1536)
+
+        # Load existing index if available
+        if (self.vector_store_path / "faiss.index").exists():
+            try:
+                self.vector_store.load(self.vector_store_path)
+                logger.info(f"Loaded persistent vector store with {self.vector_store.size} embeddings")
+            except Exception as e:
+                logger.warning(f"Failed to load vector store: {e}. Starting fresh.")
+
+        # Track which code IDs are already indexed
+        self._indexed_code_ids: set[str] = set(self.vector_store.id_map)
+
+    async def derive(
+        self,
+        user_request: str,
+        generated_code: str,
+        surrounding_files: Optional[list[str]] = None,
+        session_id: Optional[str] = None,
+    ) -> ContextDerivation:
+        """
+        Derive operational context from code and specification.
+
+        Phase 2: Searches history for similar code and reuses oracles when possible.
+
+        Args:
+            user_request: Original user specification
+            generated_code: Generated code to analyze
+            surrounding_files: Optional surrounding codebase files
+            session_id: Optional session ID for tracking
+
+        Returns:
+            ContextDerivation with all derived context (including graphs, embeddings)
+
+        Raises:
+            ValueError: If derivation fails or returns invalid JSON
+        """
+        if session_id is None:
+            session_id = str(uuid.uuid4())
+
+        # Step 1: Embed the generated code
+        logger.debug("Embedding generated code")
+        code_embedding = self.embedder.embed_code(generated_code)
+
+        # Step 2: Build code graphs
+        logger.debug("Building code relationship graphs")
+        code_graphs = self.graph_builder.build_from_code(generated_code, "<generated>")
+
+        # Step 3: Search history for similar code (if history DB available)
+        similar_oracle = None
+        retrieved_from = None
+
+        if self.history_db:
+            similar_oracle, retrieved_from = await self._search_similar_oracle(
+                code_embedding, generated_code, user_request
+            )
+
+        # Step 4: Either adapt historical oracle or derive fresh
+        if similar_oracle:
+            logger.info(f"Reusing historical oracle from session {retrieved_from}")
+            context = similar_oracle
+            context.retrieved_from = retrieved_from
+        else:
+            logger.debug("Deriving fresh oracle (no similar code in history)")
+            context = await self._derive_fresh_oracle(
+                user_request, generated_code, surrounding_files, code_graphs
+            )
+
+        # Step 5: Attach graphs and embedding
+        context.code_graphs = code_graphs
+        context.embedding = code_embedding
+
+        # Step 6: Store in history DB (if available)
+        if self.history_db:
+            await self._store_in_history(session_id, user_request, generated_code, context)
+
+        return context
+
+    def _save_vector_store(self) -> None:
+        """Save vector store to disk and reset counter."""
+        try:
+            self.vector_store.save(self.vector_store_path)
+            self._additions_since_save = 0
+            logger.info(
+                f"Saved vector store with {self.vector_store.size} embeddings "
+                f"to {self.vector_store_path}"
+            )
+        except Exception as e:
+            logger.error(f"Failed to save vector store: {e}")
+
+    async def _search_similar_oracle(
+        self,
+        code_embedding: np.ndarray,
+        new_code: str,
+        user_request: str,
+    ) -> tuple[Optional[ContextDerivation], Optional[str]]:
+        """Search history for similar code and adapt oracle if found.
+
+        Args:
+            code_embedding: Embedding of new code
+            new_code: New generated code
+            user_request: User specification
+
+        Returns:
+            Tuple of (adapted oracle, source session ID) or (None, None)
+        """
+        from .oracle_adapter import OracleAdapter
+
+        # Compute cache key from code hash
+        code_hash = hashlib.sha256(new_code.encode()).hexdigest()
+
+        # Check cache first
+        cached = self.oracle_cache.get(code_hash)
+        if cached is not None:
+            logger.debug(f"Cache HIT for code hash {code_hash[:8]}...")
+            similar_oracle, session_id = cached
+            return similar_oracle, session_id
+
+        logger.debug(f"Cache MISS for code hash {code_hash[:8]}...")
+
+        # Sync vector store with database (incremental updates)
+        code_records = self.history_db.get_all_code_embeddings(limit=10000)
+
+        if not code_records:
+            logger.debug("No historical code in database")
+            # Cache negative result
+            self.oracle_cache.put(code_hash, (None, None))
+            return None, None
+
+        # Add new embeddings incrementally
+        new_embeddings = []
+        new_ids = []
+
+        for rec in code_records:
+            if rec.code_id not in self._indexed_code_ids:
+                new_embeddings.append(rec.embedding)
+                new_ids.append(rec.code_id)
+                self._indexed_code_ids.add(rec.code_id)
+
+        if new_embeddings:
+            logger.debug(f"Adding {len(new_embeddings)} new embeddings to vector store")
+            self.vector_store.add(np.array(new_embeddings), new_ids)
+            self._additions_since_save += len(new_embeddings)
+
+            # Auto-save if threshold reached
+            if self._additions_since_save >= self.auto_save_interval:
+                self._save_vector_store()
+
+        # Search for similar code using persistent vector store
+        results = self.vector_store.search(
+            code_embedding,
+            k=3,
+            min_similarity=self.similarity_threshold,
+        )
+
+        if not results:
+            logger.debug(f"No similar code found above threshold {self.similarity_threshold}")
+            # Cache negative result
+            self.oracle_cache.put(code_hash, (None, None))
+            return None, None
+
+        # Try to adapt oracle from most similar code
+        for code_id, similarity in results:
+            logger.debug(f"Found similar code: {code_id} (similarity: {similarity:.2%})")
+
+            # Get the code record
+            code_record = next((r for r in code_records if r.code_id == code_id), None)
+            if not code_record:
+                continue
+
+            # Get the oracle for this session
+            oracle_records = self.history_db.get_all_oracle_embeddings()
+            oracle_record = next(
+                (o for o in oracle_records if o.session_id == code_record.session_id),
+                None,
+            )
+
+            if not oracle_record:
+                logger.debug(f"No oracle found for session {code_record.session_id}")
+                continue
+
+            # Reconstruct the historical oracle
+            try:
+                historical_oracle = ContextDerivation.from_dict(json.loads(oracle_record.oracle))
+            except Exception as e:
+                logger.warning(f"Failed to parse historical oracle: {e}")
+                continue
+
+            # Attempt to adapt the oracle
+            adapter = OracleAdapter(self.provider)
+            adapted_oracle = await adapter.adapt_oracle(
+                historical_oracle,
+                code_record.code,
+                new_code,
+                user_request,
+                similarity,
+            )
+
+            if adapted_oracle:
+                # Cache the successful result
+                result = (adapted_oracle, code_record.session_id)
+                self.oracle_cache.put(code_hash, result)
+                return result
+
+        logger.debug("Could not adapt any historical oracles")
+        # Cache the negative result to avoid repeated searches
+        self.oracle_cache.put(code_hash, (None, None))
+        return None, None
+
+    async def _derive_fresh_oracle(
+        self,
+        user_request: str,
+        generated_code: str,
+        surrounding_files: Optional[list[str]],
+        code_graphs: CodeGraphs,
+    ) -> ContextDerivation:
+        """Derive fresh oracle from scratch (original derivation logic).
+
+        Args:
+            user_request: User specification
+            generated_code: Generated code
+            surrounding_files: Optional surrounding files
+            code_graphs: Code relationship graphs
+
+        Returns:
+            Fresh ContextDerivation
+
+        Raises:
+            ValueError: If derivation fails
+        """
+        # Build graph metadata section
+        graph_metadata = self._build_graph_metadata(code_graphs)
+
+        # Build system prompt (from DIFFFUZZTEST.md lines 80-140)
+        system_prompt = """You are a senior systems analyst. Your job is to examine a piece of generated
+code and its originating specification, then derive the full operational
+context — even when that context wasn't explicitly stated.
+
+You will be given:
+1. The user's original prompt/specification
+2. The generated source code
+3. (Optional) Surrounding codebase files, if available
+
+Produce a CONTEXT DERIVATION REPORT with these sections:
+
+## 1. System Placement
+Where does this code live? Derive from clues in the code and spec:
+- What kind of system is this part of? (web server, CLI tool, data pipeline,
+  library, microservice, mobile app, embedded system, etc.)
+- What layer? (HTTP handler, business logic, data access, infrastructure,
+  utility, middleware, etc.)
+- What runs around it? (What likely calls this code? What does it call?)
+
+## 2. Environmental Constraints
+What can we infer about the runtime environment?
+- Language version and platform (derive from syntax, imports, idioms)
+- Concurrency model (async, threaded, single-threaded, multiprocess?)
+- Resource constraints (memory-sensitive? latency-sensitive? throughput?)
+- Deployment context (serverless, container, long-running server, edge?)
+
+## 3. Integration Contracts
+What external systems does this code interact with (explicitly or implicitly)?
+- APIs it calls or is called by (infer from HTTP clients, route handlers, etc.)
+- Databases or data stores (infer from ORMs, query builders, file I/O)
+- Message queues, caches, external services
+- For each: what are the implicit contracts? (timeouts, retry semantics,
+  idempotency requirements, ordering guarantees)
+
+## 4. Behavioral Invariants
+What MUST be true about this code's behavior, regardless of implementation?
+Derive from the spec AND from the system context:
+- Functional invariants (output properties, state transitions)
+- Safety invariants (must never: lose data, corrupt state, leak secrets, etc.)
+- Liveness invariants (must always: eventually return, release resources, etc.)
+- Idempotency requirements (is it safe to call twice with the same input?)
+- Ordering constraints (must things happen in a specific sequence?)
+
+## 5. Edge Case Surface
+What environmental edge cases should be tested?
+- What happens when dependencies are slow, down, or returning errors?
+- What happens under concurrent access?
+- What happens at resource limits (memory, file descriptors, connections)?
+- What happens with malformed or unexpected upstream data?
+- Clock skew, timezone issues, daylight savings transitions?
+- Filesystem permissions, disk full, network partitions?
+
+## 6. Implicit Assumptions
+What assumptions has the code made that are NOT stated in the spec?
+Flag each as:
+- SAFE: Reasonable assumption, likely correct
+- RISKY: Could be wrong depending on context
+- DANGEROUS: Likely to cause issues in production
+
+## 7. Function-Level Invariants
+For each function defined in the code, derive specific behavioral invariants.
+Output as a dict mapping function names to lists of invariants.
+
+Output as structured JSON with all 7 sections."""
+
+        # Build user message
+        context_section = (
+            surrounding_files
+            if surrounding_files
+            else "No surrounding codebase provided. Derive context entirely from the code and specification."
+        )
+
+        user_message = f"""## Original Specification
+{user_request}
+
+## Generated Code
+```
+{generated_code}
+```
+
+{graph_metadata}
+
+## Available Context
+{context_section}"""
+
+        # Call LLM
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_message},
+        ]
+
+        response = await self.provider.generate(messages)
+        response_text = response.get("text", "").strip()
+
+        # Parse JSON response
+        try:
+            # Extract JSON from markdown code blocks if present
+            if "```json" in response_text:
+                start = response_text.find("```json") + 7
+                end = response_text.find("```", start)
+                response_text = response_text[start:end].strip()
+            elif "```" in response_text:
+                start = response_text.find("```") + 3
+                end = response_text.find("```", start)
+                response_text = response_text[start:end].strip()
+
+            data = json.loads(response_text)
+            return ContextDerivation.from_dict(data)
+
+        except (json.JSONDecodeError, KeyError) as e:
+            raise ValueError(f"Failed to parse context derivation response: {e}\nResponse: {response_text}")
+
+    def _build_graph_metadata(self, graphs: CodeGraphs) -> str:
+        """Build graph metadata section for prompt.
+
+        Args:
+            graphs: Code relationship graphs
+
+        Returns:
+            Formatted graph metadata string
+        """
+        if graphs.function_count == 0:
+            return ""
+
+        metadata_parts = ["## Code Structure Analysis"]
+        metadata_parts.append(f"- Functions defined: {graphs.function_count}")
+        metadata_parts.append(f"- Classes defined: {graphs.class_count}")
+
+        # List functions
+        if graphs.function_count > 0:
+            functions = [
+                name
+                for name, info in graphs.export_map.items()
+                if info.symbol_type == "function"
+            ]
+            metadata_parts.append(f"- Function names: {', '.join(functions[:10])}")
+
+        # Call graph summary
+        if graphs.call_graph.number_of_edges() > 0:
+            metadata_parts.append(
+                f"- Function calls detected: {graphs.call_graph.number_of_edges()} relationships"
+            )
+
+        return "\n".join(metadata_parts)
+
+    async def _store_in_history(
+        self,
+        session_id: str,
+        user_request: str,
+        generated_code: str,
+        context: ContextDerivation,
+    ) -> None:
+        """Store derivation results in history database.
+
+        Args:
+            session_id: Session identifier
+            user_request: User specification
+            generated_code: Generated code
+            context: Derived context
+        """
+        try:
+            # Store code with embedding
+            code_record = CodeRecord(
+                code_id=f"{session_id}_code",
+                session_id=session_id,
+                code=generated_code,
+                embedding=context.embedding,
+                timestamp=datetime.now(),
+            )
+            self.history_db.store_code(code_record)
+
+            # Store oracle with embedding and versioning
+            oracle_embedding = self.embedder.embed_oracle(context.to_json())
+
+            # Determine version and parent
+            oracle_version = 1
+            parent_oracle_id = None
+
+            if context.retrieved_from:
+                # This oracle was adapted from a parent
+                parent_oracle_id = f"{context.retrieved_from}_oracle"
+
+                # Get parent oracle to determine version
+                try:
+                    parent_oracles = self.history_db.get_all_oracle_embeddings()
+                    parent = next(
+                        (o for o in parent_oracles if o.oracle_id == parent_oracle_id),
+                        None
+                    )
+                    if parent:
+                        oracle_version = parent.oracle_version + 1
+                        # Increment reuse count of parent
+                        self.history_db.increment_oracle_reuse(parent_oracle_id)
+                        logger.debug(f"Oracle adapted from v{parent.oracle_version}, creating v{oracle_version}")
+                except Exception as e:
+                    logger.warning(f"Failed to get parent oracle version: {e}")
+
+            oracle_record = OracleRecord(
+                oracle_id=f"{session_id}_oracle",
+                session_id=session_id,
+                oracle=context.to_json(),
+                embedding=oracle_embedding,
+                quality_score=1.0,  # Could be computed based on invariant count, etc.
+                timestamp=datetime.now(),
+                oracle_version=oracle_version,
+                parent_oracle_id=parent_oracle_id,
+                reuse_count=0,
+            )
+            self.history_db.store_oracle(oracle_record)
+
+            logger.debug(f"Stored derivation results for session {session_id} in history DB")
+
+        except Exception as e:
+            logger.warning(f"Failed to store in history DB: {e}")