ctrlcode 0.1.0__py3-none-any.whl

ctrlcode/embeddings/vector_store.py

@@ -0,0 +1,213 @@
+"""FAISS-based vector storage for fast similarity search."""
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+import faiss
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+
+class VectorStore:
+    """Fast approximate nearest neighbor search using FAISS HNSW index.
+
+    Supports efficient similarity search over thousands of embeddings.
+    Uses HNSW (Hierarchical Navigable Small World) algorithm for sub-millisecond queries.
+    """
+
+    def __init__(
+        self,
+        dimension: int = 384,  # Default for all-MiniLM-L6-v2
+        index_type: str = "hnsw",
+        M: int = 32,  # HNSW construction parameter (connectivity)
+        ef_construction: int = 200,  # HNSW build quality
+        ef_search: int = 64,  # HNSW search quality
+    ):
+        """Initialize vector store with FAISS index.
+
+        Args:
+            dimension: Embedding vector dimension
+            index_type: Index type (hnsw or flat)
+            M: HNSW parameter - higher = better recall, more memory
+            ef_construction: HNSW build-time search depth
+            ef_search: HNSW query-time search depth
+        """
+        self.dimension = dimension
+        self.index_type = index_type
+        self.M = M
+        self.ef_construction = ef_construction
+        self.ef_search = ef_search
+
+        # Initialize index
+        # Note: For cosine similarity with normalized vectors:
+        # - Flat index uses inner product (IP) directly
+        # - HNSW uses L2 distance (converted to cosine in search results)
+        if index_type == "hnsw":
+            # IndexHNSWFlat uses L2 distance only
+            self.index = faiss.IndexHNSWFlat(dimension, M)
+            self.index.hnsw.efConstruction = ef_construction
+            self.index.hnsw.efSearch = ef_search
+        elif index_type == "flat":
+            # Exact search for small datasets with inner product
+            self.index = faiss.IndexFlatIP(dimension)  # Inner product (cosine for normalized)
+        else:
+            raise ValueError(f"Unknown index type: {index_type}")
+
+        # Track IDs separately (FAISS uses integer IDs)
+        self.id_map: list[str] = []
+
+    def add(
+        self,
+        embeddings: np.ndarray | list[np.ndarray],
+        ids: list[str],
+    ) -> None:
+        """Add embeddings to the index.
+
+        Args:
+            embeddings: Single embedding or array of embeddings
+            ids: Corresponding string IDs
+        """
+        # Convert single embedding to batch
+        if isinstance(embeddings, list):
+            embeddings = np.array(embeddings)
+
+        if embeddings.ndim == 1:
+            embeddings = embeddings.reshape(1, -1)
+
+        if embeddings.shape[0] != len(ids):
+            raise ValueError(f"Mismatch: {embeddings.shape[0]} embeddings, {len(ids)} IDs")
+
+        if embeddings.shape[1] != self.dimension:
+            raise ValueError(f"Wrong dimension: {embeddings.shape[1]}, expected {self.dimension}")
+
+        # Ensure float32 for FAISS
+        embeddings = embeddings.astype(np.float32)
+
+        # Add to index
+        self.index.add(embeddings)
+        self.id_map.extend(ids)
+
+        logger.debug(f"Added {len(ids)} embeddings to index (total: {len(self.id_map)})")
+
+    def search(
+        self,
+        query_embedding: np.ndarray,
+        k: int = 5,
+        min_similarity: Optional[float] = None,
+    ) -> list[tuple[str, float]]:
+        """Search for k most similar embeddings.
+
+        Args:
+            query_embedding: Query vector
+            k: Number of results to return
+            min_similarity: Minimum similarity threshold (filter results)
+
+        Returns:
+            List of (id, similarity_score) tuples, sorted by similarity (descending)
+        """
+        if len(self.id_map) == 0:
+            return []
+
+        # Reshape query
+        if query_embedding.ndim == 1:
+            query_embedding = query_embedding.reshape(1, -1)
+
+        query_embedding = query_embedding.astype(np.float32)
+
+        # Limit k to available items
+        k = min(k, len(self.id_map))
+
+        # Search
+        distances, indices = self.index.search(query_embedding, k)
+
+        # Convert to (id, score) tuples
+        results = []
+        for idx, dist in zip(indices[0], distances[0]):
+            if idx < 0 or idx >= len(self.id_map):
+                continue  # Invalid index
+
+            # Convert distance to similarity based on index type
+            if self.index_type == "hnsw":
+                # For L2 distance on normalized vectors: cosine_sim = 1 - (L2²/2)
+                # FAISS returns squared L2 distance
+                similarity = float(1.0 - (dist / 2.0))
+            else:
+                # For inner product, distance IS similarity
+                similarity = float(dist)
+
+            # Filter by threshold
+            if min_similarity is not None and similarity < min_similarity:
+                continue
+
+            results.append((self.id_map[idx], similarity))
+
+        return results
+
+    def save(self, path: Path) -> None:
+        """Save index and ID map to disk.
+
+        Args:
+            path: Directory to save to
+        """
+        path = Path(path)
+        path.mkdir(parents=True, exist_ok=True)
+
+        # Save FAISS index
+        index_file = path / "faiss.index"
+        faiss.write_index(self.index, str(index_file))
+
+        # Save ID map
+        id_map_file = path / "id_map.txt"
+        with open(id_map_file, "w") as f:
+            f.write("\n".join(self.id_map))
+
+        logger.info(f"Saved vector store to {path} ({len(self.id_map)} embeddings)")
+
+    def load(self, path: Path) -> None:
+        """Load index and ID map from disk.
+
+        Args:
+            path: Directory to load from
+        """
+        path = Path(path)
+
+        # Load FAISS index
+        index_file = path / "faiss.index"
+        if not index_file.exists():
+            raise FileNotFoundError(f"Index file not found: {index_file}")
+
+        self.index = faiss.read_index(str(index_file))
+
+        # Update ef_search for HNSW
+        if self.index_type == "hnsw" and hasattr(self.index, "hnsw"):
+            self.index.hnsw.efSearch = self.ef_search
+
+        # Load ID map
+        id_map_file = path / "id_map.txt"
+        if not id_map_file.exists():
+            raise FileNotFoundError(f"ID map not found: {id_map_file}")
+
+        with open(id_map_file, "r") as f:
+            self.id_map = [line.strip() for line in f if line.strip()]
+
+        logger.info(f"Loaded vector store from {path} ({len(self.id_map)} embeddings)")
+
+    @property
+    def size(self) -> int:
+        """Number of embeddings in the index."""
+        return len(self.id_map)
+
+    def clear(self) -> None:
+        """Clear all embeddings from the index."""
+        # Reinitialize index
+        if self.index_type == "hnsw":
+            self.index = faiss.IndexHNSWFlat(self.dimension, self.M)
+            self.index.hnsw.efConstruction = self.ef_construction
+            self.index.hnsw.efSearch = self.ef_search
+        else:
+            self.index = faiss.IndexFlatIP(self.dimension)
+
+        self.id_map = []
+        logger.debug("Cleared vector store")

ctrlcode/fuzzing/__init__.py

@@ -0,0 +1,24 @@
+"""Differential fuzzing system for ctrl-code (derived context architecture)."""
+
+from .derived_orchestrator import DerivedFuzzingOrchestrator, FuzzingResult
+from .context import ContextDerivationEngine, ContextDerivation, SystemPlacement, IntegrationContract, ImplicitAssumption
+from .context_fuzzer import ContextAwareFuzzer, FuzzTestCase, EnvironmentScenario
+from .analyzer import DerivedOracleAnalyzer, DiagnosedDivergence
+from .budget import BudgetManager, BudgetConfig
+
+__all__ = [
+    "DerivedFuzzingOrchestrator",
+    "FuzzingResult",
+    "ContextDerivationEngine",
+    "ContextDerivation",
+    "SystemPlacement",
+    "IntegrationContract",
+    "ImplicitAssumption",
+    "ContextAwareFuzzer",
+    "FuzzTestCase",
+    "EnvironmentScenario",
+    "DerivedOracleAnalyzer",
+    "DiagnosedDivergence",
+    "BudgetManager",
+    "BudgetConfig",
+]

ctrlcode/fuzzing/analyzer.py

@@ -0,0 +1,280 @@
+"""Enhanced analyzer with oracle bug detection."""
+
+import json
+from dataclasses import dataclass, asdict
+from typing import Literal, Any
+
+from ..providers.base import Provider
+from .context import ContextDerivation
+from .context_fuzzer import FuzzTestCase
+
+
+@dataclass
+class DiagnosedDivergence:
+    """Analyzed divergence with root cause and fix."""
+
+    divergence_id: str
+    diagnosis: str
+    root_cause: str
+    source: Literal["MODEL_BUG", "ORACLE_BUG", "SPEC_GAP", "ENVIRONMENT_MISMATCH"]
+    confidence: float  # 0-1
+    fix: dict[str, Any]  # Patch, corrected invariant, or clarification question
+    regression_test: str
+    follow_up_guidance: list[str]
+    impact: str
+    blast_radius: str
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary."""
+        return asdict(self)
+
+
+class DerivedOracleAnalyzer:
+    """
+    Analyzes divergences in derived context testing.
+
+    Key innovation: Detects when the ORACLE is wrong, not just the code.
+    """
+
+    def __init__(self, provider: Provider):
+        """
+        Initialize analyzer.
+
+        Args:
+            provider: LLM provider for analysis
+        """
+        self.provider = provider
+
+    async def analyze_divergence(
+        self,
+        spec: str,
+        code: str,
+        context: ContextDerivation,
+        test_case: FuzzTestCase,
+        actual_output: dict,
+        expected_output: dict,
+        previous_analyses: list[DiagnosedDivergence] = [],
+    ) -> DiagnosedDivergence:
+        """
+        Analyze a divergence between actual and expected behavior.
+
+        Args:
+            spec: Original specification
+            code: Generated code
+            context: Derived context
+            test_case: The test case that revealed the divergence
+            actual_output: What the code actually produced
+            expected_output: What was expected (from derived oracle)
+            previous_analyses: Previous divergence analyses (for learning)
+
+        Returns:
+            DiagnosedDivergence with root cause and fix
+
+        This determines whether the divergence is due to:
+        - MODEL_BUG: Code is wrong
+        - ORACLE_BUG: Derived expectation was wrong
+        - SPEC_GAP: Specification is ambiguous
+        - ENVIRONMENT_MISMATCH: Wrong environmental assumptions
+        """
+        # Build system prompt (from DIFFFUZZTEST.md lines 396-445)
+        system_prompt = """You are a senior software engineer performing root cause analysis. You operate
+in a pipeline where code is tested against DERIVED behavioral expectations
+rather than a running system.
+
+This means divergences could come from three sources:
+1. The MODEL'S CODE is wrong (most common)
+2. The DERIVED EXPECTATION is wrong (the context analysis misjudged what the
+   code should do)
+3. The SPECIFICATION is ambiguous or incomplete
+4. The ENVIRONMENTAL ASSUMPTIONS don't match reality
+
+For each divergence:
+
+## 1. DIAGNOSE
+Identify the root cause. Point to specific code, specific invariant, or
+specific spec clause.
+
+## 2. DETERMINE SOURCE
+- MODEL_BUG: The code doesn't match what the spec and context require.
+- ORACLE_BUG: The derived expectation was wrong — the code is actually fine,
+  but the context derivation made an incorrect inference about what the
+  behavior should be. (This is important — flag it so the context derivation
+  improves over time.)
+- SPEC_GAP: The specification doesn't address this case. Neither the code
+  nor the derived oracle can be judged correct.
+- ENVIRONMENT_MISMATCH: The environmental assumptions in the context report
+  don't match reality (e.g., assumed async but code is sync).
+
+## 3. PROPOSE FIX
+- MODEL_BUG → Code patch (minimal diff)
+- ORACLE_BUG → Corrected invariant/expectation + note to adjust context derivation
+- SPEC_GAP → Clarification question for the user + both possible interpretations
+- ENVIRONMENT_MISMATCH → Corrected context assumptions + re-derive affected invariants
+
+## 4. CONFIDENCE & IMPACT
+- Confidence (0-1): How certain are you about the diagnosis?
+- Impact: What else could break if this bug exists in production?
+  (e.g., "If retries aren't bounded, a single failing upstream could
+  consume all connection pool slots and cascade-fail the entire service")
+- Blast radius: Just this function? Its callers? The whole service?
+
+## 5. REGRESSION TEST
+Concrete test code covering this case.
+
+## 6. FOLLOW-UP FUZZING GUIDANCE
+- Inputs/scenarios the fuzzer should try next
+- If ORACLE_BUG: note which invariants to re-derive
+
+Output as JSON."""
+
+        # Build user message
+        previous_section = (
+            json.dumps([a.to_dict() for a in previous_analyses], indent=2)
+            if previous_analyses
+            else "No previous analyses."
+        )
+
+        user_message = f"""## Specification
+{spec}
+
+## Generated Code
+```
+{code}
+```
+
+## Derived Context
+{context.to_json()}
+
+## Test Case
+{json.dumps(test_case.to_dict(), indent=2)}
+
+## Actual Output
+{json.dumps(actual_output, indent=2)}
+
+## Expected Output
+{json.dumps(expected_output, indent=2)}
+
+## Previous Analyses
+{previous_section}
+
+Analyze this divergence and provide a diagnosis with fix."""
+
+        # Call LLM
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_message},
+        ]
+
+        response = await self.provider.generate(messages)
+        response_text = response.get("text", "").strip()
+
+        # Parse JSON response
+        try:
+            # Extract JSON from markdown code blocks if present
+            if "```json" in response_text:
+                start = response_text.find("```json") + 7
+                end = response_text.find("```", start)
+                response_text = response_text[start:end].strip()
+            elif "```" in response_text:
+                start = response_text.find("```") + 3
+                end = response_text.find("```", start)
+                response_text = response_text[start:end].strip()
+
+            data = json.loads(response_text)
+
+            return DiagnosedDivergence(
+                divergence_id=test_case.id,
+                diagnosis=data["diagnosis"],
+                root_cause=data["root_cause"],
+                source=data["source"],
+                confidence=data["confidence"],
+                fix=data["fix"],
+                regression_test=data["regression_test"],
+                follow_up_guidance=data["follow_up_guidance"],
+                impact=data.get("impact", "Unknown impact"),
+                blast_radius=data.get("blast_radius", "Unknown blast radius"),
+            )
+
+        except (json.JSONDecodeError, KeyError) as e:
+            raise ValueError(f"Failed to parse analyzer response: {e}\nResponse: {response_text}")
+
+
+    async def check_invariants(
+        self,
+        output: dict,
+        invariants: list[str],
+        context: ContextDerivation,
+    ) -> dict[str, Any]:
+        """
+        Check if output satisfies behavioral invariants.
+
+        Used in MODE 2 (oracle-from-invariants) when no system is available.
+
+        Args:
+            output: Function output to check
+            invariants: List of invariants that must hold
+            context: Full context derivation
+
+        Returns:
+            Dict with invariant_results, overall status, and violations
+        """
+        system_prompt = """You are a behavioral oracle. You will receive:
+1. A function's output for a given input
+2. A set of behavioral invariants that MUST hold
+3. The system context
+
+For each invariant, determine: HOLDS or VIOLATED.
+
+Be precise. An invariant is only VIOLATED if the output definitively
+contradicts it. If the output is consistent with the invariant but you
+can't fully verify (e.g., you can't observe internal timing), mark it
+as UNVERIFIABLE rather than assuming it holds.
+
+Respond as:
+{
+  "invariant_results": [
+    {
+      "invariant": "Total retry delay must be bounded",
+      "result": "HOLDS | VIOLATED | UNVERIFIABLE",
+      "evidence": "..."
+    }
+  ],
+  "overall": "PASS | FAIL | PARTIAL",
+  "violations": ["list of violated invariant names"]
+}"""
+
+        user_message = f"""## Output
+{json.dumps(output, indent=2)}
+
+## Invariants to Check
+{json.dumps(invariants, indent=2)}
+
+## System Context
+{context.to_json()}
+
+Check each invariant and respond with results."""
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_message},
+        ]
+
+        response = await self.provider.generate(messages)
+        response_text = response.get("text", "").strip()
+
+        # Parse JSON response
+        try:
+            # Extract JSON from markdown code blocks if present
+            if "```json" in response_text:
+                start = response_text.find("```json") + 7
+                end = response_text.find("```", start)
+                response_text = response_text[start:end].strip()
+            elif "```" in response_text:
+                start = response_text.find("```") + 3
+                end = response_text.find("```", start)
+                response_text = response_text[start:end].strip()
+
+            return json.loads(response_text)
+
+        except (json.JSONDecodeError, KeyError) as e:
+            raise ValueError(f"Failed to parse invariant check response: {e}\nResponse: {response_text}")

ctrlcode/fuzzing/budget.py

@@ -0,0 +1,112 @@
+"""Budget management for fuzzing iterations."""
+
+import time
+from dataclasses import dataclass
+
+
+@dataclass
+class BudgetConfig:
+    """Budget configuration."""
+
+    max_tokens: int
+    max_seconds: float
+    max_iterations: int
+
+
+class BudgetManager:
+    """Manages time and token budgets for fuzzing."""
+
+    def __init__(self, config: BudgetConfig):
+        """
+        Initialize budget manager.
+
+        Args:
+            config: Budget configuration
+        """
+        self.config = config
+        self.start_time = time.time()
+        self.tokens_used = 0
+        self.iterations = 0
+
+    def consume(self, tokens: int, elapsed_time: float) -> None:
+        """
+        Consume budget resources.
+
+        Args:
+            tokens: Number of tokens used
+            elapsed_time: Time elapsed in seconds
+        """
+        self.tokens_used += tokens
+        self.iterations += 1
+
+    def exhausted(self) -> bool:
+        """
+        Check if budget is exhausted.
+
+        Returns:
+            True if any budget limit is reached
+        """
+        # Check iteration limit
+        if self.iterations >= self.config.max_iterations:
+            return True
+
+        # Check token limit
+        if self.tokens_used >= self.config.max_tokens:
+            return True
+
+        # Check time limit
+        elapsed = time.time() - self.start_time
+        if elapsed >= self.config.max_seconds:
+            return True
+
+        return False
+
+    def remaining_tokens(self) -> int:
+        """Get remaining token budget."""
+        return max(0, self.config.max_tokens - self.tokens_used)
+
+    def remaining_time(self) -> float:
+        """Get remaining time budget in seconds."""
+        elapsed = time.time() - self.start_time
+        return max(0.0, self.config.max_seconds - elapsed)
+
+    def remaining_iterations(self) -> int:
+        """Get remaining iteration budget."""
+        return max(0, self.config.max_iterations - self.iterations)
+
+    def progress(self) -> dict[str, float]:
+        """
+        Get budget progress.
+
+        Returns:
+            Dict with progress percentages for each budget type
+        """
+        elapsed = time.time() - self.start_time
+
+        return {
+            "tokens": self.tokens_used / self.config.max_tokens,
+            "time": elapsed / self.config.max_seconds,
+            "iterations": self.iterations / self.config.max_iterations,
+        }
+
+    def summary(self) -> dict[str, int | float]:
+        """
+        Get budget summary.
+
+        Returns:
+            Dict with used and remaining amounts
+        """
+        return {
+            "tokens_used": self.tokens_used,
+            "tokens_remaining": self.remaining_tokens(),
+            "time_elapsed": time.time() - self.start_time,
+            "time_remaining": self.remaining_time(),
+            "iterations_done": self.iterations,
+            "iterations_remaining": self.remaining_iterations(),
+        }
+
+    def reset(self) -> None:
+        """Reset budget counters."""
+        self.start_time = time.time()
+        self.tokens_used = 0
+        self.iterations = 0