ctrlcode 0.1.0__py3-none-any.whl

ctrlcode/fuzzing/derived_orchestrator.py

@@ -0,0 +1,732 @@
+"""Derived fuzzing orchestrator - coordinates context-aware differential fuzzing."""
+
+import logging
+import time
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import AsyncIterator, Optional
+
+from ..analysis.bug_detector import BugPatternDetector
+from ..embeddings.embedder import CodeEmbedder
+from ..embeddings.vector_store import VectorStore
+from ..providers.base import Provider, StreamEvent
+from ..storage.history_db import BugPattern, FuzzingSession, HistoryDB, StoredTest
+from .analyzer import DiagnosedDivergence, DerivedOracleAnalyzer
+from .budget import BudgetConfig, BudgetManager
+from .context import ContextDerivation, ContextDerivationEngine
+from .context_fuzzer import ContextAwareFuzzer, FuzzTestCase
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FuzzingResult:
+    """Result of derived fuzzing process."""
+
+    final_output: str
+    iterations: int
+    total_tests: int
+    divergences_found: int
+    divergences_fixed: int
+    oracle_corrections: int
+    context_derivation: ContextDerivation
+    quality_score: float
+    budget_used: dict[str, float]
+    analysis_report: str
+
+    # Phase 4: Historical learning metrics
+    oracle_reused: bool = False
+    reused_from_session: Optional[str] = None
+    bug_patterns_detected: int = 0
+    tests_before_deduplication: int = 0
+    tests_after_deduplication: int = 0
+    session_id: Optional[str] = None
+
+
+class DerivedFuzzingOrchestrator:
+    """
+    Main coordinator for derived context fuzzing.
+
+    Pipeline:
+    1. Derive context from code (once)
+    2. Loop (max iterations):
+       a. Generate test batch (fuzzer)
+       b. Execute tests with mocks
+       c. Check divergences
+       d. Analyze divergences
+       e. Apply fixes (code patches OR oracle corrections)
+       f. Update fuzzer history
+    3. Return patched code + analysis + corrected context
+    """
+
+    def __init__(
+        self,
+        providers: list[Provider],
+        config: dict,
+        embeddings_config: Optional[dict] = None,
+        enable_history: bool = True,
+        history_db_path: Optional[Path] = None,
+    ):
+        """
+        Initialize derived fuzzing orchestrator.
+
+        Args:
+            providers: List of LLM providers to use
+            config: Fuzzing configuration
+            embeddings_config: Embeddings endpoint config (api_key, base_url, model)
+            enable_history: Enable historical learning (default: True)
+            history_db_path: Path to history database (default: ~/.ctrlcode/history.db)
+        """
+        self.providers = providers
+        self.config = config
+        self.provider = providers[0]  # Primary provider
+        self.enable_history = enable_history
+        self.embeddings_config = embeddings_config
+
+        # Phase 4: Initialize historical learning components
+        self.history_db: Optional[HistoryDB] = None
+        self.embedder: Optional[CodeEmbedder] = None
+        self.bug_detector: Optional[BugPatternDetector] = None
+
+        if enable_history:
+            # Default history DB path
+            if history_db_path is None:
+                history_dir = Path.home() / ".ctrlcode"
+                history_dir.mkdir(parents=True, exist_ok=True)
+                history_db_path = history_dir / "history.db"
+
+            logger.info(f"Initializing history database at {history_db_path}")
+            self.history_db = HistoryDB(str(history_db_path))
+
+            # Initialize embedder if config provided
+            if embeddings_config:
+                self.embedder = CodeEmbedder(
+                    api_key=embeddings_config["api_key"],
+                    base_url=embeddings_config["base_url"],
+                    model_name=embeddings_config.get("model", "text-embedding-3-small")
+                )
+            self.bug_detector = BugPatternDetector(
+                self.history_db,
+                embedder=self.embedder,
+                similarity_threshold=0.75,
+            )
+
+        # Initialize components (with history integration if enabled)
+        self.context_engine = ContextDerivationEngine(
+            self.provider,
+            history_db=self.history_db,
+            embedder=self.embedder,
+        )
+        self.fuzzer = ContextAwareFuzzer(
+            self.provider,
+            history_db=self.history_db,
+            embedder=self.embedder,
+            bug_detector=self.bug_detector,
+        )
+        self.analyzer = DerivedOracleAnalyzer(self.provider)
+
+    async def fuzz(
+        self,
+        user_request: str,
+        generated_code: str,
+        context_messages: list[dict],
+        max_iterations: int = 10,
+    ) -> AsyncIterator[StreamEvent | FuzzingResult]:
+        """
+        Run derived fuzzing pipeline.
+
+        Args:
+            user_request: Original user specification
+            generated_code: Generated code to test
+            context_messages: Conversation context
+            max_iterations: Maximum fuzzing iterations
+
+        Yields:
+            StreamEvent for progress updates, FuzzingResult at end
+        """
+        # Generate session ID
+        session_id = str(uuid.uuid4())
+
+        # Initialize budget
+        budget = BudgetManager(
+            BudgetConfig(
+                max_tokens=self.config.get("budget_tokens", 100000),
+                max_seconds=self.config.get("budget_seconds", 30),
+                max_iterations=max_iterations,
+            )
+        )
+
+        # Phase 4: Check for similar bug patterns before fuzzing
+        bug_patterns_detected = 0
+        if self.enable_history and self.bug_detector:
+            yield StreamEvent(
+                type="fuzzing_progress",
+                data={"stage": "bug_detection", "message": "Checking for similar bug patterns..."},
+            )
+
+            try:
+                similar_bugs = self.bug_detector.check_patterns(generated_code)
+                bug_patterns_detected = len(similar_bugs)
+
+                if similar_bugs:
+                    warnings = self.bug_detector.format_warnings(similar_bugs)
+                    yield StreamEvent(
+                        type="workflow_bug_pattern_detected",
+                        data={
+                            "count": len(similar_bugs),
+                            "warnings": warnings,
+                            "patterns": [
+                                {
+                                    "description": bug.description,
+                                    "severity": bug.severity,
+                                    "confidence": bug.confidence,
+                                    "similarity": bug.similarity,
+                                }
+                                for bug in similar_bugs[:3]  # Top 3
+                            ],
+                        },
+                    )
+                    logger.warning(f"Detected {len(similar_bugs)} similar bug patterns")
+            except Exception as e:
+                logger.error(f"Bug pattern detection failed: {e}")
+
+        # Stage 1: Context Derivation
+        yield StreamEvent(
+            type="fuzzing_progress",
+            data={"stage": "context_derivation", "message": "Deriving operational context..."},
+        )
+
+        try:
+            context = await self.context_engine.derive(
+                user_request=user_request,
+                generated_code=generated_code,
+                surrounding_files=None,  # Could extract from context_messages
+                session_id=session_id,
+            )
+
+            # Track if oracle was reused
+            oracle_reused = context.retrieved_from is not None
+            reused_from_session = context.retrieved_from
+
+            # Emit oracle reuse event
+            if oracle_reused:
+                yield StreamEvent(
+                    type="workflow_oracle_reused",
+                    data={
+                        "reused_from_session": reused_from_session,
+                        "token_savings": 2500,  # Estimated tokens saved per reuse
+                        "time_savings_seconds": 25.0,  # Estimated time saved per reuse
+                    },
+                )
+
+        except Exception as e:
+            yield StreamEvent(
+                type="fuzzing_error",
+                data={"stage": "context_derivation", "error": str(e)},
+            )
+            # Return with minimal result
+            yield FuzzingResult(
+                final_output=generated_code,
+                iterations=0,
+                total_tests=0,
+                divergences_found=0,
+                divergences_fixed=0,
+                oracle_corrections=0,
+                context_derivation=None,  # type: ignore
+                quality_score=0.0,
+                budget_used=budget.summary(),
+                analysis_report=f"Context derivation failed: {e}",
+            )
+            return
+
+        yield StreamEvent(
+            type="context_derived",
+            data={
+                "system_type": context.system_placement.system_type,
+                "invariants_count": len(context.behavioral_invariants),
+                "edge_cases_count": len(context.edge_case_surface),
+            },
+        )
+
+        # Pipeline state
+        current_code = generated_code
+        current_context = context
+        all_analyses: list[DiagnosedDivergence] = []
+        test_history: list[dict] = []
+        iteration = 0
+        total_tests = 0
+        divergences_found = 0
+        divergences_fixed = 0
+        oracle_corrections = 0
+
+        # Stage 2-6: Fuzzing Loop
+        while not budget.exhausted() and iteration < max_iterations:
+            iteration += 1
+
+            yield StreamEvent(
+                type="fuzzing_progress",
+                data={
+                    "stage": "iteration_start",
+                    "iteration": iteration,
+                    "message": f"Fuzzing iteration {iteration}...",
+                },
+            )
+
+            # Stage 3: Generate test batch
+            try:
+                batch_size = 20  # Could be configurable
+                test_cases = await self.fuzzer.generate_tests(
+                    spec=user_request,
+                    code=current_code,
+                    context=current_context,
+                    previous_results=test_history[-10:] if test_history else None,  # Last 10 results
+                    batch_size=batch_size,
+                )
+
+                total_tests += len(test_cases)
+
+                yield StreamEvent(
+                    type="fuzzing_batch_generated",
+                    data={
+                        "iteration": iteration,
+                        "test_count": len(test_cases),
+                        "input_tests": sum(1 for t in test_cases if t.type == "input"),
+                        "environment_tests": sum(1 for t in test_cases if t.type == "environment"),
+                        "combined_tests": sum(1 for t in test_cases if t.type == "combined"),
+                    },
+                )
+
+            except Exception as e:
+                yield StreamEvent(
+                    type="fuzzing_error",
+                    data={"stage": "test_generation", "iteration": iteration, "error": str(e)},
+                )
+                break
+
+            # Stage 4: Execute tests and check for divergences
+            yield StreamEvent(
+                type="fuzzing_progress",
+                data={"stage": "executing_tests", "iteration": iteration, "message": "Executing tests..."},
+            )
+
+            iteration_divergences = []
+            for test_case in test_cases:
+                # Execute test (simplified - would need actual execution)
+                # For now, we'll simulate by checking if expected behavior makes sense
+                try:
+                    actual_output, expected_output, has_divergence = await self._execute_test(
+                        code=current_code,
+                        test_case=test_case,
+                        context=current_context,
+                    )
+
+                    # Record test result
+                    test_history.append(
+                        {
+                            "test_id": test_case.id,
+                            "iteration": iteration,
+                            "type": test_case.type,
+                            "divergence": has_divergence,
+                        }
+                    )
+
+                    if has_divergence:
+                        divergences_found += 1
+                        iteration_divergences.append((test_case, actual_output, expected_output))
+
+                except Exception as e:
+                    # Test execution failed
+                    test_history.append(
+                        {
+                            "test_id": test_case.id,
+                            "iteration": iteration,
+                            "error": str(e),
+                        }
+                    )
+
+            if iteration_divergences:
+                yield StreamEvent(
+                    type="divergence_found",
+                    data={
+                        "iteration": iteration,
+                        "count": len(iteration_divergences),
+                    },
+                )
+
+            # Stage 5: Analyze divergences
+            if iteration_divergences:
+                yield StreamEvent(
+                    type="fuzzing_progress",
+                    data={
+                        "stage": "analyzing_divergences",
+                        "iteration": iteration,
+                        "message": f"Analyzing {len(iteration_divergences)} divergences...",
+                    },
+                )
+
+                for test_case, actual_output, expected_output in iteration_divergences:
+                    try:
+                        analysis = await self.analyzer.analyze_divergence(
+                            spec=user_request,
+                            code=current_code,
+                            context=current_context,
+                            test_case=test_case,
+                            actual_output=actual_output,
+                            expected_output=expected_output,
+                            previous_analyses=all_analyses,
+                        )
+
+                        all_analyses.append(analysis)
+
+                        yield StreamEvent(
+                            type="analysis_complete",
+                            data={
+                                "iteration": iteration,
+                                "test_id": test_case.id,
+                                "source": analysis.source,
+                                "confidence": analysis.confidence,
+                            },
+                        )
+
+                        # Stage 6: Apply fix based on source
+                        if analysis.source == "MODEL_BUG":
+                            # Patch the code
+                            current_code = await self._apply_code_patch(current_code, analysis.fix)
+                            divergences_fixed += 1
+
+                            yield StreamEvent(
+                                type="code_patched",
+                                data={
+                                    "iteration": iteration,
+                                    "test_id": test_case.id,
+                                    "fix_type": "model_bug",
+                                },
+                            )
+
+                        elif analysis.source == "ORACLE_BUG":
+                            # Correct the oracle
+                            current_context = await self._correct_oracle(current_context, analysis.fix)
+                            oracle_corrections += 1
+
+                            yield StreamEvent(
+                                type="oracle_corrected",
+                                data={
+                                    "iteration": iteration,
+                                    "test_id": test_case.id,
+                                    "corrected_invariants": analysis.fix.get("corrected_invariants", []),
+                                },
+                            )
+
+                        elif analysis.source == "SPEC_GAP":
+                            # Flag for user clarification
+                            yield StreamEvent(
+                                type="spec_gap_found",
+                                data={
+                                    "iteration": iteration,
+                                    "test_id": test_case.id,
+                                    "question": analysis.fix.get("clarification_question", ""),
+                                },
+                            )
+
+                        elif analysis.source == "ENVIRONMENT_MISMATCH":
+                            # Re-derive context with corrected assumptions
+                            # For now, we'll update the context in place
+                            current_context = await self._correct_environment(current_context, analysis.fix)
+
+                    except Exception as e:
+                        yield StreamEvent(
+                            type="fuzzing_error",
+                            data={
+                                "stage": "analysis",
+                                "iteration": iteration,
+                                "test_id": test_case.id,
+                                "error": str(e),
+                            },
+                        )
+
+            # Check if we should continue
+            if not iteration_divergences:
+                # No divergences found, we're stable
+                break
+
+            # Update budget
+            budget.consume(tokens=1000, elapsed_time=time.time() - budget.start_time)
+
+        # Generate analysis report
+        analysis_report = self._generate_report(
+            iterations=iteration,
+            total_tests=total_tests,
+            divergences_found=divergences_found,
+            divergences_fixed=divergences_fixed,
+            oracle_corrections=oracle_corrections,
+            analyses=all_analyses,
+            context=current_context,
+        )
+
+        # Calculate quality score (simplified)
+        quality_score = 1.0 - (divergences_found - divergences_fixed) / max(total_tests, 1)
+        quality_score = max(0.0, min(1.0, quality_score))
+
+        # Phase 4: Store fuzzing session in history DB
+        if self.enable_history and self.history_db:
+            try:
+                await self._store_fuzzing_session(
+                    session_id=session_id,
+                    user_request=user_request,
+                    generated_code=generated_code,
+                    context=current_context,
+                    total_tests=total_tests,
+                    divergences=divergences_found,
+                    oracle_reused=oracle_reused,
+                    reused_from=reused_from_session,
+                    quality_score=quality_score,
+                    all_analyses=all_analyses,
+                    test_cases=test_cases if 'test_cases' in locals() else [],
+                )
+                logger.info(f"Stored fuzzing session {session_id} in history database")
+
+                # Emit history updated event
+                yield StreamEvent(
+                    type="workflow_history_updated",
+                    data={
+                        "session_id": session_id,
+                        "oracle_reused": oracle_reused,
+                        "bug_patterns_detected": bug_patterns_detected,
+                        "total_tests": total_tests,
+                    },
+                )
+            except Exception as e:
+                logger.error(f"Failed to store fuzzing session in history: {e}")
+
+        # Return final result
+        yield FuzzingResult(
+            final_output=current_code,
+            iterations=iteration,
+            total_tests=total_tests,
+            divergences_found=divergences_found,
+            divergences_fixed=divergences_fixed,
+            oracle_corrections=oracle_corrections,
+            context_derivation=current_context,
+            quality_score=quality_score,
+            budget_used=budget.summary(),
+            analysis_report=analysis_report,
+            # Phase 4: Historical learning metrics
+            oracle_reused=oracle_reused,
+            reused_from_session=reused_from_session,
+            bug_patterns_detected=bug_patterns_detected,
+            tests_before_deduplication=0,  # Would be tracked in fuzzer
+            tests_after_deduplication=total_tests,
+            session_id=session_id,
+        )
+
+    async def _execute_test(
+        self,
+        code: str,
+        test_case: FuzzTestCase,
+        context: ContextDerivation,
+    ) -> tuple[dict, dict, bool]:
+        """
+        Execute a test case and check for divergences.
+
+        Args:
+            code: Code to test
+            test_case: Test case to execute
+            context: Context derivation
+
+        Returns:
+            (actual_output, expected_output, has_divergence)
+        """
+        # This is a simplified implementation
+        # In reality, would need to:
+        # 1. Set up mocks based on test_case.environment.mock_setup
+        # 2. Execute the code with test_case.input
+        # 3. Compare actual vs expected from test_case.environment.expected_behavior
+
+        # For now, we'll simulate by saying 20% of tests have divergences
+        import random
+
+        has_divergence = random.random() < 0.2
+
+        actual_output = {"result": "simulated_output", "divergence": has_divergence}
+        expected_output = (
+            test_case.environment.expected_behavior
+            if test_case.environment
+            else {"result": "expected"}
+        )
+
+        return actual_output, expected_output, has_divergence
+
+    async def _apply_code_patch(self, code: str, fix: dict) -> str:
+        """Apply a code patch from MODEL_BUG fix."""
+        # In reality, would apply the actual patch
+        # For now, just return the code with a comment
+        patch = fix.get("patch", "# Patch applied")
+        return f"{code}\n\n{patch}"
+
+    async def _correct_oracle(self, context: ContextDerivation, fix: dict) -> ContextDerivation:
+        """Correct the derived oracle based on ORACLE_BUG fix."""
+        # In reality, would update specific invariants
+        # For now, return the context as-is
+        _corrected_invariants = fix.get("corrected_invariants", [])
+        # Update behavioral_invariants with corrections
+        return context
+
+    async def _correct_environment(self, context: ContextDerivation, fix: dict) -> ContextDerivation:
+        """Correct environmental assumptions based on ENVIRONMENT_MISMATCH fix."""
+        # In reality, would update environmental_constraints
+        # For now, return the context as-is
+        _corrected_assumptions = fix.get("corrected_assumptions", {})
+        # Update environmental_constraints with corrections
+        return context
+
+    def _generate_report(
+        self,
+        iterations: int,
+        total_tests: int,
+        divergences_found: int,
+        divergences_fixed: int,
+        oracle_corrections: int,
+        analyses: list[DiagnosedDivergence],
+        context: ContextDerivation,
+    ) -> str:
+        """Generate analysis report."""
+        report = f"""# Derived Fuzzing Analysis Report
+
+## Summary
+- Iterations: {iterations}
+- Total tests executed: {total_tests}
+- Divergences found: {divergences_found}
+- Divergences fixed: {divergences_fixed}
+- Oracle corrections: {oracle_corrections}
+
+## Context Derivation
+- System type: {context.system_placement.system_type}
+- Layer: {context.system_placement.layer}
+- Behavioral invariants: {len(context.behavioral_invariants)}
+- Integration contracts: {len(context.integration_contracts)}
+- Edge cases identified: {len(context.edge_case_surface)}
+
+## Divergence Breakdown
+"""
+
+        # Count by source
+        source_counts = {}
+        for analysis in analyses:
+            source_counts[analysis.source] = source_counts.get(analysis.source, 0) + 1
+
+        for source, count in source_counts.items():
+            report += f"- {source}: {count}\n"
+
+        if analyses:
+            report += "\n## Key Findings\n"
+            for i, analysis in enumerate(analyses[:5], 1):  # Top 5
+                report += f"\n### {i}. {analysis.diagnosis}\n"
+                report += f"- Source: {analysis.source}\n"
+                report += f"- Confidence: {analysis.confidence:.2f}\n"
+                report += f"- Impact: {analysis.impact}\n"
+
+        return report
+
+    async def _store_fuzzing_session(
+        self,
+        session_id: str,
+        user_request: str,
+        generated_code: str,
+        context: ContextDerivation,
+        total_tests: int,
+        divergences: int,
+        oracle_reused: bool,
+        reused_from: Optional[str],
+        quality_score: float,
+        all_analyses: list[DiagnosedDivergence],
+        test_cases: list[FuzzTestCase],
+    ) -> None:
+        """Store fuzzing session results in history database.
+
+        Args:
+            session_id: Session identifier
+            user_request: User specification
+            generated_code: Generated code
+            context: Derived context
+            total_tests: Number of tests executed
+            divergences: Number of divergences found
+            oracle_reused: Whether oracle was reused
+            reused_from: Source session ID if reused
+            quality_score: Fuzzing quality score
+            all_analyses: List of divergence analyses
+            test_cases: List of test cases
+        """
+        if not self.history_db or not self.embedder:
+            return
+
+        # Store session metadata
+        session = FuzzingSession(
+            session_id=session_id,
+            user_request=user_request,
+            generated_code=generated_code,
+            oracle=context.to_json(),
+            timestamp=datetime.now(),
+            num_tests=total_tests,
+            num_failures=divergences,
+            oracle_reused=oracle_reused,
+            reused_from=reused_from,
+            quality_score=quality_score,
+        )
+        self.history_db.store_session(session)
+
+        # Store bug patterns from divergence analyses (batch embed)
+        model_bug_analyses = [
+            (i, analysis) for i, analysis in enumerate(all_analyses)
+            if analysis.source == "MODEL_BUG"
+        ]
+
+        if model_bug_analyses:
+            # Batch embed all bugs (same code for all, so just embed once)
+            bug_embedding = self.embedder.embed_code(generated_code)
+
+            for i, analysis in model_bug_analyses:
+                bug_pattern = BugPattern(
+                    bug_id=f"{session_id}_bug_{i}",
+                    session_id=session_id,
+                    bug_description=analysis.diagnosis,
+                    code_snippet=generated_code[:500],  # First 500 chars
+                    embedding=bug_embedding,
+                    severity="high" if analysis.confidence > 0.8 else "medium",
+                    timestamp=datetime.now(),
+                )
+                self.history_db.store_bug(bug_pattern)
+
+        # Store test cases with embeddings (batch embed)
+        test_subset = test_cases[:50]  # Store up to 50 tests
+        if test_subset:
+            # Prepare test strings and batch embed
+            test_strs = [f"{tc.type}: {tc.rationale}" for tc in test_subset]
+            test_embeddings = self.embedder.embed_batch(test_strs)
+
+            for i, (test_case, test_str, test_embedding) in enumerate(
+                zip(test_subset, test_strs, test_embeddings)
+            ):
+                stored_test = StoredTest(
+                    test_id=f"{session_id}_test_{i}",
+                    session_id=session_id,
+                    test_code=test_str,
+                    embedding=test_embedding,
+                    passed=True,  # Would need actual execution results
+                    timestamp=datetime.now(),
+                )
+                self.history_db.store_test(stored_test)
+
+    def get_history_stats(self) -> dict:
+        """Get statistics from history database.
+
+        Returns:
+            Dictionary of statistics
+        """
+        if not self.history_db:
+            return {"enabled": False}
+
+        stats = self.history_db.get_stats()
+        stats["enabled"] = True
+        return stats