codeoptix 0.1.3__py3-none-any.whl

codeoptix/evolution/engine.py

@@ -0,0 +1,206 @@
+"""Evolution engine for CodeOptix using GEPA-style patterns."""
+
+from typing import Any
+
+from codeoptix.adapters.base import AgentAdapter
+from codeoptix.artifacts.manager import ArtifactManager
+from codeoptix.evaluation import EvaluationEngine
+from codeoptix.evolution.proposer import PromptProposer
+from codeoptix.utils.llm import LLMClient
+
+
+class EvolutionEngine:
+    """
+    Evolution engine that optimizes agent prompts using GEPA-style reflective mutation.
+
+    Uses evaluation results and reflection to evolve prompts iteratively.
+    """
+
+    def __init__(
+        self,
+        adapter: AgentAdapter,
+        evaluation_engine: EvaluationEngine,
+        llm_client: LLMClient,
+        artifact_manager: ArtifactManager | None = None,
+        config: dict[str, Any] | None = None,
+    ):
+        """
+        Initialize evolution engine.
+
+        Args:
+            adapter: Agent adapter to evolve
+            evaluation_engine: Evaluation engine for testing candidates
+            llm_client: LLM client for prompt proposal
+            artifact_manager: Artifact manager for saving evolved prompts
+            config: Configuration dictionary
+        """
+        self.adapter = adapter
+        self.evaluation_engine = evaluation_engine
+        self.llm_client = llm_client
+        self.artifact_manager = artifact_manager
+        self.config = config or {}
+
+        # Evolution parameters
+        self.max_iterations = self.config.get("max_iterations", 3)
+        self.population_size = self.config.get("population_size", 3)
+        self.minibatch_size = self.config.get("minibatch_size", 2)
+        self.improvement_threshold = self.config.get("improvement_threshold", 0.05)
+
+        # Initialize proposer
+        proposer_config = self.config.get("proposer", {})
+        self.proposer = PromptProposer(llm_client, proposer_config)
+
+    def evolve(
+        self,
+        evaluation_results: dict[str, Any],
+        reflection: str,
+        behavior_names: list[str] | None = None,
+        context: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Evolve agent prompts based on evaluation results.
+
+        Args:
+            evaluation_results: Previous evaluation results
+            reflection: Reflection markdown content
+            behavior_names: Behaviors to focus on (default: all)
+            context: Optional context for evaluation
+
+        Returns:
+            Dictionary with evolved prompts and evolution metadata
+        """
+        context = context or {}
+        behavior_names = behavior_names or list(evaluation_results.get("behaviors", {}).keys())
+
+        # Get current prompt
+        current_prompt = self.adapter.get_prompt()
+        current_candidate = {"system_prompt": current_prompt}
+
+        # Evaluate current candidate on minibatch
+        current_score = evaluation_results.get("overall_score", 0.0)
+
+        evolution_history = []
+        best_candidate = current_candidate.copy()
+        best_score = current_score
+
+        for iteration in range(self.max_iterations):
+            # Generate population of candidates
+            candidates = self._generate_candidates(
+                current_candidate=current_candidate,
+                evaluation_results=evaluation_results,
+                reflection=reflection,
+                population_size=self.population_size,
+            )
+
+            # Evaluate candidates on minibatch
+            candidate_scores = []
+            for candidate in candidates:
+                # Update adapter with candidate prompt
+                self.adapter.update_prompt(candidate["system_prompt"])
+
+                # Quick evaluation on minibatch
+                minibatch_results = self._evaluate_minibatch(
+                    behavior_names=behavior_names,
+                    num_scenarios=self.minibatch_size,
+                    context=context,
+                )
+
+                score = minibatch_results.get("overall_score", 0.0)
+                candidate_scores.append((candidate, score))
+
+            # Select best candidate
+            candidate_scores.sort(key=lambda x: x[1], reverse=True)
+            best_iteration_candidate, best_iteration_score = candidate_scores[0]
+
+            # Check for improvement
+            improvement = best_iteration_score - best_score
+
+            evolution_history.append(
+                {
+                    "iteration": iteration + 1,
+                    "candidates_tested": len(candidates),
+                    "best_score": best_iteration_score,
+                    "improvement": improvement,
+                    "candidate": best_iteration_candidate,
+                }
+            )
+
+            # Accept if improved
+            if improvement >= self.improvement_threshold:
+                best_candidate = best_iteration_candidate
+                best_score = best_iteration_score
+                current_candidate = best_iteration_candidate
+
+                # Update adapter with best prompt
+                self.adapter.update_prompt(best_candidate["system_prompt"])
+            else:
+                # No improvement, stop early
+                break
+
+        # Restore best prompt to adapter
+        self.adapter.update_prompt(best_candidate["system_prompt"])
+
+        # Prepare evolved prompts output
+        evolved_prompts = {
+            "agent": self.adapter.get_adapter_type(),
+            "prompts": best_candidate,
+            "metadata": {
+                "iterations": len(evolution_history),
+                "initial_score": current_score,
+                "final_score": best_score,
+                "improvement": best_score - current_score,
+                "evolution_history": evolution_history,
+            },
+        }
+
+        # Save evolved prompts
+        if self.artifact_manager:
+            run_id = evaluation_results.get("run_id")
+            self.artifact_manager.save_evolved_prompts(evolved_prompts, run_id=run_id)
+
+        return evolved_prompts
+
+    def _generate_candidates(
+        self,
+        current_candidate: dict[str, str],
+        evaluation_results: dict[str, Any],
+        reflection: str,
+        population_size: int,
+    ) -> list[dict[str, str]]:
+        """Generate population of candidate prompts."""
+        candidates = []
+
+        # Always include current candidate
+        candidates.append(current_candidate.copy())
+
+        # Generate new candidates
+        for _ in range(population_size - 1):
+            proposed_prompt = self.proposer.propose_improved_prompt(
+                current_prompt=current_candidate["system_prompt"],
+                evaluation_results=evaluation_results,
+                reflection=reflection,
+                component_name="system_prompt",
+            )
+
+            candidates.append({"system_prompt": proposed_prompt})
+
+        return candidates
+
+    def _evaluate_minibatch(
+        self, behavior_names: list[str], num_scenarios: int, context: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Quick evaluation on a minibatch of scenarios."""
+        # Use evaluation engine with reduced scenarios
+        config = self.evaluation_engine.config.copy()
+        config["scenario_generator"] = config.get("scenario_generator", {}).copy()
+        config["scenario_generator"]["num_scenarios"] = num_scenarios
+
+        # Create temporary evaluation engine with reduced config
+        from codeoptix.evaluation import EvaluationEngine
+
+        temp_engine = EvaluationEngine(self.adapter, self.llm_client, config=config)
+
+        # Run evaluation
+        results = temp_engine.evaluate_behaviors(behavior_names=behavior_names, context=context)
+
+        return results

codeoptix/evolution/gepa_integration.py

@@ -0,0 +1,149 @@
+"""Minimal GEPA integration for CodeOptiX evolution.
+
+This module provides a minimal integration with GEPA (Genetic-Pareto) by using
+GEPA's InstructionProposalSignature component for prompt evolution.
+
+Note: This is NOT a full GEPA framework integration. We're using GEPA's proven
+instruction proposal mechanism rather than the complete GEPA optimization engine.
+"""
+
+from typing import Any
+
+from gepa.strategies.instruction_proposal import InstructionProposalSignature
+
+
+class MinimalGEPAProposer:
+    """
+    Minimal GEPA integration for prompt proposal.
+
+    Uses GEPA's InstructionProposalSignature for prompt evolution.
+    """
+
+    def __init__(self, llm_client, config: dict[str, Any] | None = None):
+        """
+        Initialize minimal GEPA proposer.
+
+        Args:
+            llm_client: LLM client compatible with GEPA's LanguageModel interface
+            config: Configuration dictionary
+        """
+        self.llm_client = llm_client
+        self.config = config or {}
+        self.prompt_template = self.config.get(
+            "prompt_template", InstructionProposalSignature.default_prompt_template
+        )
+
+    def propose_improved_prompt(
+        self,
+        current_prompt: str,
+        reflective_dataset: list[dict[str, Any]],
+        component_name: str = "system_prompt",
+    ) -> str:
+        """
+        Propose improved prompt using GEPA's InstructionProposalSignature.
+
+        Args:
+            current_prompt: Current prompt text
+            reflective_dataset: List of failure examples with feedback
+            component_name: Name of the component
+
+        Returns:
+            Proposed improved prompt text
+        """
+        # Convert reflective dataset to GEPA format
+        gepa_dataset = self._convert_to_gepa_format(reflective_dataset)
+
+        # Use GEPA's instruction proposal
+        result = InstructionProposalSignature.run(
+            lm=self._wrap_llm_client(),
+            input_dict={
+                "current_instruction_doc": current_prompt,
+                "dataset_with_feedback": gepa_dataset,
+                "prompt_template": self.prompt_template,
+            },
+        )
+
+        return result["new_instruction"]
+
+    def _convert_to_gepa_format(
+        self, reflective_dataset: list[dict[str, Any]]
+    ) -> list[dict[str, Any]]:
+        """
+        Convert CodeOptiX reflective dataset to GEPA format.
+
+        GEPA's InstructionProposalSignature expects a dataset with this structure:
+        {
+            "Inputs": Dict[str, str],           # Task inputs
+            "Generated Outputs": Dict[str, str] | str,  # Model outputs
+            "Feedback": str                     # Feedback on performance
+        }
+
+        This format is used by GEPA's default prompt template to generate
+        improved instructions through reflective mutation.
+        """
+        gepa_format = []
+
+        for example in reflective_dataset:
+            gepa_example = {
+                "Inputs": {
+                    "task": example.get("scenario", ""),
+                    "behavior": example.get("behavior", ""),
+                },
+                "Generated Outputs": {
+                    "score": example.get("score", 0.0),
+                    "evidence": example.get("evidence", []),
+                },
+                "Feedback": self._generate_feedback(example),
+            }
+            gepa_format.append(gepa_example)
+
+        return gepa_format
+
+    def _generate_feedback(self, example: dict[str, Any]) -> str:
+        """Generate feedback string from example."""
+        behavior = example.get("behavior", "unknown")
+        score = example.get("score", 0.0)
+        evidence = example.get("evidence", [])
+
+        feedback = f"Behavior '{behavior}' scored {score:.2f}/1.0. "
+
+        if evidence:
+            feedback += "Issues found: " + "; ".join(evidence[:3])
+        else:
+            feedback += "No specific issues identified."
+
+        return feedback
+
+    def _wrap_llm_client(self):
+        """
+        Wrap CodeOptiX LLM client to GEPA's LanguageModel interface.
+
+        GEPA expects a LanguageModel Protocol with a __call__ method that takes
+        a prompt string and returns a string response.
+        """
+
+        class GEPALLMWrapper:
+            def __init__(self, llm_client):
+                self.llm_client = llm_client
+                # Get default model from config if available
+                self.default_model = "gpt-4o"  # Default fallback
+                if hasattr(llm_client, "config") and isinstance(llm_client.config, dict):
+                    self.default_model = llm_client.config.get("model", self.default_model)
+
+            def __call__(self, prompt: str) -> str:
+                """
+                Generate response using CodeOptiX LLM client.
+
+                This implements GEPA's LanguageModel Protocol:
+                - Takes a prompt string
+                - Returns a string response
+                """
+                response = self.llm_client.chat_completion(
+                    messages=[{"role": "user", "content": prompt}],
+                    model=self.default_model,
+                    temperature=0.7,
+                )
+                # Ensure we return a string
+                return str(response) if response else ""
+
+        return GEPALLMWrapper(self.llm_client)

codeoptix/evolution/proposer.py

@@ -0,0 +1,185 @@
+"""Prompt proposer using GEPA-style reflective mutation."""
+
+from typing import Any
+
+from codeoptix.evolution.gepa_integration import MinimalGEPAProposer
+from codeoptix.utils.llm import LLMClient
+
+
+class PromptProposer:
+    """
+    Proposes improved prompts using GEPA-style reflective mutation.
+
+    Uses evaluation results and reflection to generate improved prompt variants.
+    """
+
+    def __init__(self, llm_client: LLMClient, config: dict[str, Any] | None = None):
+        """
+        Initialize prompt proposer.
+
+        Args:
+            llm_client: LLM client for generating proposals
+            config: Configuration dictionary
+        """
+        self.llm_client = llm_client
+        self.config = config or {}
+        self.use_gepa = self.config.get("use_gepa", True)  # Use GEPA by default
+
+        # Initialize GEPA proposer if enabled
+        if self.use_gepa:
+            self.gepa_proposer = MinimalGEPAProposer(llm_client, config)
+        else:
+            self.gepa_proposer = None
+            self.model = self.config.get("model", "gpt-4o")
+            self.temperature = self.config.get("temperature", 0.7)
+
+    def propose_improved_prompt(
+        self,
+        current_prompt: str,
+        evaluation_results: dict[str, Any],
+        reflection: str,
+        component_name: str = "system_prompt",
+    ) -> str:
+        """
+        Propose an improved prompt based on evaluation results and reflection.
+
+        Args:
+            current_prompt: Current prompt text
+            evaluation_results: Evaluation results dictionary
+            reflection: Reflection markdown content
+            component_name: Name of the prompt component
+
+        Returns:
+            Proposed improved prompt text
+        """
+        # Build reflective dataset from evaluation results
+        reflective_data = self._build_reflective_dataset(evaluation_results)
+
+        # Use GEPA if enabled
+        if self.use_gepa and self.gepa_proposer:
+            return self.gepa_proposer.propose_improved_prompt(
+                current_prompt=current_prompt,
+                reflective_dataset=reflective_data,
+                component_name=component_name,
+            )
+
+        # Fallback to custom implementation
+        # Create proposal prompt
+        proposal_prompt = self._create_proposal_prompt(
+            current_prompt=current_prompt,
+            reflective_data=reflective_data,
+            reflection=reflection,
+            component_name=component_name,
+        )
+
+        # Get proposal from LLM
+        response = self.llm_client.chat_completion(
+            messages=[{"role": "user", "content": proposal_prompt}],
+            model=self.model,
+            temperature=self.temperature,
+        )
+
+        # Extract proposed prompt
+        proposed_prompt = self._extract_proposed_prompt(response, current_prompt)
+
+        return proposed_prompt
+
+    def _build_reflective_dataset(self, evaluation_results: dict[str, Any]) -> list[dict[str, Any]]:
+        """Build reflective dataset from evaluation results."""
+        dataset = []
+
+        behaviors = evaluation_results.get("behaviors", {})
+
+        for behavior_name, behavior_data in behaviors.items():
+            score = behavior_data.get("score", 0.0)
+            evidence = behavior_data.get("evidence", [])
+            scenario_results = behavior_data.get("scenario_results", [])
+
+            # Collect failure examples
+            if score < 0.7:  # Focus on failures
+                for scenario_result in scenario_results:
+                    behavior_result = scenario_result.get("behavior_result", {})
+                    if not behavior_result.get("passed", True):
+                        dataset.append(
+                            {
+                                "behavior": behavior_name,
+                                "score": score,
+                                "evidence": evidence[:3],  # Limit evidence
+                                "scenario": scenario_result.get("scenario", {}).get("prompt", ""),
+                            }
+                        )
+
+        return dataset
+
+    def _create_proposal_prompt(
+        self,
+        current_prompt: str,
+        reflective_data: list[dict[str, Any]],
+        reflection: str,
+        component_name: str,
+    ) -> str:
+        """Create prompt for LLM to propose improved prompt."""
+
+        # Format reflective data
+        failure_examples = []
+        for i, data in enumerate(reflective_data[:5], 1):  # Limit to 5 examples
+            failure_examples.append(
+                f"Example {i}:\n"
+                f"  Behavior: {data['behavior']}\n"
+                f"  Score: {data['score']:.2f}\n"
+                f"  Issues: {', '.join(data['evidence'][:2])}\n"
+            )
+
+        prompt = f"""You are optimizing a coding agent's prompt to improve its behavior.
+
+Current {component_name}:
+```
+{current_prompt}
+```
+
+Evaluation Results Summary:
+{reflection[:1000]}...
+
+Specific Failure Examples:
+{"".join(failure_examples)}
+
+Based on the evaluation results and reflection above, propose an improved version of the {component_name} that addresses the identified issues.
+
+The improved prompt should:
+1. Address the root causes identified in the reflection
+2. Include specific guidance to prevent the failures observed
+3. Maintain the original intent while adding necessary constraints
+4. Be clear, actionable, and specific
+
+Provide ONLY the improved prompt text, without additional explanation or markdown formatting.
+"""
+
+        return prompt
+
+    def _extract_proposed_prompt(self, response: str, fallback: str) -> str:
+        """Extract proposed prompt from LLM response."""
+        # Try to extract code block
+        import re
+
+        # Look for code blocks
+        code_blocks = re.findall(r"```(?:.*)?\n(.*?)```", response, re.DOTALL)
+        if code_blocks:
+            return code_blocks[0].strip()
+
+        # Look for text between markers
+        if "Current" in response and "Improved" in response:
+            # Try to extract after "Improved" marker
+            improved_section = response.split("Improved")[-1]
+            # Take first substantial paragraph
+            lines = [l.strip() for l in improved_section.split("\n") if l.strip()]
+            if lines:
+                return "\n".join(lines[:10])  # Take first 10 non-empty lines
+
+        # Fallback: use response as-is, but clean it up
+        cleaned = response.strip()
+        # Remove common prefixes
+        for prefix in ["Improved prompt:", "Here's the improved prompt:", "Proposed prompt:"]:
+            if cleaned.startswith(prefix):
+                cleaned = cleaned[len(prefix) :].strip()
+
+        return cleaned if cleaned else fallback

codeoptix/linters/__init__.py

@@ -0,0 +1,13 @@
+"""Linter integration for CodeOptiX - works without API keys."""
+
+from codeoptix.linters.base import LinterIssue, LinterResult, Severity
+from codeoptix.linters.language_detector import LanguageDetector
+from codeoptix.linters.runner import LinterRunner
+
+__all__ = [
+    "LanguageDetector",
+    "LinterIssue",
+    "LinterResult",
+    "LinterRunner",
+    "Severity",
+]