deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/{optimization/gepa/loop.py → optimizer/algorithms/gepa/gepa.py}

@@ -14,86 +14,118 @@ from typing import (
     Optional,
 )
 
+from deepeval.models.base_model import DeepEvalBaseLLM
+
 from deepeval.errors import DeepEvalError
-from deepeval.optimization.aggregates import Aggregator, mean_of_all
-from deepeval.optimization.types import (
+from deepeval.optimizer.utils import Aggregator, mean_of_all
+from deepeval.optimizer.types import (
     AcceptedIterationDict,
     PromptConfiguration,
     PromptConfigurationId,
     ModuleId,
     ScoreTable,
-    ScoringAdapter,
-    OptimizationResult,
+    OptimizationReport,
     RunnerStatusType,
-    RunnerStatusCallbackProtocol,
+    RunnerStatusCallback,
 )
-from deepeval.optimization.utils import (
+from deepeval.optimizer.scorer.base import BaseScorer
+from deepeval.optimizer.algorithms.base import BaseAlgorithm
+from deepeval.optimizer.utils import (
     split_goldens,
     build_prompt_config_snapshots,
 )
-from deepeval.optimization.policies import (
+from deepeval.optimizer.policies import (
     pick_best_with_ties,
     select_prompt_configuration_pareto,
+    frequency_weights,
+    pareto_frontier,
 )
 from deepeval.prompt.api import PromptType
 from deepeval.prompt.prompt import Prompt
-from deepeval.optimization.mutations.prompt_rewriter import (
-    PromptRewriter,
+from deepeval.optimizer.rewriter import Rewriter
+from deepeval.optimizer.policies import TieBreaker
+from deepeval.optimizer.algorithms.configs import (
+    GEPA_MIN_DELTA,
+    GEPA_TIE_TOLERANCE,
+    GEPA_REWRITE_INSTRUCTION_MAX_CHARS,
 )
-from .configs import GEPAConfig
 
 
 if TYPE_CHECKING:
     from deepeval.dataset.golden import Golden, ConversationalGolden
 
 
-class GEPARunner:
+class GEPA(BaseAlgorithm):
     """
     GEPA loop with sync/async execution.
 
     This runner is intentionally low level and does not know about metrics,
     models, or async configs. It relies on a preconfigured
-    ScoringAdapter and PromptRewriter, which are typically constructed by
+    Scorer and Rewriter, which are typically constructed by
     the higher-level PromptOptimizer.
+
+    Parameters
+    ----------
+    iterations : int
+        Total number of GEPA loop iterations (mutation attempts). Default is 5.
+    minibatch_size : int
+        Number of examples drawn from D_feedback per iteration. Default is 8.
+    pareto_size : int
+        Size of the Pareto validation subset D_pareto. Default is 3.
+    random_seed : int, optional
+        RNG seed for reproducibility. If None, derived from time.time_ns().
+    tie_breaker : TieBreaker
+        Policy for breaking ties. Default is TieBreaker.PREFER_CHILD.
     """
 
+    name = "GEPA"
     SINGLE_MODULE_ID: ModuleId = "__module__"
+    TieBreaker = TieBreaker
 
     def __init__(
         self,
-        *,
-        config: GEPAConfig,
+        iterations: int = 5,
+        minibatch_size: int = 8,
+        pareto_size: int = 3,
+        random_seed: Optional[int] = None,
+        tie_breaker: TieBreaker = TieBreaker.PREFER_CHILD,
         aggregate_instances: Aggregator = mean_of_all,
-        scoring_adapter: Optional[ScoringAdapter] = None,
+        scorer: Optional[BaseScorer] = None,
     ) -> None:
-        self.config = config
+        # Validate parameters
+        if iterations < 1:
+            raise ValueError("iterations must be >= 1")
+        if minibatch_size < 1:
+            raise ValueError("minibatch_size must be >= 1")
+        if pareto_size < 1:
+            raise ValueError("pareto_size must be >= 1")
+
+        self.iterations = iterations
+        self.minibatch_size = minibatch_size
+        self.pareto_size = pareto_size
+        self.tie_breaker = tie_breaker
         self.aggregate_instances = aggregate_instances
-        self.scoring_adapter = scoring_adapter
+        self.scorer = scorer
 
-        # random seeded from config is used for splits, sampling, and tie-breaking.
-        self.random_state = random.Random(config.random_seed)
+        # If no seed provided, use time-based seed
+        if random_seed is None:
+            random_seed = time.time_ns()
+        self.random_seed = random_seed
+        self.random_state = random.Random(random_seed)
 
         # runtime state to be reset between runs
         self.reset_state()
 
         # Status callback set by PromptOptimizer:
         #   (kind, step_index, total_steps, detail) -> None
-        self.status_callback: Optional[RunnerStatusCallbackProtocol] = None
-
-        # Model callback used by the rewriter set by PromptOptimizer.
-        self.model_callback: Optional[
-            Callable[
-                ...,
-                Union[
-                    str,
-                    Dict,
-                    Tuple[Union[str, Dict], float],
-                ],
-            ]
-        ] = None
+        self.status_callback: Optional[RunnerStatusCallback] = None
+
+        # Optimizer model used by the rewriter for prompt mutation.
+        # Set by PromptOptimizer.
+        self.optimizer_model: Optional["DeepEvalBaseLLM"] = None
 
         # lazy loaded
-        self._rewriter: Optional[PromptRewriter] = None
+        self._rewriter: Optional[Rewriter] = None
 
     ##############
     # Public API #
@@ -101,10 +133,9 @@ class GEPARunner:
 
     def execute(
         self,
-        *,
         prompt: Prompt,
         goldens: Union[List["Golden"], List["ConversationalGolden"]],
-    ) -> Tuple[Prompt, Dict]:
+    ) -> Tuple[Prompt, OptimizationReport]:
         """Synchronous GEPA run from a full list of goldens (splits internally)."""
         total_goldens = len(goldens)
         if total_goldens < 2:
@@ -114,12 +145,11 @@ class GEPARunner:
                 "run the optimizer."
             )
 
-        self._ensure_scoring_adapter()
-        self._ensure_rewriter()
+        self._ensure_scorer()
         self.reset_state()
 
         d_feedback, d_pareto = split_goldens(
-            goldens, self.config.pareto_size, random_state=self.random_state
+            goldens, self.pareto_size, random_state=self.random_state
         )
 
         seed_prompts_by_module = {self.SINGLE_MODULE_ID: prompt}
@@ -139,7 +169,7 @@ class GEPARunner:
             # Seed Pareto scores lazily on first iteration
             if not self.pareto_score_table:
                 self.pareto_score_table[root_prompt_configuration.id] = (
-                    self.scoring_adapter.score_on_pareto(
+                    self.scorer.score_pareto(
                         root_prompt_configuration, d_pareto
                     )
                 )
@@ -154,7 +184,7 @@ class GEPARunner:
             minibatch = self._draw_minibatch(d_feedback)
 
             # 4. Feedback
-            feedback_text = self.scoring_adapter.minibatch_feedback(
+            feedback_text = self.scorer.get_minibatch_feedback(
                 parent_prompt_configuration, selected_module_id, minibatch
             )
 
@@ -172,15 +202,16 @@ class GEPARunner:
             )
 
             # 7. Evaluate parent/child on minibatch
-            parent_score = self.scoring_adapter.minibatch_score(
+            parent_score = self.scorer.score_minibatch(
                 parent_prompt_configuration, minibatch
             )
-            child_score = self.scoring_adapter.minibatch_score(
+            child_score = self.scorer.score_minibatch(
                 child_prompt_configuration, minibatch
             )
 
             # 8. Acceptance test
-            if self._should_accept_child(parent_score, child_score):
+            accepted = self._should_accept_child(parent_score, child_score)
+            if accepted:
                 accepted_iterations.append(
                     self._accept_child(
                         selected_module_id,
@@ -199,7 +230,7 @@ class GEPARunner:
         prompt_config_snapshots = build_prompt_config_snapshots(
             self.prompt_configurations_by_id
         )
-        report = OptimizationResult(
+        report = OptimizationReport(
             optimization_id=self.optimization_id,
             best_id=best.id,
             accepted_iterations=accepted_iterations,
@@ -207,14 +238,13 @@ class GEPARunner:
             parents=self.parents_by_id,
             prompt_configurations=prompt_config_snapshots,
         )
-        return best.prompts[self.SINGLE_MODULE_ID], report.as_dict()
+        return best.prompts[self.SINGLE_MODULE_ID], report
 
     async def a_execute(
         self,
-        *,
         prompt: Prompt,
         goldens: Union[List["Golden"], List["ConversationalGolden"]],
-    ) -> Tuple[Prompt, Dict]:
+    ) -> Tuple[Prompt, OptimizationReport]:
         """Asynchronous twin of execute_gepa()."""
         total_goldens = len(goldens)
         if total_goldens < 2:
@@ -224,12 +254,11 @@ class GEPARunner:
                 "run the optimizer."
             )
 
-        self._ensure_scoring_adapter()
-        self._ensure_rewriter()
+        self._ensure_scorer()
         self.reset_state()
 
         d_feedback, d_pareto = split_goldens(
-            goldens, self.config.pareto_size, random_state=self.random_state
+            goldens, self.pareto_size, random_state=self.random_state
         )
 
         seed_prompts_by_module = {self.SINGLE_MODULE_ID: prompt}
@@ -246,13 +275,19 @@ class GEPARunner:
             if not d_feedback:
                 return False
 
+            iter_start = time.perf_counter()
+
             # Seed Pareto scores lazily on first iteration
             if not self.pareto_score_table:
+                t0 = time.perf_counter()
                 self.pareto_score_table[root_prompt_configuration.id] = (
-                    await self.scoring_adapter.a_score_on_pareto(
+                    await self.scorer.a_score_pareto(
                         root_prompt_configuration, d_pareto
                     )
                 )
+                print(
+                    f"[DEBUG] Initial pareto scoring ({len(d_pareto)} goldens): {time.perf_counter() - t0:.2f}s"
+                )
 
             # 1. Pick prompt_configuration via Pareto
             parent_prompt_configuration = self._pick_prompt_configuration()
@@ -262,18 +297,23 @@ class GEPARunner:
 
             # 3. Draw minibatch
             minibatch = self._draw_minibatch(d_feedback)
+            print(f"[DEBUG] Minibatch size: {len(minibatch)}")
 
             # 4. Feedback
-            feedback_text = await self.scoring_adapter.a_minibatch_feedback(
+            t0 = time.perf_counter()
+            feedback_text = await self.scorer.a_get_minibatch_feedback(
                 parent_prompt_configuration, selected_module_id, minibatch
             )
+            print(f"[DEBUG] Get feedback: {time.perf_counter() - t0:.2f}s")
 
             # 5. Rewrite
+            t0 = time.perf_counter()
             child_prompt = await self._a_generate_child_prompt(
                 selected_module_id, parent_prompt_configuration, feedback_text
             )
+            print(f"[DEBUG] Rewrite prompt: {time.perf_counter() - t0:.2f}s")
             if child_prompt is None:
-                # Child prompt matched parent; skip this iteration.
+                print(f"[DEBUG] Child prompt same as parent, skipping")
                 return True
 
             # 6. Child prompt_configuration
@@ -282,15 +322,29 @@ class GEPARunner:
             )
 
             # 7. Evaluate parent/child on minibatch
-            parent_score = await self.scoring_adapter.a_minibatch_score(
+            t0 = time.perf_counter()
+            parent_score = await self.scorer.a_score_minibatch(
                 parent_prompt_configuration, minibatch
             )
-            child_score = await self.scoring_adapter.a_minibatch_score(
+            print(
+                f"[DEBUG] Score parent on minibatch: {time.perf_counter() - t0:.2f}s (score={parent_score:.4f})"
+            )
+
+            t0 = time.perf_counter()
+            child_score = await self.scorer.a_score_minibatch(
                 child_prompt_configuration, minibatch
             )
+            print(
+                f"[DEBUG] Score child on minibatch: {time.perf_counter() - t0:.2f}s (score={child_score:.4f})"
+            )
 
             # 8. Acceptance test
-            if self._should_accept_child(parent_score, child_score):
+            accepted = self._should_accept_child(parent_score, child_score)
+            print(
+                f"[DEBUG] Acceptance: {'ACCEPTED' if accepted else 'REJECTED'}"
+            )
+            if accepted:
+                t0 = time.perf_counter()
                 accepted_iterations.append(
                     await self._a_accept_child(
                         selected_module_id,
@@ -301,6 +355,13 @@ class GEPARunner:
                         child_score,
                     )
                 )
+                print(
+                    f"[DEBUG] Accept child (pareto scoring): {time.perf_counter() - t0:.2f}s"
+                )
+
+            print(
+                f"[DEBUG] Total iteration time: {time.perf_counter() - iter_start:.2f}s\n"
+            )
             return True
 
         await self._a_run_loop_iteration(_one_iteration)
@@ -308,7 +369,7 @@ class GEPARunner:
         prompt_config_snapshots = build_prompt_config_snapshots(
             self.prompt_configurations_by_id
         )
-        report = OptimizationResult(
+        report = OptimizationReport(
             optimization_id=self.optimization_id,
             best_id=best.id,
             accepted_iterations=accepted_iterations,
@@ -316,7 +377,7 @@ class GEPARunner:
             parents=self.parents_by_id,
             prompt_configurations=prompt_config_snapshots,
         )
-        return best.prompts[self.SINGLE_MODULE_ID], report.as_dict()
+        return best.prompts[self.SINGLE_MODULE_ID], report
 
     ###################
     # State & helpers #
@@ -332,23 +393,14 @@ class GEPARunner:
         ] = {}
         self.pareto_score_table: ScoreTable = {}
 
-    def _ensure_scoring_adapter(self) -> None:
-        if self.scoring_adapter is None:
+    def _ensure_scorer(self) -> None:
+        if self.scorer is None:
             raise DeepEvalError(
-                "GEPARunner requires a `scoring_adapter`. "
-                "Construct one (for example, DeepEvalScoringAdapter) in "
-                "PromptOptimizer and assign it to `runner.scoring_adapter`."
+                "GEPARunner requires a `scorer`. "
+                "Construct one (for example, Scorer) in "
+                "PromptOptimizer and assign it to `runner.scorer`."
             )
 
-    def _ensure_rewriter(self) -> None:
-        if self._rewriter is not None:
-            return
-
-        # For now, always use the basic PromptRewriter. Additional
-        # variants (e.g. for GEPA Alg. 4 crossover) can be introduced
-        # later
-        self._rewriter = PromptRewriter()
-
     def _prompts_equivalent(
         self, old_prompt: Prompt, new_prompt: Prompt
     ) -> bool:
@@ -413,17 +465,16 @@ class GEPARunner:
             totals,
             self.parents_by_id,
             random_state=self.random_state,
-            tie_tolerance=float(self.config.tie_tolerance),
-            policy=self.config.tie_breaker,
+            tie_tolerance=GEPA_TIE_TOLERANCE,
+            policy=self.tie_breaker,
         )
         if self.status_callback is not None and len(tied) > 1:
             msg = (
                 f"tie on aggregate={max_val:.4f} among {len(tied)} "
                 f"prompt_configurations; using tie_breaker="
-                f"{self.config.tie_breaker.value!r} selected {chosen}. "
-                f"To change, set GEPAConfig.tie_breaker to one of: "
-                f"{[t.value for t in self.config.TieBreaker]} "
-                f"(tie_tolerance={float(self.config.tie_tolerance):g})."
+                f"{self.tie_breaker.value!r} selected {chosen}. "
+                f"To change, set GEPA tie_breaker to one of: "
+                f"{[t.value for t in self.TieBreaker]}."
             )
             self.status_callback(
                 RunnerStatusType.TIE,
@@ -433,9 +484,43 @@ class GEPARunner:
         return self.prompt_configurations_by_id[chosen]
 
     def _pick_prompt_configuration(self) -> PromptConfiguration:
+        # Log Pareto selection details
+        all_candidates = list(self.pareto_score_table.keys())
+        print(f"[DEBUG] Pareto Selection:")
+        print(f"  - Total candidates in pool: {len(all_candidates)}")
+
+        # Show score table
+        print(f"  - Score table (per-instance scores):")
+        for cid, scores in self.pareto_score_table.items():
+            is_root = self.parents_by_id.get(cid) is None
+            label = (
+                "(root)"
+                if is_root
+                else f"(child of {self.parents_by_id.get(cid)[:8]}...)"
+            )
+            mean_score = sum(scores) / len(scores) if scores else 0
+            print(
+                f"      {cid[:8]}... {label}: {[round(s, 3) for s in scores]} (mean={mean_score:.3f})"
+            )
+
+        # Show Pareto frontier
+        frontier = pareto_frontier(all_candidates, self.pareto_score_table)
+        print(f"  - Pareto frontier ({len(frontier)} non-dominated):")
+        for cid in frontier:
+            print(f"      {cid[:8]}...")
+
+        # Show frequency weights
+        freq = frequency_weights(self.pareto_score_table)
+        print(f"  - Frequency weights (how often each wins an instance):")
+        for cid, weight in freq.items():
+            print(f"      {cid[:8]}...: {weight}")
+
+        # Do the selection
         selected_prompt_configuration_id = select_prompt_configuration_pareto(
             self.pareto_score_table, random_state=self.random_state
         )
+        print(f"  - Selected: {selected_prompt_configuration_id[:8]}...\n")
+
         return self.prompt_configurations_by_id[
             selected_prompt_configuration_id
         ]
@@ -443,25 +528,13 @@ class GEPARunner:
     def _draw_minibatch(
         self, d_feedback: Union[List["Golden"], List["ConversationalGolden"]]
     ) -> Union[List["Golden"], List["ConversationalGolden"]]:
-        # Determine effective minibatch size from GEPAConfig, bounded by the
+        # Determine effective minibatch size, bounded by the
         # available feedback set.
         n_feedback = len(d_feedback)
         if n_feedback <= 0:
             return []
 
-        if self.config.minibatch_size is not None:
-            size = self.config.minibatch_size
-        else:
-            # Dynamic sizing from ratio, bounded between min and max.
-            dynamic = max(
-                1, int(round(n_feedback * self.config.minibatch_ratio))
-            )
-            size = max(
-                self.config.minibatch_min_size,
-                min(dynamic, self.config.minibatch_max_size),
-            )
-
-        size = max(1, min(size, n_feedback))
+        size = min(self.minibatch_size, n_feedback)
 
         return [
             d_feedback[self.random_state.randrange(0, n_feedback)]
@@ -479,7 +552,6 @@ class GEPARunner:
         )
 
         new_prompt = await self._rewriter.a_rewrite(
-            model_callback=self.model_callback,
             module_id=selected_module_id,
             old_prompt=old_prompt,
             feedback_text=feedback_text,
@@ -504,7 +576,6 @@ class GEPARunner:
         )
 
         new_prompt = self._rewriter.rewrite(
-            model_callback=self.model_callback,
             module_id=selected_module_id,
             old_prompt=old_prompt,
             feedback_text=feedback_text,
@@ -535,7 +606,7 @@ class GEPARunner:
         self, parent_score: float, child_score: float
     ) -> bool:
         jitter = 1e-6
-        return child_score >= parent_score + max(self.config.min_delta, jitter)
+        return child_score >= parent_score + max(GEPA_MIN_DELTA, jitter)
 
     def _accept_child(
         self,
@@ -548,9 +619,7 @@ class GEPARunner:
     ) -> AcceptedIterationDict:
         self._add_prompt_configuration(child_prompt_configuration)
         self.pareto_score_table[child_prompt_configuration.id] = (
-            self.scoring_adapter.score_on_pareto(
-                child_prompt_configuration, d_pareto
-            )
+            self.scorer.score_pareto(child_prompt_configuration, d_pareto)
         )
 
         return AcceptedIterationDict(
@@ -572,7 +641,7 @@ class GEPARunner:
     ) -> AcceptedIterationDict:
         self._add_prompt_configuration(child_prompt_configuration)
         self.pareto_score_table[child_prompt_configuration.id] = (
-            await self.scoring_adapter.a_score_on_pareto(
+            await self.scorer.a_score_pareto(
                 child_prompt_configuration, d_pareto
             )
         )
@@ -590,13 +659,12 @@ class GEPARunner:
         total_iterations: int,
         iteration: int,
         remaining_iterations: int,
-        elapsed: float,
     ):
         if self.status_callback is not None:
             detail = (
                 f"(iterations={total_iterations}) "
                 f"• iteration {iteration}/{total_iterations} "
-                f"• {elapsed:.2f}s • remaining={remaining_iterations}"
+                f"• remaining={remaining_iterations}"
             )
             self.status_callback(
                 RunnerStatusType.PROGRESS,
@@ -626,52 +694,44 @@ class GEPARunner:
         self,
         gepa_iteration: Callable[[], bool],
     ) -> None:
-        total_iterations = self.config.iterations
+        total_iterations = self.iterations
         remaining_iterations = total_iterations
         iteration = 0
-        self._update_progress(
-            total_iterations, iteration, remaining_iterations, 0
-        )
+        self._update_progress(total_iterations, iteration, remaining_iterations)
         while remaining_iterations > 0:
             iteration += 1
-            start_time = time.perf_counter()
             try:
                 ok = gepa_iteration()
             except Exception as exc:
                 # Report a user facing error event and halt optimization.
                 self._update_error(total_iterations, iteration, exc)
                 break
-            elapsed = time.perf_counter() - start_time
             if not ok:
                 break
             remaining_iterations -= 1
             self._update_progress(
-                total_iterations, iteration, remaining_iterations, elapsed
+                total_iterations, iteration, remaining_iterations
             )
 
     async def _a_run_loop_iteration(
         self,
         a_gepa_iteration: Callable[[], Awaitable[bool]],
     ) -> None:
-        total_iterations = self.config.iterations
+        total_iterations = self.iterations
         remaining_iterations = total_iterations
         iteration = 0
-        self._update_progress(
-            total_iterations, iteration, remaining_iterations, 0
-        )
+        self._update_progress(total_iterations, iteration, remaining_iterations)
         while remaining_iterations > 0:
             iteration += 1
-            start_time = time.perf_counter()
             try:
                 ok = await a_gepa_iteration()
             except Exception as exc:
                 # Report a user facing error event and halt optimization.
                 self._update_error(total_iterations, iteration, exc)
                 break
-            elapsed = time.perf_counter() - start_time
             if not ok:
                 break
             remaining_iterations -= 1
             self._update_progress(
-                total_iterations, iteration, remaining_iterations, elapsed
+                total_iterations, iteration, remaining_iterations
             )

deepeval/optimizer/algorithms/miprov2/__init__.py

@@ -0,0 +1,17 @@
+from .miprov2 import MIPROV2
+from .proposer import InstructionProposer
+from .bootstrapper import (
+    Demo,
+    DemoSet,
+    DemoBootstrapper,
+    render_prompt_with_demos,
+)
+
+__all__ = [
+    "MIPROV2",
+    "InstructionProposer",
+    "Demo",
+    "DemoSet",
+    "DemoBootstrapper",
+    "render_prompt_with_demos",
+]