deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/{optimization/copro/loop.py → optimizer/algorithms/copro/copro.py}

@@ -30,92 +30,119 @@ from typing import (
     Union,
 )
 
+from deepeval.models.base_model import DeepEvalBaseLLM
+
 from deepeval.errors import DeepEvalError
-from deepeval.optimization.aggregates import Aggregator, mean_of_all
-from deepeval.optimization.types import (
+from deepeval.optimizer.utils import Aggregator, mean_of_all
+from deepeval.optimizer.types import (
     AcceptedIterationDict,
     ModuleId,
-    OptimizationResult,
+    OptimizationReport,
     PromptConfiguration,
     PromptConfigurationId,
-    RunnerStatusCallbackProtocol,
+    RunnerStatusCallback,
     RunnerStatusType,
     ScoreTable,
-    ScoringAdapter,
 )
-from deepeval.optimization.utils import (
+from deepeval.optimizer.scorer.base import BaseScorer
+from deepeval.optimizer.utils import (
     build_prompt_config_snapshots,
 )
 from deepeval.prompt.api import PromptType
 from deepeval.prompt.prompt import Prompt
-from deepeval.optimization.mutations.prompt_rewriter import PromptRewriter
-
-from .configs import COPROConfig
+from deepeval.optimizer.rewriter import Rewriter
+from deepeval.optimizer.algorithms.configs import MIPROV2_MIN_DELTA
+from deepeval.optimizer.algorithms.base import BaseAlgorithm
 
 if TYPE_CHECKING:  # pragma: no cover - type-checking only
     from deepeval.dataset.golden import ConversationalGolden, Golden
 
 
-class COPRORunner:
+class COPRO(BaseAlgorithm):
     """
     COPRO style cooperative prompt optimization loop with sync/async execution.
 
     This runner is intentionally low level and does not know about metrics,
-    models, or async configs. It relies on a preconfigured ScoringAdapter and
-    PromptRewriter, which are typically constructed by PromptOptimizer.
-
-    - Optimizes a single Prompt (instruction) against a list of Goldens.
-    - Uses mini-batches of goldens for trial scoring and epsilon-greedy
-      selection over prompt candidates based on mean minibatch scores,
-      extended with cooperative proposals:
-        - At each iteration, a parent candidate is selected.
-        - A shared feedback string is computed on a minibatch.
-        - Multiple child prompts are proposed from that parent using the
-          same feedback but different LLM samples.
-        - Any child whose minibatch score improves over the parent by at
-          least ``min_delta`` is added to the candidate pool.
+    models, or async configs. It relies on a preconfigured Scorer and
+    Rewriter, which are typically constructed by PromptOptimizer.
+
+    Parameters
+    ----------
+    iterations : int
+        Total number of optimization trials. Default is 5.
+    minibatch_size : int
+        Number of examples drawn per iteration. Default is 8.
+    random_seed : int, optional
+        RNG seed for reproducibility. If None, derived from time.time_ns().
+    exploration_probability : float
+        Epsilon greedy exploration rate. Default is 0.2.
+    full_eval_every : int, optional
+        Fully evaluate best candidate every N trials. Default is 5.
+    population_size : int
+        Maximum number of candidates in the pool. Default is 4.
+    proposals_per_step : int
+        Number of child prompts proposed per iteration. Default is 4.
     """
 
+    name = "COPRO"
     SINGLE_MODULE_ID: ModuleId = "__module__"
 
     def __init__(
         self,
-        *,
-        config: COPROConfig,
+        iterations: int = 5,
+        minibatch_size: int = 8,
+        random_seed: Optional[int] = None,
+        exploration_probability: float = 0.2,
+        full_eval_every: Optional[int] = 5,
+        population_size: int = 4,
+        proposals_per_step: int = 4,
         aggregate_instances: Aggregator = mean_of_all,
-        scoring_adapter: Optional[ScoringAdapter] = None,
+        scorer: Optional[BaseScorer] = None,
     ) -> None:
-        self.config = config
+        # Validate parameters
+        if iterations < 1:
+            raise ValueError("iterations must be >= 1")
+        if minibatch_size < 1:
+            raise ValueError("minibatch_size must be >= 1")
+        if exploration_probability < 0.0 or exploration_probability > 1.0:
+            raise ValueError(
+                "exploration_probability must be >= 0.0 and <= 1.0"
+            )
+        if full_eval_every is not None and full_eval_every < 1:
+            raise ValueError("full_eval_every must be >= 1")
+        if population_size < 1:
+            raise ValueError("population_size must be >= 1")
+        if proposals_per_step < 1:
+            raise ValueError("proposals_per_step must be >= 1")
+
+        self.iterations = iterations
+        self.minibatch_size = minibatch_size
+        self.exploration_probability = exploration_probability
+        self.full_eval_every = full_eval_every
+        self.population_size = population_size
+        self.proposals_per_step = proposals_per_step
         self.aggregate_instances = aggregate_instances
-        self.scoring_adapter = scoring_adapter
-
-        # Random seeded from config is used for minibatch sampling and
-        # epsilon-greedy candidate selection.
-        self.random_state = random.Random(config.random_seed)
+        self.scorer = scorer
 
-        self.random_state = random.Random(config.random_seed)
+        # If no seed provided, use time-based seed
+        if random_seed is None:
+            random_seed = time.time_ns()
+        self.random_seed = random_seed
+        self.random_state = random.Random(random_seed)
 
         # Runtime state to be reset between runs
         self.reset_state()
 
         # Status callback set by PromptOptimizer:
         #   (kind, step_index, total_steps, detail) -> None
-        self.status_callback: Optional[RunnerStatusCallbackProtocol] = None
-
-        # Model callback used by the rewriter set by PromptOptimizer.
-        self.model_callback: Optional[
-            Callable[
-                ...,
-                Union[
-                    str,
-                    Dict,
-                    Tuple[Union[str, Dict], float],
-                ],
-            ]
-        ] = None
-
-        # Lazy-loaded PromptRewriter set by PromptOptimizer
-        self._rewriter: Optional[PromptRewriter] = None
+        self.status_callback: Optional[RunnerStatusCallback] = None
+
+        # Optimizer model used by the rewriter for prompt mutation.
+        # Set by PromptOptimizer.
+        self.optimizer_model: Optional["DeepEvalBaseLLM"] = None
+
+        # Lazy-loaded Rewriter set by PromptOptimizer
+        self._rewriter: Optional[Rewriter] = None
 
     ##############
     # Public API #
@@ -123,10 +150,9 @@ class COPRORunner:
 
     def execute(
         self,
-        *,
         prompt: Prompt,
         goldens: Union[List["Golden"], List["ConversationalGolden"]],
-    ) -> Tuple[Prompt, Dict]:
+    ) -> Tuple[Prompt, OptimizationReport]:
         """
         Synchronous COPRO run from a full list of goldens.
 
@@ -141,8 +167,7 @@ class COPRORunner:
                 "the optimizer."
             )
 
-        self._ensure_scoring_adapter()
-        self._ensure_rewriter()
+        self._ensure_scorer()
         self.reset_state()
 
         # Seed candidate pool with the root prompt configuration.
@@ -168,7 +193,7 @@ class COPRORunner:
             # candidate on the first iteration.
             if not self._minibatch_score_counts:
                 seed_minibatch = self._draw_minibatch(goldens)
-                root_score = self.scoring_adapter.minibatch_score(
+                root_score = self.scorer.score_minibatch(
                     root_prompt_configuration, seed_minibatch
                 )
                 self._record_minibatch_score(
@@ -183,7 +208,7 @@ class COPRORunner:
 
             # Compute shared feedback for this parent/minibatch that will be
             # used by all cooperative child proposals.
-            feedback_text = self.scoring_adapter.minibatch_feedback(
+            feedback_text = self.scorer.get_minibatch_feedback(
                 parent_prompt_configuration, selected_module_id, minibatch
             )
 
@@ -191,10 +216,10 @@ class COPRORunner:
                 parent_prompt_configuration.id
             )
             jitter = 1e-6
-            min_delta = max(self.config.min_delta, jitter)
+            min_delta = max(MIPROV2_MIN_DELTA, jitter)
 
             # 2. Generate multiple cooperative child prompts and evaluate them.
-            num_proposals = int(self.config.proposals_per_step)
+            num_proposals = int(self.proposals_per_step)
             for _ in range(num_proposals):
                 child_prompt = self._generate_child_prompt(
                     selected_module_id,
@@ -211,7 +236,7 @@ class COPRORunner:
                     child_prompt,
                 )
 
-                child_score = self.scoring_adapter.minibatch_score(
+                child_score = self.scorer.score_minibatch(
                     child_prompt_configuration, minibatch
                 )
 
@@ -236,8 +261,8 @@ class COPRORunner:
 
             self.trial_index += 1
             if (
-                self.config.full_eval_every is not None
-                and self.trial_index % self.config.full_eval_every == 0
+                self.full_eval_every is not None
+                and self.trial_index % self.full_eval_every == 0
             ):
                 self._full_evaluate_best(goldens)
 
@@ -253,7 +278,7 @@ class COPRORunner:
         prompt_config_snapshots = build_prompt_config_snapshots(
             self.prompt_configurations_by_id
         )
-        report = OptimizationResult(
+        report = OptimizationReport(
             optimization_id=self.optimization_id,
             best_id=best.id,
             accepted_iterations=accepted_iterations,
@@ -261,14 +286,13 @@ class COPRORunner:
             parents=self.parents_by_id,
             prompt_configurations=prompt_config_snapshots,
         )
-        return best.prompts[self.SINGLE_MODULE_ID], report.as_dict()
+        return best.prompts[self.SINGLE_MODULE_ID], report
 
     async def a_execute(
         self,
-        *,
         prompt: Prompt,
         goldens: Union[List["Golden"], List["ConversationalGolden"]],
-    ) -> Tuple[Prompt, Dict]:
+    ) -> Tuple[Prompt, OptimizationReport]:
         """
         Asynchronous twin of execute().
         """
@@ -280,8 +304,7 @@ class COPRORunner:
                 "the optimizer."
             )
 
-        self._ensure_scoring_adapter()
-        self._ensure_rewriter()
+        self._ensure_scorer()
         self.reset_state()
 
         seed_prompts_by_module = {self.SINGLE_MODULE_ID: prompt}
@@ -306,7 +329,7 @@ class COPRORunner:
             # candidate on the first iteration.
             if not self._minibatch_score_counts:
                 seed_minibatch = self._draw_minibatch(goldens)
-                root_score = await self.scoring_adapter.a_minibatch_score(
+                root_score = await self.scorer.a_score_minibatch(
                     root_prompt_configuration, seed_minibatch
                 )
                 self._record_minibatch_score(
@@ -318,7 +341,7 @@ class COPRORunner:
 
             minibatch = self._draw_minibatch(goldens)
 
-            feedback_text = await self.scoring_adapter.a_minibatch_feedback(
+            feedback_text = await self.scorer.a_get_minibatch_feedback(
                 parent_prompt_configuration, selected_module_id, minibatch
             )
 
@@ -326,9 +349,9 @@ class COPRORunner:
                 parent_prompt_configuration.id
             )
             jitter = 1e-6
-            min_delta = max(self.config.min_delta, jitter)
+            min_delta = max(MIPROV2_MIN_DELTA, jitter)
 
-            num_proposals = int(self.config.proposals_per_step)
+            num_proposals = int(self.proposals_per_step)
             for _ in range(num_proposals):
                 child_prompt = await self._a_generate_child_prompt(
                     selected_module_id,
@@ -344,7 +367,7 @@ class COPRORunner:
                     child_prompt,
                 )
 
-                child_score = await self.scoring_adapter.a_minibatch_score(
+                child_score = await self.scorer.a_score_minibatch(
                     child_prompt_configuration, minibatch
                 )
 
@@ -366,8 +389,8 @@ class COPRORunner:
 
             self.trial_index += 1
             if (
-                self.config.full_eval_every is not None
-                and self.trial_index % self.config.full_eval_every == 0
+                self.full_eval_every is not None
+                and self.trial_index % self.full_eval_every == 0
             ):
                 await self._a_full_evaluate_best(goldens)
 
@@ -382,7 +405,7 @@ class COPRORunner:
         prompt_config_snapshots = build_prompt_config_snapshots(
             self.prompt_configurations_by_id
         )
-        report = OptimizationResult(
+        report = OptimizationReport(
             optimization_id=self.optimization_id,
             best_id=best.id,
             accepted_iterations=accepted_iterations,
@@ -390,7 +413,7 @@ class COPRORunner:
             parents=self.parents_by_id,
             prompt_configurations=prompt_config_snapshots,
         )
-        return best.prompts[self.SINGLE_MODULE_ID], report.as_dict()
+        return best.prompts[self.SINGLE_MODULE_ID], report
 
     ###################
     # State & helpers #
@@ -414,25 +437,14 @@ class COPRORunner:
         # Trial counter (used for full_eval_every).
         self.trial_index: int = 0
 
-    def _ensure_scoring_adapter(self) -> None:
-        if self.scoring_adapter is None:
+    def _ensure_scorer(self) -> None:
+        if self.scorer is None:
             raise DeepEvalError(
-                "COPRORunner requires a `scoring_adapter`. "
-                "Construct one (for example, DeepEvalScoringAdapter) in "
-                "PromptOptimizer and assign it to `runner.scoring_adapter`."
+                "COPRORunner requires a `scorer`. "
+                "Construct one (for example, Scorer) in "
+                "PromptOptimizer and assign it to `runner.scorer`."
             )
 
-    def _ensure_rewriter(self) -> None:
-        if self._rewriter is not None:
-            return
-
-        # Default basic PromptRewriter; PromptOptimizer can override this and
-        # pass a configured instance (e.g. with list-mutation config).
-        self._rewriter = PromptRewriter(
-            max_chars=self.config.rewrite_instruction_max_chars,
-            random_state=self.random_state,
-        )
-
     def _prompts_equivalent(
         self,
         old_prompt: Prompt,
@@ -484,9 +496,7 @@ class COPRORunner:
 
         # If we exceed the population size, iteratively prune the worst
         # (by mean minibatch score), never removing the current best.
-        while (
-            len(self.prompt_configurations_by_id) > self.config.population_size
-        ):
+        while len(self.prompt_configurations_by_id) > self.population_size:
             best_id: Optional[PromptConfigurationId] = None
             best_score = float("-inf")
             for cand_id in self.prompt_configurations_by_id.keys():
@@ -611,7 +621,7 @@ class COPRORunner:
                 "COPRORunner has an empty candidate pool; this should not happen."
             )
 
-        eps = float(self.config.exploration_probability)
+        eps = float(self.exploration_probability)
         if eps > 0.0 and self.random_state.random() < eps:
             chosen_id = self.random_state.choice(candidate_ids)
         else:
@@ -624,23 +634,14 @@ class COPRORunner:
         goldens: Union[List["Golden"], List["ConversationalGolden"]],
     ) -> Union[List["Golden"], List["ConversationalGolden"]]:
         """
-        Determine effective minibatch size from COPROConfig, bounded by the
-        available goldens, and sample with replacement.
+        Determine effective minibatch size, bounded by the available goldens,
+        and sample with replacement.
         """
         n = len(goldens)
         if n <= 0:
             return []
 
-        if self.config.minibatch_size is not None:
-            size = self.config.minibatch_size
-        else:
-            dynamic = max(1, int(round(n * self.config.minibatch_ratio)))
-            size = max(
-                self.config.minibatch_min_size,
-                min(dynamic, self.config.minibatch_max_size),
-            )
-
-        size = max(1, min(size, n))
+        size = min(self.minibatch_size, n)
 
         return [goldens[self.random_state.randrange(0, n)] for _ in range(size)]
 
@@ -655,7 +656,7 @@ class COPRORunner:
         if best.id in self.pareto_score_table:
             return
 
-        scores = await self.scoring_adapter.a_score_on_pareto(best, goldens)
+        scores = await self.scorer.a_score_pareto(best, goldens)
         self.pareto_score_table[best.id] = scores
 
     def _full_evaluate_best(
@@ -669,7 +670,7 @@ class COPRORunner:
         if best.id in self.pareto_score_table:
             return
 
-        scores = self.scoring_adapter.score_on_pareto(best, goldens)
+        scores = self.scorer.score_pareto(best, goldens)
         self.pareto_score_table[best.id] = scores
 
     async def _a_generate_child_prompt(
@@ -688,7 +689,6 @@ class COPRORunner:
             ) from exc
 
         new_prompt = await self._rewriter.a_rewrite(
-            model_callback=self.model_callback,
             module_id=selected_module_id,
             old_prompt=old_prompt,
             feedback_text=feedback_text,
@@ -718,7 +718,6 @@ class COPRORunner:
             ) from exc
 
         new_prompt = self._rewriter.rewrite(
-            model_callback=self.model_callback,
             module_id=selected_module_id,
             old_prompt=old_prompt,
             feedback_text=feedback_text,
@@ -788,7 +787,7 @@ class COPRORunner:
         self,
         copro_iteration: Callable[[], bool],
     ) -> None:
-        total_iterations = self.config.iterations
+        total_iterations = self.iterations
         remaining_iterations = total_iterations
         iteration = 0
         self._update_progress(
@@ -814,7 +813,7 @@ class COPRORunner:
         self,
         a_copro_iteration: Callable[[], Awaitable[bool]],
     ) -> None:
-        total_iterations = self.config.iterations
+        total_iterations = self.iterations
         remaining_iterations = total_iterations
         iteration = 0
         self._update_progress(

deepeval/optimizer/algorithms/gepa/__init__.py

@@ -0,0 +1,5 @@
+from .gepa import GEPA
+
+__all__ = [
+    "GEPA",
+]