deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/{optimization/simba/loop.py → optimizer/algorithms/simba/simba.py}

@@ -30,59 +30,109 @@ from typing import (
     Union,
 )
 
+from deepeval.models.base_model import DeepEvalBaseLLM
+
 from deepeval.errors import DeepEvalError
 from deepeval.dataset.golden import ConversationalGolden, Golden
-from deepeval.optimization.aggregates import Aggregator, mean_of_all
-from deepeval.optimization.types import (
+from deepeval.optimizer.utils import Aggregator, mean_of_all
+from deepeval.optimizer.types import (
     AcceptedIterationDict,
     ModuleId,
-    OptimizationResult,
+    OptimizationReport,
     PromptConfiguration,
     PromptConfigurationId,
-    RunnerStatusCallbackProtocol,
+    RunnerStatusCallback,
     RunnerStatusType,
     ScoreTable,
-    ScoringAdapter,
 )
-from deepeval.optimization.utils import build_prompt_config_snapshots
+from deepeval.optimizer.scorer.base import BaseScorer
+from deepeval.optimizer.algorithms.base import BaseAlgorithm
+from deepeval.optimizer.utils import build_prompt_config_snapshots
 from deepeval.prompt.api import PromptType
 from deepeval.prompt.prompt import Prompt
-from deepeval.optimization.mutations.prompt_rewriter import PromptRewriter
+from deepeval.optimizer.rewriter import Rewriter
 
-from .configs import SIMBAConfig
-from .types import SIMBAStrategy
+from deepeval.optimizer.algorithms.configs import (
+    MIPROV2_MIN_DELTA,
+    MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS,
+    SIMBA_DEMO_INPUT_MAX_CHARS,
+)
+from deepeval.optimizer.algorithms.simba.types import SIMBAStrategy
 
 
-class SIMBARunner:
+class SIMBA(BaseAlgorithm):
     """
     SIMBA-style cooperative prompt optimization loop with sync/async execution.
 
     This runner is intentionally low level and does not know about metrics,
-    models, or async configs. It relies on a preconfigured ScoringAdapter and
-    PromptRewriter, which are typically constructed by PromptOptimizer.
-
-    - Optimizes a single Prompt (instruction) against a list of Goldens.
-    - Uses mini-batches of goldens for trial scoring and epsilon-greedy
-      selection over prompt candidates based on mean minibatch scores.
-    - At each iteration, proposes multiple child prompts using SIMBA-style
-      edit strategies (APPEND_DEMO and APPEND_RULE) by passing different
-      instructions into the PromptRewriter.
+    models, or async configs. It relies on a preconfigured Scorer and
+    Rewriter, which are typically constructed by PromptOptimizer.
+
+    Parameters
+    ----------
+    iterations : int
+        Total number of optimization trials. Default is 5.
+    minibatch_size : int
+        Number of examples drawn per iteration. Default is 8.
+    random_seed : int, optional
+        RNG seed for reproducibility. If None, derived from time.time_ns().
+    exploration_probability : float
+        Epsilon greedy exploration rate. Default is 0.2.
+    full_eval_every : int, optional
+        Fully evaluate best candidate every N trials. Default is 5.
+    population_size : int
+        Maximum number of candidates in the pool. Default is 4.
+    proposals_per_step : int
+        Number of child prompts proposed per iteration. Default is 4.
+    max_demos_per_proposal : int
+        Maximum demos from minibatch for APPEND_DEMO strategy. Default is 3.
     """
 
+    name = "SIMBA"
     SINGLE_MODULE_ID: ModuleId = "__module__"
 
     def __init__(
         self,
-        *,
-        config: SIMBAConfig,
+        iterations: int = 5,
+        minibatch_size: int = 8,
+        random_seed: Optional[int] = None,
+        exploration_probability: float = 0.2,
+        full_eval_every: Optional[int] = 5,
+        population_size: int = 4,
+        proposals_per_step: int = 4,
+        max_demos_per_proposal: int = 3,
         aggregate_instances: Aggregator = mean_of_all,
-        scoring_adapter: Optional[ScoringAdapter] = None,
+        scorer: Optional[BaseScorer] = None,
     ) -> None:
-        self.config = config
+        # Validate parameters
+        if iterations < 1:
+            raise ValueError("iterations must be >= 1")
+        if minibatch_size < 1:
+            raise ValueError("minibatch_size must be >= 1")
+        if exploration_probability < 0.0 or exploration_probability > 1.0:
+            raise ValueError(
+                "exploration_probability must be >= 0.0 and <= 1.0"
+            )
+        if full_eval_every is not None and full_eval_every < 1:
+            raise ValueError("full_eval_every must be >= 1")
+        if population_size < 1:
+            raise ValueError("population_size must be >= 1")
+        if proposals_per_step < 1:
+            raise ValueError("proposals_per_step must be >= 1")
+        if max_demos_per_proposal < 0:
+            raise ValueError("max_demos_per_proposal must be >= 0")
+
+        self.iterations = iterations
+        self.minibatch_size = minibatch_size
+        self.exploration_probability = exploration_probability
+        self.full_eval_every = full_eval_every
+        self.population_size = population_size
+        self.proposals_per_step = proposals_per_step
+        self.max_demos_per_proposal = max_demos_per_proposal
         self.aggregate_instances = aggregate_instances
-        self.scoring_adapter = scoring_adapter
+        self.scorer = scorer
 
-        if config.max_demos_per_proposal > 0:
+        if max_demos_per_proposal > 0:
             self._strategies = [
                 SIMBAStrategy.APPEND_DEMO,
                 SIMBAStrategy.APPEND_RULE,
@@ -90,31 +140,25 @@ class SIMBARunner:
         else:
             self._strategies = [SIMBAStrategy.APPEND_RULE]
 
-        # Random seeded from config is used for minibatch sampling, strategy
-        # selection, and epsilon-greedy candidate selection.
-        self.random_state = random.Random(config.random_seed)
+        # If no seed provided, use time-based seed
+        if random_seed is None:
+            random_seed = time.time_ns()
+        self.random_seed = random_seed
+        self.random_state = random.Random(random_seed)
 
         # Runtime state to be reset between runs
         self.reset_state()
 
         # Status callback set by PromptOptimizer:
         #   (kind, step_index, total_steps, detail) -> None
-        self.status_callback: Optional[RunnerStatusCallbackProtocol] = None
-
-        # Model callback used by the rewriter set by PromptOptimizer.
-        self.model_callback: Optional[
-            Callable[
-                ...,
-                Union[
-                    str,
-                    Dict,
-                    Tuple[Union[str, Dict], float],
-                ],
-            ]
-        ] = None
+        self.status_callback: Optional[RunnerStatusCallback] = None
 
-        # Lazy-loaded PromptRewriter set by PromptOptimizer
-        self._rewriter: Optional[PromptRewriter] = None
+        # Optimizer model used by the rewriter for prompt mutation.
+        # Set by PromptOptimizer.
+        self.optimizer_model: Optional["DeepEvalBaseLLM"] = None
+
+        # Lazy-loaded Rewriter set by PromptOptimizer
+        self._rewriter: Optional[Rewriter] = None
 
     ##############
     # Public API #
@@ -122,10 +166,9 @@ class SIMBARunner:
 
     def execute(
         self,
-        *,
         prompt: Prompt,
         goldens: Union[List[Golden], List[ConversationalGolden]],
-    ) -> Tuple[Prompt, Dict]:
+    ) -> Tuple[Prompt, OptimizationReport]:
         """
         Synchronous SIMBA run from a full list of goldens.
 
@@ -140,8 +183,7 @@ class SIMBARunner:
                 "the optimizer."
             )
 
-        self._ensure_scoring_adapter()
-        self._ensure_rewriter()
+        self._ensure_scorer()
         self.reset_state()
 
         # Seed candidate pool with the root prompt configuration.
@@ -167,7 +209,7 @@ class SIMBARunner:
             # candidate on the first iteration.
             if not self._minibatch_score_counts:
                 seed_minibatch = self._draw_minibatch(goldens)
-                root_score = self.scoring_adapter.minibatch_score(
+                root_score = self.scorer.score_minibatch(
                     root_prompt_configuration, seed_minibatch
                 )
                 self._record_minibatch_score(
@@ -182,7 +224,7 @@ class SIMBARunner:
 
             # Compute shared feedback for this parent/minibatch that will be
             # used by all SIMBA proposals in this iteration.
-            feedback_text = self.scoring_adapter.minibatch_feedback(
+            feedback_text = self.scorer.get_minibatch_feedback(
                 parent_prompt_configuration, selected_module_id, minibatch
             )
 
@@ -190,10 +232,10 @@ class SIMBARunner:
                 parent_prompt_configuration.id
             )
             jitter = 1e-6
-            min_delta = max(self.config.min_delta, jitter)
+            min_delta = max(MIPROV2_MIN_DELTA, jitter)
 
             # 2. Generate multiple SIMBA child prompts and evaluate them.
-            num_proposals = int(self.config.proposals_per_step)
+            num_proposals = int(self.proposals_per_step)
             for _ in range(num_proposals):
                 strategy = self._sample_strategy()
                 child_prompt = self._generate_child_prompt(
@@ -213,7 +255,7 @@ class SIMBARunner:
                     child_prompt,
                 )
 
-                child_score = self.scoring_adapter.minibatch_score(
+                child_score = self.scorer.score_minibatch(
                     child_prompt_configuration, minibatch
                 )
 
@@ -238,8 +280,8 @@ class SIMBARunner:
 
             self.trial_index += 1
             if (
-                self.config.full_eval_every is not None
-                and self.trial_index % self.config.full_eval_every == 0
+                self.full_eval_every is not None
+                and self.trial_index % self.full_eval_every == 0
             ):
                 self._full_evaluate_best(goldens)
 
@@ -255,7 +297,7 @@ class SIMBARunner:
         prompt_config_snapshots = build_prompt_config_snapshots(
             self.prompt_configurations_by_id
         )
-        report = OptimizationResult(
+        report = OptimizationReport(
             optimization_id=self.optimization_id,
             best_id=best.id,
             accepted_iterations=accepted_iterations,
@@ -263,14 +305,13 @@ class SIMBARunner:
             parents=self.parents_by_id,
             prompt_configurations=prompt_config_snapshots,
         )
-        return best.prompts[self.SINGLE_MODULE_ID], report.as_dict()
+        return best.prompts[self.SINGLE_MODULE_ID], report
 
     async def a_execute(
         self,
-        *,
         prompt: Prompt,
         goldens: Union[List[Golden], List[ConversationalGolden]],
-    ) -> Tuple[Prompt, Dict]:
+    ) -> Tuple[Prompt, OptimizationReport]:
         """
         Asynchronous twin of execute().
         """
@@ -282,8 +323,7 @@ class SIMBARunner:
                 "the optimizer."
             )
 
-        self._ensure_scoring_adapter()
-        self._ensure_rewriter()
+        self._ensure_scorer()
         self.reset_state()
 
         seed_prompts_by_module = {self.SINGLE_MODULE_ID: prompt}
@@ -303,7 +343,7 @@ class SIMBARunner:
 
             if not self._minibatch_score_counts:
                 seed_minibatch = self._draw_minibatch(goldens)
-                root_score = await self.scoring_adapter.a_minibatch_score(
+                root_score = await self.scorer.a_score_minibatch(
                     root_prompt_configuration, seed_minibatch
                 )
                 self._record_minibatch_score(
@@ -315,7 +355,7 @@ class SIMBARunner:
 
             minibatch = self._draw_minibatch(goldens)
 
-            feedback_text = await self.scoring_adapter.a_minibatch_feedback(
+            feedback_text = await self.scorer.a_get_minibatch_feedback(
                 parent_prompt_configuration, selected_module_id, minibatch
             )
 
@@ -323,9 +363,9 @@ class SIMBARunner:
                 parent_prompt_configuration.id
             )
             jitter = 1e-6
-            min_delta = max(self.config.min_delta, jitter)
+            min_delta = max(MIPROV2_MIN_DELTA, jitter)
 
-            num_proposals = int(self.config.proposals_per_step)
+            num_proposals = int(self.proposals_per_step)
             for _ in range(num_proposals):
                 strategy = self._sample_strategy()
                 child_prompt = await self._a_generate_child_prompt(
@@ -344,7 +384,7 @@ class SIMBARunner:
                     child_prompt,
                 )
 
-                child_score = await self.scoring_adapter.a_minibatch_score(
+                child_score = await self.scorer.a_score_minibatch(
                     child_prompt_configuration, minibatch
                 )
 
@@ -366,8 +406,8 @@ class SIMBARunner:
 
             self.trial_index += 1
             if (
-                self.config.full_eval_every is not None
-                and self.trial_index % self.config.full_eval_every == 0
+                self.full_eval_every is not None
+                and self.trial_index % self.full_eval_every == 0
             ):
                 await self._a_full_evaluate_best(goldens)
 
@@ -382,7 +422,7 @@ class SIMBARunner:
         prompt_config_snapshots = build_prompt_config_snapshots(
             self.prompt_configurations_by_id
         )
-        report = OptimizationResult(
+        report = OptimizationReport(
             optimization_id=self.optimization_id,
             best_id=best.id,
             accepted_iterations=accepted_iterations,
@@ -390,7 +430,7 @@ class SIMBARunner:
             parents=self.parents_by_id,
             prompt_configurations=prompt_config_snapshots,
         )
-        return best.prompts[self.SINGLE_MODULE_ID], report.as_dict()
+        return best.prompts[self.SINGLE_MODULE_ID], report
 
     ###################
     # State & helpers #
@@ -414,25 +454,14 @@ class SIMBARunner:
         # Trial counter (used for full_eval_every).
         self.trial_index: int = 0
 
-    def _ensure_scoring_adapter(self) -> None:
-        if self.scoring_adapter is None:
+    def _ensure_scorer(self) -> None:
+        if self.scorer is None:
             raise DeepEvalError(
-                "SIMBARunner requires a `scoring_adapter`. "
-                "Construct one (for example, DeepEvalScoringAdapter) in "
-                "PromptOptimizer and assign it to `runner.scoring_adapter`."
+                "SIMBARunner requires a `scorer`. "
+                "Construct one (for example, Scorer) in "
+                "PromptOptimizer and assign it to `runner.scorer`."
             )
 
-    def _ensure_rewriter(self) -> None:
-        if self._rewriter is not None:
-            return
-
-        # Default basic PromptRewriter; PromptOptimizer can override this and
-        # pass a configured instance (e.g. with list-mutation config).
-        self._rewriter = PromptRewriter(
-            max_chars=self.config.rewrite_instruction_max_chars,
-            random_state=self.random_state,
-        )
-
     def _prompts_equivalent(
         self,
         old_prompt: Prompt,
@@ -484,9 +513,7 @@ class SIMBARunner:
 
         # If we exceed the population size, iteratively prune the worst
         # (by mean minibatch score), never removing the current best.
-        while (
-            len(self.prompt_configurations_by_id) > self.config.population_size
-        ):
+        while len(self.prompt_configurations_by_id) > self.population_size:
             best_id: Optional[PromptConfigurationId] = None
             best_score = float("-inf")
             for cand_id in self.prompt_configurations_by_id.keys():
@@ -611,7 +638,7 @@ class SIMBARunner:
                 "SIMBARunner has an empty candidate pool; this should not happen."
             )
 
-        eps = float(self.config.exploration_probability)
+        eps = float(self.exploration_probability)
         if eps > 0.0 and self.random_state.random() < eps:
             chosen_id = self.random_state.choice(candidate_ids)
         else:
@@ -624,23 +651,14 @@ class SIMBARunner:
         goldens: Union[List[Golden], List[ConversationalGolden]],
     ) -> Union[List[Golden], List[ConversationalGolden]]:
         """
-        Determine effective minibatch size from SIMBAConfig, bounded by the
-        available goldens, and sample with replacement.
+        Determine effective minibatch size, bounded by the available goldens,
+        and sample with replacement.
         """
         n = len(goldens)
         if n <= 0:
             return []
 
-        if self.config.minibatch_size is not None:
-            size = self.config.minibatch_size
-        else:
-            dynamic = max(1, int(round(n * self.config.minibatch_ratio)))
-            size = max(
-                self.config.minibatch_min_size,
-                min(dynamic, self.config.minibatch_max_size),
-            )
-
-        size = max(1, min(size, n))
+        size = min(self.minibatch_size, n)
 
         return [goldens[self.random_state.randrange(0, n)] for _ in range(size)]
 
@@ -655,7 +673,7 @@ class SIMBARunner:
         if best.id in self.pareto_score_table:
             return
 
-        scores = await self.scoring_adapter.a_score_on_pareto(best, goldens)
+        scores = await self.scorer.a_score_pareto(best, goldens)
         self.pareto_score_table[best.id] = scores
 
     def _full_evaluate_best(
@@ -669,7 +687,7 @@ class SIMBARunner:
         if best.id in self.pareto_score_table:
             return
 
-        scores = self.scoring_adapter.score_on_pareto(best, goldens)
+        scores = self.scorer.score_pareto(best, goldens)
         self.pareto_score_table[best.id] = scores
 
     async def _a_generate_child_prompt(
@@ -694,7 +712,6 @@ class SIMBARunner:
         )
 
         new_prompt = await self._rewriter.a_rewrite(
-            model_callback=self.model_callback,
             module_id=selected_module_id,
             old_prompt=old_prompt,
             feedback_text=strategy_feedback,
@@ -730,7 +747,6 @@ class SIMBARunner:
         )
 
         new_prompt = self._rewriter.rewrite(
-            model_callback=self.model_callback,
             module_id=selected_module_id,
             old_prompt=old_prompt,
             feedback_text=strategy_feedback,
@@ -761,7 +777,7 @@ class SIMBARunner:
         Truncate strategy instructions + feedback to the configured character
         budget so the rewriter prompt does not explode.
         """
-        max_chars = self.config.rewrite_instruction_max_chars
+        max_chars = MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS
         if max_chars <= 0:
             return text
         if len(text) <= max_chars:
@@ -788,15 +804,15 @@ class SIMBARunner:
                 Context <- " ".join(golden.context) if present
                 Output  <- golden.expected_outcome
 
-        All text segments are independently truncated to `demo_input_max_chars`.
+        All text segments are independently truncated to `SIMBA_DEMO_INPUT_MAX_CHARS`.
         """
-        max_demos = self.config.max_demos_per_proposal
+        max_demos = self.max_demos_per_proposal
         if max_demos <= 0:
             return ""
 
         lines: List[str] = []
         demo_limit = min(max_demos, len(minibatch))
-        max_chars = self.config.demo_input_max_chars
+        max_chars = SIMBA_DEMO_INPUT_MAX_CHARS
 
         for golden in minibatch[:demo_limit]:
             if isinstance(golden, Golden):
@@ -843,7 +859,7 @@ class SIMBARunner:
     ) -> str:
         """
         Construct a strategy-specific feedback string that is passed into
-        PromptRewriter.rewrite / a_rewrite.
+        Rewriter.rewrite / a_rewrite.
 
         - APPEND_RULE: emphasize extracting a concise rule from metric feedback.
         - APPEND_DEMO: emphasize appending concrete demos built from goldens.
@@ -934,7 +950,7 @@ class SIMBARunner:
         self,
         simba_iteration: Callable[[], bool],
     ) -> None:
-        total_iterations = self.config.iterations
+        total_iterations = self.iterations
         remaining_iterations = total_iterations
         iteration = 0
         self._update_progress(
@@ -960,7 +976,7 @@ class SIMBARunner:
         self,
         a_simba_iteration: Callable[[], Awaitable[bool]],
     ) -> None:
-        total_iterations = self.config.iterations
+        total_iterations = self.iterations
         remaining_iterations = total_iterations
         iteration = 0
         self._update_progress(

deepeval/{optimization → optimizer}/configs.py

@@ -2,27 +2,24 @@ from __future__ import annotations
 from enum import Enum
 from pydantic import BaseModel, Field, conint
 from typing import Optional
+from deepeval.evaluate.configs import AsyncConfig
 
 
-class OptimizerDisplayConfig(BaseModel):
-    """Display controls used by PromptOptimizer for all algorithms."""
-
+class DisplayConfig(BaseModel):
     show_indicator: bool = True
     announce_ties: bool = Field(
         False, description="Print a one-line note when a tie is detected"
     )
 
 
-class PromptListMutationTargetType(Enum):
+class MutationTargetType(Enum):
     RANDOM = "random"
     FIXED_INDEX = "fixed_index"
 
 
 # default all messages
-class PromptListMutationConfig(BaseModel):
-    target_type: PromptListMutationTargetType = (
-        PromptListMutationTargetType.RANDOM
-    )
+class MutationConfig(BaseModel):
+    target_type: MutationTargetType = MutationTargetType.RANDOM
     # should be list
     target_role: Optional[str] = Field(
         default=None,

deepeval/{optimization/policies/selection.py → optimizer/policies.py}

@@ -1,9 +1,10 @@
 from __future__ import annotations
-from typing import Dict, List, Sequence
+from enum import Enum
 import random
+from typing import Dict, List, Sequence, Optional, Tuple
 
 from deepeval.errors import DeepEvalError
-from deepeval.optimization.types import PromptConfigurationId, ScoreTable
+from deepeval.optimizer.types import PromptConfigurationId, ScoreTable
 
 
 def _is_dominated(
@@ -164,3 +165,63 @@ def select_prompt_configuration_pareto(
     """
     freq = frequency_weights(score_table)
     return sample_by_frequency(freq, random_state=random_state)
+
+
+class TieBreaker(str, Enum):
+    PREFER_ROOT = "prefer_root"
+    PREFER_CHILD = "prefer_child"
+    RANDOM = "random"
+
+
+def pick_best_with_ties(
+    totals: Dict[PromptConfigurationId, float],
+    parents_by_id: Dict[PromptConfigurationId, Optional[PromptConfigurationId]],
+    *,
+    random_state: random.Random,
+    tie_tolerance: float = 1e-9,
+    policy: TieBreaker = TieBreaker.PREFER_ROOT,
+) -> Tuple[PromptConfigurationId, List[PromptConfigurationId], float]:
+    """
+    Choose the best candidate by aggregate score with deterministic tie handling.
+
+    Returns: (chosen_id, tied_ids, max_score)
+    - tied_ids includes everyone within tie_tolerance of max_score
+    """
+    if not totals:
+        raise DeepEvalError("No candidate prompt configuration to choose from.")
+
+    max_score = max(totals.values())
+    tied = [
+        prompt_configuration_id
+        for prompt_configuration_id, score in totals.items()
+        if abs(score - max_score) <= tie_tolerance
+    ]
+
+    if len(tied) == 1:
+        return tied[0], tied, max_score
+
+    # Resolve tie by policy
+    if policy == TieBreaker.PREFER_CHILD:
+        # Prefer any non root. When multiple children exist, use the most recent
+        child_ids = [
+            prompt_configuration_id
+            for prompt_configuration_id in tied
+            if parents_by_id.get(prompt_configuration_id) is not None
+        ]
+        if child_ids:
+            # choose the newest child deterministically by order
+            for prompt_configuration_id in reversed(list(totals.keys())):
+                if prompt_configuration_id in child_ids:
+                    return prompt_configuration_id, tied, max_score
+
+    if policy == TieBreaker.RANDOM:
+        return random_state.choice(tied), tied, max_score
+
+    # by default prefer a root if present, otherwise the first tied
+    root_ids = [
+        prompt_configuration_id
+        for prompt_configuration_id in tied
+        if parents_by_id.get(prompt_configuration_id) is None
+    ]
+    chosen = root_ids[0] if root_ids else tied[0]
+    return chosen, tied, max_score