deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/optimizer/algorithms/simba/types.py

@@ -0,0 +1,15 @@
+from enum import Enum
+
+
+class SIMBAStrategy(str, Enum):
+    """
+    Edit strategies used by SIMBA-style optimization.
+
+    - APPEND_DEMO: append one or more input/output demos distilled from the
+      current minibatch, similar in spirit to DSPy's `append_a_demo`.
+    - APPEND_RULE: append a concise natural-language rule distilled from
+      feedback, similar in spirit to DSPy's `append_a_rule`.
+    """
+
+    APPEND_DEMO = "append_demo"
+    APPEND_RULE = "append_rule"

deepeval/optimizer/configs.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+from enum import Enum
+from pydantic import BaseModel, Field, conint
+from typing import Optional
+from deepeval.evaluate.configs import AsyncConfig
+
+
+class DisplayConfig(BaseModel):
+    show_indicator: bool = True
+    announce_ties: bool = Field(
+        False, description="Print a one-line note when a tie is detected"
+    )
+
+
+class MutationTargetType(Enum):
+    RANDOM = "random"
+    FIXED_INDEX = "fixed_index"
+
+
+# default all messages
+class MutationConfig(BaseModel):
+    target_type: MutationTargetType = MutationTargetType.RANDOM
+    # should be list
+    target_role: Optional[str] = Field(
+        default=None,
+        description="If set, restricts candidates to messages with this role (case insensitive).",
+    )
+    target_index: conint(ge=0) = Field(
+        default=0,
+        description="0-based index used when target_type == FIXED_INDEX.",
+    )

deepeval/optimizer/policies.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+from enum import Enum
+import random
+from typing import Dict, List, Sequence, Optional, Tuple
+
+from deepeval.errors import DeepEvalError
+from deepeval.optimizer.types import PromptConfigurationId, ScoreTable
+
+
+def _is_dominated(
+    candidate_scores: List[float], other_scores: List[float]
+) -> bool:
+    """
+    Return True if `candidate_scores` is dominated by `other_scores`:
+    (other >= candidate on all dimensions) AND (other > candidate on at least one).
+    """
+    other_ge_everywhere = all(
+        other_score >= candidate_score
+        for candidate_score, other_score in zip(candidate_scores, other_scores)
+    )
+    other_gt_somewhere = any(
+        other_score > candidate_score
+        for candidate_score, other_score in zip(candidate_scores, other_scores)
+    )
+    return other_ge_everywhere and other_gt_somewhere
+
+
+def pareto_frontier(
+    prompt_configuration_ids: Sequence[PromptConfigurationId],
+    score_table: ScoreTable,
+) -> List[PromptConfigurationId]:
+    """
+    Compute the set of non-dominated candidates given their scores.
+    Returns PromptConfigurationIds on the Pareto frontier.
+    """
+    frontier: List[PromptConfigurationId] = []
+    for prompt_configuration_id in prompt_configuration_ids:
+        candidate_vector = score_table[prompt_configuration_id]
+        dominated = False
+
+        # If any existing frontier member dominates this candidate, skip it.
+        for frontier_id in frontier:
+            if _is_dominated(candidate_vector, score_table[frontier_id]):
+                dominated = True
+                break
+        if dominated:
+            continue
+
+        # Remove any frontier member that is dominated by this candidate.
+        frontier = [
+            f_id
+            for f_id in frontier
+            if not _is_dominated(score_table[f_id], candidate_vector)
+        ]
+        frontier.append(prompt_configuration_id)
+
+    return frontier
+
+
+def frequency_weights(
+    score_table: ScoreTable,
+) -> Dict[PromptConfigurationId, int]:
+    """
+    Build best sets, remove dominated candidates, and count appearances.
+
+    Returns:
+        A map {prompt_configuration_id -> frequency} counting how often each
+        globally non-dominated prompt configuration appears among the instance
+        Pareto sets.
+    """
+    if not score_table:
+        return {}
+
+    # Assume all score vectors have the same length.
+    example_vector = next(iter(score_table.values()))
+    num_instances = len(example_vector)
+    all_candidates = list(score_table.keys())
+
+    per_instance_frontiers: List[List[PromptConfigurationId]] = []
+    for i in range(num_instances):
+        best_score_i = max(
+            score_table[prompt_configuration_id][i]
+            for prompt_configuration_id in all_candidates
+        )
+        winners_i = [
+            prompt_configuration_id
+            for prompt_configuration_id in all_candidates
+            if score_table[prompt_configuration_id][i] == best_score_i
+        ]
+
+        # Instance frontier among winners. We pass 1-D score vectors
+        # so this reduces to "all candidates with the max score at instance i",
+        instance_frontier = pareto_frontier(
+            winners_i,
+            {
+                prompt_configuration_id: [
+                    score_table[prompt_configuration_id][i]
+                ]
+                for prompt_configuration_id in winners_i
+            },
+        )
+        per_instance_frontiers.append(instance_frontier)
+
+    # Global candidate set appearing in any winners
+    candidate_union = sorted(
+        {
+            prompt_configuration_id
+            for winners in per_instance_frontiers
+            for prompt_configuration_id in winners
+        }
+    )
+    global_frontier = pareto_frontier(candidate_union, score_table)
+
+    # Count frequency only for candidates on the global frontier
+    frequency_by_prompt_config: Dict[PromptConfigurationId, int] = {
+        prompt_configuration_id: 0
+        for prompt_configuration_id in global_frontier
+    }
+    for winners in per_instance_frontiers:
+        for prompt_configuration_id in winners:
+            if prompt_configuration_id in frequency_by_prompt_config:
+                frequency_by_prompt_config[prompt_configuration_id] += 1
+
+    return frequency_by_prompt_config
+
+
+def sample_by_frequency(
+    frequency_by_prompt_config: Dict[PromptConfigurationId, int],
+    *,
+    random_state: random.Random,
+) -> PromptConfigurationId:
+    """
+    Sample a prompt configuration id with probability proportional to its frequency.
+    Falls back to uniform if the total weight is zero.
+    """
+    if not frequency_by_prompt_config:
+        raise DeepEvalError("No prompt configurations to sample.")
+
+    items = list(frequency_by_prompt_config.items())
+    total_weight = sum(weight for _, weight in items)
+
+    if total_weight == 0:
+        # Uniform fallback
+        return random_state.choice(
+            [prompt_configuration_id for prompt_configuration_id, _ in items]
+        )
+
+    r = random_state.uniform(0, total_weight)
+    cumulative = 0.0
+    for prompt_configuration_id, weight in items:
+        cumulative += weight
+        if r <= cumulative:
+            return prompt_configuration_id
+    return items[-1][0]
+
+
+def select_prompt_configuration_pareto(
+    score_table: ScoreTable, *, random_state: random.Random
+) -> PromptConfigurationId:
+    """
+    Frequency weighted sampling over the Pareto winners,
+    restricted to globally non-dominated prompt configurations. A configuration
+    is globally non-dominated if no other configuration dominates it using
+    the full vector.
+    """
+    freq = frequency_weights(score_table)
+    return sample_by_frequency(freq, random_state=random_state)
+
+
+class TieBreaker(str, Enum):
+    PREFER_ROOT = "prefer_root"
+    PREFER_CHILD = "prefer_child"
+    RANDOM = "random"
+
+
+def pick_best_with_ties(
+    totals: Dict[PromptConfigurationId, float],
+    parents_by_id: Dict[PromptConfigurationId, Optional[PromptConfigurationId]],
+    *,
+    random_state: random.Random,
+    tie_tolerance: float = 1e-9,
+    policy: TieBreaker = TieBreaker.PREFER_ROOT,
+) -> Tuple[PromptConfigurationId, List[PromptConfigurationId], float]:
+    """
+    Choose the best candidate by aggregate score with deterministic tie handling.
+
+    Returns: (chosen_id, tied_ids, max_score)
+    - tied_ids includes everyone within tie_tolerance of max_score
+    """
+    if not totals:
+        raise DeepEvalError("No candidate prompt configuration to choose from.")
+
+    max_score = max(totals.values())
+    tied = [
+        prompt_configuration_id
+        for prompt_configuration_id, score in totals.items()
+        if abs(score - max_score) <= tie_tolerance
+    ]
+
+    if len(tied) == 1:
+        return tied[0], tied, max_score
+
+    # Resolve tie by policy
+    if policy == TieBreaker.PREFER_CHILD:
+        # Prefer any non root. When multiple children exist, use the most recent
+        child_ids = [
+            prompt_configuration_id
+            for prompt_configuration_id in tied
+            if parents_by_id.get(prompt_configuration_id) is not None
+        ]
+        if child_ids:
+            # choose the newest child deterministically by order
+            for prompt_configuration_id in reversed(list(totals.keys())):
+                if prompt_configuration_id in child_ids:
+                    return prompt_configuration_id, tied, max_score
+
+    if policy == TieBreaker.RANDOM:
+        return random_state.choice(tied), tied, max_score
+
+    # by default prefer a root if present, otherwise the first tied
+    root_ids = [
+        prompt_configuration_id
+        for prompt_configuration_id in tied
+        if parents_by_id.get(prompt_configuration_id) is None
+    ]
+    chosen = root_ids[0] if root_ids else tied[0]
+    return chosen, tied, max_score

deepeval/optimizer/prompt_optimizer.py

@@ -0,0 +1,263 @@
+from contextlib import contextmanager
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    BarColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+from deepeval.dataset.golden import Golden, ConversationalGolden
+from deepeval.errors import DeepEvalError
+from deepeval.metrics import BaseConversationalMetric, BaseMetric
+from deepeval.metrics.utils import initialize_model
+from deepeval.models.base_model import DeepEvalBaseLLM
+from deepeval.optimizer.scorer import Scorer
+from deepeval.optimizer.rewriter import Rewriter
+from deepeval.optimizer.types import (
+    ModelCallback,
+    RunnerStatusType,
+)
+from deepeval.optimizer.utils import (
+    validate_callback,
+    validate_metrics,
+)
+from deepeval.optimizer.configs import (
+    DisplayConfig,
+    MutationConfig,
+    AsyncConfig,
+)
+from deepeval.prompt.prompt import Prompt
+from deepeval.utils import get_or_create_event_loop
+from deepeval.optimizer.algorithms import (
+    GEPA,
+    MIPROV2,
+    COPRO,
+    SIMBA,
+)
+from deepeval.optimizer.algorithms.configs import (
+    GEPA_REWRITE_INSTRUCTION_MAX_CHARS,
+    MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS,
+)
+
+
+class PromptOptimizer:
+    def __init__(
+        self,
+        model_callback: ModelCallback,
+        metrics: Union[List[BaseMetric], List[BaseConversationalMetric]],
+        optimizer_model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        algorithm: Union[GEPA, MIPROV2, COPRO, SIMBA] = GEPA(),
+        async_config: Optional[AsyncConfig] = AsyncConfig(),
+        display_config: Optional[DisplayConfig] = DisplayConfig(),
+        mutation_config: Optional[MutationConfig] = MutationConfig(),
+    ):
+        self.optimizer_model, self.using_native_model = initialize_model(
+            optimizer_model
+        )
+        self.model_callback = validate_callback(
+            component="PromptOptimizer",
+            model_callback=model_callback,
+        )
+        self.metrics = validate_metrics(
+            component="PromptOptimizer", metrics=metrics
+        )
+
+        self.async_config = async_config
+        self.display_config = display_config
+        self.mutation_config = mutation_config
+        self.algorithm = algorithm
+        self.optimization_report = None
+        self._configure_algorithm()
+
+        # Internal state used only when a progress indicator is active.
+        # Tuple is (Progress instance, task_id).
+        self._progress_state: Optional[Tuple[Progress, int]] = None
+
+    ##############
+    # Public API #
+    ##############
+
+    def optimize(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Prompt:
+        if self.async_config.run_async:
+            loop = get_or_create_event_loop()
+            return loop.run_until_complete(
+                self.a_optimize(prompt=prompt, goldens=goldens)
+            )
+
+        try:
+            with self._progress_context():
+                best_prompt, self.optimization_report = self.algorithm.execute(
+                    prompt=prompt, goldens=goldens
+                )
+        except Exception as exc:
+            self._handle_optimization_error(exc)
+
+        return best_prompt
+
+    async def a_optimize(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Prompt:
+        try:
+            with self._progress_context():
+                best_prompt, self.optimization_report = (
+                    await self.algorithm.a_execute(
+                        prompt=prompt, goldens=goldens
+                    )
+                )
+        except Exception as exc:
+            self._handle_optimization_error(exc)
+
+        return best_prompt
+
+    ####################
+    # Internal helpers #
+    ####################
+
+    def _configure_algorithm(self) -> None:
+        """Configure the algorithm with scorer, rewriter, and callbacks."""
+        self.algorithm.scorer = Scorer(
+            model_callback=self.model_callback,
+            metrics=self.metrics,
+            max_concurrent=self.async_config.max_concurrent,
+            throttle_seconds=float(self.async_config.throttle_value),
+        )
+
+        # Attach rewriter for mutation behavior
+        # GEPA uses internal constant; other algorithms use MIPROV2 constant
+        if isinstance(self.algorithm, GEPA):
+            max_chars = GEPA_REWRITE_INSTRUCTION_MAX_CHARS
+        else:
+            max_chars = MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS
+        self.algorithm._rewriter = Rewriter(
+            optimizer_model=self.optimizer_model,
+            max_chars=max_chars,
+            list_mutation_config=self.mutation_config,
+            random_state=self.algorithm.random_state,
+        )
+
+        # Set status callback
+        self.algorithm.status_callback = self._on_status
+
+    @contextmanager
+    def _progress_context(self):
+        """Context manager that sets up progress indicator if enabled."""
+        if not self.display_config.show_indicator:
+            yield
+            return
+
+        with Progress(
+            SpinnerColumn(style="rgb(106,0,255)"),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(bar_width=40),
+            TimeElapsedColumn(),
+            transient=True,
+        ) as progress:
+            task = progress.add_task(
+                f"Optimizing prompt with {self.algorithm.name}..."
+            )
+            self._progress_state = (progress, task)
+            try:
+                yield
+            finally:
+                self._progress_state = None
+
+    def _handle_optimization_error(self, exc: Exception) -> None:
+        """
+        Handle optimization errors by formatting and raising a user-friendly message.
+        """
+        total_steps: Optional[int] = None
+        iterations: Optional[int] = getattr(self.algorithm, "iterations", None)
+        if iterations is not None:
+            total_steps = int(iterations)
+
+        prefix = f"(iterations={iterations}) " if iterations is not None else ""
+        detail = (
+            f"{prefix}• error {exc.__class__.__name__}: {exc} "
+            "• halted before first iteration"
+        )
+
+        self._on_status(
+            RunnerStatusType.ERROR,
+            detail=detail,
+            step_index=None,
+            total_steps=total_steps,
+        )
+
+        algo = self.algorithm.name
+        raise DeepEvalError(f"[{algo}] {detail}") from None
+
+    def _on_status(
+        self,
+        kind: RunnerStatusType,
+        detail: str,
+        step_index: Optional[int] = None,
+        total_steps: Optional[int] = None,
+    ) -> None:
+        """
+        Unified status callback used by the algorithm.
+
+        - PROGRESS: update the progress bar description and position
+        - TIE:      optionally print a tie message
+        - ERROR:    print a concise error message and allow the run to halt
+        """
+        algo = self.algorithm.name
+
+        if kind is RunnerStatusType.ERROR:
+            if self._progress_state is not None:
+                progress, task = self._progress_state
+                if total_steps is not None:
+                    progress.update(task, total=total_steps)
+                description = self._format_progress_description(detail)
+                progress.update(task, description=description)
+            print(f"[{algo}] {detail}")
+            return
+
+        if kind is RunnerStatusType.TIE:
+            if not self.display_config.announce_ties:
+                return
+            print(f"[{algo}] {detail}")
+            return
+
+        if kind is not RunnerStatusType.PROGRESS:
+            return
+
+        if self._progress_state is None:
+            return
+
+        progress, task = self._progress_state
+
+        if total_steps is not None:
+            progress.update(task, total=total_steps)
+
+        if step_index is not None and step_index > 0:
+            progress.advance(task, 1)
+
+        description = self._format_progress_description(detail)
+        progress.update(task, description=description)
+
+    def _format_progress_description(self, detail: str) -> str:
+        """
+        Compose a human readable progress line using an algorithm agnostic
+        prefix and an algorithm specific detail string provided by the algorithm.
+        """
+        algo = self.algorithm.name
+        base = f"Optimizing prompt with {algo}"
+        if detail:
+            return f"{base} [rgb(25,227,160)]{detail}[/]"
+        return base

deepeval/optimizer/rewriter/__init__.py

@@ -0,0 +1,5 @@
+from .rewriter import Rewriter
+
+__all__ = [
+    "Rewriter",
+]

deepeval/optimizer/rewriter/rewriter.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+import random
+from typing import Optional, Tuple, Union
+
+from deepeval.models.base_model import DeepEvalBaseLLM
+from deepeval.optimizer.types import (
+    ModuleId,
+)
+from deepeval.optimizer.configs import (
+    MutationConfig,
+)
+from deepeval.prompt.prompt import Prompt
+from deepeval.optimizer.rewriter.utils import (
+    _summarize_prompt_for_rewrite,
+    _compose_prompt_messages,
+    _normalize_llm_output_to_text,
+    _apply_rewritten_prompt,
+)
+
+
+class Rewriter:
+    """
+    Uses a provided DeepEval model to rewrite the prompt for a module,
+    guided by feedback_text (μ_f).
+
+    For LIST prompts, the target message to rewrite is chosen according to
+    `list_mutation_config` and `random_state`.
+    """
+
+    def __init__(
+        self,
+        optimizer_model: DeepEvalBaseLLM,
+        max_chars: int = 4000,
+        list_mutation_config: Optional[MutationConfig] = None,
+        random_state: Optional[Union[int, random.Random]] = None,
+    ):
+        self.optimizer_model = optimizer_model
+        self.max_chars = max_chars
+        self.list_mutation_config = list_mutation_config or MutationConfig()
+
+        # Accept either an int seed or a Random instance.
+        if isinstance(random_state, int):
+            self.random_state: Optional[random.Random] = random.Random(
+                random_state
+            )
+        else:
+            self.random_state = random_state or random.Random()
+
+    def _compose_messages(
+        self, *, module_id: ModuleId, old_prompt: Prompt, feedback_text: str
+    ) -> Tuple[str, str]:
+        current_prompt_block = _summarize_prompt_for_rewrite(
+            old_prompt, self.max_chars
+        )
+        system_message = (
+            "You are refining a prompt used in a multi-step LLM pipeline. "
+            "Given the current prompt and concise feedback, produce a revised prompt "
+            "that addresses the issues while preserving intent and style. "
+            "Return only the new prompt text, no explanations."
+        )
+        user_message = f"""[Current Prompt]
+{current_prompt_block}
+
+[Feedback]
+{feedback_text[:self.max_chars]}
+
+[Instruction]
+Rewrite the prompt. Keep it concise and actionable. Do not include extraneous text.
+"""
+        return system_message, user_message
+
+    def rewrite(
+        self,
+        module_id: ModuleId,
+        old_prompt: Prompt,
+        feedback_text: str,
+    ) -> Prompt:
+        if not feedback_text.strip():
+            return old_prompt
+
+        system_message, user_message = self._compose_messages(
+            module_id=module_id,
+            old_prompt=old_prompt,
+            feedback_text=feedback_text,
+        )
+        merged_prompt_text = _compose_prompt_messages(
+            system_message, user_message
+        )
+
+        out = self.optimizer_model.generate(merged_prompt_text)
+        new_text = _normalize_llm_output_to_text(out)
+        return _apply_rewritten_prompt(
+            old_prompt,
+            new_text,
+            self.random_state,
+            self.list_mutation_config,
+        )
+
+    async def a_rewrite(
+        self,
+        module_id: ModuleId,
+        old_prompt: Prompt,
+        feedback_text: str,
+    ) -> Prompt:
+        if not feedback_text.strip():
+            return old_prompt
+
+        system_message, user_message = self._compose_messages(
+            module_id=module_id,
+            old_prompt=old_prompt,
+            feedback_text=feedback_text,
+        )
+        merged_prompt_text = _compose_prompt_messages(
+            system_message, user_message
+        )
+
+        out = await self.optimizer_model.a_generate(merged_prompt_text)
+        new_text = _normalize_llm_output_to_text(out)
+        return _apply_rewritten_prompt(
+            old_prompt,
+            new_text,
+            self.random_state,
+            self.list_mutation_config,
+        )