deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/optimizer/prompt_optimizer.py

@@ -0,0 +1,263 @@
+from contextlib import contextmanager
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    BarColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+from deepeval.dataset.golden import Golden, ConversationalGolden
+from deepeval.errors import DeepEvalError
+from deepeval.metrics import BaseConversationalMetric, BaseMetric
+from deepeval.metrics.utils import initialize_model
+from deepeval.models.base_model import DeepEvalBaseLLM
+from deepeval.optimizer.scorer import Scorer
+from deepeval.optimizer.rewriter import Rewriter
+from deepeval.optimizer.types import (
+    ModelCallback,
+    RunnerStatusType,
+)
+from deepeval.optimizer.utils import (
+    validate_callback,
+    validate_metrics,
+)
+from deepeval.optimizer.configs import (
+    DisplayConfig,
+    MutationConfig,
+    AsyncConfig,
+)
+from deepeval.prompt.prompt import Prompt
+from deepeval.utils import get_or_create_event_loop
+from deepeval.optimizer.algorithms import (
+    GEPA,
+    MIPROV2,
+    COPRO,
+    SIMBA,
+)
+from deepeval.optimizer.algorithms.configs import (
+    GEPA_REWRITE_INSTRUCTION_MAX_CHARS,
+    MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS,
+)
+
+
+class PromptOptimizer:
+    def __init__(
+        self,
+        model_callback: ModelCallback,
+        metrics: Union[List[BaseMetric], List[BaseConversationalMetric]],
+        optimizer_model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        algorithm: Union[GEPA, MIPROV2, COPRO, SIMBA] = GEPA(),
+        async_config: Optional[AsyncConfig] = AsyncConfig(),
+        display_config: Optional[DisplayConfig] = DisplayConfig(),
+        mutation_config: Optional[MutationConfig] = MutationConfig(),
+    ):
+        self.optimizer_model, self.using_native_model = initialize_model(
+            optimizer_model
+        )
+        self.model_callback = validate_callback(
+            component="PromptOptimizer",
+            model_callback=model_callback,
+        )
+        self.metrics = validate_metrics(
+            component="PromptOptimizer", metrics=metrics
+        )
+
+        self.async_config = async_config
+        self.display_config = display_config
+        self.mutation_config = mutation_config
+        self.algorithm = algorithm
+        self.optimization_report = None
+        self._configure_algorithm()
+
+        # Internal state used only when a progress indicator is active.
+        # Tuple is (Progress instance, task_id).
+        self._progress_state: Optional[Tuple[Progress, int]] = None
+
+    ##############
+    # Public API #
+    ##############
+
+    def optimize(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Prompt:
+        if self.async_config.run_async:
+            loop = get_or_create_event_loop()
+            return loop.run_until_complete(
+                self.a_optimize(prompt=prompt, goldens=goldens)
+            )
+
+        try:
+            with self._progress_context():
+                best_prompt, self.optimization_report = self.algorithm.execute(
+                    prompt=prompt, goldens=goldens
+                )
+        except Exception as exc:
+            self._handle_optimization_error(exc)
+
+        return best_prompt
+
+    async def a_optimize(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Prompt:
+        try:
+            with self._progress_context():
+                best_prompt, self.optimization_report = (
+                    await self.algorithm.a_execute(
+                        prompt=prompt, goldens=goldens
+                    )
+                )
+        except Exception as exc:
+            self._handle_optimization_error(exc)
+
+        return best_prompt
+
+    ####################
+    # Internal helpers #
+    ####################
+
+    def _configure_algorithm(self) -> None:
+        """Configure the algorithm with scorer, rewriter, and callbacks."""
+        self.algorithm.scorer = Scorer(
+            model_callback=self.model_callback,
+            metrics=self.metrics,
+            max_concurrent=self.async_config.max_concurrent,
+            throttle_seconds=float(self.async_config.throttle_value),
+        )
+
+        # Attach rewriter for mutation behavior
+        # GEPA uses internal constant; other algorithms use MIPROV2 constant
+        if isinstance(self.algorithm, GEPA):
+            max_chars = GEPA_REWRITE_INSTRUCTION_MAX_CHARS
+        else:
+            max_chars = MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS
+        self.algorithm._rewriter = Rewriter(
+            optimizer_model=self.optimizer_model,
+            max_chars=max_chars,
+            list_mutation_config=self.mutation_config,
+            random_state=self.algorithm.random_state,
+        )
+
+        # Set status callback
+        self.algorithm.status_callback = self._on_status
+
+    @contextmanager
+    def _progress_context(self):
+        """Context manager that sets up progress indicator if enabled."""
+        if not self.display_config.show_indicator:
+            yield
+            return
+
+        with Progress(
+            SpinnerColumn(style="rgb(106,0,255)"),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(bar_width=40),
+            TimeElapsedColumn(),
+            transient=True,
+        ) as progress:
+            task = progress.add_task(
+                f"Optimizing prompt with {self.algorithm.name}..."
+            )
+            self._progress_state = (progress, task)
+            try:
+                yield
+            finally:
+                self._progress_state = None
+
+    def _handle_optimization_error(self, exc: Exception) -> None:
+        """
+        Handle optimization errors by formatting and raising a user-friendly message.
+        """
+        total_steps: Optional[int] = None
+        iterations: Optional[int] = getattr(self.algorithm, "iterations", None)
+        if iterations is not None:
+            total_steps = int(iterations)
+
+        prefix = f"(iterations={iterations}) " if iterations is not None else ""
+        detail = (
+            f"{prefix}• error {exc.__class__.__name__}: {exc} "
+            "• halted before first iteration"
+        )
+
+        self._on_status(
+            RunnerStatusType.ERROR,
+            detail=detail,
+            step_index=None,
+            total_steps=total_steps,
+        )
+
+        algo = self.algorithm.name
+        raise DeepEvalError(f"[{algo}] {detail}") from None
+
+    def _on_status(
+        self,
+        kind: RunnerStatusType,
+        detail: str,
+        step_index: Optional[int] = None,
+        total_steps: Optional[int] = None,
+    ) -> None:
+        """
+        Unified status callback used by the algorithm.
+
+        - PROGRESS: update the progress bar description and position
+        - TIE:      optionally print a tie message
+        - ERROR:    print a concise error message and allow the run to halt
+        """
+        algo = self.algorithm.name
+
+        if kind is RunnerStatusType.ERROR:
+            if self._progress_state is not None:
+                progress, task = self._progress_state
+                if total_steps is not None:
+                    progress.update(task, total=total_steps)
+                description = self._format_progress_description(detail)
+                progress.update(task, description=description)
+            print(f"[{algo}] {detail}")
+            return
+
+        if kind is RunnerStatusType.TIE:
+            if not self.display_config.announce_ties:
+                return
+            print(f"[{algo}] {detail}")
+            return
+
+        if kind is not RunnerStatusType.PROGRESS:
+            return
+
+        if self._progress_state is None:
+            return
+
+        progress, task = self._progress_state
+
+        if total_steps is not None:
+            progress.update(task, total=total_steps)
+
+        if step_index is not None and step_index > 0:
+            progress.advance(task, 1)
+
+        description = self._format_progress_description(detail)
+        progress.update(task, description=description)
+
+    def _format_progress_description(self, detail: str) -> str:
+        """
+        Compose a human readable progress line using an algorithm agnostic
+        prefix and an algorithm specific detail string provided by the algorithm.
+        """
+        algo = self.algorithm.name
+        base = f"Optimizing prompt with {algo}"
+        if detail:
+            return f"{base} [rgb(25,227,160)]{detail}[/]"
+        return base

deepeval/optimizer/rewriter/__init__.py

@@ -0,0 +1,5 @@
+from .rewriter import Rewriter
+
+__all__ = [
+    "Rewriter",
+]

deepeval/optimizer/rewriter/rewriter.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+import random
+from typing import Optional, Tuple, Union
+
+from deepeval.models.base_model import DeepEvalBaseLLM
+from deepeval.optimizer.types import (
+    ModuleId,
+)
+from deepeval.optimizer.configs import (
+    MutationConfig,
+)
+from deepeval.prompt.prompt import Prompt
+from deepeval.optimizer.rewriter.utils import (
+    _summarize_prompt_for_rewrite,
+    _compose_prompt_messages,
+    _normalize_llm_output_to_text,
+    _apply_rewritten_prompt,
+)
+
+
+class Rewriter:
+    """
+    Uses a provided DeepEval model to rewrite the prompt for a module,
+    guided by feedback_text (μ_f).
+
+    For LIST prompts, the target message to rewrite is chosen according to
+    `list_mutation_config` and `random_state`.
+    """
+
+    def __init__(
+        self,
+        optimizer_model: DeepEvalBaseLLM,
+        max_chars: int = 4000,
+        list_mutation_config: Optional[MutationConfig] = None,
+        random_state: Optional[Union[int, random.Random]] = None,
+    ):
+        self.optimizer_model = optimizer_model
+        self.max_chars = max_chars
+        self.list_mutation_config = list_mutation_config or MutationConfig()
+
+        # Accept either an int seed or a Random instance.
+        if isinstance(random_state, int):
+            self.random_state: Optional[random.Random] = random.Random(
+                random_state
+            )
+        else:
+            self.random_state = random_state or random.Random()
+
+    def _compose_messages(
+        self, *, module_id: ModuleId, old_prompt: Prompt, feedback_text: str
+    ) -> Tuple[str, str]:
+        current_prompt_block = _summarize_prompt_for_rewrite(
+            old_prompt, self.max_chars
+        )
+        system_message = (
+            "You are refining a prompt used in a multi-step LLM pipeline. "
+            "Given the current prompt and concise feedback, produce a revised prompt "
+            "that addresses the issues while preserving intent and style. "
+            "Return only the new prompt text, no explanations."
+        )
+        user_message = f"""[Current Prompt]
+{current_prompt_block}
+
+[Feedback]
+{feedback_text[:self.max_chars]}
+
+[Instruction]
+Rewrite the prompt. Keep it concise and actionable. Do not include extraneous text.
+"""
+        return system_message, user_message
+
+    def rewrite(
+        self,
+        module_id: ModuleId,
+        old_prompt: Prompt,
+        feedback_text: str,
+    ) -> Prompt:
+        if not feedback_text.strip():
+            return old_prompt
+
+        system_message, user_message = self._compose_messages(
+            module_id=module_id,
+            old_prompt=old_prompt,
+            feedback_text=feedback_text,
+        )
+        merged_prompt_text = _compose_prompt_messages(
+            system_message, user_message
+        )
+
+        out = self.optimizer_model.generate(merged_prompt_text)
+        new_text = _normalize_llm_output_to_text(out)
+        return _apply_rewritten_prompt(
+            old_prompt,
+            new_text,
+            self.random_state,
+            self.list_mutation_config,
+        )
+
+    async def a_rewrite(
+        self,
+        module_id: ModuleId,
+        old_prompt: Prompt,
+        feedback_text: str,
+    ) -> Prompt:
+        if not feedback_text.strip():
+            return old_prompt
+
+        system_message, user_message = self._compose_messages(
+            module_id=module_id,
+            old_prompt=old_prompt,
+            feedback_text=feedback_text,
+        )
+        merged_prompt_text = _compose_prompt_messages(
+            system_message, user_message
+        )
+
+        out = await self.optimizer_model.a_generate(merged_prompt_text)
+        new_text = _normalize_llm_output_to_text(out)
+        return _apply_rewritten_prompt(
+            old_prompt,
+            new_text,
+            self.random_state,
+            self.list_mutation_config,
+        )

deepeval/optimizer/rewriter/utils.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+import json
+import random
+from typing import List, Optional, Tuple, Union
+
+from deepeval.errors import DeepEvalError
+from deepeval.optimizer.utils import (
+    validate_int_in_range,
+    validate_instance,
+)
+from deepeval.optimizer.configs import (
+    MutationConfig,
+    MutationTargetType,
+)
+from deepeval.prompt.api import PromptType, PromptMessage
+from deepeval.prompt.prompt import Prompt
+
+
+##################
+# Common Helpers #
+##################
+def _summarize_prompt_for_rewrite(old_prompt: Prompt, max_chars: int) -> str:
+    """
+    Produce a human-readable summary of the current prompt for the
+    rewriter instruction block.
+
+    - For TEXT prompts, this is just `text_template`.
+    - For LIST prompts, this is a numbered list of (role, content) lines.
+    """
+
+    # LIST prompts: show each message with its role.
+    if old_prompt.type is PromptType.LIST and old_prompt.messages_template:
+        lines: List[str] = []
+        for message_index, message in enumerate(old_prompt.messages_template):
+            role = message.role or ""
+            content = message.content or ""
+            lines.append(f"[{message_index+1}] ({role}) {content}")
+        combined = "\n".join(lines)
+        return combined[:max_chars]
+
+    # Since it is not a LIST prompt, just use text_template.
+    text = old_prompt.text_template or ""
+    return text[:max_chars]
+
+
+def _select_list_target_index(
+    messages: List[PromptMessage],
+    config: MutationConfig,
+    random_state: random.Random,
+) -> int:
+    """
+    Select which list message index to rewrite, based on PromptListMutationConfig.
+
+    Rules:
+    - Start with all indices in scope.
+    - If target_role is set, restrict candidates to messages with that role
+      (case insensitive). If no messages match, fall back to all indices.
+    - target_type:
+        * FIRST:       pick the first candidate index.
+        * RANDOM:      pick a candidate via random_state.choice(candidates).
+        * FIXED_INDEX: use target_index when valid (and consistent with role
+                       filter), otherwise fall back to the first candidate.
+    """
+    if not messages:
+        raise DeepEvalError(
+            "Rewriter._select_list_target_index expected at least one "
+            "message, but received an empty message list."
+        )
+
+    validate_instance(
+        component="Rewriter._select_list_target_index",
+        param_name="target_type",
+        value=config.target_type,
+        expected_types=MutationTargetType,
+    )
+
+    messages_length = len(messages)
+    candidate_indices = list(range(messages_length))
+
+    # Optional case insensitive role restriction
+    if config.target_role:
+        target_role_lower = config.target_role.lower()
+        filtered = [
+            index
+            for index, message in enumerate(messages)
+            if (message.role or "").lower() == target_role_lower
+        ]
+        if filtered:
+            candidate_indices = filtered
+
+    target_type = config.target_type
+
+    if target_type is MutationTargetType.RANDOM:
+        return random_state.choice(candidate_indices)
+
+    if target_type is MutationTargetType.FIXED_INDEX:
+        index = validate_int_in_range(
+            component="Rewriter._select_list_target_index",
+            param_name="target_index",
+            value=int(config.target_index),
+            min_inclusive=0,
+            max_exclusive=len(candidate_indices),
+        )
+        return candidate_indices[index]
+
+    # if you got this error it means that a new PromptListMutationTargetType was added,
+    # but not handled above
+    raise DeepEvalError(
+        "Rewriter._select_list_target_index received unsupported "
+        f"target_type={target_type!r}. Expected RANDOM or FIXED_INDEX."
+    )
+
+
+def _apply_rewritten_prompt(
+    old_prompt: Prompt,
+    new_text: str,
+    random_state: random.Random,
+    list_mutation_config: Optional[MutationConfig] = None,
+) -> Prompt:
+    """
+    Apply the rewritten text to a Prompt, preserving representation:
+
+    - For TEXT prompts, update `text_template`.
+    - For LIST prompts, rewrite the content of a single message while
+      keeping the number of messages the same.
+    - Preserve additonal Prompt meta such as `label` and `interpolation_type`
+    """
+    if not new_text:
+        return old_prompt
+
+    if old_prompt.type is PromptType.LIST and old_prompt.messages_template:
+        messages = old_prompt.messages_template
+        config = list_mutation_config or MutationConfig()
+
+        target_index = _select_list_target_index(
+            messages=messages,
+            config=config,
+            random_state=random_state,
+        )
+
+        new_messages: List[PromptMessage] = []
+        for message_index, message in enumerate(messages):
+            if message_index == target_index:
+                # Preserve the original role; do not inject a new one.
+                new_messages.append(
+                    PromptMessage(
+                        role=message.role,
+                        content=new_text,
+                    )
+                )
+            else:
+                new_messages.append(message)
+
+        new_prompt = Prompt(
+            alias=old_prompt.alias,
+            text_template=None,
+            messages_template=new_messages,
+            model_settings=old_prompt.model_settings,
+            output_type=old_prompt.output_type,
+            output_schema=old_prompt.output_schema,
+        )
+
+    else:
+        # Since it is not LIST, it must be TEXT type
+        new_prompt = Prompt(
+            alias=old_prompt.alias,
+            text_template=new_text,
+            model_settings=old_prompt.model_settings,
+            output_type=old_prompt.output_type,
+            output_schema=old_prompt.output_schema,
+        )
+
+    new_prompt.label = old_prompt.label
+    new_prompt.interpolation_type = old_prompt.interpolation_type
+    return new_prompt
+
+
+def _compose_prompt_messages(system_message: str, user_message: str) -> str:
+    """
+    Join system and user messages into a single prompt string.
+    Strips surrounding whitespace from each part; if the system message is
+    empty or absent, returns just the user message.
+    """
+    system_text = (system_message or "").strip()
+    user_text = (user_message or "").strip()
+    return f"{system_text}\n\n{user_text}" if system_text else user_text
+
+
+def _normalize_llm_output_to_text(
+    result: Union[str, Tuple[Union[str, dict], float], dict],
+) -> str:
+    """
+    Convert a DeepEval LLM generate() / a_generate() result to a clean string.
+
+    Accepted inputs:
+      - str                        -> returned as trimmed
+      - (str|dict, float_cost)     -> first element extracted and normalized
+      - dict (e.g. JSON mode)      -> JSON serialized with ensure_ascii=False
+
+    Fallback: if serialization fails, str(value).strip() is used.
+    """
+    output_value: Union[str, dict]
+    if isinstance(result, tuple):
+        output_value = result[0]
+    else:
+        output_value = result
+
+    if isinstance(output_value, str):
+        return output_value.strip()
+
+    try:
+        return json.dumps(output_value, ensure_ascii=False)
+    except Exception:
+        return str(output_value).strip()

deepeval/optimizer/scorer/__init__.py

@@ -0,0 +1,5 @@
+from .scorer import Scorer
+
+__all__ = [
+    "Scorer",
+]

deepeval/optimizer/scorer/base.py

@@ -0,0 +1,86 @@
+from abc import ABC, abstractmethod
+from typing import Union, List
+
+from deepeval.optimizer.types import PromptConfiguration, ScoreVector
+from deepeval.dataset.golden import Golden, ConversationalGolden
+
+ModuleId = str
+
+
+class BaseScorer(ABC):
+    """
+    Base scorer contract used by optimization runners.
+
+    Runners call into this adapter to:
+    - compute scores per-instance on some subset (score_on_pareto),
+    - compute minibatch means for selection and acceptance,
+    - generate feedback text used by the Rewriter.
+    """
+
+    # Sync
+    @abstractmethod
+    def score_pareto(
+        self,
+        prompt_configuration: PromptConfiguration,
+        d_pareto: Union[List[Golden], List[ConversationalGolden]],
+    ) -> ScoreVector:
+        """Return per-instance scores on D_pareto."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def score_minibatch(
+        self,
+        prompt_configuration: PromptConfiguration,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> float:
+        """Return average score μ on a minibatch from D_feedback."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_minibatch_feedback(
+        self,
+        prompt_configuration: PromptConfiguration,
+        module: ModuleId,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> str:
+        """Return μ_f text for the module (metric.reason + traces, etc.)."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def select_module(
+        self, prompt_configuration: PromptConfiguration
+    ) -> ModuleId:
+        """Pick a module to mutate."""
+        raise NotImplementedError
+
+    # Async
+    @abstractmethod
+    async def a_score_pareto(
+        self,
+        prompt_configuration: PromptConfiguration,
+        d_pareto: Union[List[Golden], List[ConversationalGolden]],
+    ) -> ScoreVector:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_score_minibatch(
+        self,
+        prompt_configuration: PromptConfiguration,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> float:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_get_minibatch_feedback(
+        self,
+        prompt_configuration: PromptConfiguration,
+        module: ModuleId,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> str:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_select_module(
+        self, prompt_configuration: PromptConfiguration
+    ) -> ModuleId:
+        raise NotImplementedError