deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/optimizer/rewriter/utils.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+import json
+import random
+from typing import List, Optional, Tuple, Union
+
+from deepeval.errors import DeepEvalError
+from deepeval.optimizer.utils import (
+    validate_int_in_range,
+    validate_instance,
+)
+from deepeval.optimizer.configs import (
+    MutationConfig,
+    MutationTargetType,
+)
+from deepeval.prompt.api import PromptType, PromptMessage
+from deepeval.prompt.prompt import Prompt
+
+
+##################
+# Common Helpers #
+##################
+def _summarize_prompt_for_rewrite(old_prompt: Prompt, max_chars: int) -> str:
+    """
+    Produce a human-readable summary of the current prompt for the
+    rewriter instruction block.
+
+    - For TEXT prompts, this is just `text_template`.
+    - For LIST prompts, this is a numbered list of (role, content) lines.
+    """
+
+    # LIST prompts: show each message with its role.
+    if old_prompt.type is PromptType.LIST and old_prompt.messages_template:
+        lines: List[str] = []
+        for message_index, message in enumerate(old_prompt.messages_template):
+            role = message.role or ""
+            content = message.content or ""
+            lines.append(f"[{message_index+1}] ({role}) {content}")
+        combined = "\n".join(lines)
+        return combined[:max_chars]
+
+    # Since it is not a LIST prompt, just use text_template.
+    text = old_prompt.text_template or ""
+    return text[:max_chars]
+
+
+def _select_list_target_index(
+    messages: List[PromptMessage],
+    config: MutationConfig,
+    random_state: random.Random,
+) -> int:
+    """
+    Select which list message index to rewrite, based on PromptListMutationConfig.
+
+    Rules:
+    - Start with all indices in scope.
+    - If target_role is set, restrict candidates to messages with that role
+      (case insensitive). If no messages match, fall back to all indices.
+    - target_type:
+        * FIRST:       pick the first candidate index.
+        * RANDOM:      pick a candidate via random_state.choice(candidates).
+        * FIXED_INDEX: use target_index when valid (and consistent with role
+                       filter), otherwise fall back to the first candidate.
+    """
+    if not messages:
+        raise DeepEvalError(
+            "Rewriter._select_list_target_index expected at least one "
+            "message, but received an empty message list."
+        )
+
+    validate_instance(
+        component="Rewriter._select_list_target_index",
+        param_name="target_type",
+        value=config.target_type,
+        expected_types=MutationTargetType,
+    )
+
+    messages_length = len(messages)
+    candidate_indices = list(range(messages_length))
+
+    # Optional case insensitive role restriction
+    if config.target_role:
+        target_role_lower = config.target_role.lower()
+        filtered = [
+            index
+            for index, message in enumerate(messages)
+            if (message.role or "").lower() == target_role_lower
+        ]
+        if filtered:
+            candidate_indices = filtered
+
+    target_type = config.target_type
+
+    if target_type is MutationTargetType.RANDOM:
+        return random_state.choice(candidate_indices)
+
+    if target_type is MutationTargetType.FIXED_INDEX:
+        index = validate_int_in_range(
+            component="Rewriter._select_list_target_index",
+            param_name="target_index",
+            value=int(config.target_index),
+            min_inclusive=0,
+            max_exclusive=len(candidate_indices),
+        )
+        return candidate_indices[index]
+
+    # if you got this error it means that a new PromptListMutationTargetType was added,
+    # but not handled above
+    raise DeepEvalError(
+        "Rewriter._select_list_target_index received unsupported "
+        f"target_type={target_type!r}. Expected RANDOM or FIXED_INDEX."
+    )
+
+
+def _apply_rewritten_prompt(
+    old_prompt: Prompt,
+    new_text: str,
+    random_state: random.Random,
+    list_mutation_config: Optional[MutationConfig] = None,
+) -> Prompt:
+    """
+    Apply the rewritten text to a Prompt, preserving representation:
+
+    - For TEXT prompts, update `text_template`.
+    - For LIST prompts, rewrite the content of a single message while
+      keeping the number of messages the same.
+    - Preserve additonal Prompt meta such as `label` and `interpolation_type`
+    """
+    if not new_text:
+        return old_prompt
+
+    if old_prompt.type is PromptType.LIST and old_prompt.messages_template:
+        messages = old_prompt.messages_template
+        config = list_mutation_config or MutationConfig()
+
+        target_index = _select_list_target_index(
+            messages=messages,
+            config=config,
+            random_state=random_state,
+        )
+
+        new_messages: List[PromptMessage] = []
+        for message_index, message in enumerate(messages):
+            if message_index == target_index:
+                # Preserve the original role; do not inject a new one.
+                new_messages.append(
+                    PromptMessage(
+                        role=message.role,
+                        content=new_text,
+                    )
+                )
+            else:
+                new_messages.append(message)
+
+        new_prompt = Prompt(
+            alias=old_prompt.alias,
+            text_template=None,
+            messages_template=new_messages,
+            model_settings=old_prompt.model_settings,
+            output_type=old_prompt.output_type,
+            output_schema=old_prompt.output_schema,
+        )
+
+    else:
+        # Since it is not LIST, it must be TEXT type
+        new_prompt = Prompt(
+            alias=old_prompt.alias,
+            text_template=new_text,
+            model_settings=old_prompt.model_settings,
+            output_type=old_prompt.output_type,
+            output_schema=old_prompt.output_schema,
+        )
+
+    new_prompt.label = old_prompt.label
+    new_prompt.interpolation_type = old_prompt.interpolation_type
+    return new_prompt
+
+
+def _compose_prompt_messages(system_message: str, user_message: str) -> str:
+    """
+    Join system and user messages into a single prompt string.
+    Strips surrounding whitespace from each part; if the system message is
+    empty or absent, returns just the user message.
+    """
+    system_text = (system_message or "").strip()
+    user_text = (user_message or "").strip()
+    return f"{system_text}\n\n{user_text}" if system_text else user_text
+
+
+def _normalize_llm_output_to_text(
+    result: Union[str, Tuple[Union[str, dict], float], dict],
+) -> str:
+    """
+    Convert a DeepEval LLM generate() / a_generate() result to a clean string.
+
+    Accepted inputs:
+      - str                        -> returned as trimmed
+      - (str|dict, float_cost)     -> first element extracted and normalized
+      - dict (e.g. JSON mode)      -> JSON serialized with ensure_ascii=False
+
+    Fallback: if serialization fails, str(value).strip() is used.
+    """
+    output_value: Union[str, dict]
+    if isinstance(result, tuple):
+        output_value = result[0]
+    else:
+        output_value = result
+
+    if isinstance(output_value, str):
+        return output_value.strip()
+
+    try:
+        return json.dumps(output_value, ensure_ascii=False)
+    except Exception:
+        return str(output_value).strip()

deepeval/optimizer/scorer/__init__.py

@@ -0,0 +1,5 @@
+from .scorer import Scorer
+
+__all__ = [
+    "Scorer",
+]

deepeval/optimizer/scorer/base.py

@@ -0,0 +1,86 @@
+from abc import ABC, abstractmethod
+from typing import Union, List
+
+from deepeval.optimizer.types import PromptConfiguration, ScoreVector
+from deepeval.dataset.golden import Golden, ConversationalGolden
+
+ModuleId = str
+
+
+class BaseScorer(ABC):
+    """
+    Base scorer contract used by optimization runners.
+
+    Runners call into this adapter to:
+    - compute scores per-instance on some subset (score_on_pareto),
+    - compute minibatch means for selection and acceptance,
+    - generate feedback text used by the Rewriter.
+    """
+
+    # Sync
+    @abstractmethod
+    def score_pareto(
+        self,
+        prompt_configuration: PromptConfiguration,
+        d_pareto: Union[List[Golden], List[ConversationalGolden]],
+    ) -> ScoreVector:
+        """Return per-instance scores on D_pareto."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def score_minibatch(
+        self,
+        prompt_configuration: PromptConfiguration,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> float:
+        """Return average score μ on a minibatch from D_feedback."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_minibatch_feedback(
+        self,
+        prompt_configuration: PromptConfiguration,
+        module: ModuleId,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> str:
+        """Return μ_f text for the module (metric.reason + traces, etc.)."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def select_module(
+        self, prompt_configuration: PromptConfiguration
+    ) -> ModuleId:
+        """Pick a module to mutate."""
+        raise NotImplementedError
+
+    # Async
+    @abstractmethod
+    async def a_score_pareto(
+        self,
+        prompt_configuration: PromptConfiguration,
+        d_pareto: Union[List[Golden], List[ConversationalGolden]],
+    ) -> ScoreVector:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_score_minibatch(
+        self,
+        prompt_configuration: PromptConfiguration,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> float:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_get_minibatch_feedback(
+        self,
+        prompt_configuration: PromptConfiguration,
+        module: ModuleId,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> str:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_select_module(
+        self, prompt_configuration: PromptConfiguration
+    ) -> ModuleId:
+        raise NotImplementedError

deepeval/optimizer/scorer/scorer.py

@@ -0,0 +1,316 @@
+from __future__ import annotations
+import asyncio
+import copy
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Union,
+)
+
+from deepeval.dataset.golden import Golden, ConversationalGolden
+from deepeval.dataset.utils import (
+    convert_goldens_to_test_cases,
+    convert_convo_goldens_to_convo_test_cases,
+)
+from deepeval.errors import DeepEvalError
+from deepeval.metrics import (
+    BaseMetric,
+    BaseConversationalMetric,
+)
+from deepeval.metrics.utils import copy_metrics
+from deepeval.test_case import (
+    LLMTestCase,
+    ConversationalTestCase,
+    Turn,
+)
+from deepeval.prompt.prompt import Prompt
+
+from deepeval.optimizer.types import (
+    ModelCallback,
+    PromptConfiguration,
+    Objective,
+    MeanObjective,
+    ModuleId,
+)
+from deepeval.optimizer.scorer.base import BaseScorer
+from deepeval.optimizer.utils import (
+    validate_callback,
+    validate_metrics,
+    invoke_model_callback,
+    a_invoke_model_callback,
+)
+from deepeval.optimizer.scorer.utils import (
+    _measure_no_indicator,
+    _a_measure_no_indicator,
+)
+
+
+class Scorer(BaseScorer):
+    """
+    Scores prompts by running model_callback, building test cases,
+    running metrics, and aggregating scores.
+    """
+
+    DEFAULT_MODULE_ID: ModuleId = "__module__"
+
+    def __init__(
+        self,
+        model_callback: ModelCallback,
+        metrics: Union[List[BaseMetric], List[BaseConversationalMetric]],
+        max_concurrent: int,
+        throttle_seconds: float,
+        objective_scalar: Objective = MeanObjective(),
+    ):
+        self.model_callback = validate_callback(
+            component="Scorer",
+            model_callback=model_callback,
+        )
+        self.metrics = validate_metrics(component="Scorer", metrics=metrics)
+        self.objective_scalar = objective_scalar
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+        self._throttle = float(throttle_seconds)
+
+    ########################
+    # generation & scoring #
+    ########################
+
+    def generate(
+        self,
+        prompts_by_module: Dict[ModuleId, Prompt],
+        golden: Union[Golden, ConversationalGolden],
+    ) -> str:
+        module_id = self._select_module_id_from_prompts(prompts_by_module)
+        prompt = prompts_by_module.get(module_id) or next(
+            iter(prompts_by_module.values())
+        )
+
+        return invoke_model_callback(
+            model_callback=self.model_callback,
+            prompt=prompt,
+            golden=golden,
+        )
+
+    async def a_generate(
+        self,
+        prompts_by_module: Dict[ModuleId, Prompt],
+        golden: Union[Golden, ConversationalGolden],
+    ) -> str:
+        module_id = self._select_module_id_from_prompts(prompts_by_module)
+        prompt = prompts_by_module.get(module_id) or next(
+            iter(prompts_by_module.values())
+        )
+
+        return await a_invoke_model_callback(
+            model_callback=self.model_callback,
+            prompt=prompt,
+            golden=golden,
+        )
+
+    def score_pareto(
+        self,
+        prompt_configuration: PromptConfiguration,
+        d_pareto: Union[List[Golden], List[ConversationalGolden]],
+    ) -> List[float]:
+        return [
+            self._score_one(prompt_configuration, golden) for golden in d_pareto
+        ]
+
+    def score_minibatch(
+        self,
+        prompt_configuration: PromptConfiguration,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> float:
+        if not minibatch:
+            return 0.0
+
+        scores = [
+            self._score_one(prompt_configuration, golden)
+            for golden in minibatch
+        ]
+        return sum(scores) / len(scores)
+
+    def get_minibatch_feedback(
+        self,
+        prompt_configuration: PromptConfiguration,
+        module: ModuleId,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> str:
+        # default metric feedback (μ_f): concat metric.reason across minibatch and cap length
+        reasons: List[str] = []
+        for golden in minibatch:
+            actual = self.generate(prompt_configuration.prompts, golden)
+            test_case = self._golden_to_test_case(golden, actual)
+            for metric in copy_metrics(self.metrics):
+                _measure_no_indicator(metric=metric, test_case=test_case)
+                if metric.reason:
+                    reasons.append(str(metric.reason))
+        if not reasons:
+            return ""
+        unique: List[str] = []
+        seen = set()
+        for reason in reasons:
+            if reason not in seen:
+                unique.append(reason)
+                seen.add(reason)
+        return "\n---\n".join(
+            unique[:8]
+        )  # TODO: Make how much feedback configurable
+
+    async def a_score_pareto(
+        self,
+        prompt_configuration: PromptConfiguration,
+        d_pareto: Union[List[Golden], List[ConversationalGolden]],
+    ) -> List[float]:
+        tasks = [
+            self._bounded(self._a_score_one(prompt_configuration, golden))
+            for golden in d_pareto
+        ]
+        return await asyncio.gather(*tasks)
+
+    async def a_score_minibatch(
+        self,
+        prompt_configuration: PromptConfiguration,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> float:
+        tasks = [
+            self._bounded(self._a_score_one(prompt_configuration, golden))
+            for golden in minibatch
+        ]
+        scores = await asyncio.gather(*tasks)
+        return sum(scores) / len(scores) if scores else 0.0
+
+    async def a_get_minibatch_feedback(
+        self,
+        prompt_configuration: PromptConfiguration,
+        module: ModuleId,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> str:
+        async def reasons_one(golden) -> List[str]:
+            # Clone per task to avoid shared state
+            metrics = copy_metrics(self.metrics)
+            # metrics = self.metrics
+            actual = await self.a_generate(prompt_configuration.prompts, golden)
+            test_case = self._golden_to_test_case(golden, actual)
+            out: List[str] = []
+            for metric in metrics:
+                await _a_measure_no_indicator(metric, test_case)
+                if metric.reason:
+                    out.append(str(metric.reason))
+            return out
+
+        tasks = [self._bounded(reasons_one(golden)) for golden in minibatch]
+        nested = await asyncio.gather(*tasks)
+        reasons: List[str] = [reason for sub in nested for reason in sub]
+        if not reasons:
+            return ""
+        unique: List[str] = []
+        seen = set()
+        for reason in reasons:
+            if reason not in seen:
+                unique.append(reason)
+                seen.add(reason)
+        return "\n---\n".join(unique[:8])
+
+    ###################
+    # scoring helpers #
+    ###################
+
+    def _golden_to_test_case(
+        self,
+        golden: Union[Golden, ConversationalGolden],
+        actual: str,
+    ) -> Union[LLMTestCase, ConversationalTestCase]:
+        """Convert a golden + actual output into a test case for metrics."""
+        if isinstance(golden, Golden):
+            golden.actual_output = actual
+            return convert_goldens_to_test_cases([golden])[0]
+
+        if isinstance(golden, ConversationalGolden):
+            # Build turns with actual output as assistant response
+            turns: List[Turn] = list(golden.turns or [])
+            if turns and turns[-1].role == "assistant":
+                turns[-1] = Turn(role="assistant", content=actual)
+            elif turns:
+                turns.append(Turn(role="assistant", content=actual))
+            else:
+                turns = [
+                    Turn(role="assistant", content=actual),
+                ]
+
+            golden.turns = turns
+            return convert_convo_goldens_to_convo_test_cases([golden])[0]
+
+    async def _bounded(self, coro):
+        if self._semaphore is None:
+            return await coro
+        async with self._semaphore:
+            res = await coro
+        if self._throttle:
+            await asyncio.sleep(self._throttle)
+        return res
+
+    async def _a_score_one(
+        self,
+        prompt_configuration: PromptConfiguration,
+        golden: Union[Golden, ConversationalGolden],
+    ) -> float:
+        # Clone metrics to avoid shared-state
+        metrics = copy_metrics(self.metrics)
+        actual = await self.a_generate(prompt_configuration.prompts, golden)
+        test_case = self._golden_to_test_case(golden, actual)
+
+        per_metric: Dict[str, float] = {}
+        for metric in metrics:
+            score = await _a_measure_no_indicator(metric, test_case)
+            per_metric[metric.__class__.__name__] = float(score)
+        return self.objective_scalar.scalarize(per_metric)
+
+    def _score_one(
+        self,
+        prompt_configuration: PromptConfiguration,
+        golden: Union[Golden, ConversationalGolden],
+    ) -> float:
+        metrics = copy_metrics(self.metrics)
+        actual = self.generate(prompt_configuration.prompts, golden)
+        test_case = self._golden_to_test_case(golden, actual)
+
+        per_metric: Dict[str, float] = {}
+        for metric in metrics:
+            score = _measure_no_indicator(metric, test_case)
+            per_metric[metric.__class__.__name__] = float(score)
+        return self.objective_scalar.scalarize(per_metric)
+
+    def _select_module_id_from_prompts(
+        self, prompts_by_module: Dict[ModuleId, Prompt]
+    ) -> ModuleId:
+        """
+        Default module selection strategy:
+
+        - Prefer the synthetic '__module__' key when present
+        - Otherwise fall back to the first key in prompts_by_module.
+
+        Assumes `prompts_by_module` is non-empty; callers should validate that.
+        """
+        if self.DEFAULT_MODULE_ID in prompts_by_module:
+            return self.DEFAULT_MODULE_ID
+
+        # At this point we expect at least one key.
+        try:
+            return next(iter(prompts_by_module.keys()))
+        except StopIteration:
+            raise DeepEvalError(
+                "Scorer._select_module_id_from_prompts(...) "
+                "received an empty `prompts_by_module`. At least one Prompt is required."
+            )
+
+    def select_module(
+        self, prompt_configuration: PromptConfiguration
+    ) -> ModuleId:
+        return self._select_module_id_from_prompts(prompt_configuration.prompts)
+
+    async def a_select_module(
+        self, prompt_configuration: PromptConfiguration
+    ) -> ModuleId:
+        return self.select_module(prompt_configuration)

deepeval/optimizer/scorer/utils.py

@@ -0,0 +1,30 @@
+import inspect
+from typing import Callable, Union
+
+from deepeval.metrics import BaseConversationalMetric, BaseMetric
+from deepeval.test_case import ConversationalTestCase, LLMTestCase
+
+
+def _build_measure_kwargs(func: Callable) -> dict:
+    params = inspect.signature(func).parameters
+    kwargs = {}
+    for key in ("_show_indicator", "_in_component", "_log_metric_to_confident"):
+        if key in params:
+            kwargs[key] = False
+    return kwargs
+
+
+def _measure_no_indicator(
+    metric: Union[BaseMetric, BaseConversationalMetric],
+    test_case: Union[LLMTestCase, ConversationalTestCase],
+):
+    kwargs = _build_measure_kwargs(metric.measure)
+    return metric.measure(test_case, **kwargs)
+
+
+async def _a_measure_no_indicator(
+    metric: Union[BaseMetric, BaseConversationalMetric],
+    test_case: Union[LLMTestCase, ConversationalTestCase],
+):
+    kwargs = _build_measure_kwargs(metric.a_measure)
+    return await metric.a_measure(test_case, **kwargs)