deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/optimizer/scorer/scorer.py

@@ -0,0 +1,316 @@
+from __future__ import annotations
+import asyncio
+import copy
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Union,
+)
+
+from deepeval.dataset.golden import Golden, ConversationalGolden
+from deepeval.dataset.utils import (
+    convert_goldens_to_test_cases,
+    convert_convo_goldens_to_convo_test_cases,
+)
+from deepeval.errors import DeepEvalError
+from deepeval.metrics import (
+    BaseMetric,
+    BaseConversationalMetric,
+)
+from deepeval.metrics.utils import copy_metrics
+from deepeval.test_case import (
+    LLMTestCase,
+    ConversationalTestCase,
+    Turn,
+)
+from deepeval.prompt.prompt import Prompt
+
+from deepeval.optimizer.types import (
+    ModelCallback,
+    PromptConfiguration,
+    Objective,
+    MeanObjective,
+    ModuleId,
+)
+from deepeval.optimizer.scorer.base import BaseScorer
+from deepeval.optimizer.utils import (
+    validate_callback,
+    validate_metrics,
+    invoke_model_callback,
+    a_invoke_model_callback,
+)
+from deepeval.optimizer.scorer.utils import (
+    _measure_no_indicator,
+    _a_measure_no_indicator,
+)
+
+
+class Scorer(BaseScorer):
+    """
+    Scores prompts by running model_callback, building test cases,
+    running metrics, and aggregating scores.
+    """
+
+    DEFAULT_MODULE_ID: ModuleId = "__module__"
+
+    def __init__(
+        self,
+        model_callback: ModelCallback,
+        metrics: Union[List[BaseMetric], List[BaseConversationalMetric]],
+        max_concurrent: int,
+        throttle_seconds: float,
+        objective_scalar: Objective = MeanObjective(),
+    ):
+        self.model_callback = validate_callback(
+            component="Scorer",
+            model_callback=model_callback,
+        )
+        self.metrics = validate_metrics(component="Scorer", metrics=metrics)
+        self.objective_scalar = objective_scalar
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+        self._throttle = float(throttle_seconds)
+
+    ########################
+    # generation & scoring #
+    ########################
+
+    def generate(
+        self,
+        prompts_by_module: Dict[ModuleId, Prompt],
+        golden: Union[Golden, ConversationalGolden],
+    ) -> str:
+        module_id = self._select_module_id_from_prompts(prompts_by_module)
+        prompt = prompts_by_module.get(module_id) or next(
+            iter(prompts_by_module.values())
+        )
+
+        return invoke_model_callback(
+            model_callback=self.model_callback,
+            prompt=prompt,
+            golden=golden,
+        )
+
+    async def a_generate(
+        self,
+        prompts_by_module: Dict[ModuleId, Prompt],
+        golden: Union[Golden, ConversationalGolden],
+    ) -> str:
+        module_id = self._select_module_id_from_prompts(prompts_by_module)
+        prompt = prompts_by_module.get(module_id) or next(
+            iter(prompts_by_module.values())
+        )
+
+        return await a_invoke_model_callback(
+            model_callback=self.model_callback,
+            prompt=prompt,
+            golden=golden,
+        )
+
+    def score_pareto(
+        self,
+        prompt_configuration: PromptConfiguration,
+        d_pareto: Union[List[Golden], List[ConversationalGolden]],
+    ) -> List[float]:
+        return [
+            self._score_one(prompt_configuration, golden) for golden in d_pareto
+        ]
+
+    def score_minibatch(
+        self,
+        prompt_configuration: PromptConfiguration,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> float:
+        if not minibatch:
+            return 0.0
+
+        scores = [
+            self._score_one(prompt_configuration, golden)
+            for golden in minibatch
+        ]
+        return sum(scores) / len(scores)
+
+    def get_minibatch_feedback(
+        self,
+        prompt_configuration: PromptConfiguration,
+        module: ModuleId,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> str:
+        # default metric feedback (μ_f): concat metric.reason across minibatch and cap length
+        reasons: List[str] = []
+        for golden in minibatch:
+            actual = self.generate(prompt_configuration.prompts, golden)
+            test_case = self._golden_to_test_case(golden, actual)
+            for metric in copy_metrics(self.metrics):
+                _measure_no_indicator(metric=metric, test_case=test_case)
+                if metric.reason:
+                    reasons.append(str(metric.reason))
+        if not reasons:
+            return ""
+        unique: List[str] = []
+        seen = set()
+        for reason in reasons:
+            if reason not in seen:
+                unique.append(reason)
+                seen.add(reason)
+        return "\n---\n".join(
+            unique[:8]
+        )  # TODO: Make how much feedback configurable
+
+    async def a_score_pareto(
+        self,
+        prompt_configuration: PromptConfiguration,
+        d_pareto: Union[List[Golden], List[ConversationalGolden]],
+    ) -> List[float]:
+        tasks = [
+            self._bounded(self._a_score_one(prompt_configuration, golden))
+            for golden in d_pareto
+        ]
+        return await asyncio.gather(*tasks)
+
+    async def a_score_minibatch(
+        self,
+        prompt_configuration: PromptConfiguration,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> float:
+        tasks = [
+            self._bounded(self._a_score_one(prompt_configuration, golden))
+            for golden in minibatch
+        ]
+        scores = await asyncio.gather(*tasks)
+        return sum(scores) / len(scores) if scores else 0.0
+
+    async def a_get_minibatch_feedback(
+        self,
+        prompt_configuration: PromptConfiguration,
+        module: ModuleId,
+        minibatch: Union[List[Golden], List[ConversationalGolden]],
+    ) -> str:
+        async def reasons_one(golden) -> List[str]:
+            # Clone per task to avoid shared state
+            metrics = copy_metrics(self.metrics)
+            # metrics = self.metrics
+            actual = await self.a_generate(prompt_configuration.prompts, golden)
+            test_case = self._golden_to_test_case(golden, actual)
+            out: List[str] = []
+            for metric in metrics:
+                await _a_measure_no_indicator(metric, test_case)
+                if metric.reason:
+                    out.append(str(metric.reason))
+            return out
+
+        tasks = [self._bounded(reasons_one(golden)) for golden in minibatch]
+        nested = await asyncio.gather(*tasks)
+        reasons: List[str] = [reason for sub in nested for reason in sub]
+        if not reasons:
+            return ""
+        unique: List[str] = []
+        seen = set()
+        for reason in reasons:
+            if reason not in seen:
+                unique.append(reason)
+                seen.add(reason)
+        return "\n---\n".join(unique[:8])
+
+    ###################
+    # scoring helpers #
+    ###################
+
+    def _golden_to_test_case(
+        self,
+        golden: Union[Golden, ConversationalGolden],
+        actual: str,
+    ) -> Union[LLMTestCase, ConversationalTestCase]:
+        """Convert a golden + actual output into a test case for metrics."""
+        if isinstance(golden, Golden):
+            golden.actual_output = actual
+            return convert_goldens_to_test_cases([golden])[0]
+
+        if isinstance(golden, ConversationalGolden):
+            # Build turns with actual output as assistant response
+            turns: List[Turn] = list(golden.turns or [])
+            if turns and turns[-1].role == "assistant":
+                turns[-1] = Turn(role="assistant", content=actual)
+            elif turns:
+                turns.append(Turn(role="assistant", content=actual))
+            else:
+                turns = [
+                    Turn(role="assistant", content=actual),
+                ]
+
+            golden.turns = turns
+            return convert_convo_goldens_to_convo_test_cases([golden])[0]
+
+    async def _bounded(self, coro):
+        if self._semaphore is None:
+            return await coro
+        async with self._semaphore:
+            res = await coro
+        if self._throttle:
+            await asyncio.sleep(self._throttle)
+        return res
+
+    async def _a_score_one(
+        self,
+        prompt_configuration: PromptConfiguration,
+        golden: Union[Golden, ConversationalGolden],
+    ) -> float:
+        # Clone metrics to avoid shared-state
+        metrics = copy_metrics(self.metrics)
+        actual = await self.a_generate(prompt_configuration.prompts, golden)
+        test_case = self._golden_to_test_case(golden, actual)
+
+        per_metric: Dict[str, float] = {}
+        for metric in metrics:
+            score = await _a_measure_no_indicator(metric, test_case)
+            per_metric[metric.__class__.__name__] = float(score)
+        return self.objective_scalar.scalarize(per_metric)
+
+    def _score_one(
+        self,
+        prompt_configuration: PromptConfiguration,
+        golden: Union[Golden, ConversationalGolden],
+    ) -> float:
+        metrics = copy_metrics(self.metrics)
+        actual = self.generate(prompt_configuration.prompts, golden)
+        test_case = self._golden_to_test_case(golden, actual)
+
+        per_metric: Dict[str, float] = {}
+        for metric in metrics:
+            score = _measure_no_indicator(metric, test_case)
+            per_metric[metric.__class__.__name__] = float(score)
+        return self.objective_scalar.scalarize(per_metric)
+
+    def _select_module_id_from_prompts(
+        self, prompts_by_module: Dict[ModuleId, Prompt]
+    ) -> ModuleId:
+        """
+        Default module selection strategy:
+
+        - Prefer the synthetic '__module__' key when present
+        - Otherwise fall back to the first key in prompts_by_module.
+
+        Assumes `prompts_by_module` is non-empty; callers should validate that.
+        """
+        if self.DEFAULT_MODULE_ID in prompts_by_module:
+            return self.DEFAULT_MODULE_ID
+
+        # At this point we expect at least one key.
+        try:
+            return next(iter(prompts_by_module.keys()))
+        except StopIteration:
+            raise DeepEvalError(
+                "Scorer._select_module_id_from_prompts(...) "
+                "received an empty `prompts_by_module`. At least one Prompt is required."
+            )
+
+    def select_module(
+        self, prompt_configuration: PromptConfiguration
+    ) -> ModuleId:
+        return self._select_module_id_from_prompts(prompt_configuration.prompts)
+
+    async def a_select_module(
+        self, prompt_configuration: PromptConfiguration
+    ) -> ModuleId:
+        return self.select_module(prompt_configuration)

deepeval/optimizer/scorer/utils.py

@@ -0,0 +1,30 @@
+import inspect
+from typing import Callable, Union
+
+from deepeval.metrics import BaseConversationalMetric, BaseMetric
+from deepeval.test_case import ConversationalTestCase, LLMTestCase
+
+
+def _build_measure_kwargs(func: Callable) -> dict:
+    params = inspect.signature(func).parameters
+    kwargs = {}
+    for key in ("_show_indicator", "_in_component", "_log_metric_to_confident"):
+        if key in params:
+            kwargs[key] = False
+    return kwargs
+
+
+def _measure_no_indicator(
+    metric: Union[BaseMetric, BaseConversationalMetric],
+    test_case: Union[LLMTestCase, ConversationalTestCase],
+):
+    kwargs = _build_measure_kwargs(metric.measure)
+    return metric.measure(test_case, **kwargs)
+
+
+async def _a_measure_no_indicator(
+    metric: Union[BaseMetric, BaseConversationalMetric],
+    test_case: Union[LLMTestCase, ConversationalTestCase],
+):
+    kwargs = _build_measure_kwargs(metric.a_measure)
+    return await metric.a_measure(test_case, **kwargs)

deepeval/optimizer/types.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+import uuid
+from abc import ABC, abstractmethod
+
+from dataclasses import dataclass
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    TypedDict,
+    TYPE_CHECKING,
+    Union,
+)
+from enum import Enum
+from pydantic import BaseModel, ConfigDict
+
+from deepeval.prompt.prompt import Prompt
+
+if TYPE_CHECKING:
+    from deepeval.dataset.golden import Golden, ConversationalGolden
+
+PromptConfigurationId = str
+ModuleId = str
+ScoreVector = List[float]  # scores per instance on D_pareto, aligned order
+ScoreTable = Dict[PromptConfigurationId, ScoreVector]
+
+# Type alias for model callback function
+ModelCallback = Callable[[Prompt, Union["Golden", "ConversationalGolden"]], str]
+
+
+@dataclass
+class PromptConfiguration:
+    id: PromptConfigurationId
+    parent: Optional[PromptConfigurationId]
+    prompts: Dict[ModuleId, Prompt]
+
+    @staticmethod
+    def new(
+        prompts: Dict[ModuleId, Prompt],
+        parent: Optional[PromptConfigurationId] = None,
+    ) -> "PromptConfiguration":
+        return PromptConfiguration(
+            id=str(uuid.uuid4()), parent=parent, prompts=dict(prompts)
+        )
+
+
+class RunnerStatusType(str, Enum):
+    """Status events emitted by optimization runners."""
+
+    PROGRESS = "progress"
+    TIE = "tie"
+    ERROR = "error"
+
+
+# Type alias for status callback function
+RunnerStatusCallback = Callable[..., None]
+
+
+class Objective(ABC):
+    """Strategy for reducing scores per-metric to a single scalar value.
+
+    Implementations receive a mapping from metric name to score
+    (for example, {"AnswerRelevancyMetric": 0.82}) and return a
+    single float used for comparisons inside the optimizer.
+    """
+
+    @abstractmethod
+    def scalarize(self, scores_by_metric: Dict[str, float]) -> float:
+        raise NotImplementedError
+
+
+class MeanObjective(Objective):
+    """Default scalarizer: unweighted arithmetic mean.
+
+    - If `scores_by_metric` is non-empty, returns the arithmetic
+      mean of all metric scores.
+    - If `scores_by_metric` is empty, returns 0.0.
+    """
+
+    def scalarize(self, scores_by_metric: Dict[str, float]) -> float:
+        if not scores_by_metric:
+            return 0.0
+        return sum(scores_by_metric.values()) / len(scores_by_metric)
+
+
+class WeightedObjective(Objective):
+    """
+    Objective that scales each metric's score by a user-provided weight and sums them.
+
+    - `weights_by_metric` keys should match the names of the metrics passed to the
+      metric class names passed to the PromptOptimizer.
+    - Metrics not present in `weights_by_metric` receive `default_weight`.
+      This makes it easy to emphasize a subset of metrics while keeping
+      everything else at a baseline weight of 1.0, e.g.:
+
+          WeightedObjective({"AnswerRelevancyMetric": 2.0})
+
+      which treats AnswerRelevancy as 2x as important as the other metrics.
+    """
+
+    def __init__(
+        self,
+        weights_by_metric: Optional[Dict[str, float]] = None,
+        default_weight: float = 1.0,
+    ):
+        self.weights_by_metric: Dict[str, float] = dict(weights_by_metric or {})
+        self.default_weight: float = float(default_weight)
+
+    def scalarize(self, scores_by_metric: Dict[str, float]) -> float:
+        return sum(
+            self.weights_by_metric.get(name, self.default_weight) * score
+            for name, score in scores_by_metric.items()
+        )
+
+
+class AcceptedIterationDict(TypedDict):
+    parent: PromptConfigurationId
+    child: PromptConfigurationId
+    module: ModuleId
+    before: float
+    after: float
+
+
+class AcceptedIteration(BaseModel):
+    parent: str
+    child: str
+    module: str
+    before: float
+    after: float
+
+
+class PromptConfigSnapshot(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    parent: Optional[str]
+    prompts: Dict[str, Prompt]
+
+
+class OptimizationReport(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    optimization_id: str
+    best_id: str
+    accepted_iterations: List[AcceptedIteration]
+    pareto_scores: Dict[str, List[float]]
+    parents: Dict[str, Optional[str]]
+    prompt_configurations: Dict[str, PromptConfigSnapshot]