deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/optimization/types.py

@@ -1,361 +0,0 @@
-from __future__ import annotations
-import uuid
-
-from dataclasses import dataclass
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    List,
-    Literal,
-    Optional,
-    Protocol,
-    TYPE_CHECKING,
-    TypedDict,
-    Tuple,
-    Union,
-)
-from enum import Enum
-from pydantic import BaseModel as PydanticBaseModel, Field, AliasChoices
-
-from deepeval.prompt.prompt import Prompt
-from deepeval.models.base_model import DeepEvalBaseLLM
-
-
-if TYPE_CHECKING:
-    from deepeval.dataset.golden import Golden, ConversationalGolden
-
-PromptConfigurationId = str
-ModuleId = str
-ScoreVector = List[float]  # scores per instance on D_pareto, aligned order
-ScoreTable = Dict[PromptConfigurationId, ScoreVector]
-
-
-@dataclass
-class PromptConfiguration:
-    id: PromptConfigurationId
-    parent: Optional[PromptConfigurationId]
-    prompts: Dict[ModuleId, Prompt]
-
-    @staticmethod
-    def new(
-        prompts: Dict[ModuleId, Prompt],
-        parent: Optional[PromptConfigurationId] = None,
-    ) -> "PromptConfiguration":
-        return PromptConfiguration(
-            id=str(uuid.uuid4()), parent=parent, prompts=dict(prompts)
-        )
-
-
-class ScoringAdapter(Protocol):
-    """
-    Scoring adapter contract used by optimization runners.
-
-    Runners call into this adapter to:
-    - compute scores per-instance on some subset (score_on_pareto),
-    - compute minibatch means for selection and acceptance,
-    - generate feedback text used by the PromptRewriter.
-    """
-
-    # Sync
-    def score_on_pareto(
-        self,
-        prompt_configuration: PromptConfiguration,
-        d_pareto: Union[List[Golden], List[ConversationalGolden]],
-    ) -> ScoreVector:
-        """Return per-instance scores on D_pareto."""
-        ...
-
-    def minibatch_score(
-        self,
-        prompt_configuration: PromptConfiguration,
-        minibatch: Union[List[Golden], List[ConversationalGolden]],
-    ) -> float:
-        """Return average score μ on a minibatch from D_feedback."""
-        ...
-
-    def minibatch_feedback(
-        self,
-        prompt_configuration: PromptConfiguration,
-        module: ModuleId,
-        minibatch: Union[List[Golden], List[ConversationalGolden]],
-    ) -> str:
-        """Return μ_f text for the module (metric.reason + traces, etc.)."""
-        ...
-
-    def select_module(
-        self, prompt_configuration: PromptConfiguration
-    ) -> ModuleId:
-        """Pick a module to mutate."""
-        ...
-
-    # Async
-    async def a_score_on_pareto(
-        self,
-        prompt_configuration: PromptConfiguration,
-        d_pareto: Union[List[Golden], List[ConversationalGolden]],
-    ) -> ScoreVector: ...
-    async def a_minibatch_score(
-        self,
-        prompt_configuration: PromptConfiguration,
-        minibatch: Union[List[Golden], List[ConversationalGolden]],
-    ) -> float: ...
-    async def a_minibatch_feedback(
-        self,
-        prompt_configuration: PromptConfiguration,
-        module: ModuleId,
-        minibatch: Union[List[Golden], List[ConversationalGolden]],
-    ) -> str: ...
-    async def a_select_module(
-        self, prompt_configuration: PromptConfiguration
-    ) -> ModuleId: ...
-
-
-class PromptRewriterProtocol(Protocol):
-    def rewrite(
-        self,
-        *,
-        module_id: ModuleId,
-        model: Optional[DeepEvalBaseLLM] = None,
-        model_schema: Optional[PydanticBaseModel] = None,
-        model_callback: Optional[
-            Callable[
-                ...,
-                Union[
-                    str,
-                    Dict,
-                    Tuple[Union[str, Dict], float],
-                ],
-            ]
-        ] = None,
-        old_prompt: Prompt,
-        feedback_text: str,
-    ) -> Prompt: ...
-
-    async def a_rewrite(
-        self,
-        *,
-        module_id: ModuleId,
-        model: Optional[DeepEvalBaseLLM] = None,
-        model_schema: Optional[PydanticBaseModel] = None,
-        model_callback: Optional[
-            Callable[
-                ...,
-                Union[
-                    str,
-                    Dict,
-                    Tuple[Union[str, Dict], float],
-                ],
-            ]
-        ] = None,
-        old_prompt: Prompt,
-        feedback_text: str,
-    ) -> Prompt: ...
-
-
-class RunnerStatusType(str, Enum):
-    """Status events emitted by optimization runners."""
-
-    PROGRESS = "progress"
-    TIE = "tie"
-    ERROR = "error"
-
-
-class RunnerStatusCallbackProtocol(Protocol):
-    def __call__(
-        self,
-        kind: RunnerStatusType,
-        *,
-        detail: str,
-        step_index: Optional[int] = None,
-        total_steps: Optional[int] = None,
-    ) -> None: ...
-
-
-class RunnerProtocol(Protocol):
-    """
-    Contract for prompt optimization runners used by PromptOptimizer.
-
-    Runners are responsible for executing the optimization algorithm
-    and returning an optimized Prompt plus a report dict.
-    """
-
-    # status_callback is injected by PromptOptimizer
-    # A runner may call this to report:
-    # progress, ties, or errors during execution.
-    status_callback: Optional[RunnerStatusCallbackProtocol]
-    model_callback: Optional[
-        Callable[
-            ...,
-            Union[
-                str,
-                Dict,
-                Tuple[Union[str, Dict], float],
-            ],
-        ]
-    ]
-
-    scoring_adapter: Optional[ScoringAdapter]
-
-    def execute(
-        self,
-        *,
-        prompt: Prompt,
-        goldens: Union[List["Golden"], List["ConversationalGolden"]],
-    ) -> Tuple[Prompt, Dict]: ...
-
-    async def a_execute(
-        self,
-        *,
-        prompt: Prompt,
-        goldens: Union[List["Golden"], List["ConversationalGolden"]],
-    ) -> Tuple[Prompt, Dict]: ...
-
-
-class Objective(Protocol):
-    """Strategy for reducing scores per-metric to a single scalar value.
-
-    Implementations receive a mapping from metric name to score
-    (for example, {"AnswerRelevancyMetric": 0.82}) and return a
-    single float used for comparisons inside the optimizer.
-    """
-
-    def scalarize(self, scores_by_metric: Dict[str, float]) -> float: ...
-
-
-class MeanObjective(Objective):
-    """Default scalarizer: unweighted arithmetic mean.
-
-    - If `scores_by_metric` is non-empty, returns the arithmetic
-      mean of all metric scores.
-    - If `scores_by_metric` is empty, returns 0.0.
-    """
-
-    def scalarize(self, scores_by_metric: Dict[str, float]) -> float:
-        if not scores_by_metric:
-            return 0.0
-        return sum(scores_by_metric.values()) / len(scores_by_metric)
-
-
-class WeightedObjective(Objective):
-    """
-    Objective that scales each metric's score by a user-provided weight and sums them.
-
-    - `weights_by_metric` keys should match the names of the metrics passed to the
-      metric class names passed to the PromptOptimizer.
-    - Metrics not present in `weights_by_metric` receive `default_weight`.
-      This makes it easy to emphasize a subset of metrics while keeping
-      everything else at a baseline weight of 1.0, e.g.:
-
-          WeightedObjective({"AnswerRelevancyMetric": 2.0})
-
-      which treats AnswerRelevancy as 2x as important as the other metrics.
-    """
-
-    def __init__(
-        self,
-        weights_by_metric: Optional[Dict[str, float]] = None,
-        default_weight: float = 1.0,
-    ):
-        self.weights_by_metric: Dict[str, float] = dict(weights_by_metric or {})
-        self.default_weight: float = float(default_weight)
-
-    def scalarize(self, scores_by_metric: Dict[str, float]) -> float:
-        return sum(
-            self.weights_by_metric.get(name, self.default_weight) * score
-            for name, score in scores_by_metric.items()
-        )
-
-
-@dataclass
-class MetricInfo:
-    name: str
-    rubric: Optional[str] = None
-
-
-class AcceptedIterationDict(TypedDict):
-    parent: PromptConfigurationId
-    child: PromptConfigurationId
-    module: ModuleId
-    before: float
-    after: float
-
-
-class AcceptedIteration(PydanticBaseModel):
-    parent: str
-    child: str
-    module: str
-    before: float
-    after: float
-
-
-class PromptMessageSnapshot(PydanticBaseModel):
-    role: str
-    content: str
-
-
-class PromptModuleSnapshot(PydanticBaseModel):
-    type: Literal["TEXT", "LIST"]
-    # Only used when type == "TEXT"
-    text_template: Optional[str] = None
-    # Only used when type == "LIST"
-    messages: Optional[List[PromptMessageSnapshot]] = None
-
-
-class PromptConfigSnapshot(PydanticBaseModel):
-    parent: Optional[str]
-    prompts: Dict[str, PromptModuleSnapshot]
-
-
-@dataclass
-class OptimizationResult:
-    optimization_id: str
-    best_id: PromptConfigurationId
-    accepted_iterations: List[Dict]
-    pareto_scores: Dict[PromptConfigurationId, List[float]]
-    parents: Dict[PromptConfigurationId, Optional[PromptConfigurationId]]
-    prompt_configurations: Dict[PromptConfigurationId, Dict[str, Any]]
-
-    def as_dict(self) -> Dict:
-        return dict(
-            optimization_id=self.optimization_id,
-            best_id=self.best_id,
-            accepted_iterations=self.accepted_iterations,
-            pareto_scores=self.pareto_scores,
-            parents=self.parents,
-            prompt_configurations=self.prompt_configurations,
-        )
-
-
-class OptimizationReport(PydanticBaseModel):
-    optimization_id: str = Field(
-        alias="optimizationId",
-        validation_alias=AliasChoices("optimizationId", "optimization_id"),
-    )
-    best_id: str = Field(
-        alias="bestId",
-        validation_alias=AliasChoices("bestId", "best_id"),
-    )
-    accepted_iterations: list[AcceptedIteration] = Field(
-        default_factory=list,
-        alias="acceptedIterations",
-        validation_alias=AliasChoices(
-            "acceptedIterations", "accepted_iterations"
-        ),
-    )
-    pareto_scores: dict[str, list[float]] = Field(
-        alias="paretoScores",
-        validation_alias=AliasChoices("paretoScores", "pareto_scores"),
-    )
-    parents: dict[str, str | None]
-    prompt_configurations: dict[str, PromptConfigSnapshot] = Field(
-        alias="promptConfigurations",
-        validation_alias=AliasChoices(
-            "promptConfigurations", "prompt_configurations"
-        ),
-    )
-
-    @classmethod
-    def from_runtime(cls, result: dict) -> "OptimizationReport":
-        # accepts the dict from OptimizationResult.as_dict()
-        return cls(**result)

deepeval/test_case/mllm_test_case.py

@@ -1,170 +0,0 @@
-from typing import List, Optional, Dict, Union
-from urllib.parse import urlparse, unquote
-from dataclasses import dataclass, field
-from enum import Enum
-import mimetypes
-import base64
-import os
-
-from deepeval.test_case import ToolCall
-
-
-@dataclass
-class MLLMImage:
-    dataBase64: Optional[str] = None
-    mimeType: Optional[str] = None
-    url: Optional[str] = None
-    local: Optional[bool] = None
-    filename: Optional[str] = None
-
-    def __post_init__(self):
-
-        if self.url and self.dataBase64:
-            raise ValueError(
-                "You cannot provide both 'url' and 'dataBase64' at the same time when creating an MLLMImage."
-            )
-
-        if not self.url and not self.dataBase64:
-            raise ValueError(
-                "You must provide either a 'url' or both 'dataBase64' and 'mimeType' to create an MLLMImage."
-            )
-
-        if self.dataBase64 is not None:
-            if self.mimeType is None:
-                raise ValueError(
-                    "mimeType must be provided when initializing from Base64 data."
-                )
-        else:
-            is_local = self.is_local_path(self.url)
-            if self.local is not None:
-                assert self.local == is_local, "Local path mismatch"
-            else:
-                self.local = is_local
-
-            # compute filename, mime_type, and Base64 data
-            if self.local:
-                path = self.process_url(self.url)
-                self.filename = os.path.basename(path)
-                self.mimeType = (
-                    mimetypes.guess_type(path)[0] or "application/octet-stream"
-                )
-                with open(path, "rb") as f:
-                    raw = f.read()
-                self.dataBase64 = base64.b64encode(raw).decode("ascii")
-            else:
-                self.filename = None
-                self.mimeType = None
-                self.dataBase64 = None
-
-    @staticmethod
-    def process_url(url: str) -> str:
-        if os.path.exists(url):
-            return url
-        parsed = urlparse(url)
-        if parsed.scheme == "file":
-            raw_path = (
-                f"//{parsed.netloc}{parsed.path}"
-                if parsed.netloc
-                else parsed.path
-            )
-            path = unquote(raw_path)
-            return path
-        return url
-
-    @staticmethod
-    def is_local_path(url: str) -> bool:
-        if os.path.exists(url):
-            return True
-        parsed = urlparse(url)
-        if parsed.scheme == "file":
-            raw_path = (
-                f"//{parsed.netloc}{parsed.path}"
-                if parsed.netloc
-                else parsed.path
-            )
-            path = unquote(raw_path)
-            return os.path.exists(path)
-        return False
-
-    def as_data_uri(self) -> Optional[str]:
-        """Return the image as a data URI string, if Base64 data is available."""
-        if not self.dataBase64 or not self.mimeType:
-            return None
-        return f"data:{self.mimeType};base64,{self.dataBase64}"
-
-
-class MLLMTestCaseParams(Enum):
-    INPUT = "input"
-    ACTUAL_OUTPUT = "actual_output"
-    EXPECTED_OUTPUT = "expected_output"
-    CONTEXT = "context"
-    RETRIEVAL_CONTEXT = "retrieval_context"
-    TOOLS_CALLED = "tools_called"
-    EXPECTED_TOOLS = "expected_tools"
-
-
-@dataclass
-class MLLMTestCase:
-    input: List[Union[str, MLLMImage]]
-    actual_output: List[Union[str, MLLMImage]]
-    expected_output: Optional[List[Union[str, MLLMImage]]] = None
-    context: Optional[List[Union[str, MLLMImage]]] = None
-    retrieval_context: Optional[List[Union[str, MLLMImage]]] = None
-    additional_metadata: Optional[Dict] = None
-    comments: Optional[str] = None
-    tools_called: Optional[List[ToolCall]] = None
-    expected_tools: Optional[List[ToolCall]] = None
-    token_cost: Optional[float] = None
-    completion_time: Optional[float] = None
-    name: Optional[str] = field(default=None)
-    _dataset_rank: Optional[int] = field(default=None, repr=False)
-    _dataset_alias: Optional[str] = field(default=None, repr=False)
-    _dataset_id: Optional[str] = field(default=None, repr=False)
-
-    def __post_init__(self):
-        # Ensure `expected_output` is None or a list of strings or MLLMImage instances
-        if self.expected_output is not None:
-            if not isinstance(self.expected_output, list) or not all(
-                isinstance(item, (str, MLLMImage))
-                for item in self.expected_output
-            ):
-                raise TypeError(
-                    "'expected_output' must be None or a list of strings or MLLMImage instances"
-                )
-
-        # Ensure `context` is None or a list of strings or MLLMImage instances
-        if self.context is not None:
-            if not isinstance(self.context, list) or not all(
-                isinstance(item, (str, MLLMImage)) for item in self.context
-            ):
-                raise TypeError(
-                    "'context' must be None or a list of strings or MLLMImage instances"
-                )
-
-        # Ensure `retrieval_context` is None or a list of strings or MLLMImage instances
-        if self.retrieval_context is not None:
-            if not isinstance(self.retrieval_context, list) or not all(
-                isinstance(item, (str, MLLMImage))
-                for item in self.retrieval_context
-            ):
-                raise TypeError(
-                    "'retrieval_context' must be None or a list of strings or MLLMImage instances"
-                )
-
-        # Ensure `tools_called` is None or a list of strings
-        if self.tools_called is not None:
-            if not isinstance(self.tools_called, list) or not all(
-                isinstance(item, ToolCall) for item in self.tools_called
-            ):
-                raise TypeError(
-                    "'tools_called' must be None or a list of `ToolCall`"
-                )
-
-        # Ensure `expected_tools` is None or a list of strings
-        if self.expected_tools is not None:
-            if not isinstance(self.expected_tools, list) or not all(
-                isinstance(item, ToolCall) for item in self.expected_tools
-            ):
-                raise TypeError(
-                    "'expected_tools' must be None or a list of `ToolCall`"
-                )