deepeval 3.5.6__tar.gz → 3.5.8__tar.gz

{deepeval-3.5.6 → deepeval-3.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.6
+Version: 3.5.8
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -17,6 +17,7 @@ Requires-Dist: anthropic
 Requires-Dist: click (>=8.0.0,<8.3.0)
 Requires-Dist: google-genai (>=1.9.0,<2.0.0)
 Requires-Dist: grpcio (>=1.67.1,<2.0.0)
+Requires-Dist: jinja2
 Requires-Dist: nest_asyncio
 Requires-Dist: ollama
 Requires-Dist: openai

deepeval-3.5.8/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.5.8"

{deepeval-3.5.6 → deepeval-3.5.8}/deepeval/dataset/dataset.py

@@ -458,6 +458,8 @@ class EvaluationDataset:
         tools_called_col_delimiter: str = ";",
         expected_tools_col_name: Optional[str] = "expected_tools",
         expected_tools_col_delimiter: str = ";",
+        comments_key_name: str = "comments",
+        name_key_name: str = "name",
         source_file_col_name: Optional[str] = None,
         additional_metadata_col_name: Optional[str] = None,
         scenario_col_name: Optional[str] = "scenario",
@@ -526,6 +528,8 @@ class EvaluationDataset:
                 df, expected_tools_col_name, default=""
             )
         ]
+        comments = get_column_data(df, comments_key_name)
+        name = get_column_data(df, name_key_name)
         source_files = get_column_data(df, source_file_col_name)
         additional_metadatas = [
             ast.literal_eval(metadata) if metadata else None
@@ -546,6 +550,8 @@ class EvaluationDataset:
             retrieval_context,
             tools_called,
             expected_tools,
+            comments,
+            name,
             source_file,
             additional_metadata,
             scenario,
@@ -560,6 +566,8 @@ class EvaluationDataset:
             retrieval_contexts,
             tools_called,
             expected_tools,
+            comments,
+            name,
             source_files,
             additional_metadatas,
             scenarios,
@@ -569,7 +577,7 @@ class EvaluationDataset:
         ):
             if scenario:
                 self._multi_turn = True
-                parsed_turns = parse_turns(turns)
+                parsed_turns = parse_turns(turns) if turns else []
                 self.goldens.append(
                     ConversationalGolden(
                         scenario=scenario,
@@ -577,6 +585,8 @@ class EvaluationDataset:
                         expected_outcome=expected_outcome,
                         user_description=user_description,
                         context=context,
+                        comments=comments,
+                        name=name,
                     )
                 )
             else:
@@ -592,6 +602,8 @@ class EvaluationDataset:
                         expected_tools=expected_tools,
                         additional_metadata=additional_metadata,
                         source_file=source_file,
+                        comments=comments,
+                        name=name,
                     )
                 )
 
@@ -605,6 +617,8 @@ class EvaluationDataset:
         retrieval_context_key_name: Optional[str] = "retrieval_context",
         tools_called_key_name: Optional[str] = "tools_called",
         expected_tools_key_name: Optional[str] = "expected_tools",
+        comments_key_name: str = "comments",
+        name_key_name: str = "name",
         source_file_key_name: Optional[str] = "source_file",
         additional_metadata_key_name: Optional[str] = "additional_metadata",
         scenario_key_name: Optional[str] = "scenario",
@@ -628,7 +642,8 @@ class EvaluationDataset:
                 expected_outcome = json_obj.get(expected_outcome_key_name)
                 user_description = json_obj.get(user_description_key_name)
                 context = json_obj.get(context_key_name)
-
+                comments = json_obj.get(comments_key_name)
+                name = json_obj.get(name_key_name)
                 parsed_turns = parse_turns(turns) if turns else []
 
                 self._multi_turn = True
@@ -639,6 +654,8 @@ class EvaluationDataset:
                         expected_outcome=expected_outcome,
                         user_description=user_description,
                         context=context,
+                        comments=comments,
+                        name=name,
                     )
                 )
             else:
@@ -649,6 +666,8 @@ class EvaluationDataset:
                 retrieval_context = json_obj.get(retrieval_context_key_name)
                 tools_called = json_obj.get(tools_called_key_name)
                 expected_tools = json_obj.get(expected_tools_key_name)
+                comments = json_obj.get(comments_key_name)
+                name = json_obj.get(name_key_name)
                 source_file = json_obj.get(source_file_key_name)
                 additional_metadata = json_obj.get(additional_metadata_key_name)
 
@@ -663,6 +682,8 @@ class EvaluationDataset:
                         tools_called=tools_called,
                         expected_tools=expected_tools,
                         additional_metadata=additional_metadata,
+                        comments=comments,
+                        name=name,
                         source_file=source_file,
                     )
                 )
@@ -928,6 +949,8 @@ class EvaluationDataset:
                     expected_outcome=golden.expected_outcome,
                     user_description=golden.user_description,
                     context=golden.context,
+                    name=golden.name,
+                    comments=golden.comments,
                 )
                 for golden in self.goldens
             ]
@@ -939,6 +962,8 @@ class EvaluationDataset:
                     actual_output=golden.actual_output,
                     retrieval_context=golden.retrieval_context,
                     context=golden.context,
+                    name=golden.name,
+                    comments=golden.comments,
                     source_file=golden.source_file,
                 )
                 for golden in self.goldens
@@ -981,6 +1006,8 @@ class EvaluationDataset:
                             "expected_outcome": golden.expected_outcome,
                             "user_description": golden.user_description,
                             "context": golden.context,
+                            "name": golden.name,
+                            "comments": golden.comments,
                         }
                         for golden in goldens
                     ]
@@ -992,6 +1019,8 @@ class EvaluationDataset:
                             "expected_output": golden.expected_output,
                             "retrieval_context": golden.retrieval_context,
                             "context": golden.context,
+                            "name": golden.name,
+                            "comments": golden.comments,
                             "source_file": golden.source_file,
                         }
                         for golden in goldens
@@ -1010,6 +1039,8 @@ class EvaluationDataset:
                             "expected_outcome",
                             "user_description",
                             "context",
+                            "name",
+                            "comments",
                         ]
                     )
                     for golden in goldens:
@@ -1030,6 +1061,8 @@ class EvaluationDataset:
                                 golden.expected_outcome,
                                 golden.user_description,
                                 context,
+                                golden.name,
+                                golden.comments,
                             ]
                         )
                 else:
@@ -1040,6 +1073,8 @@ class EvaluationDataset:
                             "expected_output",
                             "retrieval_context",
                             "context",
+                            "name",
+                            "comments",
                             "source_file",
                         ]
                     )
@@ -1061,6 +1096,8 @@ class EvaluationDataset:
                                 golden.expected_output,
                                 retrieval_context,
                                 context,
+                                golden.name,
+                                golden.comments,
                                 golden.source_file,
                             ]
                         )

{deepeval-3.5.6 → deepeval-3.5.8}/deepeval/metrics/multimodal_metrics/multimodal_g_eval/multimodal_g_eval.py

@@ -1,6 +1,6 @@
 """LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf"""
 
-from typing import Optional, List, Tuple, Union
+from typing import Optional, List, Tuple, Type, Union
 from deepeval.models import DeepEvalBaseMLLM
 from deepeval.metrics import BaseMultimodalMetric
 from deepeval.test_case import (
@@ -10,7 +10,10 @@ from deepeval.test_case import (
 from deepeval.metrics.multimodal_metrics.multimodal_g_eval.template import (
     MultimodalGEvalTemplate,
 )
-from deepeval.metrics.multimodal_metrics.multimodal_g_eval.schema import *
+from deepeval.metrics.multimodal_metrics.multimodal_g_eval.schema import (
+    Steps,
+    ReasonScore,
+)
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.utils import (
@@ -49,6 +52,9 @@ class MultimodalGEval(BaseMultimodalMetric):
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
+        evaluation_template: Type[
+            MultimodalGEvalTemplate
+        ] = MultimodalGEvalTemplate,
         _include_g_eval_suffix: bool = True,
     ):
         validate_criteria_and_evaluation_steps(criteria, evaluation_steps)
@@ -65,6 +71,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         self.async_mode = async_mode
         self.verbose_mode = verbose_mode
         self._include_g_eval_suffix = _include_g_eval_suffix
+        self.evaluation_template = evaluation_template
 
     def measure(
         self,
@@ -167,7 +174,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         g_eval_params_str = construct_g_eval_params_string(
             self.evaluation_params
         )
-        prompt = MultimodalGEvalTemplate.generate_evaluation_steps(
+        prompt = self.evaluation_template.generate_evaluation_steps(
             criteria=self.criteria, parameters=g_eval_params_str
         )
         if self.using_native_model:
@@ -190,7 +197,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         g_eval_params_str = construct_g_eval_params_string(
             self.evaluation_params
         )
-        prompt = MultimodalGEvalTemplate.generate_evaluation_steps(
+        prompt = self.evaluation_template.generate_evaluation_steps(
             criteria=self.criteria, parameters=g_eval_params_str
         )
         if self.using_native_model:
@@ -218,7 +225,7 @@ class MultimodalGEval(BaseMultimodalMetric):
 
         if not self.strict_mode:
             rubric_str = format_rubrics(self.rubric) if self.rubric else None
-            prompt = MultimodalGEvalTemplate.generate_evaluation_results(
+            prompt = self.evaluation_template.generate_evaluation_results(
                 evaluation_steps=number_evaluation_steps(self.evaluation_steps),
                 test_case_list=test_case_list,
                 parameters=g_eval_params_str,
@@ -227,11 +234,15 @@ class MultimodalGEval(BaseMultimodalMetric):
                 _additional_context=_additional_context,
             )
         else:
-            prompt = MultimodalGEvalTemplate.generate_strict_evaluation_results(
-                evaluation_steps=number_evaluation_steps(self.evaluation_steps),
-                test_case_list=test_case_list,
-                parameters=g_eval_params_str,
-                _additional_context=_additional_context,
+            prompt = (
+                self.evaluation_template.generate_strict_evaluation_results(
+                    evaluation_steps=number_evaluation_steps(
+                        self.evaluation_steps
+                    ),
+                    test_case_list=test_case_list,
+                    parameters=g_eval_params_str,
+                    _additional_context=_additional_context,
+                )
             )
         try:
             # don't use log probabilities for unsupported gpt models
@@ -256,7 +267,7 @@ class MultimodalGEval(BaseMultimodalMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except Exception:
                 return score, reason
         except (
             AttributeError
@@ -289,7 +300,7 @@ class MultimodalGEval(BaseMultimodalMetric):
 
         if not self.strict_mode:
             rubric_str = format_rubrics(self.rubric) if self.rubric else None
-            prompt = MultimodalGEvalTemplate.generate_evaluation_results(
+            prompt = self.evaluation_template.generate_evaluation_results(
                 evaluation_steps=number_evaluation_steps(self.evaluation_steps),
                 test_case_list=test_case_list,
                 parameters=g_eval_params_str,
@@ -298,11 +309,15 @@ class MultimodalGEval(BaseMultimodalMetric):
                 _additional_context=_additional_context,
             )
         else:
-            prompt = MultimodalGEvalTemplate.generate_strict_evaluation_results(
-                evaluation_steps=number_evaluation_steps(self.evaluation_steps),
-                test_case_list=test_case_list,
-                parameters=g_eval_params_str,
-                _additional_context=_additional_context,
+            prompt = (
+                self.evaluation_template.generate_strict_evaluation_results(
+                    evaluation_steps=number_evaluation_steps(
+                        self.evaluation_steps
+                    ),
+                    test_case_list=test_case_list,
+                    parameters=g_eval_params_str,
+                    _additional_context=_additional_context,
+                )
             )
 
         try:
@@ -326,7 +341,7 @@ class MultimodalGEval(BaseMultimodalMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except Exception:
                 return score, reason
         except AttributeError:
             # This catches the case where a_generate_raw_response doesn't exist.
@@ -352,7 +367,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except Exception:
                 self.success = False
         return self.success
 

{deepeval-3.5.6 → deepeval-3.5.8}/deepeval/openai_agents/agent.py

@@ -164,13 +164,12 @@ class _ObservedModel(Model):
                 ):
 
                     if isinstance(event, ResponseCompletedEvent):
-                        observer.result = (
-                            event.response.output_text
-                        )  # TODO: support other response types
+                        observer.result = make_json_serializable(
+                            event.response.output
+                        )
 
                     yield event
 
-                observer.__exit__(None, None, None)
             except Exception as e:
                 observer.__exit__(type(e), e, e.__traceback__)
                 raise

{deepeval-3.5.6 → deepeval-3.5.8}/deepeval/prompt/api.py

@@ -8,6 +8,7 @@ class PromptInterpolationType(Enum):
     MUSTACHE_WITH_SPACE = "MUSTACHE_WITH_SPACE"
     FSTRING = "FSTRING"
     DOLLAR_BRACKETS = "DOLLAR_BRACKETS"
+    JINJA = "JINJA"
 
 
 class PromptMessage(BaseModel):

{deepeval-3.5.6 → deepeval-3.5.8}/deepeval/prompt/prompt.py

@@ -64,6 +64,10 @@ class Prompt:
             raise TypeError(
                 "Unable to create Prompt where 'alias' and 'template' are both None. Please provide at least one to continue."
             )
+        if template and messages_template:
+            raise TypeError(
+                "Unable to create Prompt where 'template' and 'messages_template' are both provided. Please provide only one to continue."
+            )
 
         self.alias = alias
         self._text_template = template
@@ -71,6 +75,10 @@ class Prompt:
         self._version = None
         self._polling_tasks: Dict[str, asyncio.Task] = {}
         self._refresh_map: Dict[str, int] = {}
+        if template:
+            self._type = PromptType.TEXT
+        elif messages_template:
+            self._type = PromptType.LIST
 
     @property
     def version(self):

{deepeval-3.5.6 → deepeval-3.5.8}/deepeval/prompt/utils.py

@@ -1,5 +1,7 @@
-from deepeval.prompt.api import PromptInterpolationType
 import re
+from jinja2 import Template
+
+from deepeval.prompt.api import PromptInterpolationType
 
 
 def interpolate_mustache(text: str, **kwargs) -> str:
@@ -25,6 +27,11 @@ def interpolate_dollar_brackets(text: str, **kwargs) -> str:
     return formatted_template.format(**kwargs)
 
 
+def interpolate_jinja(text: str, **kwargs) -> str:
+    template = Template(text)
+    return template.render(**kwargs)
+
+
 def interpolate_text(
     interpolation_type: PromptInterpolationType, text: str, **kwargs
 ) -> str:
@@ -37,5 +44,7 @@ def interpolate_text(
         return interpolate_fstring(text, **kwargs)
     elif interpolation_type == PromptInterpolationType.DOLLAR_BRACKETS:
         return interpolate_dollar_brackets(text, **kwargs)
+    elif interpolation_type == PromptInterpolationType.JINJA:
+        return interpolate_jinja(text, **kwargs)
 
     raise ValueError(f"Unsupported interpolation type: {interpolation_type}")

{deepeval-3.5.6 → deepeval-3.5.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deepeval"
-version = "3.5.6"
+version = "3.5.8"
 description = "The LLM Evaluation Framework"
 authors = ["Jeffrey Ip <jeffreyip@confident-ai.com>"]
 license = "Apache-2.0"
@@ -48,6 +48,7 @@ pyfiglet = "*"
 python-dotenv = "^1.1.1"
 pydantic = "^2.11.7"
 pydantic-settings = "^2.10.1"
+jinja2 = "*"
 
 [tool.poetry.group.dev.dependencies]
 twine = "5.1.1"

deepeval-3.5.6/deepeval/_version.py

@@ -1 +0,0 @@
-__version__: str = "3.5.6"