deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/synthesizer/synthesizer.py

@@ -555,7 +555,7 @@ class Synthesizer:
                     include_expected_output=include_expected_output,
                     max_goldens_per_context=max_goldens_per_context,
                     source_files=source_files,
-                    index=index,
+                    context_index=index,
                     progress=progress,
                     pbar_id=pbar_id,
                     context_scores=_context_scores,
@@ -577,7 +577,7 @@ class Synthesizer:
         include_expected_output: bool,
         max_goldens_per_context: int,
         source_files: Optional[List[str]],
-        index: int,
+        context_index: int,
         progress: Optional[Progress] = None,
         pbar_id: Optional[int] = None,
         context_scores: Optional[List[float]] = None,
@@ -599,7 +599,7 @@ class Synthesizer:
         # Add pbars
         pbar_generate_goldens_id = add_pbar(
             progress,
-            f"\t⚡ Generating goldens from context #{index}",
+            f"\t⚡ Generating goldens from context #{context_index}",
             total=1 + max_goldens_per_context,
         )
         pbar_generate_inputs_id = add_pbar(
@@ -643,7 +643,7 @@ class Synthesizer:
 
         # Helper function to process each input in parallel
         async def process_input(
-            index: int,
+            input_index: int,
             data: SyntheticData,
             progress: Optional[Progress] = None,
         ):
@@ -654,7 +654,7 @@ class Synthesizer:
                 num_evolutions=self.evolution_config.num_evolutions,
                 evolutions=self.evolution_config.evolutions,
                 progress=progress,
-                pbar_evolve_input_id=pbar_evolve_input_ids[index],
+                pbar_evolve_input_id=pbar_evolve_input_ids[input_index],
                 remove_pbar=False,
             )
 
@@ -672,7 +672,7 @@ class Synthesizer:
                 )
                 evolved_input = res.input
                 update_pbar(
-                    progress, pbar_evolve_input_ids[index], remove=False
+                    progress, pbar_evolve_input_ids[input_index], remove=False
                 )
 
             # Generate expected output
@@ -685,7 +685,7 @@ class Synthesizer:
                 )
                 expected_output = await self._a_generate(expected_output_prompt)
                 update_pbar(
-                    progress, pbar_evolve_input_ids[index], remove=False
+                    progress, pbar_evolve_input_ids[input_index], remove=False
                 )
 
             # Create Golden
@@ -694,13 +694,14 @@ class Synthesizer:
                 context=context,
                 expected_output=expected_output,
                 source_file=(
-                    source_files[index]
-                    if source_files is not None and index < len(source_files)
+                    source_files[context_index]
+                    if source_files is not None
+                    and context_index < len(source_files)
                     else None
                 ),
                 additional_metadata={
                     "evolutions": evolutions_used,
-                    "synthetic_input_quality": scores[index],
+                    "synthetic_input_quality": scores[input_index],
                     # "context_quality": (
                     #     context_scores[data_index]
                     #     if context_scores is not None
@@ -898,6 +899,7 @@ class Synthesizer:
                 update_pbar(progress, pbar_id)
 
                 # Evolve inputs
+                evolved_prompts = []
                 for i, data in enumerate(synthetic_data):
                     pbar_evolve_input_id = add_pbar(
                         progress,
@@ -911,14 +913,16 @@ class Synthesizer:
                         progress=progress,
                         pbar_evolve_input_id=pbar_evolve_input_id,
                     )
+                    evolved_prompts.append(evolved_prompt)
                     update_pbar(progress, pbar_id)
 
                 # Synthesize Goldens
-                golden = Golden(
-                    input=evolved_prompt,
-                    additional_metadata={"evolutions": evolutions_used},
-                )
-                goldens.append(golden)
+                for evolved_prompt in evolved_prompts:
+                    golden = Golden(
+                        input=evolved_prompt,
+                        additional_metadata={"evolutions": evolutions_used},
+                    )
+                    goldens.append(golden)
 
         # Wrap up Synthesis
         self.synthetic_goldens.extend(goldens)

deepeval/test_case/api.py

@@ -0,0 +1,131 @@
+from typing import Union, Optional
+import os
+
+from deepeval.test_run.api import (
+    LLMApiTestCase,
+    ConversationalApiTestCase,
+    TurnApi,
+    TraceApi,
+)
+from deepeval.test_case import (
+    LLMTestCase,
+    ConversationalTestCase,
+    MLLMTestCase,
+    Turn,
+)
+from deepeval.constants import PYTEST_RUN_TEST_NAME
+
+
+def create_api_turn(turn: Turn, index: int) -> TurnApi:
+    return TurnApi(
+        role=turn.role,
+        content=turn.content,
+        user_id=turn.user_id,
+        retrievalContext=turn.retrieval_context,
+        toolsCalled=turn.tools_called,
+        additionalMetadata=turn.additional_metadata,
+        order=index,
+    )
+
+
+def create_api_test_case(
+    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+    trace: Optional[TraceApi] = None,
+    index: Optional[int] = None,
+) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
+    if isinstance(test_case, ConversationalTestCase):
+        order = (
+            test_case._dataset_rank
+            if test_case._dataset_rank is not None
+            else index
+        )
+        if test_case.name:
+            name = test_case.name
+        else:
+            name = os.getenv(
+                PYTEST_RUN_TEST_NAME, f"conversational_test_case_{order}"
+            )
+
+        api_test_case = ConversationalApiTestCase(
+            name=name,
+            success=True,
+            metricsData=[],
+            runDuration=0,
+            evaluationCost=None,
+            order=order,
+            scenario=test_case.scenario,
+            expectedOutcome=test_case.expected_outcome,
+            userDescription=test_case.user_description,
+            context=test_case.context,
+            tags=test_case.tags,
+            comments=test_case.comments,
+            additionalMetadata=test_case.additional_metadata,
+        )
+        api_test_case.turns = [
+            create_api_turn(
+                turn=turn,
+                index=index,
+            )
+            for index, turn in enumerate(test_case.turns)
+        ]
+
+        return api_test_case
+    else:
+        order = (
+            test_case._dataset_rank
+            if test_case._dataset_rank is not None
+            else index
+        )
+
+        success = True
+        if test_case.name is not None:
+            name = test_case.name
+        else:
+            name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
+        metrics_data = []
+
+        if isinstance(test_case, LLMTestCase):
+            api_test_case = LLMApiTestCase(
+                name=name,
+                input=test_case.input,
+                actualOutput=test_case.actual_output,
+                expectedOutput=test_case.expected_output,
+                context=test_case.context,
+                retrievalContext=test_case.retrieval_context,
+                toolsCalled=test_case.tools_called,
+                expectedTools=test_case.expected_tools,
+                tokenCost=test_case.token_cost,
+                completionTime=test_case.completion_time,
+                tags=test_case.tags,
+                success=success,
+                metricsData=metrics_data,
+                runDuration=None,
+                evaluationCost=None,
+                order=order,
+                additionalMetadata=test_case.additional_metadata,
+                comments=test_case.comments,
+                trace=trace,
+            )
+        elif isinstance(test_case, MLLMTestCase):
+            api_test_case = LLMApiTestCase(
+                name=name,
+                input="",
+                multimodalInput=test_case.input,
+                multimodalActualOutput=test_case.actual_output,
+                multimodalExpectedOutput=test_case.expected_output,
+                multimodalRetrievalContext=test_case.retrieval_context,
+                multimodalContext=test_case.context,
+                toolsCalled=test_case.tools_called,
+                expectedTools=test_case.expected_tools,
+                tokenCost=test_case.token_cost,
+                completionTime=test_case.completion_time,
+                success=success,
+                metricsData=metrics_data,
+                runDuration=None,
+                evaluationCost=None,
+                order=order,
+                additionalMetadata=test_case.additional_metadata,
+                comments=test_case.comments,
+            )
+        # llm_test_case_lookup_map[instance_id] = api_test_case
+        return api_test_case

deepeval/test_case/llm_test_case.py

@@ -122,7 +122,9 @@ class ToolCall(BaseModel):
 
         # Handle nested fields like input_parameters
         if self.input_parameters:
-            formatted_input = json.dumps(self.input_parameters, indent=4)
+            formatted_input = json.dumps(
+                self.input_parameters, indent=4, ensure_ascii=False
+            )
             formatted_input = self._indent_nested_field(
                 "input_parameters", formatted_input
             )
@@ -130,7 +132,9 @@ class ToolCall(BaseModel):
 
         # Handle nested fields like output
         if isinstance(self.output, dict):
-            formatted_output = json.dumps(self.output, indent=4)
+            formatted_output = json.dumps(
+                self.output, indent=4, ensure_ascii=False
+            )
             formatted_output = self._indent_nested_field(
                 "output", formatted_output
             )

deepeval/test_run/__init__.py

@@ -8,6 +8,7 @@ from .test_run import (
     LLMApiTestCase,
     ConversationalApiTestCase,
     TestRunManager,
+    PromptData,
 )
 
 from .hooks import on_test_run_end, invoke_test_run_end_hook

deepeval/test_run/hyperparameters.py

@@ -1,13 +1,15 @@
-from typing import Union, Dict
-
+from typing import Union, Dict, Optional, List
 from deepeval.test_run import global_test_run_manager
 from deepeval.prompt import Prompt
 from deepeval.prompt.api import PromptApi
 from deepeval.test_run.test_run import TEMP_FILE_PATH
+from deepeval.confident.api import is_confident
+from deepeval.test_run.test_run import PromptData
 
 
 def process_hyperparameters(
-    hyperparameters,
+    hyperparameters: Optional[Dict] = None,
+    verbose: bool = True,
 ) -> Union[Dict[str, Union[str, int, float, PromptApi]], None]:
     if hyperparameters is None:
         return None
@@ -16,6 +18,7 @@ def process_hyperparameters(
         raise TypeError("Hyperparameters must be a dictionary or None")
 
     processed_hyperparameters = {}
+    prompts_version_id_map = {}
 
     for key, value in hyperparameters.items():
         if not isinstance(key, str):
@@ -30,14 +33,21 @@ def process_hyperparameters(
             )
 
         if isinstance(value, Prompt):
-            if value._prompt_version_id is not None and value._type is not None:
+            prompt_key = f"{value.alias}_{value.version}"
+            if value._prompt_version_id is not None and value.type is not None:
                 processed_hyperparameters[key] = PromptApi(
                     id=value._prompt_version_id,
-                    type=value._type,
+                    type=value.type,
                 )
-            else:
-                raise ValueError(
-                    f"Cannot log Prompt where template was not pulled from Confident AI. Please import your prompt on Confident AI to continue."
+            elif is_confident():
+                if prompt_key not in prompts_version_id_map:
+                    value.push(_verbose=verbose)
+                    prompts_version_id_map[prompt_key] = (
+                        value._prompt_version_id
+                    )
+                processed_hyperparameters[key] = PromptApi(
+                    id=prompts_version_id_map[prompt_key],
+                    type=value.type,
                 )
         else:
             processed_hyperparameters[key] = str(value)
@@ -64,3 +74,32 @@ def log_hyperparameters(func):
 
     # Return the wrapper function to be used as the decorator
     return wrapper
+
+
+def process_prompts(
+    hyperparameters: Dict[str, Union[str, int, float, Prompt]],
+) -> List[PromptData]:
+    prompts = []
+    if not hyperparameters:
+        return prompts
+    seen_prompts = set()
+    prompt_objects = [
+        value for value in hyperparameters.values() if isinstance(value, Prompt)
+    ]
+    for prompt in prompt_objects:
+        prompt_version = prompt.version if is_confident() else None
+        prompt_key = f"{prompt.alias}_{prompt_version}"
+        if prompt_key in seen_prompts:
+            continue
+        seen_prompts.add(prompt_key)
+        prompt_data = PromptData(
+            alias=prompt.alias,
+            version=prompt_version,
+            text_template=prompt.text_template,
+            messages_template=prompt.messages_template,
+            model_settings=prompt.model_settings,
+            output_type=prompt.output_type,
+            interpolation_type=prompt.interpolation_type,
+        )
+        prompts.append(prompt_data)
+    return prompts