deepeval 3.7.8__tar.gz → 3.7.9__tar.gz

{deepeval-3.7.8 → deepeval-3.7.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.7.8
+Version: 3.7.9
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -115,7 +115,7 @@ Whether your LLM applications are AI agents, RAG pipelines, or chatbots, impleme
 
 # 🔥 Metrics and Features
 
-> 🥳 You can now share DeepEval's test results on the cloud directly on [Confident AI](https://confident-ai.com?utm_source=GitHub)'s infrastructure
+> 🥳 You can now share DeepEval's test results on the cloud directly on [Confident AI](https://confident-ai.com?utm_source=GitHub)
 
 - Supports both end-to-end and component-level LLM evaluation.
 - Large variety of ready-to-use LLM evaluation metrics (all with explanations) powered by **ANY** LLM of your choice, statistical methods, or NLP models that runs **locally on your machine**:
@@ -158,7 +158,7 @@ Whether your LLM applications are AI agents, RAG pipelines, or chatbots, impleme
   - TruthfulQA
   - HumanEval
   - GSM8K
-- [100% integrated with Confident AI](https://confident-ai.com?utm_source=GitHub) for the full evaluation lifecycle:
+- [100% integrated with Confident AI](https://confident-ai.com?utm_source=GitHub) for the full evaluation & observability lifecycle:
   - Curate/annotate evaluation datasets on the cloud
   - Benchmark LLM app using dataset, and compare with previous iterations to experiment which models/prompts works best
   - Fine-tune metrics for custom results
@@ -167,7 +167,7 @@ Whether your LLM applications are AI agents, RAG pipelines, or chatbots, impleme
   - Repeat until perfection
 
 > [!NOTE]
-> Confident AI is the DeepEval platform. Create an account [here.](https://app.confident-ai.com?utm_source=GitHub)
+> DeepEval is available on Confident AI, an LLM evals platform for AI observability and quality. Create an account [here.](https://app.confident-ai.com?utm_source=GitHub)
 
 <br />
 
@@ -394,7 +394,7 @@ cp .env.example .env.local
 
 # DeepEval With Confident AI
 
-DeepEval's cloud platform, [Confident AI](https://confident-ai.com?utm_source=Github), allows you to:
+DeepEval is available on [Confident AI](https://confident-ai.com?utm_source=Github), an evals & observability platform that allows you to:
 
 1. Curate/annotate evaluation datasets on the cloud
 2. Benchmark LLM app using dataset, and compare with previous iterations to experiment which models/prompts works best

{deepeval-3.7.8 → deepeval-3.7.9}/README.md

@@ -68,7 +68,7 @@ Whether your LLM applications are AI agents, RAG pipelines, or chatbots, impleme
 
 # 🔥 Metrics and Features
 
-> 🥳 You can now share DeepEval's test results on the cloud directly on [Confident AI](https://confident-ai.com?utm_source=GitHub)'s infrastructure
+> 🥳 You can now share DeepEval's test results on the cloud directly on [Confident AI](https://confident-ai.com?utm_source=GitHub)
 
 - Supports both end-to-end and component-level LLM evaluation.
 - Large variety of ready-to-use LLM evaluation metrics (all with explanations) powered by **ANY** LLM of your choice, statistical methods, or NLP models that runs **locally on your machine**:
@@ -111,7 +111,7 @@ Whether your LLM applications are AI agents, RAG pipelines, or chatbots, impleme
   - TruthfulQA
   - HumanEval
   - GSM8K
-- [100% integrated with Confident AI](https://confident-ai.com?utm_source=GitHub) for the full evaluation lifecycle:
+- [100% integrated with Confident AI](https://confident-ai.com?utm_source=GitHub) for the full evaluation & observability lifecycle:
   - Curate/annotate evaluation datasets on the cloud
   - Benchmark LLM app using dataset, and compare with previous iterations to experiment which models/prompts works best
   - Fine-tune metrics for custom results
@@ -120,7 +120,7 @@ Whether your LLM applications are AI agents, RAG pipelines, or chatbots, impleme
   - Repeat until perfection
 
 > [!NOTE]
-> Confident AI is the DeepEval platform. Create an account [here.](https://app.confident-ai.com?utm_source=GitHub)
+> DeepEval is available on Confident AI, an LLM evals platform for AI observability and quality. Create an account [here.](https://app.confident-ai.com?utm_source=GitHub)
 
 <br />
 
@@ -347,7 +347,7 @@ cp .env.example .env.local
 
 # DeepEval With Confident AI
 
-DeepEval's cloud platform, [Confident AI](https://confident-ai.com?utm_source=Github), allows you to:
+DeepEval is available on [Confident AI](https://confident-ai.com?utm_source=Github), an evals & observability platform that allows you to:
 
 1. Curate/annotate evaluation datasets on the cloud
 2. Benchmark LLM app using dataset, and compare with previous iterations to experiment which models/prompts works best

deepeval-3.7.9/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.7.9"

{deepeval-3.7.8 → deepeval-3.7.9}/deepeval/benchmarks/drop/drop.py

@@ -279,8 +279,11 @@ class DROP(DeepEvalBaseBenchmark):
             prediction = predictions[i]
             golden = goldens[i]
             # Define Metric
-            score = self.scorer.quasi_exact_match_score(
-                golden.expected_output, prediction
+            expected_output = DROPTemplate.parse_str_to_list(
+                golden.expected_output, DELIMITER
+            )
+            score = self.scorer.quasi_contains_score(
+                expected_output, prediction
             )
             res.append({"prediction": prediction, "score": score})
 

{deepeval-3.7.8 → deepeval-3.7.9}/deepeval/benchmarks/mmlu/mmlu.py

@@ -224,10 +224,12 @@ class MMLU(DeepEvalBaseBenchmark):
             responses: List[MultipleChoiceSchema] = model.batch_generate(
                 prompts=prompts, schemas=[MultipleChoiceSchema for i in prompts]
             )
-            if isinstance(responses, (tuple, list)):
-                predictions = [res[0].answer for res in responses]
-            else:
-                predictions = [res.answer for res in responses]
+            if not isinstance(responses, list):
+                raise TypeError(
+                    "batch_generate must return List[MultipleChoiceSchema]"
+                )
+
+            predictions = [res.answer for res in responses]
         except TypeError:
             prompts = [
                 prompt

{deepeval-3.7.8 → deepeval-3.7.9}/deepeval/cli/utils.py

@@ -52,10 +52,10 @@ USE_EMBED_KEYS = [
 
 def render_login_message():
     print(
-        "🥳 Welcome to [rgb(106,0,255)]Confident AI[/rgb(106,0,255)], the DeepEval cloud platform 🏡❤️"
+        "🥳 Welcome to [rgb(106,0,255)]Confident AI[/rgb(106,0,255)], the evals cloud platform 🏡❤️"
     )
     print("")
-    print(pyfiglet.Figlet(font="big_money-ne").renderText("DeepEval Cloud"))
+    print(pyfiglet.Figlet(font="big_money-ne").renderText("Confident AI"))
 
 
 def upload_and_open_link(_span: Span):

{deepeval-3.7.8 → deepeval-3.7.9}/deepeval/models/llms/gemini_model.py

@@ -22,7 +22,7 @@ from deepeval.models.llms.constants import GEMINI_MODELS_DATA
 if TYPE_CHECKING:
     from google.genai import Client
 
-default_gemini_model = "gemini-1.5-pro"
+default_gemini_model = "gemini-2.5-pro"
 
 # consistent retry rules
 retry_gemini = create_retry_decorator(PS.GOOGLE)
@@ -371,25 +371,6 @@ class GeminiModel(DeepEvalBaseLLM):
         client_kwargs = self._client_kwargs(**self.kwargs)
 
         if self.should_use_vertexai():
-            service_account_key_json = require_secret_api_key(
-                self.service_account_key,
-                provider_label="Google Gemini",
-                env_var_name="GOOGLE_SERVICE_ACCOUNT_KEY",
-                param_hint="`service_account_key` to GeminiModel(...)",
-            )
-
-            try:
-                service_account_key = json.loads(service_account_key_json)
-            except Exception as e:
-                raise DeepEvalError(
-                    "GOOGLE_SERVICE_ACCOUNT_KEY must be valid JSON for a Google service account."
-                ) from e
-
-            if not isinstance(service_account_key, dict):
-                raise DeepEvalError(
-                    "GOOGLE_SERVICE_ACCOUNT_KEY must decode to a JSON object."
-                )
-
             if not self.project or not self.location:
                 raise DeepEvalError(
                     "When using Vertex AI API, both project and location are required. "
@@ -397,17 +378,34 @@ class GeminiModel(DeepEvalBaseLLM):
                     "GOOGLE_CLOUD_LOCATION in your DeepEval configuration."
                 )
 
-            oauth2 = self._require_oauth2()
-            credentials = (
-                oauth2.service_account.Credentials.from_service_account_info(
+            # if no service account key is provided, allow the SDK
+            # to resolve Application Default Credentials automatically.
+            credentials = None
+            if self.service_account_key is not None:
+                service_account_key_json = require_secret_api_key(
+                    self.service_account_key,
+                    provider_label="Google Gemini",
+                    env_var_name="GOOGLE_SERVICE_ACCOUNT_KEY",
+                    param_hint="`service_account_key` to GeminiModel(...)",
+                )
+
+                try:
+                    service_account_key = json.loads(service_account_key_json)
+                except Exception as e:
+                    raise DeepEvalError(
+                        "GOOGLE_SERVICE_ACCOUNT_KEY must be valid JSON for a Google service account."
+                    ) from e
+
+                if not isinstance(service_account_key, dict):
+                    raise DeepEvalError(
+                        "GOOGLE_SERVICE_ACCOUNT_KEY must decode to a JSON object."
+                    )
+
+                oauth2 = self._require_oauth2()
+                credentials = oauth2.service_account.Credentials.from_service_account_info(
                     service_account_key,
-                    scopes=[
-                        "https://www.googleapis.com/auth/cloud-platform",
-                    ],
+                    scopes=["https://www.googleapis.com/auth/cloud-platform"],
                 )
-                if service_account_key
-                else None
-            )
 
             client = self._module.Client(
                 vertexai=True,

{deepeval-3.7.8 → deepeval-3.7.9}/deepeval/synthesizer/synthesizer.py

@@ -1383,53 +1383,99 @@ class Synthesizer:
         # Prepare data for the DataFrame
         data = []
 
-        for golden in self.synthetic_goldens:
-            # Extract basic fields
-            input_text = golden.input
-            expected_output = golden.expected_output
-            context = golden.context
-            actual_output = golden.actual_output
-            retrieval_context = golden.retrieval_context
-            metadata = golden.additional_metadata
-            source_file = golden.source_file
-
-            # Calculate num_context and context_length
-            if context is not None:
-                num_context = len(context)
-                context_length = sum(len(c) for c in context)
-            else:
-                num_context = None
-                context_length = None
-
-            # Handle metadata
-            if metadata is not None:
-                evolutions = metadata.get("evolutions", None)
-                synthetic_input_quality = metadata.get(
-                    "synthetic_input_quality", None
-                )
-                context_quality = metadata.get("context_quality", None)
-            else:
-                evolutions = None
-                synthetic_input_quality = None
-                context_quality = None
-
-            # Prepare a row for the DataFrame
-            row = {
-                "input": input_text,
-                "actual_output": actual_output,
-                "expected_output": expected_output,
-                "context": context,
-                "retrieval_context": retrieval_context,
-                "n_chunks_per_context": num_context,
-                "context_length": context_length,
-                "evolutions": evolutions,
-                "context_quality": context_quality,
-                "synthetic_input_quality": synthetic_input_quality,
-                "source_file": source_file,
-            }
-
-            # Append the row to the data list
-            data.append(row)
+        if (
+            self.synthetic_goldens is not None
+            and len(self.synthetic_goldens) > 0
+        ):
+            for golden in self.synthetic_goldens:
+                # Extract basic fields
+                input_text = golden.input
+                expected_output = golden.expected_output
+                context = golden.context
+                actual_output = golden.actual_output
+                retrieval_context = golden.retrieval_context
+                metadata = golden.additional_metadata
+                source_file = golden.source_file
+
+                # Calculate num_context and context_length
+                if context is not None:
+                    num_context = len(context)
+                    context_length = sum(len(c) for c in context)
+                else:
+                    num_context = None
+                    context_length = None
+
+                # Handle metadata
+                if metadata is not None:
+                    evolutions = metadata.get("evolutions", None)
+                    synthetic_input_quality = metadata.get(
+                        "synthetic_input_quality", None
+                    )
+                    context_quality = metadata.get("context_quality", None)
+                else:
+                    evolutions = None
+                    synthetic_input_quality = None
+                    context_quality = None
+
+                # Prepare a row for the DataFrame
+                row = {
+                    "input": input_text,
+                    "actual_output": actual_output,
+                    "expected_output": expected_output,
+                    "context": context,
+                    "retrieval_context": retrieval_context,
+                    "n_chunks_per_context": num_context,
+                    "context_length": context_length,
+                    "evolutions": evolutions,
+                    "context_quality": context_quality,
+                    "synthetic_input_quality": synthetic_input_quality,
+                    "source_file": source_file,
+                }
+
+                # Append the row to the data list
+                data.append(row)
+        else:
+            for golden in self.synthetic_conversational_goldens:
+                # Extract basic fields
+                scenario = golden.scenario
+                expected_outcome = golden.expected_outcome
+                context = golden.context
+                metadata = golden.additional_metadata
+
+                # Calculate num_context and context_length
+                if context is not None:
+                    num_context = len(context)
+                    context_length = sum(len(c) for c in context)
+                else:
+                    num_context = None
+                    context_length = None
+
+                # Handle metadata
+                if metadata is not None:
+                    evolutions = metadata.get("evolutions", None)
+                    synthetic_scenario_quality = metadata.get(
+                        "synthetic_scenario_quality", None
+                    )
+                    source_files = metadata.get("source_files", None)
+                else:
+                    evolutions = None
+                    synthetic_scenario_quality = None
+                    source_files = None
+
+                # Prepare a row for the DataFrame
+                row = {
+                    "scenario": scenario,
+                    "expected_outcome": expected_outcome,
+                    "context": context,
+                    "n_chunks_per_context": num_context,
+                    "context_length": context_length,
+                    "evolutions": evolutions,
+                    "synthetic_scenario_quality": synthetic_scenario_quality,
+                    "source_files": source_files,
+                }
+
+                # Append the row to the data list
+                data.append(row)
 
         # Create the pandas DataFrame
         df = pd.DataFrame(data)
@@ -1479,7 +1525,10 @@ class Synthesizer:
                 "parameter."
             )
 
-        if len(self.synthetic_goldens) == 0:
+        if (
+            len(self.synthetic_goldens) == 0
+            and len(self.synthetic_conversational_goldens) == 0
+        ):
             raise ValueError(
                 "No synthetic goldens found. Please generate goldens before saving goldens."
             )
@@ -1494,52 +1543,111 @@ class Synthesizer:
         full_file_path = os.path.join(directory, new_filename)
         if file_type == "json":
             with open(full_file_path, "w", encoding="utf-8") as file:
-                json_data = [
-                    {
-                        "input": golden.input,
-                        "actual_output": golden.actual_output,
-                        "expected_output": golden.expected_output,
-                        "context": golden.context,
-                        "source_file": golden.source_file,
-                    }
-                    for golden in self.synthetic_goldens
-                ]
+                if (
+                    self.synthetic_goldens is not None
+                    and len(self.synthetic_goldens) > 0
+                ):
+                    json_data = [
+                        {
+                            "input": golden.input,
+                            "actual_output": golden.actual_output,
+                            "expected_output": golden.expected_output,
+                            "context": golden.context,
+                            "source_file": golden.source_file,
+                        }
+                        for golden in self.synthetic_goldens
+                    ]
+                else:
+                    json_data = [
+                        {
+                            "scenario": golden.scenario,
+                            "expected_outcome": golden.expected_outcome,
+                            "context": golden.context,
+                            "source_files": golden.additional_metadata.get(
+                                "source_files", None
+                            ),
+                        }
+                        for golden in self.synthetic_conversational_goldens
+                    ]
                 json.dump(json_data, file, indent=4, ensure_ascii=False)
         elif file_type == "csv":
             with open(
                 full_file_path, "w", newline="", encoding="utf-8"
             ) as file:
                 writer = csv.writer(file)
-                writer.writerow(
-                    [
-                        "input",
-                        "actual_output",
-                        "expected_output",
-                        "context",
-                        "source_file",
-                    ]
-                )
-                for golden in self.synthetic_goldens:
+                if (
+                    self.synthetic_goldens is not None
+                    and len(self.synthetic_goldens) > 0
+                ):
+                    writer.writerow(
+                        [
+                            "input",
+                            "actual_output",
+                            "expected_output",
+                            "context",
+                            "source_file",
+                        ]
+                    )
+                    for golden in self.synthetic_goldens:
+                        writer.writerow(
+                            [
+                                golden.input,
+                                golden.actual_output,
+                                golden.expected_output,
+                                "|".join(golden.context),
+                                golden.source_file,
+                            ]
+                        )
+                else:
                     writer.writerow(
                         [
-                            golden.input,
-                            golden.actual_output,
-                            golden.expected_output,
-                            "|".join(golden.context),
-                            golden.source_file,
+                            "scenario",
+                            "expected_outcome",
+                            "context",
+                            "source_files",
                         ]
                     )
+                    for golden in self.synthetic_conversational_goldens:
+                        writer.writerow(
+                            [
+                                golden.scenario,
+                                golden.expected_outcome,
+                                "|".join(golden.context),
+                                golden.additional_metadata.get(
+                                    "source_files", None
+                                ),
+                            ]
+                        )
         elif file_type == "jsonl":
             with open(full_file_path, "w", encoding="utf-8") as file:
-                for golden in self.synthetic_goldens:
-                    record = {
-                        "input": golden.input,
-                        "actual_output": golden.actual_output,
-                        "expected_output": golden.expected_output,
-                        "context": golden.context,
-                        "source_file": golden.source_file,
-                    }
-                    file.write(json.dumps(record, ensure_ascii=False) + "\n")
+                if (
+                    self.synthetic_goldens is not None
+                    and len(self.synthetic_goldens) > 0
+                ):
+                    for golden in self.synthetic_goldens:
+                        record = {
+                            "input": golden.input,
+                            "actual_output": golden.actual_output,
+                            "expected_output": golden.expected_output,
+                            "context": golden.context,
+                            "source_file": golden.source_file,
+                        }
+                        file.write(
+                            json.dumps(record, ensure_ascii=False) + "\n"
+                        )
+                else:
+                    for golden in self.synthetic_conversational_goldens:
+                        record = {
+                            "scenario": golden.scenario,
+                            "expected_outcome": golden.expected_outcome,
+                            "context": golden.context,
+                            "source_files": golden.additional_metadata.get(
+                                "source_files", None
+                            ),
+                        }
+                        file.write(
+                            json.dumps(record, ensure_ascii=False) + "\n"
+                        )
         if not quiet:
             print(f"Synthetic goldens saved at {full_file_path}!")
 

{deepeval-3.7.8 → deepeval-3.7.9}/deepeval/utils.py

@@ -739,14 +739,29 @@ def update_pbar(
     if progress is None or pbar_id is None:
         return
     # Get amount to advance
-    current_task = next(t for t in progress.tasks if t.id == pbar_id)
+    current_task = next((t for t in progress.tasks if t.id == pbar_id), None)
+    if current_task is None:
+        return
+
     if advance_to_end:
-        advance = current_task.remaining
+        remaining = current_task.remaining
+        if remaining is not None:
+            advance = remaining
+
     # Advance
-    progress.update(pbar_id, advance=advance, total=total)
-    # Remove if finished
-    if current_task.finished and remove:
-        progress.remove_task(pbar_id)
+    try:
+        progress.update(pbar_id, advance=advance, total=total)
+    except KeyError:
+        # progress task may be removed concurrently via callbacks which can race with teardown.
+        return
+
+    # Remove if finished and refetch before remove to avoid acting on a stale object
+    updated_task = next((t for t in progress.tasks if t.id == pbar_id), None)
+    if updated_task is not None and updated_task.finished and remove:
+        try:
+            progress.remove_task(pbar_id)
+        except KeyError:
+            pass
 
 
 def add_pbar(progress: Optional[Progress], description: str, total: int = 1):

{deepeval-3.7.8 → deepeval-3.7.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deepeval"
-version = "3.7.8"
+version = "3.7.9"
 description = "The LLM Evaluation Framework"
 authors = ["Jeffrey Ip <jeffreyip@confident-ai.com>"]
 license = "Apache-2.0"

deepeval-3.7.8/deepeval/_version.py

@@ -1 +0,0 @@
-__version__: str = "3.7.8"