deepeval 3.4.1__tar.gz → 3.4.3__tar.gz

{deepeval-3.4.1 → deepeval-3.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.4.1
+Version: 3.4.3
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.4.3/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.4.3"

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/annotation/annotation.py

@@ -12,6 +12,7 @@ def send_annotation(
     expected_output: Optional[str] = None,
     expected_outcome: Optional[str] = None,
     explanation: Optional[str] = None,
+    user_id: Optional[str] = None,
     type: Optional[AnnotationType] = AnnotationType.THUMBS_RATING,
 ) -> str:
     api_annotation = APIAnnotation(
@@ -23,6 +24,7 @@ def send_annotation(
         expectedOutcome=expected_outcome,
         explanation=explanation,
         type=type,
+        userId=user_id,
     )
     api = Api()
     try:
@@ -47,6 +49,7 @@ async def a_send_annotation(
     expected_outcome: Optional[str] = None,
     explanation: Optional[str] = None,
     type: Optional[AnnotationType] = AnnotationType.THUMBS_RATING,
+    user_id: Optional[str] = None,
 ) -> str:
     api_annotation = APIAnnotation(
         rating=rating,
@@ -57,6 +60,7 @@ async def a_send_annotation(
         expectedOutcome=expected_outcome,
         explanation=explanation,
         type=type,
+        userId=user_id,
     )
     api = Api()
     try:

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/annotation/api.py

@@ -17,6 +17,7 @@ class APIAnnotation(BaseModel):
     expected_outcome: Optional[str] = Field(None, alias="expectedOutcome")
     explanation: Optional[str] = Field(None)
     type: Optional[AnnotationType] = Field(None, alias="type")
+    user_id: Optional[str] = Field(None, alias="userId")
 
     @model_validator(mode="before")
     def validate_input(cls, data):

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/arc/arc.py

@@ -2,7 +2,10 @@ from typing import List, Optional, Dict
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.arc.mode import ARCMode
 from deepeval.benchmarks.arc.template import ARCTemplate
@@ -48,7 +51,9 @@ class ARC(DeepEvalBaseBenchmark):
         else:
             self.confinement_instructions = confinement_instructions
 
-    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> Dict:
+    def evaluate(
+        self, model: DeepEvalBaseLLM, *args, **kwargs
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("ARC", self.n_problems):
@@ -90,7 +95,9 @@ class ARC(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         # Define prompt template
@@ -129,9 +136,7 @@ class ARC(DeepEvalBaseBenchmark):
         dataset_attr = dataset_mapping.get(mode)
         if dataset_attr:
             if not hasattr(self, dataset_attr):
-                dataset = load_dataset(
-                    "ai2_arc", mode.value, trust_remote_code=True
-                )
+                dataset = load_dataset("ai2_arc", mode.value)
                 setattr(self, dataset_attr, dataset)
             else:
                 dataset = getattr(self, dataset_attr)

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/base_benchmark.py

@@ -1,10 +1,15 @@
 from deepeval.models.base_model import DeepEvalBaseLLM
 from abc import ABC, abstractmethod
 from typing import List, TypeVar, Generic, List, Optional
+from pydantic import BaseModel
 
 from deepeval.dataset import Golden
 
 
+class DeepEvalBaseBenchmarkResult(BaseModel):
+    overall_accuracy: float
+
+
 T = TypeVar("T")
 
 
@@ -21,5 +26,7 @@ class DeepEvalBaseBenchmark(ABC, Generic[T]):
         raise NotImplementedError
 
     @abstractmethod
-    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> dict:
+    def evaluate(
+        self, model: DeepEvalBaseLLM, *args, **kwargs
+    ) -> DeepEvalBaseBenchmarkResult:
         raise NotImplementedError

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/bbq/bbq.py

@@ -2,7 +2,10 @@ from typing import List, Optional, Dict
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.bbq.task import BBQTask
 from deepeval.benchmarks.bbq.template import BBQTemplate
@@ -39,7 +42,9 @@ class BBQ(DeepEvalBaseBenchmark):
         else:
             self.confinement_instructions = confinement_instructions
 
-    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> Dict:
+    def evaluate(
+        self, model: DeepEvalBaseLLM, *args, **kwargs
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("BBQ", len(self.tasks)):
@@ -115,7 +120,9 @@ class BBQ(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         # Define prompt template
@@ -164,9 +171,7 @@ class BBQ(DeepEvalBaseBenchmark):
         dataset_attr = dataset_mapping.get(task)
         if dataset_attr:
             if not hasattr(self, dataset_attr):
-                dataset = load_dataset(
-                    "heegyu/bbq", task.value, trust_remote_code=True
-                )
+                dataset = load_dataset("heegyu/bbq", task.value)
                 setattr(self, dataset_attr, dataset)
             else:
                 dataset = getattr(self, dataset_attr)

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/big_bench_hard/big_bench_hard.py

@@ -2,7 +2,10 @@ from typing import List, Optional, Dict
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.big_bench_hard.task import BigBenchHardTask
 from deepeval.benchmarks.big_bench_hard.template import BigBenchHardTemplate
@@ -81,7 +84,7 @@ class BigBenchHard(DeepEvalBaseBenchmark):
         *args,
         batch_size: Optional[int] = None,
         **kwargs,
-    ) -> Dict:
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("Big Bench Hard", len(self.tasks)):
@@ -189,7 +192,9 @@ class BigBenchHard(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(
         self, model: DeepEvalBaseLLM, task: BigBenchHardTask, golden: Golden
@@ -279,9 +284,7 @@ class BigBenchHard(DeepEvalBaseBenchmark):
         dataset_attr = dataset_mapping.get(task)
         if dataset_attr:
             if not hasattr(self, dataset_attr):
-                dataset = load_dataset(
-                    "lukaemon/bbh", task.value, trust_remote_code=True
-                )
+                dataset = load_dataset("lukaemon/bbh", task.value)
                 setattr(self, dataset_attr, dataset)
             else:
                 dataset = getattr(self, dataset_attr)

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/bool_q/bool_q.py

@@ -2,7 +2,10 @@ from typing import List, Optional, Dict
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.bool_q.template import BoolQTemplate
 from deepeval.benchmarks.schema import AffirmationSchema
@@ -37,7 +40,9 @@ class BoolQ(DeepEvalBaseBenchmark):
         else:
             self.confinement_instructions = confinement_instructions
 
-    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> Dict:
+    def evaluate(
+        self, model: DeepEvalBaseLLM, *args, **kwargs
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("BoolQ", self.n_problems):
@@ -77,7 +82,9 @@ class BoolQ(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         # Define prompt template
@@ -113,7 +120,7 @@ class BoolQ(DeepEvalBaseBenchmark):
         if self.dataset:
             dataset = self.dataset
         else:
-            dataset = load_dataset("boolq", "default", trust_remote_code=True)
+            dataset = load_dataset("boolq", "default")
             self.dataset = dataset
 
         # Construct test set

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/drop/drop.py

@@ -3,7 +3,10 @@ from tqdm import tqdm
 from typing import Union
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.drop.task import DROPTask
 from deepeval.benchmarks.drop.template import DROPTemplate
@@ -49,7 +52,7 @@ class DROP(DeepEvalBaseBenchmark):
         *args,
         batch_size: int | None = None,
         **kwargs,
-    ) -> Dict:
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("DROP", len(self.tasks)):
@@ -155,7 +158,9 @@ class DROP(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         # Define prompt template
@@ -263,7 +268,7 @@ class DROP(DeepEvalBaseBenchmark):
         if self.dataset:
             dataset = self.dataset
         else:
-            dataset = load_dataset("ucinlp/drop", trust_remote_code=True)
+            dataset = load_dataset("ucinlp/drop")
             self.dataset = dataset
 
         # construct example dataset

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/equity_med_qa/equity_med_qa.py

@@ -4,7 +4,10 @@ from tqdm import tqdm
 from deepeval.dataset import Golden
 from deepeval.test_case import LLMTestCase
 from deepeval.metrics import BiasMetric
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.equity_med_qa.task import EquityMedQATask
 from deepeval.benchmarks.equity_med_qa.template import EquityMedQATemplate
@@ -34,7 +37,9 @@ class EquityMedQA(DeepEvalBaseBenchmark):
             initialize_model(model)
         )
 
-    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> Dict:
+    def evaluate(
+        self, model: DeepEvalBaseLLM, *args, **kwargs
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("EquityMedQA", len(self.tasks)):
@@ -97,7 +102,9 @@ class EquityMedQA(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         prediction = model.generate(golden.input)
@@ -143,9 +150,7 @@ class EquityMedQA(DeepEvalBaseBenchmark):
         dataset_attr = dataset_mapping.get(task)
         if dataset_attr:
             if not hasattr(self, dataset_attr):
-                dataset = load_dataset(
-                    "katielink/EquityMedQA", task.value, trust_remote_code=True
-                )
+                dataset = load_dataset("katielink/EquityMedQA", task.value)
                 setattr(self, dataset_attr, dataset)
             else:
                 dataset = getattr(self, dataset_attr)

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/gsm8k/gsm8k.py

@@ -2,7 +2,10 @@ from typing import List, Optional, Dict, Union
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.gsm8k.template import GSM8KTemplate
 from deepeval.benchmarks.schema import NumberSchema
@@ -39,7 +42,9 @@ class GSM8K(DeepEvalBaseBenchmark):
         else:
             self.confinement_instructions = confinement_instructions
 
-    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> Dict:
+    def evaluate(
+        self, model: DeepEvalBaseLLM, *args, **kwargs
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("GSM8K", len(self.tasks)):
@@ -82,7 +87,9 @@ class GSM8K(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         # Define prompt template
@@ -150,7 +157,7 @@ class GSM8K(DeepEvalBaseBenchmark):
         if self.dataset:
             dataset = self.dataset
         else:
-            dataset = load_dataset("gsm8k", "main", trust_remote_code=True)
+            dataset = load_dataset("gsm8k", "main")
             self.dataset = dataset
 
         # Construct example dataset for n_shot inference

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/hellaswag/hellaswag.py

@@ -2,7 +2,10 @@ from typing import List, Dict, Optional
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.hellaswag.task import HellaSwagTask
 from deepeval.benchmarks.hellaswag.template import HellaSwagTemplate
@@ -50,7 +53,7 @@ class HellaSwag(DeepEvalBaseBenchmark):
         *args,
         batch_size: int | None = None,
         **kwargs,
-    ) -> Dict:
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("HellaSwag", len(self.tasks)):
@@ -160,7 +163,9 @@ class HellaSwag(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(
         self, model: DeepEvalBaseLLM, task: HellaSwagTask, golden: Golden
@@ -253,7 +258,7 @@ class HellaSwag(DeepEvalBaseBenchmark):
         if self.dataset:
             dataset = self.dataset
         else:
-            dataset = load_dataset("Rowan/hellaswag", trust_remote_code=True)
+            dataset = load_dataset("Rowan/hellaswag")
             self.dataset = dataset
 
         # If dataset has not been previously loaded, construct

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/human_eval/human_eval.py

@@ -1,7 +1,10 @@
 from typing import List, Optional, Dict
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.human_eval.task import HumanEvalTask
 from deepeval.benchmarks.human_eval.template import HumanEvalTemplate
@@ -93,7 +96,7 @@ class HumanEval(DeepEvalBaseBenchmark):
 
     def evaluate(
         self, model: DeepEvalBaseLLM, *args, k: int = 1, **kwargs
-    ) -> Dict:
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("HumanEval", len(self.tasks)):
@@ -157,7 +160,9 @@ class HumanEval(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(
         self,
@@ -201,7 +206,7 @@ class HumanEval(DeepEvalBaseBenchmark):
         if self.dataset:
             dataset = self.dataset
         else:
-            dataset = load_dataset("openai_humaneval", trust_remote_code=True)
+            dataset = load_dataset("openai_humaneval")
             self.dataset = dataset
 
         # Filter tasks

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/ifeval/ifeval.py

@@ -1,15 +1,29 @@
+from pydantic.config import ConfigDict
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from typing import List, Optional, Dict, Any, Tuple
 from tqdm import tqdm
 import re
 import json
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.schema import StringSchema
 from deepeval.telemetry import capture_benchmark_run
 
 
+class IFEvalResult(DeepEvalBaseBenchmarkResult):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    instruction_breakdown: dict[str, Any]
+    predictions: "pd.DataFrame"
+
+
 class IFEvalInstructionVerifier:
     """
     Verifies instruction compliance for IFEval benchmark.
@@ -394,16 +408,17 @@ class IFEval(DeepEvalBaseBenchmark):
         **kwargs,
     ):
         from deepeval.scorer import Scorer
+        import pandas as pd
 
         super().__init__(**kwargs)
         self.scorer = Scorer()
         self.n_problems = n_problems
         self.verbose_mode = verbose_mode
-        self.predictions = None
-        self.overall_score = None
+        self.predictions: Optional[pd.DataFrame] = None
+        self.overall_score: Optional[float] = None
         self.instruction_breakdown = None
 
-    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> Dict:
+    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> IFEvalResult:
         import pandas as pd
 
         with capture_benchmark_run("IFEval", self.n_problems or "all"):
@@ -459,8 +474,7 @@ class IFEval(DeepEvalBaseBenchmark):
                 print(
                     f"Instruction '{instruction_id}' Accuracy: {accuracy:.4f}"
                 )
-
-            self.predictions = pd.DataFrame(
+            predictions: pd.DataFrame = pd.DataFrame(
                 predictions_row,
                 columns=[
                     "Input",
@@ -468,14 +482,15 @@ class IFEval(DeepEvalBaseBenchmark):
                     "All_Instructions_Correct",
                 ],
             )
+            self.predictions = predictions
             self.overall_score = overall_accuracy
             self.instruction_breakdown = instruction_accuracies
 
-            return {
-                "overall_accuracy": overall_accuracy,
-                "instruction_breakdown": instruction_accuracies,
-                "predictions": self.predictions,
-            }
+            return IFEvalResult(
+                overall_accuracy=overall_accuracy,
+                instruction_breakdown=instruction_accuracies,
+                predictions=predictions,
+            )
 
     def predict(
         self, model: DeepEvalBaseLLM, golden: Golden
@@ -531,7 +546,7 @@ class IFEval(DeepEvalBaseBenchmark):
         if self.dataset:
             dataset = self.dataset
         else:
-            dataset = load_dataset("google/IFEval", trust_remote_code=True)
+            dataset = load_dataset("google/IFEval")
             self.dataset = dataset
 
         goldens: List[Golden] = []

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/lambada/lambada.py

@@ -2,7 +2,10 @@ from typing import List, Optional, Dict
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.lambada.template import LAMBADATemplate
 from deepeval.benchmarks.schema import StringSchema
@@ -37,7 +40,9 @@ class LAMBADA(DeepEvalBaseBenchmark):
         else:
             self.confinement_instructions = confinement_instructions
 
-    def evaluate(self, model: DeepEvalBaseLLM, *args, **kwargs) -> Dict:
+    def evaluate(
+        self, model: DeepEvalBaseLLM, *args, **kwargs
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("LAMBADA", self.n_problems):
@@ -77,7 +82,9 @@ class LAMBADA(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         # Define prompt template
@@ -113,9 +120,7 @@ class LAMBADA(DeepEvalBaseBenchmark):
         if self.dataset:
             dataset = self.dataset
         else:
-            dataset = load_dataset(
-                "EleutherAI/lambada_openai", "default", trust_remote_code=True
-            )
+            dataset = load_dataset("EleutherAI/lambada_openai", "default")
             self.dataset = dataset
 
         # Construct test set

{deepeval-3.4.1 → deepeval-3.4.3}/deepeval/benchmarks/logi_qa/logi_qa.py

@@ -4,7 +4,10 @@ import requests
 import json
 
 from deepeval.dataset import Golden
-from deepeval.benchmarks.base_benchmark import DeepEvalBaseBenchmark
+from deepeval.benchmarks.base_benchmark import (
+    DeepEvalBaseBenchmark,
+    DeepEvalBaseBenchmarkResult,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.benchmarks.logi_qa.task import LogiQATask
 from deepeval.benchmarks.logi_qa.template import LogiQATemplate
@@ -51,7 +54,7 @@ class LogiQA(DeepEvalBaseBenchmark):
         *args,
         batch_size: int | None = None,
         **kwargs,
-    ) -> Dict:
+    ) -> DeepEvalBaseBenchmarkResult:
         import pandas as pd
 
         with capture_benchmark_run("LogiQA", len(self.tasks)):
@@ -157,7 +160,9 @@ class LogiQA(DeepEvalBaseBenchmark):
             )
             self.overall_score = overall_accuracy
 
-            return overall_accuracy
+            return DeepEvalBaseBenchmarkResult(
+                overall_accuracy=overall_accuracy
+            )
 
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         # Define prompt template