mteb 2.5.2__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/deprecated_evaluator.py

@@ -5,29 +5,30 @@ import logging
 import os
 import sys
 import traceback
-from collections.abc import Iterable
+import warnings
+from collections.abc import Iterable, Sequence
 from copy import deepcopy
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
 from time import time
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 import datasets
 
 import mteb
 from mteb.abstasks import AbsTask
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.abstasks.task_metadata import TaskCategory, TaskType
 from mteb.benchmarks import Benchmark
 from mteb.models import (
     CrossEncoderWrapper,
-    EncoderProtocol,
     ModelMeta,
     MTEBModels,
     SentenceTransformerEncoderWrapper,
 )
 from mteb.results import TaskResult
-from mteb.types import ScoresDict
+from mteb.types import EncodeKwargs, ScoresDict
 
 if sys.version_info >= (3, 13):
     from warnings import deprecated
@@ -52,7 +53,7 @@ class MTEB:
     )
     def __init__(
         self,
-        tasks: Iterable[AbsTask | Benchmark],
+        tasks: Iterable[AbsTask] | Iterable[Benchmark],
         *,
         err_logs_path: str = "error_logs.txt",
     ) -> None:
@@ -63,15 +64,14 @@ class MTEB:
                 `mteb.get_tasks(["task1","task2"]) or `mteb.get_benchmark("MTEB(eng, classic)").
             err_logs_path: Path to save error logs.
         """
-        from mteb.benchmarks import Benchmark
-
-        self.tasks = list(tasks)
-        if len(self.tasks) > 0 and isinstance(self.tasks[0], Benchmark):
+        if isinstance(next(iter(tasks)), Benchmark):
             self.benchmarks = tasks
-            self.tasks = list(chain.from_iterable(self.tasks))
+            self.tasks = list(chain.from_iterable(cast(Iterable[Benchmark], tasks)))
+        elif isinstance(next(iter(tasks)), AbsTask):
+            self.tasks = list(cast(Iterable[AbsTask], tasks))
 
         self.err_logs_path = Path(err_logs_path)
-        self.last_evaluated_splits = {}
+        self._last_evaluated_splits: dict[str, list[str]] = {}
 
     @property
     def available_tasks(self) -> list[str]:
@@ -84,7 +84,7 @@ class MTEB:
         return sorted({x.metadata.type for x in self.tasks})
 
     @property
-    def available_task_categories(self) -> set[TaskCategory]:
+    def available_task_categories(self) -> set[TaskCategory | None]:
         """Set of available task categories."""
         return {x.metadata.category for x in self.tasks}
 
@@ -174,7 +174,7 @@ class MTEB:
         split: str,
         subsets_to_run: list[str] | None = None,
         *,
-        encode_kwargs: dict[str, Any],
+        encode_kwargs: EncodeKwargs,
         **kwargs: Any,
     ):
         tick = time()
@@ -231,13 +231,14 @@ class MTEB:
         merged_kg_co2_emissions = None
         if existing_kg_co2_emissions and new_kg_co2_emissions:
             merged_kg_co2_emissions = existing_kg_co2_emissions + new_kg_co2_emissions
+        existing_evaluation_time = existing_results.evaluation_time or 0
+        new_evaluation_time = new_results.evaluation_time or 0
         merged_results = TaskResult(
             dataset_revision=new_results.dataset_revision,
             task_name=new_results.task_name,
             mteb_version=new_results.mteb_version,
             scores=merged_scores,
-            evaluation_time=existing_results.evaluation_time
-            + new_results.evaluation_time,
+            evaluation_time=existing_evaluation_time + new_evaluation_time,
             kg_co2_emissions=merged_kg_co2_emissions,
         )
 
@@ -262,7 +263,7 @@ class MTEB:
         overwrite_results: bool = False,
         raise_error: bool = True,
         co2_tracker: bool = False,
-        encode_kwargs: dict[str, Any] | None = None,
+        encode_kwargs: EncodeKwargs | None = None,
         **kwargs,
     ) -> list[TaskResult]:
         """Run the evaluation pipeline on the selected tasks.
@@ -306,13 +307,16 @@ class MTEB:
         elif verbosity == 3:
             datasets.logging.set_verbosity(logging.DEBUG)
 
-        meta = self.create_model_meta(model)
-        output_path = self._create_output_folder(meta, output_folder)
-
+        mteb_model: MTEBModels
         if isinstance(model, SentenceTransformer):
-            model = SentenceTransformerEncoderWrapper(model)
+            mteb_model = SentenceTransformerEncoderWrapper(model)
         elif isinstance(model, CrossEncoder):
-            model = CrossEncoderWrapper(model)
+            mteb_model = CrossEncoderWrapper(model)
+        else:
+            mteb_model = cast(MTEBModels, model)
+
+        meta = self.create_model_meta(mteb_model)
+        output_path = self._create_output_folder(meta, output_folder)
 
         # Disable co2_tracker for API models
         if "API" in meta.framework:
@@ -333,7 +337,7 @@ class MTEB:
         )  # save them in case we re-use the object (e.g. for reranking)
 
         # To evaluate missing splits, we keep track of the task name and the corresponding splits.
-        self.last_evaluated_splits = {}
+        self._last_evaluated_splits = {}
 
         while len(self.tasks) > 0:
             task = self.tasks[0]
@@ -342,9 +346,10 @@ class MTEB:
             )
 
             if task.is_aggregate:
-                self_ = MTEB(tasks=task.metadata.tasks)
-                task_results = self_.run(
-                    model,
+                aggregated_task = cast(AbsTaskAggregate, task)
+                self_ = MTEB(tasks=aggregated_task.metadata.tasks)
+                aggregated_task_results = self_.run(
+                    mteb_model,
                     verbosity=verbosity - 1,
                     output_folder=output_folder,
                     eval_splits=eval_splits,
@@ -355,12 +360,15 @@ class MTEB:
                     encode_kwargs=encode_kwargs,
                     **kwargs,
                 )
-                new_results = task.combine_task_results(task_results)
+                new_results = aggregated_task.combine_task_results(
+                    aggregated_task_results
+                )
                 evaluation_results.append(new_results)
 
                 if output_path:
-                    save_path = output_path / f"{task.metadata.name}.json"
-                    new_results.to_disk(save_path)
+                    new_results.to_disk(
+                        output_path / f"{aggregated_task.metadata.name}.json"
+                    )
                 del self.tasks[0]
                 continue
 
@@ -382,7 +390,7 @@ class MTEB:
             task_subsets = task.hf_subsets
 
             existing_results = None
-            save_path = None
+            save_path: Path | None = None
             final_splits_to_run = task_eval_splits
             missing_evaluations = self._get_missing_evaluations(
                 existing_results,
@@ -432,7 +440,7 @@ class MTEB:
                     logger.info(
                         f"No splits to evaluate for {task.metadata.name}. Skipping evaluation."
                     )
-                self.last_evaluated_splits[task.metadata.name] = []
+                self._last_evaluated_splits[task.metadata.name] = []
                 del self.tasks[0]
                 continue
 
@@ -440,11 +448,11 @@ class MTEB:
                 task.check_if_dataset_is_superseded()
                 task.load_data()
 
-                task_results = {}
+                task_results: dict[str, dict[str, dict[str, Any]]] = {}
                 evaluation_time = 0
                 kg_co2_emissions: int | None = 0 if co2_tracker else None
 
-                self.last_evaluated_splits[task.metadata.name] = []
+                self._last_evaluated_splits[task.metadata.name] = []
 
                 for split in final_splits_to_run:
                     info = missing_evaluations[split]
@@ -465,14 +473,16 @@ class MTEB:
 
                     if co2_tracker:
                         try:
-                            from codecarbon import EmissionsTracker
+                            from codecarbon import (  # type: ignore[import-not-found,import-untyped]
+                                EmissionsTracker,
+                            )
                         except ImportError:
                             raise ImportError(
                                 "codecarbon is not installed. Please install it using `pip install 'mteb[codecarbon]'` to track CO₂ emissions."
                             )
-                        logger.warning(
-                            "Evaluating multiple MTEB runs simultaneously will produce incorrect CO₂ results"
-                        )
+                        msg = "Evaluating multiple MTEB runs simultaneously will produce incorrect CO₂ results"
+                        logger.warning(msg)
+                        warnings.warn(msg)
                         with EmissionsTracker(
                             save_to_file=False,
                             save_to_api=False,
@@ -481,7 +491,7 @@ class MTEB:
                         ) as tracker:
                             results, tick, tock = self._run_eval(
                                 task,
-                                model,
+                                mteb_model,
                                 split,
                                 encode_kwargs=encode_kwargs,
                                 subsets_to_run=subsets_to_run,
@@ -494,7 +504,7 @@ class MTEB:
                     else:
                         results, tick, tock = self._run_eval(
                             task,
-                            model,
+                            mteb_model,
                             split,
                             subsets_to_run=subsets_to_run,
                             encode_kwargs=encode_kwargs,
@@ -510,25 +520,25 @@ class MTEB:
                     if verbosity >= 1:
                         logger.info(f"Scores: {task_results[split]}")
 
-                    self.last_evaluated_splits[task.metadata.name].append(split)
+                    self._last_evaluated_splits[task.metadata.name].append(split)
 
                 # Create new TaskResult
                 new_results = TaskResult.from_task_results(
                     task,
-                    task_results,
+                    task_results,  # type: ignore[arg-type]
                     evaluation_time=evaluation_time,
                     kg_co2_emissions=kg_co2_emissions,
                 )
 
                 # Merge with existing if needed
-                if output_path and save_path.exists():
+                if output_path and save_path and save_path.exists():
                     existing_results = TaskResult.from_disk(save_path)
                 if existing_results:
                     merged_results = self._merge_results(existing_results, new_results)
                 else:
                     merged_results = new_results
 
-                if output_path:
+                if output_path and save_path:
                     merged_results.to_disk(save_path)
 
                 evaluation_results.append(merged_results)
@@ -555,7 +565,7 @@ class MTEB:
     def create_model_meta(model: MTEBModels) -> ModelMeta:
         """Create a ModelMeta object for the given model."""
         if hasattr(model, "mteb_model_meta") and model.mteb_model_meta is not None:
-            meta = model.mteb_model_meta  # type: ignore
+            meta = model.mteb_model_meta
         else:
             meta = MTEB._get_model_meta(model)
 
@@ -581,7 +591,11 @@ class MTEB:
         if output_folder is None:
             return None
 
-        model_revision: str = model_meta.revision  # type: ignore
+        model_revision: str = (
+            model_meta.revision
+            if model_meta.revision is not None
+            else "no_revision_available"
+        )
         model_path_name = model_meta.model_name_as_path()
 
         output_path = Path(output_folder) / model_path_name / model_revision
@@ -603,15 +617,15 @@ class MTEB:
              Tasks with empty lists indicate that results already existed and no splits were evaluated.
         """
         return deepcopy(
-            {task: list(splits) for task, splits in self.last_evaluated_splits.items()}
+            {task: list(splits) for task, splits in self._last_evaluated_splits.items()}
         )
 
     @staticmethod
     def _get_missing_evaluations(
         existing_results: TaskResult | None,
-        task_eval_splits: list[str],
-        task_eval_langs: list[str],
-        eval_subsets: list[str] | None,
+        task_eval_splits: Sequence[str],
+        task_eval_langs: Sequence[str],
+        eval_subsets: Sequence[str] | None,
     ) -> dict[str, dict[str, Any]]:
         """Return a dictionary for each split, indicating if the whole split is missing and which subsets are missing."""
         missing_evaluations = {
@@ -660,7 +674,7 @@ class MTEB:
         return missing_evaluations
 
     @staticmethod
-    def _get_model_meta(model: EncoderProtocol) -> ModelMeta:
+    def _get_model_meta(model: MTEBModels) -> ModelMeta:
         from sentence_transformers import CrossEncoder, SentenceTransformer
 
         if isinstance(model, CrossEncoder):

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2CybersecurityRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 1299,
+        "number_of_characters": 9254,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 2245,
+            "average_image_width": 2370.324347826087,
+            "max_image_width": 3508,
+            "min_image_height": 2481,
+            "average_image_height": 3289.8060869565215,
+            "max_image_height": 3580,
+            "unique_images": 1132
+        },
+        "queries_text_statistics": {
+            "total_text_length": 9254,
+            "min_text_length": 15,
+            "average_text_length": 62.10738255033557,
+            "max_text_length": 108,
+            "unique_texts": 149
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 409,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 2.7449664429530203,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 316
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EconomicRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 1640,
+        "number_of_characters": 8331,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 2313,
+            "average_image_width": 2347.5321597833445,
+            "max_image_width": 2481,
+            "min_image_height": 3138,
+            "average_image_height": 3214.301963439404,
+            "max_image_height": 3508,
+            "unique_images": 1442
+        },
+        "queries_text_statistics": {
+            "total_text_length": 8331,
+            "min_text_length": 23,
+            "average_text_length": 51.11042944785276,
+            "max_text_length": 110,
+            "unique_texts": 163
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 413,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 2.5337423312883436,
+            "max_relevant_docs_per_query": 6,
+            "unique_relevant_docs": 349
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EnergyRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 2166,
+        "number_of_characters": 9764,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 2221,
+            "average_image_width": 2339.4957350727545,
+            "max_image_width": 2480,
+            "min_image_height": 3036,
+            "average_image_height": 3242.8138484696437,
+            "max_image_height": 3508,
+            "unique_images": 1974
+        },
+        "queries_text_statistics": {
+            "total_text_length": 9764,
+            "min_text_length": 22,
+            "average_text_length": 56.4393063583815,
+            "max_text_length": 103,
+            "unique_texts": 173
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 525,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 3.0346820809248554,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 442
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2HrRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 2330,
+        "number_of_characters": 13131,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 1949,
+            "average_image_width": 2430.1152204836417,
+            "max_image_width": 3505,
+            "min_image_height": 2480,
+            "average_image_height": 3350.3921289710765,
+            "max_image_height": 3626,
+            "unique_images": 2096
+        },
+        "queries_text_statistics": {
+            "total_text_length": 13131,
+            "min_text_length": 21,
+            "average_text_length": 59.41628959276018,
+            "max_text_length": 112,
+            "unique_texts": 221
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 726,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 3.2850678733031673,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 575
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/ChemRxivRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 74457,
+        "number_of_characters": 76109543,
+        "documents_text_statistics": {
+            "total_text_length": 75549698,
+            "min_text_length": 121,
+            "average_text_length": 1087.7189916063176,
+            "max_text_length": 25438,
+            "unique_texts": 69150
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 559845,
+            "min_text_length": 57,
+            "average_text_length": 111.969,
+            "max_text_length": 224,
+            "unique_texts": 5000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 5000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 5000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/EuroPIRQRetrieval.json

@@ -0,0 +1,116 @@
+{
+    "test": {
+        "num_samples": 30300,
+        "number_of_characters": 17320243,
+        "documents_text_statistics": {
+            "total_text_length": 17276572,
+            "min_text_length": 316,
+            "average_text_length": 575.8857333333333,
+            "max_text_length": 1008,
+            "unique_texts": 28361
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 43671,
+            "min_text_length": 67,
+            "average_text_length": 145.57,
+            "max_text_length": 345,
+            "unique_texts": 300
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 300,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 300
+        },
+        "top_ranked_statistics": null,
+        "hf_subset_descriptive_stats": {
+            "en": {
+                "num_samples": 10100,
+                "number_of_characters": 5517678,
+                "documents_text_statistics": {
+                    "total_text_length": 5503635,
+                    "min_text_length": 316,
+                    "average_text_length": 550.3635,
+                    "max_text_length": 726,
+                    "unique_texts": 9422
+                },
+                "documents_image_statistics": null,
+                "queries_text_statistics": {
+                    "total_text_length": 14043,
+                    "min_text_length": 68,
+                    "average_text_length": 140.43,
+                    "max_text_length": 305,
+                    "unique_texts": 100
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 100,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 100
+                },
+                "top_ranked_statistics": null
+            },
+            "fi": {
+                "num_samples": 10100,
+                "number_of_characters": 5953462,
+                "documents_text_statistics": {
+                    "total_text_length": 5938809,
+                    "min_text_length": 326,
+                    "average_text_length": 593.8809,
+                    "max_text_length": 1008,
+                    "unique_texts": 9422
+                },
+                "documents_image_statistics": null,
+                "queries_text_statistics": {
+                    "total_text_length": 14653,
+                    "min_text_length": 67,
+                    "average_text_length": 146.53,
+                    "max_text_length": 345,
+                    "unique_texts": 100
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 100,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 100
+                },
+                "top_ranked_statistics": null
+            },
+            "pt": {
+                "num_samples": 10100,
+                "number_of_characters": 5849103,
+                "documents_text_statistics": {
+                    "total_text_length": 5834128,
+                    "min_text_length": 325,
+                    "average_text_length": 583.4128,
+                    "max_text_length": 774,
+                    "unique_texts": 9517
+                },
+                "documents_image_statistics": null,
+                "queries_text_statistics": {
+                    "total_text_length": 14975,
+                    "min_text_length": 69,
+                    "average_text_length": 149.75,
+                    "max_text_length": 320,
+                    "unique_texts": 100
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 100,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 100
+                },
+                "top_ranked_statistics": null
+            }
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/NanoClimateFEVER-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 102198,
+        "number_of_characters": 47870352,
+        "documents_text_statistics": {
+            "total_text_length": 47719757,
+            "min_text_length": 9,
+            "average_text_length": 472.01951591046225,
+            "max_text_length": 8686,
+            "unique_texts": 101097
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 150595,
+            "min_text_length": 30,
+            "average_text_length": 136.78019981834694,
+            "max_text_length": 404,
+            "unique_texts": 1099
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 3401,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 3.089009990917348,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 1123
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoDBPedia-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 132137,
+        "number_of_characters": 43323279,
+        "documents_text_statistics": {
+            "total_text_length": 43311486,
+            "min_text_length": 11,
+            "average_text_length": 328.5778249819823,
+            "max_text_length": 8576,
+            "unique_texts": 131814
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 11793,
+            "min_text_length": 6,
+            "average_text_length": 36.62422360248447,
+            "max_text_length": 100,
+            "unique_texts": 321
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 11620,
+            "min_relevant_docs_per_query": 31,
+            "average_relevant_docs_per_query": 36.08695652173913,
+            "max_relevant_docs_per_query": 1288,
+            "unique_relevant_docs": 32537
+        },
+        "top_ranked_statistics": null
+    }
+}