mteb 2.5.2__py3-none-any.whl → 2.7.9__py3-none-any.whl

mteb/descriptive_stats/Retrieval/ChemRxivRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 74457,
+        "number_of_characters": 76109543,
+        "documents_text_statistics": {
+            "total_text_length": 75549698,
+            "min_text_length": 121,
+            "average_text_length": 1087.7189916063176,
+            "max_text_length": 25438,
+            "unique_texts": 69150
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 559845,
+            "min_text_length": 57,
+            "average_text_length": 111.969,
+            "max_text_length": 224,
+            "unique_texts": 5000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 5000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 5000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/EuroPIRQRetrieval.json

@@ -0,0 +1,116 @@
+{
+    "test": {
+        "num_samples": 30300,
+        "number_of_characters": 17320243,
+        "documents_text_statistics": {
+            "total_text_length": 17276572,
+            "min_text_length": 316,
+            "average_text_length": 575.8857333333333,
+            "max_text_length": 1008,
+            "unique_texts": 28361
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 43671,
+            "min_text_length": 67,
+            "average_text_length": 145.57,
+            "max_text_length": 345,
+            "unique_texts": 300
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 300,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 300
+        },
+        "top_ranked_statistics": null,
+        "hf_subset_descriptive_stats": {
+            "en": {
+                "num_samples": 10100,
+                "number_of_characters": 5517678,
+                "documents_text_statistics": {
+                    "total_text_length": 5503635,
+                    "min_text_length": 316,
+                    "average_text_length": 550.3635,
+                    "max_text_length": 726,
+                    "unique_texts": 9422
+                },
+                "documents_image_statistics": null,
+                "queries_text_statistics": {
+                    "total_text_length": 14043,
+                    "min_text_length": 68,
+                    "average_text_length": 140.43,
+                    "max_text_length": 305,
+                    "unique_texts": 100
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 100,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 100
+                },
+                "top_ranked_statistics": null
+            },
+            "fi": {
+                "num_samples": 10100,
+                "number_of_characters": 5953462,
+                "documents_text_statistics": {
+                    "total_text_length": 5938809,
+                    "min_text_length": 326,
+                    "average_text_length": 593.8809,
+                    "max_text_length": 1008,
+                    "unique_texts": 9422
+                },
+                "documents_image_statistics": null,
+                "queries_text_statistics": {
+                    "total_text_length": 14653,
+                    "min_text_length": 67,
+                    "average_text_length": 146.53,
+                    "max_text_length": 345,
+                    "unique_texts": 100
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 100,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 100
+                },
+                "top_ranked_statistics": null
+            },
+            "pt": {
+                "num_samples": 10100,
+                "number_of_characters": 5849103,
+                "documents_text_statistics": {
+                    "total_text_length": 5834128,
+                    "min_text_length": 325,
+                    "average_text_length": 583.4128,
+                    "max_text_length": 774,
+                    "unique_texts": 9517
+                },
+                "documents_image_statistics": null,
+                "queries_text_statistics": {
+                    "total_text_length": 14975,
+                    "min_text_length": 69,
+                    "average_text_length": 149.75,
+                    "max_text_length": 320,
+                    "unique_texts": 100
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 100,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 100
+                },
+                "top_ranked_statistics": null
+            }
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/NanoClimateFEVER-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 102198,
+        "number_of_characters": 47870352,
+        "documents_text_statistics": {
+            "total_text_length": 47719757,
+            "min_text_length": 9,
+            "average_text_length": 472.01951591046225,
+            "max_text_length": 8686,
+            "unique_texts": 101097
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 150595,
+            "min_text_length": 30,
+            "average_text_length": 136.78019981834694,
+            "max_text_length": 404,
+            "unique_texts": 1099
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 3401,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 3.089009990917348,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 1123
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoDBPedia-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 132137,
+        "number_of_characters": 43323279,
+        "documents_text_statistics": {
+            "total_text_length": 43311486,
+            "min_text_length": 11,
+            "average_text_length": 328.5778249819823,
+            "max_text_length": 8576,
+            "unique_texts": 131814
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 11793,
+            "min_text_length": 6,
+            "average_text_length": 36.62422360248447,
+            "max_text_length": 100,
+            "unique_texts": 321
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 11620,
+            "min_relevant_docs_per_query": 31,
+            "average_relevant_docs_per_query": 36.08695652173913,
+            "max_relevant_docs_per_query": 1288,
+            "unique_relevant_docs": 32537
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoFEVER-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 106558,
+        "number_of_characters": 48164581,
+        "documents_text_statistics": {
+            "total_text_length": 47886101,
+            "min_text_length": 9,
+            "average_text_length": 472.6783768310499,
+            "max_text_length": 8689,
+            "unique_texts": 101308
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 278480,
+            "min_text_length": 11,
+            "average_text_length": 53.04380952380952,
+            "max_text_length": 196,
+            "unique_texts": 5124
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 6254,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1912380952380952,
+            "max_relevant_docs_per_query": 15,
+            "unique_relevant_docs": 1324
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoHotpotQA-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 117974,
+        "number_of_characters": 35927363,
+        "documents_text_statistics": {
+            "total_text_length": 35335613,
+            "min_text_length": 22,
+            "average_text_length": 316.47705838625023,
+            "max_text_length": 4105,
+            "unique_texts": 111651
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 591750,
+            "min_text_length": 21,
+            "average_text_length": 93.61651637399146,
+            "max_text_length": 280,
+            "unique_texts": 6321
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 12642,
+            "min_relevant_docs_per_query": 2,
+            "average_relevant_docs_per_query": 2.0,
+            "max_relevant_docs_per_query": 2,
+            "unique_relevant_docs": 11874
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoMSMARCO-VN.json

@@ -0,0 +1,30 @@
+{
+    "dev": {
+        "num_samples": 107153,
+        "number_of_characters": 33316879,
+        "documents_text_statistics": {
+            "total_text_length": 33200903,
+            "min_text_length": 2,
+            "average_text_length": 320.30199218561575,
+            "max_text_length": 1712,
+            "unique_texts": 103641
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 115976,
+            "min_text_length": 8,
+            "average_text_length": 33.15494568324757,
+            "max_text_length": 190,
+            "unique_texts": 3498
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 3700,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0577472841623785,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 3698
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NanoNQ-VN.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 104095,
+        "number_of_characters": 52312680,
+        "documents_text_statistics": {
+            "total_text_length": 52220289,
+            "min_text_length": 10,
+            "average_text_length": 510.98673124908265,
+            "max_text_length": 10245,
+            "unique_texts": 102181
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 92391,
+            "min_text_length": 22,
+            "average_text_length": 48.62684210526316,
+            "max_text_length": 113,
+            "unique_texts": 1900
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 2283,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.201578947368421,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 2283
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/TVPLRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 20561,
+        "number_of_characters": 10832770,
+        "documents_text_statistics": {
+            "total_text_length": 9929303,
+            "min_text_length": 9,
+            "average_text_length": 938.8524016641452,
+            "max_text_length": 6319,
+            "unique_texts": 10573
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 903467,
+            "min_text_length": 13,
+            "average_text_length": 90.48242363545317,
+            "max_text_length": 228,
+            "unique_texts": 9985
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 11158,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1174762143214823,
+            "max_relevant_docs_per_query": 8,
+            "unique_relevant_docs": 10576
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/evaluate.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 import logging
-from collections.abc import Iterable
+import warnings
 from pathlib import Path
 from time import time
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, cast
 
 from datasets.exceptions import DatasetNotFoundError
 from tqdm.auto import tqdm
@@ -13,25 +13,28 @@ from mteb._helpful_enum import HelpfulStrEnum
 from mteb.abstasks import AbsTaskRetrieval
 from mteb.abstasks.abstask import AbsTask
 from mteb.abstasks.aggregated_task import AbsTaskAggregate
+from mteb.benchmarks.benchmark import Benchmark
 from mteb.cache import ResultCache
 from mteb.models.model_meta import ModelMeta
-from mteb.models.models_protocols import (
-    CrossEncoderProtocol,
-    EncoderProtocol,
-    MTEBModels,
-)
 from mteb.models.sentence_transformer_wrapper import (
     CrossEncoderWrapper,
     SentenceTransformerEncoderWrapper,
 )
 from mteb.results import ModelResult, TaskResult
 from mteb.results.task_result import TaskError
-from mteb.types import HFSubset, PromptType, SplitName
-from mteb.types._metadata import ModelName, Revision
+from mteb.types import PromptType
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from sentence_transformers import CrossEncoder, SentenceTransformer
 
+    from mteb.models.models_protocols import (
+        MTEBModels,
+    )
+    from mteb.types import EncodeKwargs, HFSubset, SplitName
+    from mteb.types._metadata import ModelName, Revision
+
 logger = logging.getLogger(__name__)
 
 
@@ -57,27 +60,26 @@ def _sanitize_model(
 ) -> tuple[MTEBModels | ModelMeta, ModelMeta, ModelName, Revision]:
     from sentence_transformers import CrossEncoder, SentenceTransformer
 
+    wrapped_model: MTEBModels | ModelMeta
     if isinstance(model, SentenceTransformer):
-        _mdl = SentenceTransformerEncoderWrapper(model)
-        meta = _mdl.mteb_model_meta
-        _mdl = cast(EncoderProtocol, _mdl)
-        model = _mdl
+        wrapped_model = SentenceTransformerEncoderWrapper(model)
+        meta = wrapped_model.mteb_model_meta
     elif isinstance(model, CrossEncoder):
-        _mdl = CrossEncoderWrapper(model)
-        _mdl = cast(CrossEncoderProtocol, _mdl)
-        meta = _mdl.mteb_model_meta
-        model = _mdl
+        wrapped_model = CrossEncoderWrapper(model)
+        meta = wrapped_model.mteb_model_meta
     elif hasattr(model, "mteb_model_meta"):
-        meta = model.mteb_model_meta  # type: ignore[attr-defined]
+        meta = getattr(model, "mteb_model_meta")
         if not isinstance(meta, ModelMeta):
-            meta = ModelMeta.from_hub(None)
+            meta = ModelMeta._from_hub(None)
+        wrapped_model = cast("MTEBModels | ModelMeta", model)
     else:
-        meta = ModelMeta.from_hub(None) if not isinstance(model, ModelMeta) else model
+        meta = ModelMeta._from_hub(None) if not isinstance(model, ModelMeta) else model
+        wrapped_model = meta
 
-    model_name = cast(str, meta.name)
-    model_revision = cast(str, meta.revision)
+    model_name = cast("str", meta.name)
+    model_revision = cast("str", meta.revision)
 
-    return model, meta, model_name, model_revision
+    return wrapped_model, meta, model_name, model_revision
 
 
 def _evaluate_task(
@@ -86,9 +88,10 @@ def _evaluate_task(
     *,
     splits: dict[SplitName, list[HFSubset]],
     co2_tracker: bool | None,
-    encode_kwargs: dict[str, Any],
+    encode_kwargs: EncodeKwargs,
     prediction_folder: Path | None,
     public_only: bool | None,
+    num_proc: int = 1,
 ) -> TaskResult | TaskError:
     """The core logic to run a model on a given task. See `evaluate` for more details.
 
@@ -122,24 +125,28 @@ def _evaluate_task(
                 co2_tracker=False,
                 prediction_folder=prediction_folder,
                 public_only=public_only,
+                num_proc=num_proc,
             )
-        result.kg_co2_emissions = tracker.final_emissions
+        if isinstance(result, TaskResult):
+            result.kg_co2_emissions = tracker.final_emissions
         return result
 
     task_results = {}
 
     task.check_if_dataset_is_superseded()
 
-    data_loaded = task.data_loaded
-    if not data_loaded:
+    data_preloaded = task.data_loaded
+    if not data_preloaded:
         try:
-            task.load_data()
+            task.load_data(num_proc=num_proc)
         except DatasetNotFoundError as e:
             if not task.metadata.is_public and public_only is None:
-                logger.warning(
+                msg = (
                     f"Dataset for private task '{task.metadata.name}' not found. "
                     "Make sure you have access to the dataset and that you have set up the authentication correctly. To disable this warning set `public_only=False`"
                 )
+                logger.warning(msg)
+                warnings.warn(msg)
                 return TaskError(
                     task_name=task.metadata.name,
                     exception=str(e),
@@ -147,7 +154,7 @@ def _evaluate_task(
             if public_only is False:
                 raise e
 
-    evaluation_time = 0
+    evaluation_time = 0.0
 
     for split, hf_subsets in splits.items():
         tick = time()
@@ -157,6 +164,7 @@ def _evaluate_task(
             subsets_to_run=hf_subsets,
             encode_kwargs=encode_kwargs,
             prediction_folder=prediction_folder,
+            num_proc=num_proc,
         )
         tock = time()
 
@@ -172,7 +180,7 @@ def _evaluate_task(
         kg_co2_emissions=None,
     )
 
-    if data_loaded:  # only unload if we loaded the data
+    if not data_preloaded:  # only unload if we loaded the data
         task.unload_data()
 
     return result
@@ -194,12 +202,18 @@ def _check_model_modalities(
         return
 
     model_modalities = set(model.modalities)
+    check_tasks: Iterable[AbsTask] = []
     if isinstance(tasks, AbsTask):
-        tasks = [tasks]
+        check_tasks = [tasks]
+    elif isinstance(tasks, Benchmark):
+        benchmark = cast("Benchmark", tasks)
+        check_tasks = benchmark.tasks
+    else:
+        check_tasks = cast("Iterable[AbsTask]", tasks)
 
     warnings, errors = [], []
 
-    for task in tasks:
+    for task in check_tasks:
         # only retrieval tasks have different modalities for query and document and can be run with partial overlaps
         if isinstance(task, AbsTaskRetrieval):
             query_mods = set(task.metadata.get_modalities(PromptType.query))
@@ -262,12 +276,13 @@ def evaluate(
     *,
     co2_tracker: bool | None = None,
     raise_error: bool = True,
-    encode_kwargs: dict[str, Any] | None = None,
+    encode_kwargs: EncodeKwargs | None = None,
     cache: ResultCache | None = ResultCache(),
     overwrite_strategy: str | OverwriteStrategy = "only-missing",
     prediction_folder: Path | str | None = None,
     show_progress_bar: bool = True,
     public_only: bool | None = None,
+    num_proc: int = 1,
 ) -> ModelResult:
     """This function runs a model on a given task and returns the results.
 
@@ -276,7 +291,7 @@ def evaluate(
         tasks: A task to run.
         co2_tracker: If True, track the CO₂ emissions of the evaluation, required codecarbon to be installed, which can be installed using
             `pip install mteb[codecarbon]`. If none is passed co2 tracking will only be run if codecarbon is installed.
-        encode_kwargs: Additional keyword arguments passed to the models `encode` method.
+        encode_kwargs: Additional keyword arguments passed to the models `encode` and `load_data` methods;
         raise_error: If True, raise an error if the task fails. If False, return an empty list.
         cache: The cache to use for loading the results. If None, then no cache will be used. The default cache saved the cache in the
             `~/.cache/mteb` directory. It can be overridden by setting the `MTEB_CACHE` environment variable to a different directory or by directly
@@ -288,10 +303,11 @@ def evaluate(
                 changed.
             - "only-cache": Only load the results from the cache folder and do not run the task. Useful if you just want to load the results from the
                 cache.
-        prediction_folder: Optional folder in which to save model predictions for the task. Predictions of the tasks will be sabed in `prediction_folder/{task_name}_predictions.json`
+        prediction_folder: Optional folder in which to save model predictions for the task. Predictions of the tasks will be saved in `prediction_folder/{task_name}_predictions.json`
         show_progress_bar: Whether to show a progress bar when running the evaluation. Default is True. Setting this to False will also set the
             `encode_kwargs['show_progress_bar']` to False if encode_kwargs is unspecified.
         public_only: Run only public tasks. If None, it will attempt to run the private task.
+        num_proc: Number of processes to use during data loading and transformation. Defaults to 1.
 
     Returns:
         The results of the evaluation.
@@ -332,10 +348,10 @@ def evaluate(
 
     # AbsTaskAggregate is a special case where we have to run multiple tasks and combine the results
     if isinstance(tasks, AbsTaskAggregate):
-        task = cast(AbsTaskAggregate, tasks)
+        aggregated_task = cast("AbsTaskAggregate", tasks)
         results = evaluate(
             model,
-            task.metadata.tasks,
+            aggregated_task.metadata.tasks,
             co2_tracker=co2_tracker,
             raise_error=raise_error,
             encode_kwargs=encode_kwargs,
@@ -344,18 +360,23 @@ def evaluate(
             prediction_folder=prediction_folder,
             show_progress_bar=show_progress_bar,
             public_only=public_only,
+            num_proc=num_proc,
         )
-        result = task.combine_task_results(results.task_results)
+        combined_results = aggregated_task.combine_task_results(results.task_results)
+        if cache:
+            cache.save_to_cache(combined_results, meta)
+
         return ModelResult(
             model_name=results.model_name,
             model_revision=results.model_revision,
-            task_results=[result],
+            task_results=[combined_results],
         )
 
     if isinstance(tasks, AbsTask):
         task = tasks
     else:
-        results = []
+        tasks = cast("Iterable[AbsTask]", tasks)
+        evaluate_results = []
         exceptions = []
         tasks_tqdm = tqdm(
             tasks,
@@ -375,24 +396,25 @@ def evaluate(
                 prediction_folder=prediction_folder,
                 show_progress_bar=False,
                 public_only=public_only,
+                num_proc=num_proc,
             )
-            results.extend(_res.task_results)
+            evaluate_results.extend(_res.task_results)
             if _res.exceptions:
                 exceptions.extend(_res.exceptions)
         return ModelResult(
             model_name=_res.model_name,
             model_revision=_res.model_revision,
-            task_results=results,
+            task_results=evaluate_results,
             exceptions=exceptions,
         )
 
     overwrite_strategy = OverwriteStrategy.from_str(overwrite_strategy)
 
-    existing_results = None
+    existing_results: TaskResult | None = None
     if cache and overwrite_strategy != OverwriteStrategy.ALWAYS:
-        results = cache.load_task_result(task.metadata.name, meta)
-        if results:
-            existing_results = results
+        cache_results = cache.load_task_result(task.metadata.name, meta)
+        if cache_results:
+            existing_results = cache_results
 
     if (
         existing_results
@@ -454,6 +476,7 @@ def evaluate(
                 encode_kwargs=encode_kwargs,
                 prediction_folder=prediction_folder,
                 public_only=public_only,
+                num_proc=num_proc,
             )
         except Exception as e:
             logger.error(
@@ -469,6 +492,7 @@ def evaluate(
             encode_kwargs=encode_kwargs,
             prediction_folder=prediction_folder,
             public_only=public_only,
+            num_proc=num_proc,
         )
     logger.info(f"✓ Finished evaluation for {task.metadata.name}")