mteb 2.1.4__py3-none-any.whl → 2.5.2__py3-none-any.whl

mteb/descriptive_stats/Retrieval/SCIDOCS-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 26657,
+        "number_of_characters": 34261482,
+        "documents_text_statistics": {
+            "total_text_length": 34181168,
+            "min_text_length": 10,
+            "average_text_length": 1332.2355692403632,
+            "max_text_length": 9275,
+            "unique_texts": 25656
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 80314,
+            "min_text_length": 17,
+            "average_text_length": 80.314,
+            "max_text_length": 227,
+            "unique_texts": 1000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 4928,
+            "min_relevant_docs_per_query": 27,
+            "average_relevant_docs_per_query": 4.928,
+            "max_relevant_docs_per_query": 30,
+            "unique_relevant_docs": 25657
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/SQuADKorV1Retrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 6734,
+        "number_of_characters": 718835,
+        "documents_text_statistics": {
+            "total_text_length": 523388,
+            "min_text_length": 352,
+            "average_text_length": 545.1958333333333,
+            "max_text_length": 2952,
+            "unique_texts": 960
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 195447,
+            "min_text_length": 5,
+            "average_text_length": 33.84949774852788,
+            "max_text_length": 110,
+            "unique_texts": 5764
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 5774,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 960
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/SciFact-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 5483,
+        "number_of_characters": 8526662,
+        "documents_text_statistics": {
+            "total_text_length": 8496576,
+            "min_text_length": 228,
+            "average_text_length": 1639.3162261238665,
+            "max_text_length": 9187,
+            "unique_texts": 5183
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 30086,
+            "min_text_length": 24,
+            "average_text_length": 100.28666666666666,
+            "max_text_length": 228,
+            "unique_texts": 300
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 339,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.13,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 283
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/evaluate.py

@@ -2,11 +2,11 @@ from __future__ import annotations
 
 import logging
 from collections.abc import Iterable
-from copy import deepcopy
 from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, cast
 
+from datasets.exceptions import DatasetNotFoundError
 from tqdm.auto import tqdm
 
 from mteb._helpful_enum import HelpfulStrEnum
@@ -25,6 +25,7 @@ from mteb.models.sentence_transformer_wrapper import (
     SentenceTransformerEncoderWrapper,
 )
 from mteb.results import ModelResult, TaskResult
+from mteb.results.task_result import TaskError
 from mteb.types import HFSubset, PromptType, SplitName
 from mteb.types._metadata import ModelName, Revision
 
@@ -51,36 +52,6 @@ class OverwriteStrategy(HelpfulStrEnum):
     ONLY_CACHE = "only-cache"
 
 
-_empty_model_meta = ModelMeta(
-    loader=None,
-    name=None,
-    revision=None,
-    release_date=None,
-    languages=None,
-    framework=[],
-    similarity_fn_name=None,
-    n_parameters=None,
-    memory_usage_mb=None,
-    max_tokens=None,
-    embed_dim=None,
-    license=None,
-    open_weights=None,
-    public_training_code=None,
-    public_training_data=None,
-    use_instructions=None,
-    training_datasets=None,
-    modalities=[],
-)
-
-
-def _create_empty_model_meta() -> ModelMeta:
-    logger.warning("Model metadata is missing. Using empty metadata.")
-    meta = deepcopy(_empty_model_meta)
-    meta.revision = "no_revision_available"
-    meta.name = "no_model_name_available"
-    return meta
-
-
 def _sanitize_model(
     model: ModelMeta | MTEBModels | SentenceTransformer | CrossEncoder,
 ) -> tuple[MTEBModels | ModelMeta, ModelMeta, ModelName, Revision]:
@@ -99,9 +70,9 @@ def _sanitize_model(
     elif hasattr(model, "mteb_model_meta"):
         meta = model.mteb_model_meta  # type: ignore[attr-defined]
         if not isinstance(meta, ModelMeta):
-            meta = _create_empty_model_meta()
+            meta = ModelMeta.from_hub(None)
     else:
-        meta = _create_empty_model_meta() if not isinstance(model, ModelMeta) else model
+        meta = ModelMeta.from_hub(None) if not isinstance(model, ModelMeta) else model
 
     model_name = cast(str, meta.name)
     model_revision = cast(str, meta.revision)
@@ -117,7 +88,8 @@ def _evaluate_task(
     co2_tracker: bool | None,
     encode_kwargs: dict[str, Any],
     prediction_folder: Path | None,
-) -> TaskResult:
+    public_only: bool | None,
+) -> TaskResult | TaskError:
     """The core logic to run a model on a given task. See `evaluate` for more details.
 
     Returns:
@@ -149,6 +121,7 @@ def _evaluate_task(
                 encode_kwargs=encode_kwargs,
                 co2_tracker=False,
                 prediction_folder=prediction_folder,
+                public_only=public_only,
             )
         result.kg_co2_emissions = tracker.final_emissions
         return result
@@ -159,7 +132,20 @@ def _evaluate_task(
 
     data_loaded = task.data_loaded
     if not data_loaded:
-        task.load_data()
+        try:
+            task.load_data()
+        except DatasetNotFoundError as e:
+            if not task.metadata.is_public and public_only is None:
+                logger.warning(
+                    f"Dataset for private task '{task.metadata.name}' not found. "
+                    "Make sure you have access to the dataset and that you have set up the authentication correctly. To disable this warning set `public_only=False`"
+                )
+                return TaskError(
+                    task_name=task.metadata.name,
+                    exception=str(e),
+                )
+            if public_only is False:
+                raise e
 
     evaluation_time = 0
 
@@ -256,6 +242,20 @@ def _check_model_modalities(
         logger.warning(msg)
 
 
+def _requires_merge(task: AbsTask, existing_results: TaskResult) -> bool:
+    """Check if the existing results require merging with new results."""
+    # If the task has multiple eval splits and existing results cover only a subset, we need to merge
+    required_evals = dict.fromkeys(task.eval_splits, task.hf_subsets)
+    for split, subsets in required_evals.items():
+        res = existing_results.scores.get(split, None)
+        if res is None:
+            return True
+        hf_subsets = [r["hf_subset"] for r in res]
+        if not set(subsets).issubset(set(hf_subsets)):
+            return True
+    return False
+
+
 def evaluate(
     model: ModelMeta | MTEBModels | SentenceTransformer | CrossEncoder,
     tasks: AbsTask | Iterable[AbsTask],
@@ -267,6 +267,7 @@ def evaluate(
     overwrite_strategy: str | OverwriteStrategy = "only-missing",
     prediction_folder: Path | str | None = None,
     show_progress_bar: bool = True,
+    public_only: bool | None = None,
 ) -> ModelResult:
     """This function runs a model on a given task and returns the results.
 
@@ -290,6 +291,7 @@ def evaluate(
         prediction_folder: Optional folder in which to save model predictions for the task. Predictions of the tasks will be sabed in `prediction_folder/{task_name}_predictions.json`
         show_progress_bar: Whether to show a progress bar when running the evaluation. Default is True. Setting this to False will also set the
             `encode_kwargs['show_progress_bar']` to False if encode_kwargs is unspecified.
+        public_only: Run only public tasks. If None, it will attempt to run the private task.
 
     Returns:
         The results of the evaluation.
@@ -341,6 +343,7 @@ def evaluate(
             overwrite_strategy=overwrite_strategy,
             prediction_folder=prediction_folder,
             show_progress_bar=show_progress_bar,
+            public_only=public_only,
         )
         result = task.combine_task_results(results.task_results)
         return ModelResult(
@@ -353,6 +356,7 @@ def evaluate(
         task = tasks
     else:
         results = []
+        exceptions = []
         tasks_tqdm = tqdm(
             tasks,
             desc="Evaluating tasks",
@@ -370,12 +374,16 @@ def evaluate(
                 overwrite_strategy=overwrite_strategy,
                 prediction_folder=prediction_folder,
                 show_progress_bar=False,
+                public_only=public_only,
             )
             results.extend(_res.task_results)
+            if _res.exceptions:
+                exceptions.extend(_res.exceptions)
         return ModelResult(
             model_name=_res.model_name,
             model_revision=_res.model_revision,
             task_results=results,
+            exceptions=exceptions,
         )
 
     overwrite_strategy = OverwriteStrategy.from_str(overwrite_strategy)
@@ -388,13 +396,18 @@ def evaluate(
 
     if (
         existing_results
-        and overwrite_strategy == "only-missing"
-        and overwrite_strategy == OverwriteStrategy.ONLY_MISSING
-        and existing_results.is_mergeable(task)
+        and overwrite_strategy
+        not in (OverwriteStrategy.ALWAYS, OverwriteStrategy.NEVER)
+        and (
+            not _requires_merge(task, existing_results)
+            or existing_results.is_mergeable(task)
+        )
     ):
         missing_eval = existing_results.get_missing_evaluations(task)
     else:
         missing_eval = dict.fromkeys(task.eval_splits, task.hf_subsets)
+        # Will be fully recomputed so we set it to None to avoid merging:
+        existing_results = None
 
     if (
         existing_results
@@ -415,12 +428,13 @@ def evaluate(
         OverwriteStrategy.ONLY_CACHE,
     ]:
         raise ValueError(
-            f"overwrite_strategy is set to '{overwrite_strategy.value}' and the results file exists. However there are the following missing splits (and subsets): {missing_eval}. To rerun these set overwrite_strategy to 'only-missing'."
+            f"overwrite_strategy is set to '{overwrite_strategy.value}' and the results file exists for task {task.metadata.name}. "
+            + f"However there are the following missing splits (and subsets): {missing_eval}. To rerun these set overwrite_strategy to 'only-missing'."
         )
 
     if existing_results:
         logger.info(
-            f"Found existing results for {task.metadata.name}, only running missing splits: {list(missing_eval.keys())}"
+            f"Found existing results for {task.metadata.name}, only running missing splits (subsets): {missing_eval}"
         )
 
     if isinstance(model, ModelMeta):
@@ -439,16 +453,13 @@ def evaluate(
                 co2_tracker=co2_tracker,
                 encode_kwargs=encode_kwargs,
                 prediction_folder=prediction_folder,
+                public_only=public_only,
             )
         except Exception as e:
             logger.error(
                 f"Error while running task {task.metadata.name} on splits {list(missing_eval.keys())}: {e}"
             )
-            return ModelResult(
-                model_name=model_name,
-                model_revision=model_revision,
-                task_results=[],
-            )
+            result = TaskError(task_name=task.metadata.name, exception=str(e))
     else:
         result = _evaluate_task(
             model=model,
@@ -457,9 +468,18 @@ def evaluate(
             co2_tracker=False,
             encode_kwargs=encode_kwargs,
             prediction_folder=prediction_folder,
+            public_only=public_only,
         )
     logger.info(f"✓ Finished evaluation for {task.metadata.name}")
 
+    if isinstance(result, TaskError):
+        return ModelResult(
+            model_name=model_name,
+            model_revision=model_revision,
+            task_results=[],
+            exceptions=[result],
+        )
+
     if existing_results:
         result = result.merge(existing_results)