mteb 2.5.2__py3-none-any.whl → 2.7.9__py3-none-any.whl

mteb/cli/generate_model_card.py

@@ -1,23 +1,33 @@
+from __future__ import annotations
+
 import logging
+import warnings
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from huggingface_hub import ModelCard, ModelCardData, repo_exists
 
-from mteb import BenchmarkResults
 from mteb.abstasks.abstask import AbsTask
 from mteb.cache import ResultCache
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from mteb.abstasks.abstask import AbsTask
+    from mteb.benchmarks.benchmark import Benchmark
+
 logger = logging.getLogger(__name__)
 
 
 def generate_model_card(
     model_name: str,
-    tasks: list[AbsTask] | None = None,
+    tasks: Sequence[AbsTask] | None = None,
+    benchmarks: Sequence[Benchmark] | None = None,
     existing_model_card_id_or_path: str | Path | None = None,
     results_cache: ResultCache = ResultCache(),
     output_path: Path = Path("model_card.md"),
     add_table_to_model_card: bool = False,
-    models_to_compare: list[str] | None = None,
+    models_to_compare: Sequence[str] | None = None,
     token: str | None = None,
     push_to_hub: bool = False,
 ) -> None:
@@ -26,6 +36,7 @@ def generate_model_card(
     Args:
         model_name: Name of the model.
         tasks: List of tasks to generate results for.
+        benchmarks: A Benchmark or list of benchmarks to generate results for.
         existing_model_card_id_or_path: Path or ID of an existing model card to update.
         results_cache: Instance of ResultCache to load results from.
         output_path: Path to save the generated model card.
@@ -39,16 +50,24 @@ def generate_model_card(
     if existing_model_card_id_or_path:
         existing_model_card = ModelCard.load(existing_model_card_id_or_path)
 
+    all_tasks: list[AbsTask] = []
+    if tasks is not None:
+        all_tasks.extend(tasks)
+
+    if benchmarks is not None:
+        for b in benchmarks:
+            all_tasks.extend(b.tasks)
+
     benchmark_results = results_cache.load_results(
-        [model_name], tasks, only_main_score=True
+        [model_name], all_tasks if all_tasks else None, only_main_score=True
     )
     eval_results = []
     for models_results in benchmark_results.model_results:
         for task_result in models_results.task_results:
             eval_results.extend(task_result.get_hf_eval_results())
 
-    existing_model_card_data = (
-        existing_model_card.data if existing_model_card else ModelCardData()
+    existing_model_card_data: ModelCardData = (
+        existing_model_card.data if existing_model_card else ModelCardData()  # type: ignore[assignment]
     )
 
     if existing_model_card_data.eval_results is None:
@@ -78,35 +97,43 @@ def generate_model_card(
             card_data=existing_model_card_data
         )
 
-    if models_to_compare:
-        benchmark_results = results_cache.load_results(
-            [model_name, *models_to_compare], tasks, only_main_score=True
-        )
-
     if add_table_to_model_card:
         existing_model_card = _add_table_to_model_card(
-            benchmark_results, existing_model_card
+            results_cache,
+            existing_model_card,
+            (model_name, *models_to_compare) if models_to_compare else (model_name,),
+            benchmarks or [],
         )
 
-    if push_to_hub:
+    if push_to_hub and existing_model_card_id_or_path:
+        existing_model_card_id_or_path = str(existing_model_card_id_or_path)
         if repo_exists(existing_model_card_id_or_path):
             existing_model_card.push_to_hub(existing_model_card_id_or_path, token=token)
         else:
-            logger.warning(
-                f"Repository {existing_model_card_id_or_path} does not exist on the Hub. Skipping push to hub."
-            )
+            msg = f"Repository {existing_model_card_id_or_path} does not exist on the Hub. Skipping push to hub."
+            logger.warning(msg)
+            warnings.warn(msg)
     existing_model_card.save(output_path)
 
 
 def _add_table_to_model_card(
-    results: BenchmarkResults, model_card: ModelCard
+    results_cache: ResultCache,
+    model_card: ModelCard,
+    models: Sequence[str],
+    benchmarks: Sequence[Benchmark],
 ) -> ModelCard:
     original_content = model_card.content
-    results_df = results.to_dataframe()
-    results_df = results_df.set_index("task_name")
-    mteb_content = f"""
-# MTEB results
-{results_df.to_markdown()}
-"""
+    mteb_content = "# MTEB Results\n\n"
+
+    for benchmark in benchmarks:
+        mteb_content += f"## Benchmark: {benchmark.name}\n\n"
+        benchmark_results = results_cache.load_results(
+            tasks=benchmark,
+            models=models,
+            only_main_score=True,
+        )
+        df_results = benchmark_results.get_benchmark_result()
+        mteb_content += df_results.to_markdown(index=True) + "\n\n"
+
     model_card.content = original_content + "\n\n" + mteb_content
     return model_card

mteb/deprecated_evaluator.py

@@ -5,38 +5,43 @@ import logging
 import os
 import sys
 import traceback
-from collections.abc import Iterable
+import warnings
 from copy import deepcopy
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
 from time import time
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 import datasets
 
 import mteb
 from mteb.abstasks import AbsTask
-from mteb.abstasks.task_metadata import TaskCategory, TaskType
 from mteb.benchmarks import Benchmark
 from mteb.models import (
     CrossEncoderWrapper,
-    EncoderProtocol,
     ModelMeta,
-    MTEBModels,
     SentenceTransformerEncoderWrapper,
 )
 from mteb.results import TaskResult
-from mteb.types import ScoresDict
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
+
+    from sentence_transformers import CrossEncoder, SentenceTransformer
+
+    from mteb.abstasks.aggregated_task import AbsTaskAggregate
+    from mteb.abstasks.task_metadata import TaskCategory, TaskType
+    from mteb.models import (
+        MTEBModels,
+    )
+    from mteb.types import EncodeKwargs, ScoresDict
 
 if sys.version_info >= (3, 13):
     from warnings import deprecated
 else:
     from typing_extensions import deprecated
 
-if TYPE_CHECKING:
-    from sentence_transformers import CrossEncoder, SentenceTransformer
-
 logger = logging.getLogger(__name__)
 
 
@@ -52,7 +57,7 @@ class MTEB:
     )
     def __init__(
         self,
-        tasks: Iterable[AbsTask | Benchmark],
+        tasks: Iterable[AbsTask] | Iterable[Benchmark],
         *,
         err_logs_path: str = "error_logs.txt",
     ) -> None:
@@ -63,15 +68,14 @@ class MTEB:
                 `mteb.get_tasks(["task1","task2"]) or `mteb.get_benchmark("MTEB(eng, classic)").
             err_logs_path: Path to save error logs.
         """
-        from mteb.benchmarks import Benchmark
-
-        self.tasks = list(tasks)
-        if len(self.tasks) > 0 and isinstance(self.tasks[0], Benchmark):
+        if isinstance(next(iter(tasks)), Benchmark):
             self.benchmarks = tasks
-            self.tasks = list(chain.from_iterable(self.tasks))
+            self.tasks = list(chain.from_iterable(cast("Iterable[Benchmark]", tasks)))
+        elif isinstance(next(iter(tasks)), AbsTask):
+            self.tasks = list(cast("Iterable[AbsTask]", tasks))
 
         self.err_logs_path = Path(err_logs_path)
-        self.last_evaluated_splits = {}
+        self._last_evaluated_splits: dict[str, list[str]] = {}
 
     @property
     def available_tasks(self) -> list[str]:
@@ -84,7 +88,7 @@ class MTEB:
         return sorted({x.metadata.type for x in self.tasks})
 
     @property
-    def available_task_categories(self) -> set[TaskCategory]:
+    def available_task_categories(self) -> set[TaskCategory | None]:
         """Set of available task categories."""
         return {x.metadata.category for x in self.tasks}
 
@@ -174,7 +178,7 @@ class MTEB:
         split: str,
         subsets_to_run: list[str] | None = None,
         *,
-        encode_kwargs: dict[str, Any],
+        encode_kwargs: EncodeKwargs,
         **kwargs: Any,
     ):
         tick = time()
@@ -231,13 +235,14 @@ class MTEB:
         merged_kg_co2_emissions = None
         if existing_kg_co2_emissions and new_kg_co2_emissions:
             merged_kg_co2_emissions = existing_kg_co2_emissions + new_kg_co2_emissions
+        existing_evaluation_time = existing_results.evaluation_time or 0
+        new_evaluation_time = new_results.evaluation_time or 0
         merged_results = TaskResult(
             dataset_revision=new_results.dataset_revision,
             task_name=new_results.task_name,
             mteb_version=new_results.mteb_version,
             scores=merged_scores,
-            evaluation_time=existing_results.evaluation_time
-            + new_results.evaluation_time,
+            evaluation_time=existing_evaluation_time + new_evaluation_time,
             kg_co2_emissions=merged_kg_co2_emissions,
         )
 
@@ -262,7 +267,7 @@ class MTEB:
         overwrite_results: bool = False,
         raise_error: bool = True,
         co2_tracker: bool = False,
-        encode_kwargs: dict[str, Any] | None = None,
+        encode_kwargs: EncodeKwargs | None = None,
         **kwargs,
     ) -> list[TaskResult]:
         """Run the evaluation pipeline on the selected tasks.
@@ -306,13 +311,16 @@ class MTEB:
         elif verbosity == 3:
             datasets.logging.set_verbosity(logging.DEBUG)
 
-        meta = self.create_model_meta(model)
-        output_path = self._create_output_folder(meta, output_folder)
-
+        mteb_model: MTEBModels
         if isinstance(model, SentenceTransformer):
-            model = SentenceTransformerEncoderWrapper(model)
+            mteb_model = SentenceTransformerEncoderWrapper(model)
         elif isinstance(model, CrossEncoder):
-            model = CrossEncoderWrapper(model)
+            mteb_model = CrossEncoderWrapper(model)
+        else:
+            mteb_model = cast("MTEBModels", model)
+
+        meta = self.create_model_meta(mteb_model)
+        output_path = self._create_output_folder(meta, output_folder)
 
         # Disable co2_tracker for API models
         if "API" in meta.framework:
@@ -333,7 +341,7 @@ class MTEB:
         )  # save them in case we re-use the object (e.g. for reranking)
 
         # To evaluate missing splits, we keep track of the task name and the corresponding splits.
-        self.last_evaluated_splits = {}
+        self._last_evaluated_splits = {}
 
         while len(self.tasks) > 0:
             task = self.tasks[0]
@@ -342,9 +350,10 @@ class MTEB:
             )
 
             if task.is_aggregate:
-                self_ = MTEB(tasks=task.metadata.tasks)
-                task_results = self_.run(
-                    model,
+                aggregated_task = cast("AbsTaskAggregate", task)
+                self_ = MTEB(tasks=aggregated_task.metadata.tasks)
+                aggregated_task_results = self_.run(
+                    mteb_model,
                     verbosity=verbosity - 1,
                     output_folder=output_folder,
                     eval_splits=eval_splits,
@@ -355,12 +364,15 @@ class MTEB:
                     encode_kwargs=encode_kwargs,
                     **kwargs,
                 )
-                new_results = task.combine_task_results(task_results)
+                new_results = aggregated_task.combine_task_results(
+                    aggregated_task_results
+                )
                 evaluation_results.append(new_results)
 
                 if output_path:
-                    save_path = output_path / f"{task.metadata.name}.json"
-                    new_results.to_disk(save_path)
+                    new_results.to_disk(
+                        output_path / f"{aggregated_task.metadata.name}.json"
+                    )
                 del self.tasks[0]
                 continue
 
@@ -382,7 +394,7 @@ class MTEB:
             task_subsets = task.hf_subsets
 
             existing_results = None
-            save_path = None
+            save_path: Path | None = None
             final_splits_to_run = task_eval_splits
             missing_evaluations = self._get_missing_evaluations(
                 existing_results,
@@ -432,7 +444,7 @@ class MTEB:
                     logger.info(
                         f"No splits to evaluate for {task.metadata.name}. Skipping evaluation."
                     )
-                self.last_evaluated_splits[task.metadata.name] = []
+                self._last_evaluated_splits[task.metadata.name] = []
                 del self.tasks[0]
                 continue
 
@@ -440,11 +452,11 @@ class MTEB:
                 task.check_if_dataset_is_superseded()
                 task.load_data()
 
-                task_results = {}
+                task_results: dict[str, dict[str, dict[str, Any]]] = {}
                 evaluation_time = 0
                 kg_co2_emissions: int | None = 0 if co2_tracker else None
 
-                self.last_evaluated_splits[task.metadata.name] = []
+                self._last_evaluated_splits[task.metadata.name] = []
 
                 for split in final_splits_to_run:
                     info = missing_evaluations[split]
@@ -465,14 +477,16 @@ class MTEB:
 
                     if co2_tracker:
                         try:
-                            from codecarbon import EmissionsTracker
+                            from codecarbon import (  # type: ignore[import-not-found,import-untyped]
+                                EmissionsTracker,
+                            )
                         except ImportError:
                             raise ImportError(
                                 "codecarbon is not installed. Please install it using `pip install 'mteb[codecarbon]'` to track CO₂ emissions."
                             )
-                        logger.warning(
-                            "Evaluating multiple MTEB runs simultaneously will produce incorrect CO₂ results"
-                        )
+                        msg = "Evaluating multiple MTEB runs simultaneously will produce incorrect CO₂ results"
+                        logger.warning(msg)
+                        warnings.warn(msg)
                         with EmissionsTracker(
                             save_to_file=False,
                             save_to_api=False,
@@ -481,7 +495,7 @@ class MTEB:
                         ) as tracker:
                             results, tick, tock = self._run_eval(
                                 task,
-                                model,
+                                mteb_model,
                                 split,
                                 encode_kwargs=encode_kwargs,
                                 subsets_to_run=subsets_to_run,
@@ -494,7 +508,7 @@ class MTEB:
                     else:
                         results, tick, tock = self._run_eval(
                             task,
-                            model,
+                            mteb_model,
                             split,
                             subsets_to_run=subsets_to_run,
                             encode_kwargs=encode_kwargs,
@@ -510,25 +524,25 @@ class MTEB:
                     if verbosity >= 1:
                         logger.info(f"Scores: {task_results[split]}")
 
-                    self.last_evaluated_splits[task.metadata.name].append(split)
+                    self._last_evaluated_splits[task.metadata.name].append(split)
 
                 # Create new TaskResult
                 new_results = TaskResult.from_task_results(
                     task,
-                    task_results,
+                    task_results,  # type: ignore[arg-type]
                     evaluation_time=evaluation_time,
                     kg_co2_emissions=kg_co2_emissions,
                 )
 
                 # Merge with existing if needed
-                if output_path and save_path.exists():
+                if output_path and save_path and save_path.exists():
                     existing_results = TaskResult.from_disk(save_path)
                 if existing_results:
                     merged_results = self._merge_results(existing_results, new_results)
                 else:
                     merged_results = new_results
 
-                if output_path:
+                if output_path and save_path:
                     merged_results.to_disk(save_path)
 
                 evaluation_results.append(merged_results)
@@ -555,7 +569,7 @@ class MTEB:
     def create_model_meta(model: MTEBModels) -> ModelMeta:
         """Create a ModelMeta object for the given model."""
         if hasattr(model, "mteb_model_meta") and model.mteb_model_meta is not None:
-            meta = model.mteb_model_meta  # type: ignore
+            meta = model.mteb_model_meta
         else:
             meta = MTEB._get_model_meta(model)
 
@@ -581,7 +595,11 @@ class MTEB:
         if output_folder is None:
             return None
 
-        model_revision: str = model_meta.revision  # type: ignore
+        model_revision: str = (
+            model_meta.revision
+            if model_meta.revision is not None
+            else "no_revision_available"
+        )
         model_path_name = model_meta.model_name_as_path()
 
         output_path = Path(output_folder) / model_path_name / model_revision
@@ -603,15 +621,15 @@ class MTEB:
              Tasks with empty lists indicate that results already existed and no splits were evaluated.
         """
         return deepcopy(
-            {task: list(splits) for task, splits in self.last_evaluated_splits.items()}
+            {task: list(splits) for task, splits in self._last_evaluated_splits.items()}
         )
 
     @staticmethod
     def _get_missing_evaluations(
         existing_results: TaskResult | None,
-        task_eval_splits: list[str],
-        task_eval_langs: list[str],
-        eval_subsets: list[str] | None,
+        task_eval_splits: Sequence[str],
+        task_eval_langs: Sequence[str],
+        eval_subsets: Sequence[str] | None,
     ) -> dict[str, dict[str, Any]]:
         """Return a dictionary for each split, indicating if the whole split is missing and which subsets are missing."""
         missing_evaluations = {
@@ -660,7 +678,7 @@ class MTEB:
         return missing_evaluations
 
     @staticmethod
-    def _get_model_meta(model: EncoderProtocol) -> ModelMeta:
+    def _get_model_meta(model: MTEBModels) -> ModelMeta:
         from sentence_transformers import CrossEncoder, SentenceTransformer
 
         if isinstance(model, CrossEncoder):

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2CybersecurityRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 1299,
+        "number_of_characters": 9254,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 2245,
+            "average_image_width": 2370.324347826087,
+            "max_image_width": 3508,
+            "min_image_height": 2481,
+            "average_image_height": 3289.8060869565215,
+            "max_image_height": 3580,
+            "unique_images": 1132
+        },
+        "queries_text_statistics": {
+            "total_text_length": 9254,
+            "min_text_length": 15,
+            "average_text_length": 62.10738255033557,
+            "max_text_length": 108,
+            "unique_texts": 149
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 409,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 2.7449664429530203,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 316
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EconomicRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 1640,
+        "number_of_characters": 8331,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 2313,
+            "average_image_width": 2347.5321597833445,
+            "max_image_width": 2481,
+            "min_image_height": 3138,
+            "average_image_height": 3214.301963439404,
+            "max_image_height": 3508,
+            "unique_images": 1442
+        },
+        "queries_text_statistics": {
+            "total_text_length": 8331,
+            "min_text_length": 23,
+            "average_text_length": 51.11042944785276,
+            "max_text_length": 110,
+            "unique_texts": 163
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 413,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 2.5337423312883436,
+            "max_relevant_docs_per_query": 6,
+            "unique_relevant_docs": 349
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2EnergyRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 2166,
+        "number_of_characters": 9764,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 2221,
+            "average_image_width": 2339.4957350727545,
+            "max_image_width": 2480,
+            "min_image_height": 3036,
+            "average_image_height": 3242.8138484696437,
+            "max_image_height": 3508,
+            "unique_images": 1974
+        },
+        "queries_text_statistics": {
+            "total_text_length": 9764,
+            "min_text_length": 22,
+            "average_text_length": 56.4393063583815,
+            "max_text_length": 103,
+            "unique_texts": 173
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 525,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 3.0346820809248554,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 442
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2HrRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 2330,
+        "number_of_characters": 13131,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 1949,
+            "average_image_width": 2430.1152204836417,
+            "max_image_width": 3505,
+            "min_image_height": 2480,
+            "average_image_height": 3350.3921289710765,
+            "max_image_height": 3626,
+            "unique_images": 2096
+        },
+        "queries_text_statistics": {
+            "total_text_length": 13131,
+            "min_text_length": 21,
+            "average_text_length": 59.41628959276018,
+            "max_text_length": 112,
+            "unique_texts": 221
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 726,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 3.2850678733031673,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 575
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/BrightAopsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 188113,
+        "number_of_characters": 141769714,
+        "documents_text_statistics": {
+            "total_text_length": 141734227,
+            "min_text_length": 58,
+            "average_text_length": 753.8974425803981,
+            "max_text_length": 7334,
+            "unique_texts": 176508
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 35487,
+            "min_text_length": 85,
+            "average_text_length": 319.7027027027027,
+            "max_text_length": 1167,
+            "unique_texts": 111
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 524,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 4.7207207207207205,
+            "max_relevant_docs_per_query": 8,
+            "unique_relevant_docs": 111
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 20264921,
+            "min_top_ranked_per_query": 176954,
+            "average_top_ranked_per_query": 182566.85585585586,
+            "max_top_ranked_per_query": 186176
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightBiologyLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 627,
+        "number_of_characters": 19398082,
+        "documents_text_statistics": {
+            "total_text_length": 19344209,
+            "min_text_length": 142,
+            "average_text_length": 36916.42938931298,
+            "max_text_length": 1324201,
+            "unique_texts": 498
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 53873,
+            "min_text_length": 89,
+            "average_text_length": 523.0388349514564,
+            "max_text_length": 2195,
+            "unique_texts": 103
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 134,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.3009708737864079,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 134
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 53972,
+            "min_top_ranked_per_query": 524,
+            "average_top_ranked_per_query": 524.0,
+            "max_top_ranked_per_query": 524
+        }
+    }
+}