mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/cli/generate_model_card.py

@@ -1,10 +1,12 @@
 import logging
+import warnings
+from collections.abc import Sequence
 from pathlib import Path
 
 from huggingface_hub import ModelCard, ModelCardData, repo_exists
 
-from mteb import BenchmarkResults
 from mteb.abstasks.abstask import AbsTask
+from mteb.benchmarks.benchmark import Benchmark
 from mteb.cache import ResultCache
 
 logger = logging.getLogger(__name__)
@@ -12,12 +14,13 @@ logger = logging.getLogger(__name__)
 
 def generate_model_card(
     model_name: str,
-    tasks: list[AbsTask] | None = None,
+    tasks: Sequence[AbsTask] | None = None,
+    benchmarks: Sequence[Benchmark] | None = None,
     existing_model_card_id_or_path: str | Path | None = None,
     results_cache: ResultCache = ResultCache(),
     output_path: Path = Path("model_card.md"),
     add_table_to_model_card: bool = False,
-    models_to_compare: list[str] | None = None,
+    models_to_compare: Sequence[str] | None = None,
     token: str | None = None,
     push_to_hub: bool = False,
 ) -> None:
@@ -26,6 +29,7 @@ def generate_model_card(
     Args:
         model_name: Name of the model.
         tasks: List of tasks to generate results for.
+        benchmarks: A Benchmark or list of benchmarks to generate results for.
         existing_model_card_id_or_path: Path or ID of an existing model card to update.
         results_cache: Instance of ResultCache to load results from.
         output_path: Path to save the generated model card.
@@ -39,16 +43,24 @@ def generate_model_card(
     if existing_model_card_id_or_path:
         existing_model_card = ModelCard.load(existing_model_card_id_or_path)
 
+    all_tasks: list[AbsTask] = []
+    if tasks is not None:
+        all_tasks.extend(tasks)
+
+    if benchmarks is not None:
+        for b in benchmarks:
+            all_tasks.extend(b.tasks)
+
     benchmark_results = results_cache.load_results(
-        [model_name], tasks, only_main_score=True
+        [model_name], all_tasks if all_tasks else None, only_main_score=True
     )
     eval_results = []
     for models_results in benchmark_results.model_results:
         for task_result in models_results.task_results:
             eval_results.extend(task_result.get_hf_eval_results())
 
-    existing_model_card_data = (
-        existing_model_card.data if existing_model_card else ModelCardData()
+    existing_model_card_data: ModelCardData = (
+        existing_model_card.data if existing_model_card else ModelCardData()  # type: ignore[assignment]
     )
 
     if existing_model_card_data.eval_results is None:
@@ -78,35 +90,43 @@ def generate_model_card(
             card_data=existing_model_card_data
         )
 
-    if models_to_compare:
-        benchmark_results = results_cache.load_results(
-            [model_name, *models_to_compare], tasks, only_main_score=True
-        )
-
     if add_table_to_model_card:
         existing_model_card = _add_table_to_model_card(
-            benchmark_results, existing_model_card
+            results_cache,
+            existing_model_card,
+            (model_name, *models_to_compare) if models_to_compare else (model_name,),
+            benchmarks or [],
         )
 
-    if push_to_hub:
+    if push_to_hub and existing_model_card_id_or_path:
+        existing_model_card_id_or_path = str(existing_model_card_id_or_path)
         if repo_exists(existing_model_card_id_or_path):
             existing_model_card.push_to_hub(existing_model_card_id_or_path, token=token)
         else:
-            logger.warning(
-                f"Repository {existing_model_card_id_or_path} does not exist on the Hub. Skipping push to hub."
-            )
+            msg = f"Repository {existing_model_card_id_or_path} does not exist on the Hub. Skipping push to hub."
+            logger.warning(msg)
+            warnings.warn(msg)
     existing_model_card.save(output_path)
 
 
 def _add_table_to_model_card(
-    results: BenchmarkResults, model_card: ModelCard
+    results_cache: ResultCache,
+    model_card: ModelCard,
+    models: Sequence[str],
+    benchmarks: Sequence[Benchmark],
 ) -> ModelCard:
     original_content = model_card.content
-    results_df = results.to_dataframe()
-    results_df = results_df.set_index("task_name")
-    mteb_content = f"""
-# MTEB results
-{results_df.to_markdown()}
-"""
+    mteb_content = "# MTEB Results\n\n"
+
+    for benchmark in benchmarks:
+        mteb_content += f"## Benchmark: {benchmark.name}\n\n"
+        benchmark_results = results_cache.load_results(
+            tasks=benchmark,
+            models=models,
+            only_main_score=True,
+        )
+        df_results = benchmark_results.get_benchmark_result()
+        mteb_content += df_results.to_markdown(index=True) + "\n\n"
+
     model_card.content = original_content + "\n\n" + mteb_content
     return model_card

mteb/deprecated_evaluator.py

@@ -5,39 +5,35 @@ import logging
 import os
 import sys
 import traceback
-from collections.abc import Iterable
+import warnings
+from collections.abc import Iterable, Sequence
 from copy import deepcopy
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
 from time import time
-from typing import TYPE_CHECKING, Any
-
-from mteb.abstasks.task_metadata import TaskCategory, TaskType
-from mteb.models.get_model_meta import (
-    _model_meta_from_cross_encoder,
-    _model_meta_from_sentence_transformers,
-)
-
-if sys.version_info >= (3, 13):
-    from warnings import deprecated
-else:
-    from typing_extensions import deprecated
+from typing import TYPE_CHECKING, Any, cast
 
 import datasets
 
 import mteb
 from mteb.abstasks import AbsTask
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
+from mteb.abstasks.task_metadata import TaskCategory, TaskType
 from mteb.benchmarks import Benchmark
 from mteb.models import (
     CrossEncoderWrapper,
-    EncoderProtocol,
     ModelMeta,
     MTEBModels,
     SentenceTransformerEncoderWrapper,
 )
 from mteb.results import TaskResult
-from mteb.types import ScoresDict
+from mteb.types import EncodeKwargs, ScoresDict
+
+if sys.version_info >= (3, 13):
+    from warnings import deprecated
+else:
+    from typing_extensions import deprecated
 
 if TYPE_CHECKING:
     from sentence_transformers import CrossEncoder, SentenceTransformer
@@ -57,7 +53,7 @@ class MTEB:
     )
     def __init__(
         self,
-        tasks: Iterable[AbsTask | Benchmark],
+        tasks: Iterable[AbsTask] | Iterable[Benchmark],
         *,
         err_logs_path: str = "error_logs.txt",
     ) -> None:
@@ -68,15 +64,14 @@ class MTEB:
                 `mteb.get_tasks(["task1","task2"]) or `mteb.get_benchmark("MTEB(eng, classic)").
             err_logs_path: Path to save error logs.
         """
-        from mteb.benchmarks import Benchmark
-
-        self.tasks = list(tasks)
-        if len(self.tasks) > 0 and isinstance(self.tasks[0], Benchmark):
+        if isinstance(next(iter(tasks)), Benchmark):
             self.benchmarks = tasks
-            self.tasks = list(chain.from_iterable(self.tasks))
+            self.tasks = list(chain.from_iterable(cast(Iterable[Benchmark], tasks)))
+        elif isinstance(next(iter(tasks)), AbsTask):
+            self.tasks = list(cast(Iterable[AbsTask], tasks))
 
         self.err_logs_path = Path(err_logs_path)
-        self.last_evaluated_splits = {}
+        self._last_evaluated_splits: dict[str, list[str]] = {}
 
     @property
     def available_tasks(self) -> list[str]:
@@ -89,7 +84,7 @@ class MTEB:
         return sorted({x.metadata.type for x in self.tasks})
 
     @property
-    def available_task_categories(self) -> set[TaskCategory]:
+    def available_task_categories(self) -> set[TaskCategory | None]:
         """Set of available task categories."""
         return {x.metadata.category for x in self.tasks}
 
@@ -179,7 +174,7 @@ class MTEB:
         split: str,
         subsets_to_run: list[str] | None = None,
         *,
-        encode_kwargs: dict[str, Any],
+        encode_kwargs: EncodeKwargs,
         **kwargs: Any,
     ):
         tick = time()
@@ -236,13 +231,14 @@ class MTEB:
         merged_kg_co2_emissions = None
         if existing_kg_co2_emissions and new_kg_co2_emissions:
             merged_kg_co2_emissions = existing_kg_co2_emissions + new_kg_co2_emissions
+        existing_evaluation_time = existing_results.evaluation_time or 0
+        new_evaluation_time = new_results.evaluation_time or 0
         merged_results = TaskResult(
             dataset_revision=new_results.dataset_revision,
             task_name=new_results.task_name,
             mteb_version=new_results.mteb_version,
             scores=merged_scores,
-            evaluation_time=existing_results.evaluation_time
-            + new_results.evaluation_time,
+            evaluation_time=existing_evaluation_time + new_evaluation_time,
             kg_co2_emissions=merged_kg_co2_emissions,
         )
 
@@ -267,7 +263,7 @@ class MTEB:
         overwrite_results: bool = False,
         raise_error: bool = True,
         co2_tracker: bool = False,
-        encode_kwargs: dict[str, Any] | None = None,
+        encode_kwargs: EncodeKwargs | None = None,
         **kwargs,
     ) -> list[TaskResult]:
         """Run the evaluation pipeline on the selected tasks.
@@ -311,13 +307,16 @@ class MTEB:
         elif verbosity == 3:
             datasets.logging.set_verbosity(logging.DEBUG)
 
-        meta = self.create_model_meta(model)
-        output_path = self._create_output_folder(meta, output_folder)
-
+        mteb_model: MTEBModels
         if isinstance(model, SentenceTransformer):
-            model = SentenceTransformerEncoderWrapper(model)
+            mteb_model = SentenceTransformerEncoderWrapper(model)
         elif isinstance(model, CrossEncoder):
-            model = CrossEncoderWrapper(model)
+            mteb_model = CrossEncoderWrapper(model)
+        else:
+            mteb_model = cast(MTEBModels, model)
+
+        meta = self.create_model_meta(mteb_model)
+        output_path = self._create_output_folder(meta, output_folder)
 
         # Disable co2_tracker for API models
         if "API" in meta.framework:
@@ -338,7 +337,7 @@ class MTEB:
         )  # save them in case we re-use the object (e.g. for reranking)
 
         # To evaluate missing splits, we keep track of the task name and the corresponding splits.
-        self.last_evaluated_splits = {}
+        self._last_evaluated_splits = {}
 
         while len(self.tasks) > 0:
             task = self.tasks[0]
@@ -347,9 +346,10 @@ class MTEB:
             )
 
             if task.is_aggregate:
-                self_ = MTEB(tasks=task.metadata.tasks)
-                task_results = self_.run(
-                    model,
+                aggregated_task = cast(AbsTaskAggregate, task)
+                self_ = MTEB(tasks=aggregated_task.metadata.tasks)
+                aggregated_task_results = self_.run(
+                    mteb_model,
                     verbosity=verbosity - 1,
                     output_folder=output_folder,
                     eval_splits=eval_splits,
@@ -360,12 +360,15 @@ class MTEB:
                     encode_kwargs=encode_kwargs,
                     **kwargs,
                 )
-                new_results = task.combine_task_results(task_results)
+                new_results = aggregated_task.combine_task_results(
+                    aggregated_task_results
+                )
                 evaluation_results.append(new_results)
 
                 if output_path:
-                    save_path = output_path / f"{task.metadata.name}.json"
-                    new_results.to_disk(save_path)
+                    new_results.to_disk(
+                        output_path / f"{aggregated_task.metadata.name}.json"
+                    )
                 del self.tasks[0]
                 continue
 
@@ -387,7 +390,7 @@ class MTEB:
             task_subsets = task.hf_subsets
 
             existing_results = None
-            save_path = None
+            save_path: Path | None = None
             final_splits_to_run = task_eval_splits
             missing_evaluations = self._get_missing_evaluations(
                 existing_results,
@@ -437,7 +440,7 @@ class MTEB:
                     logger.info(
                         f"No splits to evaluate for {task.metadata.name}. Skipping evaluation."
                     )
-                self.last_evaluated_splits[task.metadata.name] = []
+                self._last_evaluated_splits[task.metadata.name] = []
                 del self.tasks[0]
                 continue
 
@@ -445,11 +448,11 @@ class MTEB:
                 task.check_if_dataset_is_superseded()
                 task.load_data()
 
-                task_results = {}
+                task_results: dict[str, dict[str, dict[str, Any]]] = {}
                 evaluation_time = 0
                 kg_co2_emissions: int | None = 0 if co2_tracker else None
 
-                self.last_evaluated_splits[task.metadata.name] = []
+                self._last_evaluated_splits[task.metadata.name] = []
 
                 for split in final_splits_to_run:
                     info = missing_evaluations[split]
@@ -470,14 +473,16 @@ class MTEB:
 
                     if co2_tracker:
                         try:
-                            from codecarbon import EmissionsTracker
+                            from codecarbon import (  # type: ignore[import-not-found,import-untyped]
+                                EmissionsTracker,
+                            )
                         except ImportError:
                             raise ImportError(
                                 "codecarbon is not installed. Please install it using `pip install 'mteb[codecarbon]'` to track CO₂ emissions."
                             )
-                        logger.warning(
-                            "Evaluating multiple MTEB runs simultaneously will produce incorrect CO₂ results"
-                        )
+                        msg = "Evaluating multiple MTEB runs simultaneously will produce incorrect CO₂ results"
+                        logger.warning(msg)
+                        warnings.warn(msg)
                         with EmissionsTracker(
                             save_to_file=False,
                             save_to_api=False,
@@ -486,7 +491,7 @@ class MTEB:
                         ) as tracker:
                             results, tick, tock = self._run_eval(
                                 task,
-                                model,
+                                mteb_model,
                                 split,
                                 encode_kwargs=encode_kwargs,
                                 subsets_to_run=subsets_to_run,
@@ -499,7 +504,7 @@ class MTEB:
                     else:
                         results, tick, tock = self._run_eval(
                             task,
-                            model,
+                            mteb_model,
                             split,
                             subsets_to_run=subsets_to_run,
                             encode_kwargs=encode_kwargs,
@@ -515,25 +520,25 @@ class MTEB:
                     if verbosity >= 1:
                         logger.info(f"Scores: {task_results[split]}")
 
-                    self.last_evaluated_splits[task.metadata.name].append(split)
+                    self._last_evaluated_splits[task.metadata.name].append(split)
 
                 # Create new TaskResult
                 new_results = TaskResult.from_task_results(
                     task,
-                    task_results,
+                    task_results,  # type: ignore[arg-type]
                     evaluation_time=evaluation_time,
                     kg_co2_emissions=kg_co2_emissions,
                 )
 
                 # Merge with existing if needed
-                if output_path and save_path.exists():
+                if output_path and save_path and save_path.exists():
                     existing_results = TaskResult.from_disk(save_path)
                 if existing_results:
                     merged_results = self._merge_results(existing_results, new_results)
                 else:
                     merged_results = new_results
 
-                if output_path:
+                if output_path and save_path:
                     merged_results.to_disk(save_path)
 
                 evaluation_results.append(merged_results)
@@ -560,7 +565,7 @@ class MTEB:
     def create_model_meta(model: MTEBModels) -> ModelMeta:
         """Create a ModelMeta object for the given model."""
         if hasattr(model, "mteb_model_meta") and model.mteb_model_meta is not None:
-            meta = model.mteb_model_meta  # type: ignore
+            meta = model.mteb_model_meta
         else:
             meta = MTEB._get_model_meta(model)
 
@@ -586,7 +591,11 @@ class MTEB:
         if output_folder is None:
             return None
 
-        model_revision: str = model_meta.revision  # type: ignore
+        model_revision: str = (
+            model_meta.revision
+            if model_meta.revision is not None
+            else "no_revision_available"
+        )
         model_path_name = model_meta.model_name_as_path()
 
         output_path = Path(output_folder) / model_path_name / model_revision
@@ -608,15 +617,15 @@ class MTEB:
              Tasks with empty lists indicate that results already existed and no splits were evaluated.
         """
         return deepcopy(
-            {task: list(splits) for task, splits in self.last_evaluated_splits.items()}
+            {task: list(splits) for task, splits in self._last_evaluated_splits.items()}
         )
 
     @staticmethod
     def _get_missing_evaluations(
         existing_results: TaskResult | None,
-        task_eval_splits: list[str],
-        task_eval_langs: list[str],
-        eval_subsets: list[str] | None,
+        task_eval_splits: Sequence[str],
+        task_eval_langs: Sequence[str],
+        eval_subsets: Sequence[str] | None,
     ) -> dict[str, dict[str, Any]]:
         """Return a dictionary for each split, indicating if the whole split is missing and which subsets are missing."""
         missing_evaluations = {
@@ -665,13 +674,13 @@ class MTEB:
         return missing_evaluations
 
     @staticmethod
-    def _get_model_meta(model: EncoderProtocol) -> ModelMeta:
+    def _get_model_meta(model: MTEBModels) -> ModelMeta:
         from sentence_transformers import CrossEncoder, SentenceTransformer
 
         if isinstance(model, CrossEncoder):
-            meta = _model_meta_from_cross_encoder(model)
+            meta = ModelMeta.from_cross_encoder(model)
         elif isinstance(model, SentenceTransformer):
-            meta = _model_meta_from_sentence_transformers(model)
+            meta = ModelMeta.from_sentence_transformer_model(model)
         else:
             meta = ModelMeta(
                 loader=None,

mteb/descriptive_stats/BitextMining/RuSciBenchBitextMining.v2.json

@@ -0,0 +1,61 @@
+{
+    "test": {
+        "num_samples": 19928,
+        "number_of_characters": 35466331,
+        "unique_pairs": 19928,
+        "sentence1_statistics": {
+            "total_text_length": 17733346,
+            "min_text_length": 103,
+            "average_text_length": 889.8708350060217,
+            "max_text_length": 11576,
+            "unique_texts": 19928
+        },
+        "sentence2_statistics": {
+            "total_text_length": 17732985,
+            "min_text_length": 103,
+            "average_text_length": 889.8527197912485,
+            "max_text_length": 11576,
+            "unique_texts": 19928
+        },
+        "hf_subset_descriptive_stats": {
+            "ru-en": {
+                "num_samples": 9965,
+                "number_of_characters": 17734926,
+                "unique_pairs": 9965,
+                "sentence1_statistics": {
+                    "total_text_length": 8685585,
+                    "min_text_length": 103,
+                    "average_text_length": 871.6091319618665,
+                    "max_text_length": 5675,
+                    "unique_texts": 9965
+                },
+                "sentence2_statistics": {
+                    "total_text_length": 9049341,
+                    "min_text_length": 106,
+                    "average_text_length": 908.1124937280482,
+                    "max_text_length": 11576,
+                    "unique_texts": 9965
+                }
+            },
+            "en-ru": {
+                "num_samples": 9963,
+                "number_of_characters": 17731405,
+                "unique_pairs": 9963,
+                "sentence1_statistics": {
+                    "total_text_length": 9047761,
+                    "min_text_length": 106,
+                    "average_text_length": 908.1362039546322,
+                    "max_text_length": 11576,
+                    "unique_texts": 9963
+                },
+                "sentence2_statistics": {
+                    "total_text_length": 8683644,
+                    "min_text_length": 103,
+                    "average_text_length": 871.5892803372478,
+                    "max_text_length": 5675,
+                    "unique_texts": 9963
+                }
+            }
+        }
+    }
+}

mteb/descriptive_stats/Classification/HebrewSentimentAnalysis.v3.json

@@ -0,0 +1,60 @@
+{
+    "test": {
+        "num_samples": 1085,
+        "number_texts_intersect_with_train": 0,
+        "text_statistics": {
+            "total_text_length": 115359,
+            "min_text_length": 8,
+            "average_text_length": 106.32165898617511,
+            "max_text_length": 2722,
+            "unique_texts": 1085
+        },
+        "image_statistics": null,
+        "label_statistics": {
+            "min_labels_per_text": 1,
+            "average_label_per_text": 1.0,
+            "max_labels_per_text": 1,
+            "unique_labels": 3,
+            "labels": {
+                "0": {
+                    "count": 868
+                },
+                "1": {
+                    "count": 190
+                },
+                "2": {
+                    "count": 27
+                }
+            }
+        }
+    },
+    "train": {
+        "num_samples": 7176,
+        "number_texts_intersect_with_train": null,
+        "text_statistics": {
+            "total_text_length": 830248,
+            "min_text_length": 5,
+            "average_text_length": 115.69788182831661,
+            "max_text_length": 4759,
+            "unique_texts": 7176
+        },
+        "image_statistics": null,
+        "label_statistics": {
+            "min_labels_per_text": 1,
+            "average_label_per_text": 1.0,
+            "max_labels_per_text": 1,
+            "unique_labels": 3,
+            "labels": {
+                "0": {
+                    "count": 4933
+                },
+                "1": {
+                    "count": 2047
+                },
+                "2": {
+                    "count": 196
+                }
+            }
+        }
+    }
+}

mteb/descriptive_stats/Classification/TurkishConstitutionalCourtViolation.json

@@ -0,0 +1,54 @@
+{
+    "test": {
+        "num_samples": 193,
+        "number_texts_intersect_with_train": 0,
+        "text_statistics": {
+            "total_text_length": 1543015,
+            "min_text_length": 492,
+            "average_text_length": 7994.896373056995,
+            "max_text_length": 49510,
+            "unique_texts": 193
+        },
+        "image_statistics": null,
+        "label_statistics": {
+            "min_labels_per_text": 1,
+            "average_label_per_text": 1.0,
+            "max_labels_per_text": 1,
+            "unique_labels": 2,
+            "labels": {
+                "1": {
+                    "count": 177
+                },
+                "0": {
+                    "count": 16
+                }
+            }
+        }
+    },
+    "train": {
+        "num_samples": 870,
+        "number_texts_intersect_with_train": null,
+        "text_statistics": {
+            "total_text_length": 6968132,
+            "min_text_length": 259,
+            "average_text_length": 8009.347126436782,
+            "max_text_length": 74490,
+            "unique_texts": 870
+        },
+        "image_statistics": null,
+        "label_statistics": {
+            "min_labels_per_text": 1,
+            "average_label_per_text": 1.0,
+            "max_labels_per_text": 1,
+            "unique_labels": 2,
+            "labels": {
+                "1": {
+                    "count": 755
+                },
+                "0": {
+                    "count": 115
+                }
+            }
+        }
+    }
+}

mteb/descriptive_stats/Image/DocumentUnderstanding/KoVidore2CybersecurityRetrieval.json

@@ -0,0 +1,32 @@
+{
+    "test": {
+        "num_samples": 1299,
+        "number_of_characters": 9254,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 2245,
+            "average_image_width": 2370.324347826087,
+            "max_image_width": 3508,
+            "min_image_height": 2481,
+            "average_image_height": 3289.8060869565215,
+            "max_image_height": 3580,
+            "unique_images": 1132
+        },
+        "queries_text_statistics": {
+            "total_text_length": 9254,
+            "min_text_length": 15,
+            "average_text_length": 62.10738255033557,
+            "max_text_length": 108,
+            "unique_texts": 149
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 409,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 2.7449664429530203,
+            "max_relevant_docs_per_query": 7,
+            "unique_relevant_docs": 316
+        },
+        "top_ranked_statistics": null
+    }
+}