mteb 2.5.2__py3-none-any.whl → 2.5.4__py3-none-any.whl

mteb/cache.py

@@ -3,8 +3,9 @@ import logging
 import os
 import shutil
 import subprocess
+import warnings
 from collections import defaultdict
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 from pathlib import Path
 from typing import cast
 
@@ -83,9 +84,9 @@ class ResultCache:
         model_path = results_folder / model_name
 
         if model_revision is None:
-            logger.warning(
-                "model_revision is not specified, attempting to load the latest revision. To disable this behavior, specify model_revision explicitly."
-            )
+            msg = "`model_revision` is not specified, attempting to load the latest revision. To disable this behavior, specify the 'model_revision` explicitly."
+            logger.warning(msg)
+            warnings.warn(msg)
             # get revs from paths
             revisions = [p for p in model_path.glob("*") if p.is_dir()]
             if not revisions:
@@ -281,15 +282,17 @@ class ResultCache:
             shutil.rmtree(self.cache_path)
             logger.info(f"Cache directory {self.cache_path} cleared.")
         else:
-            logger.warning(f"Cache directory {self.cache_path} does not exist.")
+            msg = f"Cache directory `{self.cache_path}` does not exist."
+            logger.warning(msg)
+            warnings.warn(msg)
 
     def __repr__(self) -> str:
         return f"ResultCache(cache_path={self.cache_path})"
 
     def get_cache_paths(
         self,
-        models: Sequence[str] | Sequence[ModelMeta] | None = None,
-        tasks: Sequence[str] | Sequence[AbsTask] | None = None,
+        models: Sequence[str] | Iterable[ModelMeta] | None = None,
+        tasks: Sequence[str] | Iterable[AbsTask] | None = None,
         require_model_meta: bool = True,
         include_remote: bool = True,
     ) -> list[Path]:
@@ -422,7 +425,7 @@ class ResultCache:
     @staticmethod
     def _filter_paths_by_model_and_revision(
         paths: list[Path],
-        models: Sequence[str] | Sequence[ModelMeta] | None = None,
+        models: Sequence[str] | Iterable[ModelMeta] | None = None,
     ) -> list[Path]:
         """Filter a list of paths by model name and optional revision.
 
@@ -432,8 +435,9 @@ class ResultCache:
         if not models:
             return paths
 
-        if isinstance(models[0], ModelMeta):
-            models = cast(list[ModelMeta], models)
+        first_model = next(iter(models))
+        if isinstance(first_model, ModelMeta):
+            models = cast(Iterable[ModelMeta], models)
             name_and_revision = {
                 (m.model_name_as_path(), m.revision or "no_revision_available")
                 for m in models
@@ -444,13 +448,14 @@ class ResultCache:
                 if (p.parent.parent.name, p.parent.name) in name_and_revision
             ]
 
-        model_names = {m.replace("/", "__").replace(" ", "_") for m in models}
+        str_models = cast(Sequence[str], models)
+        model_names = {m.replace("/", "__").replace(" ", "_") for m in str_models}
         return [p for p in paths if p.parent.parent.name in model_names]
 
     @staticmethod
     def _filter_paths_by_task(
         paths: list[Path],
-        tasks: Sequence[str] | Sequence[AbsTask] | None = None,
+        tasks: Sequence[str] | Iterable[AbsTask] | None = None,
     ) -> list[Path]:
         if tasks is not None:
             task_names = set()
@@ -466,8 +471,8 @@ class ResultCache:
 
     def load_results(
         self,
-        models: Sequence[str] | Sequence[ModelMeta] | None = None,
-        tasks: Sequence[str] | Sequence[AbsTask] | Benchmark | str | None = None,
+        models: Sequence[str] | Iterable[ModelMeta] | None = None,
+        tasks: Sequence[str] | Iterable[AbsTask] | str | None = None,
         require_model_meta: bool = True,
         include_remote: bool = True,
         validate_and_filter: bool = False,
@@ -511,7 +516,7 @@ class ResultCache:
         )
         models_results = defaultdict(list)
 
-        task_names = {}
+        task_names: dict[str, AbsTask | None] = {}
         if tasks is not None:
             for task in tasks:
                 if isinstance(task, AbsTask):
@@ -529,9 +534,11 @@ class ResultCache:
             )
 
             if validate_and_filter:
-                task = task_names[task_result.task_name]
+                task_instance = task_names[task_result.task_name]
                 try:
-                    task_result = task_result.validate_and_filter_scores(task=task)
+                    task_result = task_result.validate_and_filter_scores(
+                        task=task_instance
+                    )
                 except Exception as e:
                     logger.info(
                         f"Validation failed for {task_result.task_name} in {model_name} {revision}: {e}"
@@ -541,7 +548,7 @@ class ResultCache:
             models_results[(model_name, revision)].append(task_result)
 
         # create BenchmarkResults object
-        models_results = [
+        models_results_object = [
             ModelResult(
                 model_name=model_name,
                 model_revision=revision,
@@ -550,9 +557,7 @@ class ResultCache:
             for (model_name, revision), task_results in models_results.items()
         ]
 
-        benchmark_results = BenchmarkResults(
-            model_results=models_results,
+        return BenchmarkResults(
+            model_results=models_results_object,
             benchmark=tasks if isinstance(tasks, Benchmark) else None,
         )
-
-        return benchmark_results

mteb/cli/_display_tasks.py

@@ -1,4 +1,4 @@
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 
 from mteb.abstasks import AbsTask
 from mteb.benchmarks import Benchmark
@@ -31,7 +31,7 @@ def _display_benchmarks(benchmarks: Sequence[Benchmark]) -> None:
         _display_tasks(benchmark.tasks, name=name)
 
 
-def _display_tasks(task_list: Sequence[AbsTask], name: str | None = None) -> None:
+def _display_tasks(task_list: Iterable[AbsTask], name: str | None = None) -> None:
     from rich.console import Console
 
     console = Console()

mteb/cli/build_cli.py

@@ -1,18 +1,19 @@
 import argparse
 import logging
 import os
+import warnings
 from pathlib import Path
 
 import torch
 from rich.logging import RichHandler
 
 import mteb
+from mteb.abstasks.abstask import AbsTask
 from mteb.cache import ResultCache
+from mteb.cli._display_tasks import _display_benchmarks, _display_tasks
 from mteb.cli.generate_model_card import generate_model_card
 from mteb.evaluate import OverwriteStrategy
 
-from ._display_tasks import _display_benchmarks, _display_tasks
-
 logger = logging.getLogger(__name__)
 
 
@@ -53,7 +54,7 @@ def run(args: argparse.Namespace) -> None:
 
     if args.benchmarks:
         benchmarks = mteb.get_benchmarks(names=args.benchmarks)
-        tasks = [t for b in benchmarks for t in b.tasks]
+        tasks = tuple(t for b in benchmarks for t in b.tasks)
     else:
         tasks = mteb.get_tasks(
             categories=args.categories,
@@ -69,15 +70,17 @@ def run(args: argparse.Namespace) -> None:
 
     overwrite_strategy = args.overwrite_strategy
     if args.overwrite:
-        logger.warning(
-            "`--overwrite` is deprecated, please use `--overwrite-strategy 'always'` instead."
+        warnings.warn(
+            "`--overwrite` is deprecated, please use `--overwrite-strategy 'always'` instead.",
+            DeprecationWarning,
         )
         overwrite_strategy = OverwriteStrategy.ALWAYS.value
 
     prediction_folder = args.prediction_folder
     if args.save_predictions:
-        logger.warning(
-            "`--save_predictions` is deprecated, please use `--prediction-folder` instead."
+        warnings.warn(
+            "`--save_predictions` is deprecated, please use `--prediction-folder` instead.",
+            DeprecationWarning,
         )
         prediction_folder = args.output_folder
 
@@ -279,15 +282,17 @@ def _create_meta(args: argparse.Namespace) -> None:
         from_existing = Path(from_existing)
 
     if output_path.exists() and overwrite:
-        logger.warning("Output path already exists, overwriting.")
+        msg = "Output path already exists, overwriting."
+        logger.warning(msg)
+        warnings.warn(msg)
     elif output_path.exists():
         raise FileExistsError(
             "Output path already exists, use --overwrite to overwrite."
         )
 
-    tasks = []
+    tasks: list[AbsTask] = []
     if tasks_names is not None:
-        tasks = mteb.get_tasks(tasks_names)
+        tasks = list(mteb.get_tasks(tasks_names))
     if benchmarks is not None:
         benchmarks = mteb.get_benchmarks(benchmarks)
         for benchmark in benchmarks:

mteb/cli/generate_model_card.py

@@ -1,4 +1,6 @@
 import logging
+import warnings
+from collections.abc import Sequence
 from pathlib import Path
 
 from huggingface_hub import ModelCard, ModelCardData, repo_exists
@@ -12,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 def generate_model_card(
     model_name: str,
-    tasks: list[AbsTask] | None = None,
+    tasks: Sequence[AbsTask] | None = None,
     existing_model_card_id_or_path: str | Path | None = None,
     results_cache: ResultCache = ResultCache(),
     output_path: Path = Path("model_card.md"),
@@ -47,8 +49,8 @@ def generate_model_card(
         for task_result in models_results.task_results:
             eval_results.extend(task_result.get_hf_eval_results())
 
-    existing_model_card_data = (
-        existing_model_card.data if existing_model_card else ModelCardData()
+    existing_model_card_data: ModelCardData = (
+        existing_model_card.data if existing_model_card else ModelCardData()  # type: ignore[assignment]
     )
 
     if existing_model_card_data.eval_results is None:
@@ -88,13 +90,14 @@ def generate_model_card(
             benchmark_results, existing_model_card
         )
 
-    if push_to_hub:
+    if push_to_hub and existing_model_card_id_or_path:
+        existing_model_card_id_or_path = str(existing_model_card_id_or_path)
         if repo_exists(existing_model_card_id_or_path):
             existing_model_card.push_to_hub(existing_model_card_id_or_path, token=token)
         else:
-            logger.warning(
-                f"Repository {existing_model_card_id_or_path} does not exist on the Hub. Skipping push to hub."
-            )
+            msg = f"Repository {existing_model_card_id_or_path} does not exist on the Hub. Skipping push to hub."
+            logger.warning(msg)
+            warnings.warn(msg)
     existing_model_card.save(output_path)
 
 

mteb/deprecated_evaluator.py

@@ -5,23 +5,24 @@ import logging
 import os
 import sys
 import traceback
-from collections.abc import Iterable
+import warnings
+from collections.abc import Iterable, Sequence
 from copy import deepcopy
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
 from time import time
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 import datasets
 
 import mteb
 from mteb.abstasks import AbsTask
+from mteb.abstasks.aggregated_task import AbsTaskAggregate
 from mteb.abstasks.task_metadata import TaskCategory, TaskType
 from mteb.benchmarks import Benchmark
 from mteb.models import (
     CrossEncoderWrapper,
-    EncoderProtocol,
     ModelMeta,
     MTEBModels,
     SentenceTransformerEncoderWrapper,
@@ -52,7 +53,7 @@ class MTEB:
     )
     def __init__(
         self,
-        tasks: Iterable[AbsTask | Benchmark],
+        tasks: Iterable[AbsTask] | Iterable[Benchmark],
         *,
         err_logs_path: str = "error_logs.txt",
     ) -> None:
@@ -63,15 +64,14 @@ class MTEB:
                 `mteb.get_tasks(["task1","task2"]) or `mteb.get_benchmark("MTEB(eng, classic)").
             err_logs_path: Path to save error logs.
         """
-        from mteb.benchmarks import Benchmark
-
-        self.tasks = list(tasks)
-        if len(self.tasks) > 0 and isinstance(self.tasks[0], Benchmark):
+        if isinstance(next(iter(tasks)), Benchmark):
             self.benchmarks = tasks
-            self.tasks = list(chain.from_iterable(self.tasks))
+            self.tasks = list(chain.from_iterable(cast(Iterable[Benchmark], tasks)))
+        elif isinstance(next(iter(tasks)), AbsTask):
+            self.tasks = list(cast(Iterable[AbsTask], tasks))
 
         self.err_logs_path = Path(err_logs_path)
-        self.last_evaluated_splits = {}
+        self._last_evaluated_splits: dict[str, list[str]] = {}
 
     @property
     def available_tasks(self) -> list[str]:
@@ -84,7 +84,7 @@ class MTEB:
         return sorted({x.metadata.type for x in self.tasks})
 
     @property
-    def available_task_categories(self) -> set[TaskCategory]:
+    def available_task_categories(self) -> set[TaskCategory | None]:
         """Set of available task categories."""
         return {x.metadata.category for x in self.tasks}
 
@@ -231,13 +231,14 @@ class MTEB:
         merged_kg_co2_emissions = None
         if existing_kg_co2_emissions and new_kg_co2_emissions:
             merged_kg_co2_emissions = existing_kg_co2_emissions + new_kg_co2_emissions
+        existing_evaluation_time = existing_results.evaluation_time or 0
+        new_evaluation_time = new_results.evaluation_time or 0
         merged_results = TaskResult(
             dataset_revision=new_results.dataset_revision,
             task_name=new_results.task_name,
             mteb_version=new_results.mteb_version,
             scores=merged_scores,
-            evaluation_time=existing_results.evaluation_time
-            + new_results.evaluation_time,
+            evaluation_time=existing_evaluation_time + new_evaluation_time,
             kg_co2_emissions=merged_kg_co2_emissions,
         )
 
@@ -306,13 +307,16 @@ class MTEB:
         elif verbosity == 3:
             datasets.logging.set_verbosity(logging.DEBUG)
 
-        meta = self.create_model_meta(model)
-        output_path = self._create_output_folder(meta, output_folder)
-
+        mteb_model: MTEBModels
         if isinstance(model, SentenceTransformer):
-            model = SentenceTransformerEncoderWrapper(model)
+            mteb_model = SentenceTransformerEncoderWrapper(model)
         elif isinstance(model, CrossEncoder):
-            model = CrossEncoderWrapper(model)
+            mteb_model = CrossEncoderWrapper(model)
+        else:
+            mteb_model = cast(MTEBModels, model)
+
+        meta = self.create_model_meta(mteb_model)
+        output_path = self._create_output_folder(meta, output_folder)
 
         # Disable co2_tracker for API models
         if "API" in meta.framework:
@@ -333,7 +337,7 @@ class MTEB:
         )  # save them in case we re-use the object (e.g. for reranking)
 
         # To evaluate missing splits, we keep track of the task name and the corresponding splits.
-        self.last_evaluated_splits = {}
+        self._last_evaluated_splits = {}
 
         while len(self.tasks) > 0:
             task = self.tasks[0]
@@ -342,9 +346,10 @@ class MTEB:
             )
 
             if task.is_aggregate:
-                self_ = MTEB(tasks=task.metadata.tasks)
-                task_results = self_.run(
-                    model,
+                aggregated_task = cast(AbsTaskAggregate, task)
+                self_ = MTEB(tasks=aggregated_task.metadata.tasks)
+                aggregated_task_results = self_.run(
+                    mteb_model,
                     verbosity=verbosity - 1,
                     output_folder=output_folder,
                     eval_splits=eval_splits,
@@ -355,12 +360,15 @@ class MTEB:
                     encode_kwargs=encode_kwargs,
                     **kwargs,
                 )
-                new_results = task.combine_task_results(task_results)
+                new_results = aggregated_task.combine_task_results(
+                    aggregated_task_results
+                )
                 evaluation_results.append(new_results)
 
                 if output_path:
-                    save_path = output_path / f"{task.metadata.name}.json"
-                    new_results.to_disk(save_path)
+                    new_results.to_disk(
+                        output_path / f"{aggregated_task.metadata.name}.json"
+                    )
                 del self.tasks[0]
                 continue
 
@@ -382,7 +390,7 @@ class MTEB:
             task_subsets = task.hf_subsets
 
             existing_results = None
-            save_path = None
+            save_path: Path | None = None
             final_splits_to_run = task_eval_splits
             missing_evaluations = self._get_missing_evaluations(
                 existing_results,
@@ -432,7 +440,7 @@ class MTEB:
                     logger.info(
                         f"No splits to evaluate for {task.metadata.name}. Skipping evaluation."
                     )
-                self.last_evaluated_splits[task.metadata.name] = []
+                self._last_evaluated_splits[task.metadata.name] = []
                 del self.tasks[0]
                 continue
 
@@ -440,11 +448,11 @@ class MTEB:
                 task.check_if_dataset_is_superseded()
                 task.load_data()
 
-                task_results = {}
+                task_results: dict[str, dict[str, dict[str, Any]]] = {}
                 evaluation_time = 0
                 kg_co2_emissions: int | None = 0 if co2_tracker else None
 
-                self.last_evaluated_splits[task.metadata.name] = []
+                self._last_evaluated_splits[task.metadata.name] = []
 
                 for split in final_splits_to_run:
                     info = missing_evaluations[split]
@@ -465,14 +473,16 @@ class MTEB:
 
                     if co2_tracker:
                         try:
-                            from codecarbon import EmissionsTracker
+                            from codecarbon import (  # type: ignore[import-untyped]
+                                EmissionsTracker,
+                            )
                         except ImportError:
                             raise ImportError(
                                 "codecarbon is not installed. Please install it using `pip install 'mteb[codecarbon]'` to track CO₂ emissions."
                             )
-                        logger.warning(
-                            "Evaluating multiple MTEB runs simultaneously will produce incorrect CO₂ results"
-                        )
+                        msg = "Evaluating multiple MTEB runs simultaneously will produce incorrect CO₂ results"
+                        logger.warning(msg)
+                        warnings.warn(msg)
                         with EmissionsTracker(
                             save_to_file=False,
                             save_to_api=False,
@@ -481,7 +491,7 @@ class MTEB:
                         ) as tracker:
                             results, tick, tock = self._run_eval(
                                 task,
-                                model,
+                                mteb_model,
                                 split,
                                 encode_kwargs=encode_kwargs,
                                 subsets_to_run=subsets_to_run,
@@ -494,7 +504,7 @@ class MTEB:
                     else:
                         results, tick, tock = self._run_eval(
                             task,
-                            model,
+                            mteb_model,
                             split,
                             subsets_to_run=subsets_to_run,
                             encode_kwargs=encode_kwargs,
@@ -510,25 +520,25 @@ class MTEB:
                     if verbosity >= 1:
                         logger.info(f"Scores: {task_results[split]}")
 
-                    self.last_evaluated_splits[task.metadata.name].append(split)
+                    self._last_evaluated_splits[task.metadata.name].append(split)
 
                 # Create new TaskResult
                 new_results = TaskResult.from_task_results(
                     task,
-                    task_results,
+                    task_results,  # type: ignore[arg-type]
                     evaluation_time=evaluation_time,
                     kg_co2_emissions=kg_co2_emissions,
                 )
 
                 # Merge with existing if needed
-                if output_path and save_path.exists():
+                if output_path and save_path and save_path.exists():
                     existing_results = TaskResult.from_disk(save_path)
                 if existing_results:
                     merged_results = self._merge_results(existing_results, new_results)
                 else:
                     merged_results = new_results
 
-                if output_path:
+                if output_path and save_path:
                     merged_results.to_disk(save_path)
 
                 evaluation_results.append(merged_results)
@@ -555,7 +565,7 @@ class MTEB:
     def create_model_meta(model: MTEBModels) -> ModelMeta:
         """Create a ModelMeta object for the given model."""
         if hasattr(model, "mteb_model_meta") and model.mteb_model_meta is not None:
-            meta = model.mteb_model_meta  # type: ignore
+            meta = model.mteb_model_meta
         else:
             meta = MTEB._get_model_meta(model)
 
@@ -581,7 +591,11 @@ class MTEB:
         if output_folder is None:
             return None
 
-        model_revision: str = model_meta.revision  # type: ignore
+        model_revision: str = (
+            model_meta.revision
+            if model_meta.revision is not None
+            else "no_revision_available"
+        )
         model_path_name = model_meta.model_name_as_path()
 
         output_path = Path(output_folder) / model_path_name / model_revision
@@ -603,15 +617,15 @@ class MTEB:
              Tasks with empty lists indicate that results already existed and no splits were evaluated.
         """
         return deepcopy(
-            {task: list(splits) for task, splits in self.last_evaluated_splits.items()}
+            {task: list(splits) for task, splits in self._last_evaluated_splits.items()}
         )
 
     @staticmethod
     def _get_missing_evaluations(
         existing_results: TaskResult | None,
-        task_eval_splits: list[str],
-        task_eval_langs: list[str],
-        eval_subsets: list[str] | None,
+        task_eval_splits: Sequence[str],
+        task_eval_langs: Sequence[str],
+        eval_subsets: Sequence[str] | None,
     ) -> dict[str, dict[str, Any]]:
         """Return a dictionary for each split, indicating if the whole split is missing and which subsets are missing."""
         missing_evaluations = {
@@ -660,7 +674,7 @@ class MTEB:
         return missing_evaluations
 
     @staticmethod
-    def _get_model_meta(model: EncoderProtocol) -> ModelMeta:
+    def _get_model_meta(model: MTEBModels) -> ModelMeta:
         from sentence_transformers import CrossEncoder, SentenceTransformer
 
         if isinstance(model, CrossEncoder):