mteb 2.5.2__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/leaderboard/app.py

@@ -5,7 +5,7 @@ import tempfile
 import time
 import warnings
 from pathlib import Path
-from typing import Literal
+from typing import Literal, get_args
 from urllib.parse import urlencode
 
 import cachetools
@@ -29,40 +29,115 @@ from mteb.leaderboard.table import (
     apply_summary_styling_from_benchmark,
 )
 from mteb.leaderboard.text_segments import ACKNOWLEDGEMENT, FAQ
+from mteb.models.model_meta import MODEL_TYPES
 
 logger = logging.getLogger(__name__)
 
+
 LANGUAGE: list[str] = list({l for t in mteb.get_tasks() for l in t.metadata.languages})
+MODEL_TYPE_CHOICES = list(get_args(MODEL_TYPES))
 
 
 def _load_results(cache: ResultCache) -> BenchmarkResults:
+    """Load benchmark results using an optimized caching strategy.
+
+    This function implements a two-tier caching strategy for faster leaderboard startup:
+
+    1. **Primary Strategy (Fast)**: Download pre-computed cached results from the
+       'cached-data' branch as a compressed JSON file (~2MB vs ~200MB full repo).
+       This avoids the need to clone the entire results repository and provides
+       near-instantaneous loading for most users.
+
+    2. **Fallback Strategy (Slower)**: If the cached download fails, fall back to
+       the original approach of downloading the full results repository and
+       building the cache from scratch.
+
+    The cached results file contains pre-aggregated benchmark data that eliminates
+    the need for expensive operations like task selection and revision joining
+    during app startup.
+
+    Args:
+        cache: ResultCache instance used for both optimized and fallback operations
+
+    Returns:
+        BenchmarkResults: Complete benchmark results ready for leaderboard display
+
+    Raises:
+        Various exceptions related to network issues, file I/O, or data validation
+        are logged and may cause fallback to the slower repository-based approach.
+    """
     start_time = time.time()
     results_cache_path = Path(__file__).parent.joinpath("__cached_results.json")
+
     if not results_cache_path.exists():
-        logger.info("Cached results not found, downloading from remote...")
-        cache.download_from_remote()
-        download_time = time.time() - start_time
-        logger.info(f"Downloaded remote results in {download_time:.2f}s")
-
-        load_start = time.time()
-        all_model_names = [model_meta.name for model_meta in mteb.get_model_metas()]
-
-        all_results = cache.load_results(
-            models=all_model_names,
-            only_main_score=True,
-            require_model_meta=False,
-            include_remote=True,
+        # First try to download the cached results file from the cached-data branch
+        # This is faster than cloning the entire results repository
+        logger.info(
+            "Cached results not found, trying to download from cached-data branch..."
         )
-        load_time = time.time() - load_start
-        logger.info(f"Loaded results from cache in {load_time:.2f}s")
-        return all_results
-    else:
-        logger.info("Loading cached results from disk...")
+
+        try:
+            # Use ResultCache's optimized download method
+            # Default saves to mteb/leaderboard/__cached_results.json
+            results_cache_path = cache._download_cached_results_from_branch()
+            download_time = time.time() - start_time
+            logger.info(
+                f"Downloaded cached results from cached-data branch in {download_time:.2f}s"
+            )
+
+        except Exception as e:
+            logger.error(
+                f"Failed to download from cached-data branch: {type(e).__name__}: {e}"
+            )
+            logger.info("Falling back to downloading full remote repository...")
+
+            # Fall back to the original approach: clone the full repo
+            cache.download_from_remote()
+            download_time = time.time() - start_time
+            logger.info(f"Downloaded remote results in {download_time:.2f}s")
+
+            load_start = time.time()
+            all_model_names = [model_meta.name for model_meta in mteb.get_model_metas()]
+
+            all_results = cache.load_results(
+                models=all_model_names,
+                only_main_score=True,
+                require_model_meta=False,
+                include_remote=True,
+            )
+            load_time = time.time() - load_start
+            logger.info(f"Loaded results from cache in {load_time:.2f}s")
+            return all_results
+
+    # Load the cached results file (either pre-existing or just downloaded)
+    logger.info("Loading cached results from disk...")
+    try:
+        logger.info(f"Opening file: {results_cache_path}")
+
+        file_size = results_cache_path.stat().st_size
+        logger.info(f"File exists, size: {file_size} bytes")
+
         with results_cache_path.open() as cache_file:
-            results = mteb.BenchmarkResults.from_validated(**json.load(cache_file))
-        total_time = time.time() - start_time
-        logger.info(f"Loaded cached results in {total_time:.2f}s")
-        return results
+            logger.info("File opened successfully, attempting JSON parse...")
+            json_data = json.load(cache_file)
+            logger.info(
+                f"JSON parsed successfully, keys: {list(json_data.keys()) if isinstance(json_data, dict) else 'not a dict'}"
+            )
+
+        logger.info("Attempting BenchmarkResults.from_validated...")
+        results = mteb.BenchmarkResults.from_validated(**json_data)
+        logger.info("BenchmarkResults.from_validated successful")
+
+    except Exception as e:
+        # TODO: Handle the case when we fail to load cached results from disk.
+        logger.error(
+            f"Failed to load cached results from disk: {type(e).__name__}: {e}"
+        )
+        raise
+
+    total_time = time.time() - start_time
+    logger.info(f"Loaded cached results in {total_time:.2f}s")
+    return results
 
 
 def _produce_benchmark_link(benchmark_name: str, request: gr.Request) -> str:
@@ -169,7 +244,7 @@ def _update_task_info(task_names: str) -> gr.DataFrame:
     df = df.drop(columns="reference")
     return gr.DataFrame(
         df,
-        datatype=["markdown"] + ["str"] * (len(df.columns) - 1),  # type: ignore
+        datatype=["markdown"] + ["str"] * (len(df.columns) - 1),
         buttons=["copy", "fullscreen"],
         show_search="filter",
     )
@@ -187,6 +262,7 @@ def _filter_models(
     instructions: bool | None,
     max_model_size: int,
     zero_shot_setting: Literal["only_zero_shot", "allow_all", "remove_unknown"],
+    model_types: list[str] | None,
 ):
     lower, upper = 0, max_model_size
     # Setting to None, when the user doesn't specify anything
@@ -205,6 +281,7 @@ def _filter_models(
         use_instructions=instructions,
         frameworks=compatibility,
         n_parameters_range=(lower, upper),
+        model_types=model_types,
     )
 
     models_to_keep = set()
@@ -269,6 +346,7 @@ def _cache_on_benchmark_select(benchmark_name, all_benchmark_results):
         instructions=None,
         max_model_size=MAX_MODEL_SIZE,
         zero_shot_setting="allow_all",
+        model_types=MODEL_TYPE_CHOICES,
     )
     # Sort to ensure consistency with update_models
     initial_models = sorted(initial_models)
@@ -387,6 +465,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
         instructions=None,
         max_model_size=MAX_MODEL_SIZE,
         zero_shot_setting="allow_all",
+        model_types=MODEL_TYPE_CHOICES,
     )
     default_filtered_scores = [
         entry for entry in default_scores if entry["model_name"] in filtered_models
@@ -583,6 +662,12 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                                     label="Model Parameters",
                                     interactive=True,
                                 )
+                            with gr.Column():
+                                model_type_select = gr.CheckboxGroup(
+                                    MODEL_TYPE_CHOICES,
+                                    value=MODEL_TYPE_CHOICES,
+                                    label="Model Type",
+                                )
 
         with gr.Tab("Summary"):
             summary_table.render()
@@ -755,7 +840,8 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
             compatibility,
             instructions,
             max_model_size,
-            zero_shot: hash(
+            zero_shot,
+            model_type_select: hash(
                 (
                     id(scores),
                     hash(tuple(tasks)),
@@ -764,6 +850,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                     hash(instructions),
                     hash(max_model_size),
                     hash(zero_shot),
+                    hash(tuple(model_type_select)),
                 )
             ),
         )
@@ -775,6 +862,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
             instructions: bool | None,
             max_model_size: int,
             zero_shot: Literal["allow_all", "remove_unknown", "only_zero_shot"],
+            model_type_select: list[str],
         ):
             start_time = time.time()
             model_names = list({entry["model_name"] for entry in scores})
@@ -786,6 +874,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                 instructions,
                 max_model_size,
                 zero_shot_setting=zero_shot,
+                model_types=model_type_select,
             )
             elapsed = time.time() - start_time
             logger.debug(f"update_models callback: {elapsed}s")
@@ -803,6 +892,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                 instructions,
                 max_model_size,
                 zero_shot,
+                model_type_select,
             ],
             outputs=[models],
         )
@@ -817,6 +907,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                 instructions,
                 max_model_size,
                 zero_shot,
+                model_type_select,
             ],
             outputs=[models],
         )
@@ -830,6 +921,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                 instructions,
                 max_model_size,
                 zero_shot,
+                model_type_select,
             ],
             outputs=[models],
         )
@@ -843,6 +935,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                 instructions,
                 max_model_size,
                 zero_shot,
+                model_type_select,
             ],
             outputs=[models],
         )
@@ -856,6 +949,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                 instructions,
                 max_model_size,
                 zero_shot,
+                model_type_select,
             ],
             outputs=[models],
         )
@@ -869,6 +963,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                 instructions,
                 max_model_size,
                 zero_shot,
+                model_type_select,
             ],
             outputs=[models],
         )
@@ -882,6 +977,21 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
                 instructions,
                 max_model_size,
                 zero_shot,
+                model_type_select,
+            ],
+            outputs=[models],
+        )
+        model_type_select.change(
+            update_models,
+            inputs=[
+                scores,
+                task_select,
+                availability,
+                compatibility,
+                instructions,
+                max_model_size,
+                zero_shot,
+                model_type_select,
             ],
             outputs=[models],
         )
@@ -1023,16 +1133,34 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
 
 
 if __name__ == "__main__":
-    logging.getLogger("mteb.load_results.task_results").setLevel(
-        logging.ERROR
-    )  # Warnings related to task split
-    logging.getLogger("mteb.model_meta").setLevel(
-        logging.ERROR
-    )  # Warning related to model metadata (fetch_from_hf=False)
-    logging.getLogger("mteb.load_results.benchmark_results").setLevel(
-        logging.ERROR
-    )  # Warning related to model metadata (fetch_from_hf=False)
+    import os
+
+    # Add process ID to logging for multiprocessing debugging
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - PID:%(process)d - %(name)s - %(levelname)s - %(message)s",
+        force=True,  # Override any existing handlers
+    )
+
+    # Flush log handlers immediately (helpful for multiprocessing)
+    for handler in logging.root.handlers:
+        handler.flush()
+
+    logger.info(f"Starting leaderboard app in process {os.getpid()}")
+
+    # Suppress specific WARNING messages while keeping INFO level for the app
+    logging.getLogger("mteb.results.task_result").setLevel(logging.ERROR)
+    logging.getLogger("mteb.models.model_meta").setLevel(logging.ERROR)
+    logging.getLogger("mteb.results.benchmark_results").setLevel(logging.ERROR)
+
     warnings.filterwarnings("ignore", message="Couldn't get scores for .* due to .*")
+    warnings.filterwarnings("ignore", message="Could not get source model: .*")
+    warnings.filterwarnings(
+        "ignore", message="No scores data available. Returning empty DataFrame."
+    )
+    warnings.filterwarnings("ignore", message="Main score .* not found in scores")
+    warnings.filterwarnings("ignore", message=".*: Missing subsets .* for split .*")
+    warnings.filterwarnings("ignore", message=".*: Missing splits .*")
 
     app = get_leaderboard_app()
 

mteb/load_results.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import sys
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 from pathlib import Path
 
 from mteb.abstasks.abstask import AbsTask
@@ -45,8 +45,8 @@ def _model_name_and_revision(
 def load_results(
     results_repo: str = "https://github.com/embeddings-benchmark/results",
     download_latest: bool = True,
-    models: Sequence[ModelMeta] | Sequence[str] | None = None,
-    tasks: Sequence[AbsTask] | Sequence[str] | None = None,
+    models: Iterable[ModelMeta] | Sequence[str] | None = None,
+    tasks: Iterable[AbsTask] | Sequence[str] | None = None,
     validate_and_filter: bool = True,
     require_model_meta: bool = True,
     only_main_score: bool = False,
@@ -83,21 +83,21 @@ def load_results(
 
     if models is not None:
         models_to_keep = {}
-        for model_path in models:
-            if isinstance(model_path, ModelMeta):
-                models_to_keep[model_path.name] = model_path.revision
+        for model in models:
+            if isinstance(model, ModelMeta):
+                models_to_keep[model.name] = model.revision
             else:
-                models_to_keep[model_path] = None
+                models_to_keep[model] = None
     else:
         models_to_keep = None
 
-    task_names = {}
+    task_names: dict[str, AbsTask | None] = {}
     if tasks is not None:
-        for task in tasks:
-            if isinstance(task, AbsTask):
-                task_names[task.metadata.name] = task
+        for task_ in tasks:
+            if isinstance(task_, AbsTask):
+                task_names[task_.metadata.name] = task_
             else:
-                task_names[task] = None
+                task_names[task_] = None
 
     model_results = []
     for model_path in model_paths:

mteb/models/abs_encoder.py

@@ -1,9 +1,11 @@
 import logging
+import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Sequence
 from typing import Any, Literal, cast, get_args, overload
 
 from torch.utils.data import DataLoader
+from typing_extensions import Unpack
 
 import mteb
 from mteb.abstasks.task_metadata import TaskMetadata, TaskType
@@ -18,6 +20,7 @@ from mteb.similarity_functions import (
 from mteb.types import (
     Array,
     BatchedInput,
+    EncodeKwargs,
     PromptType,
 )
 
@@ -43,7 +46,7 @@ class AbsEncoder(ABC):
     model: Any
     mteb_model_meta: ModelMeta | None = None
     model_prompts: dict[str, str] | None = None
-    instruction_template: str | Callable[[str, PromptType], str] | None = None
+    instruction_template: str | Callable[[str, PromptType | None], str] | None = None
     prompts_dict: dict[str, str] | None = None
 
     def get_prompt_name(
@@ -110,7 +113,7 @@ class AbsEncoder(ABC):
         if not self.model_prompts:
             return None
         prompt_name = self.get_prompt_name(task_metadata, prompt_type)
-        return self.model_prompts.get(prompt_name)
+        return self.model_prompts.get(prompt_name) if prompt_name else None
 
     @staticmethod
     @overload
@@ -187,6 +190,7 @@ class AbsEncoder(ABC):
                 except KeyError:
                     msg = f"Task name {task_name} is not valid. {valid_keys_msg}"
                     logger.warning(msg)
+                    warnings.warn(msg)
                     invalid_task_messages.add(msg)
                     invalid_keys.add(task_key)
 
@@ -232,9 +236,9 @@ class AbsEncoder(ABC):
         if isinstance(prompt, dict) and prompt_type:
             if prompt.get(prompt_type.value):
                 return prompt[prompt_type.value]
-            logger.warning(
-                f"Prompt type '{prompt_type}' not found in task metadata for task '{task_metadata.name}'."
-            )
+            msg = f"Prompt type '{prompt_type}' not found in task metadata for task '{task_metadata.name}'."
+            logger.warning(msg)
+            warnings.warn(msg)
             return ""
 
         if prompt:
@@ -368,7 +372,7 @@ class AbsEncoder(ABC):
         hf_split: str,
         hf_subset: str,
         prompt_type: PromptType | None = None,
-        **kwargs: Any,
+        **kwargs: Unpack[EncodeKwargs],
     ) -> Array:
         """Encodes the given sentences using the encoder.
 

mteb/models/cache_wrappers/cache_backend_protocol.py

@@ -5,8 +5,6 @@ from typing import Any, Protocol, runtime_checkable
 
 import numpy as np
 
-from mteb.types import BatchedInput
-
 
 @runtime_checkable
 class CacheBackendProtocol(Protocol):
@@ -26,7 +24,7 @@ class CacheBackendProtocol(Protocol):
             **kwargs: Additional backend-specific arguments.
         """
 
-    def add(self, item: list[BatchedInput], vectors: np.ndarray) -> None:
+    def add(self, item: list[dict[str, Any]], vectors: np.ndarray) -> None:
         """Add a vector to the cache.
 
         Args:
@@ -34,7 +32,7 @@ class CacheBackendProtocol(Protocol):
             vectors: Embedding vector of shape (dim,) or (1, dim).
         """
 
-    def get_vector(self, item: BatchedInput) -> np.ndarray | None:
+    def get_vector(self, item: dict[str, Any]) -> np.ndarray | None:
         """Retrieve the cached vector for the given item.
 
         Args:
@@ -53,5 +51,5 @@ class CacheBackendProtocol(Protocol):
     def close(self) -> None:
         """Release resources or flush data."""
 
-    def __contains__(self, item: BatchedInput) -> bool:
+    def __contains__(self, item: dict[str, Any]) -> bool:
         """Check whether the cache contains an item."""

mteb/models/cache_wrappers/cache_backends/_hash_utils.py

@@ -1,12 +1,13 @@
 import hashlib
+from collections.abc import Mapping
+from typing import Any
 
-from mteb.types import BatchedInput
 
-
-def _hash_item(item: BatchedInput) -> str:
+def _hash_item(item: Mapping[str, Any]) -> str:
     item_hash = ""
     if "text" in item:
-        item_hash = hashlib.sha256(item["text"].encode()).hexdigest()
+        item_text: str = item["text"]
+        item_hash = hashlib.sha256(item_text.encode()).hexdigest()
 
     if "image" in item:
         from PIL import Image

mteb/models/cache_wrappers/cache_backends/faiss_cache.py

@@ -1,6 +1,8 @@
 import json
 import logging
+import warnings
 from pathlib import Path
+from typing import Any
 
 import numpy as np
 
@@ -36,7 +38,7 @@ class FaissCache:
         logger.info(f"Initialized FAISS VectorCacheMap in {self.directory}")
         self.load()
 
-    def add(self, items: list[BatchedInput], vectors: np.ndarray) -> None:
+    def add(self, items: list[dict[str, Any]], vectors: np.ndarray) -> None:
         """Add vector to FAISS index."""
         import faiss
 
@@ -71,7 +73,9 @@ class FaissCache:
         try:
             return self.index.reconstruct(idx)
         except Exception:
-            logger.warning(f"Vector id {idx} missing for hash {item_hash}")
+            msg = f"Vector id {idx} missing for hash {item_hash}"
+            logger.warning(msg)
+            warnings.warn(msg)
             return None
 
     def save(self) -> None: