mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/cache.py

@@ -1,14 +1,22 @@
+import gzip
+import io
 import json
 import logging
 import os
 import shutil
 import subprocess
+import warnings
 from collections import defaultdict
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 from pathlib import Path
 from typing import cast
 
+import requests
+from pydantic import ValidationError
+
+import mteb
 from mteb.abstasks import AbsTask
+from mteb.benchmarks.benchmark import Benchmark
 from mteb.models import ModelMeta
 from mteb.results import BenchmarkResults, ModelResult, TaskResult
 from mteb.types import ModelName, Revision
@@ -20,8 +28,8 @@ class ResultCache:
     """Class to handle the local cache of MTEB results.
 
     Examples:
-        >>> from mteb.cache import ResultCache
-        >>> cache = ResultCache(cache_path="~/.cache/mteb") # default
+        >>> import mteb
+        >>> cache = mteb.ResultCache(cache_path="~/.cache/mteb") # default
         >>> cache.download_from_remote() # download the latest results from the remote repository
         >>> result = cache.load_results("task_name", "model_name")
     """
@@ -62,7 +70,11 @@ class ResultCache:
         Returns:
             The path to the results of the task.
         """
-        results_folder = "results" if not remote else "remote"
+        results_folder = (
+            self.cache_path / "results"
+            if not remote
+            else self.cache_path / "remote" / "results"
+        )
 
         if isinstance(model_name, ModelMeta):
             if model_revision is not None:
@@ -74,12 +86,12 @@ class ResultCache:
         elif isinstance(model_name, str):
             model_name = model_name.replace("/", "__").replace(" ", "_")
 
-        model_path = self.cache_path / results_folder / model_name
+        model_path = results_folder / model_name
 
         if model_revision is None:
-            logger.warning(
-                "model_revision is not specified, attempting to load the latest revision. To disable this behavior, specify model_revision explicitly."
-            )
+            msg = "`model_revision` is not specified, attempting to load the latest revision. To disable this behavior, specify the 'model_revision` explicitly."
+            logger.warning(msg)
+            warnings.warn(msg)
             # get revs from paths
             revisions = [p for p in model_path.glob("*") if p.is_dir()]
             if not revisions:
@@ -191,12 +203,14 @@ class ResultCache:
         self,
         remote: str = "https://github.com/embeddings-benchmark/results",
         download_latest: bool = True,
+        revision: str | None = None,
     ) -> Path:
         """Downloads the latest version of the results repository from GitHub to a local cache directory. Required git to be installed.
 
         Args:
             remote: The URL of the results repository on GitHub.
             download_latest: If True it will download the latest version of the repository, otherwise it will only update the existing repository.
+            revision: If specified, it will checkout the given revision after cloning or pulling the repository.
 
         Returns:
             The path to the local cache directory.
@@ -224,14 +238,27 @@ class ResultCache:
                 )
                 raise ValueError(msg)
 
-            if download_latest:
+            if revision or download_latest:
                 logger.info(
-                    f"remote repository already exists in {results_directory}, updating it using git pull"
+                    f"remote repository already exists in {results_directory}, fetching updates"
+                )
+                subprocess.run(
+                    ["git", "fetch", "--all", "--tags"],
+                    cwd=results_directory,
+                    check=True,
                 )
-                subprocess.run(["git", "pull"], cwd=results_directory)
             else:
                 logger.debug(
-                    f"Results repository already exists in {results_directory}, skipping update, set download_latest=True to update it"
+                    f"Results repository already exists in {results_directory}, skipping update, "
+                    f"set download_latest=True to update it"
+                )
+
+            if revision:
+                logger.info(f"Checking out revision '{revision}'")
+                subprocess.run(
+                    ["git", "checkout", revision],
+                    cwd=results_directory,
+                    check=True,
                 )
             return results_directory
 
@@ -239,25 +266,180 @@ class ResultCache:
             f"No results repository found in {results_directory}, cloning it from {remote}"
         )
 
-        subprocess.run(["git", "clone", remote, "remote"], cwd=self.cache_path)
+        clone_cmd = ["git", "clone", "--depth", "1"]
+
+        if revision:
+            logger.info(f"Cloning repository at revision '{revision}'")
+            clone_cmd.append(f"--revision={revision}")
+        clone_cmd.extend([remote, "remote"])
+
+        subprocess.run(
+            clone_cmd,
+            cwd=self.cache_path,
+            check=True,
+        )
 
         return results_directory
 
+    def _download_cached_results_from_branch(
+        self,
+        branch: str = "cached-data",
+        filename: str = "__cached_results.json.gz",
+        output_path: Path | None = None,
+        remote: str = "https://github.com/embeddings-benchmark/results",
+        timeout: int = 60,
+        max_size_mb: int = 500,
+    ) -> Path:
+        """Download pre-computed cached results from a specific branch.
+
+        This is significantly faster than download_from_remote() since it downloads
+        only a compressed cache file instead of cloning the entire repository.
+
+        The method performs the following steps:
+        1. Downloads a gzipped JSON file from the specified branch
+        2. Validates file size and content type
+        3. Decompresses the gzip content
+        4. Writes the decompressed JSON to disk
+
+        Args:
+            branch: Branch name to download from (default: "cached-data")
+            filename: Name of the cached results file (default: "__cached_results.json.gz")
+            output_path: Where to save the file. If None, uses mteb/leaderboard/__cached_results.json
+            remote: Base URL of the results repository
+            timeout: Request timeout in seconds (default: 60)
+            max_size_mb: Maximum allowed file size in megabytes (default: 500)
+
+        Returns:
+            Path to the downloaded and decompressed cache file
+
+        Raises:
+            requests.exceptions.RequestException: On HTTP errors
+            ValueError: On validation failures (size, content-type)
+            gzip.BadGzipFile: If content is not valid gzip
+            UnicodeDecodeError: If content cannot be decoded as UTF-8
+            PermissionError: If file cannot be written due to permissions
+            OSError: On other file system errors
+
+        Examples:
+            >>> import mteb
+            >>> cache = mteb.ResultCache()
+            >>> # Download optimized cached results
+            >>> cache_file = cache._download_cached_results_from_branch()
+            >>> # Use custom output path
+            >>> cache_file = cache._download_cached_results_from_branch(
+            ...     output_path=Path("/tmp/my_cache.json")
+            ... )
+        """
+        if output_path is None:
+            # Default to saving in mteb/leaderboard/__cached_results.json
+            # Get the mteb package directory (parent of this file)
+            mteb_package_dir = Path(__file__).parent
+            output_path = mteb_package_dir / "leaderboard" / "__cached_results.json"
+
+        # Extract repository owner and name from the remote URL
+        # e.g., "https://github.com/embeddings-benchmark/results" -> "embeddings-benchmark/results"
+        repo_path = remote.replace("https://github.com/", "").replace(
+            "http://github.com/", ""
+        )
+
+        url = f"https://raw.githubusercontent.com/{repo_path}/{branch}/{filename}"
+        logger.info(f"Downloading cached results from {url}")
+
+        # Step 1: Download with validation
+        max_size_bytes = max_size_mb * 1024 * 1024
+
+        try:
+            response = requests.get(url, timeout=timeout)
+            response.raise_for_status()
+
+            # Check if this is a Git LFS pointer file
+            content_type = response.headers.get("content-type", "").lower()
+            if (
+                content_type == "text/plain; charset=utf-8"
+                and b"git-lfs" in response.content
+            ):
+                # Try Git LFS media URL instead
+                media_url = f"https://media.githubusercontent.com/media/{repo_path}/{branch}/{filename}"
+                logger.info(f"Detected Git LFS file, trying media URL: {media_url}")
+                response = requests.get(media_url, timeout=timeout)
+                response.raise_for_status()
+                content_type = response.headers.get("content-type", "").lower()
+
+            # Validate content-type header
+            expected_content_types = [
+                "application/gzip",
+                "application/octet-stream",
+                "application/x-gzip",
+            ]
+            if content_type and not any(
+                ct in content_type for ct in expected_content_types
+            ):
+                raise Exception(
+                    f"Unexpected content-type: {content_type}. Expected one of: {expected_content_types}"
+                )
+
+            # Validate file size
+            content_length = len(response.content)
+            if content_length > max_size_bytes:
+                raise ValueError(
+                    f"Downloaded file too large: {content_length} bytes (max: {max_size_bytes})"
+                )
+
+            logger.info(
+                f"HTTP request successful, content length: {content_length} bytes"
+            )
+            content = response.content
+
+        except Exception as e:
+            logger.error(f"Unexpected HTTP error: {type(e).__name__}: {e}")
+            raise e
+
+        # Step 2: Decompress gzip data
+        logger.info("Attempting gzip decompression...")
+
+        try:
+            with gzip.open(io.BytesIO(content), "rt", encoding="utf-8") as gz_file:
+                data = gz_file.read()
+            logger.info(f"Decompression successful, data length: {len(data)} chars")
+
+        except Exception as e:
+            logger.error(f"Unexpected decompression error: {type(e).__name__}: {e}")
+            raise e
+
+        # Step 3: Write to disk
+        logger.info(f"Attempting to write to: {output_path}")
+
+        # Check parent directory exists and is writable
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            output_path.write_text(data, encoding="utf-8")
+            logger.info(
+                f"File write successful, size: {output_path.stat().st_size} bytes"
+            )
+        except Exception as e:
+            logger.error(f"Unexpected file write error: {type(e).__name__}: {e}")
+            raise e
+
+        return output_path
+
     def clear_cache(self) -> None:
         """Clear the local cache directory."""
         if self.cache_path.exists() and self.cache_path.is_dir():
             shutil.rmtree(self.cache_path)
             logger.info(f"Cache directory {self.cache_path} cleared.")
         else:
-            logger.warning(f"Cache directory {self.cache_path} does not exist.")
+            msg = f"Cache directory `{self.cache_path}` does not exist."
+            logger.warning(msg)
+            warnings.warn(msg)
 
     def __repr__(self) -> str:
         return f"ResultCache(cache_path={self.cache_path})"
 
     def get_cache_paths(
         self,
-        models: Sequence[str] | Sequence[ModelMeta] | None = None,
-        tasks: Sequence[str] | Sequence[AbsTask] | None = None,
+        models: Sequence[str] | Iterable[ModelMeta] | None = None,
+        tasks: Sequence[str] | Iterable[AbsTask] | None = None,
         require_model_meta: bool = True,
         include_remote: bool = True,
     ) -> list[Path]:
@@ -279,8 +461,8 @@ class ResultCache:
             A list of paths in the cache directory.
 
         Examples:
-            >>> from mteb.cache import ResultCache
-            >>> cache = ResultCache()
+            >>> import mteb
+            >>> cache = mteb.ResultCache()
             >>>
             >>> # Get all cache paths
             >>> paths = cache.get_cache_paths()
@@ -390,7 +572,7 @@ class ResultCache:
     @staticmethod
     def _filter_paths_by_model_and_revision(
         paths: list[Path],
-        models: Sequence[str] | Sequence[ModelMeta] | None = None,
+        models: Sequence[str] | Iterable[ModelMeta] | None = None,
     ) -> list[Path]:
         """Filter a list of paths by model name and optional revision.
 
@@ -400,8 +582,9 @@ class ResultCache:
         if not models:
             return paths
 
-        if isinstance(models[0], ModelMeta):
-            models = cast(list[ModelMeta], models)
+        first_model = next(iter(models))
+        if isinstance(first_model, ModelMeta):
+            models = cast(Iterable[ModelMeta], models)
             name_and_revision = {
                 (m.model_name_as_path(), m.revision or "no_revision_available")
                 for m in models
@@ -412,13 +595,14 @@ class ResultCache:
                 if (p.parent.parent.name, p.parent.name) in name_and_revision
             ]
 
-        model_names = {m.replace("/", "__").replace(" ", "_") for m in models}
+        str_models = cast(Sequence[str], models)
+        model_names = {m.replace("/", "__").replace(" ", "_") for m in str_models}
         return [p for p in paths if p.parent.parent.name in model_names]
 
     @staticmethod
     def _filter_paths_by_task(
         paths: list[Path],
-        tasks: Sequence[str] | Sequence[AbsTask] | None = None,
+        tasks: Sequence[str] | Iterable[AbsTask] | None = None,
     ) -> list[Path]:
         if tasks is not None:
             task_names = set()
@@ -434,8 +618,8 @@ class ResultCache:
 
     def load_results(
         self,
-        models: Sequence[str] | Sequence[ModelMeta] | None = None,
-        tasks: Sequence[str] | Sequence[AbsTask] | None = None,
+        models: Sequence[str] | Iterable[ModelMeta] | None = None,
+        tasks: Sequence[str] | Iterable[AbsTask] | Benchmark | str | None = None,
         require_model_meta: bool = True,
         include_remote: bool = True,
         validate_and_filter: bool = False,
@@ -445,7 +629,9 @@ class ResultCache:
 
         Args:
             models: A list of model names to load the results for. If None it will load the results for all models.
-            tasks: A list of task names to load the results for. If None it will load the results for all tasks.
+            tasks: A list of task names to load the results for. If str is passed, then benchmark will be loaded.
+                If Benchmark is passed, then all tasks in the benchmark will be loaded.
+                If None it will load the results for all tasks.
             require_model_meta: If True it will ignore results that do not have a model_meta.json file. If false it attempt to
                 extract the model name and revision from the path.
             include_remote: If True, it will include results from the remote repository.
@@ -457,8 +643,8 @@ class ResultCache:
             A BenchmarkResults object containing the results for the specified models and tasks.
 
         Examples:
-            >>> from mteb.cache import ResultCache
-            >>> cache = ResultCache()
+            >>> import mteb
+            >>> cache = mteb.ResultCache()
             >>>
             >>> # Load results for specific models and tasks
             >>> results = cache.load_results(
@@ -467,6 +653,9 @@ class ResultCache:
             ...     require_model_meta=True,
             ... )
         """
+        if isinstance(tasks, str):
+            tasks = mteb.get_benchmark(tasks)
+
         paths = self.get_cache_paths(
             models=models,
             tasks=tasks,
@@ -475,7 +664,7 @@ class ResultCache:
         )
         models_results = defaultdict(list)
 
-        task_names = {}
+        task_names: dict[str, AbsTask | None] = {}
         if tasks is not None:
             for task in tasks:
                 if isinstance(task, AbsTask):
@@ -493,10 +682,12 @@ class ResultCache:
             )
 
             if validate_and_filter:
-                task = task_names[task_result.task_name]
+                task_instance = task_names[task_result.task_name]
                 try:
-                    task_result.validate_and_filter_scores(task=task)
-                except Exception as e:
+                    task_result = task_result.validate_and_filter_scores(
+                        task=task_instance
+                    )
+                except ValidationError as e:
                     logger.info(
                         f"Validation failed for {task_result.task_name} in {model_name} {revision}: {e}"
                     )
@@ -505,7 +696,7 @@ class ResultCache:
             models_results[(model_name, revision)].append(task_result)
 
         # create BenchmarkResults object
-        models_results = [
+        models_results_object = [
             ModelResult(
                 model_name=model_name,
                 model_revision=revision,
@@ -514,8 +705,7 @@ class ResultCache:
             for (model_name, revision), task_results in models_results.items()
         ]
 
-        benchmark_results = BenchmarkResults(
-            model_results=models_results,
+        return BenchmarkResults(
+            model_results=models_results_object,
+            benchmark=tasks if isinstance(tasks, Benchmark) else None,
         )
-
-        return benchmark_results

mteb/cli/_display_tasks.py

@@ -1,4 +1,4 @@
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 
 from mteb.abstasks import AbsTask
 from mteb.benchmarks import Benchmark
@@ -31,7 +31,7 @@ def _display_benchmarks(benchmarks: Sequence[Benchmark]) -> None:
         _display_tasks(benchmark.tasks, name=name)
 
 
-def _display_tasks(task_list: Sequence[AbsTask], name: str | None = None) -> None:
+def _display_tasks(task_list: Iterable[AbsTask], name: str | None = None) -> None:
     from rich.console import Console
 
     console = Console()

mteb/cli/build_cli.py

@@ -1,17 +1,19 @@
 import argparse
 import logging
 import os
+import warnings
 from pathlib import Path
 
 import torch
 from rich.logging import RichHandler
 
 import mteb
+from mteb.abstasks.abstask import AbsTask
 from mteb.cache import ResultCache
+from mteb.cli._display_tasks import _display_benchmarks, _display_tasks
 from mteb.cli.generate_model_card import generate_model_card
 from mteb.evaluate import OverwriteStrategy
-
-from ._display_tasks import _display_benchmarks, _display_tasks
+from mteb.types._encoder_io import EncodeKwargs
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +55,7 @@ def run(args: argparse.Namespace) -> None:
 
     if args.benchmarks:
         benchmarks = mteb.get_benchmarks(names=args.benchmarks)
-        tasks = [t for b in benchmarks for t in b.tasks]
+        tasks = tuple(t for b in benchmarks for t in b.tasks)
     else:
         tasks = mteb.get_tasks(
             categories=args.categories,
@@ -63,21 +65,23 @@ def run(args: argparse.Namespace) -> None:
             eval_splits=args.eval_splits,
         )
 
-    encode_kwargs = {}
+    encode_kwargs: EncodeKwargs = {}
     if args.batch_size is not None:
         encode_kwargs["batch_size"] = args.batch_size
 
     overwrite_strategy = args.overwrite_strategy
     if args.overwrite:
-        logger.warning(
-            "`--overwrite` is deprecated, please use `--overwrite-strategy 'always'` instead."
+        warnings.warn(
+            "`--overwrite` is deprecated, please use `--overwrite-strategy 'always'` instead.",
+            DeprecationWarning,
         )
         overwrite_strategy = OverwriteStrategy.ALWAYS.value
 
     prediction_folder = args.prediction_folder
     if args.save_predictions:
-        logger.warning(
-            "`--save_predictions` is deprecated, please use `--prediction-folder` instead."
+        warnings.warn(
+            "`--save_predictions` is deprecated, please use `--prediction-folder` instead.",
+            DeprecationWarning,
         )
         prediction_folder = args.output_folder
 
@@ -279,23 +283,25 @@ def _create_meta(args: argparse.Namespace) -> None:
         from_existing = Path(from_existing)
 
     if output_path.exists() and overwrite:
-        logger.warning("Output path already exists, overwriting.")
+        msg = "Output path already exists, overwriting."
+        logger.warning(msg)
+        warnings.warn(msg)
     elif output_path.exists():
         raise FileExistsError(
             "Output path already exists, use --overwrite to overwrite."
         )
 
-    tasks = []
+    benchmarks = None
+    tasks: list[AbsTask] = []
     if tasks_names is not None:
-        tasks = mteb.get_tasks(tasks_names)
+        tasks = list(mteb.get_tasks(tasks_names))
     if benchmarks is not None:
         benchmarks = mteb.get_benchmarks(benchmarks)
-        for benchmark in benchmarks:
-            tasks.extend(benchmark.tasks)
 
     generate_model_card(
         model_name,
-        tasks if len(tasks) > 0 else None,
+        tasks,
+        benchmarks,
         existing_model_card_id_or_path=from_existing,
         results_cache=ResultCache(results_folder),
         output_path=output_path,
@@ -356,6 +362,95 @@ def _add_create_meta_parser(subparsers) -> None:
     parser.set_defaults(func=_create_meta)
 
 
+def _add_leaderboard_parser(subparsers) -> None:
+    parser = subparsers.add_parser("leaderboard", help="Launch the MTEB leaderboard")
+
+    parser.add_argument(
+        "--cache-path",
+        type=str,
+        help="Path to the cache folder containing model results",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="0.0.0.0",
+        help="Host to run the leaderboard server on",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=7860,
+        help="Port to run the leaderboard server on",
+    )
+    parser.add_argument(
+        "--share",
+        action="store_true",
+        default=False,
+        help="Create a public URL for the leaderboard",
+    )
+
+    parser.set_defaults(func=_leaderboard)
+
+
+def _leaderboard(args: argparse.Namespace) -> None:
+    """Launch the MTEB leaderboard with specified cache path."""
+    # Import leaderboard module only when needed to avoid requiring leaderboard dependencies
+    # for other CLI commands
+    try:
+        import gradio as gr
+
+        from mteb.leaderboard import get_leaderboard_app
+    except ImportError as e:
+        raise ImportError(
+            "Seems like some dependencies are not installed. "
+            + "You can likely install these using: `pip install mteb[leaderboard]`. "
+            + f"{e}"
+        )
+
+    cache_path = args.cache_path
+
+    if cache_path:
+        logger.info(f"Using cache path: {cache_path}")
+        cache = ResultCache(cache_path)
+    else:
+        cache = ResultCache()
+        logger.info(f"Using default cache path: {cache.cache_path}")
+
+    app = get_leaderboard_app(cache)
+
+    logger.info(f"Starting leaderboard on {args.host}:{args.port}")
+    if args.share:
+        logger.info("Creating public URL...")
+
+    logging.getLogger("mteb.load_results.task_results").setLevel(
+        logging.ERROR
+    )  # Warnings related to task split
+    logging.getLogger("mteb.model_meta").setLevel(
+        logging.ERROR
+    )  # Warning related to model metadata (fetch_from_hf=False)
+    logging.getLogger("mteb.load_results.benchmark_results").setLevel(
+        logging.ERROR
+    )  # Warning related to model metadata (fetch_from_hf=False)
+    warnings.filterwarnings("ignore", message="Couldn't get scores for .* due to .*")
+
+    # Head content for Tailwind CSS
+    head = """
+    <link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
+    """
+
+    app.launch(
+        server_name=args.host,
+        server_port=args.port,
+        share=args.share,
+        theme=gr.themes.Soft(
+            font=[gr.themes.GoogleFont("Roboto Mono"), "Arial", "sans-serif"],
+        ),
+        head=head,
+    )
+
+
 def build_cli() -> argparse.ArgumentParser:
     """Builds the argument parser for the MTEB CLI.
 
@@ -375,6 +470,7 @@ def build_cli() -> argparse.ArgumentParser:
     _add_available_tasks_parser(subparsers)
     _add_available_benchmarks_parser(subparsers)
     _add_create_meta_parser(subparsers)
+    _add_leaderboard_parser(subparsers)
 
     return parser