mteb 2.3.9__py3-none-any.whl → 2.3.11__py3-none-any.whl

mteb/_evaluators/image/imagetext_pairclassification_evaluator.py

@@ -1,10 +1,11 @@
+from __future__ import annotations
+
 import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import torch
 import torch.nn.functional as F
 from datasets import Dataset
-from PIL.Image import Image
 from torch.utils.data import DataLoader
 
 from mteb._create_dataloaders import (
@@ -15,6 +16,10 @@ from mteb._requires_package import requires_image_dependencies
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models.models_protocols import EncoderProtocol
 
+if TYPE_CHECKING:
+    from PIL.Image import Image
+
+
 logger = logging.getLogger(__name__)
 
 

mteb/abstasks/_statistics_calculation.py

@@ -1,7 +1,8 @@
+from __future__ import annotations
+
 import hashlib
 from collections import Counter
-
-from PIL import Image
+from typing import TYPE_CHECKING
 
 from mteb.types import TopRankedDocumentsType
 from mteb.types.statistics import (
@@ -13,6 +14,9 @@ from mteb.types.statistics import (
     TopRankedStatistics,
 )
 
+if TYPE_CHECKING:
+    from PIL import Image
+
 
 def calculate_text_statistics(texts: list[str]) -> TextStatistics:
     """Calculate descriptive statistics for a list of texts.

mteb/abstasks/classification.py

@@ -5,7 +5,6 @@ from typing import Any, TypedDict
 
 import numpy as np
 from datasets import Dataset, DatasetDict
-from PIL import ImageFile
 from sklearn.linear_model import LogisticRegression
 from sklearn.metrics import (
     accuracy_score,
@@ -32,7 +31,6 @@ from ._statistics_calculation import (
 )
 from .abstask import AbsTask
 
-ImageFile.LOAD_TRUNCATED_IMAGES = True
 logger = logging.getLogger(__name__)
 
 

mteb/benchmarks/_create_table.py

@@ -1,5 +1,6 @@
 import re
 from collections import defaultdict
+from typing import Literal
 
 import numpy as np
 import pandas as pd
@@ -241,6 +242,65 @@ def _create_per_task_table_from_benchmark_results(
     return per_task
 
 
+def _create_per_language_table_from_benchmark_results(
+    benchmark_results: BenchmarkResults,
+    language_view: list[str] | Literal["all"],
+) -> pd.DataFrame:
+    """Create per-language table from BenchmarkResults.
+
+    Returns a DataFrame with one row per model and one column per language.
+
+    Args:
+        benchmark_results: BenchmarkResults object containing model results
+        language_view: List of languages to include in the per-language table, or "all" for all languages present in the results
+    Returns:
+        DataFrame with per-language scores, ready for styling in the leaderboard
+    """
+    if language_view != "all" and not isinstance(language_view, list):
+        raise ValueError("language_view must be a list of languages or 'all'")
+
+    data = benchmark_results.to_dataframe(aggregation_level="language", format="long")
+
+    if data.empty:
+        no_results_frame = pd.DataFrame(
+            {"No results": ["You can try relaxing your criteria"]}
+        )
+        return no_results_frame
+
+    if language_view != "all":
+        data = data[data["language"].isin(language_view)]
+
+    per_language = data.pivot_table(
+        index="model_name", columns="language", values="score", aggfunc="mean"
+    )
+
+    to_remove = per_language.isna().all(axis="columns")
+    if to_remove.all():
+        no_results_frame = pd.DataFrame(
+            {"No results": ["You can try relaxing your criteria"]}
+        )
+        return no_results_frame
+
+    models_to_remove = list(per_language[to_remove].index)
+    per_language = per_language.drop(models_to_remove, axis=0)
+
+    per_language["borda_rank"] = _get_borda_rank(per_language)
+    per_language = per_language.sort_values("borda_rank", ascending=True)
+    per_language = per_language.drop(columns=["borda_rank"])
+    per_language = per_language.reset_index()
+
+    per_language["model_name"] = per_language["model_name"].map(
+        lambda name: name.split("/")[-1]
+    )
+    per_language = per_language.rename(
+        columns={
+            "model_name": "Model",
+        }
+    )
+
+    return per_language
+
+
 def _create_summary_table_mean_public_private(
     benchmark_results: BenchmarkResults,
 ) -> pd.DataFrame:

mteb/benchmarks/benchmark.py

@@ -1,10 +1,11 @@
 from collections.abc import Iterable, Sequence
-from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Literal
 
 import pandas as pd
 
 from mteb.benchmarks._create_table import (
+    _create_per_language_table_from_benchmark_results,
     _create_per_task_table_from_benchmark_results,
     _create_summary_table_from_benchmark_results,
     _create_summary_table_mean_public_private,
@@ -50,6 +51,7 @@ class Benchmark:
     display_on_leaderboard: bool = True
     icon: str | None = None
     display_name: str | None = None
+    language_view: list[str] | Literal["all"] = field(default_factory=list)
 
     def __iter__(self) -> Iterable["AbsTask"]:
         return iter(self.tasks)
@@ -80,6 +82,28 @@ class Benchmark:
         """
         return _create_per_task_table_from_benchmark_results(benchmark_results)
 
+    def _create_per_language_table(
+        self, benchmark_results: BenchmarkResults
+    ) -> pd.DataFrame:
+        """Create per-language table. Called by the leaderboard app.
+
+        Returns:
+            A pandas DataFrame representing the per-language results.
+        """
+        if self.language_view == "all" or len(self.language_view) > 0:
+            return _create_per_language_table_from_benchmark_results(
+                benchmark_results, self.language_view
+            )
+        else:
+            no_results_frame = pd.DataFrame(
+                {
+                    "No results": [
+                        "The per-language table is not available for this benchmark."
+                    ]
+                }
+            )
+            return no_results_frame
+
 
 class RtebBenchmark(Benchmark):
     """Wrapper for RTEB benchmark."""

mteb/benchmarks/benchmarks/benchmarks.py

@@ -471,6 +471,7 @@ SEB = Benchmark(
     name="MTEB(Scandinavian, v1)",
     display_name="Scandinavian",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/dk.svg",
+    language_view=["dan-Latn", "swe-Latn", "nno-Latn", "nob-Latn"],
     tasks=get_tasks(
         tasks=[
             # Bitext
@@ -953,6 +954,28 @@ MTEB_multilingual_v1 = Benchmark(
 MTEB_multilingual_v2 = Benchmark(
     name="MTEB(Multilingual, v2)",
     display_name="Multilingual",
+    language_view=[
+        "eng-Latn",  # English
+        "zho-Hans",  # Chinese (Simplified)
+        "hin-Deva",  # Hindi
+        "spa-Latn",  # Spanish
+        "fra-Latn",  # French
+        "ara-Arab",  # Arabic
+        "ben-Beng",  # Bengali
+        "rus-Cyrl",  # Russian
+        "por-Latn",  # Portuguese
+        "urd-Arab",  # Urdu
+        "ind-Latn",  # Indonesian
+        "deu-Latn",  # German
+        "jpn-Jpan",  # Japanese
+        "swa-Latn",  # Swahili
+        "mar-Deva",  # Marathi
+        "tel-Telu",  # Telugu
+        "tur-Latn",  # Turkish
+        "tam-Taml",  # Tamil
+        "vie-Latn",  # Vietnamese
+        "kor-Hang",  # Korean
+    ],
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-globe.svg",
     tasks=mteb_multilingual_tasks,
     description="A large-scale multilingual expansion of MTEB, driven mainly by highly-curated community contributions covering 250+ languages. ",
@@ -2283,6 +2306,14 @@ VIDORE_V2 = Benchmark(
 VIDORE_V3 = VidoreBenchmark(
     name="ViDoRe(v3)",
     display_name="ViDoRe V3",
+    language_view=[
+        "deu-Latn",
+        "eng-Latn",
+        "fra-Latn",
+        "ita-Latn",
+        "por-Latn",
+        "spa-Latn",
+    ],
     icon="https://cdn-uploads.huggingface.co/production/uploads/66e16a677c2eb2da5109fb5c/x99xqw__fl2UaPbiIdC_f.png",
     tasks=get_tasks(
         tasks=[

mteb/leaderboard/app.py

@@ -24,6 +24,7 @@ from mteb.leaderboard.benchmark_selector import (
 )
 from mteb.leaderboard.figures import _performance_size_plot, _radar_chart
 from mteb.leaderboard.table import (
+    apply_per_language_styling_from_benchmark,
     apply_per_task_styling_from_benchmark,
     apply_summary_styling_from_benchmark,
 )
@@ -361,6 +362,13 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
     per_task_table = apply_per_task_styling_from_benchmark(
         default_benchmark, filtered_benchmark_results
     )
+    per_language_table = apply_per_language_styling_from_benchmark(
+        default_benchmark,
+        filtered_benchmark_results,
+    )
+
+    # Check if this benchmark displays per-language results
+    display_language_table = len(default_benchmark.language_view) > 0
 
     lang_select = gr.CheckboxGroup(
         sorted(default_results.languages),
@@ -554,6 +562,16 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
             download_per_task.click(
                 _download_table, inputs=[per_task_table], outputs=[download_per_task]
             )
+        with gr.Tab(
+            "Performance per language", visible=display_language_table
+        ) as language_tab:
+            per_language_table.render()
+            download_per_language = gr.DownloadButton("Download Table")
+            download_per_language.click(
+                _download_table,
+                inputs=[per_language_table],
+                outputs=[download_per_language],
+            )
         with gr.Tab("Task information"):
             task_info_table = gr.DataFrame(_update_task_info, inputs=[task_select])  # noqa: F841
 
@@ -879,9 +897,18 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
             per_task = apply_per_task_styling_from_benchmark(
                 benchmark, filtered_benchmark_results
             )
+            per_language = apply_per_language_styling_from_benchmark(
+                benchmark,
+                filtered_benchmark_results,
+            )
             elapsed = time.time() - start_time
             logger.debug(f"update_tables callback: {elapsed}s")
-            return summary, per_task
+            return (
+                summary,
+                per_task,
+                per_language,
+                gr.update(visible=len(benchmark.language_view) > 0),
+            )
 
         # Only update tables when models change, not when scores/tasks change directly
         # This avoids redundant updates since scores/tasks changes trigger update_models
@@ -890,7 +917,12 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
             item.change(
                 update_tables,
                 inputs=[scores, task_select, models, benchmark_select],
-                outputs=[summary_table, per_task_table],
+                outputs=[
+                    summary_table,
+                    per_task_table,
+                    per_language_table,
+                    language_tab,
+                ],
             )
 
         gr.Markdown(ACKNOWLEDGEMENT, elem_id="ack_markdown")

mteb/leaderboard/table.py

@@ -120,6 +120,31 @@ def apply_per_task_styling_from_benchmark(
     return _apply_per_task_table_styling(per_task_df)
 
 
+def apply_per_language_styling_from_benchmark(
+    benchmark_instance: Benchmark, benchmark_results: BenchmarkResults
+) -> gr.DataFrame:
+    """Apply styling to per-language table created by the benchmark instance's _create_per_language_table method.
+
+    This supports polymorphism - different benchmark classes can have different table generation logic.
+
+    Args:
+        benchmark_instance: The benchmark instance
+        benchmark_results: BenchmarkResults object containing model results (may be pre-filtered)
+
+    Returns:
+        Styled gr.DataFrame ready for display in the leaderboard
+    """
+    # Use the instance method to support polymorphism
+    per_language_df = benchmark_instance._create_per_language_table(benchmark_results)
+
+    # If it's a no-results DataFrame, return it as-is
+    if "No results" in per_language_df.columns:
+        return gr.DataFrame(per_language_df)
+
+    # Apply the styling
+    return _apply_per_language_table_styling(per_language_df)
+
+
 def _style_number_of_parameters(num_params: float) -> str:
     """Anything bigger than 1B is shown in billions with 1 decimal (e.g. 1.712 > 1.7) while anything smaller as 0.xxx B (e.g. 0.345 remains 0.345)"""
     if num_params >= 1:
@@ -237,10 +262,47 @@ def _apply_per_task_table_styling(per_task: pd.DataFrame) -> gr.DataFrame:
         "{:.2f}", subset=task_score_columns, na_rep=""
     ).highlight_max(subset=task_score_columns, props="font-weight: bold")
 
+    # setting task name column width to 250px
+    column_widths = _get_column_widths(per_task_style.data)
+    if len(column_widths) > 0:
+        column_widths[0] = "250px"
+
     return gr.DataFrame(
         per_task_style,
         interactive=False,
         pinned_columns=1,
+        column_widths=column_widths,
+        buttons=["copy", "fullscreen"],
+        show_search="filter",
+    )
+
+
+def _apply_per_language_table_styling(per_language: pd.DataFrame) -> gr.DataFrame:
+    """Apply styling to a raw per-task DataFrame
+
+    Returns:
+        Styled gr.DataFrame ready for display in the leaderboard
+    """
+    language_score_columns = per_language.select_dtypes("number").columns
+    per_language[language_score_columns] *= 100
+
+    if len(per_language.columns) > 100:  # Avoid gradio error on very wide tables
+        per_language_style = per_language.round(2)
+    else:
+        per_language_style = per_language.style.format(
+            "{:.2f}", subset=language_score_columns, na_rep=""
+        ).highlight_max(subset=language_score_columns, props="font-weight: bold")
+
+    # setting task name column width to 250px
+    column_widths = _get_column_widths(per_language_style.data)
+    if len(column_widths) > 0:
+        column_widths[0] = "250px"
+
+    return gr.DataFrame(
+        per_language_style,
+        interactive=False,
+        pinned_columns=1,
+        column_widths=column_widths,
         buttons=["copy", "fullscreen"],
         show_search="filter",
     )

mteb/models/cache_wrappers/cache_backends/_hash_utils.py

@@ -1,7 +1,5 @@
 import hashlib
 
-from PIL import Image
-
 from mteb.types import BatchedInput
 
 
@@ -11,6 +9,8 @@ def _hash_item(item: BatchedInput) -> str:
         item_hash = hashlib.sha256(item["text"].encode()).hexdigest()
 
     if "image" in item:
+        from PIL import Image
+
         image: Image.Image = item["image"]
         item_hash += hashlib.sha256(image.tobytes()).hexdigest()
 

mteb/models/model_implementations/colpali_models.py

@@ -1,8 +1,9 @@
+from __future__ import annotations
+
 import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
@@ -15,6 +16,9 @@ from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.types import Array, BatchedInput, PromptType
 
+if TYPE_CHECKING:
+    from PIL import Image
+
 logger = logging.getLogger(__name__)
 
 
@@ -89,6 +93,7 @@ class ColPaliEngineWrapper(AbsEncoder):
         **kwargs,
     ):
         import torchvision.transforms.functional as F
+        from PIL import Image
 
         all_embeds = []
 

mteb/models/model_implementations/colqwen_models.py

@@ -2,7 +2,6 @@ import logging
 from typing import Any
 
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
@@ -154,6 +153,7 @@ class ColQwen3Wrapper(AbsEncoder):
         **kwargs: Any,
     ):
         import torchvision.transforms.functional as F
+        from PIL import Image
 
         contains_image = "image" in image_texts_pairs.dataset.features
         contains_text = "text" in image_texts_pairs.dataset.features

mteb/models/model_implementations/gme_v_models.py

@@ -1,9 +1,10 @@
+from __future__ import annotations
+
 import logging
 import math
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 from tqdm.autonotebook import tqdm
 
@@ -12,6 +13,9 @@ from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.types import Array, BatchedInput, PromptType
 
+if TYPE_CHECKING:
+    from PIL import Image
+
 logger = logging.getLogger(__name__)
 
 GME_CITATION = """@misc{zhang2024gme,
@@ -267,9 +271,9 @@ def smart_resize(
     return h_bar, w_bar
 
 
-def fetch_image(
-    image: str | Image.Image, size_factor: int = IMAGE_FACTOR
-) -> Image.Image:
+def fetch_image(image: Image.Image, size_factor: int = IMAGE_FACTOR) -> Image.Image:
+    from PIL import Image
+
     image_obj = None
     if isinstance(image, Image.Image):
         image_obj = image

mteb/models/model_implementations/granite_vision_embedding_models.py

@@ -1,8 +1,9 @@
+from __future__ import annotations
+
 import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
@@ -15,6 +16,9 @@ from mteb.types import Array, BatchedInput, PromptType
 
 logger = logging.getLogger(__name__)
 
+if TYPE_CHECKING:
+    from PIL import Image
+
 
 class GraniteVisionEmbeddingWrapper:
     def __init__(

mteb/models/model_implementations/jasper_models.py

@@ -355,13 +355,13 @@ Jasper_Token_Compression_600M = ModelMeta(
     | qzhou_training_data,
     citation="""
 @misc{zhang2025jaspertokencompression600mtechnicalreport,
-      title={Jasper-Token-Compression-600M Technical Report}, 
+      title={Jasper-Token-Compression-600M Technical Report},
       author={Dun Zhang and Ziyang Zeng and Yudong Zhou and Shuyang Lu},
       year={2025},
       eprint={2511.14405},
       archivePrefix={arXiv},
       primaryClass={cs.IR},
-      url={https://arxiv.org/abs/2511.14405}, 
+      url={https://arxiv.org/abs/2511.14405},
 }
 """,
 )

mteb/models/model_implementations/jina_models.py

@@ -740,7 +740,7 @@ jina_reranker_v3 = ModelMeta(
     training_datasets=JINARerankerV3_TRAINING_DATA,
     adapted_from="Qwen/Qwen3-0.6B",
     citation="""@misc{wang2025jinarerankerv3lateinteractionlistwise,
-      title={jina-reranker-v3: Last but Not Late Interaction for Listwise Document Reranking}, 
+      title={jina-reranker-v3: Last but Not Late Interaction for Listwise Document Reranking},
       author={Feng Wang and Yuqing Li and Han Xiao},
       year={2025},
       eprint={2509.25085},

mteb/models/model_implementations/nomic_models_vision.py

@@ -1,8 +1,9 @@
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
 
 import torch
 import torch.nn.functional as F
-from PIL import Image
 from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
@@ -12,6 +13,9 @@ from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.types import Array, BatchedInput, PromptType
 
+if TYPE_CHECKING:
+    from PIL import Image
+
 NOMIC_EMBED_VISION_CITATION = """@article{nussbaum2024nomicembedvision,
       title={Nomic Embed Vision: Expanding the Latent Space},
       author={Nussbaum, Zach and Duderstadt, Brandon and Mulyar, Andriy},

mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py

@@ -1,7 +1,6 @@
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 
 from mteb.abstasks.task_metadata import TaskMetadata
@@ -9,6 +8,10 @@ from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta
 from mteb.types import Array, BatchedInput, PromptType
 
+if TYPE_CHECKING:
+    pass
+
+
 LLAMA_NEMORETRIEVER_CITATION = """@misc{xu2025llamanemoretrievercolembedtopperforming,
       title={Llama Nemoretriever Colembed: Top-Performing Text-Image Retrieval Model},
       author={Mengyao Xu and Gabriel Moreira and Ronay Ak and Radek Osmulski and Yauhen Babakhin and Zhiding Yu and Benedikt Schifferer and Even Oldridge},
@@ -53,6 +56,7 @@ class LlamaNemoretrieverColembed(AbsEncoder):
         **kwargs,
     ):
         import torchvision.transforms.functional as F
+        from PIL import Image
 
         all_images = []
         if isinstance(images, DataLoader):

mteb/models/model_implementations/pawan_models.py

@@ -0,0 +1,38 @@
+from mteb.models.model_meta import (
+    ModelMeta,
+    ScoringFunction,
+)
+from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
+
+PAWAN_EMBD_CITATION = """@misc{medhi2025pawanembd,
+    title={PawanEmbd-68M: Distilled Embedding Model},
+    author={Medhi, D.},
+    year={2025},
+    url={https://huggingface.co/dmedhi/PawanEmbd-68M}
+}"""
+
+pawan_embd_68m = ModelMeta(
+    loader=sentence_transformers_loader,
+    name="dmedhi/PawanEmbd-68M",
+    languages=["eng-Latn"],
+    open_weights=True,
+    revision="32f295145802bdbd65699ad65fd27d2a5b69a909",
+    release_date="2025-12-08",
+    n_parameters=68_000_000,
+    memory_usage_mb=260,
+    embed_dim=768,
+    license="apache-2.0",
+    max_tokens=512,
+    reference="https://huggingface.co/dmedhi/PawanEmbd-68M",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=["Sentence Transformers", "PyTorch"],
+    adapted_from="ibm-granite/granite-embedding-278m-multilingual",
+    superseded_by=None,
+    public_training_code=None,
+    public_training_data=None,
+    use_instructions=False,
+    training_datasets={
+        "AllNLI",
+    },
+    citation=PAWAN_EMBD_CITATION,
+)

mteb/models/model_implementations/pylate_models.py

@@ -328,13 +328,10 @@ class MultiVectorModel(AbsEncoder, PylateSearchEncoder):
             inputs,
             prompt_name=prompt_name,
             is_query=prompt_type == PromptType.query,
-            convert_to_tensor=True,
             **kwargs,
         )
 
-        # encode returns a list of tensors shaped (x, token_dim), pad to uniform length
-        pred = torch.nn.utils.rnn.pad_sequence(pred, batch_first=True, padding_value=0)
-        return pred.cpu().numpy()
+        return pred
 
 
 colbert_v2 = ModelMeta(

mteb/models/model_implementations/random_baseline.py

@@ -1,9 +1,10 @@
+from __future__ import annotations
+
 import hashlib
-from typing import Any, Literal
+from typing import TYPE_CHECKING, Any, Literal
 
 import numpy as np
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 
 from mteb.abstasks.task_metadata import TaskMetadata
@@ -14,6 +15,9 @@ from mteb.similarity_functions import (
 )
 from mteb.types._encoder_io import Array, BatchedInput, PromptType
 
+if TYPE_CHECKING:
+    from PIL import Image
+
 
 def _string_to_vector(text: str | None, size: int) -> np.ndarray:
     """Generate a deterministic random vector based on a string.

mteb/models/model_implementations/seed_1_6_embedding_models.py

@@ -1,14 +1,15 @@
+from __future__ import annotations
+
 import base64
 import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from io import BytesIO
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import requests
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 
 from mteb._requires_package import requires_package
@@ -19,6 +20,10 @@ from mteb.models.model_implementations.nvidia_models import nvidia_training_data
 from mteb.models.model_meta import ModelMeta
 from mteb.types import Array, BatchedInput, PromptType
 
+if TYPE_CHECKING:
+    from PIL import Image
+
+
 logger = logging.getLogger(__name__)
 
 

mteb/models/model_implementations/voyage_v.py

@@ -1,8 +1,9 @@
+from __future__ import annotations
+
 import logging
-from typing import Any, Literal
+from typing import TYPE_CHECKING, Any, Literal
 
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 from tqdm.auto import tqdm
 
@@ -12,6 +13,9 @@ from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.types import Array, BatchedInput, PromptType
 
+if TYPE_CHECKING:
+    from PIL import Image
+
 
 def _downsample_image(
     image: Image.Image, max_pixels: int = 16000000, target_longest_side: int = 4000

mteb/results/benchmark_results.py

@@ -296,7 +296,7 @@ class BenchmarkResults(BaseModel):
 
     def to_dataframe(
         self,
-        aggregation_level: Literal["subset", "split", "task"] = "task",
+        aggregation_level: Literal["subset", "split", "task", "language"] = "task",
         aggregation_fn: Callable[[list[Score]], Any] | None = None,
         include_model_revision: bool = False,
         format: Literal["wide", "long"] = "wide",
@@ -321,6 +321,7 @@ class BenchmarkResults(BaseModel):
                 - "subset"/None: No aggregation will be done. The DataFrame will have one row per model, task, split and subset.
                 - "split": Aggregates the scores by split. The DataFrame will have one row per model, task and split.
                 - "task": Aggregates the scores by task. The DataFrame will have one row per model and task.
+                - "language": Aggregates the scores by language. The DataFrame will have one row per model and language.
             aggregation_fn: The function to use for aggregation. If None, the mean will be used.
             include_model_revision: If True, the model revision will be included in the DataFrame. If False, it will be excluded.
                 If there are multiple revisions for the same model, they will be joined using the `join_revisions` method.

mteb/results/model_result.py

@@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)
 def _aggregate_and_pivot(
     df: pd.DataFrame,
     columns: list[str],
-    aggregation_level: Literal["subset", "split", "task"],
+    aggregation_level: Literal["subset", "split", "task", "language"],
     format: Literal["wide", "long"],
     aggregation_fn: Callable[[list[Score]], Any] | None,
 ) -> pd.DataFrame:
@@ -43,6 +43,12 @@ def _aggregate_and_pivot(
     elif aggregation_level == "task":
         index_columns = ["task_name"]
 
+    elif aggregation_level == "language":
+        index_columns = ["language"]
+        df = df.explode("language").reset_index(
+            drop=True
+        )  # each language in its own row before aggregation
+
     # perform aggregation
     if aggregation_fn is None:
         aggregation_fn = np.mean
@@ -227,7 +233,7 @@ class ModelResult(BaseModel):
                     )
             return entries
 
-    def _get_score_for_table(self) -> list[dict[str, str | float]]:
+    def _get_score_for_table(self) -> list[dict[str, str | float | list[str]]]:
         scores_data = []
         model_name = self.model_name
         for task_result in self.task_results:
@@ -239,10 +245,10 @@ class ModelResult(BaseModel):
                         "model_revision": self.model_revision,
                         "task_name": task_name,
                         "split": split,
+                        "language": score_item.get("languages", ["Unknown"]),
                         "subset": score_item.get("hf_subset", "default"),
                         "score": score_item.get("main_score", None),
                     }
-
                     scores_data.append(row)
 
         return scores_data

mteb/types/_encoder_io.py

@@ -1,13 +1,18 @@
+from __future__ import annotations
+
 from collections.abc import Mapping
 from enum import Enum
-from typing import TypedDict
+from typing import TYPE_CHECKING, TypedDict
 
 import numpy as np
 import torch
 from datasets import Dataset
-from PIL import Image
 from typing_extensions import NotRequired
 
+if TYPE_CHECKING:
+    from PIL import Image
+
+
 # --- Output types ---
 Array = np.ndarray | torch.Tensor
 """General array type, can be a numpy array or a torch tensor."""

{mteb-2.3.9.dist-info → mteb-2.3.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.3.9
+Version: 2.3.11
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>
@@ -34,6 +34,7 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: polars>=0.20.22
 Provides-Extra: image
 Requires-Dist: torchvision>0.2.1; extra == "image"
+Requires-Dist: transformers[torch-vision,vision]; extra == "image"
 Provides-Extra: codecarbon
 Requires-Dist: codecarbon<3.0.0,>=2.0.0; extra == "codecarbon"
 Provides-Extra: leaderboard

{mteb-2.3.9.dist-info → mteb-2.3.11.dist-info}/RECORD

@@ -24,17 +24,17 @@ mteb/_evaluators/retrieval_metrics.py,sha256=we0damQCJrdaRUD6JlU2MM7Ls9xERP_OBS5
 mteb/_evaluators/sklearn_evaluator.py,sha256=f9SgBbvgCrkltdTebQTixT7KmIagGkjQ_cNnKuHTb3w,3772
 mteb/_evaluators/zeroshot_classification_evaluator.py,sha256=dQq6g9my-0xn_0fLJXSnhN9Qu6PuJtWCKGIDrlkeyJk,2282
 mteb/_evaluators/image/__init__.py,sha256=CsQd7OMkeV2Phun7paPWjayZ5qRnvj8H0TYBFeqMxag,148
-mteb/_evaluators/image/imagetext_pairclassification_evaluator.py,sha256=RSJxEIVvOl_Za07fk2jfXE-7QOwKsLOhIyGhZtLVABs,4808
+mteb/_evaluators/image/imagetext_pairclassification_evaluator.py,sha256=lVizL_11s0yFAZzuGqv-wtkBbMaK7cArD1eUkxwG4uU,4883
 mteb/_evaluators/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mteb/_evaluators/text/bitext_mining_evaluator.py,sha256=XS7AVml5-BpQWtG1XFHf6fx8VMVPRwibg-9si4b-A_U,6308
 mteb/_evaluators/text/summarization_evaluator.py,sha256=l0AwjVO594mtzPV9Kcqf_xtHHpkx6uhDJ61KnolcVAo,10461
 mteb/abstasks/__init__.py,sha256=1iAwpYTWX7U-goak2KMmacPFCzxPchLQAmZ_uI0t-p0,1130
-mteb/abstasks/_statistics_calculation.py,sha256=gf_BhkXbCdwT1w4xza3op6FmjFsZIsNjvyUMCgRyTrg,5578
+mteb/abstasks/_statistics_calculation.py,sha256=UP2H2Cy8yqwtqeimTWfe4unmZ4iyyr5qiBNZzzFjy9o,5669
 mteb/abstasks/_stratification.py,sha256=zfwkIVmD7Aq7mR2Yt8jTeW1j5ZVV7CIweW842VzcfXc,14364
 mteb/abstasks/abstask.py,sha256=nZwiY_5d0VVtUrlCATngpFLG3JAovO5AvmD0nkkWsLE,25118
 mteb/abstasks/aggregate_task_metadata.py,sha256=vzt1z2wDl0sXD7ErZFwKojYwmFUBPAnGlXLuqLA_-6Q,5992
 mteb/abstasks/aggregated_task.py,sha256=puY6-EAqbL5ehKvFHTMriIdy3rAuqqYHF3ezog1eYxw,6671
-mteb/abstasks/classification.py,sha256=Es9pmRdjv6xbc-KnGqVdO6dR1cc7yAHhekCZES7n5o8,13441
+mteb/abstasks/classification.py,sha256=k_wrM1rq2XcVEK97RpU_uEcqhiWWbV7sm3B0dtvP5yY,13376
 mteb/abstasks/clustering.py,sha256=4KcaU8_sNLmLvMhwDpNmcY2nD3BNyx_LcM-ddSv-wtY,14410
 mteb/abstasks/clustering_legacy.py,sha256=HZY8zgBgqqs5urF_to9wzqm3MnjFivs59hU6P3NrzcI,8684
 mteb/abstasks/dataset_card_template.md,sha256=aD6l8qc3_jxwoIGJNYLzse-jpRa8hu92AxpnUtNgges,5122
@@ -56,11 +56,11 @@ mteb/abstasks/text/bitext_mining.py,sha256=8m86XHJ3TxguC9itxZRq2Bt_p0NYojojS2Btk
 mteb/abstasks/text/reranking.py,sha256=rfRGRBeSjZLgkh8pneMgRm-vd9NHr5jSFH92YfOHfmU,7776
 mteb/abstasks/text/summarization.py,sha256=KYEb8gh4JjpSsrvGUmQ2VlrVdzzVxIWcitXOJUaHhO4,6954
 mteb/benchmarks/__init__.py,sha256=MQEVeli-zLaJ7Xg0z7RhXQwsdmm7Ht_W2Ln0rZo1Szc,225
-mteb/benchmarks/_create_table.py,sha256=OAiR44ynJ2fMzoBmVITQtOTYQzxIu9KUdS_HzlBlAck,20195
-mteb/benchmarks/benchmark.py,sha256=70RlMyyg_wkWTlU_IbfLl-KaqRWXGCKTd8fWe9X-AQE,4173
+mteb/benchmarks/_create_table.py,sha256=b2RqGqi0ZonKbHecEcZiF4pkfE96smFRIzxOI82ETA8,22304
+mteb/benchmarks/benchmark.py,sha256=UEllUtZQ0L10SNnxRyKbiv4wLCMcNF2nUPhBDKY3nz8,5097
 mteb/benchmarks/get_benchmark.py,sha256=-n_O-gitRKZi48gJKNgGuI36hsP7yLVSiwulnMHN7Gw,3935
 mteb/benchmarks/benchmarks/__init__.py,sha256=Ig5dSFunzI-F-OamruuKJVSstbG3xQNkXCxRY3Bj_Ck,2180
-mteb/benchmarks/benchmarks/benchmarks.py,sha256=vWX6QZgqF9iKAE1tIQwaXw9f8q_WiBtdgo8yj4_CHFI,94767
+mteb/benchmarks/benchmarks/benchmarks.py,sha256=mZQ56KBQwnBj2qLSQFOv39Av0HBNpH9HXYsDoFmqvu4,95640
 mteb/benchmarks/benchmarks/rteb_benchmarks.py,sha256=QnCSrTTaBfcRlAQp2Nu81tgv1idMXqiM16Fp2zKJ5Ys,10607
 mteb/cli/__init__.py,sha256=v-csUr3eUZElIvrGB6QGtaIdndDfNWEe9oZchsGsJpg,64
 mteb/cli/_display_tasks.py,sha256=7A06dT9sSoTz6shyMvskPxuc5eHY_H7PGPlROzMP0yw,2196
@@ -1430,10 +1430,10 @@ mteb/languages/language_family.json,sha256=OUGcHeOIPcZPb2FWmYLhxTS0JxjK5y3Fo6x0P
 mteb/languages/language_scripts.py,sha256=5wix9HTYolNIpTiS5oXf2pGJyL7ftdGKs_m432w81V8,3998
 mteb/languages/programming_languages.py,sha256=zxAakT3OSUnAuTnQ34VyeFIECnNXMlleZmAake6jsZE,211
 mteb/leaderboard/__init__.py,sha256=991roXmtRwEQysV-37hWEzWpkvPgMCGRqZTHR-hm2io,88
-mteb/leaderboard/app.py,sha256=29MxFLKEVT-roULHG5boHmsQVhld1rDGNS94r7MWlz8,33118
+mteb/leaderboard/app.py,sha256=-sBAkZ9JTr9czhsYEbSm92MfTmB8BOQ17WDkQ1dsP90,34282
 mteb/leaderboard/benchmark_selector.py,sha256=qd-2L20RQ4ACke01UlytkhZok1dkWgfUlXzfET52kGc,7956
 mteb/leaderboard/figures.py,sha256=mPO0go_23QEhAm1RJdLiBxPFCoUiA74_ztyl6yimc7k,7553
-mteb/leaderboard/table.py,sha256=6SnrYC5GcBlvVSO6vOk6ObuqtoveBLv3JUuXqdKueG8,8333
+mteb/leaderboard/table.py,sha256=NxXAUkQRWtxjJwfIiO9yvdvw9do3ogzqmAn6az01SSc,10609
 mteb/leaderboard/text_segments.py,sha256=iMIkS04QQjPbT-SkU0x6fOcS8xRbUYevryu9HydipKM,6570
 mteb/models/__init__.py,sha256=ABTuoqiBjBtBWW3LYY7ItBHdylR6jWoy06HH0g6j6fU,910
 mteb/models/abs_encoder.py,sha256=m0JkRfRPMYadDgBR9eozRloI31ZSWkSzDFINpwbfLZk,16533
@@ -1447,7 +1447,7 @@ mteb/models/cache_wrappers/__init__.py,sha256=1w1TnMwulWJSzNkLXjbh5MY3sqgHWc6vUn
 mteb/models/cache_wrappers/cache_backend_protocol.py,sha256=TR7kD7KbN1J4piszIecpegtLZYGy7sRHZt3SDWlImKk,1665
 mteb/models/cache_wrappers/cache_wrapper.py,sha256=KLDeOCe_ndQshbZa5ep2u3jovsl--tfpQzvt9EXyxCA,6589
 mteb/models/cache_wrappers/cache_backends/__init__.py,sha256=hN2Tq7cpTxoOYSCJ1Wnpvb8dEm-kQLfCCahT1N9Bacw,123
-mteb/models/cache_wrappers/cache_backends/_hash_utils.py,sha256=WeQjXlNUpGpb-CbaWpu_EtezML7zoDYfbwIo0Jswvsc,480
+mteb/models/cache_wrappers/cache_backends/_hash_utils.py,sha256=zAp7BDuYyGETn2kX58uk8_tn1G2B7bgcsItDDxgyn-w,488
 mteb/models/cache_wrappers/cache_backends/faiss_cache.py,sha256=i9IfaCv1-_BvVokXFW1UZ9hMLCuM6rZ0tI-ZesoBkt4,3734
 mteb/models/cache_wrappers/cache_backends/numpy_cache.py,sha256=GyTVC5DLph3EeRnDMO1EEQzBDoOgk2J1hPqpl07lefM,7442
 mteb/models/model_implementations/__init__.py,sha256=BZDdde6ajKv-yroy9mqE2YS3Hw1KBdKoxBPg8aPTZEs,1164
@@ -1471,8 +1471,8 @@ mteb/models/model_implementations/codefuse_models.py,sha256=19Y-d_qetVU64quzEvuU
 mteb/models/model_implementations/codesage_models.py,sha256=D4CdISGyv5f2GMYq4_efgm5qNq80SWAX5R2u5mjEiXM,2998
 mteb/models/model_implementations/cohere_models.py,sha256=OWFClVAN4phjBoxfGGDyGDmzMu-t2VrjCGFyAIWmz4w,13832
 mteb/models/model_implementations/cohere_v.py,sha256=K6VEw1NkyM2PuMd18kHE6aqPrcByYSwEmAKjvLods_w,15760
-mteb/models/model_implementations/colpali_models.py,sha256=7PJ0SshVXasyncTfZRFIf_ZWzbqxJhhzNKAoGLhNktw,9004
-mteb/models/model_implementations/colqwen_models.py,sha256=6upaxe19V8j5Ayu03Dgj5jPtC8SJBCITK_RionJRMSE,15545
+mteb/models/model_implementations/colpali_models.py,sha256=l-0A3J5rt1bhhTKFPQ3Ti0qvWf2qXYkiv3j1si04R8I,9108
+mteb/models/model_implementations/colqwen_models.py,sha256=wxR3sqyzObuXMlm1QLoFopJK3ZpQTzd3ZB5IrkzPfZk,15553
 mteb/models/model_implementations/colsmol_models.py,sha256=O2M7Ksydh94M_Iax4KytHb-wOL18N0BIYLKSsLF8BFs,2967
 mteb/models/model_implementations/conan_models.py,sha256=G-s7xo9VtNX-f7lWKtYVGHHiMMN0Xp44PlNIp7E0LAo,6502
 mteb/models/model_implementations/dino_models.py,sha256=QFgaFHR5YKrylqJGSljXCBn2W7qHhmF6KdXkvHrQNEI,16380
@@ -1487,18 +1487,18 @@ mteb/models/model_implementations/evaclip_models.py,sha256=cPMGYLDIq4s8zJxb4vPXq
 mteb/models/model_implementations/fa_models.py,sha256=WGal70_ezITWoNdjcMdbOCTSCtoaXzuPadYstLVXxhg,7478
 mteb/models/model_implementations/facebookai.py,sha256=uhE6rB1YgxE0SIc7u8heE1U62qRFFA23IMgpjxBq_Ok,3116
 mteb/models/model_implementations/geogpt_models.py,sha256=Juv86SwhgQX80lVLjAFtim2aSiJT1AcgjniyyiKyk1Q,1923
-mteb/models/model_implementations/gme_v_models.py,sha256=NkfgR3_UdZzoBt1NnalVou6LOR-F7qXM4by9EbAVrys,13568
+mteb/models/model_implementations/gme_v_models.py,sha256=GEu1wl5q77RMM3BwtKMjkMwm38KX_r0qWxD_IEMVC2U,13657
 mteb/models/model_implementations/google_models.py,sha256=7QfsaJ5JNDRQxFl7Zh2AtiR2PR7PZcfeCBgviuOFBCo,9130
-mteb/models/model_implementations/granite_vision_embedding_models.py,sha256=uqQ5-e_a-ADv3gf3sR9Drk0S4x8Gy8mZkpL-E4X16TM,7241
+mteb/models/model_implementations/granite_vision_embedding_models.py,sha256=cvG5NliPwDVMvGuJTo8rk5yL3m6cuJZ_fMLEc0ESNfc,7315
 mteb/models/model_implementations/gritlm_models.py,sha256=aS_CuioL95JAQMYiaKlGuAWU9wZjabn268Xut3bD8-w,3005
 mteb/models/model_implementations/gte_models.py,sha256=o26Xyu_tucUlP435Q_jB4-bl0xckgj4wtbutTwhYgIo,10073
 mteb/models/model_implementations/hinvec_models.py,sha256=I_d_dSNVaGIwMIwyvTlaPAzGMpwh_PzvsfE4y47GFyg,1575
 mteb/models/model_implementations/human.py,sha256=klMpuMAtYH92EIEwNMEhne_Baf9fNiTg1DNWYD11P44,532
 mteb/models/model_implementations/ibm_granite_models.py,sha256=YCT0jbgawy19ps5l8QlxpQoJLjq8Nh-3R-e6yxS0DRM,7902
 mteb/models/model_implementations/inf_models.py,sha256=lvXUFhAYDltq2_Xa9MHcwfhh1V20rbJLSgON76tkj6w,2906
-mteb/models/model_implementations/jasper_models.py,sha256=ZY7qRRpBpD3eVryQb4rLs5E3KDXlgFBvyelataqLIWs,16213
+mteb/models/model_implementations/jasper_models.py,sha256=onX_ipI-UZbaZrjcHpZtk34tpy6DcT6Yvq6X3RMSmYA,16211
 mteb/models/model_implementations/jina_clip.py,sha256=CfiIxbhKspjQajNtObCfGPHOWPk6uLn4cuwydQHFTMo,5118
-mteb/models/model_implementations/jina_models.py,sha256=HrHm2Io3g9gHwxU5icAaudy_E8rAVkAAIFSzVYWF-dM,34859
+mteb/models/model_implementations/jina_models.py,sha256=1bkGwIaRNIun2ghkWb4FG-7js4lJ39s97Q9KAW3wkXo,34858
 mteb/models/model_implementations/kalm_models.py,sha256=FmW7Z5Qs6WYBLuKvql3u4IJW36kj4k-Ypah8qTBEBkg,59837
 mteb/models/model_implementations/kblab.py,sha256=DDh8gDEI6YPjS4_yGYWC4HatE0mFf7vhGDU83zzV7V0,866
 mteb/models/model_implementations/kennethenevoldsen_models.py,sha256=DF-9nmsewYO9ikZ0kV81ujKGr7Ot36-9iPoxN7KX2mY,2993
@@ -1521,22 +1521,23 @@ mteb/models/model_implementations/mxbai_models.py,sha256=33ta2BnhvKYBUgE89wFgPNf
 mteb/models/model_implementations/nbailab.py,sha256=bqqR0qs10IH2g5HC6K962tDMBciw8qFsNVHADNS72jk,2396
 mteb/models/model_implementations/no_instruct_sentence_models.py,sha256=6i-xbLRRNKuDpU-hwklwdQjgu1wnz5CecLSoc6kyd7Q,3976
 mteb/models/model_implementations/nomic_models.py,sha256=mT-v5Gs5-sRH8-ziCw_CtxB9ox3C6FtwWJjNghNrunw,11334
-mteb/models/model_implementations/nomic_models_vision.py,sha256=gEEieMThvw4p-QhRH0G_9-WWTvj-jqOlgFsh6O07dbc,6731
-mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py,sha256=14XSv7wGsitu0cF8P3A951gel_Py7PrKlRixkLS4qG4,6203
+mteb/models/model_implementations/nomic_models_vision.py,sha256=6aca0XVLXnkGk6GW8jVCIbbjPGq98lKq4c9Az4jbEkE,6805
+mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py,sha256=OEhVrvA-zfX2PSm76VcCDPkRyAArSFkVeweyLyzpqPI,6255
 mteb/models/model_implementations/nvidia_models.py,sha256=acVverAt77lURkILCVkCdXsWgY1BJoG1-ugB7yIhlIM,21555
 mteb/models/model_implementations/openai_models.py,sha256=loU6JByNUwRidq7lmcu8iGOtUQvzejw6HVLaF_IKCR0,9352
 mteb/models/model_implementations/openclip_models.py,sha256=W8XcokgLU1nSmMaWpYXkWWizVd3sQezcP02YtF2fXpo,11436
 mteb/models/model_implementations/opensearch_neural_sparse_models.py,sha256=fuxIjOx_kPoDps5C7LW3JllG-AZj4ktqeTNgJESHZh4,8351
 mteb/models/model_implementations/ops_moa_models.py,sha256=luWw1j2iTMx1z1ydLCjvCI89E9Yvge7ruEawivJTmfE,2413
 mteb/models/model_implementations/ordalietech_solon_embeddings_mini_beta_1_1.py,sha256=qGXv71qRjNCIFluZOwvfBlFlKKyN2bXBokwUPk4KHmM,1066
+mteb/models/model_implementations/pawan_models.py,sha256=rV2ePGIuYroocvwqDXm4VU369Y_Vr67CyAE-08K5B9c,1151
 mteb/models/model_implementations/piccolo_models.py,sha256=d8Dtkv_ZTUOCmJLLOuwquq-gX-2UfKvAtl_LvAS0Xi0,2113
 mteb/models/model_implementations/promptriever_models.py,sha256=S7uWes_P74p3OZR_KBJHJN_ezlvvRx2__46DMCWqV5M,6328
-mteb/models/model_implementations/pylate_models.py,sha256=yINGQL97S4xjj74-FTWpO4KHX-E9NDOEeyQWyRmmnaE,14772
+mteb/models/model_implementations/pylate_models.py,sha256=oNoPndZuiJahSd-ikR4dE4vL9261btXYiJbF3bk3Dco,14546
 mteb/models/model_implementations/qodo_models.py,sha256=JDqffDlQiOEariyheybOIf3iNkqot2gTkEIHWDnRbUE,2037
 mteb/models/model_implementations/qtack_models.py,sha256=biZLH5E3UWIcMZXIZNGgBZFEUvovPpAo6vUyL776W1w,1224
 mteb/models/model_implementations/qwen3_models.py,sha256=F_o6ciD-6gLFfIlQYD9MsNvcbkmGzJ39eKpFlEog1rM,5132
 mteb/models/model_implementations/qzhou_models.py,sha256=7KaZpHdap-YyK0QxOMHxU0W2aGismx7GZv_bNXkEOcI,3536
-mteb/models/model_implementations/random_baseline.py,sha256=1VNnWBSi0Ph_RLON6clOuQI-Kli5BRtiiDFZMrTj7PM,7489
+mteb/models/model_implementations/random_baseline.py,sha256=z4xNs5fbH1HUZhtf3Ry5AKa264SWk2Y4eobRu8rmPKM,7563
 mteb/models/model_implementations/rasgaard_models.py,sha256=a8F3kDSBWHH0UR7wRioOrWGQUxtloD5mU7EG27iM-68,1260
 mteb/models/model_implementations/reasonir_model.py,sha256=wSCcJpUgZ0pG2g3vTEzYNmPlPG_CVn_rR0ENVCines0,2218
 mteb/models/model_implementations/repllama_models.py,sha256=89HoqEpzkNysHeuf_-YhU8WETamHTogSRztGIRo6G1s,7321
@@ -1549,7 +1550,7 @@ mteb/models/model_implementations/salesforce_models.py,sha256=KslTK-IKeLvNG-vQir
 mteb/models/model_implementations/samilpwc_models.py,sha256=oMwKNwCxoH1jZgCy04oo2oVlBZWu253QMpnEEC6emz8,2021
 mteb/models/model_implementations/sarashina_embedding_models.py,sha256=TSmr2FEX79mJTA9mbEV3meEZYSelGv58Veiw__TTGFM,8415
 mteb/models/model_implementations/searchmap_models.py,sha256=XvVl99emIgnNUCxkTuFQXW6py2R8vgsArfpyHveCugw,1904
-mteb/models/model_implementations/seed_1_6_embedding_models.py,sha256=8J3htEddltyGTydIbnMUudgAV97FdD43-SQKaSA_Iuc,18534
+mteb/models/model_implementations/seed_1_6_embedding_models.py,sha256=Q8JTW2fjePR9dq4spuwK2lyVeL3mn1bl-H5wkQuEV_E,18609
 mteb/models/model_implementations/seed_models.py,sha256=SgK4kPVO6V33G3F1zSq06zSkWarPLEwBt1SWp4TUoVw,14142
 mteb/models/model_implementations/sentence_transformers_models.py,sha256=EtEaXg1yFFp3DQEOxu6am8bcVQR-ypcHj6DCqJGHOVU,21160
 mteb/models/model_implementations/shuu_model.py,sha256=KkcuVYjIzoha3Fvxh8ppqHQ9BfNMWeqDqn9dGCRKUjg,1167
@@ -1566,7 +1567,7 @@ mteb/models/model_implementations/vi_vn_models.py,sha256=quWmd3JT2J6SlAsFrV2gcnc
 mteb/models/model_implementations/vista_models.py,sha256=Q3I01kRtIPaoke0iMIcH4CLcCDTnMSIBFNCof7LPTX4,10832
 mteb/models/model_implementations/vlm2vec_models.py,sha256=HGGy_-z9Wc99xOKum71rBNipCPqWcM1efmmXgy5Rvxc,11724
 mteb/models/model_implementations/voyage_models.py,sha256=dOCccOQlloGrg0q44PxMQzx8dHuQ8VgkDUD01EydpJ0,19824
-mteb/models/model_implementations/voyage_v.py,sha256=WnvwYNVv3c5K0ChzGA3v2iTQX2ekXYn_3402H721c6M,8088
+mteb/models/model_implementations/voyage_v.py,sha256=vT1MXCt6-_PWA9U7lNz-Qj2zyGHwm_79WqxH4elMm90,8162
 mteb/models/model_implementations/xyz_models.py,sha256=TePlrH6EHwRPO87U_J3Yce9-XHCn_X7I2cJ_6BZ2fUY,1296
 mteb/models/model_implementations/youtu_models.py,sha256=NB74E6z-_36HyXb8GXKn8CrmRLN68uX9eH4xcS57zl0,5938
 mteb/models/model_implementations/yuan_models.py,sha256=yZ6ki6YFaoVrJ_2pPSRQaMKOsIOUo3GtmhPx1qeUl2w,939
@@ -1576,8 +1577,8 @@ mteb/models/search_encoder_index/search_backend_protocol.py,sha256=TSjlx88stJcMl
 mteb/models/search_encoder_index/search_indexes/__init__.py,sha256=Wm60_oUemUpFsvrCMW111dcPH2L2rt1iZrXMskXmG7o,88
 mteb/models/search_encoder_index/search_indexes/faiss_search_index.py,sha256=WMs3QbbYV13fRuT3dakmdVMZLFdc_9ZzSupS3QxlbVQ,5555
 mteb/results/__init__.py,sha256=EXQqK4Am5eIYzD52dpcGAFSdqnC38oE6JHN302oidHc,158
-mteb/results/benchmark_results.py,sha256=OWqeBxbNsPmOKRhxY980N5CikpdJXToDGJGTXUe64Lw,18209
-mteb/results/model_result.py,sha256=pTyGFTLg6l1wmc3Ul1CJK6ESBqWJAuU4aeT8iFygAdU,13746
+mteb/results/benchmark_results.py,sha256=b_g0QmTbwue9ZpWTtyPfgf_nyavckZHUgTVE6zqqtzM,18342
+mteb/results/model_result.py,sha256=Y6b_xfJlw8EFZq464ZVhyw0Rryv111hvMjnXbEZJpXk,14059
 mteb/results/task_result.py,sha256=DgmAw6akotjp8m8E6gE8QP9mQMxUvyzu1hnZ5o01GkU,32303
 mteb/tasks/__init__.py,sha256=izAxU0ip1F_YUwx0dFCuN35BaktdmePh6vlDiHC0kLo,503
 mteb/tasks/aggregated_tasks/__init__.py,sha256=Ufgbh1AirxCQkojO3AUhUFWM8zQG10cfdVTkj_PeyLI,104
@@ -2576,14 +2577,14 @@ mteb/tasks/zeroshot_classification/eng/sun397.py,sha256=Nls7tXM2Svu008MmAUjt-o_N
 mteb/tasks/zeroshot_classification/eng/ucf101.py,sha256=kwNRYks-_Oe4VE3GyoHIvN-2OJ6zhkwFr76WDNL9ymU,1884
 mteb/tasks/zeroshot_classification/eng/templates/__init__.py,sha256=da1PTClDMl-IBkrSvq6JC1lnS-K_BASzCvxVhNxN5Ls,13
 mteb/types/__init__.py,sha256=7_q6_84RvMuHeZK51GbLc5gbpTb3C1WmnqDHm6bnCzw,1104
-mteb/types/_encoder_io.py,sha256=YlIthHwi0o-ObANsWwVPcQc4pA1wpiRwIewDjNTMAdQ,5461
+mteb/types/_encoder_io.py,sha256=Q7llxv3FfiExFKiQGHtATvbSk4_DwdJolLMPTnAPrrI,5536
 mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=CRAUc5IvqI3_9SyXDwv-PWLCXwXdZem9RePeYESRtuw,996
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=YwJsxTf1eaCI_RE-J37a-gK5wDeGAsmkeZKoZCFihSo,3755
-mteb-2.3.9.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.3.9.dist-info/METADATA,sha256=da_FgK7mGK2HivEwQfKDyIPYzDVMFaz-lTeVQVvp2q8,13923
-mteb-2.3.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mteb-2.3.9.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.3.9.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.3.9.dist-info/RECORD,,
+mteb-2.3.11.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.3.11.dist-info/METADATA,sha256=zK0XHgO0btF1XS2eXGROlNeh8jCSj6dQV4NAT3N_Hn8,13991
+mteb-2.3.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mteb-2.3.11.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.3.11.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.3.11.dist-info/RECORD,,