mteb 2.7.15__py3-none-any.whl → 2.7.17__py3-none-any.whl

mteb/_evaluators/classification_metrics.py

@@ -1,7 +1,16 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 import numpy as np
 
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
 
-def hamming_score(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+def hamming_score(
+    y_true: NDArray[np.integer], y_pred: NDArray[np.integer | np.floating]
+) -> float:
     """Compute the Hamming score (a.k.a. label-based accuracy) for multilabel classification.
 
     The Hamming score is the fraction of labels that are correctly predicted for each sample,

mteb/_evaluators/pair_classification_evaluator.py

@@ -16,6 +16,7 @@ from mteb.similarity_functions import compute_pairwise_similarity
 
 if TYPE_CHECKING:
     from datasets import Dataset
+    from numpy.typing import NDArray
 
     from mteb.abstasks.task_metadata import TaskMetadata
     from mteb.models import EncoderProtocol
@@ -155,7 +156,7 @@ class PairClassificationEvaluator(Evaluator):
         hf_split: str,
         hf_subset: str,
         **encode_kwargs: Any,
-    ) -> np.ndarray:
+    ) -> NDArray[np.floating]:
         index_map = {}
         all_unique_texts: list[str] = []
         all_texts_indexes = []

mteb/_evaluators/retrieval_metrics.py

@@ -15,6 +15,8 @@ from mteb.types import RetrievalEvaluationResult
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
+    from numpy.typing import NDArray
+
     from mteb.types import RelevantDocumentsType
 
 logger = logging.getLogger(__name__)
@@ -273,9 +275,9 @@ def confidence_scores(sim_scores: list[float]) -> dict[str, float]:
 
 
 def nauc(
-    conf_scores: np.ndarray,
-    metrics: np.ndarray,
-    abstention_rates: np.ndarray = np.linspace(0, 1, 11)[:-1],
+    conf_scores: NDArray[np.floating],
+    metrics: NDArray[np.floating],
+    abstention_rates: NDArray[np.floating] = np.linspace(0, 1, 11)[:-1],
 ) -> float:
     """Computes normalized Area Under the Curve (nAUC) on a set of evaluated instances as presented in the paper https://arxiv.org/abs/2402.12997
 
@@ -295,10 +297,10 @@ def nauc(
     """
 
     def abstention_curve(
-        conf_scores: np.ndarray,
-        metrics: np.ndarray,
-        abstention_rates: np.ndarray = np.linspace(0, 1, 11)[:-1],
-    ) -> np.ndarray:
+        conf_scores: NDArray[np.floating],
+        metrics: NDArray[np.floating],
+        abstention_rates: NDArray[np.floating] = np.linspace(0, 1, 11)[:-1],
+    ) -> NDArray[np.floating]:
         """Computes the raw abstention curve for a given set of evaluated instances and corresponding confidence scores
 
         Args:

mteb/_evaluators/sklearn_evaluator.py

@@ -10,6 +10,7 @@ from .evaluator import Evaluator
 if TYPE_CHECKING:
     import numpy as np
     from datasets import Dataset
+    from numpy.typing import NDArray
     from torch.utils.data import DataLoader
     from typing_extensions import Self
 
@@ -21,11 +22,15 @@ logger = logging.getLogger(__name__)
 
 
 class SklearnModelProtocol(Protocol):
-    def fit(self, X: Array, y: np.ndarray | list[int]) -> None: ...  # noqa: N803
-    def predict(self, X: Array) -> np.ndarray: ...  # noqa: N803
+    def fit(
+        self, X: Array, y: NDArray[np.integer | np.floating] | list[int | float]
+    ) -> None: ...
+    def predict(self, X: Array) -> NDArray[np.integer | np.floating]: ...
     def get_params(self) -> dict[str, Any]: ...
     def set_params(self, random_state: int, **kwargs: dict[str, Any]) -> Self: ...
-    def score(self, X: Array, y: np.ndarray | list[int]) -> float: ...  # noqa: N803
+    def score(
+        self, X: Array, y: NDArray[np.integer | np.floating] | list[int | float]
+    ) -> float: ...
 
 
 class SklearnEvaluator(Evaluator):
@@ -79,7 +84,7 @@ class SklearnEvaluator(Evaluator):
         encode_kwargs: EncodeKwargs,
         test_cache: Array | None = None,
         num_proc: int = 1,
-    ) -> tuple[np.ndarray, Array]:
+    ) -> tuple[NDArray[np.integer | np.floating], Array]:
         """Classification evaluation by training a sklearn classifier on the embeddings of the training set and evaluating on the embeddings of the test set.
 
         Args:

mteb/abstasks/_stratification.py

@@ -38,21 +38,26 @@ Bibtex:
     }
 """
 
+from __future__ import annotations
+
 import itertools
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import scipy.sparse as sp
 from sklearn.model_selection._split import _BaseKFold
 from sklearn.utils import check_random_state
 
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
 
 def _iterative_train_test_split(
-    X: np.ndarray,  # noqa: N803
-    y: np.ndarray,
+    X: NDArray[np.integer],
+    y: NDArray[np.integer],
     test_size: float,
     random_state: int | None = None,
-) -> tuple[np.ndarray, np.ndarray]:
+) -> tuple[NDArray[np.integer], NDArray[np.integer]]:
     """Iteratively stratified train/test split
 
     Slighltly modified from:
@@ -79,8 +84,8 @@ def _iterative_train_test_split(
 
 
 def _fold_tie_break(
-    desired_samples_per_fold: np.ndarray,
-    M: np.ndarray,  # noqa: N803
+    desired_samples_per_fold: NDArray[np.floating],
+    M: NDArray[np.integer],  # noqa: N803
     random_state: np.random.RandomState,
 ):
     """Helper function to split a tie between folds with same desirability of a given sample
@@ -179,7 +184,7 @@ class IterativeStratification(_BaseKFold):
             ]
 
     def _prepare_stratification(
-        self, y: np.ndarray
+        self, y: NDArray[np.integer]
     ) -> tuple[
         list[list[int]],
         dict[int, bool],
@@ -301,7 +306,7 @@ class IterativeStratification(_BaseKFold):
             self.desired_samples_per_fold[fold_selected] -= 1
             folds[fold_selected].append(row)
 
-    def _iter_test_indices(self, X, y=None, groups=None):  # noqa: N803
+    def _iter_test_indices(self, X, y=None, groups=None):
         """Internal method for providing scikit-learn's split with folds
 
         Args:

mteb/abstasks/classification.py

@@ -31,6 +31,8 @@ from .abstask import AbsTask
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from numpy.typing import NDArray
+
     from mteb._evaluators.sklearn_evaluator import SklearnModelProtocol
     from mteb.models import MTEBModels
     from mteb.types import EncodeKwargs, HFSubset, ScoresDict
@@ -270,8 +272,8 @@ class AbsTaskClassification(AbsTask):
 
     def _calculate_scores(
         self,
-        y_test: np.ndarray | list[int],
-        y_pred: np.ndarray,
+        y_test: NDArray[np.integer] | list[int],
+        y_pred: NDArray[np.integer | np.floating] | list[int],
     ) -> ClassificationMetrics:
         scores = ClassificationMetrics(
             accuracy=accuracy_score(y_test, y_pred),

mteb/abstasks/multilabel_classification.py

@@ -23,6 +23,8 @@ from .classification import AbsTaskClassification
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from numpy.typing import NDArray
+
     from mteb._evaluators.sklearn_evaluator import SklearnModelProtocol
     from mteb.models import MTEBModels
     from mteb.types import Array, EncodeKwargs
@@ -32,10 +34,10 @@ logger = logging.getLogger(__name__)
 
 def _evaluate_classifier(
     embeddings_train: Array,
-    y_train: np.ndarray,
+    y_train: NDArray[np.integer],
     embeddings_test: Array,
     classifier: SklearnModelProtocol,
-) -> tuple[np.ndarray, SklearnModelProtocol]:
+) -> tuple[NDArray[np.integer | np.floating], SklearnModelProtocol]:
     classifier_copy: SklearnModelProtocol = clone(classifier)
     classifier_copy.fit(embeddings_train, y_train)
     return classifier_copy.predict(embeddings_test), classifier_copy
@@ -208,8 +210,8 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
 
     def _calculate_scores(  # type: ignore[override]
         self,
-        y_test: np.ndarray,
-        y_pred: np.ndarray,
+        y_test: NDArray[np.integer],
+        y_pred: NDArray[np.integer | np.floating],
         x_test_embedding: Array,
         current_classifier: SklearnModelProtocol,
     ) -> MultilabelClassificationMetrics:

mteb/abstasks/regression.py

@@ -24,6 +24,7 @@ from .classification import AbsTaskClassification
 
 if TYPE_CHECKING:
     from datasets import Dataset
+    from numpy.typing import NDArray
 
     from mteb._evaluators.sklearn_evaluator import SklearnModelProtocol
     from mteb.types.statistics import (
@@ -123,8 +124,8 @@ class AbsTaskRegression(AbsTaskClassification):
 
     def _calculate_scores(  # type: ignore[override]
         self,
-        y_test: np.ndarray | list[int],
-        y_pred: np.ndarray,
+        y_test: NDArray[np.floating] | list[float],
+        y_pred: NDArray[np.floating] | list[float],
     ) -> RegressionMetrics:
         mse = mean_squared_error(y_test, y_pred)
         return RegressionMetrics(

mteb/benchmarks/benchmark.py

@@ -164,14 +164,142 @@ class MIEBBenchmark(Benchmark):
 class VidoreBenchmark(Benchmark):
     """Wrapper for Vidore3 benchmark."""
 
-    def _create_summary_table(
+    def _create_vidore_summary_table(
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
+        """Create summary table from BenchmarkResults.
+
+        Returns a DataFrame with one row per model containing summary statistics
+        and task type averages. Customized for Vidore benchmark.
+
+        Args:
+            benchmark_results: BenchmarkResults object containing model results
+
+        Returns:
+            DataFrame with model summaries, ready for styling in the leaderboard
+        """
+        import mteb
         from mteb.benchmarks._create_table import (
-            _create_summary_table_mean_public_private,
+            _format_max_tokens,
+            _format_n_parameters,
+            _get_means_per_types,
+            _split_on_capital,
+        )
+        from mteb.get_tasks import get_task
+
+        data = benchmark_results.to_dataframe(format="long")
+
+        if data.empty:
+            no_results_frame = pd.DataFrame(
+                {"No results": ["You can try relaxing your criteria"]}
+            )
+            return no_results_frame
+        public_task_name = benchmark_results._filter_tasks(is_public=True).task_names
+        private_task_name = benchmark_results._filter_tasks(is_public=False).task_names
+        # Convert to DataFrame and pivot
+        per_task = data.pivot(index="model_name", columns="task_name", values="score")
+
+        # Remove models with no scores
+        to_remove = per_task.isna().all(axis="columns")
+        if to_remove.all():
+            no_results_frame = pd.DataFrame(
+                {"No results": ["You can try relaxing your criteria"]}
+            )
+            return no_results_frame
+
+        models_to_remove = list(per_task[to_remove].index)
+        per_task = per_task.drop(models_to_remove, axis=0)
+
+        # Calculate means by task type
+        mean_per_type = _get_means_per_types(per_task)
+        mean_per_type = mean_per_type.pivot(
+            index="model_name", columns="task_type", values="score"
+        )
+        mean_per_type.columns = [
+            _split_on_capital(column) for column in mean_per_type.columns
+        ]
+
+        # Calculate overall means
+        public_mean = per_task[public_task_name].mean(skipna=False, axis=1)
+        private_mean = per_task[private_task_name].mean(skipna=False, axis=1)
+
+        # Build joint table
+        joint_table = mean_per_type.copy()
+        joint_table.insert(1, "mean(public)", public_mean)
+        joint_table.insert(2, "mean(private)", private_mean)
+        task_type = get_task(
+            per_task.columns[0]
+        ).metadata.type  # "DocumentUnderstanding"
+        joint_table = joint_table.sort_values(
+            [_split_on_capital(task_type), "mean(public)", "mean(private)"],
+            ascending=False,
+        )
+
+        joint_table = joint_table.reset_index()
+
+        # Add model metadata
+        model_metas = joint_table["model_name"].map(mteb.get_model_meta)
+        joint_table = joint_table[model_metas.notna()]
+        joint_table["model_link"] = model_metas.map(lambda m: m.reference)
+
+        # Insert model metadata columns
+        joint_table.insert(
+            1,
+            "Max Tokens",
+            model_metas.map(lambda m: _format_max_tokens(m.max_tokens)),
+        )
+        joint_table.insert(
+            1,
+            "Embedding Dimensions",
+            model_metas.map(lambda m: int(m.embed_dim) if m.embed_dim else None),
+        )
+        joint_table.insert(
+            1,
+            "Number of Parameters (B)",
+            model_metas.map(lambda m: _format_n_parameters(m.n_parameters)),
+        )
+        joint_table.insert(
+            1,
+            "Memory Usage (MB)",
+            model_metas.map(
+                lambda m: int(m.memory_usage_mb) if m.memory_usage_mb else None
+            ),
+        )
+
+        # Clean up model names (remove HF organization)
+        joint_table["model_name"] = joint_table["model_name"].map(
+            lambda name: name.split("/")[-1]
+        )
+
+        # Add markdown links to model names
+        name_w_link = (
+            "[" + joint_table["model_name"] + "](" + joint_table["model_link"] + ")"
+        )
+        joint_table["model_name"] = joint_table["model_name"].mask(
+            joint_table["model_link"].notna(), name_w_link
+        )
+        joint_table = joint_table.drop(columns=["model_link"])
+
+        # Rename columns
+        rename_dict = {
+            "model_name": "Model",
+            "mean(public)": "Mean (Public)",
+            "mean(private)": "Mean (Private)",
+        }
+
+        joint_table = joint_table.rename(columns=rename_dict)
+
+        # Add Rank column
+        joint_table.insert(
+            0, "Rank (Mean Task)", [i + 1 for i in range(len(joint_table))]
         )
 
-        joint_table = _create_summary_table_mean_public_private(benchmark_results)
+        return joint_table
+
+    def _create_summary_table(
+        self, benchmark_results: BenchmarkResults
+    ) -> pd.DataFrame:
+        joint_table = self._create_vidore_summary_table(benchmark_results)
         # For ViDoRe (V1, V2, V3): all tasks are Document Understanding type, so Document Understanding column = Mean (Task)
         joint_table = joint_table.rename(
             columns={"Document Understanding": "Mean (Task)"}

mteb/leaderboard/figures.py

@@ -125,6 +125,7 @@ def _performance_size_plot(df: pd.DataFrame) -> go.Figure:
     min_score, max_score = df["Mean (Task)"].min(), df["Mean (Task)"].max()
     df["sqrt(dim)"] = np.sqrt(df["Embedding Dimensions"])
     df["Max Tokens"] = df["Max Tokens"].apply(lambda x: _process_max_tokens(x))
+    rank_column = "Rank (Borda)" if "Rank (Borda)" in df.columns else "Rank (Mean Task)"
     fig = px.scatter(
         df,
         x="Number of Parameters",
@@ -141,7 +142,7 @@ def _performance_size_plot(df: pd.DataFrame) -> go.Figure:
             "Embedding Dimensions": True,
             "Number of Parameters": True,
             "Mean (Task)": True,
-            "Rank (Borda)": True,
+            rank_column: True,
             "Log(Tokens)": False,
             "sqrt(dim)": False,
             "model_text": False,

mteb/leaderboard/table.py

@@ -156,6 +156,7 @@ def _apply_summary_table_styling(joint_table: pd.DataFrame) -> gr.DataFrame:
     """
     excluded_columns = [
         "Rank (Borda)",
+        "Rank (Mean Task)",
         "Rank",
         "Model",
         "Number of Parameters (B)",
@@ -183,10 +184,17 @@ def _apply_summary_table_styling(joint_table: pd.DataFrame) -> gr.DataFrame:
         joint_table["Zero-shot"] = joint_table["Zero-shot"].apply(_format_zero_shot)
     joint_table[score_columns] = joint_table[score_columns].map(_format_scores)
 
+    if "Rank (Borda)" in joint_table.columns:
+        rank_column = "Rank (Borda)"
+    elif "Rank (Mean Task)" in joint_table.columns:
+        rank_column = "Rank (Mean Task)"
+    else:
+        raise ValueError("No rank column found in the result table.")
+
     joint_table_style = joint_table.style.format(
         {
             **dict.fromkeys(score_columns, "{:.2f}"),
-            "Rank (Borda)": "{:.0f}",
+            rank_column: "{:.0f}",
             "Memory Usage (MB)": "{:.0f}",
             "Embedding Dimensions": "{:.0f}",
             "Max Tokens": "{:.0f}",
@@ -195,7 +203,7 @@ def _apply_summary_table_styling(joint_table: pd.DataFrame) -> gr.DataFrame:
         na_rep="",
     )
     joint_table_style = joint_table_style.highlight_min(
-        "Rank (Borda)", props="font-weight: bold"
+        rank_column, props="font-weight: bold"
     ).highlight_max(subset=score_columns, props="font-weight: bold")
 
     # Apply background gradients for each selected column

mteb/models/cache_wrappers/cache_backend_protocol.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
 if TYPE_CHECKING:
     from pathlib import Path
 
-    import numpy as np
+    from mteb.types import Array
 
 
 @runtime_checkable
@@ -26,7 +26,7 @@ class CacheBackendProtocol(Protocol):
             **kwargs: Additional backend-specific arguments.
         """
 
-    def add(self, item: list[dict[str, Any]], vectors: np.ndarray) -> None:
+    def add(self, item: list[dict[str, Any]], vectors: Array) -> None:
         """Add a vector to the cache.
 
         Args:
@@ -34,7 +34,7 @@ class CacheBackendProtocol(Protocol):
             vectors: Embedding vector of shape (dim,) or (1, dim).
         """
 
-    def get_vector(self, item: dict[str, Any]) -> np.ndarray | None:
+    def get_vector(self, item: dict[str, Any]) -> Array | None:
         """Retrieve the cached vector for the given item.
 
         Args:

mteb/models/cache_wrappers/cache_backends/faiss_cache.py

@@ -15,7 +15,7 @@ from ._hash_utils import _hash_item
 if TYPE_CHECKING:
     import faiss
 
-    from mteb.types import BatchedInput
+    from mteb.types import Array, BatchedInput
 
 logger = logging.getLogger(__name__)
 
@@ -43,7 +43,7 @@ class FaissCache:
         logger.info(f"Initialized FAISS VectorCacheMap in {self.directory}")
         self.load()
 
-    def add(self, items: list[dict[str, Any]], vectors: np.ndarray) -> None:
+    def add(self, items: list[dict[str, Any]], vectors: Array) -> None:
         """Add vector to FAISS index."""
         import faiss
 
@@ -67,7 +67,7 @@ class FaissCache:
             vectors_array = np.vstack(vectors_to_add).astype(np.float32)
             self.index.add(vectors_array)
 
-    def get_vector(self, item: BatchedInput) -> np.ndarray | None:
+    def get_vector(self, item: dict[str, Any]) -> Array | None:
         """Retrieve vector from index by hash."""
         if self.index is None:
             return None

mteb/models/cache_wrappers/cache_backends/numpy_cache.py

@@ -1,13 +1,18 @@
+from __future__ import annotations
+
 import json
 import logging
 import warnings
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
 from ._hash_utils import _hash_item
 
+if TYPE_CHECKING:
+    from mteb.types import Array
+
 logger = logging.getLogger(__name__)
 
 
@@ -27,7 +32,7 @@ class NumpyCache:
         logger.info(f"Initialized VectorCacheMap in directory: {self.directory}")
         self._initialize_vectors_file()
 
-    def add(self, items: list[dict[str, Any]], vectors: np.ndarray) -> None:
+    def add(self, items: list[dict[str, Any]], vectors: Array) -> None:
         """Add a vector to the cache."""
         try:
             if self.vector_dim is None:
@@ -178,7 +183,7 @@ class NumpyCache:
             logger.error(f"Error loading VectorCacheMap: {str(e)}")
             raise
 
-    def get_vector(self, item: dict[str, Any]) -> np.ndarray | None:
+    def get_vector(self, item: dict[str, Any]) -> Array | None:
         """Retrieve vector from index by hash."""
         if self.vectors is None:
             return None

mteb/models/cache_wrappers/cache_wrapper.py

@@ -98,7 +98,7 @@ class CachedEmbeddingWrapper:
             uncached_items: list[dict[str, Any]] = []
             uncached_indices: list[int] = []
             all_items: Dataset = inputs.dataset
-            cached_vectors: dict[int, np.ndarray] = {}
+            cached_vectors: dict[int, Array] = {}
 
             for i, item in enumerate(all_items):
                 vector = cache.get_vector(item)
@@ -108,7 +108,7 @@ class CachedEmbeddingWrapper:
                     uncached_items.append(item)
                     uncached_indices.append(i)
 
-            newly_encoded: dict[int, np.ndarray] = {}
+            newly_encoded: dict[int, Array] = {}
             if uncached_items:
                 logger.info(f"Encoding {len(uncached_items)} new items")
                 # Build a simple DataLoader with only uncached items

mteb/models/model_implementations/bedrock_models.py

@@ -86,7 +86,7 @@ class BedrockModel(AbsEncoder):
 
     def _encode_amazon(
         self, sentences: list[str], show_progress_bar: bool = False
-    ) -> np.ndarray:
+    ) -> Array:
         from botocore.exceptions import ValidationError
 
         all_embeddings = []
@@ -125,7 +125,7 @@ class BedrockModel(AbsEncoder):
         sentences: list[str],
         cohere_task_type: str,
         show_progress_bar: bool = False,
-    ) -> np.ndarray:
+    ) -> Array:
         batches = [
             sentences[i : i + self._max_batch_size]
             for i in range(0, len(sentences), self._max_batch_size)
@@ -149,7 +149,7 @@ class BedrockModel(AbsEncoder):
 
         return np.array(all_embeddings)
 
-    def _embed_amazon(self, sentence: str) -> np.ndarray:
+    def _embed_amazon(self, sentence: str) -> Array:
         response = self._client.invoke_model(
             body=json.dumps({"inputText": sentence}),
             modelId=self._model_id,
@@ -158,7 +158,7 @@ class BedrockModel(AbsEncoder):
         )
         return self._to_numpy(response)
 
-    def _to_numpy(self, embedding_response) -> np.ndarray:
+    def _to_numpy(self, embedding_response) -> Array:
         response = json.loads(embedding_response.get("body").read())
         key = "embedding" if self._provider == "amazon" else "embeddings"
         return np.array(response[key])

mteb/models/model_implementations/mcinext_models.py

@@ -13,7 +13,7 @@ from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta
 
 if TYPE_CHECKING:
-    from mteb.types import PromptType
+    from mteb.types import Array, PromptType
 logger = logging.getLogger(__name__)
 
 HAKIM_CITATION = """@article{sarmadi2025hakim,
@@ -302,7 +302,7 @@ class HakimModelWrapper(AbsEncoder):
         prompt_type: PromptType | None = None,
         batch_size: int = 32,
         **kwargs: Any,
-    ) -> np.ndarray:
+    ) -> Array:
         """Encodes sentences using the API.
 
         Returns:

mteb/models/model_implementations/misc_models.py

@@ -1007,54 +1007,6 @@ thenlper__gte_small = ModelMeta(
   year={2023}
 }""",
 )
-OrlikB__KartonBERT_USE_base_v1 = ModelMeta(
-    name="OrlikB/KartonBERT-USE-base-v1",
-    model_type=["dense"],
-    revision="1f59dd58fe57995c0e867d5e29f03763eae99645",
-    release_date="2024-09-30",
-    languages=["pol-Latn"],
-    loader=sentence_transformers_loader,
-    n_parameters=103705344,
-    n_embedding_parameters=None,
-    memory_usage_mb=396,
-    max_tokens=512.0,
-    embed_dim=768,
-    license="gpl-3.0",
-    open_weights=True,
-    public_training_code=None,
-    public_training_data=None,
-    framework=["PyTorch"],
-    reference="https://huggingface.co/OrlikB/KartonBERT-USE-base-v1",
-    similarity_fn_name=ScoringFunction.COSINE,
-    use_instructions=None,
-    training_datasets=None,
-    adapted_from="KartonBERT-USE-base-v1",
-    superseded_by=None,
-)
-OrlikB__st_polish_kartonberta_base_alpha_v1 = ModelMeta(
-    name="OrlikB/st-polish-kartonberta-base-alpha-v1",
-    model_type=["dense"],
-    revision="5590a0e2d7bb43674e44d7076b3ff157f7d4a1cb",
-    release_date="2023-11-12",
-    languages=["pol-Latn"],
-    loader=sentence_transformers_loader,
-    n_parameters=None,
-    n_embedding_parameters=None,
-    memory_usage_mb=None,
-    max_tokens=514.0,
-    embed_dim=768,
-    license="lgpl",
-    open_weights=True,
-    public_training_code=None,
-    public_training_data=None,
-    framework=["PyTorch"],
-    reference="https://huggingface.co/OrlikB/st-polish-kartonberta-base-alpha-v1",
-    similarity_fn_name=ScoringFunction.COSINE,
-    use_instructions=None,
-    training_datasets=None,
-    adapted_from="st-polish-kartonberta-base-alpha-v1",
-    superseded_by=None,
-)
 sdadas__mmlw_e5_base = ModelMeta(
     name="sdadas/mmlw-e5-base",
     model_type=["dense"],

mteb/models/model_implementations/openai_models.py

@@ -11,6 +11,7 @@ from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 
 if TYPE_CHECKING:
+    from numpy.typing import NDArray
     from torch.utils.data import DataLoader
 
     from mteb.abstasks.task_metadata import TaskMetadata
@@ -166,7 +167,7 @@ class OpenAIModel(AbsEncoder):
             all_embeddings[mask] = no_empty_embeddings
         return all_embeddings
 
-    def _to_numpy(self, embedding_response) -> np.ndarray:
+    def _to_numpy(self, embedding_response) -> NDArray[np.floating]:
         return np.array([e.embedding for e in embedding_response.data])
 
 

mteb/models/model_implementations/random_baseline.py

@@ -13,6 +13,7 @@ from mteb.similarity_functions import (
 )
 
 if TYPE_CHECKING:
+    from numpy.typing import NDArray
     from PIL import Image
     from torch.utils.data import DataLoader
 
@@ -20,7 +21,7 @@ if TYPE_CHECKING:
     from mteb.types._encoder_io import Array, BatchedInput, PromptType
 
 
-def _string_to_vector(text: str | None, size: int) -> np.ndarray:
+def _string_to_vector(text: str | None, size: int) -> NDArray[np.floating]:
     """Generate a deterministic random vector based on a string.
 
     Args:
@@ -39,7 +40,7 @@ def _string_to_vector(text: str | None, size: int) -> np.ndarray:
     return rng.random(size, dtype=np.float32)
 
 
-def _image_to_vector(image: Image.Image, size: int) -> np.ndarray:
+def _image_to_vector(image: Image.Image, size: int) -> NDArray[np.floating]:
     """Generate a deterministic random vector based on image content.
 
     Args:
@@ -80,7 +81,7 @@ _common_mock_metadata = dict(
 
 def _batch_to_embeddings(
     inputs: DataLoader[BatchedInput], embedding_dim: int
-) -> np.ndarray:
+) -> NDArray[np.floating]:
     """Convert batched text/image inputs into embeddings.
 
     Args:

mteb/models/model_implementations/rerankers_custom.py

@@ -103,68 +103,6 @@ class BGEReranker(RerankerWrapper):
         return scores
 
 
-class MonoBERTReranker(RerankerWrapper):
-    name: str = "MonoBERT"
-
-    def __init__(
-        self,
-        model_name_or_path="castorini/monobert-large-msmarco",
-        torch_compile=False,
-        **kwargs,
-    ):
-        from transformers import AutoModelForSequenceClassification, AutoTokenizer
-
-        super().__init__(model_name_or_path, **kwargs)
-        if not self.device:
-            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        model_args = {}
-        if self.fp_options:
-            model_args["torch_dtype"] = self.fp_options
-        self.model = AutoModelForSequenceClassification.from_pretrained(
-            model_name_or_path,
-            **model_args,
-        )
-        self.model.to(self.device)
-        self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
-        self.max_length = self.tokenizer.model_max_length
-        logger.info(f"Using max_length of {self.max_length}")
-
-        self.model.eval()
-
-    @torch.inference_mode()
-    def predict(
-        self,
-        inputs1: DataLoader[BatchedInput],
-        inputs2: DataLoader[BatchedInput],
-        *,
-        task_metadata: TaskMetadata,
-        hf_split: str,
-        hf_subset: str,
-        prompt_type: PromptType | None = None,
-        **kwargs: Any,
-    ) -> Array:
-        queries = [text for batch in inputs1 for text in batch["query"]]
-        instructions = None
-        if "instruction" in inputs2.dataset.features:
-            instructions = [text for batch in inputs1 for text in batch["instruction"]]
-        passages = [text for batch in inputs2 for text in batch["text"]]
-
-        if instructions is not None and instructions[0] is not None:
-            queries = [f"{q} {i}".strip() for i, q in zip(instructions, queries)]
-
-        tokens = self.tokenizer(
-            queries,
-            passages,
-            padding=True,
-            truncation="only_second",
-            return_tensors="pt",
-            max_length=self.max_length,
-        ).to(self.device)
-        output = self.model(**tokens)[0]
-        batch_scores = torch.nn.functional.log_softmax(output, dim=1)
-        return batch_scores[:, 1].exp()
-
-
 class JinaReranker(RerankerWrapper):
     name = "Jina"
 
@@ -219,31 +157,6 @@ class JinaReranker(RerankerWrapper):
         return scores
 
 
-monobert_large = ModelMeta(
-    loader=MonoBERTReranker,
-    loader_kwargs=dict(
-        fp_options="float16",
-    ),
-    name="castorini/monobert-large-msmarco",
-    model_type=["cross-encoder"],
-    languages=["eng-Latn"],
-    open_weights=True,
-    revision="0a97706f3827389da43b83348d5d18c9d53876fa",
-    release_date="2020-05-28",
-    n_parameters=None,
-    n_embedding_parameters=31_254_528,
-    memory_usage_mb=None,
-    max_tokens=None,
-    embed_dim=None,
-    license=None,
-    public_training_code=None,
-    public_training_data=None,
-    similarity_fn_name=None,
-    use_instructions=None,
-    training_datasets=None,
-    framework=["Sentence Transformers", "PyTorch", "Transformers"],
-)
-
 # languages unclear: https://huggingface.co/jinaai/jina-reranker-v2-base-multilingual/discussions/28
 jina_reranker_multilingual = ModelMeta(
     loader=JinaReranker,

mteb/models/model_implementations/rerankers_monot5_based.py

@@ -34,7 +34,6 @@ prediction_tokens = {
     "unicamp-dl/mt5-base-en-msmarco": ["▁no", "▁yes"],
     "unicamp-dl/mt5-base-mmarco-v2": ["▁no", "▁yes"],
     "unicamp-dl/mt5-base-mmarco-v1": ["▁no", "▁yes"],
-    "unicamp-dl/mt5-13b-mmarco-100k": ["▁", "▁true"],
 }
 
 
@@ -919,28 +918,3 @@ mt5_base_mmarco_v2 = ModelMeta(
     use_instructions=None,
     framework=["PyTorch", "Transformers"],
 )
-
-mt5_13b_mmarco_100k = ModelMeta(
-    loader=MonoT5Reranker,
-    loader_kwargs=dict(
-        fp_options="float16",
-    ),
-    name="unicamp-dl/mt5-13b-mmarco-100k",
-    model_type=["cross-encoder"],
-    languages=mt5_languages,
-    open_weights=True,
-    revision="e1a4317e102a525ea9e16745ad21394a4f1bffbc",
-    release_date="2022-11-04",
-    n_parameters=None,
-    n_embedding_parameters=1_024_458_752,
-    memory_usage_mb=None,
-    max_tokens=None,
-    embed_dim=None,
-    license=None,
-    public_training_code=None,
-    public_training_data=None,
-    similarity_fn_name=None,
-    use_instructions=None,
-    training_datasets=None,
-    framework=["PyTorch", "Transformers"],
-)

mteb/models/model_implementations/seed_models.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
+
 import logging
 import time
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import torch
@@ -14,6 +16,9 @@ from mteb.types import PromptType
 from .bge_models import bge_chinese_training_data
 from .nvidia_models import nvidia_training_datasets
 
+if TYPE_CHECKING:
+    from mteb.types import Array
+
 logger = logging.getLogger(__name__)
 
 
@@ -110,7 +115,7 @@ class SeedTextEmbeddingModel(AbsEncoder):
         prompt_type: PromptType | None = None,
         retries: int = 5,
         **kwargs: Any,
-    ) -> np.ndarray:
+    ) -> Array:
         trimmed_sentences = []
         for sentence in sentences:
             encoded_sentence = self._encoding.encode(sentence)

mteb/models/model_implementations/voyage_models.py

@@ -150,7 +150,7 @@ class VoyageModel(AbsEncoder):
         sentences: list[str],
         batch_size: int,
         input_type: Literal["query", "document"],
-    ) -> np.ndarray:
+    ) -> Array:
         embeddings, index = [], 0
 
         output_dtype = VOYAGE_DTYPE_TRANSLATION.get(

mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py

@@ -68,6 +68,7 @@ class Vidore3FinanceFrRetrieval(AbsTaskRetrieval):
         license="cc-by-4.0",
         annotations_creators="derived",
         dialect=[],
+        modalities=["text", "image"],
         sample_creation="created and machine-translated",
         bibtex_citation=r"""
 @article{loison2026vidorev3comprehensiveevaluation,

mteb/types/_encoder_io.py

@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, TypedDict
 import numpy as np
 import torch
 from datasets import Dataset
+from numpy.typing import NDArray
 
 if TYPE_CHECKING:
     from PIL import Image
@@ -26,8 +27,8 @@ class EncodeKwargs(TypedDict):
 
 
 # --- Output types ---
-Array = np.ndarray | torch.Tensor
-"""General array type, can be a numpy array or a torch tensor."""
+Array = NDArray[np.floating | np.integer | np.bool] | torch.Tensor
+"""General array type, can be a numpy array (float, int, or bool) or a torch tensor."""
 
 
 # --- Input types ---

{mteb-2.7.15.dist-info → mteb-2.7.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.7.15
+Version: 2.7.17
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.7.15.dist-info → mteb-2.7.17.dist-info}/RECORD

@@ -16,13 +16,13 @@ mteb/similarity_functions.py,sha256=7ENHjq35EMSO1kT73IKLbQ0jMWGxMPSMM8SPmGHzaAM,
 mteb/_evaluators/__init__.py,sha256=Ag1_RWpxBGMpujzd3FZjI40gY_KQKIpY31tJPuk-hFg,1013
 mteb/_evaluators/_download.py,sha256=jntlcURbJxcxUjTmn2D9Tu6ZnWgDc9t5bY8p9CZCqv4,586
 mteb/_evaluators/any_sts_evaluator.py,sha256=aeK6ZJ_wuGR_8L6f5B4Xcl7Fo0dgApqevsgg7hdHQk0,3918
-mteb/_evaluators/classification_metrics.py,sha256=TI-cMPWrIpMqpsNhhwSBY4bZUu2yM469fbcu44zolW0,1926
+mteb/_evaluators/classification_metrics.py,sha256=cWMU-2bCNXlk_UN0sZ7uRS5aFXESkYlJWg4OLMeFrRg,2090
 mteb/_evaluators/clustering_evaluator.py,sha256=COSG4tGz2hI3Ff3MNFxdeg9w1TPmisxhvF333zQx-ko,2226
 mteb/_evaluators/evaluator.py,sha256=8fEuBZW0sv-tpJCUT2X5lIvmF0Ji0Vuq7Z8AblipspA,1074
-mteb/_evaluators/pair_classification_evaluator.py,sha256=0_B_5LVPTgd7IsbehUoxOPfmETmTbv0DOZtwPmDLhWs,6624
+mteb/_evaluators/pair_classification_evaluator.py,sha256=g7JiwzYa_c_Ql3mOavKho2-vXdAagoZfEhref-r1luM,6671
 mteb/_evaluators/retrieval_evaluator.py,sha256=UexQCsfGMzZq-JfrdNQ4PylHdoyS6Ef_zxXokNZiR5o,3250
-mteb/_evaluators/retrieval_metrics.py,sha256=XXeI7pXHuhKDvDjMKhGSU-Uv6J2itQKeD7lUH85IL8Q,23856
-mteb/_evaluators/sklearn_evaluator.py,sha256=yjnNtwOuEU38eKOKHI7uFsIZJJDU5jmTHGH7MX0rKtA,4122
+mteb/_evaluators/retrieval_metrics.py,sha256=nO1StNLDaa_iBWTuVexo2rsDWkxYCgHZ98qgr9bydCs,23964
+mteb/_evaluators/sklearn_evaluator.py,sha256=sCGn3YOhwT2OARTlSCJIg69IMijqfVyHoTVgK1TFuYQ,4253
 mteb/_evaluators/zeroshot_classification_evaluator.py,sha256=jCf6H0LwxZGIZ-2LGF9SPbjHTtCAk9v1HtxWcmutqks,2488
 mteb/_evaluators/image/__init__.py,sha256=CsQd7OMkeV2Phun7paPWjayZ5qRnvj8H0TYBFeqMxag,148
 mteb/_evaluators/image/imagetext_pairclassification_evaluator.py,sha256=w2vJrQXIbRsdG837x1yYQkGNcTJt8rdLofQ3Jo3nIn4,5227
@@ -31,17 +31,17 @@ mteb/_evaluators/text/bitext_mining_evaluator.py,sha256=We-BQZJQGBXz_vTUZz-3OBt4
 mteb/_evaluators/text/summarization_evaluator.py,sha256=ZHn3kIFGJ1XzgbI21jEeTnP5pdIChGHVTCuyz2MSKyg,10900
 mteb/abstasks/__init__.py,sha256=1iAwpYTWX7U-goak2KMmacPFCzxPchLQAmZ_uI0t-p0,1130
 mteb/abstasks/_statistics_calculation.py,sha256=4opttohaS6LV5K0zQIqfG2IGIzQAdKAaLTpSTQ6auBc,5988
-mteb/abstasks/_stratification.py,sha256=GnqYRtkFYsB-412EvMR2iMqIinFr98NCSmxHeCXctlw,14347
+mteb/abstasks/_stratification.py,sha256=upxjHt4wjuEFGmb_vrONTh9ngZ8Oa1tY56tUdllFERQ,14490
 mteb/abstasks/abstask.py,sha256=9bpPnrwwGBWE--MGoTOb-J-RtOnoo0YUaAjAPv7JC1g,26831
 mteb/abstasks/aggregate_task_metadata.py,sha256=WXYY_DUU55s4PkxMVz7lwbdZarq6QznhbvJYdSTYZZI,5846
 mteb/abstasks/aggregated_task.py,sha256=8NY_vaqmMuYNxuB05YjU4W6aEipyKrF2iDFS3m-eXNc,6167
-mteb/abstasks/classification.py,sha256=zSA9nTplwspktPnZiN_RQrPvOgEKYxeQASm_Q1lb3ww,14052
+mteb/abstasks/classification.py,sha256=9gNSPw2fVPyoTb1tV4kR_Fzku3Z6xutJzbyNpkktPzc,14134
 mteb/abstasks/clustering.py,sha256=I8vre2f2FJFagzJEYf6hKDo3Y28xU29J_O-MhfqWqSI,14944
 mteb/abstasks/clustering_legacy.py,sha256=sbx8K6paccvzDPnmhgNE_UJE83orAJnQm3NGr-Ktjfs,9184
 mteb/abstasks/dataset_card_template.md,sha256=aD6l8qc3_jxwoIGJNYLzse-jpRa8hu92AxpnUtNgges,5122
-mteb/abstasks/multilabel_classification.py,sha256=rFa_Pw2OsUzqhZS-jh2zFD7I-TNl8bVNJ-DW7EpPapU,9708
+mteb/abstasks/multilabel_classification.py,sha256=olldnMq5a-elan2vZSQ-EFZbcS6Mt9h1nTOi8iFXjjE,9810
 mteb/abstasks/pair_classification.py,sha256=RVV5WUjs18N5PbWpyxakDNEd1UlRc4ON9I0OjD26Z78,14231
-mteb/abstasks/regression.py,sha256=ZuMZfOwU3G4hr__eHsgdagKKdrbN4-wQMLz45jr9YUc,8946
+mteb/abstasks/regression.py,sha256=2aYJvktoENLi1wLYdND35TGFfhrN_BFCs-Yr2Ex-f5I,9019
 mteb/abstasks/retrieval.py,sha256=BPyRibStAD70JfR0Z1x-VVVfzJDRVSmbOS6uREfpmok,27743
 mteb/abstasks/retrieval_dataset_loaders.py,sha256=p0y1nrWlUrt_aeoR4ocDLEQMLuD_SlMH0gBiUsOwrww,9983
 mteb/abstasks/sts.py,sha256=Xta3KVQE7hHqkPTDptemvNVEG0CsZSVjA-Z52EIBvDE,9576
@@ -58,7 +58,7 @@ mteb/abstasks/text/reranking.py,sha256=mCzy0-TnZ46_GC9Czl4zWKAPnYK5ur0qtFbPt47m9
 mteb/abstasks/text/summarization.py,sha256=bSgb0XhUzJVuLV1Wjr3HYB_Tn7SjmCDMnkBIEWHO4EQ,7381
 mteb/benchmarks/__init__.py,sha256=MQEVeli-zLaJ7Xg0z7RhXQwsdmm7Ht_W2Ln0rZo1Szc,225
 mteb/benchmarks/_create_table.py,sha256=CJL8U0adUbaxr5G26trfYo1tGx8cU2IWWx_ZHU6q6do,22407
-mteb/benchmarks/benchmark.py,sha256=YCGIvJ5Vc6GdCAYSjzwrnfj2A8MkbzNLvvtPBLMSSp8,6327
+mteb/benchmarks/benchmark.py,sha256=3AA-Zeh9Z4bTN6qVpRUfHWHxJfTei3KQBEOoVP2Mtic,10922
 mteb/benchmarks/get_benchmark.py,sha256=nzR6cu5yXu1kIJKhd4A2R62xp43Z62bluPbOpNXHMWQ,2545
 mteb/benchmarks/benchmarks/__init__.py,sha256=-o3EMWEfP0eQ8iZpWvTj5r4yuGOUuL9mHk8IgFcpPtk,2330
 mteb/benchmarks/benchmarks/benchmarks.py,sha256=IOU3Kk7TEZkhypJ7ScyfqHmYgx1nG_KPJkjXLKoSTmo,103931
@@ -1474,8 +1474,8 @@ mteb/languages/programming_languages.py,sha256=zxAakT3OSUnAuTnQ34VyeFIECnNXMlleZ
 mteb/leaderboard/__init__.py,sha256=991roXmtRwEQysV-37hWEzWpkvPgMCGRqZTHR-hm2io,88
 mteb/leaderboard/app.py,sha256=Y3G93VJq6eZMD4_CNzLwSEEGnuNJDqYEYztmzYR85eA,42549
 mteb/leaderboard/benchmark_selector.py,sha256=qd-2L20RQ4ACke01UlytkhZok1dkWgfUlXzfET52kGc,7956
-mteb/leaderboard/figures.py,sha256=cfOK82rRf-7sCjyP7GBxh4ezhOIt0OhD0_86mKtzLrg,7530
-mteb/leaderboard/table.py,sha256=U5mWtrVUTk_6t8T4KAp5qlbFgKh1PD0iKICqNMfhsoY,10462
+mteb/leaderboard/figures.py,sha256=9mHxxzL44OFE3RL0d5zvMFJwl2irnYL0YbMbfCugifs,7616
+mteb/leaderboard/table.py,sha256=egBQYgOXqUZCZa7QXkjYq45RYhAuey7KCo6MeTcx4B8,10741
 mteb/leaderboard/text_segments.py,sha256=iMIkS04QQjPbT-SkU0x6fOcS8xRbUYevryu9HydipKM,6570
 mteb/models/__init__.py,sha256=ABTuoqiBjBtBWW3LYY7ItBHdylR6jWoy06HH0g6j6fU,910
 mteb/models/abs_encoder.py,sha256=We9HlwWP61P4cMyZ080gywvDErA1eVsU9t46PtcNrCM,16830
@@ -1487,12 +1487,12 @@ mteb/models/search_wrappers.py,sha256=PXE1VVDWUd0LgTPJ-FxqIbGpIDWLRKo5CjrwIuu5nz
 mteb/models/sentence_transformer_wrapper.py,sha256=RsOxj-b7qzeYcxUTVJyb-lZDY4bINl4jEAEkPvKYB10,13578
 mteb/models/vllm_wrapper.py,sha256=rvE0mghcHRA1UwIN1mjJf1GKA6pTTcs8mZQ2UoGRJ0g,12287
 mteb/models/cache_wrappers/__init__.py,sha256=1w1TnMwulWJSzNkLXjbh5MY3sqgHWc6vUntYn49i9X8,169
-mteb/models/cache_wrappers/cache_backend_protocol.py,sha256=d00imHSJAVU1jtvwLWv3mn06_SJn-6JQbOvb0N2nlAk,1676
-mteb/models/cache_wrappers/cache_wrapper.py,sha256=ZzbtW5rVGNZ-5wrqE06C0Uy6BHT5Lq3IysHe8hAoTz4,6703
+mteb/models/cache_wrappers/cache_backend_protocol.py,sha256=vDDXnXDwWhhTvw0e0fiGB7PhPQtGx__-i6jJPvZVoAU,1676
+mteb/models/cache_wrappers/cache_wrapper.py,sha256=2hdQSFrISZB7Su2li743OIJDWUyp9z-spC1dnb8huTY,6693
 mteb/models/cache_wrappers/cache_backends/__init__.py,sha256=hN2Tq7cpTxoOYSCJ1Wnpvb8dEm-kQLfCCahT1N9Bacw,123
 mteb/models/cache_wrappers/cache_backends/_hash_utils.py,sha256=HRZ8FeXSJyK3m07gnEIFduR2hU2DSJLjo4QPs5DxDGs,620
-mteb/models/cache_wrappers/cache_backends/faiss_cache.py,sha256=ejOONSD8wZACuGt455UXn_z5ZHGMpv64ORvteKNqfDw,3899
-mteb/models/cache_wrappers/cache_backends/numpy_cache.py,sha256=V275IY-0lyh2REqZjIZOgJJ7SY05yiWdHNF2kiSdRfo,8071
+mteb/models/cache_wrappers/cache_backends/faiss_cache.py,sha256=vEGoc74mONqiaO5Tmf5OMY9HWZDUJj92SObSvifKgu4,3898
+mteb/models/cache_wrappers/cache_backends/numpy_cache.py,sha256=iWp2H63TWYH-e3sxFq-4E23RIXTdKM9GKgrbVNQDAXI,8164
 mteb/models/model_implementations/__init__.py,sha256=BZDdde6ajKv-yroy9mqE2YS3Hw1KBdKoxBPg8aPTZEs,1164
 mteb/models/model_implementations/align_models.py,sha256=82zOy5yyeaaVej81NGyjT49d1Lbg174Kpa4d8iaBi4Q,4676
 mteb/models/model_implementations/amazon_models.py,sha256=bEiOzDE2V2duMv-TCAYhMeqlgI--PXGMv6CqKLUQmp0,753
@@ -1500,7 +1500,7 @@ mteb/models/model_implementations/andersborges.py,sha256=au-947iRW4F6aq57QoYWZwQ
 mteb/models/model_implementations/ara_models.py,sha256=b-Qa5q3O8M5XbkauVm7I6D6aZSU1cd9XePT6ZVvSBtk,1517
 mteb/models/model_implementations/arctic_models.py,sha256=MfYvAkcGcb3FdbvieYmiekSvZREwu2pRJ_2sbbcUIPk,11051
 mteb/models/model_implementations/b1ade_models.py,sha256=-czgy_Ym5LHAX4-f-F7YaUGqTkfwsKmTL-tiCiihLnU,1705
-mteb/models/model_implementations/bedrock_models.py,sha256=tqfQofVHbKGY163x46CPtrLiyjpyHXf6JLNIjcmCXw4,9072
+mteb/models/model_implementations/bedrock_models.py,sha256=oY6eLVUvMYlwfx7N1VvlPjPz6mCX0qISsF6VNCrMmVA,9052
 mteb/models/model_implementations/bge_models.py,sha256=JuO1FRWrsqlsM_jslQ96oVsD3FeWVD_uHBnMv8JJyNA,28033
 mteb/models/model_implementations/bica_model.py,sha256=Yx3iZrXF6ZMJS9SH5lbzNHoUWGNH3dypRtZ7dX5o7rA,1305
 mteb/models/model_implementations/blip2_models.py,sha256=C6egwozJthHmv92I0SWID3-sQCPROPJP0TzfQVKNzlo,7898
@@ -1554,9 +1554,9 @@ mteb/models/model_implementations/linq_models.py,sha256=alq0ylE8QDbpOGgIHUcs74fj
 mteb/models/model_implementations/listconranker.py,sha256=ojzBWxaCuMmfBaCZla8ECv5aq514IlABb6um5TGoYY8,4664
 mteb/models/model_implementations/llm2clip_models.py,sha256=X3W16uipaZ0t4Mco4lhhg4s9WC9lpVzg7Djq1wTQkyY,9522
 mteb/models/model_implementations/llm2vec_models.py,sha256=n86YQ8fAHU1gVtlY7tZcXq-1ab_ISxBmuk-X4MDnY4o,13348
-mteb/models/model_implementations/mcinext_models.py,sha256=T3vO9JQSmh3BICp6Y_q7j4anuA8P8LGZ4ZWnwGnF7cs,19299
+mteb/models/model_implementations/mcinext_models.py,sha256=PHMjzPQV8haaOj25YDMntmcyhNeZLPwVa4D3nMNrWkg,19301
 mteb/models/model_implementations/mdbr_models.py,sha256=AqsRZ-IDekIjq-FDWu0zx7Nk9ySJxaWTdRb8YhUZeu4,2828
-mteb/models/model_implementations/misc_models.py,sha256=0FkvheqPYh3JwM65F4CDlQKBDQQdjyMyfJPUdP1X2Ns,74780
+mteb/models/model_implementations/misc_models.py,sha256=JkJsyha-B5M8myLvHIwFUV14yo2lnSuBzHeO5fE9i74,73191
 mteb/models/model_implementations/mixedbread_ai_models.py,sha256=1-RD4M-16M-Rcf5CTD_R7LVoLv3cNFbmEjataQ__q94,10666
 mteb/models/model_implementations/mme5_models.py,sha256=V7BCGFkfZxkZ3ANJImvSFfP7in8OSfmkbqX-zXc_iF8,1574
 mteb/models/model_implementations/moco_models.py,sha256=6eEGpGTlI4StFRYsaNtXejhYE9GCqasUYCqB_SQy9cE,5714
@@ -1570,7 +1570,7 @@ mteb/models/model_implementations/nomic_models_vision.py,sha256=AzTCWbXBonUAVub0
 mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py,sha256=-5_kd9jeDcgVv9gdwWuvX_-bNQdhAxInf1Mqo8_BdS8,10653
 mteb/models/model_implementations/nvidia_models.py,sha256=r-AW1dVQbteWjexjvZgFEt_90OHNRYer_5GLuqSXRS0,26924
 mteb/models/model_implementations/octen_models.py,sha256=5z-t2O-iIFiOOLdZ_AK9f7GrVRg-9_vx3JNAG9dJNPE,8562
-mteb/models/model_implementations/openai_models.py,sha256=y1wMknrrcu1L5CNwniG0mFThPVMON1c2Fj22jkKsw7Y,9730
+mteb/models/model_implementations/openai_models.py,sha256=fE8SfSAcl20GccR8D8s-7MR9w_kO6LlN5Pm80Iwx82c,9777
 mteb/models/model_implementations/openclip_models.py,sha256=z2gQum16O0QhJPyxqKor3oO-_uWfnep6wSXqOFQQ2Q8,11969
 mteb/models/model_implementations/opensearch_neural_sparse_models.py,sha256=J5FEvKWQUiBusL6PHcrRuRRJOQ-iMwOSu1fX0pblXhk,8941
 mteb/models/model_implementations/ops_colqwen3_models.py,sha256=5vg5d1_WfVGMgtIwkh6zf2-Paum6V35XcKEvLfRyRzs,7437
@@ -1586,12 +1586,12 @@ mteb/models/model_implementations/qtack_models.py,sha256=vw_2O4ZABR-_nYV4g1Ud8bW
 mteb/models/model_implementations/querit_models.py,sha256=P7lAw5IDe47DA_5srMwGPqxjMIFuvOW0BJ7xwB4GOro,8917
 mteb/models/model_implementations/qwen3_models.py,sha256=857UnUEil9o8xcw7vSr2fMRlEegyE2Q86e5yLeRL_mQ,5517
 mteb/models/model_implementations/qzhou_models.py,sha256=mfG70JrNJCo-s3MykRn6lg9gFPcKMeMI7Y8VrBhNo7I,3684
-mteb/models/model_implementations/random_baseline.py,sha256=YsITQoLbea_Iz2X84WNGBGkhlsQ3hB7yx1oJwXghimE,7561
+mteb/models/model_implementations/random_baseline.py,sha256=SGJEjWQn6IlfHKMg3JU8iE-f7X1FuOgi3j_lx6ZEUYA,7628
 mteb/models/model_implementations/rasgaard_models.py,sha256=_uNYP_nqJcOyoKnHNcvfJnP9gRvsv7HCWhZX2LJzQ9s,1322
 mteb/models/model_implementations/reasonir_model.py,sha256=WNWGqa9wANBL9vTdcFx51TEFXz6yHq_ygK0rij3LCL8,5217
 mteb/models/model_implementations/repllama_models.py,sha256=k6BgN2Cn41p0gQ0F1FdOTQ9OXlmFgG-2RtdvzOcCSZg,7543
-mteb/models/model_implementations/rerankers_custom.py,sha256=Bjgg_UbeHarupzzCk2rdy_Dd0_W0ZsE-DCD5v1EshnI,10953
-mteb/models/model_implementations/rerankers_monot5_based.py,sha256=6por4DPCycS8gljqKRZWUNM093bjjSVvmyQ3dzj9H6U,35321
+mteb/models/model_implementations/rerankers_custom.py,sha256=WBSA7kBRqxgb1549UwRYdtYzUovdwmW8C0PWzvGR54g,8087
+mteb/models/model_implementations/rerankers_monot5_based.py,sha256=U9ChokUEDXtkoFno-o4GeT4fXEEoFtnZn2denIafxi8,34583
 mteb/models/model_implementations/richinfoai_models.py,sha256=FsXamY-bvR5LLagtKK8fP-I5oc6B_bKp_i6_xzUYL8Y,1069
 mteb/models/model_implementations/ru_sentence_models.py,sha256=W4R985LnThJ-9XFbPnTGKb3L1QnoS3i3VXBFq94DK_w,43034
 mteb/models/model_implementations/ruri_models.py,sha256=3zYOqacB3JEnGJkMGYHqFgVkbmLo4uceJs9kzV54ivU,10819
@@ -1601,7 +1601,7 @@ mteb/models/model_implementations/sarashina_embedding_models.py,sha256=vU6rPMfUb
 mteb/models/model_implementations/searchmap_models.py,sha256=XFJMAuSick-Dh0FCLTiVIjXo_kfP_VJp28Oi9WVjjlo,1994
 mteb/models/model_implementations/seed_1_6_embedding_models.py,sha256=u7L41utKM8EY-aTVVRrpvLB924FqivM3tjy83CqDPak,18699
 mteb/models/model_implementations/seed_1_6_embedding_models_1215.py,sha256=SDQUIyXIpODFjfgMMZF0rGI1_YPTvd3MpMjEvGhq0jc,36551
-mteb/models/model_implementations/seed_models.py,sha256=dHTS3NYacl83yV8Z4L_kjBUFpsDNRquMKCN2isXrMzw,14183
+mteb/models/model_implementations/seed_models.py,sha256=DCfDfSebhGCOTrNKLAl7z7y1wuJjZ0unzFx8j46V5-I,14281
 mteb/models/model_implementations/sentence_transformers_models.py,sha256=u1GkT5Xxjw0bYZvuV1wK26-bRiqv8PPCRStbu_2hYeY,26757
 mteb/models/model_implementations/shuu_model.py,sha256=q4wZk5tYSJlnNErWpDeHGSypbAwHD5yAUHnEOuTURs8,1210
 mteb/models/model_implementations/siglip_models.py,sha256=IZj9GUlqIkvJr8_aLk7FCozWZSCXHTqaCqN44B5l0oY,13425
@@ -1617,7 +1617,7 @@ mteb/models/model_implementations/vdr_models.py,sha256=IGvpE2F42IWBN5QwKSWjsAehT
 mteb/models/model_implementations/vi_vn_models.py,sha256=7hot8CF5B1UeC4WJXnAAs1C1vbqK2lq7Bw338ztKFDE,6566
 mteb/models/model_implementations/vista_models.py,sha256=mcI0La6__LasuLd5P-nkc4Z-r9X_8sYhGFLdVPGPmkw,11033
 mteb/models/model_implementations/vlm2vec_models.py,sha256=1iq2i1ZbsPINE8nXoVZsX1Km-4dTTAd6St6J38I8Tew,11951
-mteb/models/model_implementations/voyage_models.py,sha256=g7WET4MibXN6eABrmhn8uTGXdjwf5Kk4ddqQmHA7v6A,23920
+mteb/models/model_implementations/voyage_models.py,sha256=fNWs7DBS5dEMa0Hz4Ti52Pm1JrM-oPf58eRT-tlEJOc,23915
 mteb/models/model_implementations/voyage_v.py,sha256=_mJGhskJj9zeHYebEJFuYheLPb-YDyiu6Hny_5LQcAE,8280
 mteb/models/model_implementations/xyz_models.py,sha256=69JyOCQHVq19nAG3zQFi-UYYT6I7uHmvTcmRxHvjyc8,1361
 mteb/models/model_implementations/youtu_models.py,sha256=P5fh34UJZQObJAbz3Wuzqh9Nw5S7LraqxdtwAX3sDJ8,6028
@@ -2442,7 +2442,7 @@ mteb/tasks/retrieval/multilingual/ru_sci_bench_retrieval.py,sha256=In055XTnkJqZK
 mteb/tasks/retrieval/multilingual/statcan_dialogue_dataset_retrieval.py,sha256=7EBRQ173hECqHSjzL0CkRcOGeQ0IdkJfhfYfRLdTDL4,3825
 mteb/tasks/retrieval/multilingual/vdr_multilingual_retrieval.py,sha256=847aR3nw-Ba7ErgZlMnUI2DS6Wknxp55K6I-msOCAAw,4478
 mteb/tasks/retrieval/multilingual/vidore2_bench_retrieval.py,sha256=xBUWdIf3O7Sz5WUpiQCCVdyf9WQ7VtA3_-LhtnRz19M,9509
-mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py,sha256=V3jtSlWhoKR1PCvHsH0HrONy-oFghomwqihBonQs_50,17414
+mteb/tasks/retrieval/multilingual/vidore3_bench_retrieval.py,sha256=gYNA2V7lKmQ_fCsDPe7D94Q3G4TOjf1zoFKZfUj5l0M,17452
 mteb/tasks/retrieval/multilingual/web_faq_retrieval.py,sha256=TM-Q98yXZny_PKHAFNEvw9o9ET_L6VM3aNis1NJ9DgM,2686
 mteb/tasks/retrieval/multilingual/wikipedia_retrieval_multilingual.py,sha256=zyqAt63bHXNU_I37jb891pwWUyGzZUGkXCyhWlRbed8,1569
 mteb/tasks/retrieval/multilingual/wit_t2i_retrieval.py,sha256=G5JZb7FkyNpeilYNoOM_vxAacpj3Y-AhXro5uUOxsbw,4388
@@ -2641,14 +2641,14 @@ mteb/tasks/zeroshot_classification/eng/sun397.py,sha256=Nls7tXM2Svu008MmAUjt-o_N
 mteb/tasks/zeroshot_classification/eng/ucf101.py,sha256=kwNRYks-_Oe4VE3GyoHIvN-2OJ6zhkwFr76WDNL9ymU,1884
 mteb/tasks/zeroshot_classification/eng/templates/__init__.py,sha256=da1PTClDMl-IBkrSvq6JC1lnS-K_BASzCvxVhNxN5Ls,13
 mteb/types/__init__.py,sha256=O26vXPolPReX7iVUBgUsyCkCo4w8KeLs7uueQDWp3fc,1142
-mteb/types/_encoder_io.py,sha256=jfwzUBmYNOZvYepQcW3KJeiJ7vmA-JTHUCyNAtJBXK0,5851
+mteb/types/_encoder_io.py,sha256=V7m_t7ZXm3COJ4SoHP8bcr23WgjFBRCGa9AIaqAX8v4,5939
 mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=UKNokV9pu3G74MGebocU512aU_fFU9I9nPKnrG9Q0iE,1035
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=gElgSShKBXpfcqaZHhU_d2UHln1CyzUj8FN8KFun_UA,4087
-mteb-2.7.15.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.7.15.dist-info/METADATA,sha256=EoUeroRRdre5jYbplBGCJuWs-6M7cZGpzwLqSQyJKgI,14348
-mteb-2.7.15.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-mteb-2.7.15.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.7.15.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.7.15.dist-info/RECORD,,
+mteb-2.7.17.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.7.17.dist-info/METADATA,sha256=_PjxjEK4Txl-EuqMT4hDPwGIrWRp6IFqmTTGqryialU,14348
+mteb-2.7.17.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+mteb-2.7.17.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.7.17.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.7.17.dist-info/RECORD,,