mteb 2.7.12__py3-none-any.whl → 2.7.14__py3-none-any.whl

mteb/abstasks/pair_classification.py

@@ -25,6 +25,8 @@ from mteb.types.statistics import (
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from numpy.typing import NDArray
+
     from mteb._evaluators.pair_classification_evaluator import (
         PairClassificationDistances,
     )
@@ -36,7 +38,6 @@ if TYPE_CHECKING:
         TextStatistics,
     )
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -138,7 +139,7 @@ class AbsTaskPairClassification(AbsTask):
         self, similarity_scores: PairClassificationDistances, labels: list[int]
     ) -> dict[str, float]:
         logger.info("Computing metrics...")
-        np_labels = np.asarray(labels)
+        np_labels: NDArray[np.int64] = np.asarray(labels, dtype=np.int64)
         output_scores = {}
         max_scores = defaultdict(list)
         for short_name, scores, reverse in [
@@ -281,7 +282,10 @@ class AbsTaskPairClassification(AbsTask):
         )
 
     def _compute_metrics_values(
-        self, scores: list[float], labels: np.ndarray, high_score_more_similar: bool
+        self,
+        scores: list[float],
+        labels: NDArray[np.int64],
+        high_score_more_similar: bool,
     ) -> dict[str, float]:
         """Compute the metrics for the given scores and labels.
 
@@ -315,7 +319,10 @@ class AbsTaskPairClassification(AbsTask):
         )
 
     def _find_best_acc_and_threshold(
-        self, scores: list[float], labels: np.ndarray, high_score_more_similar: bool
+        self,
+        scores: list[float],
+        labels: NDArray[np.int64],
+        high_score_more_similar: bool,
     ) -> tuple[float, float]:
         rows = list(zip(scores, labels))
         rows = sorted(rows, key=lambda x: x[0], reverse=high_score_more_similar)
@@ -323,7 +330,7 @@ class AbsTaskPairClassification(AbsTask):
         max_acc = 0
         best_threshold = -1.0
         positive_so_far = 0
-        remaining_negatives = sum(np.array(labels) == 0)
+        remaining_negatives = sum(labels == 0)
 
         for i in range(len(rows) - 1):
             score, label = rows[i]
@@ -339,10 +346,9 @@ class AbsTaskPairClassification(AbsTask):
         return max_acc, best_threshold
 
     def _find_best_f1_and_threshold(
-        self, scores, labels, high_score_more_similar: bool
+        self, scores, labels: NDArray[np.int64], high_score_more_similar: bool
     ) -> tuple[float, float, float, float]:
         scores = np.asarray(scores)
-        labels = np.asarray(labels)
 
         rows = list(zip(scores, labels))
 

mteb/models/get_model_meta.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import difflib
 import logging
+import warnings
 from typing import TYPE_CHECKING, Any
 
 from mteb.models import (
@@ -122,6 +123,11 @@ def get_model(
     return model
 
 
+_MODEL_RENAMES: dict[str, str] = {
+    "bm25s": "baseline/bm25s",
+}
+
+
 def get_model_meta(
     model_name: str,
     revision: str | None = None,
@@ -139,6 +145,12 @@ def get_model_meta(
     Returns:
         A model metadata object
     """
+    if model_name in _MODEL_RENAMES:
+        new_name = _MODEL_RENAMES[model_name]
+        msg = f"The model '{model_name}' has been renamed to '{new_name}'. To prevent this warning use the new name."
+        warnings.warn(msg, DeprecationWarning, stacklevel=2)
+        model_name = new_name
+
     if model_name in MODEL_REGISTRY:
         model_meta = MODEL_REGISTRY[model_name]
 

mteb/models/model_implementations/bm25.py

@@ -129,7 +129,7 @@ def bm25_loader(model_name, **kwargs) -> SearchProtocol:
 
 bm25_s = ModelMeta(
     loader=bm25_loader,
-    name="bm25s",
+    name="baseline/bm25s",
     model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,

mteb/models/model_implementations/human.py

@@ -2,7 +2,7 @@ from mteb.models import ModelMeta
 
 human = ModelMeta(
     loader=None,
-    name="Human",
+    name="baseline/Human",
     model_type=["dense"],
     languages=["eng-Latn", "ara-Arab", "rus-Cyrl", "dan-Latn", "nob-Latn"],
     open_weights=True,

mteb/models/model_implementations/nomic_models.py

@@ -7,6 +7,7 @@ import torch
 import torch.nn.functional as F
 from packaging.version import Version
 
+from mteb.models import sentence_transformers_loader
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.models.sentence_transformer_wrapper import SentenceTransformerEncoderWrapper
 from mteb.types import PromptType
@@ -509,3 +510,42 @@ nomic_embed_text_v2_moe = ModelMeta(
       url={https://arxiv.org/abs/2502.07972},
 }""",
 )
+
+nomic_embed_code = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs={
+        "trust_remote_code": True,
+        "model_prompts": model_prompts,
+    },
+    name="nomic-ai/nomic-embed-code",
+    revision="11114029805cee545ef111d5144b623787462a52",
+    release_date="2025-03-24",
+    languages=["eng-Latn"],
+    n_parameters=7_070_619_136,
+    n_embedding_parameters=None,
+    memory_usage_mb=26972.0,
+    max_tokens=32768,
+    embed_dim=3584,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code="https://github.com/gangiswag/cornstack/",
+    public_training_data="https://huggingface.co/collections/nomic-ai/cornstack",
+    framework=["PyTorch", "Sentence Transformers", "safetensors"],
+    reference="https://huggingface.co/nomic-ai/nomic-embed-code",
+    similarity_fn_name=ScoringFunction.COSINE,
+    use_instructions=True,
+    training_datasets={"CoRNStack"},
+    adapted_from=None,
+    superseded_by=None,
+    modalities=["text"],
+    model_type=["dense"],
+    citation="""@misc{suresh2025cornstackhighqualitycontrastivedata,
+      title={CoRNStack: High-Quality Contrastive Data for Better Code Retrieval and Reranking},
+      author={Tarun Suresh and Revanth Gangi Reddy and Yifei Xu and Zach Nussbaum and Andriy Mulyar and Brandon Duderstadt and Heng Ji},
+      year={2025},
+      eprint={2412.01007},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2412.01007},
+}""",
+)

mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Any
 
 import torch
-from packaging.version import Version
+from packaging.specifiers import SpecifierSet
 from torch.utils.data import DataLoader
 from transformers import __version__ as transformers_version
 
@@ -31,18 +31,20 @@ class LlamaNemoretrieverColembed(AbsEncoder):
         model_name_or_path: str,
         revision: str,
         trust_remote_code: bool,
+        transformers_version_constraint: str | None = None,
         device_map="cuda",
         torch_dtype=torch.bfloat16,
         attn_implementation="flash_attention_2",
         **kwargs,
     ):
-        required_transformers_version = "4.49.0"
-
-        if Version(transformers_version) != Version(required_transformers_version):
-            raise RuntimeError(
-                f"transformers version {transformers_version} is not match with required "
-                f"install version {required_transformers_version} to run `nvidia/llama-nemoretriever-colembed`"
-            )
+        if transformers_version_constraint is not None:
+            spec = SpecifierSet(transformers_version_constraint)
+            if transformers_version not in spec:
+                raise RuntimeError(
+                    f"Model `{model_name_or_path}` requires transformers{transformers_version_constraint}, "
+                    f"but {transformers_version} is installed. "
+                    f"Run: pip install 'transformers{transformers_version_constraint}'"
+                )
 
         from transformers import AutoModel
 
@@ -150,10 +152,24 @@ TRAINING_DATA = {
     "wiki-ss-nq",
 }
 
+
+TRAINING_DATA_v2 = {
+    "VidoreDocVQARetrieval",
+    "VidoreInfoVQARetrieval",
+    "VidoreTatdqaRetrieval",
+    "VidoreArxivQARetrieval",
+    "docmatix-ir",
+    "VDRMultilingualRetrieval",
+    "VisRAG-Ret-Train-Synthetic-data",
+    "VisRAG-Ret-Train-In-domain-data",
+    "wiki-ss-nq",
+}
+
 llama_nemoretriever_colembed_1b_v1 = ModelMeta(
     loader=LlamaNemoretrieverColembed,
     loader_kwargs=dict(
         trust_remote_code=True,
+        transformers_version_constraint="==4.49.0",
     ),
     name="nvidia/llama-nemoretriever-colembed-1b-v1",
     model_type=["late-interaction"],
@@ -168,7 +184,7 @@ llama_nemoretriever_colembed_1b_v1 = ModelMeta(
     embed_dim=2048,
     license="https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1/blob/main/LICENSE",
     open_weights=True,
-    public_training_code="Proprietary Code",
+    public_training_code=None,
     public_training_data="https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1#training-dataset",
     framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1",
@@ -182,6 +198,7 @@ llama_nemoretriever_colembed_3b_v1 = ModelMeta(
     loader=LlamaNemoretrieverColembed,
     loader_kwargs=dict(
         trust_remote_code=True,
+        transformers_version_constraint="==4.49.0",
     ),
     name="nvidia/llama-nemoretriever-colembed-3b-v1",
     model_type=["late-interaction"],
@@ -196,7 +213,7 @@ llama_nemoretriever_colembed_3b_v1 = ModelMeta(
     embed_dim=3072,
     license="https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1/blob/main/LICENSE",
     open_weights=True,
-    public_training_code="Proprietary Code",
+    public_training_code=None,
     public_training_data="https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1#training-dataset",
     framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/nvidia/llama-nemoretriever-colembed-3b-v1",
@@ -205,3 +222,86 @@ llama_nemoretriever_colembed_3b_v1 = ModelMeta(
     training_datasets=TRAINING_DATA,
     citation=LLAMA_NEMORETRIEVER_CITATION,
 )
+
+llama_nemotron_colembed_vl_3b_v2 = ModelMeta(
+    loader=LlamaNemoretrieverColembed,
+    loader_kwargs=dict(
+        trust_remote_code=True,
+        transformers_version_constraint="==4.49.0",
+    ),
+    name="nvidia/llama-nemotron-colembed-vl-3b-v2",
+    model_type=["late-interaction"],
+    languages=["eng-Latn"],
+    revision="75f03c712cb3a252e062295f9a0966e5d95d6156",
+    release_date="2026-01-21",
+    modalities=["image", "text"],
+    n_parameters=4_407_000_000,
+    memory_usage_mb=8403,
+    max_tokens=8192,
+    embed_dim=3072,
+    license="https://huggingface.co/nvidia/llama-nemotron-colembed-vl-3b-v2/blob/main/LICENSE",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/nvidia/llama-nemotron-colembed-vl-3b-v2#training-dataset",
+    framework=["PyTorch", "Transformers", "safetensors"],
+    reference="https://huggingface.co/nvidia/llama-nemotron-colembed-vl-3b-v2",
+    similarity_fn_name="MaxSim",
+    use_instructions=True,
+    training_datasets=TRAINING_DATA,
+    citation=LLAMA_NEMORETRIEVER_CITATION,
+)
+
+nemotron_colembed_vl_4b_v2 = ModelMeta(
+    loader=LlamaNemoretrieverColembed,
+    loader_kwargs=dict(
+        trust_remote_code=True,
+        transformers_version_constraint="==5.0.0rc0",
+    ),
+    name="nvidia/nemotron-colembed-vl-4b-v2",
+    revision="823b1625c15fe3da73fa094205e538a7a2301a2a",
+    languages=["eng-Latn"],
+    release_date="2026-01-07",
+    modalities=["image", "text"],
+    n_parameters=4_800_000_000,
+    memory_usage_mb=9206,
+    max_tokens=262144,
+    embed_dim=2560,
+    license="https://huggingface.co/nvidia/nemotron-colembed-vl-4b-v2/blob/main/LICENSE",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/nvidia/nemotron-colembed-vl-4b-v2#training-dataset",
+    framework=["PyTorch", "Transformers"],
+    reference="https://huggingface.co/nvidia/nemotron-colembed-vl-4b-v2",
+    similarity_fn_name="MaxSim",
+    use_instructions=True,
+    training_datasets=TRAINING_DATA_v2,
+    citation=LLAMA_NEMORETRIEVER_CITATION,
+)
+
+
+nemotron_colembed_vl_8b_v2 = ModelMeta(
+    loader=LlamaNemoretrieverColembed,
+    loader_kwargs=dict(
+        trust_remote_code=True,
+        transformers_version_constraint="==5.0.0rc0",
+    ),
+    name="nvidia/nemotron-colembed-vl-8b-v2",
+    revision="6cbe43579dda6237768fc373768ad372cc5cdfec",
+    languages=["eng-Latn"],
+    release_date="2026-01-07",
+    modalities=["image", "text"],
+    n_parameters=8_700_000_000,
+    memory_usage_mb=16722,
+    max_tokens=262144,
+    embed_dim=4096,
+    license="https://huggingface.co/nvidia/nemotron-colembed-vl-8b-v2/blob/main/LICENSE",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/nvidia/nemotron-colembed-vl-8b-v2#training-dataset",
+    framework=["PyTorch", "Transformers"],
+    reference="https://huggingface.co/nvidia/nemotron-colembed-vl-8b-v2",
+    similarity_fn_name="MaxSim",
+    use_instructions=True,
+    training_datasets=TRAINING_DATA_v2,
+    citation=LLAMA_NEMORETRIEVER_CITATION,
+)

mteb/models/model_implementations/opensearch_neural_sparse_models.py

@@ -134,7 +134,7 @@ class SparseEncoderWrapper(AbsEncoder):
 
 opensearch_neural_sparse_encoding_doc_v3_gte = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v3-gte",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="a8abaa916125ee512a7a8f4d706d07eb0128a8e6",
@@ -161,7 +161,7 @@ opensearch_neural_sparse_encoding_doc_v3_gte = ModelMeta(
 
 opensearch_neural_sparse_encoding_doc_v3_distill = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v3-distill",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="babf71f3c48695e2e53a978208e8aba48335e3c0",
@@ -184,7 +184,7 @@ opensearch_neural_sparse_encoding_doc_v3_distill = ModelMeta(
 
 opensearch_neural_sparse_encoding_doc_v2_distill = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v2-distill",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="8921a26c78b8559d6604eb1f5c0b74c079bee38f",
@@ -208,7 +208,7 @@ opensearch_neural_sparse_encoding_doc_v2_distill = ModelMeta(
 
 opensearch_neural_sparse_encoding_doc_v2_mini = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v2-mini",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="4af867a426867dfdd744097531046f4289a32fdd",
@@ -231,7 +231,7 @@ opensearch_neural_sparse_encoding_doc_v2_mini = ModelMeta(
 
 opensearch_neural_sparse_encoding_doc_v1 = ModelMeta(
     name="opensearch-project/opensearch-neural-sparse-encoding-doc-v1",
-    model_type=["dense"],
+    model_type=["sparse"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="98cdcbd72867c547f72f2b7b7bed9cdf9f09922d",

mteb/models/model_implementations/ops_colqwen3_models.py

@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import torch
+from tqdm.auto import tqdm
+from transformers import AutoModel, AutoProcessor
+
+from mteb._requires_package import requires_image_dependencies
+from mteb.models.abs_encoder import AbsEncoder
+from mteb.models.model_meta import ModelMeta, ScoringFunction
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import Array, BatchedInput, PromptType
+
+
+class OpsColQwen3Wrapper(AbsEncoder):
+    """Wrapper for OpsColQwen3 model."""
+
+    def __init__(
+        self,
+        model_name: str = "OpenSearch-AI/Ops-Colqwen3-4B",
+        revision: str | None = None,
+        device: str | None = None,
+        attn_implementation: str | None = None,
+        **kwargs,
+    ):
+        requires_image_dependencies()
+        from transformers.utils.import_utils import is_flash_attn_2_available
+
+        if attn_implementation is None:
+            attn_implementation = (
+                "flash_attention_2" if is_flash_attn_2_available() else None
+            )
+
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+
+        self.mdl = AutoModel.from_pretrained(
+            model_name,
+            device_map=self.device,
+            attn_implementation=attn_implementation,
+            trust_remote_code=True,
+            revision=revision,
+            **kwargs,
+        )
+        self.mdl.eval()
+
+        self.processor = AutoProcessor.from_pretrained(
+            model_name,
+            trust_remote_code=True,
+        )
+
+    def encode(
+        self,
+        inputs: DataLoader[BatchedInput],
+        *,
+        task_metadata: TaskMetadata,
+        hf_split: str,
+        hf_subset: str,
+        prompt_type: PromptType | None = None,
+        **kwargs: Any,
+    ) -> Array:
+        text_embeddings = None
+        image_embeddings = None
+
+        if "text" in inputs.dataset.features:
+            text_embeddings = self.get_text_embeddings(inputs, **kwargs)
+        if "image" in inputs.dataset.features:
+            image_embeddings = self.get_image_embeddings(inputs, **kwargs)
+
+        if text_embeddings is not None and image_embeddings is not None:
+            if len(text_embeddings) != len(image_embeddings):
+                raise ValueError(
+                    "The number of texts and images must have the same length"
+                )
+            fused_embeddings = text_embeddings + image_embeddings
+            return fused_embeddings
+        elif text_embeddings is not None:
+            return text_embeddings
+        elif image_embeddings is not None:
+            return image_embeddings
+        raise ValueError("No text or image inputs found")
+
+    def encode_input(self, inputs):
+        return self.mdl(**inputs)
+
+    def get_image_embeddings(
+        self,
+        images: DataLoader,
+        batch_size: int = 32,
+        **kwargs,
+    ) -> torch.Tensor:
+        import torchvision.transforms.functional as F
+        from PIL import Image
+
+        all_embeds = []
+
+        with torch.no_grad():
+            for batch in tqdm(images, desc="Encoding images"):
+                # batch may be list of tensors or PIL
+                imgs = [
+                    F.to_pil_image(b.to(self.device))
+                    if not isinstance(b, Image.Image)
+                    else b
+                    for b in batch["image"]
+                ]
+                inputs = self.processor.process_images(imgs)
+                inputs = {k: v.to(self.device) for k, v in inputs.items()}
+                outs = self.encode_input(inputs)
+                all_embeds.extend(outs.cpu().to(torch.float32))
+
+        padded = torch.nn.utils.rnn.pad_sequence(
+            all_embeds, batch_first=True, padding_value=0
+        )
+        return padded
+
+    def get_text_embeddings(
+        self,
+        texts: DataLoader,
+        batch_size: int = 32,
+        **kwargs,
+    ) -> torch.Tensor:
+        all_embeds = []
+
+        with torch.no_grad():
+            for batch in tqdm(texts, desc="Encoding texts"):
+                batch_texts = batch["text"]
+                inputs = self.processor.process_queries(batch_texts)
+                inputs = {k: v.to(self.device) for k, v in inputs.items()}
+                outs = self.encode_input(inputs)
+                all_embeds.extend(outs.cpu().to(torch.float32))
+
+        padded = torch.nn.utils.rnn.pad_sequence(
+            all_embeds, batch_first=True, padding_value=0
+        )
+        return padded
+
+    def similarity(self, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
+        return self.processor.score_multi_vector(a, b, device=self.device)
+
+
+OPS_COLQWEN3_TRAINING_DATA = {
+    "VDRMultilingualRetrieval",
+    # from https://huggingface.co/datasets/vidore/colpali_train_set
+    "VidoreDocVQARetrieval",
+    "VidoreInfoVQARetrieval",
+    "VidoreTatdqaRetrieval",
+    "VidoreArxivQARetrieval",
+    "docmatix-ir",
+    "HotpotQA",
+    "FEVER",
+    "NQ",
+    "MIRACLRetrieval",
+    "WebInstructSub",  # MathStackExchange and ScienceStackExchange only
+    "MrTyDi",
+}
+
+multilingual_langs = [
+    "afr-Latn",
+    "ara-Arab",
+    "aze-Latn",
+    "bel-Cyrl",
+    "bul-Cyrl",
+    "ben-Beng",
+    "cat-Latn",
+    "ceb-Latn",
+    "ces-Latn",
+    "cym-Latn",
+    "dan-Latn",
+    "deu-Latn",
+    "ell-Grek",
+    "eng-Latn",
+    "spa-Latn",
+    "est-Latn",
+    "eus-Latn",
+    "fas-Arab",
+    "fin-Latn",
+    "fra-Latn",
+    "glg-Latn",
+    "guj-Gujr",
+    "heb-Hebr",
+    "hin-Deva",
+    "hrv-Latn",
+    "hat-Latn",
+    "hun-Latn",
+    "hye-Armn",
+    "ind-Latn",
+    "isl-Latn",
+    "ita-Latn",
+    "jpn-Jpan",
+    "jav-Latn",
+    "kat-Geor",
+    "kaz-Cyrl",
+    "khm-Khmr",
+    "kan-Knda",
+    "kor-Hang",
+    "kir-Cyrl",
+    "lao-Laoo",
+    "lit-Latn",
+    "lav-Latn",
+    "mkd-Cyrl",
+    "mal-Mlym",
+    "mon-Cyrl",
+    "mar-Deva",
+    "msa-Latn",
+    "mya-Mymr",
+    "nep-Deva",
+    "nld-Latn",
+    "nor-Latn",
+    "nob-Latn",
+    "nno-Latn",
+    "pan-Guru",
+    "pol-Latn",
+    "por-Latn",
+    "que-Latn",
+    "ron-Latn",
+    "rus-Cyrl",
+    "sin-Sinh",
+    "slk-Latn",
+    "slv-Latn",
+    "swa-Latn",
+    "tam-Taml",
+    "tel-Telu",
+    "tha-Thai",
+    "tgl-Latn",
+    "tur-Latn",
+    "ukr-Cyrl",
+    "urd-Arab",
+    "vie-Latn",
+    "yor-Latn",
+    "zho-Hans",
+]
+
+OPS_COLQWEN3_CITATION = """
+@misc{ops_colqwen3_4b,
+  author       = {OpenSearch-AI},
+  title        = {Ops-ColQwen3: State-of-the-Art Multimodal Embedding Model for Visual Document Retrieval},
+  year         = {2026},
+  url          = {https://huggingface.co/OpenSearch-AI/Ops-ColQwen3-4B},
+}"""
+
+ops_colqwen3_4b = ModelMeta(
+    loader=OpsColQwen3Wrapper,
+    name="OpenSearch-AI/Ops-Colqwen3-4B",
+    loader_kwargs=dict(dtype=torch.float16, trust_remote_code=True),
+    languages=multilingual_langs,
+    revision="4894b7d451ff33981650acc693bb482dbef302d3",
+    release_date="2026-01-24",
+    modalities=["image", "text"],
+    n_parameters=4_800_000_000,
+    memory_usage_mb=9206,
+    max_tokens=32768,
+    embed_dim=2560,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code="https://github.com/illuin-tech/colpali",
+    public_training_data=None,
+    framework=["PyTorch"],
+    reference="https://huggingface.co/OpenSearch-AI/Ops-Colqwen3-4B",
+    similarity_fn_name=ScoringFunction.MAX_SIM,
+    use_instructions=True,
+    training_datasets=OPS_COLQWEN3_TRAINING_DATA,
+    citation=OPS_COLQWEN3_CITATION,
+)

mteb/models/model_implementations/querit_models.py

@@ -0,0 +1,245 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+import torch
+from tqdm.auto import tqdm
+
+from mteb.models.model_meta import ModelMeta
+
+from .rerankers_custom import RerankerWrapper
+
+if TYPE_CHECKING:
+    from torch.utils.data import DataLoader
+
+    from mteb.abstasks.task_metadata import TaskMetadata
+    from mteb.types import BatchedInput, PromptType
+
+logger = logging.getLogger(__name__)
+
+
+class QueritWrapper(RerankerWrapper):
+    """
+    Multi-GPU / multi-process reranker wrapper for mteb.mteb evaluation.
+    Supports flattening all query-passage pairs without explicit grouping.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(model_name, **kwargs)
+        from transformers import AutoModel, AutoTokenizer
+
+        if not self.device:
+            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        model_args = {}
+        if self.fp_options:
+            model_args["torch_dtype"] = self.fp_options
+        self.model = AutoModel.from_pretrained(
+            model_name, trust_remote_code=True, **model_args
+        )
+        logger.info(f"Using model {model_name}")
+
+        self.model.to(self.device)
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_name, trust_remote_code=True
+        )
+        if "[CLS]" not in self.tokenizer.get_vocab():
+            raise ValueError("Tokenizer missing required special token '[CLS]'")
+        self.cls_token_id = self.tokenizer.convert_tokens_to_ids("[CLS]")
+        self.pad_token_id = self.tokenizer.pad_token_id or 0
+
+        self.max_length = (
+            min(kwargs.get("max_length", 4096), self.tokenizer.model_max_length) - 1
+        )  # sometimes it's a v large number/max int
+        logger.info(f"Using max_length of {self.max_length}, 1 token for [CLS]")
+        self.model.eval()
+
+    def process_inputs(
+        self,
+        pairs: list[str],
+    ) -> dict[str, torch.Tensor]:
+        """
+        Encode a batch of (query, document) pairs:
+        - Concatenate prompt + Query + Content
+        - Append [CLS] at the end
+        - Left-pad to max_length
+        - Generate custom attention mask based on block types
+        """
+        # Construct input texts
+        enc = self.tokenizer(
+            pairs,
+            add_special_tokens=False,
+            truncation=True,
+            max_length=self.max_length,
+            padding=False,
+        )
+
+        input_ids_list: list[list[int]] = []
+        attn_mask_list: list[torch.Tensor] = []
+
+        for ids in enc["input_ids"]:
+            # Append [CLS] token
+            ids = ids + [self.cls_token_id]
+            block_types = [1] * (len(ids) - 1) + [2]  # content + CLS
+
+            # Pad or truncate
+            if len(ids) < self.max_length:
+                pad_len = self.max_length - len(ids)
+                ids = [self.pad_token_id] * pad_len + ids
+                block_types = [0] * pad_len + block_types
+            else:
+                ids = ids[-self.max_length :]
+                block_types = block_types[-self.max_length :]
+
+            attn = self.compute_mask_content_cls(block_types)
+            input_ids_list.append(ids)
+            attn_mask_list.append(attn)
+
+        input_ids = torch.tensor(input_ids_list, dtype=torch.long, device=self.device)
+        attention_mask = torch.stack(attn_mask_list, dim=0).to(self.device)
+
+        return {"input_ids": input_ids, "attention_mask": attention_mask}
+
+    @torch.inference_mode()
+    def predict(
+        self,
+        inputs1: DataLoader[BatchedInput],
+        inputs2: DataLoader[BatchedInput],
+        *,
+        task_metadata: TaskMetadata,
+        hf_split: str,
+        hf_subset: str,
+        prompt_type: PromptType | None = None,
+        **kwargs: Any,
+    ) -> list[float]:
+        """
+        Predict relevance scores for query-passage pairs.
+        Supports both single-process and multi-process/multi-GPU modes.
+        """
+        # Flatten all pairs from mteb.mteb DataLoaders
+        queries = [text for batch in inputs1 for text in batch["text"]]
+        passages = [text for batch in inputs2 for text in batch["text"]]
+
+        instructions = None
+        if "instruction" in inputs2.dataset.features:
+            instructions = [text for batch in inputs1 for text in batch["instruction"]]
+
+        num_pairs = len(queries)
+        if num_pairs == 0:
+            return []
+        final_scores: list[float] = []
+
+        batch_size = kwargs.get("batch_size", self.batch_size)
+        with tqdm(total=num_pairs, desc="Scoring", ncols=100) as pbar:
+            for start in range(0, num_pairs, batch_size):
+                end = min(start + batch_size, num_pairs)
+                batch_q = queries[start:end]
+                batch_d = passages[start:end]
+
+                batch_instructions = (
+                    instructions[start:end]
+                    if instructions is not None
+                    else [None] * len(batch_q)
+                )
+                pairs = [
+                    self.format_instruction(instr, query, doc)
+                    for instr, query, doc in zip(batch_instructions, batch_q, batch_d)
+                ]
+                enc = self.process_inputs(pairs)
+                out = self.model(**enc)
+                scores = out["score"].squeeze(-1).detach().float().cpu().tolist()
+
+                if not isinstance(scores, list):
+                    scores = [scores]
+
+                final_scores.extend(scores)
+                pbar.update(len(scores))
+
+        return final_scores
+
+    @staticmethod
+    def format_instruction(instruction: str | None, query: str, doc: str) -> str:
+        if instruction is None:
+            output = f"Judge whether the Content meets the requirements based on the Query. Query: {query}; Content: {doc}"
+        else:
+            output = f"{instruction} Query: {query}; Content: {doc}"
+        return output
+
+    @staticmethod
+    def compute_mask_content_cls(block_types: list[int]) -> torch.Tensor:
+        """
+        Create custom attention mask based on token block types:
+        - 0: padding   → ignored
+        - 1: content   → causal attention to previous content only
+        - 2: [CLS]     → causal attention to all non-padding tokens
+
+        Args:
+            block_types: List of token types for one sequence
+
+        Returns:
+            [1, seq_len, seq_len] boolean attention mask (True = allowed to attend)
+        """
+        pos = torch.tensor(block_types, dtype=torch.long)
+        n = pos.shape[0]
+        if n == 0:
+            return torch.empty((0, 0), dtype=torch.bool, device=pos.device)
+
+        row_types = pos.view(n, 1)
+        col_types = pos.view(1, n)
+
+        row_idx = torch.arange(n, device=pos.device).view(n, 1)
+        col_idx = torch.arange(n, device=pos.device).view(1, n)
+        causal_mask = col_idx <= row_idx
+
+        # Content tokens only attend to previous content
+        mask_content = (row_types == 1) & (col_types == 1) & causal_mask
+
+        # [CLS] attends to all non-pad tokens (causal)
+        mask_cls = (row_types == 2) & (col_types != 0) & causal_mask
+
+        type_mask = mask_content | mask_cls
+        return type_mask.unsqueeze(0)
+
+
+querit_reranker_training_data = {
+    "MIRACLRanking",  # https://huggingface.co/datasets/mteb/MIRACLReranking
+    "MrTidyRetrieval",  # https://huggingface.co/datasets/mteb/mrtidy
+    "ruri-v3-dataset-reranker",  # https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-reranker
+    "MultiLongDocReranking",  # https://huggingface.co/datasets/Shitao/MLDR
+    "MindSmallReranking",  # https://huggingface.co/datasets/mteb/MindSmallReranking
+    "MSMARCO",  # https://huggingface.co/datasets/mteb/msmarco
+    "CQADupStack",  # https://huggingface.co/datasets/mteb/cqadupstack-*
+    "AskUbuntuDupQuestions",  # https://github.com/taolei87/askubuntu & The corpus and queries that overlap with mteb/askubuntudupquestions-reranking have been removed.
+    "T2Reranking",  # https://huggingface.co/datasets/THUIR/T2Ranking & The corpus and queries that overlap with mteb/T2Reranking have been removed.
+}
+
+model_meta = ModelMeta(
+    loader=QueritWrapper,
+    loader_kwargs={
+        "fp_options": "bfloat16",
+    },
+    name="Querit/Querit",
+    model_type=["cross-encoder"],
+    languages=["eng-Latn"],
+    open_weights=True,
+    revision="5ad2649cc4defb7e1361262260e9a781f14b08bc",
+    release_date="2026-01-24",
+    n_parameters=4919636992,
+    n_embedding_parameters=131907584,
+    embed_dim=1024,
+    memory_usage_mb=9383.0,
+    max_tokens=4096,
+    reference="https://huggingface.co/Querit/Querit",
+    similarity_fn_name=None,
+    training_datasets=querit_reranker_training_data,
+    license="apache-2.0",
+    framework=["PyTorch"],
+    use_instructions=None,
+    public_training_code=None,
+    public_training_data=None,
+    citation=None,
+)

mteb/models/model_meta.py

@@ -71,7 +71,7 @@ FRAMEWORKS = Literal[
     "Transformers",
 ]
 
-MODEL_TYPES = Literal["dense", "cross-encoder", "late-interaction"]
+MODEL_TYPES = Literal["dense", "cross-encoder", "late-interaction", "sparse"]
 
 
 class ScoringFunction(HelpfulStrEnum):
@@ -266,7 +266,7 @@ class ModelMeta(BaseModel):
     @field_validator("name")
     @classmethod
     def _check_name(cls, v: str | None) -> str | None:
-        if v is None or v in ("bm25s", "Human"):
+        if v is None:
             return v
         if "/" not in v:
             raise ValueError(
@@ -302,6 +302,121 @@ class ModelMeta(BaseModel):
             raise ValueError("Model name is not set")
         return self.name.replace("/", "__").replace(" ", "_")
 
+    @classmethod
+    def _detect_cross_encoder_or_dense(
+        cls,
+        model_name: str,
+        revision: str | None,
+        sentence_transformers_loader: Callable[..., MTEBModels],
+        cross_encoder_loader: Callable[..., MTEBModels],
+    ) -> tuple[Callable[..., MTEBModels] | None, MODEL_TYPES]:
+        """Detect if model is CrossEncoder or default to dense."""
+        config = _get_json_from_hub(
+            model_name, "config.json", "model", revision=revision
+        )
+
+        if not config:
+            logger.warning(
+                f"Could not load config.json for {model_name}. "
+                "Defaulting to SentenceTransformer loader."
+            )
+            return sentence_transformers_loader, "dense"
+
+        architectures = config.get("architectures", [])
+
+        is_cross_encoder = any(
+            arch.endswith("ForSequenceClassification") for arch in architectures
+        )
+        if is_cross_encoder:
+            return cross_encoder_loader, "cross-encoder"
+
+        if cls._is_causal_lm_reranker(architectures, config, model_name):
+            return cross_encoder_loader, "cross-encoder"
+
+        logger.info(
+            f"Model {model_name} does not have modules.json or recognized architecture. "
+            "Defaulting to SentenceTransformer loader."
+        )
+        return sentence_transformers_loader, "dense"
+
+    @staticmethod
+    def _is_causal_lm_reranker(
+        architectures: list[str], config: dict[str, Any], model_name: str
+    ) -> bool:
+        """Check if model is a CausalLM-style reranker."""
+        is_causal_lm = any(arch.endswith("ForCausalLM") for arch in architectures)
+
+        if not is_causal_lm:
+            return False
+
+        num_labels = config.get("num_labels", 0)
+        model_name_lower = model_name.lower()
+
+        return (
+            num_labels > 0
+            or "rerank" in model_name_lower
+            or "cross-encoder" in model_name_lower
+        )
+
+    @classmethod
+    def _detect_model_type_and_loader(
+        cls,
+        model_name: str | None,
+        revision: str | None = None,
+    ) -> tuple[Callable[..., MTEBModels] | None, MODEL_TYPES]:
+        """Detect the model type and appropriate loader based on HuggingFace Hub configuration files.
+
+        This follows the Sentence Transformers architecture detection logic:
+        1. Check for modules.json - If present, model is a SentenceTransformer (dense encoder)
+        2. If no modules.json, check config.json for architecture:
+            - ForSequenceClassification → CrossEncoder
+            - CausalLM with reranking indicators → CrossEncoder
+        3. Default to dense (SentenceTransformer) if no clear indicators are found
+
+        Detection for CausalLM-style rerankers:
+        - Model has ForCausalLM architecture AND
+        - Has num_labels > 0 in config, OR
+        - Model name contains "rerank" or "cross-encoder"
+
+        Args:
+            model_name: The HuggingFace model name (can be None)
+            revision: The model revision
+
+        Returns:
+            A tuple of (loader_function, model_type) where:
+            - loader_function: A callable that returns MTEBModels, or None if model doesn't exist
+            - model_type: One of "dense", "cross-encoder", or "late-interaction"
+        """
+        from mteb.models import CrossEncoderWrapper, sentence_transformers_loader
+
+        if not model_name or not _repo_exists(model_name):
+            return sentence_transformers_loader, "dense"
+
+        try:
+            modules_config = _get_json_from_hub(
+                model_name, "modules.json", "model", revision=revision
+            )
+
+            if (
+                modules_config
+            ):  # SentenceTransformer/SparseEncoder (Not support for now)
+                return sentence_transformers_loader, "dense"
+            else:
+                return cls._detect_cross_encoder_or_dense(
+                    model_name,
+                    revision,
+                    sentence_transformers_loader,
+                    cross_encoder_loader=CrossEncoderWrapper,
+                )
+
+        except Exception as e:
+            logger.warning(
+                f"Error detecting model type for {model_name}: {e}. "
+                "Defaulting to SentenceTransformer loader."
+            )
+
+        return sentence_transformers_loader, "dense"
+
     @classmethod
     def _from_hub(
         cls,
@@ -319,9 +434,11 @@ class ModelMeta(BaseModel):
         Returns:
             The generated ModelMeta.
         """
-        from mteb.models import sentence_transformers_loader
+        loader: Callable[..., MTEBModels] | None
+        model_type: MODEL_TYPES
+
+        loader, model_type = cls._detect_model_type_and_loader(model_name, revision)
 
-        loader = sentence_transformers_loader
         frameworks: list[FRAMEWORKS] = ["PyTorch"]
         model_license = None
         reference = None
@@ -363,6 +480,7 @@ class ModelMeta(BaseModel):
         return cls(
             loader=loader,
             name=model_name or "no_model_name/available",
+            model_type=[model_type],
             revision=revision or "no_revision_available",
             reference=reference,
             release_date=release_date,

mteb/results/model_result.py

@@ -17,6 +17,7 @@ from .task_result import TaskError, TaskResult
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable
+    from pathlib import Path
 
     from mteb.abstasks.abstask import AbsTask
     from mteb.abstasks.task_metadata import (
@@ -417,3 +418,25 @@ class ModelResult(BaseModel):
         if not mods:
             mods = self.default_modalities
         return list(set(mods))
+
+    def to_disk(self, path: Path) -> None:
+        """Save ModelResult to disk as JSON.
+
+        Args:
+            path: The path to the file to save.
+        """
+        with path.open("w") as f:
+            f.write(self.model_dump_json(indent=2))
+
+    @classmethod
+    def from_disk(cls, path: Path) -> ModelResult:
+        """Load ModelResult from disk.
+
+        Args:
+            path: The path to the JSON file to load.
+
+        Returns:
+            The loaded ModelResult object.
+        """
+        with path.open("r", encoding="utf-8") as f:
+            return cls.model_validate_json(f.read())

mteb/results/task_result.py

@@ -337,16 +337,16 @@ class TaskResult(BaseModel):
             The loaded TaskResult object.
         """
         with path.open("r", encoding="utf-8") as f:
-            data = json.load(f)
+            json_str = f.read()
 
         if not load_historic_data:
             try:
-                return cls.model_validate(data)
+                return cls.model_validate_json(json_str)
             except Exception as e:
                 raise ValueError(
                     f"Error loading TaskResult from disk. You can try to load historic data by setting `load_historic_data=True`. Error: {e}"
                 )
-
+        data = json.loads(json_str)
         pre_1_11_load = (
             (
                 "mteb_version" in data
@@ -357,7 +357,7 @@ class TaskResult(BaseModel):
         )  # assume it is before 1.11.0 if the version is not present
 
         try:
-            obj: TaskResult = cls.model_validate(data)
+            obj: TaskResult = cls.model_validate_json(json_str)
         except Exception as e:
             if not pre_1_11_load:
                 raise e

mteb/tasks/pair_classification/fas/fars_tail.py

@@ -1,5 +1,3 @@
-import datasets
-
 from mteb.abstasks.pair_classification import AbsTaskPairClassification
 from mteb.abstasks.task_metadata import TaskMetadata
 
@@ -8,8 +6,8 @@ class FarsTail(AbsTaskPairClassification):
     metadata = TaskMetadata(
         name="FarsTail",
         dataset={
-            "path": "azarijafari/FarsTail",
-            "revision": "7335288588f14e5a687d97fc979194c2abe6f4e7",
+            "path": "mteb/FarsTail",
+            "revision": "0fa0863dc160869b5a2d78803b4440ea3c671ff5",
         },
         description="This dataset, named FarsTail, includes 10,367 samples which are provided in both the Persian language as well as the indexed format to be useful for non-Persian researchers. The samples are generated from 3,539 multiple-choice questions with the least amount of annotator interventions in a way similar to the SciTail dataset",
         reference="https://link.springer.com/article/10.1007/s00500-023-08959-3",
@@ -37,33 +35,3 @@ class FarsTail(AbsTaskPairClassification):
 }
 """,  # after removing neutral
     )
-
-    def load_data(self, num_proc: int = 1, **kwargs) -> None:
-        if self.data_loaded:
-            return
-        path = self.metadata.dataset["path"]
-        revision = self.metadata.dataset["revision"]
-        data_files = {
-            "test": f"https://huggingface.co/datasets/{path}/resolve/{revision}/data/Test-word.csv"
-        }
-        self.dataset = datasets.load_dataset(
-            "csv", data_files=data_files, delimiter="\t"
-        )
-        self.dataset_transform()
-        self.data_loaded = True
-
-    def dataset_transform(self, num_proc: int = 1):
-        _dataset = {}
-        self.dataset = self.dataset.filter(lambda x: x["label"] != "n")
-        self.dataset = self.dataset.map(
-            lambda example: {"label": 1 if example["label"] == "e" else 0}
-        )
-        for split in self.metadata.eval_splits:
-            _dataset[split] = [
-                {
-                    "sentence1": self.dataset[split]["premise"],
-                    "sentence2": self.dataset[split]["hypothesis"],
-                    "labels": self.dataset[split]["label"],
-                }
-            ]
-        self.dataset = _dataset

{mteb-2.7.12.dist-info → mteb-2.7.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.7.12
+Version: 2.7.14
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>
@@ -32,8 +32,6 @@ Requires-Dist: rich>=0.0.0
 Requires-Dist: pytrec-eval-terrier>=0.5.6
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: polars>=0.20.22
-Requires-Dist: torch; python_full_version < "3.14"
-Requires-Dist: torch>=2.9.0; python_full_version >= "3.14"
 Provides-Extra: image
 Requires-Dist: torchvision>0.2.1; extra == "image"
 Requires-Dist: transformers[torch-vision,vision]; extra == "image"

{mteb-2.7.12.dist-info → mteb-2.7.14.dist-info}/RECORD

@@ -40,7 +40,7 @@ mteb/abstasks/clustering.py,sha256=I8vre2f2FJFagzJEYf6hKDo3Y28xU29J_O-MhfqWqSI,1
 mteb/abstasks/clustering_legacy.py,sha256=sbx8K6paccvzDPnmhgNE_UJE83orAJnQm3NGr-Ktjfs,9184
 mteb/abstasks/dataset_card_template.md,sha256=aD6l8qc3_jxwoIGJNYLzse-jpRa8hu92AxpnUtNgges,5122
 mteb/abstasks/multilabel_classification.py,sha256=rFa_Pw2OsUzqhZS-jh2zFD7I-TNl8bVNJ-DW7EpPapU,9708
-mteb/abstasks/pair_classification.py,sha256=MpjOeIcM_RMqSpkXc8PFDAwDgmb54gEqj6ZOUcOa1aQ,14122
+mteb/abstasks/pair_classification.py,sha256=RVV5WUjs18N5PbWpyxakDNEd1UlRc4ON9I0OjD26Z78,14231
 mteb/abstasks/regression.py,sha256=ZuMZfOwU3G4hr__eHsgdagKKdrbN4-wQMLz45jr9YUc,8946
 mteb/abstasks/retrieval.py,sha256=BPyRibStAD70JfR0Z1x-VVVfzJDRVSmbOS6uREfpmok,27743
 mteb/abstasks/retrieval_dataset_loaders.py,sha256=p0y1nrWlUrt_aeoR4ocDLEQMLuD_SlMH0gBiUsOwrww,9983
@@ -1479,9 +1479,9 @@ mteb/leaderboard/table.py,sha256=U5mWtrVUTk_6t8T4KAp5qlbFgKh1PD0iKICqNMfhsoY,104
 mteb/leaderboard/text_segments.py,sha256=iMIkS04QQjPbT-SkU0x6fOcS8xRbUYevryu9HydipKM,6570
 mteb/models/__init__.py,sha256=ABTuoqiBjBtBWW3LYY7ItBHdylR6jWoy06HH0g6j6fU,910
 mteb/models/abs_encoder.py,sha256=We9HlwWP61P4cMyZ080gywvDErA1eVsU9t46PtcNrCM,16830
-mteb/models/get_model_meta.py,sha256=h6Z2Q3D_L4BeJbi1gPHTSgelbQjsQn1ALpI8uwNa0ac,6832
+mteb/models/get_model_meta.py,sha256=WRWnVIT1n7i63BYlBRB-8BpYNtHxn7KMJOm5mzlJ8xI,7211
 mteb/models/instruct_wrapper.py,sha256=XAvvbPnXiTxKhFbmusm2uS8E9BMq8QXRSzQQI1jqKzE,9781
-mteb/models/model_meta.py,sha256=5seQwce893SbgSywFsphLuZGQ9rRn1fLl9fj1SfKnXE,32887
+mteb/models/model_meta.py,sha256=E6mBB_inz9kMO8z3ixgGuB9QKWUYYzW44gSZwnY3ZbI,37316
 mteb/models/models_protocols.py,sha256=HTB4-SYa3SeJXMMSA8o05lHTiLBbq314VW60K_PfcZY,9509
 mteb/models/search_wrappers.py,sha256=PXE1VVDWUd0LgTPJ-FxqIbGpIDWLRKo5CjrwIuu5nzw,21567
 mteb/models/sentence_transformer_wrapper.py,sha256=RsOxj-b7qzeYcxUTVJyb-lZDY4bINl4jEAEkPvKYB10,13578
@@ -1505,7 +1505,7 @@ mteb/models/model_implementations/bge_models.py,sha256=JuO1FRWrsqlsM_jslQ96oVsD3
 mteb/models/model_implementations/bica_model.py,sha256=Yx3iZrXF6ZMJS9SH5lbzNHoUWGNH3dypRtZ7dX5o7rA,1305
 mteb/models/model_implementations/blip2_models.py,sha256=C6egwozJthHmv92I0SWID3-sQCPROPJP0TzfQVKNzlo,7898
 mteb/models/model_implementations/blip_models.py,sha256=D_9e7C8GXGST8k7dMJL20x984vMeqbITu36XASi-iUU,12149
-mteb/models/model_implementations/bm25.py,sha256=-xpeMqYi0_Ne5rL_Y1qAlYvloLlQzNDUav2ESVGHAhA,5217
+mteb/models/model_implementations/bm25.py,sha256=IAKU8syYesN7seRQLII-c1ACq6BRz5Ql6nEQEXYWLwQ,5226
 mteb/models/model_implementations/bmretriever_models.py,sha256=rijCIzX6nO5kNXqxEFbZrV7bsZtmKs8RIkMqa5cPWTk,7078
 mteb/models/model_implementations/cadet_models.py,sha256=gXIfW9MkGYFhOhsrq5a_tQcPuth13Dh1dO1KySwVxyo,2305
 mteb/models/model_implementations/cde_models.py,sha256=l4E6h1hcsNY1GTXoCgQDoeG5dRcEl7JTOiiWmp6FYqg,9373
@@ -1537,7 +1537,7 @@ mteb/models/model_implementations/granite_vision_embedding_models.py,sha256=jxyR
 mteb/models/model_implementations/gritlm_models.py,sha256=756vgZGADy5FhKlFuzuD6huevC_AYD5b88V1Y5yFht8,3241
 mteb/models/model_implementations/gte_models.py,sha256=-ASkoAuAiVytVtsYMtuKonUf39i0U69HSEnJy_-PwXA,14574
 mteb/models/model_implementations/hinvec_models.py,sha256=SYWGFr8XALmM7B9tIHEQnrqq9kZOZIBkW7m7QpzerHI,1756
-mteb/models/model_implementations/human.py,sha256=MTw560BrwuvUsHUb5_Gjq7ZRfrBmaT8dGnrubFFDB_o,591
+mteb/models/model_implementations/human.py,sha256=k7vN6WTcSWyWS9wnluzr6yCOjuMi5LupQnT-4cfzNOk,600
 mteb/models/model_implementations/ibm_granite_models.py,sha256=ipLRDBerTQiL5NaoaDho410Fzy7eNFlF3jB54hGZrwI,8687
 mteb/models/model_implementations/inf_models.py,sha256=q_hNNhzMjAxbnJnAT0N6KaNegX_3XZlmz-LXY5C891I,3093
 mteb/models/model_implementations/jasper_models.py,sha256=ourAMx1_L6b2AxX046wQcxDqvYzY1Mx3gaHww0WaMA8,16476
@@ -1565,14 +1565,15 @@ mteb/models/model_implementations/model2vec_models.py,sha256=qXcPhV0hGRFBsvRBrb8
 mteb/models/model_implementations/moka_models.py,sha256=4Esujv_fVJjHuX1nRH6sGtmrmF04A90F4Xo2uN0YTzs,5205
 mteb/models/model_implementations/nbailab.py,sha256=iv2xdqVM5HoTAlBR6e_UdzJu6rSPujqWXFYwyCv69hU,2684
 mteb/models/model_implementations/no_instruct_sentence_models.py,sha256=DTb-eHZYSY6lGJkkdkC0tZ_n0GHLQwVlUehVg59T5N4,4198
-mteb/models/model_implementations/nomic_models.py,sha256=oUaSabKEJXrg_cnmWdfczsdrqRDP7LZhX5vN1U5b-Xk,15469
+mteb/models/model_implementations/nomic_models.py,sha256=BO6XQbX4PFa5By0opAYkxz95CcHmjxbG5DYcklxJ1l8,16986
 mteb/models/model_implementations/nomic_models_vision.py,sha256=AzTCWbXBonUAVub0TTxWCsBtg4WYex3vPiLlz3ULdHc,6916
-mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py,sha256=5jYzS0qc4x82T5_c1HFt4uy9kksTt9Fs24pdLioq4Oc,7033
+mteb/models/model_implementations/nvidia_llama_nemoretriever_colemb.py,sha256=-5_kd9jeDcgVv9gdwWuvX_-bNQdhAxInf1Mqo8_BdS8,10653
 mteb/models/model_implementations/nvidia_models.py,sha256=r-AW1dVQbteWjexjvZgFEt_90OHNRYer_5GLuqSXRS0,26924
 mteb/models/model_implementations/octen_models.py,sha256=5z-t2O-iIFiOOLdZ_AK9f7GrVRg-9_vx3JNAG9dJNPE,8562
 mteb/models/model_implementations/openai_models.py,sha256=y1wMknrrcu1L5CNwniG0mFThPVMON1c2Fj22jkKsw7Y,9730
 mteb/models/model_implementations/openclip_models.py,sha256=z2gQum16O0QhJPyxqKor3oO-_uWfnep6wSXqOFQQ2Q8,11969
-mteb/models/model_implementations/opensearch_neural_sparse_models.py,sha256=O6ewgGKZtw-kpspbz8-dXArDf5tUWy_vY6AxFs7fsZ8,8936
+mteb/models/model_implementations/opensearch_neural_sparse_models.py,sha256=J5FEvKWQUiBusL6PHcrRuRRJOQ-iMwOSu1fX0pblXhk,8941
+mteb/models/model_implementations/ops_colqwen3_models.py,sha256=5vg5d1_WfVGMgtIwkh6zf2-Paum6V35XcKEvLfRyRzs,7437
 mteb/models/model_implementations/ops_moa_models.py,sha256=Ah7L78mqC9pH8t6sf1OWXOLjouVUpAutt6lZ0np7eMM,2655
 mteb/models/model_implementations/ordalietech_solon_embeddings_mini_beta_1_1.py,sha256=xv1ftJeMND4lpeKYC3RLQB4nhdiYy0wCxrzEjUj4gSg,1114
 mteb/models/model_implementations/pawan_models.py,sha256=iyzh6NSPZKU9znJYEDPjJNIqvkyuKPAol5TcILuq1Is,1225
@@ -1582,6 +1583,7 @@ mteb/models/model_implementations/promptriever_models.py,sha256=tDXysEKbvoDNtidV
 mteb/models/model_implementations/pylate_models.py,sha256=EwpDJf5yjONmmiv9zHSEbc3e7kMRJUHYv7K59QTrNAE,17525
 mteb/models/model_implementations/qodo_models.py,sha256=AwYRGctJbjEkcIh1pcSeeEXYiYiizAbfTyw6CaZBJfU,2224
 mteb/models/model_implementations/qtack_models.py,sha256=vw_2O4ZABR-_nYV4g1Ud8bW1DTP-wwtQS2eGqN70vT4,1304
+mteb/models/model_implementations/querit_models.py,sha256=P7lAw5IDe47DA_5srMwGPqxjMIFuvOW0BJ7xwB4GOro,8917
 mteb/models/model_implementations/qwen3_models.py,sha256=857UnUEil9o8xcw7vSr2fMRlEegyE2Q86e5yLeRL_mQ,5517
 mteb/models/model_implementations/qzhou_models.py,sha256=mfG70JrNJCo-s3MykRn6lg9gFPcKMeMI7Y8VrBhNo7I,3684
 mteb/models/model_implementations/random_baseline.py,sha256=YsITQoLbea_Iz2X84WNGBGkhlsQ3hB7yx1oJwXghimE,7561
@@ -1627,8 +1629,8 @@ mteb/models/search_encoder_index/search_indexes/__init__.py,sha256=Wm60_oUemUpFs
 mteb/models/search_encoder_index/search_indexes/faiss_search_index.py,sha256=jwC-3swhnILZnVHUrMR7Ts78TuYtVRxPusF02UV1g6E,5770
 mteb/results/__init__.py,sha256=EXQqK4Am5eIYzD52dpcGAFSdqnC38oE6JHN302oidHc,158
 mteb/results/benchmark_results.py,sha256=unBUBJ92ud0UXlkZJLn71WVcf-oUlF6XcITTccz5OBA,20318
-mteb/results/model_result.py,sha256=u1eUynaf_KVsdYdz29MACsCDR_48xODapkIGLG-lZa0,15321
-mteb/results/task_result.py,sha256=TL9V7P7cXqLljnCHdzzaQ9vZZSXh9qXYFnPRG049alw,34247
+mteb/results/model_result.py,sha256=h894O5-RSCOF8XNpXMuhBCqnj43T-1K5Y1el_fyrzP4,15954
+mteb/results/task_result.py,sha256=Wi5MRQBkb0Qddhc4nLzrrjm1nGlCh8aq4_VCQoxGmNg,34300
 mteb/tasks/__init__.py,sha256=izAxU0ip1F_YUwx0dFCuN35BaktdmePh6vlDiHC0kLo,503
 mteb/tasks/aggregated_tasks/__init__.py,sha256=Ufgbh1AirxCQkojO3AUhUFWM8zQG10cfdVTkj_PeyLI,104
 mteb/tasks/aggregated_tasks/eng/__init__.py,sha256=HgaSyAX8Is5CGE006RgJkLQQVxrx2FmMnm6NHQBDi-4,358
@@ -2111,7 +2113,7 @@ mteb/tasks/pair_classification/eng/twitter_sem_eval2015_pc.py,sha256=0YjKK4C47Uu
 mteb/tasks/pair_classification/eng/twitter_url_corpus_pc.py,sha256=M9B3JCFt4L6yEnd8S-o-W-FtCSMdl0h1KST3fqApEVA,1796
 mteb/tasks/pair_classification/fas/__init__.py,sha256=1Bbr5ZKSjpPuJb9zvk7OSd2Krdh1bpxJjVNLNPFT4Ck,440
 mteb/tasks/pair_classification/fas/fa_mteb_pair_classification.py,sha256=1i8phewQffpIxzWtBWQFUisHu3XhBpk9Sf-IkwM8jNg,10932
-mteb/tasks/pair_classification/fas/fars_tail.py,sha256=NPtN4mAQ6TcBJmX_GD7WZ4mE-_Bw3D9Aj-GEliZDIMo,2848
+mteb/tasks/pair_classification/fas/fars_tail.py,sha256=jb-6UW0Lk7YxdMMCZsMavY6CRiv3T6MFrbvlPd0vPPk,1676
 mteb/tasks/pair_classification/hye/__init__.py,sha256=hU4xSf6kyKhD4o4CuNMQNE1w9FKv8tkkqvYvhpMV5Kg,93
 mteb/tasks/pair_classification/hye/armenian_paraphrase_pc.py,sha256=Ezi604W-cHOLDm8O9j3yq9z-GzDt9OWI9jgyqVjY9M4,1437
 mteb/tasks/pair_classification/ind/__init__.py,sha256=iXGvZ6eNgGhyD2wgbkvV-bpPPCJNxlE5eq_qvF2Y_UI,53
@@ -2644,9 +2646,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=UKNokV9pu3G74MGebocU512aU_fFU9I9nPKnrG9Q0iE,1035
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=gElgSShKBXpfcqaZHhU_d2UHln1CyzUj8FN8KFun_UA,4087
-mteb-2.7.12.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.7.12.dist-info/METADATA,sha256=qFfnF_lPdIQWnBIiunbaPAatQI_x21ouRUpgasLHbM0,14458
-mteb-2.7.12.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-mteb-2.7.12.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.7.12.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.7.12.dist-info/RECORD,,
+mteb-2.7.14.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.7.14.dist-info/METADATA,sha256=ZTD9D9Fuy9OCRxIXSZzh1bObP0PKSXUMqI4j3XVNR_c,14348
+mteb-2.7.14.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+mteb-2.7.14.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.7.14.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.7.14.dist-info/RECORD,,