mteb-core 2.26.0__tar.gz → 2.26.2__tar.gz

{mteb_core-2.26.0 → mteb_core-2.26.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb-core
-Version: 2.26.0
+Version: 2.26.2
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>
@@ -195,6 +195,12 @@ Requires-Dist: transformers[torch]==5.0.0; extra == "nemotron-colembed-vl-v2"
 Requires-Dist: torchvision>=0.22.0; extra == "nemotron-colembed-vl-v2"
 Requires-Dist: flash-attn>=2.6.3; extra == "nemotron-colembed-vl-v2"
 Requires-Dist: accelerate; extra == "nemotron-colembed-vl-v2"
+Provides-Extra: nemotron-3-5-vl
+Requires-Dist: torch>=2.8; extra == "nemotron-3-5-vl"
+Requires-Dist: sentence-transformers>=6.1.0; extra == "nemotron-3-5-vl"
+Requires-Dist: transformers[torch]<5.17.0,>=5.12.1; extra == "nemotron-3-5-vl"
+Requires-Dist: torchvision>=0.22.0; extra == "nemotron-3-5-vl"
+Requires-Dist: accelerate>=0.26.0; extra == "nemotron-3-5-vl"
 Provides-Extra: llama-nemotron-embed-vl-1b-v2
 Requires-Dist: transformers[torch]>=4.56.0; extra == "llama-nemotron-embed-vl-1b-v2"
 Requires-Dist: torchvision>=0.22.0; extra == "llama-nemotron-embed-vl-1b-v2"

{mteb_core-2.26.0 → mteb_core-2.26.2}/mteb/_evaluators/retrieval_evaluator.py

@@ -5,6 +5,7 @@ import time
 from typing import TYPE_CHECKING, Any
 
 from mteb.models.search_wrappers import SearchCrossEncoderWrapper, SearchEncoderWrapper
+from mteb.models.sentence_transformer_wrapper import MultiVectorSearchEncoderWrapper
 
 from .evaluator import Evaluator
 from .retrieval_metrics import (
@@ -76,7 +77,7 @@ class RetrievalEvaluator(Evaluator):
         )
         end_time = time.monotonic()
         encodes_corpus_during_search = isinstance(
-            search_model, SearchCrossEncoderWrapper
+            search_model, (SearchCrossEncoderWrapper, MultiVectorSearchEncoderWrapper)
         ) or (
             isinstance(search_model, SearchEncoderWrapper)
             and search_model.index_backend is None
@@ -90,6 +91,24 @@ class RetrievalEvaluator(Evaluator):
                 subset=self.hf_subset,
             )
 
+        search_kwargs: dict[str, Any] = dict(
+            queries=self.queries,
+            top_k=self.top_k,
+            task_metadata=self.task_metadata,
+            hf_split=self.hf_split,
+            hf_subset=self.hf_subset,
+            encode_kwargs=encode_kwargs,
+            top_ranked=self.top_ranked,
+            num_proc=num_proc,
+        )
+
+        # These wrappers record finer-grained phases (queries, corpus, similarity) themselves
+        if isinstance(
+            search_model, (SearchEncoderWrapper, MultiVectorSearchEncoderWrapper)
+        ):
+            logger.info("Running retrieval task - Searching queries...")
+            return search_model.search(**search_kwargs, timer=self.timer)
+
         search_phase_name = (
             "Encoding queries and documents"
             if encodes_corpus_during_search
@@ -102,16 +121,7 @@ class RetrievalEvaluator(Evaluator):
             subset=self.hf_subset,
             log_message="Running retrieval task - Searching queries...",
         ):
-            return search_model.search(
-                queries=self.queries,
-                top_k=self.top_k,
-                task_metadata=self.task_metadata,
-                hf_split=self.hf_split,
-                hf_subset=self.hf_subset,
-                encode_kwargs=encode_kwargs,
-                top_ranked=self.top_ranked,
-                num_proc=num_proc,
-            )
+            return search_model.search(**search_kwargs)
 
     def evaluate(  # noqa: PLR6301
         self,

{mteb_core-2.26.0 → mteb_core-2.26.2}/mteb/benchmarks/benchmarks/__init__.py

@@ -58,7 +58,7 @@ from mteb.benchmarks.benchmarks.benchmarks import (
     VIDORE_V2,
     VIDORE_V3,
     VIDORE_V3_1,
-    # VIDORE_V3_1_RERANKING,  # commented out until reference models have been run
+    VIDORE_V3_1_RERANKING,
     VISUAL_DOCUMENT_RETRIEVAL,
     VN_MTEB,
     CoIR,
@@ -151,7 +151,7 @@ __all__ = [
     "VIDORE_V2",
     "VIDORE_V3",
     "VIDORE_V3_1",
-    # "VIDORE_V3_1_RERANKING",  # commented out until reference models have been run
+    "VIDORE_V3_1_RERANKING",
     "VISUAL_DOCUMENT_RETRIEVAL",
     "VN_MTEB",
     "CoIR",

{mteb_core-2.26.0 → mteb_core-2.26.2}/mteb/benchmarks/benchmarks/benchmarks.py

@@ -3055,99 +3055,98 @@ VIDORE_V3_1 = VidoreBenchmark(
 """,
 )
 
-# commented out until reference models have been run
-# _VIDORE_V3_1_RERANKING_BM25S = [
-#     "Vidore3ComputerScienceBM25sReranking.v2",
-#     "Vidore3EnergyBM25sReranking.v2",
-#     "Vidore3FinanceEnBM25sReranking.v2",
-#     "Vidore3FinanceFrBM25sReranking.v2",
-#     "Vidore3HrBM25sReranking.v2",
-#     "Vidore3IndustrialBM25sReranking.v2",
-#     "Vidore3PharmaceuticalsBM25sReranking.v2",
-#     "Vidore3PhysicsBM25sReranking.v2",
-#     "Vidore3TelecomBM25sReranking.v2",
-#     "Vidore3NuclearBM25sReranking.v2",
-# ]
-# _VIDORE_V3_1_RERANKING_BGEM3 = [
-#     "Vidore3ComputerScienceBGEm3Reranking.v2",
-#     "Vidore3EnergyBGEm3Reranking.v2",
-#     "Vidore3FinanceEnBGEm3Reranking.v2",
-#     "Vidore3FinanceFrBGEm3Reranking.v2",
-#     "Vidore3HrBGEm3Reranking.v2",
-#     "Vidore3IndustrialBGEm3Reranking.v2",
-#     "Vidore3PharmaceuticalsBGEm3Reranking.v2",
-#     "Vidore3PhysicsBGEm3Reranking.v2",
-#     "Vidore3TelecomBGEm3Reranking.v2",
-#     "Vidore3NuclearBGEm3Reranking.v2",
-# ]
-# _VIDORE_V3_1_RERANKING_QWEN3_VL_EMBEDDING_2B = [
-#     "Vidore3ComputerScienceQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3EnergyQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3FinanceEnQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3FinanceFrQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3HrQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3IndustrialQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3PharmaceuticalsQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3PhysicsQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3TelecomQwen3VLEmbedding2BReranking.v2",
-#     "Vidore3NuclearQwen3VLEmbedding2BReranking.v2",
-# ]
-#
-# VIDORE_V3_1_RERANKING = VidoreBenchmark(
-#     name="ViDoRe(v3.1, reranking, beta)",
-#     display_name="ViDoRe v3.1 Reranking",
-#     language_view=[
-#         "deu-Latn",
-#         "eng-Latn",
-#         "fra-Latn",
-#         "ita-Latn",
-#         "por-Latn",
-#         "spa-Latn",
-#     ],
-#     icon="https://cdn-uploads.huggingface.co/production/uploads/66e16a677c2eb2da5109fb5c/x99xqw__fl2UaPbiIdC_f.png",
-#     tasks=get_tasks(
-#         tasks=_VIDORE_V3_1_RERANKING_BM25S
-#         + _VIDORE_V3_1_RERANKING_BGEM3
-#         + _VIDORE_V3_1_RERANKING_QWEN3_VL_EMBEDDING_2B
-#     ),
-#     description="Reranking of visual documents on the ViDoRe v3.1 domains. For each domain, the candidates are the top-50 pages retrieved by one of three first-stage retrievers (mteb/baseline-bm25s, BAAI/bge-m3 and Qwen/Qwen3-VL-Embedding-2B), so rerankers are compared on candidate lists of varying quality. Includes both open and closed datasets; to submit results on private tasks, please [open an issue](https://github.com/embeddings-benchmark/mteb/issues?template=eval_request.yaml). This benchmark is in beta.",
-#     reference="https://arxiv.org/abs/2601.08620",
-#     citation=r"""
-# @article{loison2026vidorev3comprehensiveevaluation,
-#   archiveprefix = {arXiv},
-#   author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
-#   eprint = {2601.08620},
-#   primaryclass = {cs.AI},
-#   title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
-#   url = {https://arxiv.org/abs/2601.08620},
-#   year = {2026},
-# }
-# """,
-#     aggregations=(
-#         BenchmarkAggregation.MEAN_TASK,
-#         BenchmarkAggregation.PUBLIC_PRIVATE,
-#         CustomGrouping(
-#             name="First-stage retriever",
-#             groups=(
-#                 CustomGroup(
-#                     label="BM25s",
-#                     tasks=get_tasks(tasks=_VIDORE_V3_1_RERANKING_BM25S),
-#                     description="Mean over the domains when reranking the top-50 candidates retrieved by mteb/baseline-bm25s.",
-#                 ),
-#                 CustomGroup(
-#                     label="BGE-M3",
-#                     tasks=get_tasks(tasks=_VIDORE_V3_1_RERANKING_BGEM3),
-#                     description="Mean over the domains when reranking the top-50 candidates retrieved by BAAI/bge-m3.",
-#                 ),
-#                 CustomGroup(
-#                     label="Qwen3-VL-Embedding-2B",
-#                     tasks=get_tasks(tasks=_VIDORE_V3_1_RERANKING_QWEN3_VL_EMBEDDING_2B),
-#                     description="Mean over the domains when reranking the top-50 candidates retrieved by Qwen/Qwen3-VL-Embedding-2B.",
-#                 ),
-#             ),
-#         ),
-#     ),
-# )
+_VIDORE_V3_1_RERANKING_BM25S = [
+    "Vidore3ComputerScienceBM25sReranking.v2",
+    "Vidore3EnergyBM25sReranking.v2",
+    "Vidore3FinanceEnBM25sReranking.v2",
+    "Vidore3FinanceFrBM25sReranking.v2",
+    "Vidore3HrBM25sReranking.v2",
+    "Vidore3IndustrialBM25sReranking.v2",
+    "Vidore3PharmaceuticalsBM25sReranking.v2",
+    "Vidore3PhysicsBM25sReranking.v2",
+    "Vidore3TelecomBM25sReranking.v2",
+    "Vidore3NuclearBM25sReranking.v2",
+]
+_VIDORE_V3_1_RERANKING_BGEM3 = [
+    "Vidore3ComputerScienceBGEm3Reranking.v2",
+    "Vidore3EnergyBGEm3Reranking.v2",
+    "Vidore3FinanceEnBGEm3Reranking.v2",
+    "Vidore3FinanceFrBGEm3Reranking.v2",
+    "Vidore3HrBGEm3Reranking.v2",
+    "Vidore3IndustrialBGEm3Reranking.v2",
+    "Vidore3PharmaceuticalsBGEm3Reranking.v2",
+    "Vidore3PhysicsBGEm3Reranking.v2",
+    "Vidore3TelecomBGEm3Reranking.v2",
+    "Vidore3NuclearBGEm3Reranking.v2",
+]
+_VIDORE_V3_1_RERANKING_QWEN3_VL_EMBEDDING_2B = [
+    "Vidore3ComputerScienceQwen3VLEmbedding2BReranking.v2",
+    "Vidore3EnergyQwen3VLEmbedding2BReranking.v2",
+    "Vidore3FinanceEnQwen3VLEmbedding2BReranking.v2",
+    "Vidore3FinanceFrQwen3VLEmbedding2BReranking.v2",
+    "Vidore3HrQwen3VLEmbedding2BReranking.v2",
+    "Vidore3IndustrialQwen3VLEmbedding2BReranking.v2",
+    "Vidore3PharmaceuticalsQwen3VLEmbedding2BReranking.v2",
+    "Vidore3PhysicsQwen3VLEmbedding2BReranking.v2",
+    "Vidore3TelecomQwen3VLEmbedding2BReranking.v2",
+    "Vidore3NuclearQwen3VLEmbedding2BReranking.v2",
+]
+
+VIDORE_V3_1_RERANKING = VidoreBenchmark(
+    name="ViDoRe(v3.1, reranking, beta)",
+    display_name="ViDoRe v3.1 Reranking",
+    language_view=[
+        "deu-Latn",
+        "eng-Latn",
+        "fra-Latn",
+        "ita-Latn",
+        "por-Latn",
+        "spa-Latn",
+    ],
+    icon="https://cdn-uploads.huggingface.co/production/uploads/66e16a677c2eb2da5109fb5c/x99xqw__fl2UaPbiIdC_f.png",
+    tasks=get_tasks(
+        tasks=_VIDORE_V3_1_RERANKING_BM25S
+        + _VIDORE_V3_1_RERANKING_BGEM3
+        + _VIDORE_V3_1_RERANKING_QWEN3_VL_EMBEDDING_2B
+    ),
+    description="Reranking of visual documents on the ViDoRe v3.1 domains. For each domain, the candidates are the top-50 pages retrieved by one of three first-stage retrievers; BM25, bge-m3 and Qwen3-VL-Embedding-2B, so rerankers are compared on candidate lists from various model types. Includes both open and closed datasets; to submit results on private tasks, please [open an issue](https://github.com/embeddings-benchmark/mteb/issues?template=eval_request.yaml).",
+    reference="https://arxiv.org/abs/2601.08620",
+    citation=r"""
+@article{loison2026vidorev3comprehensiveevaluation,
+  archiveprefix = {arXiv},
+  author = {António Loison and Quentin Macé and Antoine Edy and Victor Xing and Tom Balough and Gabriel Moreira and Bo Liu and Manuel Faysse and Céline Hudelot and Gautier Viaud},
+  eprint = {2601.08620},
+  primaryclass = {cs.AI},
+  title = {ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
+  url = {https://arxiv.org/abs/2601.08620},
+  year = {2026},
+}
+""",
+    aggregations=(
+        BenchmarkAggregation.MEAN_TASK,
+        BenchmarkAggregation.PUBLIC_PRIVATE,
+        CustomGrouping(
+            name="First-stage retriever",
+            groups=(
+                CustomGroup(
+                    label="BM25s",
+                    tasks=get_tasks(tasks=_VIDORE_V3_1_RERANKING_BM25S),
+                    description="Mean over the domains when reranking the top-50 candidates retrieved by mteb/baseline-bm25s.",
+                ),
+                CustomGroup(
+                    label="BGE-M3",
+                    tasks=get_tasks(tasks=_VIDORE_V3_1_RERANKING_BGEM3),
+                    description="Mean over the domains when reranking the top-50 candidates retrieved by BAAI/bge-m3.",
+                ),
+                CustomGroup(
+                    label="Qwen3-VL-Embedding-2B",
+                    tasks=get_tasks(tasks=_VIDORE_V3_1_RERANKING_QWEN3_VL_EMBEDDING_2B),
+                    description="Mean over the domains when reranking the top-50 candidates retrieved by Qwen/Qwen3-VL-Embedding-2B.",
+                ),
+            ),
+        ),
+    ),
+)
 
 
 VISUAL_DOCUMENT_RETRIEVAL = Benchmark(