mteb 2.1.7__py3-none-any.whl → 2.1.8__py3-none-any.whl

mteb/_create_dataloaders.py

@@ -3,7 +3,7 @@ from collections.abc import Callable
 from typing import Any, cast
 
 import torch
-from datasets import Dataset
+from datasets import Dataset, Image
 from torch.utils.data import DataLoader, default_collate
 
 from mteb.abstasks.task_metadata import TaskMetadata
@@ -22,12 +22,14 @@ logger = logging.getLogger(__name__)
 def _create_dataloader_from_texts(
     text: list[str],
     batch_size: int = 32,
+    **kwargs: dict[str, Any],
 ) -> DataLoader[TextInput]:
     """Create a dataloader from a list of text.
 
     Args:
         text: A list of text to create a dataloader from.
         batch_size: Batch size for the dataloader.
+        kwargs: Not used, present catching extra arguments.
 
     Returns:
         A dataloader with the text.
@@ -244,14 +246,15 @@ def _prepare_image_dataset(
     transform: Callable[[Any], Any] | None = None,
 ) -> Dataset:
     """Prepare the image dataset by converting images to RGB and applying transformations."""
-    # If the dataset uses a different column name for images, rename it to "image".
     if (
         image_column_name
         and image_column_name in dataset.column_names
         and "image" not in dataset.column_names
     ):
         dataset = dataset.rename_column(image_column_name, "image")
-    # Map the conversion function over the dataset.
+    # don't process image if it's already in the correct format
+    if isinstance(dataset.features["image"], Image):
+        return dataset
     return dataset.map(
         _convert_images_to_rgb,
         fn_kwargs={"image_col_name": "image", "transform": transform},

mteb/_evaluators/any_sts_evaluator.py

@@ -45,16 +45,8 @@ class AnySTSEvaluator(Evaluator):
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
-        self.first_column = create_dataloader(
-            dataset,
-            task_metadata,
-            input_column=sentences_column_names[0],
-        )
-        self.second_column = create_dataloader(
-            dataset,
-            task_metadata,
-            input_column=sentences_column_names[1],
-        )
+        self.dataset = dataset
+        self.input_columns = sentences_column_names
         self.task_metadata = task_metadata
         self.hf_split = hf_split
         self.hf_subset = hf_subset
@@ -67,7 +59,12 @@ class AnySTSEvaluator(Evaluator):
     ) -> STSEvaluatorScores:
         logger.info("Running semantic similarity - Encoding samples (1/2)")
         embeddings1 = model.encode(
-            self.first_column,
+            create_dataloader(
+                self.dataset,
+                self.task_metadata,
+                input_column=self.input_columns[0],
+                **encode_kwargs,
+            ),
             task_metadata=self.task_metadata,
             hf_split=self.hf_split,
             hf_subset=self.hf_subset,
@@ -76,7 +73,12 @@ class AnySTSEvaluator(Evaluator):
 
         logger.info("Running semantic similarity - Encoding samples (2/2)...")
         embeddings2 = model.encode(
-            self.second_column,
+            create_dataloader(
+                self.dataset,
+                self.task_metadata,
+                input_column=self.input_columns[1],
+                **encode_kwargs,
+            ),
             task_metadata=self.task_metadata,
             hf_split=self.hf_split,
             hf_subset=self.hf_subset,

mteb/_evaluators/clustering_evaluator.py

@@ -44,7 +44,7 @@ class ClusteringEvaluator(Evaluator):
             self.dataset,
             self.task_metadata,
             input_column=self.input_column_name,
-            batch_size=encode_kwargs["batch_size"],
+            **encode_kwargs,
         )
 
         logger.info("Running clustering - Encoding samples...")

mteb/_evaluators/image/imagetext_pairclassification_evaluator.py

@@ -103,7 +103,7 @@ class ImageTextPairClassificationEvaluator(Evaluator):
         text_embeddings = model.encode(
             DataLoader(
                 Dataset.from_dict({"text": texts}),
-                batch_size=encode_kwargs["batch_size"],
+                **encode_kwargs,
             ),
             task_metadata=self.task_metadata,
             hf_subset=self.hf_subset,
@@ -122,8 +122,8 @@ class ImageTextPairClassificationEvaluator(Evaluator):
         image_embeddings = model.encode(
             DataLoader(
                 CustomImageDataset(images),
-                batch_size=encode_kwargs["batch_size"],
                 collate_fn=lambda x: {"image": [item["image"] for item in x]},
+                **encode_kwargs,
             ),
             task_metadata=self.task_metadata,
             hf_subset=self.hf_subset,

mteb/_evaluators/pair_classification_evaluator.py

@@ -106,6 +106,7 @@ class PairClassificationEvaluator(Evaluator):
                     self.dataset,
                     task_metadata=self.task_metadata,
                     input_column=self.input1_column_name,
+                    **encode_kwargs,
                 ),
                 task_metadata=self.task_metadata,
                 hf_split=self.hf_split,
@@ -117,6 +118,7 @@ class PairClassificationEvaluator(Evaluator):
                     self.dataset,
                     task_metadata=self.task_metadata,
                     input_column=self.input2_column_name,
+                    **encode_kwargs,
                 ),
                 task_metadata=self.task_metadata,
                 hf_split=self.hf_split,
@@ -168,7 +170,7 @@ class PairClassificationEvaluator(Evaluator):
         )
         all_unique_texts_embs = np.asarray(
             model.encode(
-                _create_dataloader_from_texts(all_unique_texts),
+                _create_dataloader_from_texts(all_unique_texts, **encode_kwargs),
                 task_metadata=task_metadata,
                 hf_split=hf_split,
                 hf_subset=hf_subset,

mteb/_evaluators/sklearn_evaluator.py

@@ -6,7 +6,7 @@ from datasets import Dataset
 from torch.utils.data import DataLoader
 from typing_extensions import Self
 
-from mteb._create_dataloaders import _create_image_dataloader
+from mteb._create_dataloaders import create_dataloader
 from mteb.abstasks.task_metadata import TaskMetadata
 from mteb.models import EncoderProtocol
 from mteb.types import BatchedInput
@@ -50,33 +50,20 @@ class SklearnEvaluator(Evaluator):
         self.evaluator_model = evaluator_model
 
     def create_dataloaders(
-        self, batch_size: int
+        self, encode_kwargs: dict[str, Any]
     ) -> tuple[DataLoader[BatchedInput], DataLoader[BatchedInput]]:
-        if self.task_metadata.modalities == ["image"]:
-            dataloader_train = _create_image_dataloader(
-                self.train_dataset,
-                image_column_name=self.values_column_name,
-                batch_size=batch_size,
-            )
-            dataloader_test = _create_image_dataloader(
-                self.eval_dataset,
-                image_column_name=self.values_column_name,
-                batch_size=batch_size,
-            )
-        elif self.task_metadata.modalities == ["text"]:
-            if self.values_column_name != "text":
-                self.train_dataset = self.train_dataset.rename_column(
-                    self.values_column_name, "text"
-                )
-                self.eval_dataset = self.eval_dataset.rename_column(
-                    self.values_column_name, "text"
-                )
-            dataloader_train = DataLoader(self.train_dataset)
-            dataloader_test = DataLoader(self.eval_dataset)
-        else:
-            raise ValueError(
-                "ClassificationEvaluator only supports image and text modalities."
-            )
+        dataloader_train = create_dataloader(
+            self.train_dataset,
+            self.task_metadata,
+            input_column=self.values_column_name,
+            **encode_kwargs,
+        )
+        dataloader_test = create_dataloader(
+            self.eval_dataset,
+            self.task_metadata,
+            input_column=self.values_column_name,
+            **encode_kwargs,
+        )
         return dataloader_train, dataloader_test
 
     def __call__(  # type: ignore[override]
@@ -98,7 +85,7 @@ class SklearnEvaluator(Evaluator):
 
         """
         dataloader_train, dataloader_test = self.create_dataloaders(
-            batch_size=encode_kwargs["batch_size"]
+            encode_kwargs=encode_kwargs,
         )
 
         logger.info("Running - Encoding samples...")

mteb/_evaluators/text/bitext_mining_evaluator.py

@@ -46,7 +46,10 @@ class BitextMiningEvaluator(Evaluator):
 
         embeddings = {}
         for sub in tqdm(subsets):
-            dataloader = _create_dataloader_from_texts(self.sentences[sub])
+            dataloader = _create_dataloader_from_texts(
+                self.sentences[sub],
+                **encode_kwargs,
+            )
             embeddings[sub] = model.encode(
                 dataloader,
                 task_metadata=self.task_metadata,

mteb/_evaluators/text/summarization_evaluator.py

@@ -109,7 +109,8 @@ class SummarizationEvaluator(Evaluator):
                     summary
                     for human_summaries in self.human_summaries
                     for summary in human_summaries
-                ]
+                ],
+                **encode_kwargs,
             ),
             task_metadata=self.task_metadata,
             hf_subset=self.hf_subset,
@@ -124,7 +125,8 @@ class SummarizationEvaluator(Evaluator):
                     summary
                     for machine_summaries in self.machine_summaries
                     for summary in machine_summaries
-                ]
+                ],
+                **encode_kwargs,
             ),
             task_metadata=self.task_metadata,
             hf_subset=self.hf_subset,

mteb/_evaluators/zeroshot_classification_evaluator.py

@@ -42,14 +42,14 @@ class ZeroShotClassificationEvaluator(Evaluator):
     ) -> Array:
         dataloader = create_dataloader(
             self.dataset,
-            batch_size=encode_kwargs["batch_size"],
             input_column=self.input_column_name,
             task_metadata=self.task_metadata,
+            **encode_kwargs,
         )
 
         logger.info("Running zero-shot classification - Encoding labels...")
         text_label_embeddings = model.encode(
-            _create_dataloader_from_texts(self.candidate_labels),
+            _create_dataloader_from_texts(self.candidate_labels, **encode_kwargs),
             task_metadata=self.task_metadata,
             hf_subset=self.hf_subset,
             hf_split=self.hf_split,

mteb/abstasks/clustering.py

@@ -200,7 +200,7 @@ class AbsTaskClustering(AbsTask):
                 downsampled_dataset,
                 self.metadata,
                 input_column=self.input_column_name,
-                batch_size=encode_kwargs["batch_size"],
+                **encode_kwargs,
             ),
             task_metadata=self.metadata,
             hf_subset=hf_subset,

mteb/abstasks/multilabel_classification.py

@@ -112,7 +112,7 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
             unique_train_dataset,
             self.metadata,
             input_column=self.input_column_name,
-            batch_size=encode_kwargs["batch_size"],
+            **encode_kwargs,
         )
 
         logger.info("Running multilabel classification - Encoding training set...")
@@ -141,7 +141,7 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
             test_dataset.select_columns(self.input_column_name),
             self.metadata,
             input_column=self.input_column_name,
-            batch_size=encode_kwargs["batch_size"],
+            **encode_kwargs,
         )
 
         logger.info("Running multilabel classification - Encoding test set...")

mteb/abstasks/task_metadata.py

@@ -107,6 +107,7 @@ The domains follow the categories used in the [Universal Dependencies project](h
 SampleCreationMethod = Literal[
     "found",
     "created",
+    "created and machine-translated",
     "human-translated and localized",
     "human-translated",
     "machine-translated",

mteb/benchmarks/benchmark.py

@@ -106,3 +106,12 @@ class MIEBBenchmark(Benchmark):
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
         return _create_summary_table_mean_task_type(benchmark_results)
+
+
+class Vidore3Benchmark(Benchmark):
+    """Wrapper for Vidore3 benchmark."""
+
+    def _create_summary_table(
+        self, benchmark_results: BenchmarkResults
+    ) -> pd.DataFrame:
+        return _create_summary_table_mean_public_private(benchmark_results)

mteb/benchmarks/benchmarks/__init__.py

@@ -38,6 +38,7 @@ from mteb.benchmarks.benchmarks.benchmarks import (
     SEB,
     VIDORE,
     VIDORE_V2,
+    VIDORE_V3,
     VISUAL_DOCUMENT_RETRIEVAL,
     VN_MTEB,
     CoIR,
@@ -108,6 +109,7 @@ __all__ = [
     "SEB",
     "VIDORE",
     "VIDORE_V2",
+    "VIDORE_V3",
     "VISUAL_DOCUMENT_RETRIEVAL",
     "VN_MTEB",
     "CoIR",

mteb/benchmarks/benchmarks/benchmarks.py

@@ -1,4 +1,9 @@
-from mteb.benchmarks.benchmark import Benchmark, HUMEBenchmark, MIEBBenchmark
+from mteb.benchmarks.benchmark import (
+    Benchmark,
+    HUMEBenchmark,
+    MIEBBenchmark,
+    Vidore3Benchmark,
+)
 from mteb.get_tasks import MTEBTasks, get_task, get_tasks
 
 MMTEB_CITATION = r"""@article{enevoldsen2025mmtebmassivemultilingualtext,
@@ -2214,6 +2219,40 @@ VIDORE_V2 = Benchmark(
 """,
 )
 
+VIDORE_V3 = Vidore3Benchmark(
+    name="ViDoRe(v3)",
+    display_name="ViDoRe V3",
+    icon="https://cdn-uploads.huggingface.co/production/uploads/66e16a677c2eb2da5109fb5c/x99xqw__fl2UaPbiIdC_f.png",
+    tasks=get_tasks(
+        tasks=[
+            "Vidore3FinanceEnRetrieval",
+            "Vidore3IndustrialRetrieval",
+            "Vidore3ComputerScienceRetrieval",
+            "Vidore3PharmaceuticalsRetrieval",
+            "Vidore3HrRetrieval",
+            "Vidore3FinanceFrRetrieval",
+            "Vidore3PhysicsRetrieval",
+            "Vidore3EnergyRetrieval",
+            "Vidore3TelecomRetrieval",
+            "Vidore3NuclearRetrieval",
+        ]
+    ),
+    description="ViDoRe V3 sets a new industry gold standard for multi-modal, enterprise document visual retrieval evaluation. It addresses a critical challenge in production RAG systems: retrieving accurate information from complex, visually-rich documents. The benchmark includes both open and closed datasets: to submit results on private tasks, please [open an issue](https://github.com/embeddings-benchmark/mteb/issues?template=eval_request.yaml).",
+    reference="https://huggingface.co/blog/QuentinJG/introducing-vidore-v3",
+    citation=r"""
+@misc{mace2025vidorev3,
+  author = {Macé, Quentin and Loison, Antonio and EDY, Antoine and Xing, Victor and Viaud, Gautier},
+  day = {5},
+  howpublished = {\url{https://huggingface.co/blog/QuentinJG/introducing-vidore-v3}},
+  journal = {Hugging Face Blog},
+  month = {November},
+  publisher = {Hugging Face},
+  title = {ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases},
+  year = {2025},
+}
+""",
+)
+
 VISUAL_DOCUMENT_RETRIEVAL = Benchmark(
     name="VisualDocumentRetrieval",
     display_name="Visual Document Retrieval",

mteb/descriptive_stats/Image/DocumentUnderstanding/Vidore3ComputerScienceRetrieval.json

@@ -0,0 +1,214 @@
+{
+    "test": {
+        "num_samples": 9450,
+        "number_of_characters": 152825,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 1700,
+            "average_image_width": 1700.0,
+            "max_image_width": 1700,
+            "min_image_height": 2200,
+            "average_image_height": 2200.0,
+            "max_image_height": 2200,
+            "unique_images": 1359
+        },
+        "queries_text_statistics": {
+            "total_text_length": 152825,
+            "min_text_length": 21,
+            "average_text_length": 118.46899224806202,
+            "max_text_length": 591,
+            "unique_texts": 1290
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 37764,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 4.879069767441861,
+            "max_relevant_docs_per_query": 21,
+            "unique_relevant_docs": 3534
+        },
+        "top_ranked_statistics": null,
+        "hf_subset_descriptive_stats": {
+            "french": {
+                "num_samples": 1575,
+                "number_of_characters": 27948,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 1700,
+                    "average_image_width": 1700.0,
+                    "max_image_width": 1700,
+                    "min_image_height": 2200,
+                    "average_image_height": 2200.0,
+                    "max_image_height": 2200,
+                    "unique_images": 1359
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 27948,
+                    "min_text_length": 28,
+                    "average_text_length": 129.9906976744186,
+                    "max_text_length": 563,
+                    "unique_texts": 215
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 6294,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 4.879069767441861,
+                    "max_relevant_docs_per_query": 21,
+                    "unique_relevant_docs": 589
+                },
+                "top_ranked_statistics": null
+            },
+            "spanish": {
+                "num_samples": 1575,
+                "number_of_characters": 26025,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 1700,
+                    "average_image_width": 1700.0,
+                    "max_image_width": 1700,
+                    "min_image_height": 2200,
+                    "average_image_height": 2200.0,
+                    "max_image_height": 2200,
+                    "unique_images": 1359
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 26025,
+                    "min_text_length": 30,
+                    "average_text_length": 121.04651162790698,
+                    "max_text_length": 565,
+                    "unique_texts": 215
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 6294,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 4.879069767441861,
+                    "max_relevant_docs_per_query": 21,
+                    "unique_relevant_docs": 589
+                },
+                "top_ranked_statistics": null
+            },
+            "english": {
+                "num_samples": 1575,
+                "number_of_characters": 22198,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 1700,
+                    "average_image_width": 1700.0,
+                    "max_image_width": 1700,
+                    "min_image_height": 2200,
+                    "average_image_height": 2200.0,
+                    "max_image_height": 2200,
+                    "unique_images": 1359
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 22198,
+                    "min_text_length": 22,
+                    "average_text_length": 103.24651162790698,
+                    "max_text_length": 486,
+                    "unique_texts": 215
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 6294,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 4.879069767441861,
+                    "max_relevant_docs_per_query": 21,
+                    "unique_relevant_docs": 589
+                },
+                "top_ranked_statistics": null
+            },
+            "german": {
+                "num_samples": 1575,
+                "number_of_characters": 26237,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 1700,
+                    "average_image_width": 1700.0,
+                    "max_image_width": 1700,
+                    "min_image_height": 2200,
+                    "average_image_height": 2200.0,
+                    "max_image_height": 2200,
+                    "unique_images": 1359
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 26237,
+                    "min_text_length": 22,
+                    "average_text_length": 122.03255813953488,
+                    "max_text_length": 542,
+                    "unique_texts": 215
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 6294,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 4.879069767441861,
+                    "max_relevant_docs_per_query": 21,
+                    "unique_relevant_docs": 589
+                },
+                "top_ranked_statistics": null
+            },
+            "italian": {
+                "num_samples": 1575,
+                "number_of_characters": 25835,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 1700,
+                    "average_image_width": 1700.0,
+                    "max_image_width": 1700,
+                    "min_image_height": 2200,
+                    "average_image_height": 2200.0,
+                    "max_image_height": 2200,
+                    "unique_images": 1359
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 25835,
+                    "min_text_length": 21,
+                    "average_text_length": 120.16279069767442,
+                    "max_text_length": 521,
+                    "unique_texts": 215
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 6294,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 4.879069767441861,
+                    "max_relevant_docs_per_query": 21,
+                    "unique_relevant_docs": 589
+                },
+                "top_ranked_statistics": null
+            },
+            "portuguese": {
+                "num_samples": 1575,
+                "number_of_characters": 24582,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 1700,
+                    "average_image_width": 1700.0,
+                    "max_image_width": 1700,
+                    "min_image_height": 2200,
+                    "average_image_height": 2200.0,
+                    "max_image_height": 2200,
+                    "unique_images": 1359
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 24582,
+                    "min_text_length": 26,
+                    "average_text_length": 114.33488372093024,
+                    "max_text_length": 591,
+                    "unique_texts": 215
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 6294,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 4.879069767441861,
+                    "max_relevant_docs_per_query": 21,
+                    "unique_relevant_docs": 589
+                },
+                "top_ranked_statistics": null
+            }
+        }
+    }
+}