mteb 2.1.4__py3-none-any.whl → 2.5.2__py3-none-any.whl

mteb/abstasks/_data_filter/task_pipelines.py

@@ -0,0 +1,102 @@
+import logging
+
+from datasets import DatasetDict
+
+from mteb import TaskMetadata
+from mteb.abstasks import AbsTaskClassification
+from mteb.abstasks._data_filter.filters import (
+    deduplicate,
+    filter_empty,
+    filter_short,
+    filter_train_leakage,
+    filter_unclear_label,
+    split_train_test,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def clean_dataset(
+    ds: DatasetDict,
+    metadata: TaskMetadata,
+    train_split: str,
+    input_column: str,
+    label_column: str,
+    subset: str | None = None,
+) -> DatasetDict:
+    """Apply the full cleaning pipeline with logging."""
+    logger.info("[clean_dataset] Starting dataset cleaning pipeline...")
+
+    transforms = [
+        ("filter_empty", filter_empty),
+        ("deduplicate", deduplicate),
+    ]
+
+    skip_cjk_codes = {"zho", "jpn", "tha", "mya", "cmn"}
+    logger.info("[clean_dataset] Applying short-text filter")
+    cur_langs = (
+        metadata.eval_langs[subset]
+        if isinstance(metadata.eval_langs, dict) and subset
+        else metadata.eval_langs
+    )
+    apply_short = not any(lang.split("-")[0] in skip_cjk_codes for lang in cur_langs)
+    if apply_short:
+        logger.info("[clean_dataset] Applying short-text filter")
+        transforms.append(("filter_short", filter_short))
+
+    for split in [train_split, *metadata.eval_splits]:
+        if split not in ds:
+            logger.warning(f"[clean_dataset] Split '{split}' missing; skipping.")
+            continue
+
+        for name, fn in transforms:
+            before = len(ds[split])
+            ds[split] = fn(ds[split], input_column=input_column)
+            logger.info(
+                f"[clean_dataset:{split}] {name} removed={before - len(ds[split])}"
+            )
+
+    ds = split_train_test(ds, metadata, train_split, label_column)
+
+    for split in metadata.eval_splits:
+        if split == train_split:
+            continue
+        before = len(ds[split])
+        ds[split] = filter_train_leakage(ds[train_split], ds[split], input_column)
+        logger.info(
+            f"[clean_dataset:{split}] leakage_removed={before - len(ds[split])}"
+        )
+
+    ds = filter_unclear_label(ds, input_column=input_column, label_column=label_column)
+
+    logger.info("[clean_dataset] Cleaning pipeline complete.")
+    return ds
+
+
+def process_classification(
+    task: AbsTaskClassification,
+) -> DatasetDict | dict[str, DatasetDict]:
+    """Process classification task dataset(s) with cleaning pipeline."""
+    if not task.data_loaded:
+        task.load_data()
+    if isinstance(task.dataset, DatasetDict):
+        return clean_dataset(
+            task.dataset,
+            task.metadata,
+            task.train_split,
+            task.input_column_name,
+            task.label_column_name,
+            subset=None,
+        )
+
+    new_ds = {}
+    for subset in task.dataset:
+        new_ds[subset] = clean_dataset(
+            task.dataset[subset],
+            task.metadata,
+            task.train_split,
+            task.input_column_name,
+            task.label_column_name,
+            subset=subset,
+        )
+    return new_ds

mteb/abstasks/_statistics_calculation.py

@@ -1,7 +1,8 @@
+from __future__ import annotations
+
 import hashlib
 from collections import Counter
-
-from PIL import Image
+from typing import TYPE_CHECKING
 
 from mteb.types import TopRankedDocumentsType
 from mteb.types.statistics import (
@@ -13,6 +14,9 @@ from mteb.types.statistics import (
     TopRankedStatistics,
 )
 
+if TYPE_CHECKING:
+    from PIL import Image
+
 
 def calculate_text_statistics(texts: list[str]) -> TextStatistics:
     """Calculate descriptive statistics for a list of texts.

mteb/abstasks/classification.py

@@ -5,7 +5,6 @@ from typing import Any, TypedDict
 
 import numpy as np
 from datasets import Dataset, DatasetDict
-from PIL import ImageFile
 from sklearn.linear_model import LogisticRegression
 from sklearn.metrics import (
     accuracy_score,
@@ -32,7 +31,6 @@ from ._statistics_calculation import (
 )
 from .abstask import AbsTask
 
-ImageFile.LOAD_TRUNCATED_IMAGES = True
 logger = logging.getLogger(__name__)
 
 

mteb/abstasks/clustering.py

@@ -200,7 +200,7 @@ class AbsTaskClustering(AbsTask):
                 downsampled_dataset,
                 self.metadata,
                 input_column=self.input_column_name,
-                batch_size=encode_kwargs["batch_size"],
+                **encode_kwargs,
             ),
             task_metadata=self.metadata,
             hf_subset=hf_subset,

mteb/abstasks/clustering_legacy.py

@@ -89,6 +89,9 @@ class AbsTaskClusteringLegacy(AbsTask):
         prediction_folder: Path | None = None,
         **kwargs: Any,
     ) -> ScoresDict:
+        data_split = data_split.select_columns(
+            [self.input_column_name, self.label_column_name]
+        )
         # MTEB text clustering requires renaming and eval per subset.
         if self.metadata.modalities == ["text"]:
             all_metrics = []

mteb/abstasks/multilabel_classification.py

@@ -14,6 +14,7 @@ from sklearn.preprocessing import MultiLabelBinarizer
 from typing_extensions import override
 
 from mteb._create_dataloaders import create_dataloader
+from mteb._evaluators.classification_metrics import hamming_score
 from mteb._evaluators.sklearn_evaluator import SklearnModelProtocol
 from mteb.models import EncoderProtocol
 
@@ -40,11 +41,13 @@ class MultilabelClassificationMetrics(TypedDict):
         accuracy: Accuracy of the classifier.
         lrap: Label Ranking Average Precision (LRAP) score.
         f1: Macro F1 score.
+        hamming: Hamming score (label-based accuracy).
     """
 
     accuracy: float
     lrap: float
     f1: float
+    hamming: float
 
 
 class FullMultilabelClassificationMetrics(MultilabelClassificationMetrics):
@@ -112,7 +115,7 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
             unique_train_dataset,
             self.metadata,
             input_column=self.input_column_name,
-            batch_size=encode_kwargs["batch_size"],
+            **encode_kwargs,
         )
 
         logger.info("Running multilabel classification - Encoding training set...")
@@ -141,7 +144,7 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
             test_dataset.select_columns(self.input_column_name),
             self.metadata,
             input_column=self.input_column_name,
-            batch_size=encode_kwargs["batch_size"],
+            **encode_kwargs,
         )
 
         logger.info("Running multilabel classification - Encoding test set...")
@@ -157,7 +160,7 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
 
         logger.info("Running multilabel classification - Evaluating classifiers...")
         all_predictions = []
-        for i_experiment, sample_indices in enumerate(train_samples):
+        for _, sample_indices in enumerate(train_samples):
             X_train = np.stack([unique_train_embeddings[idx] for idx in sample_indices])
             y_train = train_split.select(sample_indices)[self.label_column_name]
             y_train = binarizer.transform(y_train)
@@ -207,10 +210,12 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
         else:
             lrap = label_ranking_average_precision_score(y_test, y_pred)
         f1 = f1_score(y_test, y_pred, average="macro")
+        hamming = hamming_score(y_test, y_pred)
         return MultilabelClassificationMetrics(
             accuracy=accuracy,
             lrap=lrap,
             f1=f1,
+            hamming=hamming,
         )
 
     def _undersample_data_indices(
@@ -218,6 +223,8 @@ class AbsTaskMultilabelClassification(AbsTaskClassification):
     ) -> tuple[list[int], list[int]]:
         """Undersample data to have samples_per_label samples of each label.
 
+        Currently ensures that each label has at least samples_per_label samples.
+
         Returns:
             A tuple containing:
                 - List of sampled indices.

mteb/abstasks/pair_classification.py

@@ -19,6 +19,7 @@ from mteb.abstasks._statistics_calculation import (
 from mteb.abstasks.abstask import AbsTask
 from mteb.models.model_meta import ScoringFunction
 from mteb.models.models_protocols import EncoderProtocol
+from mteb.types import PromptType
 from mteb.types.statistics import (
     ImageStatistics,
     LabelStatistics,
@@ -35,7 +36,7 @@ class PairClassificationDescriptiveStatistics(SplitDescriptiveStatistics):
     Attributes:
         num_samples: number of samples in the dataset.
         number_of_characters: Total number of symbols in the dataset.
-        unique_text_pairs: Number of unique pairs
+        unique_pairs: Number of unique pairs
 
         text1_statistics: Statistics for sentence1
         text2_statistics: Statistics for sentence2
@@ -65,12 +66,16 @@ class AbsTaskPairClassification(AbsTask):
         input2_column_name: The name of the column containing the second sentence in the pair.
         label_column_name: The name of the column containing the labels for the pairs. Labels should be 0 or 1.
         abstask_prompt: Prompt to use for the task for instruction model if not prompt is provided in TaskMetadata.prompt.
+        input1_prompt_type: Type of prompt of first input. Used for asymmetric tasks.
+        input2_prompt_type: Type of prompt of second input. Used for asymmetric tasks.
     """
 
     abstask_prompt = "Retrieve text that are semantically similar to the given text."
     input1_column_name: str = "sentence1"
     input2_column_name: str = "sentence2"
     label_column_name: str = "labels"
+    input1_prompt_type: PromptType | None = None
+    input2_prompt_type: PromptType | None = None
 
     def _evaluate_subset(
         self,
@@ -93,6 +98,8 @@ class AbsTaskPairClassification(AbsTask):
             task_metadata=self.metadata,
             hf_split=hf_split,
             hf_subset=hf_subset,
+            input1_prompt_type=self.input1_prompt_type,
+            input2_prompt_type=self.input2_prompt_type,
             **kwargs,
         )
         similarity_scores = evaluator(model, encode_kwargs=encode_kwargs)

mteb/abstasks/sts.py

@@ -8,6 +8,7 @@ from scipy.stats import pearsonr, spearmanr
 from mteb._evaluators import AnySTSEvaluator
 from mteb._evaluators.any_sts_evaluator import STSEvaluatorScores
 from mteb.models import EncoderProtocol
+from mteb.types import PromptType
 from mteb.types.statistics import (
     ImageStatistics,
     ScoreStatistics,
@@ -89,12 +90,16 @@ class AbsTaskSTS(AbsTask):
         min_score: Minimum possible score in the dataset.
         max_score: Maximum possible score in the dataset.
         abstask_prompt: Prompt to use for the task for instruction model if not prompt is provided in TaskMetadata.prompt.
+        input1_prompt_type: Type of prompt of first input. Used for asymmetric tasks.
+        input2_prompt_type: Type of prompt of second input. Used for asymmetric tasks.
     """
 
     abstask_prompt = "Retrieve semantically similar text."
     column_names: tuple[str, str] = ("sentence1", "sentence2")
     min_score: int = 0
     max_score: int = 5
+    input1_prompt_type: PromptType | None = None
+    input2_prompt_type: PromptType | None = None
 
     def _evaluate_subset(
         self,
@@ -115,6 +120,8 @@ class AbsTaskSTS(AbsTask):
             task_metadata=self.metadata,
             hf_split=hf_split,
             hf_subset=hf_subset,
+            input1_prompt_type=self.input1_prompt_type,
+            input2_prompt_type=self.input2_prompt_type,
             **kwargs,
         )
         scores = evaluator(model, encode_kwargs=encode_kwargs)

mteb/abstasks/task_metadata.py

@@ -107,6 +107,7 @@ The domains follow the categories used in the [Universal Dependencies project](h
 SampleCreationMethod = Literal[
     "found",
     "created",
+    "created and machine-translated",
     "human-translated and localized",
     "human-translated",
     "machine-translated",

mteb/benchmarks/_create_table.py

@@ -1,6 +1,6 @@
-import math
 import re
 from collections import defaultdict
+from typing import Literal
 
 import numpy as np
 import pandas as pd
@@ -32,26 +32,18 @@ def _split_on_capital(s: str) -> str:
     return " ".join(re.findall(r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)", s))
 
 
-def _format_n_parameters(n_parameters) -> str:
-    if (n_parameters is None) or (not int(n_parameters)):
-        return "Unknown"
-    n_thousand = int(n_parameters // 1e3)
-    if n_thousand < 1:
-        return str(int(n_parameters))
-    n_zeros = math.log10(n_thousand)
-    if n_zeros >= 6:
-        return str(n_thousand // (10**6)) + "B"
-    if n_zeros >= 3:
-        return str(n_thousand // (10**3)) + "M"
-    return str(n_thousand) + "K"
+def _format_n_parameters(n_parameters) -> float | None:
+    """Format n_parameters to be in billions with decimals down to 1 million. I.e. 7M -> 0.007B, 1.5B -> 1.5B, None -> None"""
+    if n_parameters:
+        n_parameters = float(n_parameters)
+        return round(n_parameters / 1e9, 3)
+    return None
 
 
-def _format_max_tokens(max_tokens: float | None) -> str:
-    if max_tokens is None:
-        return "Unknown"
-    if max_tokens == np.inf:
-        return "Infinite"
-    return str(int(max_tokens))
+def _format_max_tokens(max_tokens: float | None) -> float | None:
+    if max_tokens is None or max_tokens == np.inf:
+        return None
+    return float(max_tokens)
 
 
 def _get_means_per_types(per_task: pd.DataFrame):
@@ -144,18 +136,18 @@ def _create_summary_table_from_benchmark_results(
     joint_table.insert(
         1,
         "Embedding Dimensions",
-        model_metas.map(lambda m: str(int(m.embed_dim)) if m.embed_dim else "Unknown"),
+        model_metas.map(lambda m: int(m.embed_dim) if m.embed_dim else None),
     )
     joint_table.insert(
         1,
-        "Number of Parameters",
+        "Number of Parameters (B)",
         model_metas.map(lambda m: _format_n_parameters(m.n_parameters)),
     )
     joint_table.insert(
         1,
         "Memory Usage (MB)",
         model_metas.map(
-            lambda m: str(int(m.memory_usage_mb)) if m.memory_usage_mb else "Unknown"
+            lambda m: int(m.memory_usage_mb) if m.memory_usage_mb else None
         ),
     )
 
@@ -250,6 +242,65 @@ def _create_per_task_table_from_benchmark_results(
     return per_task
 
 
+def _create_per_language_table_from_benchmark_results(
+    benchmark_results: BenchmarkResults,
+    language_view: list[str] | Literal["all"],
+) -> pd.DataFrame:
+    """Create per-language table from BenchmarkResults.
+
+    Returns a DataFrame with one row per model and one column per language.
+
+    Args:
+        benchmark_results: BenchmarkResults object containing model results
+        language_view: List of languages to include in the per-language table, or "all" for all languages present in the results
+    Returns:
+        DataFrame with per-language scores, ready for styling in the leaderboard
+    """
+    if language_view != "all" and not isinstance(language_view, list):
+        raise ValueError("language_view must be a list of languages or 'all'")
+
+    data = benchmark_results.to_dataframe(aggregation_level="language", format="long")
+
+    if data.empty:
+        no_results_frame = pd.DataFrame(
+            {"No results": ["You can try relaxing your criteria"]}
+        )
+        return no_results_frame
+
+    if language_view != "all":
+        data = data[data["language"].isin(language_view)]
+
+    per_language = data.pivot_table(
+        index="model_name", columns="language", values="score", aggfunc="mean"
+    )
+
+    to_remove = per_language.isna().all(axis="columns")
+    if to_remove.all():
+        no_results_frame = pd.DataFrame(
+            {"No results": ["You can try relaxing your criteria"]}
+        )
+        return no_results_frame
+
+    models_to_remove = list(per_language[to_remove].index)
+    per_language = per_language.drop(models_to_remove, axis=0)
+
+    per_language["borda_rank"] = _get_borda_rank(per_language)
+    per_language = per_language.sort_values("borda_rank", ascending=True)
+    per_language = per_language.drop(columns=["borda_rank"])
+    per_language = per_language.reset_index()
+
+    per_language["model_name"] = per_language["model_name"].map(
+        lambda name: name.split("/")[-1]
+    )
+    per_language = per_language.rename(
+        columns={
+            "model_name": "Model",
+        }
+    )
+
+    return per_language
+
+
 def _create_summary_table_mean_public_private(
     benchmark_results: BenchmarkResults,
 ) -> pd.DataFrame:
@@ -323,18 +374,18 @@ def _create_summary_table_mean_public_private(
     joint_table.insert(
         1,
         "Embedding Dimensions",
-        model_metas.map(lambda m: str(int(m.embed_dim)) if m.embed_dim else "Unknown"),
+        model_metas.map(lambda m: int(m.embed_dim) if m.embed_dim else None),
     )
     joint_table.insert(
         1,
-        "Number of Parameters",
+        "Number of Parameters (B)",
         model_metas.map(lambda m: _format_n_parameters(m.n_parameters)),
     )
     joint_table.insert(
         1,
         "Memory Usage (MB)",
         model_metas.map(
-            lambda m: str(int(m.memory_usage_mb)) if m.memory_usage_mb else "Unknown"
+            lambda m: int(m.memory_usage_mb) if m.memory_usage_mb else None
         ),
     )
 
@@ -358,9 +409,7 @@ def _create_summary_table_mean_public_private(
         "mean(public)": "Mean (Public)",
         "mean(private)": "Mean (Private)",
     }
-    # For RTEB: all tasks are Retrieval type, so Retrieval column = Mean (Task)
-    if "Retrieval" in joint_table.columns:
-        rename_dict["Retrieval"] = "Mean (Task)"
+
     joint_table = joint_table.rename(columns=rename_dict)
 
     # Move borda rank to front
@@ -447,18 +496,18 @@ def _create_summary_table_mean_subset(
     joint_table.insert(
         1,
         "Embedding Dimensions",
-        model_metas.map(lambda m: str(int(m.embed_dim)) if m.embed_dim else "Unknown"),
+        model_metas.map(lambda m: int(m.embed_dim) if m.embed_dim else None),
     )
     joint_table.insert(
         1,
-        "Number of Parameters",
+        "Number of Parameters (B)",
         model_metas.map(lambda m: _format_n_parameters(m.n_parameters)),
     )
     joint_table.insert(
         1,
         "Memory Usage (MB)",
         model_metas.map(
-            lambda m: str(int(m.memory_usage_mb)) if m.memory_usage_mb else "Unknown"
+            lambda m: int(m.memory_usage_mb) if m.memory_usage_mb else None
         ),
     )
 
@@ -560,25 +609,23 @@ def _create_summary_table_mean_task_type(
 
     # Insert model metadata columns
     joint_table.insert(
-        1,
-        "Max Tokens",
-        model_metas.map(lambda m: _format_max_tokens(m.max_tokens)),
+        1, "Max Tokens", model_metas.map(lambda m: _format_max_tokens(m.max_tokens))
     )
     joint_table.insert(
         1,
         "Embedding Dimensions",
-        model_metas.map(lambda m: str(int(m.embed_dim)) if m.embed_dim else "Unknown"),
+        model_metas.map(lambda m: int(m.embed_dim) if m.embed_dim else None),
     )
     joint_table.insert(
         1,
-        "Number of Parameters",
+        "Number of Parameters (B)",
         model_metas.map(lambda m: _format_n_parameters(m.n_parameters)),
     )
     joint_table.insert(
         1,
         "Memory Usage (MB)",
         model_metas.map(
-            lambda m: str(int(m.memory_usage_mb)) if m.memory_usage_mb else "Unknown"
+            lambda m: int(m.memory_usage_mb) if m.memory_usage_mb else None
         ),
     )
 

mteb/benchmarks/benchmark.py

@@ -1,21 +1,16 @@
+from __future__ import annotations
+
 from collections.abc import Iterable, Sequence
-from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Literal
 
 import pandas as pd
 
-from mteb.benchmarks._create_table import (
-    _create_per_task_table_from_benchmark_results,
-    _create_summary_table_from_benchmark_results,
-    _create_summary_table_mean_public_private,
-    _create_summary_table_mean_subset,
-    _create_summary_table_mean_task_type,
-)
-from mteb.results import BenchmarkResults
+from mteb.abstasks.abstask import AbsTask
 from mteb.types import StrURL
 
 if TYPE_CHECKING:
-    from mteb.abstasks import AbsTask
+    from mteb.results import BenchmarkResults
 
 
 @dataclass
@@ -42,7 +37,7 @@ class Benchmark:
     """
 
     name: str
-    tasks: Sequence["AbsTask"]
+    tasks: Sequence[AbsTask]
     description: str | None = None
     reference: StrURL | None = None
     citation: str | None = None
@@ -50,14 +45,15 @@ class Benchmark:
     display_on_leaderboard: bool = True
     icon: str | None = None
     display_name: str | None = None
+    language_view: list[str] | Literal["all"] = field(default_factory=list)
 
-    def __iter__(self) -> Iterable["AbsTask"]:
+    def __iter__(self) -> Iterable[AbsTask]:
         return iter(self.tasks)
 
     def __len__(self) -> int:
         return len(self.tasks)
 
-    def __getitem__(self, index: int) -> "AbsTask":
+    def __getitem__(self, index: int) -> AbsTask:
         return self.tasks[index]
 
     def _create_summary_table(
@@ -68,6 +64,10 @@ class Benchmark:
         Returns:
             A pandas DataFrame representing the summary results.
         """
+        from mteb.benchmarks._create_table import (
+            _create_summary_table_from_benchmark_results,
+        )
+
         return _create_summary_table_from_benchmark_results(benchmark_results)
 
     def _create_per_task_table(
@@ -78,8 +78,38 @@ class Benchmark:
         Returns:
             A pandas DataFrame representing the per-task results.
         """
+        from mteb.benchmarks._create_table import (
+            _create_per_task_table_from_benchmark_results,
+        )
+
         return _create_per_task_table_from_benchmark_results(benchmark_results)
 
+    def _create_per_language_table(
+        self, benchmark_results: BenchmarkResults
+    ) -> pd.DataFrame:
+        """Create per-language table. Called by the leaderboard app.
+
+        Returns:
+            A pandas DataFrame representing the per-language results.
+        """
+        from mteb.benchmarks._create_table import (
+            _create_per_language_table_from_benchmark_results,
+        )
+
+        if self.language_view == "all" or len(self.language_view) > 0:
+            return _create_per_language_table_from_benchmark_results(
+                benchmark_results, self.language_view
+            )
+        else:
+            no_results_frame = pd.DataFrame(
+                {
+                    "No results": [
+                        "The per-language table is not available for this benchmark."
+                    ]
+                }
+            )
+            return no_results_frame
+
 
 class RtebBenchmark(Benchmark):
     """Wrapper for RTEB benchmark."""
@@ -87,7 +117,14 @@ class RtebBenchmark(Benchmark):
     def _create_summary_table(
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
-        return _create_summary_table_mean_public_private(benchmark_results)
+        from mteb.benchmarks._create_table import (
+            _create_summary_table_mean_public_private,
+        )
+
+        joint_table = _create_summary_table_mean_public_private(benchmark_results)
+        # For RTEB: all tasks are Retrieval type, so Retrieval column = Mean (Task)
+        joint_table = joint_table.rename(columns={"Retrieval": "Mean (Task)"})
+        return joint_table
 
 
 class HUMEBenchmark(Benchmark):
@@ -96,6 +133,8 @@ class HUMEBenchmark(Benchmark):
     def _create_summary_table(
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
+        from mteb.benchmarks._create_table import _create_summary_table_mean_subset
+
         return _create_summary_table_mean_subset(benchmark_results)
 
 
@@ -105,4 +144,24 @@ class MIEBBenchmark(Benchmark):
     def _create_summary_table(
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
+        from mteb.benchmarks._create_table import _create_summary_table_mean_task_type
+
         return _create_summary_table_mean_task_type(benchmark_results)
+
+
+class VidoreBenchmark(Benchmark):
+    """Wrapper for Vidore3 benchmark."""
+
+    def _create_summary_table(
+        self, benchmark_results: BenchmarkResults
+    ) -> pd.DataFrame:
+        from mteb.benchmarks._create_table import (
+            _create_summary_table_mean_public_private,
+        )
+
+        joint_table = _create_summary_table_mean_public_private(benchmark_results)
+        # For ViDoRe (V1, V2, V3): all tasks are Document Understanding type, so Document Understanding column = Mean (Task)
+        joint_table = joint_table.rename(
+            columns={"Document Understanding": "Mean (Task)"}
+        )
+        return joint_table