mteb 1.39.5__tar.gz → 1.39.7__tar.gz

{mteb-1.39.5 → mteb-1.39.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 1.39.5
+Version: 1.39.7
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, nouamane@huggingface.co, info@nils-reimers.de
 License:                                  Apache License

{mteb-1.39.5 → mteb-1.39.7}/mteb/benchmarks/_create_table.py

@@ -500,3 +500,132 @@ def _create_summary_table_mean_subset(
     joint_table.insert(0, "Rank (Borda)", joint_table.pop("borda_rank"))
 
     return joint_table
+
+
+def _create_summary_table_mean_task_type(
+    benchmark_results: BenchmarkResults,
+) -> pd.DataFrame:
+    """Create summary table from BenchmarkResults.
+
+    Returns a DataFrame with one row per model containing summary statistics
+    and task type averages.
+
+    Args:
+        benchmark_results: BenchmarkResults object containing model results
+
+    Returns:
+        DataFrame with model summaries, ready for styling in the leaderboard
+    """
+    data = benchmark_results.to_dataframe(format="long")
+
+    if data.empty:
+        no_results_frame = pd.DataFrame(
+            {"No results": ["You can try relaxing your criteria"]}
+        )
+        return no_results_frame
+
+    # Convert to DataFrame and pivot
+    per_task = data.pivot(index="model_name", columns="task_name", values="score")
+
+    # Remove models with no scores
+    to_remove = per_task.isna().all(axis="columns")
+    if to_remove.all():
+        no_results_frame = pd.DataFrame(
+            {"No results": ["You can try relaxing your criteria"]}
+        )
+        return no_results_frame
+
+    models_to_remove = list(per_task[to_remove].index)
+    per_task = per_task.drop(models_to_remove, axis=0)
+
+    # Calculate means by task type
+    mean_per_type = _get_means_per_types(per_task)
+    mean_per_type = mean_per_type.pivot(
+        index="model_name", columns="task_type", values="score"
+    )
+    mean_per_type.columns = [
+        _split_on_capital(column) for column in mean_per_type.columns
+    ]
+
+    # Calculate overall means
+    typed_mean = mean_per_type.mean(skipna=False, axis=1)
+
+    # Build joint table
+    joint_table = mean_per_type.copy()
+    joint_table = joint_table.drop(models_to_remove, axis=0)
+    joint_table.insert(0, "mean_by_task_type", typed_mean)
+    joint_table = joint_table.sort_values("mean_by_task_type", ascending=False)
+    joint_table["borda_rank"] = _get_borda_rank(per_task)
+    joint_table["rank"] = [i + 1 for i in range(len(joint_table))]
+    joint_table = joint_table.reset_index()
+
+    # Add model metadata
+    model_metas = joint_table["model_name"].map(_failsafe_get_model_meta)
+    joint_table = joint_table[model_metas.notna()]
+    joint_table["model_link"] = model_metas.map(lambda m: m.reference)
+
+    # Insert model metadata columns
+    joint_table.insert(
+        1,
+        "Max Tokens",
+        model_metas.map(lambda m: _format_max_tokens(m.max_tokens)),
+    )
+    joint_table.insert(
+        1,
+        "Embedding Dimensions",
+        model_metas.map(lambda m: str(int(m.embed_dim)) if m.embed_dim else "Unknown"),
+    )
+    joint_table.insert(
+        1,
+        "Number of Parameters",
+        model_metas.map(lambda m: _format_n_parameters(m.n_parameters)),
+    )
+    joint_table.insert(
+        1,
+        "Memory Usage (MB)",
+        model_metas.map(
+            lambda m: str(int(m.memory_usage_mb)) if m.memory_usage_mb else "Unknown"
+        ),
+    )
+
+    # Add zero-shot percentage
+    tasks = get_tasks(tasks=list(data["task_name"].unique()))
+    joint_table.insert(
+        1, "Zero-shot", model_metas.map(lambda m: m.zero_shot_percentage(tasks))
+    )
+    joint_table["Zero-shot"] = joint_table["Zero-shot"].fillna(-1)
+
+    # Clean up model names (remove HF organization)
+    joint_table["model_name"] = joint_table["model_name"].map(
+        lambda name: name.split("/")[-1]
+    )
+
+    # Add markdown links to model names
+    name_w_link = (
+        "[" + joint_table["model_name"] + "](" + joint_table["model_link"] + ")"
+    )
+    joint_table["model_name"] = joint_table["model_name"].mask(
+        joint_table["model_link"].notna(), name_w_link
+    )
+    joint_table = joint_table.drop(columns=["model_link"])
+
+    # Rename columns
+    joint_table = joint_table.rename(
+        columns={
+            "model_name": "Model",
+            "mean_by_task_type": "Mean (TaskType)",
+            "borda_rank": "Rank (Borda)",
+        }
+    )
+
+    if "Any Any Multilingual Retrieval" in joint_table.columns:
+        joint_table = joint_table.rename(
+            columns={"Any Any Multilingual Retrieval": "Multilingual Retrieval"}
+        )
+    if "Any Any Retrieval" in joint_table.columns:
+        joint_table = joint_table.rename(columns={"Any Any Retrieval": "Retrieval"})
+
+    # Move borda rank to front
+    joint_table.insert(0, "Rank", joint_table.pop("rank"))
+
+    return joint_table

{mteb-1.39.5 → mteb-1.39.7}/mteb/benchmarks/benchmark.py

@@ -12,6 +12,7 @@ from mteb.benchmarks._create_table import (
     _create_summary_table_from_benchmark_results,
     _create_summary_table_mean_public_private,
     _create_summary_table_mean_subset,
+    _create_summary_table_mean_task_type,
 )
 from mteb.load_results.load_results import load_results
 
@@ -107,3 +108,11 @@ class HUMEBenchmark(Benchmark):
     ) -> pd.DataFrame:
         """Create summary table. Called by the leaderboard app."""
         return _create_summary_table_mean_subset(benchmark_results)
+
+
+class MIEBBenchmark(Benchmark):
+    def _create_summary_table(
+        self, benchmark_results: BenchmarkResults
+    ) -> pd.DataFrame:
+        """Create summary table. Called by the leaderboard app."""
+        return _create_summary_table_mean_task_type(benchmark_results)

{mteb-1.39.5 → mteb-1.39.7}/mteb/benchmarks/benchmarks/benchmarks.py

@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Annotated
 
 from pydantic import AnyUrl, BeforeValidator, TypeAdapter
 
-from mteb.benchmarks.benchmark import Benchmark, HUMEBenchmark
+from mteb.benchmarks.benchmark import Benchmark, HUMEBenchmark, MIEBBenchmark
 from mteb.overview import MTEBTasks, get_task, get_tasks
 
 if TYPE_CHECKING:
@@ -1754,7 +1754,7 @@ MIEB_common_tasks = [
     "WebQAT2TRetrieval",
 ]
 
-MIEB_ENG = Benchmark(
+MIEB_ENG = MIEBBenchmark(
     name="MIEB(eng)",
     display_name="Image-Text, English",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-picture.svg",
@@ -1783,7 +1783,7 @@ MIEB_ENG = Benchmark(
 """,
 )
 
-MIEB_MULTILINGUAL = Benchmark(
+MIEB_MULTILINGUAL = MIEBBenchmark(
     name="MIEB(Multilingual)",
     display_name="Image-Text, Multilingual",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-pictures.svg",
@@ -1818,7 +1818,7 @@ MIEB_MULTILINGUAL = Benchmark(
 """,
 )
 
-MIEB_LITE = Benchmark(
+MIEB_LITE = MIEBBenchmark(
     name="MIEB(lite)",
     display_name="Image-Text, Lite",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-map-landscape.svg",
@@ -1902,7 +1902,7 @@ MIEB_LITE = Benchmark(
 """,
 )
 
-MIEB_IMG = Benchmark(
+MIEB_IMG = MIEBBenchmark(
     name="MIEB(Img)",
     display_name="Image only",
     icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-pictures.svg",

{mteb-1.39.5 → mteb-1.39.7}/mteb/evaluation/evaluators/RegressionEvaluator.py

@@ -53,19 +53,13 @@ class LinearRegressionEvaluator(Evaluator):
         scores = {}
         X_train = model.encode(
             self.sentences_train,
-            model=model,
             task_name=self.task_name,
-            hf_split="train",
-            hf_subset=self.hf_subset,
             **encode_kwargs,
         )
         if test_cache is None:
             X_test = model.encode(
                 self.sentences_test,
-                model=model,
                 task_name=self.task_name,
-                hf_split=self.hf_split,
-                hf_subset=self.hf_subset,
                 **encode_kwargs,
             )
             test_cache = X_test

{mteb-1.39.5 → mteb-1.39.7}/mteb/leaderboard/table.py

@@ -116,6 +116,7 @@ def _apply_summary_table_styling(joint_table: pd.DataFrame) -> gr.DataFrame:
     """Apply styling to a raw summary DataFrame"""
     excluded_columns = [
         "Rank (Borda)",
+        "Rank",
         "Model",
         "Number of Parameters",
         "Embedding Dimensions",

{mteb-1.39.5 → mteb-1.39.7}/mteb/tasks/Classification/__init__.py

@@ -150,11 +150,11 @@ from .sin.SinhalaNewsClassification import *
 from .sin.SinhalaNewsSourceClassification import *
 from .slk.CSFDSKMovieReviewSentimentClassification import *
 from .slk.SlovakHateSpeechClassification import *
+from .slk.SlovakMovieReviewSentimentClassification import *
 from .slv.FrenkSlClassification import *
 from .spa.SpanishNewsClassification import *
 from .spa.SpanishSentimentClassification import *
 from .ssw.SiswatiNewsClassification import *
-from .svk.SlovakMovieReviewSentimentClassification import *
 from .swa.SwahiliNewsClassification import *
 from .swe.DalajClassification import *
 from .swe.SwedishSentimentClassification import *

{mteb-1.39.5/mteb/tasks/Classification/svk → mteb-1.39.7/mteb/tasks/Classification/slk}/SlovakMovieReviewSentimentClassification.py

@@ -18,7 +18,7 @@ class SlovakMovieReviewSentimentClassification(AbsTaskClassification):
         category="s2s",
         modalities=["text"],
         eval_splits=["test"],
-        eval_langs=["svk-Latn"],
+        eval_langs=["slk-Latn"],
         main_score="accuracy",
         date=("2002-05-21", "2020-03-05"),
         dialect=[],
@@ -59,7 +59,7 @@ class SlovakMovieReviewSentimentClassificationV2(AbsTaskClassification):
         category="s2s",
         modalities=["text"],
         eval_splits=["test"],
-        eval_langs=["svk-Latn"],
+        eval_langs=["slk-Latn"],
         main_score="accuracy",
         date=("2002-05-21", "2020-03-05"),
         dialect=[],

{mteb-1.39.5 → mteb-1.39.7}/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py

@@ -30,6 +30,39 @@ _LANGUAGES = {
     "zh": ["zho-Hans"],
 }
 
+_common_metadata = dict(
+    reference="http://miracl.ai",
+    type="Retrieval",
+    category="s2p",
+    modalities=["text"],
+    eval_splits=[_EVAL_SPLIT],
+    eval_langs=_LANGUAGES,
+    main_score="ndcg_at_10",
+    date=("2022-06-01", "2023-01-30"),
+    domains=["Encyclopaedic", "Written"],
+    task_subtypes=[],
+    license="cc-by-sa-4.0",
+    annotations_creators="expert-annotated",
+    dialect=[],
+    sample_creation="created",
+    bibtex_citation=r"""
+@article{10.1162/tacl_a_00595,
+  abstract = {{MIRACL is a multilingual dataset for ad hoc retrieval across 18 languages that collectively encompass over three billion native speakers around the world. This resource is designed to support monolingual retrieval tasks, where the queries and the corpora are in the same language. In total, we have gathered over 726k high-quality relevance judgments for 78k queries over Wikipedia in these languages, where all annotations have been performed by native speakers hired by our team. MIRACL covers languages that are both typologically close as well as distant from 10 language families and 13 sub-families, associated with varying amounts of publicly available resources. Extensive automatic heuristic verification and manual assessments were performed during the annotation process to control data quality. In total, MIRACL represents an investment of around five person-years of human annotator effort. Our goal is to spur research on improving retrieval across a continuum of languages, thus enhancing information access capabilities for diverse populations around the world, particularly those that have traditionally been underserved. MIRACL is available at http://miracl.ai/.}},
+  author = {Zhang, Xinyu and Thakur, Nandan and Ogundepo, Odunayo and Kamalloo, Ehsan and Alfonso-Hermelo, David and Li, Xiaoguang and Liu, Qun and Rezagholizadeh, Mehdi and Lin, Jimmy},
+  doi = {10.1162/tacl_a_00595},
+  eprint = {https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl\_a\_00595/2157340/tacl\_a\_00595.pdf},
+  issn = {2307-387X},
+  journal = {Transactions of the Association for Computational Linguistics},
+  month = {09},
+  pages = {1114-1131},
+  title = {{MIRACL: A Multilingual Retrieval Dataset Covering 18 Diverse Languages}},
+  url = {https://doi.org/10.1162/tacl\_a\_00595},
+  volume = {11},
+  year = {2023},
+}
+""",
+)
+
 
 def _load_miracl_data(
     path: str,
@@ -106,44 +139,15 @@ class MIRACLRetrieval(MultilingualTask, AbsTaskRetrieval):
     metadata = TaskMetadata(
         name="MIRACLRetrieval",
         description="MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual retrieval dataset that focuses on search across 18 different languages.",
-        reference="http://miracl.ai",
         dataset={
             "path": "miracl/mmteb-miracl",
             "revision": "main",
             "trust_remote_code": True,
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=[_EVAL_SPLIT],
-        eval_langs=_LANGUAGES,
-        main_score="ndcg_at_10",
-        date=("2022-06-01", "2023-01-30"),
-        domains=["Encyclopaedic", "Written"],
-        task_subtypes=[],
-        license="cc-by-sa-4.0",
-        annotations_creators="expert-annotated",
-        dialect=[],
-        sample_creation="created",
-        bibtex_citation=r"""
-@article{10.1162/tacl_a_00595,
-  abstract = {{MIRACL is a multilingual dataset for ad hoc retrieval across 18 languages that collectively encompass over three billion native speakers around the world. This resource is designed to support monolingual retrieval tasks, where the queries and the corpora are in the same language. In total, we have gathered over 726k high-quality relevance judgments for 78k queries over Wikipedia in these languages, where all annotations have been performed by native speakers hired by our team. MIRACL covers languages that are both typologically close as well as distant from 10 language families and 13 sub-families, associated with varying amounts of publicly available resources. Extensive automatic heuristic verification and manual assessments were performed during the annotation process to control data quality. In total, MIRACL represents an investment of around five person-years of human annotator effort. Our goal is to spur research on improving retrieval across a continuum of languages, thus enhancing information access capabilities for diverse populations around the world, particularly those that have traditionally been underserved. MIRACL is available at http://miracl.ai/.}},
-  author = {Zhang, Xinyu and Thakur, Nandan and Ogundepo, Odunayo and Kamalloo, Ehsan and Alfonso-Hermelo, David and Li, Xiaoguang and Liu, Qun and Rezagholizadeh, Mehdi and Lin, Jimmy},
-  doi = {10.1162/tacl_a_00595},
-  eprint = {https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl\_a\_00595/2157340/tacl\_a\_00595.pdf},
-  issn = {2307-387X},
-  journal = {Transactions of the Association for Computational Linguistics},
-  month = {09},
-  pages = {1114-1131},
-  title = {{MIRACL: A Multilingual Retrieval Dataset Covering 18 Diverse Languages}},
-  url = {https://doi.org/10.1162/tacl\_a\_00595},
-  volume = {11},
-  year = {2023},
-}
-""",
         prompt={
             "query": "Given a question, retrieve Wikipedia passages that answer the question"
         },
+        **_common_metadata,
     )
 
     def load_data(self, **kwargs):
@@ -300,45 +304,58 @@ def _load_miracl_data_hard_negatives(
 
 
 class MIRACLRetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval):
+    # in current version prompt for instruction models different from original task
+    superseded_by = "MIRACLRetrievalHardNegatives.v2"
+
     metadata = TaskMetadata(
         name="MIRACLRetrievalHardNegatives",
         description="MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual retrieval dataset that focuses on search across 18 different languages. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.",
-        reference="http://miracl.ai",
         dataset={
             "path": "mteb/miracl-hard-negatives",
             "revision": "95c8db7d4a6e9c1d8a60601afd63d553ae20a2eb",
             "trust_remote_code": True,
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=[_EVAL_SPLIT],
-        eval_langs=_LANGUAGES,
-        main_score="ndcg_at_10",
-        date=("2022-06-01", "2023-01-30"),
-        domains=["Encyclopaedic", "Written"],
-        task_subtypes=[],
-        license="cc-by-sa-4.0",
-        annotations_creators="expert-annotated",
-        dialect=[],
-        sample_creation="created",
-        bibtex_citation=r"""
-@article{10.1162/tacl_a_00595,
-  abstract = {{MIRACL is a multilingual dataset for ad hoc retrieval across 18 languages that collectively encompass over three billion native speakers around the world. This resource is designed to support monolingual retrieval tasks, where the queries and the corpora are in the same language. In total, we have gathered over 726k high-quality relevance judgments for 78k queries over Wikipedia in these languages, where all annotations have been performed by native speakers hired by our team. MIRACL covers languages that are both typologically close as well as distant from 10 language families and 13 sub-families, associated with varying amounts of publicly available resources. Extensive automatic heuristic verification and manual assessments were performed during the annotation process to control data quality. In total, MIRACL represents an investment of around five person-years of human annotator effort. Our goal is to spur research on improving retrieval across a continuum of languages, thus enhancing information access capabilities for diverse populations around the world, particularly those that have traditionally been underserved. MIRACL is available at http://miracl.ai/.}},
-  author = {Zhang, Xinyu and Thakur, Nandan and Ogundepo, Odunayo and Kamalloo, Ehsan and Alfonso-Hermelo, David and Li, Xiaoguang and Liu, Qun and Rezagholizadeh, Mehdi and Lin, Jimmy},
-  doi = {10.1162/tacl_a_00595},
-  eprint = {https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl\_a\_00595/2157340/tacl\_a\_00595.pdf},
-  issn = {2307-387X},
-  journal = {Transactions of the Association for Computational Linguistics},
-  month = {09},
-  pages = {1114-1131},
-  title = {{MIRACL: A Multilingual Retrieval Dataset Covering 18 Diverse Languages}},
-  url = {https://doi.org/10.1162/tacl\_a\_00595},
-  volume = {11},
-  year = {2023},
-}
-""",
         adapted_from=["MIRACLRetrieval"],
+        **_common_metadata,
+    )
+
+    def load_data(self, **kwargs):
+        if self.data_loaded:
+            return
+
+        self.corpus, self.queries, self.relevant_docs = (
+            _load_miracl_data_hard_negatives(
+                path=self.metadata.dataset["path"],
+                langs=self.hf_subsets,
+                splits=self.metadata_dict["eval_splits"],
+                cache_dir=kwargs.get("cache_dir", None),
+                revision=self.metadata.dataset["revision"],
+                trust_remote_code=self.metadata.dataset.get("trust_remote_code", False),
+            )
+        )
+
+        self.data_loaded = True
+
+
+class MIRACLRetrievalHardNegativesV2(MultilingualTask, AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="MIRACLRetrievalHardNegatives.v2",
+        description=(
+            "MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual retrieval "
+            "dataset that focuses on search across 18 different languages. The hard negative version has been "
+            "created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct."
+            "V2 uses a more appropriate prompt rather than the default prompt for retrieval."
+        ),
+        dataset={
+            "path": "mteb/miracl-hard-negatives",
+            "revision": "95c8db7d4a6e9c1d8a60601afd63d553ae20a2eb",
+            "trust_remote_code": True,
+        },
+        prompt={
+            "query": "Given a question, retrieve Wikipedia passages that answer the question",
+        },
+        adapted_from=["MIRACLRetrieval"],
+        **_common_metadata,
     )
 
     def load_data(self, **kwargs):

{mteb-1.39.5 → mteb-1.39.7}/mteb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 1.39.5
+Version: 1.39.7
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, nouamane@huggingface.co, info@nils-reimers.de
 License:                                  Apache License

{mteb-1.39.5 → mteb-1.39.7}/mteb.egg-info/SOURCES.txt

@@ -435,6 +435,7 @@ mteb/tasks/Classification/sin/SinhalaNewsSourceClassification.py
 mteb/tasks/Classification/sin/__init__.py
 mteb/tasks/Classification/slk/CSFDSKMovieReviewSentimentClassification.py
 mteb/tasks/Classification/slk/SlovakHateSpeechClassification.py
+mteb/tasks/Classification/slk/SlovakMovieReviewSentimentClassification.py
 mteb/tasks/Classification/slk/__init__.py
 mteb/tasks/Classification/slv/FrenkSlClassification.py
 mteb/tasks/Classification/slv/__init__.py
@@ -443,8 +444,6 @@ mteb/tasks/Classification/spa/SpanishSentimentClassification.py
 mteb/tasks/Classification/spa/__init__.py
 mteb/tasks/Classification/ssw/SiswatiNewsClassification.py
 mteb/tasks/Classification/ssw/__init__.py
-mteb/tasks/Classification/svk/SlovakMovieReviewSentimentClassification.py
-mteb/tasks/Classification/svk/__init__.py
 mteb/tasks/Classification/swa/SwahiliNewsClassification.py
 mteb/tasks/Classification/swa/__init__.py
 mteb/tasks/Classification/swe/DalajClassification.py

{mteb-1.39.5 → mteb-1.39.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mteb"
-version = "1.39.5"
+version = "1.39.7"
 description = "Massive Text Embedding Benchmark"
 readme = "README.md"
 authors = [