mteb 2.7.3__py3-none-any.whl → 2.7.5__py3-none-any.whl

mteb/abstasks/zeroshot_classification.py

@@ -127,6 +127,7 @@ class AbsTaskZeroShotClassification(AbsTask):
         hf_subset: str,
         encode_kwargs: EncodeKwargs,
         prediction_folder: Path | None = None,
+        num_proc: int = 1,
         **kwargs,
     ) -> ZeroShotClassificationMetrics:
         if not isinstance(model, EncoderProtocol):
@@ -145,7 +146,11 @@ class AbsTaskZeroShotClassification(AbsTask):
             hf_subset=hf_subset,
             **kwargs,
         )
-        probs = evaluator(model, encode_kwargs=encode_kwargs)
+        probs = evaluator(
+            model,
+            encode_kwargs=encode_kwargs,
+            num_proc=num_proc,
+        )
 
         if prediction_folder:
             self._save_task_predictions(
@@ -170,13 +175,14 @@ class AbsTaskZeroShotClassification(AbsTask):
             accuracy=metrics.accuracy_score(labels, predictions),
         )
 
-    def _push_dataset_to_hub(self, repo_name: str) -> None:
+    def _push_dataset_to_hub(self, repo_name: str, num_proc: int = 1) -> None:
         self._upload_dataset_to_hub(
             repo_name,
             [
                 self.input_column_name,
                 self.label_column_name,
             ],
+            num_proc=num_proc,
         )
         labels_dataset = Dataset.from_dict({"labels": self.get_candidate_labels()})
         labels_dataset.push_to_hub(repo_name, config_name="labels")

mteb/benchmarks/benchmarks/__init__.py

@@ -3,6 +3,7 @@ from mteb.benchmarks.benchmarks.benchmarks import (
     BEIR_NL,
     BRIGHT,
     BRIGHT_LONG,
+    BRIGHT_V1_1,
     BUILT_MTEB,
     C_MTEB,
     CHEMTEB,
@@ -69,6 +70,7 @@ __all__ = [
     "BEIR_NL",
     "BRIGHT",
     "BRIGHT_LONG",
+    "BRIGHT_V1_1",
     "BUILT_MTEB",
     "CHEMTEB",
     "CHEMTEB_V1_1",

mteb/benchmarks/benchmarks/benchmarks.py

@@ -1330,6 +1330,46 @@ This is the long version of the benchmark, which only filter longer documents.
 """,
 )
 
+BRIGHT_V1_1 = Benchmark(
+    name="BRIGHT(v1.1)",
+    display_name="Reasoning Retrieval",
+    tasks=get_tasks(
+        tasks=[
+            "BrightBiologyRetrieval",
+            "BrightEarthScienceRetrieval",
+            "BrightEconomicsRetrieval",
+            "BrightPsychologyRetrieval",
+            "BrightRoboticsRetrieval",
+            "BrightStackoverflowRetrieval",
+            "BrightSustainableLivingRetrieval",
+            "BrightPonyRetrieval",
+            "BrightLeetcodeRetrieval",
+            "BrightAopsRetrieval",
+            "BrightTheoremQATheoremsRetrieval",
+            "BrightTheoremQAQuestionsRetrieval",
+            "BrightBiologyLongRetrieval",
+            "BrightEarthScienceLongRetrieval",
+            "BrightEconomicsLongRetrieval",
+            "BrightPsychologyLongRetrieval",
+            "BrightRoboticsLongRetrieval",
+            "BrightStackoverflowLongRetrieval",
+            "BrightSustainableLivingLongRetrieval",
+            "BrightPonyLongRetrieval",
+        ],
+    ),
+    description="v1.1 refactors the BRIGHT into a different tasks and added prompt to individual tasks.",
+    reference="https://brightbenchmark.github.io/",
+    citation=r"""
+@article{su2024bright,
+  author = {Su, Hongjin and Yen, Howard and Xia, Mengzhou and Shi, Weijia and Muennighoff, Niklas and Wang, Han-yu and Liu, Haisu and Shi, Quan and Siegel, Zachary S and Tang, Michael and others},
+  journal = {arXiv preprint arXiv:2407.12883},
+  title = {Bright: A realistic and challenging benchmark for reasoning-intensive retrieval},
+  year = {2024},
+}
+""",
+)
+
+
 CODE_RAG = Benchmark(
     name="CodeRAG",
     tasks=get_tasks(
@@ -1781,8 +1821,7 @@ BEIR_NL = Benchmark(
             "TRECCOVID-NL",
         ],
     ),
-    description="BEIR-NL is a Dutch adaptation of the publicly available BEIR benchmark, created through automated "
-    "translation.",
+    description="BEIR-NL is a Dutch adaptation of the publicly available BEIR benchmark, created through automated translation.",
     reference="https://arxiv.org/abs/2412.08329",
     contacts=["nikolay-banar"],
     citation=r"""

mteb/descriptive_stats/Retrieval/BrightAopsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 188113,
+        "number_of_characters": 141769714,
+        "documents_text_statistics": {
+            "total_text_length": 141734227,
+            "min_text_length": 58,
+            "average_text_length": 753.8974425803981,
+            "max_text_length": 7334,
+            "unique_texts": 176508
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 35487,
+            "min_text_length": 85,
+            "average_text_length": 319.7027027027027,
+            "max_text_length": 1167,
+            "unique_texts": 111
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 524,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 4.7207207207207205,
+            "max_relevant_docs_per_query": 8,
+            "unique_relevant_docs": 111
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 20264921,
+            "min_top_ranked_per_query": 176954,
+            "average_top_ranked_per_query": 182566.85585585586,
+            "max_top_ranked_per_query": 186176
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightBiologyLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 627,
+        "number_of_characters": 19398082,
+        "documents_text_statistics": {
+            "total_text_length": 19344209,
+            "min_text_length": 142,
+            "average_text_length": 36916.42938931298,
+            "max_text_length": 1324201,
+            "unique_texts": 498
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 53873,
+            "min_text_length": 89,
+            "average_text_length": 523.0388349514564,
+            "max_text_length": 2195,
+            "unique_texts": 103
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 134,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.3009708737864079,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 134
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 53972,
+            "min_top_ranked_per_query": 524,
+            "average_top_ranked_per_query": 524.0,
+            "max_top_ranked_per_query": 524
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightBiologyRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 57462,
+        "number_of_characters": 18936054,
+        "documents_text_statistics": {
+            "total_text_length": 18882181,
+            "min_text_length": 1,
+            "average_text_length": 329.192994996426,
+            "max_text_length": 31130,
+            "unique_texts": 49434
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 53873,
+            "min_text_length": 89,
+            "average_text_length": 523.0388349514564,
+            "max_text_length": 2195,
+            "unique_texts": 103
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 374,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 3.6310679611650487,
+            "max_relevant_docs_per_query": 19,
+            "unique_relevant_docs": 374
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 5907977,
+            "min_top_ranked_per_query": 57359,
+            "average_top_ranked_per_query": 57359.0,
+            "max_top_ranked_per_query": 57359
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightEarthScienceLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 717,
+        "number_of_characters": 41696684,
+        "documents_text_statistics": {
+            "total_text_length": 41641374,
+            "min_text_length": 28,
+            "average_text_length": 69286.81198003328,
+            "max_text_length": 2627262,
+            "unique_texts": 587
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 55310,
+            "min_text_length": 83,
+            "average_text_length": 476.8103448275862,
+            "max_text_length": 1565,
+            "unique_texts": 116
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 187,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.6120689655172413,
+            "max_relevant_docs_per_query": 4,
+            "unique_relevant_docs": 187
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 69716,
+            "min_top_ranked_per_query": 601,
+            "average_top_ranked_per_query": 601.0,
+            "max_top_ranked_per_query": 601
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightEarthScienceRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 121365,
+        "number_of_characters": 40478259,
+        "documents_text_statistics": {
+            "total_text_length": 40422949,
+            "min_text_length": 1,
+            "average_text_length": 333.3878959826473,
+            "max_text_length": 233622,
+            "unique_texts": 117633
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 55310,
+            "min_text_length": 83,
+            "average_text_length": 476.8103448275862,
+            "max_text_length": 1565,
+            "unique_texts": 116
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 609,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 5.25,
+            "max_relevant_docs_per_query": 23,
+            "unique_relevant_docs": 609
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 14064884,
+            "min_top_ranked_per_query": 121249,
+            "average_top_ranked_per_query": 121249.0,
+            "max_top_ranked_per_query": 121249
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightEconomicsLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 619,
+        "number_of_characters": 19993261,
+        "documents_text_statistics": {
+            "total_text_length": 19917079,
+            "min_text_length": 43,
+            "average_text_length": 38598.99031007752,
+            "max_text_length": 429507,
+            "unique_texts": 515
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 76182,
+            "min_text_length": 164,
+            "average_text_length": 739.6310679611651,
+            "max_text_length": 2223,
+            "unique_texts": 103
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 109,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.058252427184466,
+            "max_relevant_docs_per_query": 3,
+            "unique_relevant_docs": 109
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 53148,
+            "min_top_ranked_per_query": 516,
+            "average_top_ranked_per_query": 516.0,
+            "max_top_ranked_per_query": 516
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightEconomicsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 50323,
+        "number_of_characters": 19882579,
+        "documents_text_statistics": {
+            "total_text_length": 19806397,
+            "min_text_length": 1,
+            "average_text_length": 394.3926125049781,
+            "max_text_length": 39672,
+            "unique_texts": 40594
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 76182,
+            "min_text_length": 164,
+            "average_text_length": 739.6310679611651,
+            "max_text_length": 2223,
+            "unique_texts": 103
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 823,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 7.990291262135922,
+            "max_relevant_docs_per_query": 85,
+            "unique_relevant_docs": 823
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 5172660,
+            "min_top_ranked_per_query": 50220,
+            "average_top_ranked_per_query": 50220.0,
+            "max_top_ranked_per_query": 50220
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightLeetcodeRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 414074,
+        "number_of_characters": 438348000,
+        "documents_text_statistics": {
+            "total_text_length": 438140779,
+            "min_text_length": 75,
+            "average_text_length": 1058.4849178125876,
+            "max_text_length": 103665,
+            "unique_texts": 413932
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 207221,
+            "min_text_length": 422,
+            "average_text_length": 1459.3028169014085,
+            "max_text_length": 3964,
+            "unique_texts": 142
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 262,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.8450704225352113,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 216
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 58744859,
+            "min_top_ranked_per_query": 412813,
+            "average_top_ranked_per_query": 413696.1901408451,
+            "max_top_ranked_per_query": 413923
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightPonyLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 689,
+        "number_of_characters": 2093720,
+        "documents_text_statistics": {
+            "total_text_length": 2050155,
+            "min_text_length": 28,
+            "average_text_length": 3553.1282495667247,
+            "max_text_length": 108885,
+            "unique_texts": 577
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 43565,
+            "min_text_length": 182,
+            "average_text_length": 388.9732142857143,
+            "max_text_length": 946,
+            "unique_texts": 112
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 769,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 6.866071428571429,
+            "max_relevant_docs_per_query": 12,
+            "unique_relevant_docs": 17
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 64624,
+            "min_top_ranked_per_query": 577,
+            "average_top_ranked_per_query": 577.0,
+            "max_top_ranked_per_query": 577
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightPonyRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 8006,
+        "number_of_characters": 2082980,
+        "documents_text_statistics": {
+            "total_text_length": 2039415,
+            "min_text_length": 5,
+            "average_text_length": 258.350012667849,
+            "max_text_length": 2583,
+            "unique_texts": 6183
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 43565,
+            "min_text_length": 182,
+            "average_text_length": 388.9732142857143,
+            "max_text_length": 946,
+            "unique_texts": 112
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 2519,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 22.491071428571427,
+            "max_relevant_docs_per_query": 32,
+            "unique_relevant_docs": 47
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 884128,
+            "min_top_ranked_per_query": 7894,
+            "average_top_ranked_per_query": 7894.0,
+            "max_top_ranked_per_query": 7894
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightPsychologyLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 613,
+        "number_of_characters": 20489389,
+        "documents_text_statistics": {
+            "total_text_length": 20419376,
+            "min_text_length": 23,
+            "average_text_length": 39881.59375,
+            "max_text_length": 669575,
+            "unique_texts": 509
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 70013,
+            "min_text_length": 166,
+            "average_text_length": 693.1980198019802,
+            "max_text_length": 2334,
+            "unique_texts": 101
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 116,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1485148514851484,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 113
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 51712,
+            "min_top_ranked_per_query": 512,
+            "average_top_ranked_per_query": 512.0,
+            "max_top_ranked_per_query": 512
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightPsychologyRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 52936,
+        "number_of_characters": 20372421,
+        "documents_text_statistics": {
+            "total_text_length": 20302408,
+            "min_text_length": 3,
+            "average_text_length": 384.26058483959497,
+            "max_text_length": 226941,
+            "unique_texts": 43756
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 70013,
+            "min_text_length": 166,
+            "average_text_length": 693.1980198019802,
+            "max_text_length": 2334,
+            "unique_texts": 101
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 742,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 7.346534653465347,
+            "max_relevant_docs_per_query": 59,
+            "unique_relevant_docs": 738
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 5336335,
+            "min_top_ranked_per_query": 52835,
+            "average_top_ranked_per_query": 52835.0,
+            "max_top_ranked_per_query": 52835
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightRoboticsLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 609,
+        "number_of_characters": 18386897,
+        "documents_text_statistics": {
+            "total_text_length": 18166762,
+            "min_text_length": 117,
+            "average_text_length": 35761.34251968504,
+            "max_text_length": 3589928,
+            "unique_texts": 505
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 220135,
+            "min_text_length": 165,
+            "average_text_length": 2179.5544554455446,
+            "max_text_length": 19341,
+            "unique_texts": 101
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 106,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0495049504950495,
+            "max_relevant_docs_per_query": 2,
+            "unique_relevant_docs": 106
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 51308,
+            "min_top_ranked_per_query": 508,
+            "average_top_ranked_per_query": 508.0,
+            "max_top_ranked_per_query": 508
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightRoboticsRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "standard": {
+        "num_samples": 62062,
+        "number_of_characters": 18167360,
+        "documents_text_statistics": {
+            "total_text_length": 17947225,
+            "min_text_length": 1,
+            "average_text_length": 289.6535724084505,
+            "max_text_length": 28637,
+            "unique_texts": 40431
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 220135,
+            "min_text_length": 165,
+            "average_text_length": 2179.5544554455446,
+            "max_text_length": 19341,
+            "unique_texts": 101
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 553,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 5.475247524752476,
+            "max_relevant_docs_per_query": 36,
+            "unique_relevant_docs": 553
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 6258061,
+            "min_top_ranked_per_query": 61961,
+            "average_top_ranked_per_query": 61961.0,
+            "max_top_ranked_per_query": 61961
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/BrightStackoverflowLongRetrieval.json

@@ -0,0 +1,35 @@
+{
+    "long": {
+        "num_samples": 1975,
+        "number_of_characters": 184326754,
+        "documents_text_statistics": {
+            "total_text_length": 184175475,
+            "min_text_length": 41,
+            "average_text_length": 99125.65931108719,
+            "max_text_length": 9182738,
+            "unique_texts": 1846
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 151279,
+            "min_text_length": 185,
+            "average_text_length": 1292.982905982906,
+            "max_text_length": 12432,
+            "unique_texts": 117
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 129,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.1025641025641026,
+            "max_relevant_docs_per_query": 2,
+            "unique_relevant_docs": 125
+        },
+        "top_ranked_statistics": {
+            "num_top_ranked": 217386,
+            "min_top_ranked_per_query": 1858,
+            "average_top_ranked_per_query": 1858.0,
+            "max_top_ranked_per_query": 1858
+        }
+    }
+}