mteb 2.1.1__py3-none-any.whl → 2.1.3__py3-none-any.whl

mteb/_evaluators/retrieval_metrics.py

@@ -5,7 +5,6 @@ from typing import Any
 import numpy as np
 import pandas as pd
 import pytrec_eval
-import torch
 from packaging.version import Version
 from sklearn.metrics import auc
 
@@ -14,14 +13,6 @@ from mteb.types import RelevantDocumentsType, RetrievalEvaluationResult
 logger = logging.getLogger(__name__)
 
 
-try:
-    # speeds up computation if available
-    torch.set_float32_matmul_precision("high")
-    logger.info("Setting torch float32 matmul precision to high for a speedup")
-except Exception:
-    pass
-
-
 def mrr(
     qrels: RelevantDocumentsType,
     results: dict[str, dict[str, float]],

mteb/benchmarks/benchmarks/benchmarks.py

@@ -1634,7 +1634,7 @@ BEIR_NL = Benchmark(
 
 MTEB_NL = Benchmark(
     name="MTEB(nld, v1)",
-    display_name="MTEB-NL",
+    display_name="Dutch",
     icon="https://github.com/lipis/flag-icons/raw/260c91531be024944c6514130c5defb2ebb02b7d/flags/4x3/nl.svg",
     tasks=MTEBTasks(
         get_tasks(
@@ -1678,7 +1678,7 @@ MTEB_NL = Benchmark(
                 "SciFact-NL",
                 "NFCorpus-NL",
                 "BelebeleRetrieval",
-                # "WebFAQRetrieval",
+                "WebFAQRetrieval",
                 "DutchNewsArticlesRetrieval",
                 "bBSARDNLRetrieval",
                 "LegalQANLRetrieval",

mteb/evaluate.py

@@ -333,7 +333,7 @@ def evaluate(
         task = cast(AbsTaskAggregate, tasks)
         results = evaluate(
             model,
-            task.metadata.task_list,
+            task.metadata.tasks,
             co2_tracker=co2_tracker,
             raise_error=raise_error,
             encode_kwargs=encode_kwargs,

mteb/leaderboard/app.py

@@ -318,7 +318,7 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
             """
         ## Embedding Leaderboard
 
-        This leaderboard compares 100+ text and image embedding models across 1000+ languages. We refer to the publication of each selectable benchmark for details on metrics, languages, tasks, and task types. Anyone is welcome [to add a model](https://github.com/embeddings-benchmark/mteb/blob/main/docs/adding_a_model.md), [add benchmarks](https://github.com/embeddings-benchmark/mteb/blob/main/docs/adding_a_benchmark.md), [help us improve zero-shot annotations](https://github.com/embeddings-benchmark/mteb/blob/06489abca007261c7e6b11f36d4844c5ed5efdcb/mteb/models/bge_models.py#L91) or [propose other changes to the leaderboard](https://github.com/embeddings-benchmark/mteb/tree/main/mteb/leaderboard).
+        This leaderboard compares 100+ text and image embedding models across 1000+ languages. We refer to the publication of each selectable benchmark for details on metrics, languages, tasks, and task types. Anyone is welcome [to add a model](https://embeddings-benchmark.github.io/mteb/contributing/adding_a_model/), [add benchmarks](https://embeddings-benchmark.github.io/mteb/contributing/adding_a_benchmark/), [help us improve zero-shot annotations](https://github.com/embeddings-benchmark/mteb/blob/06489abca007261c7e6b11f36d4844c5ed5efdcb/mteb/models/bge_models.py#L91) or [propose other changes to the leaderboard](https://github.com/embeddings-benchmark/mteb/issues/new?template=enhancement.yaml).
         """
         )
         gr.Markdown(

mteb/leaderboard/benchmark_selector.py

@@ -73,6 +73,7 @@ GP_BENCHMARK_ENTRIES = [
                         "MTEB(fra, v1)",
                         "MTEB(jpn, v1)",
                         "MTEB(kor, v1)",
+                        "MTEB(nld, v1)",
                         "MTEB(pol, v1)",
                         "MTEB(rus, v1)",
                         "MTEB(fas, v2)",

mteb/models/model_implementations/reasonir_model.py

@@ -18,7 +18,7 @@ def instruction_template(
 ) -> str:
     return (
         # https://github.com/facebookresearch/ReasonIR/blob/0aac96269e455965949df16520fab72da68ffc22/evaluation/bright/configs/reasonir/economics.json#L3
-        f"<|user|>\n{instruction}<|embed|>\n"
+        f"<|user|>\n{instruction}\n<|embed|>\n"
         if (prompt_type is None or prompt_type == PromptType.query) and instruction
         else "<|embed|>\n"
     )

{mteb-2.1.1.dist-info → mteb-2.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.1.1
+Version: 2.1.3
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.1.1.dist-info → mteb-2.1.3.dist-info}/RECORD

@@ -7,7 +7,7 @@ mteb/_requires_package.py,sha256=eHg_TD9BVZRzNCcQQrUP17d8M1DF_vOd_tVx54AmAnM,301
 mteb/_set_seed.py,sha256=HPlPRl__Pe6IG-4UgJqTfplcivJ_wA2kaClbXoHQedM,1178
 mteb/cache.py,sha256=rjpXTo9FTuYNwuZGQxZ1v9Yj8pIeMvwW7WNWM4CI6zA,20010
 mteb/deprecated_evaluator.py,sha256=t13Eluvm5ByVIOqgT7fqiVfLb8Ud3A4bbF2djRfs8iA,26901
-mteb/evaluate.py,sha256=nk7OzWuLTgcEJsrHY6KcRlcUJyeTJu3xujytmDkHXZ0,17157
+mteb/evaluate.py,sha256=a8dE6GtDC8TC4Q9rvJfzPuPNDbuNJSx8TAhR-aFMnLQ,17153
 mteb/filter_tasks.py,sha256=5XE1OYmgDDoJYnXwFf4ma_PIT_Lekzs420sQF_kpCiY,7240
 mteb/get_tasks.py,sha256=6Gc18a2bZoLQV1Ms_qdr2KieAqIXg8TDg4l7ZN8rW2I,14218
 mteb/load_results.py,sha256=Xw2ZX7BToU92WwUTQUQKPAgPhX7ucyRRdoCrxAoPHdI,6414
@@ -19,7 +19,7 @@ mteb/_evaluators/clustering_evaluator.py,sha256=uw9mNJ_swuZFwjU8b1XRMsI49nip-_fO
 mteb/_evaluators/evaluator.py,sha256=gwaeftcAKoGcIQs8jIaafynbcYrYErj6AitHBxgjn2w,807
 mteb/_evaluators/pair_classification_evaluator.py,sha256=m-xxDvSLrsqccbpmHur3h2dLl5YAC8Noy4n3-p0ewBM,6770
 mteb/_evaluators/retrieval_evaluator.py,sha256=HsowKZkqRCNzTwM7EcsHX18KhVKAjrm0sa_wFrreCb8,3031
-mteb/_evaluators/retrieval_metrics.py,sha256=EzeasJMK2aDCXLtcXKUsdZwLrJF3pV_eLh6H3pd1BD0,23803
+mteb/_evaluators/retrieval_metrics.py,sha256=we0damQCJrdaRUD6JlU2MM7Ls9xERP_OBS5gHt53u9Q,23588
 mteb/_evaluators/sklearn_evaluator.py,sha256=P6YRx4we1AHmIZJV_VTzHiZ-tx1kV7iG_t4vutlXixA,4491
 mteb/_evaluators/zeroshot_classification_evaluator.py,sha256=i_4XXqZ_naSWHWSJChzxby4x9EGG0FjOSfDzB_7IVNc,2288
 mteb/_evaluators/image/__init__.py,sha256=CsQd7OMkeV2Phun7paPWjayZ5qRnvj8H0TYBFeqMxag,148
@@ -56,7 +56,7 @@ mteb/benchmarks/_create_table.py,sha256=gYgwhACmY8JSN00y6r3HfHNQphCIuvcw_IwRZNQw
 mteb/benchmarks/benchmark.py,sha256=9pbrn5vsSB9Xo23YqQXjRdzorXnMulkm1QvJUkqvuGk,3445
 mteb/benchmarks/get_benchmark.py,sha256=Zh_C_RXl02hBxg6ffQlxEFERrokaern1RAN4yh2pa2E,3833
 mteb/benchmarks/benchmarks/__init__.py,sha256=Et7Gf7pKDxa939bIvAJB6OCirQrWAbmbbI3LHA6eMa0,2070
-mteb/benchmarks/benchmarks/benchmarks.py,sha256=_qo3bfJsEhqGdovIPxZnYCT-4sJynNMpBdV0AjhMTgw,88420
+mteb/benchmarks/benchmarks/benchmarks.py,sha256=iddOlyNIqnxd6Mp_-cQcADKbZ3gc2uZKxyEutJvv78o,88416
 mteb/benchmarks/benchmarks/rteb_benchmarks.py,sha256=QnCSrTTaBfcRlAQp2Nu81tgv1idMXqiM16Fp2zKJ5Ys,10607
 mteb/cli/__init__.py,sha256=v-csUr3eUZElIvrGB6QGtaIdndDfNWEe9oZchsGsJpg,64
 mteb/cli/_display_tasks.py,sha256=7A06dT9sSoTz6shyMvskPxuc5eHY_H7PGPlROzMP0yw,2196
@@ -1408,8 +1408,8 @@ mteb/languages/language_family.json,sha256=OUGcHeOIPcZPb2FWmYLhxTS0JxjK5y3Fo6x0P
 mteb/languages/language_scripts.py,sha256=5wix9HTYolNIpTiS5oXf2pGJyL7ftdGKs_m432w81V8,3998
 mteb/languages/programming_languages.py,sha256=zxAakT3OSUnAuTnQ34VyeFIECnNXMlleZmAake6jsZE,211
 mteb/leaderboard/__init__.py,sha256=991roXmtRwEQysV-37hWEzWpkvPgMCGRqZTHR-hm2io,88
-mteb/leaderboard/app.py,sha256=5XcuTCJUS5WPDTBejeVETWY3vpn78Fq7zVR2vv0iS7U,32575
-mteb/leaderboard/benchmark_selector.py,sha256=TnW2Y-xoDkEH3ZhR_8EW42WZN_XrtfucUbiuiNjChCA,7580
+mteb/leaderboard/app.py,sha256=xvOcK_ICmAwl1Mo5muI7fmCjYeSt1ztSgsQDrPF6OvM,32575
+mteb/leaderboard/benchmark_selector.py,sha256=1PRTDcNU4o2ZVHAzAUVp6UxWNElZDcGbnKHsNGv4aco,7621
 mteb/leaderboard/figures.py,sha256=Rq20LFpaUhQD4tuKp7P7ExQtAjonMLibgO3ud0ykMag,7491
 mteb/leaderboard/table.py,sha256=qs0H_Gt9FzRvzb-AL0YlqEe0YAsdYsVX3QlncfCBEqg,7828
 mteb/leaderboard/text_segments.py,sha256=iMIkS04QQjPbT-SkU0x6fOcS8xRbUYevryu9HydipKM,6570
@@ -1505,7 +1505,7 @@ mteb/models/model_implementations/qtack_models.py,sha256=biZLH5E3UWIcMZXIZNGgBZF
 mteb/models/model_implementations/qwen3_models.py,sha256=IpPJC_y-Kt_1Bq6nT2lX3-lMl-hl0BV6VhAL8Czfyfg,5133
 mteb/models/model_implementations/qzhou_models.py,sha256=90tsGftdCX90Gvu63bxtMyzczGi_u9bsUlg4WAVeH9Q,3536
 mteb/models/model_implementations/random_baseline.py,sha256=NH-epZ73BYvV6qYgu09BsoiFxBWypUNLdJIXRp5QPhM,8030
-mteb/models/model_implementations/reasonir_model.py,sha256=MJWHFiu4x2s3GwpuI5_E9bcwvU_KL-FIROzGaw0SMJQ,2216
+mteb/models/model_implementations/reasonir_model.py,sha256=wSCcJpUgZ0pG2g3vTEzYNmPlPG_CVn_rR0ENVCines0,2218
 mteb/models/model_implementations/repllama_models.py,sha256=89HoqEpzkNysHeuf_-YhU8WETamHTogSRztGIRo6G1s,7321
 mteb/models/model_implementations/rerankers_custom.py,sha256=ro73A9-hHudy3_qIMrhP-ja-3Xqu78r_aORm856zHQc,10651
 mteb/models/model_implementations/rerankers_monot5_based.py,sha256=rxVwzapNnHl4gCw79XVCaTXj3-wbToyj7XVL97tpAF4,34302
@@ -2536,9 +2536,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=CRAUc5IvqI3_9SyXDwv-PWLCXwXdZem9RePeYESRtuw,996
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=YwJsxTf1eaCI_RE-J37a-gK5wDeGAsmkeZKoZCFihSo,3755
-mteb-2.1.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.1.1.dist-info/METADATA,sha256=T4fFqeirUqZMgxCCsxYn0bCZDj13uyWBNKXR_DWnRKE,13573
-mteb-2.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mteb-2.1.1.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.1.1.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.1.1.dist-info/RECORD,,
+mteb-2.1.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.1.3.dist-info/METADATA,sha256=fSSp_uFAo3SSrjhhMQGHJMDIyEjeinv-7QjmK11VrPQ,13573
+mteb-2.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mteb-2.1.3.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.1.3.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.1.3.dist-info/RECORD,,