mteb 2.0.5__py3-none-any.whl → 2.1.19__py3-none-any.whl

mteb/descriptive_stats/Retrieval/HotpotQAHardNegatives.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 226621,
+        "number_of_characters": 84600866,
+        "documents_text_statistics": {
+            "total_text_length": 84508282,
+            "min_text_length": 8,
+            "average_text_length": 374.55858275603777,
+            "max_text_length": 3463,
+            "unique_texts": 225621
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 92584,
+            "min_text_length": 34,
+            "average_text_length": 92.584,
+            "max_text_length": 288,
+            "unique_texts": 1000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 2000,
+            "min_relevant_docs_per_query": 2,
+            "average_relevant_docs_per_query": 2.0,
+            "max_relevant_docs_per_query": 2,
+            "unique_relevant_docs": 1975
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/LegalQANLRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 30905,
+        "number_of_characters": 20629665,
+        "documents_text_statistics": {
+            "total_text_length": 20619027,
+            "min_text_length": 41,
+            "average_text_length": 669.3837288575788,
+            "max_text_length": 1716,
+            "unique_texts": 30172
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 10638,
+            "min_text_length": 27,
+            "average_text_length": 104.29411764705883,
+            "max_text_length": 369,
+            "unique_texts": 102
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 157,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.5392156862745099,
+            "max_relevant_docs_per_query": 8,
+            "unique_relevant_docs": 148
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/NFCorpus-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 3956,
+        "number_of_characters": 6345348,
+        "documents_text_statistics": {
+            "total_text_length": 6337710,
+            "min_text_length": 144,
+            "average_text_length": 1744.483897605285,
+            "max_text_length": 8480,
+            "unique_texts": 3593
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 7638,
+            "min_text_length": 3,
+            "average_text_length": 23.647058823529413,
+            "max_text_length": 89,
+            "unique_texts": 323
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 12334,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 38.18575851393189,
+            "max_relevant_docs_per_query": 475,
+            "unique_relevant_docs": 3128
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/OpenTenderRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 138633,
+        "number_of_characters": 59639635,
+        "documents_text_statistics": {
+            "total_text_length": 59576581,
+            "min_text_length": 2,
+            "average_text_length": 432.86552643624714,
+            "max_text_length": 16782,
+            "unique_texts": 122413
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 63054,
+            "min_text_length": 9,
+            "average_text_length": 63.054,
+            "max_text_length": 286,
+            "unique_texts": 992
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/QuoraRetrievalHardNegatives.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 178163,
+        "number_of_characters": 10498457,
+        "documents_text_statistics": {
+            "total_text_length": 10447229,
+            "min_text_length": 1,
+            "average_text_length": 58.96958732918273,
+            "max_text_length": 581,
+            "unique_texts": 176849
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 51228,
+            "min_text_length": 2,
+            "average_text_length": 51.228,
+            "max_text_length": 180,
+            "unique_texts": 1000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1641,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.641,
+            "max_relevant_docs_per_query": 34,
+            "unique_relevant_docs": 1641
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/RiaNewsRetrievalHardNegatives.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 192237,
+        "number_of_characters": 234466370,
+        "documents_text_statistics": {
+            "total_text_length": 234404032,
+            "min_text_length": 0,
+            "average_text_length": 1225.7253146619116,
+            "max_text_length": 2000,
+            "unique_texts": 191237
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 62338,
+            "min_text_length": 4,
+            "average_text_length": 62.338,
+            "max_text_length": 85,
+            "unique_texts": 1000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/SCIDOCS-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 26657,
+        "number_of_characters": 34261482,
+        "documents_text_statistics": {
+            "total_text_length": 34181168,
+            "min_text_length": 10,
+            "average_text_length": 1332.2355692403632,
+            "max_text_length": 9275,
+            "unique_texts": 25656
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 80314,
+            "min_text_length": 17,
+            "average_text_length": 80.314,
+            "max_text_length": 227,
+            "unique_texts": 1000
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 4928,
+            "min_relevant_docs_per_query": 27,
+            "average_relevant_docs_per_query": 4.928,
+            "max_relevant_docs_per_query": 30,
+            "unique_relevant_docs": 25657
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/SciFact-NL.v2.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 5483,
+        "number_of_characters": 8526662,
+        "documents_text_statistics": {
+            "total_text_length": 8496576,
+            "min_text_length": 228,
+            "average_text_length": 1639.3162261238665,
+            "max_text_length": 9187,
+            "unique_texts": 5183
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 30086,
+            "min_text_length": 24,
+            "average_text_length": 100.28666666666666,
+            "max_text_length": 228,
+            "unique_texts": 300
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 339,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.13,
+            "max_relevant_docs_per_query": 5,
+            "unique_relevant_docs": 283
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/VABBRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 10318,
+        "number_of_characters": 7839416,
+        "documents_text_statistics": {
+            "total_text_length": 7765564,
+            "min_text_length": 9,
+            "average_text_length": 833.393861343636,
+            "max_text_length": 35146,
+            "unique_texts": 9123
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 73852,
+            "min_text_length": 7,
+            "average_text_length": 73.852,
+            "max_text_length": 258,
+            "unique_texts": 999
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Retrieval/VDRMultilingualRetrieval.json

@@ -0,0 +1,184 @@
+{
+    "train": {
+        "num_samples": 16500,
+        "number_of_characters": 118992,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 447,
+            "average_image_width": 1401.1196666666667,
+            "max_image_width": 2743,
+            "min_image_height": 376,
+            "average_image_height": 1685.2892,
+            "max_image_height": 5257,
+            "unique_images": 14981
+        },
+        "queries_text_statistics": {
+            "total_text_length": 118992,
+            "min_text_length": 13,
+            "average_text_length": 79.328,
+            "max_text_length": 204,
+            "unique_texts": 1499
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1499,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1499
+        },
+        "top_ranked_statistics": null,
+        "hf_subset_descriptive_stats": {
+            "en": {
+                "num_samples": 3300,
+                "number_of_characters": 20947,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 653,
+                    "average_image_width": 1388.4603333333334,
+                    "max_image_width": 2464,
+                    "min_image_height": 878,
+                    "average_image_height": 1691.6246666666666,
+                    "max_image_height": 3533,
+                    "unique_images": 2996
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 20947,
+                    "min_text_length": 31,
+                    "average_text_length": 69.82333333333334,
+                    "max_text_length": 142,
+                    "unique_texts": 300
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 300,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 300
+                },
+                "top_ranked_statistics": null
+            },
+            "es": {
+                "num_samples": 3300,
+                "number_of_characters": 24935,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 447,
+                    "average_image_width": 1370.8263333333334,
+                    "max_image_width": 2743,
+                    "min_image_height": 376,
+                    "average_image_height": 1709.195,
+                    "max_image_height": 5257,
+                    "unique_images": 2997
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 24935,
+                    "min_text_length": 35,
+                    "average_text_length": 83.11666666666666,
+                    "max_text_length": 153,
+                    "unique_texts": 300
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 300,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 300
+                },
+                "top_ranked_statistics": null
+            },
+            "fr": {
+                "num_samples": 3300,
+                "number_of_characters": 25217,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 780,
+                    "average_image_width": 1402.3566666666666,
+                    "max_image_width": 2579,
+                    "min_image_height": 756,
+                    "average_image_height": 1689.5696666666668,
+                    "max_image_height": 2912,
+                    "unique_images": 2998
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 25217,
+                    "min_text_length": 37,
+                    "average_text_length": 84.05666666666667,
+                    "max_text_length": 152,
+                    "unique_texts": 299
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 299,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 299
+                },
+                "top_ranked_statistics": null
+            },
+            "de": {
+                "num_samples": 3300,
+                "number_of_characters": 23029,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 828,
+                    "average_image_width": 1394.5596666666668,
+                    "max_image_width": 2366,
+                    "min_image_height": 756,
+                    "average_image_height": 1686.0596666666668,
+                    "max_image_height": 2827,
+                    "unique_images": 2994
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 23029,
+                    "min_text_length": 35,
+                    "average_text_length": 76.76333333333334,
+                    "max_text_length": 143,
+                    "unique_texts": 300
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 300,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 300
+                },
+                "top_ranked_statistics": null
+            },
+            "it": {
+                "num_samples": 3300,
+                "number_of_characters": 24864,
+                "documents_text_statistics": null,
+                "documents_image_statistics": {
+                    "min_image_width": 788,
+                    "average_image_width": 1449.3953333333334,
+                    "max_image_width": 2583,
+                    "min_image_height": 804,
+                    "average_image_height": 1649.997,
+                    "max_image_height": 2168,
+                    "unique_images": 2996
+                },
+                "queries_text_statistics": {
+                    "total_text_length": 24864,
+                    "min_text_length": 13,
+                    "average_text_length": 82.88,
+                    "max_text_length": 204,
+                    "unique_texts": 300
+                },
+                "queries_image_statistics": null,
+                "relevant_docs_statistics": {
+                    "num_relevant_docs": 300,
+                    "min_relevant_docs_per_query": 1,
+                    "average_relevant_docs_per_query": 1.0,
+                    "max_relevant_docs_per_query": 1,
+                    "unique_relevant_docs": 300
+                },
+                "top_ranked_statistics": null
+            }
+        }
+    }
+}

mteb/descriptive_stats/Retrieval/WinoGrande.json

@@ -1,29 +1,29 @@
 {
     "test": {
-        "num_samples": 4872,
-        "number_of_characters": 9352943,
+        "num_samples": 6362,
+        "number_of_characters": 180770,
         "documents_text_statistics": {
-            "total_text_length": 8957572,
-            "min_text_length": 8,
-            "average_text_length": 3504.527386541471,
-            "max_text_length": 47929,
-            "unique_texts": 2556
+            "total_text_length": 39142,
+            "min_text_length": 2,
+            "average_text_length": 7.68243375858685,
+            "max_text_length": 31,
+            "unique_texts": 5095
         },
         "documents_image_statistics": null,
         "queries_text_statistics": {
-            "total_text_length": 395371,
-            "min_text_length": 8,
-            "average_text_length": 170.71286701208982,
-            "max_text_length": 2863,
-            "unique_texts": 2316
+            "total_text_length": 141628,
+            "min_text_length": 79,
+            "average_text_length": 111.78216258879242,
+            "max_text_length": 185,
+            "unique_texts": 1267
         },
         "queries_image_statistics": null,
         "relevant_docs_statistics": {
-            "num_relevant_docs": 2316,
+            "num_relevant_docs": 1267,
             "min_relevant_docs_per_query": 1,
             "average_relevant_docs_per_query": 1.0,
             "max_relevant_docs_per_query": 1,
-            "unique_relevant_docs": 988
+            "unique_relevant_docs": 478
         },
         "top_ranked_statistics": null
     }

mteb/descriptive_stats/Retrieval/bBSARDNLRetrieval.json

@@ -0,0 +1,30 @@
+{
+    "test": {
+        "num_samples": 22637,
+        "number_of_characters": 21218611,
+        "documents_text_statistics": {
+            "total_text_length": 21197901,
+            "min_text_length": 7,
+            "average_text_length": 945.7015837608744,
+            "max_text_length": 37834,
+            "unique_texts": 22415
+        },
+        "documents_image_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 20710,
+            "min_text_length": 22,
+            "average_text_length": 93.28828828828829,
+            "max_text_length": 250,
+            "unique_texts": 222
+        },
+        "queries_image_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1059,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 4.77027027027027,
+            "max_relevant_docs_per_query": 57,
+            "unique_relevant_docs": 491
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/STS/SICK-NL-STS.json

@@ -0,0 +1,28 @@
+{
+    "test": {
+        "num_samples": 4902,
+        "number_of_characters": 463327,
+        "unique_pairs": 4902,
+        "text1_statistics": {
+            "total_text_length": 233941,
+            "min_text_length": 10,
+            "average_text_length": 47.72358221134231,
+            "max_text_length": 158,
+            "unique_texts": 3378
+        },
+        "text2_statistics": {
+            "total_text_length": 229386,
+            "min_text_length": 10,
+            "average_text_length": 46.79436964504284,
+            "max_text_length": 158,
+            "unique_texts": 3327
+        },
+        "image1_statistics": null,
+        "image2_statistics": null,
+        "label_statistics": {
+            "min_score": 1.0,
+            "avg_score": 3.528012039368932,
+            "max_score": 5.0
+        }
+    }
+}

mteb/evaluate.py

@@ -256,6 +256,20 @@ def _check_model_modalities(
         logger.warning(msg)
 
 
+def _requires_merge(task: AbsTask, existing_results: TaskResult) -> bool:
+    """Check if the existing results require merging with new results."""
+    # If the task has multiple eval splits and existing results cover only a subset, we need to merge
+    required_evals = dict.fromkeys(task.eval_splits, task.hf_subsets)
+    for split, subsets in required_evals.items():
+        res = existing_results.scores.get(split, None)
+        if res is None:
+            return True
+        hf_subsets = [r["hf_subset"] for r in res]
+        if not set(subsets).issubset(set(hf_subsets)):
+            return True
+    return False
+
+
 def evaluate(
     model: ModelMeta | MTEBModels | SentenceTransformer | CrossEncoder,
     tasks: AbsTask | Iterable[AbsTask],
@@ -333,7 +347,7 @@ def evaluate(
         task = cast(AbsTaskAggregate, tasks)
         results = evaluate(
             model,
-            task.metadata.task_list,
+            task.metadata.tasks,
             co2_tracker=co2_tracker,
             raise_error=raise_error,
             encode_kwargs=encode_kwargs,
@@ -388,13 +402,18 @@ def evaluate(
 
     if (
         existing_results
-        and overwrite_strategy == "only-missing"
-        and overwrite_strategy == OverwriteStrategy.ONLY_MISSING
-        and existing_results.is_mergeable(task)
+        and overwrite_strategy
+        not in (OverwriteStrategy.ALWAYS, OverwriteStrategy.NEVER)
+        and (
+            not _requires_merge(task, existing_results)
+            or existing_results.is_mergeable(task)
+        )
     ):
         missing_eval = existing_results.get_missing_evaluations(task)
     else:
         missing_eval = dict.fromkeys(task.eval_splits, task.hf_subsets)
+        # Will be fully recomputed so we set it to None to avoid merging:
+        existing_results = None
 
     if (
         existing_results
@@ -415,12 +434,13 @@ def evaluate(
         OverwriteStrategy.ONLY_CACHE,
     ]:
         raise ValueError(
-            f"overwrite_strategy is set to '{overwrite_strategy.value}' and the results file exists. However there are the following missing splits (and subsets): {missing_eval}. To rerun these set overwrite_strategy to 'only-missing'."
+            f"overwrite_strategy is set to '{overwrite_strategy.value}' and the results file exists for task {task.metadata.name}. "
+            + f"However there are the following missing splits (and subsets): {missing_eval}. To rerun these set overwrite_strategy to 'only-missing'."
         )
 
     if existing_results:
         logger.info(
-            f"Found existing results for {task.metadata.name}, only running missing splits: {list(missing_eval.keys())}"
+            f"Found existing results for {task.metadata.name}, only running missing splits (subsets): {missing_eval}"
         )
 
     if isinstance(model, ModelMeta):

mteb/languages/check_language_code.py

@@ -13,7 +13,15 @@ def check_language_code(code: str) -> None:
     Args:
         code: The language code to check.
     """
-    lang, script = code.split("-")
+    lang = None
+    script = None
+    if "-" in code:
+        lang, script = code.split("-")
+    elif code[0].isupper():
+        script = code
+    else:
+        lang = code
+
     if script == "Code":
         if lang in PROGRAMMING_LANGS:
             return  # override for code
@@ -21,11 +29,11 @@ def check_language_code(code: str) -> None:
             raise ValueError(
                 f"Programming language {lang} is not a valid programming language."
             )
-    if lang not in ISO_TO_LANGUAGE:
+    if lang is not None and lang not in ISO_TO_LANGUAGE:
         raise ValueError(
             f"Invalid language code: {lang}, you can find valid ISO 639-3 codes in {path_to_lang_codes}"
         )
-    if script not in ISO_TO_SCRIPT:
+    if script is not None and script not in ISO_TO_SCRIPT:
         raise ValueError(
             f"Invalid script code: {script}, you can find valid ISO 15924 codes in {path_to_lang_scripts}"
         )