mteb 2.7.5__py3-none-any.whl → 2.7.7__py3-none-any.whl

mteb/abstasks/retrieval.py

@@ -263,7 +263,7 @@ class AbsTaskRetrieval(AbsTask):
             return
 
         dataset_path = self.metadata.dataset["path"]
-        eval_splits = self.metadata.eval_splits
+        eval_splits = self.eval_splits
         trust_remote_code = self.metadata.dataset.get("trust_remote_code", False)
         revision = self.metadata.dataset["revision"]
 
@@ -284,7 +284,7 @@ class AbsTaskRetrieval(AbsTask):
             )
 
         if self.metadata.is_multilingual:
-            for lang in self.metadata.eval_langs:
+            for lang in self.hf_subsets:
                 for split in eval_splits:
                     _process_data(split, lang)
         else:

mteb/abstasks/retrieval_dataset_loaders.py

@@ -139,9 +139,9 @@ class RetrievalDatasetLoader:
         )
 
     def _load_corpus(self, num_proc: int) -> CorpusDatasetType:
-        logger.info("Loading Corpus...")
-
         config = f"{self.config}-corpus" if self.config is not None else "corpus"
+        logger.info("Loading corpus subset: %s", config)
+
         corpus_ds = self._load_dataset_split(config, num_proc)
         if "_id" in corpus_ds.column_names:
             corpus_ds = corpus_ds.cast_column("_id", Value("string")).rename_column(
@@ -152,9 +152,9 @@ class RetrievalDatasetLoader:
         return corpus_ds
 
     def _load_queries(self, num_proc: int) -> QueryDatasetType:
-        logger.info("Loading Queries...")
-
         config = f"{self.config}-queries" if self.config is not None else "queries"
+        logger.info("Loading queries subset: %s", config)
+
         if "query" in self.dataset_configs:
             config = "query"
         queries_ds = self._load_dataset_split(config, num_proc)
@@ -169,9 +169,9 @@ class RetrievalDatasetLoader:
         return queries_ds
 
     def _load_qrels(self, num_proc: int) -> RelevantDocumentsType:
-        logger.info("Loading qrels...")
-
         config = f"{self.config}-qrels" if self.config is not None else "default"
+
+        logger.info("Loading qrels subset: %s", config)
         if config == "default" and config not in self.dataset_configs:
             if "qrels" in self.dataset_configs:
                 config = "qrels"
@@ -204,11 +204,10 @@ class RetrievalDatasetLoader:
         return qrels_dict
 
     def _load_top_ranked(self, num_proc: int) -> TopRankedDocumentsType:
-        logger.info("Loading Top Ranked")
-
         config = (
             f"{self.config}-top_ranked" if self.config is not None else "top_ranked"
         )
+        logger.info("Loading top ranked subset: %s", config)
         top_ranked_ds = self._load_dataset_split(config, num_proc)
         top_ranked_ds = top_ranked_ds.cast(
             Features(
@@ -228,11 +227,10 @@ class RetrievalDatasetLoader:
         return top_ranked_dict
 
     def _load_instructions(self, num_proc: int) -> InstructionDatasetType:
-        logger.info("Loading Instructions")
-
         config = (
             f"{self.config}-instruction" if self.config is not None else "instruction"
         )
+        logger.info("Loading instruction subset: %s", config)
         instructions_ds = self._load_dataset_split(config, num_proc)
         instructions_ds = instructions_ds.cast(
             Features(

mteb/benchmarks/_create_table.py

@@ -119,7 +119,6 @@ def _create_summary_table_from_benchmark_results(
 
     # Build joint table
     joint_table = mean_per_type.copy()
-    joint_table = joint_table.drop(models_to_remove, axis=0)
     joint_table.insert(0, "mean", overall_mean)
     joint_table.insert(1, "mean_by_task_type", typed_mean)
     joint_table["borda_rank"] = _get_borda_rank(per_task)
@@ -359,7 +358,6 @@ def _create_summary_table_mean_public_private(
 
     # Build joint table
     joint_table = mean_per_type.copy()
-    joint_table = joint_table.drop(models_to_remove, axis=0)
     joint_table.insert(0, "mean(public)", public_mean)
     joint_table.insert(1, "mean(private)", private_mean)
     if exclude_private_from_borda:
@@ -486,7 +484,6 @@ def _create_summary_table_mean_subset(
 
     # Build joint table
     joint_table = mean_per_type.copy()
-    joint_table = joint_table.drop(models_to_remove, axis=0)
     joint_table.insert(0, "mean(subset)", overall_subset_mean)
     joint_table["borda_rank"] = _get_borda_rank(per_subset)
     joint_table = joint_table.sort_values("mean(subset)", ascending=False)
@@ -605,7 +602,6 @@ def _create_summary_table_mean_task_type(
 
     # Build joint table
     joint_table = mean_per_type.copy()
-    joint_table = joint_table.drop(models_to_remove, axis=0)
     joint_table.insert(0, "mean_by_task_type", typed_mean)
     joint_table = joint_table.sort_values("mean_by_task_type", ascending=False)
     joint_table["borda_rank"] = _get_borda_rank(per_task)

mteb/evaluate.py

@@ -363,6 +363,9 @@ def evaluate(
             num_proc=num_proc,
         )
         combined_results = aggregated_task.combine_task_results(results.task_results)
+        if cache:
+            cache.save_to_cache(combined_results, meta)
+
         return ModelResult(
             model_name=results.model_name,
             model_revision=results.model_revision,

mteb/leaderboard/app.py

@@ -550,7 +550,10 @@ def get_leaderboard_app(cache: ResultCache = ResultCache()) -> gr.Blocks:
 
     logger.info("Step 7/7: Building Gradio interface and callbacks...")
     interface_start = time.time()
-    with gr.Blocks(fill_width=True) as demo:
+    with gr.Blocks(
+        title="MTEB Leaderboard",
+        fill_width=True,
+    ) as demo:
         with gr.Sidebar(
             position="left",
             label="Benchmark Selection and Customization",

mteb/models/get_model_meta.py

@@ -123,7 +123,10 @@ def get_model(
 
 
 def get_model_meta(
-    model_name: str, revision: str | None = None, fetch_from_hf: bool = True
+    model_name: str,
+    revision: str | None = None,
+    fetch_from_hf: bool = True,
+    fill_missing: bool = False,
 ) -> ModelMeta:
     """A function to fetch a model metadata object by name.
 
@@ -131,6 +134,7 @@ def get_model_meta(
         model_name: Name of the model to fetch
         revision: Revision of the model to fetch
         fetch_from_hf: Whether to fetch the model from HuggingFace Hub if not found in the registry
+        fill_missing: Computes missing attributes from the metadata including number of parameters and memory usage.
 
     Returns:
         A model metadata object
@@ -142,10 +146,25 @@ def get_model_meta(
             raise ValueError(
                 f"Model revision {revision} not found for model {model_name}. Expected {model_meta.revision}."
             )
+
+        if fill_missing and fetch_from_hf:
+            original_meta_dict = model_meta.model_dump()
+            new_meta = ModelMeta.from_hub(model_name)
+            new_meta_dict = new_meta.model_dump(exclude_none=True)
+
+            updates = {
+                k: v
+                for k, v in new_meta_dict.items()
+                if original_meta_dict.get(k) is None
+            }
+
+            if updates:
+                return model_meta.model_copy(update=updates)
         return model_meta
+
     if fetch_from_hf:
         logger.info(
-            "Model not found in model registry. Attempting to extract metadata by loading the model ({model_name}) using HuggingFace."
+            f"Model not found in model registry. Attempting to extract metadata by loading the model ({model_name}) using HuggingFace."
         )
         meta = ModelMeta.from_hub(model_name, revision)
         return meta

mteb/models/model_implementations/voyage_models.py

@@ -182,6 +182,7 @@ class VoyageModel(AbsEncoder):
                     model=self._model_name,
                     input_type=input_type,
                     output_dtype=output_dtype,
+                    output_dimension=self.mteb_model_meta.embed_dim,
                 ).embeddings
             )
             pbar.update(len(batch))
@@ -215,6 +216,32 @@ model_prompts = {
     PromptType.document.value: "document",
 }
 
+voyage_4_large_2048d = ModelMeta(
+    name="voyageai/voyage-4-large (embed_dim=2048)",
+    model_type=["dense"],
+    revision="1",
+    release_date="2026-01-15",
+    languages=None,  # supported languages not specified
+    loader=VoyageModel,
+    loader_kwargs=dict(
+        max_tokens=32000,
+        model_prompts=model_prompts,
+    ),
+    max_tokens=32000,
+    embed_dim=2048,
+    open_weights=False,
+    n_parameters=None,
+    memory_usage_mb=None,
+    license=None,
+    reference="https://blog.voyageai.com/2026/01/15/voyage-4/",
+    similarity_fn_name="cosine",
+    framework=["API"],
+    use_instructions=True,
+    training_datasets=VOYAGE_TRAINING_DATA,
+    public_training_code=None,
+    public_training_data=None,
+)
+
 voyage_4 = ModelMeta(
     name="voyageai/voyage-4",
     model_type=["dense"],

mteb/results/task_result.py

@@ -647,16 +647,26 @@ class TaskResult(BaseModel):
             if split not in splits:
                 continue
             seen_subsets = set()
-            # Use list comprehension for better performance
-            new_scores[split] = [
-                _scores
-                for _scores in self.scores[split]
-                if _scores["hf_subset"] in hf_subsets
-            ]
+            if task.is_aggregate:
+                # aggregate tasks only have the default subset, but in metadata can be multiple
+                new_scores[split] = [
+                    _scores
+                    for _scores in self.scores[split]
+                    if _scores["hf_subset"] == "default"
+                ]
+                seen_subsets = {"default"}
+            else:
+                new_scores[split] = [
+                    _scores
+                    for _scores in self.scores[split]
+                    if _scores["hf_subset"] in hf_subsets
+                ]
             for _scores in new_scores[split]:
                 seen_subsets.add(_scores["hf_subset"])
 
-            if seen_subsets != hf_subsets:
+            if seen_subsets != hf_subsets and not (
+                task.is_aggregate and "default" in seen_subsets
+            ):
                 missing_subsets = hf_subsets - seen_subsets
                 if len(missing_subsets) > 2:
                     subset1, subset2 = list(missing_subsets)[:2]

{mteb-2.7.5.dist-info → mteb-2.7.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.7.5
+Version: 2.7.7
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.7.5.dist-info → mteb-2.7.7.dist-info}/RECORD

@@ -7,7 +7,7 @@ mteb/_requires_package.py,sha256=eHg_TD9BVZRzNCcQQrUP17d8M1DF_vOd_tVx54AmAnM,301
 mteb/_set_seed.py,sha256=HPlPRl__Pe6IG-4UgJqTfplcivJ_wA2kaClbXoHQedM,1178
 mteb/cache.py,sha256=xE2BhrG_ypm2C0nPt6tRQCe393pN1RjDLadQcC6W1WE,27657
 mteb/deprecated_evaluator.py,sha256=F47pBHL3lM8YHGZEPLR4CuVn7aV3gzomNpyP3I3jmmc,27727
-mteb/evaluate.py,sha256=eudGyxRukbR6s7iAOPAvbOyLKK8hBwGRicYffX2ltzA,19643
+mteb/evaluate.py,sha256=XFNaxut-prQDwYw_1F4Oco9B_pumUOHI6Ke2PBLk22U,19718
 mteb/filter_tasks.py,sha256=ZFFEl7jWTSq3kDdM8oHG-RZrrBOP16zcFzezA3OsLiM,7430
 mteb/get_tasks.py,sha256=1BZJF7jADJnnL_ZzsJaTKLITO7u6inAHV0PxCAQfhfc,14402
 mteb/load_results.py,sha256=geGyArDB1CNrqse_QzPqfvY7YvF9fL57LJsuy-DDY58,6532
@@ -42,8 +42,8 @@ mteb/abstasks/dataset_card_template.md,sha256=aD6l8qc3_jxwoIGJNYLzse-jpRa8hu92Ax
 mteb/abstasks/multilabel_classification.py,sha256=rFa_Pw2OsUzqhZS-jh2zFD7I-TNl8bVNJ-DW7EpPapU,9708
 mteb/abstasks/pair_classification.py,sha256=MpjOeIcM_RMqSpkXc8PFDAwDgmb54gEqj6ZOUcOa1aQ,14122
 mteb/abstasks/regression.py,sha256=ZuMZfOwU3G4hr__eHsgdagKKdrbN4-wQMLz45jr9YUc,8946
-mteb/abstasks/retrieval.py,sha256=bw7-5QnLzfclvw5o8RO2sljhajsplmpEPdVZ9nYKj_g,27761
-mteb/abstasks/retrieval_dataset_loaders.py,sha256=XDJHh_11R3o4L0Y80kL0ia3DHMmZw3EqWslEQakvMOc,9900
+mteb/abstasks/retrieval.py,sha256=BPyRibStAD70JfR0Z1x-VVVfzJDRVSmbOS6uREfpmok,27743
+mteb/abstasks/retrieval_dataset_loaders.py,sha256=p0y1nrWlUrt_aeoR4ocDLEQMLuD_SlMH0gBiUsOwrww,9983
 mteb/abstasks/sts.py,sha256=Xta3KVQE7hHqkPTDptemvNVEG0CsZSVjA-Z52EIBvDE,9576
 mteb/abstasks/task_metadata.py,sha256=_yVgX3yjaxVvK-Cs2nw2wMEkI0xf-QhcGb_byCeyo-w,27210
 mteb/abstasks/zeroshot_classification.py,sha256=UrUErSH4owJujRn-HQI6KUe4By12oRjZeOnvB6lBnNA,6360
@@ -57,7 +57,7 @@ mteb/abstasks/text/bitext_mining.py,sha256=an9ZlcraS9ZN5tCT8ZQFtMbCrPE4-n7anAAbn
 mteb/abstasks/text/reranking.py,sha256=mCzy0-TnZ46_GC9Czl4zWKAPnYK5ur0qtFbPt47m9jQ,7917
 mteb/abstasks/text/summarization.py,sha256=bSgb0XhUzJVuLV1Wjr3HYB_Tn7SjmCDMnkBIEWHO4EQ,7381
 mteb/benchmarks/__init__.py,sha256=MQEVeli-zLaJ7Xg0z7RhXQwsdmm7Ht_W2Ln0rZo1Szc,225
-mteb/benchmarks/_create_table.py,sha256=rsckziAot5EuD-aarQ6JnGurNsq6X2zn_nl43_Ge9ng,22651
+mteb/benchmarks/_create_table.py,sha256=CJL8U0adUbaxr5G26trfYo1tGx8cU2IWWx_ZHU6q6do,22407
 mteb/benchmarks/benchmark.py,sha256=YCGIvJ5Vc6GdCAYSjzwrnfj2A8MkbzNLvvtPBLMSSp8,6327
 mteb/benchmarks/get_benchmark.py,sha256=nzR6cu5yXu1kIJKhd4A2R62xp43Z62bluPbOpNXHMWQ,2545
 mteb/benchmarks/benchmarks/__init__.py,sha256=-o3EMWEfP0eQ8iZpWvTj5r4yuGOUuL9mHk8IgFcpPtk,2330
@@ -1472,14 +1472,14 @@ mteb/languages/language_family.json,sha256=OUGcHeOIPcZPb2FWmYLhxTS0JxjK5y3Fo6x0P
 mteb/languages/language_scripts.py,sha256=AjtdlJlVRE3TWASdmREKw0P-xAoVFdl_Rut5DgX6k6c,4149
 mteb/languages/programming_languages.py,sha256=zxAakT3OSUnAuTnQ34VyeFIECnNXMlleZmAake6jsZE,211
 mteb/leaderboard/__init__.py,sha256=991roXmtRwEQysV-37hWEzWpkvPgMCGRqZTHR-hm2io,88
-mteb/leaderboard/app.py,sha256=te9F3F5u6pMYzEsLtBnOWSr-6ZQ4ZvXVSmbfxuu3cfQ,42500
+mteb/leaderboard/app.py,sha256=Y3G93VJq6eZMD4_CNzLwSEEGnuNJDqYEYztmzYR85eA,42549
 mteb/leaderboard/benchmark_selector.py,sha256=qd-2L20RQ4ACke01UlytkhZok1dkWgfUlXzfET52kGc,7956
 mteb/leaderboard/figures.py,sha256=cfOK82rRf-7sCjyP7GBxh4ezhOIt0OhD0_86mKtzLrg,7530
 mteb/leaderboard/table.py,sha256=U5mWtrVUTk_6t8T4KAp5qlbFgKh1PD0iKICqNMfhsoY,10462
 mteb/leaderboard/text_segments.py,sha256=iMIkS04QQjPbT-SkU0x6fOcS8xRbUYevryu9HydipKM,6570
 mteb/models/__init__.py,sha256=ABTuoqiBjBtBWW3LYY7ItBHdylR6jWoy06HH0g6j6fU,910
 mteb/models/abs_encoder.py,sha256=We9HlwWP61P4cMyZ080gywvDErA1eVsU9t46PtcNrCM,16830
-mteb/models/get_model_meta.py,sha256=_3gbC9I9fHJuqW4EzfM3xAv-heOBwEPswLVkCGO4BBI,6201
+mteb/models/get_model_meta.py,sha256=h6Z2Q3D_L4BeJbi1gPHTSgelbQjsQn1ALpI8uwNa0ac,6832
 mteb/models/instruct_wrapper.py,sha256=XAvvbPnXiTxKhFbmusm2uS8E9BMq8QXRSzQQI1jqKzE,9781
 mteb/models/model_meta.py,sha256=5seQwce893SbgSywFsphLuZGQ9rRn1fLl9fj1SfKnXE,32887
 mteb/models/models_protocols.py,sha256=HTB4-SYa3SeJXMMSA8o05lHTiLBbq314VW60K_PfcZY,9509
@@ -1615,7 +1615,7 @@ mteb/models/model_implementations/vdr_models.py,sha256=IGvpE2F42IWBN5QwKSWjsAehT
 mteb/models/model_implementations/vi_vn_models.py,sha256=7hot8CF5B1UeC4WJXnAAs1C1vbqK2lq7Bw338ztKFDE,6566
 mteb/models/model_implementations/vista_models.py,sha256=mcI0La6__LasuLd5P-nkc4Z-r9X_8sYhGFLdVPGPmkw,11033
 mteb/models/model_implementations/vlm2vec_models.py,sha256=1iq2i1ZbsPINE8nXoVZsX1Km-4dTTAd6St6J38I8Tew,11951
-mteb/models/model_implementations/voyage_models.py,sha256=L8-9kKgIzp_BIS_NIK6rVf4TtvjCeW4KEgR8H6LD2Wk,23125
+mteb/models/model_implementations/voyage_models.py,sha256=g7WET4MibXN6eABrmhn8uTGXdjwf5Kk4ddqQmHA7v6A,23920
 mteb/models/model_implementations/voyage_v.py,sha256=_mJGhskJj9zeHYebEJFuYheLPb-YDyiu6Hny_5LQcAE,8280
 mteb/models/model_implementations/xyz_models.py,sha256=69JyOCQHVq19nAG3zQFi-UYYT6I7uHmvTcmRxHvjyc8,1361
 mteb/models/model_implementations/youtu_models.py,sha256=P5fh34UJZQObJAbz3Wuzqh9Nw5S7LraqxdtwAX3sDJ8,6028
@@ -1628,7 +1628,7 @@ mteb/models/search_encoder_index/search_indexes/faiss_search_index.py,sha256=jwC
 mteb/results/__init__.py,sha256=EXQqK4Am5eIYzD52dpcGAFSdqnC38oE6JHN302oidHc,158
 mteb/results/benchmark_results.py,sha256=unBUBJ92ud0UXlkZJLn71WVcf-oUlF6XcITTccz5OBA,20318
 mteb/results/model_result.py,sha256=u1eUynaf_KVsdYdz29MACsCDR_48xODapkIGLG-lZa0,15321
-mteb/results/task_result.py,sha256=3j-zHyzsMRUrNWkDrgGalCc2-QmlxMYryb67P3m2VV8,33813
+mteb/results/task_result.py,sha256=TL9V7P7cXqLljnCHdzzaQ9vZZSXh9qXYFnPRG049alw,34247
 mteb/tasks/__init__.py,sha256=izAxU0ip1F_YUwx0dFCuN35BaktdmePh6vlDiHC0kLo,503
 mteb/tasks/aggregated_tasks/__init__.py,sha256=Ufgbh1AirxCQkojO3AUhUFWM8zQG10cfdVTkj_PeyLI,104
 mteb/tasks/aggregated_tasks/eng/__init__.py,sha256=HgaSyAX8Is5CGE006RgJkLQQVxrx2FmMnm6NHQBDi-4,358
@@ -2644,9 +2644,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=UKNokV9pu3G74MGebocU512aU_fFU9I9nPKnrG9Q0iE,1035
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=gElgSShKBXpfcqaZHhU_d2UHln1CyzUj8FN8KFun_UA,4087
-mteb-2.7.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.7.5.dist-info/METADATA,sha256=fZWNhj3MX_kIjecFW0X_lpEtAzgw5lbJIh4tfmlpeJ8,14457
-mteb-2.7.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mteb-2.7.5.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.7.5.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.7.5.dist-info/RECORD,,
+mteb-2.7.7.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.7.7.dist-info/METADATA,sha256=4rUFnkpk-5uGKDWjJwqBBQNLDaWwDMo0aZ7SavBbNL8,14457
+mteb-2.7.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mteb-2.7.7.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.7.7.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.7.7.dist-info/RECORD,,