mteb 2.1.8__py3-none-any.whl → 2.1.9__py3-none-any.whl

mteb/benchmarks/_create_table.py

@@ -358,9 +358,7 @@ def _create_summary_table_mean_public_private(
         "mean(public)": "Mean (Public)",
         "mean(private)": "Mean (Private)",
     }
-    # For RTEB: all tasks are Retrieval type, so Retrieval column = Mean (Task)
-    if "Retrieval" in joint_table.columns:
-        rename_dict["Retrieval"] = "Mean (Task)"
+
     joint_table = joint_table.rename(columns=rename_dict)
 
     # Move borda rank to front

mteb/benchmarks/benchmark.py

@@ -87,7 +87,10 @@ class RtebBenchmark(Benchmark):
     def _create_summary_table(
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
-        return _create_summary_table_mean_public_private(benchmark_results)
+        joint_table = _create_summary_table_mean_public_private(benchmark_results)
+        # For RTEB: all tasks are Retrieval type, so Retrieval column = Mean (Task)
+        joint_table = joint_table.rename(columns={"Retrieval": "Mean (Task)"})
+        return joint_table
 
 
 class HUMEBenchmark(Benchmark):
@@ -108,10 +111,15 @@ class MIEBBenchmark(Benchmark):
         return _create_summary_table_mean_task_type(benchmark_results)
 
 
-class Vidore3Benchmark(Benchmark):
+class VidoreBenchmark(Benchmark):
     """Wrapper for Vidore3 benchmark."""
 
     def _create_summary_table(
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
-        return _create_summary_table_mean_public_private(benchmark_results)
+        joint_table = _create_summary_table_mean_public_private(benchmark_results)
+        # For ViDoRe (V1, V2, V3): all tasks are Document Understanding type, so Document Understanding column = Mean (Task)
+        joint_table = joint_table.rename(
+            columns={"Document Understanding": "Mean (Task)"}
+        )
+        return joint_table

mteb/benchmarks/benchmarks/benchmarks.py

@@ -2,7 +2,7 @@ from mteb.benchmarks.benchmark import (
     Benchmark,
     HUMEBenchmark,
     MIEBBenchmark,
-    Vidore3Benchmark,
+    VidoreBenchmark,
 )
 from mteb.get_tasks import MTEBTasks, get_task, get_tasks
 
@@ -2219,7 +2219,7 @@ VIDORE_V2 = Benchmark(
 """,
 )
 
-VIDORE_V3 = Vidore3Benchmark(
+VIDORE_V3 = VidoreBenchmark(
     name="ViDoRe(v3)",
     display_name="ViDoRe V3",
     icon="https://cdn-uploads.huggingface.co/production/uploads/66e16a677c2eb2da5109fb5c/x99xqw__fl2UaPbiIdC_f.png",
@@ -2253,10 +2253,9 @@ VIDORE_V3 = Vidore3Benchmark(
 """,
 )
 
-VISUAL_DOCUMENT_RETRIEVAL = Benchmark(
-    name="VisualDocumentRetrieval",
-    display_name="Visual Document Retrieval",
-    icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-picture.svg",
+VISUAL_DOCUMENT_RETRIEVAL = VidoreBenchmark(
+    name="ViDoRe(v1&v2)",
+    display_name="ViDoRe (V1&V2)",
     tasks=get_tasks(
         tasks=[
             # v1

mteb/benchmarks/get_benchmark.py

@@ -39,6 +39,7 @@ def _get_previous_benchmark_names() -> dict[str, str]:
         MTEB_RETRIEVAL_MEDICAL,
         MTEB_RETRIEVAL_WITH_INSTRUCTIONS,
         SEB,
+        VISUAL_DOCUMENT_RETRIEVAL,
         MTEB_code,
         MTEB_multilingual_v2,
     )
@@ -63,6 +64,7 @@ def _get_previous_benchmark_names() -> dict[str, str]:
         "MTEB(Chinese)": C_MTEB.name,
         "FaMTEB(fas, beta)": FA_MTEB.name,
         "BRIGHT(long)": BRIGHT_LONG.name,
+        "VisualDocumentRetrieval": VISUAL_DOCUMENT_RETRIEVAL.name,
     }
     return previous_benchmark_names
 

mteb/leaderboard/benchmark_selector.py

@@ -110,10 +110,11 @@ R_BENCHMARK_ENTRIES = [
             MenuEntry(
                 "Image",
                 description=None,
-                open=False,
+                open=True,
                 benchmarks=[
-                    mteb.get_benchmark("VisualDocumentRetrieval"),
+                    mteb.get_benchmark("ViDoRe(v3)"),
                     mteb.get_benchmark("JinaVDR"),
+                    MenuEntry("Other", [mteb.get_benchmark("ViDoRe(v1&v2)")]),
                 ],
             ),
             MenuEntry(

mteb/models/model_implementations/emillykkejensen_models.py

@@ -0,0 +1,70 @@
+from mteb.models.model_meta import ModelMeta
+from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
+
+embedding_gemma_300m_scandi = ModelMeta(
+    loader=sentence_transformers_loader,  # type: ignore
+    name="emillykkejensen/EmbeddingGemma-Scandi-300m",
+    languages=["dan-Latn", "swe-Latn", "nor-Latn", "nob-Latn", "nno-Latn"],
+    open_weights=True,
+    revision="9f3307b9f601db564a9190cb475324d128dcfe86",
+    release_date="2025-10-17",
+    n_parameters=307_581_696,
+    embed_dim=768,
+    max_tokens=2048,
+    license="apache-2.0",
+    reference="https://huggingface.co/emillykkejensen/EmbeddingGemma-Scandi-300m",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/DDSC/nordic-embedding-training-data",
+    training_datasets=set(),
+    similarity_fn_name="cosine",  # type: ignore[arg-type]
+    adapted_from="emillykkejensen/EmbeddingGemma-Scandi-300m",
+    memory_usage_mb=578,
+)
+
+
+qwen_scandi = ModelMeta(
+    loader=sentence_transformers_loader,  # type: ignore
+    name="emillykkejensen/Qwen3-Embedding-Scandi-0.6B",
+    languages=["dan-Latn", "swe-Latn", "nor-Latn", "nob-Latn", "nno-Latn"],
+    open_weights=True,
+    revision="cf1e7ba36ebd3d605549d8f02930a18e17b54513",
+    release_date="2025-10-17",
+    n_parameters=595776512,
+    memory_usage_mb=2272,
+    embed_dim=1024,
+    max_tokens=32768,
+    license="apache-2.0",
+    reference="https://huggingface.co/emillykkejensen/Qwen3-Embedding-Scandi-0.6B",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/DDSC/nordic-embedding-training-data",
+    training_datasets=set(),
+    similarity_fn_name="cosine",  # type: ignore[arg-type]
+    adapted_from="Qwen/Qwen3-Embedding-0.6B",
+)
+
+
+mmbert_scandi = ModelMeta(
+    loader=sentence_transformers_loader,  # type: ignore
+    name="emillykkejensen/mmBERTscandi-base-embedding",
+    languages=["dan-Latn", "swe-Latn", "nor-Latn", "nob-Latn", "nno-Latn"],
+    open_weights=True,
+    revision="82d74c7a5d8e1ddf31b132865df2d16b2b0294ee",
+    release_date="2025-10-17",
+    n_parameters=306939648,
+    memory_usage_mb=1171,
+    embed_dim=768,
+    max_tokens=8192,
+    license="apache-2.0",
+    reference="https://huggingface.co/emillykkejensen/Qwen3-Embedding-Scandi-0.6B",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/DDSC/nordic-embedding-training-data",
+    training_datasets=set(),
+    similarity_fn_name="cosine",  # type: ignore[arg-type]
+    adapted_from="jonasaise/scandmmBERT-base-scandinavian",
+)

{mteb-2.1.8.dist-info → mteb-2.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.1.8
+Version: 2.1.9
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.1.8.dist-info → mteb-2.1.9.dist-info}/RECORD

@@ -52,11 +52,11 @@ mteb/abstasks/text/bitext_mining.py,sha256=8m86XHJ3TxguC9itxZRq2Bt_p0NYojojS2Btk
 mteb/abstasks/text/reranking.py,sha256=rfRGRBeSjZLgkh8pneMgRm-vd9NHr5jSFH92YfOHfmU,7776
 mteb/abstasks/text/summarization.py,sha256=KYEb8gh4JjpSsrvGUmQ2VlrVdzzVxIWcitXOJUaHhO4,6954
 mteb/benchmarks/__init__.py,sha256=MQEVeli-zLaJ7Xg0z7RhXQwsdmm7Ht_W2Ln0rZo1Szc,225
-mteb/benchmarks/_create_table.py,sha256=gYgwhACmY8JSN00y6r3HfHNQphCIuvcw_IwRZNQwgdc,20623
-mteb/benchmarks/benchmark.py,sha256=jet-j-Aho20Wn-cKDbh5nwNqnX64scu6YmP9PLJwINs,3704
-mteb/benchmarks/get_benchmark.py,sha256=Zh_C_RXl02hBxg6ffQlxEFERrokaern1RAN4yh2pa2E,3833
+mteb/benchmarks/_create_table.py,sha256=z3iqa5dajLk0DYxEE9EeO1qpR3VJXokg8ZQ2rdUkvdM,20452
+mteb/benchmarks/benchmark.py,sha256=70RlMyyg_wkWTlU_IbfLl-KaqRWXGCKTd8fWe9X-AQE,4173
+mteb/benchmarks/get_benchmark.py,sha256=-n_O-gitRKZi48gJKNgGuI36hsP7yLVSiwulnMHN7Gw,3935
 mteb/benchmarks/benchmarks/__init__.py,sha256=UD6YjWPDVPSQdUhmD-4rho08Gs5LU9pS_C2jX5eUns0,2102
-mteb/benchmarks/benchmarks/benchmarks.py,sha256=3IO9wtj_bx1EugkMok4r_WNYo1KMYzslTqYJ4oBoK9w,90092
+mteb/benchmarks/benchmarks/benchmarks.py,sha256=v7n2fPGOC66zzBhS1nfSthY55DQnGNg-hGa4XfT21Vg,89941
 mteb/benchmarks/benchmarks/rteb_benchmarks.py,sha256=QnCSrTTaBfcRlAQp2Nu81tgv1idMXqiM16Fp2zKJ5Ys,10607
 mteb/cli/__init__.py,sha256=v-csUr3eUZElIvrGB6QGtaIdndDfNWEe9oZchsGsJpg,64
 mteb/cli/_display_tasks.py,sha256=7A06dT9sSoTz6shyMvskPxuc5eHY_H7PGPlROzMP0yw,2196
@@ -1423,7 +1423,7 @@ mteb/languages/language_scripts.py,sha256=5wix9HTYolNIpTiS5oXf2pGJyL7ftdGKs_m432
 mteb/languages/programming_languages.py,sha256=zxAakT3OSUnAuTnQ34VyeFIECnNXMlleZmAake6jsZE,211
 mteb/leaderboard/__init__.py,sha256=991roXmtRwEQysV-37hWEzWpkvPgMCGRqZTHR-hm2io,88
 mteb/leaderboard/app.py,sha256=xvOcK_ICmAwl1Mo5muI7fmCjYeSt1ztSgsQDrPF6OvM,32575
-mteb/leaderboard/benchmark_selector.py,sha256=1PRTDcNU4o2ZVHAzAUVp6UxWNElZDcGbnKHsNGv4aco,7621
+mteb/leaderboard/benchmark_selector.py,sha256=hnXdo_Kj4UUAruFl6nZkCxAQ88IEfbaH8EADFJMMdVo,7686
 mteb/leaderboard/figures.py,sha256=Rq20LFpaUhQD4tuKp7P7ExQtAjonMLibgO3ud0ykMag,7491
 mteb/leaderboard/table.py,sha256=qs0H_Gt9FzRvzb-AL0YlqEe0YAsdYsVX3QlncfCBEqg,7828
 mteb/leaderboard/text_segments.py,sha256=iMIkS04QQjPbT-SkU0x6fOcS8xRbUYevryu9HydipKM,6570
@@ -1469,6 +1469,7 @@ mteb/models/model_implementations/dino_models.py,sha256=QFgaFHR5YKrylqJGSljXCBn2
 mteb/models/model_implementations/e5_instruct.py,sha256=9R4GoSFicgqNDCh3HhTN_8L1qhzuEKvatjHYn3T9zlU,7676
 mteb/models/model_implementations/e5_models.py,sha256=vsqkmm6XzZn9ROj_OUR0j2KiN75MEuQsOPeoyc1AeYg,10937
 mteb/models/model_implementations/e5_v.py,sha256=_9W7I0ryIzx_H9eCkzwdm8iHdGX1LIjKGXkhSh_zNv8,6690
+mteb/models/model_implementations/emillykkejensen_models.py,sha256=1DEAGdSZZXDFbbw0YH-vkLm9Y-wthgbOJCRTIpV3Jeo,2795
 mteb/models/model_implementations/en_code_retriever.py,sha256=leZ-0M6LrunocY3XQBYZU1uevDRopeyR5ujIhwqBbd8,1043
 mteb/models/model_implementations/evaclip_models.py,sha256=cPMGYLDIq4s8zJxb4vPXqJ-rqwPaq7KOh2QZSO6cDas,8000
 mteb/models/model_implementations/fa_models.py,sha256=WGal70_ezITWoNdjcMdbOCTSCtoaXzuPadYstLVXxhg,7478
@@ -2553,9 +2554,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=CRAUc5IvqI3_9SyXDwv-PWLCXwXdZem9RePeYESRtuw,996
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=YwJsxTf1eaCI_RE-J37a-gK5wDeGAsmkeZKoZCFihSo,3755
-mteb-2.1.8.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.1.8.dist-info/METADATA,sha256=-9mxdWuiDSscT1jAYxM6FVZpIjEvDPOCPC05TOIk8XI,13573
-mteb-2.1.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mteb-2.1.8.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.1.8.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.1.8.dist-info/RECORD,,
+mteb-2.1.9.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.1.9.dist-info/METADATA,sha256=yUOXi6O_wkyskXKnHDcDB6SqLZg5Q5Nc_a_qK7Pngpc,13573
+mteb-2.1.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mteb-2.1.9.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.1.9.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.1.9.dist-info/RECORD,,