mteb 2.1.3__py3-none-any.whl → 2.1.5__py3-none-any.whl

mteb/descriptive_stats/Retrieval/WinoGrande.json

@@ -1,29 +1,29 @@
 {
     "test": {
-        "num_samples": 4872,
-        "number_of_characters": 9352943,
+        "num_samples": 6362,
+        "number_of_characters": 180770,
         "documents_text_statistics": {
-            "total_text_length": 8957572,
-            "min_text_length": 8,
-            "average_text_length": 3504.527386541471,
-            "max_text_length": 47929,
-            "unique_texts": 2556
+            "total_text_length": 39142,
+            "min_text_length": 2,
+            "average_text_length": 7.68243375858685,
+            "max_text_length": 31,
+            "unique_texts": 5095
         },
         "documents_image_statistics": null,
         "queries_text_statistics": {
-            "total_text_length": 395371,
-            "min_text_length": 8,
-            "average_text_length": 170.71286701208982,
-            "max_text_length": 2863,
-            "unique_texts": 2316
+            "total_text_length": 141628,
+            "min_text_length": 79,
+            "average_text_length": 111.78216258879242,
+            "max_text_length": 185,
+            "unique_texts": 1267
         },
         "queries_image_statistics": null,
         "relevant_docs_statistics": {
-            "num_relevant_docs": 2316,
+            "num_relevant_docs": 1267,
             "min_relevant_docs_per_query": 1,
             "average_relevant_docs_per_query": 1.0,
             "max_relevant_docs_per_query": 1,
-            "unique_relevant_docs": 988
+            "unique_relevant_docs": 478
         },
         "top_ranked_statistics": null
     }

mteb/models/model_implementations/tarka_models.py

@@ -0,0 +1,58 @@
+from mteb.models.model_implementations.google_models import gemma_embedding_loader
+from mteb.models.model_meta import ModelMeta
+
+Tarka_Embedding_150M_V1_CITATION = """@misc{tarka_ai_research_2025,
+	author       = { Tarka AI Research },
+	title        = { Tarka-Embedding-150M-V1 (Revision c5f4f43) },
+	year         = 2025,
+	url          = { https://huggingface.co/Tarka-AIR/Tarka-Embedding-150M-V1 },
+	doi          = { 10.57967/hf/6875 },
+	publisher    = { Hugging Face }
+}"""
+
+MULTILINGUAL_EVALUATED_LANGUAGES = [
+    "arb-Arab",
+    "eng-Latn",
+    "spa-Latn",
+    "deu-Latn",
+    "fra-Latn",
+    "jpn-Jpan",
+    "kor-Hang",
+    "zho-Hans",
+]
+
+training_data = {
+    "T2Retrieval",
+    "DuRetrieval",
+    "MMarcoReranking",
+    "CMedQAv2-reranking",
+    "NQ",
+    "MSMARCO",
+    "HotpotQA",
+    "FEVER",
+    "MrTidyRetrieval",
+    "MIRACLRetrieval",
+    "CodeSearchNet",
+}
+
+tarka_embedding_150m_v1 = ModelMeta(
+    loader=gemma_embedding_loader,
+    name="Tarka-AIR/Tarka-Embedding-150M-V1",
+    languages=MULTILINGUAL_EVALUATED_LANGUAGES,
+    open_weights=True,
+    revision="c5f4f43",
+    release_date="2025-11-04",
+    n_parameters=155_714_304,
+    embed_dim=768,
+    max_tokens=2048,
+    license="gemma",
+    reference="https://huggingface.co/Tarka-AIR/Tarka-Embedding-150M-V1",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets=training_data,
+    similarity_fn_name="cosine",
+    memory_usage_mb=576,
+    citation=Tarka_Embedding_150M_V1_CITATION,
+)

mteb/models/model_implementations/voyage_models.py

@@ -156,16 +156,15 @@ class VoyageModel(AbsEncoder):
                 and len(batch) < batch_size
                 and batch_tokens < self._max_tokens_per_batch
             ):
-                n_tokens = len(
-                    self._client.tokenize([sentences[index]], model=self._model_name)[0]
-                )
+                txt = sentences[index] if len(sentences[index]) > 0 else " "
+                n_tokens = len(self._client.tokenize([txt], model=self._model_name)[0])
                 if (
                     batch_tokens + n_tokens > self._max_tokens_per_batch
                     and len(batch) > 0
                 ):
                     break
                 batch_tokens += n_tokens
-                batch.append(sentences[index])
+                batch.append(txt)
                 index += 1
 
             embeddings.extend(
@@ -249,7 +248,7 @@ voyage_3_5 = ModelMeta(
     n_parameters=None,
     memory_usage_mb=None,
     license=None,
-    reference="https://docs.voyageai.com/docs/embeddings",
+    reference="https://blog.voyageai.com/2025/05/20/voyage-3-5/",
     similarity_fn_name="cosine",
     framework=["API"],
     use_instructions=True,
@@ -274,7 +273,7 @@ voyage_3_5_int8 = ModelMeta(
     n_parameters=None,
     memory_usage_mb=None,
     license=None,
-    reference="https://docs.voyageai.com/docs/flexible-dimensions-and-quantization",
+    reference="https://blog.voyageai.com/2025/05/20/voyage-3-5/",
     similarity_fn_name="cosine",
     framework=["API"],
     use_instructions=True,
@@ -300,7 +299,7 @@ voyage_3_5_binary = ModelMeta(
     n_parameters=None,
     memory_usage_mb=None,
     license=None,
-    reference="https://docs.voyageai.com/docs/flexible-dimensions-and-quantization",
+    reference="https://blog.voyageai.com/2025/05/20/voyage-3-5/",
     similarity_fn_name="cosine",
     framework=["API"],
     use_instructions=True,

mteb/models/search_wrappers.py

@@ -191,6 +191,7 @@ class SearchEncoderWrapper:
             cos_scores_top_k_idx = cos_scores_top_k_idx.cpu().tolist()
             cos_scores_top_k_values = cos_scores_top_k_values.cpu().tolist()
 
+            sub_corpus_ids = list(sub_corpus_ids)
             for query_itr in range(len(query_embeddings)):
                 query_id = query_idx_to_id[query_itr]
                 for sub_corpus_id, score in zip(

mteb/tasks/retrieval/eng/wino_grande_retrieval.py

@@ -9,7 +9,7 @@ class WinoGrande(AbsTaskRetrieval):
         reference="https://winogrande.allenai.org/",
         dataset={
             "path": "mteb/WinoGrande",
-            "revision": "770abbd7f77affc005f9734996e795925cbc0f65",
+            "revision": "4dec9c5666e9f84702ac614363db6d96a68bc6de",
         },
         type="Retrieval",
         category="t2t",

{mteb-2.1.3.dist-info → mteb-2.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mteb
-Version: 2.1.3
+Version: 2.1.5
 Summary: Massive Text Embedding Benchmark
 Author-email: MTEB Contributors <niklas@huggingface.co>, Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Nouamane Tazi <nouamane@huggingface.co>, Nils Reimers <info@nils-reimers.de>
 Maintainer-email: Kenneth Enevoldsen <kenneth.enevoldsen@cas.au.dk>, Roman Solomatin <risolomatin@gmail.com>, Isaac Chung <chungisaac1217@gmail.com>

{mteb-2.1.3.dist-info → mteb-2.1.5.dist-info}/RECORD

@@ -1340,7 +1340,7 @@ mteb/descriptive_stats/Retrieval/VieQuADRetrieval.json,sha256=NHt0U-wJXBOPYOki5Y
 mteb/descriptive_stats/Retrieval/WebFAQRetrieval.json,sha256=uNkLCfiuVbdNKZ54vYGo8dPaoBlTShexDjl_IwCcT_M,60318
 mteb/descriptive_stats/Retrieval/WikiSQLRetrieval.json,sha256=JinCBRnmfMDeIwQtQfD6bD8MYNEpUgedw05A6G-W7u4,985
 mteb/descriptive_stats/Retrieval/WikipediaRetrievalMultilingual.json,sha256=v9npJOOJrcBUdRQ8EzAbOrpynAoSAJBaJoSJhS-qkww,20357
-mteb/descriptive_stats/Retrieval/WinoGrande.json,sha256=--CmwZnUf7EBw01t8yz2UGc8ifsMKtSBT-QIiMRkKm0,989
+mteb/descriptive_stats/Retrieval/WinoGrande.json,sha256=czzrnqg24MPfCrlVPLamL4LvMdWWOZC4R4tJYjc1QAA,982
 mteb/descriptive_stats/Retrieval/XMarket.json,sha256=s0odo5MWwzgQi0HwqK4APYhWIBR8WEtiC8KKt7wgzOc,4770
 mteb/descriptive_stats/Retrieval/XPQARetrieval.json,sha256=qHxeCggIwW1iw5ujbHsCc_7rf_-JIhfYRFGEKvzSTO0,44958
 mteb/descriptive_stats/Retrieval/XQuADRetrieval.json,sha256=NjNvrloDl561L_WTWBg6fQ31yBTZInYWL-SW0q35SfA,15462
@@ -1419,7 +1419,7 @@ mteb/models/get_model_meta.py,sha256=VpZZNINk-QrNeVpPZnlqzlLhtBs8G84eRwTzAb_gRD4
 mteb/models/instruct_wrapper.py,sha256=HxHmnlxkjtZhfgTZRYJBT3Nma7Dhx6a9e2Bg-cO_IYs,8844
 mteb/models/model_meta.py,sha256=b-Nel9nX5bJk4cgJnqkBzEKyMY7uXvxlCBSxmmH1Ios,14769
 mteb/models/models_protocols.py,sha256=D2hYWn_UBGMaKtRwBx3u0B0ni6lHJjSzTxX21XFNwIc,8917
-mteb/models/search_wrappers.py,sha256=0McxwGnqyiYKPHjHsxWZp1pP9qGuHemZjeX1z5ZgNAI,15588
+mteb/models/search_wrappers.py,sha256=W99EeMDQ58N0auVp2-7T39orG7GZLekSsVCdxEZLxw0,15638
 mteb/models/sentence_transformer_wrapper.py,sha256=n5CMsM6Lpg_CFHH0NkpJusMsaLUTt-L9vRmFINQ961k,12338
 mteb/models/cache_wrappers/__init__.py,sha256=j3JBHN73Tr7uMUO92FEvKXstnybxrPpGWmKXU2lAoIE,88
 mteb/models/cache_wrappers/cache_backend_protocol.py,sha256=TR7kD7KbN1J4piszIecpegtLZYGy7sRHZt3SDWlImKk,1665
@@ -1521,6 +1521,7 @@ mteb/models/model_implementations/shuu_model.py,sha256=KkcuVYjIzoha3Fvxh8ppqHQ9B
 mteb/models/model_implementations/siglip_models.py,sha256=tvi8QB2ayBoeXsxwHrl5RFlkknvE6FM9N06zSBWGQD0,12602
 mteb/models/model_implementations/sonar_models.py,sha256=Nc6kAJRWSrxA57DPRrgOPHqS1dNhz2vsE_1ZA2JtigQ,4784
 mteb/models/model_implementations/stella_models.py,sha256=NL3tk-rnuBdznsQ-nmelqun4tFO2xKoNPPOOVKqnPGU,8062
+mteb/models/model_implementations/tarka_models.py,sha256=aj4PvEzZ6ZSKcvwYVuTxf1IFOvH4rmJHtbPUcRw1fMI,1568
 mteb/models/model_implementations/text2vec_models.py,sha256=zaHWRc2W0RYZAOetinqRzug9UGW0HmY5U-jYsLXA8wo,4160
 mteb/models/model_implementations/ua_sentence_models.py,sha256=fcvXR4-Rrt-UDTlDkh2ZAO1gO_ufCOHiT6EhoeKiHx8,1224
 mteb/models/model_implementations/uae_models.py,sha256=KZxH5a3t-sfh33xUBkLizEuyFAyPlGfnRsn-S7mjq74,3112
@@ -1528,7 +1529,7 @@ mteb/models/model_implementations/vdr_models.py,sha256=lMm43BBPjZU5lxZcpmPZ8hn0P
 mteb/models/model_implementations/vi_vn_models.py,sha256=quWmd3JT2J6SlAsFrV2gcnc67M9zr58mEF2zLUF8-uw,4795
 mteb/models/model_implementations/vista_models.py,sha256=Q3I01kRtIPaoke0iMIcH4CLcCDTnMSIBFNCof7LPTX4,10832
 mteb/models/model_implementations/vlm2vec_models.py,sha256=HGGy_-z9Wc99xOKum71rBNipCPqWcM1efmmXgy5Rvxc,11724
-mteb/models/model_implementations/voyage_models.py,sha256=Qn9foyBmAGuuhispwZTFQb5ZtDM-OWMif5Ca4PBEhcw,19842
+mteb/models/model_implementations/voyage_models.py,sha256=dOCccOQlloGrg0q44PxMQzx8dHuQ8VgkDUD01EydpJ0,19824
 mteb/models/model_implementations/voyage_v.py,sha256=6i-oFnaY2D2qR1Dgb0B98ougnD1ujW9aNG9QoWyvwwY,8041
 mteb/models/model_implementations/xyz_models.py,sha256=TePlrH6EHwRPO87U_J3Yce9-XHCn_X7I2cJ_6BZ2fUY,1296
 mteb/models/model_implementations/youtu_models.py,sha256=NB74E6z-_36HyXb8GXKn8CrmRLN68uX9eH4xcS57zl0,5938
@@ -2283,7 +2284,7 @@ mteb/tasks/retrieval/eng/viz_wiz_it2t_retrieval.py,sha256=jE70T5If62lkKnbF-CMAgR
 mteb/tasks/retrieval/eng/vqa2_it2t_retrieval.py,sha256=M_g6Y6OrNRByD52-JxuO8iIO8aFUg8HHg5BxQ31-m1I,1403
 mteb/tasks/retrieval/eng/web_qa_t2it_retrieval.py,sha256=c7pJja_ii4ku9pfd-Gd3FqO6cF-0IIEb_H0FRY2A69w,1477
 mteb/tasks/retrieval/eng/web_qa_t2t_retrieval.py,sha256=rx6uoqc8yduGhuvdv2K5v2oFiQI8jP-BEt5nmaKrsac,1517
-mteb/tasks/retrieval/eng/wino_grande_retrieval.py,sha256=bRjNxz_pgH7waI7m_NMR_VLvcRxnBtMXY6CTU4RLEnc,1714
+mteb/tasks/retrieval/eng/wino_grande_retrieval.py,sha256=ou8TlZ-JPS1nh7NS7OeerUsB2WRZWWwKTuygpJNLb2A,1714
 mteb/tasks/retrieval/est/__init__.py,sha256=uNkOSKfZsO1F-xC4twL8ukxtfrI4A4eIU-oAs3Hi5Dg,46
 mteb/tasks/retrieval/est/estqa.py,sha256=sORL3KI47yXOy8GXptBtCuryOdDShdRDFpCdnnIaaCI,1418
 mteb/tasks/retrieval/fas/__init__.py,sha256=DUq1CTC_nj-201dbUNqlmqN-oR-YKjeW3O8DhtMX9rk,2213
@@ -2536,9 +2537,9 @@ mteb/types/_metadata.py,sha256=NN-W0S6a5TDV7UkpRx1pyWtGF4TyyCyoPUfHOwdeci8,2290
 mteb/types/_result.py,sha256=CRAUc5IvqI3_9SyXDwv-PWLCXwXdZem9RePeYESRtuw,996
 mteb/types/_string_validators.py,sha256=PY-dYq4E8O50VS3bLYdldPWp400fl_WzUjfVSkNWe8U,523
 mteb/types/statistics.py,sha256=YwJsxTf1eaCI_RE-J37a-gK5wDeGAsmkeZKoZCFihSo,3755
-mteb-2.1.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mteb-2.1.3.dist-info/METADATA,sha256=fSSp_uFAo3SSrjhhMQGHJMDIyEjeinv-7QjmK11VrPQ,13573
-mteb-2.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mteb-2.1.3.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
-mteb-2.1.3.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
-mteb-2.1.3.dist-info/RECORD,,
+mteb-2.1.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mteb-2.1.5.dist-info/METADATA,sha256=VunL5iKfFZLfKB-4Yruzd5vq31mhfNOT0JVjs2GibSQ,13573
+mteb-2.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mteb-2.1.5.dist-info/entry_points.txt,sha256=8IJoEJFKoDHmVnNev-qJ9pp4Ln7_1-ma9QsXnzVCzGU,39
+mteb-2.1.5.dist-info/top_level.txt,sha256=OLVIjcQAlWBz0bdmutKlWHLF42FF0hp4uVAg3ZyiG4U,5
+mteb-2.1.5.dist-info/RECORD,,