mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/models/model_implementations/sentence_transformers_models.py

@@ -1,5 +1,7 @@
 """Implementation of Sentence Transformers model validated in MTEB."""
 
+import numpy as np
+
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.models.sentence_transformer_wrapper import (
     SentenceTransformerEncoderWrapper,
@@ -113,6 +115,7 @@ sent_trf_training_dataset = {
 all_minilm_l6_v2 = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/all-MiniLM-L6-v2",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="8b3219a92973c328a8e22fadcfa821b5dc75636a",
@@ -124,7 +127,13 @@ all_minilm_l6_v2 = ModelMeta(
     max_tokens=256,
     reference="https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
@@ -137,6 +146,7 @@ all_minilm_l6_v2 = ModelMeta(
 all_minilm_l12_v2 = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/all-MiniLM-L12-v2",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="364dd28d28dcd3359b537f3cf1f5348ba679da62",
@@ -148,7 +158,13 @@ all_minilm_l12_v2 = ModelMeta(
     max_tokens=256,
     reference="https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
@@ -161,6 +177,7 @@ all_minilm_l12_v2 = ModelMeta(
 paraphrase_multilingual_minilm_l12_v2 = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
+    model_type=["dense"],
     languages=paraphrase_langs,
     open_weights=True,
     revision="bf3bf13ab40c3157080a7ab344c831b9ad18b5eb",
@@ -172,7 +189,13 @@ paraphrase_multilingual_minilm_l12_v2 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
@@ -185,6 +208,7 @@ paraphrase_multilingual_minilm_l12_v2 = ModelMeta(
 paraphrase_multilingual_mpnet_base_v2 = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+    model_type=["dense"],
     languages=paraphrase_langs,
     open_weights=True,
     revision="79f2382ceacceacdf38563d7c5d16b9ff8d725d6",
@@ -196,7 +220,13 @@ paraphrase_multilingual_mpnet_base_v2 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
@@ -220,6 +250,7 @@ paraphrase_multilingual_mpnet_base_v2 = ModelMeta(
 labse = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/LaBSE",
+    model_type=["dense"],
     languages=paraphrase_langs,
     open_weights=True,
     revision="e34fab64a3011d2176c99545a93d5cbddc9a91b7",
@@ -231,7 +262,7 @@ labse = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/LaBSE",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
@@ -257,6 +288,7 @@ labse = ModelMeta(
 multi_qa_minilm_l6_cos_v1 = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/multi-qa-MiniLM-L6-cos-v1",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="b207367332321f8e44f96e224ef15bc607f4dbf0",
@@ -268,7 +300,13 @@ multi_qa_minilm_l6_cos_v1 = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/multi-qa-MiniLM-L6-cos-v1",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=False,
     superseded_by=None,
     adapted_from="nreimers/MiniLM-L6-H384-uncased",
@@ -281,6 +319,7 @@ multi_qa_minilm_l6_cos_v1 = ModelMeta(
 all_mpnet_base_v2 = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/all-mpnet-base-v2",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="9a3225965996d404b775526de6dbfe85d3368642",
@@ -292,7 +331,13 @@ all_mpnet_base_v2 = ModelMeta(
     max_tokens=384,
     reference="https://huggingface.co/sentence-transformers/all-mpnet-base-v2",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
@@ -380,6 +425,7 @@ static_multi_languages = [
 
 static_similarity_mrl_multilingual_v1 = ModelMeta(
     name="sentence-transformers/static-similarity-mrl-multilingual-v1",
+    model_type=["dense"],
     loader=SentenceTransformerEncoderWrapper,
     loader_kwargs=dict(
         device="cpu",  # CPU is just as quick, if not quicker
@@ -395,18 +441,28 @@ static_similarity_mrl_multilingual_v1 = ModelMeta(
     max_tokens=None,
     reference="https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1",
     similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
     training_datasets=static_multi_datasets,
     public_training_code="https://huggingface.co/blog/static-embeddings",
     public_training_data="https://huggingface.co/collections/sentence-transformers/embedding-model-datasets-6644d7a3673a511914aa7552",
+    citation="""@inproceedings{reimers-2019-sentence-bert,
+    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
+    author = "Reimers, Nils and Gurevych, Iryna",
+    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
+    month = "11",
+    year = "2019",
+    publisher = "Association for Computational Linguistics",
+    url = "https://arxiv.org/abs/1908.10084",
+}""",
 )
 
 contriever = ModelMeta(
     loader=SentenceTransformerEncoderWrapper,
     name="facebook/contriever-msmarco",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="abe8c1493371369031bcb1e02acb754cf4e162fa",
@@ -418,7 +474,7 @@ contriever = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/facebook/contriever-msmarco",
     similarity_fn_name=ScoringFunction.DOT_PRODUCT,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers"],
     use_instructions=False,
     citation="""
     @misc{izacard2021contriever,
@@ -436,6 +492,7 @@ contriever = ModelMeta(
 microllama_text_embedding = ModelMeta(
     loader=sentence_transformers_loader,
     name="keeeeenw/MicroLlama-text-embedding",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="98f70f14cdf12d7ea217ed2fd4e808b0195f1e7e",
@@ -447,7 +504,7 @@ microllama_text_embedding = ModelMeta(
     max_tokens=2048,
     reference="https://huggingface.co/keeeeenw/MicroLlama-text-embedding",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     superseded_by=None,
     adapted_from=None,
@@ -467,9 +524,21 @@ microllama_text_embedding = ModelMeta(
     public_training_data=None,
 )
 
+SENTENCE_T5_CITATION = """
+@misc{ni2021sentencet5scalablesentenceencoders,
+      title={Sentence-T5: Scalable Sentence Encoders from Pre-trained Text-to-Text Models},
+      author={Jianmo Ni and Gustavo Hernández Ábrego and Noah Constant and Ji Ma and Keith B. Hall and Daniel Cer and Yinfei Yang},
+      year={2021},
+      eprint={2108.08877},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2108.08877},
+}
+"""
 sentence_t5_base = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/sentence-t5-base",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="50c53e206f8b01c9621484a3c0aafce4e55efebf",
@@ -481,16 +550,18 @@ sentence_t5_base = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/sentence-t5-base",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
     training_datasets={"SNLI", "Community QA"},
+    citation=SENTENCE_T5_CITATION,
 )
 
 sentence_t5_large = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/sentence-t5-large",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="1fc08ea477205aa54a3e5b13f0971ae16b86410a",
@@ -502,16 +573,18 @@ sentence_t5_large = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/sentence-t5-large",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
     training_datasets={"SNLI", "Community QA"},
+    citation=SENTENCE_T5_CITATION,
 )
 
 sentence_t5_xl = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/sentence-t5-xl",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="2965d31b368fb14117688e0bde77cbd720e91f53",
@@ -523,16 +596,18 @@ sentence_t5_xl = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/sentence-t5-xl",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
     training_datasets={"SNLI", "Community QA"},
+    citation=SENTENCE_T5_CITATION,
 )
 
 sentence_t5_xxl = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/sentence-t5-xxl",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="4d122282ba80e807e9e6eb8c358269e92796365d",
@@ -544,15 +619,28 @@ sentence_t5_xxl = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/sentence-t5-xxl",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
     training_datasets={"SNLI", "Community QA"},
+    citation=SENTENCE_T5_CITATION,
 )
+GTR_CITATION = """
+@misc{ni2021largedualencodersgeneralizable,
+      title={Large Dual Encoders Are Generalizable Retrievers},
+      author={Jianmo Ni and Chen Qu and Jing Lu and Zhuyun Dai and Gustavo Hernández Ábrego and Ji Ma and Vincent Y. Zhao and Yi Luan and Keith B. Hall and Ming-Wei Chang and Yinfei Yang},
+      year={2021},
+      eprint={2112.07899},
+      archivePrefix={arXiv},
+      primaryClass={cs.IR},
+      url={https://arxiv.org/abs/2112.07899},
+}
+"""
 gtr_t5_large = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/gtr-t5-large",
+    model_type=["dense"],
     languages=["eng-Latn"],  # in format eng-Latn
     open_weights=True,
     revision="a2c8ac47f998531948d4cbe32a0b577a7037a5e3",
@@ -564,7 +652,7 @@ gtr_t5_large = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/gtr-t5-large",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
@@ -581,11 +669,13 @@ gtr_t5_large = ModelMeta(
         "NQ-PL",  # translation not trained on
         "Community QA",
     },
+    citation=GTR_CITATION,
 )
 
 gtr_t5_xl = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/gtr-t5-xl",
+    model_type=["dense"],
     languages=["eng-Latn"],  # in format eng-Latn
     open_weights=True,
     revision="23a8d667a1ad2578af181ce762867003c498d1bf",
@@ -597,7 +687,7 @@ gtr_t5_xl = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/gtr-t5-xl",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
@@ -614,10 +704,12 @@ gtr_t5_xl = ModelMeta(
         "NQ-PL",  # translation not trained on
         "Community QA",
     },
+    citation=GTR_CITATION,
 )
 gtr_t5_xxl = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/gtr-t5-xxl",
+    model_type=["dense"],
     languages=["eng-Latn"],  # in format eng-Latn
     open_weights=True,
     revision="73f2a9156a3dcc2194dfdb2bf201cd7d17e17884",
@@ -629,7 +721,7 @@ gtr_t5_xxl = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/gtr-t5-xxl",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
@@ -646,11 +738,13 @@ gtr_t5_xxl = ModelMeta(
         "NQ-PL",  # translation not trained on
         "Community QA",
     },
+    citation=GTR_CITATION,
 )
 
 gtr_t5_base = ModelMeta(
     loader=sentence_transformers_loader,
     name="sentence-transformers/gtr-t5-base",
+    model_type=["dense"],
     languages=["eng-Latn"],  # in format eng-Latn
     open_weights=True,
     revision="7027e9594267928589816394bdd295273ddc0739",
@@ -662,7 +756,7 @@ gtr_t5_base = ModelMeta(
     max_tokens=512,
     reference="https://huggingface.co/sentence-transformers/gtr-t5-base",
     similarity_fn_name=ScoringFunction.COSINE,
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
     use_instructions=False,
     public_training_code=None,
     public_training_data=None,
@@ -679,4 +773,69 @@ gtr_t5_base = ModelMeta(
         "NQ-PL",  # translation not trained on
         "Community QA",
     },
+    citation=GTR_CITATION,
+)
+
+static_retrieval_mrl_en_v1 = ModelMeta(
+    loader=sentence_transformers_loader,
+    name="sentence-transformers/static-retrieval-mrl-en-v1",
+    revision="f60985c706f192d45d218078e49e5a8b6f15283a",
+    release_date="2024-10-24",
+    languages=["eng-Latn"],
+    n_parameters=3_125_4528,
+    memory_usage_mb=119,
+    max_tokens=np.inf,
+    embed_dim=1024,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code="https://huggingface.co/sentence-transformers/static-retrieval-mrl-en-v1/blob/main/train.py",
+    public_training_data=None,
+    framework=["PyTorch", "Sentence Transformers"],
+    reference="https://huggingface.co/sentence-transformers/static-retrieval-mrl-en-v1",
+    similarity_fn_name=ScoringFunction.COSINE,
+    use_instructions=False,
+    training_datasets={
+        "MSMARCO",
+        # gooaq
+        # s2orc
+        # allnli
+        # paq
+        # trivia-qa
+        # swim-ir-monolingual
+        # PubMedQA
+        # swim
+        "MIRACLRetrieval",
+        "MultiLongDocRetrieval",
+        "MrTidyRetrieval",
+    },
+    modalities=["text"],
+    model_type=["dense"],
+)
+
+multi_qa_mpnet_base_dot_v1 = ModelMeta(
+    loader=sentence_transformers_loader,
+    name="sentence-transformers/multi-qa-mpnet-base-dot-v1",
+    revision="3af7c6da5b3e1bea796ef6c97fe237538cbe6e7f",
+    release_date="2021-08-23",
+    languages=["eng-Latn"],
+    n_parameters=109486978,
+    memory_usage_mb=418.0,
+    max_tokens=512,
+    embed_dim=768,
+    license=None,
+    open_weights=True,
+    public_training_code="https://huggingface.co/sentence-transformers/multi-qa-mpnet-base-dot-v1/blob/main/train_script.py",
+    public_training_data=None,
+    framework=["PyTorch", "Sentence Transformers"],
+    reference="https://huggingface.co/sentence-transformers/multi-qa-mpnet-base-dot-v1",
+    similarity_fn_name=ScoringFunction.DOT_PRODUCT,
+    use_instructions=False,
+    training_datasets={
+        "MSMARCO",
+        "YahooAnswersTopicsClassification",
+        "NQ",
+    },
+    adapted_from="microsoft/mpnet-base",
+    modalities=["text"],
+    model_type=["dense"],
 )

mteb/models/model_implementations/shuu_model.py

@@ -1,31 +1,32 @@
-from mteb.models.model_meta import ModelMeta
-from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
-
-codemodernbert_crow_meta = ModelMeta(
-    loader=sentence_transformers_loader,
-    name="Shuu12121/CodeSearch-ModernBERT-Crow-Plus",
-    languages=["eng-Latn"],
-    open_weights=True,
-    revision="044a7a4b552f86e284817234c336bccf16f895ce",
-    release_date="2025-04-21",
-    n_parameters=151668480,
-    memory_usage_mb=607,
-    embed_dim=768,
-    license="apache-2.0",
-    max_tokens=1024,
-    reference="https://huggingface.co/Shuu12121/CodeSearch-ModernBERT-Crow-Plus",
-    similarity_fn_name="cosine",
-    framework=["Sentence Transformers", "PyTorch"],
-    use_instructions=False,
-    public_training_code=None,
-    public_training_data=None,
-    training_datasets={
-        "CodeSearchNetRetrieval",
-        # "code-search-net/code_search_net",
-        # "Shuu12121/python-codesearch-filtered",
-        # "Shuu12121/java-codesearch-filtered",
-        # "Shuu12121/javascript-codesearch-filtered",
-        # "Shuu12121/ruby-codesearch-filtered",
-        # "Shuu12121/rust-codesearch-filtered",
-    },
-)
+from mteb.models.model_meta import ModelMeta
+from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
+
+codemodernbert_crow_meta = ModelMeta(
+    loader=sentence_transformers_loader,
+    name="Shuu12121/CodeSearch-ModernBERT-Crow-Plus",
+    model_type=["dense"],
+    languages=["eng-Latn"],
+    open_weights=True,
+    revision="044a7a4b552f86e284817234c336bccf16f895ce",
+    release_date="2025-04-21",
+    n_parameters=151668480,
+    memory_usage_mb=607,
+    embed_dim=768,
+    license="apache-2.0",
+    max_tokens=1024,
+    reference="https://huggingface.co/Shuu12121/CodeSearch-ModernBERT-Crow-Plus",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
+    use_instructions=False,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets={
+        "CodeSearchNetRetrieval",
+        # "code-search-net/code_search_net",
+        # "Shuu12121/python-codesearch-filtered",
+        # "Shuu12121/java-codesearch-filtered",
+        # "Shuu12121/javascript-codesearch-filtered",
+        # "Shuu12121/ruby-codesearch-filtered",
+        # "Shuu12121/rust-codesearch-filtered",
+    },
+)