mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/models/model_implementations/mixedbread_ai_models.py

@@ -0,0 +1,332 @@
+from mteb.models.model_implementations.pylate_models import MultiVectorModel
+from mteb.models.model_meta import (
+    ModelMeta,
+    ScoringFunction,
+)
+from mteb.models.sentence_transformer_wrapper import (
+    CrossEncoderWrapper,
+    sentence_transformers_loader,
+)
+
+mixedbread_training_data = {
+    # from correspondence:
+    # as mentioned in our blog post
+    # (https://www.mixedbread.com/blog/mxbai-embed-large-v1#built-for-rag-and-real-world-use-cases:~:text=During%20the%20whole,related%20use%20cases.)
+    # We do not train on any data (except the MSMarco training split) of MTEB. We have a strong filtering process to ensure the OOD setting. That's true
+    # for all of our models. Keep up the good work and let me know if you have any questions.
+    "MSMARCO",
+}
+
+mxbai_embed_large_v1 = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts={
+            "query": "Represent this sentence for searching relevant passages: "
+        },
+    ),
+    name="mixedbread-ai/mxbai-embed-large-v1",
+    model_type=["dense"],
+    languages=["eng-Latn"],
+    open_weights=True,
+    revision="990580e27d329c7408b3741ecff85876e128e203",
+    release_date="2024-03-07",  # initial commit of hf model.
+    n_parameters=335_000_000,
+    memory_usage_mb=639,
+    max_tokens=512,
+    embed_dim=1024,
+    license="apache-2.0",
+    reference="https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "GGUF",
+        "Transformers",
+    ],
+    use_instructions=True,
+    citation="""
+    @online{emb2024mxbai,
+      title={Open Source Strikes Bread - New Fluffy Embeddings Model},
+      author={Sean Lee and Aamir Shakir and Darius Koenig and Julius Lipp},
+      year={2024},
+      url={https://www.mixedbread.ai/blog/mxbai-embed-large-v1},
+    }
+
+    @article{li2023angle,
+      title={AnglE-optimized Text Embeddings},
+      author={Li, Xianming and Li, Jing},
+      journal={arXiv preprint arXiv:2309.12871},
+      year={2023}
+    }
+    """,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets=mixedbread_training_data,
+)
+
+mxbai_embed_2d_large_v1 = ModelMeta(
+    loader=sentence_transformers_loader,
+    name="mixedbread-ai/mxbai-embed-2d-large-v1",
+    model_type=["dense"],
+    languages=["eng-Latn"],
+    open_weights=True,
+    revision="7e639ca8e344af398876ead3b19ec3c0b9068f49",
+    release_date="2024-03-04",  # initial commit of hf model.
+    n_parameters=335_000_000,
+    memory_usage_mb=None,
+    max_tokens=512,
+    embed_dim=768,
+    license="apache-2.0",
+    reference="https://huggingface.co/mixedbread-ai/mxbai-embed-2d-large-v1",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=[
+        "Sentence Transformers",
+        "PyTorch",
+        "ONNX",
+        "safetensors",
+        "Transformers",
+    ],
+    use_instructions=True,
+    adapted_from=None,
+    superseded_by=None,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets=None,
+)
+
+
+mxbai_embed_xsmall_v1 = ModelMeta(
+    loader=sentence_transformers_loader,
+    name="mixedbread-ai/mxbai-embed-xsmall-v1",
+    model_type=["dense"],
+    languages=["eng-Latn"],
+    open_weights=True,
+    revision="2f741ec33328bb57e4704e1238fc59a4a5745705",
+    release_date="2024-08-13",  # initial commit of hf model.
+    n_parameters=24_100_000,
+    memory_usage_mb=None,
+    max_tokens=512,
+    embed_dim=384,
+    license="apache-2.0",
+    reference="https://huggingface.co/mixedbread-ai/mxbai-embed-xsmall-v1",
+    similarity_fn_name=ScoringFunction.COSINE,
+    framework=["Sentence Transformers", "PyTorch", "ONNX", "safetensors", "GGUF"],
+    use_instructions=True,
+    adapted_from="sentence-transformers/all-MiniLM-L6-v2",
+    superseded_by=None,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets=mixedbread_training_data,
+    citation="""@online{xsmall2024mxbai,
+  title={Every Byte Matters: Introducing mxbai-embed-xsmall-v1},
+  author={Sean Lee and Julius Lipp and Rui Huang and Darius Koenig},
+  year={2024},
+  url={https://www.mixedbread.ai/blog/mxbai-embed-xsmall-v1},
+}""",
+)
+
+mxbai_rerank_xsmall_v1 = ModelMeta(
+    loader=CrossEncoderWrapper,
+    name="mixedbread-ai/mxbai-rerank-xsmall-v1",
+    revision="b5c6e9da73abc3711f593f705371cdbe9e0fe422",
+    release_date="2024-02-29",
+    languages=["eng-Latn"],
+    n_parameters=70830337,
+    memory_usage_mb=135.0,
+    max_tokens=512,
+    embed_dim=None,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data=None,
+    framework=[
+        "PyTorch",
+        "Sentence Transformers",
+        "Transformers",
+        "ONNX",
+        "safetensors",
+    ],
+    reference="https://huggingface.co/mixedbread-ai/mxbai-rerank-xsmall-v1",
+    similarity_fn_name=None,
+    use_instructions=None,
+    training_datasets=None,
+    adapted_from=None,
+    superseded_by=None,
+    modalities=["text"],
+    model_type=["cross-encoder"],
+    citation="""@online{rerank2024mxbai,
+  title={Boost Your Search With The Crispy Mixedbread Rerank Models},
+  author={Aamir Shakir and Darius Koenig and Julius Lipp and Sean Lee},
+  year={2024},
+  url={https://www.mixedbread.ai/blog/mxbai-rerank-v1},
+}""",
+    contacts=None,
+)
+
+mxbai_rerank_base_v1 = ModelMeta(
+    loader=CrossEncoderWrapper,
+    name="mixedbread-ai/mxbai-rerank-base-v1",
+    revision="800f24c113213a187e65bde9db00c15a2bb12738",
+    release_date="2024-02-29",
+    languages=["eng-Latn"],
+    n_parameters=184422913,
+    memory_usage_mb=352.0,
+    max_tokens=512,
+    embed_dim=None,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data=None,
+    framework=[
+        "PyTorch",
+        "Sentence Transformers",
+        "Transformers",
+        "ONNX",
+        "safetensors",
+    ],
+    reference="https://huggingface.co/mixedbread-ai/mxbai-rerank-base-v1",
+    similarity_fn_name=None,
+    use_instructions=None,
+    training_datasets=None,
+    adapted_from=None,
+    superseded_by=None,
+    modalities=["text"],
+    model_type=["cross-encoder"],
+    citation="""@online{rerank2024mxbai,
+  title={Boost Your Search With The Crispy Mixedbread Rerank Models},
+  author={Aamir Shakir and Darius Koenig and Julius Lipp and Sean Lee},
+  year={2024},
+  url={https://www.mixedbread.ai/blog/mxbai-rerank-v1},
+}""",
+    contacts=None,
+)
+
+mxbai_rerank_large_v1 = ModelMeta(
+    loader=CrossEncoderWrapper,
+    name="mixedbread-ai/mxbai-rerank-large-v1",
+    revision="98f655841d5caf0b16eaff79c2b4ca109d920d17",
+    release_date="2024-02-29",
+    languages=["eng-Latn"],
+    n_parameters=435062785,
+    memory_usage_mb=830.0,
+    max_tokens=512,
+    embed_dim=None,
+    license="apache-2.0",
+    open_weights=True,
+    public_training_code=None,
+    public_training_data=None,
+    framework=[
+        "PyTorch",
+        "Sentence Transformers",
+        "Transformers",
+        "ONNX",
+        "safetensors",
+    ],
+    reference="https://huggingface.co/mixedbread-ai/mxbai-rerank-large-v1",
+    similarity_fn_name=None,
+    use_instructions=None,
+    training_datasets=None,
+    adapted_from=None,
+    superseded_by=None,
+    modalities=["text"],
+    model_type=["cross-encoder"],
+    citation="""@online{rerank2024mxbai,
+  title={Boost Your Search With The Crispy Mixedbread Rerank Models},
+  author={Aamir Shakir and Darius Koenig and Julius Lipp and Sean Lee},
+  year={2024},
+  url={https://www.mixedbread.ai/blog/mxbai-rerank-v1},
+}""",
+    contacts=None,
+)
+
+mxbai_edge_colbert_v0_17m = ModelMeta(
+    loader=MultiVectorModel,
+    name="mixedbread-ai/mxbai-edge-colbert-v0-17m",
+    model_type=["late-interaction"],
+    languages=["eng-Latn"],
+    open_weights=True,
+    revision="23ae07f5bf028bc0d1f80c82e6e2dd2311f13a46",
+    public_training_code=None,
+    public_training_data=None,
+    release_date="2025-10-16",
+    n_parameters=int(17 * 1e6),
+    memory_usage_mb=64,
+    max_tokens=7999,
+    embed_dim=None,
+    license="apache-2.0",
+    similarity_fn_name=ScoringFunction.MAX_SIM,
+    framework=["PyLate", "ColBERT", "Transformers", "safetensors"],
+    reference="https://huggingface.co/mixedbread-ai/mxbai-edge-colbert-v0-17m",
+    use_instructions=False,
+    adapted_from="https://huggingface.co/jhu-clsp/ettin-encoder-17m",
+    superseded_by=None,
+    training_datasets={
+        "CornStack",
+        "MSMARCO",
+        "NQ",
+        "HotpotQA",
+        "AmazonQA",
+        "LoTTE",
+        "MultiLongDocRetrieval",
+        # "FineWeb",
+        # "PubMedQA",
+        # "TriviaQA",
+    },
+    citation="""@misc{takehi2025fantasticsmallretrieverstrain,
+      title={Fantastic (small) Retrievers and How to Train Them: mxbai-edge-colbert-v0 Tech Report},
+      author={Rikiya Takehi and Benjamin Clavié and Sean Lee and Aamir Shakir},
+      year={2025},
+      eprint={2510.14880},
+      archivePrefix={arXiv},
+      primaryClass={cs.IR},
+      url={https://arxiv.org/abs/2510.14880},
+}""",
+    contacts=None,
+)
+
+mxbai_edge_colbert_v0_32m = ModelMeta(
+    loader=MultiVectorModel,
+    name="mixedbread-ai/mxbai-edge-colbert-v0-32m",
+    model_type=["late-interaction"],
+    languages=["eng-Latn"],
+    open_weights=True,
+    revision="2f12870a85dae80680b9babc59992c9a2bc59e4a",
+    public_training_code=None,
+    public_training_data=None,
+    release_date="2025-10-16",
+    n_parameters=int(32 * 1e6),
+    memory_usage_mb=122,
+    max_tokens=511,
+    embed_dim=None,
+    license="apache-2.0",
+    similarity_fn_name=ScoringFunction.MAX_SIM,
+    framework=["PyLate", "ColBERT", "Transformers", "safetensors"],
+    reference="https://huggingface.co/mixedbread-ai/mxbai-edge-colbert-v0-32m",
+    use_instructions=False,
+    adapted_from="https://huggingface.co/jhu-clsp/ettin-encoder-32m",
+    superseded_by=None,
+    training_datasets={
+        "CornStack",
+        "MSMARCO",
+        "NQ",
+        "HotpotQA",
+        "AmazonQA",
+        "LoTTE",
+        "MultiLongDocRetrieval",
+        # "FineWeb",
+        # "PubMedQA",
+        # "TriviaQA",
+    },
+    citation="""@misc{takehi2025fantasticsmallretrieverstrain,
+      title={Fantastic (small) Retrievers and How to Train Them: mxbai-edge-colbert-v0 Tech Report},
+      author={Rikiya Takehi and Benjamin Clavié and Sean Lee and Aamir Shakir},
+      year={2025},
+      eprint={2510.14880},
+      archivePrefix={arXiv},
+      primaryClass={cs.IR},
+      url={https://arxiv.org/abs/2510.14880},
+}""",
+    contacts=None,
+)

mteb/models/model_implementations/mme5_models.py

@@ -12,6 +12,7 @@ mme5_mllama = ModelMeta(
         "trust_remote_code": True,
     },
     name="intfloat/mmE5-mllama-11b-instruct",
+    model_type=["dense"],
     revision="cbb328b9bf9ff5362c852c3166931903226d46f1",
     release_date="2025-02-12",
     languages=["eng-Latn"],
@@ -24,7 +25,7 @@ mme5_mllama = ModelMeta(
     open_weights=True,
     public_training_code=None,
     public_training_data="https://huggingface.co/datasets/intfloat/mmE5-MMEB-hardneg, https://huggingface.co/datasets/intfloat/mmE5-synthetic",
-    framework=["Sentence Transformers", "PyTorch"],
+    framework=["Sentence Transformers", "PyTorch", "Transformers", "safetensors"],
     reference="https://huggingface.co/intfloat/mmE5-mllama-11b-instruct",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=True,

mteb/models/model_implementations/moco_models.py

@@ -10,6 +10,13 @@ from mteb.models.abs_encoder import AbsEncoder
 from mteb.models.model_meta import ModelMeta, ScoringFunction
 from mteb.types import Array, BatchedInput, PromptType
 
+MOCOV3_CITATION = """@Article{chen2021mocov3,
+    author  = {Xinlei Chen* and Saining Xie* and Kaiming He},
+    title   = {An Empirical Study of Training Self-Supervised Vision Transformers},
+    journal = {arXiv preprint arXiv:2104.02057},
+    year    = {2021},
+}"""
+
 
 def mocov3_loader(model_name, **kwargs):
     requires_package(mocov3_loader, "timm", model_name, "pip install 'mteb[timm]'")
@@ -110,8 +117,9 @@ mocov3_training_datasets = set(
 )
 
 mocov3_vit_base = ModelMeta(
-    loader=mocov3_loader,  # type: ignore
+    loader=mocov3_loader,
     name="nyu-visionx/moco-v3-vit-b",
+    model_type=["dense"],
     languages=["eng-Latn"],
     revision="7d091cd70772c5c0ecf7f00b5f12ca609a99d69d",
     release_date="2024-06-03",
@@ -124,16 +132,18 @@ mocov3_vit_base = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/facebookresearch/moco-v3",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://github.com/facebookresearch/moco-v3",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
     training_datasets=mocov3_training_datasets,
+    citation=MOCOV3_CITATION,
 )
 
 mocov3_vit_large = ModelMeta(
-    loader=mocov3_loader,  # type: ignore
+    loader=mocov3_loader,
     name="nyu-visionx/moco-v3-vit-l",
+    model_type=["dense"],
     languages=["eng-Latn"],
     revision="7bf75358d616f39b9716148bf4e3425f3bd35b47",
     release_date="2024-06-03",
@@ -146,9 +156,10 @@ mocov3_vit_large = ModelMeta(
     open_weights=True,
     public_training_code="https://github.com/facebookresearch/moco-v3",
     public_training_data=None,
-    framework=["PyTorch"],
+    framework=["PyTorch", "Transformers", "safetensors"],
     reference="https://github.com/facebookresearch/moco-v3",
     similarity_fn_name=ScoringFunction.COSINE,
     use_instructions=False,
     training_datasets=mocov3_training_datasets,
+    citation=MOCOV3_CITATION,
 )

mteb/models/model_implementations/mod_models.py

@@ -0,0 +1,191 @@
+from mteb.models.instruct_wrapper import InstructSentenceTransformerModel
+from mteb.models.model_meta import ModelMeta
+from mteb.models.models_protocols import PromptType
+
+
+def instruction_template(
+    instruction: str, prompt_type: PromptType | None = None
+) -> str:
+    if not instruction or prompt_type == PromptType.document:
+        return ""
+    if isinstance(instruction, dict):
+        if prompt_type is None:
+            instruction = next(iter(instruction.values()))  # TODO
+        else:
+            instruction = instruction[prompt_type]
+    return f"Instruct: {instruction}\nQuery:"
+
+
+multilingual_langs = [
+    "afr-Latn",
+    "ara-Arab",
+    "aze-Latn",
+    "bel-Cyrl",
+    "bul-Cyrl",
+    "ben-Beng",
+    "cat-Latn",
+    "ceb-Latn",
+    "ces-Latn",
+    "cym-Latn",
+    "dan-Latn",
+    "deu-Latn",
+    "ell-Grek",
+    "eng-Latn",
+    "spa-Latn",
+    "est-Latn",
+    "eus-Latn",
+    "fas-Arab",
+    "fin-Latn",
+    "fra-Latn",
+    "glg-Latn",
+    "guj-Gujr",
+    "heb-Hebr",
+    "hin-Deva",
+    "hrv-Latn",
+    "hat-Latn",
+    "hun-Latn",
+    "hye-Armn",
+    "ind-Latn",
+    "isl-Latn",
+    "ita-Latn",
+    "jpn-Jpan",
+    "jav-Latn",
+    "kat-Geor",
+    "kaz-Cyrl",
+    "khm-Khmr",
+    "kan-Knda",
+    "kor-Hang",
+    "kir-Cyrl",
+    "lao-Laoo",
+    "lit-Latn",
+    "lav-Latn",
+    "mkd-Cyrl",
+    "mal-Mlym",
+    "mon-Cyrl",
+    "mar-Deva",
+    "msa-Latn",
+    "mya-Mymr",
+    "nep-Deva",
+    "nld-Latn",
+    "nor-Latn",
+    "nob-Latn",
+    "nno-Latn",
+    "pan-Guru",
+    "pol-Latn",
+    "por-Latn",
+    "que-Latn",
+    "ron-Latn",
+    "rus-Cyrl",
+    "sin-Sinh",
+    "slk-Latn",
+    "slv-Latn",
+    "swa-Latn",
+    "tam-Taml",
+    "tel-Telu",
+    "tha-Thai",
+    "tgl-Latn",
+    "tur-Latn",
+    "ukr-Cyrl",
+    "urd-Arab",
+    "vie-Latn",
+    "yor-Latn",
+    "zho-Hans",
+]
+
+MOD_CITATION = """@misc{mod-embedding-2025,
+  title={MoD-Embedding: A Fine-tuned Multilingual Text Embedding Model},
+  author={MoD Team},
+  year={2025},
+  url={https://huggingface.co/bflhc/MoD-Embedding}
+}"""
+
+training_data = {
+    "T2Retrieval",
+    "DuRetrieval",
+    "MMarcoReranking",
+    "CMedQAv2-reranking",
+    "NQ",
+    "MSMARCO",
+    "HotpotQA",
+    "FEVER",
+    "MrTidyRetrieval",
+    "MIRACLRetrieval",
+    "CodeSearchNet",
+}
+
+# Predefined prompts for various RTEB tasks
+_PREDEFINED_PROMPTS = {
+    # ========== Open Datasets ==========
+    # Legal domain
+    "AILACasedocs": "Given a legal case scenario, retrieve the most relevant case documents",
+    "AILAStatutes": "Given a legal scenario, retrieve the most relevant statute documents",
+    "LegalQuAD": "Given a legal question, retrieve relevant legal documents that answer the question",
+    "LegalSummarization": "Given a query, retrieve relevant legal documents for summarization",
+    # Code domain
+    "AppsRetrieval": "Given a query about mobile applications, retrieve relevant app information",
+    "HumanEvalRetrieval": "Given a code problem description, retrieve relevant code examples",
+    "MBPPRetrieval": "Given a programming problem description, retrieve relevant code solutions",
+    "DS1000Retrieval": "Given a data science problem, retrieve relevant code snippets",
+    "FreshStackRetrieval": "Given a programming question, retrieve relevant Stack Overflow posts",
+    # Finance domain
+    "FinQARetrieval": "Given a financial question, retrieve relevant financial documents",
+    "FinanceBenchRetrieval": "Given a financial query, retrieve relevant financial information",
+    "HC3FinanceRetrieval": "Given a finance-related query, retrieve relevant documents",
+    # Medical domain
+    "CUREv1": "Given a medical query, retrieve relevant clinical documents",
+    "ChatDoctorRetrieval": "Given a medical question, retrieve relevant medical information",
+    # SQL domain
+    "WikiSQLRetrieval": "Given a natural language query, retrieve relevant SQL examples",
+    # Multilingual
+    "MIRACLRetrievalHardNegatives": "Given a question, retrieve Wikipedia passages that answer the question",
+    # ========== Private/Closed Datasets ==========
+    # Code domain (Private)
+    "Code1Retrieval": "Given a code problem description, retrieve relevant code examples",
+    "JapaneseCode1Retrieval": "Given a code problem description, retrieve relevant code examples",
+    # Finance domain (Private)
+    "EnglishFinance1Retrieval": "Given a financial query, retrieve relevant financial documents",
+    "EnglishFinance2Retrieval": "Given a financial query, retrieve relevant financial documents",
+    "EnglishFinance3Retrieval": "Given a financial query, retrieve relevant financial documents",
+    "EnglishFinance4Retrieval": "Given a financial query, retrieve relevant financial documents",
+    # Healthcare domain (Private)
+    "EnglishHealthcare1Retrieval": "Given a medical question, retrieve relevant medical information",
+    "GermanHealthcare1Retrieval": "Given a medical question, retrieve relevant medical information",
+    # Legal domain (Private)
+    "FrenchLegal1Retrieval": "Given a legal query, retrieve relevant legal documents",
+    "GermanLegal1Retrieval": "Given a legal query, retrieve relevant legal documents",
+    "JapaneseLegal1Retrieval": "Given a legal query, retrieve relevant legal documents",
+    # General/Multilingual (Private)
+    "French1Retrieval": "Given a query, retrieve relevant passages",
+    "German1Retrieval": "Given a query, retrieve relevant passages",
+}
+
+
+MoD_Embedding = ModelMeta(
+    loader=InstructSentenceTransformerModel,
+    loader_kwargs=dict(
+        instruction_template=instruction_template,
+        apply_instruction_to_passages=False,
+        prompts_dict=_PREDEFINED_PROMPTS,
+        max_seq_length=18480,
+        model_kwargs={"torch_dtype": "bfloat16"},
+    ),
+    name="bflhc/MoD-Embedding",
+    languages=multilingual_langs,
+    open_weights=True,
+    revision="acbb5b70fdab262226a6af2bc62001de8021b05c",
+    release_date="2025-12-14",
+    n_parameters=4021774336,
+    memory_usage_mb=7671,
+    embed_dim=2560,
+    max_tokens=32768,
+    license="apache-2.0",
+    reference="https://huggingface.co/bflhc/MoD-Embedding",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch", "safetensors"],
+    use_instructions=True,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets=training_data,
+    citation=MOD_CITATION,
+    adapted_from="Qwen/Qwen3-Embedding-4B",
+)