mteb 2.1.4__py3-none-any.whl → 2.5.2__py3-none-any.whl

mteb/models/model_implementations/ruri_models.py

@@ -0,0 +1,322 @@
+from mteb.models.model_meta import ModelMeta
+from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
+
+RURI_V3_PROMPTS = {
+    "Retrieval-query": "検索クエリ: ",
+    "Retrieval-document": "検索文書: ",
+    "Reranking-query": "検索クエリ: ",
+    "Reranking-document": "検索文書: ",
+    "Classification": "トピック: ",
+    "Clustering": "トピック: ",
+}
+
+RURI_V1_V2_PROMPTS = {
+    "query": "クエリ: ",
+    "document": "文章: ",
+}
+
+
+RURI_CITATION = r"""@misc{Ruri,
+  title={{Ruri: Japanese General Text Embeddings}},
+  author={Hayato Tsukagoshi and Ryohei Sasano},
+  year={2024},
+  eprint={2409.07737},
+  archivePrefix={arXiv},
+  primaryClass={cs.CL},
+  url={https://arxiv.org/abs/2409.07737},
+}"""
+
+cl_nagoya_ruri_v3_30m = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V3_PROMPTS,
+    ),
+    name="cl-nagoya/ruri-v3-30m",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="24899e5de370b56d179604a007c0d727bf144504",
+    release_date="2025-04-07",
+    n_parameters=36_705_536,
+    memory_usage_mb=140,
+    embed_dim=256,
+    license="apache-2.0",
+    max_tokens=8192,
+    reference="https://huggingface.co/cl-nagoya/ruri-v3-30m",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    superseded_by=None,
+    training_datasets={
+        "cl-nagoya/ruri-v3-dataset-ft",
+    },
+    adapted_from="sbintuitions/modernbert-ja-30m",
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+cl_nagoya_ruri_v3_70m = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V3_PROMPTS,
+    ),
+    name="cl-nagoya/ruri-v3-70m",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="07a8b0aba47d29d2ca21f89b915c1efe2c23d1cc",
+    release_date="2025-04-09",
+    n_parameters=36_705_536,
+    memory_usage_mb=140,
+    embed_dim=256,
+    license="apache-2.0",
+    max_tokens=8192,
+    reference="https://huggingface.co/cl-nagoya/ruri-v3-70m",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    superseded_by=None,
+    training_datasets={"MrTidyRetrieval", "MIRACLRetrieval"},
+    adapted_from="sbintuitions/modernbert-ja-70m",
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+cl_nagoya_ruri_v3_130m = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V3_PROMPTS,
+    ),
+    name="cl-nagoya/ruri-v3-130m",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="e3114c6ee10dbab8b4b235fbc6dcf9dd4d5ac1a6",
+    release_date="2025-04-09",
+    n_parameters=132_140_544,
+    memory_usage_mb=504,
+    embed_dim=512,
+    license="apache-2.0",
+    max_tokens=8192,
+    reference="https://huggingface.co/cl-nagoya/ruri-v3-130m",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    superseded_by=None,
+    training_datasets={"MrTidyRetrieval", "MIRACLRetrieval"},
+    adapted_from="sbintuitions/modernbert-ja-130m",
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+cl_nagoya_ruri_v3_310m = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V3_PROMPTS,
+    ),
+    name="cl-nagoya/ruri-v3-310m",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="18b60fb8c2b9df296fb4212bb7d23ef94e579cd3",
+    release_date="2025-04-09",
+    n_parameters=314_611_968,
+    memory_usage_mb=1200,
+    embed_dim=768,
+    license="apache-2.0",
+    max_tokens=8192,
+    reference="https://huggingface.co/cl-nagoya/ruri-v3-310m",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    superseded_by=None,
+    training_datasets={"MrTidyRetrieval", "MIRACLRetrieval"},
+    adapted_from="sbintuitions/modernbert-ja-310m",
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-v3-dataset-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+cl_nagoya_ruri_small_v2 = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V1_V2_PROMPTS,
+        trust_remote_code=True,
+    ),
+    name="cl-nagoya/ruri-small-v2",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="db18646e673b713cd0518a5bb0fefdce21e77cd9",
+    release_date="2024-12-05",
+    n_parameters=68_087_808,
+    memory_usage_mb=260,
+    embed_dim=768,
+    license="apache-2.0",
+    max_tokens=512,
+    reference="https://huggingface.co/cl-nagoya/ruri-small-v2",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    adapted_from="line-corporation/line-distilbert-base-japanese",
+    superseded_by=None,
+    training_datasets={"MrTidyRetrieval", "MIRACLRetrieval"},
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-v2-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+cl_nagoya_ruri_base_v2 = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V1_V2_PROMPTS,
+    ),
+    name="cl-nagoya/ruri-base-v2",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="8ce03882903668a01c83ca3b8111ac025a3bc734",
+    release_date="2024-12-05",
+    n_parameters=111_207_168,
+    memory_usage_mb=424,
+    embed_dim=768,
+    license="apache-2.0",
+    max_tokens=512,
+    reference="https://huggingface.co/cl-nagoya/ruri-base-v2",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    adapted_from="tohoku-nlp/bert-base-japanese-v3",
+    superseded_by=None,
+    training_datasets=None,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-v2-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+cl_nagoya_ruri_large_v2 = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V1_V2_PROMPTS,
+    ),
+    name="cl-nagoya/ruri-large-v2",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="42898ef34a5574977380ebf0dfd28cbfbd36438b",
+    release_date="2024-12-06",
+    n_parameters=337_441_792,
+    memory_usage_mb=1287,
+    embed_dim=1024,
+    license="apache-2.0",
+    max_tokens=512,
+    reference="https://huggingface.co/cl-nagoya/ruri-large-v2",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    adapted_from="tohoku-nlp/bert-large-japanese-v2",
+    superseded_by=None,
+    training_datasets=None,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-v2-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+cl_nagoya_ruri_small_v1 = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V1_V2_PROMPTS,
+        trust_remote_code=True,
+    ),
+    name="cl-nagoya/ruri-small",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="bc56ce90cd7a979f6eb199fc52dfe700bfd94bc3",
+    release_date="2024-08-28",
+    n_parameters=68_087_808,
+    memory_usage_mb=130,
+    embed_dim=768,
+    license="apache-2.0",
+    max_tokens=512,
+    reference="https://huggingface.co/cl-nagoya/ruri-small",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    adapted_from="line-corporation/line-distilbert-base-japanese",
+    superseded_by="cl-nagoya/ruri-small-v2",
+    training_datasets=None,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+cl_nagoya_ruri_base_v1 = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V1_V2_PROMPTS,
+    ),
+    name="cl-nagoya/ruri-base",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="1ae40b8b6c78518a499425086bab8fc16c2e4b0e",
+    release_date="2024-08-28",
+    n_parameters=111_207_168,
+    memory_usage_mb=212,
+    embed_dim=768,
+    license="apache-2.0",
+    max_tokens=512,
+    reference="https://huggingface.co/cl-nagoya/ruri-base",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    adapted_from="tohoku-nlp/bert-base-japanese-v3",
+    superseded_by="cl-nagoya/ruri-base-v2",
+    training_datasets=None,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)
+
+
+cl_nagoya_ruri_large_v1 = ModelMeta(
+    loader=sentence_transformers_loader,
+    loader_kwargs=dict(
+        model_prompts=RURI_V1_V2_PROMPTS,
+    ),
+    name="cl-nagoya/ruri-large",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="a011c39b13e8bc137ee13c6bc82191ece46c414c",
+    release_date="2024-08-28",
+    n_parameters=337_441_792,
+    memory_usage_mb=644,
+    embed_dim=1024,
+    license="apache-2.0",
+    max_tokens=512,
+    reference="https://huggingface.co/cl-nagoya/ruri-large",
+    similarity_fn_name="cosine",
+    framework=["PyTorch", "Sentence Transformers"],
+    use_instructions=True,
+    adapted_from="tohoku-nlp/bert-large-japanese-v2",
+    superseded_by="cl-nagoya/ruri-large-v2",
+    training_datasets=None,
+    public_training_code=None,
+    public_training_data="https://huggingface.co/datasets/cl-nagoya/ruri-dataset-ft",
+    citation=RURI_CITATION,
+    contacts=["hpprc"],
+)

mteb/models/model_implementations/salesforce_models.py

@@ -46,6 +46,7 @@ SFR_Embedding_2_R = ModelMeta(
         normalized=True,
     ),
     name="Salesforce/SFR-Embedding-2_R",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="91762139d94ed4371a9fa31db5551272e0b83818",
@@ -83,6 +84,7 @@ SFR_Embedding_Code_2B_R = ModelMeta(
         normalized=True,
     ),
     name="Salesforce/SFR-Embedding-Code-2B_R",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="c73d8631a005876ed5abde34db514b1fb6566973",
@@ -120,6 +122,7 @@ SFR_Embedding_Mistral = ModelMeta(
         normalized=True,
     ),
     name="Salesforce/SFR-Embedding-Mistral",
+    model_type=["dense"],
     languages=["eng-Latn"],
     open_weights=True,
     revision="938c560d1c236aa563b2dbdf084f28ab28bccb11",

mteb/models/model_implementations/samilpwc_models.py

@@ -43,6 +43,7 @@ samilpwc_expr = ModelMeta(
         apply_instruction_to_passages=False,
     ),
     name="SamilPwC-AXNode-GenAI/PwC-Embedding_expr",
+    model_type=["dense"],
     languages=[
         "kor-Hang",
     ],

mteb/models/model_implementations/sarashina_embedding_models.py

@@ -0,0 +1,168 @@
+from mteb.models.instruct_wrapper import InstructSentenceTransformerModel
+from mteb.models.model_meta import ModelMeta
+from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader
+from mteb.types import PromptType
+
+SARASHINA_V2_INSTRUCTIONS = {
+    "Retrieval": {
+        "query": "クエリを与えるので、もっともクエリに意味が似ている一節を探してください。",
+        "document": "text: ",
+    },
+    "Reranking": {
+        "query": "クエリを与えるので、もっともクエリに意味が似ている一節を探してください。",
+        "document": "text: ",
+    },
+    "Classification": "与えられたドキュメントを適切なカテゴリに分類してください。",
+    "Clustering": "与えられたドキュメントのトピックまたはテーマを特定してください。",
+    # optimization regarding JMTEB
+    "LivedoorNewsClustering.v2": "与えられたニュース記事のトピックを特定してください。",
+    "MewsC16JaClustering": "与えられたニュース記事のトピックを特定してください。",
+    "SIB200ClusteringS2S": "与えられたテキストのトピックを特定してください。",
+    "AmazonReviewsClassification": "与えられたAmazonレビューを適切な評価カテゴリに分類してください。",
+    "AmazonCounterfactualClassification": "与えられたAmazonのカスタマーレビューのテキストを反事実か反事実でないかに分類してください。",
+    "MassiveIntentClassification": "ユーザーの発話をクエリとして与えるので、ユーザーの意図を見つけてください。",
+    "MassiveScenarioClassification": "ユーザーの発話をクエリとして与えるので、ユーザーシナリオを見つけてください。",
+    "JapaneseSentimentClassification": "与えられたテキストの感情極性をポジティブ(1)かネガティブか(0)に分類してください。",
+    "SIB200Classification": "与えられたテキストのトピックを特定してください。",
+    "WRIMEClassification": "与えられたテキストの感情極性（-2:強いネガティブ、-1:ネガティブ、0:ニュートラル、1:ポジティブ、2:強いポジティブ）を分類してください。",
+    "JSTS": "クエリを与えるので，もっともクエリに意味が似ている一節を探してください。",
+    "JSICK": "クエリを与えるので，もっともクエリに意味が似ている一節を探してください。",
+    "JaqketRetrieval": {
+        "query": "質問を与えるので、その質問に答えるのに役立つWikipediaの文章を検索してください。",
+        "document": "text: ",
+    },
+    "MrTidyRetrieval": {
+        "query": "質問を与えるので、その質問に答えるWikipediaの文章を検索するしてください。",
+        "document": "text: ",
+    },
+    "JaGovFaqsRetrieval": {
+        "query": "質問を与えるので、その質問に答えるのに役立つ関連文書を検索してください。",
+        "document": "text: ",
+    },
+    "NLPJournalTitleAbsRetrieval.V2": {
+        "query": "論文のタイトルを与えるので、タイトルに対応する要約を検索してください。",
+        "document": "text: ",
+    },
+    "NLPJournalTitleIntroRetrieval.V2": {
+        "query": "論文のタイトルを与えるので、タイトルに対応する要約を検索してください。",
+        "document": "text: ",
+    },
+    "NLPJournalAbsIntroRetrieval.V2": {
+        "query": "論文の序論を与えるので、序論に対応する全文を検索してください。",
+        "document": "text: ",
+    },
+    "NLPJournalAbsArticleRetrieval.V2": {
+        "query": "論文の序論を与えるので、序論に対応する全文を検索してください。",
+        "document": "text: ",
+    },
+    "JaCWIRRetrieval": {
+        "query": "記事のタイトルを与えるので、そのタイトルと合っている記事の中身を検索してください。",
+        "document": "text: ",
+    },
+    "MIRACLRetrieval": {
+        "query": "質問を与えるので、その質問に答えるのに役立つ関連文書を検索してください。",
+        "document": "text: ",
+    },
+    "MintakaRetrieval": {
+        "query": "質問を与えるので、その質問に答えられるテキストを検索してください。",
+        "document": "text: ",
+    },
+    "MultiLongDocRetrieval": {
+        "query": "質問を与えるので、その質問に答えるのに役立つWikipediaの文章を検索してください。",
+        "document": "text: ",
+    },
+    "ESCIReranking": {
+        "query": "クエリを与えるので、与えられたWeb検索クエリに答える関連文章を検索してください。",
+        "document": "text: ",
+    },
+    "JQaRAReranking": {
+        "query": "質問を与えるので、その質問に答えるのに役立つWikipediaの文章を検索してください。",
+        "document": "text: ",
+    },
+    "JaCWIRReranking": {
+        "query": "記事のタイトルを与えるので、そのタイトルと合っている記事の中身を検索してください。",
+        "document": "text: ",
+    },
+    "MIRACLReranking": {
+        "query": "質問を与えるので、その質問に答えるのに役立つ関連文書を検索してください。",
+        "document": "text: ",
+    },
+    "MultiLongDocReranking": {
+        "query": "質問を与えるので、その質問に答えるのに役立つWikipediaの文章を検索してください。",
+        "document": "text: ",
+    },
+}
+
+
+def sarashina_instruction_template(
+    instruction: str, prompt_type: PromptType | None = None
+) -> str:
+    """Instruction template for Sarashina v2 model.
+
+    Returns the instruction as-is since the prompts already contain the full format.
+    For document prompts, returns the instruction directly (e.g., "text: ").
+    """
+    if not instruction:
+        return ""
+    if prompt_type == PromptType.document:
+        return "text: "
+    return f"task: {instruction}\nquery: "
+
+
+sbintuitions_sarashina_embedding_v2_1b = ModelMeta(
+    loader=InstructSentenceTransformerModel,
+    loader_kwargs=dict(
+        instruction_template=sarashina_instruction_template,
+        apply_instruction_to_passages=True,
+        prompts_dict=SARASHINA_V2_INSTRUCTIONS,
+        max_seq_length=8192,
+    ),
+    name="sbintuitions/sarashina-embedding-v2-1b",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="1f3408afaa7b617e3445d891310a9c26dd0c68a5",
+    release_date="2025-07-30",
+    n_parameters=1_224_038_144,
+    memory_usage_mb=4669,
+    embed_dim=1792,
+    license="https://huggingface.co/sbintuitions/sarashina-embedding-v2-1b/blob/main/LICENSE",
+    max_tokens=8192,
+    reference="https://huggingface.co/sbintuitions/sarashina-embedding-v2-1b",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    adapted_from="sbintuitions/sarashina2.2-1b",
+    superseded_by=None,
+    training_datasets={"NQ", "MrTidyRetrieval"},
+    public_training_code=None,
+    public_training_data="https://huggingface.co/sbintuitions/sarashina-embedding-v1-1b",
+    citation=None,
+    contacts=["Sraym1217", "akiFQC", "lsz05"],
+)
+
+sbintuitions_sarashina_embedding_v1_1b = ModelMeta(
+    loader=sentence_transformers_loader,
+    name="sbintuitions/sarashina-embedding-v1-1b",
+    model_type=["dense"],
+    languages=["jpn-Jpan"],
+    open_weights=True,
+    revision="d060fcd8984075071e7fad81baff035cbb3b6c7e",
+    release_date="2024-11-22",
+    n_parameters=1_224_038_144,
+    memory_usage_mb=4669,
+    embed_dim=1792,
+    license="https://huggingface.co/sbintuitions/sarashina-embedding-v1-1b/blob/main/LICENSE",
+    max_tokens=8192,
+    reference="https://huggingface.co/sbintuitions/sarashina-embedding-v1-1b",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=False,
+    adapted_from="sbintuitions/sarashina2.1-1b",
+    superseded_by="sbintuitions/sarashina-embedding-v2-1b",
+    training_datasets={"NQ", "MrTidyRetrieval"},
+    public_training_code=None,
+    public_training_data="https://huggingface.co/sbintuitions/sarashina-embedding-v1-1b",
+    citation=None,
+    contacts=["akiFQC", "lsz05"],
+)

mteb/models/model_implementations/searchmap_models.py

@@ -20,6 +20,7 @@ searchmap_preview = ModelMeta(
         "model_prompts": task_instructions,
     },
     name="VPLabs/SearchMap_Preview",
+    model_type=["dense"],
     revision="69de17ef48278ed08ba1a4e65ead8179912b696e",
     languages=["eng-Latn"],
     open_weights=True,

mteb/models/model_implementations/seed_1_6_embedding_models.py

@@ -1,14 +1,15 @@
+from __future__ import annotations
+
 import base64
 import logging
 import os
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from io import BytesIO
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import requests
 import torch
-from PIL import Image
 from torch.utils.data import DataLoader
 
 from mteb._requires_package import requires_package
@@ -19,6 +20,10 @@ from mteb.models.model_implementations.nvidia_models import nvidia_training_data
 from mteb.models.model_meta import ModelMeta
 from mteb.types import Array, BatchedInput, PromptType
 
+if TYPE_CHECKING:
+    from PIL import Image
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -408,6 +413,7 @@ TASK_NAME_TO_INSTRUCTION = {
 
 seed_embedding = ModelMeta(
     name="Bytedance/Seed1.6-embedding",
+    model_type=["dense"],
     revision="1",
     release_date="2025-06-18",
     languages=[