mteb 2.1.4__py3-none-any.whl → 2.5.2__py3-none-any.whl

mteb/tasks/clustering/nld/open_tender_clustering_s2s.py

@@ -38,4 +38,7 @@ class OpenTenderClusteringS2S(AbsTaskClustering):
   year = {2025},
 }
 """,
+        prompt={
+            "query": "Identificeer de hoofdcategorie van aanbestedingen op basis van de titels"
+        },
     )

mteb/tasks/clustering/nld/vabb_clustering_p2p.py

@@ -39,6 +39,9 @@ class VABBClusteringP2P(AbsTaskClustering):
   year = {2024},
 }
 """,
+        prompt={
+            "query": "Identificeer de hoofdcategorie van wetenschappelijke artikelen op basis van de titels en abstracts"
+        },
     )
 
     def dataset_transform(self):

mteb/tasks/clustering/nld/vabb_clustering_s2s.py

@@ -39,6 +39,9 @@ class VABBClusteringS2S(AbsTaskClustering):
   year = {2024},
 }
 """,
+        prompt={
+            "query": "Identificeer de hoofdcategorie van wetenschappelijke artikelen op basis van de titels"
+        },
     )
 
     def dataset_transform(self):

mteb/tasks/clustering/vie/reddit_clustering_p2p_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class RedditClusteringP2PVN(AbsTaskClusteringLegacy):
     metadata = TaskMetadata(
         name="RedditClusteringP2P-VN",
-        description="""A translated dataset from Clustering of title+posts from reddit. Clustering of 10 sets of 50k paragraphs and 40 sets of 10k paragraphs.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Clustering of title+posts from reddit. Clustering of 10 sets of 50k paragraphs and 40 sets of 10k paragraphs. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://arxiv.org/abs/2104.07081",
         dataset={
             "path": "GreenNode/reddit-clustering-p2p-vn",

mteb/tasks/clustering/vie/reddit_clustering_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class RedditClusteringVN(AbsTaskClusteringLegacy):
     metadata = TaskMetadata(
         name="RedditClustering-VN",
-        description="""A translated dataset from Clustering of titles from 199 subreddits. Clustering of 25 sets, each with 10-50 classes, and each class with 100 - 1000 sentences.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Clustering of titles from 199 subreddits. Clustering of 25 sets, each with 10-50 classes, and each class with 100 - 1000 sentences. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://arxiv.org/abs/2104.07081",
         dataset={
             "path": "GreenNode/reddit-clustering-vn",

mteb/tasks/clustering/vie/stack_exchange_clustering_p2p_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class StackExchangeClusteringP2PVN(AbsTaskClusteringLegacy):
     metadata = TaskMetadata(
         name="StackExchangeClusteringP2P-VN",
-        description="""A translated Clustering of title+body from stackexchange. Clustering of 5 sets of 10k paragraphs and 5 sets of 5k paragraphs.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated Clustering of title+body from stackexchange. Clustering of 5 sets of 10k paragraphs and 5 sets of 5k paragraphs. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://arxiv.org/abs/2104.07081",
         dataset={
             "path": "GreenNode/stackexchange-clustering-p2p-vn",

mteb/tasks/clustering/vie/stack_exchange_clustering_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class StackExchangeClusteringVN(AbsTaskClusteringLegacy):
     metadata = TaskMetadata(
         name="StackExchangeClustering-VN",
-        description="""A translated dataset from Clustering of titles from 121 stackexchanges. Clustering of 25 sets, each with 10-50 classes, and each class with 100 - 1000 sentences.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Clustering of titles from 121 stackexchanges. Clustering of 25 sets, each with 10-50 classes, and each class with 100 - 1000 sentences. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://arxiv.org/abs/2104.07081",
         dataset={
             "path": "GreenNode/stackexchange-clustering-vn",

mteb/tasks/clustering/vie/twenty_newsgroups_clustering_vn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class TwentyNewsgroupsClusteringVN(AbsTaskClusteringLegacy):
     metadata = TaskMetadata(
         name="TwentyNewsgroupsClustering-VN",
-        description="""A translated dataset from Clustering of the 20 Newsgroups dataset (subject only).
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Clustering of the 20 Newsgroups dataset (subject only). The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html",
         dataset={
             "path": "GreenNode/twentynewsgroups-clustering-vn",

mteb/tasks/multilabel_classification/ita/emit_classification.py

@@ -7,11 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class EmitClassification(AbsTaskMultilabelClassification):
     metadata = TaskMetadata(
         name="EmitClassification",
-        description="""The EMit dataset is a comprehensive resource for the detection of emotions in Italian social media texts.
-        The EMit dataset consists of social media messages about TV shows, TV series, music videos, and advertisements.
-        Each message is annotated with one or more of the 8 primary emotions defined by Plutchik
-        (anger, anticipation, disgust, fear, joy, sadness, surprise, trust), as well as an additional label “love.”
-        """,
+        description="The EMit dataset is a comprehensive resource for the detection of emotions in Italian social media texts. The EMit dataset consists of social media messages about TV shows, TV series, music videos, and advertisements. Each message is annotated with one or more of the 8 primary emotions defined by Plutchik (anger, anticipation, disgust, fear, joy, sadness, surprise, trust), as well as an additional label “love.”",
         reference="https://github.com/oaraque/emit",
         dataset={
             "path": "MattiaSangermano/emit",

mteb/tasks/multilabel_classification/kor/kor_hate_speech_ml_classification.py

@@ -7,15 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class KorHateSpeechMLClassification(AbsTaskMultilabelClassification):
     metadata = TaskMetadata(
         name="KorHateSpeechMLClassification",
-        description="""
-        The Korean Multi-label Hate Speech Dataset, K-MHaS, consists of 109,692 utterances from Korean online news comments,
-        labelled with 8 fine-grained hate speech classes (labels: Politics, Origin, Physical, Age, Gender, Religion, Race, Profanity)
-        or Not Hate Speech class. Each utterance provides from a single to four labels that can handles Korean language patterns effectively.
-        For more details, please refer to the paper about K-MHaS, published at COLING 2022.
-        This dataset is based on the Korean online news comments available on Kaggle and Github.
-        The unlabeled raw data was collected between January 2018 and June 2020.
-        The language producers are users who left the comments on the Korean online news platform between 2018 and 2020.
-        """,
+        description="The Korean Multi-label Hate Speech Dataset, K-MHaS, consists of 109,692 utterances from Korean online news comments, labelled with 8 fine-grained hate speech classes (labels: Politics, Origin, Physical, Age, Gender, Religion, Race, Profanity) or Not Hate Speech class. Each utterance provides from a single to four labels that can handles Korean language patterns effectively. For more details, please refer to the paper about K-MHaS, published at COLING 2022. This dataset is based on the Korean online news comments available on Kaggle and Github. The unlabeled raw data was collected between January 2018 and June 2020. The language producers are users who left the comments on the Korean online news platform between 2018 and 2020.",
         dataset={
             "path": "mteb/KorHateSpeechMLClassification",
             "revision": "47cd2e61b64f2f11ccb006a579cda71318c6de9b",

mteb/tasks/multilabel_classification/mlt/maltese_news_classification.py

@@ -7,12 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class MalteseNewsClassification(AbsTaskMultilabelClassification):
     metadata = TaskMetadata(
         name="MalteseNewsClassification",
-        description="""A multi-label topic classification dataset for Maltese News
-        Articles. The data was collected from the press_mt subset from Korpus
-        Malti v4.0. Article contents were cleaned to filter out JavaScript, CSS,
-        & repeated non-Maltese sub-headings. The labels are based on the category
-        field from this corpus.
-        """,
+        description="A multi-label topic classification dataset for Maltese News Articles. The data was collected from the press_mt subset from Korpus Malti v4.0. Article contents were cleaned to filter out JavaScript, CSS, & repeated non-Maltese sub-headings. The labels are based on the category field from this corpus.",
         reference="https://huggingface.co/datasets/MLRS/maltese_news_categories",
         dataset={
             "path": "MLRS/maltese_news_categories",

mteb/tasks/multilabel_classification/nld/covid_disinformation_nl_multi_label_classification.py

@@ -61,6 +61,9 @@ Yih, Scott Wen-tau},
   year = {2021},
 }
 """,
+        prompt={
+            "query": "Classificeer COVID-19-gerelateerde sociale media-berichten in alle toepasselijke desinformatiecategorieën"
+        },
     )
 
     def dataset_transform(self) -> None:

mteb/tasks/multilabel_classification/nld/vabb_multi_label_classification.py

@@ -41,4 +41,7 @@ class VABBMultiLabelClassification(AbsTaskMultilabelClassification):
   year = {2024},
 }
 """,
+        prompt={
+            "query": "Classificeer de onderwerpen van een wetenschappelijk artikel op basis van de abstract"
+        },
     )

mteb/tasks/multilabel_classification/por/brazilian_toxic_tweets_classification.py

@@ -7,12 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class BrazilianToxicTweetsClassification(AbsTaskMultilabelClassification):
     metadata = TaskMetadata(
         name="BrazilianToxicTweetsClassification",
-        description="""
-        ToLD-Br is the biggest dataset for toxic tweets in Brazilian Portuguese, crowdsourced by 42 annotators selected from
-        a pool of 129 volunteers. Annotators were selected aiming to create a plural group in terms of demographics (ethnicity,
-        sexual orientation, age, gender). Each tweet was labeled by three annotators in 6 possible categories: LGBTQ+phobia,
-        Xenophobia, Obscene, Insult, Misogyny and Racism.
-        """,
+        description="ToLD-Br is the biggest dataset for toxic tweets in Brazilian Portuguese, crowdsourced by 42 annotators selected from a pool of 129 volunteers. Annotators were selected aiming to create a plural group in terms of demographics (ethnicity, sexual orientation, age, gender). Each tweet was labeled by three annotators in 6 possible categories: LGBTQ+phobia, Xenophobia, Obscene, Insult, Misogyny and Racism.",
         reference="https://paperswithcode.com/dataset/told-br",
         dataset={
             "path": "mteb/BrazilianToxicTweetsClassification",

mteb/tasks/multilabel_classification/swe/swedish_patent_cpc_group_classification.py

@@ -7,7 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SwedishPatentCPCGroupClassification(AbsTaskMultilabelClassification):
     metadata = TaskMetadata(
         name="SwedishPatentCPCGroupClassification",
-        description="""This dataset contains historical Swedish patent documents (1885-1972) classified according to the Cooperative Patent Classification (CPC) system at the group level. Each document can have multiple labels, making this a challenging multi-label classification task with significant class imbalance and data sparsity characteristics. The dataset includes patent claims text extracted from digitally recreated versions of historical Swedish patents, generated using Optical Character Recognition (OCR) from original paper documents. The text quality varies due to OCR limitations, but all CPC labels were manually assigned by patent engineers at PRV (Swedish Patent and Registration Office), ensuring high reliability for machine learning applications.""",
+        description="This dataset contains historical Swedish patent documents (1885-1972) classified according to the Cooperative Patent Classification (CPC) system at the group level. Each document can have multiple labels, making this a challenging multi-label classification task with significant class imbalance and data sparsity characteristics. The dataset includes patent claims text extracted from digitally recreated versions of historical Swedish patents, generated using Optical Character Recognition (OCR) from original paper documents. The text quality varies due to OCR limitations, but all CPC labels were manually assigned by patent engineers at PRV (Swedish Patent and Registration Office), ensuring high reliability for machine learning applications.",
         reference="https://urn.kb.se/resolve?urn=urn:nbn:se:kth:diva-368254",
         type="MultilabelClassification",
         category="t2t",

mteb/tasks/multilabel_classification/swe/swedish_patent_cpc_subclass_classification.py

@@ -7,8 +7,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SwedishPatentCPCSubclassClassification(AbsTaskMultilabelClassification):
     metadata = TaskMetadata(
         name="SwedishPatentCPCSubclassClassification",
-        description="""This dataset contains historical Swedish patent documents (1885-1972) classified according to the Cooperative Patent Classification (CPC) system. Each document can have multiple labels, making this a multi-label classification task with significant implications for patent retrieval and prior art search.
-		The dataset includes patent claims text extracted from digitally recreated versions of historical Swedish patents, generated using Optical Character Recognition (OCR) from original paper documents. The text quality varies due to OCR limitations, but all CPC labels were manually assigned by patent engineers at PRV (Swedish Patent and Registration Office), ensuring high reliability for machine learning applications.""",
+        description="This dataset contains historical Swedish patent documents (1885-1972) classified according to the Cooperative Patent Classification (CPC) system. Each document can have multiple labels, making this a multi-label classification task with significant implications for patent retrieval and prior art search. The dataset includes patent claims text extracted from digitally recreated versions of historical Swedish patents, generated using Optical Character Recognition (OCR) from original paper documents. The text quality varies due to OCR limitations, but all CPC labels were manually assigned by patent engineers at PRV (Swedish Patent and Registration Office), ensuring high reliability for machine learning applications.",
         reference="https://urn.kb.se/resolve?urn=urn:nbn:se:kth:diva-368254",
         type="MultilabelClassification",
         category="t2t",

mteb/tasks/pair_classification/dan/talemaader_pc.py

@@ -5,12 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class TalemaaderPC(AbsTaskPairClassification):
     metadata = TaskMetadata(
         name="TalemaaderPC",
-        description="""\
-The Danish Language and Literature Society has developed a dataset for evaluating language models in Danish.
-The dataset contains a total of 1000 Danish idioms and fixed expressions with transferred meanings based on the Danish Dictionary's collection of fixed expressions with associated definitions.
-For each of the 1000 idioms and fixed expressions, three false definitions have also been prepared.
-The dataset can be used to test the performance of language models in identifying correct definitions for Danish idioms and fixed expressions.
-""",
+        description="\\ The Danish Language and Literature Society has developed a dataset for evaluating language models in Danish. The dataset contains a total of 1000 Danish idioms and fixed expressions with transferred meanings based on the Danish Dictionary's collection of fixed expressions with associated definitions. For each of the 1000 idioms and fixed expressions, three false definitions have also been prepared. The dataset can be used to test the performance of language models in identifying correct definitions for Danish idioms and fixed expressions.",
         reference="https://sprogteknologi.dk/dataset/1000-talemader-evalueringsdatasaet",
         dataset={
             "path": "mteb/talemaader_pc",

mteb/tasks/pair_classification/eng/legal_bench_pc.py

@@ -50,15 +50,7 @@ _DATASET_COLUMN_MAP = [
 class LegalBenchPC(AbsTaskPairClassification):
     metadata = TaskMetadata(
         name="LegalBenchPC",
-        description="""This LegalBench pair classification task is a combination of the following datasets:
-
-        - Citation Prediction Classification: Given a legal statement and a case citation, determine if the citation is supportive of the legal statement.
-        - Consumer Contracts QA: The task consists of 400 yes/no questions relating to consumer contracts (specifically, online terms of service) and is relevant to the legal skill of contract interpretation.
-        - Contract QA: Answer yes/no questions about whether contractual clauses discuss particular issues like confidentiality requirements, BIPA consent, PII data breaches, breach of contract etc.
-        - Hearsay: Classify if a particular piece of evidence qualifies as hearsay. Each sample in the dataset describes (1) an issue being litigated or an assertion a party wishes to prove, and (2) a piece of evidence a party wishes to introduce. The goal is to determine if—as it relates to the issue—the evidence would be considered hearsay under the definition provided above.
-        - Privacy Policy Entailment: Given a privacy policy clause and a description of the clause, determine if the description is correct. This is a binary classification task in which the LLM is provided with a clause from a privacy policy, and a description of that clause (e.g., “The policy describes collection of the user’s HTTP cookies, flash cookies, pixel tags, or similar identifiers by a party to the contract.”).
-        - Privacy Policy QA: Given a question and a clause from a privacy policy, determine if the clause contains enough information to answer the question. This is a binary classification task in which the LLM is provided with a question (e.g., “do you publish my data”) and a clause from a privacy policy. The LLM must determine if the clause contains an answer to the question, and classify the question-clause pair.
-        """,
+        description="This LegalBench pair classification task is a combination of the following datasets: - Citation Prediction Classification: Given a legal statement and a case citation, determine if the citation is supportive of the legal statement. - Consumer Contracts QA: The task consists of 400 yes/no questions relating to consumer contracts (specifically, online terms of service) and is relevant to the legal skill of contract interpretation. - Contract QA: Answer yes/no questions about whether contractual clauses discuss particular issues like confidentiality requirements, BIPA consent, PII data breaches, breach of contract etc. - Hearsay: Classify if a particular piece of evidence qualifies as hearsay. Each sample in the dataset describes (1) an issue being litigated or an assertion a party wishes to prove, and (2) a piece of evidence a party wishes to introduce. The goal is to determine if—as it relates to the issue—the evidence would be considered hearsay under the definition provided above. - Privacy Policy Entailment: Given a privacy policy clause and a description of the clause, determine if the description is correct. This is a binary classification task in which the LLM is provided with a clause from a privacy policy, and a description of that clause (e.g., “The policy describes collection of the user’s HTTP cookies, flash cookies, pixel tags, or similar identifiers by a party to the contract.”). - Privacy Policy QA: Given a question and a clause from a privacy policy, determine if the clause contains enough information to answer the question. This is a binary classification task in which the LLM is provided with a question (e.g., “do you publish my data”) and a clause from a privacy policy. The LLM must determine if the clause contains an answer to the question, and classify the question-clause pair.",
         reference="https://huggingface.co/datasets/nguha/legalbench",
         dataset={
             "path": "mteb/LegalBenchPC",

mteb/tasks/pair_classification/nld/sick_nl_pair_classification.py

@@ -33,4 +33,7 @@ class SICKNLPairClassification(AbsTaskPairClassification):
   year = {2021},
 }
 """,
+        prompt={
+            "query": "Zoek tekst die semantisch vergelijkbaar is met de gegeven tekst."
+        },
     )

mteb/tasks/pair_classification/nld/xlwic_nl_pair_classification.py

@@ -38,4 +38,7 @@ class XLWICNLPairClassification(AbsTaskPairClassification):
   year = {2020},
 }
 """,
+        prompt={
+            "query": "Zoek tekst die semantisch vergelijkbaar is met de gegeven tekst."
+        },
     )

mteb/tasks/pair_classification/rus/__init__.py

@@ -1,3 +1,3 @@
-from .terra import TERRa
+from .terra import TERRa, TERRaV2
 
-__all__ = ["TERRa"]
+__all__ = ["TERRa", "TERRaV2"]

mteb/tasks/pair_classification/rus/terra.py

@@ -1,31 +1,27 @@
 from mteb.abstasks.pair_classification import AbsTaskPairClassification
 from mteb.abstasks.task_metadata import TaskMetadata
+from mteb.types import PromptType
 
-
-class TERRa(AbsTaskPairClassification):
-    metadata = TaskMetadata(
-        name="TERRa",
-        dataset={
-            "path": "ai-forever/terra-pairclassification",
-            "revision": "7b58f24536063837d644aab9a023c62199b2a612",
-        },
-        description="Textual Entailment Recognition for Russian. This task requires to recognize, given two text fragments, "
-        + "whether the meaning of one text is entailed (can be inferred) from the other text.",
-        reference="https://arxiv.org/pdf/2010.15925",
-        type="PairClassification",
-        category="t2t",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["rus-Cyrl"],
-        main_score="max_ap",
-        date=("2000-01-01", "2018-01-01"),
-        domains=["News", "Web", "Written"],
-        task_subtypes=[],
-        license="mit",
-        annotations_creators="human-annotated",
-        dialect=[],
-        sample_creation="found",
-        bibtex_citation=r"""
+_terra_metadata = dict(
+    dataset={
+        "path": "ai-forever/terra-pairclassification",
+        "revision": "7b58f24536063837d644aab9a023c62199b2a612",
+    },
+    reference="https://arxiv.org/pdf/2010.15925",
+    type="PairClassification",
+    category="t2t",
+    modalities=["text"],
+    eval_splits=["dev"],
+    eval_langs=["rus-Cyrl"],
+    main_score="max_ap",
+    date=("2000-01-01", "2018-01-01"),
+    domains=["News", "Web", "Written"],
+    task_subtypes=[],
+    license="mit",
+    annotations_creators="human-annotated",
+    dialect=[],
+    sample_creation="found",
+    bibtex_citation=r"""
 @article{shavrina2020russiansuperglue,
   author = {Shavrina, Tatiana
 and Fenogenova, Alena
@@ -42,7 +38,37 @@ and Evlampiev, Andrey},
   year = {2020},
 }
 """,
+)
+
+
+class TERRa(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="TERRa",
+        description="Textual Entailment Recognition for Russian. This task requires to recognize, given two text fragments, "
+        + "whether the meaning of one text is entailed (can be inferred) from the other text.",
         prompt="Given a premise, retrieve a hypothesis that is entailed by the premise",
+        **_terra_metadata,
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.dataset.rename_column("sent1", "sentence1")
+        self.dataset = self.dataset.rename_column("sent2", "sentence2")
+
+
+class TERRaV2(AbsTaskPairClassification):
+    input1_prompt_type = PromptType.document
+    input2_prompt_type = PromptType.query
+
+    metadata = TaskMetadata(
+        name="TERRa.V2",
+        description="Textual Entailment Recognition for Russian. This task requires to recognize, given two text fragments, "
+        + "whether the meaning of one text is entailed (can be inferred) from the other text."
+        + " Version 2 uses different prompt types for the two inputs.",
+        adapted_from=["TERRa"],
+        prompt={
+            PromptType.query.value: "Given a premise, retrieve a hypothesis that is entailed by the premise"
+        },
+        **_terra_metadata,
     )
 
     def dataset_transform(self):

mteb/tasks/pair_classification/vie/sprint_duplicate_questions_pcvn.py

@@ -5,11 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class SprintDuplicateQuestionsPCVN(AbsTaskPairClassification):
     metadata = TaskMetadata(
         name="SprintDuplicateQuestions-VN",
-        description="""A translated dataset from Duplicate questions from the Sprint community.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Duplicate questions from the Sprint community. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://www.aclweb.org/anthology/D18-1131/",
         dataset={
             "path": "GreenNode/sprintduplicatequestions-pairclassification-vn",

mteb/tasks/pair_classification/vie/twitter_sem_eval2015_pcvn.py

@@ -9,11 +9,7 @@ class TwitterSemEval2015PCVN(AbsTaskPairClassification):
             "path": "GreenNode/twittersemeval2015-pairclassification-vn",
             "revision": "9215a3c954078fd15c2bbecca914477d53944de1",
         },
-        description="""A translated dataset from Paraphrase-Pairs of Tweets from the SemEval 2015 workshop.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Paraphrase-Pairs of Tweets from the SemEval 2015 workshop. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://alt.qcri.org/semeval2015/task1/",
         category="t2c",
         type="PairClassification",

mteb/tasks/pair_classification/vie/twitter_url_corpus_pcvn.py

@@ -9,11 +9,7 @@ class TwitterURLCorpusPC(AbsTaskPairClassification):
             "path": "GreenNode/twitterurlcorpus-pairclassification-vn",
             "revision": "6e6a40aaade2129f70432f2156a6d24b63d72be3",
         },
-        description="""A translated dataset from Paraphrase-Pairs of Tweets.
-            The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system:
-            - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation.
-            - Applies advanced embedding models to filter the translations.
-            - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.""",
+        description="A translated dataset from Paraphrase-Pairs of Tweets. The process of creating the VN-MTEB (Vietnamese Massive Text Embedding Benchmark) from English samples involves a new automated system: - The system uses large language models (LLMs), specifically Coherence's Aya model, for translation. - Applies advanced embedding models to filter the translations. - Use LLM-as-a-judge to scoring the quality of the samples base on multiple criteria.",
         reference="https://languagenet.github.io/",
         category="t2c",
         type="PairClassification",

mteb/tasks/regression/multilingual/ru_sci_bench_regression.py

@@ -5,9 +5,7 @@ from mteb.abstasks.task_metadata import TaskMetadata
 class RuSciBenchCitedCountRegression(AbsTaskRegression):
     metadata = TaskMetadata(
         name="RuSciBenchCitedCountRegression",
-        description="""Predicts the number of times a scientific article has been cited by other papers.
-        The prediction is based on the article's title and abstract. The data is sourced from the Russian electronic
-        library of scientific publications (eLibrary.ru) and includes papers with both Russian and English abstracts.""",
+        description="Predicts the number of times a scientific article has been cited by other papers. The prediction is based on the article's title and abstract. The data is sourced from the Russian electronic library of scientific publications (eLibrary.ru) and includes papers with both Russian and English abstracts.",
         reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
         dataset={
             "path": "mlsa-iai-msu-lab/ru_sci_bench_mteb",
@@ -51,9 +49,7 @@ class RuSciBenchCitedCountRegression(AbsTaskRegression):
 class RuSciBenchYearPublRegression(AbsTaskRegression):
     metadata = TaskMetadata(
         name="RuSciBenchYearPublRegression",
-        description="""Predicts the publication year of a scientific article. The prediction is based on the
-        article's title and abstract. The data is sourced from the Russian electronic library of scientific
-        publications (eLibrary.ru) and includes papers with both Russian and English abstracts.""",
+        description="Predicts the publication year of a scientific article. The prediction is based on the article's title and abstract. The data is sourced from the Russian electronic library of scientific publications (eLibrary.ru) and includes papers with both Russian and English abstracts.",
         reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
         dataset={
             "path": "mlsa-iai-msu-lab/ru_sci_bench_mteb",

mteb/tasks/reranking/jpn/__init__.py

@@ -1,5 +1,13 @@
 from .j_qa_ra_reranking import JQaRAReranking
+from .j_qa_ra_reranking_lite import JQaRARerankingLite
 from .ja_cwir_reranking import JaCWIRReranking
+from .ja_cwir_reranking_lite import JaCWIRRerankingLite
 from .m_marco_reranking import VoyageMMarcoReranking
 
-__all__ = ["JQaRAReranking", "JaCWIRReranking", "VoyageMMarcoReranking"]
+__all__ = [
+    "JQaRAReranking",
+    "JQaRARerankingLite",
+    "JaCWIRReranking",
+    "JaCWIRRerankingLite",
+    "VoyageMMarcoReranking",
+]

mteb/tasks/reranking/jpn/j_qa_ra_reranking_lite.py

@@ -0,0 +1,49 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class JQaRARerankingLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JQaRARerankingLite",
+        dataset={
+            "path": "mteb/JQaRARerankingLite",
+            "revision": "d23d3ad479f74824ed126052e810eac47e685558",
+        },
+        description=(
+            "JQaRA (Japanese Question Answering with Retrieval Augmentation) is a reranking dataset "
+            "consisting of questions from JAQKET and corpus from Japanese Wikipedia. This is the lightweight "
+            "version with a reduced corpus (172,897 documents) constructed using hard negatives from "
+            "5 high-performance models."
+        ),
+        reference="https://huggingface.co/datasets/hotchpotch/JQaRA",
+        type="Reranking",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2020-01-01", "2025-01-01"),
+        domains=["Encyclopaedic", "Non-fiction", "Written"],
+        task_subtypes=["Question answering"],
+        license="cc-by-sa-4.0",
+        annotations_creators="derived",
+        dialect=["jpn-Jpan"],
+        sample_creation="found",
+        adapted_from=["JQaRAReranking"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@misc{yuichi-tateno-2024-jqara,
+  author = {Yuichi Tateno},
+  title = {JQaRA: Japanese Question Answering with Retrieval Augmentation
+- 検索拡張(RAG)評価のための日本語Q&Aデータセット},
+  url = {https://huggingface.co/datasets/hotchpotch/JQaRA},
+}
+""",
+    )

mteb/tasks/reranking/jpn/ja_cwir_reranking_lite.py

@@ -0,0 +1,47 @@
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class JaCWIRRerankingLite(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="JaCWIRRerankingLite",
+        dataset={
+            "path": "mteb/JaCWIRRerankingLite",
+            "revision": "b7c738193fb9b20c97c2b5d9a8fa3f3d28503dc0",
+        },
+        description=(
+            "JaCWIR (Japanese Casual Web IR) is a dataset consisting of questions and webpage meta descriptions "
+            "collected from Hatena Bookmark. This is the lightweight reranking version with a reduced corpus "
+            "(188,033 documents) constructed using hard negatives from 5 high-performance models."
+        ),
+        reference="https://huggingface.co/datasets/hotchpotch/JaCWIR",
+        type="Reranking",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["jpn-Jpan"],
+        main_score="ndcg_at_10",
+        date=("2020-01-01", "2025-01-01"),
+        domains=["Web", "Written"],
+        task_subtypes=["Article retrieval"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="found",
+        adapted_from=["JaCWIRReranking"],
+        bibtex_citation=r"""
+@misc{jmteb_lite,
+  author = {Li, Shengzhe and Ohagi, Masaya and Ri, Ryokan and Fukuchi, Akihiko and Shibata, Tomohide
+and Kawahara, Daisuke},
+  howpublished = {\url{https://huggingface.co/datasets/sbintuitions/JMTEB-lite}},
+  title = {{J}{M}{T}{E}{B}-lite: {T}he {L}ightweight {V}ersion of {JMTEB}},
+  year = {2025},
+}
+
+@misc{yuichi-tateno-2024-jacwir,
+  author = {Yuichi Tateno},
+  title = {JaCWIR: Japanese Casual Web IR - 日本語情報検索評価のための小規模でカジュアルなWebタイトルと概要のデータセット},
+  url = {https://huggingface.co/datasets/hotchpotch/JaCWIR},
+}
+""",
+    )

mteb/tasks/reranking/multilingual/__init__.py

@@ -1,6 +1,7 @@
 from .esci_reranking import ESCIReranking
 from .hume_wikipedia_reranking_multilingual import HUMEWikipediaRerankingMultilingual
 from .miracl_reranking import MIRACLReranking
+from .multi_long_doc_reranking import MultiLongDocReranking
 from .wikipedia_reranking_multilingual import WikipediaRerankingMultilingual
 from .x_glue_wpr_reranking import XGlueWPRReranking
 
@@ -8,6 +9,7 @@ __all__ = [
     "ESCIReranking",
     "HUMEWikipediaRerankingMultilingual",
     "MIRACLReranking",
+    "MultiLongDocReranking",
     "WikipediaRerankingMultilingual",
     "XGlueWPRReranking",
 ]