mteb 2.1.4__py3-none-any.whl → 2.7.2__py3-none-any.whl

mteb/similarity_functions.py

@@ -1,6 +1,7 @@
 import torch
 
 from mteb.models import EncoderProtocol
+from mteb.models.model_meta import ScoringFunction
 from mteb.types import Array
 
 
@@ -38,6 +39,54 @@ def compute_pairwise_similarity(
     return pairwise_cos_sim(embedding1, embedding2)
 
 
+def select_similarity(
+    embedding1: Array,
+    embedding2: Array,
+    similarity_fn: ScoringFunction,
+) -> Array:
+    """Compute similarity between two sets of embeddings using the specified similarity function.
+
+    Args:
+        embedding1: The first set of embeddings.
+        embedding2: The second set of embeddings.
+        similarity_fn: The similarity function to use (COSINE, DOT_PRODUCT, EUCLIDEAN).
+
+    Returns:
+        Array: The computed similarity scores.
+    """
+    if similarity_fn is ScoringFunction.COSINE:
+        return cos_sim(embedding1, embedding2)
+    elif similarity_fn is ScoringFunction.DOT_PRODUCT:
+        return dot_score(embedding1, embedding2)
+    elif similarity_fn is ScoringFunction.EUCLIDEAN:
+        return euclidean_sim(embedding1, embedding2)
+    raise ValueError(f"Unsupported similarity function: {similarity_fn}")
+
+
+def select_pairwise_similarity(
+    embedding1: Array,
+    embedding2: Array,
+    similarity_fn: ScoringFunction,
+) -> Array:
+    """Compute pairwise similarity between two sets of embeddings using the specified similarity function.
+
+    Args:
+        embedding1: The first set of embeddings.
+        embedding2: The second set of embeddings.
+        similarity_fn: The similarity function to use (COSINE, DOT_PRODUCT, EUCLIDEAN).
+
+    Returns:
+        Array: The computed pairwise similarity scores.
+    """
+    if similarity_fn is ScoringFunction.COSINE:
+        return pairwise_cos_sim(embedding1, embedding2)
+    elif similarity_fn is ScoringFunction.DOT_PRODUCT:
+        return pairwise_dot_score(embedding1, embedding2)
+    elif similarity_fn is ScoringFunction.EUCLIDEAN:
+        return pairwise_euclidean_sim(embedding1, embedding2)
+    raise ValueError(f"Unsupported similarity function: {similarity_fn}")
+
+
 def _normalize_embeddings(embeddings: Array) -> torch.Tensor:
     """Normalizes the embeddings matrix, so that each sentence embedding has unit length.
 
@@ -137,7 +186,7 @@ def max_sim(a: Array, b: Array) -> torch.Tensor:
         b,
     )
 
-    return scores.max(axis=-1).values.sum(axis=-1)
+    return scores.max(axis=-1).values.sum(axis=-1)  # type: ignore[call-overload]
 
 
 # https://github.com/lightonai/pylate/blob/2d094a724866d6e15701781528368438081c0157/pylate/scores/scores.py#L67C1-L122C38
@@ -168,7 +217,7 @@ def pairwise_max_sim(
             document_embedding,
         )
 
-        scores.append(query_document_score.max(axis=-1).values.sum())
+        scores.append(query_document_score.max(axis=-1).values.sum())  # type: ignore[call-overload]
 
     return torch.stack(scores, dim=0)
 
@@ -268,11 +317,15 @@ def similarity(text_embeddings: Array, input_embeddings: Array) -> Array:
     Returns:
         Matrix with similarities
     """
-    text_embeddings = _convert_to_tensor(text_embeddings)
-    input_embeddings = _convert_to_tensor(input_embeddings)
+    text_embeddings_tensor = _convert_to_tensor(text_embeddings)
+    input_embeddings_tensor = _convert_to_tensor(input_embeddings)
 
-    text_embeddings = text_embeddings / text_embeddings.norm(dim=-1, keepdim=True)
-    input_embeddings = input_embeddings / input_embeddings.norm(dim=-1, keepdim=True)
-    logits = torch.matmul(input_embeddings, text_embeddings.T)
+    text_embeddings_tensor = text_embeddings_tensor / text_embeddings_tensor.norm(
+        dim=-1, keepdim=True
+    )
+    input_embeddings_tensor = input_embeddings_tensor / input_embeddings_tensor.norm(
+        dim=-1, keepdim=True
+    )
+    logits = torch.matmul(input_embeddings_tensor, text_embeddings_tensor.T)
     probs = (logits * 100).softmax(dim=-1)
     return probs

mteb/tasks/bitext_mining/multilingual/__init__.py

@@ -16,7 +16,7 @@ from .nusa_translation_bitext_mining import NusaTranslationBitextMining
 from .nusa_x_bitext_mining import NusaXBitextMining
 from .phinc_bitext_mining import PhincBitextMining
 from .roma_tales_bitext_mining import RomaTalesBitextMining
-from .ru_sci_bench_bitext_mining import RuSciBenchBitextMining
+from .ru_sci_bench_bitext_mining import RuSciBenchBitextMining, RuSciBenchBitextMiningV2
 from .tatoeba_bitext_mining import TatoebaBitextMining
 from .web_faq_bitext_mining import WebFAQBitextMiningQAs, WebFAQBitextMiningQuestions
 
@@ -40,6 +40,7 @@ __all__ = [
     "PhincBitextMining",
     "RomaTalesBitextMining",
     "RuSciBenchBitextMining",
+    "RuSciBenchBitextMiningV2",
     "TatoebaBitextMining",
     "WebFAQBitextMiningQAs",
     "WebFAQBitextMiningQuestions",

mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining.py

@@ -23,7 +23,7 @@ class BUCCBitextMining(AbsTaskBitextMining):
             "path": "mteb/BUCC",
             "revision": "414572247440f0ccacf7eb0bb70a31533a0e5443",
         },
-        description="BUCC bitext mining dataset",
+        description="BUCC bitext mining dataset train split.",
         reference="https://comparable.limsi.fr/bucc2018/bucc2018-task.html",
         type="BitextMining",
         category="t2t",
@@ -71,7 +71,9 @@ Rapp, Reinhard},
 
                 sentence1 = data["sentence1"][0]
                 sentence2 = data["sentence2"][0]
-                sentence1 = [sentence1[i] for (i, j) in gold]
+                sentence1 = [
+                    sentence1[i] for (i, j) in gold
+                ]  # keep only sentences in gold. The 2nd value is meant for sentence2 but not used here. This is fixed in BUCC.v2.
                 logger.info(f"Lang {lang} num gold {len(gold)}")
                 logger.info(f"Lang {lang} num sentence1 {len(sentence1)}")
                 logger.info(f"Lang {lang} num sentence2 {len(sentence2)}")

mteb/tasks/bitext_mining/multilingual/bucc_bitext_mining_fast.py

@@ -20,7 +20,7 @@ class BUCCBitextMiningFast(AbsTaskBitextMining):
             "path": "mteb/bucc-bitext-mining",
             "revision": "1739dc11ffe9b7bfccd7f3d585aeb4c544fc6677",
         },
-        description="BUCC bitext mining dataset",
+        description="BUCC bitext mining dataset train split, gold set only.",
         reference="https://comparable.limsi.fr/bucc2018/bucc2018-task.html",
         type="BitextMining",
         category="t2t",

mteb/tasks/bitext_mining/multilingual/ru_sci_bench_bitext_mining.py

@@ -10,11 +10,53 @@ class RuSciBenchBitextMining(AbsTaskBitextMining):
             "path": "mlsa-iai-msu-lab/ru_sci_bench_bitext_mining",
             "revision": "e5840033c5cf2573932db027ac8001fe0a7eb6fa",
         },
-        description="""This task focuses on finding translations of scientific articles.
-        The dataset is sourced from eLibrary, Russia's largest electronic library of scientific publications.
-        Russian authors often provide English translations for their abstracts and titles,
-        and the data consists of these paired titles and abstracts. The task evaluates a model's ability
-        to match an article's Russian title and abstract to its English counterpart, or vice versa.""",
+        description="This task focuses on finding translations of scientific articles. The dataset is sourced from eLibrary, Russia's largest electronic library of scientific publications. Russian authors often provide English translations for their abstracts and titles, and the data consists of these paired titles and abstracts. The task evaluates a model's ability to match an article's Russian title and abstract to its English counterpart, or vice versa.",
+        reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
+        type="BitextMining",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs={
+            "ru-en": ["rus-Cyrl", "eng-Latn"],
+            "en-ru": ["eng-Latn", "rus-Cyrl"],
+        },
+        main_score="f1",
+        date=("2007-01-01", "2023-01-01"),
+        domains=["Academic", "Non-fiction", "Written"],
+        task_subtypes=[],
+        license="not specified",
+        dialect=[],
+        sample_creation="found",
+        annotations_creators="derived",
+        bibtex_citation=r"""
+@article{vatolin2024ruscibench,
+  author = {Vatolin, A. and Gerasimenko, N. and Ianina, A. and Vorontsov, K.},
+  doi = {10.1134/S1064562424602191},
+  issn = {1531-8362},
+  journal = {Doklady Mathematics},
+  month = {12},
+  number = {1},
+  pages = {S251--S260},
+  title = {RuSciBench: Open Benchmark for Russian and English Scientific Document Representations},
+  url = {https://doi.org/10.1134/S1064562424602191},
+  volume = {110},
+  year = {2024},
+}
+""",
+        prompt="Given the following title and abstract of the scientific article, find its translation",
+        superseded_by="RuSciBenchBitextMining.v2",
+    )
+
+
+class RuSciBenchBitextMiningV2(AbsTaskBitextMining):
+    fast_loading = True
+    metadata = TaskMetadata(
+        name="RuSciBenchBitextMining.v2",
+        dataset={
+            "path": "mlsa-iai-msu-lab/ru_sci_bench_bitext_mining",
+            "revision": "20e815e8ac8787331546386dfd177821510f79a3",
+        },
+        description="This task focuses on finding translations of scientific articles. The dataset is sourced from eLibrary, Russia's largest electronic library of scientific publications. Russian authors often provide English translations for their abstracts and titles, and the data consists of these paired titles and abstracts. The task evaluates a model's ability to match an article's Russian title and abstract to its English counterpart, or vice versa. Compared to the previous version, 6 erroneous examples have been removed.",
         reference="https://github.com/mlsa-iai-msu-lab/ru_sci_bench_mteb",
         type="BitextMining",
         category="t2c",

mteb/tasks/bitext_mining/multilingual/web_faq_bitext_mining.py

@@ -198,9 +198,7 @@ _SPLITS = ["default"]
 class WebFAQBitextMiningQuestions(AbsTaskBitextMining):
     metadata = TaskMetadata(
         name="WebFAQBitextMiningQuestions",
-        description="""The WebFAQ Bitext Dataset consists of natural FAQ-style Question-Answer pairs that align across languages.
-A sentence in the "WebFAQBitextMiningQuestions" task is the question originating from an aligned QA.
-The dataset is sourced from FAQ pages on the web.""",
+        description='The WebFAQ Bitext Dataset consists of natural FAQ-style Question-Answer pairs that align across languages. A sentence in the "WebFAQBitextMiningQuestions" task is the question originating from an aligned QA. The dataset is sourced from FAQ pages on the web.',
         reference="https://huggingface.co/PaDaS-Lab",
         dataset={
             "path": "PaDaS-Lab/webfaq-bitexts",
@@ -254,9 +252,7 @@ The dataset is sourced from FAQ pages on the web.""",
 class WebFAQBitextMiningQAs(AbsTaskBitextMining):
     metadata = TaskMetadata(
         name="WebFAQBitextMiningQAs",
-        description="""The WebFAQ Bitext Dataset consists of natural FAQ-style Question-Answer pairs that align across languages.
-A sentence in the "WebFAQBitextMiningQAs" task is a concatenation of a question and its corresponding answer.
-The dataset is sourced from FAQ pages on the web.""",
+        description='The WebFAQ Bitext Dataset consists of natural FAQ-style Question-Answer pairs that align across languages. A sentence in the "WebFAQBitextMiningQAs" task is a concatenation of a question and its corresponding answer. The dataset is sourced from FAQ pages on the web.',
         reference="https://huggingface.co/PaDaS-Lab",
         dataset={
             "path": "PaDaS-Lab/webfaq-bitexts",

mteb/tasks/classification/ara/ajgt.py

@@ -45,8 +45,7 @@ class AJGTV2(AbsTaskClassification):
             "path": "mteb/ajgt",
             "revision": "0a3dea7301ee0c051891f04d32f3e8577a9eae36",
         },
-        description="""Arabic Jordanian General Tweets (AJGT) Corpus consisted of 1,800 tweets (900 for training and 900 for testing) annotated as positive and negative. Modern Standard Arabic (MSA) or Jordanian dialect.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="Arabic Jordanian General Tweets (AJGT) Corpus consisted of 1,800 tweets (900 for training and 900 for testing) annotated as positive and negative. Modern Standard Arabic (MSA) or Jordanian dialect. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://link.springer.com/chapter/10.1007/978-3-319-60042-0_66/",
         type="Classification",
         category="t2c",

mteb/tasks/classification/ara/hotel_review_sentiment_classification.py

@@ -45,8 +45,7 @@ class HotelReviewSentimentClassificationV2(AbsTaskClassification):
             "path": "mteb/HotelReviewSentimentClassification",
             "revision": "f5e6a24acbed4182114ffdf46747090b3f51e836",
         },
-        description="""HARD is a dataset of Arabic hotel reviews collected from the Booking.com website.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="HARD is a dataset of Arabic hotel reviews collected from the Booking.com website. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://link.springer.com/chapter/10.1007/978-3-319-67056-0_3",
         type="Classification",
         category="t2c",

mteb/tasks/classification/ara/online_store_review_sentiment_classification.py

@@ -41,8 +41,7 @@ class OnlineStoreReviewSentimentClassificationV2(AbsTaskClassification):
             "path": "mteb/online_store_review_sentiment",
             "revision": "de0e8eed65adf1cbc58f8743a5f5c5df556de4c4",
         },
-        description="""This dataset contains Arabic reviews of products from the SHEIN online store.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="This dataset contains Arabic reviews of products from the SHEIN online store. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://huggingface.co/datasets/Ruqiya/Arabic_Reviews_of_SHEIN",
         type="Classification",
         category="t2c",

mteb/tasks/classification/ara/restaurant_review_sentiment_classification.py

@@ -52,8 +52,7 @@ class RestaurantReviewSentimentClassificationV2(AbsTaskClassification):
             "path": "mteb/restaurant_review_sentiment",
             "revision": "5d28c1e8fb393173a849696ed178b90a6f78754a",
         },
-        description="""Dataset of 8156 restaurant reviews from qaym.com in Arabic for sentiment analysis
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="Dataset of 8156 restaurant reviews from qaym.com in Arabic for sentiment analysis This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://link.springer.com/chapter/10.1007/978-3-319-18117-2_2",
         type="Classification",
         category="t2c",

mteb/tasks/classification/ara/tweet_emotion_classification.py

@@ -45,8 +45,7 @@ class TweetEmotionClassificationV2(AbsTaskClassification):
             "path": "mteb/TweetEmotionClassification",
             "revision": "930d65840c089406ceed5241b1a9ba7294e5eeae",
         },
-        description="""A dataset of 10,012 tweets that was created with the aim of covering the most frequently used emotion categories in Arabic tweets.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="A dataset of 10,012 tweets that was created with the aim of covering the most frequently used emotion categories in Arabic tweets. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://link.springer.com/chapter/10.1007/978-3-319-77116-8_8",
         type="Classification",
         category="t2c",

mteb/tasks/classification/ara/tweet_sarcasm_classification.py

@@ -62,8 +62,7 @@ class TweetSarcasmClassificationV2(AbsTaskClassification):
             "path": "mteb/tweet_sarcasm",
             "revision": "3a20898e2ea3303844e907d55f7a815a7644150d",
         },
-        description="""Arabic sarcasm detection dataset, which was created through the reannotation of available Arabic sentiment analysis datasets.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="Arabic sarcasm detection dataset, which was created through the reannotation of available Arabic sentiment analysis datasets. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://aclanthology.org/2020.osact-1.5/",
         type="Classification",
         category="t2c",

mteb/tasks/classification/ben/bengali_document_classification.py

@@ -55,8 +55,7 @@ Islam, Tanvir},
 class BengaliDocumentClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="BengaliDocumentClassification.v2",
-        description="""Dataset for News Classification, categorized with 13 domains.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="Dataset for News Classification, categorized with 13 domains. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://aclanthology.org/2023.eacl-main.4",
         dataset={
             "path": "mteb/bengali_document",

mteb/tasks/classification/ben/bengali_hate_speech_classification.py

@@ -45,8 +45,7 @@ class BengaliHateSpeechClassification(AbsTaskClassification):
 class BengaliHateSpeechClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="BengaliHateSpeechClassification.v2",
-        description="""The Bengali Hate Speech Dataset is a Bengali-language dataset of news articles collected from various Bengali media sources and categorized based on the type of hate in the text.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="The Bengali Hate Speech Dataset is a Bengali-language dataset of news articles collected from various Bengali media sources and categorized based on the type of hate in the text. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://huggingface.co/datasets/bn_hate_speech",
         dataset={
             "path": "mteb/bengali_hate_speech",

mteb/tasks/classification/ben/bengali_sentiment_analysis.py

@@ -45,8 +45,7 @@ class BengaliSentimentAnalysis(AbsTaskClassification):
 class BengaliSentimentAnalysisV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="BengaliSentimentAnalysis.v2",
-        description="""dataset contains 2854 Negative reviews and 7238 Positive reviews collected and manually annotated from Youtube Bengali drama.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)""",
+        description="dataset contains 2854 Negative reviews and 7238 Positive reviews collected and manually annotated from Youtube Bengali drama. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2632)",
         reference="https://data.mendeley.com/datasets/p6zc7krs37/4",
         dataset={
             "path": "mteb/bengali_sentiment_analysis",

mteb/tasks/classification/ces/csfdcz_movie_review_sentiment_classification.py

@@ -51,8 +51,7 @@ class CSFDCZMovieReviewSentimentClassification(AbsTaskClassification):
 class CSFDCZMovieReviewSentimentClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="CSFDCZMovieReviewSentimentClassification.v2",
-        description="""The dataset contains 30k user reviews from csfd.cz in Czech.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="The dataset contains 30k user reviews from csfd.cz in Czech. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://arxiv.org/abs/2304.01922",
         dataset={
             "path": "mteb/csfdcz_movie_review_sentiment",

mteb/tasks/classification/ces/czech_product_review_sentiment_classification.py

@@ -58,8 +58,7 @@ Montoyo, Andres},
 class CzechProductReviewSentimentClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="CzechProductReviewSentimentClassification.v2",
-        description="""User reviews of products on Czech e-shop Mall.cz with 3 sentiment classes (positive, neutral, negative)
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="User reviews of products on Czech e-shop Mall.cz with 3 sentiment classes (positive, neutral, negative) This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://aclanthology.org/W13-1609/",
         dataset={
             "path": "mteb/czech_product_review_sentiment",

mteb/tasks/classification/ces/czech_so_me_sentiment_classification.py

@@ -55,8 +55,7 @@ Montoyo, Andres},
 class CzechSoMeSentimentClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="CzechSoMeSentimentClassification.v2",
-        description="""User comments on Facebook
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="User comments on Facebook This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://aclanthology.org/W13-1609/",
         dataset={
             "path": "mteb/czech_so_me_sentiment",

mteb/tasks/classification/dan/angry_tweets_classification.py

@@ -47,8 +47,7 @@ class AngryTweetsClassificationV2(AbsTaskClassification):
             "path": "mteb/angry_tweets",
             "revision": "b9475fb66a13befda4fa9871cd92343bb2c0eb77",
         },
-        description="""A sentiment dataset with 3 classes (positive, negative, neutral) for Danish tweets
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="A sentiment dataset with 3 classes (positive, negative, neutral) for Danish tweets This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://aclanthology.org/2021.nodalida-main.53/",
         type="Classification",
         category="t2c",

mteb/tasks/classification/dan/danish_political_comments_classification.py

@@ -49,8 +49,7 @@ class DanishPoliticalCommentsClassificationV2(AbsTaskClassification):
             "path": "mteb/danish_political_comments",
             "revision": "476a9e7327aba70ad3e97a169d7310b86be9b245",
         },
-        description="""A dataset of Danish political comments rated for sentiment
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="A dataset of Danish political comments rated for sentiment This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://huggingface.co/datasets/danish_political_comments",
         type="Classification",
         category="t2c",

mteb/tasks/classification/dan/ddisco_cohesion_classification.py

@@ -69,8 +69,7 @@ class DdiscoCohesionClassificationV2(AbsTaskClassification):
             "path": "mteb/ddisco_cohesion",
             "revision": "b5a05bdecdfc6efc14eebc8f7a86e0986edaf5ff",
         },
-        description="""A Danish Discourse dataset with values for coherence and source (Wikipedia or Reddit)
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="A Danish Discourse dataset with values for coherence and source (Wikipedia or Reddit) This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://aclanthology.org/2022.lrec-1.260/",
         type="Classification",
         category="t2c",

mteb/tasks/classification/dan/dk_hate_classification.py

@@ -62,7 +62,7 @@ Piperidis, Stelios},
 
     def dataset_transform(self):
         # convert label to a 0/1 label
-        labels = self.dataset["train"]["label"]  # type: ignore
+        labels = self.dataset["train"]["label"]
         lab2idx = {lab: idx for idx, lab in enumerate(set(labels))}
         self.dataset = self.dataset.map(
             lambda x: {"label": lab2idx[x["label"]]}, remove_columns=["label"]
@@ -76,8 +76,7 @@ class DKHateClassificationV2(AbsTaskClassification):
             "path": "mteb/dk_hate",
             "revision": "0468ff11393992d8347cf4282fb706fe970608d4",
         },
-        description="""Danish Tweets annotated for Hate Speech either being Offensive or not
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="Danish Tweets annotated for Hate Speech either being Offensive or not This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://aclanthology.org/2020.lrec-1.430/",
         type="Classification",
         category="t2c",

mteb/tasks/classification/deu/german_politicians_twitter_sentiment_classification.py

@@ -56,8 +56,7 @@ Zesch, Torsten},
 class GermanPoliticiansTwitterSentimentClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="GermanPoliticiansTwitterSentimentClassification.v2",
-        description="""GermanPoliticiansTwitterSentiment is a dataset of German tweets categorized with their sentiment (3 classes).
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="GermanPoliticiansTwitterSentiment is a dataset of German tweets categorized with their sentiment (3 classes). This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://aclanthology.org/2022.konvens-1.9",
         dataset={
             "path": "mteb/german_politicians_twitter_sentiment",

mteb/tasks/classification/deu/ten_k_gnad_classification.py

@@ -43,8 +43,7 @@ class TenKGnadClassification(AbsTaskClassification):
 class TenKGnadClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="TenKGnadClassification.v2",
-        description="""10k German News Articles Dataset (10kGNAD) contains news articles from the online Austrian newspaper website DER Standard with their topic classification (9 classes).
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="10k German News Articles Dataset (10kGNAD) contains news articles from the online Austrian newspaper website DER Standard with their topic classification (9 classes). This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://tblock.github.io/10kGNAD/",
         dataset={
             "path": "mteb/ten_k_gnad",

mteb/tasks/classification/eng/amazon_polarity_classification.py

@@ -44,8 +44,7 @@ class AmazonPolarityClassification(AbsTaskClassification):
 class AmazonPolarityClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="AmazonPolarityClassification.v2",
-        description="""Amazon Polarity Classification Dataset.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="Amazon Polarity Classification Dataset. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://huggingface.co/datasets/amazon_polarity",
         dataset={
             "path": "mteb/amazon_polarity",

mteb/tasks/classification/eng/arxiv_classification.py

@@ -43,8 +43,7 @@ class ArxivClassification(AbsTaskClassification):
 class ArxivClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="ArxivClassification.v2",
-        description="""Classification Dataset of Arxiv Papers
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="Classification Dataset of Arxiv Papers This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         dataset={
             "path": "mteb/arxiv",
             "revision": "202e10e9a5d37a5068397b48184d0728346a7b4a",

mteb/tasks/classification/eng/banking77_classification.py

@@ -61,8 +61,7 @@ Shah, Rushin},
 class Banking77ClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="Banking77Classification.v2",
-        description="""Dataset composed of online banking queries annotated with their corresponding intents.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="Dataset composed of online banking queries annotated with their corresponding intents. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://arxiv.org/abs/2003.04807",
         dataset={
             "path": "mteb/banking77",

mteb/tasks/classification/eng/dbpedia_classification.py

@@ -50,8 +50,7 @@ class DBpediaClassification(AbsTaskClassification):
 class DBpediaClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="DBpediaClassification.v2",
-        description="""DBpedia14 is a dataset of English texts from Wikipedia articles, categorized into 14 non-overlapping classes based on their DBpedia ontology.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="DBpedia14 is a dataset of English texts from Wikipedia articles, categorized into 14 non-overlapping classes based on their DBpedia ontology. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://arxiv.org/abs/1509.01626",
         dataset={
             "path": "mteb/d_bpedia",

mteb/tasks/classification/eng/emotion_classification.py

@@ -59,8 +59,7 @@ Tsujii, Jun{'}ichi},
 class EmotionClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="EmotionClassification.v2",
-        description="""Emotion is a dataset of English Twitter messages with six basic emotions: anger, fear, joy, love, sadness, and surprise.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="Emotion is a dataset of English Twitter messages with six basic emotions: anger, fear, joy, love, sadness, and surprise. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://www.aclweb.org/anthology/D18-1404",
         dataset={
             "path": "mteb/emotion",

mteb/tasks/classification/eng/financial_phrasebank_classification.py

@@ -40,8 +40,7 @@ class FinancialPhrasebankClassification(AbsTaskClassification):
 class FinancialPhrasebankClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="FinancialPhrasebankClassification.v2",
-        description="""Polar sentiment dataset of sentences from financial news, categorized by sentiment into positive, negative, or neutral.
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="Polar sentiment dataset of sentences from financial news, categorized by sentiment into positive, negative, or neutral. This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         reference="https://arxiv.org/abs/1307.5336",
         dataset={
             "path": "mteb/financial_phrasebank",

mteb/tasks/classification/eng/frenk_en_classification.py

@@ -42,8 +42,7 @@ class FrenkEnClassification(AbsTaskClassification):
 class FrenkEnClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="FrenkEnClassification.v2",
-        description="""English subset of the FRENK dataset
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="English subset of the FRENK dataset This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         dataset={
             "path": "mteb/frenk_en",
             "revision": "630d941b6e0879a7238da89af6bfe1b1eb27ca0f",

mteb/tasks/classification/eng/gtsrb_classification.py

@@ -10,7 +10,7 @@ class GTSRBClassification(AbsTaskClassification):
 
     metadata = TaskMetadata(
         name="GTSRB",
-        description="""The German Traffic Sign Recognition Benchmark (GTSRB) is a multi-class classification dataset for traffic signs. It consists of dataset of more than 50,000 traffic sign images. The dataset comprises 43 classes with unbalanced class frequencies.""",
+        description="The German Traffic Sign Recognition Benchmark (GTSRB) is a multi-class classification dataset for traffic signs. It consists of dataset of more than 50,000 traffic sign images. The dataset comprises 43 classes with unbalanced class frequencies.",
         reference="https://benchmark.ini.rub.de/",
         dataset={
             "path": "clip-benchmark/wds_gtsrb",

mteb/tasks/classification/eng/imdb_classification.py

@@ -55,8 +55,7 @@ Mihalcea, Rada},
 class ImdbClassificationV2(AbsTaskClassification):
     metadata = TaskMetadata(
         name="ImdbClassification.v2",
-        description="""Large Movie Review Dataset
-        This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)""",
+        description="Large Movie Review Dataset This version corrects errors found in the original data. For details, see [pull request](https://github.com/embeddings-benchmark/mteb/pull/2900)",
         dataset={
             "path": "mteb/imdb",
             "revision": "d05f0155defa7991dad75bc68c5ccb6774b1fdc5",