evalscope 0.7.2__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/backend/rag_eval/cmteb/tasks/Classification.py

@@ -4,19 +4,19 @@ from mteb.abstasks.TaskMetadata import TaskMetadata
 
 class TNews(AbsTaskClassification):
     metadata = TaskMetadata(
-        name="TNews",
-        description="Short Text Classification for News",
-        reference="https://www.cluebenchmarks.com/introduce.html",
+        name='TNews',
+        description='Short Text Classification for News',
+        reference='https://www.cluebenchmarks.com/introduce.html',
         dataset={
-            "path": "C-MTEB/TNews-classification",
-            "revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
+            'path': 'C-MTEB/TNews-classification',
+            'revision': '317f262bf1e6126357bbe89e875451e4b0938fe4',
         },
-        type="Classification",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation"],
-        eval_langs=["cmn-Hans"],
-        main_score="accuracy",
+        type='Classification',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['validation'],
+        eval_langs=['cmn-Hans'],
+        main_score='accuracy',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -67,31 +67,34 @@ class TNews(AbsTaskClassification):
  doi = "10.18653/v1/2020.coling-main.419",
  pages = "4762--4772",
 }""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
     @property
     def metadata_dict(self) -> dict[str, str]:
         metadata_dict = super().metadata_dict
-        metadata_dict["samples_per_label"] = 32
+        metadata_dict['samples_per_label'] = 32
         return metadata_dict
 
 
 class IFlyTek(AbsTaskClassification):
     metadata = TaskMetadata(
-        name="IFlyTek",
-        description="Long Text classification for the description of Apps",
-        reference="https://www.cluebenchmarks.com/introduce.html",
+        name='IFlyTek',
+        description='Long Text classification for the description of Apps',
+        reference='https://www.cluebenchmarks.com/introduce.html',
         dataset={
-            "path": "C-MTEB/IFlyTek-classification",
-            "revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
+            'path': 'C-MTEB/IFlyTek-classification',
+            'revision': '421605374b29664c5fc098418fe20ada9bd55f8a',
         },
-        type="Classification",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation"],
-        eval_langs=["cmn-Hans"],
-        main_score="accuracy",
+        type='Classification',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['validation'],
+        eval_langs=['cmn-Hans'],
+        main_score='accuracy',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -143,32 +146,36 @@ class IFlyTek(AbsTaskClassification):
  pages = "4762--4772",
  abstract = "The advent of natural language understanding (NLU) benchmarks for English, such as GLUE and SuperGLUE allows new NLU models to be evaluated across a diverse set of tasks. These comprehensive benchmarks have facilitated a broad range of research and applications in natural language processing (NLP). The problem, however, is that most such benchmarks are limited to English, which has made it difficult to replicate many of the successes in English NLU for other languages. To help remedy this issue, we introduce the first large-scale Chinese Language Understanding Evaluation (CLUE) benchmark. CLUE is an open-ended, community-driven project that brings together 9 tasks spanning several well-established single-sentence/sentence-pair classification tasks, as well as machine reading comprehension, all on original Chinese text. To establish results on these tasks, we report scores using an exhaustive set of current state-of-the-art pre-trained Chinese models (9 in total). We also introduce a number of supplementary datasets and additional tools to help facilitate further progress on Chinese NLU. Our benchmark is released at https://www.cluebenchmarks.com",
 }""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
     @property
     def metadata_dict(self) -> dict[str, str]:
         metadata_dict = super().metadata_dict
-        metadata_dict["samples_per_label"] = 32
-        metadata_dict["n_experiments"] = 5
+        metadata_dict['samples_per_label'] = 32
+        metadata_dict['n_experiments'] = 5
         return metadata_dict
 
 
 class MultilingualSentiment(AbsTaskClassification):
     metadata = TaskMetadata(
-        name="MultilingualSentiment",
-        description="A collection of multilingual sentiments datasets grouped into 3 classes -- positive, neutral, negative",
-        reference="https://github.com/tyqiangz/multilingual-sentiment-datasets",
+        name='MultilingualSentiment',
+        description=
+        'A collection of multilingual sentiments datasets grouped into 3 classes -- positive, neutral, negative',
+        reference='https://github.com/tyqiangz/multilingual-sentiment-datasets',
         dataset={
-            "path": "C-MTEB/MultilingualSentiment-classification",
-            "revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
+            'path': 'C-MTEB/MultilingualSentiment-classification',
+            'revision': '46958b007a63fdbf239b7672c25d0bea67b5ea1a',
         },
-        type="Classification",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation", "test"],
-        eval_langs=["cmn-Hans"],
-        main_score="accuracy",
+        type='Classification',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['validation', 'test'],
+        eval_langs=['cmn-Hans'],
+        main_score='accuracy',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -177,31 +184,34 @@ class MultilingualSentiment(AbsTaskClassification):
         dialect=None,
         sample_creation=None,
         bibtex_citation=None,
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
     @property
     def metadata_dict(self) -> dict[str, str]:
         metadata_dict = super().metadata_dict
-        metadata_dict["samples_per_label"] = 32
+        metadata_dict['samples_per_label'] = 32
         return metadata_dict
 
 
 class JDReview(AbsTaskClassification):
     metadata = TaskMetadata(
-        name="JDReview",
-        description="review for iphone",
-        reference="https://aclanthology.org/2023.nodalida-1.20/",
+        name='JDReview',
+        description='review for iphone',
+        reference='https://aclanthology.org/2023.nodalida-1.20/',
         dataset={
-            "path": "C-MTEB/JDReview-classification",
-            "revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
+            'path': 'C-MTEB/JDReview-classification',
+            'revision': 'b7c64bd89eb87f8ded463478346f76731f07bf8b',
         },
-        type="Classification",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="accuracy",
+        type='Classification',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='accuracy',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -215,31 +225,34 @@ class JDReview(AbsTaskClassification):
   journal={arXiv preprint arXiv:2309.07597},
   year={2023}
 }""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
     @property
     def metadata_dict(self) -> dict[str, str]:
         metadata_dict = super().metadata_dict
-        metadata_dict["samples_per_label"] = 32
+        metadata_dict['samples_per_label'] = 32
         return metadata_dict
 
 
 class OnlineShopping(AbsTaskClassification):
     metadata = TaskMetadata(
-        name="OnlineShopping",
-        description="Sentiment Analysis of User Reviews on Online Shopping Websites",
-        reference="https://aclanthology.org/2023.nodalida-1.20/",
+        name='OnlineShopping',
+        description='Sentiment Analysis of User Reviews on Online Shopping Websites',
+        reference='https://aclanthology.org/2023.nodalida-1.20/',
         dataset={
-            "path": "C-MTEB/OnlineShopping-classification",
-            "revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
+            'path': 'C-MTEB/OnlineShopping-classification',
+            'revision': 'e610f2ebd179a8fda30ae534c3878750a96db120',
         },
-        type="Classification",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="accuracy",
+        type='Classification',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='accuracy',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -253,31 +266,34 @@ class OnlineShopping(AbsTaskClassification):
   journal={arXiv preprint arXiv:2309.07597},
   year={2023}
 }""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
     @property
     def metadata_dict(self) -> dict[str, str]:
         metadata_dict = super().metadata_dict
-        metadata_dict["samples_per_label"] = 32
+        metadata_dict['samples_per_label'] = 32
         return metadata_dict
 
 
 class Waimai(AbsTaskClassification):
     metadata = TaskMetadata(
-        name="Waimai",
-        description="Sentiment Analysis of user reviews on takeaway platforms",
-        reference="https://aclanthology.org/2023.nodalida-1.20/",
+        name='Waimai',
+        description='Sentiment Analysis of user reviews on takeaway platforms',
+        reference='https://aclanthology.org/2023.nodalida-1.20/',
         dataset={
-            "path": "C-MTEB/waimai-classification",
-            "revision": "339287def212450dcaa9df8c22bf93e9980c7023",
+            'path': 'C-MTEB/waimai-classification',
+            'revision': '339287def212450dcaa9df8c22bf93e9980c7023',
         },
-        type="Classification",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="accuracy",
+        type='Classification',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='accuracy',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -291,12 +307,15 @@ class Waimai(AbsTaskClassification):
   journal={arXiv preprint arXiv:2309.07597},
   year={2023}
 }""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
     @property
     def metadata_dict(self) -> dict[str, str]:
         metadata_dict = super().metadata_dict
-        metadata_dict["samples_per_label"] = 32
+        metadata_dict['samples_per_label'] = 32
 
         return metadata_dict

evalscope/backend/rag_eval/cmteb/tasks/Clustering.py

@@ -1,12 +1,7 @@
 import itertools
-
 from datasets import Dataset, DatasetDict
-
 from mteb.abstasks.AbsTaskClustering import AbsTaskClustering
-from mteb.abstasks.AbsTaskClusteringFast import (
-    AbsTaskClusteringFast,
-    check_label_distribution,
-)
+from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast, check_label_distribution
 from mteb.abstasks.TaskMetadata import TaskMetadata
 
 NUM_SAMPLES = 2048
@@ -46,7 +41,9 @@ class CLSClusteringFastS2S(AbsTaskClusteringFast):
             primaryClass={cs.CL}
         }""",  # noqa
         descriptive_stats={
-            'n_samples': {'test': NUM_SAMPLES},
+            'n_samples': {
+                'test': NUM_SAMPLES
+            },
             'avg_character_length': {},
         },
     )
@@ -55,9 +52,7 @@ class CLSClusteringFastS2S(AbsTaskClusteringFast):
         ds = {}
         for split in self.metadata.eval_splits:
             labels = list(itertools.chain.from_iterable(self.dataset[split]['labels']))
-            sentences = list(
-                itertools.chain.from_iterable(self.dataset[split]['sentences'])
-            )
+            sentences = list(itertools.chain.from_iterable(self.dataset[split]['sentences']))
 
             check_label_distribution(self.dataset[split])
 
@@ -106,7 +101,9 @@ class CLSClusteringFastP2P(AbsTaskClusteringFast):
             primaryClass={cs.CL}
         }""",  # noqa
         descriptive_stats={
-            'n_samples': {'test': NUM_SAMPLES},
+            'n_samples': {
+                'test': NUM_SAMPLES
+            },
             'avg_character_length': {},
         },
     )
@@ -115,9 +112,7 @@ class CLSClusteringFastP2P(AbsTaskClusteringFast):
         ds = {}
         for split in self.metadata.eval_splits:
             labels = list(itertools.chain.from_iterable(self.dataset[split]['labels']))
-            sentences = list(
-                itertools.chain.from_iterable(self.dataset[split]['sentences'])
-            )
+            sentences = list(itertools.chain.from_iterable(self.dataset[split]['sentences']))
 
             check_label_distribution(self.dataset[split])
 
@@ -166,7 +161,9 @@ class ThuNewsClusteringFastS2S(AbsTaskClusteringFast):
   url = {https://github.com/thunlp/THUCTC}
 }""",
         descriptive_stats={
-            'n_samples': {'test': NUM_SAMPLES},
+            'n_samples': {
+                'test': NUM_SAMPLES
+            },
             'avg_character_length': {},
         },
     )
@@ -175,9 +172,7 @@ class ThuNewsClusteringFastS2S(AbsTaskClusteringFast):
         ds = {}
         for split in self.metadata.eval_splits:
             labels = list(itertools.chain.from_iterable(self.dataset[split]['labels']))
-            sentences = list(
-                itertools.chain.from_iterable(self.dataset[split]['sentences'])
-            )
+            sentences = list(itertools.chain.from_iterable(self.dataset[split]['sentences']))
 
             check_label_distribution(self.dataset[split])
 
@@ -226,7 +221,9 @@ class ThuNewsClusteringFastP2P(AbsTaskClusteringFast):
   url = {https://github.com/thunlp/THUCTC}
 }""",
         descriptive_stats={
-            'n_samples': {'test': NUM_SAMPLES},
+            'n_samples': {
+                'test': NUM_SAMPLES
+            },
             'avg_character_length': {},
         },
     )
@@ -235,9 +232,7 @@ class ThuNewsClusteringFastP2P(AbsTaskClusteringFast):
         ds = {}
         for split in self.metadata.eval_splits:
             labels = list(itertools.chain.from_iterable(self.dataset[split]['labels']))
-            sentences = list(
-                itertools.chain.from_iterable(self.dataset[split]['sentences'])
-            )
+            sentences = list(itertools.chain.from_iterable(self.dataset[split]['sentences']))
 
             check_label_distribution(self.dataset[split])
 

evalscope/backend/rag_eval/cmteb/tasks/CustomTask.py

@@ -1,31 +1,29 @@
-from typing import Optional
 from mteb import AbsTaskRetrieval
 from mteb import HFDataLoader as CustomDataLoader
 from mteb.abstasks.TaskMetadata import TaskMetadata
+from typing import Optional
 
 
 class CustomRetrieval(AbsTaskRetrieval):
     metadata: TaskMetadata
     ignore_identical_ids: bool = True
 
-    def __init__(
-        self, dataset_path: Optional[str] = "custom_eval/text/retrieval", **kwargs
-    ):
+    def __init__(self, dataset_path: Optional[str] = 'custom_eval/text/retrieval', **kwargs):
         super().__init__(**kwargs)
         self.metadata = TaskMetadata(
-            name="CustomRetrieval",
-            description="CustomRetrieval Task",
+            name='CustomRetrieval',
+            description='CustomRetrieval Task',
             reference=None,
             dataset={
-                "path": dataset_path,
-                "revision": "v1",
+                'path': dataset_path,
+                'revision': 'v1',
             },
-            type="Retrieval",
-            category="s2p",
-            modalities=["text"],
-            eval_splits=["test"],
-            eval_langs=["cmn-Hans"],
-            main_score="recall_at_5",
+            type='Retrieval',
+            category='s2p',
+            modalities=['text'],
+            eval_splits=['test'],
+            eval_langs=['cmn-Hans'],
+            main_score='recall_at_5',
             date=None,
             domains=None,
             task_subtypes=None,
@@ -33,7 +31,7 @@ class CustomRetrieval(AbsTaskRetrieval):
             annotations_creators=None,
             dialect=None,
             sample_creation=None,
-            bibtex_citation="",
+            bibtex_citation='',
             descriptive_stats={},
         )
 
@@ -41,17 +39,17 @@ class CustomRetrieval(AbsTaskRetrieval):
         if self.data_loaded:
             return
         self.corpus, self.queries, self.relevant_docs = {}, {}, {}
-        dataset_path = self.metadata_dict["dataset"]["path"]
+        dataset_path = self.metadata_dict['dataset']['path']
 
-        for split in kwargs.get("eval_splits", self.metadata_dict["eval_splits"]):
+        for split in kwargs.get('eval_splits', self.metadata_dict['eval_splits']):
             corpus, queries, qrels = CustomDataLoader(
                 data_folder=dataset_path,
                 streaming=False,
                 keep_in_memory=False,
             ).load(split=split)
             # Conversion from DataSet
-            queries = {query["id"]: query["text"] for query in queries}
-            corpus = {doc["id"]: {"text": doc["text"]} for doc in corpus}
+            queries = {query['id']: query['text'] for query in queries}
+            corpus = {doc['id']: {'text': doc['text']} for doc in corpus}
             self.corpus[split], self.queries[split], self.relevant_docs[split] = (
                 corpus,
                 queries,

evalscope/backend/rag_eval/cmteb/tasks/PairClassification.py

@@ -4,19 +4,19 @@ from mteb.abstasks.TaskMetadata import TaskMetadata
 
 class Ocnli(AbsTaskPairClassification):
     metadata = TaskMetadata(
-        name="Ocnli",
-        description="Original Chinese Natural Language Inference dataset",
-        reference="https://arxiv.org/abs/2010.05444",
+        name='Ocnli',
+        description='Original Chinese Natural Language Inference dataset',
+        reference='https://arxiv.org/abs/2010.05444',
         dataset={
-            "path": "C-MTEB/OCNLI",
-            "revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
+            'path': 'C-MTEB/OCNLI',
+            'revision': '66e76a618a34d6d565d5538088562851e6daa7ec',
         },
-        type="PairClassification",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation"],
-        eval_langs=["cmn-Hans"],
-        main_score="max_accuracy",
+        type='PairClassification',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['validation'],
+        eval_langs=['cmn-Hans'],
+        main_score='max_accuracy',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -25,36 +25,39 @@ class Ocnli(AbsTaskPairClassification):
         dialect=None,
         sample_creation=None,
         bibtex_citation="""@misc{hu2020ocnli,
-            title={OCNLI: Original Chinese Natural Language Inference}, 
+            title={OCNLI: Original Chinese Natural Language Inference},
             author={Hai Hu and Kyle Richardson and Liang Xu and Lu Li and Sandra Kuebler and Lawrence S. Moss},
             year={2020},
             eprint={2010.05444},
             archivePrefix={arXiv},
             primaryClass={cs.CL}
         }""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
     def dataset_transform(self):
-        self.dataset = self.dataset.rename_column("sent1", "sentence1")
-        self.dataset = self.dataset.rename_column("sent2", "sentence2")
+        self.dataset = self.dataset.rename_column('sent1', 'sentence1')
+        self.dataset = self.dataset.rename_column('sent2', 'sentence2')
 
 
 class Cmnli(AbsTaskPairClassification):
     metadata = TaskMetadata(
-        name="Cmnli",
-        description="Chinese Multi-Genre NLI",
-        reference="https://huggingface.co/datasets/clue/viewer/cmnli",
+        name='Cmnli',
+        description='Chinese Multi-Genre NLI',
+        reference='https://huggingface.co/datasets/clue/viewer/cmnli',
         dataset={
-            "path": "C-MTEB/CMNLI",
-            "revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
+            'path': 'C-MTEB/CMNLI',
+            'revision': '41bc36f332156f7adc9e38f53777c959b2ae9766',
         },
-        type="PairClassification",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["validation", "test"],
-        eval_langs=["cmn-Hans"],
-        main_score="max_accuracy",
+        type='PairClassification',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['validation', 'test'],
+        eval_langs=['cmn-Hans'],
+        main_score='max_accuracy',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -105,9 +108,12 @@ class Cmnli(AbsTaskPairClassification):
             doi = "10.18653/v1/2020.coling-main.419",
             pages = "4762--4772",
         }""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
     def dataset_transform(self):
-        self.dataset = self.dataset.rename_column("sent1", "sentence1")
-        self.dataset = self.dataset.rename_column("sent2", "sentence2")
+        self.dataset = self.dataset.rename_column('sent1', 'sentence1')
+        self.dataset = self.dataset.rename_column('sent2', 'sentence2')

evalscope/backend/rag_eval/cmteb/tasks/Reranking.py

@@ -33,7 +33,10 @@ class T2Reranking(AbsTaskReranking):
       archivePrefix={arXiv},
       primaryClass={cs.IR}
 }""",  # noqa
-        descriptive_stats={'n_samples': None, 'avg_character_length': None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
 
@@ -68,7 +71,10 @@ class MMarcoReranking(AbsTaskReranking):
       archivePrefix={arXiv},
       primaryClass={cs.CL}
 }""",  # noqa
-        descriptive_stats={'n_samples': None, 'avg_character_length': None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )
 
 
@@ -105,8 +111,12 @@ class CMedQAv1(AbsTaskReranking):
   publisher={Multidisciplinary Digital Publishing Institute}
 }""",
         descriptive_stats={
-            'n_samples': {'test': 2000},
-            'avg_character_length': {'test': 165},
+            'n_samples': {
+                'test': 2000
+            },
+            'avg_character_length': {
+                'test': 165
+            },
         },
     )
 
@@ -146,5 +156,8 @@ keywords={Biomedical imaging;Data mining;Semantics;Medical services;Feature extr
 doi={10.1109/ACCESS.2018.2883637},
 ISSN={2169-3536},
 month={},}""",  # noqa
-        descriptive_stats={'n_samples': None, 'avg_character_length': None},
+        descriptive_stats={
+            'n_samples': None,
+            'avg_character_length': None
+        },
     )