evalscope 0.6.0__tar.gz → 0.6.0rc0__tar.gz

{evalscope-0.6.0 → evalscope-0.6.0rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.6.0
+Version: 0.6.0rc0
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -28,7 +28,7 @@ Requires-Dist: nltk>=3.9
 Requires-Dist: openai
 Requires-Dist: pandas
 Requires-Dist: plotly
-Requires-Dist: pyarrow
+Requires-Dist: pyarrow<=17.0.0
 Requires-Dist: pympler
 Requires-Dist: pyyaml
 Requires-Dist: regex
@@ -48,12 +48,12 @@ Requires-Dist: transformers_stream_generator
 Requires-Dist: jieba
 Requires-Dist: rouge-chinese
 Provides-Extra: opencompass
-Requires-Dist: ms-opencompass>=0.1.1; extra == "opencompass"
+Requires-Dist: ms-opencompass>=0.1.3; extra == "opencompass"
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.5; extra == "vlmeval"
 Provides-Extra: rag
-Requires-Dist: mteb>=0.14.16; extra == "rag"
-Requires-Dist: ragas<0.3,>=0.2.3; extra == "rag"
+Requires-Dist: mteb==1.19.4; extra == "rag"
+Requires-Dist: ragas==0.2.3; extra == "rag"
 Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: inner
 Requires-Dist: absl-py; extra == "inner"
@@ -96,7 +96,7 @@ Requires-Dist: nltk>=3.9; extra == "all"
 Requires-Dist: openai; extra == "all"
 Requires-Dist: pandas; extra == "all"
 Requires-Dist: plotly; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: pyarrow<=17.0.0; extra == "all"
 Requires-Dist: pympler; extra == "all"
 Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: regex; extra == "all"
@@ -115,10 +115,10 @@ Requires-Dist: transformers>=4.33; extra == "all"
 Requires-Dist: transformers_stream_generator; extra == "all"
 Requires-Dist: jieba; extra == "all"
 Requires-Dist: rouge-chinese; extra == "all"
-Requires-Dist: ms-opencompass>=0.1.1; extra == "all"
+Requires-Dist: ms-opencompass>=0.1.3; extra == "all"
 Requires-Dist: ms-vlmeval>=0.0.5; extra == "all"
-Requires-Dist: mteb>=0.14.16; extra == "all"
-Requires-Dist: ragas<0.3,>=0.2.3; extra == "all"
+Requires-Dist: mteb==1.19.4; extra == "all"
+Requires-Dist: ragas==0.2.3; extra == "all"
 Requires-Dist: webdataset>0.2.0; extra == "all"
 
 

{evalscope-0.6.0 → evalscope-0.6.0rc0}/evalscope/backend/opencompass/tasks/eval_datasets.py

@@ -50,12 +50,13 @@ with read_base():
     from opencompass.configs.datasets.nq.nq_gen_c788f6 import nq_datasets
     from opencompass.configs.datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
     from opencompass.configs.datasets.cmb.cmb_gen_dfb5c4 import cmb_datasets
+    from opencompass.configs.datasets.cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
+    from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
 
     # Note: to be supported
     # from opencompass.configs.datasets.flores.flores_gen_806ede import flores_datasets
     # from opencompass.configs.datasets.TheoremQA.TheoremQA_5shot_gen_6f0af8 import TheoremQA_datasets
     # from opencompass.configs.datasets.commonsenseqa.commonsenseqa_gen_c946f2 import commonsenseqa_datasets
-    # from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
 
 
 datasets = []

{evalscope-0.6.0 → evalscope-0.6.0rc0}/evalscope/backend/rag_eval/cmteb/tasks/Clustering.py

@@ -17,57 +17,57 @@ class CLSClusteringFastS2S(AbsTaskClusteringFast):
     max_fraction_of_documents_to_embed = None
 
     metadata = TaskMetadata(
-        name="CLSClusteringS2S",
-        description="Clustering of titles from CLS dataset. Clustering of 13 sets on the main category.",
-        reference="https://arxiv.org/abs/2209.05034",
+        name='CLSClusteringS2S',
+        description='Clustering of titles from CLS dataset. Clustering of 13 sets on the main category.',
+        reference='https://arxiv.org/abs/2209.05034',
         dataset={
-            "path": "C-MTEB/CLSClusteringS2S",
-            "revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
+            'path': 'C-MTEB/CLSClusteringS2S',
+            'revision': 'e458b3f5414b62b7f9f83499ac1f5497ae2e869f',
         },
-        type="Clustering",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="v_measure",
-        date=("2022-01-01", "2022-09-12"),
-        domains=["Academic", "Written"],
-        task_subtypes=["Thematic clustering", "Topic classification"],
-        license="Apache-2.0",
-        annotations_creators="derived",
+        type='Clustering',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='v_measure',
+        date=('2022-01-01', '2022-09-12'),
+        domains=['Academic', 'Written'],
+        task_subtypes=['Thematic clustering', 'Topic classification'],
+        license='apache-2.0',
+        annotations_creators='derived',
         dialect=[],
-        sample_creation="found",
+        sample_creation='found',
         bibtex_citation="""@misc{li2022csl,
-            title={CSL: A Large-scale Chinese Scientific Literature Dataset}, 
+            title={CSL: A Large-scale Chinese Scientific Literature Dataset},
             author={Yudong Li and Yuqing Zhang and Zhe Zhao and Linlin Shen and Weijie Liu and Weiquan Mao and Hui Zhang},
             year={2022},
             eprint={2209.05034},
             archivePrefix={arXiv},
             primaryClass={cs.CL}
-        }""",
+        }""",  # noqa
         descriptive_stats={
-            "n_samples": {"test": NUM_SAMPLES},
-            "avg_character_length": {},
+            'n_samples': {'test': NUM_SAMPLES},
+            'avg_character_length': {},
         },
     )
 
     def dataset_transform(self):
         ds = {}
         for split in self.metadata.eval_splits:
-            labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"]))
+            labels = list(itertools.chain.from_iterable(self.dataset[split]['labels']))
             sentences = list(
-                itertools.chain.from_iterable(self.dataset[split]["sentences"])
+                itertools.chain.from_iterable(self.dataset[split]['sentences'])
             )
 
             check_label_distribution(self.dataset[split])
 
-            ds[split] = Dataset.from_dict({"labels": labels, "sentences": sentences})
+            ds[split] = Dataset.from_dict({'labels': labels, 'sentences': sentences})
         self.dataset = DatasetDict(ds)
         self.dataset = self.stratified_subsampling(
             self.dataset,
             self.seed,
             self.metadata.eval_splits,
-            label="labels",
+            label='labels',
             n_samples=NUM_SAMPLES,
         )
 
@@ -77,57 +77,57 @@ class CLSClusteringFastP2P(AbsTaskClusteringFast):
     max_fraction_of_documents_to_embed = None
 
     metadata = TaskMetadata(
-        name="CLSClusteringP2P",
-        description="Clustering of titles + abstract from CLS dataset. Clustering of 13 sets on the main category.",
-        reference="https://arxiv.org/abs/2209.05034",
+        name='CLSClusteringP2P',
+        description='Clustering of titles + abstract from CLS dataset. Clustering of 13 sets on the main category.',
+        reference='https://arxiv.org/abs/2209.05034',
         dataset={
-            "path": "C-MTEB/CLSClusteringP2P",
-            "revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
+            'path': 'C-MTEB/CLSClusteringP2P',
+            'revision': '4b6227591c6c1a73bc76b1055f3b7f3588e72476',
         },
-        type="Clustering",
-        category="p2p",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="v_measure",
-        date=("2022-01-01", "2022-09-12"),
-        domains=["Academic", "Written"],
-        task_subtypes=["Thematic clustering", "Topic classification"],
-        license="Apache-2.0",
-        annotations_creators="derived",
+        type='Clustering',
+        category='p2p',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='v_measure',
+        date=('2022-01-01', '2022-09-12'),
+        domains=['Academic', 'Written'],
+        task_subtypes=['Thematic clustering', 'Topic classification'],
+        license='apache-2.0',
+        annotations_creators='derived',
         dialect=[],
-        sample_creation="found",
+        sample_creation='found',
         bibtex_citation="""@misc{li2022csl,
-            title={CSL: A Large-scale Chinese Scientific Literature Dataset}, 
+            title={CSL: A Large-scale Chinese Scientific Literature Dataset},
             author={Yudong Li and Yuqing Zhang and Zhe Zhao and Linlin Shen and Weijie Liu and Weiquan Mao and Hui Zhang},
             year={2022},
             eprint={2209.05034},
             archivePrefix={arXiv},
             primaryClass={cs.CL}
-        }""",
+        }""",  # noqa
         descriptive_stats={
-            "n_samples": {"test": NUM_SAMPLES},
-            "avg_character_length": {},
+            'n_samples': {'test': NUM_SAMPLES},
+            'avg_character_length': {},
         },
     )
 
     def dataset_transform(self):
         ds = {}
         for split in self.metadata.eval_splits:
-            labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"]))
+            labels = list(itertools.chain.from_iterable(self.dataset[split]['labels']))
             sentences = list(
-                itertools.chain.from_iterable(self.dataset[split]["sentences"])
+                itertools.chain.from_iterable(self.dataset[split]['sentences'])
             )
 
             check_label_distribution(self.dataset[split])
 
-            ds[split] = Dataset.from_dict({"labels": labels, "sentences": sentences})
+            ds[split] = Dataset.from_dict({'labels': labels, 'sentences': sentences})
         self.dataset = DatasetDict(ds)
         self.dataset = self.stratified_subsampling(
             self.dataset,
             self.seed,
             self.metadata.eval_splits,
-            label="labels",
+            label='labels',
             n_samples=NUM_SAMPLES,
         )
 
@@ -137,26 +137,26 @@ class ThuNewsClusteringFastS2S(AbsTaskClusteringFast):
     max_fraction_of_documents_to_embed = None
 
     metadata = TaskMetadata(
-        name="ThuNewsClusteringS2S",
+        name='ThuNewsClusteringS2S',
         dataset={
-            "path": "C-MTEB/ThuNewsClusteringS2S",
-            "revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
+            'path': 'C-MTEB/ThuNewsClusteringS2S',
+            'revision': '8a8b2caeda43f39e13c4bc5bea0f8a667896e10d',
         },
-        description="Clustering of titles from the THUCNews dataset",
-        reference="http://thuctc.thunlp.org/",
-        type="Clustering",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="v_measure",
-        date=("2006-01-01", "2007-01-01"),
-        domains=["News", "Written"],
-        task_subtypes=["Thematic clustering", "Topic classification"],
-        license="Not specified",
-        annotations_creators="derived",
+        description='Clustering of titles from the THUCNews dataset',
+        reference='http://thuctc.thunlp.org/',
+        type='Clustering',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='v_measure',
+        date=('2006-01-01', '2007-01-01'),
+        domains=['News', 'Written'],
+        task_subtypes=['Thematic clustering', 'Topic classification'],
+        license='apache-2.0',
+        annotations_creators='derived',
         dialect=[],
-        sample_creation="found",
+        sample_creation='found',
         bibtex_citation="""@software{THUCTC,
   author = {Sun, M. and Li, J. and Guo, Z. and Yu, Z. and Zheng, Y. and Si, X. and Liu, Z.},
   title = {THUCTC: An Efficient Chinese Text Classifier},
@@ -166,28 +166,28 @@ class ThuNewsClusteringFastS2S(AbsTaskClusteringFast):
   url = {https://github.com/thunlp/THUCTC}
 }""",
         descriptive_stats={
-            "n_samples": {"test": NUM_SAMPLES},
-            "avg_character_length": {},
+            'n_samples': {'test': NUM_SAMPLES},
+            'avg_character_length': {},
         },
     )
 
     def dataset_transform(self):
         ds = {}
         for split in self.metadata.eval_splits:
-            labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"]))
+            labels = list(itertools.chain.from_iterable(self.dataset[split]['labels']))
             sentences = list(
-                itertools.chain.from_iterable(self.dataset[split]["sentences"])
+                itertools.chain.from_iterable(self.dataset[split]['sentences'])
             )
 
             check_label_distribution(self.dataset[split])
 
-            ds[split] = Dataset.from_dict({"labels": labels, "sentences": sentences})
+            ds[split] = Dataset.from_dict({'labels': labels, 'sentences': sentences})
         self.dataset = DatasetDict(ds)
         self.dataset = self.stratified_subsampling(
             self.dataset,
             self.seed,
             self.metadata.eval_splits,
-            label="labels",
+            label='labels',
             n_samples=NUM_SAMPLES,
         )
 
@@ -197,26 +197,26 @@ class ThuNewsClusteringFastP2P(AbsTaskClusteringFast):
     max_fraction_of_documents_to_embed = None
 
     metadata = TaskMetadata(
-        name="ThuNewsClusteringP2P",
+        name='ThuNewsClusteringP2P',
         dataset={
-            "path": "C-MTEB/ThuNewsClusteringP2P",
-            "revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
+            'path': 'C-MTEB/ThuNewsClusteringP2P',
+            'revision': '5798586b105c0434e4f0fe5e767abe619442cf93',
         },
-        description="Clustering of titles + abstracts from the THUCNews dataset",
-        reference="http://thuctc.thunlp.org/",
-        type="Clustering",
-        category="p2p",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="v_measure",
-        date=("2006-01-01", "2007-01-01"),
-        domains=["News", "Written"],
-        task_subtypes=["Thematic clustering", "Topic classification"],
-        license="Not specified",
-        annotations_creators="derived",
+        description='Clustering of titles + abstracts from the THUCNews dataset',
+        reference='http://thuctc.thunlp.org/',
+        type='Clustering',
+        category='p2p',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='v_measure',
+        date=('2006-01-01', '2007-01-01'),
+        domains=['News', 'Written'],
+        task_subtypes=['Thematic clustering', 'Topic classification'],
+        license='apache-2.0',
+        annotations_creators='derived',
         dialect=[],
-        sample_creation="found",
+        sample_creation='found',
         bibtex_citation="""@software{THUCTC,
   author = {Sun, M. and Li, J. and Guo, Z. and Yu, Z. and Zheng, Y. and Si, X. and Liu, Z.},
   title = {THUCTC: An Efficient Chinese Text Classifier},
@@ -226,27 +226,27 @@ class ThuNewsClusteringFastP2P(AbsTaskClusteringFast):
   url = {https://github.com/thunlp/THUCTC}
 }""",
         descriptive_stats={
-            "n_samples": {"test": NUM_SAMPLES},
-            "avg_character_length": {},
+            'n_samples': {'test': NUM_SAMPLES},
+            'avg_character_length': {},
         },
     )
 
     def dataset_transform(self):
         ds = {}
         for split in self.metadata.eval_splits:
-            labels = list(itertools.chain.from_iterable(self.dataset[split]["labels"]))
+            labels = list(itertools.chain.from_iterable(self.dataset[split]['labels']))
             sentences = list(
-                itertools.chain.from_iterable(self.dataset[split]["sentences"])
+                itertools.chain.from_iterable(self.dataset[split]['sentences'])
             )
 
             check_label_distribution(self.dataset[split])
 
-            ds[split] = Dataset.from_dict({"labels": labels, "sentences": sentences})
+            ds[split] = Dataset.from_dict({'labels': labels, 'sentences': sentences})
         self.dataset = DatasetDict(ds)
         self.dataset = self.stratified_subsampling(
             self.dataset,
             self.seed,
             self.metadata.eval_splits,
-            label="labels",
+            label='labels',
             n_samples=NUM_SAMPLES,
         )

{evalscope-0.6.0 → evalscope-0.6.0rc0}/evalscope/backend/rag_eval/cmteb/tasks/Reranking.py

@@ -2,22 +2,21 @@ from mteb.abstasks.AbsTaskReranking import AbsTaskReranking
 from mteb.abstasks.TaskMetadata import HFSubset, TaskMetadata
 
 
-
 class T2Reranking(AbsTaskReranking):
     metadata = TaskMetadata(
-        name="T2Reranking",
-        description="T2Ranking: A large-scale Chinese Benchmark for Passage Ranking",
-        reference="https://arxiv.org/abs/2304.03679",
+        name='T2Reranking',
+        description='T2Ranking: A large-scale Chinese Benchmark for Passage Ranking',
+        reference='https://arxiv.org/abs/2304.03679',
         dataset={
-            "path": "C-MTEB/T2Reranking",
-            "revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
+            'path': 'C-MTEB/T2Reranking',
+            'revision': '76631901a18387f85eaa53e5450019b87ad58ef9',
         },
-        type="Reranking",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="map",
+        type='Reranking',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='map',
         date=None,
         form=None,
         domains=None,
@@ -27,32 +26,32 @@ class T2Reranking(AbsTaskReranking):
         dialect=None,
         sample_creation=None,
         bibtex_citation="""@misc{xie2023t2ranking,
-      title={T2Ranking: A large-scale Chinese Benchmark for Passage Ranking}, 
+      title={T2Ranking: A large-scale Chinese Benchmark for Passage Ranking},
       author={Xiaohui Xie and Qian Dong and Bingning Wang and Feiyang Lv and Ting Yao and Weinan Gan and Zhijing Wu and Xiangsheng Li and Haitao Li and Yiqun Liu and Jin Ma},
       year={2023},
       eprint={2304.03679},
       archivePrefix={arXiv},
       primaryClass={cs.IR}
-}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+}""",  # noqa
+        descriptive_stats={'n_samples': None, 'avg_character_length': None},
     )
 
 
 class MMarcoReranking(AbsTaskReranking):
     metadata = TaskMetadata(
-        name="MMarcoReranking",
-        description="mMARCO is a multilingual version of the MS MARCO passage ranking dataset",
-        reference="https://github.com/unicamp-dl/mMARCO",
+        name='MMarcoReranking',
+        description='mMARCO is a multilingual version of the MS MARCO passage ranking dataset',
+        reference='https://github.com/unicamp-dl/mMARCO',
         dataset={
-            "path": "C-MTEB/Mmarco-reranking",
-            "revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
+            'path': 'C-MTEB/Mmarco-reranking',
+            'revision': '8e0c766dbe9e16e1d221116a3f36795fbade07f6',
         },
-        type="Reranking",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="map",
+        type='Reranking',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='map',
         date=None,
         form=None,
         domains=None,
@@ -62,39 +61,39 @@ class MMarcoReranking(AbsTaskReranking):
         dialect=None,
         sample_creation=None,
         bibtex_citation="""@misc{bonifacio2021mmarco,
-      title={mMARCO: A Multilingual Version of MS MARCO Passage Ranking Dataset}, 
+      title={mMARCO: A Multilingual Version of MS MARCO Passage Ranking Dataset},
       author={Luiz Henrique Bonifacio and Vitor Jeronymo and Hugo Queiroz Abonizio and Israel Campiotti and Marzieh Fadaee and  and Roberto Lotufo and Rodrigo Nogueira},
       year={2021},
       eprint={2108.13897},
       archivePrefix={arXiv},
       primaryClass={cs.CL}
-}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+}""",  # noqa
+        descriptive_stats={'n_samples': None, 'avg_character_length': None},
     )
 
 
 class CMedQAv1(AbsTaskReranking):
     metadata = TaskMetadata(
-        name="CMedQAv1",
-        description="Chinese community medical question answering",
-        reference="https://github.com/zhangsheng93/cMedQA",
+        name='CMedQAv1',
+        description='Chinese community medical question answering',
+        reference='https://github.com/zhangsheng93/cMedQA',
         dataset={
-            "path": "C-MTEB/CMedQAv1-reranking",
-            "revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
+            'path': 'C-MTEB/CMedQAv1-reranking',
+            'revision': '8d7f1e942507dac42dc58017c1a001c3717da7df',
         },
-        type="Reranking",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="map",
-        date=("2017-01-01", "2017-07-26"),
-        domains=["Medical", "Written"],
+        type='Reranking',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='map',
+        date=('2017-01-01', '2017-07-26'),
+        domains=['Medical', 'Written'],
         task_subtypes=[],
-        license="Not specified",
-        annotations_creators="expert-annotated",
+        license='apache-2.0',
+        annotations_creators='expert-annotated',
         dialect=[],
-        sample_creation="found",
+        sample_creation='found',
         bibtex_citation="""@article{zhang2017chinese,
   title={Chinese Medical Question Answer Matching Using End-to-End Character-Level Multi-Scale CNNs},
   author={Zhang, Sheng and Zhang, Xin and Wang, Hui and Cheng, Jiajun and Li, Pei and Ding, Zhaoyun},
@@ -106,27 +105,27 @@ class CMedQAv1(AbsTaskReranking):
   publisher={Multidisciplinary Digital Publishing Institute}
 }""",
         descriptive_stats={
-            "n_samples": {"test": 2000},
-            "avg_character_length": {"test": 165},
+            'n_samples': {'test': 2000},
+            'avg_character_length': {'test': 165},
         },
     )
 
 
 class CMedQAv2(AbsTaskReranking):
     metadata = TaskMetadata(
-        name="CMedQAv2",
-        description="Chinese community medical question answering",
-        reference="https://github.com/zhangsheng93/cMedQA2",
+        name='CMedQAv2',
+        description='Chinese community medical question answering',
+        reference='https://github.com/zhangsheng93/cMedQA2',
         dataset={
-            "path": "C-MTEB/CMedQAv2-reranking",
-            "revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
+            'path': 'C-MTEB/CMedQAv2-reranking',
+            'revision': '23d186750531a14a0357ca22cd92d712fd512ea0',
         },
-        type="Reranking",
-        category="s2s",
-        modalities=["text"],
-        eval_splits=["test"],
-        eval_langs=["cmn-Hans"],
-        main_score="map",
+        type='Reranking',
+        category='s2s',
+        modalities=['text'],
+        eval_splits=['test'],
+        eval_langs=['cmn-Hans'],
+        main_score='map',
         date=None,
         form=None,
         domains=None,
@@ -135,17 +134,17 @@ class CMedQAv2(AbsTaskReranking):
         annotations_creators=None,
         dialect=None,
         sample_creation=None,
-        bibtex_citation="""@ARTICLE{8548603, 
-author={S. Zhang and X. Zhang and H. Wang and L. Guo and S. Liu}, 
-journal={IEEE Access}, 
-title={Multi-Scale Attentive Interaction Networks for Chinese Medical Question Answer Selection}, 
-year={2018}, 
-volume={6}, 
-number={}, 
-pages={74061-74071}, 
-keywords={Biomedical imaging;Data mining;Semantics;Medical services;Feature extraction;Knowledge discovery;Medical question answering;interactive attention;deep learning;deep neural networks}, 
-doi={10.1109/ACCESS.2018.2883637}, 
-ISSN={2169-3536}, 
-month={},}""",
-        descriptive_stats={"n_samples": None, "avg_character_length": None},
+        bibtex_citation="""@ARTICLE{8548603,
+author={S. Zhang and X. Zhang and H. Wang and L. Guo and S. Liu},
+journal={IEEE Access},
+title={Multi-Scale Attentive Interaction Networks for Chinese Medical Question Answer Selection},
+year={2018},
+volume={6},
+number={},
+pages={74061-74071},
+keywords={Biomedical imaging;Data mining;Semantics;Medical services;Feature extraction;Knowledge discovery;Medical question answering;interactive attention;deep learning;deep neural networks},
+doi={10.1109/ACCESS.2018.2883637},
+ISSN={2169-3536},
+month={},}""",  # noqa
+        descriptive_stats={'n_samples': None, 'avg_character_length': None},
     )

evalscope-0.6.0rc0/evalscope/version.py

@@ -0,0 +1,4 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+__version__ = "0.6.0rc0"
+__release_datetime__ = "2099-01-01 00:00:00"