evalscope 0.5.5rc0__py3-none-any.whl → 0.6.0__py3-none-any.whl

evalscope/backend/rag_eval/cmteb/tasks/PairClassification.py

@@ -0,0 +1,113 @@
+from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class Ocnli(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="Ocnli",
+        description="Original Chinese Natural Language Inference dataset",
+        reference="https://arxiv.org/abs/2010.05444",
+        dataset={
+            "path": "C-MTEB/OCNLI",
+            "revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
+        },
+        type="PairClassification",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["validation"],
+        eval_langs=["cmn-Hans"],
+        main_score="max_accuracy",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{hu2020ocnli,
+            title={OCNLI: Original Chinese Natural Language Inference}, 
+            author={Hai Hu and Kyle Richardson and Liang Xu and Lu Li and Sandra Kuebler and Lawrence S. Moss},
+            year={2020},
+            eprint={2010.05444},
+            archivePrefix={arXiv},
+            primaryClass={cs.CL}
+        }""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.dataset.rename_column("sent1", "sentence1")
+        self.dataset = self.dataset.rename_column("sent2", "sentence2")
+
+
+class Cmnli(AbsTaskPairClassification):
+    metadata = TaskMetadata(
+        name="Cmnli",
+        description="Chinese Multi-Genre NLI",
+        reference="https://huggingface.co/datasets/clue/viewer/cmnli",
+        dataset={
+            "path": "C-MTEB/CMNLI",
+            "revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
+        },
+        type="PairClassification",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["validation", "test"],
+        eval_langs=["cmn-Hans"],
+        main_score="max_accuracy",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@inproceedings{xu-etal-2020-clue,
+            title = "{CLUE}: A {C}hinese Language Understanding Evaluation Benchmark",
+            author = "Xu, Liang  and
+            Hu, Hai  and
+            Zhang, Xuanwei  and
+            Li, Lu  and
+            Cao, Chenjie  and
+            Li, Yudong  and
+            Xu, Yechen  and
+            Sun, Kai  and
+            Yu, Dian  and
+            Yu, Cong  and
+            Tian, Yin  and
+            Dong, Qianqian  and
+            Liu, Weitang  and
+            Shi, Bo  and
+            Cui, Yiming  and
+            Li, Junyi  and
+            Zeng, Jun  and
+            Wang, Rongzhao  and
+            Xie, Weijian  and
+            Li, Yanting  and
+            Patterson, Yina  and
+            Tian, Zuoyu  and
+            Zhang, Yiwen  and
+            Zhou, He  and
+            Liu, Shaoweihua  and
+            Zhao, Zhe  and
+            Zhao, Qipeng  and
+            Yue, Cong  and
+            Zhang, Xinrui  and
+            Yang, Zhengliang  and
+            Richardson, Kyle  and
+            Lan, Zhenzhong",
+            booktitle = "Proceedings of the 28th International Conference on Computational Linguistics",
+            month = dec,
+            year = "2020",
+            address = "Barcelona, Spain (Online)",
+            publisher = "International Committee on Computational Linguistics",
+            url = "https://aclanthology.org/2020.coling-main.419",
+            doi = "10.18653/v1/2020.coling-main.419",
+            pages = "4762--4772",
+        }""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+    def dataset_transform(self):
+        self.dataset = self.dataset.rename_column("sent1", "sentence1")
+        self.dataset = self.dataset.rename_column("sent2", "sentence2")

evalscope/backend/rag_eval/cmteb/tasks/Reranking.py

@@ -0,0 +1,151 @@
+from mteb.abstasks.AbsTaskReranking import AbsTaskReranking
+from mteb.abstasks.TaskMetadata import HFSubset, TaskMetadata
+
+
+
+class T2Reranking(AbsTaskReranking):
+    metadata = TaskMetadata(
+        name="T2Reranking",
+        description="T2Ranking: A large-scale Chinese Benchmark for Passage Ranking",
+        reference="https://arxiv.org/abs/2304.03679",
+        dataset={
+            "path": "C-MTEB/T2Reranking",
+            "revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
+        },
+        type="Reranking",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="map",
+        date=None,
+        form=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{xie2023t2ranking,
+      title={T2Ranking: A large-scale Chinese Benchmark for Passage Ranking}, 
+      author={Xiaohui Xie and Qian Dong and Bingning Wang and Feiyang Lv and Ting Yao and Weinan Gan and Zhijing Wu and Xiangsheng Li and Haitao Li and Yiqun Liu and Jin Ma},
+      year={2023},
+      eprint={2304.03679},
+      archivePrefix={arXiv},
+      primaryClass={cs.IR}
+}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+
+class MMarcoReranking(AbsTaskReranking):
+    metadata = TaskMetadata(
+        name="MMarcoReranking",
+        description="mMARCO is a multilingual version of the MS MARCO passage ranking dataset",
+        reference="https://github.com/unicamp-dl/mMARCO",
+        dataset={
+            "path": "C-MTEB/Mmarco-reranking",
+            "revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
+        },
+        type="Reranking",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="map",
+        date=None,
+        form=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{bonifacio2021mmarco,
+      title={mMARCO: A Multilingual Version of MS MARCO Passage Ranking Dataset}, 
+      author={Luiz Henrique Bonifacio and Vitor Jeronymo and Hugo Queiroz Abonizio and Israel Campiotti and Marzieh Fadaee and  and Roberto Lotufo and Rodrigo Nogueira},
+      year={2021},
+      eprint={2108.13897},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )
+
+
+class CMedQAv1(AbsTaskReranking):
+    metadata = TaskMetadata(
+        name="CMedQAv1",
+        description="Chinese community medical question answering",
+        reference="https://github.com/zhangsheng93/cMedQA",
+        dataset={
+            "path": "C-MTEB/CMedQAv1-reranking",
+            "revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
+        },
+        type="Reranking",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["cmn-Hans"],
+        main_score="map",
+        date=("2017-01-01", "2017-07-26"),
+        domains=["Medical", "Written"],
+        task_subtypes=[],
+        license="Not specified",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@article{zhang2017chinese,
+  title={Chinese Medical Question Answer Matching Using End-to-End Character-Level Multi-Scale CNNs},
+  author={Zhang, Sheng and Zhang, Xin and Wang, Hui and Cheng, Jiajun and Li, Pei and Ding, Zhaoyun},
+  journal={Applied Sciences},
+  volume={7},
+  number={8},
+  pages={767},
+  year={2017},
+  publisher={Multidisciplinary Digital Publishing Institute}
+}""",
+        descriptive_stats={
+            "n_samples": {"test": 2000},
+            "avg_character_length": {"test": 165},
+        },
+    )
+
+
+class CMedQAv2(AbsTaskReranking):
+    metadata = TaskMetadata(
+        name="CMedQAv2",
+        description="Chinese community medical question answering",
+        reference="https://github.com/zhangsheng93/cMedQA2",
+        dataset={
+            "path": "C-MTEB/CMedQAv2-reranking",
+            "revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
+        },
+        type="Reranking",
+        category="s2s",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["cmn-Hans"],
+        main_score="map",
+        date=None,
+        form=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@ARTICLE{8548603, 
+author={S. Zhang and X. Zhang and H. Wang and L. Guo and S. Liu}, 
+journal={IEEE Access}, 
+title={Multi-Scale Attentive Interaction Networks for Chinese Medical Question Answer Selection}, 
+year={2018}, 
+volume={6}, 
+number={}, 
+pages={74061-74071}, 
+keywords={Biomedical imaging;Data mining;Semantics;Medical services;Feature extraction;Knowledge discovery;Medical question answering;interactive attention;deep learning;deep neural networks}, 
+doi={10.1109/ACCESS.2018.2883637}, 
+ISSN={2169-3536}, 
+month={},}""",
+        descriptive_stats={"n_samples": None, "avg_character_length": None},
+    )

evalscope/backend/rag_eval/cmteb/tasks/Retrieval.py

@@ -0,0 +1,345 @@
+from mteb import AbsTaskRetrieval
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class T2Retrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="T2Retrieval",
+        description="T2Ranking: A large-scale Chinese Benchmark for Passage Ranking",
+        reference="https://arxiv.org/abs/2304.03679",
+        dataset={
+            "path": "C-MTEB/T2Retrieval",
+            "revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
+            "qrel_revision": "1c83b8d1544e529875e3f6930f3a1fcf749a8e97",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="ndcg_at_10",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{xie2023t2ranking,
+      title={T2Ranking: A large-scale Chinese Benchmark for Passage Ranking}, 
+      author={Xiaohui Xie and Qian Dong and Bingning Wang and Feiyang Lv and Ting Yao and Weinan Gan and Zhijing Wu and Xiangsheng Li and Haitao Li and Yiqun Liu and Jin Ma},
+      year={2023},
+      eprint={2304.03679},
+      archivePrefix={arXiv},
+      primaryClass={cs.IR}
+}""",
+        descriptive_stats={
+            "n_samples": None,
+            "avg_character_length": {
+                "dev": {
+                    "average_document_length": 874.1184182791619,
+                    "average_query_length": 10.938847974750132,
+                    "num_documents": 118605,
+                    "num_queries": 22812,
+                    "average_relevant_docs_per_query": 5.213571804313519,
+                }
+            },
+        },
+    )
+
+
+class MMarcoRetrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="MMarcoRetrieval",
+        description="MMarcoRetrieval",
+        reference="https://arxiv.org/abs/2309.07597",
+        dataset={
+            "path": "C-MTEB/MMarcoRetrieval",
+            "revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
+            "qrel_revision": "bae08bb7bddbedb96c7e7db52018a55167b67f89",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="ndcg_at_10",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{xiao2024cpack,
+      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
+      author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
+      year={2024},
+      eprint={2309.07597},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}""",
+        descriptive_stats={
+            "n_samples": None,
+            "avg_character_length": {
+                "dev": {
+                    "average_document_length": 114.41787048392986,
+                    "average_query_length": 10.51131805157593,
+                    "num_documents": 106813,
+                    "num_queries": 6980,
+                    "average_relevant_docs_per_query": 1.0654727793696275,
+                }
+            },
+        },
+    )
+
+
+class DuRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="DuRetrieval",
+        description="A Large-scale Chinese Benchmark for Passage Retrieval from Web Search Engine",
+        reference="https://aclanthology.org/2022.emnlp-main.357.pdf",
+        dataset={
+            "path": "C-MTEB/DuRetrieval",
+            "revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
+            "qrel_revision": "497b7bd1bbb25cb3757ff34d95a8be50a3de2279",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="ndcg_at_10",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation="""@misc{qiu2022dureaderretrieval,
+      title={DuReader_retrieval: A Large-scale Chinese Benchmark for Passage Retrieval from Web Search Engine}, 
+      author={Yifu Qiu and Hongyu Li and Yingqi Qu and Ying Chen and Qiaoqiao She and Jing Liu and Hua Wu and Haifeng Wang},
+      year={2022},
+      eprint={2203.10232},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}""",
+        descriptive_stats={
+            "n_samples": None,
+            "avg_character_length": {
+                "dev": {
+                    "average_document_length": 331.3219967800322,
+                    "average_query_length": 9.289,
+                    "num_documents": 100001,
+                    "num_queries": 2000,
+                    "average_relevant_docs_per_query": 4.9195,
+                }
+            },
+        },
+    )
+
+
+class CovidRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="CovidRetrieval",
+        description="COVID-19 news articles",
+        reference="https://arxiv.org/abs/2203.03367",
+        dataset={
+            "path": "C-MTEB/CovidRetrieval",
+            "revision": "1271c7809071a13532e05f25fb53511ffce77117",
+            "qrel_revision": "a9f41b7cdf24785531d12417ce0d1157ed4b39ca",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="ndcg_at_10",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation=None,
+        descriptive_stats={
+            "n_samples": None,
+            "avg_character_length": {
+                "dev": {
+                    "average_document_length": 332.4152658473415,
+                    "average_query_length": 25.9304531085353,
+                    "num_documents": 100001,
+                    "num_queries": 949,
+                    "average_relevant_docs_per_query": 1.0105374077976819,
+                }
+            },
+        },
+    )
+
+
+class CmedqaRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="CmedqaRetrieval",
+        description="Online medical consultation text. Used the CMedQAv2 as its underlying dataset.",
+        reference="https://aclanthology.org/2022.emnlp-main.357.pdf",
+        dataset={
+            "path": "C-MTEB/CmedqaRetrieval",
+            "revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
+            "qrel_revision": "279d737f36c731c8ff6e2b055f31fe02216fa23d",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="ndcg_at_10",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation=None,
+        descriptive_stats={
+            "n_samples": None,
+            "avg_character_length": {
+                "dev": {
+                    "average_document_length": 307.7710222897771,
+                    "average_query_length": 48.470367591897976,
+                    "num_documents": 100001,
+                    "num_queries": 3999,
+                    "average_relevant_docs_per_query": 1.86271567891973,
+                }
+            },
+        },
+    )
+
+
+class EcomRetrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="EcomRetrieval",
+        description="EcomRetrieval",
+        reference="https://arxiv.org/abs/2203.03367",
+        dataset={
+            "path": "C-MTEB/EcomRetrieval",
+            "revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
+            "qrel_revision": "39c90699b034ec22ac45b3abf5b0bbb5ffd421f9",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="ndcg_at_10",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation=None,
+        descriptive_stats={
+            "n_samples": None,
+            "avg_character_length": {
+                "dev": {
+                    "average_document_length": 32.98041664189015,
+                    "average_query_length": 6.798,
+                    "num_documents": 100902,
+                    "num_queries": 1000,
+                    "average_relevant_docs_per_query": 1.0,
+                }
+            },
+        },
+    )
+
+
+class MedicalRetrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="MedicalRetrieval",
+        description="MedicalRetrieval",
+        reference="https://arxiv.org/abs/2203.03367",
+        dataset={
+            "path": "C-MTEB/MedicalRetrieval",
+            "revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
+            "qrel_revision": "37b8efec53c54c3d9c6af212f6710b62ccdf895c",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="ndcg_at_10",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation=None,
+        descriptive_stats={
+            "n_samples": None,
+            "avg_character_length": {
+                "dev": {
+                    "average_document_length": 122.04231725066585,
+                    "average_query_length": 17.938,
+                    "num_documents": 100999,
+                    "num_queries": 1000,
+                    "average_relevant_docs_per_query": 1.0,
+                }
+            },
+        },
+    )
+
+
+class VideoRetrieval(AbsTaskRetrieval):
+    ignore_identical_ids = True
+
+    metadata = TaskMetadata(
+        name="VideoRetrieval",
+        description="VideoRetrieval",
+        reference="https://arxiv.org/abs/2203.03367",
+        dataset={
+            "path": "C-MTEB/VideoRetrieval",
+            "revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
+            "qrel_revision": "faa71382b6a29cf1778d1f436b963e75cb5b927c",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["dev"],
+        eval_langs=["cmn-Hans"],
+        main_score="ndcg_at_10",
+        date=None,
+        domains=None,
+        task_subtypes=None,
+        license=None,
+        annotations_creators=None,
+        dialect=None,
+        sample_creation=None,
+        bibtex_citation=None,
+        descriptive_stats={
+            "n_samples": None,
+            "avg_character_length": {
+                "dev": {
+                    "average_document_length": 31.048855642524522,
+                    "average_query_length": 7.365,
+                    "num_documents": 100930,
+                    "num_queries": 1000,
+                    "average_relevant_docs_per_query": 1.0,
+                }
+            },
+        },
+    )