evalscope 0.7.2__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/backend/rag_eval/cmteb/tasks/Retrieval.py

@@ -6,20 +6,20 @@ class T2Retrieval(AbsTaskRetrieval):
     ignore_identical_ids = True
 
     metadata = TaskMetadata(
-        name="T2Retrieval",
-        description="T2Ranking: A large-scale Chinese Benchmark for Passage Ranking",
-        reference="https://arxiv.org/abs/2304.03679",
+        name='T2Retrieval',
+        description='T2Ranking: A large-scale Chinese Benchmark for Passage Ranking',
+        reference='https://arxiv.org/abs/2304.03679',
         dataset={
-            "path": "C-MTEB/T2Retrieval",
-            "revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
-            "qrel_revision": "1c83b8d1544e529875e3f6930f3a1fcf749a8e97",
+            'path': 'C-MTEB/T2Retrieval',
+            'revision': '8731a845f1bf500a4f111cf1070785c793d10e64',
+            'qrel_revision': '1c83b8d1544e529875e3f6930f3a1fcf749a8e97',
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="ndcg_at_10",
+        type='Retrieval',
+        category='s2p',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='ndcg_at_10',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -28,7 +28,7 @@ class T2Retrieval(AbsTaskRetrieval):
         dialect=None,
         sample_creation=None,
         bibtex_citation="""@misc{xie2023t2ranking,
-      title={T2Ranking: A large-scale Chinese Benchmark for Passage Ranking}, 
+      title={T2Ranking: A large-scale Chinese Benchmark for Passage Ranking},
       author={Xiaohui Xie and Qian Dong and Bingning Wang and Feiyang Lv and Ting Yao and Weinan Gan and Zhijing Wu and Xiangsheng Li and Haitao Li and Yiqun Liu and Jin Ma},
       year={2023},
       eprint={2304.03679},
@@ -36,14 +36,14 @@ class T2Retrieval(AbsTaskRetrieval):
       primaryClass={cs.IR}
 }""",
         descriptive_stats={
-            "n_samples": None,
-            "avg_character_length": {
-                "dev": {
-                    "average_document_length": 874.1184182791619,
-                    "average_query_length": 10.938847974750132,
-                    "num_documents": 118605,
-                    "num_queries": 22812,
-                    "average_relevant_docs_per_query": 5.213571804313519,
+            'n_samples': None,
+            'avg_character_length': {
+                'dev': {
+                    'average_document_length': 874.1184182791619,
+                    'average_query_length': 10.938847974750132,
+                    'num_documents': 118605,
+                    'num_queries': 22812,
+                    'average_relevant_docs_per_query': 5.213571804313519,
                 }
             },
         },
@@ -54,20 +54,20 @@ class MMarcoRetrieval(AbsTaskRetrieval):
     ignore_identical_ids = True
 
     metadata = TaskMetadata(
-        name="MMarcoRetrieval",
-        description="MMarcoRetrieval",
-        reference="https://arxiv.org/abs/2309.07597",
+        name='MMarcoRetrieval',
+        description='MMarcoRetrieval',
+        reference='https://arxiv.org/abs/2309.07597',
         dataset={
-            "path": "C-MTEB/MMarcoRetrieval",
-            "revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
-            "qrel_revision": "bae08bb7bddbedb96c7e7db52018a55167b67f89",
+            'path': 'C-MTEB/MMarcoRetrieval',
+            'revision': '539bbde593d947e2a124ba72651aafc09eb33fc2',
+            'qrel_revision': 'bae08bb7bddbedb96c7e7db52018a55167b67f89',
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="ndcg_at_10",
+        type='Retrieval',
+        category='s2p',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='ndcg_at_10',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -76,7 +76,7 @@ class MMarcoRetrieval(AbsTaskRetrieval):
         dialect=None,
         sample_creation=None,
         bibtex_citation="""@misc{xiao2024cpack,
-      title={C-Pack: Packaged Resources To Advance General Chinese Embedding}, 
+      title={C-Pack: Packaged Resources To Advance General Chinese Embedding},
       author={Shitao Xiao and Zheng Liu and Peitian Zhang and Niklas Muennighoff and Defu Lian and Jian-Yun Nie},
       year={2024},
       eprint={2309.07597},
@@ -84,14 +84,14 @@ class MMarcoRetrieval(AbsTaskRetrieval):
       primaryClass={cs.CL}
 }""",
         descriptive_stats={
-            "n_samples": None,
-            "avg_character_length": {
-                "dev": {
-                    "average_document_length": 114.41787048392986,
-                    "average_query_length": 10.51131805157593,
-                    "num_documents": 106813,
-                    "num_queries": 6980,
-                    "average_relevant_docs_per_query": 1.0654727793696275,
+            'n_samples': None,
+            'avg_character_length': {
+                'dev': {
+                    'average_document_length': 114.41787048392986,
+                    'average_query_length': 10.51131805157593,
+                    'num_documents': 106813,
+                    'num_queries': 6980,
+                    'average_relevant_docs_per_query': 1.0654727793696275,
                 }
             },
         },
@@ -100,20 +100,20 @@ class MMarcoRetrieval(AbsTaskRetrieval):
 
 class DuRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
-        name="DuRetrieval",
-        description="A Large-scale Chinese Benchmark for Passage Retrieval from Web Search Engine",
-        reference="https://aclanthology.org/2022.emnlp-main.357.pdf",
+        name='DuRetrieval',
+        description='A Large-scale Chinese Benchmark for Passage Retrieval from Web Search Engine',
+        reference='https://aclanthology.org/2022.emnlp-main.357.pdf',
         dataset={
-            "path": "C-MTEB/DuRetrieval",
-            "revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
-            "qrel_revision": "497b7bd1bbb25cb3757ff34d95a8be50a3de2279",
+            'path': 'C-MTEB/DuRetrieval',
+            'revision': 'a1a333e290fe30b10f3f56498e3a0d911a693ced',
+            'qrel_revision': '497b7bd1bbb25cb3757ff34d95a8be50a3de2279',
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="ndcg_at_10",
+        type='Retrieval',
+        category='s2p',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='ndcg_at_10',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -122,7 +122,7 @@ class DuRetrieval(AbsTaskRetrieval):
         dialect=None,
         sample_creation=None,
         bibtex_citation="""@misc{qiu2022dureaderretrieval,
-      title={DuReader_retrieval: A Large-scale Chinese Benchmark for Passage Retrieval from Web Search Engine}, 
+      title={DuReader_retrieval: A Large-scale Chinese Benchmark for Passage Retrieval from Web Search Engine},
       author={Yifu Qiu and Hongyu Li and Yingqi Qu and Ying Chen and Qiaoqiao She and Jing Liu and Hua Wu and Haifeng Wang},
       year={2022},
       eprint={2203.10232},
@@ -130,14 +130,14 @@ class DuRetrieval(AbsTaskRetrieval):
       primaryClass={cs.CL}
 }""",
         descriptive_stats={
-            "n_samples": None,
-            "avg_character_length": {
-                "dev": {
-                    "average_document_length": 331.3219967800322,
-                    "average_query_length": 9.289,
-                    "num_documents": 100001,
-                    "num_queries": 2000,
-                    "average_relevant_docs_per_query": 4.9195,
+            'n_samples': None,
+            'avg_character_length': {
+                'dev': {
+                    'average_document_length': 331.3219967800322,
+                    'average_query_length': 9.289,
+                    'num_documents': 100001,
+                    'num_queries': 2000,
+                    'average_relevant_docs_per_query': 4.9195,
                 }
             },
         },
@@ -146,20 +146,20 @@ class DuRetrieval(AbsTaskRetrieval):
 
 class CovidRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
-        name="CovidRetrieval",
-        description="COVID-19 news articles",
-        reference="https://arxiv.org/abs/2203.03367",
+        name='CovidRetrieval',
+        description='COVID-19 news articles',
+        reference='https://arxiv.org/abs/2203.03367',
         dataset={
-            "path": "C-MTEB/CovidRetrieval",
-            "revision": "1271c7809071a13532e05f25fb53511ffce77117",
-            "qrel_revision": "a9f41b7cdf24785531d12417ce0d1157ed4b39ca",
+            'path': 'C-MTEB/CovidRetrieval',
+            'revision': '1271c7809071a13532e05f25fb53511ffce77117',
+            'qrel_revision': 'a9f41b7cdf24785531d12417ce0d1157ed4b39ca',
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="ndcg_at_10",
+        type='Retrieval',
+        category='s2p',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='ndcg_at_10',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -169,14 +169,14 @@ class CovidRetrieval(AbsTaskRetrieval):
         sample_creation=None,
         bibtex_citation=None,
         descriptive_stats={
-            "n_samples": None,
-            "avg_character_length": {
-                "dev": {
-                    "average_document_length": 332.4152658473415,
-                    "average_query_length": 25.9304531085353,
-                    "num_documents": 100001,
-                    "num_queries": 949,
-                    "average_relevant_docs_per_query": 1.0105374077976819,
+            'n_samples': None,
+            'avg_character_length': {
+                'dev': {
+                    'average_document_length': 332.4152658473415,
+                    'average_query_length': 25.9304531085353,
+                    'num_documents': 100001,
+                    'num_queries': 949,
+                    'average_relevant_docs_per_query': 1.0105374077976819,
                 }
             },
         },
@@ -185,20 +185,20 @@ class CovidRetrieval(AbsTaskRetrieval):
 
 class CmedqaRetrieval(AbsTaskRetrieval):
     metadata = TaskMetadata(
-        name="CmedqaRetrieval",
-        description="Online medical consultation text. Used the CMedQAv2 as its underlying dataset.",
-        reference="https://aclanthology.org/2022.emnlp-main.357.pdf",
+        name='CmedqaRetrieval',
+        description='Online medical consultation text. Used the CMedQAv2 as its underlying dataset.',
+        reference='https://aclanthology.org/2022.emnlp-main.357.pdf',
         dataset={
-            "path": "C-MTEB/CmedqaRetrieval",
-            "revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
-            "qrel_revision": "279d737f36c731c8ff6e2b055f31fe02216fa23d",
+            'path': 'C-MTEB/CmedqaRetrieval',
+            'revision': 'cd540c506dae1cf9e9a59c3e06f42030d54e7301',
+            'qrel_revision': '279d737f36c731c8ff6e2b055f31fe02216fa23d',
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="ndcg_at_10",
+        type='Retrieval',
+        category='s2p',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='ndcg_at_10',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -208,14 +208,14 @@ class CmedqaRetrieval(AbsTaskRetrieval):
         sample_creation=None,
         bibtex_citation=None,
         descriptive_stats={
-            "n_samples": None,
-            "avg_character_length": {
-                "dev": {
-                    "average_document_length": 307.7710222897771,
-                    "average_query_length": 48.470367591897976,
-                    "num_documents": 100001,
-                    "num_queries": 3999,
-                    "average_relevant_docs_per_query": 1.86271567891973,
+            'n_samples': None,
+            'avg_character_length': {
+                'dev': {
+                    'average_document_length': 307.7710222897771,
+                    'average_query_length': 48.470367591897976,
+                    'num_documents': 100001,
+                    'num_queries': 3999,
+                    'average_relevant_docs_per_query': 1.86271567891973,
                 }
             },
         },
@@ -226,20 +226,20 @@ class EcomRetrieval(AbsTaskRetrieval):
     ignore_identical_ids = True
 
     metadata = TaskMetadata(
-        name="EcomRetrieval",
-        description="EcomRetrieval",
-        reference="https://arxiv.org/abs/2203.03367",
+        name='EcomRetrieval',
+        description='EcomRetrieval',
+        reference='https://arxiv.org/abs/2203.03367',
         dataset={
-            "path": "C-MTEB/EcomRetrieval",
-            "revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
-            "qrel_revision": "39c90699b034ec22ac45b3abf5b0bbb5ffd421f9",
+            'path': 'C-MTEB/EcomRetrieval',
+            'revision': '687de13dc7294d6fd9be10c6945f9e8fec8166b9',
+            'qrel_revision': '39c90699b034ec22ac45b3abf5b0bbb5ffd421f9',
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="ndcg_at_10",
+        type='Retrieval',
+        category='s2p',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='ndcg_at_10',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -249,14 +249,14 @@ class EcomRetrieval(AbsTaskRetrieval):
         sample_creation=None,
         bibtex_citation=None,
         descriptive_stats={
-            "n_samples": None,
-            "avg_character_length": {
-                "dev": {
-                    "average_document_length": 32.98041664189015,
-                    "average_query_length": 6.798,
-                    "num_documents": 100902,
-                    "num_queries": 1000,
-                    "average_relevant_docs_per_query": 1.0,
+            'n_samples': None,
+            'avg_character_length': {
+                'dev': {
+                    'average_document_length': 32.98041664189015,
+                    'average_query_length': 6.798,
+                    'num_documents': 100902,
+                    'num_queries': 1000,
+                    'average_relevant_docs_per_query': 1.0,
                 }
             },
         },
@@ -267,20 +267,20 @@ class MedicalRetrieval(AbsTaskRetrieval):
     ignore_identical_ids = True
 
     metadata = TaskMetadata(
-        name="MedicalRetrieval",
-        description="MedicalRetrieval",
-        reference="https://arxiv.org/abs/2203.03367",
+        name='MedicalRetrieval',
+        description='MedicalRetrieval',
+        reference='https://arxiv.org/abs/2203.03367',
         dataset={
-            "path": "C-MTEB/MedicalRetrieval",
-            "revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
-            "qrel_revision": "37b8efec53c54c3d9c6af212f6710b62ccdf895c",
+            'path': 'C-MTEB/MedicalRetrieval',
+            'revision': '2039188fb5800a9803ba5048df7b76e6fb151fc6',
+            'qrel_revision': '37b8efec53c54c3d9c6af212f6710b62ccdf895c',
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="ndcg_at_10",
+        type='Retrieval',
+        category='s2p',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='ndcg_at_10',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -290,14 +290,14 @@ class MedicalRetrieval(AbsTaskRetrieval):
         sample_creation=None,
         bibtex_citation=None,
         descriptive_stats={
-            "n_samples": None,
-            "avg_character_length": {
-                "dev": {
-                    "average_document_length": 122.04231725066585,
-                    "average_query_length": 17.938,
-                    "num_documents": 100999,
-                    "num_queries": 1000,
-                    "average_relevant_docs_per_query": 1.0,
+            'n_samples': None,
+            'avg_character_length': {
+                'dev': {
+                    'average_document_length': 122.04231725066585,
+                    'average_query_length': 17.938,
+                    'num_documents': 100999,
+                    'num_queries': 1000,
+                    'average_relevant_docs_per_query': 1.0,
                 }
             },
         },
@@ -308,20 +308,20 @@ class VideoRetrieval(AbsTaskRetrieval):
     ignore_identical_ids = True
 
     metadata = TaskMetadata(
-        name="VideoRetrieval",
-        description="VideoRetrieval",
-        reference="https://arxiv.org/abs/2203.03367",
+        name='VideoRetrieval',
+        description='VideoRetrieval',
+        reference='https://arxiv.org/abs/2203.03367',
         dataset={
-            "path": "C-MTEB/VideoRetrieval",
-            "revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
-            "qrel_revision": "faa71382b6a29cf1778d1f436b963e75cb5b927c",
+            'path': 'C-MTEB/VideoRetrieval',
+            'revision': '58c2597a5943a2ba48f4668c3b90d796283c5639',
+            'qrel_revision': 'faa71382b6a29cf1778d1f436b963e75cb5b927c',
         },
-        type="Retrieval",
-        category="s2p",
-        modalities=["text"],
-        eval_splits=["dev"],
-        eval_langs=["cmn-Hans"],
-        main_score="ndcg_at_10",
+        type='Retrieval',
+        category='s2p',
+        modalities=['text'],
+        eval_splits=['dev'],
+        eval_langs=['cmn-Hans'],
+        main_score='ndcg_at_10',
         date=None,
         domains=None,
         task_subtypes=None,
@@ -331,14 +331,14 @@ class VideoRetrieval(AbsTaskRetrieval):
         sample_creation=None,
         bibtex_citation=None,
         descriptive_stats={
-            "n_samples": None,
-            "avg_character_length": {
-                "dev": {
-                    "average_document_length": 31.048855642524522,
-                    "average_query_length": 7.365,
-                    "num_documents": 100930,
-                    "num_queries": 1000,
-                    "average_relevant_docs_per_query": 1.0,
+            'n_samples': None,
+            'avg_character_length': {
+                'dev': {
+                    'average_document_length': 31.048855642524522,
+                    'average_query_length': 7.365,
+                    'num_documents': 100930,
+                    'num_queries': 1000,
+                    'average_relevant_docs_per_query': 1.0,
                 }
             },
         },