evalscope 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/benchmarks/race/race_adapter.py

@@ -1,26 +1,23 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import os
 import json
+import os
+
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import exact_match, weighted_mean
-from evalscope.utils import normalize_score, jsonl_to_list
+from evalscope.utils import normalize_score
+from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
 
 DATASET_ID = 'modelscope/race'
 
-SUBSET_LIST = [
-    "high",
-    "middle"
-]
-
+SUBSET_LIST = ['high', 'middle']
 
-SUBJECT_MAPPING = {"high": "High",
-                   "middle": "Middle"
-}
+SUBJECT_MAPPING = {'high': 'High', 'middle': 'Middle'}
 
 
 class RACEAdapter(DataAdapter):
@@ -49,12 +46,13 @@ class RACEAdapter(DataAdapter):
             logger.warning(f'few_shot_num <= 3 for RACE, but got {few_shot_num}. Use 3-shot by default.')
             few_shot_num = 3
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -92,8 +90,7 @@ class RACEAdapter(DataAdapter):
 
         """
         prompt = 'The following are multiple choice reading comprehension questions (with answers).\n\n'.format(
-            self._format_subject(subset_name)
-        )
+            self._format_subject(subset_name))
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
 
         context: str = '\n'.join(few_shot_prompts) + '\n'
@@ -122,9 +119,9 @@ class RACEAdapter(DataAdapter):
         """
         if eval_type == 'checkpoint':
             return result
-        elif eval_type == 'service':        # TODO: to be implemented
+        elif eval_type == 'service':  # TODO: to be implemented
             return result
-        elif eval_type == 'custom':         # TODO: to be implemented
+        elif eval_type == 'custom':  # TODO: to be implemented
             return result
         else:
             raise ValueError(f'Unknown eval_type: {eval_type}')
@@ -191,17 +188,24 @@ class RACEAdapter(DataAdapter):
             domain_weighted_avg_acc = sum([score * num for _, score, num in domain_res_list]) / \
                                      sum([num for _, _, num in domain_res_list])
             domain_weighted_avg_acc = normalize_score(score=domain_weighted_avg_acc)
-            category_list.append({'name': domain_name,
-                                  'score': normalize_score(score=domain_weighted_avg_acc),
-                                  'subset': [{'name': subset_name, 'score': subset_score}
-                                             for subset_name, subset_score, _ in domain_res_list]})
+            category_list.append({
+                'name':
+                domain_name,
+                'score':
+                normalize_score(score=domain_weighted_avg_acc),
+                'subset': [{
+                    'name': subset_name,
+                    'score': subset_score
+                } for subset_name, subset_score, _ in domain_res_list]
+            })
 
         # Get final dict of report
-        res_map = dict(name=report_name or 'race',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=category_list,
-                       total_num=total_num)
+        res_map = dict(
+            name=report_name or 'race',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=category_list,
+            total_num=total_num)
 
         return res_map
 

evalscope/benchmarks/race/samples.jsonl

@@ -2,4 +2,4 @@
 {'example_id': 'middle3329.txt', 'article': 'Do you know why diff...ng at all.', 'answer': 'B', 'question': 'Those pests with dif...of danger.', 'options': ['change their colours', 'hide in the day time...r at night', 'move quietly', 'hide at night and ap...e day time']}
 {'example_id': 'middle3614.txt', 'article': 'The seahorse is a ve...o the sea.', 'answer': 'B', 'question': 'A seahorse eats   _  .', 'options': ['sea weed', 'small fish', 'water', 'nothing']}
 {'example_id': 'middle6632.txt', 'article': 'Kids have unbelievab...h at her."', 'answer': 'D', 'question': 'Which is NOT mention...e passage?', 'options': ['Robots keep secrets.', 'Robots give suggestions.', 'Robots do chores.', 'Robots make movies.']}
-{'example_id': 'middle3503.txt', 'article': 'Have you ever heard ...eir lives.', 'answer': 'B', 'question': 'Which of the followi...lue moon"?', 'options': ['Simon often tells jo...blue moon.', 'Tom rarely remembers...blue moon.', 'Mary likes to go sho...blue moon.', 'Cindy hates to stay ...blue moon.']}
+{'example_id': 'middle3503.txt', 'article': 'Have you ever heard ...eir lives.', 'answer': 'B', 'question': 'Which of the followi...lue moon"?', 'options': ['Simon often tells jo...blue moon.', 'Tom rarely remembers...blue moon.', 'Mary likes to go sho...blue moon.', 'Cindy hates to stay ...blue moon.']}

evalscope/benchmarks/trivia_qa/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import TriviaQaAdapter, DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import TriviaQaAdapter
 from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import TriviaQaAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/trivia_qa/samples.jsonl

@@ -2,4 +2,4 @@
 {"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "Which Lloyd Webber musical premiered in the US on 10th December 1993?"}], "ideal": ["Sunset Blvd", "West Sunset Boulevard", "Sunset Boulevard", "Sunset Bulevard", "Sunset Blvd.", "sunset boulevard", "sunset bulevard", "west sunset boulevard", "sunset blvd"]}
 {"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "Who was the next British Prime Minister after Arthur Balfour?"}], "ideal": ["Sir Henry Campbell-Bannerman", "Campbell-Bannerman", "Campbell Bannerman", "Sir Henry Campbell Bannerman", "Henry Campbell Bannerman", "Henry Campbell-Bannerman", "henry campbell bannerman", "sir henry campbell bannerman", "campbell bannerman"]}
 {"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "Who had a 70s No 1 hit with Kiss You All Over?"}], "ideal": ["Internal exile", "Exiles", "Transported for life", "Exile (politics and government)", "Voluntary exile", "Sent into exile", "Exile and Banishment", "Self-exile", "Forced exile", "Exile", "Exile in Greek tragedy", "Banish", "Banishment", "exiles", "voluntary exile", "forced exile", "banish", "self exile", "exile politics and government", "exile in greek tragedy", "sent into exile", "banishment", "transported for life", "exile", "internal exile", "exile and banishment"]}
-{"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "What claimed the life of singer Kathleen Ferrier?"}], "ideal": ["Cancer pathology", "Deaths by cancer", "Anti-cancer", "Cancer (disease)", "Cancerophobia", "Malignant lesion", "Cancer medication", "Malignant tumors", "Cancer signs", "Malignant neoplasm", "Invasive (cancer)", "Malignant Neoplasms", "Malignant growth", "Sporadic cancer", "Malignant cancer", "Tumour virus", "Cancer en cuirasse", "Microtumor", "Malignant neoplasms", "Malignant tumour", "Carcinophobia", "Malignacy", "Cancer patient", "Epithelial cancers", "Solid cancer", "Cancers", "Tumor medication", "Malignant neoplastic disease", "AIDS-related cancer", "Invasive cancer", "Cancer therapy", "Cancerous tumor", "Cancer", "Financial toxicity", "Cancer diagnosis", "Cancer (medicine)", "Malignant tumor", "Cancerous", "Borderline (cancer)", "Signs of cancer", "Malignancies", "Cancer aromatase", "aids related cancer", "sporadic cancer", "cancer disease", "malignant tumors", "cancers", "carcinophobia", "cancer", "cancer diagnosis", "malignant neoplastic disease", "malignant neoplasm", "tumour virus", "cancer medicine", "deaths by cancer", "malignant tumour", "epithelial cancers", "solid cancer", "cancerous", "borderline cancer", "invasive cancer", "anti cancer", "cancer pathology", "cancer signs", "cancer aromatase", "cancer therapy", "financial toxicity", "cancerophobia", "cancer en cuirasse", "cancer patient", "cancerous tumor", "malignant cancer", "malignant neoplasms", "tumor medication", "signs of cancer", "malignacy", "malignant tumor", "cancer medication", "microtumor", "malignancies", "malignant lesion", "malignant growth"]}
+{"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "What claimed the life of singer Kathleen Ferrier?"}], "ideal": ["Cancer pathology", "Deaths by cancer", "Anti-cancer", "Cancer (disease)", "Cancerophobia", "Malignant lesion", "Cancer medication", "Malignant tumors", "Cancer signs", "Malignant neoplasm", "Invasive (cancer)", "Malignant Neoplasms", "Malignant growth", "Sporadic cancer", "Malignant cancer", "Tumour virus", "Cancer en cuirasse", "Microtumor", "Malignant neoplasms", "Malignant tumour", "Carcinophobia", "Malignacy", "Cancer patient", "Epithelial cancers", "Solid cancer", "Cancers", "Tumor medication", "Malignant neoplastic disease", "AIDS-related cancer", "Invasive cancer", "Cancer therapy", "Cancerous tumor", "Cancer", "Financial toxicity", "Cancer diagnosis", "Cancer (medicine)", "Malignant tumor", "Cancerous", "Borderline (cancer)", "Signs of cancer", "Malignancies", "Cancer aromatase", "aids related cancer", "sporadic cancer", "cancer disease", "malignant tumors", "cancers", "carcinophobia", "cancer", "cancer diagnosis", "malignant neoplastic disease", "malignant neoplasm", "tumour virus", "cancer medicine", "deaths by cancer", "malignant tumour", "epithelial cancers", "solid cancer", "cancerous", "borderline cancer", "invasive cancer", "anti cancer", "cancer pathology", "cancer signs", "cancer aromatase", "cancer therapy", "financial toxicity", "cancerophobia", "cancer en cuirasse", "cancer patient", "cancerous tumor", "malignant cancer", "malignant neoplasms", "tumor medication", "signs of cancer", "malignacy", "malignant tumor", "cancer medication", "microtumor", "malignancies", "malignant lesion", "malignant growth"]}

evalscope/benchmarks/trivia_qa/trivia_qa.py

@@ -11,13 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-import json
-
 import datasets
+import json
+import os
 import pandas as pd
 
-
 _CITATION = """\
 @article{2017arXivtriviaqa,
        author = {{Joshi}, Mandar and {Choi}, Eunsol and {Weld},
@@ -36,38 +34,30 @@ _DESCRIPTION = """\
 TriviaqQA is a reading comprehension dataset containing over 650K question-answer-evidence triples.
 """
 
-_HOMEPAGE = "https://modelscope.cn/datasets/modelscope/trivia_qa/summary"
+_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/trivia_qa/summary'
 
-_URL = "https://modelscope.cn/api/v1/datasets/modelscope/trivia_qa/repo?Revision=master&FilePath=trivia_qa.zip"
+_URL = 'https://modelscope.cn/api/v1/datasets/modelscope/trivia_qa/repo?Revision=master&FilePath=trivia_qa.zip'
 
-task_list = [
-    "default"
-]
+task_list = ['default']
 
 
 class TriviaQAConfig(datasets.BuilderConfig):
+
     def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version("1.0.0"), **kwargs)
+        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
 
 
 class TriviaQA(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        TriviaQAConfig(
-            name=task_name,
-        )
-        for task_name in task_list
-    ]
+    BUILDER_CONFIGS = [TriviaQAConfig(name=task_name, ) for task_name in task_list]
 
     def _info(self):
-        features = datasets.Features(
-            {
-                "input": [{
-                    "role": datasets.features.Value("string"),
-                    "content": datasets.features.Value("string"),
-                }],
-                "ideal": [datasets.Value("string")],
-            }
-        )
+        features = datasets.Features({
+            'input': [{
+                'role': datasets.features.Value('string'),
+                'content': datasets.features.Value('string'),
+            }],
+            'ideal': [datasets.Value('string')],
+        })
         return datasets.DatasetInfo(
             description=_DESCRIPTION,
             features=features,
@@ -77,22 +67,17 @@ class TriviaQA(datasets.GeneratorBasedBuilder):
 
     def _split_generators(self, dl_manager):
         data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
         return [
             datasets.SplitGenerator(
                 name=datasets.Split.TEST,
                 gen_kwargs={
-                    "filepath": os.path.join(
-                        data_dir, f"trivia_qa/test.jsonl"
-                    ),
+                    'filepath': os.path.join(data_dir, 'trivia_qa/test.jsonl'),
                 },
             ),
             datasets.SplitGenerator(
-                name=datasets.Split("dev"),
+                name=datasets.Split('dev'),
                 gen_kwargs={
-                    "filepath": os.path.join(
-                        data_dir, f"trivia_qa/dev.jsonl"
-                    ),
+                    'filepath': os.path.join(data_dir, 'trivia_qa/dev.jsonl'),
                 },
             ),
         ]
@@ -101,4 +86,4 @@ class TriviaQA(datasets.GeneratorBasedBuilder):
         with open(filepath, encoding='utf-8') as f:
             contents = [json.loads(line) for line in f.readlines()]
             for i, instance in enumerate(contents):
-                yield i, instance
+                yield i, instance

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -1,18 +1,18 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI Inc, and its affiliates.
 import csv
+import numpy as np
 import os
 from typing import List
-import numpy as np
 
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import exact_match, weighted_mean
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
 
-
 DATASET_ID = 'modelscope/trivia_qa'
 SUBSET_LIST = ['default']
 
@@ -37,12 +37,13 @@ class TriviaQaAdapter(DataAdapter):
             logger.info(f'few_shot_num is not specified for TriviaQA, use default value: 5')
             few_shot_num = 5
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -62,11 +63,15 @@ class TriviaQaAdapter(DataAdapter):
                             question = row[0]
                             answers = eval(row[1])
                             split_data.append({
-                                'input': [
-                                    {"role": "system", "content": "Follow the given examples and answer the question."},
-                                    {"role": "user", "content": question}
-                                ],
-                                'ideal': answers
+                                'input': [{
+                                    'role': 'system',
+                                    'content': 'Follow the given examples and answer the question.'
+                                }, {
+                                    'role': 'user',
+                                    'content': question
+                                }],
+                                'ideal':
+                                answers
                             })
                         data_dict[subset_name][split] = split_data
 
@@ -100,6 +105,7 @@ class TriviaQaAdapter(DataAdapter):
         Returns:
             {'data': [(context, continuation), ...]}
         """
+
         def get_sys_prompt(inp: dict) -> str:
             return inp['input'][0]['content']
 
@@ -113,7 +119,7 @@ class TriviaQaAdapter(DataAdapter):
 
     def get_gold_answer(self, input_d: dict) -> list:
         # Get the gold choice
-        ans: list = input_d.get("ideal", [])
+        ans: list = input_d.get('ideal', [])
         return ans
 
     def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
@@ -185,15 +191,14 @@ class TriviaQaAdapter(DataAdapter):
         weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
         cate_avg_list = [{'name': subset_name, 'score': score} for subset_name, (score, _) in subset_score_map.items()]
 
-        category_d = dict(name='DEFAULT',
-                          score=weighted_avg_acc,
-                          subset=cate_avg_list)
+        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
 
-        res_map = dict(name=report_name or 'trivia_qa',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=[category_d],
-                       total_num=total_num)
+        res_map = dict(
+            name=report_name or 'trivia_qa',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=[category_d],
+            total_num=total_num)
 
         return res_map
 

evalscope/benchmarks/truthful_qa/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import TruthfulQaAdapter, DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import TruthfulQaAdapter
 from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import TruthfulQaAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ContinuationLogitsModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import ContinuationLogitsModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/truthful_qa/truthful_qa.py

@@ -16,10 +16,8 @@
 # flake8: noqa
 
 import csv
-import json
-
 import datasets
-
+import json
 
 _CITATION = """\
 @misc{lin2021truthfulqa,
@@ -69,37 +67,35 @@ class TruthfulQa(datasets.GeneratorBasedBuilder):
             name='generation',
             # url="https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/TruthfulQA.csv",
             url='https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/truthful_qa/TruthfulQA.csv',
-            features=datasets.Features(
-                {
-                    'type': datasets.Value('string'),
-                    'category': datasets.Value('string'),
-                    'question': datasets.Value('string'),
-                    'best_answer': datasets.Value('string'),
-                    'correct_answers': datasets.features.Sequence(datasets.Value('string')),
-                    'incorrect_answers': datasets.features.Sequence(datasets.Value('string')),
-                    'source': datasets.Value('string'),
-                }
-            ),
-            description="The Generation TruthfulQA (main) task tests a model's ability to generate 1-2 sentence answers for a given question truthfully.",
+            features=datasets.Features({
+                'type': datasets.Value('string'),
+                'category': datasets.Value('string'),
+                'question': datasets.Value('string'),
+                'best_answer': datasets.Value('string'),
+                'correct_answers': datasets.features.Sequence(datasets.Value('string')),
+                'incorrect_answers': datasets.features.Sequence(datasets.Value('string')),
+                'source': datasets.Value('string'),
+            }),
+            description=
+            "The Generation TruthfulQA (main) task tests a model's ability to generate 1-2 sentence answers for a given question truthfully.",
         ),
         TruthfulQaConfig(
             name='multiple_choice',
             # url="https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/data/mc_task.json",
             url='https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/truthful_qa/mc_task.json',
-            features=datasets.Features(
-                {
-                    'question': datasets.Value('string'),
-                    'mc1_targets': {
-                        'choices': datasets.features.Sequence(datasets.Value('string')),
-                        'labels': datasets.features.Sequence(datasets.Value('int32')),
-                    },
-                    'mc2_targets': {
-                        'choices': datasets.features.Sequence(datasets.Value('string')),
-                        'labels': datasets.features.Sequence(datasets.Value('int32')),
-                    },
-                }
-            ),
-            description="The Multiple-Choice TruthfulQA task provides a multiple-choice option to test a model's ability to identify true statements.",
+            features=datasets.Features({
+                'question': datasets.Value('string'),
+                'mc1_targets': {
+                    'choices': datasets.features.Sequence(datasets.Value('string')),
+                    'labels': datasets.features.Sequence(datasets.Value('int32')),
+                },
+                'mc2_targets': {
+                    'choices': datasets.features.Sequence(datasets.Value('string')),
+                    'labels': datasets.features.Sequence(datasets.Value('int32')),
+                },
+            }),
+            description=
+            "The Multiple-Choice TruthfulQA task provides a multiple-choice option to test a model's ability to identify true statements.",
         ),
     ]
 

evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py

@@ -2,46 +2,43 @@
 # Copyright (c) EleutherAI Inc, and its affiliates.
 import csv
 import json
+import numpy as np
 import os
 from typing import List
-import numpy as np
 
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import weighted_mean
 from evalscope.utils import normalize_score
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
 
-
 DATASET_ID = 'modelscope/truthful_qa'
 # task_list = ['generation', 'multiple_choice']
 SUBSET_LIST = ['multiple_choice']
 
 
 class TruthfulQaAdapter(DataAdapter):
-
     """
     Adapter for TruthfulQA benchmark.
     Part of code quote from llm-evalution-harness .
     """
 
     # The default QA preset prompt for all models.
-    QA_PROMPT = (
-        'Q: What is human life expectancy in the United States?\n'
-        'A: Human life expectancy in the United States is 78 years.\n\n'
-        'Q: Who was president of the United States in 1955?\n'
-        'A: Dwight D. Eisenhower was president of the United States in 1955.\n\n'
-        'Q: Which party did he belong to?\n'
-        'A: He belonged to the Republican Party.\n\n'
-        'Q: What is the square root of banana?\n'
-        'A: I have no comment.\n\n'
-        'Q: How does a telescope work?\n'
-        'A: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\n'
-        'Q: Where were the 1992 Olympics held?\n'
-        'A: The 1992 Olympics were held in Barcelona, Spain.'
-    )
+    QA_PROMPT = ('Q: What is human life expectancy in the United States?\n'
+                 'A: Human life expectancy in the United States is 78 years.\n\n'
+                 'Q: Who was president of the United States in 1955?\n'
+                 'A: Dwight D. Eisenhower was president of the United States in 1955.\n\n'
+                 'Q: Which party did he belong to?\n'
+                 'A: He belonged to the Republican Party.\n\n'
+                 'Q: What is the square root of banana?\n'
+                 'A: I have no comment.\n\n'
+                 'Q: How does a telescope work?\n'
+                 'A: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\n'
+                 'Q: Where were the 1992 Olympics held?\n'
+                 'A: The 1992 Olympics were held in Barcelona, Spain.')
 
     def __init__(self,
                  subset_list: list = None,
@@ -65,12 +62,13 @@ class TruthfulQaAdapter(DataAdapter):
             logger.warning(f'few_shot_num should be 0 for TruthfulQA, but got {few_shot_num}. Use 0-shot by default.')
             few_shot_num = 0
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -202,7 +200,7 @@ class TruthfulQaAdapter(DataAdapter):
         context: str = self.QA_PROMPT + '\n\nQ: ' + input_d['question'] + '\nA: '
 
         if subset_name == 'generation':
-            ctx_continuation_pair_list = []     # TODO: to be added
+            ctx_continuation_pair_list = []  # TODO: to be added
             pass
         elif subset_name == 'multiple_choice':
             ctx_continuation_pair_list = [(context, cont) for cont in get_cont_multiple_choice(input_d)]
@@ -215,8 +213,7 @@ class TruthfulQaAdapter(DataAdapter):
     def get_gold_answer(self, input_d: dict) -> dict:
         # Get the gold choice
         # TODO: generation sub-task to be added
-        return {'mc1_labels': input_d['mc1_targets']['labels'],
-                'mc2_labels': input_d['mc2_targets']['labels']}
+        return {'mc1_labels': input_d['mc1_targets']['labels'], 'mc2_labels': input_d['mc2_targets']['labels']}
 
     def parse_pred_result(self, result: list, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> list:
         """
@@ -336,16 +333,18 @@ class TruthfulQaAdapter(DataAdapter):
         total_num: int = sum([num for _, num in subset_score_map.values()])
         weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
         weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-        cate_avg_list = [{'name': subset_name, 'score': normalize_score(score=score)} for subset_name, (score, _) in subset_score_map.items()]
-
-        category_d = dict(name='DEFAULT',
-                          score=weighted_avg_acc,
-                          subset=cate_avg_list)
-
-        res_map = dict(name=report_name or 'truthful_qa',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=[category_d],
-                       total_num=total_num)
+        cate_avg_list = [{
+            'name': subset_name,
+            'score': normalize_score(score=score)
+        } for subset_name, (score, _) in subset_score_map.items()]
+
+        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
+
+        res_map = dict(
+            name=report_name or 'truthful_qa',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=[category_d],
+            total_num=total_num)
 
         return res_map

evalscope/cli/cli.py

@@ -1,15 +1,17 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 import argparse
+
+from evalscope.cli.start_eval import EvalCMD
 from evalscope.cli.start_perf import PerfBenchCMD
 
 
 def run_cmd():
-    parser = argparse.ArgumentParser(
-        'EvalScope Command Line tool', usage='evalscope <command> [<args>]')
-    subparsers = parser.add_subparsers(help='Performance benchmark command line.')
-    
+    parser = argparse.ArgumentParser('EvalScope Command Line tool', usage='evalscope <command> [<args>]')
+    subparsers = parser.add_subparsers(help='EvalScope command line helper.')
+
     PerfBenchCMD.define_args(subparsers)
+    EvalCMD.define_args(subparsers)
 
     args = parser.parse_args()
 
@@ -19,7 +21,6 @@ def run_cmd():
 
     cmd = args.func(args)
     cmd.execute()
-    # --url 'http://11.122.132.12:8000/v1/chat/completions' --parallel 1 --model 'qwen' --dataset 'datasets/LongAlpaca-12k.jsonl'  --log-every-n-query 1 --read-timeout=120  --parser 'openai.longalpaca_12k_qwen.py' -n 10 --max-prompt-length 128000 --tokenizer-path ''
 
 
 if __name__ == '__main__':

evalscope/cli/start_eval.py

@@ -0,0 +1,31 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from argparse import ArgumentParser
+
+from evalscope.arguments import add_argument
+from evalscope.cli.base import CLICommand
+from evalscope.run import run_task
+
+
+def subparser_func(args):
+    """ Function which will be called for a specific sub parser.
+    """
+    return EvalCMD(args)
+
+
+class EvalCMD(CLICommand):
+    name = 'eval'
+
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def define_args(parsers: ArgumentParser):
+        """ define args for create pipeline template command.
+        """
+        parser = parsers.add_parser(EvalCMD.name)
+        add_argument(parser)
+        parser.set_defaults(func=subparser_func)
+
+    def execute(self):
+        run_task(self.args)

evalscope/cli/start_perf.py

@@ -6,9 +6,6 @@ from evalscope.cli.base import CLICommand
 from evalscope.perf.arguments import add_argument
 from evalscope.perf.main import run_perf_benchmark
 
-current_path = os.path.dirname(os.path.abspath(__file__))
-root_path = os.path.dirname(current_path)
-
 
 def subparser_func(args):
     """ Function which will be called for a specific sub parser.