evalscope 0.7.2__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/benchmarks/trivia_qa/trivia_qa.py

@@ -11,13 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-import json
-
 import datasets
+import json
+import os
 import pandas as pd
 
-
 _CITATION = """\
 @article{2017arXivtriviaqa,
        author = {{Joshi}, Mandar and {Choi}, Eunsol and {Weld},
@@ -36,38 +34,30 @@ _DESCRIPTION = """\
 TriviaqQA is a reading comprehension dataset containing over 650K question-answer-evidence triples.
 """
 
-_HOMEPAGE = "https://modelscope.cn/datasets/modelscope/trivia_qa/summary"
+_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/trivia_qa/summary'
 
-_URL = "https://modelscope.cn/api/v1/datasets/modelscope/trivia_qa/repo?Revision=master&FilePath=trivia_qa.zip"
+_URL = 'https://modelscope.cn/api/v1/datasets/modelscope/trivia_qa/repo?Revision=master&FilePath=trivia_qa.zip'
 
-task_list = [
-    "default"
-]
+task_list = ['default']
 
 
 class TriviaQAConfig(datasets.BuilderConfig):
+
     def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version("1.0.0"), **kwargs)
+        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
 
 
 class TriviaQA(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        TriviaQAConfig(
-            name=task_name,
-        )
-        for task_name in task_list
-    ]
+    BUILDER_CONFIGS = [TriviaQAConfig(name=task_name, ) for task_name in task_list]
 
     def _info(self):
-        features = datasets.Features(
-            {
-                "input": [{
-                    "role": datasets.features.Value("string"),
-                    "content": datasets.features.Value("string"),
-                }],
-                "ideal": [datasets.Value("string")],
-            }
-        )
+        features = datasets.Features({
+            'input': [{
+                'role': datasets.features.Value('string'),
+                'content': datasets.features.Value('string'),
+            }],
+            'ideal': [datasets.Value('string')],
+        })
         return datasets.DatasetInfo(
             description=_DESCRIPTION,
             features=features,
@@ -77,22 +67,17 @@ class TriviaQA(datasets.GeneratorBasedBuilder):
 
     def _split_generators(self, dl_manager):
         data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
         return [
             datasets.SplitGenerator(
                 name=datasets.Split.TEST,
                 gen_kwargs={
-                    "filepath": os.path.join(
-                        data_dir, f"trivia_qa/test.jsonl"
-                    ),
+                    'filepath': os.path.join(data_dir, 'trivia_qa/test.jsonl'),
                 },
             ),
             datasets.SplitGenerator(
-                name=datasets.Split("dev"),
+                name=datasets.Split('dev'),
                 gen_kwargs={
-                    "filepath": os.path.join(
-                        data_dir, f"trivia_qa/dev.jsonl"
-                    ),
+                    'filepath': os.path.join(data_dir, 'trivia_qa/dev.jsonl'),
                 },
             ),
         ]
@@ -101,4 +86,4 @@ class TriviaQA(datasets.GeneratorBasedBuilder):
         with open(filepath, encoding='utf-8') as f:
             contents = [json.loads(line) for line in f.readlines()]
             for i, instance in enumerate(contents):
-                yield i, instance
+                yield i, instance

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -1,18 +1,18 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI Inc, and its affiliates.
 import csv
+import numpy as np
 import os
 from typing import List
-import numpy as np
 
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import exact_match, weighted_mean
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
 
-
 DATASET_ID = 'modelscope/trivia_qa'
 SUBSET_LIST = ['default']
 
@@ -37,12 +37,13 @@ class TriviaQaAdapter(DataAdapter):
             logger.info(f'few_shot_num is not specified for TriviaQA, use default value: 5')
             few_shot_num = 5
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -62,11 +63,15 @@ class TriviaQaAdapter(DataAdapter):
                             question = row[0]
                             answers = eval(row[1])
                             split_data.append({
-                                'input': [
-                                    {"role": "system", "content": "Follow the given examples and answer the question."},
-                                    {"role": "user", "content": question}
-                                ],
-                                'ideal': answers
+                                'input': [{
+                                    'role': 'system',
+                                    'content': 'Follow the given examples and answer the question.'
+                                }, {
+                                    'role': 'user',
+                                    'content': question
+                                }],
+                                'ideal':
+                                answers
                             })
                         data_dict[subset_name][split] = split_data
 
@@ -100,6 +105,7 @@ class TriviaQaAdapter(DataAdapter):
         Returns:
             {'data': [(context, continuation), ...]}
         """
+
         def get_sys_prompt(inp: dict) -> str:
             return inp['input'][0]['content']
 
@@ -113,7 +119,7 @@ class TriviaQaAdapter(DataAdapter):
 
     def get_gold_answer(self, input_d: dict) -> list:
         # Get the gold choice
-        ans: list = input_d.get("ideal", [])
+        ans: list = input_d.get('ideal', [])
         return ans
 
     def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
@@ -185,15 +191,14 @@ class TriviaQaAdapter(DataAdapter):
         weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
         cate_avg_list = [{'name': subset_name, 'score': score} for subset_name, (score, _) in subset_score_map.items()]
 
-        category_d = dict(name='DEFAULT',
-                          score=weighted_avg_acc,
-                          subset=cate_avg_list)
+        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
 
-        res_map = dict(name=report_name or 'trivia_qa',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=[category_d],
-                       total_num=total_num)
+        res_map = dict(
+            name=report_name or 'trivia_qa',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=[category_d],
+            total_num=total_num)
 
         return res_map
 

evalscope/benchmarks/truthful_qa/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import TruthfulQaAdapter, DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import TruthfulQaAdapter
 from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import TruthfulQaAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ContinuationLogitsModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import ContinuationLogitsModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/truthful_qa/truthful_qa.py

@@ -16,10 +16,8 @@
 # flake8: noqa
 
 import csv
-import json
-
 import datasets
-
+import json
 
 _CITATION = """\
 @misc{lin2021truthfulqa,
@@ -69,37 +67,35 @@ class TruthfulQa(datasets.GeneratorBasedBuilder):
             name='generation',
             # url="https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/TruthfulQA.csv",
             url='https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/truthful_qa/TruthfulQA.csv',
-            features=datasets.Features(
-                {
-                    'type': datasets.Value('string'),
-                    'category': datasets.Value('string'),
-                    'question': datasets.Value('string'),
-                    'best_answer': datasets.Value('string'),
-                    'correct_answers': datasets.features.Sequence(datasets.Value('string')),
-                    'incorrect_answers': datasets.features.Sequence(datasets.Value('string')),
-                    'source': datasets.Value('string'),
-                }
-            ),
-            description="The Generation TruthfulQA (main) task tests a model's ability to generate 1-2 sentence answers for a given question truthfully.",
+            features=datasets.Features({
+                'type': datasets.Value('string'),
+                'category': datasets.Value('string'),
+                'question': datasets.Value('string'),
+                'best_answer': datasets.Value('string'),
+                'correct_answers': datasets.features.Sequence(datasets.Value('string')),
+                'incorrect_answers': datasets.features.Sequence(datasets.Value('string')),
+                'source': datasets.Value('string'),
+            }),
+            description=
+            "The Generation TruthfulQA (main) task tests a model's ability to generate 1-2 sentence answers for a given question truthfully.",
         ),
         TruthfulQaConfig(
             name='multiple_choice',
             # url="https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/data/mc_task.json",
             url='https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/truthful_qa/mc_task.json',
-            features=datasets.Features(
-                {
-                    'question': datasets.Value('string'),
-                    'mc1_targets': {
-                        'choices': datasets.features.Sequence(datasets.Value('string')),
-                        'labels': datasets.features.Sequence(datasets.Value('int32')),
-                    },
-                    'mc2_targets': {
-                        'choices': datasets.features.Sequence(datasets.Value('string')),
-                        'labels': datasets.features.Sequence(datasets.Value('int32')),
-                    },
-                }
-            ),
-            description="The Multiple-Choice TruthfulQA task provides a multiple-choice option to test a model's ability to identify true statements.",
+            features=datasets.Features({
+                'question': datasets.Value('string'),
+                'mc1_targets': {
+                    'choices': datasets.features.Sequence(datasets.Value('string')),
+                    'labels': datasets.features.Sequence(datasets.Value('int32')),
+                },
+                'mc2_targets': {
+                    'choices': datasets.features.Sequence(datasets.Value('string')),
+                    'labels': datasets.features.Sequence(datasets.Value('int32')),
+                },
+            }),
+            description=
+            "The Multiple-Choice TruthfulQA task provides a multiple-choice option to test a model's ability to identify true statements.",
         ),
     ]
 

evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py

@@ -2,46 +2,43 @@
 # Copyright (c) EleutherAI Inc, and its affiliates.
 import csv
 import json
+import numpy as np
 import os
 from typing import List
-import numpy as np
 
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import weighted_mean
 from evalscope.utils import normalize_score
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
 
-
 DATASET_ID = 'modelscope/truthful_qa'
 # task_list = ['generation', 'multiple_choice']
 SUBSET_LIST = ['multiple_choice']
 
 
 class TruthfulQaAdapter(DataAdapter):
-
     """
     Adapter for TruthfulQA benchmark.
     Part of code quote from llm-evalution-harness .
     """
 
     # The default QA preset prompt for all models.
-    QA_PROMPT = (
-        'Q: What is human life expectancy in the United States?\n'
-        'A: Human life expectancy in the United States is 78 years.\n\n'
-        'Q: Who was president of the United States in 1955?\n'
-        'A: Dwight D. Eisenhower was president of the United States in 1955.\n\n'
-        'Q: Which party did he belong to?\n'
-        'A: He belonged to the Republican Party.\n\n'
-        'Q: What is the square root of banana?\n'
-        'A: I have no comment.\n\n'
-        'Q: How does a telescope work?\n'
-        'A: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\n'
-        'Q: Where were the 1992 Olympics held?\n'
-        'A: The 1992 Olympics were held in Barcelona, Spain.'
-    )
+    QA_PROMPT = ('Q: What is human life expectancy in the United States?\n'
+                 'A: Human life expectancy in the United States is 78 years.\n\n'
+                 'Q: Who was president of the United States in 1955?\n'
+                 'A: Dwight D. Eisenhower was president of the United States in 1955.\n\n'
+                 'Q: Which party did he belong to?\n'
+                 'A: He belonged to the Republican Party.\n\n'
+                 'Q: What is the square root of banana?\n'
+                 'A: I have no comment.\n\n'
+                 'Q: How does a telescope work?\n'
+                 'A: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\n'
+                 'Q: Where were the 1992 Olympics held?\n'
+                 'A: The 1992 Olympics were held in Barcelona, Spain.')
 
     def __init__(self,
                  subset_list: list = None,
@@ -65,12 +62,13 @@ class TruthfulQaAdapter(DataAdapter):
             logger.warning(f'few_shot_num should be 0 for TruthfulQA, but got {few_shot_num}. Use 0-shot by default.')
             few_shot_num = 0
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -202,7 +200,7 @@ class TruthfulQaAdapter(DataAdapter):
         context: str = self.QA_PROMPT + '\n\nQ: ' + input_d['question'] + '\nA: '
 
         if subset_name == 'generation':
-            ctx_continuation_pair_list = []     # TODO: to be added
+            ctx_continuation_pair_list = []  # TODO: to be added
             pass
         elif subset_name == 'multiple_choice':
             ctx_continuation_pair_list = [(context, cont) for cont in get_cont_multiple_choice(input_d)]
@@ -215,8 +213,7 @@ class TruthfulQaAdapter(DataAdapter):
     def get_gold_answer(self, input_d: dict) -> dict:
         # Get the gold choice
         # TODO: generation sub-task to be added
-        return {'mc1_labels': input_d['mc1_targets']['labels'],
-                'mc2_labels': input_d['mc2_targets']['labels']}
+        return {'mc1_labels': input_d['mc1_targets']['labels'], 'mc2_labels': input_d['mc2_targets']['labels']}
 
     def parse_pred_result(self, result: list, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> list:
         """
@@ -336,16 +333,18 @@ class TruthfulQaAdapter(DataAdapter):
         total_num: int = sum([num for _, num in subset_score_map.values()])
         weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
         weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-        cate_avg_list = [{'name': subset_name, 'score': normalize_score(score=score)} for subset_name, (score, _) in subset_score_map.items()]
-
-        category_d = dict(name='DEFAULT',
-                          score=weighted_avg_acc,
-                          subset=cate_avg_list)
-
-        res_map = dict(name=report_name or 'truthful_qa',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=[category_d],
-                       total_num=total_num)
+        cate_avg_list = [{
+            'name': subset_name,
+            'score': normalize_score(score=score)
+        } for subset_name, (score, _) in subset_score_map.items()]
+
+        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
+
+        res_map = dict(
+            name=report_name or 'truthful_qa',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=[category_d],
+            total_num=total_num)
 
         return res_map

evalscope/cli/cli.py

@@ -1,15 +1,17 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 import argparse
+
+from evalscope.cli.start_eval import EvalCMD
 from evalscope.cli.start_perf import PerfBenchCMD
 
 
 def run_cmd():
-    parser = argparse.ArgumentParser(
-        'EvalScope Command Line tool', usage='evalscope <command> [<args>]')
-    subparsers = parser.add_subparsers(help='Performance benchmark command line.')
-    
+    parser = argparse.ArgumentParser('EvalScope Command Line tool', usage='evalscope <command> [<args>]')
+    subparsers = parser.add_subparsers(help='EvalScope command line helper.')
+
     PerfBenchCMD.define_args(subparsers)
+    EvalCMD.define_args(subparsers)
 
     args = parser.parse_args()
 
@@ -19,7 +21,6 @@ def run_cmd():
 
     cmd = args.func(args)
     cmd.execute()
-    # --url 'http://11.122.132.12:8000/v1/chat/completions' --parallel 1 --model 'qwen' --dataset 'datasets/LongAlpaca-12k.jsonl'  --log-every-n-query 1 --read-timeout=120  --parser 'openai.longalpaca_12k_qwen.py' -n 10 --max-prompt-length 128000 --tokenizer-path ''
 
 
 if __name__ == '__main__':

evalscope/cli/start_eval.py

@@ -0,0 +1,31 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from argparse import ArgumentParser
+
+from evalscope.arguments import add_argument
+from evalscope.cli.base import CLICommand
+from evalscope.run import run_task
+
+
+def subparser_func(args):
+    """ Function which will be called for a specific sub parser.
+    """
+    return EvalCMD(args)
+
+
+class EvalCMD(CLICommand):
+    name = 'eval'
+
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def define_args(parsers: ArgumentParser):
+        """ define args for create pipeline template command.
+        """
+        parser = parsers.add_parser(EvalCMD.name)
+        add_argument(parser)
+        parser.set_defaults(func=subparser_func)
+
+    def execute(self):
+        run_task(self.args)

evalscope/cli/start_perf.py

@@ -6,9 +6,6 @@ from evalscope.cli.base import CLICommand
 from evalscope.perf.arguments import add_argument
 from evalscope.perf.main import run_perf_benchmark
 
-current_path = os.path.dirname(os.path.abspath(__file__))
-root_path = os.path.dirname(current_path)
-
 
 def subparser_func(args):
     """ Function which will be called for a specific sub parser.

evalscope/cli/start_server.py

@@ -1,67 +1,56 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os, sys, time
-from argparse import ArgumentParser
+import os
 import subprocess
-
+import sys
+import time
+from argparse import ArgumentParser
 
 from evalscope.cli.base import CLICommand
 
-
 current_path = os.path.dirname(os.path.abspath(__file__))
 print(current_path)
 root_path = os.path.dirname(current_path)
 print(root_path)
 
+
 def subparser_func(args):
     """ Function which will be called for a specific sub parser.
     """
     return PerfServerCMD(args)
 
+
 def add_perf_args(parser):
+    parser.add_argument('--server-command', required=True, type=str, help='The start server command.')
     parser.add_argument(
-        '--server-command', required=True, type=str, help='The start server command.')
-    parser.add_argument(
-        '--logdir', required=True, type=str, help='The monitor log save dir, tensorboard start at this path for display!')
-    parser.add_argument(
-        '--host', type=str, default='0.0.0.0', help='The tensorboard host'
-    )    
-    parser.add_argument(
-        '--tensorboard-port', type=str, default='6006', help='The tensorboard port'
-    )
+        '--logdir',
+        required=True,
+        type=str,
+        help='The monitor log save dir, tensorboard start at this path for display!')
+    parser.add_argument('--host', type=str, default='0.0.0.0', help='The tensorboard host')
+    parser.add_argument('--tensorboard-port', type=str, default='6006', help='The tensorboard port')
+
 
 def async_run_command_with_popen(cmd):
     sub_process = subprocess.Popen(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        bufsize=1,
-        universal_newlines=True,
-        encoding='utf8')
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True, encoding='utf8')
     return sub_process
 
+
 def start_monitor(args):
-    cmd = ['python',
-           '%s/perf/monitor.py'%root_path,
-           '--logdir',
-           args.logdir]
+    cmd = ['python', '%s/perf/monitor.py' % root_path, '--logdir', args.logdir]
     print(cmd)
     p = async_run_command_with_popen(cmd)
     os.set_blocking(p.stdout.fileno(), False)
     return p
 
+
 def start_tensorboard(args):
-    cmd = ['tensorboard',
-           '--logdir',
-           args.logdir,
-           '--host',
-           args.host,
-           '--port',
-           args.tensorboard_port
-           ]
+    cmd = ['tensorboard', '--logdir', args.logdir, '--host', args.host, '--port', args.tensorboard_port]
     p = async_run_command_with_popen(cmd)
     os.set_blocking(p.stdout.fileno(), False)
     return p
 
+
 def start_server(args):
     cmd = args.server_command
     print(cmd)
@@ -76,7 +65,7 @@ def start_server(args):
 
     os.set_blocking(sub_process.stdout.fileno(), False)
     return sub_process
- 
+
 
 def wait_for_workers(workers):
     while True:
@@ -91,12 +80,12 @@ def wait_for_workers(workers):
                     else:
                         break
             else:
-                print('Worker %s completed!'%idx)
+                print('Worker %s completed!' % idx)
                 for line in iter(worker.stdout.readline, ''):
                     if line != '':
                         sys.stdout.write(line)
                     else:
-                        break                
+                        break
                 workers[idx] = None
 
         is_all_completed = True
@@ -108,7 +97,8 @@ def wait_for_workers(workers):
         if is_all_completed:
             break
         time.sleep(0.1)
-        
+
+
 class PerfServerCMD(CLICommand):
     name = 'server'
 
@@ -127,12 +117,8 @@ class PerfServerCMD(CLICommand):
         # start monitor
         p_monitor = start_monitor(self.args)
         # start tensorboard
-        p_tensorboard = start_tensorboard(self.args)        
+        p_tensorboard = start_tensorboard(self.args)
         # start server
         p_server = start_server(self.args)
-        
+
         wait_for_workers([p_monitor, p_tensorboard, p_server])
-        
-        
-        
-