evalscope 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

evalscope/perf/plugin/api/openai_api.py

@@ -1,7 +1,7 @@
 import json
 import os
 from transformers import AutoTokenizer
-from typing import Any, Dict, Iterator, List
+from typing import Any, Dict, Iterator, List, Union
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.api.base import ApiPluginBase
@@ -29,7 +29,7 @@ class OpenaiPlugin(ApiPluginBase):
         else:
             self.tokenizer = None
 
-    def build_request(self, messages: List[Dict] | str, param: Arguments) -> Dict:
+    def build_request(self, messages: Union[List[Dict], str], param: Arguments) -> Dict:
         """Build the openai format request based on prompt, dataset
 
         Args:
@@ -96,60 +96,64 @@ class OpenaiPlugin(ApiPluginBase):
 
     def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
         """Parser responses and return number of request and response tokens.
-           sample of the output delta:
-           {"id":"4","object":"chat.completion.chunk","created":1714030870,"model":"llama3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
+        Only one response for non-stream, multiple responses for stream.
+        """
 
+        # when stream, the last response is the full usage
+        # when non-stream, the last response is the first response
+        last_response_js = json.loads(responses[-1])
+        if 'usage' in last_response_js and last_response_js['usage']:
+            input_tokens = last_response_js['usage']['prompt_tokens']
+            output_tokens = last_response_js['usage']['completion_tokens']
+            return input_tokens, output_tokens
 
-        Args:
-            responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, for general only one.
-            kwargs: (Any): The command line --parameter content.
-        Returns:
-            Tuple: Return number of prompt token and number of completion tokens.
-        """
-        full_response_content = ''
+        # no usage information in the response, parse the response to get the tokens
         delta_contents = {}
-        input_tokens = None
-        output_tokens = None
         for response in responses:
             js = json.loads(response)
-            if js['object'] == 'chat.completion':
-                for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['message']['content']]
-                input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']
-            elif js['object'] == 'text_completion':
-                for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['text']]
-                input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']
-            elif js['object'] == 'chat.completion.chunk':
-                if 'choices' in js:
-                    for choice in js['choices']:
-                        if 'delta' in choice and 'index' in choice:
-                            delta = choice['delta']
-                            idx = choice['index']
-                            if 'content' in delta:
-                                delta_content = delta['content']
-                                if idx in delta_contents:
-                                    delta_contents[idx].append(delta_content)
-                                else:
-                                    delta_contents[idx] = [delta_content]
-                # usage in chunk: {"id":"","object":"chat.completion.chunk","created":1718269986,"model":"llama3",
-                # "choices":[],"usage":{"prompt_tokens":32,"total_tokens":384,"completion_tokens":352}}
-                if 'usage' in js and js['usage']:
-                    input_tokens = js['usage']['prompt_tokens']
-                    output_tokens = js['usage']['completion_tokens']
-        if (input_tokens is None and output_tokens is None and self.tokenizer is not None):
-            input_tokens = 0
-            output_tokens = 0
+            if 'object' in js:
+                self.__process_response_object(js, delta_contents)
+            else:
+                self.__process_no_object(js, delta_contents)
+
+        input_tokens, output_tokens = self.__calculate_tokens_from_content(request, delta_contents)
+        return input_tokens, output_tokens
+
+    def __process_response_object(self, js, delta_contents):
+        if js['object'] == 'chat.completion':
+            for choice in js['choices']:
+                delta_contents[choice['index']] = [choice['message']['content']]
+        elif js['object'] == 'text_completion':
+            for choice in js['choices']:
+                delta_contents[choice['index']] = [choice['text']]
+        elif js['object'] == 'chat.completion.chunk':
+            for choice in js.get('choices', []):
+                if 'delta' in choice and 'index' in choice:
+                    delta = choice['delta']
+                    idx = choice['index']
+                    if 'content' in delta:
+                        delta_content = delta['content']
+                        delta_contents.setdefault(idx, []).append(delta_content)
+
+    def __process_no_object(self, js, delta_contents):
+        #  assume the response is a single choice
+        for choice in js['choices']:
+            if 'delta' in choice:
+                delta = choice['delta']
+                idx = choice['index']
+                if 'content' in delta:
+                    delta_content = delta['content']
+                    delta_contents.setdefault(idx, []).append(delta_content)
+            else:
+                delta_contents[choice['index']] = [choice['message']['content']]
+
+    def __calculate_tokens_from_content(self, request, delta_contents):
+        input_tokens = output_tokens = 0
+        if self.tokenizer is not None:
             for idx, choice_contents in delta_contents.items():
-                full_response_content = ''.join([m for m in choice_contents])
+                full_response_content = ''.join(choice_contents)
                 input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
                 output_tokens += len(self.tokenizer.encode(full_response_content))
-        elif input_tokens is None and output_tokens is None:  # no usage info get.
-            input_tokens = 0
-            output_tokens = 0
+        else:
             logger.warning('No usage information found. Please specify `--tokenizer-path` to generate usage details.')
-
         return input_tokens, output_tokens

evalscope/perf/plugin/registry.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Type
+from typing import Any, List, Type, Union
 
 
 class PluginRegistry:
@@ -20,7 +20,7 @@ class PluginRegistry:
         return self.get_class(name)
 
 
-def register_dataset(name: str | List[str]):
+def register_dataset(name: Union[str, List[str]]):
 
     def class_decorator(cls: Type):
         if isinstance(name, str):
@@ -35,7 +35,7 @@ def register_dataset(name: str | List[str]):
     return class_decorator
 
 
-def register_api(name: str | List[str]):
+def register_api(name: Union[str, List[str]]):
 
     def class_decorator(cls: Type):
         if isinstance(name, str):

evalscope/perf/utils/benchmark_util.py

@@ -116,19 +116,19 @@ class BenchmarkMetrics:
 
     def create_message(self, default_ndigits=3):
         message = {
-            'Time taken for tests (senconds)': round(self.total_time, default_ndigits),
+            'Time taken for tests (s)': round(self.total_time, default_ndigits),
             'Number of concurrency': self.concurrency,
             'Total requests': int(self.n_total_queries),
             'Succeed requests': self.n_succeed_queries,
             'Failed requests': self.n_failed_queries,
+            'Throughput(average tokens/s)': round(self.avg_token_per_seconds, default_ndigits),
             'Average QPS': round(self.qps, default_ndigits),
             'Average latency (s)': round(self.avg_latency, default_ndigits),
             'Average time to first token (s)': round(self.avg_first_chunk_latency, default_ndigits),
             'Average time per output token (s)': round(self.avg_time_per_token, 5),
-            'Average package latency (s)': round(self.avg_chunk_time, default_ndigits),
-            'Average package per request': round(self.n_avg_chunks, default_ndigits),
-            'Throughput(average output tokens per second)': round(self.avg_token_per_seconds, default_ndigits),
             'Average input tokens per request': round(self.avg_prompt_tokens, default_ndigits),
             'Average output tokens per request': round(self.avg_completion_tokens, default_ndigits),
+            'Average package latency (s)': round(self.avg_chunk_time, default_ndigits),
+            'Average package per request': round(self.n_avg_chunks, default_ndigits),
         }
         return message

evalscope/perf/utils/db_util.py

@@ -6,6 +6,7 @@ import sqlite3
 import sys
 from datetime import datetime
 from tabulate import tabulate
+from typing import Dict, List
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
@@ -107,44 +108,87 @@ def get_result_db_path(args: Arguments):
     return result_db_path
 
 
-def get_percentile_results(result_db_path: str):
+def calculate_percentiles(data: List[float], percentiles: List[int]) -> Dict[int, float]:
+    """
+    Calculate the percentiles for a specific list of data.
 
-    def percentile_results(rows, index, percentiles):
-        results = {}
-        n_success_queries = len(rows)
-        for percentile in percentiles:
+    :param data: List of values for a specific metric.
+    :param percentiles: List of percentiles to calculate.
+    :return: Dictionary of calculated percentiles.
+    """
+    results = {}
+    n_success_queries = len(data)
+    data.sort()
+    for percentile in percentiles:
+        try:
             idx = int(n_success_queries * percentile / 100)
-            row = rows[idx]
-            value = row[index] if row[index] is not None else float('inf')
+            value = data[idx] if data[idx] is not None else float('nan')
             results[percentile] = round(value, 4)
-        return results
+        except IndexError:
+            results[percentile] = float('nan')
+    return results
+
+
+def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
+    """
+    Compute and return quantiles for various metrics from the database results.
+
+    :param result_db_path: Path to the SQLite database file.
+    :return: Dictionary of percentiles for various metrics.
+    """
+
+    def inter_token_latencies(chunk_times_json: str) -> List[float]:
+        try:
+            chunk_times = json.loads(chunk_times_json)
+            return [t2 - t1 for t1, t2 in zip(chunk_times[:-1], chunk_times[1:])]
+        except (json.JSONDecodeError, TypeError) as e:
+            logger.error(f'Error parsing chunk times: {e}')
+            return []
 
     query_sql = ('SELECT start_time, chunk_times, success, completed_time, latency, first_chunk_latency, '
                  'n_chunks, chunk_time, prompt_tokens, completion_tokens '
-                 'FROM result WHERE success=1 ORDER BY first_chunk_latency ASC')
+                 'FROM result WHERE success=1')
+
     percentiles = [10, 25, 50, 66, 75, 80, 90, 95, 98, 99]
 
     with sqlite3.connect(result_db_path) as con:
         rows = con.execute(query_sql).fetchall()
 
-    if len(rows) <= len(percentiles):
+    if len(rows) < len(percentiles):
         logger.info('Too little data to calculate quantiles!')
         return {}
 
-    # Calculate percentiles for first chunk latency and latency
-    first_chunk_latency_index = 5
-    latency_index = 4
+    # Define index variables for columns
+    CHUNK_TIMES_INDEX = 1
+    LATENCY_INDEX = 4
+    FIRST_CHUNK_LATENCY_INDEX = 5
+    PROMPT_TOKENS_INDEX = 8
+    COMPLETION_TOKENS_INDEX = 9
+
+    # Prepare data for each metric
+    inter_token_latencies_all = []
+    for row in rows:
+        inter_token_latencies_all.extend(inter_token_latencies(row[CHUNK_TIMES_INDEX]))
+
+    metrics = {
+        'TTFT (s)': [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
+        'TPOT (s)':
+        inter_token_latencies_all,
+        'Latency (s)': [row[LATENCY_INDEX] for row in rows],
+        'Input tokens': [row[PROMPT_TOKENS_INDEX] for row in rows],
+        'Output tokens': [row[COMPLETION_TOKENS_INDEX] for row in rows],
+        'Throughput(tokens/s)':
+        [(row[COMPLETION_TOKENS_INDEX] / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan')
+         for row in rows]
+    }
 
-    first_chunk_latency_results = percentile_results(rows, first_chunk_latency_index, percentiles)
-    rows.sort(key=lambda x: x[latency_index])
-    latency_results = percentile_results(rows, latency_index, percentiles)
+    # Calculate percentiles for each metric
+    results = {'Percentile': [f'{p}%' for p in percentiles]}
+    for metric_name, data in metrics.items():
+        metric_percentiles = calculate_percentiles(data, percentiles)
+        results[metric_name] = [metric_percentiles[p] for p in percentiles]
 
-    # Prepare data for tabulation
-    return {
-        'Percentile': [f'{p}%' for p in percentiles],
-        'First Chunk Latency (s)': [first_chunk_latency_results[p] for p in percentiles],
-        'Latency (s)': [latency_results[p] for p in percentiles]
-    }
+    return results
 
 
 def summary_result(args: Arguments, metrics: BenchmarkMetrics, expected_number_of_queries: int, result_db_path: str):

evalscope/perf/utils/local_server.py

@@ -102,6 +102,8 @@ def start_app(args: Arguments):
 
     elif args.api == 'local_vllm':
         os.environ['VLLM_USE_MODELSCOPE'] = 'True'
+        os.environ['VLLM_ALLOW_LONG_MAX_MODEL_LEN'] = '1'
+        os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
         # yapf: disable
         proc = subprocess.Popen([
             'python', '-m', 'vllm.entrypoints.openai.api_server',
@@ -111,7 +113,8 @@ def start_app(args: Arguments):
             '--max-model-len', '32768',
             '--gpu-memory-utilization', '0.9',
             '--host', '0.0.0.0',
-            '--port', args.port,
+            '--port', str(args.port),
+            '--trust-remote-code',
             '--disable-log-requests',
             '--disable-log-stats',
         ])

evalscope/run.py

@@ -10,12 +10,13 @@ from datetime import datetime
 from typing import List, Optional, Union
 
 from evalscope.arguments import parse_args
-from evalscope.config import TaskConfig
-from evalscope.constants import DEFAULT_MODEL_REVISION, DEFAULT_WORK_DIR, EvalBackend, EvalType, OutputsStructure
-from evalscope.evaluator import Evaluator, HumanevalEvaluator
+from evalscope.config import TaskConfig, parse_task_config
+from evalscope.constants import DEFAULT_MODEL_REVISION, DEFAULT_WORK_DIR, EvalBackend, EvalType
+from evalscope.evaluator import Evaluator
 from evalscope.models.custom import CustomModel
 from evalscope.utils import import_module_util, seed_everything
-from evalscope.utils.logger import get_logger
+from evalscope.utils.io_utils import OutputsStructure, are_paths_same
+from evalscope.utils.logger import configure_logging, get_logger
 
 logger = get_logger()
 
@@ -23,15 +24,6 @@ BENCHMARK_PATH_PREFIX = 'evalscope.benchmarks.'
 MEMBERS_TO_IMPORT = ['DATASET_ID', 'SUBSET_LIST', 'DataAdapterClass', 'ModelAdapterClass']
 
 
-def configure_logging(debug: bool, outputs: Optional[OutputsStructure]):
-    """Configure logging level based on the debug flag."""
-    if outputs:
-        log_file = os.path.join(outputs.logs_dir, 'eval_log.log')
-        get_logger(log_file=log_file, force=True)
-    if debug:
-        get_logger(log_level=logging.DEBUG, force=True)
-
-
 def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig], Namespace]) -> Union[dict, List[dict]]:
     """Run evaluation task(s) based on the provided configuration."""
     run_time = datetime.now().strftime('%Y%m%d_%H%M%S')
@@ -48,35 +40,15 @@ def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     """Run a single evaluation task."""
     seed_everything(task_cfg.seed)
     outputs = setup_work_directory(task_cfg, run_time)
-    configure_logging(task_cfg.debug, outputs)
+    configure_logging(task_cfg.debug, os.path.join(outputs.logs_dir, 'eval_log.log'))
 
+    task_cfg.dump_yaml(outputs.configs_dir)
     logger.info(task_cfg)
 
-    return evaluate_model(task_cfg, outputs)
-
-
-def parse_task_config(task_cfg) -> TaskConfig:
-    """Parse task configuration from various formats into a TaskConfig object."""
-    if isinstance(task_cfg, TaskConfig):
-        logger.info('Args: Task config is provided with TaskConfig type.')
-    elif isinstance(task_cfg, dict):
-        logger.info('Args: Task config is provided with dictionary type.')
-        task_cfg = TaskConfig.from_dict(task_cfg)
-    elif isinstance(task_cfg, Namespace):
-        logger.info('Args: Task config is provided with CommandLine type.')
-        task_cfg = TaskConfig.from_args(task_cfg)
-    elif isinstance(task_cfg, str):
-        extension = task_cfg.split('.')[-1]
-        logger.info(f'Args: Task config is provided with {extension} file type.')
-        if extension in ['yaml', 'yml']:
-            task_cfg = TaskConfig.from_yaml(task_cfg)
-        elif extension == 'json':
-            task_cfg = TaskConfig.from_json(task_cfg)
-        else:
-            raise ValueError('Args: Unsupported file extension.')
+    if task_cfg.eval_backend != EvalBackend.NATIVE:
+        return run_non_native_backend(task_cfg)
     else:
-        raise ValueError('Args: Please provide a valid task config.')
-    return task_cfg
+        return evaluate_model(task_cfg, outputs)
 
 
 def setup_work_directory(task_cfg: TaskConfig, run_time: str):
@@ -84,10 +56,15 @@ def setup_work_directory(task_cfg: TaskConfig, run_time: str):
     if task_cfg.use_cache:
         task_cfg.work_dir = task_cfg.use_cache
         logger.info(f'Set resume from {task_cfg.work_dir}')
-    elif task_cfg.work_dir == DEFAULT_WORK_DIR:
+    elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
         task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
 
     outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
+
+    if task_cfg.eval_backend == EvalBackend.OPEN_COMPASS:
+        task_cfg.eval_config['time_str'] = run_time
+    elif task_cfg.eval_backend == EvalBackend.VLM_EVAL_KIT:
+        task_cfg.eval_config['work_dir'] = task_cfg.work_dir
     return outputs
 
 
@@ -125,10 +102,6 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
     # Initialize evaluator
     eval_results = {}
-    task_cfg.dump_yaml(outputs.configs_dir)
-
-    if task_cfg.eval_backend != EvalBackend.NATIVE:
-        return run_non_native_backend(task_cfg)
 
     for dataset_name in task_cfg.datasets:
         evaluator = create_evaluator(task_cfg, dataset_name, outputs)
@@ -143,45 +116,35 @@ def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsSt
     imported_modules = import_module_util(BENCHMARK_PATH_PREFIX, dataset_name, MEMBERS_TO_IMPORT)
     model_adapter = initialize_model_adapter(task_cfg, dataset_name, imported_modules)
 
-    if dataset_name == 'humaneval':
-        problem_file = task_cfg.dataset_args.get('humaneval', {}).get('local_path')
-        return HumanevalEvaluator(
-            problem_file=problem_file,
-            model_id=task_cfg.model,
-            model_revision=task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION),
-            model_adapter=model_adapter,
-            outputs=outputs,
-            is_custom_outputs_dir=False,
-        )
-    else:
-        dataset_config = task_cfg.dataset_args.get(dataset_name, {})
-        dataset_name_or_path = dataset_config.get('local_path') or imported_modules['DATASET_ID']
-        in_prompt_template = dataset_config.get('prompt_template', '')
-        few_shot_num = dataset_config.get('few_shot_num', None)
-        few_shot_random = dataset_config.get('few_shot_random', True)
-
-        data_adapter = imported_modules['DataAdapterClass'](
-            few_shot_num=few_shot_num,
-            few_shot_random=few_shot_random,
-            prompt_template=in_prompt_template,
-        )
-        in_subset_list = dataset_config.get('subset_list', imported_modules['SUBSET_LIST'])
-
-        logger.info(f'Evaluating on subsets for {dataset_name}: {in_subset_list}\n')
-
-        return Evaluator(
-            dataset_name_or_path=dataset_name_or_path,
-            subset_list=in_subset_list,
-            data_adapter=data_adapter,
-            model_adapter=model_adapter,
-            use_cache=task_cfg.use_cache,
-            outputs=outputs,
-            datasets_dir=task_cfg.dataset_dir,
-            datasets_hub=task_cfg.dataset_hub,
-            stage=task_cfg.stage,
-            eval_type=task_cfg.eval_type,
-            overall_task_cfg=task_cfg,
-        )
+    dataset_config = task_cfg.dataset_args.get(dataset_name, {})
+    dataset_name_or_path = dataset_config.get('local_path') or imported_modules['DATASET_ID']
+    in_prompt_template = dataset_config.get('prompt_template', '')
+    few_shot_num = dataset_config.get('few_shot_num', None)
+    few_shot_random = dataset_config.get('few_shot_random', True)
+
+    data_adapter = imported_modules['DataAdapterClass'](
+        few_shot_num=few_shot_num,
+        few_shot_random=few_shot_random,
+        prompt_template=in_prompt_template,
+        outputs=outputs,
+    )
+    in_subset_list = dataset_config.get('subset_list', imported_modules['SUBSET_LIST'])
+
+    logger.info(f'Evaluating on subsets for {dataset_name}: {in_subset_list}\n')
+
+    return Evaluator(
+        dataset_name_or_path=dataset_name_or_path,
+        subset_list=in_subset_list,
+        data_adapter=data_adapter,
+        model_adapter=model_adapter,
+        use_cache=task_cfg.use_cache,
+        outputs=outputs,
+        datasets_dir=task_cfg.dataset_dir,
+        datasets_hub=task_cfg.dataset_hub,
+        stage=task_cfg.stage,
+        eval_type=task_cfg.eval_type,
+        overall_task_cfg=task_cfg,
+    )
 
 
 def initialize_model_adapter(task_cfg: TaskConfig, dataset_name: str, imported_modules):

evalscope/run_arena.py

@@ -11,7 +11,8 @@ from tqdm import tqdm
 from evalscope.constants import EvalConfigKeys
 from evalscope.evaluator.rating_eval import RatingEvaluate
 from evalscope.models.model_adapter import ChatGenerationModelAdapter
-from evalscope.utils import dump_jsonl_data, get_obj_from_cfg, jsonl_to_list, yaml_to_dict
+from evalscope.utils import get_obj_from_cfg
+from evalscope.utils.io_utils import dump_jsonl_data, jsonl_to_list, yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/summarizer.py

@@ -4,10 +4,11 @@ import json
 import os
 from typing import List, Union
 
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalBackend, OutputsStructure
+from evalscope.config import TaskConfig, parse_task_config
+from evalscope.constants import EvalBackend
 from evalscope.tools.combine_reports import gen_table
-from evalscope.utils import csv_to_list, get_latest_folder_path, json_to_dict, yaml_to_dict
+from evalscope.utils import csv_to_list, get_latest_folder_path
+from evalscope.utils.io_utils import OutputsStructure, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -24,7 +25,7 @@ class Summarizer:
         if reports_dir is None:
             raise ValueError(f'No reports directory in {outputs_dir}')
 
-        report_files: list = glob.glob(os.path.join(reports_dir, '*.json'))
+        report_files: list = glob.glob(os.path.join(reports_dir, '**/*.json'))
         for report_file in report_files:
             with open(report_file, 'r') as f:
                 res_list.append(json.load(f))
@@ -47,33 +48,20 @@ class Summarizer:
             A report dict is overall report on a benchmark for specific model.
         """
         final_res_list: List[dict] = []
-        candidate_task_cfgs: List[dict] = []
-
-        if isinstance(task_cfg, dict):
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, str):
-            task_cfg: dict = yaml_to_dict(task_cfg)
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, TaskConfig):
-            task_cfg: dict = task_cfg.to_dict()
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, list):
+        candidate_task_cfgs: List[TaskConfig] = []
+
+        if isinstance(task_cfg, list):
             for task_cfg_item in task_cfg:
-                if isinstance(task_cfg_item, str):
-                    task_cfg_item: dict = yaml_to_dict(task_cfg_item)
-                elif isinstance(task_cfg_item, TaskConfig):
-                    task_cfg_item: dict = task_cfg_item.to_dict()
-                candidate_task_cfgs.append(task_cfg_item)
+                candidate_task_cfgs.append(parse_task_config(task_cfg_item))
         else:
-            raise ValueError(f'Invalid task_cfg: {task_cfg}')
+            candidate_task_cfgs.append(parse_task_config(task_cfg))
 
         for candidate_task in candidate_task_cfgs:
             logger.info(f'**Loading task cfg for summarizer: {candidate_task}')
-            eval_backend = candidate_task.get('eval_backend') or EvalBackend.NATIVE
+            eval_backend = candidate_task.eval_backend
 
             if eval_backend == EvalBackend.NATIVE:
-                outputs_dir: str = candidate_task.get('outputs')
-                outputs_dir: str = os.path.expanduser(outputs_dir)
+                outputs_dir: str = os.path.expanduser(candidate_task.work_dir)
                 if outputs_dir is None:
                     raise ValueError(f'No outputs_dir in {task_cfg}')
                 res_list: list = Summarizer.get_report(outputs_dir=outputs_dir)
@@ -128,8 +116,8 @@ class Summarizer:
         return final_res_list
 
     @staticmethod
-    def parse_eval_config(candidate_task):
-        eval_config: Union[str, dict] = candidate_task.get('eval_config')
+    def parse_eval_config(candidate_task: TaskConfig):
+        eval_config: Union[str, dict] = candidate_task.eval_config
         assert eval_config is not None, 'Please provide eval_config for specific evaluation backend.'
 
         if isinstance(eval_config, str):

evalscope/third_party/longbench_write/eval.py

@@ -10,7 +10,8 @@ import requests
 from concurrent.futures import ThreadPoolExecutor
 from tqdm import tqdm
 
-from evalscope.utils import get_logger, jsonl_to_list
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import jsonl_to_list
 
 logger = get_logger()
 

evalscope/third_party/longbench_write/longbench_write.py

@@ -4,7 +4,8 @@ from typing import Union
 
 from evalscope.third_party.longbench_write.eval import run_eval
 from evalscope.third_party.longbench_write.infer import run_infer
-from evalscope.utils import get_logger, json_to_dict, yaml_to_dict
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
 
 logger = get_logger()
 

evalscope/third_party/longbench_write/tools/data_etl.py

@@ -6,7 +6,7 @@ from typing import List
 
 from evalscope.third_party.longbench_write.eval import EvalLength
 from evalscope.third_party.longbench_write.utils import chinese_to_arabic, count_words
-from evalscope.utils import jsonl_to_list
+from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/third_party/toolbench_static/toolbench_static.py

@@ -5,7 +5,8 @@ from typing import Union
 
 from evalscope.third_party.toolbench_static.eval import EvalArgs, run_eval
 from evalscope.third_party.toolbench_static.infer import InferArgs, run_infer
-from evalscope.utils import get_logger, json_to_dict, yaml_to_dict
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
 
 logger = get_logger()
 

evalscope/tools/combine_reports.py

@@ -19,16 +19,14 @@ def get_report(report_file: str):
     dataset_name = data_d['dataset_name']
     model_name = data_d['model_name']
     score = data_d['score']  # float or dict
+    metric = data_d['metric']
     score_d = {}
     if isinstance(score, dict):
-        # score_d = dict([(k, round(v, 4) * 100) for k, v in score.items()])
         score_d = score
     elif isinstance(score, float):
-        # score_d['acc'] = round(score, 4) * 100
-        score_d['acc'] = score
+        score_d[metric] = score
     else:
         raise ValueError(f'Unknown score type: {type(score)}')
-    # score_str = '\n'.join([str(v) + ' (' + k + ')' for k, v in score_d.items()])
     score_str = '\n'.join(['(' + dataset_name + '/' + k + ') ' + str(v) for k, v in score_d.items()])
 
     return model_name, {'dataset_name': dataset_name, 'score': score_str}