evalscope 0.8.0__py3-none-any.whl → 0.10.1__py3-none-any.whl

evalscope/report/combinator.py

@@ -0,0 +1,73 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import glob
+import os
+import pandas as pd
+from tabulate import tabulate
+from typing import List, Tuple
+
+from evalscope.report.utils import Report
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+"""
+Combine and generate table for reports of LLMs.
+"""
+
+
+def get_report_list(reports_path_list: List[str]) -> List[Report]:
+    report_list: List[Report] = []
+    # Iterate over each report path
+    for report_path in reports_path_list:
+        model_report_dir = os.path.normpath(report_path)
+        report_files = glob.glob(os.path.join(model_report_dir, '**', '*.json'), recursive=True)
+        # Iterate over each report file
+        for file_path in report_files:
+            try:
+                report = Report.from_json(file_path)
+                report_list.append(report)
+            except Exception as e:
+                logger.error(f'Error loading report from {file_path}: {e}')
+    report_list = sorted(report_list, key=lambda x: (x.model_name, x.dataset_name))
+    return report_list
+
+
+def get_data_frame(report_list: List[Report],
+                   flatten_metrics: bool = True,
+                   flatten_categories: bool = True) -> pd.DataFrame:
+    tables = []
+    for report in report_list:
+        df = report.to_dataframe(flatten_metrics=flatten_metrics, flatten_categories=flatten_categories)
+        tables.append(df)
+    return pd.concat(tables, ignore_index=True)
+
+
+def gen_table(reports_path_list: list) -> str:
+    report_list = get_report_list(reports_path_list)
+    table = get_data_frame(report_list)
+    return tabulate(table, headers=table.columns, tablefmt='grid', showindex=False)
+
+
+class ReportsRecorder:
+    COMMON_DATASET_PATH = []
+    CUSTOM_DATASET_PATH = []
+
+    def __init__(self, oss_url: str = '', endpoint: str = ''):
+        pass
+
+
+if __name__ == '__main__':
+    report_dir_1 = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250117_151926'
+    # report_dir_2 = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250107_204445/reports'
+
+    report_table = gen_table([report_dir_1])
+    print(report_table)
+
+    # ALL VALUES ONLY FOR EXAMPLE
+    # +--------------------------+-------------------+-------------+
+    # | Model                    | CompetitionMath   | GSM8K       |
+    # +==========================+===================+=============+
+    # | ZhipuAI_chatglm2-6b-base | 25.0 (acc)        | 30.50 (acc) |
+    # +--------------------------+-------------------+-------------+
+    # | ZhipuAI_chatglm2-6b      | 30.5 (acc)        | 40.50 (acc) |
+    # +--------------------------+-------------------+-------------+

evalscope/report/generator.py

@@ -0,0 +1,80 @@
+import pandas as pd
+from pandas import DataFrame
+
+from evalscope.constants import DataCollection
+from evalscope.report.utils import *
+
+
+class ReportGenerator:
+
+    @staticmethod
+    def gen_report(subset_score_map: dict, report_name: str, **kwargs) -> Report:
+        """
+        Generate report for specific dataset.
+        subset_score_map: e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}, {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}]}
+        category_map: e.g. {'subset_name': ['category_name1', 'category_name2'], ...}
+        metric_list: e.g. [{'object': AverageAccuracy, 'name': 'AverageAccuracy'}, {'object': 'WeightedAverageAccuracy', 'name': 'WeightedAverageAccuracy'}]
+        """  # noqa: E501
+
+        dataset_name = kwargs.get('dataset_name', None)
+        model_name = kwargs.get('model_name', None)
+        category_map = kwargs.get('category_map', {})
+
+        def flatten_subset() -> DataFrame:
+            """
+            Flatten subset score map to a DataFrame.
+
+            Example:
+                        name  score  num   categories      metric_name
+            0       ARC-Easy    0.5    2    [default]  AverageAccuracy
+            1  ARC-Challenge    0.5    2    [default]  AverageAccuracy
+            """
+            subsets = []
+            for subset_name, scores in subset_score_map.items():
+                for score_item in scores:
+                    categories = category_map.get(subset_name, ['default'])
+                    if isinstance(categories, str):
+                        categories = [categories]
+                    subsets.append(
+                        dict(
+                            name=subset_name,
+                            score=score_item['score'],
+                            num=score_item['num'],
+                            metric_name=score_item['metric_name'],
+                            categories=tuple(categories)))
+            df = pd.DataFrame(subsets)
+            return df
+
+        df = flatten_subset()
+
+        metrics_list = []
+        for metric_name, group_metric in df.groupby('metric_name'):
+            categories = []
+            for category_name, group_category in group_metric.groupby('categories'):
+                subsets = []
+                for _, row in group_category.iterrows():
+                    subsets.append(Subset(name=row['name'], score=row['score'], num=row['num']))
+
+                categories.append(Category(name=category_name, subsets=subsets))
+
+            metrics_list.append(Metric(name=metric_name, categories=categories))
+
+        report = Report(name=report_name, metrics=metrics_list, dataset_name=dataset_name, model_name=model_name)
+        return report
+
+    @staticmethod
+    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
+        categories = []
+        for category_name, group_category in df.groupby('categories'):
+            subsets = []
+            for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name', 'subset_name']):
+                avg_score = group_subset['score'].mean()
+                num = group_subset['score'].count()
+                subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
+
+            categories.append(Category(name=category_name, subsets=subsets))
+        return Report(
+            name=DataCollection.NAME,
+            metrics=[Metric(name='Average', categories=categories)],
+            dataset_name=all_dataset_name,
+            model_name=model_name)

evalscope/report/utils.py

@@ -0,0 +1,133 @@
+import json
+import pandas as pd
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List
+
+from evalscope.metrics import macro_mean, micro_mean
+from evalscope.utils import normalize_score
+
+
+@dataclass
+class Subset:
+    name: str = 'default_subset'
+    score: float = 0.0
+    num: int = 0
+
+    def __post_init__(self):
+        self.score = normalize_score(self.score)
+
+
+@dataclass
+class Category:
+    name: tuple[str] = field(default_factory=tuple)
+    num: int = 0
+    score: float = 0.0
+    macro_score: float = 0.0
+    subsets: List[Subset] = field(default_factory=list)
+
+    def __post_init__(self):
+        if isinstance(self.name, str):
+            # ensure name is tuple format
+            self.name = (self.name, )
+        self.num = sum(subset.num for subset in self.subsets)
+        self.score = normalize_score(micro_mean(self.subsets))
+        self.macro_score = normalize_score(macro_mean(self.subsets))
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        subsets = [Subset(**subset) for subset in data.get('subsets', [])]
+        return cls(name=data['name'], subsets=subsets)
+
+
+@dataclass
+class Metric:
+    name: str = 'default_metric'
+    num: int = 0
+    score: float = 0.0
+    macro_score: float = 0.0
+    categories: List[Category] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.num = sum(category.num for category in self.categories)
+        self.score = normalize_score(micro_mean(self.categories))
+        self.macro_score = normalize_score(macro_mean(self.categories))
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        categories = [Category.from_dict(category) for category in data.get('categories', [])]
+        return cls(name=data['name'], categories=categories)
+
+
+class ReportKey:
+    model_name = 'Model'
+    dataset_name = 'Dataset'
+    metric_name = 'Metric'
+    category_name = 'Category'
+    category_prefix = 'Cat.'
+    subset_name = 'Subset'
+    num = 'Num'
+    score = 'Score'
+
+
+@dataclass
+class Report:
+    name: str = 'default_report'
+    dataset_name: str = 'default_dataset'
+    model_name: str = 'default_model'
+    score: float = 0.0
+    metrics: List[Metric] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.score = self.metrics[0].score  # NOTE: only use the first metric by default
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        metrics = [Metric.from_dict(metric) for metric in data.get('metrics', [])]
+        return cls(
+            name=data['name'],
+            score=data['score'],
+            metrics=metrics,
+            dataset_name=data['dataset_name'],
+            model_name=data['model_name'])
+
+    @classmethod
+    def from_json(cls, json_file: str):
+        with open(json_file, 'r') as f:
+            data = json.load(f)
+        return cls.from_dict(data)
+
+    def to_dataframe(self, flatten_metrics: bool = True, flatten_categories: bool = True):
+        table = defaultdict(list)
+        for metric in self.metrics:
+            for category in metric.categories:
+                for subset in category.subsets:
+                    table[ReportKey.model_name].append(self.model_name)
+                    table[ReportKey.dataset_name].append(self.dataset_name)
+                    table[ReportKey.metric_name].append(metric.name)
+                    table[ReportKey.category_name].append(category.name)
+                    table[ReportKey.subset_name].append(subset.name)
+                    table[ReportKey.num].append(subset.num)
+                    table[ReportKey.score].append(subset.score)  # TODO: convert to percentage
+            # NOTE: only flatten metrics if needed, use the first metric by default
+            if not flatten_metrics:
+                break
+        df = pd.DataFrame.from_dict(table, orient='columns')
+        if flatten_categories:
+            df = self._flatten_categories(df)
+        return df
+
+    def _flatten_categories(self, df: pd.DataFrame):
+        # expand categories to multiple rows
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories[ReportKey.category_name].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'{ReportKey.category_prefix}{level}'] = df_categories[ReportKey.category_name].apply(
+                lambda x: x[level] if len(x) > level else None)
+
+        df_categories.drop(columns=[ReportKey.category_name], inplace=True)
+        return df_categories

evalscope/run.py

@@ -2,34 +2,21 @@
 """
 Run evaluation for LLMs.
 """
-import logging
 import os.path
-import torch
 from argparse import Namespace
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Union
 
-from evalscope.arguments import parse_args
-from evalscope.config import TaskConfig
-from evalscope.constants import DEFAULT_MODEL_REVISION, DEFAULT_WORK_DIR, EvalBackend, EvalType, OutputsStructure
-from evalscope.evaluator import Evaluator, HumanevalEvaluator
-from evalscope.models.custom import CustomModel
-from evalscope.utils import import_module_util, seed_everything
-from evalscope.utils.logger import get_logger
+from evalscope.config import TaskConfig, parse_task_config
+from evalscope.constants import DataCollection, EvalBackend
+from evalscope.utils import seed_everything
+from evalscope.utils.io_utils import OutputsStructure
+from evalscope.utils.logger import configure_logging, get_logger
 
-logger = get_logger()
-
-BENCHMARK_PATH_PREFIX = 'evalscope.benchmarks.'
-MEMBERS_TO_IMPORT = ['DATASET_ID', 'SUBSET_LIST', 'DataAdapterClass', 'ModelAdapterClass']
+if TYPE_CHECKING:
+    from evalscope.models import LocalModel
 
-
-def configure_logging(debug: bool, outputs: Optional[OutputsStructure]):
-    """Configure logging level based on the debug flag."""
-    if outputs:
-        log_file = os.path.join(outputs.logs_dir, 'eval_log.log')
-        get_logger(log_file=log_file, force=True)
-    if debug:
-        get_logger(log_level=logging.DEBUG, force=True)
+logger = get_logger()
 
 
 def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig], Namespace]) -> Union[dict, List[dict]]:
@@ -46,37 +33,15 @@ def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig], Namespace]
 
 def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     """Run a single evaluation task."""
-    seed_everything(task_cfg.seed)
+    if task_cfg.seed is not None:
+        seed_everything(task_cfg.seed)
     outputs = setup_work_directory(task_cfg, run_time)
-    configure_logging(task_cfg.debug, outputs)
+    configure_logging(task_cfg.debug, os.path.join(outputs.logs_dir, 'eval_log.log'))
 
-    logger.info(task_cfg)
-
-    return evaluate_model(task_cfg, outputs)
-
-
-def parse_task_config(task_cfg) -> TaskConfig:
-    """Parse task configuration from various formats into a TaskConfig object."""
-    if isinstance(task_cfg, TaskConfig):
-        logger.info('Args: Task config is provided with TaskConfig type.')
-    elif isinstance(task_cfg, dict):
-        logger.info('Args: Task config is provided with dictionary type.')
-        task_cfg = TaskConfig.from_dict(task_cfg)
-    elif isinstance(task_cfg, Namespace):
-        logger.info('Args: Task config is provided with CommandLine type.')
-        task_cfg = TaskConfig.from_args(task_cfg)
-    elif isinstance(task_cfg, str):
-        extension = task_cfg.split('.')[-1]
-        logger.info(f'Args: Task config is provided with {extension} file type.')
-        if extension in ['yaml', 'yml']:
-            task_cfg = TaskConfig.from_yaml(task_cfg)
-        elif extension == 'json':
-            task_cfg = TaskConfig.from_json(task_cfg)
-        else:
-            raise ValueError('Args: Unsupported file extension.')
+    if task_cfg.eval_backend != EvalBackend.NATIVE:
+        return run_non_native_backend(task_cfg, outputs)
     else:
-        raise ValueError('Args: Please provide a valid task config.')
-    return task_cfg
+        return evaluate_model(task_cfg, outputs)
 
 
 def setup_work_directory(task_cfg: TaskConfig, run_time: str):
@@ -84,14 +49,19 @@ def setup_work_directory(task_cfg: TaskConfig, run_time: str):
     if task_cfg.use_cache:
         task_cfg.work_dir = task_cfg.use_cache
         logger.info(f'Set resume from {task_cfg.work_dir}')
-    elif task_cfg.work_dir == DEFAULT_WORK_DIR:
-        task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
+    # elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
+    task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
 
     outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
+
+    if task_cfg.eval_backend == EvalBackend.OPEN_COMPASS:
+        task_cfg.eval_config['time_str'] = run_time
+    elif task_cfg.eval_backend == EvalBackend.VLM_EVAL_KIT:
+        task_cfg.eval_config['work_dir'] = task_cfg.work_dir
     return outputs
 
 
-def run_non_native_backend(task_cfg: TaskConfig) -> dict:
+def run_non_native_backend(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Run evaluation using a non-native backend."""
     eval_backend = task_cfg.eval_backend
     eval_config = task_cfg.eval_config
@@ -101,6 +71,10 @@ def run_non_native_backend(task_cfg: TaskConfig) -> dict:
 
     backend_manager_class = get_backend_manager_class(eval_backend)
     backend_manager = backend_manager_class(config=eval_config)
+
+    task_cfg.dump_yaml(outputs.configs_dir)
+    logger.info(task_cfg)
+
     backend_manager.run()
 
     return dict()
@@ -123,92 +97,57 @@ def get_backend_manager_class(eval_backend: EvalBackend):
 
 def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
+    from evalscope.models import get_local_model
+
     # Initialize evaluator
     eval_results = {}
-    task_cfg.dump_yaml(outputs.configs_dir)
+    base_model = get_local_model(task_cfg)
+    evaluators = []
+    for dataset_name in task_cfg.datasets:
+        evaluator = create_evaluator(task_cfg, dataset_name, outputs, base_model)
+        evaluators.append(evaluator)
 
-    if task_cfg.eval_backend != EvalBackend.NATIVE:
-        return run_non_native_backend(task_cfg)
+    # dump task_cfg to outputs.configs_dir after creating evaluators
+    task_cfg.dump_yaml(outputs.configs_dir)
+    logger.info(task_cfg)
 
-    for dataset_name in task_cfg.datasets:
-        evaluator = create_evaluator(task_cfg, dataset_name, outputs)
+    for evaluator in evaluators:
         res_dict = evaluator.eval(infer_cfg=task_cfg.generation_config, debug=task_cfg.debug, limit=task_cfg.limit)
         eval_results[dataset_name] = res_dict
 
     return eval_results
 
 
-def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure):
+def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure, base_model: 'LocalModel'):
     """Create an evaluator object for the specified dataset."""
-    imported_modules = import_module_util(BENCHMARK_PATH_PREFIX, dataset_name, MEMBERS_TO_IMPORT)
-    model_adapter = initialize_model_adapter(task_cfg, dataset_name, imported_modules)
-
-    if dataset_name == 'humaneval':
-        problem_file = task_cfg.dataset_args.get('humaneval', {}).get('local_path')
-        return HumanevalEvaluator(
-            problem_file=problem_file,
-            model_id=task_cfg.model,
-            model_revision=task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION),
-            model_adapter=model_adapter,
-            outputs=outputs,
-            is_custom_outputs_dir=False,
-        )
-    else:
-        dataset_config = task_cfg.dataset_args.get(dataset_name, {})
-        dataset_name_or_path = dataset_config.get('local_path') or imported_modules['DATASET_ID']
-        in_prompt_template = dataset_config.get('prompt_template', '')
-        few_shot_num = dataset_config.get('few_shot_num', None)
-        few_shot_random = dataset_config.get('few_shot_random', True)
-
-        data_adapter = imported_modules['DataAdapterClass'](
-            few_shot_num=few_shot_num,
-            few_shot_random=few_shot_random,
-            prompt_template=in_prompt_template,
-        )
-        in_subset_list = dataset_config.get('subset_list', imported_modules['SUBSET_LIST'])
-
-        logger.info(f'Evaluating on subsets for {dataset_name}: {in_subset_list}\n')
-
-        return Evaluator(
-            dataset_name_or_path=dataset_name_or_path,
-            subset_list=in_subset_list,
-            data_adapter=data_adapter,
-            model_adapter=model_adapter,
-            use_cache=task_cfg.use_cache,
-            outputs=outputs,
-            datasets_dir=task_cfg.dataset_dir,
-            datasets_hub=task_cfg.dataset_hub,
-            stage=task_cfg.stage,
-            eval_type=task_cfg.eval_type,
-            overall_task_cfg=task_cfg,
-        )
-
-
-def initialize_model_adapter(task_cfg: TaskConfig, dataset_name: str, imported_modules):
-    """Initialize the model adapter based on the task configuration."""
-    if task_cfg.dry_run:
-        from evalscope.models.dummy_chat_model import DummyChatModel
-        return DummyChatModel(model_cfg=dict())
-    elif task_cfg.eval_type == EvalType.CUSTOM:
-        if not isinstance(task_cfg.model, CustomModel):
-            raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
-        from evalscope.models.model_adapter import CustomModelAdapter
-        return CustomModelAdapter(custom_model=task_cfg.model)
-    else:
-        device_map = task_cfg.model_args.get('device_map', 'auto') if torch.cuda.is_available() else None
-        model_precision = task_cfg.model_args.get('precision', torch.float16)
-        if isinstance(model_precision, str) and model_precision != 'auto':
-            model_precision = eval(model_precision)
-        return imported_modules['ModelAdapterClass'](
-            model_id=task_cfg.model,
-            model_revision=task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION),
-            device_map=device_map,
-            torch_dtype=model_precision,
-            generation_config=task_cfg.generation_config,
-            chat_template=task_cfg.chat_template)
+    from evalscope.benchmarks import Benchmark, BenchmarkMeta
+    from evalscope.evaluator import Evaluator
+    from evalscope.models import initialize_model_adapter
+
+    if dataset_name == DataCollection.NAME:
+        # EvaluatorCollection is a collection of evaluators
+        from evalscope.collections import EvaluatorCollection
+        return EvaluatorCollection(task_cfg, outputs)
+
+    benchmark: BenchmarkMeta = Benchmark.get(dataset_name)
+
+    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
+    model_adapter = initialize_model_adapter(task_cfg, benchmark.model_adapter, base_model)
+
+    # update task_cfg.dataset_args
+    task_cfg.dataset_args[dataset_name] = benchmark.to_string_dict()
+
+    return Evaluator(
+        dataset_name_or_path=benchmark.dataset_id,
+        data_adapter=data_adapter,
+        model_adapter=model_adapter,
+        outputs=outputs,
+        task_cfg=task_cfg,
+    )
 
 
 def main():
+    from evalscope.arguments import parse_args
     args = parse_args()
     run_task(args)
 

evalscope/run_arena.py

@@ -10,8 +10,9 @@ from tqdm import tqdm
 
 from evalscope.constants import EvalConfigKeys
 from evalscope.evaluator.rating_eval import RatingEvaluate
-from evalscope.models.model_adapter import ChatGenerationModelAdapter
-from evalscope.utils import dump_jsonl_data, get_obj_from_cfg, jsonl_to_list, yaml_to_dict
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils import get_obj_from_cfg
+from evalscope.utils.io_utils import dump_jsonl_data, jsonl_to_list, yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/summarizer.py

@@ -4,10 +4,11 @@ import json
 import os
 from typing import List, Union
 
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalBackend, OutputsStructure
-from evalscope.tools.combine_reports import gen_table
-from evalscope.utils import csv_to_list, get_latest_folder_path, json_to_dict, yaml_to_dict
+from evalscope.config import TaskConfig, parse_task_config
+from evalscope.constants import EvalBackend
+from evalscope.report import gen_table
+from evalscope.utils import csv_to_list, get_latest_folder_path
+from evalscope.utils.io_utils import OutputsStructure, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -24,7 +25,7 @@ class Summarizer:
         if reports_dir is None:
             raise ValueError(f'No reports directory in {outputs_dir}')
 
-        report_files: list = glob.glob(os.path.join(reports_dir, '*.json'))
+        report_files: list = glob.glob(os.path.join(reports_dir, '**/*.json'))
         for report_file in report_files:
             with open(report_file, 'r') as f:
                 res_list.append(json.load(f))
@@ -47,33 +48,20 @@ class Summarizer:
             A report dict is overall report on a benchmark for specific model.
         """
         final_res_list: List[dict] = []
-        candidate_task_cfgs: List[dict] = []
-
-        if isinstance(task_cfg, dict):
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, str):
-            task_cfg: dict = yaml_to_dict(task_cfg)
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, TaskConfig):
-            task_cfg: dict = task_cfg.to_dict()
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, list):
+        candidate_task_cfgs: List[TaskConfig] = []
+
+        if isinstance(task_cfg, list):
             for task_cfg_item in task_cfg:
-                if isinstance(task_cfg_item, str):
-                    task_cfg_item: dict = yaml_to_dict(task_cfg_item)
-                elif isinstance(task_cfg_item, TaskConfig):
-                    task_cfg_item: dict = task_cfg_item.to_dict()
-                candidate_task_cfgs.append(task_cfg_item)
+                candidate_task_cfgs.append(parse_task_config(task_cfg_item))
         else:
-            raise ValueError(f'Invalid task_cfg: {task_cfg}')
+            candidate_task_cfgs.append(parse_task_config(task_cfg))
 
         for candidate_task in candidate_task_cfgs:
             logger.info(f'**Loading task cfg for summarizer: {candidate_task}')
-            eval_backend = candidate_task.get('eval_backend') or EvalBackend.NATIVE
+            eval_backend = candidate_task.eval_backend
 
             if eval_backend == EvalBackend.NATIVE:
-                outputs_dir: str = candidate_task.get('outputs')
-                outputs_dir: str = os.path.expanduser(outputs_dir)
+                outputs_dir: str = os.path.expanduser(candidate_task.work_dir)
                 if outputs_dir is None:
                     raise ValueError(f'No outputs_dir in {task_cfg}')
                 res_list: list = Summarizer.get_report(outputs_dir=outputs_dir)
@@ -128,8 +116,8 @@ class Summarizer:
         return final_res_list
 
     @staticmethod
-    def parse_eval_config(candidate_task):
-        eval_config: Union[str, dict] = candidate_task.get('eval_config')
+    def parse_eval_config(candidate_task: TaskConfig):
+        eval_config: Union[str, dict] = candidate_task.eval_config
         assert eval_config is not None, 'Please provide eval_config for specific evaluation backend.'
 
         if isinstance(eval_config, str):

evalscope/third_party/longbench_write/eval.py

@@ -10,7 +10,8 @@ import requests
 from concurrent.futures import ThreadPoolExecutor
 from tqdm import tqdm
 
-from evalscope.utils import get_logger, jsonl_to_list
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import jsonl_to_list
 
 logger = get_logger()
 

evalscope/third_party/longbench_write/longbench_write.py

@@ -4,7 +4,8 @@ from typing import Union
 
 from evalscope.third_party.longbench_write.eval import run_eval
 from evalscope.third_party.longbench_write.infer import run_infer
-from evalscope.utils import get_logger, json_to_dict, yaml_to_dict
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
 
 logger = get_logger()
 

evalscope/third_party/longbench_write/tools/data_etl.py

@@ -6,7 +6,7 @@ from typing import List
 
 from evalscope.third_party.longbench_write.eval import EvalLength
 from evalscope.third_party.longbench_write.utils import chinese_to_arabic, count_words
-from evalscope.utils import jsonl_to_list
+from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/third_party/toolbench_static/toolbench_static.py

@@ -5,7 +5,8 @@ from typing import Union
 
 from evalscope.third_party.toolbench_static.eval import EvalArgs, run_eval
 from evalscope.third_party.toolbench_static.infer import InferArgs, run_infer
-from evalscope.utils import get_logger, json_to_dict, yaml_to_dict
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
 
 logger = get_logger()