evalscope 0.8.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/perf/utils/db_util.py

@@ -6,6 +6,7 @@ import sqlite3
 import sys
 from datetime import datetime
 from tabulate import tabulate
+from typing import Dict, List
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
@@ -107,44 +108,87 @@ def get_result_db_path(args: Arguments):
     return result_db_path
 
 
-def get_percentile_results(result_db_path: str):
+def calculate_percentiles(data: List[float], percentiles: List[int]) -> Dict[int, float]:
+    """
+    Calculate the percentiles for a specific list of data.
 
-    def percentile_results(rows, index, percentiles):
-        results = {}
-        n_success_queries = len(rows)
-        for percentile in percentiles:
+    :param data: List of values for a specific metric.
+    :param percentiles: List of percentiles to calculate.
+    :return: Dictionary of calculated percentiles.
+    """
+    results = {}
+    n_success_queries = len(data)
+    data.sort()
+    for percentile in percentiles:
+        try:
             idx = int(n_success_queries * percentile / 100)
-            row = rows[idx]
-            value = row[index] if row[index] is not None else float('inf')
+            value = data[idx] if data[idx] is not None else float('nan')
             results[percentile] = round(value, 4)
-        return results
+        except IndexError:
+            results[percentile] = float('nan')
+    return results
+
+
+def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
+    """
+    Compute and return quantiles for various metrics from the database results.
+
+    :param result_db_path: Path to the SQLite database file.
+    :return: Dictionary of percentiles for various metrics.
+    """
+
+    def inter_token_latencies(chunk_times_json: str) -> List[float]:
+        try:
+            chunk_times = json.loads(chunk_times_json)
+            return [t2 - t1 for t1, t2 in zip(chunk_times[:-1], chunk_times[1:])]
+        except (json.JSONDecodeError, TypeError) as e:
+            logger.error(f'Error parsing chunk times: {e}')
+            return []
 
     query_sql = ('SELECT start_time, chunk_times, success, completed_time, latency, first_chunk_latency, '
                  'n_chunks, chunk_time, prompt_tokens, completion_tokens '
-                 'FROM result WHERE success=1 ORDER BY first_chunk_latency ASC')
+                 'FROM result WHERE success=1')
+
     percentiles = [10, 25, 50, 66, 75, 80, 90, 95, 98, 99]
 
     with sqlite3.connect(result_db_path) as con:
         rows = con.execute(query_sql).fetchall()
 
-    if len(rows) <= len(percentiles):
+    if len(rows) < len(percentiles):
         logger.info('Too little data to calculate quantiles!')
         return {}
 
-    # Calculate percentiles for first chunk latency and latency
-    first_chunk_latency_index = 5
-    latency_index = 4
+    # Define index variables for columns
+    CHUNK_TIMES_INDEX = 1
+    LATENCY_INDEX = 4
+    FIRST_CHUNK_LATENCY_INDEX = 5
+    PROMPT_TOKENS_INDEX = 8
+    COMPLETION_TOKENS_INDEX = 9
+
+    # Prepare data for each metric
+    inter_token_latencies_all = []
+    for row in rows:
+        inter_token_latencies_all.extend(inter_token_latencies(row[CHUNK_TIMES_INDEX]))
+
+    metrics = {
+        'TTFT (s)': [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
+        'TPOT (s)':
+        inter_token_latencies_all,
+        'Latency (s)': [row[LATENCY_INDEX] for row in rows],
+        'Input tokens': [row[PROMPT_TOKENS_INDEX] for row in rows],
+        'Output tokens': [row[COMPLETION_TOKENS_INDEX] for row in rows],
+        'Throughput(tokens/s)':
+        [(row[COMPLETION_TOKENS_INDEX] / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan')
+         for row in rows]
+    }
 
-    first_chunk_latency_results = percentile_results(rows, first_chunk_latency_index, percentiles)
-    rows.sort(key=lambda x: x[latency_index])
-    latency_results = percentile_results(rows, latency_index, percentiles)
+    # Calculate percentiles for each metric
+    results = {'Percentile': [f'{p}%' for p in percentiles]}
+    for metric_name, data in metrics.items():
+        metric_percentiles = calculate_percentiles(data, percentiles)
+        results[metric_name] = [metric_percentiles[p] for p in percentiles]
 
-    # Prepare data for tabulation
-    return {
-        'Percentile': [f'{p}%' for p in percentiles],
-        'First Chunk Latency (s)': [first_chunk_latency_results[p] for p in percentiles],
-        'Latency (s)': [latency_results[p] for p in percentiles]
-    }
+    return results
 
 
 def summary_result(args: Arguments, metrics: BenchmarkMetrics, expected_number_of_queries: int, result_db_path: str):

evalscope/perf/utils/local_server.py

@@ -102,6 +102,7 @@ def start_app(args: Arguments):
 
     elif args.api == 'local_vllm':
         os.environ['VLLM_USE_MODELSCOPE'] = 'True'
+        os.environ['VLLM_ALLOW_LONG_MAX_MODEL_LEN'] = '1'
         # yapf: disable
         proc = subprocess.Popen([
             'python', '-m', 'vllm.entrypoints.openai.api_server',
@@ -111,7 +112,8 @@ def start_app(args: Arguments):
             '--max-model-len', '32768',
             '--gpu-memory-utilization', '0.9',
             '--host', '0.0.0.0',
-            '--port', args.port,
+            '--port', str(args.port),
+            '--trust-remote-code',
             '--disable-log-requests',
             '--disable-log-stats',
         ])

evalscope/run.py

@@ -10,12 +10,13 @@ from datetime import datetime
 from typing import List, Optional, Union
 
 from evalscope.arguments import parse_args
-from evalscope.config import TaskConfig
-from evalscope.constants import DEFAULT_MODEL_REVISION, DEFAULT_WORK_DIR, EvalBackend, EvalType, OutputsStructure
-from evalscope.evaluator import Evaluator, HumanevalEvaluator
+from evalscope.config import TaskConfig, parse_task_config
+from evalscope.constants import DEFAULT_MODEL_REVISION, DEFAULT_WORK_DIR, EvalBackend, EvalType
+from evalscope.evaluator import Evaluator
 from evalscope.models.custom import CustomModel
 from evalscope.utils import import_module_util, seed_everything
-from evalscope.utils.logger import get_logger
+from evalscope.utils.io_utils import OutputsStructure, are_paths_same
+from evalscope.utils.logger import configure_logging, get_logger
 
 logger = get_logger()
 
@@ -23,15 +24,6 @@ BENCHMARK_PATH_PREFIX = 'evalscope.benchmarks.'
 MEMBERS_TO_IMPORT = ['DATASET_ID', 'SUBSET_LIST', 'DataAdapterClass', 'ModelAdapterClass']
 
 
-def configure_logging(debug: bool, outputs: Optional[OutputsStructure]):
-    """Configure logging level based on the debug flag."""
-    if outputs:
-        log_file = os.path.join(outputs.logs_dir, 'eval_log.log')
-        get_logger(log_file=log_file, force=True)
-    if debug:
-        get_logger(log_level=logging.DEBUG, force=True)
-
-
 def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig], Namespace]) -> Union[dict, List[dict]]:
     """Run evaluation task(s) based on the provided configuration."""
     run_time = datetime.now().strftime('%Y%m%d_%H%M%S')
@@ -48,35 +40,15 @@ def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     """Run a single evaluation task."""
     seed_everything(task_cfg.seed)
     outputs = setup_work_directory(task_cfg, run_time)
-    configure_logging(task_cfg.debug, outputs)
+    configure_logging(task_cfg.debug, os.path.join(outputs.logs_dir, 'eval_log.log'))
 
+    task_cfg.dump_yaml(outputs.configs_dir)
     logger.info(task_cfg)
 
-    return evaluate_model(task_cfg, outputs)
-
-
-def parse_task_config(task_cfg) -> TaskConfig:
-    """Parse task configuration from various formats into a TaskConfig object."""
-    if isinstance(task_cfg, TaskConfig):
-        logger.info('Args: Task config is provided with TaskConfig type.')
-    elif isinstance(task_cfg, dict):
-        logger.info('Args: Task config is provided with dictionary type.')
-        task_cfg = TaskConfig.from_dict(task_cfg)
-    elif isinstance(task_cfg, Namespace):
-        logger.info('Args: Task config is provided with CommandLine type.')
-        task_cfg = TaskConfig.from_args(task_cfg)
-    elif isinstance(task_cfg, str):
-        extension = task_cfg.split('.')[-1]
-        logger.info(f'Args: Task config is provided with {extension} file type.')
-        if extension in ['yaml', 'yml']:
-            task_cfg = TaskConfig.from_yaml(task_cfg)
-        elif extension == 'json':
-            task_cfg = TaskConfig.from_json(task_cfg)
-        else:
-            raise ValueError('Args: Unsupported file extension.')
+    if task_cfg.eval_backend != EvalBackend.NATIVE:
+        return run_non_native_backend(task_cfg)
     else:
-        raise ValueError('Args: Please provide a valid task config.')
-    return task_cfg
+        return evaluate_model(task_cfg, outputs)
 
 
 def setup_work_directory(task_cfg: TaskConfig, run_time: str):
@@ -84,10 +56,15 @@ def setup_work_directory(task_cfg: TaskConfig, run_time: str):
     if task_cfg.use_cache:
         task_cfg.work_dir = task_cfg.use_cache
         logger.info(f'Set resume from {task_cfg.work_dir}')
-    elif task_cfg.work_dir == DEFAULT_WORK_DIR:
+    elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
         task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
 
     outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
+
+    if task_cfg.eval_backend == EvalBackend.OPEN_COMPASS:
+        task_cfg.eval_config['time_str'] = run_time
+    elif task_cfg.eval_backend == EvalBackend.VLM_EVAL_KIT:
+        task_cfg.eval_config['work_dir'] = task_cfg.work_dir
     return outputs
 
 
@@ -125,10 +102,6 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
     # Initialize evaluator
     eval_results = {}
-    task_cfg.dump_yaml(outputs.configs_dir)
-
-    if task_cfg.eval_backend != EvalBackend.NATIVE:
-        return run_non_native_backend(task_cfg)
 
     for dataset_name in task_cfg.datasets:
         evaluator = create_evaluator(task_cfg, dataset_name, outputs)
@@ -143,45 +116,35 @@ def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsSt
     imported_modules = import_module_util(BENCHMARK_PATH_PREFIX, dataset_name, MEMBERS_TO_IMPORT)
     model_adapter = initialize_model_adapter(task_cfg, dataset_name, imported_modules)
 
-    if dataset_name == 'humaneval':
-        problem_file = task_cfg.dataset_args.get('humaneval', {}).get('local_path')
-        return HumanevalEvaluator(
-            problem_file=problem_file,
-            model_id=task_cfg.model,
-            model_revision=task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION),
-            model_adapter=model_adapter,
-            outputs=outputs,
-            is_custom_outputs_dir=False,
-        )
-    else:
-        dataset_config = task_cfg.dataset_args.get(dataset_name, {})
-        dataset_name_or_path = dataset_config.get('local_path') or imported_modules['DATASET_ID']
-        in_prompt_template = dataset_config.get('prompt_template', '')
-        few_shot_num = dataset_config.get('few_shot_num', None)
-        few_shot_random = dataset_config.get('few_shot_random', True)
-
-        data_adapter = imported_modules['DataAdapterClass'](
-            few_shot_num=few_shot_num,
-            few_shot_random=few_shot_random,
-            prompt_template=in_prompt_template,
-        )
-        in_subset_list = dataset_config.get('subset_list', imported_modules['SUBSET_LIST'])
-
-        logger.info(f'Evaluating on subsets for {dataset_name}: {in_subset_list}\n')
-
-        return Evaluator(
-            dataset_name_or_path=dataset_name_or_path,
-            subset_list=in_subset_list,
-            data_adapter=data_adapter,
-            model_adapter=model_adapter,
-            use_cache=task_cfg.use_cache,
-            outputs=outputs,
-            datasets_dir=task_cfg.dataset_dir,
-            datasets_hub=task_cfg.dataset_hub,
-            stage=task_cfg.stage,
-            eval_type=task_cfg.eval_type,
-            overall_task_cfg=task_cfg,
-        )
+    dataset_config = task_cfg.dataset_args.get(dataset_name, {})
+    dataset_name_or_path = dataset_config.get('local_path') or imported_modules['DATASET_ID']
+    in_prompt_template = dataset_config.get('prompt_template', '')
+    few_shot_num = dataset_config.get('few_shot_num', None)
+    few_shot_random = dataset_config.get('few_shot_random', True)
+
+    data_adapter = imported_modules['DataAdapterClass'](
+        few_shot_num=few_shot_num,
+        few_shot_random=few_shot_random,
+        prompt_template=in_prompt_template,
+        outputs=outputs,
+    )
+    in_subset_list = dataset_config.get('subset_list', imported_modules['SUBSET_LIST'])
+
+    logger.info(f'Evaluating on subsets for {dataset_name}: {in_subset_list}\n')
+
+    return Evaluator(
+        dataset_name_or_path=dataset_name_or_path,
+        subset_list=in_subset_list,
+        data_adapter=data_adapter,
+        model_adapter=model_adapter,
+        use_cache=task_cfg.use_cache,
+        outputs=outputs,
+        datasets_dir=task_cfg.dataset_dir,
+        datasets_hub=task_cfg.dataset_hub,
+        stage=task_cfg.stage,
+        eval_type=task_cfg.eval_type,
+        overall_task_cfg=task_cfg,
+    )
 
 
 def initialize_model_adapter(task_cfg: TaskConfig, dataset_name: str, imported_modules):

evalscope/run_arena.py

@@ -11,7 +11,8 @@ from tqdm import tqdm
 from evalscope.constants import EvalConfigKeys
 from evalscope.evaluator.rating_eval import RatingEvaluate
 from evalscope.models.model_adapter import ChatGenerationModelAdapter
-from evalscope.utils import dump_jsonl_data, get_obj_from_cfg, jsonl_to_list, yaml_to_dict
+from evalscope.utils import get_obj_from_cfg
+from evalscope.utils.io_utils import dump_jsonl_data, jsonl_to_list, yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/summarizer.py

@@ -4,10 +4,11 @@ import json
 import os
 from typing import List, Union
 
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalBackend, OutputsStructure
+from evalscope.config import TaskConfig, parse_task_config
+from evalscope.constants import EvalBackend
 from evalscope.tools.combine_reports import gen_table
-from evalscope.utils import csv_to_list, get_latest_folder_path, json_to_dict, yaml_to_dict
+from evalscope.utils import csv_to_list, get_latest_folder_path
+from evalscope.utils.io_utils import OutputsStructure, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -24,7 +25,7 @@ class Summarizer:
         if reports_dir is None:
             raise ValueError(f'No reports directory in {outputs_dir}')
 
-        report_files: list = glob.glob(os.path.join(reports_dir, '*.json'))
+        report_files: list = glob.glob(os.path.join(reports_dir, '**/*.json'))
         for report_file in report_files:
             with open(report_file, 'r') as f:
                 res_list.append(json.load(f))
@@ -47,33 +48,20 @@ class Summarizer:
             A report dict is overall report on a benchmark for specific model.
         """
         final_res_list: List[dict] = []
-        candidate_task_cfgs: List[dict] = []
-
-        if isinstance(task_cfg, dict):
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, str):
-            task_cfg: dict = yaml_to_dict(task_cfg)
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, TaskConfig):
-            task_cfg: dict = task_cfg.to_dict()
-            candidate_task_cfgs = [task_cfg]
-        elif isinstance(task_cfg, list):
+        candidate_task_cfgs: List[TaskConfig] = []
+
+        if isinstance(task_cfg, list):
             for task_cfg_item in task_cfg:
-                if isinstance(task_cfg_item, str):
-                    task_cfg_item: dict = yaml_to_dict(task_cfg_item)
-                elif isinstance(task_cfg_item, TaskConfig):
-                    task_cfg_item: dict = task_cfg_item.to_dict()
-                candidate_task_cfgs.append(task_cfg_item)
+                candidate_task_cfgs.append(parse_task_config(task_cfg_item))
         else:
-            raise ValueError(f'Invalid task_cfg: {task_cfg}')
+            candidate_task_cfgs.append(parse_task_config(task_cfg))
 
         for candidate_task in candidate_task_cfgs:
             logger.info(f'**Loading task cfg for summarizer: {candidate_task}')
-            eval_backend = candidate_task.get('eval_backend') or EvalBackend.NATIVE
+            eval_backend = candidate_task.eval_backend
 
             if eval_backend == EvalBackend.NATIVE:
-                outputs_dir: str = candidate_task.get('outputs')
-                outputs_dir: str = os.path.expanduser(outputs_dir)
+                outputs_dir: str = os.path.expanduser(candidate_task.work_dir)
                 if outputs_dir is None:
                     raise ValueError(f'No outputs_dir in {task_cfg}')
                 res_list: list = Summarizer.get_report(outputs_dir=outputs_dir)
@@ -128,8 +116,8 @@ class Summarizer:
         return final_res_list
 
     @staticmethod
-    def parse_eval_config(candidate_task):
-        eval_config: Union[str, dict] = candidate_task.get('eval_config')
+    def parse_eval_config(candidate_task: TaskConfig):
+        eval_config: Union[str, dict] = candidate_task.eval_config
         assert eval_config is not None, 'Please provide eval_config for specific evaluation backend.'
 
         if isinstance(eval_config, str):

evalscope/third_party/longbench_write/eval.py

@@ -10,7 +10,8 @@ import requests
 from concurrent.futures import ThreadPoolExecutor
 from tqdm import tqdm
 
-from evalscope.utils import get_logger, jsonl_to_list
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import jsonl_to_list
 
 logger = get_logger()
 

evalscope/third_party/longbench_write/longbench_write.py

@@ -4,7 +4,8 @@ from typing import Union
 
 from evalscope.third_party.longbench_write.eval import run_eval
 from evalscope.third_party.longbench_write.infer import run_infer
-from evalscope.utils import get_logger, json_to_dict, yaml_to_dict
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
 
 logger = get_logger()
 

evalscope/third_party/longbench_write/tools/data_etl.py

@@ -6,7 +6,7 @@ from typing import List
 
 from evalscope.third_party.longbench_write.eval import EvalLength
 from evalscope.third_party.longbench_write.utils import chinese_to_arabic, count_words
-from evalscope.utils import jsonl_to_list
+from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/third_party/toolbench_static/toolbench_static.py

@@ -5,7 +5,8 @@ from typing import Union
 
 from evalscope.third_party.toolbench_static.eval import EvalArgs, run_eval
 from evalscope.third_party.toolbench_static.infer import InferArgs, run_infer
-from evalscope.utils import get_logger, json_to_dict, yaml_to_dict
+from evalscope.utils import get_logger
+from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
 
 logger = get_logger()
 

evalscope/tools/combine_reports.py

@@ -19,16 +19,14 @@ def get_report(report_file: str):
     dataset_name = data_d['dataset_name']
     model_name = data_d['model_name']
     score = data_d['score']  # float or dict
+    metric = data_d['metric']
     score_d = {}
     if isinstance(score, dict):
-        # score_d = dict([(k, round(v, 4) * 100) for k, v in score.items()])
         score_d = score
     elif isinstance(score, float):
-        # score_d['acc'] = round(score, 4) * 100
-        score_d['acc'] = score
+        score_d[metric] = score
     else:
         raise ValueError(f'Unknown score type: {type(score)}')
-    # score_str = '\n'.join([str(v) + ' (' + k + ')' for k, v in score_d.items()])
     score_str = '\n'.join(['(' + dataset_name + '/' + k + ') ' + str(v) for k, v in score_d.items()])
 
     return model_name, {'dataset_name': dataset_name, 'score': score_str}

evalscope/tools/rewrite_eval_results.py

@@ -4,7 +4,7 @@ import time
 
 from evalscope.models.custom import CustomModel
 from evalscope.run import run_task
-from evalscope.utils import yaml_to_dict
+from evalscope.utils.io_utils import yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

evalscope/utils/__init__.py

@@ -1,3 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+from evalscope.constants import *
 from evalscope.utils.utils import *

evalscope/utils/chat_service.py

@@ -43,7 +43,7 @@ class DeltaMessage(BaseModel):
 
 class ChatCompletionRequest(BaseModel):
     model: str
-    messages: List[ChatMessage] | str
+    messages: Union[List[ChatMessage], str]
     temperature: Optional[float] = None
     top_p: Optional[float] = None
     max_tokens: Optional[int] = 2048

evalscope/utils/io_utils.py

@@ -0,0 +1,162 @@
+import json
+import jsonlines as jsonl
+import os
+import yaml
+
+from evalscope.constants import DumpMode
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class OutputsStructure:
+    LOGS_DIR = 'logs'
+    PREDICTIONS_DIR = 'predictions'
+    REVIEWS_DIR = 'reviews'
+    REPORTS_DIR = 'reports'
+    CONFIGS_DIR = 'configs'
+
+    def __init__(self, outputs_dir: str, is_make=True):
+        self.outputs_dir = outputs_dir
+        self.is_make = is_make
+        self._dirs = {
+            'logs_dir': None,
+            'predictions_dir': None,
+            'reviews_dir': None,
+            'reports_dir': None,
+            'configs_dir': None
+        }
+
+    def _get_dir(self, attr_name, dir_name):
+        if self._dirs[attr_name] is None:
+            dir_path = os.path.join(self.outputs_dir, dir_name)
+            if self.is_make:
+                os.makedirs(dir_path, exist_ok=True)
+            self._dirs[attr_name] = dir_path
+        return self._dirs[attr_name]
+
+    @property
+    def logs_dir(self):
+        return self._get_dir('logs_dir', OutputsStructure.LOGS_DIR)
+
+    @property
+    def predictions_dir(self):
+        return self._get_dir('predictions_dir', OutputsStructure.PREDICTIONS_DIR)
+
+    @property
+    def reviews_dir(self):
+        return self._get_dir('reviews_dir', OutputsStructure.REVIEWS_DIR)
+
+    @property
+    def reports_dir(self):
+        return self._get_dir('reports_dir', OutputsStructure.REPORTS_DIR)
+
+    @property
+    def configs_dir(self):
+        return self._get_dir('configs_dir', OutputsStructure.CONFIGS_DIR)
+
+
+def jsonl_to_list(jsonl_file):
+    """
+    Read jsonl file to list.
+
+    Args:
+        jsonl_file: jsonl file path.
+
+    Returns:
+        list: list of lines. Each line is a dict.
+    """
+    res_list = []
+    with jsonl.open(jsonl_file, mode='r') as reader:
+        for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
+            res_list.append(line)
+    return res_list
+
+
+def jsonl_to_reader(jsonl_file):
+    """
+    Read jsonl file to reader object.
+
+    Args:
+        jsonl_file: jsonl file path.
+
+    Returns:
+        reader: jsonl reader object.
+    """
+    with jsonl.open(jsonl_file, mode='r') as reader:
+        return reader
+
+
+def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
+    """
+    Dump data to jsonl file.
+
+    Args:
+        data_list: data list to be dumped.  [{'a': 'aaa'}, ...]
+        jsonl_file: jsonl file path.
+        dump_mode: dump mode. It can be 'overwrite' or 'append'.
+    """
+    if not jsonl_file:
+        raise ValueError('output file must be provided.')
+
+    jsonl_file = os.path.expanduser(jsonl_file)
+
+    if not isinstance(data_list, list):
+        data_list = [data_list]
+
+    if dump_mode == DumpMode.OVERWRITE:
+        dump_mode = 'w'
+    elif dump_mode == DumpMode.APPEND:
+        dump_mode = 'a'
+    with jsonl.open(jsonl_file, mode=dump_mode) as writer:
+        writer.write_all(data_list)
+
+
+def jsonl_to_csv():
+    pass
+
+
+def yaml_to_dict(yaml_file) -> dict:
+    """
+    Read yaml file to dict.
+    """
+    with open(yaml_file, 'r') as f:
+        try:
+            stream = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            logger.error(f'{e}')
+            raise e
+
+    return stream
+
+
+def dict_to_yaml(d: dict, yaml_file: str):
+    """
+    Dump dict to yaml file.
+    """
+    with open(yaml_file, 'w') as f:
+        yaml.dump(d, f, default_flow_style=False)
+
+
+def json_to_dict(json_file) -> dict:
+    """
+    Read json file to dict.
+    """
+    with open(json_file, 'r') as f:
+        try:
+            stream = json.load(f)
+        except json.JSONDecodeError as e:
+            logger.error(f'{e}')
+            raise e
+
+    return stream
+
+
+def are_paths_same(path1, path2):
+    """
+    Check if two paths are the same.
+    """
+    real_path1 = os.path.realpath(os.path.abspath(os.path.expanduser(path1)))
+    real_path2 = os.path.realpath(os.path.abspath(os.path.expanduser(path2)))
+
+    return real_path1 == real_path2

evalscope/utils/logger.py

@@ -75,6 +75,14 @@ def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, f
     return logger
 
 
+def configure_logging(debug: bool, log_file: Optional[str] = None):
+    """Configure logging level based on the debug flag."""
+    if log_file:
+        get_logger(log_file=log_file, force=True)
+    if debug:
+        get_logger(log_level=logging.DEBUG, force=True)
+
+
 def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
     for handler in logger.handlers:
         if isinstance(handler, logging.FileHandler):