evalscope 0.8.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/utils/utils.py

@@ -5,19 +5,13 @@ import functools
 import hashlib
 import importlib
 import importlib.util
-import json
-import jsonlines as jsonl
 import numpy as np
 import os
 import random
 import re
-import sys
 import torch
-import torch.nn.functional as F
-import yaml
 from typing import Any, Dict, List, Tuple, Union
 
-from evalscope.constants import DumpMode
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -36,102 +30,6 @@ def test_level_list():
     return TEST_LEVEL_LIST
 
 
-def jsonl_to_list(jsonl_file):
-    """
-    Read jsonl file to list.
-
-    Args:
-        jsonl_file: jsonl file path.
-
-    Returns:
-        list: list of lines. Each line is a dict.
-    """
-    res_list = []
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
-            res_list.append(line)
-    return res_list
-
-
-def jsonl_to_reader(jsonl_file):
-    """
-    Read jsonl file to reader object.
-
-    Args:
-        jsonl_file: jsonl file path.
-
-    Returns:
-        reader: jsonl reader object.
-    """
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        return reader
-
-
-def jsonl_to_csv():
-    pass
-
-
-def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
-    """
-    Dump data to jsonl file.
-
-    Args:
-        data_list: data list to be dumped.  [{'a': 'aaa'}, ...]
-        jsonl_file: jsonl file path.
-        dump_mode: dump mode. It can be 'overwrite' or 'append'.
-    """
-    if not jsonl_file:
-        raise ValueError('output file must be provided.')
-
-    jsonl_file = os.path.expanduser(jsonl_file)
-
-    if not isinstance(data_list, list):
-        data_list = [data_list]
-
-    if dump_mode == DumpMode.OVERWRITE:
-        dump_mode = 'w'
-    elif dump_mode == DumpMode.APPEND:
-        dump_mode = 'a'
-    with jsonl.open(jsonl_file, mode=dump_mode) as writer:
-        writer.write_all(data_list)
-
-
-def yaml_to_dict(yaml_file) -> dict:
-    """
-    Read yaml file to dict.
-    """
-    with open(yaml_file, 'r') as f:
-        try:
-            stream = yaml.safe_load(f)
-        except yaml.YAMLError as e:
-            logger.error(f'{e}')
-            raise e
-
-    return stream
-
-
-def dict_to_yaml(d: dict, yaml_file: str):
-    """
-    Dump dict to yaml file.
-    """
-    with open(yaml_file, 'w') as f:
-        yaml.dump(d, f, default_flow_style=False)
-
-
-def json_to_dict(json_file) -> dict:
-    """
-    Read json file to dict.
-    """
-    with open(json_file, 'r') as f:
-        try:
-            stream = json.load(f)
-        except json.JSONDecodeError as e:
-            logger.error(f'{e}')
-            raise e
-
-    return stream
-
-
 def get_obj_from_cfg(eval_class_ref: Any, *args, **kwargs) -> Any:
     module_name, spliter, cls_name = eval_class_ref.partition(':')
 
@@ -300,18 +198,6 @@ class ResponseParser:
         return ''
 
 
-def make_outputs_dir(root_dir: str, datasets: list, model_id: str, model_revision: str):
-    if not model_id:
-        model_id = 'default'
-    model_id = model_id.replace('/', '_')
-
-    if not model_revision:
-        model_revision = 'default'
-
-    outputs_dir = os.path.join(root_dir, model_id, model_revision, f"eval_{'-'.join(datasets)}")
-
-    return outputs_dir
-
 
 def import_module_util(import_path_prefix: str, module_name: str, members_to_import: list) -> dict:
     """
@@ -355,67 +241,6 @@ def normalize_score(score: Union[float, dict], keep_num: int = 4) -> Union[float
     return score
 
 
-def split_str_parts_by(text: str, delimiters: List[str]):
-    """Split the text field into parts.
-    Args:
-        text: A text to be split.
-        delimiters: The delimiters.
-    Returns:
-        The split text in list of dicts.
-    """
-    all_start_chars = [d[0] for d in delimiters]
-    all_length = [len(d) for d in delimiters]
-
-    text_list = []
-    last_words = ''
-
-    while len(text) > 0:
-        for char_idx, char in enumerate(text):
-            match_index = [idx for idx, start_char in enumerate(all_start_chars) if start_char == char]
-            is_delimiter = False
-            for index in match_index:
-                if text[char_idx:char_idx + all_length[index]] == delimiters[index]:
-                    if last_words:
-                        if text_list:
-                            text_list[-1]['content'] = last_words
-                        else:
-                            text_list.append({'key': '', 'content': last_words})
-                    last_words = ''
-                    text_list.append({'key': delimiters[index]})
-                    text = text[char_idx + all_length[index]:]
-                    is_delimiter = True
-                    break
-            if not is_delimiter:
-                last_words += char
-            else:
-                break
-        if last_words == text:
-            text = ''
-
-    text_list[-1]['content'] = last_words
-    return text_list
-
-
-def get_bucket_sizes(max_length: int) -> List[int]:
-    return [max_length // 4 * (i + 1) for i in range(4)]
-
-
-def _get_closet_bucket(bucket_sizes, data_length):
-    """Select the one from bucket_sizes that is closest in distance to
-    data_length. This is required for TorchAcc.
-    """
-    cloest_length = sys.maxsize
-    for b in bucket_sizes:
-        if b == data_length or ((b < cloest_length) and (b > data_length)):
-            cloest_length = b
-
-    if cloest_length == sys.maxsize:
-        bucket_sizes.append(data_length)
-        cloest_length = data_length
-
-    return cloest_length
-
-
 def is_module_installed(module_name):
     try:
         importlib.import_module(module_name)

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.8.0'
-__release_datetime__ = '2024-12-15 00:00:00'
+__version__ = '0.8.1'
+__release_datetime__ = '2024-12-17 20:00:00'

{evalscope-0.8.0.dist-info → evalscope-0.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.8.0
+Version: 0.8.1
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team

{evalscope-0.8.0.dist-info → evalscope-0.8.1.dist-info}/RECORD

@@ -1,13 +1,13 @@
 evalscope/__init__.py,sha256=RY0EjssSquqqsysRobElYm9Ix6E41uTXeaeh7lI7kqs,106
 evalscope/arguments.py,sha256=nozBnog45l77jxTFH_lyyJkj04ER3yyIpICepc2tC1Y,3783
-evalscope/config.py,sha256=KYS_O0RdAbruQhqP6mp3rQL0003Oaskx03IroZUGRps,6897
-evalscope/constants.py,sha256=D2MU7bs_qwmcHQ1ge05C5Ekk04XqMyiGxssvKwAecxI,4515
-evalscope/run.py,sha256=5cG81qfdpMN_GtPphvJ7BHboD6LBYHWyodX8ViR1XL4,8874
-evalscope/run_arena.py,sha256=Kmzak4TGdATbOhOCe_zLLRxDvgtkOfs6e4VaxOAzPKk,8550
-evalscope/summarizer.py,sha256=Eq7ZqGKuvrhWVeGriLxHCGupgnJmtvmIGqZYzRNaY8I,6480
-evalscope/version.py,sha256=Xha7v5_YH0Oppyh6iO7HrpSsmv1WCPdQPFtzYTJvG4A,118
+evalscope/config.py,sha256=ZDN0XVCCXMSSD675Smzm57fNDOx-cZTsNvPboMtYVow,8407
+evalscope/constants.py,sha256=M5qJ8b7kp-RF52IwBjx5EMjeuiH1e1jdollCsbIT-c4,3753
+evalscope/run.py,sha256=s_qE1ukrt4HBfRVAPJjC1XiqD9k7rSH7lX8yysyf5do,7279
+evalscope/run_arena.py,sha256=6nc_S8KL7B3V4SsnpIexfvczHN9kQwHR9R1GXb2sqgI,8586
+evalscope/summarizer.py,sha256=FgdYz7LlNs5XpDMlj2ULkVQGIg5XVeeWdWJ1_OMweq0,5882
+evalscope/version.py,sha256=OXwZDg6ML1mbsIw-CBhWRf4zVz2ArW2PFzzLK9FVAZk,118
 evalscope/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/backend/base.py,sha256=l7zUHXX2XToIfU_hkVeTSHT9wWURYumyohXCIgywZBI,1021
+evalscope/backend/base.py,sha256=qYu8Shokrtrx-N6T_BAJk_6OCpovUBYuN0p3wngt-dw,1030
 evalscope/backend/opencompass/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
 evalscope/backend/opencompass/api_meta_template.py,sha256=DaBJg15ZSIjxroXiygl3-4RdmIe_FD7xHbXvjSZmkQA,1706
 evalscope/backend/opencompass/backend_manager.py,sha256=y5NnAIY1pI7E1ZSeKU3acrD-oyH3uMGL7M3nPp1WiHU,10381
@@ -68,8 +68,8 @@ evalscope/backend/rag_eval/ragas/tasks/build_transform.py,sha256=GtAYqdVOy7BxIGy
 evalscope/backend/rag_eval/ragas/tasks/testset_generation.py,sha256=B5ZETlQw5XTEDnO-VR5yXjSbbg1eUtjGts7M5msK2ik,5618
 evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py,sha256=aP8U9zjIDl26X_YF82SXLpkxoJ4nUurmdKSEoJ-qsLY,2129
 evalscope/backend/rag_eval/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/backend/rag_eval/utils/clip.py,sha256=WZovQJGyPI33Y-9bUnanR6fIYJzrXgnjD4zVwUJSgCw,5002
-evalscope/backend/rag_eval/utils/embedding.py,sha256=XWI07YeWDALc2etP4DGluYqrid85nKz1tjM91JLZRmM,6252
+evalscope/backend/rag_eval/utils/clip.py,sha256=GLHhPCac2AH35AvRLvVqePA1gIMAewHTFmCJCDZzvqU,5015
+evalscope/backend/rag_eval/utils/embedding.py,sha256=x9HAEfZSSAnT2Tdbf-9a5UmBVagCr__ay5A2nMCPMpg,6258
 evalscope/backend/rag_eval/utils/llm.py,sha256=619eP8pXUcwIBaktBrGNA17j53j9jfg_1JeFDYzMCIE,2582
 evalscope/backend/rag_eval/utils/tools.py,sha256=FU7tNu-8y8V_o_kArFVTTLM_GzL12KBNeXiwQw5SpJA,1529
 evalscope/backend/vlm_eval_kit/__init__.py,sha256=R-GuBm8dAwvDF73XHaGpPSjlt7Y4tycyy-FJgzLdjeY,84
@@ -122,23 +122,23 @@ evalscope/benchmarks/competition_math/__init__.py,sha256=CDK03RXT-X21WcIAlkrCs0r
 evalscope/benchmarks/competition_math/competition_math.py,sha256=Cehyokift7oDKjc8TdmfblZ6mMc39wQWtqqbUi34QLc,2629
 evalscope/benchmarks/competition_math/competition_math_adapter.py,sha256=cHWJ6LLIWvftFXjGrOidMlZ1RGUFxPgDjs4wmBPSm1Y,18862
 evalscope/benchmarks/general_qa/__init__.py,sha256=N2t-ehNrl9eVAarlSgJvRapm9yOjhfCWhNPPfcUUy-s,409
-evalscope/benchmarks/general_qa/general_qa_adapter.py,sha256=Y7_d6hmh94W2XbzUnDMX9_uKWcarK0zv4Q4mQWUfSZ8,5869
+evalscope/benchmarks/general_qa/general_qa_adapter.py,sha256=cSW0Mq9__-gh-tVoVXD9Rk6h3h2iZW-Fu3RQ16haJhQ,5878
 evalscope/benchmarks/gsm8k/__init__.py,sha256=CtcG_QM8m5zmvMs2N53d7kcm4_hIgsO2qYPyx-71aLw,313
 evalscope/benchmarks/gsm8k/gsm8k.py,sha256=ZDN5lfeZyc_pkTDVY0voC_zUExHE1ZoEgEaTvt5hpXg,4233
-evalscope/benchmarks/gsm8k/gsm8k_adapter.py,sha256=gg65W_pz4mPOBUOwaYIgfUxGKzrmRZRuoEg5xtS8bYg,13830
+evalscope/benchmarks/gsm8k/gsm8k_adapter.py,sha256=KBI9t5F7XW1Cs44QUA7ultkfsXxLyucH9zNYe-jOQQk,13866
 evalscope/benchmarks/hellaswag/__init__.py,sha256=cY1kluaTqC7AvyzwlQYc3BF_kB3LD1gOpg6i7RDr0cI,415
 evalscope/benchmarks/hellaswag/hellaswag.py,sha256=5_c9WbaS1LIdvgXzqEcvjAEtKi2V2Yn0YtszPlFqhXI,4610
-evalscope/benchmarks/hellaswag/hellaswag_adapter.py,sha256=7REJeC8vD8OVtmcqI5TP6cTn88-KOzBs5oOKEZEmESs,8459
+evalscope/benchmarks/hellaswag/hellaswag_adapter.py,sha256=IIesSMPw1Yya4-LjqJt1QVkpOx8RGKwBYTQtmc0VfaQ,8495
 evalscope/benchmarks/humaneval/__init__.py,sha256=lqSlAf1-8Nzhc1j89sj6yAcaLt9pGhqu15M84bmzamc,333
 evalscope/benchmarks/humaneval/humaneval.py,sha256=2Exsg6u8FEu0buADY2tETJluSM8tWacvX06nykKKLSE,3395
-evalscope/benchmarks/humaneval/humaneval_adapter.py,sha256=HxAjkIA-Wt5-wb8kNSDMzZRoHflgsNxIfa1BoeVzwog,1660
+evalscope/benchmarks/humaneval/humaneval_adapter.py,sha256=VAO7siedusq9z3b1J3ztFE4XDopYKqmwe2n-Numg7HY,9149
 evalscope/benchmarks/mmlu/__init__.py,sha256=OGiN1J80WDM72y242o7diYT9Rl-jkVEqTNntCl8Vt4M,385
 evalscope/benchmarks/mmlu/mmlu.py,sha256=sA8AC0bN7iURrSazqkY31s_reNVbDZSUCB-NCTQsVeI,5042
 evalscope/benchmarks/mmlu/mmlu_adapter.py,sha256=8T-fN_Az0gWOyME9nHl3MvcD144TjWknFKcEOMHppAI,15494
 evalscope/benchmarks/mmlu/samples.jsonl,sha256=f5Y2vwbEvNtpE7vrl9BHoJzsdceI4vUAo1frexYyX2o,1345
 evalscope/benchmarks/race/__init__.py,sha256=HVda-CB-Q-N8RbwiVLADXYNY6VLUH-frJ8VCc3jm0Mk,385
 evalscope/benchmarks/race/race.py,sha256=TtFC3opqEA6q8AQIAFQRGx07FjD9z7iW8wmtxeO61nU,3608
-evalscope/benchmarks/race/race_adapter.py,sha256=Ppo7bttx15zB-m-UtguIwIXgqpEKAi_ClIOol0hPQiE,9805
+evalscope/benchmarks/race/race_adapter.py,sha256=WgnWYSctc3VtWm2FAeVDTlxR2hwXsF2tala7n66f5mw,9841
 evalscope/benchmarks/race/samples.jsonl,sha256=bhSktBgU6axYQCClRtQ7nN8D1x815AU8xMAIG1oflG0,1243
 evalscope/benchmarks/trivia_qa/__init__.py,sha256=eLMVC6tfwty5HqrQuGyWeAF2IhRNajWoO1SkLVemQj4,409
 evalscope/benchmarks/trivia_qa/samples.jsonl,sha256=1isBD62PGhCiNbzQa-GFrHHL4XLHIkojWfgSvn7ktf8,3445
@@ -153,12 +153,11 @@ evalscope/cli/cli.py,sha256=yNL3ZeolBc-cVr5D4GByGZWKrmpKIK-48R6wXOXO7Y0,641
 evalscope/cli/start_eval.py,sha256=2lyD2WSQ0DnP6T31VvTimQ-6POnwxeEP9GLPFnT7Tfo,767
 evalscope/cli/start_perf.py,sha256=lEHJBSpzNsO4KGlWfQc-EfZGXq1M_FpOwtRxRdb4fso,813
 evalscope/cli/start_server.py,sha256=DQRIfbsHaOAsVcLGF6iRyJnxmd5Sf_tgytpJNfiWCeE,3662
-evalscope/evaluator/__init__.py,sha256=h_EyZm7vDqBsGx6CkoQVLg0aMy0tE_IG5uEnheubb0s,174
-evalscope/evaluator/evaluator.py,sha256=MGkuJi9o5Hdbj_fN7qolDqP0B47i9i0ksGd1uc-TMn0,18365
-evalscope/evaluator/humaneval_evaluator.py,sha256=245XRxwulGQpjdapwU8CiYJn1xT0XKxl7hdWvzFxLG0,5964
-evalscope/evaluator/rating_eval.py,sha256=VuDIZcmSlsv1tc8znDGesz8ZwpQ7NvZJPv823Quvht0,5566
+evalscope/evaluator/__init__.py,sha256=S6MU1O_iiNAaKxNIhO9MEmdW-BSNf_YH2l6NQ9lxVNo,103
+evalscope/evaluator/evaluator.py,sha256=nRR6aaa9J8nRfB8QPZwexSrfKDvPkPSGQpFVpbWLeW0,18380
+evalscope/evaluator/rating_eval.py,sha256=uo0uj9z_TDsxdYlT8WIfNZhFLAfRkW9zn_wlu-F72O0,5575
 evalscope/evaluator/reviewer/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-evalscope/evaluator/reviewer/auto_reviewer.py,sha256=YVTJAHK0uz9hNupsdeTXMM2PISECf8phXq0GYPr4law,16378
+evalscope/evaluator/reviewer/auto_reviewer.py,sha256=nL8k-i92L1iMwjPOnNxzQyZICfukZKJul4ZBvOWkHGw,16414
 evalscope/metrics/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/metrics/code_metric.py,sha256=EXE2BZAc9JJT_cpd6eCb0Jo9wwtnzXzHBZxmLzG5Jpw,3300
 evalscope/metrics/math_accuracy.py,sha256=WqLfACuIeVFrX4q6_c2exnTLn2t10-rjv6sfxcqJJ14,1965
@@ -178,17 +177,17 @@ evalscope/models/api/openai_api.py,sha256=PiIvvDYJkn041SJkLoroXwl1B8TtwpB7licVfq
 evalscope/models/custom/__init__.py,sha256=wb6f_Bi39s5sj-VO7EXRDXB2WhyFb49BUtEMk77ksNQ,102
 evalscope/models/custom/custom_model.py,sha256=rBQLAuPEw_OPUtRSCEmxEfpcA8jPj8bAdsmtKs4ygus,1566
 evalscope/perf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/perf/arguments.py,sha256=_gW1tq7SbrAZd05N-FbY_oWrQB0Djs4KUaFdXSfFsr8,9112
-evalscope/perf/benchmark.py,sha256=ff9PFFMY5UucuUihcdo6lSf1X9XXoaOmrpBvjDk5Mrw,9599
+evalscope/perf/arguments.py,sha256=J067vNJF-RObJNZ0oE2RBIBNjliCYcflWtt6aGAt40g,9205
+evalscope/perf/benchmark.py,sha256=h151QXsVbg7lMe09aH_mxUdPRALIl1A35I9VO2zryEo,9615
 evalscope/perf/http_client.py,sha256=TfnQT9OaBlUCpGwi4ifSJBaaGsn3P2KVBPMGuw-Rqkk,7073
-evalscope/perf/main.py,sha256=-8NsvJZ7uyVfJT9N2lX36KfsHkVTy0r8OcsWPYoKms0,1316
+evalscope/perf/main.py,sha256=2GrE9wHibprzaw4gmcovdc5ods_EHwoSwwmkFDLTUjQ,1257
 evalscope/perf/plugin/__init__.py,sha256=1sl5s-csrwKb_LVTnpF3HqArz06TRD5LYJ0hpqvokUA,85
-evalscope/perf/plugin/registry.py,sha256=PyK3E1AqQFuU4Bs9COvFFCJOaCtmHbfeQOVGtjVYh-I,1304
+evalscope/perf/plugin/registry.py,sha256=w1IAt6GDdluzSYK5i-yrntvx3_EvIIqJamEL0xZv3zA,1323
 evalscope/perf/plugin/api/__init__.py,sha256=Ckzbq4CkSMVQTedQcDHCYlRd6FTwQAElt2mHB-VXJac,195
 evalscope/perf/plugin/api/base.py,sha256=B_H04qKx7eRTn155rnDrbTYur7PK1mvxfQKYcqYbndU,2118
 evalscope/perf/plugin/api/custom_api.py,sha256=IplmkCu8v9yQrY5CeqBEQDWdOfOp3vRkiDYUcvhw2yY,3775
 evalscope/perf/plugin/api/dashscope_api.py,sha256=V5fwn-p_fLH0dWKzhN9TvYSHRgla4INfXC4NDaIjoQ8,3825
-evalscope/perf/plugin/api/openai_api.py,sha256=KRN6EjObTG08mcI82kJD3dGK7DoVMUZzrUZ1AgoLEp0,7007
+evalscope/perf/plugin/api/openai_api.py,sha256=raa4SaatEphNfWuK6_3ecfe49Vg4yftD6C-enhufJuE,7020
 evalscope/perf/plugin/datasets/__init__.py,sha256=9mz2TnVHhxbEKAS9pLbKMQuIoShNlZpGiRo9e2RQLUs,490
 evalscope/perf/plugin/datasets/base.py,sha256=Z-INWueeYjfEZhP4lbTlBMVwIa6BcXZKWx-w7Pop3mA,1786
 evalscope/perf/plugin/datasets/custom.py,sha256=_GSC5yR90_BjcRjdJqrJT2vHQAzskz5XxYOxngUM2Pg,815
@@ -199,10 +198,10 @@ evalscope/perf/plugin/datasets/openqa.py,sha256=2pv7yyPSFYTjPhvAGBsHl0eQO8gt7Wk1
 evalscope/perf/plugin/datasets/speed_benchmark.py,sha256=ef8MXhT6756y6LsXSpYeWjmwswu2hRXe2BOVS2_OgVM,1968
 evalscope/perf/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/perf/utils/analysis_result.py,sha256=ig0zPwbUODGh1GUr3GmnNF4lJJp9SQvW0awWiXEIkCI,1212
-evalscope/perf/utils/benchmark_util.py,sha256=xFZSSUoBoFpHRZC69-KS9cK2vqJlL7rIuCEz_MnpnGA,5564
-evalscope/perf/utils/db_util.py,sha256=A2K3otCrNw3K1SMwoYo8a6jekT5nAVvWJepqi31DH28,7479
+evalscope/perf/utils/benchmark_util.py,sha256=T_pXpSCwCNLJgfzgv3IO7kG61ghTLthVMsXZhBCGP_4,5541
+evalscope/perf/utils/db_util.py,sha256=PSBq16uWyzXx0zyoEE4wazWKN19UAA8_GjobS7rTPso,9001
 evalscope/perf/utils/handler.py,sha256=HyKIxbzC0XCyQanlbb7UEY7yaeqjJTePNea8kMV3Sdc,1192
-evalscope/perf/utils/local_server.py,sha256=31EQZ8S_SzgSiBFpc9zRU13GXm2jREvRmPDN5qWKgbg,4468
+evalscope/perf/utils/local_server.py,sha256=A26gqBbxsnZA8CqQospyO50x3prVnD9XiT2l--ERxK0,4566
 evalscope/registry/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/registry/config/cfg_arena.yaml,sha256=rub6ceaQxxB1mbSjdoFf0IaVgGfbOonV2nYRebv2OKo,3292
 evalscope/registry/config/cfg_arena_zhihu.yaml,sha256=tvvihBwvoTjoezwTSaZwoGOB44ysofpnin4pNyY9TfQ,2755
@@ -229,9 +228,9 @@ evalscope/third_party/longbench_write/README.md,sha256=1yLKeSVIcihpoc4KXr8NpK86J
 evalscope/third_party/longbench_write/__init__.py,sha256=GNbBDc7HAh_V2Hfy5HhND_u7z6OI79czoBlP8lX4PVo,126
 evalscope/third_party/longbench_write/default_task.json,sha256=d_NPShtW10Mc02U3pAuxX9hXd09tZw7QJAr1SvrECcM,694
 evalscope/third_party/longbench_write/default_task.yaml,sha256=YjU8EeyH9UtM8e7_fhrwJNChQdszOAcrKmOi--Awvhk,578
-evalscope/third_party/longbench_write/eval.py,sha256=bZrpaKg9sPXv2VkUxLpfJiNqMIoIj7Pf3eFMqmDncyY,11229
+evalscope/third_party/longbench_write/eval.py,sha256=39McZSDHL7bA5Dg-BSyZ4EiAF1nfTiYJAnx5FqbNYok,11265
 evalscope/third_party/longbench_write/infer.py,sha256=bFsOp--8Qn6qQ-NpdLY0bennQGQl5TMGEngvGda8k7g,4937
-evalscope/third_party/longbench_write/longbench_write.py,sha256=1caNiJvmZL2vwDU6oHUE4cdCViZGYE8yBo9EsMcA-Qw,3955
+evalscope/third_party/longbench_write/longbench_write.py,sha256=nIR1toB1hvUXR7Lrs3xcY9wqaI-bjeADg_Oscf3HdaY,3991
 evalscope/third_party/longbench_write/utils.py,sha256=nd-YslsOyNGAuyBfAWb2pnTMaGLMQ58lbnJJdrCndeI,815
 evalscope/third_party/longbench_write/resources/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/third_party/longbench_write/resources/judge.txt,sha256=Go1ISY4bUBmEDXXY_DItjAmskuHSaRj5WTNMNH98FSk,1885
@@ -239,7 +238,7 @@ evalscope/third_party/longbench_write/resources/longbench_write.jsonl,sha256=H26
 evalscope/third_party/longbench_write/resources/longbench_write_en.jsonl,sha256=h4AJJ3YfNA5IiZ5N9dR_tyEa1JNqY0INv6l5ZgQUJZ8,24235
 evalscope/third_party/longbench_write/resources/longwrite_ruler.jsonl,sha256=odTr8N8PoWAFZ2kdEcmlLeMDfEo3KXDtLo9S8oieCmI,5718
 evalscope/third_party/longbench_write/tools/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-evalscope/third_party/longbench_write/tools/data_etl.py,sha256=nmWKOrD-GeZi0ZGH5jLCGuW3qiLTui8ASSxI2z8l6ls,5962
+evalscope/third_party/longbench_write/tools/data_etl.py,sha256=T7a-4PwZg5alZQh-oTi1zjMxjGmVVZYVwSR9-diZlF8,5971
 evalscope/third_party/toolbench_static/README.md,sha256=Osdnt0_K-setbmYwDPCPRp2LXxamGp2mE8KsOByPPOY,3944
 evalscope/third_party/toolbench_static/__init__.py,sha256=BO936RxwodHr4OEpV6W3S_keC91OfOd41_msIJ2d0fs,128
 evalscope/third_party/toolbench_static/config_default.json,sha256=KrUzeHL2DNiM5FwY7cH3KZlxTwELCQZ6e39nilfUi0M,368
@@ -247,26 +246,27 @@ evalscope/third_party/toolbench_static/config_default.yaml,sha256=-6n6Zyg9eHN2ee
 evalscope/third_party/toolbench_static/eval.py,sha256=do_-lVi_vEoljeLYvt3b_AYSMqpdKzgYnTek9WLSKe8,8236
 evalscope/third_party/toolbench_static/infer.py,sha256=rsADLhEd2IBcC6EI9aD7hSJmo6Oo5b22mnHWBCZLDPs,9010
 evalscope/third_party/toolbench_static/requirements.txt,sha256=OW91Z8hfzh7yQUYgP1Di_E6DgNgGoGP1UcvnqrdCR68,22
-evalscope/third_party/toolbench_static/toolbench_static.py,sha256=y4nC9WCBCgBg378aWYAdhmrFte_r_XOkigJs7XJ_iXQ,1930
+evalscope/third_party/toolbench_static/toolbench_static.py,sha256=ABb9Gy09zMt30tY50AZGxSZ46k3NVEsvuDj6xlLOjeA,1966
 evalscope/third_party/toolbench_static/llm/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/third_party/toolbench_static/llm/swift_infer.py,sha256=usmVelh0ogBlCtSUL0dqp89w2mAqH1Ptv9MURVoGrc8,1209
 evalscope/tools/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-evalscope/tools/combine_reports.py,sha256=1BJ29IEUKoZLM3NAzg_IpU8B9uhljO9-b_hqAYi9RpA,5078
+evalscope/tools/combine_reports.py,sha256=JFf3P_GJLPdlSqpv30D8ioPb7dup3tOTktsELmsKXLI,4900
 evalscope/tools/gen_mmlu_subject_mapping.py,sha256=CUmRdReEU7QfMyprh9I56KmHoRww_zUda_JuyxmCL1A,3277
-evalscope/tools/rewrite_eval_results.py,sha256=2lbDHfF_9abK1tUk2UYZZRwzO68eoiE36dXyh_b-mwg,2011
-evalscope/utils/__init__.py,sha256=hDS1xpoAxtVH4-ZQOXstdg7WYmjcGPQ62Kh54FIgkwU,87
+evalscope/tools/rewrite_eval_results.py,sha256=ODD6pt9FvZq_a54oYsehBDslRKHOsk9zsC9iAZvi5Yg,2020
+evalscope/utils/__init__.py,sha256=ZOri8VHx8LpJBJS90uw8h0Z7gPhtxhjWlBPWuuZgoRE,121
 evalscope/utils/arena_utils.py,sha256=Gf8VpH4C_oF2Abif_QeL0rAP6tvTzsc0gglpdNkUE48,7155
-evalscope/utils/chat_service.py,sha256=N8lJPiVtzdqsHypa42wzb15T7hduXUrRPtU3Atf8yg4,8641
+evalscope/utils/chat_service.py,sha256=VdNPXdFSf-4zxe0Ht74LBcdRNbpb9vzVi86HDEqfXHc,8647
 evalscope/utils/completion_parsers.py,sha256=YWHkLkSfURTcUjNNlCL6PPDICd4F2Ns9figgPN4C97c,2933
-evalscope/utils/logger.py,sha256=IkY0oxkWSvfA0z1m79crioTiqQcnxulNF5HtJNlV0Fc,3174
+evalscope/utils/io_utils.py,sha256=MnEi4llOYtXK81bUQ_XE_WP5qIsVrJ4MlKmWMH9vzFs,3993
+evalscope/utils/logger.py,sha256=4OGlkBsut_wzq-1UcM2DKQKdKs1FRNYGHw538TGvypU,3440
 evalscope/utils/model_utils.py,sha256=zMS1YRu4CzU4CVLZS6e_lgfHIDBqv3YBTJbPF1R2M90,443
-evalscope/utils/utils.py,sha256=PVtpv3WAIm6Bs66Vz4KBDiAiXp8y6Oejxxr1LWHTRsI,15146
+evalscope/utils/utils.py,sha256=lZl5lt4WqjoY5SEfsum8Sc-s_c9GSlmIZlkTAQkMnjE,10485
 tests/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 tests/test_run_all.py,sha256=YcMTlWoFpvWY8jevWyIf2G_tz8hgDD1cAwSvmyZt96M,429
 tests/cli/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-tests/cli/test_run.py,sha256=lXR35DDLQjdb-XGA6pKnQC9pJTfTOHjknAN7PEaw8G4,4334
+tests/cli/test_run.py,sha256=pMZvI3b0Vs-UFfciDoPwCYFAaYJzocQjxEaMLFTxYSo,4289
 tests/perf/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-tests/perf/test_perf.py,sha256=GD5nInXpQG7H1E8wI6dvy4DFSvTEddGDzv-Cu8YV1ts,2995
+tests/perf/test_perf.py,sha256=AQB2QuMwJ1TnenHFPBF4YAtifbR0D0pSobP6xmDysqw,3023
 tests/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/rag/test_clip_benchmark.py,sha256=Ar8Br2CoAFYT2h4zCv_JKMKCGJKbKGYZgNwJ410ZaoU,2597
 tests/rag/test_mteb.py,sha256=CaEJ0f1M06Z90c72FQb9z23IC_KZtkURWsc_oRMgQn8,4609
@@ -276,10 +276,10 @@ tests/swift/test_run_swift_eval.py,sha256=JKG-0BwTxkbg-XeiXxujPqnVIM3f2EFaJ_9a7p
 tests/swift/test_run_swift_vlm_eval.py,sha256=C8DftjewnZaerQWfERI70bU3sQLWQ-ejZUQhtYO5e0o,4898
 tests/swift/test_run_swift_vlm_jugde_eval.py,sha256=THZEXUOSqm9rWslwJHmZyh-Ytv5c_QKpgRW5J2s_69E,6017
 tests/vlm/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-tests/vlm/test_vlmeval.py,sha256=21xi0nu4ghDB6_X-Pol7pTfK7aYkAYOp82TQ-MSQv-I,1757
-evalscope-0.8.0.dist-info/LICENSE,sha256=K_2M03pN0PxVMyx9IQUKsHGhhDMkw5ryQ02rlMvzj3I,11416
-evalscope-0.8.0.dist-info/METADATA,sha256=5RKZaNBwuJj84sdAXlNmT11Bm8kGYha6EYnqszwZ1Qk,23190
-evalscope-0.8.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-evalscope-0.8.0.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
-evalscope-0.8.0.dist-info/top_level.txt,sha256=Yv0iprOqZQ4rfUO-AWJp7Ni6m0Twxny1yvZwO-8hUDM,16
-evalscope-0.8.0.dist-info/RECORD,,
+tests/vlm/test_vlmeval.py,sha256=nzWXjw49SlxXgDnYS9N5JSFtcUp8xPOW2YNNzupvtt4,1806
+evalscope-0.8.1.dist-info/LICENSE,sha256=K_2M03pN0PxVMyx9IQUKsHGhhDMkw5ryQ02rlMvzj3I,11416
+evalscope-0.8.1.dist-info/METADATA,sha256=HydrEYb1OxbvVUMl11oLekV2sjvlgQQvtEpkcNAiW5A,23190
+evalscope-0.8.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+evalscope-0.8.1.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
+evalscope-0.8.1.dist-info/top_level.txt,sha256=Yv0iprOqZQ4rfUO-AWJp7Ni6m0Twxny1yvZwO-8hUDM,16
+evalscope-0.8.1.dist-info/RECORD,,

tests/cli/test_run.py

@@ -70,7 +70,7 @@ class TestRun(unittest.TestCase):
 
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_task(self):
-        task_cfg = {'model': 'qwen/Qwen2-0.5B-Instruct', 'datasets': ['gsm8k'], 'limit': 2, 'debug': False}
+        task_cfg = {'model': 'qwen/Qwen2-0.5B-Instruct', 'datasets': ['bbh', 'gsm8k', 'arc'], 'limit': 2, 'debug': False}
         run_task(task_cfg=task_cfg)
 
 
@@ -80,33 +80,32 @@ class TestRun(unittest.TestCase):
 
         task_cfg = TaskConfig(
             model='qwen/Qwen2-0.5B-Instruct',
-            datasets=['ceval'],  # 数据格式，选择题格式固定为 'ceval'
+            datasets=['ceval', 'general_qa'],  # 数据格式，选择题格式固定为 'ceval'
             dataset_args={
                 'ceval': {
                     'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
                     'subset_list': [
                         'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
                     ]
+                },
+                'general_qa': {
+                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
+                    'subset_list': [
+                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
+                    ]
                 }
             },
         )
         run_task(task_cfg=task_cfg)
 
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom_qa(self):
+    def test_run_humaneval(self):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
             model='qwen/Qwen2-0.5B-Instruct',
-            datasets=['general_qa'],  # 数据格式，选择题格式固定为 'ceval'
-            dataset_args={
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ]
-                }
-            },
+            datasets=['humaneval'],
+            limit=2
         )
 
         run_task(task_cfg=task_cfg)

tests/perf/test_perf.py

@@ -25,6 +25,7 @@ class TestPerf(unittest.TestCase):
             'number': 15,
             'api': 'openai',
             'dataset': 'openqa',
+            'stream': True,
             'debug': True,
         }
         run_perf_benchmark(task_cfg)
@@ -46,7 +47,7 @@ class TestPerf(unittest.TestCase):
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf_speed_benchmark(self):
         task_cfg = {
-            'url': 'http://127.0.0.1:8000/v1/completions',
+            'url': 'http://127.0.0.1:8801/v1/completions',
             'parallel': 1,
             'model': 'qwen2.5',
             'api': 'openai',

tests/vlm/test_vlmeval.py

@@ -40,8 +40,9 @@ class TestVLMEval(unittest.TestCase):
                 }],  # model name for VLMEval config
                 'nproc': 1,
                 'reuse': True,
-                'work_dir': 'outputs'
-            }
+            },
+            'work_dir': 'outputs',
+            'use_cache': 'outputs/20241216_142838'
         }
 
         logger.info(f'>> Start to run task: {task_cfg}')