evalscope 0.17.0__py3-none-any.whl → 0.17.1__py3-none-any.whl

{evalscope-0.17.0.dist-info → evalscope-0.17.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
 evalscope/__init__.py,sha256=XZYDn3ShhM_48je5qQgwymtSdpTt8zYEnNfanYnpBdA,181
 evalscope/arguments.py,sha256=QkxE8eGSryiyo9uDiNQNZUI3l_hGPYmhVz1-KHgtB6E,6044
 evalscope/config.py,sha256=1YfHXlIyYH70FQfi8TiUtpUH3VIRCh5YcbaayKZo5s4,6781
-evalscope/constants.py,sha256=1CYghe0fGccyiVgzMIHd2HIb6lOo9fmB-8pH_l99iI4,4014
+evalscope/constants.py,sha256=Tc74W89SxeeEzISDzO5IoxSo9A_F0LqjH0mOrcAYJXc,3737
 evalscope/run.py,sha256=dL1deJ0J1RHW6X6ZStXzAVL7NwbjW6McfdOMkCpWrtc,7012
 evalscope/summarizer.py,sha256=ZLFDHmi0Bgo18ouQsxuUl9vmIES9zkoapLLWRLhy19Q,5911
-evalscope/version.py,sha256=IZr-isfEmPkZ2eTCGlS5vvkiE5fMlg3HeXpgVmjGGJY,119
+evalscope/version.py,sha256=wsTu-_Fq9Dmfg7bXg6eDVtNwZA5ui-MZ6IPs4EhytAc,119
 evalscope/app/__init__.py,sha256=HWLXld_JXcBDsdL4L_4E8JsKyuBwwPUSwlejKnZ3HKc,579
 evalscope/app/app.py,sha256=8mSBp8qUCCmqupV4FEPMPdT9jL-bYu4DdH2qj8P0ktk,776
 evalscope/app/arguments.py,sha256=1wHTLeFx1G94cKXYOeOVe_wTiOY2D929UctIRGOtRaQ,699
@@ -69,9 +69,9 @@ evalscope/backend/vlm_eval_kit/__init__.py,sha256=R-GuBm8dAwvDF73XHaGpPSjlt7Y4ty
 evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=jlwM13Ty-Ax6AeMsNlo9xIBupNFgnceYuXtCmh0hNTQ,6160
 evalscope/benchmarks/__init__.py,sha256=NVd_VvmkY36LxdHNmgeogSBwMFfWoLJAZF8vDg-CoFc,1308
 evalscope/benchmarks/benchmark.py,sha256=uZ_-Y_wPhy6TxufWiElF4BwEWN93azT1JHtGRW8tR-w,2633
-evalscope/benchmarks/data_adapter.py,sha256=t_IOA6hvPrF_mrAzwgS-HP1aRQ_sI-3s9oSpRxmtFLg,23475
+evalscope/benchmarks/data_adapter.py,sha256=UI4HpnJNYo18GXRiU0HwNUxjRfoSXlCB-xEBIGs2ckg,23914
 evalscope/benchmarks/filters.py,sha256=x_NX40uWMmUsVrAGHCeeV2e63HZZFugWUgdUhk64ivM,1523
-evalscope/benchmarks/utils.py,sha256=37Pn9SxRqF0WoLR7LcGJF9xASh4VxcUL93v03WHmnh8,1813
+evalscope/benchmarks/utils.py,sha256=mIk8n6zVMICQ5JWMyEwUqwlkxva4L-oD5SZzpIKw1sI,1851
 evalscope/benchmarks/aigc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/benchmarks/aigc/t2i/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/benchmarks/aigc/t2i/base.py,sha256=4GFAvceT1Gpt5teDLRCZi62RwvPazuhG3zwft3gN3X4,2102
@@ -121,7 +121,7 @@ evalscope/benchmarks/bbh/cot_prompts/tracking_shuffled_objects_three_objects.txt
 evalscope/benchmarks/bbh/cot_prompts/web_of_lies.txt,sha256=s_x6u5MLeKpuAHZj3GNQqY1I8vWqQIfJasOp9XcM7Ck,2945
 evalscope/benchmarks/bbh/cot_prompts/word_sorting.txt,sha256=qfTZafCzNiz9ULBaDlfy_LISL617NyH5Nc0-nO0K0LE,2164
 evalscope/benchmarks/bfcl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/benchmarks/bfcl/bfcl_adapter.py,sha256=MQPlfMvTQYHA4EP5g7eNzXDs4A4QvgYOiGC458Z39q4,10080
+evalscope/benchmarks/bfcl/bfcl_adapter.py,sha256=ThDOYrJY_RdXMLSC1S9lP-8zYd1syZWpcrXXV1ZPLVs,10100
 evalscope/benchmarks/ceval/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/benchmarks/ceval/ceval_adapter.py,sha256=V_TC_E0lKXaFcV_qIdrg2_iddmGJ4um8iIdaXVaK_EM,11146
 evalscope/benchmarks/ceval/ceval_exam.py,sha256=ngOvb6Fymt7iPWIb2fzrUVpqmUT2VBoqh7X_IH8Bcsc,4824
@@ -149,9 +149,9 @@ evalscope/benchmarks/general_arena/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 evalscope/benchmarks/general_arena/general_arena_adapter.py,sha256=j2aDzikz9obxvrR-damdvSCXR0rfjEo-OzX8vujj2N0,19887
 evalscope/benchmarks/general_arena/utils.py,sha256=u0q4FNIOFka1_gC344OCvBXUz89Ah6M8asjIXbNSweM,7188
 evalscope/benchmarks/general_mcq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/benchmarks/general_mcq/general_mcq_adapter.py,sha256=B5zMiywH06NWOZsxAOwP-4aE3DbJB3Oyi9tlbM2BEHU,5181
+evalscope/benchmarks/general_mcq/general_mcq_adapter.py,sha256=cPN-p0tndjocQYqfc6OFkT5k8KL7kkVklmOtps-F08Y,5391
 evalscope/benchmarks/general_qa/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-evalscope/benchmarks/general_qa/general_qa_adapter.py,sha256=qdiH_XoWy3JNrBaUxl9S0bH16k0gXcx8dexZQflH74o,5443
+evalscope/benchmarks/general_qa/general_qa_adapter.py,sha256=dpIGe635CoW4ejVohVwcarBxSckqvlnxcJ2ElpRlQ9o,5669
 evalscope/benchmarks/gpqa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/benchmarks/gpqa/chain_of_thought.txt,sha256=pgoT5a-DMPJaMhoH_M8zfU5s80ibWDTVW6vnonITd8k,5610
 evalscope/benchmarks/gpqa/gpqa_adapter.py,sha256=J6RfxpUT1l8Jj3vT_Vtsn1z8MKCg32XTlKn_eihCI50,5071
@@ -161,9 +161,11 @@ evalscope/benchmarks/gsm8k/gsm8k_adapter.py,sha256=IBMdsvQ1w45_raCiACTBm7DVHtOYf
 evalscope/benchmarks/hellaswag/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/benchmarks/hellaswag/hellaswag.py,sha256=5_c9WbaS1LIdvgXzqEcvjAEtKi2V2Yn0YtszPlFqhXI,4610
 evalscope/benchmarks/hellaswag/hellaswag_adapter.py,sha256=kgHz-n8_93J8DdR7XBlzfM2KDRoKcvg80h6CCjWv_Xk,6191
+evalscope/benchmarks/hle/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalscope/benchmarks/hle/hle_adapter.py,sha256=ts38e-AqtUcbfc6VqRtWLacZDh7KzSm4rj7xKm9vTFc,4445
 evalscope/benchmarks/humaneval/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/benchmarks/humaneval/humaneval.py,sha256=2Exsg6u8FEu0buADY2tETJluSM8tWacvX06nykKKLSE,3395
-evalscope/benchmarks/humaneval/humaneval_adapter.py,sha256=O6muXpiBrQ9RGSglnl3gS0yO6BSkQtXASMR9yXUfhEE,5515
+evalscope/benchmarks/humaneval/humaneval_adapter.py,sha256=ZqNG3L8yMY44B7HleUjlSbVG-GLk9RBsvaGWOm2fQVw,4788
 evalscope/benchmarks/ifeval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/benchmarks/ifeval/ifeval_adapter.py,sha256=2oStqiTD4w2f2n0kbjcbg7GJQfKCsHFieokQcNndWb4,2041
 evalscope/benchmarks/ifeval/instructions.py,sha256=oaJ9D_4rvS67BraHBNPpDtFd4TblFAnR4A3YW9HWfuY,56304
@@ -186,7 +188,7 @@ evalscope/benchmarks/math_500/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
 evalscope/benchmarks/math_500/math_500_adapter.py,sha256=Oc9XnBgMAjEerYAk3GtY2TTKm1QH_UI896kUuW2_a5Y,2324
 evalscope/benchmarks/mmlu/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/benchmarks/mmlu/mmlu.py,sha256=sA8AC0bN7iURrSazqkY31s_reNVbDZSUCB-NCTQsVeI,5042
-evalscope/benchmarks/mmlu/mmlu_adapter.py,sha256=58z0BQDYq1TKTzFVDbVdpJVlBOv0pJtuAu7uS8gVwbA,12111
+evalscope/benchmarks/mmlu/mmlu_adapter.py,sha256=Rhi-J6oGWawRVBk38ZgXk8-XrZ7wL8sf4zrncU73jgs,12111
 evalscope/benchmarks/mmlu/samples.jsonl,sha256=f5Y2vwbEvNtpE7vrl9BHoJzsdceI4vUAo1frexYyX2o,1345
 evalscope/benchmarks/mmlu_pro/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py,sha256=uglOOZBZfQBIuJOG7iT4THk2LNcfHQoakxQDpS4jB1U,4554
@@ -211,8 +213,10 @@ evalscope/benchmarks/super_gpqa/five_shot_prompt.txt,sha256=CQxRszzUrSIygOSd1G10
 evalscope/benchmarks/super_gpqa/super_gpqa_adapter.py,sha256=ce99v28wkhlGnfmihwpv3ikTqy3aumT8Jzm1LGxz-ck,10147
 evalscope/benchmarks/super_gpqa/utils.py,sha256=ftYPP9ODvLBlQSd9ltACx9iRIvjB8u1bg4AtgcJ4JAI,3360
 evalscope/benchmarks/super_gpqa/zero_shot_prompt.txt,sha256=XZb0CN83YbfH2dF-iIV-ciNLbIb3ON220qHe7zf8KF0,247
+evalscope/benchmarks/tau_bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalscope/benchmarks/tau_bench/tau_bench_adapter.py,sha256=5_VgRUtEjeZ-8gRZj4cnwwso1GUqf2GB49AlI4xqyDM,4221
 evalscope/benchmarks/tool_bench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/benchmarks/tool_bench/tool_bench_adapter.py,sha256=_QNncuCCMhhjsWzB934sYF-k010fKUdhhAOWrJ9LKDA,2813
+evalscope/benchmarks/tool_bench/tool_bench_adapter.py,sha256=fy6Hb84cm6s-pOoQXmT-N8D1OUYVGCuq77-2xwM_WLA,3093
 evalscope/benchmarks/tool_bench/utils.py,sha256=led0d-Pa3rvmWkSWhEnZWP00fceudgESq5HXAQzJGls,7042
 evalscope/benchmarks/trivia_qa/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/benchmarks/trivia_qa/samples.jsonl,sha256=1isBD62PGhCiNbzQa-GFrHHL4XLHIkojWfgSvn7ktf8,3445
@@ -234,13 +238,13 @@ evalscope/collections/__init__.py,sha256=3v7tVLcJk86FeNBrxw3pWhu_lcpKYrnT_dDACCe
 evalscope/collections/evaluator.py,sha256=RJ337S0sy8dsV25I2OAxeWgSx_HrmXTyuuHKSt9vQtM,17474
 evalscope/collections/sampler.py,sha256=2NwvhJVdi-mrDeK7RWwEGOoE7DdxtpyASRUZU_D6hWw,4855
 evalscope/collections/schema.py,sha256=mjJfNmy_athJ1TmnuJRkrKRlefzefuQXZuTtjn8SHKo,4073
-evalscope/evaluator/__init__.py,sha256=S6MU1O_iiNAaKxNIhO9MEmdW-BSNf_YH2l6NQ9lxVNo,103
-evalscope/evaluator/evaluator.py,sha256=0vxXyNt1v2wjzUXwC_7nc_-3wmT18yQ-QKDA9DDVXpA,22441
-evalscope/metrics/__init__.py,sha256=LlqXdOPiWTTAzxuKdUwTYO0KgN3Zh1zs18H2sM_5o5I,1709
-evalscope/metrics/completion_parsers.py,sha256=LJnHD_ea7SLfRVXIAHuzeJx0mAgbpzmf3VYcQRL-CdA,8733
-evalscope/metrics/llm_judge.py,sha256=9DCT4p9llNSVUc-K6inrHGpHVdReHlEz_kJQyRezvz8,8268
+evalscope/evaluator/__init__.py,sha256=XqPnEp5MvfRwC5M5cEeOAC0-MMEPxBIESqiSa3YMBgo,84
+evalscope/evaluator/evaluator.py,sha256=HKEF2k0S_dJR8cF9lrqf_W4diXbb6H3L81pD6XcmLiA,22481
+evalscope/metrics/__init__.py,sha256=CH3bNyRx9dJ3gOqNwKDlaZ7zan4MShM0h8SnzarjokU,1851
+evalscope/metrics/completion_parsers.py,sha256=56ZNzOfNU0O1ba9fs9Cyi4Vk_YUmcgWUbxW0SJ2KrlU,8974
+evalscope/metrics/llm_judge.py,sha256=1hPFnGc3Szszqo21O618a7mxOgkdba3KsbZ66vvTbSA,8380
 evalscope/metrics/math_parser.py,sha256=JtOkj28XOtwoUACXOXLzCeRYz0rx0tBsQLQDU8cbC20,17311
-evalscope/metrics/metrics.py,sha256=f-KFVBJi6hOm9K7dFJSPCQDe5opEOzeb0z1YvhkKXb0,13797
+evalscope/metrics/metrics.py,sha256=OLfvEljGbQnv-bBiFD-GR2On4mpZ0xhKxiKkjZfoDX8,14268
 evalscope/metrics/named_metrics.py,sha256=PrzU_1mGTeRFxVJFT1aXxIOiS7MnNoWyZsb8uCRVDeE,2278
 evalscope/metrics/rouge_metric.py,sha256=bqvSotuDdC0MEKmt8v6y6tBTBx0S3Ma-tfF-cMCckA4,4645
 evalscope/metrics/bundled_rouge_score/__init__.py,sha256=PwbTdk8168FwDJe_l8XIqDuBgZQooDsP31vj7di05Fs,650
@@ -346,46 +350,49 @@ evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/processors/__init__.p
 evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/processors/base_processor.py,sha256=LqMHlUTy2LEzoVwjALtrAw0UYmzIuHnFjQiVmn5nv-I,605
 evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/processors/blip_processors.py,sha256=d4HInkL_Phk0Bgg2cWaOvhsPa6lkqDeovFW86PL0I18,6371
 evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/processors/randaugment.py,sha256=XzebAHBAjOpkIMZm43dd55PESgmyq_J45Ji6bogYR3s,11204
-evalscope/models/__init__.py,sha256=yB4NuKvSd3Jd4GRQvJeGPxwigd8RJErdop5PzSQhsMY,1565
+evalscope/models/__init__.py,sha256=x0Sna8mbujdOVqIYSGwIULbiPOue_Ifp-2JElSZsuMs,1481
 evalscope/models/local_model.py,sha256=UWsmZlWpT8JNGjijzZQKirvq4YywBkKOS9G-U2cuxAw,4115
-evalscope/models/model.py,sha256=MxvJAUNkuT7IA3bchnmJDur_YCKj9ShOD2Uq40dBcGc,6308
-evalscope/models/register.py,sha256=WiylzfL-vb6Bl3H3_RdIaBabVOAc9tiuhsQzYJDVzTg,1948
-evalscope/models/adapters/__init__.py,sha256=zmldx8yC_KTI8NDRcxNLyPzv19wc57UvOVvzwyuYnG4,647
-evalscope/models/adapters/base_adapter.py,sha256=TfINK84g4mqmHcnqvvHmk-MXRN2Pkan4yVlVd4j0nVY,3166
-evalscope/models/adapters/bfcl_adapter.py,sha256=KtreuJ21X1lcUGGhVgW3U62p3P65_oydMdBPtE5um-I,10332
-evalscope/models/adapters/chat_adapter.py,sha256=PAClyBL_nQ1I1kmjeeZ3sdC-y5ZmfFj8rjCigh_vr40,7885
-evalscope/models/adapters/choice_adapter.py,sha256=4fuz3MFEqK8ln4mMs3goMCdRPBwYmmgN70HTdr_sW_U,8005
-evalscope/models/adapters/custom_adapter.py,sha256=w8cD0b3xgcdhSZelcat67CGJnALOfz5IALzURnLjab8,2275
-evalscope/models/adapters/server_adapter.py,sha256=4fyC7fM_L_hn1SDqqDJAvMjEvBHVbTNF7xZHrO9bnhI,9616
-evalscope/models/adapters/t2i_adapter.py,sha256=xkMRyZ61yTiJfmULK-p9du4nNox41pkHiV2CTFBO3qM,2659
+evalscope/models/register.py,sha256=G35J6BULFWwuqZO_rTkKBru1llZAyfPztcAASp_cb8M,1257
+evalscope/models/adapters/__init__.py,sha256=WRaZsHlnz0MvGg9Jq565-XJjED-4cAyu4KbmrOhrHO4,688
+evalscope/models/adapters/base_adapter.py,sha256=P4aicNmz1nsX9QLY9t4c6OIQPzIYfOhcrqjlAjR-ENY,3477
+evalscope/models/adapters/bfcl_adapter.py,sha256=cG0vsQ3H2pmabo6tC0Y5Gonw0ng5-RFljDyRBMSj6xE,10422
+evalscope/models/adapters/chat_adapter.py,sha256=epxA_on9ipsak8Lnkweh9en2AjVm5G0L1ARXYmDEEbk,8026
+evalscope/models/adapters/choice_adapter.py,sha256=wIXnDcgnKaIMdhToaqy6fidhuZDpEz2vhxIB_V9u3Z8,8203
+evalscope/models/adapters/custom_adapter.py,sha256=W8DIBiMWvHHcc0Mn9Frjj1YbpHRi7w-UQVJDiU2PakU,2400
+evalscope/models/adapters/server_adapter.py,sha256=W6SXrPy-hZXpnISDjupu_j7bnmt-cP55sDojPXThitc,9701
+evalscope/models/adapters/t2i_adapter.py,sha256=d6OviQFi_uN8PPXKrFpivk5Awm1O6wd_Gii8t3hVahY,2806
+evalscope/models/adapters/tau_bench_adapter.py,sha256=jYGaj2L2wxtEiTdiSwZdY1XNkSzm6os7IvkxgK4msR0,6889
 evalscope/models/custom/__init__.py,sha256=MZylegALg1HerOYtp-qbzu4Wb6PW3JbrxwONHU-PAVs,131
 evalscope/models/custom/custom_model.py,sha256=rBccFVpCIfTGt9cgXLcxeUWc7w1sTRtbTO5w5qqQIQE,1405
 evalscope/models/custom/dummy_model.py,sha256=WpfrS3kvwRRdyThx9baaJ5vodYYh29VGRKsGKMWFflI,3124
 evalscope/perf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/perf/arguments.py,sha256=4YNmgTl4c76dvcL1GqxHsfRrs5cx6pvsT-6ss7weRC4,10415
-evalscope/perf/benchmark.py,sha256=cjUpJ3SRnZVBs_H24yqLh4WG_hcCADrniLG1VsmByb8,7901
-evalscope/perf/http_client.py,sha256=-c3-N7bxKsj3d5DVsKSaYA3XAHJDzZgoqZBbhuDYIGk,7419
-evalscope/perf/main.py,sha256=UdtzFWG5M9VDeuM2EvD6pqRNw5EubRadU74K-PnCLpU,3400
-evalscope/perf/plugin/__init__.py,sha256=1sl5s-csrwKb_LVTnpF3HqArz06TRD5LYJ0hpqvokUA,85
-evalscope/perf/plugin/registry.py,sha256=w1IAt6GDdluzSYK5i-yrntvx3_EvIIqJamEL0xZv3zA,1323
-evalscope/perf/plugin/api/__init__.py,sha256=Ckzbq4CkSMVQTedQcDHCYlRd6FTwQAElt2mHB-VXJac,195
-evalscope/perf/plugin/api/base.py,sha256=B_H04qKx7eRTn155rnDrbTYur7PK1mvxfQKYcqYbndU,2118
-evalscope/perf/plugin/api/custom_api.py,sha256=ssE4J8AynA0n5SnXSQyk7K5Co3dwUN6Opph08clZna0,3785
-evalscope/perf/plugin/api/dashscope_api.py,sha256=V5fwn-p_fLH0dWKzhN9TvYSHRgla4INfXC4NDaIjoQ8,3825
-evalscope/perf/plugin/api/openai_api.py,sha256=PmjBfIzzSuzcKiVOUeA2aPxihV0dZEzFlgmbrD2isME,7773
-evalscope/perf/plugin/datasets/__init__.py,sha256=Z6Jc0RxJS_z0nBBV1-b0-56Ija60AtQ7I_67gY6ZfdQ,568
-evalscope/perf/plugin/datasets/base.py,sha256=Z-INWueeYjfEZhP4lbTlBMVwIa6BcXZKWx-w7Pop3mA,1786
-evalscope/perf/plugin/datasets/custom.py,sha256=-meul2hRmYvYAo--c_EtCnItRi5DvN7xxFOpq6vqdts,1346
-evalscope/perf/plugin/datasets/flickr8k.py,sha256=MbJKEB0XqZE0nDEenwYs0FLH9QL658Vn9uQmUH4hPvk,1605
-evalscope/perf/plugin/datasets/line_by_line.py,sha256=AqZYG6tVL3BIGnzh_2Tev8lDYezJG_1gqJY8bSNQl3Q,957
-evalscope/perf/plugin/datasets/longalpaca.py,sha256=XelLris0-c3StLInQ-Oav4jqGcXPNfJxEDeYvaetEbI,1297
-evalscope/perf/plugin/datasets/openqa.py,sha256=4Pnx5duFJzoiTUfZCbcK7LO8f-skmcpYNUUrtNR_UUc,1463
-evalscope/perf/plugin/datasets/random_dataset.py,sha256=SIlsjAE_Stknfr6o1CBFvANBGCSgSExFbscLwSM_Gmk,2958
+evalscope/perf/arguments.py,sha256=lG2IOOzxg29pdnF6IobzPcqEcYqopulFpVU2QzRaEJA,11429
+evalscope/perf/benchmark.py,sha256=ZVmsSeKDUKkApt3y5tIMMFZAyAj3UNVT7JPp1fh5mhE,7880
+evalscope/perf/http_client.py,sha256=l_OKL80kTP6sM_PEBvsJ1_TejYJdUQnE2UlB-ud1WQM,4588
+evalscope/perf/main.py,sha256=WZbBgFhIj9KqxzC7_NZxDlou019_EXatsHRt5vqDhFg,3439
+evalscope/perf/plugin/__init__.py,sha256=Ztj4h1_JYJqbbWkeuDTj5aTRyGQf5Woc4xEIyjcokVU,94
+evalscope/perf/plugin/registry.py,sha256=GhLe-h1rGzya2bgIUaV5VymQIaHqI7h5SG_i4PoGAm8,1967
+evalscope/perf/plugin/api/__init__.py,sha256=7RsGdYTSfnW6iVpveEzNu8v4x8Yc8H-Kk39DqOHMrd4,152
+evalscope/perf/plugin/api/base.py,sha256=9cX4xwTzy5ycnWqmQqRGMLasTEX6jVlobtADkh1KwXE,2782
+evalscope/perf/plugin/api/custom_api.py,sha256=f8rUixcV9mTxoYyabu3wedEC4YVB70Yw6Az1NpfeWPQ,10375
+evalscope/perf/plugin/api/dashscope_api.py,sha256=Miv2pzMa6sxZyYYJhCzcbOI_QHuZx7tazKpb6Not7ck,3627
+evalscope/perf/plugin/api/default_api.py,sha256=kjuHQ-zRHe5WU4ofSzWBpWbIxBQBOh_ucu1z2g62gWg,4315
+evalscope/perf/plugin/api/openai_api.py,sha256=Mt_VedJUaCH3g-oVSJ_fsGcPk0KkspSzIMkrkih2Zb0,7777
+evalscope/perf/plugin/datasets/__init__.py,sha256=qzeQ9BrJhiJJm1wHaFeOQkvXXdSd15Ucspbn5zjs-6Q,495
+evalscope/perf/plugin/datasets/base.py,sha256=-3Ihnp2hYvZyPnP8Gh2Pu8ovlLNFHyZnNgRu3WHG4d0,2714
+evalscope/perf/plugin/datasets/custom.py,sha256=UuOk8xYfSYyyYZL3U4grUjtfQhWHHZeAEC63n_4Siuw,1376
+evalscope/perf/plugin/datasets/flickr8k.py,sha256=IXz5uu5SlqF1l_tJ_ITr2vx_R_d7gxWzqPuyEOx7rYo,1043
+evalscope/perf/plugin/datasets/kontext_bench.py,sha256=XjKzr7nMzI3cfk83IH0PH1TNJaQMRXUpACnzFfP2n6g,1091
+evalscope/perf/plugin/datasets/line_by_line.py,sha256=c3ydW4GqxkG0vl2g64jG0vBMql2FuFPyWh3mgkIh9Do,987
+evalscope/perf/plugin/datasets/longalpaca.py,sha256=VnMjdHl_JV3NmZ6wRxVlJ99e8PYSjQTcVxoTkl21Ei0,1327
+evalscope/perf/plugin/datasets/openqa.py,sha256=33AR419IrH-FxZRjjcYdAIEZXaX4TKEoirVVfX--N9I,1493
+evalscope/perf/plugin/datasets/random_dataset.py,sha256=NNAXvgFPkLDOSpYNex1DyE4X-ELtQRm13_oBooO30j8,3514
+evalscope/perf/plugin/datasets/random_vl_dataset.py,sha256=F3yA9Ih3YO895lZKCo3i85LeKTzjvGcvhzc8UNN-gUI,3240
 evalscope/perf/plugin/datasets/speed_benchmark.py,sha256=J6q7AF_Re5eHLVejXEw9c1jlk1T1PPmist0yO9UFTPE,2432
 evalscope/perf/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/perf/utils/analysis_result.py,sha256=aoT7JD2zAzBeuZUfncKhJ2odX_7KnymwOmNB1Upam2c,935
-evalscope/perf/utils/benchmark_util.py,sha256=pv__38XjxkTqOfcREnod40WxeMJe4okDuVcYjyySDtg,7258
-evalscope/perf/utils/db_util.py,sha256=xqrXZapP_WwUdzkgFBTh3LDBWzr_UoU8v13rOjQ8TT4,9876
+evalscope/perf/utils/benchmark_util.py,sha256=7bHpa5oaqcPJX7DSUkzK9assoFSHC27Q7-QylUOiklQ,7136
+evalscope/perf/utils/db_util.py,sha256=TCdmoEx5iScL6h8wzucPojPwn6J1wTmQqX4sVk-ilHo,11630
 evalscope/perf/utils/handler.py,sha256=HyKIxbzC0XCyQanlbb7UEY7yaeqjJTePNea8kMV3Sdc,1192
 evalscope/perf/utils/local_server.py,sha256=RL9rGd5tEniZ0aErhHcbVXMX22YmujfE11T3j37VL8k,4684
 evalscope/perf/utils/log_utils.py,sha256=NWSK_ITG4yoVx5GMLbIRGDoXSs90s7X3mftdm37Os2U,1666
@@ -434,7 +441,7 @@ evalscope/utils/argument_utils.py,sha256=D7qOH85wf7LKh_cJ2X51OEaL7CMaddydmHZkfoY
 evalscope/utils/chat_service.py,sha256=U2jtrkOa2asRp16Zam0zIi_38mCyWQqql_L6JSwii4I,8749
 evalscope/utils/deprecation_utils.py,sha256=WyeiLWSi5ti6FkuMbhimcPPUB43paa1FZ5-JOAWNFZI,1344
 evalscope/utils/import_utils.py,sha256=BSdp7RQSZu67129TBbtJvMWU0CfCFu864K31eiM3pr8,2975
-evalscope/utils/io_utils.py,sha256=xvFvVu3Hy2HJFvemvREdFh6en7SNmfrsnikK-Mj-q6Q,6828
+evalscope/utils/io_utils.py,sha256=2eEkLx4jhekgIV4vYL8yTN0PT6dbHUERMBZwmvxuiEc,7109
 evalscope/utils/logger.py,sha256=Q2IeV_0jxz8L34b5GddPeCKXVh0UClbuhjyLe5Wtj7M,3648
 evalscope/utils/model_utils.py,sha256=F1_WBHvBehWqrTd6kPtKICeeYucaZn5H0Gc3cCplYB8,2329
 tests/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
@@ -443,12 +450,12 @@ tests/utils.py,sha256=Fgm0CU6ilZjCGOfOMJH-Trxy0UIAGbhvy0Ijy_zDGUk,323
 tests/aigc/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 tests/aigc/test_t2i.py,sha256=XtVknpwlVMb6FSw3_WMFxMq0gZX6iG-ffdSQkcW2Fzw,3856
 tests/cli/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-tests/cli/test_all.py,sha256=GmY6g-EQCb_RJY4R76MeF9pvgYyzQHBxwn7Y_9BMwns,5866
+tests/cli/test_all.py,sha256=IT0mxjiuHCC0PpT4z3oN1Bbr_0viMcm8GnShZ02kp8w,6333
 tests/cli/test_collection.py,sha256=bXWzccH822Y2B1Ed251U6TE8G_osI6MXYNxzmfv9kBI,4197
 tests/cli/test_custom.py,sha256=0YE-TCAeaQMRVRFla_TIvTd8d0USvvsSeqvYAD3NDNg,8796
-tests/cli/test_run.py,sha256=Al8-CZeoWZH-c-YIg6qUIKtSIfRdzlEBsgVsl-WMosk,17570
+tests/cli/test_run.py,sha256=YKX2XCHPxnStgzzP67U90RV9r1MC2GM3JoGQqfZKqrI,17324
 tests/perf/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-tests/perf/test_perf.py,sha256=u82U7QYkA5JvR-iw0f4MNpWuEQOYid2g9cQ11ma7NAU,4844
+tests/perf/test_perf.py,sha256=AEWvpN3ID6s-9MEoaZjQqUM8VVsqgk_v9KX8pDgvozA,5864
 tests/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/rag/test_clip_benchmark.py,sha256=13pcY3gYHNQh2KfEHCqtCSqiOcbngSJ1BlVZzI58JCE,2694
 tests/rag/test_mteb.py,sha256=fdNQIyUEzE7puPCKw5QhCHTEu7hz-ieHeq1xCWGh6IM,7246
@@ -459,9 +466,9 @@ tests/swift/test_run_swift_vlm_eval.py,sha256=RwrKkc1WHEZxetM11cGL81G4faKCn7SYn4
 tests/swift/test_run_swift_vlm_jugde_eval.py,sha256=UAUtOCQ72xbm8s-sov3cBEpYVDy189wpB-qOL3KoU7M,6053
 tests/vlm/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 tests/vlm/test_vlmeval.py,sha256=EDQRkYfSyOICUwo_tm3p-puaE_xdFmqOPkrt5etxsqM,3307
-evalscope-0.17.0.dist-info/LICENSE,sha256=K_2M03pN0PxVMyx9IQUKsHGhhDMkw5ryQ02rlMvzj3I,11416
-evalscope-0.17.0.dist-info/METADATA,sha256=EwH2JsfnfzkG-OSyvu-8lGjQ0aU6lsTuHMUwC_RDDTU,36893
-evalscope-0.17.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-evalscope-0.17.0.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
-evalscope-0.17.0.dist-info/top_level.txt,sha256=Yv0iprOqZQ4rfUO-AWJp7Ni6m0Twxny1yvZwO-8hUDM,16
-evalscope-0.17.0.dist-info/RECORD,,
+evalscope-0.17.1.dist-info/LICENSE,sha256=K_2M03pN0PxVMyx9IQUKsHGhhDMkw5ryQ02rlMvzj3I,11416
+evalscope-0.17.1.dist-info/METADATA,sha256=1PRiimjOBZgSWjvT3iL4VcvdaWk8v3fGp9xCXLpM1Dw,38469
+evalscope-0.17.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+evalscope-0.17.1.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
+evalscope-0.17.1.dist-info/top_level.txt,sha256=Yv0iprOqZQ4rfUO-AWJp7Ni6m0Twxny1yvZwO-8hUDM,16
+evalscope-0.17.1.dist-info/RECORD,,

tests/cli/test_all.py

@@ -39,7 +39,7 @@ datasets=[
         'general_mcq',
         'general_qa',
         'super_gpqa',
-        'live_code_bench',
+        # 'live_code_bench',
         'mmlu_redux',
         'simple_qa',
         'chinese_simpleqa',
@@ -53,8 +53,13 @@ datasets=[
         'docmath',
         'needle_haystack',
         'bfcl_v3',
+        'hle',
+        'tau_bench',
 ]
 
+# Reverse the datasets list to ensure the order is from most recent to oldest
+datasets.reverse()
+
 dataset_args={
     'mmlu': {
         'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
@@ -132,7 +137,18 @@ dataset_args={
     },
     'bfcl_v3':{
         'subset_list': ['simple', 'multiple']
-    }
+    },
+    'hle': {
+        'subset_list': ['Math', 'Other'],
+    },
+    'tau_bench': {
+        'extra_params': {
+            'user_model': 'qwen-plus',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+        },
+        'subset_list': ['airline'],
+    },
 }
 
 class TestRun(unittest.TestCase):

tests/cli/test_run.py

@@ -259,14 +259,14 @@ class TestRun(unittest.TestCase):
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=[
-                'iquiz',
+                # 'iquiz',
                 # 'ifeval',
                 # 'mmlu',
                 # 'mmlu_pro',
                 # 'musr',
                 # 'process_bench',
                 # 'race',
-                # 'trivia_qa',
+                'trivia_qa',
                 # 'cmmlu',
                 # 'humaneval',
                 # 'gsm8k',
@@ -289,6 +289,8 @@ class TestRun(unittest.TestCase):
                 # 'frames',
                 # 'bfcl_v3',
                 # 'truthful_qa',
+                # 'tau_bench',
+                # 'hle'
             ],
             dataset_args={
                 'mmlu': {
@@ -297,7 +299,7 @@ class TestRun(unittest.TestCase):
                 },
                 'mmlu_pro': {
                     'subset_list': ['math', 'health'],
-                    'few_shot_num': 4
+                    'few_shot_num': 0
                 },
                 'ceval': {
                     'subset_list': [
@@ -360,13 +362,23 @@ class TestRun(unittest.TestCase):
                         # 'is_fc_model': False,
                     }
                 },
+                'tau_bench': {
+                    'extra_params': {
+                        'user_model': 'qwen-plus',
+                        'api_key': env.get('DASHSCOPE_API_KEY'),
+                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                    }
+                },
+                'hle': {
+                    'subset_list': ['Math', 'Other'],
+                },
             },
-            eval_batch_size=1,
-            limit=5,
+            eval_batch_size=10,
+            limit=10,
             # debug=True,
             stream=True,
             generation_config={
-                'temperature': 0,
+                'temperature': 0.6,
                 'n': 1,
                 'max_tokens': 4096,
                 # 'extra_headers':{'key': 'value'},
@@ -377,35 +389,6 @@ class TestRun(unittest.TestCase):
         run_task(task_cfg=task_cfg)
 
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_batch_eval(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='LLM-Research/Llama-3.2-1B-Instruct',
-            datasets=[
-                # 'math_500',
-                # 'aime24',
-                # 'competition_math'
-                # 'arc',
-                'gsm8k'
-                # 'truthful_qa'
-            ],
-            dataset_args={
-                'competition_math': {
-                    'subset_list': ['Level 4', 'Level 5']
-                }
-            },
-            eval_batch_size=2,
-            limit=5,
-            generation_config={
-                'max_new_tokens': 2048,
-                'temperature': 0.7,
-                'num_return_sequences': 2,
-            }
-        )
-
-        run_task(task_cfg=task_cfg)
 
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_judge_model(self):
@@ -417,7 +400,7 @@ class TestRun(unittest.TestCase):
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=[
-                'math_500',
+                # 'math_500',
                 # 'aime24',
                 # 'competition_math',
                 # 'arc',
@@ -434,6 +417,7 @@ class TestRun(unittest.TestCase):
                 # 'docmath',
                 # 'needle_haystack',
                 # 'ifeval',
+                'hle'
             ],
             dataset_args={
                 'needle_haystack': {
@@ -466,7 +450,10 @@ class TestRun(unittest.TestCase):
                 },
                 'frames': {
                     'local_path': '/root/.cache/modelscope/hub/datasets/iic/frames'
-                }
+                },
+                'hle': {
+                    'subset_list': ['Math', 'Other'],
+                },
             },
             eval_batch_size=10,
             limit=3,
@@ -489,6 +476,7 @@ class TestRun(unittest.TestCase):
             },
             timeout=60000,
             stream=True,
+            use_cache='outputs/20250714_150626'
             # analysis_report=True,
             # debug=True,
             # use_cache='outputs/20250616_161931'

tests/perf/test_perf.py

@@ -35,9 +35,9 @@ class TestPerf(unittest.TestCase):
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf_stream(self):
         task_cfg = {
-            'url': 'http://127.0.0.1:8000/v1/chat/completions',
+            'url': 'http://127.0.0.1:8801/v1/chat/completions',
             'parallel': 1,
-            'model': 'qwen2.5',
+            'model': 'Qwen2.5-0.5B-Instruct',
             'number': 15,
             'api': 'openai',
             'dataset': 'openqa',
@@ -145,5 +145,32 @@ class TestPerf(unittest.TestCase):
         print(metrics_result)
         print(percentile_result)
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_random_vl(self):
+        from evalscope.perf.arguments import Arguments
+        task_cfg = Arguments(
+            parallel=[1, 2],
+            number=[2, 4],
+            model='qwen-vl-max',
+            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+            api_key=env.get('DASHSCOPE_API_KEY'),
+            api='openai',
+            dataset='kontext_bench',
+            min_tokens=100,
+            max_tokens=100,
+            prefix_length=0,
+            min_prompt_length=100,
+            max_prompt_length=100,
+            image_height=512,
+            image_width=512,
+            image_num=2,
+            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
+            seed=None,
+            extra_args={'ignore_eos': True}
+        )
+        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
+        print(metrics_result)
+        print(percentile_result)
+
 if __name__ == '__main__':
     unittest.main(buffer=False)

evalscope/models/model.py

@@ -1,189 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os
-import time
-from abc import ABC, abstractmethod
-from typing import Any, List
-
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class BaseModel(ABC):
-
-    def __init__(self, model_cfg: dict, **kwargs):
-        """
-        Base model class.
-
-        Args:
-            model_cfg (dict): The model configuration. Depending on the specific model. Example:
-                {'model_id': 'modelscope/Llama-2-7b-chat-ms', 'revision': 'v1.0.0'}
-
-            **kwargs: kwargs
-        """
-        self.model_cfg: dict = model_cfg
-        self.kwargs = kwargs
-
-    @abstractmethod
-    def predict(self, *args, **kwargs) -> Any:
-        """
-        Model prediction func.
-        """
-        raise NotImplementedError
-
-
-class ChatBaseModel(BaseModel):
-
-    def __init__(self, model_cfg: dict, **kwargs):
-        """
-        Chat base model class. Depending on the specific model.
-
-        Args:
-            model_cfg (dict):
-                {'model_id': 'modelscope/Llama-2-7b-chat-ms', 'revision': 'v1.0.0', 'device_map': 'auto'}
-
-            **kwargs: kwargs
-        """
-        super(ChatBaseModel, self).__init__(model_cfg=model_cfg, **kwargs)
-
-    @abstractmethod
-    def predict(self, inputs: dict, **kwargs) -> dict:
-        """
-        Model prediction func. The inputs and outputs are compatible with OpenAI Chat Completions APIs.
-        Refer to: https://platform.openai.com/docs/guides/gpt/chat-completions-api
-
-        # TODO: follow latest OpenAI API
-
-        Args:
-            inputs (dict): The input prompts and history. Input format:
-                {'messages': [
-                    {'role': 'system', 'content': 'You are a helpful assistant.'},
-                    {'role': 'user', 'content': 'Who won the world series in 2020?'},
-                    {'role': 'assistant', 'content': 'The Los Angeles Dodgers won the World Series in 2020.'},
-                ]
-                'history': [
-                    {'role': 'system', 'content': 'Hello'},
-                    {'role': 'user', 'content': 'Hi'}]
-                }
-
-            kwargs (dict): Could be inference configuration. Default: None.
-                cfg format: {'max_length': 1024}
-
-        Returns: The result format:
-                {
-                  'choices': [
-                    {
-                      'index': 0,
-                      'message': {
-                        'content': 'The 2020 World Series was played in Texas at Globe Life Field in Arlington.',
-                        'role': 'assistant'
-                      }
-                    }
-                  ],
-                  'created': 1677664795,
-                  # For models on the ModelScope or HuggingFace, concat model_id and revision with "-".
-                  'model': 'gpt-3.5-turbo-0613',
-                  'object': 'chat.completion',
-                  'usage': {
-                    'completion_tokens': 17,
-                    'prompt_tokens': 57,
-                    'total_tokens': 74
-                  }
-                }
-        """
-        raise NotImplementedError
-
-
-# TODO: Remove this class after refactoring all models
-class OpenAIModel(ChatBaseModel):
-    """
-    APIs of OpenAI models.
-    Available models: gpt-3.5-turbo, gpt-4
-    """
-
-    MAX_RETRIES = 3
-
-    def __init__(self, model_cfg: dict, **kwargs):
-        super(OpenAIModel, self).__init__(model_cfg=model_cfg, **kwargs)
-
-        openai_api_key = os.environ.get('OPENAI_API_KEY', None)
-        self.api_key = self.model_cfg.get('api_key', openai_api_key)
-
-        if not self.api_key:
-            logger.error('OpenAI API key is not provided, please set it in environment variable OPENAI_API_KEY')
-            # raise ValueError(
-            #     'OpenAI API key is not provided, '
-            #     'please set it in environment variable OPENAI_API_KEY')
-
-    def predict(self, model_id: str, inputs: dict, **kwargs) -> dict:
-
-        sys_prompt: str = inputs.get('sys_prompt', '')
-        user_prompt: str = inputs.get('user_prompt', '')
-
-        # model_id: str = kwargs.get('model_id', '')
-        temperature: float = kwargs.pop('temperature', 0.2)
-        max_tokens: int = kwargs.pop('max_tokens', 1024)
-        mode: str = kwargs.pop('mode', 'chat.completion')
-
-        logger.info(f'Using OpenAI model_id: {model_id}')
-
-        res = self._predict(
-            model_id=model_id,
-            sys_prompt=sys_prompt,
-            user_prompt=user_prompt,
-            temperature=temperature,
-            max_tokens=max_tokens,
-            mode=mode)
-
-        return res
-
-    def _predict(
-        self,
-        model_id,
-        sys_prompt,
-        user_prompt,
-        temperature,
-        max_tokens,
-        mode: str = 'chat.completion',
-    ) -> dict:
-        import openai
-
-        res = {}
-        openai.api_key = self.api_key
-
-        for i in range(self.MAX_RETRIES):
-            try:
-                if mode == 'chat.completion':
-                    resp = openai.ChatCompletion.create(
-                        model=model_id,
-                        messages=[{
-                            'role': 'system',
-                            'content': sys_prompt
-                        }, {
-                            'role': 'user',
-                            'content': user_prompt
-                        }],
-                        temperature=temperature,
-                        max_tokens=max_tokens)
-
-                    if resp:
-                        ans_text = resp['choices'][0]['message']['content']
-                        model_id = resp['model']
-                    else:
-                        logger.warning(f'OpenAI GPT API call failed: got empty response '
-                                       f'for input {sys_prompt} {user_prompt}')
-                        ans_text = ''
-                        model_id = ''
-
-                    res['ans_text'] = ans_text
-                    res['model_id'] = model_id
-                else:
-                    raise ValueError(f'Invalid mode: {mode}')
-
-                return res
-
-            except Exception as e:
-                logger.warning(f'OpenAI API call failed: {e}')
-                time.sleep(3)
-        logger.error(f'OpenAI API call failed after {self.MAX_RETRIES} retries')
-        return res