evalscope 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

{evalscope-0.6.1.dist-info → evalscope-0.7.0.dist-info}/RECORD

@@ -6,7 +6,7 @@ evalscope/run.py,sha256=uAXtaxIBcR94jyfHGFAecuzn0y71oLgu-d9VOohCJAw,18738
 evalscope/run_arena.py,sha256=BCWCAiX0BQ9pLMIq08svEcd-IoFr75gFShpV88robIY,8963
 evalscope/run_ms.py,sha256=UtJoGnah64SXigTawJQWTi_TEGjr7Td0rjCTaO-htL8,6028
 evalscope/summarizer.py,sha256=rIyML8HpjQxIpXg8KvQ0CzOS6xMS-JHZh6kUZzkaRsk,6640
-evalscope/version.py,sha256=o4SLhBjhMLzVbUK1flGxf-kiqIBLnLnJbxG06BmvkyU,118
+evalscope/version.py,sha256=KtYzabHQSFovvIs99J7glrRj7yrdxYKzKpbzzM2lORk,118
 evalscope/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/backend/base.py,sha256=5BLrDNNwxsGp35zorD-kphmN15tlBbkuuqwkz8jWZq0,876
 evalscope/backend/opencompass/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
@@ -25,6 +25,8 @@ evalscope/backend/rag_eval/clip_benchmark/tasks/__init__.py,sha256=47DEQpj8HBSa-
 evalscope/backend/rag_eval/clip_benchmark/tasks/image_caption.py,sha256=Bj2ysvM0JT-6T40v0rffeZgJIRht5KVX0GzMOiUphf0,2578
 evalscope/backend/rag_eval/clip_benchmark/tasks/zeroshot_classification.py,sha256=ZrUYDbQ75eo0vmIwXh5Bb9c4nyEwd4AO2oURaIqjIII,7502
 evalscope/backend/rag_eval/clip_benchmark/tasks/zeroshot_retrieval.py,sha256=Bcs64xece4BMNhxuaFimOwMJnlpjNxfGrdSCWOYItko,5977
+evalscope/backend/rag_eval/clip_benchmark/utils/webdataset_convert.py,sha256=3wW-AigMx5rygsI47rr8Kym_t0GWO4eio7zSAavSr6A,8765
+evalscope/backend/rag_eval/clip_benchmark/utils/webdatasets.txt,sha256=eiiAaxhS48b5rVLy5O9VvFfV2AfxY86ITu_iqT7ZLkQ,649
 evalscope/backend/rag_eval/cmteb/__init__.py,sha256=ajVz6XP5hqPq-jm66hp2poA2qKj1V19ZGoqjrGUlO7U,279
 evalscope/backend/rag_eval/cmteb/arguments.py,sha256=wZvnVir2tSxYCV_DPR3TSDj4VxtUn3wLhBPqyMJYKno,2330
 evalscope/backend/rag_eval/cmteb/base.py,sha256=fYrIjKwOLwBAHb2rlNkEjYScjZ5Qpyv2LdMmWZYWREA,2830
@@ -43,6 +45,26 @@ evalscope/backend/rag_eval/ragas/task_template.py,sha256=nv2i9-NE2SXpLrVKo5zhadY
 evalscope/backend/rag_eval/ragas/metrics/__init__.py,sha256=HgY5nrcNtWpQ7gBi5lCEJXJVINd_R57dsmI8ldS2rd0,160
 evalscope/backend/rag_eval/ragas/metrics/multi_modal_faithfulness.py,sha256=Uqz5qWZ76Gos95_QlhwncbATXyk0YX4wkI0LiAdPElU,3838
 evalscope/backend/rag_eval/ragas/metrics/multi_modal_relevance.py,sha256=CdLnWHq1eTna6j3F5-pncW5YusxD_v3ScjzeCsZ7mng,3967
+evalscope/backend/rag_eval/ragas/prompts/persona_prompt.py,sha256=1m8FBVga_uetCkahL_mwhGS8nAXG8V4jmnT4iP_6QYo,794
+evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/correctness_prompt_chinese.json,sha256=YaqCbIynnRtPQHng6AzlD4l7KA-TPAi4ayjnhZj6gw0,3940
+evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerCorrectness/long_form_answer_prompt_chinese.json,sha256=-BjIwLy3QOiQbFGqjhYTNfhLTLeaBeOtpKBKfpjlf7E,1736
+evalscope/backend/rag_eval/ragas/prompts/chinese/AnswerRelevancy/question_generation_chinese.json,sha256=eyUasvFvtwXAcpeUaOOBVuvxhGl-u_dndV-qsjnqsF4,981
+evalscope/backend/rag_eval/ragas/prompts/chinese/ContextPrecision/context_precision_prompt_chinese.json,sha256=KXr3hmd49n1KsgYWrjTuYY9xBFIcTSksueVTUEwfEm0,3188
+evalscope/backend/rag_eval/ragas/prompts/chinese/Faithfulness/nli_statements_message_chinese.json,sha256=1A9KlwbQr8WqNxdLEa4nU1HlPzF-q2KflQ591pJA0To,2475
+evalscope/backend/rag_eval/ragas/prompts/chinese/Faithfulness/statement_prompt_chinese.json,sha256=YQFk8o0esRyOF9m2aJBR_Nwn40D6LAr7YrfhQdHae_s,1739
+evalscope/backend/rag_eval/ragas/prompts/chinese/HeadlinesExtractor/prompt_chinese.json,sha256=xH4kduv1OUJIl_xcGGh-StK_zOlZa4G-pCrIt1M8Hbc,1025
+evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/concept_combination_prompt_chinese.json,sha256=Q4sf2Xud4NpVrbEIYZJEE_VVjMy-fgwX_AK0OnMQpDg,992
+evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/generate_query_reference_prompt_chinese.json,sha256=FGGqRlNgvEXnH-YcNPk5pzoRZXwtaS5cMtbIBQyEPyU,669
+evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopAbstractQuerySynthesizer/theme_persona_matching_prompt_chinese.json,sha256=4JTUCczH-7UjH5nlz13w-srcTC3usqiXjJwLwxu-MIg,919
+evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json,sha256=PJ2IHm3zXHe_XnT_DPxL5TNqJGJ-jjX2owVShw9V9kA,672
+evalscope/backend/rag_eval/ragas/prompts/chinese/MultiHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json,sha256=4JTUCczH-7UjH5nlz13w-srcTC3usqiXjJwLwxu-MIg,919
+evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalFaithfulness/faithfulness_prompt_chinese.json,sha256=nZ7VIz6R1XyyKtP0Vq5jPFNfHaN6M1Z9rFPOCVRChBE,1374
+evalscope/backend/rag_eval/ragas/prompts/chinese/MultiModalRelevance/relevance_prompt_chinese.json,sha256=5IKDA_hPmyuDXMhzK7aACrZGrYNT3wuqhzsHYC7Vkt4,1496
+evalscope/backend/rag_eval/ragas/prompts/chinese/NERExtractor/prompt_chinese.json,sha256=uY_4P9OloNHP2IdvIuoTFCuUEHMyEqx9TzCoC6tj8G8,774
+evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/generate_query_reference_prompt_chinese.json,sha256=-0BwNQgPeH3dIIHsgNSL9OCMsg03oqtWtqm6HJG6gOk,663
+evalscope/backend/rag_eval/ragas/prompts/chinese/SingleHopSpecificQuerySynthesizer/theme_persona_matching_prompt_chinese.json,sha256=9oSmEYvqor920jXByeNynyOSXagAukFK_e4jnMuDZQU,916
+evalscope/backend/rag_eval/ragas/prompts/chinese/SummaryExtractor/prompt_chinese.json,sha256=ukF4AaOn8Su0uZ5E_uszzZFC1_MY2M9OymOSZ15w0BQ,688
+evalscope/backend/rag_eval/ragas/prompts/chinese/ThemesExtractor/prompt_chinese.json,sha256=dH-etTJrQ0gQIS97QCZ5IhQR223gLS0_QZjUEW91fOA,657
 evalscope/backend/rag_eval/ragas/tasks/__init__.py,sha256=WO2xja0g0JSiYGdu2uAEDQgDceuFcgPWwPoqFnwDU0s,172
 evalscope/backend/rag_eval/ragas/tasks/testset_generation.py,sha256=nX-dG0Fm1629pSASujuEmMODFZf1955WncNNykRrNtI,9305
 evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py,sha256=bXOqik6qKWzbrEz21ykdkqeqqPrmoUIhTwW6eRQXy0M,2222
@@ -52,7 +74,7 @@ evalscope/backend/rag_eval/utils/embedding.py,sha256=RZf0JlovZY_cCBsq8MMUqC_Sy78
 evalscope/backend/rag_eval/utils/llm.py,sha256=9tFwMNoTf3jNomgDu5qqVLO92HtEtelH3DXpny9_B2g,2552
 evalscope/backend/rag_eval/utils/tools.py,sha256=LpcYoeIBj1btzQ1_P84u1dYCdRWhMtiltxihmZCvWKk,1528
 evalscope/backend/vlm_eval_kit/__init__.py,sha256=xTgHM95lWzh4s0W7zxLwYkgUbPAZfAb0UoGGmyyBXrs,83
-evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=ewhpE9yzsqf5ED6kqsqek2YEgg96GBQOupxtVNhaXxI,6046
+evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=ZB0wYzPBPyIM0zjfp5C71GcjAgKxKoWxF7RrhuDQBYM,5931
 evalscope/backend/vlm_eval_kit/custom_dataset.py,sha256=Yz2A5kB1E8DYBnjuVCA6TTPtLjhg8vYKeJTh6FU_Ecw,1645
 evalscope/benchmarks/__init__.py,sha256=6TKP35wfKf7R_h870fsEtcIlIAgomKOcukNL9M-5I1Y,162
 evalscope/benchmarks/benchmark.py,sha256=EmwYyFdrAHBGMkSbsMZQOR_62Q0CSKl8zeLlr7xvJdQ,2159
@@ -92,9 +114,11 @@ evalscope/benchmarks/bbh/cot_prompts/word_sorting.txt,sha256=uhRRz8y0hfHI96olJS9
 evalscope/benchmarks/ceval/__init__.py,sha256=SatTco8Ks6wD0jh9LUN5chf21VaJnwW1SG4cGG8OYAo,343
 evalscope/benchmarks/ceval/ceval_adapter.py,sha256=FBUTdmW4a5TY7atBjE_H1h_ST2_WoPWMMTvfHNvusNU,15852
 evalscope/benchmarks/ceval/ceval_exam.py,sha256=S32eMfGUBMrUDP39HzO6XfvSir0tthHCPItNtriE-hc,5063
+evalscope/benchmarks/ceval/samples.jsonl,sha256=dyWhGAdt4eq6Amgu2Ykx8RevUJVFtbhGFSTbDAeUgHc,448
 evalscope/benchmarks/cmmlu/__init__.py,sha256=mIMlXA_BHb_bF71Oi5XJwhV_sZKN2b_lBTOXhU5h6Bg,342
 evalscope/benchmarks/cmmlu/cmmlu.py,sha256=q_6ONrjdcHNqpXTUmSVbNOfl1yMd0zEQZWnh0PMQmYY,5153
 evalscope/benchmarks/cmmlu/cmmlu_adapter.py,sha256=jqVghYwex2Awx7THgka0wQ7dFY0EdzfnI7n0aMXGPro,15216
+evalscope/benchmarks/cmmlu/samples.jsonl,sha256=l842nKaAfeRE69jcX_E5N1gstWrHYpoNZjP-5D6Aq_k,1721
 evalscope/benchmarks/competition_math/__init__.py,sha256=hXO0DTtrA_0YDYUcyrL4XOyPGvPEa0sy2miHTF1Cxrg,393
 evalscope/benchmarks/competition_math/competition_math.py,sha256=0p5iKUfU6WpXgplb44YgVWZUYkeWLLmOdj66_dapdDc,2678
 evalscope/benchmarks/competition_math/competition_math_adapter.py,sha256=FijGL1FlEWJAy34tp3bIapiglT7KBJ8AvU8bjP4CGAw,19087
@@ -112,10 +136,13 @@ evalscope/benchmarks/humaneval/humaneval_adapter.py,sha256=1YiAvNbWRUcaTu9oGwpDM
 evalscope/benchmarks/mmlu/__init__.py,sha256=fZicGcLq67XOc5cofGCi6WrV4FdubLupKb7nMdCUQSA,337
 evalscope/benchmarks/mmlu/mmlu.py,sha256=GhjZFOgX5qG041eVrSWggOcRcMyl0oAI_yGXmufwEzc,5256
 evalscope/benchmarks/mmlu/mmlu_adapter.py,sha256=9lg_3s3QjGKC794O-RogU9cdvcCP7_Vp4ve9U9dRhz8,16401
+evalscope/benchmarks/mmlu/samples.jsonl,sha256=f5Y2vwbEvNtpE7vrl9BHoJzsdceI4vUAo1frexYyX2o,1345
 evalscope/benchmarks/race/__init__.py,sha256=htMZhgk40CsvNF7HXaHeAejUnGbUtU6Nu2yATOiMfaU,337
 evalscope/benchmarks/race/race.py,sha256=giY44Vr6CePdVQxpi0x4CLsaknye47Gdlc_PVqN9VCA,3835
 evalscope/benchmarks/race/race_adapter.py,sha256=3zHfz3tFzCVKoYLtzpGek338ZnIGT7ejq_xSaMxiIjU,9900
+evalscope/benchmarks/race/samples.jsonl,sha256=GMwF5IPRWrsq6cfYNGS5yt_woXz687HObA0IkB6k3V4,1242
 evalscope/benchmarks/trivia_qa/__init__.py,sha256=oslov-n_oV3bhEhrPXLJoQwmHE8_vYR2JTerxoHq29A,351
+evalscope/benchmarks/trivia_qa/samples.jsonl,sha256=9OsKC9uuBbw9MHghOOMMALeGkFOY_QTNWZYAr0ASPQ0,3444
 evalscope/benchmarks/trivia_qa/trivia_qa.py,sha256=xrebA71r_Ek9NvwkDfsmWTuRCsae2HZEGmTBtZMGwfM,3296
 evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py,sha256=0g0xSWorXiHrZ3PKTqOO6g18kK2tUop1HWaAjmCKRwg,7659
 evalscope/benchmarks/truthful_qa/__init__.py,sha256=4bRdnHOceaEvn20jZj0yLCg5wpOHpzP3LRjkYm5u-Fs,367
@@ -124,7 +151,7 @@ evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py,sha256=Cavimjnc6NPMC1TDO
 evalscope/cli/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/cli/base.py,sha256=m1DFlF16L0Lyrn0YNuFj8ByGjVJIoI0jKzAoodIXjRk,404
 evalscope/cli/cli.py,sha256=uZ-qC8WBsLd5-Hn94d43sSGg0UC_12RebSD4ToKjypg,844
-evalscope/cli/start_perf.py,sha256=TL6bMXYl3ln-tfs5uBmzb9x94uxz6f3PBFIt1l7g3VA,994
+evalscope/cli/start_perf.py,sha256=yIE3sP13_yoTXQD3DBNzRVY6L_5p-Ix0J1VBvZFYdVU,914
 evalscope/cli/start_server.py,sha256=ATGLP2TE0aImJNicpehdzBuFlNb50F7KhyL4A_ZSoGU,3885
 evalscope/evaluator/__init__.py,sha256=S6MU1O_iiNAaKxNIhO9MEmdW-BSNf_YH2l6NQ9lxVNo,103
 evalscope/evaluator/evaluator.py,sha256=eSCgPPDGfIJfKu0cthhbDLFm1xMhj_869iT3ngcQkPc,30817
@@ -135,9 +162,11 @@ evalscope/metrics/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw
 evalscope/metrics/code_metric.py,sha256=zK1tpNDZbvmSHt3a_JJ5Y2Hdu2cqeFriy__wUOl2tSw,3462
 evalscope/metrics/math_accuracy.py,sha256=1PCy1VUNYg48JcGy-6SUmUDZNwPeAkMW1QQ_lXomdWw,1988
 evalscope/metrics/metrics.py,sha256=sDZljGiZwgHsFZ5eNi65-3z3BLCdIwWUzPcq2QpKf1k,12545
-evalscope/metrics/rouge_metric.py,sha256=sN0r-sXXc-nJUdFrthQPAv1VFdOCrF6zzIYDKaLSgrU,4522
+evalscope/metrics/rouge_metric.py,sha256=VNdy86ZGZL6thVDFg0nKedp6dPApV7_yoIupMe0f6hk,4518
 evalscope/metrics/bundled_rouge_score/__init__.py,sha256=PwbTdk8168FwDJe_l8XIqDuBgZQooDsP31vj7di05Fs,650
 evalscope/metrics/bundled_rouge_score/rouge_scorer.py,sha256=MXcHwmsXnh9mQZR1Bt5St6DNwXY-mfz4dNM8y6a23dc,12236
+evalscope/metrics/resources/gpt2-zhcn3-v4.bpe,sha256=J_K-oSt9usFEw87Av6j7ETn3J48EsmFuY5_iVvY6xjc,524464
+evalscope/metrics/resources/gpt2-zhcn3-v4.json,sha256=WkM4J_FDPPNQwYi0kj5sM5SVjk2_6bci7tqf8dV9p_U,1289341
 evalscope/models/__init__.py,sha256=zG27J2HSeKPGiAIUE7QLPHEPLyXLsfaDwYI_TDXjpCg,145
 evalscope/models/dummy_chat_model.py,sha256=xE8wcFVSCkvizEJ-B8ojX0Ir01Q5KrN5mapjMQaQtbg,1325
 evalscope/models/model.py,sha256=ZzzVzZHVzuzdt5F1r-rEBT44ZfW9B7R1spsrV-T8nSw,3020
@@ -149,25 +178,42 @@ evalscope/models/api/openai_api.py,sha256=uBicJPaFLOhIrB5PKI8FE-SItb7v-fuDwBgkgn
 evalscope/models/custom/__init__.py,sha256=K4Ewo7Qrs73-jBuPq4ffxd8hMnttKhic-Zj0amH3wiU,103
 evalscope/models/custom/custom_model.py,sha256=2ivxfGQs5V5HDnQEhTBi5v8KNBxJDbzPVJdNOGo3iSg,1566
 evalscope/perf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/perf/_logging.py,sha256=v-a1uhqUt8116OEGXa-uhCPNE3mLxBaJZaKo2ReJgp8,1034
-evalscope/perf/api_plugin_base.py,sha256=ieAE-WjJLfgKIz0lDv1TkoKU3oPAW4pMseOJLmuHxCo,2243
-evalscope/perf/custom_api.py,sha256=H2IgM-LMjqXxVhbrtkXuiREb-p14zwMmllgl26a-jgw,3712
-evalscope/perf/dashscope_api.py,sha256=_XUF3czkYdPdVgtP7nqzRxROKxlqDjWs4DQnTyocNvM,3410
-evalscope/perf/dataset_plugin_base.py,sha256=6veUTyZ38W1Iig65vxNV9SfmqrsR8ID_UHgNiUO9Bv4,1814
-evalscope/perf/how_to_analysis_result.py,sha256=UVd_aYJ_7N5hl_wK9oIZig1vSwfgzodxW7XC6IWqbdg,1044
-evalscope/perf/http_client.py,sha256=4ppaZAIwrajJ9nzdgdwc3EdjmGSJz1_dg7Q6wQYELgw,34537
-evalscope/perf/openai_api.py,sha256=rJSGlXtnHgMNYcgO0bJQCsSLhKChUxklTk4cI63YTMQ,6066
-evalscope/perf/plugin_registry.py,sha256=D2MG2AXDBScjuKxB4g_Hg026pSRO752dBimonYtaAzM,782
-evalscope/perf/query_parameters.py,sha256=HfGRZJSzRMVfPezWTvbWhYeprCetGNPX_M_paoDtuOY,1346
-evalscope/perf/server_sent_event.py,sha256=s2UqUr1qAMWzBG1XWCFxhulyztd6FM0tGqVvPC8jD5o,1153
-evalscope/perf/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/perf/datasets/line_by_line.py,sha256=vbBNh0GcR-BfbFZMT6Z_3NqXe4y-uVfyaoooBRE7gjc,830
-evalscope/perf/datasets/longalpaca_12k.py,sha256=OaOzksyBBbeYwO0tFnKZ6UZ9PQO2RdMRD4HyCVBxnX4,934
-evalscope/perf/datasets/openqa.py,sha256=Dz5__mcYjP81Mc2NCGDAy-JFTVvif1slP7iWQflayFY,1018
-evalscope/preprocess/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
-evalscope/preprocess/tokenizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-evalscope/preprocess/tokenizers/gpt2_tokenizer.py,sha256=8dOPVWrzAXhzmzSKBWdWjfDqPlRaMH9slK8v5aWhwcQ,7810
+evalscope/perf/arguments.py,sha256=ixiWx16qAL1gU7JTwoYOnvvc3IrwVWGz2uVno38gywA,8671
+evalscope/perf/benchmark.py,sha256=Yiqcg5N03KmBa-5aWYNyklbYJ9Hqiuu1oaD8kBkFPSQ,9659
+evalscope/perf/http_client.py,sha256=OpTgYl4obSpmyi5bOkTRSIQxp0aVdO08EcIVFAv-znU,7192
+evalscope/perf/main.py,sha256=ljJDJVsD9hGWgF5bJCW-mfUGohc4LofaxiyAUfMa2WQ,997
+evalscope/perf/plugin/__init__.py,sha256=1sl5s-csrwKb_LVTnpF3HqArz06TRD5LYJ0hpqvokUA,85
+evalscope/perf/plugin/registry.py,sha256=PyK3E1AqQFuU4Bs9COvFFCJOaCtmHbfeQOVGtjVYh-I,1304
+evalscope/perf/plugin/api/__init__.py,sha256=Ckzbq4CkSMVQTedQcDHCYlRd6FTwQAElt2mHB-VXJac,195
+evalscope/perf/plugin/api/base.py,sha256=B_H04qKx7eRTn155rnDrbTYur7PK1mvxfQKYcqYbndU,2118
+evalscope/perf/plugin/api/custom_api.py,sha256=NQ2LDKsFQfExVRx2prcmfORCBzxxibfhpVHhB-lxAO4,3776
+evalscope/perf/plugin/api/dashscope_api.py,sha256=0p9f6ujppS_H6w7wsIbRVNnCkHXtRemIai5Bhdogla4,3826
+evalscope/perf/plugin/api/openai_api.py,sha256=I9yM4ouY1-xlBz4bYQ_62FZHKX4F3YCsg5GCqLU_9xA,6938
+evalscope/perf/plugin/datasets/__init__.py,sha256=9mz2TnVHhxbEKAS9pLbKMQuIoShNlZpGiRo9e2RQLUs,490
+evalscope/perf/plugin/datasets/base.py,sha256=1U_efZuU2ZdWV9UVAqFu1fx9_0PST_sJnaSIqbNvTF4,1787
+evalscope/perf/plugin/datasets/custom.py,sha256=_GSC5yR90_BjcRjdJqrJT2vHQAzskz5XxYOxngUM2Pg,815
+evalscope/perf/plugin/datasets/flickr8k.py,sha256=39jbcZde4cOY6PpJHeb20v5PIg58ezFMoXjYO7U6Z2A,1582
+evalscope/perf/plugin/datasets/line_by_line.py,sha256=IKVZMpKei6XW9DTm9VEssWHE96i1lTqMf0621dA_img,836
+evalscope/perf/plugin/datasets/longalpaca.py,sha256=ohmq3Mp0JKeG8h8ef9GYqN7pBLTHzpF8g9KrrriRbwM,1165
+evalscope/perf/plugin/datasets/openqa.py,sha256=l9vCnEKBYU1a8uo49kArwSXu-ZaOXDHa2Pl3gp4yXE4,1395
+evalscope/perf/plugin/datasets/speed_benchmark.py,sha256=ef8MXhT6756y6LsXSpYeWjmwswu2hRXe2BOVS2_OgVM,1968
+evalscope/perf/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalscope/perf/utils/analysis_result.py,sha256=o0wMcr9U0Gwd5lh5tAFCFpp3FmfwsaMppyJOLI2_sJ8,1213
+evalscope/perf/utils/benchmark_util.py,sha256=-wZyZnWrXsQOzPrWdxQVbQUVUAljzsfWV4-2Hw_xzpQ,5565
+evalscope/perf/utils/chat_service.py,sha256=ncMmeUDpOo7Kjkhe_TPDZY8ffoHTCl-B5szHJ4gipEo,8642
+evalscope/perf/utils/db_util.py,sha256=TeZzcGoWDde81EjpDOyV6c2B1ZM7NzRv-0cEmeorGjE,7356
+evalscope/perf/utils/handler.py,sha256=HyKIxbzC0XCyQanlbb7UEY7yaeqjJTePNea8kMV3Sdc,1192
+evalscope/perf/utils/local_server.py,sha256=AezbEdPGuE1esCBxXtXJWjFYTZfFb6SYC6bAfcaX5Gk,4316
 evalscope/registry/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
+evalscope/registry/config/cfg_arena.yaml,sha256=rub6ceaQxxB1mbSjdoFf0IaVgGfbOonV2nYRebv2OKo,3292
+evalscope/registry/config/cfg_arena_zhihu.yaml,sha256=IQmfcwkzCCV-bMbIC9M2fd-X99bHJ_r_qfIJjClClx0,2760
+evalscope/registry/config/cfg_pairwise_baseline.yaml,sha256=d05pBiqOk1ejcdd9XE-opZ_ersyttAesF3Iwa2df8O8,3580
+evalscope/registry/config/cfg_single.yaml,sha256=zjsUC3zhU8z7JURaJiz7npkUbFpP82q1ycqUmObC-hc,3056
+evalscope/registry/data/question.jsonl,sha256=WQw5FXvFYerdfwPK1L4YwrWX-TApeAr2X4Zxjznq-oc,12885
+evalscope/registry/data/prompt_template/lmsys_v2.jsonl,sha256=F3PcsoO_UOCztLNmGDYd90K4z4eVufBWz5prKrcqHG0,10554
+evalscope/registry/data/prompt_template/prompt_templates.jsonl,sha256=F3PcsoO_UOCztLNmGDYd90K4z4eVufBWz5prKrcqHG0,10554
+evalscope/registry/data/qa_browser/battle.jsonl,sha256=2MXcYoMItBmttQxSMh2Oa0x51xxqJaWEgSuERUx1O_0,1185590
+evalscope/registry/data/qa_browser/category_mapping.yaml,sha256=3r9nUIciW9205qbtOQF7aI_etM191cM3vlWU8ueG2Co,484
 evalscope/registry/tasks/arc.yaml,sha256=phXsBLsAgvHWmU31J89QMnJJnUioRphraQrF9SrJ53c,863
 evalscope/registry/tasks/bbh.yaml,sha256=Ircb_-_eVri2B1MHeSrFs9vIol7RY8ZaWwdz1j57NHA,701
 evalscope/registry/tasks/bbh_mini.yaml,sha256=eZYash__XJcfJau0VqujehuYE2WnFzrWr9s9jCkNT8Q,775
@@ -180,7 +226,10 @@ evalscope/registry/tasks/gsm8k.yaml,sha256=KYLK-xtv_3qtgCZiwwP4-rP_ftc_qUmtsl1Tf
 evalscope/registry/tasks/mmlu.yaml,sha256=504yhHVfi9pvUBk_SGPs-Yx7R2hx_2_-nAFiGIiFGx4,726
 evalscope/registry/tasks/mmlu_mini.yaml,sha256=wVbosZ5Tm9pwLG5nCphalezXilIjcq5j33nz3MR7_BE,778
 evalscope/third_party/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
+evalscope/third_party/longbench_write/README.md,sha256=p7C5StphFFzEeMA2lcfKyeBlJgJiIjTSXvzwhw9md2k,3248
 evalscope/third_party/longbench_write/__init__.py,sha256=GNbBDc7HAh_V2Hfy5HhND_u7z6OI79czoBlP8lX4PVo,126
+evalscope/third_party/longbench_write/default_task.json,sha256=HPSnI7Ar7cqe86wzQnH2XsDtqmAuCDLy3sZm3MeNyKc,711
+evalscope/third_party/longbench_write/default_task.yaml,sha256=aQB-Cn-gEkdoI_26yOaeJWGpoI3-FxHBclZGAmxeBcc,579
 evalscope/third_party/longbench_write/eval.py,sha256=_fwV3f-Yq0qrkuZ6LBXvBiXnM6lpz6sOqd7BfYxEU80,11163
 evalscope/third_party/longbench_write/infer.py,sha256=MB0MdSM1qDx15FyrPSU6BXPbSGnBjxuTWqrcHAgbj9o,8318
 evalscope/third_party/longbench_write/longbench_write.py,sha256=MQzlIzv3sGlNgxgX0FPHtDIuAmgwThfBkMeKNcsR3U8,3926
@@ -192,9 +241,13 @@ evalscope/third_party/longbench_write/resources/longbench_write_en.jsonl,sha256=
 evalscope/third_party/longbench_write/resources/longwrite_ruler.jsonl,sha256=odTr8N8PoWAFZ2kdEcmlLeMDfEo3KXDtLo9S8oieCmI,5718
 evalscope/third_party/longbench_write/tools/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/third_party/longbench_write/tools/data_etl.py,sha256=fSc4iT7_bdTvW20TbjlWme-k1pLqj_e2wXV8z831_Yw,5963
+evalscope/third_party/toolbench_static/README.md,sha256=Osdnt0_K-setbmYwDPCPRp2LXxamGp2mE8KsOByPPOY,3944
 evalscope/third_party/toolbench_static/__init__.py,sha256=BO936RxwodHr4OEpV6W3S_keC91OfOd41_msIJ2d0fs,128
+evalscope/third_party/toolbench_static/config_default.json,sha256=KrUzeHL2DNiM5FwY7cH3KZlxTwELCQZ6e39nilfUi0M,368
+evalscope/third_party/toolbench_static/config_default.yaml,sha256=-6n6Zyg9eHN2eexlehSi9LI4F3EPk-3JacrAb6ZoyxI,451
 evalscope/third_party/toolbench_static/eval.py,sha256=TqjMuuYePnD3bGRhQe1_9bIOlAW41kiFSztaEuppRLM,8237
 evalscope/third_party/toolbench_static/infer.py,sha256=WogwVXqDabdcsJ4uftZxAwR2wncp6HYpkS-fACEvjT4,9331
+evalscope/third_party/toolbench_static/requirements.txt,sha256=JMIbWAfKRYcQh771IT-EjroMagXchYDSgfgY7gcqx08,21
 evalscope/third_party/toolbench_static/toolbench_static.py,sha256=uXvyeyNWTZHFVASnOeMf1sqHUjy9NQ3r8wbkhUQJL1g,1930
 evalscope/third_party/toolbench_static/llm/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/third_party/toolbench_static/llm/swift_infer.py,sha256=UywM8SU2ByFTzD4YkbB17SXJyxmzY1QDwARDuGzbCvs,1452
@@ -205,12 +258,29 @@ evalscope/tools/rewrite_eval_results.py,sha256=ZVi2hVjiTOmR_O5IaLv6qnQNpMz6FnDb9
 evalscope/utils/__init__.py,sha256=6RjACRYUSpGj6fkZ7NzYpl0lFppQCp9KVn5ktZe626s,128
 evalscope/utils/arena_utils.py,sha256=RMkymUv9Cxs37arUntzgDY5P0Dand2jGpsb7uy6wZmg,7670
 evalscope/utils/completion_parsers.py,sha256=61l8CTh1VxHgRoMDhtznpAhuJp47MssGgS-LdEe_h80,2997
-evalscope/utils/logger.py,sha256=cf3U400Mx1speMMNXorjwEE8noDz5Mbd-9PNgaulGeY,3013
+evalscope/utils/logger.py,sha256=Nhm8u_Wpd5BlVPdv9IBW_M3XMEcp5UbkOf1oN2HvGG0,3060
 evalscope/utils/task_cfg_parser.py,sha256=LiNQ2X8lbZU0cODpaY_PbKyUhNoxZIC495UsLJigX64,138
 evalscope/utils/task_utils.py,sha256=IMtBSBUp3H95Ko0vn8Q55Wmz2SFZXSfjVy49tyomL_g,537
-evalscope/utils/utils.py,sha256=zHo9hfxGBUVKE2xNMR7lDoEvfRnk4V4946DEfXQhlq4,20509
-evalscope-0.6.1.dist-info/METADATA,sha256=n4CpTzJGnhgqEsfbL1UfZtXHULmeNCGnKChyi6eT8Fw,21237
-evalscope-0.6.1.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-evalscope-0.6.1.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
-evalscope-0.6.1.dist-info/top_level.txt,sha256=jNR-HMn3TR8Atolq7_4rW8IWVX6GhvYV5_1Y_KbJKlY,10
-evalscope-0.6.1.dist-info/RECORD,,
+evalscope/utils/utils.py,sha256=bv_5zDNNzsODSwXz6M7TFkdfVJT6rw_orn_BG-qkijM,20567
+tests/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
+tests/test_run_all.py,sha256=YcMTlWoFpvWY8jevWyIf2G_tz8hgDD1cAwSvmyZt96M,429
+tests/cli/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
+tests/cli/test_run.py,sha256=9GTF21NaUgERcF1Rkm9almO5-5pxsDF86Nw8fs8X7Hg,2926
+tests/perf/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
+tests/perf/test_perf.py,sha256=Mn3nw2UJoR4qDLZ3Jhna3m52gD4mouc63uY_DLyXkG0,2889
+tests/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/rag/test_clip_benchmark.py,sha256=7NsOzgrpU9ou22M7fXtSFEnYt0iy2Q-ShIDL26Kp2gw,2597
+tests/rag/test_mteb.py,sha256=MOksxYseIQ6SD_iFFxMC9BinvDtB0vlNSFEGJt0SGl8,4608
+tests/rag/test_ragas.py,sha256=g3rAHymUzTyM6usIce6kItwyh1IocummK0BBPZiJPmY,4024
+tests/swift/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
+tests/swift/test_run_swift_eval.py,sha256=Qop40c8jsHUbDTJe-Y8b_Aa8qn4Xstmu-FNGG14Gqik,5749
+tests/swift/test_run_swift_vlm_eval.py,sha256=p2i2ZRj-vG1YsQGsemvQLHcyhjy1EmUChyAjFEmVbCE,4899
+tests/swift/test_run_swift_vlm_jugde_eval.py,sha256=c31jwQle_97ru5Dep91qsAqYjR1HDm1O9YZihRr0u0s,6018
+tests/vlm/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
+tests/vlm/test_vlmeval.py,sha256=21xi0nu4ghDB6_X-Pol7pTfK7aYkAYOp82TQ-MSQv-I,1757
+evalscope-0.7.0.dist-info/LICENSE,sha256=K_2M03pN0PxVMyx9IQUKsHGhhDMkw5ryQ02rlMvzj3I,11416
+evalscope-0.7.0.dist-info/METADATA,sha256=W-NWOZwX9X-VN_LDI16aW6TxcOLJ3Um9dvms8bs28Bw,23796
+evalscope-0.7.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+evalscope-0.7.0.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
+evalscope-0.7.0.dist-info/top_level.txt,sha256=Yv0iprOqZQ4rfUO-AWJp7Ni6m0Twxny1yvZwO-8hUDM,16
+evalscope-0.7.0.dist-info/RECORD,,

{evalscope-0.6.1.dist-info → evalscope-0.7.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{evalscope-0.6.1.dist-info → evalscope-0.7.0.dist-info}/top_level.txt

@@ -1 +1,2 @@
 evalscope
+tests

tests/cli/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

tests/cli/test_run.py

@@ -0,0 +1,76 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import subprocess
+import unittest
+from evalscope.utils import test_level_list, is_module_installed
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class TestRun(unittest.TestCase):
+
+    def setUp(self) -> None:
+        logger.info(f'Init env for evalscope native run UTs ...\n')
+        self._check_env('evalscope')
+
+    def tearDown(self) -> None:
+        pass
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f'{module_name} is installed.')
+        else:
+            raise ModuleNotFoundError(f'run: pip install {module_name}')
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_simple_eval(self):
+        model = 'ZhipuAI/chatglm3-6b'
+        template_type = 'chatglm3'
+        datasets = 'arc'  # arc ceval
+        limit = 100
+
+        cmd_simple = f'python3 -m evalscope.run ' \
+                     f'--model {model} ' \
+                     f'--template-type {template_type} ' \
+                     f'--datasets {datasets} ' \
+                     f'--limit {limit}'
+
+        logger.info(f'Start to run command: {cmd_simple}')
+        run_res = subprocess.run(cmd_simple, text=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        assert run_res.returncode == 0, f'Failed to run command: {cmd_simple}'
+        logger.info(f'>>test_run_simple_eval stdout: {run_res.stdout}')
+        logger.error(f'>>test_run_simple_eval stderr: {run_res.stderr}')
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_eval_with_args(self):
+        model = 'ZhipuAI/chatglm3-6b'
+        template_type = 'chatglm3'
+        datasets = 'arc ceval'  # arc ceval
+        limit = 5
+        dataset_args = '{"ceval": {"few_shot_num": 0, "few_shot_random": false}}'
+
+        cmd_with_args = f'python3 -m evalscope.run ' \
+                        f'--model {model} ' \
+                        f'--template-type {template_type} ' \
+                        f'--datasets {datasets} ' \
+                        f'--limit {limit} ' \
+                        f'--generation-config do_sample=false,temperature=0.0 ' \
+                        f"""--dataset-args \'{dataset_args}\' """
+
+        logger.info(f'Start to run command: {cmd_with_args}')
+        run_res = subprocess.run(cmd_with_args, text=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        assert run_res.returncode == 0, f'Failed to run command: {cmd_with_args}'
+        logger.info(f'>>test_run_eval_with_args stdout: {run_res.stdout}')
+        logger.error(f'>>test_run_eval_with_args stderr: {run_res.stderr}')
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_eval_local(self):
+        ...
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/perf/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

tests/perf/test_perf.py

@@ -0,0 +1,96 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import unittest
+
+from evalscope.perf.main import run_perf_benchmark
+from evalscope.utils import test_level_list
+
+
+class TestPerf(unittest.TestCase):
+
+    def setUp(self) -> None:
+        pass
+
+    def tearDown(self) -> None:
+        pass
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf(self):
+        task_cfg = {
+            'url': 'http://127.0.0.1:8000/v1/chat/completions',
+            'parallel': 1,
+            'model': 'qwen2.5',
+            'number': 15,
+            'api': 'openai',
+            'dataset': 'openqa',
+            'debug': True,
+        }
+        run_perf_benchmark(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_stream(self):
+        task_cfg = {
+            'url': 'http://127.0.0.1:8000/v1/chat/completions',
+            'parallel': 1,
+            'model': 'qwen2.5',
+            'number': 15,
+            'api': 'openai',
+            'dataset': 'openqa',
+            'stream': True,
+            'debug': True,
+        }
+        run_perf_benchmark(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_speed_benchmark(self):
+        task_cfg = {
+            'url': 'http://127.0.0.1:8000/v1/completions',
+            'parallel': 1,
+            'model': 'qwen2.5',
+            'api': 'openai',
+            'dataset': 'speed_benchmark',
+            'debug': True,
+        }
+        run_perf_benchmark(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_local(self):
+        task_cfg = {
+            'parallel': 1,
+            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
+            'number': 5,
+            'api': 'local',
+            'dataset': 'openqa',
+            'debug': True,
+        }
+        run_perf_benchmark(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_local_stream(self):
+        task_cfg = {
+            'parallel': 1,
+            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
+            'number': 5,
+            'api': 'local',
+            'dataset': 'openqa',
+            'stream': True,
+            'debug': True,
+        }
+        run_perf_benchmark(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_local_speed_benchmark(self):
+        task_cfg = {
+            'parallel': 1,
+            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
+            'api': 'local_vllm',
+            'dataset': 'speed_benchmark',
+            'min_tokens': 2048,
+            'max_tokens': 2048,
+            'debug': True,
+        }
+        run_perf_benchmark(task_cfg)
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=False)

tests/rag/test_clip_benchmark.py

@@ -0,0 +1,85 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+
+# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
+import subprocess
+import unittest
+from evalscope.utils import test_level_list, is_module_installed
+from evalscope.utils.logger import get_logger
+from evalscope.run import run_task
+
+logger = get_logger()
+
+
+class TestCLIPBenchmark(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self._check_env("webdataset")
+
+    def tearDown(self) -> None:
+        pass
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f"{module_name} is installed.")
+        else:
+            raise ModuleNotFoundError(f"run: pip install {module_name}")
+
+    @unittest.skipUnless(0 in test_level_list(), "skip test in current test level")
+    def test_run_task(self):
+        task_cfg = {
+            "eval_backend": "RAGEval",
+            "eval_config": {
+                "tool": "clip_benchmark",
+                "eval": {
+                    "models": [
+                        {
+                            "model_name": "AI-ModelScope/chinese-clip-vit-large-patch14-336px",
+                        }
+                    ],
+                    "dataset_name": ["muge", "mnist"],
+                    "split": "test",
+                    "batch_size": 128,
+                    "num_workers": 1,
+                    "verbose": True,
+                    "skip_existing": False,
+                    "output_dir": "outputs",
+                    "cache_dir": "cache",
+                    "limit": 1000,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), "skip test in current test level")
+    def test_run_custom(self):
+        task_cfg = {
+            "eval_backend": "RAGEval",
+            "eval_config": {
+                "tool": "clip_benchmark",
+                "eval": {
+                    "models": [
+                        {
+                            "model_name": "AI-ModelScope/chinese-clip-vit-large-patch14-336px",
+                        }
+                    ],
+                    "dataset_name": ["custom"],
+                    "data_dir": "custom_eval/multimodal/text-image-retrieval",
+                    "split": "test",
+                    "batch_size": 128,
+                    "num_workers": 1,
+                    "verbose": True,
+                    "skip_existing": False,
+                    "limit": 1000,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+
+if __name__ == "__main__":
+    unittest.main(buffer=False)

tests/rag/test_mteb.py

@@ -0,0 +1,136 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import subprocess
+import unittest
+from evalscope.utils import test_level_list, is_module_installed
+from evalscope.utils.logger import get_logger
+from evalscope.run import run_task
+
+logger = get_logger()
+
+
+class TestMTEB(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self._check_env('mteb')
+
+    def tearDown(self) -> None:
+        pass
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f'{module_name} is installed.')
+        else:
+            raise ModuleNotFoundError(f'run: pip install {module_name}')
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_one_stage_mteb(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'MTEB',
+                'model': [
+                    {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                        'pooling_mode': None,  # load from model config
+                        'max_seq_length': 512,
+                        'prompt': '',
+                        'model_kwargs': {'torch_dtype': 'auto'},
+                        'encode_kwargs': {
+                            'batch_size': 128,
+                        },
+                    }
+                ],
+                'eval': {
+                    'tasks': [
+                        'TNews',
+                        'CLSClusteringS2S',
+                        'T2Reranking',
+                        'T2Retrieval',
+                        'ATEC',
+                    ],
+                    'verbosity': 2,
+                    'output_folder': 'outputs',
+                    'overwrite_results': True,
+                    'limits': 500,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_two_stage_mteb(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'MTEB',
+                'model': [
+                    {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                        'is_cross_encoder': False,
+                        'max_seq_length': 512,
+                        'prompt': '',
+                        'model_kwargs': {'torch_dtype': 'auto'},
+                        'encode_kwargs': {
+                            'batch_size': 64,
+                        },
+                    },
+                    {
+                        'model_name_or_path': 'OpenBMB/MiniCPM-Reranker',
+                        'is_cross_encoder': True,
+                        'max_seq_length': 512,
+                        'prompt': '为这个问题生成一个检索用的表示',
+                        'model_kwargs': {'torch_dtype': 'auto'},
+                        'encode_kwargs': {
+                            'batch_size': 32,
+                        },
+                    },
+                ],
+                'eval': {
+                    'tasks': ['MedicalRetrieval', 'T2Retrieval'],
+                    'verbosity': 2,
+                    'output_folder': 'outputs',
+                    'overwrite_results': True,
+                    'limits': 10,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_custom(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'MTEB',
+                'model': [
+                    {
+                        'model_name_or_path': 'AI-ModelScope/m3e-base',
+                        'pooling_mode': None,  # load from model config
+                        'max_seq_length': 512,
+                        'prompt': '',
+                        'model_kwargs': {'torch_dtype': 'auto'},
+                        'encode_kwargs': {
+                            'batch_size': 128,
+                        },
+                    }
+                ],
+                'eval': {
+                    'tasks': ['CustomRetrieval'],
+                    'dataset_path': 'custom_eval/text/retrieval',
+                    'verbosity': 2,
+                    'output_folder': 'outputs',
+                    'overwrite_results': True,
+                    'limits': 500,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=False)