evalscope 0.16.2__tar.gz → 0.17.0__tar.gz

{evalscope-0.16.2/evalscope.egg-info → evalscope-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.16.2
+Version: 0.17.0
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -15,116 +15,15 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: accelerate
-Requires-Dist: datasets>=3.0
-Requires-Dist: immutabledict
-Requires-Dist: jieba
-Requires-Dist: jsonlines
-Requires-Dist: langdetect
-Requires-Dist: latex2sympy2_extended
-Requires-Dist: matplotlib
-Requires-Dist: modelscope[framework]
-Requires-Dist: nltk>=3.9
-Requires-Dist: openai
-Requires-Dist: pandas
-Requires-Dist: pillow
-Requires-Dist: pyarrow
-Requires-Dist: pyyaml>=5.1
-Requires-Dist: requests
-Requires-Dist: rouge-chinese
-Requires-Dist: rouge-score>=0.1.0
-Requires-Dist: sacrebleu
-Requires-Dist: scikit-learn
-Requires-Dist: seaborn
-Requires-Dist: sympy
-Requires-Dist: tabulate
-Requires-Dist: torch
-Requires-Dist: tqdm
-Requires-Dist: transformers>=4.33
-Requires-Dist: word2number
 Provides-Extra: opencompass
-Requires-Dist: ms-opencompass>=0.1.6; extra == "opencompass"
 Provides-Extra: vlmeval
-Requires-Dist: ms-vlmeval>=0.0.17; extra == "vlmeval"
 Provides-Extra: rag
-Requires-Dist: langchain<0.4.0,>=0.3.0; extra == "rag"
-Requires-Dist: langchain-community<0.4.0,>=0.3.0; extra == "rag"
-Requires-Dist: langchain-core<0.4.0,>=0.3.0; extra == "rag"
-Requires-Dist: langchain-openai<0.4.0,>=0.3.0; extra == "rag"
-Requires-Dist: mteb==1.38.20; extra == "rag"
-Requires-Dist: ragas==0.2.14; extra == "rag"
-Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: perf
-Requires-Dist: aiohttp; extra == "perf"
-Requires-Dist: fastapi; extra == "perf"
-Requires-Dist: numpy; extra == "perf"
-Requires-Dist: rich; extra == "perf"
-Requires-Dist: sse_starlette; extra == "perf"
-Requires-Dist: transformers; extra == "perf"
-Requires-Dist: uvicorn; extra == "perf"
 Provides-Extra: app
-Requires-Dist: gradio==5.4.0; extra == "app"
-Requires-Dist: plotly<6.0.0,>=5.23.0; extra == "app"
 Provides-Extra: aigc
-Requires-Dist: diffusers; extra == "aigc"
-Requires-Dist: iopath; extra == "aigc"
-Requires-Dist: omegaconf; extra == "aigc"
-Requires-Dist: open_clip_torch; extra == "aigc"
-Requires-Dist: opencv-python; extra == "aigc"
-Requires-Dist: torchvision; extra == "aigc"
+Provides-Extra: dev
 Provides-Extra: all
-Requires-Dist: accelerate; extra == "all"
-Requires-Dist: datasets>=3.0; extra == "all"
-Requires-Dist: immutabledict; extra == "all"
-Requires-Dist: jieba; extra == "all"
-Requires-Dist: jsonlines; extra == "all"
-Requires-Dist: langdetect; extra == "all"
-Requires-Dist: latex2sympy2_extended; extra == "all"
-Requires-Dist: matplotlib; extra == "all"
-Requires-Dist: modelscope[framework]; extra == "all"
-Requires-Dist: nltk>=3.9; extra == "all"
-Requires-Dist: openai; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: pillow; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: pyyaml>=5.1; extra == "all"
-Requires-Dist: requests; extra == "all"
-Requires-Dist: rouge-chinese; extra == "all"
-Requires-Dist: rouge-score>=0.1.0; extra == "all"
-Requires-Dist: sacrebleu; extra == "all"
-Requires-Dist: scikit-learn; extra == "all"
-Requires-Dist: seaborn; extra == "all"
-Requires-Dist: sympy; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: torch; extra == "all"
-Requires-Dist: tqdm; extra == "all"
-Requires-Dist: transformers>=4.33; extra == "all"
-Requires-Dist: word2number; extra == "all"
-Requires-Dist: ms-opencompass>=0.1.6; extra == "all"
-Requires-Dist: ms-vlmeval>=0.0.17; extra == "all"
-Requires-Dist: langchain<0.4.0,>=0.3.0; extra == "all"
-Requires-Dist: langchain-community<0.4.0,>=0.3.0; extra == "all"
-Requires-Dist: langchain-core<0.4.0,>=0.3.0; extra == "all"
-Requires-Dist: langchain-openai<0.4.0,>=0.3.0; extra == "all"
-Requires-Dist: mteb==1.38.20; extra == "all"
-Requires-Dist: ragas==0.2.14; extra == "all"
-Requires-Dist: webdataset>0.2.0; extra == "all"
-Requires-Dist: aiohttp; extra == "all"
-Requires-Dist: fastapi; extra == "all"
-Requires-Dist: numpy; extra == "all"
-Requires-Dist: rich; extra == "all"
-Requires-Dist: sse_starlette; extra == "all"
-Requires-Dist: transformers; extra == "all"
-Requires-Dist: uvicorn; extra == "all"
-Requires-Dist: gradio==5.4.0; extra == "all"
-Requires-Dist: plotly<6.0.0,>=5.23.0; extra == "all"
-Requires-Dist: diffusers; extra == "all"
-Requires-Dist: iopath; extra == "all"
-Requires-Dist: omegaconf; extra == "all"
-Requires-Dist: open_clip_torch; extra == "all"
-Requires-Dist: opencv-python; extra == "all"
-Requires-Dist: torchvision; extra == "all"
+License-File: LICENSE
 
 <p align="center">
     <br>
@@ -198,24 +97,33 @@ EvalScope is not merely an evaluation tool; it is a valuable ally in your model
 Below is the overall architecture diagram of EvalScope:
 
 <p align="center">
-  <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
+  <img src="https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/doc/EvalScope%E6%9E%B6%E6%9E%84%E5%9B%BE.png" width="70%">
   <br>EvalScope Framework.
 </p>
 
 <details><summary>Framework Description</summary>
 
 The architecture includes the following modules:
-1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
-2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
-3. **Evaluation Backend**:
-    - **Native**: EvalScope’s own **default evaluation framework**, supporting various evaluation modes, including single model evaluation, arena mode, baseline model comparison mode, etc.
-    - **OpenCompass**: Supports [OpenCompass](https://github.com/open-compass/opencompass) as the evaluation backend, providing advanced encapsulation and task simplification, allowing you to submit tasks for evaluation more easily.
-    - **VLMEvalKit**: Supports [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) as the evaluation backend, enabling easy initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
-    - **RAGEval**: Supports RAG evaluation, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
-    - **ThirdParty**: Other third-party evaluation tasks, such as ToolBench.
-4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
-5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
-6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
+1. Input Layer
+- **Model Sources**: API models (OpenAI API), local models (ModelScope)
+- **Datasets**: Standard evaluation benchmarks (MMLU/GSM8k, etc.), custom data (MCQ/QA)
+
+2. Core Functions
+- **Multi-backend Evaluation**
+   - Native backends: Unified evaluation for LLM/VLM/Embedding/T2I models
+   - Integrated frameworks: OpenCompass/MTEB/VLMEvalKit/RAGAS
+
+- **Performance Monitoring**
+   - Model plugins: Supports various model service APIs
+   - Data plugins: Supports multiple data formats
+   - Metric tracking: TTFT/TPOP/Stability and other metrics
+
+- **Tool Extensions**
+   - Integration: Tool-Bench/Needle-in-a-Haystack/BFCL-v3
+
+3. Output Layer
+- **Structured Reports**: Supports JSON/Tables/Logs
+- **Visualization Platforms**: Supports Gradio/Wandb/SwanLab
 
 </details>
 
@@ -230,7 +138,9 @@ Please scan the QR code below to join our community groups:
 
 ## 🎉 News
 
-- 🔥 **[2025.06.19]** Added support for the BFCL-v3 benchmark, designed to evaluate model function-calling capabilities across various scenarios. For more information, refer to the [documentation](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html).
+- 🔥 **[2025.07.03]** Refactored Arena Mode: now supports custom model battles, outputs a model leaderboard, and provides battle result visualization. See [reference](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html) for details.
+- 🔥 **[2025.06.28]** Optimized custom dataset evaluation: now supports evaluation without reference answers. Enhanced LLM judge usage, with built-in modes for "scoring directly without reference answers" and "checking answer consistency with reference answers". See [reference](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/llm.html#qa) for details.
+- 🔥 **[2025.06.19]** Added support for the [BFCL-v3](https://modelscope.cn/datasets/AI-ModelScope/bfcl_v3) benchmark, designed to evaluate model function-calling capabilities across various scenarios. For more information, refer to the [documentation](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html).
 - 🔥 **[2025.06.02]** Added support for the Needle-in-a-Haystack test. Simply specify `needle_haystack` to conduct the test, and a corresponding heatmap will be generated in the `outputs/reports` folder, providing a visual representation of the model's performance. Refer to the [documentation](https://evalscope.readthedocs.io/en/latest/third_party/needle_haystack.html) for more details.
 - 🔥 **[2025.05.29]** Added support for two long document evaluation benchmarks: [DocMath](https://modelscope.cn/datasets/yale-nlp/DocMath-Eval/summary) and [FRAMES](https://modelscope.cn/datasets/iic/frames/summary). For usage guidelines, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html).
 - 🔥 **[2025.05.16]** Model service performance stress testing now supports setting various levels of concurrency and outputs a performance test report. [Reference example](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/quick_start.html#id3).
@@ -248,12 +158,12 @@ Please scan the QR code below to join our community groups:
 - 🔥 **[2025.03.03]** Added support for evaluating the IQ and EQ of models. Refer to [📖 Best Practices for IQ and EQ Evaluation](https://evalscope.readthedocs.io/en/latest/best_practice/iquiz.html) to find out how smart your AI is!
 - 🔥 **[2025.02.27]** Added support for evaluating the reasoning efficiency of models. Refer to [📖 Best Practices for Evaluating Thinking Efficiency](https://evalscope.readthedocs.io/en/latest/best_practice/think_eval.html). This implementation is inspired by the works [Overthinking](https://doi.org/10.48550/arXiv.2412.21187) and [Underthinking](https://doi.org/10.48550/arXiv.2501.18585).
 - 🔥 **[2025.02.25]** Added support for two model inference-related evaluation benchmarks: [MuSR](https://modelscope.cn/datasets/AI-ModelScope/MuSR) and [ProcessBench](https://www.modelscope.cn/datasets/Qwen/ProcessBench/summary). To use them, simply specify `musr` and `process_bench` respectively in the datasets parameter.
+<details><summary>More</summary>
+
 - 🔥 **[2025.02.18]** Supports the AIME25 dataset, which contains 15 questions (Grok3 scored 93 on this dataset).
 - 🔥 **[2025.02.13]** Added support for evaluating DeepSeek distilled models, including AIME24, MATH-500, and GPQA-Diamond datasets，refer to [best practice](https://evalscope.readthedocs.io/en/latest/best_practice/deepseek_r1_distill.html); Added support for specifying the `eval_batch_size` parameter to accelerate model evaluation.
 - 🔥 **[2025.01.20]** Support for visualizing evaluation results, including single model evaluation results and multi-model comparison, refer to the [📖 Visualizing Evaluation Results](https://evalscope.readthedocs.io/en/latest/get_started/visualization.html) for more details; Added [`iquiz`](https://modelscope.cn/datasets/AI-ModelScope/IQuiz/summary) evaluation example, evaluating the IQ and EQ of the model.
 - 🔥 **[2025.01.07]** Native backend: Support for model API evaluation is now available. Refer to the [📖 Model API Evaluation Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html#api) for more details. Additionally, support for the `ifeval` evaluation benchmark has been added.
-<details><summary>More</summary>
-
 - 🔥🔥 **[2024.12.31]** Support for adding benchmark evaluations, refer to the [📖 Benchmark Evaluation Addition Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/add_benchmark.html); support for custom mixed dataset evaluations, allowing for more comprehensive model evaluations with less data, refer to the [📖 Mixed Dataset Evaluation Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/collection/index.html).
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.
 - 🔥 **[2024.11.26]** The model inference service performance evaluator has been completely refactored: it now supports local inference service startup and Speed Benchmark; asynchronous call error handling has been optimized. For more details, refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/index.html).
@@ -572,10 +482,17 @@ Speed Benchmark Results:
 EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/index.html)
 
 
-## 🏟️ Arena Mode
-The Arena mode allows multiple candidate models to be evaluated through pairwise battles, and can choose to use the AI Enhanced Auto-Reviewer (AAR) automatic evaluation process or manual evaluation to obtain the evaluation report.
+## ⚔️ Arena Mode
 
-Refer to: Arena Mode [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html)
+Arena mode allows you to configure multiple candidate models and specify a baseline model. Evaluation is performed by pairwise battles between each candidate model and the baseline model, with the final output including each model's win rate and ranking. This method is suitable for comparative evaluation among multiple models, providing an intuitive reflection of each model's strengths and weaknesses. Refer to: Arena Mode [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html)
+
+```text
+Model           WinRate (%)  CI (%)
+------------  -------------  ---------------
+qwen2.5-72b            69.3  (-13.3 / +12.2)
+qwen2.5-7b             50    (+0.0 / +0.0)
+qwen2.5-0.5b            4.7  (-2.5 / +4.4)
+```
 
 ## 👷‍♂️ Contribution
 
@@ -601,7 +518,7 @@ EvalScope, as the official evaluation tool of [ModelScope](https://modelscope.cn
 - [ ] Distributed evaluating
 - [x] Multi-modal evaluation
 - [ ] Benchmarks
-  - [ ] GAIA
+  - [x] BFCL-v3
   - [x] GPQA
   - [x] MBPP
 

{evalscope-0.16.2 → evalscope-0.17.0}/README.md

@@ -70,24 +70,33 @@ EvalScope is not merely an evaluation tool; it is a valuable ally in your model
 Below is the overall architecture diagram of EvalScope:
 
 <p align="center">
-  <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
+  <img src="https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/doc/EvalScope%E6%9E%B6%E6%9E%84%E5%9B%BE.png" width="70%">
   <br>EvalScope Framework.
 </p>
 
 <details><summary>Framework Description</summary>
 
 The architecture includes the following modules:
-1. **Model Adapter**: The model adapter is used to convert the outputs of specific models into the format required by the framework, supporting both API call models and locally run models.
-2. **Data Adapter**: The data adapter is responsible for converting and processing input data to meet various evaluation needs and formats.
-3. **Evaluation Backend**:
-    - **Native**: EvalScope’s own **default evaluation framework**, supporting various evaluation modes, including single model evaluation, arena mode, baseline model comparison mode, etc.
-    - **OpenCompass**: Supports [OpenCompass](https://github.com/open-compass/opencompass) as the evaluation backend, providing advanced encapsulation and task simplification, allowing you to submit tasks for evaluation more easily.
-    - **VLMEvalKit**: Supports [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) as the evaluation backend, enabling easy initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
-    - **RAGEval**: Supports RAG evaluation, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
-    - **ThirdParty**: Other third-party evaluation tasks, such as ToolBench.
-4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
-5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
-6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
+1. Input Layer
+- **Model Sources**: API models (OpenAI API), local models (ModelScope)
+- **Datasets**: Standard evaluation benchmarks (MMLU/GSM8k, etc.), custom data (MCQ/QA)
+
+2. Core Functions
+- **Multi-backend Evaluation**
+   - Native backends: Unified evaluation for LLM/VLM/Embedding/T2I models
+   - Integrated frameworks: OpenCompass/MTEB/VLMEvalKit/RAGAS
+
+- **Performance Monitoring**
+   - Model plugins: Supports various model service APIs
+   - Data plugins: Supports multiple data formats
+   - Metric tracking: TTFT/TPOP/Stability and other metrics
+
+- **Tool Extensions**
+   - Integration: Tool-Bench/Needle-in-a-Haystack/BFCL-v3
+
+3. Output Layer
+- **Structured Reports**: Supports JSON/Tables/Logs
+- **Visualization Platforms**: Supports Gradio/Wandb/SwanLab
 
 </details>
 
@@ -102,7 +111,9 @@ Please scan the QR code below to join our community groups:
 
 ## 🎉 News
 
-- 🔥 **[2025.06.19]** Added support for the BFCL-v3 benchmark, designed to evaluate model function-calling capabilities across various scenarios. For more information, refer to the [documentation](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html).
+- 🔥 **[2025.07.03]** Refactored Arena Mode: now supports custom model battles, outputs a model leaderboard, and provides battle result visualization. See [reference](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html) for details.
+- 🔥 **[2025.06.28]** Optimized custom dataset evaluation: now supports evaluation without reference answers. Enhanced LLM judge usage, with built-in modes for "scoring directly without reference answers" and "checking answer consistency with reference answers". See [reference](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/llm.html#qa) for details.
+- 🔥 **[2025.06.19]** Added support for the [BFCL-v3](https://modelscope.cn/datasets/AI-ModelScope/bfcl_v3) benchmark, designed to evaluate model function-calling capabilities across various scenarios. For more information, refer to the [documentation](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html).
 - 🔥 **[2025.06.02]** Added support for the Needle-in-a-Haystack test. Simply specify `needle_haystack` to conduct the test, and a corresponding heatmap will be generated in the `outputs/reports` folder, providing a visual representation of the model's performance. Refer to the [documentation](https://evalscope.readthedocs.io/en/latest/third_party/needle_haystack.html) for more details.
 - 🔥 **[2025.05.29]** Added support for two long document evaluation benchmarks: [DocMath](https://modelscope.cn/datasets/yale-nlp/DocMath-Eval/summary) and [FRAMES](https://modelscope.cn/datasets/iic/frames/summary). For usage guidelines, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html).
 - 🔥 **[2025.05.16]** Model service performance stress testing now supports setting various levels of concurrency and outputs a performance test report. [Reference example](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/quick_start.html#id3).
@@ -120,12 +131,12 @@ Please scan the QR code below to join our community groups:
 - 🔥 **[2025.03.03]** Added support for evaluating the IQ and EQ of models. Refer to [📖 Best Practices for IQ and EQ Evaluation](https://evalscope.readthedocs.io/en/latest/best_practice/iquiz.html) to find out how smart your AI is!
 - 🔥 **[2025.02.27]** Added support for evaluating the reasoning efficiency of models. Refer to [📖 Best Practices for Evaluating Thinking Efficiency](https://evalscope.readthedocs.io/en/latest/best_practice/think_eval.html). This implementation is inspired by the works [Overthinking](https://doi.org/10.48550/arXiv.2412.21187) and [Underthinking](https://doi.org/10.48550/arXiv.2501.18585).
 - 🔥 **[2025.02.25]** Added support for two model inference-related evaluation benchmarks: [MuSR](https://modelscope.cn/datasets/AI-ModelScope/MuSR) and [ProcessBench](https://www.modelscope.cn/datasets/Qwen/ProcessBench/summary). To use them, simply specify `musr` and `process_bench` respectively in the datasets parameter.
+<details><summary>More</summary>
+
 - 🔥 **[2025.02.18]** Supports the AIME25 dataset, which contains 15 questions (Grok3 scored 93 on this dataset).
 - 🔥 **[2025.02.13]** Added support for evaluating DeepSeek distilled models, including AIME24, MATH-500, and GPQA-Diamond datasets，refer to [best practice](https://evalscope.readthedocs.io/en/latest/best_practice/deepseek_r1_distill.html); Added support for specifying the `eval_batch_size` parameter to accelerate model evaluation.
 - 🔥 **[2025.01.20]** Support for visualizing evaluation results, including single model evaluation results and multi-model comparison, refer to the [📖 Visualizing Evaluation Results](https://evalscope.readthedocs.io/en/latest/get_started/visualization.html) for more details; Added [`iquiz`](https://modelscope.cn/datasets/AI-ModelScope/IQuiz/summary) evaluation example, evaluating the IQ and EQ of the model.
 - 🔥 **[2025.01.07]** Native backend: Support for model API evaluation is now available. Refer to the [📖 Model API Evaluation Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html#api) for more details. Additionally, support for the `ifeval` evaluation benchmark has been added.
-<details><summary>More</summary>
-
 - 🔥🔥 **[2024.12.31]** Support for adding benchmark evaluations, refer to the [📖 Benchmark Evaluation Addition Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/add_benchmark.html); support for custom mixed dataset evaluations, allowing for more comprehensive model evaluations with less data, refer to the [📖 Mixed Dataset Evaluation Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/collection/index.html).
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.
 - 🔥 **[2024.11.26]** The model inference service performance evaluator has been completely refactored: it now supports local inference service startup and Speed Benchmark; asynchronous call error handling has been optimized. For more details, refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/index.html).
@@ -444,10 +455,17 @@ Speed Benchmark Results:
 EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/index.html)
 
 
-## 🏟️ Arena Mode
-The Arena mode allows multiple candidate models to be evaluated through pairwise battles, and can choose to use the AI Enhanced Auto-Reviewer (AAR) automatic evaluation process or manual evaluation to obtain the evaluation report.
+## ⚔️ Arena Mode
+
+Arena mode allows you to configure multiple candidate models and specify a baseline model. Evaluation is performed by pairwise battles between each candidate model and the baseline model, with the final output including each model's win rate and ranking. This method is suitable for comparative evaluation among multiple models, providing an intuitive reflection of each model's strengths and weaknesses. Refer to: Arena Mode [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html)
 
-Refer to: Arena Mode [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html)
+```text
+Model           WinRate (%)  CI (%)
+------------  -------------  ---------------
+qwen2.5-72b            69.3  (-13.3 / +12.2)
+qwen2.5-7b             50    (+0.0 / +0.0)
+qwen2.5-0.5b            4.7  (-2.5 / +4.4)
+```
 
 ## 👷‍♂️ Contribution
 
@@ -473,7 +491,7 @@ EvalScope, as the official evaluation tool of [ModelScope](https://modelscope.cn
 - [ ] Distributed evaluating
 - [x] Multi-modal evaluation
 - [ ] Benchmarks
-  - [ ] GAIA
+  - [x] BFCL-v3
   - [x] GPQA
   - [x] MBPP
 

evalscope-0.17.0/evalscope/app/app.py

@@ -0,0 +1,35 @@
+"""
+Main application module for the Evalscope dashboard.
+"""
+import argparse
+
+from evalscope.utils.logger import configure_logging
+from .arguments import add_argument
+from .ui import create_app_ui
+
+
+def create_app(args: argparse.Namespace):
+    """
+    Create and launch the Evalscope dashboard application.
+
+    Args:
+        args: Command line arguments.
+    """
+    configure_logging(debug=args.debug)
+
+    demo = create_app_ui(args)
+
+    demo.launch(
+        share=args.share,
+        server_name=args.server_name,
+        server_port=args.server_port,
+        debug=args.debug,
+        allowed_paths=args.allowed_paths,
+    )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    add_argument(parser)
+    args = parser.parse_args()
+    create_app(args)

{evalscope-0.16.2 → evalscope-0.17.0}/evalscope/app/constants.py

@@ -2,6 +2,7 @@ PLOTLY_THEME = 'plotly_dark'
 REPORT_TOKEN = '@@'
 MODEL_TOKEN = '::'
 DATASET_TOKEN = ', '
+DEFAULT_BAR_WIDTH = 0.2
 LATEX_DELIMITERS = [{
     'left': '$$',
     'right': '$$',

evalscope-0.17.0/evalscope/app/ui/__init__.py

@@ -0,0 +1,20 @@
+"""
+UI components for the Evalscope dashboard.
+"""
+from .app_ui import create_app_ui
+from .multi_model import MultiModelComponents, create_multi_model_tab
+from .sidebar import SidebarComponents, create_sidebar
+from .single_model import SingleModelComponents, create_single_model_tab
+from .visualization import VisualizationComponents, create_visualization
+
+__all__ = [
+    'create_app_ui',
+    'SidebarComponents',
+    'create_sidebar',
+    'VisualizationComponents',
+    'create_visualization',
+    'SingleModelComponents',
+    'create_single_model_tab',
+    'MultiModelComponents',
+    'create_multi_model_tab',
+]

evalscope-0.17.0/evalscope/app/ui/app_ui.py

@@ -0,0 +1,52 @@
+"""
+Main UI application for the Evalscope dashboard.
+"""
+import argparse
+import gradio as gr
+
+from evalscope.version import __version__
+from ..utils.localization import get_app_locale
+from .sidebar import create_sidebar
+from .visualization import create_visualization
+
+
+def create_app_ui(args: argparse.Namespace):
+    lang = args.lang
+    locale_dict = get_app_locale(lang)
+
+    with gr.Blocks(title='Evalscope Dashboard') as demo:
+        gr.HTML(f'<h1 style="text-align: left;">{locale_dict["title"]} (v{__version__})</h1>')
+        with gr.Row():
+            with gr.Column(scale=0, min_width=35):
+                toggle_btn = gr.Button('<')
+            with gr.Column(scale=1):
+                gr.HTML(f'<h3 style="text-align: left;">{locale_dict["star_beggar"]}</h3>')
+
+        with gr.Row():
+            with gr.Column(scale=1) as sidebar_column:
+                sidebar_visible = gr.State(True)
+                sidebar = create_sidebar(args.outputs, lang)
+
+            with gr.Column(scale=5):
+                visualization = create_visualization(sidebar, lang)
+
+        @sidebar.load_btn.click(
+            inputs=[sidebar.reports_dropdown],
+            outputs=[visualization.single_model.report_name, visualization.multi_model.multi_report_name])
+        def update_displays(reports_dropdown):
+            if not reports_dropdown:
+                gr.Warning(locale_dict['note'], duration=3)
+                return gr.skip()
+
+            return (
+                gr.update(choices=reports_dropdown, value=reports_dropdown[0]),  # update single model dropdown
+                gr.update(choices=reports_dropdown, value=reports_dropdown)  # update multi model dropdown
+            )
+
+        @toggle_btn.click(inputs=[sidebar_visible], outputs=[sidebar_column, sidebar_visible, toggle_btn])
+        def toggle_sidebar(visible):
+            new_visible = not visible
+            text = '<' if new_visible else '>'
+            return gr.update(visible=new_visible), new_visible, gr.update(value=text)
+
+    return demo