evalscope 0.5.5rc1__tar.gz → 0.6.0rc0__tar.gz

{evalscope-0.5.5rc1 → evalscope-0.6.0rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.5.5rc1
+Version: 0.6.0rc0
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -19,22 +19,22 @@ Requires-Dist: torch
 Requires-Dist: absl-py
 Requires-Dist: accelerate
 Requires-Dist: cachetools
-Requires-Dist: datasets<3.0.0,>=2.18.0
+Requires-Dist: datasets<=3.0.1,>=3.0.0
 Requires-Dist: editdistance
 Requires-Dist: jsonlines
 Requires-Dist: matplotlib
 Requires-Dist: modelscope[framework]
-Requires-Dist: nltk
+Requires-Dist: nltk>=3.9
 Requires-Dist: openai
 Requires-Dist: pandas
 Requires-Dist: plotly
-Requires-Dist: pyarrow
+Requires-Dist: pyarrow<=17.0.0
 Requires-Dist: pympler
 Requires-Dist: pyyaml
 Requires-Dist: regex
 Requires-Dist: requests
 Requires-Dist: requests-toolbelt
-Requires-Dist: rouge-score
+Requires-Dist: rouge-score>=0.1.0
 Requires-Dist: sacrebleu
 Requires-Dist: scikit-learn
 Requires-Dist: seaborn
@@ -48,9 +48,13 @@ Requires-Dist: transformers_stream_generator
 Requires-Dist: jieba
 Requires-Dist: rouge-chinese
 Provides-Extra: opencompass
-Requires-Dist: ms-opencompass>=0.1.1; extra == "opencompass"
+Requires-Dist: ms-opencompass>=0.1.3; extra == "opencompass"
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.5; extra == "vlmeval"
+Provides-Extra: rag
+Requires-Dist: mteb==1.19.4; extra == "rag"
+Requires-Dist: ragas==0.2.3; extra == "rag"
+Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: inner
 Requires-Dist: absl-py; extra == "inner"
 Requires-Dist: accelerate; extra == "inner"
@@ -83,22 +87,22 @@ Requires-Dist: torch; extra == "all"
 Requires-Dist: absl-py; extra == "all"
 Requires-Dist: accelerate; extra == "all"
 Requires-Dist: cachetools; extra == "all"
-Requires-Dist: datasets<3.0.0,>=2.18.0; extra == "all"
+Requires-Dist: datasets<=3.0.1,>=3.0.0; extra == "all"
 Requires-Dist: editdistance; extra == "all"
 Requires-Dist: jsonlines; extra == "all"
 Requires-Dist: matplotlib; extra == "all"
 Requires-Dist: modelscope[framework]; extra == "all"
-Requires-Dist: nltk; extra == "all"
+Requires-Dist: nltk>=3.9; extra == "all"
 Requires-Dist: openai; extra == "all"
 Requires-Dist: pandas; extra == "all"
 Requires-Dist: plotly; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: pyarrow<=17.0.0; extra == "all"
 Requires-Dist: pympler; extra == "all"
 Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: regex; extra == "all"
 Requires-Dist: requests; extra == "all"
 Requires-Dist: requests-toolbelt; extra == "all"
-Requires-Dist: rouge-score; extra == "all"
+Requires-Dist: rouge-score>=0.1.0; extra == "all"
 Requires-Dist: sacrebleu; extra == "all"
 Requires-Dist: scikit-learn; extra == "all"
 Requires-Dist: seaborn; extra == "all"
@@ -111,14 +115,20 @@ Requires-Dist: transformers>=4.33; extra == "all"
 Requires-Dist: transformers_stream_generator; extra == "all"
 Requires-Dist: jieba; extra == "all"
 Requires-Dist: rouge-chinese; extra == "all"
-Requires-Dist: ms-opencompass>=0.1.1; extra == "all"
+Requires-Dist: ms-opencompass>=0.1.3; extra == "all"
 Requires-Dist: ms-vlmeval>=0.0.5; extra == "all"
+Requires-Dist: mteb==1.19.4; extra == "all"
+Requires-Dist: ragas==0.2.3; extra == "all"
+Requires-Dist: webdataset>0.2.0; extra == "all"
 
-English | [简体中文](README_zh.md)
 
 
 ![](docs/en/_static/images/evalscope_logo.png)
 
+<p align="center">
+    English | <a href="README_zh.md">简体中文</a>
+</p>
+
 <p align="center">
 <a href="https://badge.fury.io/py/evalscope"><img src="https://badge.fury.io/py/evalscope.svg" alt="PyPI version" height="18"></a>
 <a href="https://pypi.org/project/evalscope"><img alt="PyPI - Downloads" src="https://static.pepy.tech/badge/evalscope">
@@ -127,7 +137,7 @@ English | [简体中文](README_zh.md)
     <img src='https://readthedocs.org/projects/evalscope-en/badge/?version=latest' alt='Documentation Status' />
 </a>
 <br>
- <a href="https://evalscope.readthedocs.io/en/latest/"><span style="font-size: 16px;">📖 Documents</span></a> &nbsp | &nbsp<a href="https://evalscope.readthedocs.io/zh-cn/latest/"><span style="font-size: 16px;"> 📖  中文文档</span></a>
+ <a href="https://evalscope.readthedocs.io/en/latest/">📖 Documents</a>
 <p>
 
 
@@ -141,34 +151,15 @@ English | [简体中文](README_zh.md)
 - [Offline Evaluation](#offline-evaluation)
 - [Arena Mode](#arena-mode)
 - [Model Serving Performance Evaluation](#Model-Serving-Performance-Evaluation)
-- [Leaderboard](#leaderboard)
-
-## 📝 Introduction
-
-Large Model (including Large Language Models, Multi-modal Large Language Models) evaluation has become a critical process for assessing and improving LLMs. To better support the evaluation of large models, we propose the EvalScope framework.
 
-### Framework Features
-- **Benchmark Datasets**: Preloaded with several commonly used test benchmarks, including MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, HumanEval, etc.
-- **Evaluation Metrics**: Implements various commonly used evaluation metrics.
-- **Model Access**: A unified model access mechanism that is compatible with the Generate and Chat interfaces of multiple model families.
-- **Automated Evaluation**: Includes automatic evaluation of objective questions and complex task evaluation using expert models.
-- **Evaluation Reports**: Automatically generates evaluation reports.
-- **Arena Mode**: Used for comparisons between models and objective evaluation of models, supporting various evaluation modes, including:
-  - **Single mode**: Scoring a single model.
-  - **Pairwise-baseline mode**: Comparing against a baseline model.
-  - **Pairwise (all) mode**: Pairwise comparison among all models.
-- **Visualization Tools**: Provides intuitive displays of evaluation results.
-- **Model Performance Evaluation**: Offers a performance testing tool for model inference services and detailed statistics, see [Model Performance Evaluation Documentation](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test.html).
-- **OpenCompass Integration**: Supports OpenCompass as the evaluation backend, providing advanced encapsulation and task simplification, allowing for easier task submission for evaluation.
-- **VLMEvalKit Integration**: Supports VLMEvalKit as the evaluation backend, facilitating the initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
-- **Full-Link Support**: Through seamless integration with the [ms-swift](https://github.com/modelscope/ms-swift) training framework, provides a one-stop development process for model training, model deployment, model evaluation, and report viewing, enhancing user development efficiency.
 
+## 📝 Introduction
 
-<details><summary>Overall Architecture</summary>
+EvalScope is the official model evaluation and performance benchmarking framework launched by the [ModelScope](https://modelscope.cn/) community. It comes with built-in common benchmarks and evaluation metrics, such as MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, and HumanEval. EvalScope supports various types of model evaluations, including LLMs, multimodal LLMs, embedding models, and reranker models. It is also applicable to multiple evaluation scenarios, such as end-to-end RAG evaluation, arena mode, and model inference performance stress testing. Moreover, with the seamless integration of the ms-swift training framework, evaluations can be initiated with a single click, providing full end-to-end support from model training to evaluation 🚀
 
 <p align="center">
   <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
-  <br>Fig 1. EvalScope Framework.
+  <br>EvalScope Framework.
 </p>
 
 The architecture includes the following modules:
@@ -178,14 +169,17 @@ The architecture includes the following modules:
     - **Native**: EvalScope’s own **default evaluation framework**, supporting various evaluation modes, including single model evaluation, arena mode, baseline model comparison mode, etc.
     - **OpenCompass**: Supports [OpenCompass](https://github.com/open-compass/opencompass) as the evaluation backend, providing advanced encapsulation and task simplification, allowing you to submit tasks for evaluation more easily.
     - **VLMEvalKit**: Supports [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) as the evaluation backend, enabling easy initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
+    - **RAGEval**: Supports RAG evaluation, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
     - **ThirdParty**: Other third-party evaluation tasks, such as ToolBench.
 4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
-</details>
 
 
 ## 🎉 News
+- 🔥 **[2024.10.31]** The best practice for evaluating Multimodal-RAG has been updated, please check the [📖 Blog](https://evalscope.readthedocs.io/zh-cn/latest/blog/RAG/multimodal_RAG.html#multimodal-rag) for more details.
+- 🔥 **[2024.10.23]** Supports multimodal RAG evaluation, including the assessment of image-text retrieval using [CLIP_Benchmark](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/clip_benchmark.html), and extends [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html) to support end-to-end multimodal metrics evaluation.
+- 🔥 **[2024.10.8]** Support for RAG evaluation, including independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
 - 🔥 **[2024.09.18]** Our documentation has been updated to include a blog module, featuring some technical research and discussions related to evaluations. We invite you to [📖 read it](https://evalscope.readthedocs.io/en/refact_readme/blog/index.html).
 - 🔥 **[2024.09.12]** Support for LongWriter evaluation, which supports 10,000+ word generation. You can use the benchmark [LongBench-Write](evalscope/third_party/longbench_write/README.md) to measure the long output quality as well as the output length.
 - 🔥 **[2024.08.30]** Support for custom dataset evaluations, including text datasets and multimodal image-text datasets.
@@ -356,9 +350,10 @@ run_task(task_cfg=your_task_cfg)
 ## Evaluation Backend
 EvalScope supports using third-party evaluation frameworks to initiate evaluation tasks, which we call Evaluation Backend. Currently supported Evaluation Backend includes:
 - **Native**: EvalScope's own **default evaluation framework**, supporting various evaluation modes including single model evaluation, arena mode, and baseline model comparison mode.
-- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/opencompass_backend.html)
-- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/vlmevalkit_backend.html)
-- **ThirdParty**: The third-party task, e.g. [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html), you can contribute your own evaluation task to EvalScope as third-party backend.
+- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/opencompass_backend.html)
+- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/vlmevalkit_backend.html)
+- **RAGEval**: Initiate RAG evaluation tasks through EvalScope, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html): [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/index.html)
+- **ThirdParty**: Third-party evaluation tasks, such as [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html) and [LongBench-Write](https://evalscope.readthedocs.io/en/latest/third_party/longwriter.html).
 
 ## Custom Dataset Evaluation
 EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset.html)
@@ -380,13 +375,10 @@ A stress testing tool that focuses on large language models and can be customize
 Refer to : Model Serving Performance Evaluation [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test.html)
 
 
-## Leaderboard
-The LLM Leaderboard aims to provide an objective and comprehensive evaluation standard and platform to help researchers and developers understand and compare the performance of models on various tasks on ModelScope.
-
-Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
-
 
 ## TO-DO List
+- [x] RAG evaluation
+- [x] VLM evaluation
 - [x] Agents evaluation
 - [x] vLLM
 - [ ] Distributed evaluating
@@ -398,3 +390,7 @@ Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
 - [ ] Auto-reviewer
   - [ ] Qwen-max
 
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=modelscope/evalscope&type=Date)](https://star-history.com/#modelscope/evalscope&Date)

{evalscope-0.5.5rc1 → evalscope-0.6.0rc0}/README.md

@@ -1,8 +1,11 @@
-English | [简体中文](README_zh.md)
 
 
 ![](docs/en/_static/images/evalscope_logo.png)
 
+<p align="center">
+    English | <a href="README_zh.md">简体中文</a>
+</p>
+
 <p align="center">
 <a href="https://badge.fury.io/py/evalscope"><img src="https://badge.fury.io/py/evalscope.svg" alt="PyPI version" height="18"></a>
 <a href="https://pypi.org/project/evalscope"><img alt="PyPI - Downloads" src="https://static.pepy.tech/badge/evalscope">
@@ -11,7 +14,7 @@ English | [简体中文](README_zh.md)
     <img src='https://readthedocs.org/projects/evalscope-en/badge/?version=latest' alt='Documentation Status' />
 </a>
 <br>
- <a href="https://evalscope.readthedocs.io/en/latest/"><span style="font-size: 16px;">📖 Documents</span></a> &nbsp | &nbsp<a href="https://evalscope.readthedocs.io/zh-cn/latest/"><span style="font-size: 16px;"> 📖  中文文档</span></a>
+ <a href="https://evalscope.readthedocs.io/en/latest/">📖 Documents</a>
 <p>
 
 
@@ -25,34 +28,15 @@ English | [简体中文](README_zh.md)
 - [Offline Evaluation](#offline-evaluation)
 - [Arena Mode](#arena-mode)
 - [Model Serving Performance Evaluation](#Model-Serving-Performance-Evaluation)
-- [Leaderboard](#leaderboard)
-
-## 📝 Introduction
-
-Large Model (including Large Language Models, Multi-modal Large Language Models) evaluation has become a critical process for assessing and improving LLMs. To better support the evaluation of large models, we propose the EvalScope framework.
 
-### Framework Features
-- **Benchmark Datasets**: Preloaded with several commonly used test benchmarks, including MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, HumanEval, etc.
-- **Evaluation Metrics**: Implements various commonly used evaluation metrics.
-- **Model Access**: A unified model access mechanism that is compatible with the Generate and Chat interfaces of multiple model families.
-- **Automated Evaluation**: Includes automatic evaluation of objective questions and complex task evaluation using expert models.
-- **Evaluation Reports**: Automatically generates evaluation reports.
-- **Arena Mode**: Used for comparisons between models and objective evaluation of models, supporting various evaluation modes, including:
-  - **Single mode**: Scoring a single model.
-  - **Pairwise-baseline mode**: Comparing against a baseline model.
-  - **Pairwise (all) mode**: Pairwise comparison among all models.
-- **Visualization Tools**: Provides intuitive displays of evaluation results.
-- **Model Performance Evaluation**: Offers a performance testing tool for model inference services and detailed statistics, see [Model Performance Evaluation Documentation](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test.html).
-- **OpenCompass Integration**: Supports OpenCompass as the evaluation backend, providing advanced encapsulation and task simplification, allowing for easier task submission for evaluation.
-- **VLMEvalKit Integration**: Supports VLMEvalKit as the evaluation backend, facilitating the initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
-- **Full-Link Support**: Through seamless integration with the [ms-swift](https://github.com/modelscope/ms-swift) training framework, provides a one-stop development process for model training, model deployment, model evaluation, and report viewing, enhancing user development efficiency.
 
+## 📝 Introduction
 
-<details><summary>Overall Architecture</summary>
+EvalScope is the official model evaluation and performance benchmarking framework launched by the [ModelScope](https://modelscope.cn/) community. It comes with built-in common benchmarks and evaluation metrics, such as MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, and HumanEval. EvalScope supports various types of model evaluations, including LLMs, multimodal LLMs, embedding models, and reranker models. It is also applicable to multiple evaluation scenarios, such as end-to-end RAG evaluation, arena mode, and model inference performance stress testing. Moreover, with the seamless integration of the ms-swift training framework, evaluations can be initiated with a single click, providing full end-to-end support from model training to evaluation 🚀
 
 <p align="center">
   <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
-  <br>Fig 1. EvalScope Framework.
+  <br>EvalScope Framework.
 </p>
 
 The architecture includes the following modules:
@@ -62,14 +46,17 @@ The architecture includes the following modules:
     - **Native**: EvalScope’s own **default evaluation framework**, supporting various evaluation modes, including single model evaluation, arena mode, baseline model comparison mode, etc.
     - **OpenCompass**: Supports [OpenCompass](https://github.com/open-compass/opencompass) as the evaluation backend, providing advanced encapsulation and task simplification, allowing you to submit tasks for evaluation more easily.
     - **VLMEvalKit**: Supports [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) as the evaluation backend, enabling easy initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
+    - **RAGEval**: Supports RAG evaluation, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
     - **ThirdParty**: Other third-party evaluation tasks, such as ToolBench.
 4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
-</details>
 
 
 ## 🎉 News
+- 🔥 **[2024.10.31]** The best practice for evaluating Multimodal-RAG has been updated, please check the [📖 Blog](https://evalscope.readthedocs.io/zh-cn/latest/blog/RAG/multimodal_RAG.html#multimodal-rag) for more details.
+- 🔥 **[2024.10.23]** Supports multimodal RAG evaluation, including the assessment of image-text retrieval using [CLIP_Benchmark](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/clip_benchmark.html), and extends [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html) to support end-to-end multimodal metrics evaluation.
+- 🔥 **[2024.10.8]** Support for RAG evaluation, including independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
 - 🔥 **[2024.09.18]** Our documentation has been updated to include a blog module, featuring some technical research and discussions related to evaluations. We invite you to [📖 read it](https://evalscope.readthedocs.io/en/refact_readme/blog/index.html).
 - 🔥 **[2024.09.12]** Support for LongWriter evaluation, which supports 10,000+ word generation. You can use the benchmark [LongBench-Write](evalscope/third_party/longbench_write/README.md) to measure the long output quality as well as the output length.
 - 🔥 **[2024.08.30]** Support for custom dataset evaluations, including text datasets and multimodal image-text datasets.
@@ -240,9 +227,10 @@ run_task(task_cfg=your_task_cfg)
 ## Evaluation Backend
 EvalScope supports using third-party evaluation frameworks to initiate evaluation tasks, which we call Evaluation Backend. Currently supported Evaluation Backend includes:
 - **Native**: EvalScope's own **default evaluation framework**, supporting various evaluation modes including single model evaluation, arena mode, and baseline model comparison mode.
-- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/opencompass_backend.html)
-- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/vlmevalkit_backend.html)
-- **ThirdParty**: The third-party task, e.g. [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html), you can contribute your own evaluation task to EvalScope as third-party backend.
+- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/opencompass_backend.html)
+- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/vlmevalkit_backend.html)
+- **RAGEval**: Initiate RAG evaluation tasks through EvalScope, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html): [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/index.html)
+- **ThirdParty**: Third-party evaluation tasks, such as [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html) and [LongBench-Write](https://evalscope.readthedocs.io/en/latest/third_party/longwriter.html).
 
 ## Custom Dataset Evaluation
 EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset.html)
@@ -264,13 +252,10 @@ A stress testing tool that focuses on large language models and can be customize
 Refer to : Model Serving Performance Evaluation [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test.html)
 
 
-## Leaderboard
-The LLM Leaderboard aims to provide an objective and comprehensive evaluation standard and platform to help researchers and developers understand and compare the performance of models on various tasks on ModelScope.
-
-Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
-
 
 ## TO-DO List
+- [x] RAG evaluation
+- [x] VLM evaluation
 - [x] Agents evaluation
 - [x] vLLM
 - [ ] Distributed evaluating
@@ -282,3 +267,7 @@ Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
 - [ ] Auto-reviewer
   - [ ] Qwen-max
 
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=modelscope/evalscope&type=Date)](https://star-history.com/#modelscope/evalscope&Date)

{evalscope-0.5.5rc1 → evalscope-0.6.0rc0}/evalscope/backend/opencompass/tasks/eval_datasets.py

@@ -50,6 +50,7 @@ with read_base():
     from opencompass.configs.datasets.nq.nq_gen_c788f6 import nq_datasets
     from opencompass.configs.datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
     from opencompass.configs.datasets.cmb.cmb_gen_dfb5c4 import cmb_datasets
+    from opencompass.configs.datasets.cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
     from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
 
     # Note: to be supported

evalscope-0.6.0rc0/evalscope/backend/rag_eval/__init__.py

@@ -0,0 +1,4 @@
+from evalscope.backend.rag_eval.utils.embedding import EmbeddingModel
+from evalscope.backend.rag_eval.utils.llm import LLM, LocalLLM, ChatOpenAI
+from evalscope.backend.rag_eval.utils.clip import VisionModel
+from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager

evalscope-0.6.0rc0/evalscope/backend/rag_eval/backend_manager.py

@@ -0,0 +1,80 @@
+import os
+from typing import Optional, Union
+from evalscope.utils import is_module_installed, get_valid_list
+from evalscope.backend.base import BackendManager
+from evalscope.utils.logger import get_logger
+
+
+logger = get_logger()
+
+
+class RAGEvalBackendManager(BackendManager):
+    def __init__(self, config: Union[str, dict], **kwargs):
+        """BackendManager for VLM Evaluation Kit
+
+        Args:
+            config (Union[str, dict]): the configuration yaml-file or the configuration dictionary
+        """
+        super().__init__(config, **kwargs)
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f"Check `{module_name}` Installed")
+        else:
+            logger.error(f"Please install `{module_name}` first")
+
+    @staticmethod
+    def run_mteb(model_args, eval_args):
+        from evalscope.backend.rag_eval.cmteb import ModelArguments, EvalArguments
+        from evalscope.backend.rag_eval.cmteb import one_stage_eval, two_stage_eval
+
+        if len(model_args) > 2:
+            raise ValueError("Not support multiple models yet")
+
+        # Convert arguments to dictionary
+        model_args_list = [ModelArguments(**args).to_dict() for args in model_args]
+        eval_args = EvalArguments(**eval_args).to_dict()
+
+        if len(model_args_list) == 1:
+            one_stage_eval(model_args_list[0], eval_args)
+        else:  # len(model_args_list) == 2
+            two_stage_eval(model_args_list[0], model_args_list[1], eval_args)
+
+    @staticmethod
+    def run_ragas(testset_args, eval_args):
+        from evalscope.backend.rag_eval.ragas import rag_eval
+        from evalscope.backend.rag_eval.ragas.tasks import generate_testset
+        from evalscope.backend.rag_eval.ragas import (
+            TestsetGenerationArguments,
+            EvaluationArguments,
+        )
+
+        if testset_args is not None:
+            generate_testset(TestsetGenerationArguments(**testset_args))
+        if eval_args is not None:
+            rag_eval(EvaluationArguments(**eval_args))
+
+    @staticmethod
+    def run_clip_benchmark(args):
+        from evalscope.backend.rag_eval.clip_benchmark import Arguments, evaluate
+
+        evaluate(Arguments(**args))
+
+    def run(self, *args, **kwargs):
+        tool = self.config_d.pop("tool")
+        if tool.lower() == "mteb":
+            self._check_env("mteb")
+            model_args = self.config_d["model"]
+            eval_args = self.config_d["eval"]
+            self.run_mteb(model_args, eval_args)
+        elif tool.lower() == "ragas":
+            self._check_env("ragas")
+            testset_args = self.config_d.get("testset_generation", None)
+            eval_args = self.config_d.get("eval", None)
+            self.run_ragas(testset_args, eval_args)
+        elif tool.lower() == "clip_benchmark":
+            self._check_env("webdataset")
+            self.run_clip_benchmark(self.config_d["eval"])
+        else:
+            raise ValueError(f"Unknown tool: {tool}")

evalscope-0.6.0rc0/evalscope/backend/rag_eval/clip_benchmark/__init__.py

@@ -0,0 +1,2 @@
+from evalscope.backend.rag_eval.clip_benchmark.task_template import evaluate
+from evalscope.backend.rag_eval.clip_benchmark.arguments import Arguments

evalscope-0.6.0rc0/evalscope/backend/rag_eval/clip_benchmark/arguments.py

@@ -0,0 +1,34 @@
+from dataclasses import dataclass, field
+from typing import List, Dict
+
+
+@dataclass
+class Arguments:
+    # fmt: off
+    """
+    A dataclass to store and manage the arguments for the model configuration and data processing.
+    """
+    """
+    For CLIP model support, you can use the following fields:
+        model_name: str
+        revision: str = "master"
+        hub: str = "modelscope"
+        
+    For API VLM model support, you can use the following fields, (image caption only):
+        model_name="gpt-4o-mini"
+        api_base: str = "",
+        api_key: Optional[str] = None
+        prompt: str = None
+    """
+    models: List[Dict] = field(default_factory=dict)  # List of paths to the pre-trained models or model identifiers
+    dataset_name: List[str] = field(default_factory=list)  # List of dataset names to be used
+    data_dir: str = None  # Root directory where the datasets are stored
+    split: str = "test"  # Split of the dataset to be used (e.g., 'train', 'validation', 'test')
+    task: str = None
+    batch_size: int = 128  # Batch size for data loading
+    num_workers: int = 1  # Number of workers for data loading
+    verbose: bool = True  # Flag to enable verbose logging
+    output_dir: str = "outputs"  # Directory where the outputs (e.g., predictions, logs) will be saved
+    cache_dir: str = "cache"  # Directory where the dataset cache will be stored
+    skip_existing: bool = False  # Flag to skip processing if outputs already exist
+    limit: int = None # Limit the number of samples to be processed