evalscope 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

evalscope/backend/rag_eval/cmteb/tasks/__init__.py

@@ -0,0 +1,64 @@
+from .Classification import *
+from .Clustering import *
+from .PairClassification import *
+from .Reranking import *
+from .Retrieval import *
+from .STS import *
+
+CLS_CLASSIFICATION = {
+    "TNews": TNews,
+    "IFlyTek": IFlyTek,
+    "MultilingualSentiment": MultilingualSentiment,
+    "JDReview": JDReview,
+    "OnlineShopping": OnlineShopping,
+    "Waimai": Waimai,
+}
+
+CLS_CLUSTERING = {
+    "CLSClusteringS2S": CLSClusteringFastS2S,
+    "CLSClusteringP2P": CLSClusteringFastP2P,
+    "ThuNewsClusteringS2S": ThuNewsClusteringFastS2S,
+    "ThuNewsClusteringP2P": ThuNewsClusteringFastP2P,
+}
+
+CLS_PAIR_CLASSIFICATION = {
+    "Ocnli": Ocnli,
+    "Cmnli": Cmnli,
+}
+
+CLS_RERANKING = {
+    "T2Reranking": T2Reranking,
+    "MMarcoReranking": MMarcoReranking,
+    "CMedQAv1": CMedQAv1,
+    "CMedQAv2": CMedQAv2,
+}
+
+CLS_RETRIEVAL = {
+    "T2Retrieval": T2Retrieval,
+    "MMarcoRetrieval": MMarcoRetrieval,
+    "DuRetrieval": DuRetrieval,
+    "CovidRetrieval": CovidRetrieval,
+    "CmedqaRetrieval": CmedqaRetrieval,
+    "EcomRetrieval": EcomRetrieval,
+    "MedicalRetrieval": MedicalRetrieval,
+    "VideoRetrieval": VideoRetrieval,
+}
+
+CLS_STS = {
+    "ATEC": ATEC,
+    "BQ": BQ,
+    "LCQMC": LCQMC,
+    "PAWSX": PAWSX,
+    "STSB": STSB,
+    "AFQMC": AFQMC,
+    "QBQTC": QBQTC,
+}
+
+CLS_DICT = {
+    **CLS_CLASSIFICATION,
+    **CLS_CLUSTERING,
+    **CLS_PAIR_CLASSIFICATION,
+    **CLS_RERANKING,
+    **CLS_RETRIEVAL,
+    **CLS_STS,
+}

evalscope/backend/rag_eval/ragas/__init__.py

@@ -0,0 +1,2 @@
+from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments, EvaluationArguments
+from evalscope.backend.rag_eval.ragas.task_template import testset_generation, rag_eval

evalscope/backend/rag_eval/ragas/arguments.py

@@ -0,0 +1,37 @@
+from dataclasses import dataclass, field
+from typing import List, Optional, Union, Dict, Any
+
+
+@dataclass
+class TestsetGenerationArguments:
+    docs: List[str] = field(default_factory=list)
+    test_size: int = 10
+    output_file: str = "outputs/testset.json"
+    """
+    For local LLM support, you can use the following fields:
+        model_name_or_path: str
+        model_revision: str = "master"
+        template_type: str = "default"
+        generation_config: Optional[Dict]
+        
+    For API LLM support, you can use the following fields:
+        model_name="gpt-4o-mini"
+        api_base: str = "",
+        api_key: Optional[str] = None
+    """
+    generator_llm: Dict = field(default_factory=dict)
+    critic_llm: Dict = field(default_factory=dict)
+    embeddings: Dict = field(default_factory=dict)
+    distribution: str = field(
+        default_factory=lambda: {"simple": 0.5, "multi_context": 0.4, "reasoning": 0.1}
+    )
+
+
+@dataclass
+class EvaluationArguments:
+    testset_file: str
+    critic_llm: Dict = field(default_factory=dict)
+    embeddings: Dict = field(default_factory=dict)
+    metrics: List[str] = field(
+        default_factory=lambda: ["answer_relevancy", "faithfulness"]
+    )

evalscope/backend/rag_eval/ragas/task_template.py

@@ -0,0 +1,117 @@
+import os
+from evalscope.backend.rag_eval import EmbeddingModel, LLM
+from evalscope.utils.logger import get_logger
+from .arguments import TestsetGenerationArguments, EvaluationArguments
+
+logger = get_logger()
+
+
+def testset_generation(args: TestsetGenerationArguments) -> None:
+    from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
+    from ragas.testset.generator import TestsetGenerator
+    from ragas.testset.evolutions import simple, reasoning, multi_context
+    from ragas import RunConfig
+
+    # load data
+    file_path = args.docs
+    loader = UnstructuredFileLoader(file_path, mode="elements")
+    data = loader.load()
+
+    # generator with models
+    generator_llm = LLM.load(**args.generator_llm)
+    critic_llm = LLM.load(**args.critic_llm)
+    embeddings = EmbeddingModel.load(**args.embeddings)
+
+    # Change resulting question type distribution
+    distributions = {
+        simple: args.distribution["simple"],
+        multi_context: args.distribution["multi_context"],
+        reasoning: args.distribution["reasoning"],
+    }
+
+    generator = TestsetGenerator.from_langchain(generator_llm, critic_llm, embeddings)
+
+    runconfig = RunConfig(timeout=30, max_retries=1, max_wait=30, max_workers=1)
+    testset = generator.generate_with_langchain_docs(
+        data,
+        args.test_size,
+        distributions,
+        with_debugging_logs=True,
+        is_async=False,
+        run_config=runconfig,
+    )
+
+    # save file
+    testset_df = testset.to_pandas()
+    output_path = os.path.dirname(args.output_file)
+    os.makedirs(output_path, exist_ok=True)
+    testset_df.to_json(args.output_file, indent=4, index=False, orient="records")
+
+    # get answer
+    testset_with_answer = get_answer(testset_df, generator_llm)
+    testset_with_answer.to_json(
+        args.output_file, indent=4, index=False, orient="records"
+    )
+
+
+def get_answer(testset_df, generator_llm):
+    template = """You are an assistant for question-answering tasks. 
+Use the following pieces of retrieved context to answer the question. 
+If you don't know the answer, just say that you don't know. 
+Use two sentences maximum and keep the answer concise.
+Question: {question} 
+Context: {contexts} 
+Answer:
+"""
+    answers = []
+    for index, row in testset_df.iterrows():
+        question = row["question"]
+        contexts = "\n".join(row["contexts"])
+
+        # Combine question and contexts as input for the LLM
+        input_text = template.format(question=question, contexts=contexts)
+
+        # Generate the answer using the generator LLM
+        answer = generator_llm.invoke(input_text)
+        answers.append(answer)
+
+    testset_df["answer"] = answers
+    return testset_df
+
+
+def rag_eval(
+    args: EvaluationArguments,
+) -> None:
+    from datasets import Dataset
+    from ragas import evaluate
+    from evalscope.backend.rag_eval import EmbeddingModel, LLM
+    from ragas import RunConfig
+    import importlib
+
+    def dynamic_import(module_name, *function_names):
+        # 动态导入指定模块
+        module = importlib.import_module(module_name)
+
+        functions = [getattr(module, name) for name in function_names]
+        return functions
+
+    llm = LLM.load(**args.critic_llm)
+    embedding = EmbeddingModel.load(**args.embeddings)
+
+    dataset = Dataset.from_json(args.testset_file)
+
+    runconfig = RunConfig(timeout=30, max_retries=1, max_wait=30, max_workers=1)
+    score = evaluate(
+        dataset,
+        metrics=dynamic_import("ragas.metrics", *args.metrics),
+        llm=llm,
+        embeddings=embedding,
+        run_config=runconfig,
+    )
+    score_df = score.to_pandas()
+    # logger.info(score_df.to_string())
+
+    output_path = args.testset_file.split(".")[0] + "_score.json"
+    score_df.to_json(output_path, indent=4, index=False, orient="records")
+
+    logger.info(f"Eval score saved to {output_path}")

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -4,7 +4,6 @@ from evalscope.backend.base import BackendManager
 from evalscope.utils.logger import get_logger
 from functools import partial
 import subprocess
-from dataclasses import dataclass
 import copy
 
 logger = get_logger()

evalscope/backend/vlm_eval_kit/custom_dataset.py

@@ -8,7 +8,7 @@ class CustomDataset:
 
     def load_data(self, dataset):
         # customize the loading of the dataset
-        data_path = os.path.join("~/LMUData", f'{dataset}.tsv')
+        data_path = os.path.join(os.path.expanduser("~/LMUData"), f'{dataset}.tsv')
         return load(data_path)
 
 

evalscope/evaluator/evaluator.py

@@ -174,6 +174,7 @@ class Evaluator(object):
         """
         assert self.data_adapter is not None, 'data_adapter must be provided when calling func get_answers() !'
         assert self.model_adapter is not None, 'model must be provided when calling func get_answers() !'
+        assert len(prompts_list) > 0, 'prompts_list must not be empty when calling func get_answers() !'
 
         answers_list = []
         pred_dir: str = self.outputs_structure.get(OutputsStructure.PREDICTIONS_DIR)

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -31,6 +31,7 @@ In these examples settings.xml lists input files and formats.
 from __future__ import absolute_import, division, print_function
 import collections
 import re
+import os
 
 import nltk
 import numpy as np
@@ -38,6 +39,24 @@ import six
 from absl import logging
 from rouge_score import scoring, tokenizers
 from six.moves import map, range
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+# Deal with nltk punkt_tab.zip tokenizer file to avoid downloading issue
+try:
+    nltk_dir = os.path.join(os.path.expanduser('~'), 'nltk_data/tokenizers')
+    os.makedirs(nltk_dir, exist_ok=True)
+    punkt_path = os.path.join(nltk_dir, 'punkt_tab.zip')
+    punkt_tab_url = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip'
+
+    if not os.path.exists(punkt_path):
+        os.system(f'wget -P {nltk_dir} {punkt_tab_url}')
+        os.system(f'unzip {punkt_path} -d {nltk_dir}')
+    else:
+        logger.info(f'{punkt_path} already exists, skipping download')
+except Exception as e:
+    logger.error(f'Try to download punkt_tab.zip for nltk failed: {e}')
 
 
 class RougeScorer(scoring.BaseScorer):

evalscope/run.py

@@ -207,6 +207,10 @@ def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig]]) -> Union[
             from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
             vlm_eval_kit_backend_manager = VLMEvalKitBackendManager(config=eval_config)
             vlm_eval_kit_backend_manager.run()
+        elif eval_backend == EvalBackend.RAG_EVAL.value:
+            from evalscope.backend.rag_eval import RAGEvalBackendManager
+            rag_eval_backend_manager = RAGEvalBackendManager(config=eval_config)
+            rag_eval_backend_manager.run()
         # TODO: Add other evaluation backends
         elif eval_backend == EvalBackend.THIRD_PARTY.value:
             raise NotImplementedError(f'Not implemented for evaluation backend {eval_backend}')

evalscope/utils/logger.py

@@ -1,18 +1,20 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+import importlib.util as iutil
 import logging
 from typing import Optional
 
 init_loggers = {}
+format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+formatter = logging.Formatter(format)
 
-formatter = logging.Formatter(
-    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logging.basicConfig(format=format, level=logging.INFO)
 
 
-def get_logger(log_file: Optional[str] = None,
-               log_level: int = logging.INFO,
-               file_mode: str = 'w'):
-    """ Get logging logger
+def get_logger(
+    log_file: Optional[str] = None, log_level: int = logging.INFO, file_mode: str = "w"
+):
+    """Get logging logger
 
     Args:
         log_file: Log filename, if specified, file handler will be added to
@@ -22,21 +24,39 @@ def get_logger(log_file: Optional[str] = None,
             specified (if filemode is unspecified, it defaults to 'w').
     """
 
-    logger_name = __name__.split('.')[0]
+    logger_name = __name__.split(".")[0]
     logger = logging.getLogger(logger_name)
-
+    logger.propagate = False
     if logger_name in init_loggers:
         add_file_handler_if_needed(logger, log_file, file_mode, log_level)
+        if logger.level != log_level:
+            logger.setLevel(log_level)
         return logger
 
-    for handler in logger.root.handlers:
-        if type(handler) is logging.StreamHandler:
-            handler.setLevel(logging.ERROR)
+    # handle duplicate logs to the console
+    # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler <stderr> (NOTSET)
+    # to the root logger. As logger.propagate is True by default, this root
+    # level handler causes logging messages from rank>0 processes to
+    # unexpectedly show up on the console, creating much unwanted clutter.
+    # To fix this issue, we set the root logger's StreamHandler, if any, to log
+    # at the ERROR level.
+    torch_dist = False
+    is_worker0 = True
+    if iutil.find_spec("torch") is not None:
+        from modelscope.utils.torch_utils import is_dist, is_master
+
+        torch_dist = is_dist()
+        is_worker0 = is_master()
+
+    if torch_dist:
+        for handler in logger.root.handlers:
+            if type(handler) is logging.StreamHandler:
+                handler.setLevel(logging.ERROR)
 
     stream_handler = logging.StreamHandler()
     handlers = [stream_handler]
 
-    if log_file is not None:
+    if is_worker0 and log_file is not None:
         file_handler = logging.FileHandler(log_file, file_mode)
         handlers.append(file_handler)
 
@@ -45,7 +65,10 @@ def get_logger(log_file: Optional[str] = None,
         handler.setLevel(log_level)
         logger.addHandler(handler)
 
-    logger.setLevel(log_level)
+    if is_worker0:
+        logger.setLevel(log_level)
+    else:
+        logger.setLevel(logging.ERROR)
 
     init_loggers[logger_name] = True
 
@@ -57,7 +80,14 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
         if isinstance(handler, logging.FileHandler):
             return
 
-    if log_file is not None:
+    if iutil.find_spec("torch") is not None:
+        from modelscope.utils.torch_utils import is_master
+
+        is_worker0 = is_master()
+    else:
+        is_worker0 = True
+
+    if is_worker0 and log_file is not None:
         file_handler = logging.FileHandler(log_file, file_mode)
         file_handler.setFormatter(formatter)
         file_handler.setLevel(log_level)

evalscope/utils/task_utils.py

@@ -11,6 +11,9 @@ class EvalBackend(Enum):
     
     # Use VLM Eval Kit as the multi-modal model evaluation backend
     VLM_EVAL_KIT = 'VLMEvalKit'
+    
+    # Use RAGEval as the RAG evaluation backend
+    RAG_EVAL = 'RAGEval'
 
     # Use third-party evaluation backend/modules
     THIRD_PARTY = 'ThirdParty'

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.5.4'
-__release_datetime__ = '2024-09-19 08:00:00'
+__version__ = "0.5.5"
+__release_datetime__ = "2024-10-15 08:00:00"

{evalscope-0.5.4.dist-info → evalscope-0.5.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.5.4
+Version: 0.5.5
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -24,7 +24,7 @@ Requires-Dist: editdistance
 Requires-Dist: jsonlines
 Requires-Dist: matplotlib
 Requires-Dist: modelscope[framework]
-Requires-Dist: nltk
+Requires-Dist: nltk (>=3.9)
 Requires-Dist: openai
 Requires-Dist: pandas
 Requires-Dist: plotly
@@ -34,7 +34,7 @@ Requires-Dist: pyyaml
 Requires-Dist: regex
 Requires-Dist: requests
 Requires-Dist: requests-toolbelt
-Requires-Dist: rouge-score
+Requires-Dist: rouge-score (>=0.1.0)
 Requires-Dist: sacrebleu
 Requires-Dist: scikit-learn
 Requires-Dist: seaborn
@@ -57,7 +57,7 @@ Requires-Dist: editdistance ; extra == 'all'
 Requires-Dist: jsonlines ; extra == 'all'
 Requires-Dist: matplotlib ; extra == 'all'
 Requires-Dist: modelscope[framework] ; extra == 'all'
-Requires-Dist: nltk ; extra == 'all'
+Requires-Dist: nltk (>=3.9) ; extra == 'all'
 Requires-Dist: openai ; extra == 'all'
 Requires-Dist: pandas ; extra == 'all'
 Requires-Dist: plotly ; extra == 'all'
@@ -67,7 +67,7 @@ Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: regex ; extra == 'all'
 Requires-Dist: requests ; extra == 'all'
 Requires-Dist: requests-toolbelt ; extra == 'all'
-Requires-Dist: rouge-score ; extra == 'all'
+Requires-Dist: rouge-score (>=0.1.0) ; extra == 'all'
 Requires-Dist: sacrebleu ; extra == 'all'
 Requires-Dist: scikit-learn ; extra == 'all'
 Requires-Dist: seaborn ; extra == 'all'
@@ -80,8 +80,10 @@ Requires-Dist: transformers (>=4.33) ; extra == 'all'
 Requires-Dist: transformers-stream-generator ; extra == 'all'
 Requires-Dist: jieba ; extra == 'all'
 Requires-Dist: rouge-chinese ; extra == 'all'
-Requires-Dist: ms-opencompass (>=0.1.0) ; extra == 'all'
+Requires-Dist: ms-opencompass (>=0.1.1) ; extra == 'all'
 Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'all'
+Requires-Dist: ragas ; extra == 'all'
+Requires-Dist: mteb (>=0.14.16) ; extra == 'all'
 Provides-Extra: inner
 Requires-Dist: absl-py ; extra == 'inner'
 Requires-Dist: accelerate ; extra == 'inner'
@@ -109,7 +111,10 @@ Requires-Dist: tqdm ; extra == 'inner'
 Requires-Dist: transformers (<4.43,>=4.33) ; extra == 'inner'
 Requires-Dist: transformers-stream-generator ; extra == 'inner'
 Provides-Extra: opencompass
-Requires-Dist: ms-opencompass (>=0.1.0) ; extra == 'opencompass'
+Requires-Dist: ms-opencompass (>=0.1.1) ; extra == 'opencompass'
+Provides-Extra: rag
+Requires-Dist: ragas ; extra == 'rag'
+Requires-Dist: mteb (>=0.14.16) ; extra == 'rag'
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'vlmeval'
 
@@ -144,30 +149,11 @@ English | [简体中文](README_zh.md)
 
 ## 📝 Introduction
 
-Large Model (including Large Language Models, Multi-modal Large Language Models) evaluation has become a critical process for assessing and improving LLMs. To better support the evaluation of large models, we propose the EvalScope framework.
-
-### Framework Features
-- **Benchmark Datasets**: Preloaded with several commonly used test benchmarks, including MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, HumanEval, etc.
-- **Evaluation Metrics**: Implements various commonly used evaluation metrics.
-- **Model Access**: A unified model access mechanism that is compatible with the Generate and Chat interfaces of multiple model families.
-- **Automated Evaluation**: Includes automatic evaluation of objective questions and complex task evaluation using expert models.
-- **Evaluation Reports**: Automatically generates evaluation reports.
-- **Arena Mode**: Used for comparisons between models and objective evaluation of models, supporting various evaluation modes, including:
-  - **Single mode**: Scoring a single model.
-  - **Pairwise-baseline mode**: Comparing against a baseline model.
-  - **Pairwise (all) mode**: Pairwise comparison among all models.
-- **Visualization Tools**: Provides intuitive displays of evaluation results.
-- **Model Performance Evaluation**: Offers a performance testing tool for model inference services and detailed statistics, see [Model Performance Evaluation Documentation](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test.html).
-- **OpenCompass Integration**: Supports OpenCompass as the evaluation backend, providing advanced encapsulation and task simplification, allowing for easier task submission for evaluation.
-- **VLMEvalKit Integration**: Supports VLMEvalKit as the evaluation backend, facilitating the initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
-- **Full-Link Support**: Through seamless integration with the [ms-swift](https://github.com/modelscope/ms-swift) training framework, provides a one-stop development process for model training, model deployment, model evaluation, and report viewing, enhancing user development efficiency.
-
-
-<details><summary>Overall Architecture</summary>
+EvalScope is the official model evaluation and performance benchmarking framework launched by the [ModelScope](https://modelscope.cn/) community. It comes with built-in common benchmarks and evaluation metrics, such as MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, and HumanEval. EvalScope supports various types of model evaluations, including LLMs, multimodal LLMs, embedding models, and reranker models. It is also applicable to multiple evaluation scenarios, such as end-to-end RAG evaluation, arena mode, and model inference performance stress testing. Moreover, with the seamless integration of the ms-swift training framework, evaluations can be initiated with a single click, providing full end-to-end support from model training to evaluation 🚀
 
 <p align="center">
   <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
-  <br>Fig 1. EvalScope Framework.
+  <br>EvalScope Framework.
 </p>
 
 The architecture includes the following modules:
@@ -177,14 +163,15 @@ The architecture includes the following modules:
     - **Native**: EvalScope’s own **default evaluation framework**, supporting various evaluation modes, including single model evaluation, arena mode, baseline model comparison mode, etc.
     - **OpenCompass**: Supports [OpenCompass](https://github.com/open-compass/opencompass) as the evaluation backend, providing advanced encapsulation and task simplification, allowing you to submit tasks for evaluation more easily.
     - **VLMEvalKit**: Supports [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) as the evaluation backend, enabling easy initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
+    - **RAGEval**: Supports RAG evaluation, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
     - **ThirdParty**: Other third-party evaluation tasks, such as ToolBench.
 4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
-</details>
 
 
 ## 🎉 News
+- 🔥 **[2024.10.8]** Support for RAG evaluation, including independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
 - 🔥 **[2024.09.18]** Our documentation has been updated to include a blog module, featuring some technical research and discussions related to evaluations. We invite you to [📖 read it](https://evalscope.readthedocs.io/en/refact_readme/blog/index.html).
 - 🔥 **[2024.09.12]** Support for LongWriter evaluation, which supports 10,000+ word generation. You can use the benchmark [LongBench-Write](evalscope/third_party/longbench_write/README.md) to measure the long output quality as well as the output length.
 - 🔥 **[2024.08.30]** Support for custom dataset evaluations, including text datasets and multimodal image-text datasets.
@@ -355,9 +342,10 @@ run_task(task_cfg=your_task_cfg)
 ## Evaluation Backend
 EvalScope supports using third-party evaluation frameworks to initiate evaluation tasks, which we call Evaluation Backend. Currently supported Evaluation Backend includes:
 - **Native**: EvalScope's own **default evaluation framework**, supporting various evaluation modes including single model evaluation, arena mode, and baseline model comparison mode.
-- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/opencompass_backend.html)
-- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/vlmevalkit_backend.html)
-- **ThirdParty**: The third-party task, e.g. [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html), you can contribute your own evaluation task to EvalScope as third-party backend.
+- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/opencompass_backend.html)
+- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/vlmevalkit_backend.html)
+- **RAGEval**: Initiate RAG evaluation tasks through EvalScope, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html): [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/index.html)
+- **ThirdParty**: Third-party evaluation tasks, such as [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html) and [LongBench-Write](https://evalscope.readthedocs.io/en/latest/third_party/longwriter.html).
 
 ## Custom Dataset Evaluation
 EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset.html)
@@ -386,6 +374,8 @@ Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
 
 
 ## TO-DO List
+- [x] RAG evaluation
+- [x] VLM evaluation
 - [x] Agents evaluation
 - [x] vLLM
 - [ ] Distributed evaluating
@@ -397,3 +387,7 @@ Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
 - [ ] Auto-reviewer
   - [ ] Qwen-max
 
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=modelscope/evalscope&type=Date)](https://star-history.com/#modelscope/evalscope&Date)

{evalscope-0.5.4.dist-info → evalscope-0.5.5.dist-info}/RECORD

@@ -2,22 +2,38 @@ evalscope/__init__.py,sha256=3eLMMrjkAIAs3vGluXNZn5-xTSbO_vfba9yNPbkVtg8,105
 evalscope/cache.py,sha256=zpGjL9JMosqjk_dkODVwvIGiUC0WAMmMTHDNJOvBQU8,3288
 evalscope/config.py,sha256=G_rpSn5Kd1aPlFJO6asnZu5FUggZmwcYdAxxpuq0yDs,6972
 evalscope/constants.py,sha256=g8lGYlpA4Wk88HwtqId1-jJX_z8Lr2k02gWLsyofyj0,2670
-evalscope/run.py,sha256=T-2zoJpBx6YxLnLJH-iFF3UxUGYTU36PMV_DQ9e8tSM,18484
+evalscope/run.py,sha256=uAXtaxIBcR94jyfHGFAecuzn0y71oLgu-d9VOohCJAw,18738
 evalscope/run_arena.py,sha256=BCWCAiX0BQ9pLMIq08svEcd-IoFr75gFShpV88robIY,8963
 evalscope/run_ms.py,sha256=UtJoGnah64SXigTawJQWTi_TEGjr7Td0rjCTaO-htL8,6028
 evalscope/summarizer.py,sha256=rIyML8HpjQxIpXg8KvQ0CzOS6xMS-JHZh6kUZzkaRsk,6640
-evalscope/version.py,sha256=yeILiwZ11YPUOCKr-BHfiOuKFIWppiHTIqg22wbfo_g,118
-evalscope/backend/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
+evalscope/version.py,sha256=UZ6qDTtcyaqkwS2_IkU2Kzop4lG9AL9cYpfEpYzfCrc,118
+evalscope/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalscope/backend/base.py,sha256=5BLrDNNwxsGp35zorD-kphmN15tlBbkuuqwkz8jWZq0,876
 evalscope/backend/opencompass/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
 evalscope/backend/opencompass/api_meta_template.py,sha256=sBW0XbVDOKeJ7mVUDLhmcG4e0yClw3eluazdp_8wtgQ,1753
 evalscope/backend/opencompass/backend_manager.py,sha256=_eg82FLAVxQ6t5e1OqlyuxZcngqD8rxvI5EijLUh_zI,10294
 evalscope/backend/opencompass/tasks/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/backend/opencompass/tasks/eval_api.py,sha256=12lrgDpMzZ1XBRboq5TEOovDPCMDwwGCJoRT78Ox_yo,1108
-evalscope/backend/opencompass/tasks/eval_datasets.py,sha256=EizugDMt-ontWsTOaM61XGLUkx-S9rzdLf2Ssfmw3Yc,5263
+evalscope/backend/opencompass/tasks/eval_datasets.py,sha256=bYFHkjiwZqh2FVRo1I88xEDZ6nYmZjAgG5ZODbthKFI,5241
+evalscope/backend/rag_eval/__init__.py,sha256=yRCcfxhzC7wIYbgb-w76i4D9v8wXI7JmYNM6IZUn064,199
+evalscope/backend/rag_eval/backend_manager.py,sha256=tWkFzYO9LQjaI7paD5yz1c-HtNJUbnAr0a-4biYSZvg,2562
+evalscope/backend/rag_eval/cmteb/__init__.py,sha256=ajVz6XP5hqPq-jm66hp2poA2qKj1V19ZGoqjrGUlO7U,279
+evalscope/backend/rag_eval/cmteb/arguments.py,sha256=698UvPVZp5Ncq_p25P_67SQkYaW2tLSCHenUOZ0n5OI,2217
+evalscope/backend/rag_eval/cmteb/base.py,sha256=sJqTRCej7vk5ASirk21hOobX1_Hz7BO1LIHJFOGLuE4,2731
+evalscope/backend/rag_eval/cmteb/task_template.py,sha256=HreVxyRiF2QUe4Dy9_zKNp1WU74342RWHV5_B8ycXG0,2537
+evalscope/backend/rag_eval/cmteb/tasks/Classification.py,sha256=7adR40W6Uu58-QR9jCUP4k7TdAnG0oT225v4xHXah2g,10635
+evalscope/backend/rag_eval/cmteb/tasks/Clustering.py,sha256=7j1Hts_r4Nv8DlbIiPFMaU1JDxCYgu0wO0JI8T_Y6X8,8969
+evalscope/backend/rag_eval/cmteb/tasks/PairClassification.py,sha256=2WkaTE-jF8jqsu1UcNDqN8A4567UzW5boD_0B83j-9A,4008
+evalscope/backend/rag_eval/cmteb/tasks/Reranking.py,sha256=50h-lXaRcb5s6ZpIgnfk5mU7iZur8ZDxwsaFbrqSZ_o,5462
+evalscope/backend/rag_eval/cmteb/tasks/Retrieval.py,sha256=wUxiQH5aOmWNS4YswACyHqBn5xqP5eyvsq6U9WSp5R0,11457
+evalscope/backend/rag_eval/cmteb/tasks/STS.py,sha256=6GMaoCANM-IKYLk4srHOYr_eurav3DGihHMQeJPXR6k,12054
+evalscope/backend/rag_eval/cmteb/tasks/__init__.py,sha256=FrtoBosHq9iRp3yfZEAxWa5NkYhHtA20NmHDG6eiPNU,1421
+evalscope/backend/rag_eval/ragas/__init__.py,sha256=tHB7XGREmcrx8ulF-JZWWoHsEbn2s-PFyWFdGzOZQcw,190
+evalscope/backend/rag_eval/ragas/arguments.py,sha256=plVc2_3auVG5z91ExzBdkbNIhMVjyi_xQYbEzlV0iNw,1208
+evalscope/backend/rag_eval/ragas/task_template.py,sha256=795NHXzGdeqa15ONV1AgDZywpMHucaIlvk_EBF0CK98,3868
 evalscope/backend/vlm_eval_kit/__init__.py,sha256=xTgHM95lWzh4s0W7zxLwYkgUbPAZfAb0UoGGmyyBXrs,83
-evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=k52qTUqkp1kJivKn8bVrKoF8cng4xYTQLUmjnH_CWPM,6080
-evalscope/backend/vlm_eval_kit/custom_dataset.py,sha256=zC40Jw9bIqcGKuWS9oKPAlQdBARc-zY3sJlSiU-u-sI,1625
+evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=ewhpE9yzsqf5ED6kqsqek2YEgg96GBQOupxtVNhaXxI,6046
+evalscope/backend/vlm_eval_kit/custom_dataset.py,sha256=Yz2A5kB1E8DYBnjuVCA6TTPtLjhg8vYKeJTh6FU_Ecw,1645
 evalscope/benchmarks/__init__.py,sha256=6TKP35wfKf7R_h870fsEtcIlIAgomKOcukNL9M-5I1Y,162
 evalscope/benchmarks/benchmark.py,sha256=EmwYyFdrAHBGMkSbsMZQOR_62Q0CSKl8zeLlr7xvJdQ,2159
 evalscope/benchmarks/data_adapter.py,sha256=eVQvOQYQOQbIl8UlvOEUqRThL3FP3aUD6DSlqF1bqO0,10395
@@ -91,7 +107,7 @@ evalscope/cli/cli.py,sha256=uZ-qC8WBsLd5-Hn94d43sSGg0UC_12RebSD4ToKjypg,844
 evalscope/cli/start_perf.py,sha256=TL6bMXYl3ln-tfs5uBmzb9x94uxz6f3PBFIt1l7g3VA,994
 evalscope/cli/start_server.py,sha256=ATGLP2TE0aImJNicpehdzBuFlNb50F7KhyL4A_ZSoGU,3885
 evalscope/evaluator/__init__.py,sha256=S6MU1O_iiNAaKxNIhO9MEmdW-BSNf_YH2l6NQ9lxVNo,103
-evalscope/evaluator/evaluator.py,sha256=gB408byOpu269Psh6MjYC9-a_uv9GvThoT7t07Oqh6w,30712
+evalscope/evaluator/evaluator.py,sha256=eSCgPPDGfIJfKu0cthhbDLFm1xMhj_869iT3ngcQkPc,30817
 evalscope/evaluator/rating_eval.py,sha256=cJbkyXIuwFUZoe7ZJZM6eUskNd9zlORgndckuon2OQ8,5768
 evalscope/evaluator/reviewer/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/evaluator/reviewer/auto_reviewer.py,sha256=JycPYti9h1j_8DRcu_rc5U0wkEASHYg-XBqrUUoiO-Q,17054
@@ -101,7 +117,7 @@ evalscope/metrics/math_accuracy.py,sha256=1PCy1VUNYg48JcGy-6SUmUDZNwPeAkMW1QQ_lX
 evalscope/metrics/metrics.py,sha256=sDZljGiZwgHsFZ5eNi65-3z3BLCdIwWUzPcq2QpKf1k,12545
 evalscope/metrics/rouge_metric.py,sha256=sN0r-sXXc-nJUdFrthQPAv1VFdOCrF6zzIYDKaLSgrU,4522
 evalscope/metrics/bundled_rouge_score/__init__.py,sha256=PwbTdk8168FwDJe_l8XIqDuBgZQooDsP31vj7di05Fs,650
-evalscope/metrics/bundled_rouge_score/rouge_scorer.py,sha256=xSLis-zx1hnHuj_9JI7HuUKMS4ZQsX-D8wECZg4D2bg,11450
+evalscope/metrics/bundled_rouge_score/rouge_scorer.py,sha256=vhzIMSQezhZuJzGndymWjB_iRbDdECoEidOIdNL3NAM,12213
 evalscope/models/__init__.py,sha256=zG27J2HSeKPGiAIUE7QLPHEPLyXLsfaDwYI_TDXjpCg,145
 evalscope/models/dummy_chat_model.py,sha256=xE8wcFVSCkvizEJ-B8ojX0Ir01Q5KrN5mapjMQaQtbg,1325
 evalscope/models/model.py,sha256=ZzzVzZHVzuzdt5F1r-rEBT44ZfW9B7R1spsrV-T8nSw,3020
@@ -169,12 +185,12 @@ evalscope/tools/rewrite_eval_results.py,sha256=ZVi2hVjiTOmR_O5IaLv6qnQNpMz6FnDb9
 evalscope/utils/__init__.py,sha256=6RjACRYUSpGj6fkZ7NzYpl0lFppQCp9KVn5ktZe626s,128
 evalscope/utils/arena_utils.py,sha256=RMkymUv9Cxs37arUntzgDY5P0Dand2jGpsb7uy6wZmg,7670
 evalscope/utils/completion_parsers.py,sha256=61l8CTh1VxHgRoMDhtznpAhuJp47MssGgS-LdEe_h80,2997
-evalscope/utils/logger.py,sha256=Ycd0W17Z_oiByPuPX3_umNrOCHjT9O_e_Kws7ZWUSvU,1855
+evalscope/utils/logger.py,sha256=cf3U400Mx1speMMNXorjwEE8noDz5Mbd-9PNgaulGeY,3013
 evalscope/utils/task_cfg_parser.py,sha256=LiNQ2X8lbZU0cODpaY_PbKyUhNoxZIC495UsLJigX64,138
-evalscope/utils/task_utils.py,sha256=Mv_u_f4Z91zcUeko6acZCmnOAPRfk61kf_dliLzG5Yk,459
+evalscope/utils/task_utils.py,sha256=IMtBSBUp3H95Ko0vn8Q55Wmz2SFZXSfjVy49tyomL_g,537
 evalscope/utils/utils.py,sha256=zHo9hfxGBUVKE2xNMR7lDoEvfRnk4V4946DEfXQhlq4,20509
-evalscope-0.5.4.dist-info/METADATA,sha256=WRfHi_dmJzaF5WXqB2vPz_HhciWfNZ8Su5j_dxbiKQE,20705
-evalscope-0.5.4.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-evalscope-0.5.4.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
-evalscope-0.5.4.dist-info/top_level.txt,sha256=jNR-HMn3TR8Atolq7_4rW8IWVX6GhvYV5_1Y_KbJKlY,10
-evalscope-0.5.4.dist-info/RECORD,,
+evalscope-0.5.5.dist-info/METADATA,sha256=scs7UaBcWE2qpewo_oe6ZB8HX5CtbohPBvom6UjUY5w,20943
+evalscope-0.5.5.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+evalscope-0.5.5.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
+evalscope-0.5.5.dist-info/top_level.txt,sha256=jNR-HMn3TR8Atolq7_4rW8IWVX6GhvYV5_1Y_KbJKlY,10
+evalscope-0.5.5.dist-info/RECORD,,