evalscope 0.5.5__py3-none-any.whl → 0.5.5rc0__py3-none-any.whl

evalscope/backend/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.backend.opencompass.backend_manager import OpenCompassBackendManager

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -4,6 +4,7 @@ from evalscope.backend.base import BackendManager
 from evalscope.utils.logger import get_logger
 from functools import partial
 import subprocess
+from dataclasses import dataclass
 import copy
 
 logger = get_logger()

evalscope/backend/vlm_eval_kit/custom_dataset.py

@@ -8,7 +8,7 @@ class CustomDataset:
 
     def load_data(self, dataset):
         # customize the loading of the dataset
-        data_path = os.path.join(os.path.expanduser("~/LMUData"), f'{dataset}.tsv')
+        data_path = os.path.join("~/LMUData", f'{dataset}.tsv')
         return load(data_path)
 
 

evalscope/evaluator/evaluator.py

@@ -174,7 +174,6 @@ class Evaluator(object):
         """
         assert self.data_adapter is not None, 'data_adapter must be provided when calling func get_answers() !'
         assert self.model_adapter is not None, 'model must be provided when calling func get_answers() !'
-        assert len(prompts_list) > 0, 'prompts_list must not be empty when calling func get_answers() !'
 
         answers_list = []
         pred_dir: str = self.outputs_structure.get(OutputsStructure.PREDICTIONS_DIR)

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -31,7 +31,6 @@ In these examples settings.xml lists input files and formats.
 from __future__ import absolute_import, division, print_function
 import collections
 import re
-import os
 
 import nltk
 import numpy as np
@@ -39,24 +38,6 @@ import six
 from absl import logging
 from rouge_score import scoring, tokenizers
 from six.moves import map, range
-from evalscope.utils import get_logger
-
-logger = get_logger()
-
-# Deal with nltk punkt_tab.zip tokenizer file to avoid downloading issue
-try:
-    nltk_dir = os.path.join(os.path.expanduser('~'), 'nltk_data/tokenizers')
-    os.makedirs(nltk_dir, exist_ok=True)
-    punkt_path = os.path.join(nltk_dir, 'punkt_tab.zip')
-    punkt_tab_url = 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip'
-
-    if not os.path.exists(punkt_path):
-        os.system(f'wget -P {nltk_dir} {punkt_tab_url}')
-        os.system(f'unzip {punkt_path} -d {nltk_dir}')
-    else:
-        logger.info(f'{punkt_path} already exists, skipping download')
-except Exception as e:
-    logger.error(f'Try to download punkt_tab.zip for nltk failed: {e}')
 
 
 class RougeScorer(scoring.BaseScorer):

evalscope/run.py

@@ -207,10 +207,6 @@ def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig]]) -> Union[
             from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
             vlm_eval_kit_backend_manager = VLMEvalKitBackendManager(config=eval_config)
             vlm_eval_kit_backend_manager.run()
-        elif eval_backend == EvalBackend.RAG_EVAL.value:
-            from evalscope.backend.rag_eval import RAGEvalBackendManager
-            rag_eval_backend_manager = RAGEvalBackendManager(config=eval_config)
-            rag_eval_backend_manager.run()
         # TODO: Add other evaluation backends
         elif eval_backend == EvalBackend.THIRD_PARTY.value:
             raise NotImplementedError(f'Not implemented for evaluation backend {eval_backend}')

evalscope/utils/logger.py

@@ -1,20 +1,18 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import importlib.util as iutil
 import logging
 from typing import Optional
 
 init_loggers = {}
-format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-formatter = logging.Formatter(format)
 
-logging.basicConfig(format=format, level=logging.INFO)
+formatter = logging.Formatter(
+    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 
 
-def get_logger(
-    log_file: Optional[str] = None, log_level: int = logging.INFO, file_mode: str = "w"
-):
-    """Get logging logger
+def get_logger(log_file: Optional[str] = None,
+               log_level: int = logging.INFO,
+               file_mode: str = 'w'):
+    """ Get logging logger
 
     Args:
         log_file: Log filename, if specified, file handler will be added to
@@ -24,39 +22,21 @@ def get_logger(
             specified (if filemode is unspecified, it defaults to 'w').
     """
 
-    logger_name = __name__.split(".")[0]
+    logger_name = __name__.split('.')[0]
     logger = logging.getLogger(logger_name)
-    logger.propagate = False
+
     if logger_name in init_loggers:
         add_file_handler_if_needed(logger, log_file, file_mode, log_level)
-        if logger.level != log_level:
-            logger.setLevel(log_level)
         return logger
 
-    # handle duplicate logs to the console
-    # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler <stderr> (NOTSET)
-    # to the root logger. As logger.propagate is True by default, this root
-    # level handler causes logging messages from rank>0 processes to
-    # unexpectedly show up on the console, creating much unwanted clutter.
-    # To fix this issue, we set the root logger's StreamHandler, if any, to log
-    # at the ERROR level.
-    torch_dist = False
-    is_worker0 = True
-    if iutil.find_spec("torch") is not None:
-        from modelscope.utils.torch_utils import is_dist, is_master
-
-        torch_dist = is_dist()
-        is_worker0 = is_master()
-
-    if torch_dist:
-        for handler in logger.root.handlers:
-            if type(handler) is logging.StreamHandler:
-                handler.setLevel(logging.ERROR)
+    for handler in logger.root.handlers:
+        if type(handler) is logging.StreamHandler:
+            handler.setLevel(logging.ERROR)
 
     stream_handler = logging.StreamHandler()
     handlers = [stream_handler]
 
-    if is_worker0 and log_file is not None:
+    if log_file is not None:
         file_handler = logging.FileHandler(log_file, file_mode)
         handlers.append(file_handler)
 
@@ -65,10 +45,7 @@ def get_logger(
         handler.setLevel(log_level)
         logger.addHandler(handler)
 
-    if is_worker0:
-        logger.setLevel(log_level)
-    else:
-        logger.setLevel(logging.ERROR)
+    logger.setLevel(log_level)
 
     init_loggers[logger_name] = True
 
@@ -80,14 +57,7 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
         if isinstance(handler, logging.FileHandler):
             return
 
-    if iutil.find_spec("torch") is not None:
-        from modelscope.utils.torch_utils import is_master
-
-        is_worker0 = is_master()
-    else:
-        is_worker0 = True
-
-    if is_worker0 and log_file is not None:
+    if log_file is not None:
         file_handler = logging.FileHandler(log_file, file_mode)
         file_handler.setFormatter(formatter)
         file_handler.setLevel(log_level)

evalscope/utils/task_utils.py

@@ -11,9 +11,6 @@ class EvalBackend(Enum):
     
     # Use VLM Eval Kit as the multi-modal model evaluation backend
     VLM_EVAL_KIT = 'VLMEvalKit'
-    
-    # Use RAGEval as the RAG evaluation backend
-    RAG_EVAL = 'RAGEval'
 
     # Use third-party evaluation backend/modules
     THIRD_PARTY = 'ThirdParty'

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = "0.5.5"
-__release_datetime__ = "2024-10-15 08:00:00"
+__version__ = '0.5.5rc0'
+__release_datetime__ = '2024-09-24 08:00:00'

{evalscope-0.5.5.dist-info → evalscope-0.5.5rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.5.5
+Version: 0.5.5rc0
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -24,7 +24,7 @@ Requires-Dist: editdistance
 Requires-Dist: jsonlines
 Requires-Dist: matplotlib
 Requires-Dist: modelscope[framework]
-Requires-Dist: nltk (>=3.9)
+Requires-Dist: nltk
 Requires-Dist: openai
 Requires-Dist: pandas
 Requires-Dist: plotly
@@ -34,7 +34,7 @@ Requires-Dist: pyyaml
 Requires-Dist: regex
 Requires-Dist: requests
 Requires-Dist: requests-toolbelt
-Requires-Dist: rouge-score (>=0.1.0)
+Requires-Dist: rouge-score
 Requires-Dist: sacrebleu
 Requires-Dist: scikit-learn
 Requires-Dist: seaborn
@@ -57,7 +57,7 @@ Requires-Dist: editdistance ; extra == 'all'
 Requires-Dist: jsonlines ; extra == 'all'
 Requires-Dist: matplotlib ; extra == 'all'
 Requires-Dist: modelscope[framework] ; extra == 'all'
-Requires-Dist: nltk (>=3.9) ; extra == 'all'
+Requires-Dist: nltk ; extra == 'all'
 Requires-Dist: openai ; extra == 'all'
 Requires-Dist: pandas ; extra == 'all'
 Requires-Dist: plotly ; extra == 'all'
@@ -67,7 +67,7 @@ Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: regex ; extra == 'all'
 Requires-Dist: requests ; extra == 'all'
 Requires-Dist: requests-toolbelt ; extra == 'all'
-Requires-Dist: rouge-score (>=0.1.0) ; extra == 'all'
+Requires-Dist: rouge-score ; extra == 'all'
 Requires-Dist: sacrebleu ; extra == 'all'
 Requires-Dist: scikit-learn ; extra == 'all'
 Requires-Dist: seaborn ; extra == 'all'
@@ -82,8 +82,6 @@ Requires-Dist: jieba ; extra == 'all'
 Requires-Dist: rouge-chinese ; extra == 'all'
 Requires-Dist: ms-opencompass (>=0.1.1) ; extra == 'all'
 Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'all'
-Requires-Dist: ragas ; extra == 'all'
-Requires-Dist: mteb (>=0.14.16) ; extra == 'all'
 Provides-Extra: inner
 Requires-Dist: absl-py ; extra == 'inner'
 Requires-Dist: accelerate ; extra == 'inner'
@@ -112,9 +110,6 @@ Requires-Dist: transformers (<4.43,>=4.33) ; extra == 'inner'
 Requires-Dist: transformers-stream-generator ; extra == 'inner'
 Provides-Extra: opencompass
 Requires-Dist: ms-opencompass (>=0.1.1) ; extra == 'opencompass'
-Provides-Extra: rag
-Requires-Dist: ragas ; extra == 'rag'
-Requires-Dist: mteb (>=0.14.16) ; extra == 'rag'
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval (>=0.0.5) ; extra == 'vlmeval'
 
@@ -149,11 +144,30 @@ English | [简体中文](README_zh.md)
 
 ## 📝 Introduction
 
-EvalScope is the official model evaluation and performance benchmarking framework launched by the [ModelScope](https://modelscope.cn/) community. It comes with built-in common benchmarks and evaluation metrics, such as MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, and HumanEval. EvalScope supports various types of model evaluations, including LLMs, multimodal LLMs, embedding models, and reranker models. It is also applicable to multiple evaluation scenarios, such as end-to-end RAG evaluation, arena mode, and model inference performance stress testing. Moreover, with the seamless integration of the ms-swift training framework, evaluations can be initiated with a single click, providing full end-to-end support from model training to evaluation 🚀
+Large Model (including Large Language Models, Multi-modal Large Language Models) evaluation has become a critical process for assessing and improving LLMs. To better support the evaluation of large models, we propose the EvalScope framework.
+
+### Framework Features
+- **Benchmark Datasets**: Preloaded with several commonly used test benchmarks, including MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, HumanEval, etc.
+- **Evaluation Metrics**: Implements various commonly used evaluation metrics.
+- **Model Access**: A unified model access mechanism that is compatible with the Generate and Chat interfaces of multiple model families.
+- **Automated Evaluation**: Includes automatic evaluation of objective questions and complex task evaluation using expert models.
+- **Evaluation Reports**: Automatically generates evaluation reports.
+- **Arena Mode**: Used for comparisons between models and objective evaluation of models, supporting various evaluation modes, including:
+  - **Single mode**: Scoring a single model.
+  - **Pairwise-baseline mode**: Comparing against a baseline model.
+  - **Pairwise (all) mode**: Pairwise comparison among all models.
+- **Visualization Tools**: Provides intuitive displays of evaluation results.
+- **Model Performance Evaluation**: Offers a performance testing tool for model inference services and detailed statistics, see [Model Performance Evaluation Documentation](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test.html).
+- **OpenCompass Integration**: Supports OpenCompass as the evaluation backend, providing advanced encapsulation and task simplification, allowing for easier task submission for evaluation.
+- **VLMEvalKit Integration**: Supports VLMEvalKit as the evaluation backend, facilitating the initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
+- **Full-Link Support**: Through seamless integration with the [ms-swift](https://github.com/modelscope/ms-swift) training framework, provides a one-stop development process for model training, model deployment, model evaluation, and report viewing, enhancing user development efficiency.
+
+
+<details><summary>Overall Architecture</summary>
 
 <p align="center">
   <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
-  <br>EvalScope Framework.
+  <br>Fig 1. EvalScope Framework.
 </p>
 
 The architecture includes the following modules:
@@ -163,15 +177,14 @@ The architecture includes the following modules:
     - **Native**: EvalScope’s own **default evaluation framework**, supporting various evaluation modes, including single model evaluation, arena mode, baseline model comparison mode, etc.
     - **OpenCompass**: Supports [OpenCompass](https://github.com/open-compass/opencompass) as the evaluation backend, providing advanced encapsulation and task simplification, allowing you to submit tasks for evaluation more easily.
     - **VLMEvalKit**: Supports [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) as the evaluation backend, enabling easy initiation of multi-modal evaluation tasks, supporting various multi-modal models and datasets.
-    - **RAGEval**: Supports RAG evaluation, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
     - **ThirdParty**: Other third-party evaluation tasks, such as ToolBench.
 4. **Performance Evaluator**: Model performance evaluation, responsible for measuring model inference service performance, including performance testing, stress testing, performance report generation, and visualization.
 5. **Evaluation Report**: The final generated evaluation report summarizes the model's performance, which can be used for decision-making and further model optimization.
 6. **Visualization**: Visualization results help users intuitively understand evaluation results, facilitating analysis and comparison of different model performances.
+</details>
 
 
 ## 🎉 News
-- 🔥 **[2024.10.8]** Support for RAG evaluation, including independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
 - 🔥 **[2024.09.18]** Our documentation has been updated to include a blog module, featuring some technical research and discussions related to evaluations. We invite you to [📖 read it](https://evalscope.readthedocs.io/en/refact_readme/blog/index.html).
 - 🔥 **[2024.09.12]** Support for LongWriter evaluation, which supports 10,000+ word generation. You can use the benchmark [LongBench-Write](evalscope/third_party/longbench_write/README.md) to measure the long output quality as well as the output length.
 - 🔥 **[2024.08.30]** Support for custom dataset evaluations, including text datasets and multimodal image-text datasets.
@@ -342,10 +355,9 @@ run_task(task_cfg=your_task_cfg)
 ## Evaluation Backend
 EvalScope supports using third-party evaluation frameworks to initiate evaluation tasks, which we call Evaluation Backend. Currently supported Evaluation Backend includes:
 - **Native**: EvalScope's own **default evaluation framework**, supporting various evaluation modes including single model evaluation, arena mode, and baseline model comparison mode.
-- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/opencompass_backend.html)
-- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/vlmevalkit_backend.html)
-- **RAGEval**: Initiate RAG evaluation tasks through EvalScope, supporting independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html): [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/index.html)
-- **ThirdParty**: Third-party evaluation tasks, such as [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html) and [LongBench-Write](https://evalscope.readthedocs.io/en/latest/third_party/longwriter.html).
+- [OpenCompass](https://github.com/open-compass/opencompass): Initiate OpenCompass evaluation tasks through EvalScope. Lightweight, easy to customize, supports seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/opencompass_backend.html)
+- [VLMEvalKit](https://github.com/open-compass/VLMEvalKit): Initiate VLMEvalKit multimodal evaluation tasks through EvalScope. Supports various multimodal models and datasets, and offers seamless integration with the LLM fine-tuning framework ms-swift. [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/vlmevalkit_backend.html)
+- **ThirdParty**: The third-party task, e.g. [ToolBench](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html), you can contribute your own evaluation task to EvalScope as third-party backend.
 
 ## Custom Dataset Evaluation
 EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset.html)
@@ -374,8 +386,6 @@ Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
 
 
 ## TO-DO List
-- [x] RAG evaluation
-- [x] VLM evaluation
 - [x] Agents evaluation
 - [x] vLLM
 - [ ] Distributed evaluating
@@ -387,7 +397,3 @@ Refer to : [Leaderboard](https://modelscope.cn/leaderboard/58/ranking?type=free)
 - [ ] Auto-reviewer
   - [ ] Qwen-max
 
-
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=modelscope/evalscope&type=Date)](https://star-history.com/#modelscope/evalscope&Date)

{evalscope-0.5.5.dist-info → evalscope-0.5.5rc0.dist-info}/RECORD

@@ -2,12 +2,12 @@ evalscope/__init__.py,sha256=3eLMMrjkAIAs3vGluXNZn5-xTSbO_vfba9yNPbkVtg8,105
 evalscope/cache.py,sha256=zpGjL9JMosqjk_dkODVwvIGiUC0WAMmMTHDNJOvBQU8,3288
 evalscope/config.py,sha256=G_rpSn5Kd1aPlFJO6asnZu5FUggZmwcYdAxxpuq0yDs,6972
 evalscope/constants.py,sha256=g8lGYlpA4Wk88HwtqId1-jJX_z8Lr2k02gWLsyofyj0,2670
-evalscope/run.py,sha256=uAXtaxIBcR94jyfHGFAecuzn0y71oLgu-d9VOohCJAw,18738
+evalscope/run.py,sha256=T-2zoJpBx6YxLnLJH-iFF3UxUGYTU36PMV_DQ9e8tSM,18484
 evalscope/run_arena.py,sha256=BCWCAiX0BQ9pLMIq08svEcd-IoFr75gFShpV88robIY,8963
 evalscope/run_ms.py,sha256=UtJoGnah64SXigTawJQWTi_TEGjr7Td0rjCTaO-htL8,6028
 evalscope/summarizer.py,sha256=rIyML8HpjQxIpXg8KvQ0CzOS6xMS-JHZh6kUZzkaRsk,6640
-evalscope/version.py,sha256=UZ6qDTtcyaqkwS2_IkU2Kzop4lG9AL9cYpfEpYzfCrc,118
-evalscope/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalscope/version.py,sha256=bZN6I8gZRA-2x6Q_o4vAmsbfomUm2v86wl1-AhMHnR4,121
+evalscope/backend/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
 evalscope/backend/base.py,sha256=5BLrDNNwxsGp35zorD-kphmN15tlBbkuuqwkz8jWZq0,876
 evalscope/backend/opencompass/__init__.py,sha256=UP_TW5KBq6V_Nvqkeb7PGvGGX3rVYussT43npwCwDgE,135
 evalscope/backend/opencompass/api_meta_template.py,sha256=sBW0XbVDOKeJ7mVUDLhmcG4e0yClw3eluazdp_8wtgQ,1753
@@ -15,25 +15,9 @@ evalscope/backend/opencompass/backend_manager.py,sha256=_eg82FLAVxQ6t5e1OqlyuxZc
 evalscope/backend/opencompass/tasks/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/backend/opencompass/tasks/eval_api.py,sha256=12lrgDpMzZ1XBRboq5TEOovDPCMDwwGCJoRT78Ox_yo,1108
 evalscope/backend/opencompass/tasks/eval_datasets.py,sha256=bYFHkjiwZqh2FVRo1I88xEDZ6nYmZjAgG5ZODbthKFI,5241
-evalscope/backend/rag_eval/__init__.py,sha256=yRCcfxhzC7wIYbgb-w76i4D9v8wXI7JmYNM6IZUn064,199
-evalscope/backend/rag_eval/backend_manager.py,sha256=tWkFzYO9LQjaI7paD5yz1c-HtNJUbnAr0a-4biYSZvg,2562
-evalscope/backend/rag_eval/cmteb/__init__.py,sha256=ajVz6XP5hqPq-jm66hp2poA2qKj1V19ZGoqjrGUlO7U,279
-evalscope/backend/rag_eval/cmteb/arguments.py,sha256=698UvPVZp5Ncq_p25P_67SQkYaW2tLSCHenUOZ0n5OI,2217
-evalscope/backend/rag_eval/cmteb/base.py,sha256=sJqTRCej7vk5ASirk21hOobX1_Hz7BO1LIHJFOGLuE4,2731
-evalscope/backend/rag_eval/cmteb/task_template.py,sha256=HreVxyRiF2QUe4Dy9_zKNp1WU74342RWHV5_B8ycXG0,2537
-evalscope/backend/rag_eval/cmteb/tasks/Classification.py,sha256=7adR40W6Uu58-QR9jCUP4k7TdAnG0oT225v4xHXah2g,10635
-evalscope/backend/rag_eval/cmteb/tasks/Clustering.py,sha256=7j1Hts_r4Nv8DlbIiPFMaU1JDxCYgu0wO0JI8T_Y6X8,8969
-evalscope/backend/rag_eval/cmteb/tasks/PairClassification.py,sha256=2WkaTE-jF8jqsu1UcNDqN8A4567UzW5boD_0B83j-9A,4008
-evalscope/backend/rag_eval/cmteb/tasks/Reranking.py,sha256=50h-lXaRcb5s6ZpIgnfk5mU7iZur8ZDxwsaFbrqSZ_o,5462
-evalscope/backend/rag_eval/cmteb/tasks/Retrieval.py,sha256=wUxiQH5aOmWNS4YswACyHqBn5xqP5eyvsq6U9WSp5R0,11457
-evalscope/backend/rag_eval/cmteb/tasks/STS.py,sha256=6GMaoCANM-IKYLk4srHOYr_eurav3DGihHMQeJPXR6k,12054
-evalscope/backend/rag_eval/cmteb/tasks/__init__.py,sha256=FrtoBosHq9iRp3yfZEAxWa5NkYhHtA20NmHDG6eiPNU,1421
-evalscope/backend/rag_eval/ragas/__init__.py,sha256=tHB7XGREmcrx8ulF-JZWWoHsEbn2s-PFyWFdGzOZQcw,190
-evalscope/backend/rag_eval/ragas/arguments.py,sha256=plVc2_3auVG5z91ExzBdkbNIhMVjyi_xQYbEzlV0iNw,1208
-evalscope/backend/rag_eval/ragas/task_template.py,sha256=795NHXzGdeqa15ONV1AgDZywpMHucaIlvk_EBF0CK98,3868
 evalscope/backend/vlm_eval_kit/__init__.py,sha256=xTgHM95lWzh4s0W7zxLwYkgUbPAZfAb0UoGGmyyBXrs,83
-evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=ewhpE9yzsqf5ED6kqsqek2YEgg96GBQOupxtVNhaXxI,6046
-evalscope/backend/vlm_eval_kit/custom_dataset.py,sha256=Yz2A5kB1E8DYBnjuVCA6TTPtLjhg8vYKeJTh6FU_Ecw,1645
+evalscope/backend/vlm_eval_kit/backend_manager.py,sha256=k52qTUqkp1kJivKn8bVrKoF8cng4xYTQLUmjnH_CWPM,6080
+evalscope/backend/vlm_eval_kit/custom_dataset.py,sha256=zC40Jw9bIqcGKuWS9oKPAlQdBARc-zY3sJlSiU-u-sI,1625
 evalscope/benchmarks/__init__.py,sha256=6TKP35wfKf7R_h870fsEtcIlIAgomKOcukNL9M-5I1Y,162
 evalscope/benchmarks/benchmark.py,sha256=EmwYyFdrAHBGMkSbsMZQOR_62Q0CSKl8zeLlr7xvJdQ,2159
 evalscope/benchmarks/data_adapter.py,sha256=eVQvOQYQOQbIl8UlvOEUqRThL3FP3aUD6DSlqF1bqO0,10395
@@ -107,7 +91,7 @@ evalscope/cli/cli.py,sha256=uZ-qC8WBsLd5-Hn94d43sSGg0UC_12RebSD4ToKjypg,844
 evalscope/cli/start_perf.py,sha256=TL6bMXYl3ln-tfs5uBmzb9x94uxz6f3PBFIt1l7g3VA,994
 evalscope/cli/start_server.py,sha256=ATGLP2TE0aImJNicpehdzBuFlNb50F7KhyL4A_ZSoGU,3885
 evalscope/evaluator/__init__.py,sha256=S6MU1O_iiNAaKxNIhO9MEmdW-BSNf_YH2l6NQ9lxVNo,103
-evalscope/evaluator/evaluator.py,sha256=eSCgPPDGfIJfKu0cthhbDLFm1xMhj_869iT3ngcQkPc,30817
+evalscope/evaluator/evaluator.py,sha256=gB408byOpu269Psh6MjYC9-a_uv9GvThoT7t07Oqh6w,30712
 evalscope/evaluator/rating_eval.py,sha256=cJbkyXIuwFUZoe7ZJZM6eUskNd9zlORgndckuon2OQ8,5768
 evalscope/evaluator/reviewer/__init__.py,sha256=I_ANdxdcIHpkIzIXc1yKOlWwzb4oY0FwTPq1kYtgzQw,50
 evalscope/evaluator/reviewer/auto_reviewer.py,sha256=JycPYti9h1j_8DRcu_rc5U0wkEASHYg-XBqrUUoiO-Q,17054
@@ -117,7 +101,7 @@ evalscope/metrics/math_accuracy.py,sha256=1PCy1VUNYg48JcGy-6SUmUDZNwPeAkMW1QQ_lX
 evalscope/metrics/metrics.py,sha256=sDZljGiZwgHsFZ5eNi65-3z3BLCdIwWUzPcq2QpKf1k,12545
 evalscope/metrics/rouge_metric.py,sha256=sN0r-sXXc-nJUdFrthQPAv1VFdOCrF6zzIYDKaLSgrU,4522
 evalscope/metrics/bundled_rouge_score/__init__.py,sha256=PwbTdk8168FwDJe_l8XIqDuBgZQooDsP31vj7di05Fs,650
-evalscope/metrics/bundled_rouge_score/rouge_scorer.py,sha256=vhzIMSQezhZuJzGndymWjB_iRbDdECoEidOIdNL3NAM,12213
+evalscope/metrics/bundled_rouge_score/rouge_scorer.py,sha256=xSLis-zx1hnHuj_9JI7HuUKMS4ZQsX-D8wECZg4D2bg,11450
 evalscope/models/__init__.py,sha256=zG27J2HSeKPGiAIUE7QLPHEPLyXLsfaDwYI_TDXjpCg,145
 evalscope/models/dummy_chat_model.py,sha256=xE8wcFVSCkvizEJ-B8ojX0Ir01Q5KrN5mapjMQaQtbg,1325
 evalscope/models/model.py,sha256=ZzzVzZHVzuzdt5F1r-rEBT44ZfW9B7R1spsrV-T8nSw,3020
@@ -185,12 +169,12 @@ evalscope/tools/rewrite_eval_results.py,sha256=ZVi2hVjiTOmR_O5IaLv6qnQNpMz6FnDb9
 evalscope/utils/__init__.py,sha256=6RjACRYUSpGj6fkZ7NzYpl0lFppQCp9KVn5ktZe626s,128
 evalscope/utils/arena_utils.py,sha256=RMkymUv9Cxs37arUntzgDY5P0Dand2jGpsb7uy6wZmg,7670
 evalscope/utils/completion_parsers.py,sha256=61l8CTh1VxHgRoMDhtznpAhuJp47MssGgS-LdEe_h80,2997
-evalscope/utils/logger.py,sha256=cf3U400Mx1speMMNXorjwEE8noDz5Mbd-9PNgaulGeY,3013
+evalscope/utils/logger.py,sha256=Ycd0W17Z_oiByPuPX3_umNrOCHjT9O_e_Kws7ZWUSvU,1855
 evalscope/utils/task_cfg_parser.py,sha256=LiNQ2X8lbZU0cODpaY_PbKyUhNoxZIC495UsLJigX64,138
-evalscope/utils/task_utils.py,sha256=IMtBSBUp3H95Ko0vn8Q55Wmz2SFZXSfjVy49tyomL_g,537
+evalscope/utils/task_utils.py,sha256=Mv_u_f4Z91zcUeko6acZCmnOAPRfk61kf_dliLzG5Yk,459
 evalscope/utils/utils.py,sha256=zHo9hfxGBUVKE2xNMR7lDoEvfRnk4V4946DEfXQhlq4,20509
-evalscope-0.5.5.dist-info/METADATA,sha256=scs7UaBcWE2qpewo_oe6ZB8HX5CtbohPBvom6UjUY5w,20943
-evalscope-0.5.5.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-evalscope-0.5.5.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
-evalscope-0.5.5.dist-info/top_level.txt,sha256=jNR-HMn3TR8Atolq7_4rW8IWVX6GhvYV5_1Y_KbJKlY,10
-evalscope-0.5.5.dist-info/RECORD,,
+evalscope-0.5.5rc0.dist-info/METADATA,sha256=DZDMgIPMwCBputs0_kU74reMXftr7Y9ERi2dCmC29Gs,20708
+evalscope-0.5.5rc0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+evalscope-0.5.5rc0.dist-info/entry_points.txt,sha256=Qr4oTgGhg_K-iUtKwVH6lWUhFHDUiH9trIqydHGTEug,56
+evalscope-0.5.5rc0.dist-info/top_level.txt,sha256=jNR-HMn3TR8Atolq7_4rW8IWVX6GhvYV5_1Y_KbJKlY,10
+evalscope-0.5.5rc0.dist-info/RECORD,,

evalscope/backend/rag_eval/__init__.py

@@ -1,3 +0,0 @@
-from evalscope.backend.rag_eval.utils.embedding import EmbeddingModel
-from evalscope.backend.rag_eval.utils.llm import LLM
-from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager

evalscope/backend/rag_eval/backend_manager.py

@@ -1,68 +0,0 @@
-import os
-from typing import Optional, Union
-from evalscope.utils import is_module_installed, get_valid_list
-from evalscope.backend.base import BackendManager
-from evalscope.utils.logger import get_logger
-
-
-logger = get_logger()
-
-
-class RAGEvalBackendManager(BackendManager):
-    def __init__(self, config: Union[str, dict], **kwargs):
-        """BackendManager for VLM Evaluation Kit
-
-        Args:
-            config (Union[str, dict]): the configuration yaml-file or the configuration dictionary
-        """
-        super().__init__(config, **kwargs)
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f"Check `{module_name}` Installed")
-        else:
-            logger.error(f"Please install `{module_name}` first")
-
-    def run_mteb(self):
-        from evalscope.backend.rag_eval.cmteb import ModelArguments, EvalArguments
-        from evalscope.backend.rag_eval.cmteb import one_stage_eval, two_stage_eval
-
-        if len(self.model_args) > 2:
-            raise ValueError("Not support multiple models yet")
-
-        # Convert arguments to dictionary
-        model_args_list = [ModelArguments(**args).to_dict() for args in self.model_args]
-        eval_args = EvalArguments(**self.eval_args).to_dict()
-
-        if len(model_args_list) == 1:
-            one_stage_eval(model_args_list[0], eval_args)
-        else:  # len(model_args_list) == 2
-            two_stage_eval(model_args_list[0], model_args_list[1], eval_args)
-
-    def run_ragas(self):
-        from evalscope.backend.rag_eval.ragas import rag_eval, testset_generation
-        from evalscope.backend.rag_eval.ragas import (
-            TestsetGenerationArguments,
-            EvaluationArguments,
-        )
-
-        if self.testset_args is not None:
-            testset_generation(TestsetGenerationArguments(**self.testset_args))
-        if self.eval_args is not None:
-            rag_eval(EvaluationArguments(**self.eval_args))
-
-    def run(self, *args, **kwargs):
-        tool = self.config_d.pop("tool")
-        if tool.lower() == "mteb":
-            self._check_env("mteb")
-            self.model_args = self.config_d["model"]
-            self.eval_args = self.config_d["eval"]
-            self.run_mteb()
-        elif tool.lower() == "ragas":
-            self._check_env("ragas")
-            self.testset_args = self.config_d.get("testset_generation", None)
-            self.eval_args = self.config_d.get("eval", None)
-            self.run_ragas()
-        else:
-            raise ValueError(f"Unknown tool: {tool}")

evalscope/backend/rag_eval/cmteb/__init__.py

@@ -1,4 +0,0 @@
-from evalscope.backend.rag_eval.cmteb.tasks import *
-from evalscope.backend.rag_eval.cmteb.base import *
-from evalscope.backend.rag_eval.cmteb.arguments import ModelArguments, EvalArguments
-from evalscope.backend.rag_eval.cmteb.task_template import one_stage_eval, two_stage_eval

evalscope/backend/rag_eval/cmteb/arguments.py

@@ -1,59 +0,0 @@
-from dataclasses import dataclass, field
-from typing import List, Optional, Union, Dict, Any
-
-
-@dataclass
-class ModelArguments:
-    # Arguments for embeding model: sentence transformer or cross encoder
-    model_name_or_path: str = ""  # model name or path
-    is_cross_encoder: bool = False  # whether the model is a cross encoder
-    # pooling mode: Either “cls”, “lasttoken”, “max”, “mean”, “mean_sqrt_len_tokens”, or “weightedmean”.
-    pooling_mode: Optional[str] = None
-    max_seq_length: int = 512  # max sequence length
-    # prompt for llm based model
-    prompt: str = ""
-    # model kwargs
-    model_kwargs: dict = field(default_factory=lambda: {"torch_dtype": "auto"})
-    # config kwargs
-    config_kwargs: Dict[str, Any] = field(default_factory=dict)
-    # encode kwargs
-    encode_kwargs: dict = field(
-        default_factory=lambda: {"show_progress_bar": True, "batch_size": 32}
-    )
-    hub: str = "modelscope"  # modelscope or huggingface
-
-    def to_dict(self) -> Dict[str, Any]:
-        return {
-            "model_name_or_path": self.model_name_or_path,
-            "is_cross_encoder": self.is_cross_encoder,
-            "pooling_mode": self.pooling_mode,
-            "max_seq_length": self.max_seq_length,
-            "prompt": self.prompt,
-            "model_kwargs": self.model_kwargs,
-            "config_kwargs": self.config_kwargs,
-            "encode_kwargs": self.encode_kwargs,
-            "hub": self.hub,
-        }
-
-
-@dataclass
-class EvalArguments:
-    # Evaluation
-    tasks: List[str] = field(default_factory=list)  # task names
-    verbosity: int = 2  # verbosity level 0-3
-    output_folder: str = "outputs"  # output folder
-    overwrite_results: bool = True  # overwrite results
-    limits: Optional[int] = None  # limit number of samples
-    hub: str = "modelscope"  # modelscope or huggingface
-    top_k: int = 5
-
-    def to_dict(self) -> Dict[str, Any]:
-        return {
-            "tasks": self.tasks,
-            "verbosity": self.verbosity,
-            "output_folder": self.output_folder,
-            "overwrite_results": self.overwrite_results,
-            "limits": self.limits,
-            "hub": self.hub,
-            "top_k": 5,
-        }