evalscope 0.13.2__py3-none-any.whl → 0.15.0__py3-none-any.whl

evalscope/utils/import_utils.py

@@ -0,0 +1,66 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Copyright 2023-present the HuggingFace Inc. team.
+
+import importlib
+import os
+from itertools import chain
+from types import ModuleType
+from typing import Any
+
+from .logger import get_logger
+
+logger = get_logger()  # pylint: disable=invalid-name
+
+
+class _LazyModule(ModuleType):
+    """
+    Module class that surfaces all objects but only performs associated imports when the objects are requested.
+    """
+
+    # Very heavily inspired by optuna.integration._IntegrationModule
+    # https://github.com/optuna/optuna/blob/master/optuna/integration/__init__.py
+    def __init__(self, name, module_file, import_structure, module_spec=None, extra_objects=None):
+        super().__init__(name)
+        self._modules = set(import_structure.keys())
+        self._class_to_module = {}
+        for key, values in import_structure.items():
+            for value in values:
+                self._class_to_module[value] = key
+        # Needed for autocompletion in an IDE
+        self.__all__ = list(import_structure.keys()) + list(chain(*import_structure.values()))
+        self.__file__ = module_file
+        self.__spec__ = module_spec
+        self.__path__ = [os.path.dirname(module_file)]
+        self._objects = {} if extra_objects is None else extra_objects
+        self._name = name
+        self._import_structure = import_structure
+
+    # Needed for autocompletion in an IDE
+    def __dir__(self):
+        result = super().__dir__()
+        # The elements of self.__all__ that are submodules may or may not be in the dir already, depending on whether
+        # they have been accessed or not. So we only add the elements of self.__all__ that are not already in the dir.
+        for attr in self.__all__:
+            if attr not in result:
+                result.append(attr)
+        return result
+
+    def __getattr__(self, name: str) -> Any:
+        if name in self._objects:
+            return self._objects[name]
+        if name in self._modules:
+            value = self._get_module(name)
+        elif name in self._class_to_module.keys():
+            module = self._get_module(self._class_to_module[name])
+            value = getattr(module, name)
+        else:
+            raise AttributeError(f'module {self.__name__} has no attribute {name}')
+
+        setattr(self, name, value)
+        return value
+
+    def _get_module(self, module_name: str):
+        return importlib.import_module('.' + module_name, self.__name__)
+
+    def __reduce__(self):
+        return self.__class__, (self._name, self.__file__, self._import_structure)

evalscope/utils/utils.py

@@ -76,21 +76,21 @@ def dict_torch_dtype_to_str(d: Dict[str, Any]) -> dict:
 class ResponseParser:
 
     @staticmethod
-    def parse_first_capital(text: str) -> str:
+    def parse_first_capital(text: str, options: list[str]) -> str:
         for t in text:
-            if t.isupper():
+            if t.isupper() and (t in options):
                 return t
         return ''
 
     @staticmethod
-    def parse_last_capital(text: str) -> str:
+    def parse_last_capital(text: str, options: list[str]) -> str:
         for t in text[::-1]:
-            if t.isupper():
+            if t.isupper() and (t in options):
                 return t
         return ''
 
     @staticmethod
-    def parse_first_option_with_choices(text: str, options: list) -> str:
+    def parse_first_option_with_choices(text: str, options: list[str]) -> str:
         """
         Find first valid option for text.
 
@@ -98,7 +98,7 @@ class ResponseParser:
             text: The text to parse.
             options: The options to find. e.g. ['A', 'B', 'C', 'D']
         """
-        options_concat = '|'.join([str(i) for i in options])
+        options_concat = ResponseParser.process_options(options)
 
         patterns = [
             rf'答案是?\s?([{options_concat}])',
@@ -155,48 +155,61 @@ class ResponseParser:
                 for i in options:
                     if i in outputs:
                         return i
-        return ''
+        # If no match found, try to find the last capital letter in the text
+        last_capital = ResponseParser.parse_last_capital(text, options)
+        if last_capital:
+            return last_capital
+        return 'No valid option found'
 
     @staticmethod
-    def parse_first_option(text: str) -> str:
+    def parse_first_option(text: str, options: list[str]) -> str:
         """
         Find first valid option for text.
 
         Args:
             text: The text to parse.
         """
+        options_pattern = ResponseParser.process_options(options)
+
         patterns = [
-            r'answer is \(?(\w+)\)?',
-            r'[Aa]nswer:\s*(\w+)',
-            r'[Tt]he correct answer is:\s*(\w+)',
-            r'[Tt]he correct answer is:\n\s*(\w+)',
-            r'[Tt]he correct answer is:\n\n-\s*(\w+)',
-            r'[Tt]he answer might be:\n\n-\s*(\w+)',
-            r'[Tt]he answer is \s*(\w+)',
+            rf'[Aa]nswer:\s*({options_pattern})',
+            rf'ANSWER:\s*({options_pattern})',
+            rf'answer is \(?({options_pattern})\)?',
+            rf'[Tt]he correct answer is:\s*({options_pattern})',
+            rf'[Tt]he correct answer is:\n\s*({options_pattern})',
+            rf'[Tt]he correct answer is:\n\n-\s*({options_pattern})',
+            rf'[Tt]he answer might be:\n\n-\s*({options_pattern})',
+            rf'[Tt]he answer is \s*({options_pattern})',
         ]
 
         regexes = [re.compile(pattern) for pattern in patterns]
         for regex in regexes:
-            match = regex.search(text)
-            if match:
-                return match.group(1)
-        return ''
+            matches = regex.search(text)
+            if matches:
+                return matches.group(1)
+        # If no match found, try to find the last capital letter in the text
+        last_capital = ResponseParser.parse_last_capital(text, options)
+        if last_capital:
+            return last_capital
+        return 'No valid option found'
+
 
     @staticmethod
-    def parse_first_capital_multi(text: str) -> str:
-        match = re.search(r'([A-D]+)', text)
+    def parse_bracketed_answer(text: str, options: list[str]) -> str:
+        options = ResponseParser.process_options(options)
+        # Match the first occurrence of the options in angle brackets
+        match = re.search(rf'<({options})>', text)
         if match:
             return match.group(1)
-        return ''
+        return 'No valid option found'
 
     @staticmethod
-    def parse_last_option(text: str, options: str) -> str:
-        match = re.findall(rf'([{options}])', text)
-        if match:
-            return match[-1]
-        return ''
-
-
+    def process_options(options: list[str]) -> str:
+        # Escape each option to ensure special characters in options are treated literally
+        escaped_options = [re.escape(option) for option in options]
+        # Join options into a regex pattern separated by '|', to match any of the options
+        options_pattern = '|'.join(escaped_options)
+        return options_pattern
 
 def normalize_score(score: Union[float, dict], keep_num: int = 4) -> Union[float, dict]:
     """
@@ -299,3 +312,9 @@ def seed_everything(seed: int):
         torch.cuda.manual_seed_all(seed)
         torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.benchmark = False
+
+if __name__ == '__main__':
+    options = ['A', 'B', 'C', 'D']
+    answers = ['Context .... ANSWER: A', 'answer: A']
+    for answer in answers:
+        print(ResponseParser.parse_first_option(answer, options))

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.13.2'
-__release_datetime__ = '2025-04-01 20:00:00'
+__version__ = '0.15.0'
+__release_datetime__ = '2025-04-29 00:00:00'

{evalscope-0.13.2.dist-info → evalscope-0.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.13.2
+Version: 0.15.0
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -26,8 +26,10 @@ Requires-Dist: latex2sympy2
 Requires-Dist: matplotlib
 Requires-Dist: modelscope[framework]
 Requires-Dist: nltk>=3.9
+Requires-Dist: omegaconf
 Requires-Dist: openai
 Requires-Dist: pandas
+Requires-Dist: pillow
 Requires-Dist: pyarrow
 Requires-Dist: pyyaml
 Requires-Dist: requests
@@ -39,9 +41,15 @@ Requires-Dist: seaborn
 Requires-Dist: sympy
 Requires-Dist: tabulate
 Requires-Dist: torch
+Requires-Dist: torchvision
 Requires-Dist: tqdm
 Requires-Dist: transformers>=4.33
 Requires-Dist: word2number
+Provides-Extra: aigc
+Requires-Dist: diffusers; extra == "aigc"
+Requires-Dist: iopath; extra == "aigc"
+Requires-Dist: open-clip-torch; extra == "aigc"
+Requires-Dist: opencv-python; extra == "aigc"
 Provides-Extra: all
 Requires-Dist: accelerate; extra == "all"
 Requires-Dist: datasets<=3.2.0,>=3.0.0; extra == "all"
@@ -53,8 +61,10 @@ Requires-Dist: latex2sympy2; extra == "all"
 Requires-Dist: matplotlib; extra == "all"
 Requires-Dist: modelscope[framework]; extra == "all"
 Requires-Dist: nltk>=3.9; extra == "all"
+Requires-Dist: omegaconf; extra == "all"
 Requires-Dist: openai; extra == "all"
 Requires-Dist: pandas; extra == "all"
+Requires-Dist: pillow; extra == "all"
 Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: requests; extra == "all"
@@ -66,17 +76,18 @@ Requires-Dist: seaborn; extra == "all"
 Requires-Dist: sympy; extra == "all"
 Requires-Dist: tabulate; extra == "all"
 Requires-Dist: torch; extra == "all"
+Requires-Dist: torchvision; extra == "all"
 Requires-Dist: tqdm; extra == "all"
 Requires-Dist: transformers>=4.33; extra == "all"
 Requires-Dist: word2number; extra == "all"
 Requires-Dist: ms-opencompass>=0.1.4; extra == "all"
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "all"
-Requires-Dist: langchain<0.3.0; extra == "all"
-Requires-Dist: langchain-community<0.3.0; extra == "all"
-Requires-Dist: langchain-core<0.3.0; extra == "all"
-Requires-Dist: langchain-openai<0.3.0; extra == "all"
+Requires-Dist: langchain<0.4.0,>=0.3.0; extra == "all"
+Requires-Dist: langchain-community<0.4.0,>=0.3.0; extra == "all"
+Requires-Dist: langchain-core<0.4.0,>=0.3.0; extra == "all"
+Requires-Dist: langchain-openai<0.4.0,>=0.3.0; extra == "all"
 Requires-Dist: mteb==1.19.4; extra == "all"
-Requires-Dist: ragas==0.2.9; extra == "all"
+Requires-Dist: ragas==0.2.14; extra == "all"
 Requires-Dist: webdataset>0.2.0; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: fastapi; extra == "all"
@@ -86,6 +97,10 @@ Requires-Dist: transformers; extra == "all"
 Requires-Dist: unicorn; extra == "all"
 Requires-Dist: gradio==5.4.0; extra == "all"
 Requires-Dist: plotly<6.0.0,>=5.23.0; extra == "all"
+Requires-Dist: diffusers; extra == "all"
+Requires-Dist: iopath; extra == "all"
+Requires-Dist: open-clip-torch; extra == "all"
+Requires-Dist: opencv-python; extra == "all"
 Provides-Extra: app
 Requires-Dist: gradio==5.4.0; extra == "app"
 Requires-Dist: plotly<6.0.0,>=5.23.0; extra == "app"
@@ -99,12 +114,12 @@ Requires-Dist: sse-starlette; extra == "perf"
 Requires-Dist: transformers; extra == "perf"
 Requires-Dist: unicorn; extra == "perf"
 Provides-Extra: rag
-Requires-Dist: langchain<0.3.0; extra == "rag"
-Requires-Dist: langchain-community<0.3.0; extra == "rag"
-Requires-Dist: langchain-core<0.3.0; extra == "rag"
-Requires-Dist: langchain-openai<0.3.0; extra == "rag"
+Requires-Dist: langchain<0.4.0,>=0.3.0; extra == "rag"
+Requires-Dist: langchain-community<0.4.0,>=0.3.0; extra == "rag"
+Requires-Dist: langchain-core<0.4.0,>=0.3.0; extra == "rag"
+Requires-Dist: langchain-openai<0.4.0,>=0.3.0; extra == "rag"
 Requires-Dist: mteb==1.19.4; extra == "rag"
-Requires-Dist: ragas==0.2.9; extra == "rag"
+Requires-Dist: ragas==0.2.14; extra == "rag"
 Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "vlmeval"
@@ -121,7 +136,7 @@ Requires-Dist: ms-vlmeval>=0.0.9; extra == "vlmeval"
 </p>
 
 <p align="center">
-<img src="https://img.shields.io/badge/python-%E2%89%A53.8-5be.svg">
+<img src="https://img.shields.io/badge/python-%E2%89%A53.9-5be.svg">
 <a href="https://badge.fury.io/py/evalscope"><img src="https://badge.fury.io/py/evalscope.svg" alt="PyPI version" height="18"></a>
 <a href="https://pypi.org/project/evalscope"><img alt="PyPI - Downloads" src="https://static.pepy.tech/badge/evalscope"></a>
 <a href="https://github.com/modelscope/evalscope/pulls"><img src="https://img.shields.io/badge/PR-welcome-55EB99.svg"></a>
@@ -199,6 +214,10 @@ Please scan the QR code below to join our community groups:
 
 ## 🎉 News
 
+- 🔥 **[2025.04.29]** Added Qwen3 Evaluation Best Practices, [welcome to read 📖](https://evalscope.readthedocs.io/en/latest/best_practice/qwen3.html)
+- 🔥 **[2025.04.27]** Support for text-to-image evaluation: Supports 8 metrics including MPS, HPSv2.1Score, etc., and evaluation benchmarks such as EvalMuse, GenAI-Bench. Refer to the [user documentation](https://evalscope.readthedocs.io/en/latest/user_guides/aigc/t2i.html) for more details.
+- 🔥 **[2025.04.10]** Model service stress testing tool now supports the `/v1/completions` endpoint (the default endpoint for vLLM benchmarking)
+- 🔥 **[2025.04.08]** Support for evaluating embedding model services compatible with the OpenAI API has been added. For more details, check the [user guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html#configure-evaluation-parameters).
 - 🔥 **[2025.03.27]** Added support for [AlpacaEval](https://www.modelscope.cn/datasets/AI-ModelScope/alpaca_eval/dataPeview) and [ArenaHard](https://modelscope.cn/datasets/AI-ModelScope/arena-hard-auto-v0.1/summary) evaluation benchmarks. For usage notes, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html)
 - 🔥 **[2025.03.20]** The model inference service stress testing now supports generating prompts of specified length using random values. Refer to the [user guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/examples.html#using-the-random-dataset) for more details.
 - 🔥 **[2025.03.13]** Added support for the [LiveCodeBench](https://www.modelscope.cn/datasets/AI-ModelScope/code_generation_lite/summary) code evaluation benchmark, which can be used by specifying `live_code_bench`. Supports evaluating QwQ-32B on LiveCodeBench, refer to the [best practices](https://evalscope.readthedocs.io/en/latest/best_practice/eval_qwq.html).
@@ -212,15 +231,14 @@ Please scan the QR code below to join our community groups:
 - 🔥 **[2025.02.13]** Added support for evaluating DeepSeek distilled models, including AIME24, MATH-500, and GPQA-Diamond datasets，refer to [best practice](https://evalscope.readthedocs.io/en/latest/best_practice/deepseek_r1_distill.html); Added support for specifying the `eval_batch_size` parameter to accelerate model evaluation.
 - 🔥 **[2025.01.20]** Support for visualizing evaluation results, including single model evaluation results and multi-model comparison, refer to the [📖 Visualizing Evaluation Results](https://evalscope.readthedocs.io/en/latest/get_started/visualization.html) for more details; Added [`iquiz`](https://modelscope.cn/datasets/AI-ModelScope/IQuiz/summary) evaluation example, evaluating the IQ and EQ of the model.
 - 🔥 **[2025.01.07]** Native backend: Support for model API evaluation is now available. Refer to the [📖 Model API Evaluation Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html#api) for more details. Additionally, support for the `ifeval` evaluation benchmark has been added.
+<details><summary>More</summary>
+
 - 🔥🔥 **[2024.12.31]** Support for adding benchmark evaluations, refer to the [📖 Benchmark Evaluation Addition Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/add_benchmark.html); support for custom mixed dataset evaluations, allowing for more comprehensive model evaluations with less data, refer to the [📖 Mixed Dataset Evaluation Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/collection/index.html).
 - 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.
 - 🔥 **[2024.11.26]** The model inference service performance evaluator has been completely refactored: it now supports local inference service startup and Speed Benchmark; asynchronous call error handling has been optimized. For more details, refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/index.html).
 - 🔥 **[2024.10.31]** The best practice for evaluating Multimodal-RAG has been updated, please check the [📖 Blog](https://evalscope.readthedocs.io/zh-cn/latest/blog/RAG/multimodal_RAG.html#multimodal-rag) for more details.
 - 🔥 **[2024.10.23]** Supports multimodal RAG evaluation, including the assessment of image-text retrieval using [CLIP_Benchmark](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/clip_benchmark.html), and extends [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html) to support end-to-end multimodal metrics evaluation.
 - 🔥 **[2024.10.8]** Support for RAG evaluation, including independent evaluation of embedding models and rerankers using [MTEB/CMTEB](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html), as well as end-to-end evaluation using [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html).
-
-<details><summary>More</summary>
-
 - 🔥 **[2024.09.18]** Our documentation has been updated to include a blog module, featuring some technical research and discussions related to evaluations. We invite you to [📖 read it](https://evalscope.readthedocs.io/en/refact_readme/blog/index.html).
 - 🔥 **[2024.09.12]** Support for LongWriter evaluation, which supports 10,000+ word generation. You can use the benchmark [LongBench-Write](evalscope/third_party/longbench_write/README.md) to measure the long output quality as well as the output length.
 - 🔥 **[2024.08.30]** Support for custom dataset evaluations, including text datasets and multimodal image-text datasets.
@@ -503,6 +521,10 @@ Reference: Performance Testing [📖 User Guide](https://evalscope.readthedocs.i
 
 ![wandb sample](https://modelscope.oss-cn-beijing.aliyuncs.com/resource/wandb_sample.png)
 
+**Supports swanlab for recording results**
+
+![swanlab sample](https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/swanlab.png)
+
 **Supports Speed Benchmark**
 
 It supports speed testing and provides speed benchmarks similar to those found in the [official Qwen](https://qwen.readthedocs.io/en/latest/benchmark/speed_benchmark.html) reports: