evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/third_party/thinkbench/infer.py

@@ -0,0 +1,130 @@
+import os
+
+from evalscope import TaskConfig, run_task
+
+DASHSCOPE_API_KEY = 'sk-723135c241x'
+
+def eval_distill_qwen():
+    model_name = 'DeepSeek-R1-Distill-Qwen-7B'
+    dataset_name = 'math_500'
+    subsets = ['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5']
+
+    task_config = TaskConfig(
+        api_url='http://0.0.0.0:8801/v1/chat/completions',
+        model=model_name,
+        eval_type='service',
+        datasets=[dataset_name],
+        dataset_args={dataset_name: {'few_shot_num': 0, 'subset_list': subsets}},
+        eval_batch_size=32,
+        generation_config={
+            'max_tokens': 20000,  # avoid exceed max length
+            'temperature': 0.6,
+            'top_p': 0.95,
+            'n': 1,
+        },
+    )
+    run_task(task_config)
+
+
+def eval_math_qwen():
+    model_name = 'Qwen2.5-Math-7B-Instruct'
+    dataset_name = 'math_500'
+    subsets = ['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5']
+
+    task_config = TaskConfig(
+        api_url='http://0.0.0.0:8801/v1/chat/completions',
+        model=model_name,
+        eval_type='service',
+        datasets=[dataset_name],
+        dataset_args={dataset_name: {'few_shot_num': 0, 'subset_list': subsets}},
+        eval_batch_size=32,
+        generation_config={
+            'max_tokens': 3000,  # avoid exceed max length
+            'temperature': 0.6,
+            'top_p': 0.95,
+            'n': 3,
+        },
+    )
+    run_task(task_config)
+
+def eval_r1():
+    model_name = 'deepseek-r1'
+    dataset_name = 'math_500'
+    subsets = ['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5']
+
+    task_config = TaskConfig(
+        api_url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+        api_key=DASHSCOPE_API_KEY,
+        model=model_name,
+        eval_type='service',
+        datasets=[dataset_name],
+        dataset_args={dataset_name: {'few_shot_num': 0, 'subset_list': subsets}},
+        eval_batch_size=8,
+        generation_config={
+            'max_tokens': 20000,  # avoid exceed max length
+            'temperature': 0.6,
+            'top_p': 0.95,
+            'n': 1,
+        },
+        use_cache='./outputs/20250307_000404',
+        timeout=36000,
+        stream=True
+    )
+    run_task(task_config)
+
+
+def eval_distill_32b():
+    model_name = 'deepseek-r1-distill-qwen-32b'
+    dataset_name = 'math_500'
+    subsets = ['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5']
+
+    task_config = TaskConfig(
+        api_url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+        api_key=DASHSCOPE_API_KEY,
+        model=model_name,
+        eval_type='service',
+        datasets=[dataset_name],
+        dataset_args={dataset_name: {'few_shot_num': 0, 'subset_list': subsets}},
+        eval_batch_size=5,
+        generation_config={
+            'max_tokens': 12000,  # avoid exceed max length
+            'temperature': 0.6,
+            'top_p': 0.95,
+            'n': 1,
+        },
+        use_cache='./outputs/20250306_235951',
+        timeout=32000,
+        stream=True
+
+    )
+    run_task(task_config)
+
+def eval_qwq():
+    model_name = 'qwq-32b-preview'
+    dataset_name = 'math_500'
+    subsets = ['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5']
+
+    task_config = TaskConfig(
+        api_url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+        api_key=os.environ['DASHSCOPE_API_KEY'],
+        model=model_name,
+        eval_type='service',
+        datasets=[dataset_name],
+        dataset_args={dataset_name: {'few_shot_num': 0, 'subset_list': subsets}},
+        eval_batch_size=32,
+        generation_config={
+            'max_tokens': 8000,  # avoid exceed max length
+            'temperature': 0.6,
+            'top_p': 0.95,
+            'n': 1,
+        },
+        use_cache='./outputs/20250221_105911'
+    )
+    run_task(task_config)
+
+if __name__ == '__main__':
+    # eval_distill_qwen()
+    # eval_math_qwen()
+    eval_r1()
+    # eval_qwq()
+    # eval_distill_32b()

evalscope/third_party/thinkbench/resources/critique_template.txt

@@ -0,0 +1,17 @@
+The following is a math problem and a solution (split into paragraphs, enclosed with tags and indexed from 0):
+
+[Math Problem]
+
+{problem}
+
+[Correct Answer]
+
+{answer}
+
+[Solution]
+
+{tagged_response}
+
+Your task is to review and critique the solution paragraph by paragraph. Once you identify an correct answer in a paragraph, return the index of the paragraph where the earliest correct answer occurs. Otherwise, return the index of -1 (which typically denotes "not found").
+
+Please put your final answer (i.e., the index) in \boxed{{}}.

evalscope/third_party/thinkbench/resources/reformat_template.txt

@@ -0,0 +1,31 @@
+I will present you with a solution to a math problem. Unfortunately, the solution lacks proper paragraphing, making it hard to read. Your task is to improve readability by reformatting the solution into well-structured paragraphs. Follow these specific guidelines:
+
+* Insert \n\n for paragraph breaks within the original solution. Do **NOT** alter any content of the original solution (the only exception is for itemized lists; see below).
+
+  - Each paragraph should represent a distinct, concise reasoning step that logically advances the solution.
+
+  - Reasoning steps can include case discussions, formula simplifications, or formula derivations. Each of these should be treated as an individual reasoning step and paragraphed accordingly.
+
+  - If an introductory analysis exists in the original solution, treat it as an initial reasoning step and place it as the first paragraph.
+
+  - Do **NOT** place any mathematical formulas in their own separate paragraphs; instead, include them within the same paragraph as the preceding text to form a cohesive reasoning step.
+
+* For any itemized lists (ordered or unordered), convert them into a written format, such as "First/Second/Third." This is the **ONLY** content modification allowed.
+
+* Avoid making paragraphs too lengthy, as long paragraphs might contain multiple reasoning steps that should be paragraphed separately.
+
+* Disregard the accuracy of the solution content. Do **NOT** alter any of the original solution's content; focus solely on structuring it into logical, readable paragraphs.
+
+* Reply with the reformatted solution directly.
+
+--------------------------------------------------
+
+Here is the math problem, and the solution that needs to be reformatted:
+
+[Math Problem]
+
+{problem}
+
+[Solution]
+
+{response}

evalscope/third_party/thinkbench/tools/llm.py

@@ -0,0 +1,48 @@
+import os
+from openai import OpenAI
+
+
+def request_url(llm_config, content):
+    try:
+        client = OpenAI(
+            api_key=llm_config['api_key'],
+            base_url=llm_config['base_url'],
+        )
+        completion = client.chat.completions.create(
+            model=llm_config['model_name'],
+            messages=[{'role': 'user', 'content': content}]
+        )
+        return completion.choices[0].message.content
+    except Exception as e:
+        print(e)
+        return None
+
+def request_qwen(content):
+    try:
+        client = OpenAI(
+            api_key=os.getenv('DASHSCOPE_API_KEY'),
+            base_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+        )
+
+        completion = client.chat.completions.create(
+            model='qwen-max',
+            messages=[{'role': 'user', 'content': content}]
+        )
+        return completion.choices[0].message.content
+    except Exception as e:
+        print(e)
+
+
+def request_local(content):
+    try:
+        client = OpenAI(
+            api_key='EMPTY',
+            base_url='http://0.0.0.0:8801/v1',
+        )
+        completion = client.chat.completions.create(
+            model='Qwen2.5-72B-Instruct',
+            messages=[{'role': 'user', 'content': content}]
+        )
+        return completion.choices[0].message.content
+    except Exception as e:
+        print(e)

evalscope/third_party/thinkbench/tools/utils.py

@@ -0,0 +1,13 @@
+import re
+
+
+def extract_answer(solution_text: str):
+    boxed_pattern = r'\\boxed\{([^}]*)\}'
+    matches = re.findall(boxed_pattern, solution_text)
+    if matches:
+        last_boxed_content = matches[-1]
+        number_pattern = r'-?\d+'
+        number_matches = re.findall(number_pattern, last_boxed_content)
+        if number_matches:
+            return number_matches[-1].strip()
+    return None

evalscope/third_party/toolbench_static/llm/swift_infer.py

@@ -1,37 +1,63 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
 from dataclasses import dataclass
-from swift.llm import get_default_template_type, get_model_tokenizer, get_template, inference
-from swift.utils import seed_everything
-
-# TODO: Support custom model for swift infer
 
 
 @dataclass
 class SwiftInferArgs:
     model_id_or_path: str
     model_type: str
+    infer_backend: str = 'vllm'  # 可选 'pt', 'vllm', 'lmdeploy'
     max_new_tokens: int = 2048
-
+    temperature: float = 0.1
+    max_batch_size: int = 16
 
 class SwiftInfer:
 
     def __init__(self, args: SwiftInferArgs):
-        model_type = args.model_type
-        template_type = get_default_template_type(model_type)
-        model, tokenizer = get_model_tokenizer(
-            model_type, model_id_or_path=args.model_id_or_path, model_kwargs={'device_map': 'auto'})
-        model.generation_config.max_new_tokens = args.max_new_tokens
-        print(f'** Generation config: {model.generation_config}')
+        # infer backend模型初始化
+        if args.infer_backend == 'pt':
+            self.engine: InferEngine = PtEngine(args.model_id_or_path, max_batch_size=args.max_batch_size)
+        elif args.infer_backend == 'vllm':
+            from swift.llm import VllmEngine
+            self.engine: InferEngine = VllmEngine(args.model_id_or_path, max_model_len=8192)
+        elif args.infer_backend == 'lmdeploy':
+            from swift.llm import LmdeployEngine
+            self.engine: InferEngine = LmdeployEngine(args.model_id_or_path)
+        else:
+            raise ValueError(f'Unsupported infer_backend: {args.infer_backend}')
+
+        # 基本配置获取 （可选）
+        self.request_config = RequestConfig(
+            max_tokens=args.max_new_tokens,
+            temperature=args.temperature,
+            stream=False  # 可以透传参数改为True进行流式推理
+        )
 
-        template = get_template(template_type, tokenizer)
-        seed_everything(42)
+    def predict(self, system: str, query: str, history: list):
+        # Swift 3.0标准接口中，消息传入的格式是：
+        # messages: [{"role": "system", "content": "<SYSTEM_PROMPT>"},
+        #            {"role": "user", "content": "用户问题内容"},
+        #            {"role": "assistant", "content": "助手回答内容"}, ...]
 
-        self.tokenizer = tokenizer
-        self.model = model
-        self.template = template
+        messages = []
+        if system.strip():
+            messages.append({'role': 'system', 'content': system})
 
-    def predict(self, system: str, query: str, history: list):
+        # 将历史对话拼接进message中
+        for qa_pair in history:
+            # 假定 history 中每个元素形如 ("user input", "model response")，请根据你的数据格式进行调整。
+            user_answer, model_response = qa_pair
+            messages.append({'role': 'user', 'content': user_answer})
+            messages.append({'role': 'assistant', 'content': model_response})
+
+        # 添加本次用户问题
+        messages.append({'role': 'user', 'content': query})
+
+        infer_request = InferRequest(messages=messages)
+
+        # 进行推理
+        response = self.engine.infer([infer_request], self.request_config)
 
-        response, history = inference(self.model, self.template, query=query, system=system, history=history)
+        # 提取模型返回的文本结果（假设非stream模式）
+        result_text = response[0].choices[0].message.content.strip()
 
-        return response
+        return result_text

evalscope/third_party/toolbench_static/toolbench_static.py

@@ -6,11 +6,12 @@ from typing import Union
 from evalscope.third_party.toolbench_static.eval import EvalArgs, run_eval
 from evalscope.third_party.toolbench_static.infer import InferArgs, run_infer
 from evalscope.utils import get_logger
+from evalscope.utils.deprecation_utils import deprecated
 from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
 
 logger = get_logger()
 
-
+@deprecated(since='0.15.1', remove_in='0.18.0', alternative='Native implementation of ToolBench')
 def run_task(task_cfg: Union[str, dict]):
 
     if isinstance(task_cfg, str):

evalscope/utils/__init__.py

@@ -1,4 +1,84 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.utils.model_utils import EvalBackend
-from evalscope.utils.utils import *
+from typing import TYPE_CHECKING
+
+from .import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .argument_utils import BaseArgument, get_supported_params, parse_int_or_float
+    from .deprecation_utils import deprecated
+    from .function_utils import run_once, thread_safe
+    from .import_utils import get_module_path, is_module_installed
+    from .io_utils import (
+        OutputsStructure,
+        csv_to_jsonl,
+        csv_to_list,
+        dict_to_yaml,
+        gen_hash,
+        get_latest_folder_path,
+        get_valid_list,
+        json_to_dict,
+        jsonl_to_csv,
+        jsonl_to_list,
+        safe_filename,
+        yaml_to_dict,
+    )
+    from .logger import configure_logging, get_logger
+    from .model_utils import EvalBackend, dict_torch_dtype_to_str, fix_do_sample_warning, get_device, seed_everything
+
+else:
+    _import_structure = {
+        'argument_utils': [
+            'BaseArgument',
+            'parse_int_or_float',
+            'get_supported_params',
+        ],
+        'model_utils': [
+            'EvalBackend',
+            'get_device',
+            'seed_everything',
+            'dict_torch_dtype_to_str',
+            'fix_do_sample_warning',
+        ],
+        'import_utils': [
+            'is_module_installed',
+            'get_module_path',
+        ],
+        'function_utils': [
+            'thread_safe',
+            'run_once',
+        ],
+        'io_utils': [
+            'OutputsStructure',
+            'csv_to_list',
+            'json_to_dict',
+            'yaml_to_dict',
+            'get_latest_folder_path',
+            'gen_hash',
+            'dict_to_yaml',
+            'csv_to_jsonl',
+            'jsonl_to_csv',
+            'jsonl_to_list',
+            'gen_hash',
+            'get_valid_list',
+            'safe_filename',
+            'thread_safe',
+        ],
+        'deprecation_utils': [
+            'deprecated',
+        ],
+        'logger': [
+            'get_logger',
+            'configure_logging',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/utils/argument_utils.py

@@ -0,0 +1,64 @@
+import json
+from argparse import Namespace
+from inspect import signature
+
+from evalscope.utils.io_utils import json_to_dict, yaml_to_dict
+
+
+class BaseArgument:
+    """
+    BaseArgument is a base class designed to facilitate the creation and manipulation
+    of argument classes in the evalscope framework. It provides utility methods for
+    instantiating objects from various data formats and converting objects back into
+    dictionary representations.
+    """
+
+    @classmethod
+    def from_dict(cls, d: dict):
+        """Instantiate the class from a dictionary."""
+        return cls(**d)
+
+    @classmethod
+    def from_json(cls, json_file: str):
+        """Instantiate the class from a JSON file."""
+        return cls.from_dict(json_to_dict(json_file))
+
+    @classmethod
+    def from_yaml(cls, yaml_file: str):
+        """Instantiate the class from a YAML file."""
+        return cls.from_dict(yaml_to_dict(yaml_file))
+
+    @classmethod
+    def from_args(cls, args: Namespace):
+        """
+        Instantiate the class from an argparse.Namespace object.
+        Filters out None values and removes 'func' if present.
+        """
+        args_dict = {k: v for k, v in vars(args).items() if v is not None}
+
+        if 'func' in args_dict:
+            del args_dict['func']  # Note: compat CLI arguments
+
+        return cls.from_dict(args_dict)
+
+    def to_dict(self):
+        """Convert the instance to a dictionary."""
+        result = self.__dict__.copy()
+        return result
+
+    def __str__(self):
+        """Return a JSON-formatted string representation of the instance."""
+        return json.dumps(self.to_dict(), indent=4, default=str, ensure_ascii=False)
+
+
+def parse_int_or_float(num):
+    number = float(num)
+    if number.is_integer():
+        return int(number)
+    return number
+
+
+def get_supported_params(func):
+    """Get the supported parameters of a function."""
+    sig = signature(func)
+    return set(sig.parameters.keys())

evalscope/utils/chat_service.py

@@ -1,6 +1,5 @@
 import os
 import time
-import torch
 from contextlib import contextmanager
 from functools import partial
 from pydantic import BaseModel, Field
@@ -32,6 +31,7 @@ class ModelList(BaseModel):
 class ChatMessage(BaseModel):
     role: Literal['user', 'assistant', 'system']
     content: str
+    reasoning_content: Optional[str] = None
 
 
 class DeltaMessage(BaseModel):
@@ -63,10 +63,10 @@ class ChatCompletionResponseStreamChoice(BaseModel):
 
 class ChatCompletionResponse(BaseModel):
     model: str
-    object: Literal['chat.completion', 'chat.completion.chunk']
+    object: Literal['chat.completion', 'chat.completion.chunk', 'images.generations']
     choices: List[Union[ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, Any]]
     created: Optional[int] = Field(default_factory=lambda: int(time.time()))
-    usage: Optional[Usage]
+    usage: Optional[Usage] = None
 
 
 class TextCompletionRequest(BaseModel):
@@ -94,6 +94,7 @@ class TextCompletionResponse(BaseModel):
 class ChatService:
 
     def __init__(self, model_path, attn_implementation):
+        import torch
         from modelscope import AutoModelForCausalLM, AutoTokenizer
         from transformers import TextIteratorStreamer
 
@@ -174,7 +175,7 @@ class ChatService:
         )
 
     def _prepare_text_inputs(self, request: TextCompletionRequest):
-        inputs = self.tokenizer(request.prompt, return_tensors='pt', padding=True).to(self.device)
+        inputs = self.tokenizer(request.prompt, return_tensors='pt', padding=False).to(self.device)
         prompt_tokens = len(inputs['input_ids'][0])
         return inputs, prompt_tokens
 
@@ -203,8 +204,9 @@ class ChatService:
 
     def _prepare_chat_inputs(self, request: ChatCompletionRequest):
         formatted_prompt = self.tokenizer.apply_chat_template(
-            request.messages, tokenize=False, add_generation_prompt=True)
-        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=True).to(self.device)
+            request.messages, tokenize=False, add_generation_prompt=True
+        )
+        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=False).to(self.device)
         prompt_tokens = len(inputs['input_ids'][0])
         return formatted_prompt, inputs, prompt_tokens
 

evalscope/utils/deprecation_utils.py

@@ -0,0 +1,53 @@
+import functools
+import inspect
+import os
+from typing import Callable, Optional
+
+from .logger import get_logger
+
+logger = get_logger()
+
+
+def deprecated(since: str, remove_in: Optional[str] = None, alternative: Optional[str] = None) -> Callable:
+    """
+    Decorator to mark functions as deprecated.
+
+    :param since: String indicating the version since deprecation
+    :param remove_in: Optional string indicating the version when it will be removed
+    :param alternative: Optional string suggesting an alternative
+    :return: Decorated function
+    """
+
+    def decorator(func: Callable) -> Callable:
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            # Get the file name where the function is defined
+            file_name = os.path.basename(inspect.getfile(func))
+
+            # Construct the warning message
+            warning_parts = [
+                f'{func.__name__} in {file_name} has been deprecated since version {since}',
+                f'and will be removed in version {remove_in}' if remove_in else None,
+                f'Use {alternative} instead' if alternative else None
+            ]
+            warning_message = '. '.join(filter(None, warning_parts))
+
+            # Log the warning
+            logger.warning(warning_message)
+
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+def deprecated_warning(logger, message: str):
+    """
+    Log a deprecation warning.
+
+    :param logger: Logger instance to log the warning
+    :param message: Warning message to log
+    """
+    logger.warning(f'Deprecated: {message}')