evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/super_gpqa/utils.py

@@ -0,0 +1,86 @@
+# flake8: noqa
+import re
+from collections.abc import Sequence
+
+
+def safe_regex_search(pattern, text, flags=0):
+    try:
+        return re.search(pattern, text, flags)
+    except Exception as e:
+        print(f'Regex match error: {str(e)}')
+        return None
+
+
+def extract_option_labels(text, options='ABCDEFGHIJ'):
+    if not isinstance(text, str) or not isinstance(options, str):
+        return 'error'
+
+    text = text.rstrip()
+    last_line = text.split('\n')[-1]
+
+    option_str = ''.join([chr(65 + i) for i in range(len(options))]) if options else 'ABCDEFGHIJ'
+
+    patterns = [
+        # e.g. "The final answer to this question is: A."
+        #      "The best option is $\boxed{B}:"
+        #      "The correct answer is (C)."
+        f'[Tt]he\s+(?:\w+\s+)?(?:answer|option)(?:\w+\s+)?\s+is?:?\s*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+
+        # e.g. "ANSWER: A"
+        #      "Answer: $\boxed{B}."
+        #      "ANSWER: (C):"
+        f'(?i:Answer)[\*\s]*:\s*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+
+        # e.g. "A"
+        #      "$\boxed{B}$"
+        #      "(C)."
+        #      "[D]:"
+        f'^[^\w\r\n]*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+    ]
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, last_line, re.IGNORECASE)
+        if match:
+            return match.group(1)
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, text, re.IGNORECASE)
+        if match:
+            return match.group(1)
+
+    return None
+
+
+def extract_option_content(text, options_content=None):
+    if not isinstance(text, str) or not isinstance(options_content, Sequence):
+        return 'error'
+
+    escaped_options_content = [re.escape(option_content) for option_content in options_content]
+    escaped_options_content_str = '|'.join(escaped_options_content)
+
+    text = text.rstrip()
+    last_line = text.split('\n')[-1]
+
+    patterns = [
+        f'[Tt]he\s+(?:\w+\s+)?(?:answer|option)(?:\w+\s+)?\s+is:?\s*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+        f'(?i:Answer)\s*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+        f'^[^\w\r\n]*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+    ]
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, last_line)
+        if match:
+            if match.group(1) in escaped_options_content:
+                return options_content[escaped_options_content.index(match.group(1))]
+            else:
+                return match.group(1)
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, text)
+        if match:
+            if match.group(1) in escaped_options_content:
+                return options_content[escaped_options_content.index(match.group(1))]
+            else:
+                return match.group(1)
+
+    return None

evalscope/benchmarks/tau_bench/tau2_bench/generation.py

@@ -0,0 +1,158 @@
+import json
+import sys
+import tau2.utils.llm_utils as tau_llm_utils
+from tau2.data_model.message import AssistantMessage, Message, ToolCall
+from tau2.data_model.tasks import Task
+from tau2.environment.tool import Tool
+from tau2.run import run_task
+from tau2.utils.llm_utils import to_litellm_messages
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+from evalscope.api.dataset.dataset import Sample
+from evalscope.api.messages.chat_message import dict_to_chat_message
+from evalscope.api.model import GenerateConfig, get_model
+from evalscope.api.model.model import Model
+from evalscope.api.model.model_output import ChatCompletionChoice, ModelOutput
+from evalscope.api.tool.tool_info import ToolInfo
+from evalscope.constants import EvalType
+from evalscope.models.utils.openai import openai_chat_choices
+from evalscope.utils.function_utils import run_once
+
+MODEL_DICT: Dict[str, Model] = {
+    'agent': None,
+    'user': None,
+}
+
+_MODEL_PATCHED: bool = False
+_ORIGINAL_TAU2_GENERATE: Optional[Callable[..., Any]] = None
+
+
+def _patch_tau2_generate(new_generate: Callable[..., Any]) -> None:
+    """Fan-out monkey patch for Tau2 when consumers did `from ... import generate`."""
+    global _MODEL_PATCHED, _ORIGINAL_TAU2_GENERATE
+    if _MODEL_PATCHED:
+        return
+
+    original = getattr(tau_llm_utils, 'generate', None)
+    if original is None:
+        raise RuntimeError('tau2.utils.llm_utils.generate not found')
+
+    # Replace on the source module first
+    if original is not new_generate:
+        tau_llm_utils.generate = new_generate
+
+    # Fan-out to all tau2 submodules that may hold a direct reference
+    for mod_name, mod in list(sys.modules.items()):
+        if not (isinstance(mod_name, str) and mod_name.startswith('tau2')):
+            continue
+        mod_obj = sys.modules.get(mod_name)
+        if mod_obj is None:
+            continue
+        try:
+            # Common direct binding: `generate` at module top-level
+            if getattr(mod_obj, 'generate', None) is original:
+                setattr(mod_obj, 'generate', new_generate)
+            # Replace any other aliases that equal the original function
+            for attr, val in list(vars(mod_obj).items()):
+                if val is original:
+                    setattr(mod_obj, attr, new_generate)
+        except Exception:
+            # Best-effort: ignore modules that disallow setattr or have weird loaders
+            pass
+
+    _ORIGINAL_TAU2_GENERATE = original
+    _MODEL_PATCHED = True
+
+
+@run_once
+def build_model(agent_model, adapter_instance):
+
+    user_server = get_model(
+        model=adapter_instance.user_model,
+        eval_type=EvalType.SERVICE,
+        base_url=adapter_instance.api_base,
+        api_key=adapter_instance.api_key,
+        config=GenerateConfig(**adapter_instance.generation_config)
+    )
+    MODEL_DICT['user'] = user_server
+    MODEL_DICT['agent'] = agent_model
+    # Patch Tau2 generate function for `from ... import generate` consumers
+    _patch_tau2_generate(patched_generate)
+
+
+def patched_generate(
+    model: str,
+    messages: List[Message],
+    tools: Optional[List[Tool]] = None,
+    tool_choice: Optional[Any] = None,
+    **kwargs: Any,
+) -> AssistantMessage:
+    """
+    Generate a response via an OpenAI-compatible /chat/completions call.
+
+    - Reads EVALSCOPE_API_KEY and EVALSCOPE_BASE_URL from environment.
+    - Uses OpenAI chat format for messages/tools/tool_choice.
+    - Returns Tau2 AssistantMessage with optional tool_calls and usage.
+    """
+    global MODEL_DICT
+
+    oa_model = MODEL_DICT.get(model)
+    assert oa_model is not None, f'Model {model} not found in MODEL_DICT'
+
+    oa_messages = to_litellm_messages(messages)
+    tools = [tool.openai_schema for tool in tools] if tools else None
+    if tools and tool_choice is None:
+        tool_choice = 'auto'
+
+    # Perform request
+    completion = oa_model.generate(
+        input=[dict_to_chat_message(msg) for msg in oa_messages],
+        tools=[ToolInfo.model_validate(tool['function']) for tool in tools] if tools else None,
+        tool_choice=tool_choice,
+    )
+
+    oa_choices = openai_chat_choices(completion.choices, include_reasoning=False)
+    choice = oa_choices[0]
+    msg = choice.message
+
+    tool_calls = msg.tool_calls or []
+    tool_calls = [
+        ToolCall(
+            id=tool_call.id,
+            name=tool_call.function.name,
+            arguments=json.loads(tool_call.function.arguments),
+        ) for tool_call in tool_calls
+    ]
+    tool_calls = tool_calls or None
+    usage = completion.usage.model_dump(exclude_none=True)
+
+    return AssistantMessage(
+        role='assistant',
+        content=msg.content,
+        tool_calls=tool_calls,
+        cost=None,
+        usage=usage,
+        raw_data=completion.model_dump(),
+    )
+
+
+def predict(model: Model, sample: Sample, adapter_instance) -> ModelOutput:
+
+    build_model(agent_model=model, adapter_instance=adapter_instance)
+
+    domain = sample.subset_key
+    task = Task.model_validate(sample.metadata)
+    res = run_task(
+        domain=domain,
+        task=task,
+        agent='llm_agent_gt',
+        user='user_simulator',
+        llm_agent='agent',
+        llm_user='user',
+    )
+
+    sample.metadata['task_result'] = res.reward_info.model_dump()
+    return ModelOutput(
+        model=model.name,
+        choices=[ChatCompletionChoice.from_content(res.model_dump_json(indent=2))],
+    )

evalscope/benchmarks/tau_bench/tau2_bench/tau2_bench_adapter.py

@@ -0,0 +1,146 @@
+import os
+from collections import defaultdict
+from typing import Dict, List
+
+from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict
+from evalscope.api.dataset.loader import DictDataLoader
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.model import Model, ModelOutput
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import run_once
+from evalscope.utils.import_utils import check_import
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='tau2_bench',
+        pretty_name='τ²-bench',
+        tags=[Tags.FUNCTION_CALLING, Tags.REASONING, Tags.AGENT],
+        description='τ²-bench (Tau Squared Bench) is an extension and enhancement of the original '
+        'τ-bench (Tau Bench), which is a benchmark designed to evaluate conversational AI agents '
+        'that interact with users through domain-specific API tools and guidelines. '
+        'Please install it with `pip install git+https://github.com/sierra-research/tau2-bench@v0.2.0` '
+        'before evaluating and set a user model. [Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/tau2_bench.html)',  # noqa: E501
+        dataset_id='evalscope/tau2-bench-data',
+        subset_list=['airline', 'retail', 'telecom'],
+        aggregation='mean_and_pass_hat_k',
+        eval_split='test',
+        extra_params={
+            'user_model': 'qwen-plus',
+            'api_key': 'EMPTY',
+            'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'generation_config': {
+                'temperature': 0.0,
+                'max_tokens': 4096,
+            }
+        }
+    )
+)
+class Tau2BenchAdapter(AgentAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        check_import(
+            'tau2',
+            package='git+https://github.com/sierra-research/tau2-bench@v0.2.0',
+            raise_error=True,
+            feature_name=self.pretty_name
+        )
+
+        # setup user model args
+        self.user_model = self.extra_params.get('user_model', 'qwen-plus')
+        self.api_key = self.extra_params.get('api_key', 'EMPTY')
+        self.api_base = self.extra_params.get('api_base', 'https://dashscope.aliyuncs.com/compatible-mode/v1')
+        self.generation_config = self.extra_params.get('generation_config', {'temperature': 0.0, 'max_tokens': 4096})
+
+    def load(self):
+        # Load dataset
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            dataset_path = dataset_snapshot_download(dataset_name_or_path)
+
+        # Set Tau2 data dir
+        os.environ['TAU2_DATA_DIR'] = dataset_path
+
+        # Load data for each domain
+        from tau2.agent.llm_agent import LLMGTAgent
+        from tau2.registry import registry
+
+        data_dict = defaultdict(dict)
+        for domain_name in self.subset_list:
+            logger.info(f'Loading Tau2-Bench environment: {domain_name}')
+            # Get tasks
+            task_loader = registry.get_tasks_loader(domain_name)
+            tasks = task_loader()
+            tasks = [task for task in tasks if LLMGTAgent.check_valid_task(task)]
+            tasks = [task.model_dump(exclude_unset=True) for task in tasks]
+
+            # load dataset
+            dataset = DictDataLoader(
+                dict_list=tasks,
+                sample_fields=self.record_to_sample,
+                limit=self.limit,
+                repeats=self.repeats,
+                shuffle=self.shuffle,
+            ).load()
+
+            data_dict[domain_name] = dataset
+
+        test_dataset = DatasetDict(data_dict)
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict) -> Sample:
+        """Convert a data record to a Sample object."""
+        return Sample(
+            input=[ChatMessageUser(content=record['description']['purpose'] or '')],
+            target='',  # Will use the record for evaluation
+            subset_key=record['user_scenario']['instructions']['domain'],
+            metadata=record  # Store the full record for evaluation
+        )
+
+    def _on_inference(self, model: Model, sample: Sample) -> ModelOutput:
+        from .generation import predict
+        return predict(model, sample, adapter_instance=self)
+
+    def match_score(self, original_prediction: str, filtered_prediction: str, reference: str, task_state) -> Score:
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        try:
+            # Parse the prediction to get the reward
+            task_result = task_state.metadata['task_result']
+            reward = task_result['reward']
+
+            score.value = {
+                'acc': float(reward),
+            }
+            score.explanation = f'Task completed with reward: {reward}'
+            score.metadata = {
+                'task_result': task_result,
+            }
+            score.main_score_name = 'acc'
+
+        except Exception as e:
+            score.value = {'acc': 0.0}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+            score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/tau_bench/tau_bench/generation.py

@@ -0,0 +1,147 @@
+from typing import Any, Dict, List, Optional
+
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import dict_to_chat_message
+from evalscope.api.model import Model, ModelOutput
+from evalscope.api.model.model_output import ChatCompletionChoice
+from evalscope.api.tool import ToolInfo
+from evalscope.models.utils.openai import openai_chat_choices
+from evalscope.utils.function_utils import run_once
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@run_once
+def _patch_agent_solve(model: Model):
+    """Patch ToolCallingAgent.solve method to use custom model configuration"""
+    from tau_bench.agents.tool_calling_agent import ToolCallingAgent, message_to_action
+    from tau_bench.envs.base import Env
+    from tau_bench.types import RESPOND_ACTION_NAME, Action, SolveResult
+
+    def patched_solve(
+        self,
+        env: Env,
+        task_index: Optional[int] = None,
+        max_num_steps: int = 30,
+    ) -> SolveResult:
+        env_reset_res = env.reset(task_index=task_index)
+        obs = env_reset_res.observation
+        info = env_reset_res.info.model_dump()
+        reward = 0.0
+        messages: List[Dict[str, Any]] = [
+            {
+                'role': 'system',
+                'content': self.wiki
+            },
+            {
+                'role': 'user',
+                'content': obs
+            },
+        ]
+
+        for step_index in range(max_num_steps):
+            res = model.generate(
+                input=[dict_to_chat_message(msg) for msg in messages],
+                tools=[ToolInfo.model_validate(tool['function']) for tool in self.tools_info]
+            )
+            oai_res = openai_chat_choices(res.choices, include_reasoning=False)
+
+            next_message = oai_res[0].message.model_dump(exclude_none=True)
+
+            action = message_to_action(next_message)
+
+            env_response = env.step(action)
+            reward = env_response.reward
+            info = {**info, **env_response.info.model_dump()}
+
+            if action.name != RESPOND_ACTION_NAME:
+                next_message['tool_calls'] = next_message['tool_calls'][:1]
+                messages.extend([
+                    next_message,
+                    {
+                        'role': 'tool',
+                        'tool_call_id': next_message['tool_calls'][0]['id'],
+                        'name': next_message['tool_calls'][0]['function']['name'],
+                        'content': env_response.observation,
+                    },
+                ])
+            else:
+                messages.extend([
+                    next_message,
+                    {
+                        'role': 'user',
+                        'content': env_response.observation
+                    },
+                ])
+            logger.debug(f'Task: {task_index} Step: {step_index} finished')
+
+            if env_response.done:
+                break
+
+        return SolveResult(
+            reward=reward,
+            info=info,
+            messages=messages,
+            total_cost=0,
+        )
+
+    ToolCallingAgent.solve = patched_solve
+
+    return 'ToolCallingAgent.solve patched successfully'
+
+
+def predict(model: Model, sample: Sample) -> ModelOutput:
+    """
+    Generate predictions for tau_bench tasks using the model.
+
+    Args:
+        model: The model to use for prediction
+        sample: The sample containing task metadata
+
+    Returns:
+        ModelOutput containing the prediction results
+    """
+    from tau_bench.agents.tool_calling_agent import ToolCallingAgent
+    from tau_bench.envs import get_env
+
+    _patch_agent_solve(model)
+    try:
+        # Extract task information from sample metadata
+        task_data = sample.metadata
+        env_name = task_data['env_name']
+        task_index = task_data['task_index']
+
+        # Direct call to tau_bench_server adapter's solve method
+        # This method can be implemented to solve specific tasks in the TauBench environment
+        isolated_env = get_env(
+            env_name=env_name,
+            user_strategy='llm',
+            user_model='dummy',  # Use dummy model to prevent errors
+            user_provider='openai',  # Use dummy provider to prevent errors
+            task_split='test',
+            task_index=task_index,
+        )
+        agent = ToolCallingAgent(
+            tools_info=isolated_env.tools_info,
+            wiki=isolated_env.wiki,
+            model='dummy',  # Use dummy model to prevent errors
+            provider='dummy',  # Use dummy provider to prevent errors
+            temperature=0,  # dummy temperature to prevent errors
+        )
+
+        res = agent.solve(env=isolated_env, task_index=task_index)
+
+        sample.metadata['task_result'] = res.model_dump(exclude_none=True)
+        return ModelOutput(
+            model=model.name,
+            choices=[ChatCompletionChoice.from_content(res.model_dump_json(indent=2))],
+        )
+
+    except Exception as e:
+        logger.error(f'Error in tau_bench prediction: {str(e)}')
+        sample.metadata['task_result'] = {'reward': 0, 'error': str(e)}
+        return ModelOutput(
+            model=model.name,
+            choices=[ChatCompletionChoice.from_content('')],
+        )