evalscope 0.17.0__py3-none-any.whl → 0.17.1__py3-none-any.whl

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -35,7 +35,7 @@ SUBJECT_MAPPING = {
 @Benchmark.register(
     name='bfcl_v3',
     pretty_name='BFCL-v3',
-    tags=['Agent'],
+    tags=['Agent', 'Function Calling'],
     description=
     'Berkeley Function Calling Leaderboard (BFCL), the **first comprehensive and executable function call evaluation** '
     'dedicated to assessing Large Language Models\' (LLMs) ability to invoke functions. Unlike previous evaluations, '

evalscope/benchmarks/data_adapter.py

@@ -168,6 +168,11 @@ class DataAdapter(ABC):
         If you want to support local dataset, please rewrite this method in xxx_data_adapter.
         Use modelscope.msdatasets.MsDataset.load to load the dataset from local by default.
         """
+        # remove dataset_infos.json file if exists, since MsDataset will occur an error if it exists.
+        dataset_infos_path = os.path.join(dataset_name_or_path, 'dataset_infos.json')
+        if os.path.exists(dataset_infos_path):
+            logger.info(f'Removing dataset_infos.json file at {dataset_infos_path} to avoid MsDataset errors.')
+            os.remove(dataset_infos_path)
         return self.load_from_hub(dataset_name_or_path, subset_list, None, **kwargs)
 
     def load_with_snapshot(self,
@@ -382,7 +387,7 @@ class DataAdapter(ABC):
         pass
 
     def gen_prompt_data(self,
-                        prompt: str,
+                        prompt: str = '',
                         system_prompt: Optional[str] = None,
                         choices: Optional[List[str]] = None,
                         index: Optional[Union[int, str]] = None,
@@ -413,7 +418,8 @@ class DataAdapter(ABC):
             system_prompt=system_prompt or self.system_prompt,
             index=index or 0,
             id=id,
-            messages=messages)
+            messages=messages,
+            extra_data=kwargs.get('extra_data', None))
         return prompt_data.to_dict()
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
@@ -477,7 +483,6 @@ class DataAdapter(ABC):
         """
         return result
 
-    @abstractmethod
     def match(self, gold: Any, pred: Any) -> Any:
         """
         Match the gold answer and the predicted answer.
@@ -491,7 +496,7 @@ class DataAdapter(ABC):
         Returns:
             The match result. Usually a score (float) for chat/multiple-choice-questions.
         """
-        raise NotImplementedError
+        return 1.0 if gold == pred else 0.0
 
     def llm_match(self, gold: Any, pred: Any, judge: Optional[LLMJudge] = None, **kwargs) -> float:
         """

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -17,7 +17,8 @@ logger = get_logger()
 @Benchmark.register(
     name='general_mcq',
     pretty_name='General-MCQ',
-    description='A general multiple-choice question answering dataset.',
+    description='A general multiple-choice question answering dataset for custom evaluation. '
+    'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#mcq).',
     tags=['MCQ', 'Custom'],
     dataset_id='general_mcq',
     model_adapter=OutputType.GENERATION,

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -14,7 +14,8 @@ logger = get_logger()
 @Benchmark.register(
     name='general_qa',
     pretty_name='General-QA',
-    description='General Question Answering dataset',
+    description='A general question answering dataset for custom evaluation. '
+    'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#qa).',  # noqa: E501
     tags=['QA', 'Custom'],
     dataset_id='general_qa',
     subset_list=['default'],

evalscope/benchmarks/hle/hle_adapter.py

@@ -0,0 +1,118 @@
+import re
+from collections import defaultdict
+from typing import Any, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics import DEFAULT_PROMPT_TEMPLATE, LLMJudge, exact_match, mean
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'Biology/Medicine',
+    'Chemistry',
+    'Computer Science/AI',
+    'Engineering',
+    'Humanities/Social Science',
+    'Math',
+    'Physics',
+    'Other',
+]
+
+
+@Benchmark.register(
+    name='hle',
+    pretty_name="Humanity's-Last-Exam",
+    tags=['Knowledge', 'QA'],
+    description=
+    'Humanity\'s Last Exam (HLE) is a language model benchmark consisting of 2,500 questions across a broad range of subjects. It was created jointly by the Center for AI Safety and Scale AI. The benchmark classifies the questions into the following broad subjects: mathematics (41%), physics (9%), biology/medicine (11%), humanities/social science (9%), computer science/artificial intelligence (10%), engineering (4%), chemistry (7%), and other (9%). Around 14% of the questions require the ability to understand both text and images, i.e., multi-modality. 24% of the questions are multiple-choice; the rest are short-answer, exact-match questions.',  # noqa: E501
+    dataset_id='cais/hle',
+    subset_list=SUBSET_LIST,
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template='{query}\n\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+)
+class HLEAdapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.llm_as_a_judge = True
+
+    def load(self, **kwargs):
+        kwargs['subset_list'] = ['default']
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='category', format='{}')
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        # remove image preview
+        input_d.pop('image_preview', None)
+        input_d.pop('rationale_image', None)
+        # generate prompt
+        question = input_d['question']
+        prompt = self.prompt_template.format(query=question)
+        image = input_d.get('image', None)
+        # build messages for multi-modal input
+        messages = []
+        if self.system_prompt:
+            messages.append({'role': 'system', 'content': self.system_prompt})
+        if image:
+            messages.append({
+                'role':
+                'user',
+                'content': [{
+                    'type': 'text',
+                    'text': prompt
+                }, {
+                    'type': 'image_url',
+                    'image_url': {
+                        'url': image
+                    }
+                }]
+            })
+        else:
+            messages.append({'role': 'user', 'content': prompt})
+        return self.gen_prompt_data(prompt='', messages=messages)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        return input_d['answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
+        # Extract the answer from the model output \boxed{answer}
+        match = re.search(r'\\boxed{([^}]*)}', result)
+        if match:
+            return match.group(1).strip()
+        else:
+            logger.warning(f'No answer found in the model output: {result}')
+            return ''
+
+    def llm_parse_pred_result(self, result, raw_input_d=None, **kwargs) -> str:
+        return result.strip()
+
+    def match(self, gold: str, pred: str) -> dict:
+        # simple match
+        return {
+            'AverageAccuracy': 1.0 if exact_match(gold, pred) else 0.0,
+        }
+
+    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
+        raw_input = kwargs.get('raw_input', None)
+        question = raw_input['question']
+        # get grading response
+        prompt = judge.build_prompt(pred, gold, question)
+        judge_response = judge(prompt)
+        score = judge.get_score(judge_response)
+        return {
+            'AverageAccuracy': score,
+            'response': judge_response,
+        }
+
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
+        # zip dict answers
+        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
+
+        return super().compute_metric(res_dict, **kwargs)

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -22,7 +22,8 @@ logger = get_logger()
     few_shot_num=0,
     train_split=None,
     eval_split='test',
-    prompt_template='Complete the following python code:\n{query}',
+    prompt_template=
+    'Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{query}',  # noqa: E501
     extra_params={
         'num_workers': 4,
         'timeout': 4
@@ -76,26 +77,9 @@ class HumanevalAdapter(DataAdapter):
 
     @classmethod
     def _postprocess(cls, text: str) -> str:
-        if '```' in text:
-            blocks = re.findall(r'```(.*?)```', text, re.DOTALL)
-            if len(blocks) == 0:
-                text = text.split('```')[1]  # fall back to default strategy
-            else:
-                text = blocks[0]  # fetch the first code block
-                if not text.startswith('\n'):  # in case starting with ```python
-                    text = text[max(text.find('\n') + 1, 0):]
-        if text.strip().startswith('from') or text.strip().startswith('import'):
-            def_idx = text.find('def')
-            if def_idx != -1:
-                text = text[max(text.find('\n', def_idx) + 1, 0):]
-        text = text.split('\n\n')[0]
-        if text.strip().startswith('def'):
-            text = '\n'.join(text.split('\n')[1:])
-        if not text.startswith('    '):
-            if text.startswith(' '):
-                text = '    ' + text.lstrip()
-            else:
-                text = '\n'.join(['    ' + line for line in text.split('\n')])
+        blocks = re.findall(r'```\w*\n(.*?)```', text, re.DOTALL)
+        if len(blocks) >= 1:
+            text = blocks[0]
         return text
 
     def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -144,7 +144,7 @@ SUBJECT_MAPPING = {
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
-    few_shot_num=5,
+    few_shot_num=0,
     train_split='train',
     eval_split='test',
     prompt_template=

evalscope/benchmarks/tau_bench/tau_bench_adapter.py

@@ -0,0 +1,110 @@
+import importlib
+from collections import defaultdict
+from typing import Dict, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics import Metric, mean, metric_registry
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='tau_bench',
+    pretty_name='τ-bench',
+    tags=['Reasoning', 'Agent', 'Function Calling'],
+    description='A benchmark emulating dynamic conversations between a user (simulated by language models) '
+    'and a language agent provided with domain-specific API tools and policy guidelines. '
+    'Please install it with `pip install git+https://github.com/sierra-research/tau-bench` before evaluating and set a user model. ',  # noqa: E501
+    dataset_id='https://github.com/sierra-research/tau-bench',
+    model_adapter='tau_bench_server',
+    subset_list=['airline', 'retail'],
+    metric_list=['Pass^1'],
+    eval_split='test',
+    extra_params={
+        'user_model': 'qwen-plus',
+        'api_key': 'EMPTY',
+        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+        'generation_config': {
+            'temperature': 0.7,
+            'max_new_tokens': 1024
+        }
+    })
+class TauBenchAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        spec = importlib.util.find_spec('tau_bench')
+        if spec is None:
+            raise ImportError(
+                '`tau_bench` not found, please install it with `pip install git+https://github.com/sierra-research/tau-bench` before evaluating.'  # noqa: E501
+            )
+
+        metric_registry.register(Metric(name='Pass^1', object=mean))
+
+        # setup user model args
+        extra_params = kwargs.get('extra_params', {})
+        self.user_model = extra_params.get('user_model', 'qwen-plus')
+        self.api_key = extra_params.get('api_key', 'EMPTY')
+        self.api_base = extra_params.get('api_base', 'https://dashscope.aliyuncs.com/compatible-mode/v1')
+        self.generation_config = extra_params.get('generation_config', {'temperature': 0.7, 'max_new_tokens': 1024})
+
+        self._patch_env_completion()
+
+    def _patch_env_completion(self) -> str:
+        from tau_bench.envs.user import LLMUserSimulationEnv
+
+        def new_generate_next_message(self, messages):
+            from evalscope.models import ServerModelAdapter
+
+            user_server = ServerModelAdapter(
+                api_url=adapter_instance.api_base,
+                model_id=adapter_instance.user_model,
+                api_key=adapter_instance.api_key)
+            request_json = user_server.make_request(
+                input_item={'messages': messages}, infer_cfg=adapter_instance.generation_config)
+            res = user_server.send_request(request_json)
+
+            message = res['choices'][0]['message']
+            self.messages.append(message)
+            self.total_cost = 0
+            return message['content']
+
+        # get the current instance of TauBenchAdapter
+        adapter_instance = self
+        LLMUserSimulationEnv.generate_next_message = new_generate_next_message
+
+    def load(self, **kwargs):
+        from tau_bench.envs import get_env
+
+        data_dict = defaultdict(dict)
+        for env_name in self.subset_list:
+            logger.info(f'Loading TauBench environment: {env_name}')
+            env = get_env(
+                env_name=env_name,
+                user_strategy='llm',
+                user_model='dummy',  # Use dummy model to prevent errors
+                user_provider='openai',  # Use dummy provider to prevent errors
+                task_split=self.eval_split,
+            )
+            tasks = []
+            for i in range(len(env.tasks)):
+                tasks.append({
+                    'task_index': i,
+                    'env_name': env_name,
+                })
+            data_dict[env_name][self.eval_split] = tasks
+
+        return data_dict
+
+    def gen_prompt(self, input_d, subset_name, few_shot_list, **kwargs):
+        return self.gen_prompt_data(extra_data=input_d)
+
+    def get_gold_answer(self, input_d):
+        return ''
+
+    def match(self, gold, pred):
+        import json
+        res = json.loads(pred)
+        return res.get('reward', 0.0)

evalscope/benchmarks/tool_bench/tool_bench_adapter.py

@@ -1,3 +1,4 @@
+import json
 from typing import Dict, List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
@@ -8,7 +9,7 @@ from evalscope.metrics import Metric, mean, metric_registry
 @Benchmark.register(
     name='tool_bench',
     pretty_name='ToolBench-Static',
-    tags=['Reasoning', 'Agent'],
+    tags=['Reasoning', 'Agent', 'Function Calling'],
     description='ToolBench is a benchmark for evaluating AI models on tool use tasks. '
     'It includes various subsets such as in-domain and out-of-domain, '
     'each with its own set of problems that require step-by-step reasoning to arrive at the correct answer. '
@@ -40,6 +41,11 @@ class ToolBenchAdapter(DataAdapter):
         for message in messages:
             if 'name' in message:
                 del message['name']
+            if 'role' in message:
+                if message['role'] == 'function':
+                    content = json.dumps(message, ensure_ascii=False)
+                    message['role'] = 'user'
+                    message['content'] = content
         return self.gen_prompt_data(prompt='', messages=messages)
 
     def get_gold_answer(self, input_d: dict) -> str:

evalscope/benchmarks/utils.py

@@ -13,6 +13,7 @@ class PromptData:
     multi_choices: Optional[List[str]] = None
     id: Optional[str] = None
     messages: Optional[List[dict]] = None
+    extra_data: Optional[Dict] = None
 
     def to_dict(self) -> Dict:
         return {k: v for k, v in asdict(self).items() if v is not None}

evalscope/constants.py

@@ -41,27 +41,6 @@ class MetricsConstant:
     ]
 
 
-class MetricMembers:
-
-    # Math accuracy metric
-    MATH_ACCURACY = 'math_accuracy'
-
-    # Code pass@k metric
-    CODE_PASS_K = 'code_pass_k'
-
-    # Code rouge metric
-    ROUGE = 'rouge'
-
-    # ELO rating system for pairwise comparison
-    ELO = 'elo'
-
-    # Pairwise comparison win/lose and tie(optional)
-    PAIRWISE = 'pairwise'
-
-    # Rating score for single model
-    SCORE = 'score'
-
-
 class ArenaWinner:
 
     MODEL_A = 'model_a'
@@ -172,6 +151,11 @@ class JudgeStrategy:
     LLM_RECALL = 'llm_recall'
 
 
+class JudgeScoreType:
+    NUMERIC = 'numeric'  # numeric score
+    PATTERN = 'pattern'  # pattern matching score
+
+
 class ModelTask:
     TEXT_GENERATION = 'text_generation'
     IMAGE_GENERATION = 'image_generation'

evalscope/evaluator/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.evaluator.evaluator import Evaluator
+from .evaluator import Evaluator

evalscope/evaluator/evaluator.py

@@ -7,17 +7,19 @@ from collections import OrderedDict, defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from copy import deepcopy
 from tqdm import tqdm
-from typing import Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from evalscope.benchmarks import DataAdapter
 from evalscope.config import TaskConfig
 from evalscope.constants import AnswerKeys, DumpMode, EvalStage, EvalType, JudgeStrategy, ReviewKeys
-from evalscope.models import BaseModelAdapter
 from evalscope.report import Report, gen_table
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, gen_hash, jsonl_to_list
 from evalscope.utils.logger import get_logger
 from evalscope.utils.model_utils import dict_torch_dtype_to_str
 
+if TYPE_CHECKING:
+    from evalscope.models import BaseModelAdapter
+
 logger = get_logger()
 
 
@@ -38,7 +40,7 @@ class Evaluator(object):
 
     def __init__(self,
                  data_adapter: DataAdapter,
-                 model_adapter: BaseModelAdapter,
+                 model_adapter: 'BaseModelAdapter',
                  outputs: OutputsStructure = None,
                  task_cfg: TaskConfig = None,
                  **kwargs):

evalscope/metrics/__init__.py

@@ -5,7 +5,7 @@ from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
     from .completion_parsers import ResponseParser, lmsys_parser, ranking_parser
-    from .llm_judge import LLMJudge
+    from .llm_judge import DEFAULT_NUMERIC_SCORE_TEMPLATE, DEFAULT_PROMPT_TEMPLATE, LLMJudge
     from .math_parser import extract_answer, math_equal, strip_answer_string
     from .metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, simple_f1_score,
                           weighted_mean)
@@ -34,6 +34,8 @@ else:
         ],
         'llm_judge': [
             'LLMJudge',
+            'DEFAULT_PROMPT_TEMPLATE',
+            'DEFAULT_NUMERIC_SCORE_TEMPLATE',
         ],
         'math_parser': [
             'extract_answer',

evalscope/metrics/completion_parsers.py

@@ -218,3 +218,10 @@ class ResponseParser:
         # Join options into a regex pattern separated by '|', to match any of the options
         options_pattern = '|'.join(escaped_options)
         return options_pattern
+
+
+if __name__ == '__main__':
+    result = '**Answer: A **Answer: C**'
+    options = ['A', 'B', 'C', 'D']
+    parsed_result = ResponseParser.parse_first_option(result, options)
+    print(f'Parsed result: {parsed_result}')  # Should print 'C'

evalscope/metrics/llm_judge.py

@@ -2,6 +2,7 @@ import os
 import re
 from typing import Any, Dict, List, Optional
 
+from evalscope.constants import JudgeScoreType
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -56,7 +57,7 @@ class LLMJudge:
             generation_config: Optional[Dict[str, Any]] = None,
             score_pattern: Optional[str] = None,
             score_mapping: Optional[Dict[str, float]] = None,
-            score_type: str = 'pattern',  # 'pattern', 'numeric'
+            score_type: str = JudgeScoreType.PATTERN,  # 'pattern', 'numeric'
             **kwargs):
         """
         Initialize LLMJudge metric.
@@ -82,11 +83,11 @@ class LLMJudge:
 
         # Default score mapping for A/B pattern
         self.score_type = score_type
-        if self.score_type == 'numeric':
+        if self.score_type == JudgeScoreType.NUMERIC:
             self.score_pattern = score_pattern or r'\[\[(\d+(?:\.\d+)?)\]\]'
             self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE',
                                                                      DEFAULT_NUMERIC_SCORE_TEMPLATE)
-        elif self.score_type == 'pattern':
+        elif self.score_type == JudgeScoreType.PATTERN:
             self.score_pattern = score_pattern or r'(A|B)'
             self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
         else:
@@ -159,9 +160,9 @@ class LLMJudge:
             return 0.0
 
         # choose extraction method based on score_type
-        if self.score_type == 'numeric':
+        if self.score_type == JudgeScoreType.NUMERIC:
             return self._extract_numeric_score(response)
-        elif self.score_type == 'pattern':
+        elif self.score_type == JudgeScoreType.PATTERN:
             return self._extract_pattern_score(response)
 
     def _extract_numeric_score(self, response: str) -> Optional[float]:

evalscope/metrics/metrics.py

@@ -9,7 +9,7 @@ import random
 import sacrebleu
 from collections import defaultdict
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Dict, List, Union
+from typing import Dict, List, Union
 
 
 def mean(arr: list):
@@ -22,16 +22,28 @@ def mean(arr: list):
 
 
 def pass_at_k(arr: Union[List[int], List[List[int]]], k: int = 1) -> float:
+    """
+    Calculates the pass@k metric using the calculate_pass_at_k function.
+
+    Args:
+        arr: List of binary values (1 for correct, 0 for incorrect) or list of such lists
+        k: Number of attempts allowed
+
+    Returns:
+        The average pass@k score across all problems
+    """
     if not arr:
         return 0.0
+    if not isinstance(arr[0], list):
+        # If arr is a simple list of binary results, convert it to a list of lists
+        arr = [arr]
 
-    def sub_pass_at_k(sub_arr: List[int]) -> float:
-        return 1.0 if any(sub_arr[:k]) else 0.0
+    # For list of lists case, each inner list represents attempts for one problem
+    num_samples = [len(sub_arr) for sub_arr in arr]
+    num_correct = [sum(sub_arr) for sub_arr in arr]
+    pass_at_k_values = calculate_pass_at_k(num_samples, num_correct, k)
 
-    if isinstance(arr[0], list):
-        return sum(sub_pass_at_k(sub_arr) for sub_arr in arr) / len(arr)
-    else:
-        return sum(arr) / len(arr)
+    return float(np.mean(pass_at_k_values))
 
 
 def pop_stddev(arr):

evalscope/models/__init__.py

@@ -4,12 +4,11 @@ from typing import TYPE_CHECKING
 from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
-    from .adapters import (BaseModelAdapter, ChatGenerationModelAdapter, ContinuationLogitsModelAdapter,
+    from .adapters import (BaseModelAdapter, BFCLAdapter, ChatGenerationModelAdapter, ContinuationLogitsModelAdapter,
                            CustomModelAdapter, MultiChoiceModelAdapter, ServerModelAdapter, T2IModelAdapter,
-                           initialize_model_adapter)
+                           TauBenchAdapter, initialize_model_adapter)
     from .custom import CustomModel, DummyCustomModel
     from .local_model import LocalModel, get_local_model
-    from .model import BaseModel, ChatBaseModel, OpenAIModel
     from .register import get_model_adapter
 
 else:
@@ -23,6 +22,8 @@ else:
             'CustomModelAdapter',
             'ServerModelAdapter',
             'T2IModelAdapter',
+            'TauBenchAdapter',
+            'BFCLAdapter',
         ],
         'custom': [
             'CustomModel',
@@ -32,11 +33,6 @@ else:
             'LocalModel',
             'get_local_model',
         ],
-        'model': [
-            'BaseModel',
-            'ChatBaseModel',
-            'OpenAIModel',
-        ],
         'register': [
             'get_model_adapter',
         ],

evalscope/models/adapters/__init__.py

@@ -5,15 +5,10 @@ from .choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdap
 from .custom_adapter import CustomModelAdapter
 from .server_adapter import ServerModelAdapter
 from .t2i_adapter import T2IModelAdapter
+from .tau_bench_adapter import TauBenchAdapter
 
 __all__ = [
-    'initialize_model_adapter',
-    'BaseModelAdapter',
-    'ChatGenerationModelAdapter',
-    'ContinuationLogitsModelAdapter',
-    'MultiChoiceModelAdapter',
-    'CustomModelAdapter',
-    'ServerModelAdapter',
-    'BFCLAdapter',
-    'T2IModelAdapter',
+    'initialize_model_adapter', 'BaseModelAdapter', 'ChatGenerationModelAdapter', 'ContinuationLogitsModelAdapter',
+    'MultiChoiceModelAdapter', 'CustomModelAdapter', 'ServerModelAdapter', 'BFCLAdapter', 'T2IModelAdapter',
+    'TauBenchAdapter'
 ]