evalscope 0.16.3__py3-none-any.whl → 0.17.1__py3-none-any.whl

evalscope/metrics/llm_judge.py

@@ -2,17 +2,21 @@ import os
 import re
 from typing import Any, Dict, List, Optional
 
+from evalscope.constants import JudgeScoreType
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 DEFAULT_PROMPT_TEMPLATE = """Your job is to look at a question, a gold target, and a predicted answer, and return a letter "A" or "B" to indicate whether the predicted answer is correct or incorrect.
 
-Question: {question}
+[Question]
+{question}
 
-Reference Answer: {gold}
+[Reference Answer]
+{gold}
 
-Model Answer: {pred}
+[Predicted Answer]
+{pred}
 
 Evaluate the model's answer based on correctness compared to the reference answer.
 Grade the predicted answer of this new question as one of:
@@ -22,6 +26,18 @@ B: INCORRECT
 Just return the letters "A" or "B", with no text around it.
 """  # noqa: E501
 
+
+DEFAULT_NUMERIC_SCORE_TEMPLATE = """Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response.
+Begin your evaluation by providing a short explanation. Be as objective as possible.
+After providing your explanation, you must rate the response on a scale of 0 (worst) to 1 (best) by strictly following this format: \"[[rating]]\", for example: \"Rating: [[0.5]]\"
+
+[Question]
+{question}
+
+[Response]
+{pred}
+"""  # noqa: E501
+
 DEFAULT_JUDGE_MODEL = 'Qwen/Qwen3-235B-A22B'
 DEFAULT_API_URL = 'https://api-inference.modelscope.cn/v1/'
 
@@ -31,14 +47,18 @@ class LLMJudge:
     A metric that uses LLM to judge the quality of model predictions by comparing them with reference answers.
     """
 
-    def __init__(self,
-                 api_key: Optional[str] = None,
-                 api_url: Optional[str] = None,
-                 model_id: Optional[str] = None,
-                 system_prompt: Optional[str] = None,
-                 prompt_template: Optional[str] = None,
-                 generation_config: Optional[Dict[str, Any]] = None,
-                 **kwargs):
+    def __init__(
+            self,
+            api_key: Optional[str] = None,
+            api_url: Optional[str] = None,
+            model_id: Optional[str] = None,
+            system_prompt: Optional[str] = None,
+            prompt_template: Optional[str] = None,
+            generation_config: Optional[Dict[str, Any]] = None,
+            score_pattern: Optional[str] = None,
+            score_mapping: Optional[Dict[str, float]] = None,
+            score_type: str = JudgeScoreType.PATTERN,  # 'pattern', 'numeric'
+            **kwargs):
         """
         Initialize LLMJudge metric.
 
@@ -49,14 +69,34 @@ class LLMJudge:
             system_prompt (str, optional): System prompt for the judge
             prompt_template (str, optional): Prompt template for the judge
             generation_config (dict, optional): Generation configuration for the judge
+            score_pattern (str, optional): Regex pattern to extract score from LLM response
+            score_mapping (dict, optional): Mapping from extracted score to float value
+            score_type (str, optional): Type of score extraction strategy ('pattern', 'numeric') defaults to 'pattern'.
+                - 'pattern': Use score_pattern and score_mapping to extract categorical scores
+                - 'numeric': Treat the extracted value as a direct numerical score
         """
         self.api_key = api_key or os.environ.get('MODELSCOPE_SDK_TOKEN', 'EMPTY')
         self.api_url = api_url or os.environ.get('MODELSCOPE_API_BASE', DEFAULT_API_URL)
         self.model_id = model_id or os.environ.get('MODELSCOPE_JUDGE_LLM', DEFAULT_JUDGE_MODEL)
         self.system_prompt = system_prompt or os.environ.get('JUDGE_SYSTEM_PROMPT', None)
-        self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
         self.generation_config = generation_config or {}
 
+        # Default score mapping for A/B pattern
+        self.score_type = score_type
+        if self.score_type == JudgeScoreType.NUMERIC:
+            self.score_pattern = score_pattern or r'\[\[(\d+(?:\.\d+)?)\]\]'
+            self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE',
+                                                                     DEFAULT_NUMERIC_SCORE_TEMPLATE)
+        elif self.score_type == JudgeScoreType.PATTERN:
+            self.score_pattern = score_pattern or r'(A|B)'
+            self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
+        else:
+            raise ValueError(f"Invalid score_type: {self.score_type}. Must be 'pattern' or 'numeric'.")
+        self.score_mapping = score_mapping or {'A': 1.0, 'B': 0.0}
+
+        self._init_server_adapter()
+
+    def _init_server_adapter(self):
         from evalscope.models import ServerModelAdapter
 
         # Initialize ServerModelAdapter
@@ -95,17 +135,63 @@ class LLMJudge:
     def build_prompt(self, pred: str, gold: str, question: Optional[str] = None):
         if question is None:
             question = 'Not provided'
-        return self.prompt_template.format(question=question, pred=pred, gold=gold)
+
+        # check variables in prompt_template
+        prompt = self.prompt_template
+        if '{question}' in self.prompt_template:
+            prompt = prompt.replace('{question}', question)
+        if '{pred}' in self.prompt_template:
+            prompt = prompt.replace('{pred}', pred)
+        if '{gold}' in self.prompt_template:
+            prompt = prompt.replace('{gold}', gold)
+        return prompt
 
     def get_score(self, response: str) -> float:
+        """
+        Extract score from LLM response using the configured pattern and mapping.
+
+        Args:
+            response (str): The response from the LLM
+
+        Returns:
+            float: The numeric score extracted from the response
+        """
         if response is None:
-            return 0
-        match = re.search(r'(A|B)', response)
+            return 0.0
+
+        # choose extraction method based on score_type
+        if self.score_type == JudgeScoreType.NUMERIC:
+            return self._extract_numeric_score(response)
+        elif self.score_type == JudgeScoreType.PATTERN:
+            return self._extract_pattern_score(response)
+
+    def _extract_numeric_score(self, response: str) -> Optional[float]:
+        """extract numeric score from the response using the score_pattern"""
+        match = re.search(self.score_pattern, response)
+
+        if match:
+            # try to convert each captured group to float
+            for group in match.groups():
+                if group is not None:
+                    try:
+                        return float(group)
+                    except (ValueError, TypeError):
+                        continue
+
+            # if not found in groups, try the whole match
+            try:
+                return float(match.group(0))
+            except (ValueError, TypeError):
+                logger.warning(f'Failed to convert any extracted value to float from: {match.group(0)}')
+
+        return None
+
+    def _extract_pattern_score(self, response: str) -> float:
+        """use the score_pattern to extract categorical scores"""
+        match = re.search(self.score_pattern, response)
         if match:
             answer = match.group(0)
-            if answer == 'A':
-                return 1
-            elif answer == 'B':
-                return 0
+            return self.score_mapping.get(answer, 0.0)
         else:
-            return 0
+            logger.warning(f"No match found for pattern '{self.score_pattern}' in response: {response}")
+            return 0.0

evalscope/metrics/metrics.py

@@ -9,7 +9,7 @@ import random
 import sacrebleu
 from collections import defaultdict
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Dict, List, Union
+from typing import Dict, List, Union
 
 
 def mean(arr: list):
@@ -22,16 +22,28 @@ def mean(arr: list):
 
 
 def pass_at_k(arr: Union[List[int], List[List[int]]], k: int = 1) -> float:
+    """
+    Calculates the pass@k metric using the calculate_pass_at_k function.
+
+    Args:
+        arr: List of binary values (1 for correct, 0 for incorrect) or list of such lists
+        k: Number of attempts allowed
+
+    Returns:
+        The average pass@k score across all problems
+    """
     if not arr:
         return 0.0
+    if not isinstance(arr[0], list):
+        # If arr is a simple list of binary results, convert it to a list of lists
+        arr = [arr]
 
-    def sub_pass_at_k(sub_arr: List[int]) -> float:
-        return 1.0 if any(sub_arr[:k]) else 0.0
+    # For list of lists case, each inner list represents attempts for one problem
+    num_samples = [len(sub_arr) for sub_arr in arr]
+    num_correct = [sum(sub_arr) for sub_arr in arr]
+    pass_at_k_values = calculate_pass_at_k(num_samples, num_correct, k)
 
-    if isinstance(arr[0], list):
-        return sum(sub_pass_at_k(sub_arr) for sub_arr in arr) / len(arr)
-    else:
-        return sum(arr) / len(arr)
+    return float(np.mean(pass_at_k_values))
 
 
 def pop_stddev(arr):
@@ -223,7 +235,7 @@ def chrf(items):
     Source: https://github.com/m-popovic/chrF
     Paper: https://www.aclweb.org/anthology/W15-3049.pdf
 
-    Higher is better  # TODO I think
+    Higher is better
     """
     refs = list(zip(*items))[0]
     preds = list(zip(*items))[1]

evalscope/models/__init__.py

@@ -4,12 +4,11 @@ from typing import TYPE_CHECKING
 from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
-    from .adapters import (BaseModelAdapter, ChatGenerationModelAdapter, ContinuationLogitsModelAdapter,
+    from .adapters import (BaseModelAdapter, BFCLAdapter, ChatGenerationModelAdapter, ContinuationLogitsModelAdapter,
                            CustomModelAdapter, MultiChoiceModelAdapter, ServerModelAdapter, T2IModelAdapter,
-                           initialize_model_adapter)
+                           TauBenchAdapter, initialize_model_adapter)
     from .custom import CustomModel, DummyCustomModel
     from .local_model import LocalModel, get_local_model
-    from .model import BaseModel, ChatBaseModel, OpenAIModel
     from .register import get_model_adapter
 
 else:
@@ -23,6 +22,8 @@ else:
             'CustomModelAdapter',
             'ServerModelAdapter',
             'T2IModelAdapter',
+            'TauBenchAdapter',
+            'BFCLAdapter',
         ],
         'custom': [
             'CustomModel',
@@ -32,11 +33,6 @@ else:
             'LocalModel',
             'get_local_model',
         ],
-        'model': [
-            'BaseModel',
-            'ChatBaseModel',
-            'OpenAIModel',
-        ],
         'register': [
             'get_model_adapter',
         ],

evalscope/models/adapters/__init__.py

@@ -5,15 +5,10 @@ from .choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdap
 from .custom_adapter import CustomModelAdapter
 from .server_adapter import ServerModelAdapter
 from .t2i_adapter import T2IModelAdapter
+from .tau_bench_adapter import TauBenchAdapter
 
 __all__ = [
-    'initialize_model_adapter',
-    'BaseModelAdapter',
-    'ChatGenerationModelAdapter',
-    'ContinuationLogitsModelAdapter',
-    'MultiChoiceModelAdapter',
-    'CustomModelAdapter',
-    'ServerModelAdapter',
-    'BFCLAdapter',
-    'T2IModelAdapter',
+    'initialize_model_adapter', 'BaseModelAdapter', 'ChatGenerationModelAdapter', 'ContinuationLogitsModelAdapter',
+    'MultiChoiceModelAdapter', 'CustomModelAdapter', 'ServerModelAdapter', 'BFCLAdapter', 'T2IModelAdapter',
+    'TauBenchAdapter'
 ]

evalscope/models/adapters/base_adapter.py

@@ -53,7 +53,10 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'DataAdapter', b
         if task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
 
             if 'server' not in model_adapter_cls_str:
+                logger.warning(f'Output type {model_adapter_cls_str} is not supported for service evaluation. '
+                               f'Using server model adapter instead.')
                 model_adapter_cls_str = 'server'
+                benchmark.model_adapter = model_adapter_cls_str
 
             # init server model adapter
             model_adapter_cls = get_model_adapter(model_adapter_cls_str)
@@ -71,6 +74,7 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'DataAdapter', b
                 logger.warning(f'Output type {model_adapter_cls_str} is not supported for benchmark {benchmark.name}.'
                                f'Using {benchmark.output_types[0]} instead.')
                 model_adapter_cls_str = benchmark.output_types[0]
+                benchmark.model_adapter = model_adapter_cls_str
 
             model_adapter_cls = get_model_adapter(model_adapter_cls_str)
             return model_adapter_cls(

evalscope/models/adapters/bfcl_adapter.py

@@ -4,11 +4,13 @@ import uuid
 from typing import Any, List, Optional, Union
 
 from evalscope.utils.logger import get_logger
+from ..register import register_model_adapter
 from .server_adapter import ServerModelAdapter
 
 logger = get_logger()
 
 
+@register_model_adapter(name='bfcl_server')
 class BFCLAdapter(ServerModelAdapter):
     """
     BFCL model adapter to request remote API model and generate results for BFCL evaluation.

evalscope/models/adapters/chat_adapter.py

@@ -3,15 +3,18 @@ import time
 import torch
 from typing import Any, Dict, List, Optional, Tuple, Union
 
+from evalscope.constants import OutputType
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage, Usage
 from evalscope.utils.logger import get_logger
 from evalscope.utils.model_utils import fix_do_sample_warning
 from ..local_model import LocalModel
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
 
 
+@register_model_adapter(name=OutputType.GENERATION)
 class ChatGenerationModelAdapter(BaseModelAdapter):
     """
     Chat generation model adapter.

evalscope/models/adapters/choice_adapter.py

@@ -3,11 +3,14 @@ import time
 import torch
 from typing import List
 
+from evalscope.constants import OutputType
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 from ..local_model import LocalModel
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
 
+@register_model_adapter(name=OutputType.MULTIPLE_CHOICE)
 class MultiChoiceModelAdapter(BaseModelAdapter):
     """ The multi-choice model adapter. """
 
@@ -110,6 +113,7 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         return log_probs, {'tokens': tokens}
 
 
+@register_model_adapter(name=OutputType.CONTINUOUS)
 class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
     """
     Continuation-logits model adapter.

evalscope/models/adapters/custom_adapter.py

@@ -1,12 +1,16 @@
-from typing import Any, Dict, List, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Union
 
-from ..custom import CustomModel
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
+if TYPE_CHECKING:
+    from ..custom import CustomModel
 
+
+@register_model_adapter(name='custom')
 class CustomModelAdapter(BaseModelAdapter):
 
-    def __init__(self, custom_model: CustomModel, **kwargs):
+    def __init__(self, custom_model: 'CustomModel', **kwargs):
         """
         Custom model adapter.
 

evalscope/models/adapters/server_adapter.py

@@ -5,13 +5,15 @@ from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
 from typing import List, Optional, Union
 
+from evalscope.utils.argument_utils import get_supported_params
 from evalscope.utils.logger import get_logger
-from evalscope.utils.utils import get_supported_params
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
 
 
+@register_model_adapter(name='server')
 class ServerModelAdapter(BaseModelAdapter):
     """
     Server model adapter to request remote API model and generate results.
@@ -29,7 +31,7 @@ class ServerModelAdapter(BaseModelAdapter):
         self.api_key = api_key
 
         self.client = openai.OpenAI(
-            api_key=api_key,
+            api_key=self.api_key,
             base_url=self.api_url,
         )
         self.supported_params = get_supported_params(self.client.chat.completions.create)

evalscope/models/adapters/t2i_adapter.py

@@ -3,15 +3,18 @@ import time
 import torch
 from typing import Any, Dict, List, Optional, Tuple, Union
 
+from evalscope.constants import OutputType
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 from evalscope.utils.io_utils import OutputsStructure
 from evalscope.utils.logger import get_logger
 from ..local_model import LocalModel
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
 
 
+@register_model_adapter(name=OutputType.IMAGE_GENERATION)
 class T2IModelAdapter(BaseModelAdapter):
     """
     Text to image model adapter.

evalscope/models/adapters/tau_bench_adapter.py

@@ -0,0 +1,189 @@
+import json
+import time
+from typing import Any, Dict, List, Optional, Union
+
+from evalscope.utils.logger import get_logger
+from ..register import register_model_adapter
+from .server_adapter import ServerModelAdapter
+
+logger = get_logger()
+
+
+@register_model_adapter(name='tau_bench_server')
+class TauBenchAdapter(ServerModelAdapter):
+    """
+    TauBench model adapter to request remote API model and generate results for TauBench evaluation.
+    Support multi-turn and single-turn function calling tasks.
+    """
+
+    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
+        """
+        Args:
+            api_url: The URL of the remote API model.
+            model_id: The ID of the remote API model.
+            api_key: The API key of the remote API model.
+        """
+        super().__init__(api_url=api_url, model_id=model_id, api_key=api_key, **kwargs)
+
+        self._patch_agent_solve()
+
+    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
+        """
+        Model prediction func. For multi-turn evals, we pass a list[list[message]] to the model
+        where each list is a follow up turn in the conversation
+        each turn is a List[List[Message]]
+
+        Args:
+            inputs (List[dict]): The input data.
+            infer_cfg (dict): Inference configuration.
+
+        Returns:
+            res (List[dict]): The model prediction results.
+        """
+        infer_cfg = infer_cfg or {}
+        results = []
+
+        for input_item in inputs:
+            raw_input = input_item.get('raw_input')
+
+            res_d = self.solve(env_name=raw_input['env_name'], task_index=raw_input['task_index'], infer_cfg=infer_cfg)
+
+            wrapper_res = {
+                'choices': [{
+                    'index': 0,
+                    'message': {
+                        'content': json.dumps(res_d, ensure_ascii=False),
+                        'role': 'assistant'
+                    }
+                }],
+                'created':
+                time.time(),
+                'model':
+                self.model_id,
+                'object':
+                'chat.completion',
+                'usage': {
+                    'completion_tokens': 0,
+                    'prompt_tokens': 0,
+                    'total_tokens': 0
+                }
+            }
+
+            results.append(wrapper_res)
+
+        return results
+
+    def _patch_agent_solve(self):
+        """Patch ToolCallingAgent.solve method to use custom model configuration"""
+        from tau_bench.agents.tool_calling_agent import ToolCallingAgent, message_to_action
+        from tau_bench.envs.base import Env
+        from tau_bench.types import RESPOND_ACTION_NAME, SolveResult
+        from typing import List, Optional
+
+        def patched_solve(self,
+                          env: Env,
+                          task_index: Optional[int] = None,
+                          max_num_steps: int = 30,
+                          infer_cfg: Optional[dict] = {}) -> SolveResult:
+            env_reset_res = env.reset(task_index=task_index)
+            obs = env_reset_res.observation
+            info = env_reset_res.info.model_dump()
+            reward = 0.0
+            messages: List[Dict[str, Any]] = [
+                {
+                    'role': 'system',
+                    'content': self.wiki
+                },
+                {
+                    'role': 'user',
+                    'content': obs
+                },
+            ]
+
+            for step_index in range(max_num_steps):
+                # Use adapter's model configuration instead of agent's
+                request_json = adapter_instance.make_request(
+                    input_item={
+                        'messages': messages,
+                        'tools': self.tools_info
+                    }, infer_cfg=infer_cfg)
+                res = adapter_instance.send_request(request_json)
+
+                next_message = res['choices'][0]['message']
+                action = message_to_action(next_message)
+                env_response = env.step(action)
+                reward = env_response.reward
+                info = {**info, **env_response.info.model_dump()}
+
+                if action.name != RESPOND_ACTION_NAME:
+                    next_message['tool_calls'] = next_message['tool_calls'][:1]
+                    messages.extend([
+                        next_message,
+                        {
+                            'role': 'tool',
+                            'tool_call_id': next_message['tool_calls'][0]['id'],
+                            'name': next_message['tool_calls'][0]['function']['name'],
+                            'content': env_response.observation,
+                        },
+                    ])
+                else:
+                    messages.extend([
+                        next_message,
+                        {
+                            'role': 'user',
+                            'content': env_response.observation
+                        },
+                    ])
+                logger.debug(f'Task: {task_index} Step: {step_index} finished')
+
+                if env_response.done:
+                    break
+
+            return SolveResult(
+                reward=reward,
+                info=info,
+                messages=messages,
+                total_cost=0,
+            )
+
+        adapter_instance = self
+
+        ToolCallingAgent.solve = patched_solve
+
+        return 'ToolCallingAgent.solve patched successfully'
+
+    def solve(self, env_name, task_index, infer_cfg, **kwargs):
+        """
+        Solve a specific task in the TauBench environment.
+
+        Args:
+            env_name (str): The name of the TauBench environment.
+            task_index (int): The index of the task to solve.
+            **kwargs: Additional arguments for the task.
+
+        Returns:
+            dict: The result of the task.
+        """
+        from tau_bench.agents.tool_calling_agent import ToolCallingAgent
+        from tau_bench.envs import get_env
+
+        # This method can be implemented to solve specific tasks in the TauBench environment
+        isolated_env = get_env(
+            env_name=env_name,
+            user_strategy='llm',
+            user_model='dummy',  # Use dummy model to prevent errors
+            user_provider='openai',  # Use dummy provider to prevent errors
+            task_split='test',
+            task_index=task_index,
+        )
+        agent = ToolCallingAgent(
+            tools_info=isolated_env.tools_info,
+            wiki=isolated_env.wiki,
+            model='dummy',  # Use dummy model to prevent errors
+            provider='dummy',  # Use dummy provider to prevent errors
+            temperature=0,  # dummy temperature to prevent errors
+        )
+
+        res = agent.solve(env=isolated_env, task_index=task_index, infer_cfg=infer_cfg)
+
+        return res.model_dump()

evalscope/models/custom/dummy_model.py

@@ -50,14 +50,14 @@ class DummyCustomModel(CustomModel):
         # Must return a list of dicts with the same format as the OpenAI API.
         responses = []
         for input_item in original_inputs:
-            message = self.make_request_messages(input_item)
-            response = f'Dummy response for prompt: {message}'
+            # message = self.make_request_messages(input_item)
+            # response = f'Dummy response for prompt: {message}'
 
             res_d = {
                 'choices': [{
                     'index': 0,
                     'message': {
-                        'content': response,
+                        'content': '*PlaceHolder*',
                         'role': 'assistant'
                     }
                 }],

evalscope/models/register.py

@@ -1,6 +1,3 @@
-from evalscope.constants import OutputType
-from .adapters import *
-
 MODEL_ADAPTERS = {}
 
 
@@ -42,14 +39,3 @@ def register_model_adapter_class(cls, name=None):
     if name in MODEL_ADAPTERS:
         raise ValueError(f"Model adapter class '{name}' is already registered.")
     MODEL_ADAPTERS[name] = cls
-
-
-# register all model adapters
-register_model_adapter_class(BaseModelAdapter, name='base')
-register_model_adapter_class(ChatGenerationModelAdapter, name=OutputType.GENERATION)
-register_model_adapter_class(ContinuationLogitsModelAdapter, name=OutputType.CONTINUOUS)
-register_model_adapter_class(MultiChoiceModelAdapter, name=OutputType.MULTIPLE_CHOICE)
-register_model_adapter_class(CustomModelAdapter, name='custom')
-register_model_adapter_class(ServerModelAdapter, name='server')
-register_model_adapter_class(BFCLAdapter, name='bfcl_server')
-register_model_adapter_class(T2IModelAdapter, name=OutputType.IMAGE_GENERATION)