evalscope 0.16.0__py3-none-any.whl → 0.16.2__py3-none-any.whl

evalscope/cli/start_app.py

@@ -21,13 +21,13 @@ class StartAppCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
-        from evalscope.report import add_argument
+        from evalscope.app import add_argument
 
         parser = parsers.add_parser(StartAppCMD.name)
         add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
-        from evalscope.report.app import create_app
+        from evalscope.app import create_app
 
         create_app(self.args)

evalscope/collections/__init__.py

@@ -1,3 +1,35 @@
-from evalscope.collections.evaluator import EvaluatorCollection
-from evalscope.collections.sampler import StratifiedSampler, UniformSampler, WeightedSampler
-from evalscope.collections.schema import CollectionSchema, DatasetInfo
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .evaluator import EvaluatorCollection
+    from .sampler import StratifiedSampler, UniformSampler, WeightedSampler
+    from .schema import CollectionSchema, DatasetInfo
+
+else:
+    _import_structure = {
+        'evaluator': [
+            'EvaluatorCollection',
+        ],
+        'sampler': [
+            'StratifiedSampler',
+            'UniformSampler',
+            'WeightedSampler',
+        ],
+        'schema': [
+            'CollectionSchema',
+            'DatasetInfo',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/collections/evaluator.py

@@ -32,11 +32,22 @@ class SimpleEvaluator(Evaluator):
             task_cfg=task_cfg,
             outputs=outputs)
 
-    def get_answer(self, samples, infer_cfg) -> List[dict]:
+    def get_answer(self, samples: List[DatasetEntry], infer_cfg: dict) -> List[dict]:
         input_prompts = [sample.prompt for sample in samples]
         subset_name = samples[0].subset_name
+        try:
+            # get answer from model
+            answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
+        except Exception as e:
+            logger.error(f'Failed to get answer for {input_prompts}, due to {e}')
+            # if ignore_errors is True, continue to next input
+            if self.task_cfg.ignore_errors:
+                logger.warning('`ignore_errors` is set to True. Dropping this prompt and continuing with evaluation.')
+                return [None] * len(samples), samples
+            else:
+                raise e
+        # process answers
         answers_list = []
-        answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
         for answer_d, input_prompt in zip(answer_ds, input_prompts):
             answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
             processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
@@ -66,13 +77,17 @@ class EvaluatorCollection:
         self.dataset_id_map = EvaluatorCollection._init_id_map(self.dataset)
         self.evaluators = self._initialize_evaluators()
 
-    def load(self) -> tuple[list[DatasetEntry], str]:
+    def load(self) -> tuple[List[DatasetEntry], str]:
         dataset_name = os.path.splitext(os.path.basename(self.data_adapter.dataset_id))[0]
         raw_dataset = self.data_adapter.load()
         # random limit the dataset
-        if self.task_cfg.limit:
-            raw_dataset = random.sample(raw_dataset,
-                                        self.task_cfg.limit) if len(raw_dataset) > self.task_cfg.limit else raw_dataset
+        limit = len(raw_dataset)
+        if self.task_cfg.limit is not None:
+            if isinstance(self.task_cfg.limit, int):
+                limit = self.task_cfg.limit
+            elif isinstance(self.task_cfg.limit, float):
+                limit = int(len(raw_dataset) * self.task_cfg.limit)
+            raw_dataset = random.sample(raw_dataset, min(limit, len(raw_dataset)))
         # index dataset
         datasets = []
         for sample in raw_dataset:
@@ -82,7 +97,7 @@ class EvaluatorCollection:
         return datasets, dataset_name
 
     @staticmethod
-    def _init_name_map(dataset):
+    def _init_name_map(dataset: List[DatasetEntry]) -> Dict[str, Dict[str, List[int]]]:
         dataset_name_map = defaultdict(lambda: defaultdict(list))
         for sample in dataset:
             dataset_name, subset_name = sample.dataset_name, sample.subset_name
@@ -90,13 +105,13 @@ class EvaluatorCollection:
         return dataset_name_map
 
     @staticmethod
-    def _init_id_map(dataset):
+    def _init_id_map(dataset: List[DatasetEntry]) -> Dict[int, DatasetEntry]:
         dataset_id_map = {}
         for sample in dataset:
             dataset_id_map[sample.index] = sample
         return dataset_id_map
 
-    def _initialize_evaluators(self):
+    def _initialize_evaluators(self) -> Dict[str, SimpleEvaluator]:
         evaluators = {}
         # load dataset args
         dataset_args = deepcopy(self.task_cfg.dataset_args)
@@ -114,6 +129,8 @@ class EvaluatorCollection:
         return evaluators
 
     def get_report(self, scores):
+        if not scores:
+            return
 
         def get_dataframe(scores):
             data = []
@@ -179,11 +196,19 @@ class EvaluatorCollection:
             logger.info(f'{level} Report:\n{table}')
 
         report = ReportGenerator.gen_collection_report(df, self.dataset_name, self.task_cfg.model_id)
+        # Make report analysis
+        if self.task_cfg.analysis_report:
+            logger.info('Generating report analysis, please wait ...')
+            analysis = report.generate_analysis(self.task_cfg.judge_model_args)
+            logger.info('Report analysis:\n%s', analysis)
+        else:
+            logger.info('Skipping report analysis (`analysis_report=False`).')
+
         # save report to JSON file
         report_file_path = os.path.join(self.outputs.reports_dir, self.task_cfg.model_id, f'{self.dataset_name}.json')
-        os.makedirs(os.path.dirname(report_file_path), exist_ok=True)
-        with open(report_file_path, 'w', encoding='utf-8') as f:
-            json.dump(report.to_dict(), f, ensure_ascii=False, indent=4)
+        report.to_json(report_file_path)
+
+        logger.info(f'Report saved to {report_file_path}')
         return report
 
     def _filter_answer(self, pred_file_path):
@@ -229,9 +254,12 @@ class EvaluatorCollection:
                     # Process completed tasks
                     for future in as_completed(futures):
                         answer_list, samples = future.result()
-                        answers[samples[0].index] = answer_list[0]
-                        dump_jsonl_data(answer_list, pred_file_path, dump_mode=DumpMode.APPEND)
-                        pbar.update(1)
+                        for answer_d, sample in zip(answer_list, samples):
+                            if answer_d is None:
+                                continue
+                            answers[sample.index] = answer_d
+                            dump_jsonl_data([answer_d], pred_file_path, dump_mode=DumpMode.APPEND)
+                            pbar.update(1)
             else:
                 for dataset_name, data_map in dataset_name_map.items():
                     # get evaluator for the dataset
@@ -241,13 +269,14 @@ class EvaluatorCollection:
                             # get batch samples
                             batch_ids = ids[i:i + eval_batch_size]
                             batch_samples = [self.dataset_id_map[_id] for _id in batch_ids]
-                            answer_list, _ = evaluator.get_answer(batch_samples, self.task_cfg.generation_config)
+                            answer_list, samples = evaluator.get_answer(batch_samples, self.task_cfg.generation_config)
                             # update answers
-                            for j, _id in enumerate(batch_ids):
-                                answers[_id] = answer_list[j]
-                            dump_jsonl_data(answer_list, pred_file_path, dump_mode=DumpMode.APPEND)
-
-                            pbar.update(len(batch_ids))
+                            for answer_d, sample in zip(answer_list, samples):
+                                if answer_d is None:
+                                    continue
+                                answers[sample.index] = answer_d
+                                dump_jsonl_data([answer_d], pred_file_path, dump_mode=DumpMode.APPEND)
+                                pbar.update(1)
         return answers
 
     def get_reviews(self, answers: Dict[int, Any]) -> Dict[int, Any]:
@@ -277,19 +306,22 @@ class EvaluatorCollection:
 
         reviews = {}
         for sample in tqdm(self.dataset, desc='Getting reviews'):
-            file_name = f'{self.dataset_name}_{sample.dataset_name}_{sample.subset_name}.jsonl'
-
-            if self.task_cfg.use_cache and sample.index in review_history_map.get(file_name, {}):
-                # Use cached review if available
-                review_d = review_history_map[file_name][sample.index]
-            else:
-                # Generate new review
-                evaluator = self.evaluators[sample.dataset_name]
-                review_d = evaluator.get_review(answers[sample.index])
-                # Only save the review if it's not in the cache
-                self._save_review(review_file_path, file_name, review_d)
-
-            reviews[sample.index] = review_d
+            try:
+                file_name = f'{self.dataset_name}_{sample.dataset_name}_{sample.subset_name}.jsonl'
+
+                if self.task_cfg.use_cache and sample.index in review_history_map.get(file_name, {}):
+                    # Use cached review if available
+                    review_d = review_history_map[file_name][sample.index]
+                else:
+                    # Generate new review
+                    evaluator = self.evaluators[sample.dataset_name]
+                    review_d = evaluator.get_review(answers[sample.index])
+                    # Only save the review if it's not in the cache
+                    self._save_review(review_file_path, file_name, review_d)
+
+                reviews[sample.index] = review_d
+            except Exception as e:
+                logger.error(f'Error getting review for sample index {sample.index}: {e}. Skipping this sample.')
 
         return reviews
 
@@ -327,6 +359,8 @@ class EvaluatorCollection:
         scores = defaultdict(dict)
         for sample in tqdm(self.dataset, desc='Getting scores'):
             evaluator = self.evaluators[sample.dataset_name]
+            if sample.index not in reviews:
+                continue
             review_d = reviews[sample.index]
             score = evaluator.get_score(review_d)
             scores[sample.index] = score

evalscope/config.py

@@ -13,6 +13,7 @@ from evalscope.models import CustomModel, DummyCustomModel
 from evalscope.utils import gen_hash
 from evalscope.utils.io_utils import dict_to_yaml, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger
+from evalscope.utils.utils import parse_int_or_float
 
 logger = get_logger()
 
@@ -45,7 +46,7 @@ class TaskConfig:
     eval_backend: str = EvalBackend.NATIVE
     eval_config: Union[str, Dict, None] = None
     stage: str = EvalStage.ALL
-    limit: Optional[int] = None
+    limit: Optional[Union[int, float]] = None
     eval_batch_size: Optional[int] = None
 
     # Cache and working directory arguments
@@ -67,7 +68,8 @@ class TaskConfig:
     # LLMJudge arguments
     judge_strategy: str = JudgeStrategy.AUTO
     judge_worker_num: int = 1
-    judge_model_args: Optional[Dict] = field(default_factory=lambda: {})
+    judge_model_args: Optional[Dict] = field(default_factory=dict)
+    analysis_report: bool = False
 
     def __post_init__(self):
         if self.model is None:
@@ -86,6 +88,10 @@ class TaskConfig:
         if self.eval_batch_size is None:
             self.eval_batch_size = 8 if self.eval_type == EvalType.SERVICE else 1
 
+        # Post process limit
+        if self.limit is not None:
+            self.limit = parse_int_or_float(self.limit)
+
         # Set default generation_config and model_args
         self.__init_default_generation_config()
         self.__init_default_model_args()

evalscope/constants.py

@@ -146,7 +146,7 @@ class EvalType:
 
 
 class OutputType:
-    LOGITS = 'logits'  # for multiple choice tasks
+    LOGITS = 'logits'  # for logits output tasks
     GENERATION = 'generation'  # for text generation tasks and general tasks
     MULTIPLE_CHOICE = 'multiple_choice_logits'  # for multiple choice tasks
     CONTINUOUS = 'continuous_logits'  # for continuous tasks

evalscope/evaluator/evaluator.py

@@ -46,7 +46,6 @@ class Evaluator(object):
         self.dataset_name = data_adapter.name
         self.dataset_name_or_path = os.path.expanduser(data_adapter.dataset_id)
         self.model_name = task_cfg.model_id
-        self.custom_task_name = f'{self.model_name}_{self.dataset_name}'
 
         self.data_adapter = data_adapter
         self.model_adapter = model_adapter
@@ -79,8 +78,16 @@ class Evaluator(object):
         # Limit and index prompts
         limited_prompts = defaultdict(list)
         for subset_name, prompts_list in prompts.items():
-            limit = self.task_cfg.limit or len(prompts_list)
-            for index, prompt in enumerate(prompts_list[:limit]):
+            # If limit is None, use all prompts
+            if self.task_cfg.limit is None:
+                limit = len(prompts_list)
+            else:
+                if isinstance(self.task_cfg.limit, int):
+                    limit = self.task_cfg.limit
+                elif isinstance(self.task_cfg.limit, float):
+                    limit = int(len(prompts_list) * self.task_cfg.limit)
+            # Limit the number of prompts
+            for index, prompt in enumerate(prompts_list[:min(limit, len(prompts_list))]):
                 prompt[AnswerKeys.INDEX] = index
                 limited_prompts[subset_name].append(prompt)
 
@@ -101,7 +108,6 @@ class Evaluator(object):
         return answer_d
 
     def _get_answer(self, input_prompts, subset_name, infer_cfg) -> List[dict]:
-        answers_list = []
         try:
             # get answer from model
             answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
@@ -110,10 +116,11 @@ class Evaluator(object):
             # if ignore_errors is True, continue to next input
             if self.task_cfg.ignore_errors:
                 logger.warning('`ignore_errors` is set to True. Dropping this prompt and continuing with evaluation.')
-                return answers_list
+                return []
             else:
                 raise e
         # process answer
+        answers_list = []
         for answer_d, input_prompt in zip(answer_ds, input_prompts):
             answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
             processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
@@ -371,41 +378,46 @@ class Evaluator(object):
 
         return metric_score
 
-    def dump_report(self, reviews_score_all: List[dict], use_table: bool = True):
+    def dump_report(self, reviews_score_all: List[dict]):
         """
         Get report for total reviews of specific dataset.
         It is required to rewrite this method to support your own evaluator.
 
         Args:
             reviews_score_all: reviews score list. Generated by func self.data_adapter.compute_metric().
-            use_table: whether to generate table for reports. Default to True.
 
         Returns: None
         """
+        report_path = os.path.join(self.outputs_structure.reports_dir, self.model_name)
+        os.makedirs(report_path, exist_ok=True)
         # Get report map
         report_map: Report = self.data_adapter.gen_report(
-            subset_score_map=reviews_score_all,
-            report_name=self.custom_task_name,
-            model_name=self.model_name,
-            dataset_name=self.dataset_name)
-
-        # Dump report
-        report_path: str = os.path.join(self.outputs_structure.reports_dir, self.model_name,
-                                        self.dataset_name + '.json')
-        os.makedirs(os.path.dirname(report_path), exist_ok=True)
+            subset_score_map=reviews_score_all, model_name=self.model_name)
 
-        # Write report
-        with open(report_path, 'w', encoding='utf-8') as f:
-            f.write(json.dumps(report_map.to_dict(), ensure_ascii=False, indent=4))
-        logger.info(f'Dump report: {report_path} \n')
+        # Post process report
+        self.data_adapter.post_process_report(report_map, report_path=report_path)
 
         # Make table
-        if use_table:
-            try:
-                report_table: str = gen_table([self.outputs_structure.reports_dir])
-                logger.info(f'Report table: \n{report_table} \n')
-            except Exception:
-                logger.error('Failed to generate report table.')
+        try:
+            report_table = gen_table(report_list=[report_map], add_overall_metric=True)
+            logger.info(f'\n{self.dataset_name_or_path} report table:'
+                        f'\n{report_table} \n')
+        except Exception:
+            logger.error('Failed to generate report table.')
+
+        # Make report analysis
+        if self.task_cfg.analysis_report:
+            logger.info('Generating report analysis, please wait ...')
+            analysis = report_map.generate_analysis(self.task_cfg.judge_model_args)
+            logger.info('Report analysis:\n%s', analysis)
+        else:
+            logger.info('Skipping report analysis (`analysis_report=False`).')
+
+        # Dump report
+        report_file = os.path.join(report_path, f'{self.dataset_name}.json')
+        report_map.to_json(report_file)
+        logger.info(f'Dump report to: {report_file} \n')
+
         return report_map
 
     def eval(self, **kwargs) -> dict:
@@ -431,7 +443,7 @@ class Evaluator(object):
             stage == 'review': return the reviews_map
         """
 
-        logger.info(f'**** Start evaluating on dataset {self.dataset_name_or_path} ****')
+        logger.info(f'Start evaluating on dataset {self.dataset_name_or_path}')
 
         reviews_score_all = {}  # {subset_name: (score, num)}
         stage_answers_dict = {}
@@ -461,6 +473,6 @@ class Evaluator(object):
         # Generate report
         report_map = self.dump_report(reviews_score_all)
 
-        logger.info(f'**** Evaluation finished on {self.dataset_name_or_path} ****\n')
+        logger.info(f'Evaluation finished on {self.dataset_name_or_path}')
 
         return report_map

evalscope/metrics/__init__.py

@@ -9,7 +9,7 @@ if TYPE_CHECKING:
     from .metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, simple_f1_score,
                           weighted_mean)
     from .named_metrics import Metric, metric_registry
-    from .rouge_metric import compute_rouge_score_one_sample_zh
+    from .rouge_metric import compute_rouge_score, compute_rouge_score_one_sample, compute_rouge_score_one_sample_zh
 
 else:
     _import_structure = {
@@ -28,6 +28,8 @@ else:
         ],
         'rouge_metric': [
             'compute_rouge_score_one_sample_zh',
+            'compute_rouge_score',
+            'compute_rouge_score_one_sample',
         ],
         'llm_judge': [
             'LLMJudge',

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -88,11 +88,11 @@ class RougeScorer(scoring.BaseScorer):
     """
 
     def __init__(self, rouge_types, use_stemmer=False, split_summaries=False, tokenizer=None):
-        check_nltk_data()
         self.rouge_types = rouge_types
         if tokenizer:
             self._tokenizer = tokenizer
         else:
+            check_nltk_data()
             self._tokenizer = tokenizers.DefaultTokenizer(use_stemmer)
             logging.info('Using default tokenizer.')
 

evalscope/metrics/llm_judge.py

@@ -22,6 +22,9 @@ B: INCORRECT
 Just return the letters "A" or "B", with no text around it.
 """  # noqa: E501
 
+DEFAULT_JUDGE_MODEL = 'Qwen/Qwen3-235B-A22B'
+DEFAULT_API_URL = 'https://api-inference.modelscope.cn/v1/'
+
 
 class LLMJudge:
     """
@@ -47,12 +50,12 @@ class LLMJudge:
             prompt_template (str, optional): Prompt template for the judge
             generation_config (dict, optional): Generation configuration for the judge
         """
-        self.api_key = api_key or os.environ.get('OPENAI_API_KEY', 'EMPTY')
-        self.api_url = api_url or os.environ.get('OPENAI_API_BASE', 'https://api.openai.com/v1')
-        self.model_id = model_id or os.environ.get('LOCAL_LLM', 'gpt-4')
+        self.api_key = api_key or os.environ.get('MODELSCOPE_SDK_TOKEN', 'EMPTY')
+        self.api_url = api_url or os.environ.get('MODELSCOPE_API_BASE', DEFAULT_API_URL)
+        self.model_id = model_id or os.environ.get('MODELSCOPE_JUDGE_LLM', DEFAULT_JUDGE_MODEL)
         self.system_prompt = system_prompt or os.environ.get('JUDGE_SYSTEM_PROMPT', None)
         self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
-        self.generation_config = generation_config
+        self.generation_config = generation_config or {}
 
         from evalscope.models import ServerModelAdapter
 
@@ -74,6 +77,10 @@ class LLMJudge:
         if self.generation_config:
             infer_cfg.update(self.generation_config)
 
+        if self.model_id == DEFAULT_JUDGE_MODEL:
+            # Disable thinking for the default judge model
+            infer_cfg['enable_thinking'] = self.generation_config.get('enable_thinking', False)
+
         try:
             # Send request using ServerModelAdapter
             response = self.server_adapter.process_single_input(input_data, infer_cfg)
@@ -82,7 +89,7 @@ class LLMJudge:
             llm_response = response.get('choices', [{}])[0].get('message', {}).get('content', '')
             return llm_response
         except Exception as e:
-            logger.error(f'Error during LLM evaluation: {e}')
+            logger.error(f'Error occurred during {self.model_id}@{self.api_url} LLM judge evaluation: {e}')
             return ''
 
     def build_prompt(self, pred: str, gold: str, question: Optional[str] = None):

evalscope/metrics/math_parser.py

@@ -4,7 +4,7 @@ The logic in this file largely borrows from Qwen2.5-Math codebase at https://git
 # flake8: noqa
 import re
 import regex
-from latex2sympy2 import latex2sympy
+from latex2sympy2_extended import latex2sympy
 from math import isclose
 from sympy import N, simplify
 from sympy.parsing.latex import parse_latex

evalscope/metrics/t2v_metrics/__init__.py

@@ -1,66 +1,52 @@
-from __future__ import absolute_import, division, print_function
-
-from .clipscore import CLIPScore, list_all_clipscore_models
-from .constants import CACHE_DIR
-from .itmscore import ITMScore, list_all_itmscore_models
-from .vqascore import VQAScore, list_all_vqascore_models
-
-
-def list_all_models():
-    return list_all_vqascore_models() + list_all_clipscore_models() + list_all_itmscore_models()
-
-
-def get_score_model(model='clip-flant5-xxl', device='cuda', cache_dir=CACHE_DIR, **kwargs):
-    if model in list_all_vqascore_models():
-        return VQAScore(model, device=device, cache_dir=cache_dir, **kwargs)
-    elif model in list_all_clipscore_models():
-        return CLIPScore(model, device=device, cache_dir=cache_dir, **kwargs)
-    elif model in list_all_itmscore_models():
-        return ITMScore(model, device=device, cache_dir=cache_dir, **kwargs)
-    else:
-        raise NotImplementedError()
-
-
 def clip_flant5_score():
+    from .vqascore import VQAScore
     clip_flant5_score = VQAScore(model='clip-flant5-xxl')
     return clip_flant5_score
 
 
 def pick_score():
+    from .clipscore import CLIPScore
     pick_score = CLIPScore(model='pickscore-v1')
     return pick_score
 
 
 def clip_score():
+    from .clipscore import CLIPScore
     clip_score = CLIPScore(model='openai:ViT-L-14-336')
     return clip_score
 
 
 def blip2_score():
+    from .itmscore import ITMScore
     blip_itm_score = ITMScore(model='blip2-itm')
     return blip_itm_score
 
 
 def hpsv2_score():
+    from .clipscore import CLIPScore
     hpsv2_score = CLIPScore(model='hpsv2')
     return hpsv2_score
 
 
 def hpsv2_1_score():
+    from .clipscore import CLIPScore
     hpsv2_1_score = CLIPScore(model='hpsv2.1')
     return hpsv2_1_score
 
 
 def image_reward_score():
+    from .itmscore import ITMScore
     image_reward_score = ITMScore(model='image-reward-v1')
     return image_reward_score
 
 
 def fga_blip2_score():
+    from .itmscore import ITMScore
     fga_blip2_score = ITMScore(model='fga_blip2')
     return fga_blip2_score
 
 
 def mps_score():
+    from .clipscore import CLIPScore
     mps_score = CLIPScore(model='mps')
     return mps_score

evalscope/models/adapters/__init__.py

@@ -1,4 +1,5 @@
 from .base_adapter import BaseModelAdapter, initialize_model_adapter
+from .bfcl_adapter import BFCLAdapter
 from .chat_adapter import ChatGenerationModelAdapter
 from .choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdapter
 from .custom_adapter import CustomModelAdapter
@@ -13,5 +14,6 @@ __all__ = [
     'MultiChoiceModelAdapter',
     'CustomModelAdapter',
     'ServerModelAdapter',
+    'BFCLAdapter',
     'T2IModelAdapter',
 ]

evalscope/models/adapters/base_adapter.py

@@ -44,35 +44,39 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'DataAdapter', b
             raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
         from evalscope.models import CustomModelAdapter
         return CustomModelAdapter(custom_model=task_cfg.model)
-    elif task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
-        from evalscope.models import ServerModelAdapter
-
-        if benchmark.model_adapter in [OutputType.CONTINUOUS, OutputType.MULTIPLE_CHOICE]:
-            logger.warning('Output type is set to logits. This is not supported for service evaluation. '
-                           'Setting output type to generation by default.')
-            benchmark.model_adapter = OutputType.GENERATION
-
-        return ServerModelAdapter(
-            api_url=task_cfg.api_url,
-            model_id=task_cfg.model,
-            api_key=task_cfg.api_key,
-            seed=task_cfg.seed,
-            timeout=task_cfg.timeout,
-            stream=task_cfg.stream,
-        )
     else:
         from ..register import get_model_adapter
 
-        # for local model, we need to determine the model adapter class based on the output type
+        # we need to determine the model adapter class based on the output type
         model_adapter_cls_str = benchmark.model_adapter
-        if model_adapter_cls_str not in benchmark.output_types:
-            logger.warning(f'Output type {model_adapter_cls_str} is not supported for benchmark {benchmark.name}. '
-                           f'Using {benchmark.output_types[0]} instead.')
-            model_adapter_cls_str = benchmark.output_types[0]
 
-        model_adapter_cls = get_model_adapter(model_adapter_cls_str)
-        return model_adapter_cls(
-            model=base_model,
-            generation_config=task_cfg.generation_config,
-            chat_template=task_cfg.chat_template,
-            task_cfg=task_cfg)
+        if task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
+
+            if 'server' not in model_adapter_cls_str:
+                model_adapter_cls_str = 'server'
+                logger.info(
+                    f'Using {model_adapter_cls.__name__} for api model evaluation for benchmark {benchmark.name}.')
+
+            # init server model adapter
+            model_adapter_cls = get_model_adapter(model_adapter_cls_str)
+
+            return model_adapter_cls(
+                api_url=task_cfg.api_url,
+                model_id=task_cfg.model,
+                api_key=task_cfg.api_key,
+                seed=task_cfg.seed,
+                timeout=task_cfg.timeout,
+                stream=task_cfg.stream,
+            )
+        else:
+            if model_adapter_cls_str not in benchmark.output_types:
+                logger.warning(f'Output type {model_adapter_cls_str} is not supported for benchmark {benchmark.name}.'
+                               f'Using {benchmark.output_types[0]} instead.')
+                model_adapter_cls_str = benchmark.output_types[0]
+
+            model_adapter_cls = get_model_adapter(model_adapter_cls_str)
+            return model_adapter_cls(
+                model=base_model,
+                generation_config=task_cfg.generation_config,
+                chat_template=task_cfg.chat_template,
+                task_cfg=task_cfg)