evalscope 0.5.0rc0__py3-none-any.whl → 0.5.3__py3-none-any.whl

evalscope/backend/opencompass/backend_manager.py

@@ -76,9 +76,7 @@ class OpenCompassBackendManager(BackendManager):
     @staticmethod
     def _check_env():
         if is_module_installed('opencompass'):
-            logger.info('Please make sure you have installed the `ms-opencompass`: `pip install ms-opencompass`')
-        else:
-            raise ModuleNotFoundError('Please install the `ms-opencompass` first: `pip install ms-opencompass`')
+            logger.info('Check the OpenCompass environment: OK')
 
     @staticmethod
     def get_restore_arg(arg_name: str, arg_val: bool):

evalscope/backend/opencompass/tasks/eval_api.py

@@ -6,6 +6,7 @@ from opencompass.tasks import OpenICLInferTask
 
 
 with read_base():
+    from opencompass.configs.summarizers.medium import summarizer
     from evalscope.backend.opencompass.tasks.eval_datasets import datasets
 
 # 1. Get datasets

evalscope/backend/opencompass/tasks/eval_datasets.py

@@ -49,6 +49,7 @@ with read_base():
     from opencompass.configs.datasets.obqa.obqa_gen_9069e4 import obqa_datasets
     from opencompass.configs.datasets.nq.nq_gen_c788f6 import nq_datasets
     from opencompass.configs.datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
+    from opencompass.configs.datasets.cmb.cmb_gen_dfb5c4 import cmb_datasets
     from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
 
     # Note: to be supported

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -31,12 +31,13 @@ class VLMEvalKitBackendManager(BackendManager):
         from vlmeval.utils.arguments import Arguments as VLMEvalArguments
         self.args = VLMEvalArguments(**self.config_d)
 
-        self.valid_models = self.list_supported_VLMs()
+        self.valid_models = self.list_supported_models()
         self.valid_model_names = list(self.valid_models.keys())
         self.valid_datasets = self.list_supported_datasets()
 
         self._check_valid()
 
+    
     def _check_valid(self):
         # Ensure not both model and datasets are empty
         if not self.args.data or not self.args.model:
@@ -44,9 +45,9 @@ class VLMEvalKitBackendManager(BackendManager):
 
         # Check datasets
         valid_datasets, invalid_datasets = get_valid_list(self.args.data, self.valid_datasets)
-        assert len(invalid_datasets) == 0, f'Invalid datasets: {invalid_datasets}, ' \
-            f'refer to the following list to get proper dataset name: {self.valid_datasets}'
-
+        if len(invalid_datasets) != 0:
+            logger.warning(f"Using custom dataset: {invalid_datasets}, ")
+            
         # Check model
         if isinstance(self.args.model[0], dict):
             model_names = [model['name'] for model in self.args.model]
@@ -61,10 +62,14 @@ class VLMEvalKitBackendManager(BackendManager):
                 model_class = self.valid_models[model_name]
                 if model_name == 'CustomAPIModel':
                     model_type = model_cfg['type']
+                    remain_cfg = copy.deepcopy(model_cfg)
+                    del remain_cfg['name'] # remove not used args
+                    del remain_cfg['type'] # remove not used args
+
                     self.valid_models.update({
                                 model_type: partial(model_class, 
                                                    model=model_type,
-                                                   **model_cfg)
+                                                   **remain_cfg)
                                 })
                     new_model_names.append(model_type)
                 else:
@@ -78,15 +83,15 @@ class VLMEvalKitBackendManager(BackendManager):
 
         elif isinstance(self.args.model[0], str):
             valid_model_names, invalid_model_names = get_valid_list(self.args.model, self.valid_model_names)
-            assert len(invalid_model_names) == 0, f'Invalid models: {invalid_model_names}, ' \
-                f'refer to the following list to get proper model name: {self.valid_model_names}'
+            if len(invalid_datasets) != 0:
+                logger.warning(f"Using custom dataset: {invalid_datasets}, ")
 
     @property
     def cmd(self):
         return self.get_cmd()
 
     @staticmethod
-    def list_supported_VLMs():
+    def list_supported_models():
         from vlmeval.config import supported_VLM
         return supported_VLM
 
@@ -98,9 +103,7 @@ class VLMEvalKitBackendManager(BackendManager):
     @staticmethod
     def _check_env():
         if is_module_installed('vlmeval'):
-            logger.info('Please make sure you have installed the `ms-vlmeval`: `pip install ms-vlmeval`')
-        else:
-            raise ModuleNotFoundError('Please install the `ms-vlmeval` first: `pip install ms-vlmeval`')
+            logger.info('Check VLM Evaluation Kit: Installed')
 
     @staticmethod
     def get_restore_arg(arg_name: str, arg_val: bool):

evalscope/backend/vlm_eval_kit/custom_dataset.py

@@ -0,0 +1,47 @@
+import os
+import numpy as np
+from vlmeval.dataset.image_base import ImageBaseDataset
+from vlmeval.dataset.image_vqa import CustomVQADataset
+from vlmeval.smp import load, dump, d2df
+
+class CustomDataset:
+
+    def load_data(self, dataset):
+        # customize the loading of the dataset
+        data_path = os.path.join("~/LMUData", f'{dataset}.tsv')
+        return load(data_path)
+
+
+    def build_prompt(self, line):
+        msgs = ImageBaseDataset.build_prompt(self, line)
+        # add a hint or custom instruction here
+        msgs[-1]['value'] += '\nAnswer the question using a single word or phrase.'
+        return msgs
+    
+    
+    def evaluate(self, eval_file, **judge_kwargs):
+        data = load(eval_file)
+        assert 'answer' in data and 'prediction' in data
+        data['prediction'] = [str(x) for x in data['prediction']]
+        data['answer'] = [str(x).lower() for x in data['answer']]
+        
+        print(data)
+        
+        # ========compute the evaluation metrics as you need =========
+        # exact match
+        result = np.mean(data['answer'] == data['prediction'])
+        ret = {'Overall': result}
+        ret = d2df(ret).round(2)
+
+        # save the result
+        suffix = eval_file.split('.')[-1]
+        result_file = eval_file.replace(f'.{suffix}', '_acc.csv')
+        dump(ret, result_file)
+        return ret
+        # ============================================================
+
+
+# override the default dataset class
+CustomVQADataset.load_data = CustomDataset.load_data
+CustomVQADataset.build_prompt = CustomDataset.build_prompt
+CustomVQADataset.evaluate = CustomDataset.evaluate

evalscope/config.py

@@ -33,6 +33,7 @@ registry_tasks = {
 @dataclass
 class TaskConfig:
     model_args: Optional[dict] = field(default_factory=dict)
+    template_type: Optional[str] = 'default-generation'
     generation_config: Optional[dict] = field(default_factory=dict)
     dataset_args: Optional[dict] = field(default_factory=dict)
     dry_run: bool = False

evalscope/models/model_adapter.py

@@ -362,6 +362,8 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
             torch_dtype: The torch dtype for model inference. Default: torch.float16.
             **kwargs: Other args.
         """
+
+        custom_generation_config = kwargs.pop('generation_config', None)
         model_cache_dir = get_model_cache_dir(root_cache_dir=cache_dir)
 
         self.model_id: str = model_id
@@ -414,6 +416,10 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         self.origin_tokenizer = deepcopy(tokenizer)
 
         self.generation_config, self.generation_template = self._parse_generation_config(tokenizer, model)
+
+        if custom_generation_config:
+            logger.info('**Updating generation config ...')
+            self.generation_config.update(**custom_generation_config.to_dict())
         logger.info(f'**Generation config init: {self.generation_config.to_dict()}')
 
         super().__init__(model=model, tokenizer=self.generation_template.tokenizer, model_cfg=model_cfg)

evalscope/run_arena.py

@@ -100,17 +100,18 @@ class ArenaWorkflow:
             model_revision = cfg_d.get(EvalConfigKeys.MODEL_REVISION, None)
             precision = cfg_d.get(EvalConfigKeys.PRECISION, torch.float16)
             precision = eval(precision) if isinstance(precision, str) else precision
-            generation_config = cfg_d.get(EvalConfigKeys.GENERATION_CONFIG, {})
-            generation_config = GenerationConfig(**generation_config)
+            custom_generation_config = cfg_d.get(EvalConfigKeys.GENERATION_CONFIG, {})
+            custom_generation_config = GenerationConfig(**custom_generation_config)
             ans_output_file = os.path.join(WORK_DIR, cfg_d.get(EvalConfigKeys.OUTPUT_FILE))
             template_type = cfg_d.get(EvalConfigKeys.TEMPLATE_TYPE)
 
             answers_list = self._predict_answers(model_id_or_path=model_id_or_path,
                                                  model_revision=model_revision,
                                                  precision=precision,
-                                                 generation_config=generation_config,
+                                                 generation_config=custom_generation_config,
                                                  template_type=template_type)
 
+            os.makedirs(os.path.dirname(ans_output_file), exist_ok=True)
             dump_jsonl_data(answers_list, ans_output_file)
             logger.info(f'Answers generated by model {model_name} and saved to {ans_output_file}')
 
@@ -168,6 +169,7 @@ class ArenaWorkflow:
             res_list = ae.run(self.review_file)
             rating_df = res_list[0]
             logger.info(f'Rating results:\n{rating_df.to_csv()}')
+            os.makedirs(os.path.dirname(report_file), exist_ok=True)
             rating_df.to_csv(report_file, index=True)
             logger.info(f'Rating results are saved to {report_file}')
         else:

evalscope/summarizer.py

@@ -99,19 +99,25 @@ class Summarizer:
             elif eval_backend == EvalBackend.VLM_EVAL_KIT.value:
                 eval_config = Summarizer.parse_eval_config(candidate_task)
 
-                work_dir = eval_config.get('work_dir') or 'outputs/default'
+                work_dir = eval_config.get('work_dir') or 'outputs'
                 if not os.path.exists(work_dir):
                     raise ValueError(f'work_dir {work_dir} does not exist.')
                 
-                # TODO: parse summary files: acc.csv, score.csv, score.json for different models
                 for model in eval_config['model']:
                     if model['name'] == 'CustomAPIModel':
                         model_name = model['type']
                     else:
                         model_name = model['name']
-                    summary_files = glob.glob(os.path.join(work_dir, model_name, '*.csv'))
+                    
+                    csv_files = glob.glob(os.path.join(work_dir, model_name, '*.csv'))
+                    json_files = glob.glob(os.path.join(work_dir, model_name, '*.json'))
+                    
+                    summary_files = csv_files + json_files
                     for summary_file_path in summary_files:
-                        summary_res: dict = csv_to_list(file_path=summary_file_path)[0]
+                        if summary_file_path.endswith('csv'):
+                            summary_res: dict = csv_to_list(summary_file_path)[0]
+                        elif summary_file_path.endswith('json'):
+                            summary_res: dict = json_to_dict(summary_file_path)
                         file_name = os.path.basename(summary_file_path).split('.')[0]
                         final_res_list.append({file_name: summary_res})
                 

evalscope/utils/task_utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 
 class EvalBackend(Enum):
-    # Use native evaluation pipeline of Eval-Scope
+    # Use native evaluation pipeline of EvalScope
     NATIVE = 'Native'
 
     # Use OpenCompass framework as the evaluation backend

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.5.0rc0'
-__release_datetime__ = '2024-08-01 08:00:00'
+__version__ = '0.5.3'
+__release_datetime__ = '2024-08-29 08:00:00'