evalscope 0.8.2__py3-none-any.whl → 0.9.0__py3-none-any.whl

evalscope/collections/schema.py

@@ -0,0 +1,122 @@
+import copy
+import json
+from dataclasses import asdict, dataclass, field
+from typing import List, Union
+
+
+@dataclass
+class DatasetInfo:
+    name: str
+    weight: float = 1.0  # sample weight in each collection
+    task_type: str = ''
+    tags: List[str] = field(default_factory=list)
+    args: dict = field(default_factory=dict)
+
+    def get_data(self) -> dict:
+        from evalscope.benchmarks import Benchmark
+
+        benchmark_meta = Benchmark.get(self.name)
+
+        data_adapter = benchmark_meta.get_data_adapter(config=self.args)
+        data_dict = data_adapter.load(
+            dataset_name_or_path=benchmark_meta.dataset_id, subset_list=benchmark_meta.subset_list)
+        prompts = data_adapter.gen_prompts(data_dict)
+        return prompts
+
+
+def flatten_weight(collection: 'CollectionSchema', base_weight=1):
+    total_weight = sum(dataset.weight for dataset in collection.datasets)
+    for dataset in collection.datasets:
+        current_weight = dataset.weight / total_weight * base_weight
+        if isinstance(dataset, CollectionSchema):
+            flatten_weight(dataset, current_weight)
+        else:
+            dataset.weight = current_weight
+
+
+def flatten_tags(collection: 'CollectionSchema', parent_names=None):
+    if parent_names is None:
+        parent_names = []
+    current_names = parent_names + [collection.name]
+    for dataset in collection.datasets:
+        if isinstance(dataset, CollectionSchema):
+            flatten_tags(dataset, current_names)
+        else:
+            # Add all parent CollectionSchema names to the tags of each DatasetInfo
+            for name in current_names:
+                if name not in dataset.tags:
+                    dataset.tags.append(name)
+
+
+def flatten_datasets(collection: 'CollectionSchema') -> List[DatasetInfo]:
+    flat_datasets = []
+    for dataset in collection.datasets:
+        if isinstance(dataset, CollectionSchema):
+            flat_datasets.extend(flatten_datasets(dataset))
+        else:
+            flat_datasets.append(dataset)
+    return flat_datasets
+
+
+@dataclass
+class CollectionSchema:
+    name: str
+    weight: float = 1.0
+    datasets: List[Union[DatasetInfo, 'CollectionSchema']] = field(default_factory=list)
+
+    def __str__(self):
+        return json.dumps(self.to_dict(), ensure_ascii=False, indent=4)
+
+    def to_dict(self):
+        return {
+            'name':
+            self.name,
+            'weight':
+            self.weight,
+            'datasets':
+            [asdict(dataset) if isinstance(dataset, DatasetInfo) else dataset.to_dict() for dataset in self.datasets],
+        }
+
+    @classmethod
+    def from_dict(cls, data):
+        instance = cls(name=data.get('name', ''), weight=data.get('weight', 1))
+        for dataset in data.get('datasets', []):
+            if 'datasets' in dataset:
+                instance.datasets.append(CollectionSchema.from_dict(dataset))
+            else:
+                instance.datasets.append(DatasetInfo(**dataset))
+        return instance
+
+    def dump_json(self, file_path):
+        d = self.to_dict()
+        with open(file_path, 'w') as f:
+            json.dump(d, f, ensure_ascii=False, indent=4)
+
+    @classmethod
+    def from_json(cls, file_path):
+        with open(file_path, 'r') as f:
+            data = json.load(f)
+        return cls.from_dict(data)
+
+    def flatten(self) -> List[DatasetInfo]:
+        collection = copy.deepcopy(self)
+        flatten_tags(collection)
+        flatten_weight(collection)
+        return flatten_datasets(collection)
+
+
+if __name__ == '__main__':
+    schema = CollectionSchema(
+        name='reasoning',
+        datasets=[
+            DatasetInfo(name='arc', weight=1, task_type='reasoning', tags=['en']),
+            DatasetInfo(name='ceval', weight=1, task_type='reasoning', tags=['zh'], args={'subset_list': ['logic']})
+        ])
+    print(schema)
+    print(schema.flatten())
+    schema.dump_json('outputs/schema.json')
+
+    schema = CollectionSchema.from_json('outputs/schema.json')
+    print(schema)
+    for dataset in schema.flatten():
+        print(dataset)

evalscope/config.py

@@ -31,7 +31,7 @@ DEFAULT_GENERATION_CONFIG = {
 @dataclass
 class TaskConfig:
     # Model-related arguments
-    model: Union[str, CustomModel, None] = None
+    model: Union[str, 'CustomModel', None] = None
     model_id: Optional[str] = None
     model_args: Optional[Dict] = field(default_factory=lambda: DEFAULT_MODEL_ARGS | {})
 
@@ -40,8 +40,8 @@ class TaskConfig:
     chat_template: Optional[str] = None
 
     # Dataset-related arguments
-    datasets: Optional[List[str]] = None
-    dataset_args: Optional[Dict] = field(default_factory=dict)
+    datasets: List[str] = field(default_factory=list)
+    dataset_args: Dict = field(default_factory=dict)
     dataset_dir: str = DEFAULT_DATASET_CACHE_DIR
     dataset_hub: str = HubType.MODELSCOPE
 
@@ -64,7 +64,9 @@ class TaskConfig:
     # Debug and runtime mode arguments
     debug: bool = False
     dry_run: bool = False
-    seed: int = 42
+    seed: Optional[int] = 42
+    api_url: Optional[str] = None  # Only used for server model
+    api_key: Optional[str] = 'EMPTY'  # Only used for server model
 
     def __post_init__(self):
         if (not self.model_id) and self.model:
@@ -74,7 +76,6 @@ class TaskConfig:
                 self.model_id = os.path.basename(self.model).rstrip(os.sep)
 
     def to_dict(self):
-        # Note: to avoid serialization error for some model instance
         return self.__dict__
 
     def __str__(self):
@@ -130,6 +131,7 @@ class TaskConfig:
                 continue
 
             task.model = custom_model
+            task.model_args = custom_model.config
             task.model_id = type(custom_model).__name__
             res_list.append(task)
 

evalscope/constants.py

@@ -135,34 +135,13 @@ class EvalStage:
 class EvalType:
 
     CUSTOM = 'custom'
-    CHECKPOINT = 'checkpoint'
+    CHECKPOINT = 'checkpoint'  # native model checkpoint
+    SERVICE = 'service'  # model service
 
 
 class EvalBackend:
-
-    class _Backend:
-        #  compatible with old version, set 'value'
-
-        def __init__(self, value):
-            self._value = value
-
-        @property
-        def value(self):
-            return self._value
-
-        def __str__(self):
-            return self._value
-
-        def __repr__(self):
-            return f"'{self._value}'"
-
-        def __eq__(self, other):
-            if isinstance(other, str):
-                return self._value == other
-            return NotImplemented
-
-    NATIVE = _Backend('Native')
-    OPEN_COMPASS = _Backend('OpenCompass')
-    VLM_EVAL_KIT = _Backend('VLMEvalKit')
-    RAG_EVAL = _Backend('RAGEval')
-    THIRD_PARTY = _Backend('ThirdParty')
+    NATIVE = 'Native'
+    OPEN_COMPASS = 'OpenCompass'
+    VLM_EVAL_KIT = 'VLMEvalKit'
+    RAG_EVAL = 'RAGEval'
+    THIRD_PARTY = 'ThirdParty'

evalscope/evaluator/evaluator.py

@@ -10,9 +10,8 @@ from typing import Any, Dict, List, Optional, Union
 
 from evalscope.benchmarks import DataAdapter
 from evalscope.config import TaskConfig
-from evalscope.constants import (DEFAULT_DATASET_CACHE_DIR, AnswerKeys, DumpMode, EvalStage, EvalType, HubType,
-                                 ReviewKeys)
-from evalscope.models.model_adapter import BaseModelAdapter, CustomModelAdapter
+from evalscope.constants import AnswerKeys, DumpMode, EvalStage, ReviewKeys
+from evalscope.models import BaseModelAdapter, CustomModelAdapter
 from evalscope.tools.combine_reports import gen_table
 from evalscope.utils import dict_torch_dtype_to_str, gen_hash
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
@@ -30,73 +29,63 @@ class Evaluator(object):
                 if the dataset is a local path, e.g. /path/to/your_dataset_name,
                 then the task name will be the basename of the path, which is `your_dataset_name`.
         data_adapter: DataAdapter, the data adapter for the dataset.
-        subset_list: list, the subset list for the dataset.
         model_adapter: BaseModelAdapter, the model adapter for the model.
-        use_cache: str, path to local cache. Default: None
-        outputs_dir: OutputsStructure, the outputs dir. Default: None
-        datasets_dir: str, the datasets dir. Default: DEFAULT_ROOT_CACHE_DIR
-        datasets_hub: str, the datasets hub. `Local`, `ModelScope` or `HuggingFace`. Default: 'ModelScope'
-        stage: str, the stage of evaluation. `all` or `infer` or `review`. Default: 'all'
-        eval_type: str, the evaluation type. `checkpoint` or `service` or `custom`. Default: 'checkpoint'
-        overall_task_cfg: dict, the overall task config. Default: None
+        outputs: OutputsStructure, the outputs dir. Default: None
+        task_cfg: TaskConfig, the overall task config. Default: None
         **kwargs: kwargs.
     """
 
     def __init__(self,
                  dataset_name_or_path: str,
                  data_adapter: DataAdapter,
-                 subset_list: Optional[list] = None,
-                 model_adapter: Optional[BaseModelAdapter] = None,
-                 use_cache: Optional[str] = None,
-                 outputs: Optional[OutputsStructure] = None,
-                 datasets_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
-                 datasets_hub: Optional[str] = HubType.MODELSCOPE,
-                 stage: Optional[str] = EvalStage.ALL,
-                 eval_type: Optional[str] = EvalType.CHECKPOINT,
-                 overall_task_cfg: Optional[TaskConfig] = None,
+                 model_adapter: BaseModelAdapter,
+                 outputs: OutputsStructure = None,
+                 task_cfg: TaskConfig = None,
                  **kwargs):
 
         self.dataset_name_or_path = os.path.expanduser(dataset_name_or_path)
         self.dataset_name = os.path.basename(self.dataset_name_or_path.rstrip(os.sep)).split('.')[0]
-        self.model_name = overall_task_cfg.model_id
+        self.model_name = task_cfg.model_id
         self.custom_task_name = f'{self.model_name}_{self.dataset_name}'
 
-        self.datasets_dir = os.path.expanduser(datasets_dir)
-        self.kwargs = kwargs
         self.data_adapter = data_adapter
         self.model_adapter = model_adapter
-        self.eval_type = eval_type
-        self.stage = stage
-        self.use_cache = use_cache
-        self.overall_task_cfg = overall_task_cfg
-        if isinstance(self.model_adapter, CustomModelAdapter):
-            self.overall_task_cfg.model_args = self.model_adapter.custom_model.config
-
-        self.model_cfg = self.model_adapter.model_cfg
-
+        self.model_cfg = model_adapter.model_cfg
+        self.eval_type = task_cfg.eval_type
+        self.dataset_hub = task_cfg.dataset_hub
+        self.stage = task_cfg.stage
+        self.use_cache = task_cfg.use_cache
+        self.task_cfg = task_cfg
         # Deal with the output paths
         self.outputs_structure = outputs
 
-        # Load dataset
-        self.dataset = self.data_adapter.load(
-            dataset_name_or_path=dataset_name_or_path,
-            subset_list=subset_list,
-            work_dir=self.datasets_dir,
-            datasets_hub=datasets_hub,
-            **kwargs)
-
-        # Get prompts from dataset
-        # TODO: support sampler
-        self.prompts = self.data_adapter.gen_prompts(data_dict=self.dataset)
-        del self.dataset
-
-    def _pred_answer(self, input_d: dict, infer_cfg: dict, subset_name: str, answer_id: str = None) -> dict:
+        self.kwargs = kwargs
 
-        ans: dict = self.model_adapter.predict(inputs=input_d, infer_cfg=infer_cfg)
-        ans[AnswerKeys.ANSWER_ID] = answer_id
-        ans[AnswerKeys.SUBSET_NAME] = subset_name
+    def load_dataset(self):
+        dataset = self.data_adapter.load(
+            dataset_name_or_path=self.dataset_name_or_path,
+            subset_list=self.data_adapter.subset_list,
+            work_dir=os.path.expanduser(self.task_cfg.dataset_dir),
+            datasets_hub=self.dataset_hub,
+            **self.kwargs)
 
-        return ans
+        # Get prompts from dataset
+        prompts = self.data_adapter.gen_prompts(data_dict=dataset)
+        return prompts
+
+    def _generate_answer_id(self, model_cfg, input_d, infer_cfg):
+        model_cfg_str = json.dumps(OrderedDict(sorted(dict_torch_dtype_to_str(model_cfg).items())), ensure_ascii=False)
+        input_prompt_str = json.dumps(OrderedDict(sorted(dict_torch_dtype_to_str(input_d).items())), ensure_ascii=False)
+        infer_cfg_str = json.dumps(OrderedDict(sorted(dict_torch_dtype_to_str(infer_cfg).items())), ensure_ascii=False)
+        return 'answer-' + gen_hash(model_cfg_str + input_prompt_str + infer_cfg_str)
+
+    def _process_answer(self, answer_d, input_d, subset_name, answer_id):
+        answer_d[AnswerKeys.MODEL_SPEC] = self.model_adapter.model_cfg
+        answer_d[AnswerKeys.ANSWER_ID] = answer_id
+        answer_d[AnswerKeys.SUBSET_NAME] = subset_name
+        answer_d[AnswerKeys.RAW_INPUT] = input_d[AnswerKeys.RAW_INPUT]
+        answer_d[AnswerKeys.ORIGIN_PROMPT] = input_d
+        return answer_d
 
     def get_answers(self,
                     subset_name: str,
@@ -147,57 +136,24 @@ class Evaluator(object):
             resp_answers_list: List[Dict[str, Any]] = self.model_adapter.predict(
                 inputs=prompts_list, infer_cfg=infer_cfg)
 
-            assert len(prompts_list) == len(resp_answers_list), \
-                f'Length of prompts_list({len(prompts_list)}) != Length of resp_answers_list({len(resp_answers_list)})'
-
-            for in_d, resp_d in zip(prompts_list, resp_answers_list):
-
-                # Gen answer_id (concat: model_cfg + input_prompt + infer_cfg)
-                model_cfg_str = json.dumps(
-                    OrderedDict(sorted(dict_torch_dtype_to_str(self.model_adapter.model_cfg).items())),
-                    ensure_ascii=False)
-                input_prompt_str = json.dumps(
-                    OrderedDict(sorted(dict_torch_dtype_to_str(in_d).items())), ensure_ascii=False)
-                infer_cfg_str = json.dumps(
-                    OrderedDict(sorted(dict_torch_dtype_to_str(infer_cfg).items())), ensure_ascii=False)
-                answer_id = 'answer-' + gen_hash(model_cfg_str + input_prompt_str + infer_cfg_str)
-
-                resp_d[AnswerKeys.MODEL_SPEC] = self.model_adapter.model_cfg
-                resp_d[AnswerKeys.ANSWER_ID] = answer_id
-                resp_d[AnswerKeys.SUBSET_NAME] = subset_name
-                resp_d[AnswerKeys.RAW_INPUT] = in_d[AnswerKeys.RAW_INPUT]
-                resp_d[AnswerKeys.ORIGIN_PROMPT] = in_d
-
-                answers_list.append(resp_d)
-                dump_jsonl_data(resp_d, pred_file_path, dump_mode=DumpMode.APPEND)
+            for input_prompt, answer_d in zip(prompts_list, resp_answers_list):
+                answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
+                processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
+                answers_list.append(processed_answer)
+                dump_jsonl_data(processed_answer, pred_file_path, dump_mode=DumpMode.APPEND)
 
         else:
             for input_prompt in tqdm(prompts_list, total=len(prompts_list), desc=f'Predicting({subset_name}): '):
-
-                # Gen answer_id (concat: model_cfg + input_prompt + infer_cfg)
-                model_cfg_str = json.dumps(
-                    OrderedDict(sorted(dict_torch_dtype_to_str(self.model_adapter.model_cfg).items())),
-                    ensure_ascii=False)
-                input_prompt_str = json.dumps(
-                    OrderedDict(sorted(dict_torch_dtype_to_str(input_prompt).items())), ensure_ascii=False)
-                infer_cfg_str = json.dumps(
-                    OrderedDict(sorted(dict_torch_dtype_to_str(infer_cfg).items())), ensure_ascii=False)
-                answer_id = 'answer-' + gen_hash(model_cfg_str + input_prompt_str + infer_cfg_str)
-
-                # Get answers
-                answer_d: dict = self._pred_answer(
-                    input_d=input_prompt, infer_cfg=infer_cfg, subset_name=subset_name, answer_id=answer_id)
-
-                answer_d[AnswerKeys.MODEL_SPEC] = self.model_adapter.model_cfg
-                answer_d[AnswerKeys.RAW_INPUT] = input_prompt[AnswerKeys.RAW_INPUT]
-                answer_d[AnswerKeys.ORIGIN_PROMPT] = input_prompt
+                answer_d: dict = self.model_adapter.predict(inputs=input_prompt, infer_cfg=infer_cfg)
+                answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
+                processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
 
                 if debug:
                     logger.info(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
-                    logger.info(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
+                    logger.info(f'**predicted ans: {json.dumps(processed_answer, ensure_ascii=False)} \n')
 
-                answers_list.append(answer_d)
-                dump_jsonl_data(answer_d, pred_file_path, dump_mode=DumpMode.APPEND)
+                answers_list.append(processed_answer)
+                dump_jsonl_data(processed_answer, pred_file_path, dump_mode=DumpMode.APPEND)
 
         logger.info(f'Dump predictions to {pred_file_path}.')
         return answers_list
@@ -241,6 +197,19 @@ class Evaluator(object):
 
         return review_res
 
+    def _generate_review_id(self, answer_d):
+        # Gen review_id (concat: answer_id + reviewer_spec)
+        answer_id = answer_d[AnswerKeys.ANSWER_ID]
+        reviewer_spec = {
+            'metric': [metric_d['name'] for metric_d in self.data_adapter.metric_list],
+            'reviewer': ['Evaluator'],
+            'revision': ['default']
+        }
+        reviewer_spec_str = json.dumps(
+            OrderedDict(sorted(dict_torch_dtype_to_str(reviewer_spec).items())), ensure_ascii=False)
+        review_id = 'review-' + gen_hash(answer_id + reviewer_spec_str)
+        return review_id, reviewer_spec
+
     def get_reviews(self, subset_name: str, answers_list: List[dict], debug: bool = False, **kwargs) -> list:
         """
         Get reviews from answers.
@@ -264,19 +233,7 @@ class Evaluator(object):
             logger.warning(f'Ignore use_cache={self.use_cache}, updating the review file: {review_file_path} ...')
 
         for answer_d in tqdm(answers_list, total=len(answers_list), desc=f'Reviewing({subset_name}): '):
-
-            # Gen review_id (concat: answer_id + reviewer_spec)
-            answer_id = answer_d[AnswerKeys.ANSWER_ID]
-
-            reviewer_spec: dict = {
-                'metric': [metric_d['name'] for metric_d in self.data_adapter.metric_list],
-                'reviewer': ['Evaluator'],
-                'revision': ['default']
-            }
-            reviewer_spec_str = json.dumps(
-                OrderedDict(sorted(dict_torch_dtype_to_str(reviewer_spec).items())), ensure_ascii=False)
-            review_id = 'review-' + gen_hash(answer_id + reviewer_spec_str)
-
+            review_id, reviewer_spec = self._generate_review_id(answer_d)
             # Get review
             review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
 
@@ -284,7 +241,6 @@ class Evaluator(object):
                 logger.info(review_d)
 
             reviews_list.append(review_d)
-
             # Dump reviews
             dump_jsonl_data(review_d, review_file_path, dump_mode=DumpMode.APPEND)
 
@@ -380,7 +336,8 @@ class Evaluator(object):
         stage_answers_dict = {}
         stage_reviews_dict = {}
 
-        for subset_name, prompts_list in self.prompts.items():
+        prompts = self.load_dataset()
+        for subset_name, prompts_list in prompts.items():
             limit = kwargs.get('limit', len(prompts_list))
             prompts_list = prompts_list[:limit]
 

evalscope/evaluator/reviewer/auto_reviewer.py

@@ -8,10 +8,10 @@ import sys
 import time
 from abc import ABC, abstractmethod
 from functools import partial
-from typing import Any, List
+from typing import Any, List, Tuple
 
 from evalscope.constants import ArenaMode, EvalConfigKeys, FnCompletionParser, PositionBiasMitigation
-from evalscope.models.openai_model import OpenAIModel
+from evalscope.models.model import OpenAIModel
 from evalscope.utils import completion_parsers, random_seeded_choice
 from evalscope.utils.arena_utils import get_battle_pairs, merge_ques_ans, shuffle_pairwise_preferences
 from evalscope.utils.io_utils import dump_jsonl_data, jsonl_to_list
@@ -240,7 +240,15 @@ class AutoReviewerGpt4(BaseReviewer):
             review_text=review_text)
         return review_result
 
-    def _get_review_pair(self, model_a, model_b, question, category, ans1, ans2, dry_run=False, **kwargs) -> (str, Any):
+    def _get_review_pair(self,
+                         model_a,
+                         model_b,
+                         question,
+                         category,
+                         ans1,
+                         ans2,
+                         dry_run=False,
+                         **kwargs) -> Tuple[str, Any]:
         input_msg = dict(ques=question, category=category, ans1=ans1, ans2=ans2)
 
         if self.reference_list:
@@ -263,7 +271,7 @@ class AutoReviewerGpt4(BaseReviewer):
             result = (result, None)
         return review_text, *result
 
-    def _get_review_single(self, model, question, category, answer, dry_run=False, **kwargs) -> (str, Any):
+    def _get_review_single(self, model, question, category, answer, dry_run=False, **kwargs) -> Tuple[str, Any]:
         input_msg = dict(ques=question, category=category, ans1=answer)
 
         if self.reference_list:

evalscope/metrics/__init__.py

@@ -1 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from evalscope.metrics.metrics import bleu_ngram_one_sample, exact_match, weighted_mean
+from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
+
+WeightedAverageAccuracy = {'name': 'WeightedAverageAccuracy', 'object': weighted_mean}
+WeightedAverageBLEU = {'name': 'WeightedAverageBLEU', 'object': weighted_mean}
+Pass1 = {'name': 'Pass@1', 'object': weighted_mean}

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -55,7 +55,7 @@ try:
         os.system(f'wget --timeout=10 --tries=3 -P {nltk_dir} {punkt_tab_url}')
         os.system(f'unzip {punkt_path} -d {nltk_dir}')
     else:
-        logger.info(f'{punkt_path} already exists, skipping download')
+        logger.debug(f'{punkt_path} already exists, skipping download')
 except Exception as e:
     logger.error(f'Try to download punkt_tab.zip for nltk failed: {e}')