evalscope 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

evalscope/collections/evaluator.py

@@ -0,0 +1,198 @@
+import json
+import os
+import pandas as pd
+from collections import defaultdict
+from tabulate import tabulate
+from tqdm import tqdm
+from typing import List
+
+from evalscope.benchmarks import Benchmark
+from evalscope.collections.sampler import DatasetEntry
+from evalscope.config import TaskConfig
+from evalscope.constants import DataCollection, DumpMode
+from evalscope.evaluator import Evaluator
+from evalscope.models import get_local_model, initialize_model_adapter
+from evalscope.report import ReportGenerator
+from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class SimpleEvaluator(Evaluator):
+
+    def __init__(self, dataset_name, data_adapter, model_adapter, task_cfg, outputs):
+        super().__init__(
+            dataset_name_or_path=dataset_name,
+            data_adapter=data_adapter,
+            model_adapter=model_adapter,
+            task_cfg=task_cfg,
+            outputs=outputs)
+
+    def get_answer(self, input_prompt, subset_name, infer_cfg) -> dict:
+        answer_d: dict = self.model_adapter.predict(inputs=input_prompt, infer_cfg=infer_cfg)
+        answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
+        processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
+        return processed_answer
+
+    def get_review(self, answer_d) -> dict:
+        review_id, reviewer_spec = self._generate_review_id(answer_d)
+        review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
+        return review_d
+
+    def get_score(self, review_d) -> float:
+        metric_score: List[dict] = self.compute_metrics(reviews_list=[review_d])
+        # use the first metric by default
+        score = metric_score[0]['score']
+        return score
+
+
+class EvaluatorCollection:
+
+    def __init__(self, task_cfg: TaskConfig, outputs: OutputsStructure):
+        self.task_cfg = task_cfg
+        self.outputs = outputs
+        self.model = get_local_model(task_cfg)
+        self.dataset, self.dataset_name = self.load()
+        self.dataset_name_map, self.dataset_id_map = self._parse_dataset()
+        self.evaluators = self._initialize_evaluators()
+
+    def load(self) -> tuple[list[DatasetEntry], str]:
+        dataset_path = self.task_cfg.dataset_args[DataCollection.NAME]['local_path']
+        dataset_name = os.path.basename(dataset_path).split('.')[0]
+        raw_dataset = jsonl_to_list(dataset_path)
+        datasets = []
+        for sample in raw_dataset:
+            datasets.append(DatasetEntry(**sample))
+        return datasets, dataset_name
+
+    def _parse_dataset(self):
+        dataset_name_map = defaultdict(lambda: defaultdict(list))
+        dataset_id_map = {}
+        for sample in self.dataset:
+            dataset_name, subset_name = sample.dataset_name, sample.subset_name
+            dataset_name_map[dataset_name][subset_name].append(sample.index)
+            dataset_id_map[sample.index] = sample
+        return dataset_name_map, dataset_id_map
+
+    def _initialize_evaluators(self):
+        evaluators = {}
+        for dataset_name in self.dataset_name_map.keys():
+            benchmark = Benchmark.get(dataset_name)
+            data_adapter = benchmark.get_data_adapter()
+            model_adapter = initialize_model_adapter(self.task_cfg, benchmark.model_adapter, self.model)
+            evaluators[dataset_name] = SimpleEvaluator(dataset_name, data_adapter, model_adapter, self.task_cfg,
+                                                       self.outputs)
+        return evaluators
+
+    def get_report(self, scores):
+
+        def get_dataframe(scores):
+            data = []
+            for dataset_name, data_map in self.dataset_name_map.items():
+                for subset_name, ids in data_map.items():
+                    for _id in ids:
+                        row_data: DatasetEntry = self.dataset_id_map[_id]
+                        score = scores[_id]
+                        data.append(
+                            dict(
+                                task_type=row_data.task_type,
+                                categories=tuple(row_data.categories),
+                                dataset_name=dataset_name,
+                                subset_name=subset_name,
+                                tags=row_data.tags,
+                                score=score))
+            return pd.DataFrame(data)
+
+        def aggregate_and_sort(df, group_by_cols):
+            # aggregate by group_by_cols, and calculate average_score and count
+            report_df = df.groupby(group_by_cols) \
+                .agg(average_score=('score', 'mean'), count=('score', 'size')) \
+                .reset_index()
+            report_df['average_score'] = report_df['average_score'].round(4)
+            report_df = report_df.sort_values(by='count', ascending=False) \
+                .to_dict(orient='records')
+            return report_df
+
+        df = get_dataframe(scores)
+
+        # multi-level aggregation
+        subset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name', 'subset_name'])
+        dataset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name'])
+        task_report_df = aggregate_and_sort(df, ['task_type'])
+
+        # explode tags to multiple rows
+        df_exploded_tags = df.explode('tags')
+        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags'])
+
+        # process multi-level categories
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories['categories'].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'category{level}'] = df_categories['categories'].apply(lambda x: x[level]
+                                                                                  if len(x) > level else '')
+        category_report_df = aggregate_and_sort(df_categories, [f'category{level}' for level in range(max_depth)])
+
+        # convert to dict format
+        report_dict = {
+            'subset_level': subset_report_df,
+            'dataset_level': dataset_report_df,
+            'task_level': task_report_df,
+            'tag_level': tag_report_df,
+            'category_level': category_report_df,
+        }
+
+        # record report
+        for level, data in report_dict.items():
+            table = tabulate(data, headers='keys', tablefmt='pretty', showindex=False)
+            logger.info(f'{level} Report:\n{table}')
+
+        report = ReportGenerator.gen_collection_report(df, self.dataset_name, self.task_cfg.model_id)
+        # save report to JSON file
+        report_file_path = os.path.join(self.outputs.reports_dir, self.task_cfg.model_id, f'{self.dataset_name}.json')
+        os.makedirs(os.path.dirname(report_file_path), exist_ok=True)
+        with open(report_file_path, 'w', encoding='utf-8') as f:
+            json.dump(report.to_dict(), f, ensure_ascii=False, indent=4)
+
+    def get_answers(self):
+        pred_file_path = os.path.join(self.outputs.predictions_dir, self.task_cfg.model_id,
+                                      f'{self.dataset_name}.jsonl')
+        os.makedirs(os.path.dirname(pred_file_path), exist_ok=True)
+        answers = defaultdict(dict)
+        for sample in tqdm(self.dataset, desc='Getting answers'):
+            evaluator = self.evaluators[sample.dataset_name]
+            answer_d = evaluator.get_answer(sample.prompt, sample.subset_name, self.task_cfg.generation_config)
+            answers[sample.index] = answer_d
+            dump_jsonl_data(answer_d, pred_file_path, dump_mode=DumpMode.APPEND)
+        return answers
+
+    def get_reviews(self, answers):
+        review_file_path = os.path.join(self.outputs.reviews_dir, self.task_cfg.model_id)
+        os.makedirs(review_file_path, exist_ok=True)
+        reviews = defaultdict(dict)
+        for sample in tqdm(self.dataset, desc='Getting reviews'):
+            evaluator = self.evaluators[sample.dataset_name]
+            review_d = evaluator.get_review(answers[sample.index])
+            reviews[sample.index] = review_d
+            dump_jsonl_data(
+                review_d,
+                os.path.join(review_file_path, f'{self.dataset_name}_{sample.dataset_name}_{sample.subset_name}.jsonl'),
+                dump_mode=DumpMode.APPEND)
+        return reviews
+
+    def get_scores(self, reviews) -> float:
+        scores = defaultdict(dict)
+        for sample in tqdm(self.dataset, desc='Getting scores'):
+            evaluator = self.evaluators[sample.dataset_name]
+            review_d = reviews[sample.index]
+            score = evaluator.get_score(review_d)
+            scores[sample.index] = score
+
+        return scores
+
+    def eval(self, **kwargs):
+        answers = self.get_answers()
+        reviews = self.get_reviews(answers)
+        scores = self.get_scores(reviews)
+        self.get_report(scores)

evalscope/collections/sampler.py

@@ -0,0 +1,138 @@
+import random
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass, field
+from tqdm import tqdm
+from typing import List, Optional
+
+from evalscope.collections.schema import CollectionSchema, DatasetInfo
+
+
+@dataclass
+class DatasetEntry:
+    index: int = 0
+    prompt: dict = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
+    categories: List[str] = field(default_factory=list)
+    task_type: str = ''
+    weight: float = 0.0
+    dataset_name: str = ''
+    subset_name: str = ''
+
+
+# Define an abstract base class for Samplers
+class Sampler(ABC):
+
+    def __init__(self, schema: CollectionSchema):
+        self.schema = schema
+
+    @abstractmethod
+    def sample(self) -> List[dict]:
+        raise NotImplementedError
+
+    def _sample_dataset(self, dataset: DatasetInfo, count: int) -> List[DatasetEntry]:
+        all_data = []
+        data_dict = dataset.get_data()
+        for subset_name, subset_data in data_dict.items():
+            for prompt in subset_data:
+                all_data.append(
+                    DatasetEntry(
+                        prompt=prompt,
+                        tags=dataset.tags,
+                        categories=dataset.hierarchy,
+                        task_type=dataset.task_type,
+                        weight=dataset.weight,
+                        dataset_name=dataset.name,
+                        subset_name=subset_name,
+                    ))
+        sampled_data = random.choices(all_data, k=count)
+        return sampled_data
+
+    def _update_index(self, all_data: List[DatasetEntry]) -> List[dict]:
+        result = []
+        for i, entry in enumerate(all_data):
+            entry.index = i
+            result.append(asdict(entry))
+        return result
+
+
+class WeightedSampler(Sampler):
+    """
+    Weighted sampler, according to the weight of each dataset, sample data from each dataset.
+    """
+
+    def sample(self, count: int) -> List[dict]:
+        dataset_info_list = self.schema.flatten()
+        sampled_data = []
+        remaining_count = count
+
+        for i, dataset in enumerate(tqdm(dataset_info_list, desc='Sampling data')):
+            if i == len(dataset_info_list) - 1:
+                dataset_sample_count = remaining_count
+            else:
+                dataset_sample_count = int(dataset.weight * count)
+                remaining_count -= dataset_sample_count
+
+            sampled_data.extend(self._sample_dataset(dataset, dataset_sample_count))
+
+        return self._update_index(sampled_data)
+
+
+class UniformSampler(Sampler):
+    """
+    Uniform sampler, sample data from each dataset with the same number of samples.
+    """
+
+    def sample(self, count: int) -> List[dict]:
+        dataset_info_list = self.schema.flatten()
+        num_datasets = len(dataset_info_list)
+        remaining_count = count
+        sampled_data = []
+
+        for i, dataset in enumerate(tqdm(dataset_info_list, desc='Sampling data')):
+            if i == len(dataset_info_list) - 1:
+                dataset_sample_count = remaining_count
+            else:
+                dataset_sample_count = count // num_datasets
+                remaining_count -= dataset_sample_count
+
+            sampled_data.extend(self._sample_dataset(dataset, dataset_sample_count))
+
+        return self._update_index(sampled_data)
+
+
+class StratifiedSampler(Sampler):
+    """
+    Stratified sampler, sample data from each dataset according to the number of samples of each dataset.
+    """
+
+    def sample(self, count: int) -> List[dict]:
+        dataset_info_list = self.schema.flatten()
+
+        total_samples = sum(len(dataset.get_data()) for dataset in dataset_info_list)
+        remaining_count = count
+        sampled_data = []
+
+        for i, dataset in enumerate(tqdm(dataset_info_list, desc='Sampling data')):
+            if i == len(dataset_info_list) - 1:
+                dataset_sample_count = remaining_count
+            else:
+                dataset_sample_count = int((len(dataset.get_data()) / total_samples) * count)
+                remaining_count -= dataset_sample_count
+
+            sampled_data.extend(self._sample_dataset(dataset, dataset_sample_count))
+        return self._update_index(sampled_data)
+
+
+if __name__ == '__main__':
+    from evalscope.utils.io_utils import dump_jsonl_data
+
+    schema = CollectionSchema.from_json('outputs/schema.json')
+    print(schema.to_dict())
+    mixed_data = WeightedSampler(schema).sample(10)
+    dump_jsonl_data(mixed_data, 'outputs/weighted_mixed_data.jsonl')
+
+    # mixed_data = UniformSampler(schema, 100).sample()
+    # dump_jsonl_data(mixed_data, 'outputs/uniform_mixed_data.jsonl')
+
+    # mixed_data = StratifiedSampler(schema, 100).sample()
+    # dump_jsonl_data(mixed_data, 'outputs/stratified_mixed_data.jsonl')

evalscope/collections/schema.py

@@ -0,0 +1,126 @@
+import copy
+import json
+from dataclasses import asdict, dataclass, field
+from typing import List, Union
+
+
+@dataclass
+class DatasetInfo:
+    name: str
+    weight: float = 1.0  # sample weight in each collection
+    task_type: str = ''
+    tags: List[str] = field(default_factory=list)
+    args: dict = field(default_factory=dict)
+    hierarchy: List[str] = field(default_factory=list)
+
+    def get_data(self) -> dict:
+        from evalscope.benchmarks import Benchmark
+
+        benchmark_meta = Benchmark.get(self.name)
+
+        data_adapter = benchmark_meta.get_data_adapter(config=self.args)
+        data_dict = data_adapter.load(
+            dataset_name_or_path=benchmark_meta.dataset_id, subset_list=benchmark_meta.subset_list)
+        prompts = data_adapter.gen_prompts(data_dict)
+        return prompts
+
+
+def flatten_weight(collection: 'CollectionSchema', base_weight=1):
+    total_weight = sum(dataset.weight for dataset in collection.datasets)
+    for dataset in collection.datasets:
+        current_weight = dataset.weight / total_weight * base_weight
+        if isinstance(dataset, CollectionSchema):
+            flatten_weight(dataset, current_weight)
+        else:
+            dataset.weight = current_weight
+
+
+def flatten_name(collection: 'CollectionSchema', parent_names=None):
+    if parent_names is None:
+        parent_names = []
+    current_names = parent_names + [collection.name]
+    for dataset in collection.datasets:
+        if isinstance(dataset, CollectionSchema):
+            flatten_name(dataset, current_names)
+        else:
+            dataset.hierarchy = current_names.copy()
+
+
+def flatten_datasets(collection: 'CollectionSchema') -> List[DatasetInfo]:
+    flat_datasets = []
+    for dataset in collection.datasets:
+        if isinstance(dataset, CollectionSchema):
+            flat_datasets.extend(flatten_datasets(dataset))
+        else:
+            flat_datasets.append(dataset)
+    return flat_datasets
+
+
+@dataclass
+class CollectionSchema:
+    name: str
+    weight: float = 1.0
+    datasets: List[Union[DatasetInfo, 'CollectionSchema']] = field(default_factory=list)
+
+    def __str__(self):
+        return json.dumps(self.to_dict(), ensure_ascii=False, indent=4)
+
+    def to_dict(self):
+        return {
+            'name':
+            self.name,
+            'weight':
+            self.weight,
+            'datasets':
+            [asdict(dataset) if isinstance(dataset, DatasetInfo) else dataset.to_dict() for dataset in self.datasets],
+        }
+
+    @classmethod
+    def from_dict(cls, data):
+        instance = cls(name=data.get('name', ''), weight=data.get('weight', 1))
+        for dataset in data.get('datasets', []):
+            if 'datasets' in dataset:
+                instance.datasets.append(CollectionSchema.from_dict(dataset))
+            else:
+                instance.datasets.append(DatasetInfo(**dataset))
+        return instance
+
+    def dump_json(self, file_path):
+        d = self.to_dict()
+        with open(file_path, 'w') as f:
+            json.dump(d, f, ensure_ascii=False, indent=4)
+
+    @classmethod
+    def from_json(cls, file_path):
+        with open(file_path, 'r') as f:
+            data = json.load(f)
+        return cls.from_dict(data)
+
+    def flatten(self) -> List[DatasetInfo]:
+        collection = copy.deepcopy(self)
+        flatten_name(collection)
+        flatten_weight(collection)
+        return flatten_datasets(collection)
+
+
+if __name__ == '__main__':
+    schema = CollectionSchema(
+        name='reasoning',
+        datasets=[
+            CollectionSchema(name='english', datasets=[
+                DatasetInfo(name='arc', weight=1, tags=['en']),
+            ]),
+            CollectionSchema(
+                name='chinese',
+                datasets=[DatasetInfo(name='ceval', weight=1, tags=['zh'], args={'subset_list': ['logic']})])
+        ])
+    print(schema)
+    print(schema.flatten())
+    schema.dump_json('outputs/schema.json')
+
+    schema = CollectionSchema.from_json('outputs/schema.json')
+    print(schema)
+    # 打印扁平化后的结果
+    for dataset in schema.flatten():
+        print(f'Dataset: {dataset.name}')
+        print(f"Hierarchy: {' -> '.join(dataset.hierarchy)}")

evalscope/config.py

@@ -31,7 +31,7 @@ DEFAULT_GENERATION_CONFIG = {
 @dataclass
 class TaskConfig:
     # Model-related arguments
-    model: Union[str, CustomModel, None] = None
+    model: Union[str, 'CustomModel', None] = None
     model_id: Optional[str] = None
     model_args: Optional[Dict] = field(default_factory=lambda: DEFAULT_MODEL_ARGS | {})
 
@@ -40,8 +40,8 @@ class TaskConfig:
     chat_template: Optional[str] = None
 
     # Dataset-related arguments
-    datasets: Optional[List[str]] = None
-    dataset_args: Optional[Dict] = field(default_factory=dict)
+    datasets: List[str] = field(default_factory=list)
+    dataset_args: Dict = field(default_factory=dict)
     dataset_dir: str = DEFAULT_DATASET_CACHE_DIR
     dataset_hub: str = HubType.MODELSCOPE
 
@@ -64,7 +64,9 @@ class TaskConfig:
     # Debug and runtime mode arguments
     debug: bool = False
     dry_run: bool = False
-    seed: int = 42
+    seed: Optional[int] = 42
+    api_url: Optional[str] = None  # Only used for server model
+    api_key: Optional[str] = 'EMPTY'  # Only used for server model
 
     def __post_init__(self):
         if (not self.model_id) and self.model:
@@ -74,7 +76,6 @@ class TaskConfig:
                 self.model_id = os.path.basename(self.model).rstrip(os.sep)
 
     def to_dict(self):
-        # Note: to avoid serialization error for some model instance
         return self.__dict__
 
     def __str__(self):
@@ -130,6 +131,7 @@ class TaskConfig:
                 continue
 
             task.model = custom_model
+            task.model_args = custom_model.config
             task.model_id = type(custom_model).__name__
             res_list.append(task)
 

evalscope/constants.py

@@ -135,34 +135,17 @@ class EvalStage:
 class EvalType:
 
     CUSTOM = 'custom'
-    CHECKPOINT = 'checkpoint'
+    CHECKPOINT = 'checkpoint'  # native model checkpoint
+    SERVICE = 'service'  # model service
 
 
 class EvalBackend:
+    NATIVE = 'Native'
+    OPEN_COMPASS = 'OpenCompass'
+    VLM_EVAL_KIT = 'VLMEvalKit'
+    RAG_EVAL = 'RAGEval'
+    THIRD_PARTY = 'ThirdParty'
 
-    class _Backend:
-        #  compatible with old version, set 'value'
 
-        def __init__(self, value):
-            self._value = value
-
-        @property
-        def value(self):
-            return self._value
-
-        def __str__(self):
-            return self._value
-
-        def __repr__(self):
-            return f"'{self._value}'"
-
-        def __eq__(self, other):
-            if isinstance(other, str):
-                return self._value == other
-            return NotImplemented
-
-    NATIVE = _Backend('Native')
-    OPEN_COMPASS = _Backend('OpenCompass')
-    VLM_EVAL_KIT = _Backend('VLMEvalKit')
-    RAG_EVAL = _Backend('RAGEval')
-    THIRD_PARTY = _Backend('ThirdParty')
+class DataCollection:
+    NAME = 'data_collection'