evalscope 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -5,45 +5,34 @@ import numpy as np
 import os
 from typing import List
 
+from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.metrics.metrics import exact_match, weighted_mean
-from evalscope.utils.logger import get_logger
+from evalscope.constants import EvalType
+from evalscope.metrics import WeightedAverageAccuracy
+from evalscope.metrics.metrics import exact_match
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils import get_logger
 
 # flake8: noqa
 
 logger = get_logger()
 
-DATASET_ID = 'modelscope/trivia_qa'
-SUBSET_LIST = ['default']
-
 
+@Benchmark.register(
+    name='trivia_qa',
+    dataset_id='modelscope/trivia_qa',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['default'],
+    metric_list=[WeightedAverageAccuracy],
+    few_shot_num=5,
+    train_split='dev',
+    eval_split='test',
+)
 class TriviaQaAdapter(DataAdapter):
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = None,
-                 train_split: str = 'dev',
-                 eval_split: str = 'test',
-                 **kwargs):
-
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
+    def __init__(self, **kwargs):
 
-        if few_shot_num is None:
-            logger.info(f'few_shot_num is not specified for TriviaQA, use default value: 5')
-            few_shot_num = 5
-
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            **kwargs)
+        super().__init__(**kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -113,16 +102,16 @@ class TriviaQaAdapter(DataAdapter):
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
         context: str = '\n'.join(few_shot_prompts) + '\n'
         context += self._generate_prompt(input_d=input_d, include_answer=False)
-        full_prompt = prompt + context
+        full_prompt = context
 
-        return {'data': [full_prompt]}
+        return {'data': [full_prompt], 'system_prompt': prompt}
 
     def get_gold_answer(self, input_d: dict) -> list:
         # Get the gold choice
         ans: list = input_d.get('ideal', [])
         return ans
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
         """
         Parse the model output to get the answer.
 
@@ -134,73 +123,11 @@ class TriviaQaAdapter(DataAdapter):
         Returns:
             The predicted answer.
         """
-        if eval_type == 'checkpoint':
-            return result
-        elif eval_type == 'service':  # TODO: to be implemented
-            return result
-        elif eval_type == 'custom':  # TODO: to be implemented
-            return result
-        else:
-            raise ValueError(f'Unknown eval_type: {eval_type}')
+        return result
 
     def match(self, gold: list, pred: str) -> float:
-        return max([exact_match(gold=ref, pred=pred) for ref in gold])
-
-    def compute_metric(self, review_res_list: list) -> float:
-        """
-        Compute evaluation result by specific metric.
-
-        Args:
-            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
-
-        Returns:
-            The metric score.
-        """
-        items = [(score, 1.0) for score in review_res_list]
-        return weighted_mean(items)
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Generate the report for the model output.
-
-        Args:
-            subset_score_map: {subset_name: (score, num), ...}
-            report_name: The user-defined report name.
-
-        Returns:
-        {
-            "name":"TriviaQA",
-            "metric":"WeightedAverageAccuracy",
-            "score":0.3389,
-            "category":[
-                {
-                    "name":"DEFAULT",
-                    "score":0.3389,
-                    "subset":[
-                        {
-                            "name":"default",
-                            "score":0.3389
-                        }
-                    ]
-                }
-            ],
-            "total_num":100
-        }
-        """
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-        cate_avg_list = [{'name': subset_name, 'score': score} for subset_name, (score, _) in subset_score_map.items()]
-
-        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
-
-        res_map = dict(
-            name=report_name or 'trivia_qa',
-            metric=self.metric_list[0]['name'],
-            score=weighted_avg_acc,
-            category=[category_d],
-            total_num=total_num)
-
-        return res_map
+        is_correct = any([cand in pred for cand in gold])
+        return 1 if is_correct else 0
 
     @classmethod
     def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:

evalscope/benchmarks/truthful_qa/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import DATASET_ID, SUBSET_LIST
-from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import TruthfulQaAdapter
-from evalscope.benchmarks.truthful_qa.truthful_qa_adapter import TruthfulQaAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ContinuationLogitsModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py

@@ -6,20 +6,31 @@ import numpy as np
 import os
 from typing import List
 
+from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics import WeightedAverageAccuracy
 from evalscope.metrics.metrics import weighted_mean
-from evalscope.utils import normalize_score
-from evalscope.utils.logger import get_logger
+from evalscope.models import ContinuationLogitsModelAdapter
+from evalscope.utils import get_logger, normalize_score
 
 # flake8: noqa
 
 logger = get_logger()
 
-DATASET_ID = 'modelscope/truthful_qa'
 # task_list = ['generation', 'multiple_choice']
-SUBSET_LIST = ['multiple_choice']
 
 
+@Benchmark.register(
+    name='truthful_qa',
+    dataset_id='modelscope/truthful_qa',
+    model_adapter=ContinuationLogitsModelAdapter,
+    subset_list=['multiple_choice'],
+    metric_list=[WeightedAverageAccuracy],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='validation',
+)
 class TruthfulQaAdapter(DataAdapter):
     """
     Adapter for TruthfulQA benchmark.
@@ -40,35 +51,14 @@ class TruthfulQaAdapter(DataAdapter):
                  'Q: Where were the 1992 Olympics held?\n'
                  'A: The 1992 Olympics were held in Barcelona, Spain.')
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = None,
-                 train_split: str = None,
-                 eval_split: str = 'validation',
-                 **kwargs):
-
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
-
-        if few_shot_num is None:
-            logger.info(f'Set 0-shot examples by system for TruthfulQA.')
-            few_shot_num = 0
+    def __init__(self, **kwargs):
 
+        few_shot_num = kwargs.get('few_shot_num', 0)
         if few_shot_num != 0:
             logger.warning(f'few_shot_num should be 0 for TruthfulQA, but got {few_shot_num}. Use 0-shot by default.')
-            few_shot_num = 0
+            kwargs['few_shot_num'] = 0
 
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            **kwargs)
+        super().__init__(**kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -215,7 +205,7 @@ class TruthfulQaAdapter(DataAdapter):
         # TODO: generation sub-task to be added
         return {'mc1_labels': input_d['mc1_targets']['labels'], 'mc2_labels': input_d['mc2_targets']['labels']}
 
-    def parse_pred_result(self, result: list, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> list:
+    def parse_pred_result(self, result: list, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> list:
         """
         Parse the model output to get the answer.
 
@@ -227,11 +217,11 @@ class TruthfulQaAdapter(DataAdapter):
         Returns:
             The predicted answer.
         """
-        if eval_type == 'checkpoint':
+        if eval_type == EvalType.CHECKPOINT:
             return result
-        elif eval_type == 'service':  # TODO: to be supported !
+        elif eval_type == EvalType.SERVICE:  # TODO: to be supported !
             return result
-        elif eval_type == 'custom':  # TODO: to be supported !
+        elif eval_type == EvalType.CUSTOM:  # TODO: to be supported !
             return result
         else:
             raise ValueError(f'Invalid eval_type: {eval_type}')

evalscope/collections/__init__.py

@@ -0,0 +1,3 @@
+from evalscope.collections.evaluator import EvaluatorCollection
+from evalscope.collections.sampler import StratifiedSampler, UniformSampler, WeightedSampler
+from evalscope.collections.schema import CollectionSchema, DatasetInfo

evalscope/collections/evaluator.py

@@ -0,0 +1,178 @@
+import json
+import os
+import pandas as pd
+from collections import defaultdict
+from tabulate import tabulate
+from tqdm import tqdm
+
+from evalscope.benchmarks import Benchmark
+from evalscope.collections.sampler import DatasetEntry
+from evalscope.config import TaskConfig
+from evalscope.constants import AnswerKeys, DumpMode, EvalType, ReviewKeys
+from evalscope.evaluator import Evaluator
+from evalscope.models import get_local_model, initialize_model_adapter
+from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class SimpleEvaluator(Evaluator):
+
+    def __init__(self, dataset_name, data_adapter, model_adapter, task_cfg, outputs):
+        super().__init__(
+            dataset_name_or_path=dataset_name,
+            data_adapter=data_adapter,
+            model_adapter=model_adapter,
+            task_cfg=task_cfg,
+            outputs=outputs)
+
+    def get_answer(self, input_prompt, subset_name, infer_cfg) -> dict:
+        answer_d: dict = self.model_adapter.predict(inputs=input_prompt, infer_cfg=infer_cfg)
+        answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
+        processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
+        return processed_answer
+
+    def get_review(self, answer_d) -> dict:
+        review_id, reviewer_spec = self._generate_review_id(answer_d)
+        review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
+        return review_d
+
+
+class EvaluatorCollection:
+
+    def __init__(self, task_cfg: TaskConfig, outputs: OutputsStructure):
+        self.task_cfg = task_cfg
+        self.outputs = outputs
+        self.model = get_local_model(task_cfg)
+        self.dataset = self.load()
+        self.dataset_name_map, self.dataset_id_map = self._parse_dataset()
+        self.evaluators = self._initialize_evaluators()
+
+    def load(self) -> list[DatasetEntry]:
+        raw_dataset = jsonl_to_list(self.task_cfg.dataset_args['data_collection']['local_path'])
+        datasets = []
+        for sample in raw_dataset:
+            datasets.append(DatasetEntry(**sample))
+        return datasets
+
+    def _parse_dataset(self):
+        dataset_name_map = defaultdict(lambda: defaultdict(list))
+        dataset_id_map = {}
+        for sample in self.dataset:
+            dataset_name, subset_name = sample.dataset_name, sample.subset_name
+            dataset_name_map[dataset_name][subset_name].append(sample.index)
+            dataset_id_map[sample.index] = sample
+        return dataset_name_map, dataset_id_map
+
+    def _initialize_evaluators(self):
+        evaluators = {}
+        for dataset_name in self.dataset_name_map.keys():
+            benchmark = Benchmark.get(dataset_name)
+            data_adapter = benchmark.get_data_adapter()
+            model_adapter = initialize_model_adapter(self.task_cfg, benchmark.model_adapter, self.model)
+            evaluators[dataset_name] = SimpleEvaluator(dataset_name, data_adapter, model_adapter, self.task_cfg,
+                                                       self.outputs)
+        return evaluators
+
+    def get_report(self, reviews):
+        data = []
+        for dataset_name, data_map in self.dataset_name_map.items():
+            for subset_name, ids in data_map.items():
+                for _id in ids:
+                    review_d = reviews[_id]
+                    row_data: DatasetEntry = self.dataset_id_map[_id]
+                    score = self.get_pred_score(review_d)
+                    data.append({
+                        'task_type': row_data.task,
+                        'dataset_name': dataset_name,
+                        'subset_name': subset_name,
+                        'tags': row_data.tags,
+                        'score': score
+                    })
+
+        df = pd.DataFrame(data)
+        # Explode tags to multiple rows
+        df_exploded = df.explode('tags')
+
+        # Helper function for aggregation and sorting
+        def aggregate_and_sort(df, group_by_cols):
+            report_df = df.groupby(group_by_cols) \
+                .agg(average_score=('score', 'mean'), count=('score', 'size')) \
+                .reset_index()
+
+            # Round average_score to 4 decimal places
+            report_df['average_score'] = report_df['average_score'].round(4)
+
+            report_df = report_df.sort_values(by='count', ascending=False) \
+                .to_dict(orient='records')
+            return report_df
+
+        # Multi-level aggregation
+        subset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name', 'subset_name'])
+        dataset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name'])
+        task_report_df = aggregate_and_sort(df, ['task_type'])
+        tag_report_df = aggregate_and_sort(df_exploded, ['tags'])
+
+        # Convert sorted DataFrames to Dict
+        report = {
+            'subset_level': subset_report_df,
+            'dataset_level': dataset_report_df,
+            'task_level': task_report_df,
+            'tag_level': tag_report_df
+        }
+
+        # Log the report
+        for level, data in report.items():
+            table = tabulate(data, headers='keys', tablefmt='pretty', showindex=False)
+            logger.info(f'{level} Report:\n{table}')
+
+        # Save the report to a JSON file
+        report_file_path = os.path.join(self.outputs.reports_dir, 'data_collection.json')
+        with open(report_file_path, 'w', encoding='utf-8') as f:
+            json.dump(report, f, ensure_ascii=False, indent=4)
+
+    def get_answers(self):
+        pred_file_path = os.path.join(self.outputs.predictions_dir, 'data_collection.jsonl')
+        answers = defaultdict(dict)
+        for sample in tqdm(self.dataset, desc='Getting answers'):
+            evaluator = self.evaluators[sample.dataset_name]
+            answer_d = evaluator.get_answer(sample.prompt, sample.subset_name, self.task_cfg.generation_config)
+            answers[sample.index] = answer_d
+            dump_jsonl_data(answer_d, pred_file_path, dump_mode=DumpMode.APPEND)
+        return answers
+
+    def get_reviews(self, answers):
+        review_file_path = os.path.join(self.outputs.reviews_dir, 'data_collection.jsonl')
+        reviews = defaultdict(dict)
+        for sample in tqdm(self.dataset, desc='Getting reviews'):
+            evaluator = self.evaluators[sample.dataset_name]
+            review_d = evaluator.get_review(answers[sample.index])
+            reviews[sample.index] = review_d
+            dump_jsonl_data(review_d, review_file_path, dump_mode=DumpMode.APPEND)
+        return reviews
+
+    @staticmethod
+    def get_pred_score(review_d) -> float:
+        return float(review_d[AnswerKeys.CHOICES][0][ReviewKeys.REVIEW][ReviewKeys.RESULT])
+
+    def eval(self, **kwargs):
+        answers = self.get_answers()
+        reviews = self.get_reviews(answers)
+        self.get_report(reviews)
+
+
+if __name__ == '__main__':
+    task_cfg = TaskConfig(
+        model='qwen2.5',
+        api_url='http://127.0.0.1:8801/v1/chat/completions',
+        api_key='EMPTY',
+        eval_type=EvalType.SERVICE,
+        datasets=['data_collection'],
+        dataset_args={'data_collection': {
+            'local_path': 'outputs/mixed_data.jsonl'
+        }},
+    )
+
+    evaluator_collection = EvaluatorCollection(task_cfg)
+    evaluator_collection.eval()

evalscope/collections/sampler.py

@@ -0,0 +1,132 @@
+import random
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass, field
+from tqdm import tqdm
+from typing import List, Optional
+
+from evalscope.collections.schema import CollectionSchema, DatasetInfo
+
+
+@dataclass
+class DatasetEntry:
+    index: int = 0
+    prompt: dict = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
+    task: str = ''
+    weight: float = 0.0
+    dataset_name: str = ''
+    subset_name: str = ''
+
+
+# Define an abstract base class for Samplers
+class Sampler(ABC):
+
+    def __init__(self, schema: CollectionSchema, count: Optional[int] = None):
+        self.schema = schema
+        self.count = count
+
+    @abstractmethod
+    def sample(self) -> List[dict]:
+        raise NotImplementedError
+
+    def _collect_dataset_data(self, dataset_info_list: List[DatasetInfo]) -> List[DatasetEntry]:
+        all_data = []
+        for dataset in tqdm(dataset_info_list, desc='Collecting dataset data'):
+            data_dict = dataset.get_data()
+            for subset_name, subset_data in data_dict.items():
+                for prompt in subset_data:
+                    all_data.append(
+                        DatasetEntry(
+                            prompt=prompt,
+                            tags=dataset.tags,
+                            task=dataset.task_type,
+                            weight=dataset.weight,
+                            dataset_name=dataset.name,
+                            subset_name=subset_name,
+                        ))
+        return all_data
+
+    def _update_index(self, all_data: List[DatasetEntry]) -> List[dict]:
+        result = []
+        for i, entry in enumerate(all_data):
+            entry.index = i
+            result.append(asdict(entry))
+        return result
+
+
+class WeightedSampler(Sampler):
+    """
+    Weighted sampler, according to the weight of each dataset, sample data from each dataset.
+    """
+
+    def sample(self) -> List[dict]:
+        dataset_info_list = self.schema.flatten()
+        all_data = self._collect_dataset_data(dataset_info_list)
+
+        remaining_count = self.count
+        sampled_data = []
+
+        for i, dataset in enumerate(tqdm(dataset_info_list, desc='Sampling data')):
+            if i == len(dataset_info_list) - 1:
+                dataset_sample_count = remaining_count
+            else:
+                dataset_sample_count = int(dataset.weight * self.count)
+                remaining_count -= dataset_sample_count
+
+            sampled_data.extend(random.choices(all_data, k=dataset_sample_count))
+
+        return self._update_index(sampled_data)
+
+
+class UniformSampler(Sampler):
+    """
+    Uniform sampler, sample data from each dataset with the same number of samples.
+    """
+
+    def sample(self) -> List[dict]:
+        dataset_info_list = self.schema.flatten()
+        all_data = self._collect_dataset_data(dataset_info_list)
+
+        num_datasets = len(dataset_info_list)
+        samples_per_dataset = self.count // num_datasets
+        sampled_data = []
+
+        for _ in tqdm(dataset_info_list, desc='Sampling data'):
+            sampled_data.extend(random.choices(all_data, k=samples_per_dataset))
+
+        return self._update_index(sampled_data)
+
+
+class StratifiedSampler(Sampler):
+    """
+    Stratified sampler, sample data from each dataset according to the number of samples of each dataset.
+    """
+
+    def sample(self) -> List[dict]:
+        dataset_info_list = self.schema.flatten()
+        all_data = self._collect_dataset_data(dataset_info_list)
+
+        total_samples = sum(len(dataset.get_data()) for dataset in dataset_info_list)
+        sampled_data = []
+
+        for dataset in tqdm(dataset_info_list, desc='Sampling data'):
+            dataset_samples = len(dataset.get_data())
+            samples_for_dataset = int((dataset_samples / total_samples) * self.count)
+            sampled_data.extend(random.choices(all_data, k=samples_for_dataset))
+
+        return self._update_index(sampled_data)
+
+
+if __name__ == '__main__':
+    from evalscope.utils.io_utils import dump_jsonl_data
+
+    schema = CollectionSchema.from_json('outputs/schema.json')
+    print(schema.to_dict())
+    mixed_data = WeightedSampler(schema, 100).sample()
+    dump_jsonl_data(mixed_data, 'outputs/weighted_mixed_data.jsonl')
+
+    mixed_data = UniformSampler(schema, 100).sample()
+    dump_jsonl_data(mixed_data, 'outputs/uniform_mixed_data.jsonl')
+
+    mixed_data = StratifiedSampler(schema, 100).sample()
+    dump_jsonl_data(mixed_data, 'outputs/stratified_mixed_data.jsonl')